@opengeni/worker-bundle 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7113 -1652
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4337 -1052
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
|
@@ -6,9 +6,65 @@
|
|
|
6
6
|
// `selectCodexCredentialForTurn` precedence gate (pin > active). Keeping the
|
|
7
7
|
// decision pure makes the whole rotation correctness story unit-testable in
|
|
8
8
|
// isolation (see codex-rotation.test.ts).
|
|
9
|
-
import type {
|
|
9
|
+
import type {
|
|
10
|
+
CodexAccountStatus,
|
|
11
|
+
CodexCredentialLeaseSelectionContext,
|
|
12
|
+
CodexLeaseAccountStatus,
|
|
13
|
+
CodexPinSource,
|
|
14
|
+
} from "@opengeni/db";
|
|
10
15
|
|
|
11
|
-
export type
|
|
16
|
+
export type CodexRotationAccount = CodexAccountStatus | CodexLeaseAccountStatus;
|
|
17
|
+
|
|
18
|
+
export type CodexRotationStrategy =
|
|
19
|
+
| "most_remaining"
|
|
20
|
+
| "round_robin"
|
|
21
|
+
| "drain_then_next"
|
|
22
|
+
| "sharded";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* How a session's codex pin governs THIS turn's account selection (pure). Encodes the
|
|
26
|
+
* policy-pin LIFECYCLE rule: a 'policy' pin is meaningful ONLY while the sharded policy
|
|
27
|
+
* is active.
|
|
28
|
+
* • "manual" — a manual pin: honored under EVERY strategy; never moved or cleared.
|
|
29
|
+
* • "sharded" — the sharded policy is active and the pin is non-manual: assign / keep
|
|
30
|
+
* / re-shard the deterministic home (covers an unpinned first turn AND
|
|
31
|
+
* an existing policy pin).
|
|
32
|
+
* • "clearStale" — a 'policy' pin while the sharded policy is NOT active: IGNORE it
|
|
33
|
+
* (never honor it as a sticky pin — that is the no-escape trap) and
|
|
34
|
+
* clear it lazily so the session converges to the active strategy.
|
|
35
|
+
* • "unpinned" — no pin (or none that applies): follow the active strategy / workspace
|
|
36
|
+
* active pointer, unchanged.
|
|
37
|
+
*/
|
|
38
|
+
export type CodexPinDisposition = "manual" | "sharded" | "clearStale" | "unpinned";
|
|
39
|
+
|
|
40
|
+
/** Classify a session's codex pin against the active rotation regime (see {@link CodexPinDisposition}). */
|
|
41
|
+
export function classifyCodexPin(args: {
|
|
42
|
+
pinnedCredentialId: string | null;
|
|
43
|
+
pinSource: CodexPinSource | null;
|
|
44
|
+
strategy: CodexRotationStrategy;
|
|
45
|
+
rotationEnabled: boolean;
|
|
46
|
+
}): CodexPinDisposition {
|
|
47
|
+
const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
|
|
48
|
+
const pinned = pinnedCredentialId != null;
|
|
49
|
+
// A manual pin is sacrosanct under every strategy — checked FIRST so sharded never
|
|
50
|
+
// touches it. DEFENSE-IN-DEPTH (fail-safe toward sacredness): a pin whose source is
|
|
51
|
+
// anything OTHER than the explicit 'policy' — including a NULL source (a pre-backfill
|
|
52
|
+
// row, or any pin an unforeseen path wrote without labeling it) — is treated as
|
|
53
|
+
// MANUAL. An unlabeled pin must NEVER be policy-moved; only an explicitly-'policy' pin
|
|
54
|
+
// is re-shardable.
|
|
55
|
+
if (pinned && pinSource !== "policy") {
|
|
56
|
+
return "manual";
|
|
57
|
+
}
|
|
58
|
+
const shardedActive = rotationEnabled && strategy === "sharded";
|
|
59
|
+
if (shardedActive) {
|
|
60
|
+
return "sharded";
|
|
61
|
+
}
|
|
62
|
+
// A policy pin outside the sharded regime is stale → clear it.
|
|
63
|
+
if (pinned && pinSource === "policy") {
|
|
64
|
+
return "clearStale";
|
|
65
|
+
}
|
|
66
|
+
return "unpinned";
|
|
67
|
+
}
|
|
12
68
|
|
|
13
69
|
export type RotationDecision =
|
|
14
70
|
// The chosen account. `moved` ⇒ it differs from the current active pointer, so
|
|
@@ -24,6 +80,13 @@ export type RotationDecision =
|
|
|
24
80
|
// No connected accounts at all (preserves today's relogin-fail path).
|
|
25
81
|
| { kind: "none" };
|
|
26
82
|
|
|
83
|
+
export type CodexTurnLeaseSelection = {
|
|
84
|
+
credentialId: string | null;
|
|
85
|
+
decision: RotationDecision;
|
|
86
|
+
/** Policy/manual homes never move the workspace-global legacy pointer. */
|
|
87
|
+
advanceActivePointer?: boolean;
|
|
88
|
+
};
|
|
89
|
+
|
|
27
90
|
// Invariant 4 (NO THRASH / BOUNDED) safety floors. An all-capped idle MUST be a
|
|
28
91
|
// positive, bounded wait — a null / elapsed / unknown reset can NEVER collapse it
|
|
29
92
|
// into a 0 or a past instant (which the caller would turn into a 0 continueDelayMs
|
|
@@ -39,9 +102,20 @@ export const MIN_IDLE_MS = 60_000; // 60s
|
|
|
39
102
|
*/
|
|
40
103
|
export const DEFAULT_RESET_COOLDOWN_MS = 60_000; // 60s
|
|
41
104
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
105
|
+
function effectiveWindowUsed(usedPercent: number | null, resetAt: Date | null, now: Date): number {
|
|
106
|
+
// A completed provider window is eligible immediately even if its last cache
|
|
107
|
+
// sample still says 100%. This is the five-hour reset boundary—not a TTL
|
|
108
|
+
// heuristic—and avoids stranding a reset subscription while another remains
|
|
109
|
+
// healthy. A future/unknown reset keeps the cached percentage authoritative.
|
|
110
|
+
return resetAt && resetAt.getTime() <= now.getTime() ? 0 : (usedPercent ?? 0);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** The worse live window: weekly binds as hard as 5h. */
|
|
114
|
+
function bindingUsedPct(acct: CodexRotationAccount, now: Date): number {
|
|
115
|
+
return Math.max(
|
|
116
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
|
|
117
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now),
|
|
118
|
+
);
|
|
45
119
|
}
|
|
46
120
|
|
|
47
121
|
/**
|
|
@@ -49,13 +123,15 @@ function bindingUsedPct(acct: CodexAccountStatus): number {
|
|
|
49
123
|
* buildCodexUsageWindowFromCache's `remaining = 100 - percent`, taking the MIN
|
|
50
124
|
* across both windows (the scarcer of 5h/weekly). null percent ⇒ 100 remaining.
|
|
51
125
|
*/
|
|
52
|
-
function bindingRemaining(acct:
|
|
53
|
-
const primaryRemaining =
|
|
54
|
-
|
|
126
|
+
function bindingRemaining(acct: CodexRotationAccount, now: Date): number {
|
|
127
|
+
const primaryRemaining =
|
|
128
|
+
100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
|
|
129
|
+
const secondaryRemaining =
|
|
130
|
+
100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
|
|
55
131
|
return Math.min(primaryRemaining, secondaryRemaining);
|
|
56
132
|
}
|
|
57
133
|
|
|
58
|
-
function cooling(acct:
|
|
134
|
+
function cooling(acct: CodexRotationAccount, now: Date): boolean {
|
|
59
135
|
return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
|
|
60
136
|
}
|
|
61
137
|
|
|
@@ -66,7 +142,7 @@ function cooling(acct: CodexAccountStatus, now: Date): boolean {
|
|
|
66
142
|
* A null (never-probed) set is UNKNOWN: never credited as covering (Tier 2 only),
|
|
67
143
|
* but — critically — never excluded, so failover is fully preserved.
|
|
68
144
|
*/
|
|
69
|
-
function covers(acct:
|
|
145
|
+
function covers(acct: CodexRotationAccount, usedConnectors: string[]): boolean {
|
|
70
146
|
if (usedConnectors.length === 0) {
|
|
71
147
|
return true;
|
|
72
148
|
}
|
|
@@ -83,7 +159,7 @@ function covers(acct: CodexAccountStatus, usedConnectors: string[]): boolean {
|
|
|
83
159
|
* connector" note). Empty when the account covers (superset) or nothing was used.
|
|
84
160
|
* A null (unknown) set surfaces ALL used connectors — we can't prove it covers them.
|
|
85
161
|
*/
|
|
86
|
-
function droppedConnectorsFor(acct:
|
|
162
|
+
function droppedConnectorsFor(acct: CodexRotationAccount, usedConnectors: string[]): string[] {
|
|
87
163
|
if (usedConnectors.length === 0) {
|
|
88
164
|
return [];
|
|
89
165
|
}
|
|
@@ -96,11 +172,139 @@ function droppedConnectorsFor(acct: CodexAccountStatus, usedConnectors: string[]
|
|
|
96
172
|
}
|
|
97
173
|
|
|
98
174
|
/**
|
|
99
|
-
*
|
|
100
|
-
* not cooling
|
|
175
|
+
* Healthy for an already-frozen/in-flight turn = connected/usable (status
|
|
176
|
+
* "active", excludes needs_relogin/error), not cooling, and under the
|
|
177
|
+
* near-exhaustion threshold on BOTH windows. Allocator eligibility is
|
|
178
|
+
* deliberately separate so disabling NEW work cannot break refresh/resume.
|
|
179
|
+
*/
|
|
180
|
+
export function isCodexCredentialHealthy(
|
|
181
|
+
acct: CodexRotationAccount,
|
|
182
|
+
nearExhaustionPct: number,
|
|
183
|
+
now: Date,
|
|
184
|
+
): boolean {
|
|
185
|
+
return (
|
|
186
|
+
acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < nearExhaustionPct
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** New automatic allocation requires both health and explicit allocator eligibility. */
|
|
191
|
+
export function isCodexCredentialEligible(
|
|
192
|
+
acct: CodexRotationAccount,
|
|
193
|
+
nearExhaustionPct: number,
|
|
194
|
+
now: Date,
|
|
195
|
+
): boolean {
|
|
196
|
+
return acct.allocatorEnabled && isCodexCredentialHealthy(acct, nearExhaustionPct, now);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Public eligibility predicate (same definition as the private `eligible`): an
|
|
201
|
+
* account is usable this turn iff it is connected/active, not cooling, and under
|
|
202
|
+
* the near-exhaustion threshold on BOTH windows. The worker's sharded home
|
|
203
|
+
* health-check uses this to decide whether a session's existing policy pin is still
|
|
204
|
+
* a valid home or must be re-sharded.
|
|
205
|
+
*/
|
|
206
|
+
export function isCodexAccountEligible(
|
|
207
|
+
acct: CodexRotationAccount,
|
|
208
|
+
nearExhaustionPct: number,
|
|
209
|
+
now: Date,
|
|
210
|
+
): boolean {
|
|
211
|
+
return isCodexCredentialEligible(acct, nearExhaustionPct, now);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Deterministic 32-bit FNV-1a over a UTF-16 code-unit stream. Pure, allocation-free. */
|
|
215
|
+
function fnv1a32(input: string): number {
|
|
216
|
+
let hash = 0x811c9dc5;
|
|
217
|
+
for (let i = 0; i < input.length; i++) {
|
|
218
|
+
hash ^= input.charCodeAt(i);
|
|
219
|
+
// 32-bit FNV prime multiply via Math.imul; `>>> 0` keeps it unsigned.
|
|
220
|
+
hash = Math.imul(hash, 0x01000193);
|
|
221
|
+
}
|
|
222
|
+
return hash >>> 0;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Session-sharded HOME account (AM-6): the deterministic account a session runs on
|
|
227
|
+
* under the "sharded" strategy — `stableAccountList[ hash(sessionId) % N ]` over the
|
|
228
|
+
* ELIGIBLE (connected, not cooling, under-threshold) accounts in stable created_at
|
|
229
|
+
* order (`listCodexAccountStatuses`).
|
|
230
|
+
*
|
|
231
|
+
* Why deterministic hash over "least-loaded by session count" (AM-6): it needs ZERO
|
|
232
|
+
* coordination — every worker computes the same home for a given (sessionId,
|
|
233
|
+
* eligible-set), so a burst of concurrent first-turns can't all read the same
|
|
234
|
+
* "least-loaded" account and stampede it (read-then-write skew), and there is no
|
|
235
|
+
* shared round-robin cursor to contend on. It is balanced in expectation.
|
|
236
|
+
*
|
|
237
|
+
* Re-shard (AM-5): pass the just-capped account as already-cooling (or simply let it
|
|
238
|
+
* fall over its threshold) and this reshuffles the SURVIVORS deterministically — the
|
|
239
|
+
* sessions that shared a capped account spread across the remaining pool by their own
|
|
240
|
+
* hashes instead of all re-concentrating on one first-eligible failover.
|
|
241
|
+
*
|
|
242
|
+
* Returns null when NO account is eligible (the caller idles until reset).
|
|
101
243
|
*/
|
|
102
|
-
function
|
|
103
|
-
|
|
244
|
+
export function shardCredentialForSession(args: {
|
|
245
|
+
sessionId: string;
|
|
246
|
+
accounts: CodexRotationAccount[];
|
|
247
|
+
nearExhaustionPct: number;
|
|
248
|
+
now: Date;
|
|
249
|
+
}): string | null {
|
|
250
|
+
const { sessionId, accounts, nearExhaustionPct, now } = args;
|
|
251
|
+
const eligibles = accounts.filter((acct) =>
|
|
252
|
+
isCodexCredentialEligible(acct, nearExhaustionPct, now),
|
|
253
|
+
);
|
|
254
|
+
if (eligibles.length === 0) {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
const index = fnv1a32(sessionId) % eligibles.length;
|
|
258
|
+
return eligibles[index]!.id;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** earliestResetAt across ALL connected accounts — exported for the sharded all-capped idle. */
|
|
262
|
+
export function earliestCodexReset(
|
|
263
|
+
accounts: CodexRotationAccount[],
|
|
264
|
+
nearExhaustionPct: number,
|
|
265
|
+
now: Date,
|
|
266
|
+
): Date {
|
|
267
|
+
return earliestReset(accounts, nearExhaustionPct, now);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The proactive SHARDED home decision (pure — zero I/O, like {@link chooseRotationActive}).
|
|
272
|
+
* Decides the account a session should run on THIS turn under the "sharded" strategy,
|
|
273
|
+
* given its current POLICY pin (null on the first turn) and the accounts snapshot:
|
|
274
|
+
*
|
|
275
|
+
* - keepPin: the current policy pin is still ELIGIBLE → run there, no rewrite. This
|
|
276
|
+
* is the steady-state cache-warm path (a session stays on its one home account).
|
|
277
|
+
* - reshard: no policy pin yet (first-turn lazy assignment, AM-7) OR the policy pin
|
|
278
|
+
* is capped/ineligible (proactive re-shard, AM-4) → deterministically shard over the
|
|
279
|
+
* ELIGIBLE set (AM-6) and durably (re)write the pin (`rewritePin:true`, AM-3/AM-5).
|
|
280
|
+
* - allCapped: no account is eligible → the caller idles until the earliest reset.
|
|
281
|
+
*
|
|
282
|
+
* A MANUAL pin is handled by the CALLER (it never reaches here); this function only
|
|
283
|
+
* governs policy homes. Pure so the assignment/re-shard/keep logic is unit-testable
|
|
284
|
+
* without a worker/db env; the caller wraps it with a self-heal usage refresh between
|
|
285
|
+
* two evaluations exactly like chooseRotationActive's allCapped path.
|
|
286
|
+
*/
|
|
287
|
+
export function chooseShardedHome(args: {
|
|
288
|
+
sessionId: string;
|
|
289
|
+
currentPolicyPin: string | null;
|
|
290
|
+
accounts: CodexRotationAccount[];
|
|
291
|
+
nearExhaustionPct: number;
|
|
292
|
+
now: Date;
|
|
293
|
+
}):
|
|
294
|
+
| { kind: "home"; credentialId: string; rewritePin: boolean }
|
|
295
|
+
| { kind: "allCapped"; earliestResetAt: Date } {
|
|
296
|
+
const { sessionId, currentPolicyPin, accounts, nearExhaustionPct, now } = args;
|
|
297
|
+
const pinRow = currentPolicyPin
|
|
298
|
+
? (accounts.find((acct) => acct.id === currentPolicyPin) ?? null)
|
|
299
|
+
: null;
|
|
300
|
+
if (pinRow && isCodexCredentialEligible(pinRow, nearExhaustionPct, now)) {
|
|
301
|
+
return { kind: "home", credentialId: currentPolicyPin!, rewritePin: false };
|
|
302
|
+
}
|
|
303
|
+
const home = shardCredentialForSession({ sessionId, accounts, nearExhaustionPct, now });
|
|
304
|
+
if (home == null) {
|
|
305
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
|
|
306
|
+
}
|
|
307
|
+
return { kind: "home", credentialId: home, rewritePin: true };
|
|
104
308
|
}
|
|
105
309
|
|
|
106
310
|
/**
|
|
@@ -115,7 +319,11 @@ function eligible(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date
|
|
|
115
319
|
* default cooldown (now + DEFAULT_RESET_COOLDOWN_MS), so the caller always idles a
|
|
116
320
|
* bounded positive time and re-checks (which refreshes usage and self-heals).
|
|
117
321
|
*/
|
|
118
|
-
export function availableAt(
|
|
322
|
+
export function availableAt(
|
|
323
|
+
acct: CodexRotationAccount,
|
|
324
|
+
nearExhaustionPct: number,
|
|
325
|
+
now: Date,
|
|
326
|
+
): Date {
|
|
119
327
|
const nowMs = now.getTime();
|
|
120
328
|
// An unknown/elapsed block clears after a default cooldown, never in the past.
|
|
121
329
|
const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
|
|
@@ -131,8 +339,14 @@ export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number,
|
|
|
131
339
|
// reset is unknown → default cooldown (never a past instant).
|
|
132
340
|
candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
|
|
133
341
|
};
|
|
134
|
-
windowClear(
|
|
135
|
-
|
|
342
|
+
windowClear(
|
|
343
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= nearExhaustionPct,
|
|
344
|
+
acct.primaryResetAt,
|
|
345
|
+
);
|
|
346
|
+
windowClear(
|
|
347
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= nearExhaustionPct,
|
|
348
|
+
acct.secondaryResetAt,
|
|
349
|
+
);
|
|
136
350
|
// Ineligible for a non-quota reason (needs_relogin / error) with no known block, or
|
|
137
351
|
// a cleared cooldown: still idle a bounded cooldown before re-check — never the past.
|
|
138
352
|
if (candidates.length === 0) {
|
|
@@ -143,12 +357,54 @@ export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number,
|
|
|
143
357
|
}
|
|
144
358
|
|
|
145
359
|
/** earliestResetAt across ALL connected accounts (min of each account's availableAt). */
|
|
146
|
-
function earliestReset(
|
|
360
|
+
function earliestReset(
|
|
361
|
+
accounts: CodexRotationAccount[],
|
|
362
|
+
nearExhaustionPct: number,
|
|
363
|
+
now: Date,
|
|
364
|
+
): Date {
|
|
147
365
|
return accounts
|
|
148
366
|
.map((acct) => availableAt(acct, nearExhaustionPct, now))
|
|
149
367
|
.reduce((a, b) => (b.getTime() < a.getTime() ? b : a));
|
|
150
368
|
}
|
|
151
369
|
|
|
370
|
+
/**
|
|
371
|
+
* Earliest provider/cooldown timestamp that is actually authoritative. The
|
|
372
|
+
* ranker's `availableAt` deliberately synthesizes a positive default for
|
|
373
|
+
* unknown/stale state; durable capacity waits must distinguish that bounded
|
|
374
|
+
* refresh backoff from a real provider reset timer.
|
|
375
|
+
*/
|
|
376
|
+
export function authoritativeCodexCapacityResetAt(
|
|
377
|
+
accounts: CodexRotationAccount[],
|
|
378
|
+
nearExhaustionPct: number,
|
|
379
|
+
now: Date,
|
|
380
|
+
): Date | null {
|
|
381
|
+
const future: Date[] = [];
|
|
382
|
+
for (const account of accounts) {
|
|
383
|
+
if (!account.allocatorEnabled || account.status !== "active") continue;
|
|
384
|
+
if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
|
|
385
|
+
future.push(account.exhaustedUntil);
|
|
386
|
+
}
|
|
387
|
+
if (
|
|
388
|
+
(account.primaryUsedPercent ?? 0) >= nearExhaustionPct &&
|
|
389
|
+
account.primaryResetAt &&
|
|
390
|
+
account.primaryResetAt.getTime() > now.getTime()
|
|
391
|
+
) {
|
|
392
|
+
future.push(account.primaryResetAt);
|
|
393
|
+
}
|
|
394
|
+
if (
|
|
395
|
+
(account.secondaryUsedPercent ?? 0) >= nearExhaustionPct &&
|
|
396
|
+
account.secondaryResetAt &&
|
|
397
|
+
account.secondaryResetAt.getTime() > now.getTime()
|
|
398
|
+
) {
|
|
399
|
+
future.push(account.secondaryResetAt);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
if (future.length === 0) return null;
|
|
403
|
+
return future.reduce((earliest, candidate) =>
|
|
404
|
+
candidate.getTime() < earliest.getTime() ? candidate : earliest,
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
|
|
152
408
|
/**
|
|
153
409
|
* The bounded all-capped idle delay: clamp(earliestResetAt − now) into
|
|
154
410
|
* [MIN_IDLE_MS, maxMs]. NEVER 0 and NEVER negative, so a null / elapsed / unknown
|
|
@@ -237,7 +493,7 @@ export function chooseRotationActive(args: {
|
|
|
237
493
|
rotationStrategy: CodexRotationStrategy;
|
|
238
494
|
activeCredentialId: string | null;
|
|
239
495
|
priorCredentialId: string | null;
|
|
240
|
-
accounts:
|
|
496
|
+
accounts: CodexRotationAccount[];
|
|
241
497
|
nearExhaustionPct: number;
|
|
242
498
|
now: Date;
|
|
243
499
|
// P4 (Part B): the connectors the leaving session has access to (the leaving
|
|
@@ -246,25 +502,36 @@ export function chooseRotationActive(args: {
|
|
|
246
502
|
// P3 (every account trivially covers → Tier 1 == all eligibles). Self-gating.
|
|
247
503
|
usedConnectors?: string[];
|
|
248
504
|
}): RotationDecision {
|
|
249
|
-
const {
|
|
505
|
+
const {
|
|
506
|
+
rotationStrategy,
|
|
507
|
+
activeCredentialId,
|
|
508
|
+
priorCredentialId,
|
|
509
|
+
accounts,
|
|
510
|
+
nearExhaustionPct,
|
|
511
|
+
now,
|
|
512
|
+
} = args;
|
|
513
|
+
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
|
|
250
514
|
const usedConnectors = args.usedConnectors ?? [];
|
|
251
515
|
|
|
252
|
-
if (
|
|
516
|
+
if (allocatableAccounts.length === 0) {
|
|
253
517
|
return { kind: "none" };
|
|
254
518
|
}
|
|
255
519
|
|
|
256
|
-
const eligibles =
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
// session whose active account is still eligible does NOT rotate — no pointer
|
|
260
|
-
// move, no switch event. Steady-state stays as cheap as a non-rotation turn
|
|
261
|
-
// save the in-memory ranking. (Skipped for round_robin/drain which anchor on
|
|
262
|
-
// the prior account, but most_remaining is the default + correctness path.)
|
|
263
|
-
const activeRow = activeCredentialId ? accounts.find((acct) => acct.id === activeCredentialId) ?? null : null;
|
|
520
|
+
const eligibles = allocatableAccounts.filter((acct) =>
|
|
521
|
+
isCodexCredentialEligible(acct, nearExhaustionPct, now),
|
|
522
|
+
);
|
|
264
523
|
|
|
265
|
-
|
|
524
|
+
// The active pointer is a cursor/manual preference, NOT a sticky lease. In
|
|
525
|
+
// particular, most_remaining must rank the whole eligible pool every turn;
|
|
526
|
+
// keeping a healthy active account until 90% was the production monopolization
|
|
527
|
+
// defect fixed by OPE-21. drain_then_next remains the one explicitly sticky
|
|
528
|
+
// strategy, and a manual pin is handled by the caller as explicit policy.
|
|
529
|
+
const decide = (chosen: CodexRotationAccount | undefined): RotationDecision => {
|
|
266
530
|
if (!chosen) {
|
|
267
|
-
return {
|
|
531
|
+
return {
|
|
532
|
+
kind: "allCapped",
|
|
533
|
+
earliestResetAt: earliestReset(allocatableAccounts, nearExhaustionPct, now),
|
|
534
|
+
};
|
|
268
535
|
}
|
|
269
536
|
// P4: surface the dropped-connector note when the chosen account doesn't cover
|
|
270
537
|
// the session's used connectors (a Tier-2/unknown failover pick). Empty ⇒ omit.
|
|
@@ -281,46 +548,275 @@ export function chooseRotationActive(args: {
|
|
|
281
548
|
// Next eligible AFTER the prior account in list order (wrap around). When the
|
|
282
549
|
// prior account isn't found, start from the head.
|
|
283
550
|
if (eligibles.length === 0) {
|
|
284
|
-
return {
|
|
551
|
+
return {
|
|
552
|
+
kind: "allCapped",
|
|
553
|
+
earliestResetAt: earliestReset(allocatableAccounts, nearExhaustionPct, now),
|
|
554
|
+
};
|
|
285
555
|
}
|
|
286
|
-
const priorIdx = priorCredentialId
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
556
|
+
const priorIdx = priorCredentialId
|
|
557
|
+
? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId)
|
|
558
|
+
: -1;
|
|
559
|
+
const ordered =
|
|
560
|
+
priorIdx >= 0
|
|
561
|
+
? [
|
|
562
|
+
...allocatableAccounts.slice(priorIdx + 1),
|
|
563
|
+
...allocatableAccounts.slice(0, priorIdx + 1),
|
|
564
|
+
]
|
|
565
|
+
: allocatableAccounts;
|
|
566
|
+
const chosen = ordered.find((acct) => isCodexCredentialEligible(acct, nearExhaustionPct, now));
|
|
291
567
|
return decide(chosen);
|
|
292
568
|
}
|
|
293
569
|
|
|
294
570
|
if (rotationStrategy === "drain_then_next") {
|
|
295
571
|
// Stay on the prior account while it is eligible (drain it), else first eligible.
|
|
296
|
-
const priorRow = priorCredentialId
|
|
297
|
-
|
|
572
|
+
const priorRow = priorCredentialId
|
|
573
|
+
? accounts.find((acct) => acct.id === priorCredentialId)
|
|
574
|
+
: undefined;
|
|
575
|
+
if (priorRow && isCodexCredentialEligible(priorRow, nearExhaustionPct, now)) {
|
|
298
576
|
return decide(priorRow);
|
|
299
577
|
}
|
|
300
578
|
return decide(eligibles[0]);
|
|
301
579
|
}
|
|
302
580
|
|
|
303
|
-
// most_remaining (default + the correctness path).
|
|
304
|
-
//
|
|
305
|
-
// is still eligible never switches — not even for connector coverage. Coverage
|
|
306
|
-
// matters ONLY on the must-move branch below.
|
|
307
|
-
if (activeRow && eligible(activeRow, nearExhaustionPct, now)) {
|
|
308
|
-
return { kind: "active", credentialId: activeRow.id, moved: false };
|
|
309
|
-
}
|
|
310
|
-
// Active is capped/near-cap/cooling/missing → must move. Two-tier coverage pick
|
|
311
|
-
// over the SAME eligible set (prefer-not-require):
|
|
581
|
+
// most_remaining (default + the correctness path). Two-tier coverage pick over
|
|
582
|
+
// the SAME eligible set (prefer-not-require):
|
|
312
583
|
// Tier 1 — eligibles whose connector set COVERS the session's used set.
|
|
313
584
|
// Tier 2 — ALL eligibles, only if Tier 1 is empty (failover fully preserved).
|
|
314
|
-
// Within the chosen tier
|
|
315
|
-
//
|
|
316
|
-
//
|
|
585
|
+
// Within the chosen tier:
|
|
586
|
+
// 1. least active leases (concurrent turns spread before quota metadata moves),
|
|
587
|
+
// 2. most remaining binding quota,
|
|
588
|
+
// 3. fewest historical selections,
|
|
589
|
+
// 4. least-recently selected, then stable created_at/id input order.
|
|
590
|
+
// Base CodexAccountStatus values (pure legacy tests/reactive rank) default the
|
|
591
|
+
// lease/cursor metadata to zero/null, preserving deterministic stable ties.
|
|
317
592
|
const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
|
|
318
593
|
const pool = covering.length > 0 ? covering : eligibles;
|
|
319
|
-
const
|
|
594
|
+
const activeLeases = (acct: CodexRotationAccount): number =>
|
|
595
|
+
"activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
|
|
596
|
+
const selections = (acct: CodexRotationAccount): number =>
|
|
597
|
+
"selectionCount" in acct ? acct.selectionCount : 0;
|
|
598
|
+
const lastSelected = (acct: CodexRotationAccount): number =>
|
|
599
|
+
"lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
|
|
600
|
+
const chosen = pool.reduce<CodexRotationAccount | undefined>((best, acct) => {
|
|
320
601
|
if (!best) {
|
|
321
602
|
return acct;
|
|
322
603
|
}
|
|
323
|
-
|
|
604
|
+
if (activeLeases(acct) !== activeLeases(best)) {
|
|
605
|
+
return activeLeases(acct) < activeLeases(best) ? acct : best;
|
|
606
|
+
}
|
|
607
|
+
if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
|
|
608
|
+
return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
|
|
609
|
+
}
|
|
610
|
+
if (selections(acct) !== selections(best)) {
|
|
611
|
+
return selections(acct) < selections(best) ? acct : best;
|
|
612
|
+
}
|
|
613
|
+
return lastSelected(acct) < lastSelected(best) ? acct : best;
|
|
324
614
|
}, undefined);
|
|
325
615
|
return decide(chosen);
|
|
326
616
|
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Exact pre-0053 selector used while the lease cutover is off.
|
|
620
|
+
*
|
|
621
|
+
* Old workers keep a healthy `most_remaining` active pointer sticky and know
|
|
622
|
+
* nothing about lease counts or fairness cursors. A compatible worker must do
|
|
623
|
+
* the same until BOTH the deployment flag and workspace cutover are enabled;
|
|
624
|
+
* otherwise a rolling fleet would run two allocators at once. When the active
|
|
625
|
+
* pointer is no longer eligible, zeroing the additive metadata delegates to the
|
|
626
|
+
* same capacity/connector ranking that the old selector used.
|
|
627
|
+
*/
|
|
628
|
+
export function chooseLegacyRotationActive(
|
|
629
|
+
args: Parameters<typeof chooseRotationActive>[0],
|
|
630
|
+
): RotationDecision {
|
|
631
|
+
if (args.rotationStrategy === "most_remaining") {
|
|
632
|
+
const active = args.activeCredentialId
|
|
633
|
+
? args.accounts.find((account) => account.id === args.activeCredentialId)
|
|
634
|
+
: undefined;
|
|
635
|
+
if (active && isCodexCredentialEligible(active, args.nearExhaustionPct, args.now)) {
|
|
636
|
+
return { kind: "active", credentialId: active.id, moved: false };
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
return chooseRotationActive({
|
|
640
|
+
...args,
|
|
641
|
+
accounts: args.accounts.map((account) => ({
|
|
642
|
+
...account,
|
|
643
|
+
activeLeaseCount: 0,
|
|
644
|
+
selectionCount: 0,
|
|
645
|
+
lastSelectedAt: null,
|
|
646
|
+
})),
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Full pure policy at the transaction boundary. A live same-turn lease is
|
|
652
|
+
* idempotent; the rollout-off/manual path preserves pin>pointer behavior; a
|
|
653
|
+
* healthy pin wins only while eligible, so quota/auth quarantine cannot trap a
|
|
654
|
+
* session on one subscription.
|
|
655
|
+
*/
|
|
656
|
+
export function selectCodexCredentialLeaseForTurn(args: {
|
|
657
|
+
context: CodexCredentialLeaseSelectionContext;
|
|
658
|
+
leasingEnabled: boolean;
|
|
659
|
+
sessionId: string;
|
|
660
|
+
sessionPinnedCredentialId: string | null;
|
|
661
|
+
sessionPinSource: CodexPinSource | null;
|
|
662
|
+
sessionLastCredentialId: string | null;
|
|
663
|
+
/** Same-turn checkpoint/frozen account; temporary disable cannot move it. */
|
|
664
|
+
continuationCredentialId?: string | null;
|
|
665
|
+
nearExhaustionPct: number;
|
|
666
|
+
now: Date;
|
|
667
|
+
}): CodexTurnLeaseSelection {
|
|
668
|
+
const {
|
|
669
|
+
accounts,
|
|
670
|
+
activeCredentialId,
|
|
671
|
+
rotationEnabled,
|
|
672
|
+
leaseRotationEnabled,
|
|
673
|
+
rotationStrategy,
|
|
674
|
+
existingCredentialId,
|
|
675
|
+
} = args.context;
|
|
676
|
+
const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
|
|
677
|
+
const strategy = rotationStrategy as CodexRotationStrategy;
|
|
678
|
+
const existing = existingCredentialId
|
|
679
|
+
? accounts.find((account) => account.id === existingCredentialId)
|
|
680
|
+
: undefined;
|
|
681
|
+
if (existing && isCodexCredentialHealthy(existing, args.nearExhaustionPct, args.now)) {
|
|
682
|
+
return {
|
|
683
|
+
credentialId: existing.id,
|
|
684
|
+
advanceActivePointer: false,
|
|
685
|
+
decision: {
|
|
686
|
+
kind: "active",
|
|
687
|
+
credentialId: existing.id,
|
|
688
|
+
moved: existing.id !== activeCredentialId,
|
|
689
|
+
},
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const continuation = args.continuationCredentialId
|
|
694
|
+
? accounts.find((account) => account.id === args.continuationCredentialId)
|
|
695
|
+
: undefined;
|
|
696
|
+
if (continuation && isCodexCredentialHealthy(continuation, args.nearExhaustionPct, args.now)) {
|
|
697
|
+
return {
|
|
698
|
+
credentialId: continuation.id,
|
|
699
|
+
advanceActivePointer: false,
|
|
700
|
+
decision: {
|
|
701
|
+
kind: "active",
|
|
702
|
+
credentialId: continuation.id,
|
|
703
|
+
moved: continuation.id !== activeCredentialId,
|
|
704
|
+
},
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const connectedIds = new Set(
|
|
709
|
+
accounts.filter((account) => account.allocatorEnabled).map((account) => account.id),
|
|
710
|
+
);
|
|
711
|
+
|
|
712
|
+
const pinDisposition = classifyCodexPin({
|
|
713
|
+
pinnedCredentialId: args.sessionPinnedCredentialId,
|
|
714
|
+
pinSource: args.sessionPinSource,
|
|
715
|
+
strategy,
|
|
716
|
+
rotationEnabled,
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
// A manual pin is user intent, not a policy hint. Never silently fail it over.
|
|
720
|
+
// We still reject allocator-disabled rows for a NEW turn; healthy live/frozen
|
|
721
|
+
// same-turn continuity already returned above before this admission filter.
|
|
722
|
+
if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
|
|
723
|
+
const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
|
|
724
|
+
if (!pinned?.allocatorEnabled) {
|
|
725
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
726
|
+
}
|
|
727
|
+
if (isCodexCredentialHealthy(pinned, args.nearExhaustionPct, args.now)) {
|
|
728
|
+
return {
|
|
729
|
+
credentialId: pinned.id,
|
|
730
|
+
decision: { kind: "active", credentialId: pinned.id, moved: false },
|
|
731
|
+
advanceActivePointer: false,
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
return {
|
|
735
|
+
credentialId: null,
|
|
736
|
+
decision: {
|
|
737
|
+
kind: "allCapped",
|
|
738
|
+
earliestResetAt: availableAt(pinned, args.nearExhaustionPct, args.now),
|
|
739
|
+
},
|
|
740
|
+
advanceActivePointer: false,
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// OPE-31 policy homes compose with OPE-21 by sharding only the candidate list
|
|
745
|
+
// handed to this selector. A future OPE-32 filter may therefore choose one
|
|
746
|
+
// primary/fallback pool before this ranker runs; accounts from different pools
|
|
747
|
+
// are never union-ranked here.
|
|
748
|
+
if (pinDisposition === "sharded") {
|
|
749
|
+
if (accounts.length === 0) {
|
|
750
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
751
|
+
}
|
|
752
|
+
const shard = chooseShardedHome({
|
|
753
|
+
sessionId: args.sessionId,
|
|
754
|
+
currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
|
|
755
|
+
accounts,
|
|
756
|
+
nearExhaustionPct: args.nearExhaustionPct,
|
|
757
|
+
now: args.now,
|
|
758
|
+
});
|
|
759
|
+
if (shard.kind === "allCapped") {
|
|
760
|
+
return {
|
|
761
|
+
credentialId: null,
|
|
762
|
+
decision: shard,
|
|
763
|
+
advanceActivePointer: false,
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
return {
|
|
767
|
+
credentialId: shard.credentialId,
|
|
768
|
+
decision: {
|
|
769
|
+
kind: "active",
|
|
770
|
+
credentialId: shard.credentialId,
|
|
771
|
+
moved: shard.credentialId !== activeCredentialId,
|
|
772
|
+
},
|
|
773
|
+
advanceActivePointer: false,
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// A stale policy pin is intentionally ignored here and cleared by the caller
|
|
778
|
+
// after the atomic selection. Rotation-off/unpinned behavior otherwise keeps
|
|
779
|
+
// the pre-lease pin > active-pointer contract.
|
|
780
|
+
if (!rotationEnabled) {
|
|
781
|
+
const credentialId =
|
|
782
|
+
activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
|
|
783
|
+
return {
|
|
784
|
+
credentialId,
|
|
785
|
+
decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" },
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// Before the workspace cutover bit is enabled (or after the deployment kill
|
|
790
|
+
// switch is turned off), preserve the exact legacy policy used by old workers:
|
|
791
|
+
// a pin stays sticky; otherwise an enabled workspace runs the pure legacy
|
|
792
|
+
// rotation ranker without touching lease/cursor state.
|
|
793
|
+
if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
|
|
794
|
+
const credentialId = connectedIds.has(args.sessionPinnedCredentialId)
|
|
795
|
+
? args.sessionPinnedCredentialId
|
|
796
|
+
: activeCredentialId && connectedIds.has(activeCredentialId)
|
|
797
|
+
? activeCredentialId
|
|
798
|
+
: null;
|
|
799
|
+
return {
|
|
800
|
+
credentialId,
|
|
801
|
+
decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" },
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
const priorId = args.sessionLastCredentialId ?? activeCredentialId;
|
|
806
|
+
const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
|
|
807
|
+
const decision = choose({
|
|
808
|
+
rotationStrategy: strategy,
|
|
809
|
+
activeCredentialId,
|
|
810
|
+
// Per-session continuity is the round-robin/drain cursor. Falling back to
|
|
811
|
+
// the workspace pointer is only correct when this session has never run.
|
|
812
|
+
priorCredentialId: priorId,
|
|
813
|
+
accounts,
|
|
814
|
+
nearExhaustionPct: args.nearExhaustionPct,
|
|
815
|
+
now: args.now,
|
|
816
|
+
usedConnectors: accounts.find((account) => account.id === priorId)?.connectorNamespaces ?? [],
|
|
817
|
+
});
|
|
818
|
+
return {
|
|
819
|
+
credentialId: decision.kind === "active" ? decision.credentialId : null,
|
|
820
|
+
decision,
|
|
821
|
+
};
|
|
822
|
+
}
|