@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { environmentsEncryptionKeyBytes, parseModelProvidersJson, type RegistryProvider, type Settings } from "@opengeni/config";
|
|
2
|
+
import {
|
|
3
|
+
CODEX_APPS_MCP_SERVER_ID,
|
|
4
|
+
CODEX_APPS_MCP_SERVER_NAME,
|
|
5
|
+
CODEX_APPS_MCP_URL,
|
|
6
|
+
CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
7
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
8
|
+
CODEX_MODEL_ID_PREFIX,
|
|
9
|
+
CODEX_PROVIDER_BASE_URL,
|
|
10
|
+
CODEX_PROVIDER_ID,
|
|
11
|
+
} from "@opengeni/codex";
|
|
12
|
+
import {
|
|
13
|
+
decryptedCapabilityHeaders,
|
|
14
|
+
listEnabledMcpCapabilityServers,
|
|
15
|
+
workspaceCodexSubscriptionActive,
|
|
16
|
+
type Database,
|
|
17
|
+
type EnabledMcpCapabilityServer,
|
|
18
|
+
} from "@opengeni/db";
|
|
19
|
+
|
|
20
|
+
export async function settingsWithEnabledCapabilityMcpServers(db: Database, workspaceId: string, settings: Settings): Promise<Settings> {
|
|
21
|
+
const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
|
|
22
|
+
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* When the workspace has an active Codex subscription connected and the feature
|
|
27
|
+
* is enabled, inject a synthetic "codex-subscription" registry provider so a
|
|
28
|
+
* `codex/<slug>` model id routes through the ChatGPT backend. No secrets touch
|
|
29
|
+
* this overlay (metadata-only read); the per-request bearer is resolved later via
|
|
30
|
+
* codexRequestStorage. Idempotent and a no-op when not applicable.
|
|
31
|
+
*/
|
|
32
|
+
export async function settingsWithCodexCredential(db: Database, workspaceId: string, settings: Settings, activeOverride?: boolean): Promise<Settings> {
|
|
33
|
+
// Same active-credential predicate the billing bypass uses, so provider
|
|
34
|
+
// injection and billing can never disagree on what an "active codex" turn is.
|
|
35
|
+
// The caller may pass `activeOverride` (a single, shared read; P2-b) so routing
|
|
36
|
+
// and billing decide from the exact same observation, immune to a concurrent
|
|
37
|
+
// disconnect/reconnect landing between two independent reads.
|
|
38
|
+
const active = activeOverride ?? await workspaceCodexSubscriptionActive(db, settings, workspaceId);
|
|
39
|
+
if (!active) {
|
|
40
|
+
return settings; // disabled / not connected / needs_relogin / error -> leave settings unchanged
|
|
41
|
+
}
|
|
42
|
+
const withProvider = withCodexProvider(settings);
|
|
43
|
+
// Additive: append the synthetic codex_apps connectors MCP server for ANY
|
|
44
|
+
// active credential. Connector access is gated SERVER-SIDE per ChatGPT account
|
|
45
|
+
// (via chatgpt-account-id), NOT by token scopes — confirmed live: a `pro` token
|
|
46
|
+
// whose only scopes are openid/profile/email/offline_access still lists all 217
|
|
47
|
+
// connector tools at .../ps/mcp. So we inject unconditionally and let
|
|
48
|
+
// runtime-discovery decide: an account with no connectors yields an empty
|
|
49
|
+
// tools/list, and a connect failure best-effort-drops the server without
|
|
50
|
+
// failing the turn. The bearer is injected dynamically at connect time
|
|
51
|
+
// (runtime/codexAppsMcpRequestInit).
|
|
52
|
+
return withCodexAppsMcpServer(withProvider);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Pure: append the synthetic codex_apps MCP server, idempotently. Connector
|
|
57
|
+
* access is gated SERVER-SIDE per ChatGPT account (chatgpt-account-id), not by
|
|
58
|
+
* token scopes, so we inject for any active credential and let runtime-discovery
|
|
59
|
+
* resolve the actual tool set (empty list / dropped server when unavailable).
|
|
60
|
+
* No secrets here — the refreshing bearer is injected at connect time from
|
|
61
|
+
* codexRequestStorage (runtime/codexAppsMcpRequestInit). The connectors backend
|
|
62
|
+
* tolerates serial and parallel tool invocation, so no per-server serialization
|
|
63
|
+
* is enforced (the SDK exposes no per-server parallel-tool-calls flag in
|
|
64
|
+
* @openai/agents 0.11.6).
|
|
65
|
+
*/
|
|
66
|
+
export function withCodexAppsMcpServer(settings: Settings): Settings {
|
|
67
|
+
if (settings.mcpServers.some((server) => server.id === CODEX_APPS_MCP_SERVER_ID)) {
|
|
68
|
+
return settings; // already injected
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
...settings,
|
|
72
|
+
mcpServers: [
|
|
73
|
+
...settings.mcpServers,
|
|
74
|
+
{
|
|
75
|
+
id: CODEX_APPS_MCP_SERVER_ID,
|
|
76
|
+
name: CODEX_APPS_MCP_SERVER_NAME,
|
|
77
|
+
url: CODEX_APPS_MCP_URL,
|
|
78
|
+
timeoutMs: CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
79
|
+
// Connector availability is per-credential and must re-discover each
|
|
80
|
+
// run; never poison a process-global tools-list cache.
|
|
81
|
+
cacheToolsList: false,
|
|
82
|
+
// deliberately NO `headers` — the refreshing bearer is dynamic
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Pure: append the synthetic codex-subscription provider, idempotently. */
|
|
89
|
+
export function withCodexProvider(settings: Settings): Settings {
|
|
90
|
+
const providers = parseModelProvidersJson(settings.modelProvidersJson);
|
|
91
|
+
if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
|
|
92
|
+
return settings; // already injected
|
|
93
|
+
}
|
|
94
|
+
const codexProvider: RegistryProvider = {
|
|
95
|
+
kind: "codex-subscription",
|
|
96
|
+
id: CODEX_PROVIDER_ID,
|
|
97
|
+
label: "Codex (ChatGPT subscription)",
|
|
98
|
+
api: "responses",
|
|
99
|
+
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
100
|
+
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({ id: `${CODEX_MODEL_ID_PREFIX}${slug}`, label: slug, reasoningEffort: true })),
|
|
101
|
+
};
|
|
102
|
+
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function settingsWithMcpCapabilityServers(settings: Settings, enabled: EnabledMcpCapabilityServer[]): Settings {
|
|
106
|
+
if (enabled.length === 0) {
|
|
107
|
+
return settings;
|
|
108
|
+
}
|
|
109
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
110
|
+
const existingIds = new Set(settings.mcpServers.map((server) => server.id));
|
|
111
|
+
const dynamicServers = enabled
|
|
112
|
+
.filter((server) => !existingIds.has(server.id))
|
|
113
|
+
.flatMap((server) => {
|
|
114
|
+
const headers = decryptedCapabilityHeaders(server, encryptionKey);
|
|
115
|
+
if (headers === "unavailable") {
|
|
116
|
+
// Without its credential headers this server can only fail auth at
|
|
117
|
+
// connect time and break agent turns; leave it out of the run.
|
|
118
|
+
return [];
|
|
119
|
+
}
|
|
120
|
+
return [{
|
|
121
|
+
id: server.id,
|
|
122
|
+
name: server.name,
|
|
123
|
+
url: server.url,
|
|
124
|
+
...(server.allowedTools ? { allowedTools: server.allowedTools } : {}),
|
|
125
|
+
...(server.timeoutMs ? { timeoutMs: server.timeoutMs } : {}),
|
|
126
|
+
cacheToolsList: server.cacheToolsList ?? false,
|
|
127
|
+
...(headers ? { headers } : {}),
|
|
128
|
+
}];
|
|
129
|
+
});
|
|
130
|
+
return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
|
|
131
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// db-backed Codex token lifecycle for a turn: proactive refresh, single-flight
|
|
2
|
+
// per credential instance, and permanent-failure status surfacing.
|
|
3
|
+
//
|
|
4
|
+
// The implementation now lives in @opengeni/db (packages/db/src/codex-token-resolver.ts)
|
|
5
|
+
// so the API process can drive the SAME refreshing resolver for the P2 /wham/usage
|
|
6
|
+
// quota-bar reads without duplicating the refresh-CAS / single-flight / RLS logic.
|
|
7
|
+
// This module re-exports it for back-compat, keeping the agent-turn.ts call site
|
|
8
|
+
// (buildCodexTokenResolver) unchanged.
|
|
9
|
+
export { buildCodexTokenResolver, type CodexAuthDeps } from "@opengeni/db";
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// Multi-account P3 — the PURE rotation ranker. Zero I/O: no provider calls, no
|
|
2
|
+
// decrypts, no db. It consumes the already-loaded, metadata-only account list
|
|
3
|
+
// (cached usage columns + the exhausted_until cooldown column) and returns the
|
|
4
|
+
// account a turn should run on. The two call sites (turn-start pre-emption and
|
|
5
|
+
// the reactive 429 catch in agent-turn.ts) feed its result into the unchanged
|
|
6
|
+
// `selectCodexCredentialForTurn` precedence gate (pin > active). Keeping the
|
|
7
|
+
// decision pure makes the whole rotation correctness story unit-testable in
|
|
8
|
+
// isolation (see codex-rotation.test.ts).
|
|
9
|
+
import type { CodexAccountStatus } from "@opengeni/db";
|
|
10
|
+
|
|
11
|
+
export type CodexRotationStrategy = "most_remaining" | "round_robin" | "drain_then_next";
|
|
12
|
+
|
|
13
|
+
export type RotationDecision =
|
|
14
|
+
// The chosen account. `moved` ⇒ it differs from the current active pointer, so
|
|
15
|
+
// the caller must persist the pointer move (and the switch is a "rotation").
|
|
16
|
+
// `droppedConnectors` (P4 Part B): the leaving session's used connectors that the
|
|
17
|
+
// chosen account does NOT cover — populated ONLY on a Tier-2/unknown pick that
|
|
18
|
+
// can't cover (prefer-not-require: failover still happens, but the pill warns).
|
|
19
|
+
| { kind: "active"; credentialId: string; moved: boolean; droppedConnectors?: string[] }
|
|
20
|
+
// Every eligible account is capped/cooling: idle until the soonest instant ANY
|
|
21
|
+
// account clears every blocking condition (the multi-account generalization of
|
|
22
|
+
// the single-account idle-until-reset).
|
|
23
|
+
| { kind: "allCapped"; earliestResetAt: Date }
|
|
24
|
+
// No connected accounts at all (preserves today's relogin-fail path).
|
|
25
|
+
| { kind: "none" };
|
|
26
|
+
|
|
27
|
+
// Invariant 4 (NO THRASH / BOUNDED) safety floors. An all-capped idle MUST be a
|
|
28
|
+
// positive, bounded wait — a null / elapsed / unknown reset can NEVER collapse it
|
|
29
|
+
// into a 0 or a past instant (which the caller would turn into a 0 continueDelayMs
|
|
30
|
+
// and a tight re-dispatch loop that hammers CPU/DB and never runs the model, so the
|
|
31
|
+
// stale usage cache never self-heals).
|
|
32
|
+
/** The minimum all-capped idle: continueDelayMs is clamped to at least this (never 0). */
|
|
33
|
+
export const MIN_IDLE_MS = 60_000; // 60s
|
|
34
|
+
/**
|
|
35
|
+
* Cooldown applied to an over-threshold account whose cached reset is null / unknown
|
|
36
|
+
* / already-elapsed (the turn hot path never refreshes usage, so a capped window's
|
|
37
|
+
* reset reads stale). Treating it as "available after a default cooldown" keeps
|
|
38
|
+
* availableAt — and therefore earliestReset — ALWAYS in the future.
|
|
39
|
+
*/
|
|
40
|
+
export const DEFAULT_RESET_COOLDOWN_MS = 60_000; // 60s
|
|
41
|
+
|
|
42
|
+
/** The worse-window used percent: weekly binds as hard as 5h, so take the max. null ⇒ 0. */
|
|
43
|
+
function bindingUsedPct(acct: CodexAccountStatus): number {
|
|
44
|
+
return Math.max(acct.primaryUsedPercent ?? 0, acct.secondaryUsedPercent ?? 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Remaining quota across the binding window — the P3 rotation key. Mirrors
|
|
49
|
+
* buildCodexUsageWindowFromCache's `remaining = 100 - percent`, taking the MIN
|
|
50
|
+
* across both windows (the scarcer of 5h/weekly). null percent ⇒ 100 remaining.
|
|
51
|
+
*/
|
|
52
|
+
function bindingRemaining(acct: CodexAccountStatus): number {
|
|
53
|
+
const primaryRemaining = 100 - (acct.primaryUsedPercent ?? 0);
|
|
54
|
+
const secondaryRemaining = 100 - (acct.secondaryUsedPercent ?? 0);
|
|
55
|
+
return Math.min(primaryRemaining, secondaryRemaining);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function cooling(acct: CodexAccountStatus, now: Date): boolean {
|
|
59
|
+
return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* P4 connector coverage (prefer-not-require). `acct` COVERS `usedConnectors` iff its
|
|
64
|
+
* cached connector set is a SUPERSET of the session's used set. An empty used set is
|
|
65
|
+
* trivially covered by everyone (→ Tier 1 == all eligibles == byte-identical to P3).
|
|
66
|
+
* A null (never-probed) set is UNKNOWN: never credited as covering (Tier 2 only),
|
|
67
|
+
* but — critically — never excluded, so failover is fully preserved.
|
|
68
|
+
*/
|
|
69
|
+
function covers(acct: CodexAccountStatus, usedConnectors: string[]): boolean {
|
|
70
|
+
if (usedConnectors.length === 0) {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
const owned = acct.connectorNamespaces;
|
|
74
|
+
if (owned === null) {
|
|
75
|
+
return false; // unprobed ⇒ unknown ⇒ never Tier 1 (but still a Tier-2 candidate)
|
|
76
|
+
}
|
|
77
|
+
const ownedSet = new Set(owned);
|
|
78
|
+
return usedConnectors.every((connector) => ownedSet.has(connector));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The used connectors the chosen account does NOT cover (the "switch dropped a
|
|
83
|
+
* connector" note). Empty when the account covers (superset) or nothing was used.
|
|
84
|
+
* A null (unknown) set surfaces ALL used connectors — we can't prove it covers them.
|
|
85
|
+
*/
|
|
86
|
+
function droppedConnectorsFor(acct: CodexAccountStatus, usedConnectors: string[]): string[] {
|
|
87
|
+
if (usedConnectors.length === 0) {
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
const owned = acct.connectorNamespaces;
|
|
91
|
+
if (owned === null) {
|
|
92
|
+
return [...usedConnectors];
|
|
93
|
+
}
|
|
94
|
+
const ownedSet = new Set(owned);
|
|
95
|
+
return usedConnectors.filter((connector) => !ownedSet.has(connector));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Eligible = connected/usable (status "active", excludes needs_relogin/error) AND
|
|
100
|
+
* not cooling AND under the near-exhaustion threshold on BOTH windows.
|
|
101
|
+
*/
|
|
102
|
+
function eligible(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date): boolean {
|
|
103
|
+
return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct) < nearExhaustionPct;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The soonest instant `acct` clears EVERY blocking condition: its cooldown end,
|
|
108
|
+
* and each window's reset (only when that window is at/over the threshold). The
|
|
109
|
+
* literal multi-account generalization of #143's single-account resetsInSeconds.
|
|
110
|
+
*
|
|
111
|
+
* GUARANTEE (invariant 4): the result is ALWAYS in the future. A blocking window
|
|
112
|
+
* whose cached reset is null / unknown / already-elapsed does NOT contribute a past
|
|
113
|
+
* instant (the old EPOCH0 seed bug that made earliestReset land in 1970 → a 0
|
|
114
|
+
* continueDelayMs → a tight idle loop). Such a window is treated as clearing after a
|
|
115
|
+
* default cooldown (now + DEFAULT_RESET_COOLDOWN_MS), so the caller always idles a
|
|
116
|
+
* bounded positive time and re-checks (which refreshes usage and self-heals).
|
|
117
|
+
*/
|
|
118
|
+
export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date): Date {
|
|
119
|
+
const nowMs = now.getTime();
|
|
120
|
+
// An unknown/elapsed block clears after a default cooldown, never in the past.
|
|
121
|
+
const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
|
|
122
|
+
const candidates: Date[] = [];
|
|
123
|
+
// An active cooldown only blocks while it is still in the future; a past cooldown
|
|
124
|
+
// self-clears (matches `cooling()`), so it must not pin availableAt to the past.
|
|
125
|
+
if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
|
|
126
|
+
candidates.push(acct.exhaustedUntil);
|
|
127
|
+
}
|
|
128
|
+
const windowClear = (over: boolean, resetAt: Date | null | undefined) => {
|
|
129
|
+
if (!over) return;
|
|
130
|
+
// Over-threshold window: wait for its KNOWN future reset; a null/elapsed cached
|
|
131
|
+
// reset is unknown → default cooldown (never a past instant).
|
|
132
|
+
candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
|
|
133
|
+
};
|
|
134
|
+
windowClear((acct.primaryUsedPercent ?? 0) >= nearExhaustionPct, acct.primaryResetAt);
|
|
135
|
+
windowClear((acct.secondaryUsedPercent ?? 0) >= nearExhaustionPct, acct.secondaryResetAt);
|
|
136
|
+
// Ineligible for a non-quota reason (needs_relogin / error) with no known block, or
|
|
137
|
+
// a cleared cooldown: still idle a bounded cooldown before re-check — never the past.
|
|
138
|
+
if (candidates.length === 0) {
|
|
139
|
+
return defaultClear;
|
|
140
|
+
}
|
|
141
|
+
// Clears EVERY blocking condition ⇒ the MAX of the per-condition clear instants.
|
|
142
|
+
return candidates.reduce((a, b) => (b.getTime() > a.getTime() ? b : a));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** earliestResetAt across ALL connected accounts (min of each account's availableAt). */
|
|
146
|
+
function earliestReset(accounts: CodexAccountStatus[], nearExhaustionPct: number, now: Date): Date {
|
|
147
|
+
return accounts
|
|
148
|
+
.map((acct) => availableAt(acct, nearExhaustionPct, now))
|
|
149
|
+
.reduce((a, b) => (b.getTime() < a.getTime() ? b : a));
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The bounded all-capped idle delay: clamp(earliestResetAt − now) into
|
|
154
|
+
* [MIN_IDLE_MS, maxMs]. NEVER 0 and NEVER negative, so a null / elapsed / unknown
|
|
155
|
+
* reset cannot collapse the hold into a tight re-dispatch loop (invariant 4). The two
|
|
156
|
+
* agent-turn call sites (proactive turn-start + reactive 429) feed allCapped's
|
|
157
|
+
* earliestResetAt through this before returning continueDelayMs.
|
|
158
|
+
*/
|
|
159
|
+
export function computeIdleDelayMs(earliestResetAt: Date, now: Date, maxMs: number): number {
|
|
160
|
+
const delta = earliestResetAt.getTime() - now.getTime();
|
|
161
|
+
return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// --- P3 reactive-rotation boundedness (Finding 1). The reactive 429 catch, on a
|
|
165
|
+
// LIVE failover candidate, normally returns continueDelayMs:0 (skip the hold and
|
|
166
|
+
// re-dispatch NOW — the just-served account is cooling so the next rank cannot
|
|
167
|
+
// re-pick it). Two independent second-order faults can break that "cannot
|
|
168
|
+
// re-pick it" premise and turn the 0-delay re-dispatch into a hot loop, so this
|
|
169
|
+
// pure decision applies two backstops before allowing the 0.
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Slow-retry floor (Finding 1a) for the double-fault where the just-served
|
|
173
|
+
* account's cooldown write could NOT be confirmed persisted: the next proactive
|
|
174
|
+
* rank might re-select the same capped account (stale-low cached usedPercent, not
|
|
175
|
+
* cooling). A positive floor degrades that to a SLOW retry instead of a
|
|
176
|
+
* model-paced hot loop. A few seconds — long enough to break the loop, short
|
|
177
|
+
* enough to stay responsive once the transient write fault clears.
|
|
178
|
+
*/
|
|
179
|
+
export const REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5_000; // 5s
|
|
180
|
+
/**
|
|
181
|
+
* Circuit-breaker slack (Finding 1b) added to the connected-account count to bound
|
|
182
|
+
* consecutive reactive failovers. Each legitimate failover cools one account, so
|
|
183
|
+
* after at most N failovers every account is capped and the reactive path takes the
|
|
184
|
+
* all-capped idle instead; the +margin absorbs benign re-ranks (a connector-covering
|
|
185
|
+
* pick order) without tripping the breaker in normal use.
|
|
186
|
+
*/
|
|
187
|
+
export const REACTIVE_ROTATION_MARGIN = 2;
|
|
188
|
+
/**
|
|
189
|
+
* The FIXED positive idle the reactive path falls to once consecutive reactive
|
|
190
|
+
* failovers exceed the bound (a double-fault where cooldown writes are not sticking
|
|
191
|
+
* AND the 429s carry no usage headers, so the same account keeps getting re-picked).
|
|
192
|
+
* Mirrors the all-capped floor — a mandatory, bounded hold, never another 0-delay
|
|
193
|
+
* re-dispatch — so the loop can never hammer the capped backend + DB (invariant 4).
|
|
194
|
+
*/
|
|
195
|
+
export const REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS; // 60s
|
|
196
|
+
|
|
197
|
+
/** The reactive-resume shape returned to the goal-continuation caller. */
|
|
198
|
+
export type ReactiveRotationResume = { continueDelayMs: number; idleUntilReset: boolean };
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Decide the continueDelayMs for a reactive 429 failover onto a LIVE candidate,
|
|
202
|
+
* bounding the (otherwise 0-delay) re-dispatch against two second-order faults:
|
|
203
|
+
*
|
|
204
|
+
* 1. Circuit breaker (1b): if the consecutive reactive-failover streak (prior
|
|
205
|
+
* rotated 429 failovers since the last successful turn, PLUS this one) exceeds
|
|
206
|
+
* `connectedAccountCount + REACTIVE_ROTATION_MARGIN`, fall to a FIXED positive,
|
|
207
|
+
* MANDATORY idle. Covers the double-fault (cooldown write not persisted + a
|
|
208
|
+
* header-less cap 429) where the same capped account is re-picked every turn.
|
|
209
|
+
* 2. Persistence-fault floor (1a): if the just-served account's cooldown could NOT
|
|
210
|
+
* be confirmed persisted, use a positive floor (slow retry) instead of 0.
|
|
211
|
+
*
|
|
212
|
+
* Otherwise (a confirmed cooldown + an in-bounds streak) returns the unchanged
|
|
213
|
+
* 0-delay fast re-dispatch — the happy single-rotation path is byte-identical.
|
|
214
|
+
* Pure so the boundedness contract is unit-testable without a worker/db env.
|
|
215
|
+
*/
|
|
216
|
+
export function computeReactiveRotationResume(args: {
|
|
217
|
+
cooldownPersisted: boolean;
|
|
218
|
+
priorConsecutiveRotations: number; // reactive rotated failovers since last success (excludes this one)
|
|
219
|
+
connectedAccountCount: number;
|
|
220
|
+
}): ReactiveRotationResume {
|
|
221
|
+
const streak = args.priorConsecutiveRotations + 1; // include the failover about to be published
|
|
222
|
+
const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
|
|
223
|
+
if (streak > bound) {
|
|
224
|
+
return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
|
|
225
|
+
}
|
|
226
|
+
if (!args.cooldownPersisted) {
|
|
227
|
+
return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
|
|
228
|
+
}
|
|
229
|
+
return { continueDelayMs: 0, idleUntilReset: false };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* THE pure rotation ranker. `accounts` arrives in stable created_at order
|
|
234
|
+
* (listCodexAccountStatuses), which deterministically breaks ranking ties.
|
|
235
|
+
*/
|
|
236
|
+
export function chooseRotationActive(args: {
|
|
237
|
+
rotationStrategy: CodexRotationStrategy;
|
|
238
|
+
activeCredentialId: string | null;
|
|
239
|
+
priorCredentialId: string | null;
|
|
240
|
+
accounts: CodexAccountStatus[];
|
|
241
|
+
nearExhaustionPct: number;
|
|
242
|
+
now: Date;
|
|
243
|
+
// P4 (Part B): the connectors the leaving session has access to (the leaving
|
|
244
|
+
// account's cached connector set, used as the "session needs these" proxy).
|
|
245
|
+
// Optional + defaults to [] ⇒ when absent/empty the ranker is BYTE-IDENTICAL to
|
|
246
|
+
// P3 (every account trivially covers → Tier 1 == all eligibles). Self-gating.
|
|
247
|
+
usedConnectors?: string[];
|
|
248
|
+
}): RotationDecision {
|
|
249
|
+
const { rotationStrategy, activeCredentialId, priorCredentialId, accounts, nearExhaustionPct, now } = args;
|
|
250
|
+
const usedConnectors = args.usedConnectors ?? [];
|
|
251
|
+
|
|
252
|
+
if (accounts.length === 0) {
|
|
253
|
+
return { kind: "none" };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const eligibles = accounts.filter((acct) => eligible(acct, nearExhaustionPct, now));
|
|
257
|
+
|
|
258
|
+
// Healthy-active fast path (minimal churn, all strategies): a rotation-enabled
|
|
259
|
+
// session whose active account is still eligible does NOT rotate — no pointer
|
|
260
|
+
// move, no switch event. Steady-state stays as cheap as a non-rotation turn
|
|
261
|
+
// save the in-memory ranking. (Skipped for round_robin/drain which anchor on
|
|
262
|
+
// the prior account, but most_remaining is the default + correctness path.)
|
|
263
|
+
const activeRow = activeCredentialId ? accounts.find((acct) => acct.id === activeCredentialId) ?? null : null;
|
|
264
|
+
|
|
265
|
+
const decide = (chosen: CodexAccountStatus | undefined): RotationDecision => {
|
|
266
|
+
if (!chosen) {
|
|
267
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
|
|
268
|
+
}
|
|
269
|
+
// P4: surface the dropped-connector note when the chosen account doesn't cover
|
|
270
|
+
// the session's used connectors (a Tier-2/unknown failover pick). Empty ⇒ omit.
|
|
271
|
+
const dropped = droppedConnectorsFor(chosen, usedConnectors);
|
|
272
|
+
return {
|
|
273
|
+
kind: "active",
|
|
274
|
+
credentialId: chosen.id,
|
|
275
|
+
moved: chosen.id !== activeCredentialId,
|
|
276
|
+
...(dropped.length > 0 ? { droppedConnectors: dropped } : {}),
|
|
277
|
+
};
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
if (rotationStrategy === "round_robin") {
|
|
281
|
+
// Next eligible AFTER the prior account in list order (wrap around). When the
|
|
282
|
+
// prior account isn't found, start from the head.
|
|
283
|
+
if (eligibles.length === 0) {
|
|
284
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
|
|
285
|
+
}
|
|
286
|
+
const priorIdx = priorCredentialId ? accounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
|
|
287
|
+
const ordered = priorIdx >= 0
|
|
288
|
+
? [...accounts.slice(priorIdx + 1), ...accounts.slice(0, priorIdx + 1)]
|
|
289
|
+
: accounts;
|
|
290
|
+
const chosen = ordered.find((acct) => eligible(acct, nearExhaustionPct, now));
|
|
291
|
+
return decide(chosen);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
if (rotationStrategy === "drain_then_next") {
|
|
295
|
+
// Stay on the prior account while it is eligible (drain it), else first eligible.
|
|
296
|
+
const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : undefined;
|
|
297
|
+
if (priorRow && eligible(priorRow, nearExhaustionPct, now)) {
|
|
298
|
+
return decide(priorRow);
|
|
299
|
+
}
|
|
300
|
+
return decide(eligibles[0]);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// most_remaining (default + the correctness path).
|
|
304
|
+
// Healthy-active fast path UNCHANGED (no-thrash): a session whose active account
|
|
305
|
+
// is still eligible never switches — not even for connector coverage. Coverage
|
|
306
|
+
// matters ONLY on the must-move branch below.
|
|
307
|
+
if (activeRow && eligible(activeRow, nearExhaustionPct, now)) {
|
|
308
|
+
return { kind: "active", credentialId: activeRow.id, moved: false };
|
|
309
|
+
}
|
|
310
|
+
// Active is capped/near-cap/cooling/missing → must move. Two-tier coverage pick
|
|
311
|
+
// over the SAME eligible set (prefer-not-require):
|
|
312
|
+
// Tier 1 — eligibles whose connector set COVERS the session's used set.
|
|
313
|
+
// Tier 2 — ALL eligibles, only if Tier 1 is empty (failover fully preserved).
|
|
314
|
+
// Within the chosen tier, max bindingRemaining, ties broken by list (created_at)
|
|
315
|
+
// order via a stable reduce — byte-identical to P3 when usedConnectors is empty
|
|
316
|
+
// (Tier 1 == all eligibles).
|
|
317
|
+
const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
|
|
318
|
+
const pool = covering.length > 0 ? covering : eligibles;
|
|
319
|
+
const chosen = pool.reduce<CodexAccountStatus | undefined>((best, acct) => {
|
|
320
|
+
if (!best) {
|
|
321
|
+
return acct;
|
|
322
|
+
}
|
|
323
|
+
return bindingRemaining(acct) > bindingRemaining(best) ? acct : best;
|
|
324
|
+
}, undefined);
|
|
325
|
+
return decide(chosen);
|
|
326
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { ResourceRef, SessionStatus, ToolRef } from "@opengeni/contracts";
|
|
2
|
+
|
|
3
|
+
export {
|
|
4
|
+
mergeResourceRefs,
|
|
5
|
+
mergeToolRefs,
|
|
6
|
+
reasoningEffortForMetadata as reasoningEffortForSession,
|
|
7
|
+
} from "@opengeni/contracts";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Refuse to revive a terminal reusable session on a scheduled fire. Mirrors the
|
|
11
|
+
* API-level guard in apps/api/src/domain/sessions.ts (post a user message ->
|
|
12
|
+
* 409 when cancelled): "cancelled" is the one terminal state, an explicit user
|
|
13
|
+
* act, so a recurring task must NOT silently resurrect and re-bill it. Failed
|
|
14
|
+
* and idle sessions stay revivable (talking to a session is how it resumes), so
|
|
15
|
+
* only "cancelled" is rejected. Throwing inside the FOR UPDATE locked-update
|
|
16
|
+
* callback aborts the whole append+enqueue transaction atomically, and the
|
|
17
|
+
* dispatch catch marks the scheduled_task_run "failed" rather than dispatched.
|
|
18
|
+
*/
|
|
19
|
+
export function assertReusableSessionRevivable(status: SessionStatus): void {
|
|
20
|
+
if (status === "cancelled") {
|
|
21
|
+
throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function scheduledUserMessagePayload(prompt: string, resources: ResourceRef[], tools: ToolRef[], taskId: string, runId: string): Record<string, unknown> {
|
|
26
|
+
return {
|
|
27
|
+
text: prompt,
|
|
28
|
+
scheduledTaskId: taskId,
|
|
29
|
+
scheduledTaskRunId: runId,
|
|
30
|
+
...(resources.length ? { resources } : {}),
|
|
31
|
+
...(tools.length ? { tools } : {}),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function workflowIdForSession(sessionId: string): string {
|
|
36
|
+
return `session-${sessionId}`;
|
|
37
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import {
|
|
2
|
+
applyContextCompaction,
|
|
3
|
+
getActiveSessionHistoryItems,
|
|
4
|
+
type Database,
|
|
5
|
+
} from "@opengeni/db";
|
|
6
|
+
import {
|
|
7
|
+
buildCompactionMessages,
|
|
8
|
+
buildSummaryItem,
|
|
9
|
+
planCompaction,
|
|
10
|
+
summarizeForCompaction,
|
|
11
|
+
type CompactionItem,
|
|
12
|
+
} from "@opengeni/runtime";
|
|
13
|
+
import {
|
|
14
|
+
contextInputBudgetTokens,
|
|
15
|
+
resolveContextCompactionMode,
|
|
16
|
+
type Settings,
|
|
17
|
+
} from "@opengeni/config";
|
|
18
|
+
|
|
19
|
+
export type MaybeCompactResult =
|
|
20
|
+
| { compacted: false; reason: string }
|
|
21
|
+
| { compacted: true; supersededFrom: number; summaryPosition: number; hardForced: boolean };
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Pre-turn client-side context compaction (the Azure path).
|
|
25
|
+
*
|
|
26
|
+
* Runs BEFORE the model call when the resolved compaction mode is "client".
|
|
27
|
+
* Reads the active history rows + the last turn's actual input tokens, asks the
|
|
28
|
+
* pure planner whether/where to compact, and — when it should — summarizes the
|
|
29
|
+
* orphan-safe prefix into one plain user message via a model call and writes it
|
|
30
|
+
* with applyContextCompaction (supersede prefix rows, insert the summary).
|
|
31
|
+
*
|
|
32
|
+
* Best-effort by design: any failure (planner says no, summarize fails, DB
|
|
33
|
+
* hiccup) returns without compacting and never throws — the turn proceeds with
|
|
34
|
+
* the un-compacted history. The read path's existing sanitizer keeps that safe.
|
|
35
|
+
*
|
|
36
|
+
* The boundary the planner picks is the start of a kept user-message turn, so
|
|
37
|
+
* no tool-call pair straddles the cut. The summary row is inserted at a
|
|
38
|
+
* fractional position a half-step before the kept tail (boundaryPosition - 0.5),
|
|
39
|
+
* which sorts ahead of the tail without overwriting any real prefix row, so the
|
|
40
|
+
* active read path returns [summary, ...recent tail] in order and the full
|
|
41
|
+
* superseded prefix survives untouched as an audit trail.
|
|
42
|
+
*/
|
|
43
|
+
export type CompactionSummarizer = (
|
|
44
|
+
settings: Settings,
|
|
45
|
+
messages: { system: string; user: string },
|
|
46
|
+
) => Promise<string | null>;
|
|
47
|
+
|
|
48
|
+
export async function maybeCompactContext(
|
|
49
|
+
db: Database,
|
|
50
|
+
settings: Settings,
|
|
51
|
+
scope: { accountId: string; workspaceId: string; sessionId: string; turnId?: string | null },
|
|
52
|
+
lastInputTokens: number | null,
|
|
53
|
+
// Injectable for tests; defaults to the real provider-aware model call.
|
|
54
|
+
summarize: CompactionSummarizer = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: s.contextSummaryMaxTokens }),
|
|
55
|
+
// Operator-forced (the /compact command): bypass the budget trigger and
|
|
56
|
+
// compact now if there is anything to summarize. Structural guards still hold.
|
|
57
|
+
options: { force?: boolean } = {},
|
|
58
|
+
): Promise<MaybeCompactResult> {
|
|
59
|
+
if (resolveContextCompactionMode(settings) !== "client") {
|
|
60
|
+
return { compacted: false, reason: "mode_not_client" };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const active = await getActiveSessionHistoryItems(db, scope.workspaceId, scope.sessionId);
|
|
64
|
+
if (active.length === 0) {
|
|
65
|
+
return { compacted: false, reason: "no_history" };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const items = active.map((row) => row.item) as CompactionItem[];
|
|
69
|
+
const plan = planCompaction({
|
|
70
|
+
items,
|
|
71
|
+
lastInputTokens,
|
|
72
|
+
inputBudgetTokens: contextInputBudgetTokens(settings),
|
|
73
|
+
softFraction: settings.contextCompactSoftFraction,
|
|
74
|
+
hardFraction: settings.contextCompactHardFraction,
|
|
75
|
+
keepRecentTokens: settings.contextKeepRecentTokens,
|
|
76
|
+
...(options.force ? { force: true } : {}),
|
|
77
|
+
});
|
|
78
|
+
if (!plan.shouldCompact) {
|
|
79
|
+
return { compacted: false, reason: plan.reason };
|
|
80
|
+
}
|
|
81
|
+
if (plan.hardForced) {
|
|
82
|
+
// At/over the hard ceiling (hardFraction*B): the session is past the soft
|
|
83
|
+
// trigger by a wide margin (often because last_input_tokens was stale-low
|
|
84
|
+
// and the actual history overflowed). Logged distinctly so the hard-force
|
|
85
|
+
// backstop firing is observable in production.
|
|
86
|
+
console.warn(
|
|
87
|
+
`context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const messages = buildCompactionMessages(plan);
|
|
92
|
+
const summaryBody = await summarize(settings, messages);
|
|
93
|
+
if (!summaryBody) {
|
|
94
|
+
return { compacted: false, reason: "summarize_failed" };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Boundary is an index into the active rows; map to absolute positions. The
|
|
98
|
+
// kept tail starts at the boundary row's position; the summary takes a
|
|
99
|
+
// FRACTIONAL position a half-step before it. Positions are whole numbers, so
|
|
100
|
+
// boundaryPosition - 0.5 sorts immediately ahead of the kept tail and behind
|
|
101
|
+
// the last superseded prefix row while colliding with NO real row — the
|
|
102
|
+
// earlier `boundaryPosition - 1` always landed on (and overwrote) the real
|
|
103
|
+
// prefix row there.
|
|
104
|
+
const boundaryRow = active[plan.boundaryIndex];
|
|
105
|
+
if (!boundaryRow) {
|
|
106
|
+
return { compacted: false, reason: "boundary_out_of_range" };
|
|
107
|
+
}
|
|
108
|
+
const boundaryPosition = boundaryRow.position;
|
|
109
|
+
if (boundaryPosition <= 0) {
|
|
110
|
+
return { compacted: false, reason: "boundary_at_origin" };
|
|
111
|
+
}
|
|
112
|
+
const summaryPosition = boundaryPosition - 0.5;
|
|
113
|
+
|
|
114
|
+
await applyContextCompaction(db, {
|
|
115
|
+
accountId: scope.accountId,
|
|
116
|
+
workspaceId: scope.workspaceId,
|
|
117
|
+
sessionId: scope.sessionId,
|
|
118
|
+
turnId: scope.turnId ?? null,
|
|
119
|
+
boundaryPosition,
|
|
120
|
+
summaryPosition,
|
|
121
|
+
summaryItem: buildSummaryItem(summaryBody),
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
return { compacted: true, supersededFrom: boundaryPosition, summaryPosition, hardForced: plan.hardForced };
|
|
125
|
+
}
|