@opengeni/worker-bundle 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -14
- package/dist/activities/agent-turn/errors.d.ts +3 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/turn-context.d.ts +5 -0
- package/dist/activities/goals.d.ts +1 -17
- package/dist/activities/types.d.ts +1 -1
- package/dist/{activities-control-BNPTRTOE.js → activities-control-XZAIMFQJ.js} +23 -7
- package/dist/activities-control-XZAIMFQJ.js.map +1 -0
- package/dist/{activities-turn-5LGZOSVU.js → activities-turn-4EEVNZCC.js} +97 -5
- package/dist/activities-turn-4EEVNZCC.js.map +1 -0
- package/dist/{chunk-LJX7Z2KP.js → chunk-OT64XARC.js} +38 -162
- package/dist/chunk-OT64XARC.js.map +1 -0
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +1 -1
- package/package.json +21 -21
- package/src/activities/agent-run-admission.ts +1 -98
- package/src/activities/agent-turn/claim.ts +11 -0
- package/src/activities/agent-turn/compaction-prep.ts +15 -0
- package/src/activities/agent-turn/errors.ts +30 -15
- package/src/activities/agent-turn/failure-settlement.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/stream-attempt.ts +10 -0
- package/src/activities/agent-turn/tool-environment.ts +9 -2
- package/src/activities/agent-turn/turn-context.ts +4 -0
- package/src/activities/goals.ts +17 -113
- package/src/activities/scheduled-tasks.ts +18 -3
- package/src/activities/types.ts +1 -0
- package/src/index.ts +0 -2
- package/dist/activities-control-BNPTRTOE.js.map +0 -1
- package/dist/activities-turn-5LGZOSVU.js.map +0 -1
- package/dist/chunk-LJX7Z2KP.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -54,26 +54,26 @@
|
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@llamaindex/liteparse": "2.14.2",
|
|
57
|
-
"@opengeni/agent-proto": "1.0
|
|
58
|
-
"@opengeni/capabilities": "1.0
|
|
59
|
-
"@opengeni/codemode": "1.0
|
|
60
|
-
"@opengeni/codex": "1.0
|
|
61
|
-
"@opengeni/config": "1.0
|
|
62
|
-
"@opengeni/contracts": "1.0
|
|
63
|
-
"@opengeni/core": "1.0
|
|
64
|
-
"@opengeni/db": "1.0
|
|
65
|
-
"@opengeni/documents": "1.0
|
|
66
|
-
"@opengeni/events": "1.0
|
|
67
|
-
"@opengeni/github": "1.0
|
|
68
|
-
"@opengeni/interaction": "1.0
|
|
69
|
-
"@opengeni/jev": "1.0
|
|
70
|
-
"@opengeni/network": "1.0
|
|
71
|
-
"@opengeni/observability": "1.0
|
|
72
|
-
"@opengeni/runtime": "1.0
|
|
73
|
-
"@opengeni/sdk": "1.0
|
|
74
|
-
"@opengeni/storage": "1.0
|
|
75
|
-
"@opengeni/tool-gateway": "1.0
|
|
76
|
-
"@opengeni/xai-subscription": "1.0
|
|
57
|
+
"@opengeni/agent-proto": "1.1.0",
|
|
58
|
+
"@opengeni/capabilities": "1.1.0",
|
|
59
|
+
"@opengeni/codemode": "1.1.0",
|
|
60
|
+
"@opengeni/codex": "1.1.0",
|
|
61
|
+
"@opengeni/config": "1.1.0",
|
|
62
|
+
"@opengeni/contracts": "1.1.0",
|
|
63
|
+
"@opengeni/core": "1.1.0",
|
|
64
|
+
"@opengeni/db": "1.1.0",
|
|
65
|
+
"@opengeni/documents": "1.1.0",
|
|
66
|
+
"@opengeni/events": "1.1.0",
|
|
67
|
+
"@opengeni/github": "1.1.0",
|
|
68
|
+
"@opengeni/interaction": "1.1.0",
|
|
69
|
+
"@opengeni/jev": "1.1.0",
|
|
70
|
+
"@opengeni/network": "1.1.0",
|
|
71
|
+
"@opengeni/observability": "1.1.0",
|
|
72
|
+
"@opengeni/runtime": "1.1.0",
|
|
73
|
+
"@opengeni/sdk": "1.1.0",
|
|
74
|
+
"@opengeni/storage": "1.1.0",
|
|
75
|
+
"@opengeni/tool-gateway": "1.1.0",
|
|
76
|
+
"@opengeni/xai-subscription": "1.1.0",
|
|
77
77
|
"@temporalio/activity": "^1.17.0",
|
|
78
78
|
"@temporalio/client": "^1.17.0",
|
|
79
79
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -1,98 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { modelFundingForAdmission } from "@opengeni/core";
|
|
3
|
-
import {
|
|
4
|
-
checkWorkspaceAllowance,
|
|
5
|
-
getBillingBalance,
|
|
6
|
-
isCodexBilledTurn,
|
|
7
|
-
sumUsageQuantity,
|
|
8
|
-
} from "@opengeni/db";
|
|
9
|
-
import type { ControlActivityServices } from "./types";
|
|
10
|
-
|
|
11
|
-
export type AgentRunAdmissionDenial =
|
|
12
|
-
| "insufficient_credits"
|
|
13
|
-
| "allowance_exhausted"
|
|
14
|
-
| "monthly_model_cost_limit"
|
|
15
|
-
| "monthly_agent_run_limit";
|
|
16
|
-
|
|
17
|
-
/** One worker-side admission boundary for service-authored agent runs. */
|
|
18
|
-
export async function agentRunAdmissionDenial(
|
|
19
|
-
services: Pick<ControlActivityServices, "db" | "entitlements"> & { settings: Settings },
|
|
20
|
-
input: {
|
|
21
|
-
accountId: string;
|
|
22
|
-
workspaceId: string;
|
|
23
|
-
model: string;
|
|
24
|
-
requestedAgentRuns: number;
|
|
25
|
-
/** The accepted work's causal human, not the scheduler/service caller. */
|
|
26
|
-
initiatingHumanSubjectId?: string | null;
|
|
27
|
-
},
|
|
28
|
-
): Promise<AgentRunAdmissionDenial | null> {
|
|
29
|
-
const codexBilled = await isCodexBilledTurn({
|
|
30
|
-
db: services.db,
|
|
31
|
-
settings: services.settings,
|
|
32
|
-
workspaceId: input.workspaceId,
|
|
33
|
-
model: input.model,
|
|
34
|
-
});
|
|
35
|
-
const externallyBilled = modelFundingForAdmission(
|
|
36
|
-
services.settings,
|
|
37
|
-
input.model,
|
|
38
|
-
codexBilled,
|
|
39
|
-
).fundedWithoutCredits;
|
|
40
|
-
if (
|
|
41
|
-
!externallyBilled &&
|
|
42
|
-
(services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")
|
|
43
|
-
) {
|
|
44
|
-
if (services.entitlements) {
|
|
45
|
-
const decision = await services.entitlements.admitRun({
|
|
46
|
-
accountId: input.accountId,
|
|
47
|
-
workspaceId: input.workspaceId,
|
|
48
|
-
action: "agent_run:create",
|
|
49
|
-
quantity: input.requestedAgentRuns,
|
|
50
|
-
});
|
|
51
|
-
if (!decision.allowed) return "insufficient_credits";
|
|
52
|
-
} else {
|
|
53
|
-
const balance = await getBillingBalance(services.db, input.accountId);
|
|
54
|
-
if (balance.balanceMicros <= 0) return "insufficient_credits";
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
if (!externallyBilled) {
|
|
58
|
-
const refusal = await checkWorkspaceAllowance(services.db, {
|
|
59
|
-
accountId: input.accountId,
|
|
60
|
-
workspaceId: input.workspaceId,
|
|
61
|
-
subjectId: input.initiatingHumanSubjectId ?? null,
|
|
62
|
-
});
|
|
63
|
-
if (refusal) return refusal.code;
|
|
64
|
-
}
|
|
65
|
-
if (
|
|
66
|
-
services.settings.usageLimitsMode !== "static" &&
|
|
67
|
-
services.settings.usageLimitsMode !== "managed"
|
|
68
|
-
) {
|
|
69
|
-
return null;
|
|
70
|
-
}
|
|
71
|
-
const limits = configuredStaticUsageLimits(services.settings);
|
|
72
|
-
if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
|
|
73
|
-
const used = await sumUsageQuantity(services.db, {
|
|
74
|
-
accountId: input.accountId,
|
|
75
|
-
eventType: "model.cost",
|
|
76
|
-
since: startOfUtcMonth(),
|
|
77
|
-
});
|
|
78
|
-
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
79
|
-
return "monthly_model_cost_limit";
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
83
|
-
const used = await sumUsageQuantity(services.db, {
|
|
84
|
-
workspaceId: input.workspaceId,
|
|
85
|
-
eventType: "agent_run.created",
|
|
86
|
-
since: startOfUtcMonth(),
|
|
87
|
-
});
|
|
88
|
-
if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
89
|
-
return "monthly_agent_run_limit";
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
return null;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function startOfUtcMonth(): Date {
|
|
96
|
-
const now = new Date();
|
|
97
|
-
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
98
|
-
}
|
|
1
|
+
export { agentRunAdmissionDenial, type AgentRunAdmissionDenial } from "@opengeni/core";
|
|
@@ -60,6 +60,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
|
|
|
60
60
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
61
61
|
import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
62
62
|
import { turnCredentialRestriction } from "./credential-restriction";
|
|
63
|
+
import { readProviderRecoveryObservation } from "./provider-recovery-metrics";
|
|
63
64
|
|
|
64
65
|
import {
|
|
65
66
|
credentialSubjectIdForTurnInitiator,
|
|
@@ -237,6 +238,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
237
238
|
attempt.dispatchId = dispatchId;
|
|
238
239
|
attempt.executionGeneration = turn.executionGeneration;
|
|
239
240
|
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
241
|
+
attempt.providerRecoveryObservation = readProviderRecoveryObservation(turn.metadata ?? {});
|
|
240
242
|
const authRecovery = turn.metadata?.claudeAuthRecovery;
|
|
241
243
|
attempt.claudeAuthRecovery =
|
|
242
244
|
authRecovery &&
|
|
@@ -391,6 +393,11 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
391
393
|
latencyMode: turn.latencyMode,
|
|
392
394
|
},
|
|
393
395
|
);
|
|
396
|
+
// The durable same-turn recovery lane owns provider retries. Hidden SDK
|
|
397
|
+
// retries multiply that budget and keep the UI looking active during backoff.
|
|
398
|
+
// Apply before configuring/resolving clients so main, compaction and title
|
|
399
|
+
// requests all share this policy; standalone runtime consumers keep theirs.
|
|
400
|
+
capabilitySettings = { ...capabilitySettings, openaiMaxRetries: 0 };
|
|
394
401
|
runtime.configure(capabilitySettings);
|
|
395
402
|
const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
|
|
396
403
|
capabilitySettings,
|
|
@@ -402,6 +409,10 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
402
409
|
},
|
|
403
410
|
);
|
|
404
411
|
const turnExecutionPolicy = verifiedExecutionPolicy.policy;
|
|
412
|
+
attempt.modelMetricRoute = {
|
|
413
|
+
provider: turnExecutionPolicy.providerId,
|
|
414
|
+
model: turnExecutionPolicy.productModelId,
|
|
415
|
+
};
|
|
405
416
|
assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
|
|
406
417
|
const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
|
|
407
418
|
billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
processCompactionModelUsageEvent,
|
|
47
47
|
} from "./model-usage";
|
|
48
48
|
import { waitForTurnOperation } from "./sandbox-provision";
|
|
49
|
+
import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
|
|
49
50
|
|
|
50
51
|
import type { ClaimTurnOk } from "./claim";
|
|
51
52
|
import type { GovernanceModelOk } from "./governance-model";
|
|
@@ -330,6 +331,20 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
|
|
|
330
331
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
|
|
331
332
|
};
|
|
332
333
|
const publishCompactionOutcomeEvents = async (events: SessionEvent[]) => {
|
|
334
|
+
if (events.some((event) => event.type === "session.context.compacted")) {
|
|
335
|
+
// The summary and recovery reset already committed under the attempt
|
|
336
|
+
// fence. Skipped compaction does not prove successful model progress.
|
|
337
|
+
attempt.providerRecoveryCount = 0;
|
|
338
|
+
if (attempt.providerRecoveryObservation) {
|
|
339
|
+
recordProviderRecoveryOutcome(observability, {
|
|
340
|
+
route: attempt.modelMetricRoute,
|
|
341
|
+
cause: attempt.providerRecoveryObservation.cause,
|
|
342
|
+
outcome: "recovered",
|
|
343
|
+
elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
|
|
344
|
+
});
|
|
345
|
+
attempt.providerRecoveryObservation = undefined;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
333
348
|
// `compaction.started` was already fanout via publishCompactionLiveEvents.
|
|
334
349
|
await publishDurableSessionEvents(
|
|
335
350
|
bus,
|
|
@@ -100,6 +100,11 @@ export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_M
|
|
|
100
100
|
* alone would spend every automatic recovery before the window resets.
|
|
101
101
|
*/
|
|
102
102
|
export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
|
|
103
|
+
/** Positive-only spread: never shorten the provider's minimum delay. */
|
|
104
|
+
export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
|
|
105
|
+
const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
|
|
106
|
+
return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
|
|
107
|
+
}
|
|
103
108
|
export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
104
109
|
|
|
105
110
|
export class PostCompactionContinuationEmptyError extends Error {
|
|
@@ -126,6 +131,7 @@ export function providerRecoveryResult(input: {
|
|
|
126
131
|
failureCode: string | undefined;
|
|
127
132
|
attemptNumber: number;
|
|
128
133
|
retryAfterMs?: number | null;
|
|
134
|
+
jitterSample?: number;
|
|
129
135
|
}): ProviderRecoveryResult {
|
|
130
136
|
if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
|
|
131
137
|
return {
|
|
@@ -171,7 +177,13 @@ export function providerRecoveryResult(input: {
|
|
|
171
177
|
: PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
172
178
|
return {
|
|
173
179
|
status: "recovering",
|
|
174
|
-
continueDelayMs
|
|
180
|
+
continueDelayMs:
|
|
181
|
+
continueDelayMs +
|
|
182
|
+
(input.failureCode === "provider_rate_limited" ||
|
|
183
|
+
input.failureCode === "provider_unavailable" ||
|
|
184
|
+
input.failureCode === "upstream_connectivity_unavailable"
|
|
185
|
+
? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
|
|
186
|
+
: 0),
|
|
175
187
|
};
|
|
176
188
|
}
|
|
177
189
|
|
|
@@ -1086,19 +1098,28 @@ function isProviderSafetyRefusal(error: unknown): boolean {
|
|
|
1086
1098
|
return providerSafetyRefusalDiagnostic(error) !== undefined;
|
|
1087
1099
|
}
|
|
1088
1100
|
|
|
1101
|
+
/** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
|
|
1102
|
+
function providerHttpStatus(error: unknown): number | undefined {
|
|
1103
|
+
let current = error;
|
|
1104
|
+
const seen = new Set<unknown>();
|
|
1105
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1106
|
+
if (seen.has(current)) break;
|
|
1107
|
+
seen.add(current);
|
|
1108
|
+
const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
|
|
1109
|
+
const status = Number(value.status ?? value.statusCode);
|
|
1110
|
+
if (Number.isInteger(status) && status >= 100 && status < 600) return status;
|
|
1111
|
+
current = value.cause;
|
|
1112
|
+
}
|
|
1113
|
+
return undefined;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1089
1116
|
export function isTransientProviderError(error: unknown): boolean {
|
|
1090
1117
|
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1091
1118
|
return error.category === "unavailable";
|
|
1092
1119
|
}
|
|
1093
1120
|
// A semantic refusal can arrive inside a 5xx transport envelope.
|
|
1094
1121
|
if (isProviderSafetyRefusal(error)) return false;
|
|
1095
|
-
const status =
|
|
1096
|
-
typeof error === "object" && error !== null
|
|
1097
|
-
? Number(
|
|
1098
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
1099
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
1100
|
-
)
|
|
1101
|
-
: undefined;
|
|
1122
|
+
const status = providerHttpStatus(error);
|
|
1102
1123
|
// A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
|
|
1103
1124
|
// (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
|
|
1104
1125
|
// a request fault that must NOT auto-retry — even if its body happens to read like
|
|
@@ -1399,13 +1420,7 @@ function baseAgentRunFailurePayload(
|
|
|
1399
1420
|
};
|
|
1400
1421
|
}
|
|
1401
1422
|
const message = error instanceof Error ? error.message : String(error);
|
|
1402
|
-
const status =
|
|
1403
|
-
typeof error === "object" && error !== null
|
|
1404
|
-
? Number(
|
|
1405
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
1406
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
1407
|
-
)
|
|
1408
|
-
: undefined;
|
|
1423
|
+
const status = providerHttpStatus(error);
|
|
1409
1424
|
const code =
|
|
1410
1425
|
typeof error === "object" && error !== null && "code" in error
|
|
1411
1426
|
? String((error as { code?: unknown }).code)
|
|
@@ -100,6 +100,7 @@ import type {
|
|
|
100
100
|
} from "./turn-context";
|
|
101
101
|
import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
|
|
102
102
|
import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
|
|
103
|
+
import { providerRecoveryCause, recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
|
|
103
104
|
|
|
104
105
|
export type TurnFailureDeps = {
|
|
105
106
|
error: unknown;
|
|
@@ -1995,6 +1996,7 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
|
|
|
1995
1996
|
failureCode: failure.code,
|
|
1996
1997
|
attemptNumber: nextProviderRecoveryCount,
|
|
1997
1998
|
retryAfterMs: providerRetryAfterMs(error),
|
|
1999
|
+
jitterSample: Math.random(),
|
|
1998
2000
|
});
|
|
1999
2001
|
const setupRecoveryExhausted =
|
|
2000
2002
|
earlyCommandStartUnavailable &&
|
|
@@ -2064,9 +2066,29 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
|
|
|
2064
2066
|
control.turnMetricOutcome = "recovering";
|
|
2065
2067
|
control.activityStatus = "recovering";
|
|
2066
2068
|
control.activityError = error;
|
|
2069
|
+
const recoveryCause = providerRecoveryCause(failure.code);
|
|
2070
|
+
if (recoveryCause) {
|
|
2071
|
+
recordProviderRecoveryOutcome(observability, {
|
|
2072
|
+
route: attempt.modelMetricRoute,
|
|
2073
|
+
cause: recoveryCause,
|
|
2074
|
+
outcome: "scheduled",
|
|
2075
|
+
delayMs: recoveryResult.continueDelayMs,
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
2067
2078
|
return claimedResult(recoveryResult);
|
|
2068
2079
|
}
|
|
2069
2080
|
failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
|
|
2081
|
+
const recoveryCause = providerRecoveryCause(failure.code);
|
|
2082
|
+
if (recoveryCause) {
|
|
2083
|
+
recordProviderRecoveryOutcome(observability, {
|
|
2084
|
+
route: attempt.modelMetricRoute,
|
|
2085
|
+
cause: recoveryCause,
|
|
2086
|
+
outcome: "exhausted",
|
|
2087
|
+
...(attempt.providerRecoveryObservation
|
|
2088
|
+
? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt }
|
|
2089
|
+
: {}),
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2070
2092
|
if (earlyRecoverableSetup) {
|
|
2071
2093
|
// Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
|
|
2072
2094
|
// through Temporal into exact-attempt workflow failure settlement.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
|
|
3
|
+
export type ProviderRecoveryObservation = {
|
|
4
|
+
startedAt: number;
|
|
5
|
+
cause: "rate_limited" | "unavailable" | "connectivity";
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export function providerRecoveryCause(code: unknown): ProviderRecoveryObservation["cause"] | null {
|
|
9
|
+
if (code === "provider_rate_limited") return "rate_limited";
|
|
10
|
+
if (code === "provider_unavailable") return "unavailable";
|
|
11
|
+
if (code === "upstream_connectivity_unavailable") return "connectivity";
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function readProviderRecoveryObservation(
|
|
16
|
+
metadata: Record<string, unknown>,
|
|
17
|
+
): ProviderRecoveryObservation | undefined {
|
|
18
|
+
const cause = providerRecoveryCause(metadata.providerRecoveryReason);
|
|
19
|
+
const startedAt =
|
|
20
|
+
typeof metadata.providerRecoveryStartedAt === "string"
|
|
21
|
+
? Date.parse(metadata.providerRecoveryStartedAt)
|
|
22
|
+
: Number.NaN;
|
|
23
|
+
return cause && Number.isFinite(startedAt) ? { cause, startedAt } : undefined;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Operational observations only; never retry authority, billing or diagnostics. */
|
|
27
|
+
export function recordProviderRecoveryOutcome(
|
|
28
|
+
observability: Observability,
|
|
29
|
+
input: {
|
|
30
|
+
route?: { provider: string; model: string } | undefined;
|
|
31
|
+
cause: ProviderRecoveryObservation["cause"];
|
|
32
|
+
outcome: "scheduled" | "recovered" | "exhausted";
|
|
33
|
+
delayMs?: number;
|
|
34
|
+
elapsedMs?: number;
|
|
35
|
+
},
|
|
36
|
+
): void {
|
|
37
|
+
if (!input.route) return;
|
|
38
|
+
const labels = { ...input.route, cause: input.cause, outcome: input.outcome };
|
|
39
|
+
try {
|
|
40
|
+
observability.incrementCounter({
|
|
41
|
+
name: "opengeni_model_recovery_total",
|
|
42
|
+
help: "Observed model recovery decisions and successful resumptions.",
|
|
43
|
+
labels,
|
|
44
|
+
});
|
|
45
|
+
if (input.delayMs !== undefined) {
|
|
46
|
+
observability.observeHistogram({
|
|
47
|
+
name: "opengeni_model_recovery_delay_seconds",
|
|
48
|
+
help: "Scheduled model recovery delay, including provider hints and jitter.",
|
|
49
|
+
labels: { ...input.route, cause: input.cause },
|
|
50
|
+
value: Math.max(0, input.delayMs) / 1_000,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
if (input.elapsedMs !== undefined) {
|
|
54
|
+
observability.observeHistogram({
|
|
55
|
+
name: "opengeni_model_recovery_duration_seconds",
|
|
56
|
+
help: "Elapsed recovery episode including backoff, preparation and model requests.",
|
|
57
|
+
labels,
|
|
58
|
+
value: Math.max(0, input.elapsedMs) / 1_000,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
} catch {
|
|
62
|
+
// Metrics cannot interrupt settlement or model progress.
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
updateSessionTitleWithEvent,
|
|
19
19
|
} from "@opengeni/db";
|
|
20
20
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
21
|
+
import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
|
|
21
22
|
import {
|
|
22
23
|
AssistantMessagePhaseTracker,
|
|
23
24
|
normalizeModelCallUsage,
|
|
@@ -1132,6 +1133,15 @@ export async function runTurnStreamAttempt(
|
|
|
1132
1133
|
]);
|
|
1133
1134
|
attempt.providerRecoveryCount = 0;
|
|
1134
1135
|
}
|
|
1136
|
+
if (attempt.providerRecoveryObservation) {
|
|
1137
|
+
recordProviderRecoveryOutcome(observability, {
|
|
1138
|
+
route: attempt.modelMetricRoute,
|
|
1139
|
+
cause: attempt.providerRecoveryObservation.cause,
|
|
1140
|
+
outcome: "recovered",
|
|
1141
|
+
elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
|
|
1142
|
+
});
|
|
1143
|
+
attempt.providerRecoveryObservation = undefined;
|
|
1144
|
+
}
|
|
1135
1145
|
const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
|
|
1136
1146
|
if (Array.isArray(rawStreamHistory)) {
|
|
1137
1147
|
// The completed image item is normally retained from its own
|
|
@@ -82,6 +82,7 @@ import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
|
82
82
|
import { SandboxChannelAService } from "@opengeni/runtime/sandbox";
|
|
83
83
|
import { sandboxRunAs } from "@opengeni/runtime";
|
|
84
84
|
import {
|
|
85
|
+
bundledSkillSelectionForAgentConfig,
|
|
85
86
|
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
86
87
|
resolveAgentToolFamilies,
|
|
87
88
|
type ResourceRef,
|
|
@@ -596,7 +597,11 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
596
597
|
session.firstPartyMcpPermissions,
|
|
597
598
|
linkedAuthority,
|
|
598
599
|
);
|
|
599
|
-
|
|
600
|
+
// Background-command tools need compute: the effective route of this turn,
|
|
601
|
+
// a managed sandbox or an attached Connected Machine, not the durable home.
|
|
602
|
+
const toolFamilies = resolveAgentToolFamilies(session.agent, {
|
|
603
|
+
sandboxAttached: (activeSandboxBackend ?? groupBoxBackend) !== "none",
|
|
604
|
+
});
|
|
600
605
|
const selectedFirstPartyMcpTools = toolFamilies.firstPartyTools(
|
|
601
606
|
allowedFirstPartyMcpToolsForSession(runSettings, session.firstPartyMcpTools),
|
|
602
607
|
);
|
|
@@ -669,7 +674,9 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
669
674
|
}),
|
|
670
675
|
]);
|
|
671
676
|
const bundledSkills = loadConfiguredBundledSkills({
|
|
672
|
-
|
|
677
|
+
// Rows that could not freeze the "none" default at create (scheduled
|
|
678
|
+
// generated sessions, pre-existing rows) get the same rule here.
|
|
679
|
+
bundledSkillIds: bundledSkillSelectionForAgentConfig(session.bundledSkillIds, session.agent),
|
|
673
680
|
firstPartyTools: selectedFirstPartyMcpTools,
|
|
674
681
|
videoGenerationEnabled:
|
|
675
682
|
skillConfiguration.defaultModelId !== null && skillConfiguration.enabledModelIds.length > 0,
|
|
@@ -68,6 +68,10 @@ export type AttemptIdentityState = {
|
|
|
68
68
|
triggerEventId: string | undefined;
|
|
69
69
|
executionGeneration: number;
|
|
70
70
|
providerRecoveryCount: number;
|
|
71
|
+
providerRecoveryObservation?:
|
|
72
|
+
| import("./provider-recovery-metrics").ProviderRecoveryObservation
|
|
73
|
+
| undefined;
|
|
74
|
+
modelMetricRoute?: { provider: string; model: string };
|
|
71
75
|
claudeAuthRecovery?: { credentialId: string; credentialVersion: number } | undefined;
|
|
72
76
|
modelRequestStarted: boolean;
|
|
73
77
|
redispatchesAtDispatch: number;
|
package/src/activities/goals.ts
CHANGED
|
@@ -1,27 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
allowedFirstPartyMcpToolsForSession,
|
|
3
|
-
configuredStaticUsageLimits,
|
|
4
|
-
isModelAvailableForNewSelection,
|
|
5
|
-
policyProviderIdForModel,
|
|
6
|
-
resolveModelProvider,
|
|
7
3
|
resolveTurnExecutionPolicyV1,
|
|
8
|
-
withCodexCatalogProvider,
|
|
9
|
-
withXaiSubscriptionCatalogProvider,
|
|
10
|
-
WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
|
|
11
|
-
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
|
|
12
4
|
type Settings,
|
|
13
5
|
} from "@opengeni/config";
|
|
14
6
|
import {
|
|
15
|
-
evaluateWorkspaceModelPolicy,
|
|
16
7
|
mergeToolRefs,
|
|
17
8
|
readTurnExecutionPolicyV1,
|
|
18
9
|
type SessionGoal,
|
|
19
10
|
type ToolRef,
|
|
20
11
|
} from "@opengeni/contracts";
|
|
21
|
-
import { isCodexBilledModel } from "@opengeni/codex";
|
|
22
12
|
import {
|
|
23
13
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
24
|
-
getWorkspaceModelPolicy,
|
|
25
14
|
getSessionGoal,
|
|
26
15
|
getSessionTurn,
|
|
27
16
|
materializeGoalContinuation,
|
|
@@ -34,10 +23,10 @@ import type {
|
|
|
34
23
|
} from "./types";
|
|
35
24
|
import {
|
|
36
25
|
modelFundingForAdmission,
|
|
37
|
-
|
|
38
|
-
|
|
26
|
+
goalRunBudgetBlocked,
|
|
27
|
+
resolveGoalModelAdmission,
|
|
39
28
|
} from "@opengeni/core";
|
|
40
|
-
|
|
29
|
+
export { goalContinuationModelDecision, goalRunBudgetBlocked } from "@opengeni/core";
|
|
41
30
|
import { turnCredentialRestriction } from "./agent-turn/credential-restriction";
|
|
42
31
|
|
|
43
32
|
export function createGoalActivities(services: () => Promise<ControlActivityServices>) {
|
|
@@ -80,42 +69,18 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
|
|
|
80
69
|
if (session.status === "failed" || session.status === "cancelled") {
|
|
81
70
|
return { action: "none" };
|
|
82
71
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
72
|
+
const modelDecision = await resolveGoalModelAdmission(db, catalogSourceSettings, {
|
|
73
|
+
accountId: input.accountId,
|
|
74
|
+
workspaceId: input.workspaceId,
|
|
75
|
+
model: session.model,
|
|
76
|
+
codexCompactionMode: session.codexCompactionMode,
|
|
77
|
+
latencyMode: session.latencyMode,
|
|
78
|
+
});
|
|
79
|
+
const settings = modelDecision.settings;
|
|
80
|
+
const continuationModel = modelDecision.model;
|
|
86
81
|
const continuationReasoningEffort = session.reasoningEffort;
|
|
87
82
|
const continuationLatencyMode = session.latencyMode;
|
|
88
|
-
const
|
|
89
|
-
if (
|
|
90
|
-
inheritedContinuationModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
|
|
91
|
-
inheritedContinuationModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ||
|
|
92
|
-
session.model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
|
|
93
|
-
session.model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX)
|
|
94
|
-
) {
|
|
95
|
-
settings = (
|
|
96
|
-
await resolveWorkspaceCatalogSettings(db, catalogSourceSettings, {
|
|
97
|
-
accountId: input.accountId,
|
|
98
|
-
workspaceId: input.workspaceId,
|
|
99
|
-
retainedProductModelIds: [inheritedContinuationModel, session.model],
|
|
100
|
-
})
|
|
101
|
-
).settings;
|
|
102
|
-
}
|
|
103
|
-
const modelDecision = goalContinuationModelDecision({
|
|
104
|
-
settings,
|
|
105
|
-
workspaceModelPolicy,
|
|
106
|
-
inheritedModel: inheritedContinuationModel,
|
|
107
|
-
});
|
|
108
|
-
continuationModel = modelDecision.model;
|
|
109
|
-
let modelPolicyBlocked = modelDecision.blocked;
|
|
110
|
-
// remote_v2 sessions may only continue on Codex models — refuse synthesis
|
|
111
|
-
// that would leave the portable/non-Codex path (and mixed history shapes).
|
|
112
|
-
if (
|
|
113
|
-
!modelPolicyBlocked &&
|
|
114
|
-
session.codexCompactionMode === "remote_v2" &&
|
|
115
|
-
!isCodexBilledModel(continuationModel)
|
|
116
|
-
) {
|
|
117
|
-
modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
|
|
118
|
-
}
|
|
83
|
+
const modelPolicyBlocked = modelDecision.blocked;
|
|
119
84
|
const turnExecutionPolicy = modelPolicyBlocked
|
|
120
85
|
? undefined
|
|
121
86
|
: resolveTurnExecutionPolicyV1(settings, {
|
|
@@ -183,11 +148,12 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
|
|
|
183
148
|
initiatingHumanSubjectId: causalTurn?.initiatingHumanSubjectId ?? null,
|
|
184
149
|
},
|
|
185
150
|
);
|
|
151
|
+
const pausedReason = modelPolicyBlocked
|
|
152
|
+
? modelDecision.pausedReason
|
|
153
|
+
: budgetBlocked?.pausedReason;
|
|
186
154
|
return {
|
|
187
155
|
budgetBlocked: modelPolicyBlocked ?? budgetBlocked?.message ?? null,
|
|
188
|
-
budgetPausedReason:
|
|
189
|
-
? "limits"
|
|
190
|
-
: (budgetBlocked?.pausedReason ?? "limits"),
|
|
156
|
+
...(pausedReason ? { budgetPausedReason: pausedReason } : {}),
|
|
191
157
|
};
|
|
192
158
|
},
|
|
193
159
|
policy: continuationPolicy,
|
|
@@ -220,47 +186,6 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
|
|
|
220
186
|
};
|
|
221
187
|
}
|
|
222
188
|
|
|
223
|
-
export function goalContinuationModelDecision(input: {
|
|
224
|
-
settings: Settings;
|
|
225
|
-
workspaceModelPolicy: Awaited<ReturnType<typeof getWorkspaceModelPolicy>>;
|
|
226
|
-
inheritedModel: string;
|
|
227
|
-
}): { model: string; blocked: string | null } {
|
|
228
|
-
const catalogSettings = input.settings.supergrokSubscriptionEnabled
|
|
229
|
-
? withXaiSubscriptionCatalogProvider(
|
|
230
|
-
input.settings.codexSubscriptionEnabled
|
|
231
|
-
? withCodexCatalogProvider(input.settings)
|
|
232
|
-
: input.settings,
|
|
233
|
-
)
|
|
234
|
-
: input.settings.codexSubscriptionEnabled
|
|
235
|
-
? withCodexCatalogProvider(input.settings)
|
|
236
|
-
: input.settings;
|
|
237
|
-
const policyBlocks = (modelId: string): boolean =>
|
|
238
|
-
input.workspaceModelPolicy !== null &&
|
|
239
|
-
!evaluateWorkspaceModelPolicy(input.workspaceModelPolicy, {
|
|
240
|
-
providerId: policyProviderIdForModel(catalogSettings, modelId),
|
|
241
|
-
modelId,
|
|
242
|
-
}).allowed;
|
|
243
|
-
if (!resolveModelProvider(catalogSettings, input.inheritedModel)) {
|
|
244
|
-
return {
|
|
245
|
-
model: input.inheritedModel,
|
|
246
|
-
blocked: `model "${input.inheritedModel}" is no longer in the deployment or workspace catalog; choose an available model before resuming the goal`,
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
|
-
if (!isModelAvailableForNewSelection(catalogSettings, input.inheritedModel)) {
|
|
250
|
-
return {
|
|
251
|
-
model: input.inheritedModel,
|
|
252
|
-
blocked: `model "${input.inheritedModel}" is retired from new selection; choose an available model before resuming the goal`,
|
|
253
|
-
};
|
|
254
|
-
}
|
|
255
|
-
if (!policyBlocks(input.inheritedModel)) {
|
|
256
|
-
return { model: input.inheritedModel, blocked: null };
|
|
257
|
-
}
|
|
258
|
-
return {
|
|
259
|
-
model: input.inheritedModel,
|
|
260
|
-
blocked: `workspace model policy blocks model "${input.inheritedModel}"; pick an allowed model or change the workspace model policy`,
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
|
|
264
189
|
export function goalContinuationFundedWithoutCredits(
|
|
265
190
|
settings: Settings,
|
|
266
191
|
model: string,
|
|
@@ -364,24 +289,3 @@ export function withFirstPartyTools(settings: Settings, tools: ToolRef[]): ToolR
|
|
|
364
289
|
}
|
|
365
290
|
return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
|
|
366
291
|
}
|
|
367
|
-
|
|
368
|
-
/**
|
|
369
|
-
* Goals share scheduled admission and pause visibly without synthesizing work.
|
|
370
|
-
*/
|
|
371
|
-
export async function goalRunBudgetBlocked(
|
|
372
|
-
services: Parameters<typeof agentRunAdmissionDenial>[0],
|
|
373
|
-
input: Omit<Parameters<typeof agentRunAdmissionDenial>[1], "requestedAgentRuns">,
|
|
374
|
-
): Promise<{ pausedReason: "limits" | "allowance"; message: string } | null> {
|
|
375
|
-
const denial = await agentRunAdmissionDenial(services, { ...input, requestedAgentRuns: 1 });
|
|
376
|
-
if (denial === null) return null;
|
|
377
|
-
if (denial === "allowance_exhausted") {
|
|
378
|
-
return { pausedReason: "allowance", message: "Opengeni usage allowance exhausted" };
|
|
379
|
-
}
|
|
380
|
-
const limits = configuredStaticUsageLimits(services.settings);
|
|
381
|
-
const messages = {
|
|
382
|
-
insufficient_credits: "insufficient Opengeni credits",
|
|
383
|
-
monthly_model_cost_limit: `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`,
|
|
384
|
-
monthly_agent_run_limit: `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`,
|
|
385
|
-
};
|
|
386
|
-
return { pausedReason: "limits", message: messages[denial] };
|
|
387
|
-
}
|