@opengeni/worker-bundle 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -14
  2. package/dist/activities/agent-turn/errors.d.ts +3 -0
  3. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  4. package/dist/activities/agent-turn/turn-context.d.ts +5 -0
  5. package/dist/activities/goals.d.ts +1 -17
  6. package/dist/activities/types.d.ts +1 -1
  7. package/dist/{activities-control-BNPTRTOE.js → activities-control-XZAIMFQJ.js} +23 -7
  8. package/dist/activities-control-XZAIMFQJ.js.map +1 -0
  9. package/dist/{activities-turn-5LGZOSVU.js → activities-turn-4EEVNZCC.js} +97 -5
  10. package/dist/activities-turn-4EEVNZCC.js.map +1 -0
  11. package/dist/{chunk-LJX7Z2KP.js → chunk-OT64XARC.js} +38 -162
  12. package/dist/chunk-OT64XARC.js.map +1 -0
  13. package/dist/index.js +2 -3
  14. package/dist/index.js.map +1 -1
  15. package/dist/workflow-bundle.js +1 -1
  16. package/package.json +21 -21
  17. package/src/activities/agent-run-admission.ts +1 -98
  18. package/src/activities/agent-turn/claim.ts +11 -0
  19. package/src/activities/agent-turn/compaction-prep.ts +15 -0
  20. package/src/activities/agent-turn/errors.ts +30 -15
  21. package/src/activities/agent-turn/failure-settlement.ts +22 -0
  22. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  23. package/src/activities/agent-turn/stream-attempt.ts +10 -0
  24. package/src/activities/agent-turn/tool-environment.ts +9 -2
  25. package/src/activities/agent-turn/turn-context.ts +4 -0
  26. package/src/activities/goals.ts +17 -113
  27. package/src/activities/scheduled-tasks.ts +18 -3
  28. package/src/activities/types.ts +1 -0
  29. package/src/index.ts +0 -2
  30. package/dist/activities-control-BNPTRTOE.js.map +0 -1
  31. package/dist/activities-turn-5LGZOSVU.js.map +0 -1
  32. package/dist/chunk-LJX7Z2KP.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "1.0.2",
3
+ "version": "1.1.0",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -54,26 +54,26 @@
54
54
  },
55
55
  "dependencies": {
56
56
  "@llamaindex/liteparse": "2.14.2",
57
- "@opengeni/agent-proto": "1.0.2",
58
- "@opengeni/capabilities": "1.0.2",
59
- "@opengeni/codemode": "1.0.2",
60
- "@opengeni/codex": "1.0.2",
61
- "@opengeni/config": "1.0.2",
62
- "@opengeni/contracts": "1.0.2",
63
- "@opengeni/core": "1.0.2",
64
- "@opengeni/db": "1.0.2",
65
- "@opengeni/documents": "1.0.2",
66
- "@opengeni/events": "1.0.2",
67
- "@opengeni/github": "1.0.2",
68
- "@opengeni/interaction": "1.0.2",
69
- "@opengeni/jev": "1.0.2",
70
- "@opengeni/network": "1.0.2",
71
- "@opengeni/observability": "1.0.2",
72
- "@opengeni/runtime": "1.0.2",
73
- "@opengeni/sdk": "1.0.2",
74
- "@opengeni/storage": "1.0.2",
75
- "@opengeni/tool-gateway": "1.0.2",
76
- "@opengeni/xai-subscription": "1.0.2",
57
+ "@opengeni/agent-proto": "1.1.0",
58
+ "@opengeni/capabilities": "1.1.0",
59
+ "@opengeni/codemode": "1.1.0",
60
+ "@opengeni/codex": "1.1.0",
61
+ "@opengeni/config": "1.1.0",
62
+ "@opengeni/contracts": "1.1.0",
63
+ "@opengeni/core": "1.1.0",
64
+ "@opengeni/db": "1.1.0",
65
+ "@opengeni/documents": "1.1.0",
66
+ "@opengeni/events": "1.1.0",
67
+ "@opengeni/github": "1.1.0",
68
+ "@opengeni/interaction": "1.1.0",
69
+ "@opengeni/jev": "1.1.0",
70
+ "@opengeni/network": "1.1.0",
71
+ "@opengeni/observability": "1.1.0",
72
+ "@opengeni/runtime": "1.1.0",
73
+ "@opengeni/sdk": "1.1.0",
74
+ "@opengeni/storage": "1.1.0",
75
+ "@opengeni/tool-gateway": "1.1.0",
76
+ "@opengeni/xai-subscription": "1.1.0",
77
77
  "@temporalio/activity": "^1.17.0",
78
78
  "@temporalio/client": "^1.17.0",
79
79
  "@temporalio/worker": "^1.17.0",
@@ -1,98 +1 @@
1
- import { configuredStaticUsageLimits, type Settings } from "@opengeni/config";
2
- import { modelFundingForAdmission } from "@opengeni/core";
3
- import {
4
- checkWorkspaceAllowance,
5
- getBillingBalance,
6
- isCodexBilledTurn,
7
- sumUsageQuantity,
8
- } from "@opengeni/db";
9
- import type { ControlActivityServices } from "./types";
10
-
11
- export type AgentRunAdmissionDenial =
12
- | "insufficient_credits"
13
- | "allowance_exhausted"
14
- | "monthly_model_cost_limit"
15
- | "monthly_agent_run_limit";
16
-
17
- /** One worker-side admission boundary for service-authored agent runs. */
18
- export async function agentRunAdmissionDenial(
19
- services: Pick<ControlActivityServices, "db" | "entitlements"> & { settings: Settings },
20
- input: {
21
- accountId: string;
22
- workspaceId: string;
23
- model: string;
24
- requestedAgentRuns: number;
25
- /** The accepted work's causal human, not the scheduler/service caller. */
26
- initiatingHumanSubjectId?: string | null;
27
- },
28
- ): Promise<AgentRunAdmissionDenial | null> {
29
- const codexBilled = await isCodexBilledTurn({
30
- db: services.db,
31
- settings: services.settings,
32
- workspaceId: input.workspaceId,
33
- model: input.model,
34
- });
35
- const externallyBilled = modelFundingForAdmission(
36
- services.settings,
37
- input.model,
38
- codexBilled,
39
- ).fundedWithoutCredits;
40
- if (
41
- !externallyBilled &&
42
- (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")
43
- ) {
44
- if (services.entitlements) {
45
- const decision = await services.entitlements.admitRun({
46
- accountId: input.accountId,
47
- workspaceId: input.workspaceId,
48
- action: "agent_run:create",
49
- quantity: input.requestedAgentRuns,
50
- });
51
- if (!decision.allowed) return "insufficient_credits";
52
- } else {
53
- const balance = await getBillingBalance(services.db, input.accountId);
54
- if (balance.balanceMicros <= 0) return "insufficient_credits";
55
- }
56
- }
57
- if (!externallyBilled) {
58
- const refusal = await checkWorkspaceAllowance(services.db, {
59
- accountId: input.accountId,
60
- workspaceId: input.workspaceId,
61
- subjectId: input.initiatingHumanSubjectId ?? null,
62
- });
63
- if (refusal) return refusal.code;
64
- }
65
- if (
66
- services.settings.usageLimitsMode !== "static" &&
67
- services.settings.usageLimitsMode !== "managed"
68
- ) {
69
- return null;
70
- }
71
- const limits = configuredStaticUsageLimits(services.settings);
72
- if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
73
- const used = await sumUsageQuantity(services.db, {
74
- accountId: input.accountId,
75
- eventType: "model.cost",
76
- since: startOfUtcMonth(),
77
- });
78
- if (used >= limits.maxMonthlyCostMicrosPerAccount) {
79
- return "monthly_model_cost_limit";
80
- }
81
- }
82
- if (limits.maxMonthlyAgentRunsPerWorkspace) {
83
- const used = await sumUsageQuantity(services.db, {
84
- workspaceId: input.workspaceId,
85
- eventType: "agent_run.created",
86
- since: startOfUtcMonth(),
87
- });
88
- if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
89
- return "monthly_agent_run_limit";
90
- }
91
- }
92
- return null;
93
- }
94
-
95
- function startOfUtcMonth(): Date {
96
- const now = new Date();
97
- return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
98
- }
1
+ export { agentRunAdmissionDenial, type AgentRunAdmissionDenial } from "@opengeni/core";
@@ -60,6 +60,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
60
60
  import { createTurnMediaArtifacts } from "./media-artifacts";
61
61
  import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
62
62
  import { turnCredentialRestriction } from "./credential-restriction";
63
+ import { readProviderRecoveryObservation } from "./provider-recovery-metrics";
63
64
 
64
65
  import {
65
66
  credentialSubjectIdForTurnInitiator,
@@ -237,6 +238,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
237
238
  attempt.dispatchId = dispatchId;
238
239
  attempt.executionGeneration = turn.executionGeneration;
239
240
  attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
241
+ attempt.providerRecoveryObservation = readProviderRecoveryObservation(turn.metadata ?? {});
240
242
  const authRecovery = turn.metadata?.claudeAuthRecovery;
241
243
  attempt.claudeAuthRecovery =
242
244
  authRecovery &&
@@ -391,6 +393,11 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
391
393
  latencyMode: turn.latencyMode,
392
394
  },
393
395
  );
396
+ // The durable same-turn recovery lane owns provider retries. Hidden SDK
397
+ // retries multiply that budget and keep the UI looking active during backoff.
398
+ // Apply before configuring/resolving clients so main, compaction and title
399
+ // requests all share this policy; standalone runtime consumers keep theirs.
400
+ capabilitySettings = { ...capabilitySettings, openaiMaxRetries: 0 };
394
401
  runtime.configure(capabilitySettings);
395
402
  const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
396
403
  capabilitySettings,
@@ -402,6 +409,10 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
402
409
  },
403
410
  );
404
411
  const turnExecutionPolicy = verifiedExecutionPolicy.policy;
412
+ attempt.modelMetricRoute = {
413
+ provider: turnExecutionPolicy.providerId,
414
+ model: turnExecutionPolicy.productModelId,
415
+ };
405
416
  assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
406
417
  const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
407
418
  billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
@@ -46,6 +46,7 @@ import {
46
46
  processCompactionModelUsageEvent,
47
47
  } from "./model-usage";
48
48
  import { waitForTurnOperation } from "./sandbox-provision";
49
+ import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
49
50
 
50
51
  import type { ClaimTurnOk } from "./claim";
51
52
  import type { GovernanceModelOk } from "./governance-model";
@@ -330,6 +331,20 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
330
331
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
331
332
  };
332
333
  const publishCompactionOutcomeEvents = async (events: SessionEvent[]) => {
334
+ if (events.some((event) => event.type === "session.context.compacted")) {
335
+ // The summary and recovery reset already committed under the attempt
336
+ // fence. Skipped compaction does not prove successful model progress.
337
+ attempt.providerRecoveryCount = 0;
338
+ if (attempt.providerRecoveryObservation) {
339
+ recordProviderRecoveryOutcome(observability, {
340
+ route: attempt.modelMetricRoute,
341
+ cause: attempt.providerRecoveryObservation.cause,
342
+ outcome: "recovered",
343
+ elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
344
+ });
345
+ attempt.providerRecoveryObservation = undefined;
346
+ }
347
+ }
333
348
  // `compaction.started` was already fanout via publishCompactionLiveEvents.
334
349
  await publishDurableSessionEvents(
335
350
  bus,
@@ -100,6 +100,11 @@ export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_M
100
100
  * alone would spend every automatic recovery before the window resets.
101
101
  */
102
102
  export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
103
+ /** Positive-only spread: never shorten the provider's minimum delay. */
104
+ export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
105
+ const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
106
+ return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
107
+ }
103
108
  export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
104
109
 
105
110
  export class PostCompactionContinuationEmptyError extends Error {
@@ -126,6 +131,7 @@ export function providerRecoveryResult(input: {
126
131
  failureCode: string | undefined;
127
132
  attemptNumber: number;
128
133
  retryAfterMs?: number | null;
134
+ jitterSample?: number;
129
135
  }): ProviderRecoveryResult {
130
136
  if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
131
137
  return {
@@ -171,7 +177,13 @@ export function providerRecoveryResult(input: {
171
177
  : PROVIDER_BACKPRESSURE_DELAY_MS;
172
178
  return {
173
179
  status: "recovering",
174
- continueDelayMs,
180
+ continueDelayMs:
181
+ continueDelayMs +
182
+ (input.failureCode === "provider_rate_limited" ||
183
+ input.failureCode === "provider_unavailable" ||
184
+ input.failureCode === "upstream_connectivity_unavailable"
185
+ ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
186
+ : 0),
175
187
  };
176
188
  }
177
189
 
@@ -1086,19 +1098,28 @@ function isProviderSafetyRefusal(error: unknown): boolean {
1086
1098
  return providerSafetyRefusalDiagnostic(error) !== undefined;
1087
1099
  }
1088
1100
 
1101
+ /** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
1102
+ function providerHttpStatus(error: unknown): number | undefined {
1103
+ let current = error;
1104
+ const seen = new Set<unknown>();
1105
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
1106
+ if (seen.has(current)) break;
1107
+ seen.add(current);
1108
+ const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
1109
+ const status = Number(value.status ?? value.statusCode);
1110
+ if (Number.isInteger(status) && status >= 100 && status < 600) return status;
1111
+ current = value.cause;
1112
+ }
1113
+ return undefined;
1114
+ }
1115
+
1089
1116
  export function isTransientProviderError(error: unknown): boolean {
1090
1117
  if (error instanceof ResponsesStreamingTerminalError) {
1091
1118
  return error.category === "unavailable";
1092
1119
  }
1093
1120
  // A semantic refusal can arrive inside a 5xx transport envelope.
1094
1121
  if (isProviderSafetyRefusal(error)) return false;
1095
- const status =
1096
- typeof error === "object" && error !== null
1097
- ? Number(
1098
- (error as { status?: unknown; statusCode?: unknown }).status ??
1099
- (error as { statusCode?: unknown }).statusCode,
1100
- )
1101
- : undefined;
1122
+ const status = providerHttpStatus(error);
1102
1123
  // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
1103
1124
  // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
1104
1125
  // a request fault that must NOT auto-retry — even if its body happens to read like
@@ -1399,13 +1420,7 @@ function baseAgentRunFailurePayload(
1399
1420
  };
1400
1421
  }
1401
1422
  const message = error instanceof Error ? error.message : String(error);
1402
- const status =
1403
- typeof error === "object" && error !== null
1404
- ? Number(
1405
- (error as { status?: unknown; statusCode?: unknown }).status ??
1406
- (error as { statusCode?: unknown }).statusCode,
1407
- )
1408
- : undefined;
1423
+ const status = providerHttpStatus(error);
1409
1424
  const code =
1410
1425
  typeof error === "object" && error !== null && "code" in error
1411
1426
  ? String((error as { code?: unknown }).code)
@@ -100,6 +100,7 @@ import type {
100
100
  } from "./turn-context";
101
101
  import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
102
102
  import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
103
+ import { providerRecoveryCause, recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
103
104
 
104
105
  export type TurnFailureDeps = {
105
106
  error: unknown;
@@ -1995,6 +1996,7 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
1995
1996
  failureCode: failure.code,
1996
1997
  attemptNumber: nextProviderRecoveryCount,
1997
1998
  retryAfterMs: providerRetryAfterMs(error),
1999
+ jitterSample: Math.random(),
1998
2000
  });
1999
2001
  const setupRecoveryExhausted =
2000
2002
  earlyCommandStartUnavailable &&
@@ -2064,9 +2066,29 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
2064
2066
  control.turnMetricOutcome = "recovering";
2065
2067
  control.activityStatus = "recovering";
2066
2068
  control.activityError = error;
2069
+ const recoveryCause = providerRecoveryCause(failure.code);
2070
+ if (recoveryCause) {
2071
+ recordProviderRecoveryOutcome(observability, {
2072
+ route: attempt.modelMetricRoute,
2073
+ cause: recoveryCause,
2074
+ outcome: "scheduled",
2075
+ delayMs: recoveryResult.continueDelayMs,
2076
+ });
2077
+ }
2067
2078
  return claimedResult(recoveryResult);
2068
2079
  }
2069
2080
  failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
2081
+ const recoveryCause = providerRecoveryCause(failure.code);
2082
+ if (recoveryCause) {
2083
+ recordProviderRecoveryOutcome(observability, {
2084
+ route: attempt.modelMetricRoute,
2085
+ cause: recoveryCause,
2086
+ outcome: "exhausted",
2087
+ ...(attempt.providerRecoveryObservation
2088
+ ? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt }
2089
+ : {}),
2090
+ });
2091
+ }
2070
2092
  if (earlyRecoverableSetup) {
2071
2093
  // Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
2072
2094
  // through Temporal into exact-attempt workflow failure settlement.
@@ -0,0 +1,64 @@
1
+ import type { Observability } from "@opengeni/observability";
2
+
3
+ export type ProviderRecoveryObservation = {
4
+ startedAt: number;
5
+ cause: "rate_limited" | "unavailable" | "connectivity";
6
+ };
7
+
8
+ export function providerRecoveryCause(code: unknown): ProviderRecoveryObservation["cause"] | null {
9
+ if (code === "provider_rate_limited") return "rate_limited";
10
+ if (code === "provider_unavailable") return "unavailable";
11
+ if (code === "upstream_connectivity_unavailable") return "connectivity";
12
+ return null;
13
+ }
14
+
15
+ export function readProviderRecoveryObservation(
16
+ metadata: Record<string, unknown>,
17
+ ): ProviderRecoveryObservation | undefined {
18
+ const cause = providerRecoveryCause(metadata.providerRecoveryReason);
19
+ const startedAt =
20
+ typeof metadata.providerRecoveryStartedAt === "string"
21
+ ? Date.parse(metadata.providerRecoveryStartedAt)
22
+ : Number.NaN;
23
+ return cause && Number.isFinite(startedAt) ? { cause, startedAt } : undefined;
24
+ }
25
+
26
+ /** Operational observations only; never retry authority, billing or diagnostics. */
27
+ export function recordProviderRecoveryOutcome(
28
+ observability: Observability,
29
+ input: {
30
+ route?: { provider: string; model: string } | undefined;
31
+ cause: ProviderRecoveryObservation["cause"];
32
+ outcome: "scheduled" | "recovered" | "exhausted";
33
+ delayMs?: number;
34
+ elapsedMs?: number;
35
+ },
36
+ ): void {
37
+ if (!input.route) return;
38
+ const labels = { ...input.route, cause: input.cause, outcome: input.outcome };
39
+ try {
40
+ observability.incrementCounter({
41
+ name: "opengeni_model_recovery_total",
42
+ help: "Observed model recovery decisions and successful resumptions.",
43
+ labels,
44
+ });
45
+ if (input.delayMs !== undefined) {
46
+ observability.observeHistogram({
47
+ name: "opengeni_model_recovery_delay_seconds",
48
+ help: "Scheduled model recovery delay, including provider hints and jitter.",
49
+ labels: { ...input.route, cause: input.cause },
50
+ value: Math.max(0, input.delayMs) / 1_000,
51
+ });
52
+ }
53
+ if (input.elapsedMs !== undefined) {
54
+ observability.observeHistogram({
55
+ name: "opengeni_model_recovery_duration_seconds",
56
+ help: "Elapsed recovery episode including backoff, preparation and model requests.",
57
+ labels,
58
+ value: Math.max(0, input.elapsedMs) / 1_000,
59
+ });
60
+ }
61
+ } catch {
62
+ // Metrics cannot interrupt settlement or model progress.
63
+ }
64
+ }
@@ -18,6 +18,7 @@ import {
18
18
  updateSessionTitleWithEvent,
19
19
  } from "@opengeni/db";
20
20
  import { publishDurableSessionEvents } from "@opengeni/events";
21
+ import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
21
22
  import {
22
23
  AssistantMessagePhaseTracker,
23
24
  normalizeModelCallUsage,
@@ -1132,6 +1133,15 @@ export async function runTurnStreamAttempt(
1132
1133
  ]);
1133
1134
  attempt.providerRecoveryCount = 0;
1134
1135
  }
1136
+ if (attempt.providerRecoveryObservation) {
1137
+ recordProviderRecoveryOutcome(observability, {
1138
+ route: attempt.modelMetricRoute,
1139
+ cause: attempt.providerRecoveryObservation.cause,
1140
+ outcome: "recovered",
1141
+ elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
1142
+ });
1143
+ attempt.providerRecoveryObservation = undefined;
1144
+ }
1135
1145
  const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
1136
1146
  if (Array.isArray(rawStreamHistory)) {
1137
1147
  // The completed image item is normally retained from its own
@@ -82,6 +82,7 @@ import { createTurnMediaArtifacts } from "./media-artifacts";
82
82
  import { SandboxChannelAService } from "@opengeni/runtime/sandbox";
83
83
  import { sandboxRunAs } from "@opengeni/runtime";
84
84
  import {
85
+ bundledSkillSelectionForAgentConfig,
85
86
  DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
86
87
  resolveAgentToolFamilies,
87
88
  type ResourceRef,
@@ -596,7 +597,11 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
596
597
  session.firstPartyMcpPermissions,
597
598
  linkedAuthority,
598
599
  );
599
- const toolFamilies = resolveAgentToolFamilies(session.agent);
600
+ // Background-command tools need compute: the effective route of this turn,
601
+ // a managed sandbox or an attached Connected Machine, not the durable home.
602
+ const toolFamilies = resolveAgentToolFamilies(session.agent, {
603
+ sandboxAttached: (activeSandboxBackend ?? groupBoxBackend) !== "none",
604
+ });
600
605
  const selectedFirstPartyMcpTools = toolFamilies.firstPartyTools(
601
606
  allowedFirstPartyMcpToolsForSession(runSettings, session.firstPartyMcpTools),
602
607
  );
@@ -669,7 +674,9 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
669
674
  }),
670
675
  ]);
671
676
  const bundledSkills = loadConfiguredBundledSkills({
672
- bundledSkillIds: session.bundledSkillIds,
677
+ // Rows that could not freeze the "none" default at create (scheduled
678
+ // generated sessions, pre-existing rows) get the same rule here.
679
+ bundledSkillIds: bundledSkillSelectionForAgentConfig(session.bundledSkillIds, session.agent),
673
680
  firstPartyTools: selectedFirstPartyMcpTools,
674
681
  videoGenerationEnabled:
675
682
  skillConfiguration.defaultModelId !== null && skillConfiguration.enabledModelIds.length > 0,
@@ -68,6 +68,10 @@ export type AttemptIdentityState = {
68
68
  triggerEventId: string | undefined;
69
69
  executionGeneration: number;
70
70
  providerRecoveryCount: number;
71
+ providerRecoveryObservation?:
72
+ | import("./provider-recovery-metrics").ProviderRecoveryObservation
73
+ | undefined;
74
+ modelMetricRoute?: { provider: string; model: string };
71
75
  claudeAuthRecovery?: { credentialId: string; credentialVersion: number } | undefined;
72
76
  modelRequestStarted: boolean;
73
77
  redispatchesAtDispatch: number;
@@ -1,27 +1,16 @@
1
1
  import {
2
2
  allowedFirstPartyMcpToolsForSession,
3
- configuredStaticUsageLimits,
4
- isModelAvailableForNewSelection,
5
- policyProviderIdForModel,
6
- resolveModelProvider,
7
3
  resolveTurnExecutionPolicyV1,
8
- withCodexCatalogProvider,
9
- withXaiSubscriptionCatalogProvider,
10
- WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
11
- WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
12
4
  type Settings,
13
5
  } from "@opengeni/config";
14
6
  import {
15
- evaluateWorkspaceModelPolicy,
16
7
  mergeToolRefs,
17
8
  readTurnExecutionPolicyV1,
18
9
  type SessionGoal,
19
10
  type ToolRef,
20
11
  } from "@opengeni/contracts";
21
- import { isCodexBilledModel } from "@opengeni/codex";
22
12
  import {
23
13
  enqueueSessionWorkflowWakeIfRunnable,
24
- getWorkspaceModelPolicy,
25
14
  getSessionGoal,
26
15
  getSessionTurn,
27
16
  materializeGoalContinuation,
@@ -34,10 +23,10 @@ import type {
34
23
  } from "./types";
35
24
  import {
36
25
  modelFundingForAdmission,
37
- resolveCatalogSettings,
38
- resolveWorkspaceCatalogSettings,
26
+ goalRunBudgetBlocked,
27
+ resolveGoalModelAdmission,
39
28
  } from "@opengeni/core";
40
- import { agentRunAdmissionDenial } from "./agent-run-admission";
29
+ export { goalContinuationModelDecision, goalRunBudgetBlocked } from "@opengeni/core";
41
30
  import { turnCredentialRestriction } from "./agent-turn/credential-restriction";
42
31
 
43
32
  export function createGoalActivities(services: () => Promise<ControlActivityServices>) {
@@ -80,42 +69,18 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
80
69
  if (session.status === "failed" || session.status === "cancelled") {
81
70
  return { action: "none" };
82
71
  }
83
- let settings = (await resolveCatalogSettings(db, catalogSourceSettings)).settings;
84
- const inheritedContinuationModel = session.model;
85
- let continuationModel = inheritedContinuationModel;
72
+ const modelDecision = await resolveGoalModelAdmission(db, catalogSourceSettings, {
73
+ accountId: input.accountId,
74
+ workspaceId: input.workspaceId,
75
+ model: session.model,
76
+ codexCompactionMode: session.codexCompactionMode,
77
+ latencyMode: session.latencyMode,
78
+ });
79
+ const settings = modelDecision.settings;
80
+ const continuationModel = modelDecision.model;
86
81
  const continuationReasoningEffort = session.reasoningEffort;
87
82
  const continuationLatencyMode = session.latencyMode;
88
- const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
89
- if (
90
- inheritedContinuationModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
91
- inheritedContinuationModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ||
92
- session.model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) ||
93
- session.model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX)
94
- ) {
95
- settings = (
96
- await resolveWorkspaceCatalogSettings(db, catalogSourceSettings, {
97
- accountId: input.accountId,
98
- workspaceId: input.workspaceId,
99
- retainedProductModelIds: [inheritedContinuationModel, session.model],
100
- })
101
- ).settings;
102
- }
103
- const modelDecision = goalContinuationModelDecision({
104
- settings,
105
- workspaceModelPolicy,
106
- inheritedModel: inheritedContinuationModel,
107
- });
108
- continuationModel = modelDecision.model;
109
- let modelPolicyBlocked = modelDecision.blocked;
110
- // remote_v2 sessions may only continue on Codex models — refuse synthesis
111
- // that would leave the portable/non-Codex path (and mixed history shapes).
112
- if (
113
- !modelPolicyBlocked &&
114
- session.codexCompactionMode === "remote_v2" &&
115
- !isCodexBilledModel(continuationModel)
116
- ) {
117
- modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
118
- }
83
+ const modelPolicyBlocked = modelDecision.blocked;
119
84
  const turnExecutionPolicy = modelPolicyBlocked
120
85
  ? undefined
121
86
  : resolveTurnExecutionPolicyV1(settings, {
@@ -183,11 +148,12 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
183
148
  initiatingHumanSubjectId: causalTurn?.initiatingHumanSubjectId ?? null,
184
149
  },
185
150
  );
151
+ const pausedReason = modelPolicyBlocked
152
+ ? modelDecision.pausedReason
153
+ : budgetBlocked?.pausedReason;
186
154
  return {
187
155
  budgetBlocked: modelPolicyBlocked ?? budgetBlocked?.message ?? null,
188
- budgetPausedReason: modelPolicyBlocked
189
- ? "limits"
190
- : (budgetBlocked?.pausedReason ?? "limits"),
156
+ ...(pausedReason ? { budgetPausedReason: pausedReason } : {}),
191
157
  };
192
158
  },
193
159
  policy: continuationPolicy,
@@ -220,47 +186,6 @@ export function createGoalActivities(services: () => Promise<ControlActivityServ
220
186
  };
221
187
  }
222
188
 
223
- export function goalContinuationModelDecision(input: {
224
- settings: Settings;
225
- workspaceModelPolicy: Awaited<ReturnType<typeof getWorkspaceModelPolicy>>;
226
- inheritedModel: string;
227
- }): { model: string; blocked: string | null } {
228
- const catalogSettings = input.settings.supergrokSubscriptionEnabled
229
- ? withXaiSubscriptionCatalogProvider(
230
- input.settings.codexSubscriptionEnabled
231
- ? withCodexCatalogProvider(input.settings)
232
- : input.settings,
233
- )
234
- : input.settings.codexSubscriptionEnabled
235
- ? withCodexCatalogProvider(input.settings)
236
- : input.settings;
237
- const policyBlocks = (modelId: string): boolean =>
238
- input.workspaceModelPolicy !== null &&
239
- !evaluateWorkspaceModelPolicy(input.workspaceModelPolicy, {
240
- providerId: policyProviderIdForModel(catalogSettings, modelId),
241
- modelId,
242
- }).allowed;
243
- if (!resolveModelProvider(catalogSettings, input.inheritedModel)) {
244
- return {
245
- model: input.inheritedModel,
246
- blocked: `model "${input.inheritedModel}" is no longer in the deployment or workspace catalog; choose an available model before resuming the goal`,
247
- };
248
- }
249
- if (!isModelAvailableForNewSelection(catalogSettings, input.inheritedModel)) {
250
- return {
251
- model: input.inheritedModel,
252
- blocked: `model "${input.inheritedModel}" is retired from new selection; choose an available model before resuming the goal`,
253
- };
254
- }
255
- if (!policyBlocks(input.inheritedModel)) {
256
- return { model: input.inheritedModel, blocked: null };
257
- }
258
- return {
259
- model: input.inheritedModel,
260
- blocked: `workspace model policy blocks model "${input.inheritedModel}"; pick an allowed model or change the workspace model policy`,
261
- };
262
- }
263
-
264
189
  export function goalContinuationFundedWithoutCredits(
265
190
  settings: Settings,
266
191
  model: string,
@@ -364,24 +289,3 @@ export function withFirstPartyTools(settings: Settings, tools: ToolRef[]): ToolR
364
289
  }
365
290
  return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
366
291
  }
367
-
368
- /**
369
- * Goals share scheduled admission and pause visibly without synthesizing work.
370
- */
371
- export async function goalRunBudgetBlocked(
372
- services: Parameters<typeof agentRunAdmissionDenial>[0],
373
- input: Omit<Parameters<typeof agentRunAdmissionDenial>[1], "requestedAgentRuns">,
374
- ): Promise<{ pausedReason: "limits" | "allowance"; message: string } | null> {
375
- const denial = await agentRunAdmissionDenial(services, { ...input, requestedAgentRuns: 1 });
376
- if (denial === null) return null;
377
- if (denial === "allowance_exhausted") {
378
- return { pausedReason: "allowance", message: "Opengeni usage allowance exhausted" };
379
- }
380
- const limits = configuredStaticUsageLimits(services.settings);
381
- const messages = {
382
- insufficient_credits: "insufficient Opengeni credits",
383
- monthly_model_cost_limit: `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`,
384
- monthly_agent_run_limit: `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`,
385
- };
386
- return { pausedReason: "limits", message: messages[denial] };
387
- }