@opengeni/worker-bundle 0.22.8 → 0.23.3-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +1 -4
  2. package/dist/activities/agent-turn/claim.d.ts +2 -4
  3. package/dist/activities/agent-turn/codex.d.ts +6 -5
  4. package/dist/activities/agent-turn/compaction-prep.d.ts +2 -0
  5. package/dist/activities/agent-turn/errors.d.ts +13 -0
  6. package/dist/activities/agent-turn/finalization.d.ts +1 -3
  7. package/dist/activities/agent-turn/quiescence.d.ts +39 -0
  8. package/dist/activities/agent-turn/sandbox-route.d.ts +14 -24
  9. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -4
  10. package/dist/activities/agent-turn/sandbox-shared-preparation.d.ts +30 -0
  11. package/dist/activities/agent-turn/stream-attempt.d.ts +7 -0
  12. package/dist/activities/agent-turn/tool-policy.d.ts +3 -32
  13. package/dist/activities/agent-turn/turn-context.d.ts +1 -8
  14. package/dist/activities/agent-turn/turn-sandbox-access.d.ts +12 -0
  15. package/dist/activities/agent-turn/video-reference-sandbox.d.ts +7 -0
  16. package/dist/activities/codex-rotation.d.ts +2 -0
  17. package/dist/activities/session-state.d.ts +2 -1
  18. package/dist/{activities-control-VZIIV7RX.js → activities-control-52W5IDWM.js} +125 -13
  19. package/dist/activities-control-52W5IDWM.js.map +1 -0
  20. package/dist/{activities-turn-5RGQDBFV.js → activities-turn-A2G2KLSY.js} +534 -482
  21. package/dist/activities-turn-A2G2KLSY.js.map +1 -0
  22. package/dist/{chunk-NAVSAZ56.js → chunk-GFNXVZVQ.js} +160 -1
  23. package/dist/chunk-GFNXVZVQ.js.map +1 -0
  24. package/dist/{chunk-O4HDX5SC.js → chunk-IPVMCAW5.js} +144 -15
  25. package/dist/chunk-IPVMCAW5.js.map +1 -0
  26. package/dist/index.js +42 -2
  27. package/dist/index.js.map +1 -1
  28. package/dist/observability-metrics.d.ts +28 -0
  29. package/dist/personal-github-git-credentials.d.ts +1 -0
  30. package/dist/unhandled-rejection-boundary.d.ts +29 -0
  31. package/package.json +18 -18
  32. package/src/activities/agent-turn/agent-build.ts +15 -31
  33. package/src/activities/agent-turn/claim.ts +29 -68
  34. package/src/activities/agent-turn/codex.ts +19 -10
  35. package/src/activities/agent-turn/compaction-prep.ts +50 -1
  36. package/src/activities/agent-turn/errors.ts +78 -0
  37. package/src/activities/agent-turn/failure-settlement.ts +30 -19
  38. package/src/activities/agent-turn/finalization.ts +24 -21
  39. package/src/activities/agent-turn/history.ts +5 -1
  40. package/src/activities/agent-turn/quiescence.ts +86 -0
  41. package/src/activities/agent-turn/run-credentials.ts +12 -1
  42. package/src/activities/agent-turn/run.ts +14 -11
  43. package/src/activities/agent-turn/sandbox-establish.ts +46 -11
  44. package/src/activities/agent-turn/sandbox-route.ts +21 -25
  45. package/src/activities/agent-turn/sandbox-runtime.ts +1 -116
  46. package/src/activities/agent-turn/sandbox-shared-preparation.ts +206 -0
  47. package/src/activities/agent-turn/stream-attempt.ts +98 -14
  48. package/src/activities/agent-turn/tool-environment.ts +10 -13
  49. package/src/activities/agent-turn/tool-policy.ts +3 -55
  50. package/src/activities/agent-turn/turn-context.ts +8 -14
  51. package/src/activities/agent-turn/{image-reference-sandbox.ts → turn-sandbox-access.ts} +15 -13
  52. package/src/activities/agent-turn/video-reference-sandbox.ts +23 -0
  53. package/src/activities/codex-capacity.ts +13 -5
  54. package/src/activities/codex-rotation.ts +17 -0
  55. package/src/activities/goals.ts +10 -4
  56. package/src/activities/retained-screenshots.ts +104 -21
  57. package/src/activities/sandbox-lease.ts +7 -4
  58. package/src/activities/scheduled-tasks.ts +128 -4
  59. package/src/activities/session-state.ts +8 -1
  60. package/src/github-rest-mcp.ts +6 -3
  61. package/src/http.ts +3 -0
  62. package/src/index.ts +19 -0
  63. package/src/observability-metrics.ts +200 -0
  64. package/src/personal-github-git-credentials.ts +45 -4
  65. package/src/sandbox-resume.ts +1 -1
  66. package/src/sandbox-routing.ts +7 -4
  67. package/src/unhandled-rejection-boundary.ts +73 -0
  68. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +0 -14
  69. package/dist/activities/recording.d.ts +0 -90
  70. package/dist/activities-control-VZIIV7RX.js.map +0 -1
  71. package/dist/activities-turn-5RGQDBFV.js.map +0 -1
  72. package/dist/chunk-NAVSAZ56.js.map +0 -1
  73. package/dist/chunk-O4HDX5SC.js.map +0 -1
  74. package/src/activities/recording.ts +0 -279
@@ -1,10 +1,12 @@
1
1
  import { type SessionEventType } from "@opengeni/contracts";
2
+ import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
2
3
  import type { EventLogger } from "@opengeni/events";
3
4
  import type { Observability } from "@opengeni/observability";
4
5
  import type { CompanyBrainContributionReceipt } from "./model-context-contributions.js";
5
6
  import { type OpenSandboxKubernetesInventory } from "./opensandbox-kubernetes-inventory.js";
6
7
  import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
7
8
  export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
9
+ export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
8
10
  export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
9
11
  export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
10
12
  export type CreditBalanceGauge = {
@@ -18,6 +20,10 @@ export type TurnTaskQueueStats = {
18
20
  tasksAddRate: number;
19
21
  tasksDispatchRate: number;
20
22
  };
23
+ export type SessionRecoveryBacklog = {
24
+ quiescence_missing: number;
25
+ projection_stale: number;
26
+ };
21
27
  export type TemporalTurnTaskQueueStats = {
22
28
  approximateBacklogCount?: unknown;
23
29
  approximateBacklogAge?: {
@@ -64,6 +70,11 @@ export declare function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {
64
70
  drainEvents(): MachineOpSessionEvent[];
65
71
  };
66
72
  export declare function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics;
73
+ export declare function initializeWorkerOutcomeMetrics(observability: Observability): void;
74
+ export declare function recordWorkerDeathRecoveryMetrics(observability: Observability, input: {
75
+ outcome: WorkerDeathRecoveryOutcome;
76
+ timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
77
+ }): void;
67
78
  export declare class TurnLifecycleMetrics {
68
79
  private readonly observability;
69
80
  private readonly options;
@@ -122,6 +133,15 @@ export declare function startTurnCapacityMonitor(input: {
122
133
  }): {
123
134
  close: () => Promise<void>;
124
135
  };
136
+ export declare function recordSessionRecoveryBacklogGauges(observability: Observability, counts: SessionRecoveryBacklog): void;
137
+ export declare function startSessionRecoveryMonitor(input: {
138
+ observability: Observability;
139
+ read: () => Promise<SessionRecoveryBacklog>;
140
+ intervalMs?: number;
141
+ now?: () => number;
142
+ }): {
143
+ close: () => Promise<void>;
144
+ };
125
145
  export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
126
146
  export declare function recordOpenSandboxKubernetesInventoryGauges(observability: Observability, inventory: OpenSandboxKubernetesInventory): void;
127
147
  export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains", "opensandbox_kubernetes"];
@@ -224,6 +244,11 @@ export declare function recordSandboxLogicalProvision(observability: Observabili
224
244
  internalAttempts: number;
225
245
  durationSeconds: number;
226
246
  }): void;
247
+ export declare function recordSandboxSharedPreparation(observability: Observability, measurement: {
248
+ path: "owner" | "joined" | "reused";
249
+ outcome: "completed" | "failed";
250
+ durationSeconds: number;
251
+ }): void;
227
252
  export declare function recordTurnWorkerPreparationTotal(observability: Observability, input: {
228
253
  provider: string;
229
254
  backend: string;
@@ -285,6 +310,9 @@ export declare function recordBatchFlush(observability: Observability, input: {
285
310
  export declare function recordSessionEventAppendLatency(observability: Observability, input: {
286
311
  durationSeconds: number;
287
312
  }): void;
313
+ export declare function sessionEventBatchSizeClass(eventCount: number): string;
314
+ /** Bounded database-phase attribution for exact-attempt event appends. */
315
+ export declare function recordSessionEventAppendPhase(observability: Observability, observation: SessionEventAppendPhaseObservation): void;
288
316
  /** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
289
317
  * here (with append healthy) is delivery, not the write path. */
290
318
  export declare function recordSessionEventPublishLatency(observability: Observability, input: {
@@ -7,3 +7,4 @@ import { type Database } from "@opengeni/db";
7
7
  * the API/database process and never crosses into a sandbox or worker result.
8
8
  */
9
9
  export declare function buildPersonalGitHubGitCredentials(db: Database, settings: Settings): NonNullable<ConnectionCredentialsPort["gitCredentials"]> | null;
10
+ export declare function personalGitBrokerOrigin(settings: Settings): string | null;
@@ -0,0 +1,29 @@
1
+ export type WorkerUnhandledRejectionProcess = {
2
+ on: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
3
+ off: (event: "unhandledRejection", listener: (reason: unknown) => void) => void;
4
+ };
5
+ export type WorkerUnhandledRejectionClassification = "detached_runtime_mcp_lifecycle" | "detached_unknown";
6
+ type RuntimeMcpLifecycleError = {
7
+ name: "McpLifecycleError";
8
+ code: "mcp_connect_failed" | "mcp_close_failed";
9
+ origin: "runtime";
10
+ serverId: string;
11
+ };
12
+ /**
13
+ * The Agents SDK owns MCP connect promises and also installs a process-global
14
+ * unhandled-rejection listener. A best-effort lifecycle rejection can surface
15
+ * twice under concurrent teardown: once through the awaited MCP result and once
16
+ * through the SDK's detached worker promise. The first path already records and
17
+ * degrades the server; the second must not terminate the whole turn-worker pod.
18
+ */
19
+ export declare function isDetachedRuntimeMcpLifecycleRejection(reason: unknown): reason is RuntimeMcpLifecycleError;
20
+ /**
21
+ * Keep process-global promise rejection handling observational. Turn activities
22
+ * have their own durable failure/lease boundary; killing the shared worker here
23
+ * converts one detached SDK/transport promise into unrelated lease losses across
24
+ * every session assigned to the pod. The Agents SDK also checks for a consumer
25
+ * listener before applying its own process.exit(1), so this boundary must never
26
+ * reintroduce that fail-fast behavior for an unknown rejection shape.
27
+ */
28
+ export declare function installWorkerUnhandledRejectionBoundary(runtimeProcess?: WorkerUnhandledRejectionProcess, report?: (classification: WorkerUnhandledRejectionClassification) => void): () => void;
29
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.22.8",
3
+ "version": "0.23.3-canary.0",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -53,23 +53,23 @@
53
53
  "prepublishOnly": "bash ../../scripts/prepublish-guard"
54
54
  },
55
55
  "dependencies": {
56
- "@llamaindex/liteparse": "^1.5.3",
57
- "@opengeni/agent-proto": "^0.5.1",
58
- "@opengeni/capabilities": "^0.3.1",
59
- "@opengeni/codemode": "^0.4.17",
60
- "@opengeni/codex": "^0.2.19",
61
- "@opengeni/config": "^0.22.0",
62
- "@opengeni/contracts": "^2.7.0",
63
- "@opengeni/core": "^2.5.3",
64
- "@opengeni/db": "^3.6.0",
65
- "@opengeni/documents": "^0.8.6",
66
- "@opengeni/events": "^0.4.4",
67
- "@opengeni/github": "^0.6.1",
68
- "@opengeni/network": "^0.2.3",
69
- "@opengeni/observability": "^0.8.9",
70
- "@opengeni/runtime": "^1.4.2",
71
- "@opengeni/storage": "^0.2.110",
72
- "@opengeni/xai-subscription": "^0.1.2",
56
+ "@llamaindex/liteparse": "2.14.2",
57
+ "@opengeni/agent-proto": "^0.5.1-canary.6",
58
+ "@opengeni/capabilities": "^0.3.1-canary.5",
59
+ "@opengeni/codemode": "^0.4.22-canary.0",
60
+ "@opengeni/codex": "^0.2.20-canary.0",
61
+ "@opengeni/config": "^0.22.5-canary.0",
62
+ "@opengeni/contracts": "^2.9.2-canary.0",
63
+ "@opengeni/core": "^2.6.3-canary.0",
64
+ "@opengeni/db": "^3.7.3-canary.0",
65
+ "@opengeni/documents": "^0.8.13-canary.0",
66
+ "@opengeni/events": "^0.4.11-canary.0",
67
+ "@opengeni/github": "^0.6.7-canary.0",
68
+ "@opengeni/network": "^0.2.3-canary.5",
69
+ "@opengeni/observability": "^0.8.14-canary.0",
70
+ "@opengeni/runtime": "^2.0.1-canary.0",
71
+ "@opengeni/storage": "^0.2.115-canary.0",
72
+ "@opengeni/xai-subscription": "^0.1.2-canary.5",
73
73
  "@temporalio/activity": "^1.17.0",
74
74
  "@temporalio/client": "^1.17.0",
75
75
  "@temporalio/worker": "^1.17.0",
@@ -54,7 +54,6 @@ import { sandboxRunAs } from "@opengeni/runtime";
54
54
  import { VideoGenerationRejectedResult } from "@opengeni/contracts";
55
55
 
56
56
  import {
57
- computerToolModeForTurn,
58
57
  structuredToolTransportForTurn,
59
58
  hostedWebSearchForTurn,
60
59
  connectedSubscriptionImageGenerationAuthority,
@@ -62,7 +61,6 @@ import {
62
61
  import type { ClaimTurnOk } from "./claim";
63
62
  import type { GovernanceModelOk } from "./governance-model";
64
63
  import type { CompactionPrepOk } from "./compaction-prep";
65
- import type { SandboxTurnRuntime } from "./sandbox-runtime";
66
64
  import type { runtimeResourcesForTurn } from "./file-resources";
67
65
  import type { sandboxArtifactRuntimeAdmission } from "./sandbox-route";
68
66
  import type {
@@ -73,11 +71,11 @@ import type {
73
71
  AttemptIdentityState,
74
72
  EventingState,
75
73
  ProviderTurnState,
76
- RecordingState,
77
74
  SandboxRuntimeState,
78
75
  } from "./turn-context";
79
76
  import { SESSION_TITLE_MODEL_TOOL_NAME } from "./session-title";
80
- import { resolveImageReferenceSandboxSession } from "./image-reference-sandbox";
77
+ import { resolveTurnSandboxAccess } from "./turn-sandbox-access";
78
+ import { resolveVideoReferenceSandboxAccess } from "./video-reference-sandbox";
81
79
 
82
80
  export type BuildTurnAgentDeps = {
83
81
  input: RunAgentTurnInput;
@@ -90,8 +88,6 @@ export type BuildTurnAgentDeps = {
90
88
  eventing: EventingState;
91
89
  attempt: AttemptIdentityState;
92
90
  sandboxState: SandboxRuntimeState;
93
- recordingState: RecordingState;
94
- maybeStartOnTurnRecording: SandboxTurnRuntime["maybeStartOnTurnRecording"];
95
91
  providerTurn: ProviderTurnState;
96
92
  media: ReturnType<typeof createTurnMediaArtifacts>;
97
93
  leases: ReturnType<typeof createTurnCredentialLeases>;
@@ -160,8 +156,6 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
160
156
  runtimeCancellationSignal,
161
157
  eventing,
162
158
  sandboxState,
163
- recordingState,
164
- maybeStartOnTurnRecording,
165
159
  providerTurn,
166
160
  media,
167
161
  leases,
@@ -254,9 +248,10 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
254
248
  subjectId: fileAuthoritySubjectId,
255
249
  references,
256
250
  readSandboxFile: async (path, maxBytes) => {
257
- const imageReferenceSandbox = await resolveImageReferenceSandboxSession(
251
+ const imageReferenceSandbox = await resolveTurnSandboxAccess(
258
252
  sandboxState,
259
253
  media.sdkOwnedSandboxSession,
254
+ "Sandbox image reference is unavailable",
260
255
  );
261
256
  const relativePath = path.slice("/workspace/".length);
262
257
  const referenceRunAs = sandboxRunAs(eventing.modelRunSettings);
@@ -508,9 +503,15 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
508
503
  videoGeneration: {
509
504
  capabilities: async () => capabilities,
510
505
  execute: async (toolInput, { toolCallId }) => {
511
- const sessionForReference =
512
- sandboxState.resolvedSandbox?.established.session ?? media.sdkOwnedSandboxSession;
506
+ const referenceSandbox = await resolveVideoReferenceSandboxAccess(
507
+ toolInput,
508
+ sandboxState,
509
+ media.sdkOwnedSandboxSession,
510
+ );
513
511
  const fence = eventing.toolCancellationFenceRef.current;
512
+ if (referenceSandbox && !fence) {
513
+ throw new Error("Video reference command fence is unavailable");
514
+ }
514
515
  const runAs = sandboxRunAs(eventing.modelRunSettings);
515
516
  let accepted: Awaited<ReturnType<typeof admitVideoGenerationRequest>>;
516
517
  try {
@@ -527,11 +528,11 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
527
528
  toolInput,
528
529
  policy: videoGenerationPolicy,
529
530
  credential: videoGenerationCredential,
530
- ...(sessionForReference && fence
531
+ ...(referenceSandbox && fence
531
532
  ? {
532
533
  runCommand: async (command) =>
533
534
  await fence.runSandboxCommandStructured(
534
- sessionForReference as TurnSandboxCommandSession,
535
+ referenceSandbox.session as TurnSandboxCommandSession,
535
536
  {
536
537
  ...command,
537
538
  ...(runAs ? { runAs } : {}),
@@ -710,31 +711,14 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
710
711
  // typed input_image content. Chat wires have no proven typed image
711
712
  // result transport and therefore receive no view_image tool.
712
713
  structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
713
- // EXPLICIT computer-use tool transport. See {@link computerToolModeForTurn}.
714
- computerToolMode: computerToolModeForTurn(resolvedModel),
715
714
  ...(promptCacheKey ? { promptCacheKey } : {}),
716
715
  }
717
716
  : // LEGACY global-client fallback (resolveTurnModel returned null → the model
718
717
  // is not in the registry, served by the built-in OpenAI/Azure Responses
719
- // client). Pin computerToolMode to function-image EXPLICITLY rather than
720
- // leaving the runtime to sniff the instance.
718
+ // client).
721
719
  {
722
- computerToolMode: computerToolModeForTurn(null),
723
720
  promptCacheKey: input.sessionId,
724
721
  }),
725
- // Lazy computer-use seam: runtime first brings up :0 only after the model
726
- // selects a computer tool, then this hook begins the optional proof
727
- // recording. Shell/filesystem turns never invoke either operation.
728
- onComputerUseReady: async () => {
729
- if (!sandboxState.resolvedSandbox) {
730
- throw new Error("Computer-use display became ready without a resolved sandbox");
731
- }
732
- // This callback is the authoritative execution boundary. Record the
733
- // action before async ffmpeg startup so transport-event ordering cannot
734
- // make settlement misclassify a real computer turn as unused.
735
- recordingState.didComputerUse = true;
736
- await maybeStartOnTurnRecording(sandboxState.resolvedSandbox, activeSandboxBackend);
737
- },
738
722
  onRetainableSessionImageOutput: media.retainSessionImageAtToolBoundary,
739
723
  ...(runtimeSkillActivations.length > 0
740
724
  ? { skillActivations: runtimeSkillActivations }
@@ -10,12 +10,11 @@ import {
10
10
  type AppendEventInput,
11
11
  type ApiIntegrationRuntime,
12
12
  type CanonicalTurnStartupMilestoneReceipt,
13
+ type ClaimSessionWorkForAttemptInput,
13
14
  type SessionTurnForExecution,
14
- type SessionTurnRecordingSettlement,
15
15
  } from "@opengeni/db";
16
16
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
17
17
  import { deliverChildRequiresActionToParent } from "../parent-wake";
18
- import { deleteRecordingArtifacts } from "@opengeni/runtime";
19
18
  import {
20
19
  assertTurnExecutionPolicyMatchesConfigV1,
21
20
  resolveTurnExecutionPolicyV1,
@@ -42,12 +41,12 @@ import type {
42
41
  import { makeTurnOpJournal, type TurnHeartbeatDetails } from "../../op-journal";
43
42
  import {
44
43
  recordSessionEventAppendLatency,
44
+ recordSessionEventAppendPhase,
45
45
  recordSessionEventPublishLatency,
46
46
  recordTurnStartupPhase,
47
47
  recordTurnStartupMilestone,
48
48
  turnLifecycleMetricsFor,
49
49
  } from "../../observability-metrics";
50
- import { prepareRecordingForSettlement, type ActiveRecording } from "../recording";
51
50
  import { createTurnCredentialLeases } from "./credential-leases";
52
51
  import { createTurnMediaArtifacts } from "./media-artifacts";
53
52
  import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
@@ -66,7 +65,6 @@ import type {
66
65
  BillingState,
67
66
  ClaimedResult,
68
67
  EventingState,
69
- RecordingState,
70
68
  SandboxRuntimeState,
71
69
  TurnControlState,
72
70
  } from "./turn-context";
@@ -77,7 +75,6 @@ export type ClaimTurnDeps = {
77
75
  db: ActivityServices["db"];
78
76
  bus: ActivityServices["bus"];
79
77
  runtime: ActivityServices["runtime"];
80
- objectStorage: ActivityServices["objectStorage"];
81
78
  observability: ActivityServices["observability"];
82
79
  entitlements: ActivityServices["entitlements"];
83
80
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
@@ -89,13 +86,11 @@ export type ClaimTurnDeps = {
89
86
  attempt: AttemptIdentityState;
90
87
  billingState: BillingState;
91
88
  sandboxState: SandboxRuntimeState;
92
- recordingState: RecordingState;
93
89
  eventing: EventingState;
94
90
  leases: ReturnType<typeof createTurnCredentialLeases>;
95
91
  media: ReturnType<typeof createTurnMediaArtifacts>;
96
92
  claimedResult: ClaimedResult;
97
93
  acknowledgeLostAttemptOwnership: () => void;
98
- abandonActiveRecording: (reason: string, disposition?: "failed" | "discard") => Promise<void>;
99
94
  };
100
95
 
101
96
  export type ClaimTurnOk = {
@@ -112,6 +107,7 @@ export type ClaimTurnOk = {
112
107
  interactionInterventionResume: Awaited<
113
108
  ReturnType<typeof getInteractionInterventionResumeForEvent>
114
109
  >;
110
+ attachPendingUpdatesAfterOpenSuffix: () => Promise<boolean>;
115
111
  throwIfWorkerShuttingDown: () => void;
116
112
  throwIfTurnCancelled: () => void;
117
113
  opJournal: ReturnType<typeof makeTurnOpJournal>;
@@ -142,7 +138,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
142
138
  db,
143
139
  bus,
144
140
  runtime,
145
- objectStorage,
146
141
  observability,
147
142
  entitlements,
148
143
  wakeSessionWorkflow,
@@ -154,15 +149,23 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
154
149
  attempt,
155
150
  billingState,
156
151
  sandboxState,
157
- recordingState,
158
152
  eventing,
159
153
  leases,
160
154
  media,
161
155
  claimedResult,
162
156
  acknowledgeLostAttemptOwnership,
163
- abandonActiveRecording,
164
157
  } = deps;
165
158
 
159
+ const validatePendingSystemUpdateAuthority: NonNullable<
160
+ ClaimSessionWorkForAttemptInput["validatePendingSystemUpdateAuthority"]
161
+ > = async (tx, update) =>
162
+ await validateIncidentTelemetrySystemUpdateAuthority({
163
+ db: tx,
164
+ settings,
165
+ workspaceId: input.workspaceId,
166
+ sessionId: input.sessionId,
167
+ update,
168
+ });
166
169
  const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
167
170
  sessionId: input.sessionId,
168
171
  workflowId: input.workflowId,
@@ -170,14 +173,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
170
173
  attemptId: input.attemptId,
171
174
  dispatchId,
172
175
  trigger: input.trigger,
173
- validatePendingSystemUpdateAuthority: async (tx, update) =>
174
- await validateIncidentTelemetrySystemUpdateAuthority({
175
- db: tx,
176
- settings,
177
- workspaceId: input.workspaceId,
178
- sessionId: input.sessionId,
179
- update,
180
- }),
176
+ validatePendingSystemUpdateAuthority,
181
177
  });
182
178
  if (claim.action === "unclaimed") {
183
179
  control.activityStatus = "unclaimed";
@@ -289,6 +285,19 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
289
285
  trigger,
290
286
  );
291
287
  attempt.triggerType = trigger.type;
288
+ const attachPendingUpdatesAfterOpenSuffix = async (): Promise<boolean> => {
289
+ const attached = await claimSessionWorkForAttempt(db, input.workspaceId, {
290
+ sessionId: input.sessionId,
291
+ workflowId: input.workflowId,
292
+ workflowRunId: input.workflowRunId,
293
+ attemptId: input.attemptId,
294
+ dispatchId,
295
+ trigger: input.trigger,
296
+ validatePendingSystemUpdateAuthority,
297
+ attachPendingUpdatesToRunningAttempt: true,
298
+ });
299
+ return attached.action === "claimed" && attached.turn.id === turn.id;
300
+ };
292
301
  turnLifecycleMetricsFor(observability).start(attempt.turnId);
293
302
  // §7.5 P3 — pass the accepted billing attribution (externally funded turns
294
303
  // bypass OpenGeni credit/token gates)
@@ -380,6 +389,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
380
389
  recordSessionEventAppendLatency(observability, {
381
390
  durationSeconds,
382
391
  }),
392
+ onAppendPhase: (observation) => recordSessionEventAppendPhase(observability, observation),
383
393
  onPublish: ({ durationSeconds }) =>
384
394
  recordSessionEventPublishLatency(observability, {
385
395
  durationSeconds,
@@ -404,36 +414,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
404
414
  return appended;
405
415
  };
406
416
  eventing.settle = async (inputSettlement) => {
407
- const attemptClosing = ["completed", "failed", "cancelled", "requires_action"].includes(
408
- inputSettlement.turnStatus,
409
- );
410
- const recordingForSettlement =
411
- attemptClosing && recordingState.activeRecording && sandboxState.resolvedSandbox
412
- ? (recordingState.activeRecording as ActiveRecording)
413
- : null;
414
- const preparedRecording = recordingForSettlement
415
- ? await prepareRecordingForSettlement({
416
- settings,
417
- objectStorage,
418
- workspaceId: input.workspaceId,
419
- sessionId: input.sessionId,
420
- active: recordingForSettlement,
421
- session: sandboxState.resolvedSandbox!.established.session,
422
- didComputerUse: recordingState.didComputerUse,
423
- })
424
- : null;
425
- let recordingMutation: SessionTurnRecordingSettlement | undefined;
426
- if (preparedRecording) {
427
- const mutation = preparedRecording.mutation;
428
- recordingMutation =
429
- mutation.action === "discard"
430
- ? mutation
431
- : {
432
- ...mutation,
433
- producerId,
434
- producerSeq: ++producerSeq,
435
- };
436
- }
437
417
  const compactionRequestFailure = inputSettlement.consumeRequestedCompactionFailure
438
418
  ? {
439
419
  reason: "summarization_failed" as const,
@@ -466,7 +446,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
466
446
  activeTurnId: inputSettlement.activeTurnId,
467
447
  events: inputs,
468
448
  ...(runState ? { runState } : {}),
469
- ...(recordingMutation ? { recording: recordingMutation } : {}),
470
449
  ...(compactionRequestFailure ? { compactionRequestFailure } : {}),
471
450
  });
472
451
  if (result.action === "stale") {
@@ -478,31 +457,12 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
478
457
  // TurnAttemptFencedError. If ownership was lost for an unrelated
479
458
  // reason, allowUninterrupted makes the receipt transaction a no-op.
480
459
  acknowledgeLostAttemptOwnership();
481
- if (recordingForSettlement) {
482
- await abandonActiveRecording(
483
- "recording settlement lost attempt ownership",
484
- preparedRecording?.mutation.action === "discard" ? "discard" : "failed",
485
- );
486
- }
487
460
  control.activityStatus = "cancelled";
488
461
  control.turnMetricOutcome = "cancelled";
489
462
  return false;
490
463
  }
491
464
  recordCanonicalStartupMilestones(result.canonicalStartupMilestones);
492
465
  turnLifecycleMetricsFor(observability).progress(attempt.turnId!);
493
- if (recordingForSettlement && preparedRecording) {
494
- if (result.recordingMutationApplied) {
495
- recordingState.activeRecording = null;
496
- if (preparedRecording.deleteArtifactsAfterCommit) {
497
- await deleteRecordingArtifacts(
498
- sandboxState.resolvedSandbox!.established.session,
499
- recordingForSettlement.proc,
500
- );
501
- }
502
- } else {
503
- await abandonActiveRecording("recording row was unavailable during turn settlement");
504
- }
505
- }
506
466
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
507
467
  if (inputSettlement.turnStatus === "requires_action") {
508
468
  // The settlement transaction committed the parent's child_requires_action
@@ -579,6 +539,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
579
539
  trigger,
580
540
  humanInputResume,
581
541
  interactionInterventionResume,
542
+ attachPendingUpdatesAfterOpenSuffix,
582
543
  throwIfWorkerShuttingDown,
583
544
  throwIfTurnCancelled,
584
545
  opJournal,
@@ -4,7 +4,7 @@ import {
4
4
  type CodexAccountStatus,
5
5
  } from "@opengeni/db";
6
6
  import { type RegistryProviderKind, type Settings } from "@opengeni/config";
7
- import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
7
+ import { codexAccountNeedsLiveCapacityRefresh } from "../codex-rotation";
8
8
  import {
9
9
  refreshCodexUsageAndRepairCapacityWaiters,
10
10
  signalPendingCodexCapacityWakeTargets,
@@ -37,9 +37,10 @@ export function acceptsPromptCacheKeyForTurn(
37
37
  * would idle-loop forever. Before idling, refresh LIVE usage for every connected account
38
38
  * the cache marks exhausted (bounded to the account count), which re-writes the
39
39
  * cache columns, then return the re-read rows so the ranker can pick up a genuinely-reset
40
- * window THIS turn. A refresh/read failure is swallowed (fall back to the pre-refresh rows
41
- * + the bounded idle). Cooling (429'd) accounts are NOT refreshed: their exhaustedUntil
42
- * cooldown is authoritative, and refreshing them would burn a provider call for nothing.
40
+ * window THIS turn. Typed quota cooldowns are also refreshed because the provider may
41
+ * replace an allowance cycle before its original reset; revision fencing makes that
42
+ * repair safe. Generic backpressure and legacy-unknown cooldowns are never reconciled.
43
+ * A refresh/read failure is swallowed (fall back to the pre-refresh rows + bounded idle).
43
44
  */
44
45
  export async function refreshCappedCodexUsageRows(
45
46
  db: ActivityServices["db"],
@@ -48,7 +49,12 @@ export async function refreshCappedCodexUsageRows(
48
49
  accounts: Array<
49
50
  Pick<
50
51
  CodexAccountStatus,
51
- "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
52
+ | "id"
53
+ | "status"
54
+ | "primaryUsedPercent"
55
+ | "secondaryUsedPercent"
56
+ | "exhaustedUntil"
57
+ | "exhaustedKind"
52
58
  >
53
59
  >,
54
60
  capacitySignals: {
@@ -59,15 +65,18 @@ export async function refreshCappedCodexUsageRows(
59
65
  Array<
60
66
  Pick<
61
67
  CodexAccountStatus,
62
- "id" | "status" | "primaryUsedPercent" | "secondaryUsedPercent" | "exhaustedUntil"
68
+ | "id"
69
+ | "status"
70
+ | "primaryUsedPercent"
71
+ | "secondaryUsedPercent"
72
+ | "exhaustedUntil"
73
+ | "exhaustedKind"
63
74
  >
64
75
  >
65
76
  > {
77
+ const now = new Date();
66
78
  const stale = accounts.filter(
67
- (a) =>
68
- a.status === "active" &&
69
- ((a.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT ||
70
- (a.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT),
79
+ (account) => account.status === "active" && codexAccountNeedsLiveCapacityRefresh(account, now),
71
80
  );
72
81
  if (stale.length === 0) {
73
82
  return accounts;
@@ -1,4 +1,4 @@
1
- import { isSessionCompactionRequested } from "@opengeni/db";
1
+ import { hasPendingSteerAfterContextCompaction, isSessionCompactionRequested } from "@opengeni/db";
2
2
  import { publishDurableSessionEvents } from "@opengeni/events";
3
3
  import {
4
4
  appendSessionInstructions,
@@ -102,6 +102,7 @@ export type CompactionPrepOk = {
102
102
  compactionModeOptions: NonNullable<Parameters<typeof maybeCompactContext>[5]>;
103
103
  compactionOnlyTurn: boolean;
104
104
  compactionSummarizerFor: (systemInstructions?: string) => CompactionSummarizer;
105
+ settleDeferredSteerAfterCompaction: () => Promise<RunAgentTurnResult | null>;
105
106
  };
106
107
 
107
108
  export type CompactionPrepOutcome = { exit: RunAgentTurnResult } | { ok: CompactionPrepOk };
@@ -114,6 +115,7 @@ export type PostAgentCompactionDeps = CompactionPrepDeps & {
114
115
  compactionModeOptions: CompactionPrepOk["compactionModeOptions"];
115
116
  compactionOnlyTurn: boolean;
116
117
  compactionSummarizerFor: CompactionPrepOk["compactionSummarizerFor"];
118
+ settleDeferredSteerAfterCompaction: CompactionPrepOk["settleDeferredSteerAfterCompaction"];
117
119
  agent: ReturnType<ActivityServices["runtime"]["buildAgent"]>;
118
120
  };
119
121
 
@@ -298,6 +300,39 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
298
300
  const compactionOnlyTurn = turn.source === "compaction";
299
301
  const remoteV2CompactionNeedsAgentPrefix =
300
302
  Boolean(remoteCompactionRequester) && session.codexCompactionMode === "remote_v2";
303
+ const settleDeferredSteerAfterCompaction = async (): Promise<RunAgentTurnResult | null> => {
304
+ if (compactionOnlyTurn) return null;
305
+ const pending = await hasPendingSteerAfterContextCompaction(db, {
306
+ accountId: input.accountId,
307
+ workspaceId: input.workspaceId,
308
+ sessionId: input.sessionId,
309
+ turnId: turn.id,
310
+ executionGeneration: attempt.executionGeneration,
311
+ attemptId: input.attemptId,
312
+ });
313
+ if (!pending) return null;
314
+ if (
315
+ !(await eventing.settle({
316
+ events: [
317
+ {
318
+ type: "turn.superseded",
319
+ payload: { reason: "steer", deferredUntilCompaction: true },
320
+ },
321
+ { type: "session.status.changed", payload: { status: "queued" } },
322
+ ],
323
+ turnStatus: "superseded",
324
+ sessionStatus: "queued",
325
+ activeTurnId: null,
326
+ }))
327
+ ) {
328
+ return claimedResult({ status: "cancelled" });
329
+ }
330
+ // Metrics currently group user-directed supersession with cancellation;
331
+ // the durable turn outcome remains the precise `superseded` truth.
332
+ control.turnMetricOutcome = "cancelled";
333
+ control.activityStatus = "idle";
334
+ return claimedResult({ status: "idle" });
335
+ };
301
336
  if (compactionOnlyTurn && !remoteV2CompactionNeedsAgentPrefix) {
302
337
  const compactionInstructions = appendWorkspaceMemory(
303
338
  appendSessionInstructions(
@@ -443,6 +478,7 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
443
478
  compactionModeOptions,
444
479
  compactionOnlyTurn,
445
480
  compactionSummarizerFor,
481
+ settleDeferredSteerAfterCompaction,
446
482
  },
447
483
  };
448
484
  }
@@ -468,6 +504,7 @@ export async function runPostAgentCompaction(
468
504
  compactionModeOptions,
469
505
  compactionOnlyTurn,
470
506
  compactionSummarizerFor,
507
+ settleDeferredSteerAfterCompaction,
471
508
  compactionModelHistoryProjector,
472
509
  media,
473
510
  agent,
@@ -661,6 +698,16 @@ export async function runPostAgentCompaction(
661
698
  }
662
699
  await publishCompactionOutcomeEvents(outcome.events);
663
700
  }
701
+ if (
702
+ outcome.events.some(
703
+ (event) =>
704
+ event.type === "session.context.compacted" ||
705
+ event.type === "session.context.compaction.skipped",
706
+ )
707
+ ) {
708
+ const deferredSteer = await settleDeferredSteerAfterCompaction();
709
+ if (deferredSteer) return { exit: deferredSteer };
710
+ }
664
711
  } catch (compactError) {
665
712
  if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
666
713
  if (compactError instanceof TurnAttemptFencedError) throw compactError;
@@ -680,6 +727,8 @@ export async function runPostAgentCompaction(
680
727
  },
681
728
  );
682
729
  if (!isCompactionSummaryFailure(compactError)) throw compactError;
730
+ const deferredSteer = await settleDeferredSteerAfterCompaction();
731
+ if (deferredSteer) return { exit: deferredSteer };
683
732
  const errorMessage = String(compactionFailureReasonFromError(compactError));
684
733
  observability.error("context compaction failed", {
685
734
  sessionId: input.sessionId,