@opengeni/worker-bundle 6.5.2-canary.41607687019001 → 6.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +12 -1
  2. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +11 -0
  3. package/dist/activities/agent-turn/sandbox-runtime.d.ts +21 -2
  4. package/dist/activities/agent-turn/sandbox-setup-agent-failure.d.ts +39 -0
  5. package/dist/activities/agent-turn/stream-attempt.d.ts +2 -1
  6. package/dist/activities/agent-turn/tool-environment.d.ts +3 -1
  7. package/dist/activities/agent-turn/turn-context.d.ts +11 -0
  8. package/dist/{activities-control-SZIXMQR2.js → activities-control-LJVIL5FP.js} +4 -4
  9. package/dist/{activities-turn-KAFMA5CX.js → activities-turn-GTZPMF6T.js} +178 -72
  10. package/dist/activities-turn-GTZPMF6T.js.map +1 -0
  11. package/dist/artifact-materializer-entry.js +0 -0
  12. package/dist/artifact-outbox-entry.js +0 -0
  13. package/dist/{chunk-6AMYPT2B.js → chunk-C5RYXGXF.js} +2 -2
  14. package/dist/{chunk-ESC7J52Z.js → chunk-Q4KXCK36.js} +104 -2
  15. package/dist/chunk-Q4KXCK36.js.map +1 -0
  16. package/dist/{chunk-LQ352XGC.js → chunk-W2IHRF7L.js} +8 -1
  17. package/dist/chunk-W2IHRF7L.js.map +1 -0
  18. package/dist/github-rest-mcp.d.ts +13 -1
  19. package/dist/index.js +3 -3
  20. package/dist/{sandbox-lease-GIEUWXRI.js → sandbox-lease-ILEZ6RVI.js} +3 -3
  21. package/dist/workflow-bundle.js +1 -1
  22. package/package.json +29 -30
  23. package/src/activities/agent-turn/agent-build.ts +38 -0
  24. package/src/activities/agent-turn/errors.ts +28 -0
  25. package/src/activities/agent-turn/finalization.ts +4 -0
  26. package/src/activities/agent-turn/owned-sandbox-setup.ts +29 -19
  27. package/src/activities/agent-turn/prompt-tool-availability.ts +50 -2
  28. package/src/activities/agent-turn/provider-dispatch-barrier.ts +19 -0
  29. package/src/activities/agent-turn/run.ts +3 -4
  30. package/src/activities/agent-turn/sandbox-establish.ts +30 -18
  31. package/src/activities/agent-turn/sandbox-runtime.ts +51 -4
  32. package/src/activities/agent-turn/sandbox-setup-agent-failure.ts +110 -0
  33. package/src/activities/agent-turn/skill-selection.ts +2 -1
  34. package/src/activities/agent-turn/stream-attempt.ts +15 -1
  35. package/src/activities/agent-turn/tool-environment.ts +6 -0
  36. package/src/activities/agent-turn/turn-context.ts +13 -0
  37. package/src/github-rest-mcp.ts +61 -6
  38. package/src/observability-metrics.ts +7 -0
  39. package/dist/activities-turn-KAFMA5CX.js.map +0 -1
  40. package/dist/chunk-ESC7J52Z.js.map +0 -1
  41. package/dist/chunk-LQ352XGC.js.map +0 -1
  42. /package/dist/{activities-control-SZIXMQR2.js.map → activities-control-LJVIL5FP.js.map} +0 -0
  43. /package/dist/{chunk-6AMYPT2B.js.map → chunk-C5RYXGXF.js.map} +0 -0
  44. /package/dist/{sandbox-lease-GIEUWXRI.js.map → sandbox-lease-ILEZ6RVI.js.map} +0 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "6.5.2-canary.41607687019001",
3
+ "version": "6.7.0",
4
4
  "description": "Opengeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -56,33 +56,33 @@
56
56
  "@1password/sdk": "0.5.0",
57
57
  "@keeper-security/secrets-manager-core": "17.6.0",
58
58
  "@llamaindex/liteparse": "2.14.2",
59
- "@opengeni/agent-proto": "6.5.2-canary.41607687019001",
60
- "@opengeni/capabilities": "6.5.2-canary.41607687019001",
61
- "@opengeni/codemode": "6.5.2-canary.41607687019001",
62
- "@opengeni/codex": "6.5.2-canary.41607687019001",
63
- "@opengeni/config": "6.5.2-canary.41607687019001",
64
- "@opengeni/contracts": "6.5.2-canary.41607687019001",
65
- "@opengeni/core": "6.5.2-canary.41607687019001",
66
- "@opengeni/db": "6.5.2-canary.41607687019001",
67
- "@opengeni/decisions": "6.5.2-canary.41607687019001",
68
- "@opengeni/documents": "6.5.2-canary.41607687019001",
69
- "@opengeni/events": "6.5.2-canary.41607687019001",
70
- "@opengeni/github": "6.5.2-canary.41607687019001",
71
- "@opengeni/interaction": "6.5.2-canary.41607687019001",
72
- "@opengeni/jev": "6.5.2-canary.41607687019001",
73
- "@opengeni/network": "6.5.2-canary.41607687019001",
74
- "@opengeni/observability": "6.5.2-canary.41607687019001",
75
- "@opengeni/runtime": "6.5.2-canary.41607687019001",
76
- "@opengeni/sdk": "6.5.2-canary.41607687019001",
77
- "@opengeni/storage": "6.5.2-canary.41607687019001",
78
- "@opengeni/subscriptions": "6.5.2-canary.41607687019001",
79
- "@opengeni/tool-gateway": "6.5.2-canary.41607687019001",
80
- "@opengeni/vaults": "6.5.2-canary.41607687019001",
81
- "@opengeni/vaults-bitwarden": "6.5.2-canary.41607687019001",
82
- "@opengeni/vaults-keeper": "6.5.2-canary.41607687019001",
83
- "@opengeni/vaults-onepassword": "6.5.2-canary.41607687019001",
84
- "@opengeni/vaults-protonpass": "6.5.2-canary.41607687019001",
85
- "@opengeni/xai-subscription": "6.5.2-canary.41607687019001",
59
+ "@opengeni/agent-proto": "6.7.0",
60
+ "@opengeni/capabilities": "6.7.0",
61
+ "@opengeni/codemode": "6.7.0",
62
+ "@opengeni/codex": "6.7.0",
63
+ "@opengeni/config": "6.7.0",
64
+ "@opengeni/contracts": "6.7.0",
65
+ "@opengeni/core": "6.7.0",
66
+ "@opengeni/db": "6.7.0",
67
+ "@opengeni/decisions": "6.7.0",
68
+ "@opengeni/documents": "6.7.0",
69
+ "@opengeni/events": "6.7.0",
70
+ "@opengeni/github": "6.7.0",
71
+ "@opengeni/interaction": "6.7.0",
72
+ "@opengeni/jev": "6.7.0",
73
+ "@opengeni/network": "6.7.0",
74
+ "@opengeni/observability": "6.7.0",
75
+ "@opengeni/runtime": "6.7.0",
76
+ "@opengeni/sdk": "6.7.0",
77
+ "@opengeni/storage": "6.7.0",
78
+ "@opengeni/subscriptions": "6.7.0",
79
+ "@opengeni/tool-gateway": "6.7.0",
80
+ "@opengeni/vaults": "6.7.0",
81
+ "@opengeni/vaults-bitwarden": "6.7.0",
82
+ "@opengeni/vaults-keeper": "6.7.0",
83
+ "@opengeni/vaults-onepassword": "6.7.0",
84
+ "@opengeni/vaults-protonpass": "6.7.0",
85
+ "@opengeni/xai-subscription": "6.7.0",
86
86
  "@temporalio/activity": "^1.17.0",
87
87
  "@temporalio/client": "^1.17.0",
88
88
  "@temporalio/worker": "^1.17.0",
@@ -92,6 +92,5 @@
92
92
  },
93
93
  "engines": {
94
94
  "node": ">=18"
95
- },
96
- "gitHead": "041e773bc7a4e5e1ecd9f3667659f29f4599e441"
95
+ }
97
96
  }
@@ -20,10 +20,13 @@ import {
20
20
  type AgentPromptToolAvailability,
21
21
  type AttemptConnectorActionBinding,
22
22
  type BuildAgentOptions,
23
+ CommandCompletionDelivery,
23
24
  type ConnectorActionPolicyHooks,
25
+ modelToolTextBudgetBytes,
24
26
  type SandboxFileDownload,
25
27
  type TurnSandboxCommandSession,
26
28
  } from "@opengeni/runtime";
29
+ import { claimPendingBackgroundCommandResults } from "@opengeni/db/session-background-commands";
27
30
  import {
28
31
  serviceTierForLatencyMode,
29
32
  WORKSPACE_GATEWAY_PROVIDER_ID,
@@ -899,6 +902,41 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
899
902
  }),
900
903
  // Every tool result is masked with the attempt's known secret values.
901
904
  secretValueMemory: currentAttemptSecretValues() ?? null,
905
+ // Commands that finish while the model works arrive with its next
906
+ // plain-text tool result instead of waiting for a read or a later turn.
907
+ ...(eventing.modelRunSettings.sandboxBackend !== "none"
908
+ ? {
909
+ commandCompletionDelivery: new CommandCompletionDelivery({
910
+ take: async ({ maxBytes }) =>
911
+ await claimPendingBackgroundCommandResults(db, {
912
+ accountId: input.accountId,
913
+ workspaceId: input.workspaceId,
914
+ sessionId: input.sessionId,
915
+ maxBytes,
916
+ }),
917
+ maxTextBytes: () =>
918
+ modelToolTextBudgetBytes(
919
+ eventing.modelRunSettings.modelToolOutputTruncationTokens,
920
+ ),
921
+ // Masked here too: `apply_patch` results bypass the function-tool masking.
922
+ mask: async (text) => {
923
+ const memory = currentAttemptSecretValues();
924
+ if (!memory) return text;
925
+ await memory.refresh();
926
+ await memory.captureShapes(text, "tool_result");
927
+ return memory.maskToolOutputText(text, "tool_result");
928
+ },
929
+ signal: cancellationSignal,
930
+ onDelivered: (appended) => {
931
+ (eventing.commandCompletionNotices ??= new Set()).add(appended);
932
+ },
933
+ onError: (error) =>
934
+ observability.warn("Finished background commands were not attached", {
935
+ error: error instanceof Error ? error.message : String(error),
936
+ }),
937
+ }),
938
+ }
939
+ : {}),
902
940
  onToolCancellationFence: (fence) => {
903
941
  eventing.toolCancellationFenceRef.current = fence;
904
942
  },
@@ -1,6 +1,7 @@
1
1
  import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
2
2
  import { DrizzleQueryError } from "drizzle-orm";
3
3
  import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
4
+ import { SandboxSetupAgentUnavailableError } from "./sandbox-setup-agent-failure";
4
5
  import { heldSessionUsageReceiptsForTurn } from "./session-usage-custody";
5
6
  import {
6
7
  ActiveSessionHistoryLimitExceededError,
@@ -1731,6 +1732,31 @@ function uncertainMutationOutcome(
1731
1732
  return { uncertain: node instanceof Error ? node.message : String(node) };
1732
1733
  }
1733
1734
 
1735
+ /**
1736
+ * Platform sandbox setup failed only because the machine agent did not answer
1737
+ * (see SandboxSetupAgentUnavailableError). No model-requested command reached
1738
+ * the box, so the same turn retries with the bounded sandbox-setup recovery
1739
+ * budget. An uncertain mutation outcome in the same failure, or an unreadable
1740
+ * graph, still forbids the replay.
1741
+ */
1742
+ function sandboxSetupAgentUnavailableFailure(
1743
+ error: unknown,
1744
+ ): ReturnType<typeof baseAgentRunFailurePayload> | null {
1745
+ const graph = structuredRecoveryCauseGraph(error);
1746
+ const nodes = graph ? [...graph.keys()] : [error];
1747
+ const setup = nodes.find((node) => node instanceof SandboxSetupAgentUnavailableError);
1748
+ if (!setup) return null;
1749
+ const { uncertain } = uncertainMutationOutcome(graph, nodes);
1750
+ if (uncertain !== undefined) return null;
1751
+ return {
1752
+ error:
1753
+ "The sandbox's agent did not answer while the sandbox was being set up, and nothing the model asked for ran. The same turn will retry after a short delay.",
1754
+ code: "sandbox_command_start_unavailable",
1755
+ retryable: true,
1756
+ detail: setup.message,
1757
+ };
1758
+ }
1759
+
1734
1760
  const SANDBOX_FAILURE_SUBJECT = {
1735
1761
  [SANDBOX_CAPACITY_UNAVAILABLE_CODE]: "No sandbox capacity is available right now",
1736
1762
  [SANDBOX_SERVICE_UNAVAILABLE_CODE]: "The sandbox service is temporarily unavailable",
@@ -2240,6 +2266,8 @@ function baseAgentRunFailurePayload(
2240
2266
  // provider 5xx below. Retry authority follows the error's own kind.
2241
2267
  const sandboxService = findSandboxServiceFailure(error);
2242
2268
  if (sandboxService) return sandboxServiceFailurePayload(sandboxService);
2269
+ const setupAgentUnavailable = sandboxSetupAgentUnavailableFailure(error);
2270
+ if (setupAgentUnavailable) return setupAgentUnavailable;
2243
2271
  if (error instanceof AnthropicSizeRecoveryExhaustedError) {
2244
2272
  return { error: error.message, code: error.code, retryable: false };
2245
2273
  }
@@ -306,6 +306,10 @@ async function finalizeTurnAttemptSteps(
306
306
  },
307
307
  });
308
308
  }
309
+ // The drain above cancelled and quiesced this exact fence. Platform setup
310
+ // writes only through it, so a setup still waiting on something else (a
311
+ // read, an unanswered agent) cannot write any more either.
312
+ sandboxState.drainedCommandFence = toolCancellationFence ?? null;
309
313
  sandboxState.attemptWritersDrained = true;
310
314
  if (control.acknowledgeQuiescence) {
311
315
  // A cancellation before sandbox-backed capabilities exist still has
@@ -10,6 +10,7 @@ import {
10
10
  recordTurnStartupPhase,
11
11
  } from "../../observability-metrics";
12
12
  import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
13
+ import { createSandboxSetupReplayGuard, sandboxSetupFailure } from "./sandbox-setup-agent-failure";
13
14
  import type { TurnStreamAttemptDeps } from "./stream-attempt";
14
15
  import { ownedTurnSandboxForAgent } from "./turn-sandbox-access";
15
16
  import { sandboxStartupPhaseObserver } from "./sandbox-startup-phase";
@@ -81,6 +82,7 @@ export async function runEagerOwnedSandboxSetup(
81
82
  });
82
83
  const ownedSandboxSetupStartedAt = performance.now();
83
84
  let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
85
+ const setupReplayGuard = createSandboxSetupReplayGuard();
84
86
  try {
85
87
  const eagerSetupSession = sandboxState.setupBoxSession ?? ownedEstablished.session;
86
88
  // The worker owns platform setup for owned boxes, so the runtime skips its
@@ -97,6 +99,10 @@ export async function runEagerOwnedSandboxSetup(
97
99
  const eagerCredentialSetupSession = deps.initialRunCredentialMaterial
98
100
  ? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
99
101
  : eagerSetupSession;
102
+ // Every setup write runs as a command through this fence, which lets the
103
+ // attempt writer drain settle the setup admission even when the setup
104
+ // itself never returns (an unanswered agent).
105
+ const setupCommandFence = eventing.toolCancellationFenceRef.current;
100
106
  await deps.runWorkspaceMutationForSandbox(
101
107
  eagerResolvedSandbox,
102
108
  "eagerOwnedSandboxSetup",
@@ -118,33 +124,35 @@ export async function runEagerOwnedSandboxSetup(
118
124
  }
119
125
  await eventing.publish([{ type: event.type, payload: event.payload }], true);
120
126
  },
121
- ...(eventing.toolCancellationFenceRef.current
127
+ ...(setupCommandFence
122
128
  ? {
123
- commandRunner: eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
124
- eventing.toolCancellationFenceRef.current,
125
- ),
129
+ commandRunner: setupCommandFence.runSandboxCommand.bind(setupCommandFence),
126
130
  }
127
131
  : {}),
128
132
  ...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
129
133
  ? {
130
- coordinateSharedRigSetup: createSharedRigSetupCoordinator({
131
- db: deps.db,
132
- accountId: input.accountId,
133
- workspaceId: input.workspaceId,
134
- sandboxGroupId: sandboxState.sandboxGroupId,
135
- attemptId: input.attemptId,
136
- holderId: sandboxState.sandboxHolderId,
137
- sandbox: eagerResolvedSandbox,
138
- ...(deps.runtimeCancellationSignal
139
- ? { signal: deps.runtimeCancellationSignal }
140
- : {}),
141
- observe: (measurement) =>
142
- recordSandboxSharedPreparation(observability, measurement),
143
- }),
134
+ coordinateSharedRigSetup: setupReplayGuard.trackRigSetup(
135
+ createSharedRigSetupCoordinator({
136
+ db: deps.db,
137
+ accountId: input.accountId,
138
+ workspaceId: input.workspaceId,
139
+ sandboxGroupId: sandboxState.sandboxGroupId,
140
+ attemptId: input.attemptId,
141
+ holderId: sandboxState.sandboxHolderId,
142
+ sandbox: eagerResolvedSandbox,
143
+ ...(deps.runtimeCancellationSignal
144
+ ? { signal: deps.runtimeCancellationSignal }
145
+ : {}),
146
+ observe: (measurement) =>
147
+ recordSandboxSharedPreparation(observability, measurement),
148
+ }),
149
+ ),
144
150
  }
145
151
  : {}),
146
152
  },
147
153
  ),
154
+ undefined,
155
+ setupCommandFence ? { commandFence: setupCommandFence } : {},
148
156
  );
149
157
  await deps.attachGitCredentialRenewal(
150
158
  eagerSetupSession as GitCredentialTokenWriterSession,
@@ -152,7 +160,9 @@ export async function runEagerOwnedSandboxSetup(
152
160
  );
153
161
  } catch (error) {
154
162
  ownedSandboxSetupOutcome = "failed";
155
- throw error;
163
+ // The model stream has not started, and every setup step is idempotent,
164
+ // so an unanswered agent lets the same turn retry instead of failing it.
165
+ throw sandboxSetupFailure(setupReplayGuard, error);
156
166
  } finally {
157
167
  await measure.flush();
158
168
  recordTurnStartupPhase(observability, {
@@ -2,7 +2,33 @@ import {
2
2
  deriveAgentPromptToolAvailability,
3
3
  type AgentPromptToolAvailability,
4
4
  } from "@opengeni/runtime";
5
- import type { FirstPartyMcpToolName, Permission, ResolvedAgentConfig } from "@opengeni/contracts";
5
+ import type {
6
+ AgentLearningDefaults,
7
+ FirstPartyMcpToolName,
8
+ Permission,
9
+ ResolvedAgentConfig,
10
+ } from "@opengeni/contracts";
11
+
12
+ /**
13
+ * Tools that serve Knowledge authoring. With the turn's effective Knowledge
14
+ * learning mode Off, every Knowledge write is refused (docs/knowledge.md: the
15
+ * stricter of the chat's and the destination's mode governs), so guidance for
16
+ * these save-only tools (including the read-only knowledge_prepare_save step)
17
+ * only describes saves that cannot succeed.
18
+ */
19
+ const KNOWLEDGE_AUTHORING_TOOLS = [
20
+ "knowledge_save",
21
+ "knowledge_prepare_save",
22
+ "knowledge_retain_message",
23
+ "knowledge_retain_file",
24
+ "knowledge_archive",
25
+ "task_note_promote_knowledge",
26
+ ] as const satisfies readonly FirstPartyMcpToolName[];
27
+
28
+ /** With workspace-instruction learning Off, an agent instruction change is refused. */
29
+ const INSTRUCTION_AUTHORING_TOOLS = [
30
+ "instruction_policy_save",
31
+ ] as const satisfies readonly FirstPartyMcpToolName[];
6
32
 
7
33
  /**
8
34
  * The frozen clause-availability view for one attempt's modular instructions.
@@ -11,6 +37,12 @@ import type { FirstPartyMcpToolName, Permission, ResolvedAgentConfig } from "@op
11
37
  * so it can only describe tools the executable catalog already excludes. It is
12
38
  * a rendering input, never authority.
13
39
  *
40
+ * The turn's frozen learning modes are a rendering input too: with a category
41
+ * Off, its authoring clauses are omitted as if those tools were absent. The
42
+ * tools themselves stay in the catalog and still answer with the refusal, so
43
+ * nothing executable changes. Learning settings change rarely, so the prefix
44
+ * stays stable between turns.
45
+ *
14
46
  * Sessions without an agent configuration keep the legacy composition, which
15
47
  * does not read availability; they receive undefined so nothing changes.
16
48
  */
@@ -18,10 +50,26 @@ export function promptToolAvailabilityForTurn(input: {
18
50
  agentConfig: ResolvedAgentConfig | null | undefined;
19
51
  selectedFirstPartyMcpTools: readonly FirstPartyMcpToolName[];
20
52
  firstPartyPermissions: readonly Permission[] | null | undefined;
53
+ /** Frozen on the session; an exhausted depth hides `session_create`. */
54
+ nestedAgentDepth?: number | undefined;
55
+ effectiveMaxNestedAgentDepth?: number | undefined;
56
+ /** The accepted turn's effective learning modes; omitted keeps every clause. */
57
+ learning?: AgentLearningDefaults | undefined;
21
58
  }): AgentPromptToolAvailability | undefined {
22
59
  if (!input.agentConfig) return undefined;
23
- return deriveAgentPromptToolAvailability({
60
+ const availability = deriveAgentPromptToolAvailability({
24
61
  selectedFirstPartyTools: input.selectedFirstPartyMcpTools,
25
62
  firstPartyPermissions: input.firstPartyPermissions,
63
+ nestedAgentDepth: input.nestedAgentDepth,
64
+ effectiveMaxNestedAgentDepth: input.effectiveMaxNestedAgentDepth,
26
65
  });
66
+ const refused: FirstPartyMcpToolName[] = [
67
+ ...(input.learning?.knowledge === "off" ? KNOWLEDGE_AUTHORING_TOOLS : []),
68
+ ...(input.learning?.instructions === "off" ? INSTRUCTION_AUTHORING_TOOLS : []),
69
+ ];
70
+ if (refused.length === 0) return availability;
71
+ return {
72
+ ...availability,
73
+ unavailable: [...new Set([...availability.unavailable, ...refused])].sort(),
74
+ };
27
75
  }
@@ -1,5 +1,7 @@
1
1
  import type { TurnHistorySink } from "./history-sink";
2
2
  import type { Settings } from "@opengeni/config";
3
+ import { ownedTurnSandboxForAgent } from "./turn-sandbox-access";
4
+ import type { SandboxRuntimeState } from "./turn-context";
3
5
 
4
6
  export class SandboxCapabilitiesChangedError extends Error {
5
7
  readonly code = "native_capabilities_changed_this_attempt";
@@ -12,6 +14,23 @@ export class SandboxCapabilitiesChangedError extends Error {
12
14
 
13
15
  type ActiveRoutePointer = { activeSandboxId: string | null; activeEpoch: number };
14
16
 
17
+ type RouteContinuableSession = { assertActiveRouteContinuable?: () => Promise<void> };
18
+
19
+ /**
20
+ * The routed session the agent's own tools use, which is the one that can tell
21
+ * whether a moved route is still servable. A home box provisioned on demand
22
+ * fills `resolvedSandbox` with the raw provider box, so reading that would see
23
+ * no route at all and never end the attempt after a move.
24
+ */
25
+ export function activeRouteSessionForBoundary(
26
+ sandboxState: Pick<
27
+ SandboxRuntimeState,
28
+ "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner"
29
+ >,
30
+ ): RouteContinuableSession | undefined {
31
+ return ownedTurnSandboxForAgent(sandboxState)?.session as RouteContinuableSession | undefined;
32
+ }
33
+
15
34
  /**
16
35
  * Ends an attempt between tool batches when its route moved somewhere it can
17
36
  * no longer serve. A swap to a machine with another workspace root (or back
@@ -88,6 +88,7 @@ import { createTurnMediaArtifacts } from "./media-artifacts";
88
88
  import { createTurnHistorySink } from "./history-sink";
89
89
  import {
90
90
  ActiveRouteDispatchBoundary,
91
+ activeRouteSessionForBoundary,
91
92
  checkpointHistoryBeforeProviderDispatch,
92
93
  } from "./provider-dispatch-barrier";
93
94
  import {
@@ -495,10 +496,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
495
496
  recoveryStartedAt: attempt.providerRecoveryStartedAt,
496
497
  });
497
498
  const activeRoute = new ActiveRouteDispatchBoundary({
498
- session: () =>
499
- sandboxState.resolvedSandbox?.established.session as
500
- | { assertActiveRouteContinuable?: () => Promise<void> }
501
- | undefined,
499
+ session: () => activeRouteSessionForBoundary(sandboxState),
502
500
  isRouteTransition: (error) => sandboxRouteTransitionCode(error) !== null,
503
501
  });
504
502
  const checkpointBeforeProviderDispatch = async () => {
@@ -1747,6 +1745,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1747
1745
  });
1748
1746
  const toolRuntime = await prepareTurnToolRuntime({
1749
1747
  independentSessionTitles: !!startSessionTitleWorkflow,
1748
+ learningModes: learning.effective,
1750
1749
  vaultAccess,
1751
1750
  runCredentialRenewals: runCredentialResolver ? renewals : undefined,
1752
1751
  runMcpCredentials,
@@ -90,6 +90,7 @@ import type { GovernanceModelOk } from "./governance-model";
90
90
  import type { SandboxTurnRuntime } from "./sandbox-runtime";
91
91
  import type { AttemptIdentityState, EventingState, SandboxRuntimeState } from "./turn-context";
92
92
  import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
93
+ import { createSandboxSetupReplayGuard, sandboxSetupFailure } from "./sandbox-setup-agent-failure";
93
94
  import { sandboxStartupPhaseObserver } from "./sandbox-startup-phase";
94
95
  import { errorMessage } from "@opengeni/contracts/values";
95
96
 
@@ -1145,6 +1146,7 @@ export async function bindLazySandboxProvisioner(
1145
1146
  publish: eventing.publish,
1146
1147
  publishInBackground: eventing.publishInBackground,
1147
1148
  });
1149
+ const setupReplayGuard = createSandboxSetupReplayGuard();
1148
1150
  try {
1149
1151
  // Post-provision prerequisites: independent authorities and storage.
1150
1152
  // Join every started operation before failure/disposal; in particular
@@ -1192,6 +1194,10 @@ export async function bindLazySandboxProvisioner(
1192
1194
  input.sessionId,
1193
1195
  )
1194
1196
  : provisioned.established.session;
1197
+ // Every setup write runs as a command through this fence, which
1198
+ // lets the attempt writer drain settle the setup admission even
1199
+ // when the setup itself never returns (an unanswered agent).
1200
+ const setupCommandFence = eventing.toolCancellationFenceRef.current;
1195
1201
  await runWorkspaceMutationForSandbox(
1196
1202
  provisioned,
1197
1203
  "lazyOwnedSandboxSetup",
@@ -1225,30 +1231,30 @@ export async function bindLazySandboxProvisioner(
1225
1231
  : {}),
1226
1232
  }
1227
1233
  : {}),
1228
- ...(eventing.toolCancellationFenceRef.current
1234
+ ...(setupCommandFence
1229
1235
  ? {
1230
1236
  commandRunner:
1231
- eventing.toolCancellationFenceRef.current.runSandboxCommand.bind(
1232
- eventing.toolCancellationFenceRef.current,
1233
- ),
1237
+ setupCommandFence.runSandboxCommand.bind(setupCommandFence),
1234
1238
  }
1235
1239
  : {}),
1236
1240
  ...(sandboxState.sandboxGroupId && sandboxState.sandboxHolderId
1237
1241
  ? {
1238
- coordinateSharedRigSetup: createSharedRigSetupCoordinator({
1239
- db,
1240
- accountId: input.accountId,
1241
- workspaceId: input.workspaceId,
1242
- sandboxGroupId: sandboxState.sandboxGroupId,
1243
- attemptId: input.attemptId,
1244
- holderId: sandboxState.sandboxHolderId,
1245
- sandbox: provisioned,
1246
- ...(runtimeCancellationSignal
1247
- ? { signal: runtimeCancellationSignal }
1248
- : {}),
1249
- observe: (measurement) =>
1250
- recordSandboxSharedPreparation(observability, measurement),
1251
- }),
1242
+ coordinateSharedRigSetup: setupReplayGuard.trackRigSetup(
1243
+ createSharedRigSetupCoordinator({
1244
+ db,
1245
+ accountId: input.accountId,
1246
+ workspaceId: input.workspaceId,
1247
+ sandboxGroupId: sandboxState.sandboxGroupId,
1248
+ attemptId: input.attemptId,
1249
+ holderId: sandboxState.sandboxHolderId,
1250
+ sandbox: provisioned,
1251
+ ...(runtimeCancellationSignal
1252
+ ? { signal: runtimeCancellationSignal }
1253
+ : {}),
1254
+ observe: (measurement) =>
1255
+ recordSandboxSharedPreparation(observability, measurement),
1256
+ }),
1257
+ ),
1252
1258
  }
1253
1259
  : {}),
1254
1260
  },
@@ -1259,6 +1265,7 @@ export async function bindLazySandboxProvisioner(
1259
1265
  measurement.durationSeconds * 1_000;
1260
1266
  sandboxState.firstModelPreparationNestedSandboxPhases.push(measurement);
1261
1267
  },
1268
+ setupCommandFence ? { commandFence: setupCommandFence } : {},
1262
1269
  );
1263
1270
  await attachCodemodeTokenRenewal(
1264
1271
  provisioned.established.session as CodemodeTokenWriterSession,
@@ -1283,6 +1290,11 @@ export async function bindLazySandboxProvisioner(
1283
1290
  }
1284
1291
  await toolResultSpill.materializeDeferred();
1285
1292
  throwIfTurnOperationCancelled(sandboxResumeSignal);
1293
+ } catch (error) {
1294
+ // Nothing the model asked for has reached the box yet, and every
1295
+ // setup step above is idempotent, so an unanswered agent lets the
1296
+ // same turn retry instead of failing it.
1297
+ throw sandboxSetupFailure(setupReplayGuard, error);
1286
1298
  } finally {
1287
1299
  await measure.flush();
1288
1300
  }
@@ -23,6 +23,7 @@ import {
23
23
  type EstablishedSandboxSession,
24
24
  type ProviderCommandSession,
25
25
  type RoutingRetainedProcess,
26
+ type TurnToolCancellationFence,
26
27
  } from "@opengeni/runtime";
27
28
  import { ModalRouterProviderCommand } from "@opengeni/contracts";
28
29
  import { isDeepStrictEqual } from "node:util";
@@ -65,6 +66,36 @@ export type SandboxTurnRuntimeDeps = {
65
66
 
66
67
  export type SandboxTurnRuntime = ReturnType<typeof createSandboxTurnRuntime>;
67
68
 
69
+ export type WorkspaceMutationOptions = {
70
+ /** Every write this mutation makes runs as a command through this fence
71
+ * (platform setup passes it as its command runner). Once the attempt's
72
+ * writer drain has cancelled and quiesced this same fence, the mutation can
73
+ * no longer write, so the proof-bearing release may settle its admission
74
+ * even while the mutation itself has not returned yet. */
75
+ commandFence?: TurnToolCancellationFence;
76
+ };
77
+
78
+ type PendingWorkspaceMutation = {
79
+ operation: string;
80
+ /** Set once the provider outcome is unknown; such a mutation stays fenced. */
81
+ cause?: unknown;
82
+ commandFence?: TurnToolCancellationFence;
83
+ };
84
+
85
+ /** Whether the attempt's writer drain proves that none of these in-flight
86
+ * mutations can still write: each writes only through the exact fence the
87
+ * drain cancelled and quiesced, and none has an unknown provider outcome. */
88
+ export function pendingMutationsCoveredByWriterDrain(
89
+ pending: Iterable<PendingWorkspaceMutation>,
90
+ drainedCommandFence: TurnToolCancellationFence | null,
91
+ ): boolean {
92
+ if (!drainedCommandFence) return false;
93
+ for (const mutation of pending) {
94
+ if ("cause" in mutation || mutation.commandFence !== drainedCommandFence) return false;
95
+ }
96
+ return true;
97
+ }
98
+
68
99
  const retainInternalProviderCommand = createProviderCommandRetainer(
69
100
  retainWorkspaceMutationProcess,
70
101
  (error) => (error instanceof SandboxRetainedProcessPromotionFencedError ? error.process : null),
@@ -215,10 +246,18 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
215
246
  // The exact existing private release closure is the holder authority. Keep
216
247
  // its ordinary eager release, but never pass a false writer-drain proof that
217
248
  // would let releaseLeaseHolder settle an ambiguous null-outcome admission.
249
+ //
250
+ // A mutation still in flight is covered by the writer drain only when every
251
+ // write it makes runs as a command through the fence that drain cancelled
252
+ // and quiesced (platform setup): the drain stopped or durably handed off
253
+ // each of its commands, and the cancelled fence refuses any later one, so
254
+ // whatever the mutation still waits on (a read, an agent that stopped
255
+ // answering) cannot write to the box. Any other in-flight or outcome-unknown
256
+ // mutation keeps its admission fenced.
218
257
  const releaseGuards = new WeakMap<
219
258
  ResumedTurnSandbox["release"],
220
259
  {
221
- pending: Map<string, { operation: string; cause?: unknown }>;
260
+ pending: Map<string, PendingWorkspaceMutation>;
222
261
  release: ResumedTurnSandbox["release"];
223
262
  }
224
263
  >();
@@ -227,9 +266,13 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
227
266
  const original = sandbox.release;
228
267
  let guard = releaseGuards.get(original);
229
268
  if (!guard) {
230
- const pending = new Map<string, { operation: string; cause?: unknown }>();
269
+ const pending = new Map<string, PendingWorkspaceMutation>();
231
270
  const release: ResumedTurnSandbox["release"] = async (options) => {
232
- if (options?.workspaceWritersQuiesced && pending.size) {
271
+ if (
272
+ options?.workspaceWritersQuiesced &&
273
+ pending.size &&
274
+ !pendingMutationsCoveredByWriterDrain(pending.values(), sandboxState.drainedCommandFence)
275
+ ) {
233
276
  await original();
234
277
  throw new RoutingMutationOutcomeUnknownError(
235
278
  "internalWorkspaceMutationDrain",
@@ -363,6 +406,7 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
363
406
  outcome: "completed" | "failed";
364
407
  durationSeconds: number;
365
408
  }) => void,
409
+ options: WorkspaceMutationOptions = {},
366
410
  ): Promise<T> => {
367
411
  const observeMutationPhase = (
368
412
  phase: "admission" | "provider" | "settlement" | "snapshot_wait",
@@ -489,7 +533,10 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
489
533
  };
490
534
  let result: T;
491
535
  const releaseGuard = guardSandboxRelease(sandbox);
492
- releaseGuard?.pending.set(admission.id, { operation });
536
+ releaseGuard?.pending.set(admission.id, {
537
+ operation,
538
+ ...(options.commandFence ? { commandFence: options.commandFence } : {}),
539
+ });
493
540
  const providerStartedAt = performance.now();
494
541
  let providerOutcome: "completed" | "failed" = "failed";
495
542
  try {