@opengeni/worker-bundle 0.10.8 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -39,6 +39,8 @@ import { createObjectStorage as createObjectStorage2 } from "@opengeni/storage";
39
39
 
40
40
  // src/activities/agent-turn.ts
41
41
  import {
42
+ advanceWorkspaceGeneration as advanceWorkspaceGeneration2,
43
+ verifyWorkspaceMutationSettlement as verifyWorkspaceMutationSettlement2,
42
44
  applySessionTurnSettlement,
43
45
  requestSessionTurnRecovery,
44
46
  claimSessionWorkForAttempt,
@@ -79,7 +81,7 @@ import {
79
81
  withCodexCapacityMutation,
80
82
  countConsecutiveReactiveRotations,
81
83
  requireSession as requireSession2,
82
- recordUsageEvent as recordUsageEvent2,
84
+ recordUsageEvent,
83
85
  registerPendingSessionToolCall,
84
86
  recordPendingSessionToolCallResult,
85
87
  clearDurablePendingSessionToolCalls,
@@ -134,6 +136,7 @@ import {
134
136
  sandboxFileDownloadFailureNote,
135
137
  SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2,
136
138
  runOwnedSandboxSetup,
139
+ RoutingMutationOutcomeUnknownError,
137
140
  swapTargetEstablishability,
138
141
  deleteRecordingArtifacts as deleteRecordingArtifacts2,
139
142
  stopRecording as stopRecordingOnBox2
@@ -173,7 +176,7 @@ import {
173
176
  settingsWithResolvedModelContext,
174
177
  resolveTurnExecutionPolicyV1
175
178
  } from "@opengeni/config";
176
- import { CancelledFailure } from "@temporalio/activity";
179
+ import { CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
177
180
 
178
181
  // src/activities/capabilities.ts
179
182
  import {
@@ -674,6 +677,206 @@ function selectCodexCredentialLeaseForTurn(args) {
674
677
  };
675
678
  }
676
679
 
680
+ // src/activities/codex-fleet-shadow.ts
681
+ import {
682
+ DEFAULT_CODEX_FLEET_POLICY_V1,
683
+ compareCodexFleetCanonicalStringsV1,
684
+ createCodexFleetReplayRecordV1
685
+ } from "@opengeni/contracts";
686
+ import { CancelledFailure } from "@temporalio/activity";
687
+ import { createHmac } from "crypto";
688
+
689
+ // src/activities/turn-attempt-fenced.ts
690
+ var TurnAttemptFencedError = class extends Error {
691
+ constructor(message) {
692
+ super(message);
693
+ this.name = "TurnAttemptFencedError";
694
+ }
695
+ };
696
+
697
+ // src/activities/codex-fleet-shadow.ts
698
+ var CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1024;
699
+ var CODEX_FLEET_SHADOW_DECISION_MAX_METRIC_SERIES = 3 * 3 * 4 * 4 * 2;
700
+ var CODEX_FLEET_SHADOW_ERROR_MAX_METRIC_SERIES = 3 * 2;
701
+ function codexFleetShadowDecisionMetricLabelsV1(payload) {
702
+ return {
703
+ actual_outcome: payload.actual.outcome,
704
+ shadow_outcome: payload.replay.decision.outcome,
705
+ comparison: payload.comparison,
706
+ confidence: payload.replay.decision.confidence,
707
+ truncated: payload.replay.truncatedCandidateCount > 0 ? "true" : "false"
708
+ };
709
+ }
710
+ function codexFleetShadowErrorMetricLabelsV1(result) {
711
+ return { stage: result.stage, reason: result.reason };
712
+ }
713
+ async function publishCodexFleetShadowDecisionV1(input) {
714
+ if (!input.enabled) return { outcome: "disabled" };
715
+ let stage = "build";
716
+ let payloadBytes = null;
717
+ try {
718
+ const payload = buildCodexFleetShadowPayloadV1(input.decision);
719
+ stage = "serialize";
720
+ payloadBytes = new TextEncoder().encode(JSON.stringify(payload)).byteLength;
721
+ if (payloadBytes > CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES) {
722
+ return {
723
+ outcome: "failed",
724
+ stage,
725
+ reason: "payload_too_large",
726
+ errorName: "CodexFleetShadowPayloadTooLargeError",
727
+ payloadBytes
728
+ };
729
+ }
730
+ stage = "publish";
731
+ await input.publish([{ type: "codex.fleet.decision", payload }]);
732
+ return { outcome: "published", payload, payloadBytes };
733
+ } catch (error) {
734
+ if (error instanceof TurnAttemptFencedError || error instanceof CancelledFailure) throw error;
735
+ return {
736
+ outcome: "failed",
737
+ stage,
738
+ reason: "unexpected",
739
+ errorName: safeErrorName(error),
740
+ payloadBytes
741
+ };
742
+ }
743
+ }
744
+ function buildCodexFleetShadowPayloadV1(input) {
745
+ const eventAccounts = [...input.accounts].sort(
746
+ (left, right) => compareCodexFleetCanonicalStringsV1(
747
+ aliasSortKey(input.aliasSeed, left.id),
748
+ aliasSortKey(input.aliasSeed, right.id)
749
+ ) || compareCodexFleetCanonicalStringsV1(left.id, right.id)
750
+ );
751
+ const aliases = new Map(
752
+ eventAccounts.map((account, index) => [account.id, candidateAlias(index)])
753
+ );
754
+ const actualCandidateKey = input.actualCredentialId ? aliases.get(input.actualCredentialId) ?? null : null;
755
+ const currentCandidateKey = input.affinityCredentialId ? aliases.get(input.affinityCredentialId) ?? null : null;
756
+ const decisionInput = {
757
+ observedAtMs: input.now.getTime(),
758
+ request: {
759
+ placement: input.fencedInFlight ? "fenced_in_flight" : "new",
760
+ priority: "standard",
761
+ currentCandidateKey: input.fencedInFlight ? actualCandidateKey : currentCandidateKey,
762
+ waitAgeMs: 0,
763
+ overlayKey: null,
764
+ overlayMode: "none"
765
+ },
766
+ admission: {
767
+ // The shadow policy does not invent hard capacity from account count. A later typed
768
+ // admission snapshot supplies dynamic capacity; shadow v1 records unknown.
769
+ dynamicCapacityUnits: null,
770
+ inUseUnits: input.accounts.reduce((sum, account) => sum + account.activeLeaseCount, 0),
771
+ queuedManagerCount: 0,
772
+ emergencyFuseActive: false
773
+ },
774
+ candidates: eventAccounts.map((account, index) => ({
775
+ key: candidateAlias(index),
776
+ status: normalizeStatus(account.status),
777
+ allocatorEnabled: account.allocatorEnabled,
778
+ cooldownRemainingMs: remainingMs(account.exhaustedUntil, input.now),
779
+ activeLeaseCount: account.activeLeaseCount,
780
+ quota: {
781
+ primary: {
782
+ usedPercent: account.primaryUsedPercent,
783
+ resetRemainingMs: remainingMs(account.primaryResetAt, input.now)
784
+ },
785
+ secondary: {
786
+ usedPercent: account.secondaryUsedPercent,
787
+ resetRemainingMs: remainingMs(account.secondaryResetAt, input.now)
788
+ },
789
+ checkedAgeMs: ageMs(account.usageCheckedAt, input.now),
790
+ confidence: quotaConfidence(account)
791
+ },
792
+ // Current production cache evidence is aggregate metric + opaque log data,
793
+ // not allocator state. Preserve that gap as unknown instead of fabricating 0.
794
+ cache: {
795
+ hitRatio: null,
796
+ sampledTokens: null,
797
+ checkedAgeMs: null,
798
+ confidence: "unknown",
799
+ state: "unknown",
800
+ thresholdObservedForMs: null
801
+ },
802
+ // The shadow policy has no typed workspace-local burn feed at this boundary yet.
803
+ observedBurn: {
804
+ primaryPercentPerHour: null,
805
+ secondaryPercentPerHour: null,
806
+ confidence: "unknown"
807
+ },
808
+ // Provider accounting has not supplied a typed burn observation yet.
809
+ // Unknown inference is explicit and contributes no fabricated tenant truth.
810
+ inferredUnexplainedBurn: {
811
+ primaryPercentPerHour: null,
812
+ secondaryPercentPerHour: null,
813
+ confidence: "unknown"
814
+ },
815
+ overlayKeys: []
816
+ }))
817
+ };
818
+ const replay = createCodexFleetReplayRecordV1(decisionInput, {
819
+ ...DEFAULT_CODEX_FLEET_POLICY_V1,
820
+ placementUsageCeilingPercent: input.nearExhaustionPct
821
+ });
822
+ const actual = {
823
+ outcome: input.actualOutcome,
824
+ candidateKey: actualCandidateKey,
825
+ reason: input.actualReason
826
+ };
827
+ return {
828
+ schemaVersion: 1,
829
+ mode: "shadow",
830
+ actual,
831
+ comparison: compareDecision(actual, replay),
832
+ replay
833
+ };
834
+ }
835
+ function compareDecision(actual, replay) {
836
+ if (replay.truncatedCandidateCount > 0 && actual.candidateKey !== null && !replay.input.candidates.some((candidate) => candidate.key === actual.candidateKey)) {
837
+ return "not_comparable_truncated";
838
+ }
839
+ const shadowOutcome = replay.decision.outcome === "selected" ? "selected" : "none";
840
+ const actualComparableOutcome = actual.outcome === "selected" ? "selected" : "none";
841
+ if (shadowOutcome !== actualComparableOutcome) return "different_outcome";
842
+ if (shadowOutcome === "selected") {
843
+ return replay.decision.selectedCandidateKey === actual.candidateKey ? "match" : "different_candidate";
844
+ }
845
+ return "match";
846
+ }
847
+ function normalizeStatus(status) {
848
+ if (status === "active" || status === "needs_relogin" || status === "error") return status;
849
+ return "unknown";
850
+ }
851
+ function candidateAlias(index) {
852
+ return `c${index.toString(36).padStart(2, "0")}`;
853
+ }
854
+ function aliasSortKey(seed, credentialId) {
855
+ return createHmac("sha256", seed).update(credentialId).digest("hex");
856
+ }
857
+ function quotaConfidence(account) {
858
+ if (!account.usageCheckedAt) return "unknown";
859
+ const windows = [
860
+ [account.primaryUsedPercent, account.primaryResetAt],
861
+ [account.secondaryUsedPercent, account.secondaryResetAt]
862
+ ];
863
+ const complete = windows.filter(([used, reset]) => used !== null && reset !== null).length;
864
+ const partial = windows.some(([used, reset]) => used === null !== (reset === null));
865
+ if (complete === 0) return "unknown";
866
+ if (partial) return "low";
867
+ return complete === windows.length ? "high" : "medium";
868
+ }
869
+ function remainingMs(value, now) {
870
+ return value ? Math.max(0, value.getTime() - now.getTime()) : null;
871
+ }
872
+ function ageMs(value, now) {
873
+ return value ? Math.max(0, now.getTime() - value.getTime()) : null;
874
+ }
875
+ function safeErrorName(error) {
876
+ if (!(error instanceof Error)) return "UnknownError";
877
+ return /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(error.name) ? error.name : "Error";
878
+ }
879
+
677
880
  // src/activities/codex-auth.ts
678
881
  import { buildCodexTokenResolver } from "@opengeni/db";
679
882
 
@@ -880,16 +1083,6 @@ import {
880
1083
  summarizeForCompaction
881
1084
  } from "@opengeni/runtime";
882
1085
  import { contextInputBudgetTokens } from "@opengeni/config";
883
-
884
- // src/activities/turn-attempt-fenced.ts
885
- var TurnAttemptFencedError = class extends Error {
886
- constructor(message) {
887
- super(message);
888
- this.name = "TurnAttemptFencedError";
889
- }
890
- };
891
-
892
- // src/activities/context-compaction.ts
893
1086
  async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, {
894
1087
  maxOutputTokens: SUMMARY_BUFFER_TOKENS
895
1088
  }), options = {}) {
@@ -1950,19 +2143,16 @@ import {
1950
2143
  reasoningEffortForMetadata as reasoningEffortForMetadata2
1951
2144
  } from "@opengeni/contracts";
1952
2145
  import {
1953
- addSessionSystemUpdate,
1954
2146
  enqueueSessionWorkflowWakeIfRunnable,
1955
- evaluateGoalContinuation,
1956
2147
  getBillingBalance,
1957
2148
  getLatestStartedSessionTurn,
1958
2149
  getWorkspaceModelPolicy,
1959
2150
  getSessionGoal,
1960
2151
  isCodexBilledTurn,
1961
- recordUsageEvent,
2152
+ materializeGoalContinuation,
1962
2153
  requireSession,
1963
2154
  sumUsageQuantity
1964
2155
  } from "@opengeni/db";
1965
- import { appendAndPublishEvents } from "@opengeni/events";
1966
2156
  function createGoalActivities(services) {
1967
2157
  async function enqueueGoalRetryWake2(input) {
1968
2158
  const { db } = await services();
@@ -2019,84 +2209,29 @@ function createGoalActivities(services) {
2019
2209
  input.workspaceId,
2020
2210
  isCodexRun
2021
2211
  );
2022
- const decision = await evaluateGoalContinuation(db, {
2212
+ const decision = await materializeGoalContinuation(db, {
2213
+ accountId: input.accountId,
2023
2214
  workspaceId: input.workspaceId,
2024
2215
  sessionId: input.sessionId,
2216
+ workflowId: input.workflowId,
2025
2217
  defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
2026
2218
  noProgressLimit: settings.goalNoProgressLimit,
2027
2219
  // A model-policy block takes precedence: it is deterministic (a budget
2028
2220
  // pause can clear on its own; a policy pause needs a model/policy change)
2029
2221
  // and rides the same visible-pause channel.
2030
- budgetBlocked: modelPolicyBlocked ?? budgetBlocked
2031
- });
2032
- if (decision.decision === "none" || decision.decision === "queue") {
2033
- return { action: decision.decision };
2034
- }
2035
- if (decision.decision === "paused") {
2036
- await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
2037
- {
2038
- type: "goal.paused",
2039
- payload: {
2040
- goalId: decision.goal.id,
2041
- actor: "system",
2042
- reason: decision.reason,
2043
- ...decision.goal.rationale ? { rationale: decision.goal.rationale } : {},
2044
- autoContinuations: decision.goal.autoContinuations,
2045
- noProgressStreak: decision.goal.noProgressStreak
2046
- }
2047
- }
2048
- ]);
2049
- return { action: "paused" };
2050
- }
2051
- const recheck = await getSessionGoal(db, input.workspaceId, input.sessionId);
2052
- if (!recheck || recheck.status !== "active" || recheck.version !== decision.goal.version) {
2053
- return { action: "none" };
2054
- }
2055
- const prompt = goalContinuationPrompt(decision.goal, decision.autoContinuation, decision.cap);
2056
- const update = await addSessionSystemUpdate(db, {
2057
- accountId: input.accountId,
2058
- workspaceId: input.workspaceId,
2059
- sessionId: input.sessionId,
2060
- kind: "goal_continuation",
2061
- classification: "info",
2062
- sourceId: decision.goal.id,
2063
- dedupeKey: `goal-continuation:${decision.goal.id}:${decision.goal.version}:${decision.autoContinuation}`,
2064
- summary: prompt,
2065
- payload: {
2066
- type: "goal_continuation",
2067
- goalId: decision.goal.id,
2068
- goalVersion: decision.goal.version,
2069
- autoContinuation: decision.autoContinuation,
2070
- maxAutoContinuations: decision.cap,
2071
- prompt,
2072
- policy: {
2073
- model: continuationModel,
2074
- reasoningEffort: continuationReasoningEffort,
2075
- tools: withFirstPartyTools(settings, session.tools),
2076
- sandboxBackend: session.sandboxBackend
2077
- }
2222
+ budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
2223
+ policy: {
2224
+ model: continuationModel,
2225
+ reasoningEffort: continuationReasoningEffort,
2226
+ tools: withFirstPartyTools(settings, session.tools),
2227
+ sandboxBackend: session.sandboxBackend
2078
2228
  },
2079
- lineage: { goalId: decision.goal.id }
2229
+ prompt: goalContinuationPrompt
2080
2230
  });
2081
- if (update.reason === "session_cancelled") return { action: "none" };
2082
- if (update.added && update.events.length > 0) {
2083
- await bus.publish(input.workspaceId, input.sessionId, update.events);
2231
+ if (decision.events.length > 0) {
2232
+ await bus.publish(input.workspaceId, input.sessionId, decision.events);
2084
2233
  }
2085
- await recordUsageEvent(db, {
2086
- accountId: input.accountId,
2087
- workspaceId: input.workspaceId,
2088
- eventType: "agent_run.created",
2089
- quantity: 1,
2090
- unit: "run",
2091
- sourceResourceType: "session_system_update",
2092
- sourceResourceId: update.update.id,
2093
- sessionId: input.sessionId,
2094
- initiator: { kind: "service", subjectId: "goal-continuation" },
2095
- initiatorContext: { goalId: decision.goal.id },
2096
- origin: "goal",
2097
- idempotencyKey: `agent_run.created:goal:${input.workspaceId}:${update.update.id}`
2098
- });
2099
- return { action: "continue" };
2234
+ return { action: decision.action };
2100
2235
  }
2101
2236
  return {
2102
2237
  enqueueGoalRetryWake: enqueueGoalRetryWake2,
@@ -2580,6 +2715,7 @@ function isPlainObject(value) {
2580
2715
  }
2581
2716
 
2582
2717
  // src/activities/run-input.ts
2718
+ import { createHash } from "crypto";
2583
2719
  import {
2584
2720
  getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
2585
2721
  getLatestRunState,
@@ -2624,6 +2760,104 @@ function resumeRunStateForCodexAccount(state, current) {
2624
2760
  stripReasoningIdentityFromSerializedRunState(state.serializedRunState)
2625
2761
  );
2626
2762
  }
2763
+ var MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
2764
+ var MODEL_IMAGE_CONTENT_TYPES = /* @__PURE__ */ new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]);
2765
+ var MODEL_FILE_CONTENT_TYPES = /* @__PURE__ */ new Set([
2766
+ "application/json",
2767
+ "application/pdf",
2768
+ "application/x-yaml",
2769
+ "application/yaml"
2770
+ ]);
2771
+ var BLOCKED_TEXT_CONTENT_TYPES = /* @__PURE__ */ new Set([
2772
+ "text/css",
2773
+ "text/html",
2774
+ "text/javascript",
2775
+ "text/xml"
2776
+ ]);
2777
+ function safeErrorType(error) {
2778
+ if (!(error instanceof Error)) return typeof error;
2779
+ const name = error.name.trim();
2780
+ return /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(name) ? name : "Error";
2781
+ }
2782
+ function modelAttachmentDescriptor(contentType) {
2783
+ const normalized = contentType.toLowerCase().split(";", 1)[0]?.trim() ?? "";
2784
+ if (MODEL_IMAGE_CONTENT_TYPES.has(normalized)) {
2785
+ return { kind: "image", contentType: normalized };
2786
+ }
2787
+ if (MODEL_FILE_CONTENT_TYPES.has(normalized) || normalized.startsWith("text/") && !BLOCKED_TEXT_CONTENT_TYPES.has(normalized)) {
2788
+ return { kind: "file", contentType: normalized };
2789
+ }
2790
+ return null;
2791
+ }
2792
+ async function modelAttachmentContentForFiles(files, readFileBytes) {
2793
+ const attachments = [];
2794
+ let remainingBytes = MAX_INLINE_MODEL_ATTACHMENT_BYTES;
2795
+ for (const file of files) {
2796
+ const descriptor = modelAttachmentDescriptor(file.contentType);
2797
+ const checksum = file.sha256?.trim().toLowerCase() ?? "";
2798
+ if (file.status !== "ready" || !descriptor || file.sizeBytes > remainingBytes || !/^[a-f0-9]{64}$/.test(checksum)) {
2799
+ continue;
2800
+ }
2801
+ try {
2802
+ const bytes = await readFileBytes(file);
2803
+ if (bytes.byteLength !== file.sizeBytes || bytes.byteLength > remainingBytes) {
2804
+ console.error("model attachment bytes did not match finalized metadata", {
2805
+ fileId: file.id,
2806
+ expectedSizeBytes: file.sizeBytes,
2807
+ actualSizeBytes: bytes.byteLength
2808
+ });
2809
+ continue;
2810
+ }
2811
+ if (createHash("sha256").update(bytes).digest("hex") !== checksum) {
2812
+ console.error("model attachment checksum did not match finalized metadata", {
2813
+ fileId: file.id
2814
+ });
2815
+ continue;
2816
+ }
2817
+ attachments.push({
2818
+ kind: descriptor.kind,
2819
+ fileId: file.id,
2820
+ filename: file.safeFilename,
2821
+ contentType: descriptor.contentType,
2822
+ dataUrl: `data:${descriptor.contentType};base64,${Buffer.from(bytes).toString("base64")}`
2823
+ });
2824
+ remainingBytes -= bytes.byteLength;
2825
+ } catch (error) {
2826
+ console.error("model attachment content read failed; retaining sandbox path fallback", {
2827
+ fileId: file.id,
2828
+ errorType: safeErrorType(error)
2829
+ });
2830
+ }
2831
+ }
2832
+ return attachments;
2833
+ }
2834
+ function withCurrentUserAttachmentContent(historyItems, attachments) {
2835
+ if (attachments.length === 0) return historyItems;
2836
+ let currentUserIndex = -1;
2837
+ for (let index = historyItems.length - 1; index >= 0; index -= 1) {
2838
+ const item = historyItems[index];
2839
+ if (item?.type === "message" && item.role === "user") {
2840
+ currentUserIndex = index;
2841
+ break;
2842
+ }
2843
+ }
2844
+ if (currentUserIndex < 0) return historyItems;
2845
+ const currentUser = historyItems[currentUserIndex];
2846
+ const existingContent = Array.isArray(currentUser.content) ? [...currentUser.content] : [{ type: "input_text", text: String(currentUser.content ?? "") }];
2847
+ const attachmentContent = attachments.map(
2848
+ (attachment) => attachment.kind === "image" ? { type: "input_image", image: attachment.dataUrl } : {
2849
+ type: "input_file",
2850
+ file: attachment.dataUrl,
2851
+ filename: attachment.filename
2852
+ }
2853
+ );
2854
+ const projected = [...historyItems];
2855
+ projected[currentUserIndex] = {
2856
+ ...currentUser,
2857
+ content: [...existingContent, ...attachmentContent]
2858
+ };
2859
+ return projected;
2860
+ }
2627
2861
  async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN, options) {
2628
2862
  if (!trigger) {
2629
2863
  throw new Error("Missing trigger event");
@@ -2649,11 +2883,17 @@ async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN,
2649
2883
  if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
2650
2884
  throw new Error("user.message payload is missing text");
2651
2885
  }
2652
- const attachmentContext = await userMessageAttachmentsContext(
2886
+ const resources = Array.isArray(payload.resources) ? payload.resources : [];
2887
+ const fileAttachments = await resolveUserMessageFileAttachments(
2653
2888
  db,
2654
2889
  trigger.workspaceId,
2655
- Array.isArray(payload.resources) ? payload.resources : []
2890
+ resources
2656
2891
  );
2892
+ const attachmentContext = userMessageAttachmentsContext(fileAttachments);
2893
+ const modelAttachments = options.readFileBytesForModel ? await modelAttachmentContentForFiles(
2894
+ fileAttachments.map((attachment) => attachment.file),
2895
+ options.readFileBytesForModel
2896
+ ) : [];
2657
2897
  return await messageInput(
2658
2898
  db,
2659
2899
  runtime,
@@ -2661,7 +2901,8 @@ async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN,
2661
2901
  trigger,
2662
2902
  void 0,
2663
2903
  joinInternalContext(internalContext, attachmentContext),
2664
- current
2904
+ current,
2905
+ modelAttachments
2665
2906
  );
2666
2907
  }
2667
2908
  if (trigger.type === "system.update.delivered") {
@@ -2736,10 +2977,13 @@ function systemUpdateContext(updates) {
2736
2977
  })
2737
2978
  ].join("\n");
2738
2979
  }
2739
- async function messageInput(db, runtime, agent, trigger, text, internalContext, current = NON_CODEX_TURN) {
2980
+ async function messageInput(db, runtime, agent, trigger, text, internalContext, current = NON_CODEX_TURN, modelAttachments = []) {
2740
2981
  const stored = await getActiveSessionHistoryItems2(db, trigger.workspaceId, trigger.sessionId);
2741
2982
  const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
2742
- const historyItems = applyCodexHistoryStrip(stored, current);
2983
+ const historyItems = withCurrentUserAttachmentContent(
2984
+ applyCodexHistoryStrip(stored, current),
2985
+ modelAttachments
2986
+ );
2743
2987
  return {
2744
2988
  input: await runtime.prepareInput(agent, {
2745
2989
  kind: "message",
@@ -2751,17 +2995,19 @@ async function messageInput(db, runtime, agent, trigger, text, internalContext,
2751
2995
  modelHistoryFromItems: true
2752
2996
  };
2753
2997
  }
2754
- async function userMessageAttachmentsContext(db, workspaceId, resources) {
2755
- const attachedFiles = [];
2998
+ async function resolveUserMessageFileAttachments(db, workspaceId, resources) {
2999
+ const attachments = [];
2756
3000
  for (const resource of resources) {
2757
- if (resource.kind !== "file") {
2758
- continue;
2759
- }
3001
+ if (resource.kind !== "file") continue;
2760
3002
  const file = await requireFile(db, workspaceId, resource.fileId);
2761
- attachedFiles.push(
2762
- `- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`
2763
- );
3003
+ attachments.push({ resource, file });
2764
3004
  }
3005
+ return attachments;
3006
+ }
3007
+ function userMessageAttachmentsContext(attachments) {
3008
+ const attachedFiles = attachments.map(
3009
+ ({ resource, file }) => `- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`
3010
+ );
2765
3011
  if (attachedFiles.length === 0) {
2766
3012
  return void 0;
2767
3013
  }
@@ -2918,24 +3164,32 @@ async function nextStreamEvent(iterator, context) {
2918
3164
  // src/sandbox-resume.ts
2919
3165
  import {
2920
3166
  acquireLease,
3167
+ beginSandboxRematerialization,
2921
3168
  commitWarmingToWarm,
3169
+ failSandboxRematerialization,
2922
3170
  failWarmingToCold,
2923
3171
  getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
3172
+ markSandboxRestoreVerifying,
2924
3173
  markWarmLeaseInstanceLost,
2925
3174
  persistWarmSnapshot,
3175
+ readWorkspaceArchiveCapturePreflight,
2926
3176
  readLease,
2927
3177
  recordWarmingSandboxCreated,
2928
3178
  releaseLeaseHolder,
2929
3179
  touchLeaseHolder,
3180
+ SandboxLeaseRecoveryBlockedError,
2930
3181
  SandboxLeaseSupersededError
2931
3182
  } from "@opengeni/db";
2932
3183
  import {
3184
+ captureVerifiedWorkspaceArchive,
3185
+ WorkspaceArchiveIntegrityError,
2933
3186
  establishSandboxSessionFromEnvelope,
2934
3187
  isProviderSandboxNotFoundError,
2935
- SandboxResumeStateUnavailableError,
2936
- serializeEstablishedSandboxEnvelope,
3188
+ requirePersistableReplacementSandboxEnvelope,
3189
+ serializeReplacementSandboxEnvelope,
2937
3190
  deletePriorPersistedSnapshot,
2938
- tagModalSandbox
3191
+ tagModalSandbox,
3192
+ verifySandboxExecReadiness
2939
3193
  } from "@opengeni/runtime";
2940
3194
  function sandboxLeaseHolderIdForAttempt(attemptId) {
2941
3195
  const normalized = attemptId.trim();
@@ -2968,30 +3222,13 @@ async function sleep(ms) {
2968
3222
  await new Promise((resolve) => setTimeout(resolve, ms));
2969
3223
  }
2970
3224
  async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS) {
2971
- if (established.backendId !== "modal") return;
2972
- const session = established.session;
2973
- const run = session.exec ?? session.execCommand;
2974
- if (!run) {
2975
- throw new Error("Modal sandbox session does not expose exec");
2976
- }
2977
- let timer;
2978
3225
  try {
2979
- await Promise.race([
2980
- run.call(session, {
2981
- cmd: "true",
2982
- yieldTimeMs: 1e3,
2983
- maxOutputTokens: 1e3
2984
- }),
2985
- new Promise((_, reject) => {
2986
- timer = setTimeout(
2987
- () => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
2988
- timeoutMs
2989
- );
2990
- if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
2991
- })
2992
- ]);
2993
- } finally {
2994
- if (timer) clearTimeout(timer);
3226
+ await verifySandboxExecReadiness(established, timeoutMs);
3227
+ } catch (error) {
3228
+ if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
3229
+ throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
3230
+ }
3231
+ throw error;
2995
3232
  }
2996
3233
  }
2997
3234
  var SnapshotTimeoutError = class extends Error {
@@ -3045,7 +3282,10 @@ async function terminateEstablishedSandbox(established) {
3045
3282
  try {
3046
3283
  await client.delete(established.sessionState);
3047
3284
  return true;
3048
- } catch {
3285
+ } catch (error) {
3286
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
3287
+ return true;
3288
+ }
3049
3289
  return false;
3050
3290
  }
3051
3291
  }
@@ -3064,7 +3304,10 @@ async function terminateEstablishedSandbox(established) {
3064
3304
  return true;
3065
3305
  }
3066
3306
  return false;
3067
- } catch {
3307
+ } catch (error) {
3308
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
3309
+ return true;
3310
+ }
3068
3311
  return false;
3069
3312
  }
3070
3313
  }
@@ -3088,27 +3331,23 @@ function workspaceArchiveFieldsFromEnvelope(envelope) {
3088
3331
  return null;
3089
3332
  }
3090
3333
  const previous = sessionState?.workspaceArchivePrev;
3334
+ const metadata = sessionState?.workspaceArchiveMeta;
3335
+ const previousMetadata = sessionState?.workspaceArchivePrevMeta;
3091
3336
  const capturedAt = sessionState?.workspaceArchiveAt;
3092
3337
  return {
3093
3338
  workspaceArchive: archive,
3339
+ ...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
3094
3340
  ...typeof previous === "string" && previous.length > 0 ? { workspaceArchivePrev: previous } : {},
3341
+ ...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
3095
3342
  ...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
3096
3343
  };
3097
3344
  }
3098
- function preserveWorkspaceArchivesOnResumeState(resumeState, archiveSource) {
3099
- const archiveFields = workspaceArchiveFieldsFromEnvelope(archiveSource);
3100
- if (!archiveFields) {
3101
- return resumeState;
3102
- }
3103
- const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
3104
- return {
3105
- ...resumeState ?? {},
3106
- ...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
3107
- sessionState: {
3108
- ...existingSessionState,
3109
- ...archiveFields
3110
- }
3111
- };
3345
+ function withoutProviderIdentity(envelope) {
3346
+ if (!envelope) return null;
3347
+ const sessionState = envelope.sessionState && typeof envelope.sessionState === "object" ? envelope.sessionState : null;
3348
+ if (!sessionState) return envelope;
3349
+ const { providerState: _providerState, ...providerIndependentState } = sessionState;
3350
+ return { ...envelope, sessionState: providerIndependentState };
3112
3351
  }
3113
3352
  async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal) {
3114
3353
  const { db, settings } = services;
@@ -3125,24 +3364,33 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3125
3364
  }
3126
3365
  try {
3127
3366
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
3128
- if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
3367
+ if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm" || lease.instanceId === null) {
3129
3368
  return false;
3130
3369
  }
3370
+ const preflight = await readWorkspaceArchiveCapturePreflight(db, {
3371
+ accountId: ids.accountId,
3372
+ workspaceId: ids.workspaceId,
3373
+ sandboxGroupId: ids.sandboxGroupId,
3374
+ expectedEpoch: leaseEpoch,
3375
+ expectedInstanceId: lease.instanceId,
3376
+ liveness: "warm"
3377
+ });
3378
+ if (!preflight) return false;
3131
3379
  if (signal?.aborted) {
3132
3380
  return false;
3133
3381
  }
3134
3382
  const sessionState = lease.resumeState && typeof lease.resumeState === "object" ? lease.resumeState.sessionState : void 0;
3135
3383
  const priorAtRaw = sessionState && typeof sessionState === "object" ? sessionState.workspaceArchiveAt : void 0;
3136
3384
  const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
3137
- if (Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
3385
+ if (preflight.archiveComplete && Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
3138
3386
  return false;
3139
3387
  }
3140
3388
  let timeout;
3141
3389
  const capturedAtMs = Date.now();
3142
- const capture = persistable.persistWorkspace();
3390
+ const capture = captureVerifiedWorkspaceArchive(session, capturedAtMs);
3143
3391
  capture.catch(() => void 0);
3144
3392
  let cancelListener;
3145
- const bytes = await Promise.race([
3393
+ const archive = await Promise.race([
3146
3394
  capture,
3147
3395
  ...signal ? [
3148
3396
  new Promise((resolve) => {
@@ -3163,7 +3411,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3163
3411
  }
3164
3412
  if (cancelListener) signal?.removeEventListener("abort", cancelListener);
3165
3413
  });
3166
- if (!bytes || bytes.length === 0) {
3414
+ if (!archive) {
3167
3415
  return false;
3168
3416
  }
3169
3417
  if (signal?.aborted) {
@@ -3177,7 +3425,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
3177
3425
  attemptId: ids.attemptId,
3178
3426
  sandboxGroupId: ids.sandboxGroupId,
3179
3427
  expectedEpoch: leaseEpoch,
3180
- workspaceArchive: Buffer.from(bytes).toString("base64"),
3428
+ expectedInstanceId: lease.instanceId,
3429
+ expectedWorkspaceGeneration: preflight.workspaceGeneration,
3430
+ workspaceArchive: archive.base64,
3431
+ workspaceArchiveMeta: archive.descriptor,
3181
3432
  minIntervalMs: intervalMs,
3182
3433
  capturedAtMs
3183
3434
  });
@@ -3247,6 +3498,15 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3247
3498
  if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
3248
3499
  holderLivenessTimer.unref();
3249
3500
  }
3501
+ if (acquired.role === "blocked") {
3502
+ await release();
3503
+ throw new SandboxLeaseRecoveryBlockedError(
3504
+ ids.sandboxGroupId,
3505
+ acquired.lease.leaseEpoch,
3506
+ acquired.code,
3507
+ acquired.lease.recovery
3508
+ );
3509
+ }
3250
3510
  if (acquired.role === "fenced") {
3251
3511
  await release();
3252
3512
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
@@ -3254,9 +3514,52 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3254
3514
  if (acquired.role === "spawner") {
3255
3515
  const expectedEpoch = acquired.lease.leaseEpoch;
3256
3516
  let createdEstablished = null;
3517
+ let rematerialization = null;
3257
3518
  try {
3258
3519
  const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
3259
- const spawnEnvelope = acquired.lease.resumeState ?? envelope;
3520
+ const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutProviderIdentity(envelope) : null;
3521
+ const spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
3522
+ const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
3523
+ if (acquired.lease.recovery.archive.status === "available" || workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
3524
+ const rematerializationId = crypto.randomUUID();
3525
+ const begun = await beginSandboxRematerialization(db, {
3526
+ accountId: ids.accountId,
3527
+ workspaceId: ids.workspaceId,
3528
+ sandboxGroupId: ids.sandboxGroupId,
3529
+ expectedEpoch,
3530
+ rematerializationId,
3531
+ archiveSource
3532
+ });
3533
+ if (begun.status !== "started") {
3534
+ if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
3535
+ throw new SandboxLeaseSupersededError(
3536
+ ids.sandboxGroupId,
3537
+ begun.lease?.leaseEpoch ?? expectedEpoch
3538
+ );
3539
+ }
3540
+ throw new SandboxLeaseRecoveryBlockedError(
3541
+ ids.sandboxGroupId,
3542
+ begun.lease?.leaseEpoch ?? expectedEpoch,
3543
+ begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
3544
+ begun.lease?.recovery ?? acquired.lease.recovery
3545
+ );
3546
+ }
3547
+ const selectedRevision = begun.lease.recovery.restore.selectedRevision;
3548
+ if (!selectedRevision) {
3549
+ throw new WorkspaceArchiveIntegrityError(
3550
+ "archive_metadata_invalid",
3551
+ "sandbox rematerialization selected no durable archive revision"
3552
+ );
3553
+ }
3554
+ rematerialization = { id: rematerializationId, selectedRevision };
3555
+ } else if (acquired.lease.recovery.archive.status !== "none") {
3556
+ throw new SandboxLeaseRecoveryBlockedError(
3557
+ ids.sandboxGroupId,
3558
+ expectedEpoch,
3559
+ "restore_degraded",
3560
+ acquired.lease.recovery
3561
+ );
3562
+ }
3260
3563
  const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
3261
3564
  sessionId: ids.sessionId,
3262
3565
  recovery: "create-or-restore",
@@ -3265,15 +3568,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3265
3568
  ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
3266
3569
  onSandboxCreated: async (created) => {
3267
3570
  createdEstablished = created;
3268
- const resumeEnvelope2 = preserveWorkspaceArchivesOnResumeState(
3269
- await serializeEstablishedSandboxEnvelope(created) ?? null,
3270
- spawnEnvelope
3571
+ const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
3572
+ await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
3573
+ created.backendId
3271
3574
  );
3272
3575
  const recorded = await recordWarmingSandboxCreated(db, {
3273
3576
  accountId: ids.accountId,
3274
3577
  workspaceId: ids.workspaceId,
3275
3578
  sandboxGroupId: ids.sandboxGroupId,
3276
3579
  expectedEpoch,
3580
+ rematerializationId: rematerialization?.id ?? null,
3277
3581
  instanceId: created.instanceId,
3278
3582
  resumeBackendId: created.backendId,
3279
3583
  resumeState: resumeEnvelope2,
@@ -3292,12 +3596,38 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3292
3596
  sandboxGroupId: ids.sandboxGroupId
3293
3597
  }).catch(() => void 0);
3294
3598
  }
3599
+ },
3600
+ onWorkspaceRestoreVerifying: async (descriptor) => {
3601
+ if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
3602
+ throw new WorkspaceArchiveIntegrityError(
3603
+ "archive_metadata_invalid",
3604
+ `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
3605
+ );
3606
+ }
3607
+ const verifying = await markSandboxRestoreVerifying(db, {
3608
+ accountId: ids.accountId,
3609
+ workspaceId: ids.workspaceId,
3610
+ sandboxGroupId: ids.sandboxGroupId,
3611
+ expectedEpoch,
3612
+ rematerializationId: rematerialization.id
3613
+ });
3614
+ if (!verifying.wrote) {
3615
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3616
+ }
3295
3617
  }
3296
3618
  });
3297
3619
  createdEstablished = established;
3298
3620
  await waitForSandboxExecReadiness(established);
3299
- const serializedResumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope;
3300
- const resumeEnvelope = established.origin === "restored" ? preserveWorkspaceArchivesOnResumeState(serializedResumeEnvelope, spawnEnvelope) : serializedResumeEnvelope;
3621
+ const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
3622
+ await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
3623
+ established.backendId
3624
+ );
3625
+ if (rematerialization && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
3626
+ throw new WorkspaceArchiveIntegrityError(
3627
+ "workspace_fingerprint_mismatch",
3628
+ "sandbox restore completed without the exact selected durable archive revision"
3629
+ );
3630
+ }
3301
3631
  const committed = await commitWarmingToWarm(db, {
3302
3632
  accountId: ids.accountId,
3303
3633
  workspaceId: ids.workspaceId,
@@ -3309,10 +3639,27 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3309
3639
  dataPlaneUrl: null,
3310
3640
  resumeBackendId: established.backendId,
3311
3641
  resumeState: resumeEnvelope,
3642
+ ...rematerialization ? {
3643
+ rematerialization: {
3644
+ id: rematerialization.id,
3645
+ verifiedRevision: rematerialization.selectedRevision
3646
+ }
3647
+ } : {},
3312
3648
  leaseTtlMs
3313
3649
  });
3314
3650
  if (!committed.committed || !committed.lease) {
3315
- await terminateEstablishedSandbox(established);
3651
+ const terminated = await terminateEstablishedSandbox(established);
3652
+ if (terminated && rematerialization) {
3653
+ await failSandboxRematerialization(db, {
3654
+ accountId: ids.accountId,
3655
+ workspaceId: ids.workspaceId,
3656
+ sandboxGroupId: ids.sandboxGroupId,
3657
+ expectedEpoch,
3658
+ rematerializationId: rematerialization.id,
3659
+ failureCode: committed.reason ?? "warm_commit_rejected",
3660
+ retryable: false
3661
+ });
3662
+ }
3316
3663
  await release();
3317
3664
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
3318
3665
  }
@@ -3325,12 +3672,24 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3325
3672
  }
3326
3673
  const terminated = await terminateEstablishedSandbox(createdEstablished);
3327
3674
  if (terminated) {
3328
- await failWarmingToCold(db, {
3329
- accountId: ids.accountId,
3330
- workspaceId: ids.workspaceId,
3331
- sandboxGroupId: ids.sandboxGroupId,
3332
- expectedEpoch
3333
- });
3675
+ if (rematerialization) {
3676
+ await failSandboxRematerialization(db, {
3677
+ accountId: ids.accountId,
3678
+ workspaceId: ids.workspaceId,
3679
+ sandboxGroupId: ids.sandboxGroupId,
3680
+ expectedEpoch,
3681
+ rematerializationId: rematerialization.id,
3682
+ failureCode: error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
3683
+ retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true
3684
+ });
3685
+ } else {
3686
+ await failWarmingToCold(db, {
3687
+ accountId: ids.accountId,
3688
+ workspaceId: ids.workspaceId,
3689
+ sandboxGroupId: ids.sandboxGroupId,
3690
+ expectedEpoch
3691
+ });
3692
+ }
3334
3693
  }
3335
3694
  await release();
3336
3695
  const warmingError = asSandboxWarmingError(
@@ -3361,7 +3720,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
3361
3720
  ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
3362
3721
  });
3363
3722
  } catch (error) {
3364
- if (!(error instanceof SandboxResumeStateUnavailableError) && !isProviderSandboxNotFoundError(ids.backend, error)) {
3723
+ if (!isProviderSandboxNotFoundError(ids.backend, error)) {
3365
3724
  throw error;
3366
3725
  }
3367
3726
  const marked = await markWarmLeaseInstanceLost(db, {
@@ -3430,6 +3789,15 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
3430
3789
  leaseTtlMs,
3431
3790
  warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs
3432
3791
  });
3792
+ if (acquired.role === "blocked") {
3793
+ await release();
3794
+ throw new SandboxLeaseRecoveryBlockedError(
3795
+ ids.sandboxGroupId,
3796
+ acquired.lease.leaseEpoch,
3797
+ acquired.code,
3798
+ acquired.lease.recovery
3799
+ );
3800
+ }
3433
3801
  if (acquired.role === "fenced") {
3434
3802
  await release();
3435
3803
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
@@ -3471,6 +3839,14 @@ async function waitForWarm(services, ids) {
3471
3839
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3472
3840
  }
3473
3841
  if (lease.liveness === "cold") {
3842
+ if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
3843
+ throw new SandboxLeaseRecoveryBlockedError(
3844
+ ids.sandboxGroupId,
3845
+ lease.leaseEpoch,
3846
+ lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
3847
+ lease.recovery
3848
+ );
3849
+ }
3474
3850
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
3475
3851
  }
3476
3852
  }
@@ -3478,13 +3854,29 @@ async function waitForWarm(services, ids) {
3478
3854
  }
3479
3855
 
3480
3856
  // src/sandbox-routing.ts
3481
- import { getSandbox, readActiveSandbox } from "@opengeni/db";
3857
+ import {
3858
+ advanceWorkspaceGenerationForRetainedProcess,
3859
+ advanceWorkspaceGeneration,
3860
+ retainWorkspaceMutationProcess,
3861
+ settleRetainedProcess,
3862
+ verifyRetainedProcessMutationSettlement,
3863
+ verifyWorkspaceMutationSettlement,
3864
+ getSandbox,
3865
+ markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
3866
+ readLease as readLease2,
3867
+ readActiveSandbox
3868
+ } from "@opengeni/db";
3482
3869
  import {
3483
3870
  buildSelfhostedBackendSession,
3871
+ establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
3872
+ isProviderSandboxGoneDuringRoutedOperation,
3873
+ isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
3484
3874
  makeActiveBackendResolver,
3485
3875
  NatsControlRpc,
3486
3876
  NatsOpStreamTransport,
3487
- RoutingSandboxSession
3877
+ RoutingBackendRecoveryRequiredError,
3878
+ RoutingSandboxSession,
3879
+ verifySandboxExecReadiness as verifySandboxExecReadiness2
3488
3880
  } from "@opengeni/runtime";
3489
3881
  function relayConfigFromSettings(settings) {
3490
3882
  const raw = settings.selfhostedRelayUrl?.trim();
@@ -3510,6 +3902,231 @@ function selfhostedResolverTimeouts(settings) {
3510
3902
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
3511
3903
  return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
3512
3904
  }
3905
+ function homeRouteRecoveryDisposition(lease) {
3906
+ if (!lease) return "unrecoverable";
3907
+ if (lease.liveness === "warming") return "pending";
3908
+ if (lease.liveness === "draining") return "superseded";
3909
+ if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
3910
+ return "pending";
3911
+ }
3912
+ if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
3913
+ return "degraded";
3914
+ }
3915
+ return "unrecoverable";
3916
+ }
3917
+ function homeRouteRecoveryError(lease, fallbackEpoch) {
3918
+ return new RoutingBackendRecoveryRequiredError(
3919
+ "resolve_home_backend",
3920
+ lease?.leaseEpoch ?? fallbackEpoch,
3921
+ homeRouteRecoveryDisposition(lease)
3922
+ );
3923
+ }
3924
+ function providerIdentityFromResumeState(resumeState) {
3925
+ const sessionState = resumeState.sessionState && typeof resumeState.sessionState === "object" ? resumeState.sessionState : null;
3926
+ const providerState = sessionState?.providerState && typeof sessionState.providerState === "object" ? sessionState.providerState : null;
3927
+ for (const field of [
3928
+ "sandboxId",
3929
+ "instanceId",
3930
+ "id",
3931
+ "hostId",
3932
+ "containerId",
3933
+ "workspaceRootPath",
3934
+ "agentId"
3935
+ ]) {
3936
+ const value = providerState?.[field];
3937
+ if (typeof value === "string" && value.length > 0) return value;
3938
+ }
3939
+ return null;
3940
+ }
3941
+ async function resolveCurrentHomeBackend(services, ids, established) {
3942
+ const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
3943
+ const fallbackEpoch = lease?.leaseEpoch ?? 0;
3944
+ if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
3945
+ throw homeRouteRecoveryError(lease, fallbackEpoch);
3946
+ }
3947
+ if (lease.instanceId === established.instanceId) {
3948
+ services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
3949
+ return {
3950
+ session: established.session,
3951
+ sandboxId: null,
3952
+ kind: established.backendId,
3953
+ leaseEpoch: lease.leaseEpoch,
3954
+ providerInstanceId: lease.instanceId
3955
+ };
3956
+ }
3957
+ const resumeState = lease.resumeState;
3958
+ const resumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
3959
+ if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
3960
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
3961
+ }
3962
+ let rebound;
3963
+ try {
3964
+ rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
3965
+ sessionId: ids.sessionId,
3966
+ recovery: "resume-only",
3967
+ backendOverride: resumeBackend
3968
+ });
3969
+ await verifySandboxExecReadiness2(rebound);
3970
+ } catch (error) {
3971
+ if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
3972
+ throw new RoutingBackendRecoveryRequiredError(
3973
+ "resolve_home_backend",
3974
+ lease.leaseEpoch,
3975
+ "pending"
3976
+ );
3977
+ }
3978
+ const marked = await markWarmLeaseInstanceLost2(services.db, {
3979
+ accountId: ids.accountId,
3980
+ workspaceId: ids.workspaceId,
3981
+ sandboxGroupId: ids.sandboxGroupId,
3982
+ expectedEpoch: lease.leaseEpoch,
3983
+ expectedInstanceId: lease.instanceId,
3984
+ diagnostic: "provider_not_found_during_home_route_rebind"
3985
+ });
3986
+ if (marked.status === "marked") {
3987
+ await services.onHomeSandboxLost?.({
3988
+ sandboxGroupId: ids.sandboxGroupId,
3989
+ instanceId: lease.instanceId,
3990
+ leaseEpoch: marked.lease.leaseEpoch
3991
+ });
3992
+ }
3993
+ throw new RoutingBackendRecoveryRequiredError(
3994
+ "resolve_home_backend",
3995
+ marked.lease?.leaseEpoch ?? lease.leaseEpoch,
3996
+ marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
3997
+ );
3998
+ }
3999
+ if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
4000
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
4001
+ }
4002
+ services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
4003
+ Object.assign(established, rebound);
4004
+ return {
4005
+ session: rebound.session,
4006
+ sandboxId: null,
4007
+ kind: rebound.backendId,
4008
+ leaseEpoch: lease.leaseEpoch,
4009
+ providerInstanceId: lease.instanceId
4010
+ };
4011
+ }
4012
+ function beforePersistableHomeMutation(services, ids, home) {
4013
+ const fence = ids.workspaceMutationFence;
4014
+ if (!home || !fence) return void 0;
4015
+ return async ({ op, backend }) => {
4016
+ if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
4017
+ return null;
4018
+ }
4019
+ return await advanceWorkspaceGeneration(services.db, {
4020
+ accountId: fence.accountId,
4021
+ workspaceId: ids.workspaceId,
4022
+ sessionId: ids.sessionId,
4023
+ turnId: fence.turnId,
4024
+ executionGeneration: fence.executionGeneration,
4025
+ attemptId: fence.attemptId,
4026
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
4027
+ sandboxGroupId: home.sandboxGroupId,
4028
+ expectedEpoch: backend.leaseEpoch,
4029
+ expectedInstanceId: backend.providerInstanceId,
4030
+ operation: op
4031
+ });
4032
+ };
4033
+ }
4034
+ function afterPersistableHomeMutation(services, ids, home) {
4035
+ const fence = ids.workspaceMutationFence;
4036
+ if (!home || !fence) return void 0;
4037
+ return async ({ op, backend, admission, outcome, retainedProcess }) => {
4038
+ if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
4039
+ return;
4040
+ }
4041
+ const exactAdmission = admission;
4042
+ if (outcome === "resolved" && retainedProcess) {
4043
+ await retainWorkspaceMutationProcess(services.db, {
4044
+ accountId: fence.accountId,
4045
+ workspaceId: ids.workspaceId,
4046
+ sessionId: ids.sessionId,
4047
+ processId: retainedProcess.id,
4048
+ providerSessionId: retainedProcess.providerSessionId,
4049
+ admissionId: exactAdmission.id,
4050
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
4051
+ operation: op,
4052
+ owner: {
4053
+ kind: "turn",
4054
+ turnId: fence.turnId,
4055
+ executionGeneration: fence.executionGeneration,
4056
+ attemptId: fence.attemptId,
4057
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
4058
+ sandboxGroupId: home.sandboxGroupId,
4059
+ expectedEpoch: backend.leaseEpoch,
4060
+ expectedInstanceId: backend.providerInstanceId,
4061
+ routeKind: exactAdmission.routeKind,
4062
+ routeTargetId: exactAdmission.routeTargetId,
4063
+ routeEpoch: exactAdmission.routeEpoch
4064
+ }
4065
+ });
4066
+ return;
4067
+ }
4068
+ await verifyWorkspaceMutationSettlement(services.db, {
4069
+ accountId: fence.accountId,
4070
+ workspaceId: ids.workspaceId,
4071
+ sessionId: ids.sessionId,
4072
+ turnId: fence.turnId,
4073
+ executionGeneration: fence.executionGeneration,
4074
+ attemptId: fence.attemptId,
4075
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
4076
+ sandboxGroupId: home.sandboxGroupId,
4077
+ expectedEpoch: backend.leaseEpoch,
4078
+ expectedInstanceId: backend.providerInstanceId,
4079
+ admission: exactAdmission,
4080
+ operation: op,
4081
+ outcome
4082
+ });
4083
+ };
4084
+ }
4085
+ function beforeRetainedProcessMutation(services, ids) {
4086
+ const fence = ids.workspaceMutationFence;
4087
+ if (!fence) return void 0;
4088
+ return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
4089
+ accountId: fence.accountId,
4090
+ workspaceId: ids.workspaceId,
4091
+ sessionId: ids.sessionId,
4092
+ processId: process2.id,
4093
+ operation: op
4094
+ });
4095
+ }
4096
+ function afterRetainedProcessMutation(services, ids) {
4097
+ const fence = ids.workspaceMutationFence;
4098
+ if (!fence) return void 0;
4099
+ return async ({ op, process: process2, admission, outcome }) => {
4100
+ if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
4101
+ throw new Error("Retained-process mutation settlement lacked its exact admission");
4102
+ }
4103
+ await verifyRetainedProcessMutationSettlement(services.db, {
4104
+ accountId: fence.accountId,
4105
+ workspaceId: ids.workspaceId,
4106
+ sessionId: ids.sessionId,
4107
+ processId: process2.id,
4108
+ admission,
4109
+ operation: op,
4110
+ outcome
4111
+ });
4112
+ };
4113
+ }
4114
+ function settleRetainedProcessForTurn(services, ids) {
4115
+ const fence = ids.workspaceMutationFence;
4116
+ if (!fence) return void 0;
4117
+ return async ({ process: process2, proof }) => {
4118
+ await settleRetainedProcess(services.db, {
4119
+ accountId: fence.accountId,
4120
+ workspaceId: ids.workspaceId,
4121
+ sessionId: ids.sessionId,
4122
+ processId: process2.id,
4123
+ outcome: proof.outcome,
4124
+ exitCode: proof.exitCode,
4125
+ reason: proof.reason,
4126
+ idleGraceMs: services.settings.sandboxIdleGraceMs
4127
+ });
4128
+ };
4129
+ }
3513
4130
  function controlRpcFactory(bus) {
3514
4131
  return () => new NatsControlRpc(async () => {
3515
4132
  if (!bus) {
@@ -3520,6 +4137,11 @@ function controlRpcFactory(bus) {
3520
4137
  }
3521
4138
  function wrapTurnBoxWithRouting(services, ids, established) {
3522
4139
  const { db, settings, bus, onOp } = services;
4140
+ const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
4141
+ const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
4142
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
4143
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
4144
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
3523
4145
  const resolver = makeActiveBackendResolver({
3524
4146
  workspaceId: ids.workspaceId,
3525
4147
  defaultBackend: established.session,
@@ -3565,7 +4187,20 @@ function wrapTurnBoxWithRouting(services, ids, established) {
3565
4187
  // group box established this turn), a mid-turn clear-to-null must NOT fall back to
3566
4188
  // the pinned machine — passing defaultIsHome:false makes the null branch throw typed
3567
4189
  // `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
3568
- ...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {}
4190
+ ...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
4191
+ ...ids.homeLease ? {
4192
+ resolveDefaultBackend: () => resolveCurrentHomeBackend(
4193
+ services,
4194
+ {
4195
+ workspaceId: ids.workspaceId,
4196
+ sessionId: ids.sessionId,
4197
+ accountId: ids.homeLease.accountId,
4198
+ sandboxGroupId: ids.homeLease.sandboxGroupId,
4199
+ backend: ids.homeLease.backend
4200
+ },
4201
+ established
4202
+ )
4203
+ } : {}
3569
4204
  });
3570
4205
  const proxy = new RoutingSandboxSession({
3571
4206
  // Seed the DEFAULT backend (the established group box) at construction so
@@ -3577,18 +4212,67 @@ function wrapTurnBoxWithRouting(services, ids, established) {
3577
4212
  defaultResolved: {
3578
4213
  session: established.session,
3579
4214
  sandboxId: null,
3580
- kind: established.backendId
4215
+ kind: established.backendId,
4216
+ ...ids.homeLease ? {
4217
+ leaseEpoch: ids.homeLease.leaseEpoch,
4218
+ providerInstanceId: ids.homeLease.instanceId
4219
+ } : {}
3581
4220
  },
3582
4221
  readPointer: async () => {
4222
+ if (!routingEnabled(settings)) {
4223
+ return { activeSandboxId: null, activeEpoch: 0 };
4224
+ }
3583
4225
  const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
3584
4226
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
3585
4227
  },
3586
- resolveActiveBackend: resolver
4228
+ resolveActiveBackend: resolver,
4229
+ ...beforeMutation ? { beforeMutation } : {},
4230
+ ...afterMutation ? { afterMutation } : {},
4231
+ ...beforeProcessMutation ? { beforeProcessMutation } : {},
4232
+ ...afterProcessMutation ? { afterProcessMutation } : {},
4233
+ ...settleProcess ? { settleProcess } : {},
4234
+ ...ids.homeLease ? {
4235
+ onDefaultBackendError: async ({
4236
+ error,
4237
+ backend
4238
+ }) => {
4239
+ const home = ids.homeLease;
4240
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
4241
+ const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
4242
+ const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
4243
+ const marked = await markWarmLeaseInstanceLost2(db, {
4244
+ accountId: home.accountId,
4245
+ workspaceId: ids.workspaceId,
4246
+ sandboxGroupId: home.sandboxGroupId,
4247
+ expectedEpoch,
4248
+ expectedInstanceId,
4249
+ diagnostic: "provider_not_found_during_routed_operation"
4250
+ });
4251
+ if (marked.status === "marked") {
4252
+ await services.onHomeSandboxLost?.({
4253
+ sandboxGroupId: home.sandboxGroupId,
4254
+ instanceId: expectedInstanceId,
4255
+ leaseEpoch: marked.lease.leaseEpoch
4256
+ });
4257
+ }
4258
+ const lease = marked.lease;
4259
+ const restore = lease?.recovery.restore.status;
4260
+ return {
4261
+ leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
4262
+ recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
4263
+ };
4264
+ }
4265
+ } : {}
3587
4266
  });
3588
4267
  return { ...established, session: proxy };
3589
4268
  }
3590
4269
  function wrapLazyTurnBoxWithRouting(services, ids, args) {
3591
4270
  const { db, settings, bus, onOp } = services;
4271
+ const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
4272
+ const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
4273
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
4274
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
4275
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
3592
4276
  const syntheticSession = {
3593
4277
  state: { manifest: args.agentDefaultManifest }
3594
4278
  };
@@ -3628,6 +4312,19 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
3628
4312
  resolveActiveBackend: async (pointer) => {
3629
4313
  if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
3630
4314
  const provisioned = await args.provisioner.get();
4315
+ if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
4316
+ return resolveCurrentHomeBackend(
4317
+ services,
4318
+ {
4319
+ workspaceId: ids.workspaceId,
4320
+ sessionId: ids.sessionId,
4321
+ accountId: args.homeLeaseIdentity.accountId,
4322
+ sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
4323
+ backend: args.homeLeaseIdentity.backend
4324
+ },
4325
+ provisioned.established
4326
+ );
4327
+ }
3631
4328
  return {
3632
4329
  session: provisioned.established.session,
3633
4330
  sandboxId: null,
@@ -3635,7 +4332,45 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
3635
4332
  };
3636
4333
  }
3637
4334
  return routedResolver(pointer);
3638
- }
4335
+ },
4336
+ ...beforeMutation ? { beforeMutation } : {},
4337
+ ...afterMutation ? { afterMutation } : {},
4338
+ ...beforeProcessMutation ? { beforeProcessMutation } : {},
4339
+ ...afterProcessMutation ? { afterProcessMutation } : {},
4340
+ ...settleProcess ? { settleProcess } : {},
4341
+ ...args.homeLeaseIdentity ? {
4342
+ onDefaultBackendError: async ({
4343
+ error,
4344
+ backend
4345
+ }) => {
4346
+ const home = args.homeLeaseIdentity;
4347
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
4348
+ if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
4349
+ return null;
4350
+ }
4351
+ const marked = await markWarmLeaseInstanceLost2(db, {
4352
+ accountId: home.accountId,
4353
+ workspaceId: ids.workspaceId,
4354
+ sandboxGroupId: home.sandboxGroupId,
4355
+ expectedEpoch: backend.leaseEpoch,
4356
+ expectedInstanceId: backend.providerInstanceId,
4357
+ diagnostic: "provider_not_found_during_routed_operation"
4358
+ });
4359
+ if (marked.status === "marked") {
4360
+ await services.onHomeSandboxLost?.({
4361
+ sandboxGroupId: home.sandboxGroupId,
4362
+ instanceId: backend.providerInstanceId,
4363
+ leaseEpoch: marked.lease.leaseEpoch
4364
+ });
4365
+ }
4366
+ const lease = marked.lease;
4367
+ const restore = lease?.recovery.restore.status;
4368
+ return {
4369
+ leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
4370
+ recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
4371
+ };
4372
+ }
4373
+ } : {}
3639
4374
  });
3640
4375
  return {
3641
4376
  client: args.client,
@@ -3705,7 +4440,7 @@ function makeTurnOpJournal(context, details) {
3705
4440
  }
3706
4441
 
3707
4442
  // src/observability-metrics.ts
3708
- import { createHash } from "crypto";
4443
+ import { createHash as createHash2 } from "crypto";
3709
4444
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3710
4445
  import {
3711
4446
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
@@ -4196,7 +4931,7 @@ function stableAccountHash(credentialId) {
4196
4931
  if (!credentialId) {
4197
4932
  return "none";
4198
4933
  }
4199
- return createHash("sha256").update(credentialId).digest("hex").slice(0, 12);
4934
+ return createHash2("sha256").update(credentialId).digest("hex").slice(0, 12);
4200
4935
  }
4201
4936
  function modelCallAccountContext(input) {
4202
4937
  const accountChangedFromPrevCall = input.isFirstCallOfTurn && input.servingCredentialId !== null && input.priorSessionCredentialId !== null && input.priorSessionCredentialId !== input.servingCredentialId;
@@ -4370,7 +5105,7 @@ function scrubFreeText(value) {
4370
5105
  }
4371
5106
 
4372
5107
  // src/activities/workspace-capture.ts
4373
- import { createHash as createHash2 } from "crypto";
5108
+ import { createHash as createHash3 } from "crypto";
4374
5109
  import {
4375
5110
  deleteWorkspaceCaptureRows,
4376
5111
  insertFailedWorkspaceCapture,
@@ -4991,7 +5726,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
4991
5726
  return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
4992
5727
  }
4993
5728
  function sha256(bytes) {
4994
- return createHash2("sha256").update(bytes).digest("hex");
5729
+ return createHash3("sha256").update(bytes).digest("hex");
4995
5730
  }
4996
5731
  function utf8(s) {
4997
5732
  return new TextEncoder().encode(s);
@@ -5071,7 +5806,7 @@ import {
5071
5806
  evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
5072
5807
  readTurnExecutionPolicyV1
5073
5808
  } from "@opengeni/contracts";
5074
- import { createHash as createHash3, randomUUID } from "crypto";
5809
+ import { createHash as createHash4, randomUUID } from "crypto";
5075
5810
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
5076
5811
  function turnExecutionPolicyBillingIdentity(policy) {
5077
5812
  return {
@@ -5102,17 +5837,17 @@ function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds
5102
5837
  return downloads.filter((download) => !materializedFileIds.has(download.fileId));
5103
5838
  }
5104
5839
  function codexWorkspaceMetricKey(workspaceId) {
5105
- return createHash3("sha256").update(workspaceId).digest("hex").slice(0, 12);
5840
+ return createHash4("sha256").update(workspaceId).digest("hex").slice(0, 12);
5106
5841
  }
5107
5842
  function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
5108
- const hex = createHash3("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5843
+ const hex = createHash4("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
5109
5844
  hex[12] = "5";
5110
5845
  hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
5111
5846
  const value = hex.join("");
5112
5847
  return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
5113
5848
  }
5114
5849
  function isWorkerShutdownCancellation(error) {
5115
- return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
5850
+ return error instanceof CancelledFailure2 && error.message === "WORKER_SHUTDOWN";
5116
5851
  }
5117
5852
  function shouldRunTurnEndWorkspacePersistence(input) {
5118
5853
  return input.activityStatus !== "cancelled" && !input.cancellationRequested;
@@ -5179,6 +5914,7 @@ async function drainAttemptOwnedSandboxWriters(input) {
5179
5914
  );
5180
5915
  await input.toolCancellationFence.waitForQuiescence();
5181
5916
  }
5917
+ await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
5182
5918
  await input.toolspaceTokenRenewal?.stop();
5183
5919
  await input.runCredentialRenewal?.stop();
5184
5920
  }
@@ -5670,6 +6406,9 @@ function computerToolModeForTurn(resolvedModel) {
5670
6406
  }
5671
6407
  return "hosted";
5672
6408
  }
6409
+ function modelAcceptsTypedAttachmentContentForTurn(resolvedModel) {
6410
+ return resolvedModel === null || resolvedModel.provider.api === "responses";
6411
+ }
5673
6412
  var TurnOperationCancelledError = class extends Error {
5674
6413
  constructor(reason) {
5675
6414
  super("Turn operation was cancelled with its owning turn", {
@@ -5680,9 +6419,9 @@ var TurnOperationCancelledError = class extends Error {
5680
6419
  name = "TurnOperationCancelledError";
5681
6420
  };
5682
6421
  function turnOperationCancellationFailure(error) {
5683
- if (error instanceof CancelledFailure) return error;
6422
+ if (error instanceof CancelledFailure2) return error;
5684
6423
  if (!(error instanceof TurnOperationCancelledError)) return null;
5685
- return error.reason instanceof CancelledFailure ? error.reason : new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
6424
+ return error.reason instanceof CancelledFailure2 ? error.reason : new CancelledFailure2("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
5686
6425
  }
5687
6426
  function throwIfTurnOperationCancelled(signal) {
5688
6427
  if (signal?.aborted) {
@@ -5992,12 +6731,89 @@ function createRunAgentTurnActivity(services) {
5992
6731
  codexLeaseHeartbeatTimer.unref?.();
5993
6732
  };
5994
6733
  let resolvedSandbox = null;
6734
+ const requireResolvedSandboxForMutation = (message) => {
6735
+ if (!resolvedSandbox) throw new Error(message);
6736
+ return resolvedSandbox;
6737
+ };
5995
6738
  let machinePrimarySession = null;
5996
6739
  let lazyOwnedSandbox = null;
5997
6740
  let turnSandboxProvisioner = null;
5998
6741
  let setupBoxSession = null;
6742
+ const onHomeSandboxRebound = (rebound) => {
6743
+ const current = resolvedSandbox;
6744
+ const previousSession = current?.established.session;
6745
+ const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
6746
+ setupBoxSession = rebound.established.session;
6747
+ if (!current) return;
6748
+ current.leaseEpoch = rebound.leaseEpoch;
6749
+ current.established = preserveRoutingProxy ? {
6750
+ ...current.established,
6751
+ client: rebound.established.client,
6752
+ // Keep the stable SDK-facing proxy; only its resolver changes the
6753
+ // underlying backend. The worker's setupBoxSession above is raw.
6754
+ session: previousSession,
6755
+ sessionState: rebound.established.sessionState,
6756
+ instanceId: rebound.established.instanceId,
6757
+ backendId: rebound.established.backendId,
6758
+ ...rebound.established.origin ? { origin: rebound.established.origin } : {},
6759
+ ...rebound.established.restoredArchive ? { restoredArchive: rebound.established.restoredArchive } : {}
6760
+ } : rebound.established;
6761
+ };
5999
6762
  let sandboxHolderId = null;
6000
6763
  let sandboxGroupId = null;
6764
+ const runWorkspaceMutationForSandbox = async (sandbox, operation, mutation) => {
6765
+ if (sandbox.established.backendId === "selfhosted") return await mutation();
6766
+ if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
6767
+ throw new Error("Workspace mutation attempted before exact turn sandbox admission");
6768
+ }
6769
+ const identity = {
6770
+ accountId: input.accountId,
6771
+ workspaceId: input.workspaceId,
6772
+ sessionId: input.sessionId,
6773
+ turnId,
6774
+ executionGeneration,
6775
+ attemptId: input.attemptId,
6776
+ holderId: sandboxHolderId,
6777
+ sandboxGroupId,
6778
+ expectedEpoch: sandbox.leaseEpoch,
6779
+ expectedInstanceId: sandbox.established.instanceId,
6780
+ operation
6781
+ };
6782
+ const admission = await advanceWorkspaceGeneration2(db, identity);
6783
+ let result;
6784
+ try {
6785
+ result = await mutation();
6786
+ } catch (providerError) {
6787
+ try {
6788
+ await verifyWorkspaceMutationSettlement2(db, {
6789
+ ...identity,
6790
+ admission,
6791
+ outcome: "rejected"
6792
+ });
6793
+ } catch (settlementError) {
6794
+ throw new RoutingMutationOutcomeUnknownError(
6795
+ operation,
6796
+ `Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
6797
+ { cause: settlementError }
6798
+ );
6799
+ }
6800
+ throw providerError;
6801
+ }
6802
+ try {
6803
+ await verifyWorkspaceMutationSettlement2(db, {
6804
+ ...identity,
6805
+ admission,
6806
+ outcome: "resolved"
6807
+ });
6808
+ } catch (settlementError) {
6809
+ throw new RoutingMutationOutcomeUnknownError(
6810
+ operation,
6811
+ `Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
6812
+ { cause: settlementError }
6813
+ );
6814
+ }
6815
+ return result;
6816
+ };
6001
6817
  let leaseHeartbeatTimer;
6002
6818
  let gitCredentialRenewals = [];
6003
6819
  let gitCredentialRenewalClosed = false;
@@ -6715,6 +7531,57 @@ function createRunAgentTurnActivity(services) {
6715
7531
  codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
6716
7532
  codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
6717
7533
  if (codexLeaseHeld) startCodexLeaseHeartbeat();
7534
+ const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
7535
+ const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
7536
+ const fencedInFlight = leased.reused || continuationCodexCredentialId !== null && continuationCodexCredentialId === effectiveCodexCredentialId;
7537
+ const shadowResult = await publishCodexFleetShadowDecisionV1({
7538
+ enabled: settings.codexFleetPolicyShadowEnabled,
7539
+ decision: {
7540
+ accounts: leased.accounts,
7541
+ actualCredentialId: effectiveCodexCredentialId,
7542
+ actualOutcome,
7543
+ actualReason,
7544
+ affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
7545
+ fencedInFlight,
7546
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
7547
+ now: /* @__PURE__ */ new Date(),
7548
+ aliasSeed: randomUUID()
7549
+ },
7550
+ publish
7551
+ });
7552
+ if (shadowResult.outcome === "published") {
7553
+ const shadowPayload = shadowResult.payload;
7554
+ observability.incrementCounter({
7555
+ name: "opengeni_codex_fleet_shadow_decisions_total",
7556
+ help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
7557
+ labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
7558
+ });
7559
+ observability.info("Codex adaptive fleet shadow decision", {
7560
+ workspaceId: input.workspaceId,
7561
+ policyVersion: shadowPayload.replay.policyVersion,
7562
+ inputFingerprint: shadowPayload.replay.inputFingerprint,
7563
+ decisionFingerprint: shadowPayload.replay.decisionFingerprint,
7564
+ actualOutcome: shadowPayload.actual.outcome,
7565
+ shadowOutcome: shadowPayload.replay.decision.outcome,
7566
+ comparison: shadowPayload.comparison,
7567
+ candidateCount: shadowPayload.replay.input.candidates.length,
7568
+ truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
7569
+ payloadBytes: shadowResult.payloadBytes
7570
+ });
7571
+ } else if (shadowResult.outcome === "failed") {
7572
+ observability.incrementCounter({
7573
+ name: "opengeni_codex_fleet_shadow_errors_total",
7574
+ help: "Shadow decision build/publication failures.",
7575
+ labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
7576
+ });
7577
+ observability.warn("Codex adaptive fleet shadow decision failed open", {
7578
+ workspaceId: input.workspaceId,
7579
+ stage: shadowResult.stage,
7580
+ reason: shadowResult.reason,
7581
+ errorName: shadowResult.errorName,
7582
+ payloadBytes: shadowResult.payloadBytes
7583
+ });
7584
+ }
6718
7585
  const eligibleCount = leased.accounts.filter(
6719
7586
  (account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
6720
7587
  ).length;
@@ -7437,7 +8304,7 @@ function createRunAgentTurnActivity(services) {
7437
8304
  gitTokens: sandboxGitTokens ?? {},
7438
8305
  expiresAt: sandboxGitTokenExpiresAt ?? {}
7439
8306
  } : void 0;
7440
- const attachGitCredentialRenewal = async (tokenSession, initial) => {
8307
+ const attachGitCredentialRenewal = async (tokenSession, initial, initialSandbox) => {
7441
8308
  if (!initial || initial.bindings.length === 0) return;
7442
8309
  const previous = gitCredentialRenewals;
7443
8310
  gitCredentialRenewals = [];
@@ -7476,14 +8343,22 @@ function createRunAgentTurnActivity(services) {
7476
8343
  throw new Error("credential renewal produced no binding token");
7477
8344
  }
7478
8345
  const runAs = sandboxRunAs(runSettings);
7479
- await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
7480
- ...runAs ? { runAs } : {},
7481
- ...toolCancellationFenceRef.current ? {
7482
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
7483
- toolCancellationFenceRef.current
7484
- )
7485
- } : {}
7486
- });
8346
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
8347
+ if (!targetSandbox) {
8348
+ throw new Error("Git credential renewal has no exact sandbox lease target");
8349
+ }
8350
+ await runWorkspaceMutationForSandbox(
8351
+ targetSandbox,
8352
+ "gitCredentialRenewal",
8353
+ async () => await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
8354
+ ...runAs ? { runAs } : {},
8355
+ ...toolCancellationFenceRef.current ? {
8356
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8357
+ toolCancellationFenceRef.current
8358
+ )
8359
+ } : {}
8360
+ })
8361
+ );
7487
8362
  },
7488
8363
  onSuccess: ({ providers: renewedProviders }) => {
7489
8364
  for (const provider of renewedProviders) {
@@ -7518,7 +8393,7 @@ function createRunAgentTurnActivity(services) {
7518
8393
  }
7519
8394
  gitCredentialRenewals = controllers;
7520
8395
  };
7521
- const attachToolspaceTokenRenewal = async (tokenSession, initialExpiresAt = sandboxToolspaceTokenExpiresAt) => {
8396
+ const attachToolspaceTokenRenewal = async (tokenSession, initialExpiresAt = sandboxToolspaceTokenExpiresAt, initialSandbox) => {
7522
8397
  if (!sandboxToolspaceToken || !initialExpiresAt) return;
7523
8398
  const previous = toolspaceTokenRenewal;
7524
8399
  toolspaceTokenRenewal = null;
@@ -7527,18 +8402,26 @@ function createRunAgentTurnActivity(services) {
7527
8402
  const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
7528
8403
  const write = async (material) => {
7529
8404
  const runAs = sandboxRunAs(runSettings);
7530
- await refreshToolspaceTokenFile(tokenSession, material.token, {
7531
- ...runAs ? { runAs } : {},
7532
- ...sandboxToolspaceTokenFile ? {
7533
- tokenFile: sandboxToolspaceTokenFile,
7534
- legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE
7535
- } : {},
7536
- ...toolCancellationFenceRef.current ? {
7537
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
7538
- toolCancellationFenceRef.current
7539
- )
7540
- } : {}
7541
- });
8405
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
8406
+ if (!targetSandbox) {
8407
+ throw new Error("Toolspace token renewal has no exact sandbox lease target");
8408
+ }
8409
+ await runWorkspaceMutationForSandbox(
8410
+ targetSandbox,
8411
+ "toolspaceTokenRenewal",
8412
+ async () => await refreshToolspaceTokenFile(tokenSession, material.token, {
8413
+ ...runAs ? { runAs } : {},
8414
+ ...sandboxToolspaceTokenFile ? {
8415
+ tokenFile: sandboxToolspaceTokenFile,
8416
+ legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE
8417
+ } : {},
8418
+ ...toolCancellationFenceRef.current ? {
8419
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8420
+ toolCancellationFenceRef.current
8421
+ )
8422
+ } : {}
8423
+ })
8424
+ );
7542
8425
  };
7543
8426
  let renewalExpiresAt = initialExpiresAt;
7544
8427
  if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
@@ -7580,46 +8463,65 @@ function createRunAgentTurnActivity(services) {
7580
8463
  }
7581
8464
  toolspaceTokenRenewal = controller;
7582
8465
  };
7583
- const attachRunCredentialRenewal = async (credentialSession) => {
8466
+ const attachRunCredentialRenewal = async (credentialSession, initialSandbox) => {
7584
8467
  if (!runCredentialResolver) return;
7585
8468
  const previous = runCredentialRenewal;
7586
8469
  runCredentialRenewal = null;
7587
8470
  await previous?.stop();
7588
8471
  if (runCredentialRenewalClosed) return;
7589
8472
  runCredentialSession = credentialSession;
8473
+ const requireTargetSandbox = () => {
8474
+ const targetSandbox = resolvedSandbox ?? initialSandbox;
8475
+ if (!targetSandbox) {
8476
+ throw new Error("Run credential mutation has no exact sandbox lease target");
8477
+ }
8478
+ return targetSandbox;
8479
+ };
7590
8480
  if (!initialRunCredentialMaterial) {
7591
- await clearRunCredentials(
7592
- credentialSession,
7593
- input.sessionId,
7594
- toolCancellationFenceRef.current ? toolCancellationFenceRef.current.runSandboxCommand.bind(
7595
- toolCancellationFenceRef.current
7596
- ) : void 0
8481
+ await runWorkspaceMutationForSandbox(
8482
+ requireTargetSandbox(),
8483
+ "runCredentialClear",
8484
+ async () => await clearRunCredentials(
8485
+ credentialSession,
8486
+ input.sessionId,
8487
+ toolCancellationFenceRef.current ? toolCancellationFenceRef.current.runSandboxCommand.bind(
8488
+ toolCancellationFenceRef.current
8489
+ ) : void 0
8490
+ )
7597
8491
  );
7598
8492
  return;
7599
8493
  }
7600
8494
  const write = async (material, pruneOtherAttempts = false) => {
7601
8495
  if (!material) {
7602
- await clearRunCredentialsForAttempt(credentialSession, {
7603
- sessionId: input.sessionId,
7604
- attemptId: input.attemptId,
7605
- executionGeneration
7606
- });
8496
+ await runWorkspaceMutationForSandbox(
8497
+ requireTargetSandbox(),
8498
+ "runCredentialAttemptClear",
8499
+ async () => await clearRunCredentialsForAttempt(credentialSession, {
8500
+ sessionId: input.sessionId,
8501
+ attemptId: input.attemptId,
8502
+ executionGeneration
8503
+ })
8504
+ );
7607
8505
  return;
7608
8506
  }
7609
8507
  registerSecretRedactions(material.redactions);
7610
- await materializeRunCredentials(credentialSession, material, {
7611
- sessionId: input.sessionId,
7612
- attemptId: input.attemptId,
7613
- executionGeneration,
7614
- ...pruneOtherAttempts ? { pruneOtherAttempts: true } : {},
7615
- ...!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {},
7616
- ...material.authNeeded.length > 0 && Object.keys(material.environment).length === 0 && material.files.length === 0 ? { prunePreviousGenerations: true } : {},
7617
- ...toolCancellationFenceRef.current ? {
7618
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
7619
- toolCancellationFenceRef.current
7620
- )
7621
- } : {}
7622
- });
8508
+ await runWorkspaceMutationForSandbox(
8509
+ requireTargetSandbox(),
8510
+ "runCredentialMaterialization",
8511
+ async () => await materializeRunCredentials(credentialSession, material, {
8512
+ sessionId: input.sessionId,
8513
+ attemptId: input.attemptId,
8514
+ executionGeneration,
8515
+ ...pruneOtherAttempts ? { pruneOtherAttempts: true } : {},
8516
+ ...!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {},
8517
+ ...material.authNeeded.length > 0 && Object.keys(material.environment).length === 0 && material.files.length === 0 ? { prunePreviousGenerations: true } : {},
8518
+ ...toolCancellationFenceRef.current ? {
8519
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8520
+ toolCancellationFenceRef.current
8521
+ )
8522
+ } : {}
8523
+ })
8524
+ );
7623
8525
  for (const payload of runCredentialAuthNeededPayloads(material)) {
7624
8526
  const key = JSON.stringify(payload);
7625
8527
  if (publishedRunCredentialNotices.has(key)) continue;
@@ -7718,12 +8620,19 @@ function createRunAgentTurnActivity(services) {
7718
8620
  db,
7719
8621
  settings,
7720
8622
  bus,
7721
- onOp: machineOpObserver.observer
8623
+ onOp: machineOpObserver.observer,
8624
+ onHomeSandboxRebound
7722
8625
  },
7723
8626
  {
7724
8627
  workspaceId: input.workspaceId,
7725
8628
  sessionId: input.sessionId,
7726
8629
  environment: sandboxEnvironment,
8630
+ workspaceMutationFence: {
8631
+ accountId: input.accountId,
8632
+ turnId: turn.id,
8633
+ executionGeneration,
8634
+ attemptId: input.attemptId
8635
+ },
7727
8636
  pinnedSelfhosted: {
7728
8637
  sandboxId: activeSandboxPointer.activeSandboxId,
7729
8638
  epoch: activeSandboxPointer.activeEpoch
@@ -7789,24 +8698,41 @@ function createRunAgentTurnActivity(services) {
7789
8698
  );
7790
8699
  setupBoxSession = resolvedSandbox.established.session;
7791
8700
  await publishSandboxLifecycleEvents(resolvedSandbox);
7792
- if (routingEnabled(settings)) {
7793
- resolvedSandbox = {
7794
- ...resolvedSandbox,
7795
- established: wrapTurnBoxWithRouting(
7796
- { db, settings, bus },
7797
- // Thread the SAME declared environment the group box was created with
7798
- // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
7799
- // carries it too — the SDK's per-turn manifest-env delta stays empty
7800
- // (no "cannot change manifest environment variables" throw).
7801
- {
7802
- workspaceId: input.workspaceId,
7803
- sessionId: input.sessionId,
7804
- environment: sandboxEnvironment
8701
+ resolvedSandbox = {
8702
+ ...resolvedSandbox,
8703
+ established: wrapTurnBoxWithRouting(
8704
+ {
8705
+ db,
8706
+ settings,
8707
+ bus,
8708
+ onHomeSandboxLost: publishSandboxLost,
8709
+ onHomeSandboxRebound
8710
+ },
8711
+ // Thread the SAME declared environment the group box was created with
8712
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
8713
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
8714
+ // (no "cannot change manifest environment variables" throw).
8715
+ {
8716
+ workspaceId: input.workspaceId,
8717
+ sessionId: input.sessionId,
8718
+ environment: sandboxEnvironment,
8719
+ workspaceMutationFence: {
8720
+ accountId: input.accountId,
8721
+ turnId: turn.id,
8722
+ executionGeneration,
8723
+ attemptId: input.attemptId
7805
8724
  },
7806
- resolvedSandbox.established
7807
- )
7808
- };
7809
- }
8725
+ homeLease: {
8726
+ accountId: input.accountId,
8727
+ sandboxGroupId: session.sandboxGroupId,
8728
+ leaseEpoch: resolvedSandbox.leaseEpoch,
8729
+ instanceId: resolvedSandbox.established.instanceId,
8730
+ backend: groupBoxBackend
8731
+ }
8732
+ },
8733
+ resolvedSandbox.established
8734
+ )
8735
+ };
7810
8736
  }
7811
8737
  if (resolvedSandbox) {
7812
8738
  startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
@@ -8047,39 +8973,46 @@ function createRunAgentTurnActivity(services) {
8047
8973
  );
8048
8974
  await publishSandboxLifecycleEvents(provisioned);
8049
8975
  await attachRunCredentialRenewal(
8050
- provisioned.established.session
8976
+ provisioned.established.session,
8977
+ provisioned
8051
8978
  );
8052
8979
  const provisionedSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(
8053
8980
  provisioned.established.session,
8054
8981
  input.sessionId
8055
8982
  ) : provisioned.established.session;
8056
- await runOwnedSandboxSetup(
8057
- agent,
8058
- provisioned.established.session,
8059
- provisionedSetupSession,
8060
- {
8061
- settings: runSettings,
8062
- environment: sandboxEnvironment,
8063
- onRuntimeEvent: async (event) => {
8064
- await publish?.([{ type: event.type, payload: event.payload }], true);
8065
- },
8066
- ...lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {},
8067
- ...lazyGitCredentials?.bindings ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings } : {},
8068
- ...lazyToolspaceToken ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token } : {},
8069
- ...toolCancellationFenceRef.current ? {
8070
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8071
- toolCancellationFenceRef.current
8072
- )
8073
- } : {}
8074
- }
8983
+ await runWorkspaceMutationForSandbox(
8984
+ provisioned,
8985
+ "lazyOwnedSandboxSetup",
8986
+ async () => await runOwnedSandboxSetup(
8987
+ agent,
8988
+ provisioned.established.session,
8989
+ provisionedSetupSession,
8990
+ {
8991
+ settings: runSettings,
8992
+ environment: sandboxEnvironment,
8993
+ onRuntimeEvent: async (event) => {
8994
+ await publish?.([{ type: event.type, payload: event.payload }], true);
8995
+ },
8996
+ ...lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {},
8997
+ ...lazyGitCredentials?.bindings ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings } : {},
8998
+ ...lazyToolspaceToken ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token } : {},
8999
+ ...toolCancellationFenceRef.current ? {
9000
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
9001
+ toolCancellationFenceRef.current
9002
+ )
9003
+ } : {}
9004
+ }
9005
+ )
8075
9006
  );
8076
9007
  await attachToolspaceTokenRenewal(
8077
9008
  provisioned.established.session,
8078
- lazyToolspaceToken?.expiresAt
9009
+ lazyToolspaceToken?.expiresAt,
9010
+ provisioned
8079
9011
  );
8080
9012
  await attachGitCredentialRenewal(
8081
9013
  provisioned.established.session,
8082
- lazyGitCredentials
9014
+ lazyGitCredentials,
9015
+ provisioned
8083
9016
  );
8084
9017
  return provisioned;
8085
9018
  },
@@ -8138,18 +9071,31 @@ function createRunAgentTurnActivity(services) {
8138
9071
  db,
8139
9072
  settings,
8140
9073
  bus,
8141
- onOp: machineOpObserver.observer
9074
+ onOp: machineOpObserver.observer,
9075
+ onHomeSandboxLost: publishSandboxLost,
9076
+ onHomeSandboxRebound
8142
9077
  },
8143
9078
  {
8144
9079
  workspaceId: input.workspaceId,
8145
9080
  sessionId: input.sessionId,
8146
- environment: sandboxEnvironment
9081
+ environment: sandboxEnvironment,
9082
+ workspaceMutationFence: {
9083
+ accountId: input.accountId,
9084
+ turnId: turn.id,
9085
+ executionGeneration,
9086
+ attemptId: input.attemptId
9087
+ }
8147
9088
  },
8148
9089
  {
8149
9090
  client: lazyClient,
8150
9091
  backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
8151
9092
  agentDefaultManifest,
8152
- provisioner: turnSandboxProvisioner
9093
+ provisioner: turnSandboxProvisioner,
9094
+ homeLeaseIdentity: {
9095
+ accountId: input.accountId,
9096
+ sandboxGroupId: session.sandboxGroupId,
9097
+ backend: groupBoxBackend
9098
+ }
8153
9099
  }
8154
9100
  );
8155
9101
  }
@@ -8251,20 +9197,24 @@ function createRunAgentTurnActivity(services) {
8251
9197
  );
8252
9198
  const runAs = sandboxRunAs(runSettings);
8253
9199
  if (downloadsToMaterialize.length > 0) {
8254
- const materialized = await materializeSandboxFileDownloads(
8255
- setupBoxSession,
8256
- downloadsToMaterialize,
8257
- {
8258
- onRuntimeEvent: async (event) => {
8259
- await publish([{ type: event.type, payload: event.payload }], true);
8260
- },
8261
- ...runAs ? { runAs } : {},
8262
- ...toolCancellationFenceRef.current ? {
8263
- commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
8264
- toolCancellationFenceRef.current
8265
- )
8266
- } : {}
8267
- }
9200
+ const materialized = await runWorkspaceMutationForSandbox(
9201
+ resolvedSandbox,
9202
+ "fileMaterialization",
9203
+ async () => await materializeSandboxFileDownloads(
9204
+ setupBoxSession,
9205
+ downloadsToMaterialize,
9206
+ {
9207
+ onRuntimeEvent: async (event) => {
9208
+ await publish([{ type: event.type, payload: event.payload }], true);
9209
+ },
9210
+ ...runAs ? { runAs } : {},
9211
+ ...toolCancellationFenceRef.current ? {
9212
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
9213
+ toolCancellationFenceRef.current
9214
+ )
9215
+ } : {}
9216
+ }
9217
+ )
8268
9218
  );
8269
9219
  fileMaterializationFailures = materialized.failures;
8270
9220
  const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
@@ -8301,7 +9251,10 @@ function createRunAgentTurnActivity(services) {
8301
9251
  turnId: activeTurnId,
8302
9252
  recovering: turn.executionGeneration > 1,
8303
9253
  ...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
8304
- ...runCredentialsNote ? { runCredentialsNote } : {}
9254
+ ...runCredentialsNote ? { runCredentialsNote } : {},
9255
+ ...modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage ? {
9256
+ readFileBytesForModel: (file) => objectStorage.getFileBytes(file)
9257
+ } : {}
8305
9258
  }
8306
9259
  );
8307
9260
  runInput = prepared.input;
@@ -8375,57 +9328,90 @@ function createRunAgentTurnActivity(services) {
8375
9328
  throwIfWorkerShuttingDown();
8376
9329
  throwIfTurnCancelled();
8377
9330
  const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
8378
- const runStreamOnce = () => runtime.runStream(agent, runInput, modelRunSettings, {
8379
- ...activityContext ? { signal: activityContext.cancellationSignal } : {},
8380
- sandboxEnvironment,
8381
- onRuntimeEvent: async (event) => {
8382
- await renewCodexLease("runtime_event");
8383
- if (codexLeaseLost) {
8384
- throw new Error("Codex credential lease expired during the active turn");
8385
- }
8386
- await publish([{ type: event.type, payload: event.payload }], true);
8387
- },
8388
- // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
8389
- // keystone). Absent when the flag is off -> legacy build-and-discard.
8390
- ...ownedEstablished ? {
8391
- ownedSandbox: {
8392
- client: ownedEstablished.client,
8393
- session: ownedEstablished.session,
8394
- ...resolvedSandbox?.established.sessionState ? {
8395
- sessionState: resolvedSandbox.established.sessionState
8396
- } : {},
8397
- // Pin platform setup (hooks + file materialization) to the un-proxied
8398
- // established box — never through the routing proxy, which would
8399
- // re-route those execs onto a machine swapped in mid-turn.
8400
- ...setupBoxSession ? { setupSession: setupBoxSession } : {},
8401
- ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
8402
- ...lazyOwnedSandbox ? { deferredSetup: true } : {}
8403
- }
8404
- } : {},
8405
- ...initialGitCredentials ? {
8406
- onGitCredentialSessionReady: async (tokenSession) => {
8407
- await attachGitCredentialRenewal(tokenSession, initialGitCredentials);
8408
- }
8409
- } : {},
8410
- ...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
8411
- onToolspaceTokenSessionReady: async (tokenSession) => {
8412
- const renewalSession = activeSandboxBackend === "selfhosted" ? tokenSession : setupBoxSession ?? tokenSession;
8413
- await attachToolspaceTokenRenewal(renewalSession);
8414
- }
8415
- } : {},
8416
- ...runCredentialResolver ? {
8417
- runCredentialSessionId: input.sessionId,
8418
- ...!lazyOwnedSandbox ? {
8419
- onRunCredentialSessionReady: async (credentialSession) => {
8420
- const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
8421
- await attachRunCredentialRenewal(pinnedCredentialSession);
9331
+ const runStreamOnce = async () => {
9332
+ if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
9333
+ const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
9334
+ await runWorkspaceMutationForSandbox(
9335
+ resolvedSandbox,
9336
+ "eagerOwnedSandboxSetup",
9337
+ async () => await runOwnedSandboxSetup(
9338
+ agent,
9339
+ ownedEstablished.session,
9340
+ eagerSetupSession,
9341
+ {
9342
+ settings: runSettings,
9343
+ environment: sandboxEnvironment,
9344
+ preparedInput: runInput,
9345
+ ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
9346
+ onRuntimeEvent: async (event) => {
9347
+ await renewCodexLease("runtime_event");
9348
+ if (codexLeaseLost) {
9349
+ throw new Error("Codex credential lease expired during sandbox setup");
9350
+ }
9351
+ await publish([{ type: event.type, payload: event.payload }], true);
9352
+ },
9353
+ ...toolCancellationFenceRef.current ? {
9354
+ commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
9355
+ toolCancellationFenceRef.current
9356
+ )
9357
+ } : {}
9358
+ }
9359
+ )
9360
+ );
9361
+ await attachGitCredentialRenewal(
9362
+ eagerSetupSession,
9363
+ initialGitCredentials
9364
+ );
9365
+ }
9366
+ return await runtime.runStream(agent, runInput, modelRunSettings, {
9367
+ ...activityContext ? { signal: activityContext.cancellationSignal } : {},
9368
+ sandboxEnvironment,
9369
+ onRuntimeEvent: async (event) => {
9370
+ await renewCodexLease("runtime_event");
9371
+ if (codexLeaseLost) {
9372
+ throw new Error("Codex credential lease expired during the active turn");
8422
9373
  }
8423
- } : {}
8424
- } : {},
8425
- contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
8426
- contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
8427
- ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
8428
- });
9374
+ await publish([{ type: event.type, payload: event.payload }], true);
9375
+ },
9376
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
9377
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
9378
+ ...ownedEstablished ? {
9379
+ ownedSandbox: {
9380
+ client: ownedEstablished.client,
9381
+ session: ownedEstablished.session,
9382
+ ...resolvedSandbox?.established.sessionState ? {
9383
+ sessionState: resolvedSandbox.established.sessionState
9384
+ } : {},
9385
+ // Pin platform setup (hooks + file materialization) to the un-proxied
9386
+ // established box — never through the routing proxy, which would
9387
+ // re-route those execs onto a machine swapped in mid-turn.
9388
+ ...setupBoxSession ? { setupSession: setupBoxSession } : {},
9389
+ ...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
9390
+ // Both owned paths execute setup outside runStream: eager just
9391
+ // above under its exact admission, lazy in the provisioner.
9392
+ deferredSetup: true
9393
+ }
9394
+ } : {},
9395
+ ...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
9396
+ onToolspaceTokenSessionReady: async (tokenSession) => {
9397
+ const renewalSession = activeSandboxBackend === "selfhosted" ? tokenSession : setupBoxSession ?? tokenSession;
9398
+ await attachToolspaceTokenRenewal(renewalSession);
9399
+ }
9400
+ } : {},
9401
+ ...runCredentialResolver ? {
9402
+ runCredentialSessionId: input.sessionId,
9403
+ ...!lazyOwnedSandbox ? {
9404
+ onRunCredentialSessionReady: async (credentialSession) => {
9405
+ const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
9406
+ await attachRunCredentialRenewal(pinnedCredentialSession);
9407
+ }
9408
+ } : {}
9409
+ } : {},
9410
+ contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
9411
+ contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
9412
+ ...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
9413
+ });
9414
+ };
8429
9415
  if (codexLeaseLost) {
8430
9416
  throw new Error("Codex credential lease expired before the model run");
8431
9417
  }
@@ -8514,6 +9500,7 @@ function createRunAgentTurnActivity(services) {
8514
9500
  turnId: activeTurnId,
8515
9501
  executionGeneration,
8516
9502
  attemptId: input.attemptId,
9503
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
8517
9504
  ...pendingToolCall
8518
9505
  });
8519
9506
  if (!registered.accepted) {
@@ -8533,6 +9520,7 @@ function createRunAgentTurnActivity(services) {
8533
9520
  executionGeneration,
8534
9521
  attemptId: input.attemptId,
8535
9522
  callId: completedToolCall.callId,
9523
+ modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
8536
9524
  resultItem: completedToolCall.resultItem
8537
9525
  });
8538
9526
  if (!recorded.accepted) {
@@ -8749,7 +9737,7 @@ function createRunAgentTurnActivity(services) {
8749
9737
  return claimedResult({ status: "cancelled" });
8750
9738
  }
8751
9739
  turnMetricOutcome = "completed";
8752
- await recordUsageEvent2(db, {
9740
+ await recordUsageEvent(db, {
8753
9741
  accountId: input.accountId,
8754
9742
  workspaceId: input.workspaceId,
8755
9743
  eventType: "agent_run.completed",
@@ -8936,7 +9924,7 @@ function createRunAgentTurnActivity(services) {
8936
9924
  turnFinalizerCancellationSignal(cancellationSignal, activityStatus)
8937
9925
  );
8938
9926
  turnMetricOutcome = "cancelled";
8939
- throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
9927
+ throw new CancelledFailure2("TURN_ATTEMPT_FENCED", [], error);
8940
9928
  }
8941
9929
  if (cancellationFailure) {
8942
9930
  activityStatus = "cancelled";
@@ -8969,7 +9957,7 @@ function createRunAgentTurnActivity(services) {
8969
9957
  return claimedResult({ status: "cancelled" });
8970
9958
  }
8971
9959
  turnMetricOutcome = "completed";
8972
- await recordUsageEvent2(db, {
9960
+ await recordUsageEvent(db, {
8973
9961
  accountId: input.accountId,
8974
9962
  workspaceId: input.workspaceId,
8975
9963
  eventType: "agent_run.completed",
@@ -9456,7 +10444,7 @@ function createRunAgentTurnActivity(services) {
9456
10444
  return claimedResult({ status: "cancelled" });
9457
10445
  }
9458
10446
  turnMetricOutcome = "completed";
9459
- await recordUsageEvent2(db, {
10447
+ await recordUsageEvent(db, {
9460
10448
  accountId: input.accountId,
9461
10449
  workspaceId: input.workspaceId,
9462
10450
  eventType: "agent_run.completed",
@@ -9557,15 +10545,34 @@ function createRunAgentTurnActivity(services) {
9557
10545
  const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
9558
10546
  try {
9559
10547
  const toolCancellationFence = toolCancellationFenceRef.current;
10548
+ gitCredentialRenewalClosed = true;
10549
+ const gitRenewalsToStop = gitCredentialRenewals;
10550
+ gitCredentialRenewals = [];
9560
10551
  toolspaceTokenRenewalClosed = true;
9561
10552
  const toolspaceRenewalToStop = toolspaceTokenRenewal;
9562
10553
  toolspaceTokenRenewal = null;
9563
10554
  runCredentialRenewalClosed = true;
9564
10555
  const runRenewalToStop = runCredentialRenewal;
9565
10556
  runCredentialRenewal = null;
10557
+ const credentialSessionToClear = runCredentialSession;
10558
+ runCredentialSession = null;
10559
+ if (credentialSessionToClear) {
10560
+ await runWorkspaceMutationForSandbox(
10561
+ requireResolvedSandboxForMutation(
10562
+ "Run credential cleanup has no exact sandbox lease target"
10563
+ ),
10564
+ "runCredentialAttemptClear",
10565
+ async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
10566
+ sessionId: input.sessionId,
10567
+ attemptId: input.attemptId,
10568
+ executionGeneration
10569
+ })
10570
+ );
10571
+ }
9566
10572
  await drainAttemptOwnedSandboxWriters({
9567
10573
  toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
9568
10574
  cancellationReason: cancellationSignal?.reason,
10575
+ gitCredentialRenewals: gitRenewalsToStop,
9569
10576
  toolspaceTokenRenewal: toolspaceRenewalToStop,
9570
10577
  runCredentialRenewal: runRenewalToStop
9571
10578
  });
@@ -9635,37 +10642,6 @@ function createRunAgentTurnActivity(services) {
9635
10642
  });
9636
10643
  }
9637
10644
  }
9638
- const credentialSessionToClear = runCredentialSession;
9639
- runCredentialSession = null;
9640
- if (credentialSessionToClear) {
9641
- const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
9642
- sessionId: input.sessionId,
9643
- attemptId: input.attemptId,
9644
- executionGeneration
9645
- }).catch((error) => {
9646
- try {
9647
- observability.incrementCounter({
9648
- name: "opengeni_run_credential_cleanup_total",
9649
- help: "Attempt-owned run credential cleanup outcomes.",
9650
- labels: { outcome: "error" }
9651
- });
9652
- observability.warn("Attempt-owned run credential cleanup failed", {
9653
- sessionId: input.sessionId,
9654
- turnId,
9655
- attemptId: input.attemptId,
9656
- errorClass: error instanceof Error ? error.name : "UnknownError"
9657
- });
9658
- } catch {
9659
- }
9660
- });
9661
- await waitForTurnFinalizerStep(cleanup, finalizerSignal);
9662
- }
9663
- gitCredentialRenewalClosed = true;
9664
- const renewalsToStop = gitCredentialRenewals;
9665
- gitCredentialRenewals = [];
9666
- for (const renewal of renewalsToStop) {
9667
- await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
9668
- }
9669
10645
  const machineOpEvents = machineOpObserver.drainEvents();
9670
10646
  if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
9671
10647
  await waitForTurnFinalizerStep(
@@ -10192,7 +11168,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
10192
11168
  const outputTokens = sanitizedUsage.outputTokens ?? 0;
10193
11169
  const totalTokens = sanitizedUsage.totalTokens ?? 0;
10194
11170
  if (input.externallyBilled) {
10195
- await recordUsageEvent2(db, {
11171
+ await recordUsageEvent(db, {
10196
11172
  accountId: input.accountId,
10197
11173
  workspaceId: input.workspaceId,
10198
11174
  eventType: "model.cost",
@@ -10208,7 +11184,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
10208
11184
  return;
10209
11185
  }
10210
11186
  if (totalTokens > 0) {
10211
- await recordUsageEvent2(db, {
11187
+ await recordUsageEvent(db, {
10212
11188
  accountId: input.accountId,
10213
11189
  workspaceId: input.workspaceId,
10214
11190
  eventType: "model.tokens",
@@ -10230,7 +11206,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
10230
11206
  throw new Error(`Missing model pricing for ${input.model}`);
10231
11207
  }
10232
11208
  const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
10233
- await recordUsageEvent2(db, {
11209
+ await recordUsageEvent(db, {
10234
11210
  accountId: input.accountId,
10235
11211
  workspaceId: input.workspaceId,
10236
11212
  eventType: "model.cost",
@@ -10334,7 +11310,7 @@ import { indexDocumentNow } from "@opengeni/documents";
10334
11310
  import { configuredStaticUsageLimits as configuredStaticUsageLimits3 } from "@opengeni/config";
10335
11311
  import {
10336
11312
  getBillingBalance as getBillingBalance3,
10337
- recordUsageEvent as recordUsageEvent3,
11313
+ recordUsageEvent as recordUsageEvent2,
10338
11314
  sumUsageQuantity as sumUsageQuantity3,
10339
11315
  withWorkspaceUsageLock
10340
11316
  } from "@opengeni/db";
@@ -10379,7 +11355,7 @@ function createDocumentActivities(services) {
10379
11355
  }
10380
11356
  );
10381
11357
  if (document.status === "ready") {
10382
- await recordUsageEvent3(lockedDb, {
11358
+ await recordUsageEvent2(lockedDb, {
10383
11359
  accountId: input.accountId,
10384
11360
  workspaceId: input.workspaceId,
10385
11361
  eventType: "document.indexed",
@@ -10464,17 +11440,19 @@ import {
10464
11440
  listLiveModalSandboxLeaseAttributions,
10465
11441
  listMeterableWarmLeases,
10466
11442
  persistDrainSnapshot,
10467
- readLease as readLease2,
11443
+ readWorkspaceArchiveCapturePreflight as readWorkspaceArchiveCapturePreflight2,
11444
+ readLease as readLease3,
10468
11445
  reapExpiredSessionListSnapshots,
10469
11446
  reapStaleLeaseHoldersGlobal,
10470
11447
  rlsContextForWorkspace
10471
11448
  } from "@opengeni/db";
10472
11449
  import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
10473
11450
  import {
11451
+ captureVerifiedWorkspaceArchive as captureVerifiedWorkspaceArchive2,
10474
11452
  createSandboxClientForBackend,
10475
11453
  deletePriorPersistedSnapshot as deletePriorPersistedSnapshot2,
10476
11454
  deserializeSandboxSessionStateEnvelope,
10477
- isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
11455
+ isProviderSandboxNotFoundError as isProviderSandboxNotFoundError3,
10478
11456
  sweepModalOrphanSandboxes,
10479
11457
  terminateModalSandboxById
10480
11458
  } from "@opengeni/runtime";
@@ -10746,28 +11724,45 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
10746
11724
  }
10747
11725
  async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
10748
11726
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
10749
- const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
11727
+ const lease = await readLease3(db, row.workspaceId, row.sandboxGroupId);
10750
11728
  if (!lease) {
10751
11729
  return false;
10752
11730
  }
10753
11731
  if (lease.liveness !== "draining" || lease.refcount !== 0 || lease.leaseEpoch !== row.leaseEpoch) {
10754
11732
  return false;
10755
11733
  }
11734
+ const capturePreflight = lease.instanceId ? await readWorkspaceArchiveCapturePreflight2(db, {
11735
+ accountId,
11736
+ workspaceId: row.workspaceId,
11737
+ sandboxGroupId: row.sandboxGroupId,
11738
+ expectedEpoch: row.leaseEpoch,
11739
+ expectedInstanceId: lease.instanceId,
11740
+ liveness: "draining"
11741
+ }) : null;
10756
11742
  let persisted = false;
10757
- const persistArchive = async (archiveBase64) => {
11743
+ let archiveRevision = null;
11744
+ const persistArchive = async (archiveBase64, archiveMetadata) => {
11745
+ if (!lease.instanceId || !capturePreflight) {
11746
+ return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
11747
+ }
10758
11748
  const result = await persistDrainSnapshot(db, {
10759
11749
  accountId,
10760
11750
  workspaceId: row.workspaceId,
10761
11751
  sandboxGroupId: row.sandboxGroupId,
10762
11752
  expectedEpoch: row.leaseEpoch,
10763
- workspaceArchive: archiveBase64
11753
+ expectedInstanceId: lease.instanceId,
11754
+ expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
11755
+ workspaceArchive: archiveBase64,
11756
+ ...archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}
10764
11757
  });
10765
11758
  if (result.wrote && archiveBase64 !== null) {
10766
11759
  persisted = true;
11760
+ archiveRevision = result.archiveRevision;
10767
11761
  }
10768
11762
  return result;
10769
11763
  };
10770
- const terminated = await terminateBox(settings, lease, observability, persistArchive);
11764
+ const termination = await terminateBox(settings, lease, observability, persistArchive);
11765
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
10771
11766
  if (!terminated) {
10772
11767
  return false;
10773
11768
  }
@@ -10775,13 +11770,20 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
10775
11770
  accountId,
10776
11771
  workspaceId: row.workspaceId,
10777
11772
  sandboxGroupId: row.sandboxGroupId,
10778
- expectedEpoch: row.leaseEpoch
11773
+ expectedEpoch: row.leaseEpoch,
11774
+ providerMissingBeforeCapture: typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture
10779
11775
  });
10780
11776
  if (wentCold) {
10781
11777
  try {
10782
11778
  await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
10783
11779
  type: "sandbox.box.terminated",
10784
- payload: { actor: "reaper", persisted, instanceId: lease.instanceId }
11780
+ payload: {
11781
+ actor: "reaper",
11782
+ persisted,
11783
+ archiveRevision,
11784
+ instanceId: lease.instanceId,
11785
+ providerMissingBeforeCapture: typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture
11786
+ }
10785
11787
  });
10786
11788
  } catch (eventError) {
10787
11789
  observability.warn(
@@ -10798,7 +11800,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
10798
11800
  async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
10799
11801
  const backend = lease.resumeBackendId ?? lease.backend;
10800
11802
  if (!backend || backend === "none") {
10801
- return true;
11803
+ return { terminated: true, providerMissingBeforeCapture: false };
10802
11804
  }
10803
11805
  if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
10804
11806
  observability.info(
@@ -10808,7 +11810,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10808
11810
  backend
10809
11811
  }
10810
11812
  );
10811
- return true;
11813
+ return { terminated: true, providerMissingBeforeCapture: false };
10812
11814
  }
10813
11815
  if (backend === "modal" && !lease.resumeState && lease.instanceId) {
10814
11816
  const { wrote } = await persistArchive(null);
@@ -10821,12 +11823,12 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10821
11823
  instanceId: lease.instanceId
10822
11824
  }
10823
11825
  );
10824
- return false;
11826
+ return { terminated: false, providerMissingBeforeCapture: false };
10825
11827
  }
10826
11828
  try {
10827
11829
  await terminateModalSandboxById(settings, lease.instanceId);
10828
11830
  } catch (error) {
10829
- if (!isProviderSandboxNotFoundError2("modal", error)) {
11831
+ if (!isProviderSandboxNotFoundError3("modal", error)) {
10830
11832
  throw error;
10831
11833
  }
10832
11834
  observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
@@ -10835,20 +11837,20 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10835
11837
  instanceId: lease.instanceId
10836
11838
  });
10837
11839
  }
10838
- return true;
11840
+ return { terminated: true, providerMissingBeforeCapture: false };
10839
11841
  }
10840
11842
  if (!lease.resumeState) {
10841
- return true;
11843
+ return { terminated: true, providerMissingBeforeCapture: false };
10842
11844
  }
10843
11845
  const client = createClientForBackend(backend, settings);
10844
11846
  if (!client) {
10845
- return true;
11847
+ return { terminated: true, providerMissingBeforeCapture: false };
10846
11848
  }
10847
11849
  let session;
10848
11850
  let sessionState;
10849
11851
  try {
10850
11852
  if (!client.resume || !client.deserializeSessionState) {
10851
- return true;
11853
+ throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
10852
11854
  }
10853
11855
  const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
10854
11856
  const resumedState = await deserializeSandboxSessionStateEnvelope(
@@ -10856,36 +11858,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10856
11858
  envelopeSessionState
10857
11859
  );
10858
11860
  if (resumedState === void 0) {
10859
- return true;
11861
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
10860
11862
  }
10861
11863
  session = await client.resume(resumedState);
10862
11864
  sessionState = resumedState;
10863
11865
  } catch (error) {
10864
- if (isProviderSandboxNotFoundError2(client.backendId, error)) {
10865
- observability.info(
10866
- "sandbox reaper: drainable box already gone (NotFound on resume) \u2014 proceeding to cold",
10867
- {
10868
- sandboxGroupId: lease.sandboxGroupId,
10869
- backend
10870
- }
10871
- );
10872
- return true;
11866
+ if (isProviderSandboxNotFoundError3(client.backendId, error)) {
11867
+ observability.info("sandbox reaper: drainable box already gone before workspace capture", {
11868
+ sandboxGroupId: lease.sandboxGroupId,
11869
+ backend
11870
+ });
11871
+ return { terminated: true, providerMissingBeforeCapture: true };
10873
11872
  }
10874
11873
  throw error;
10875
11874
  }
10876
- let archiveBytes;
11875
+ let verifiedArchive;
10877
11876
  try {
10878
- archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : void 0;
11877
+ verifiedArchive = session?.persistWorkspace ? await captureVerifiedWorkspaceArchive2(session) : void 0;
10879
11878
  } catch (error) {
10880
- if (isProviderSandboxNotFoundError2(client.backendId, error)) {
10881
- observability.info(
10882
- "sandbox reaper: box gone during persist (NotFound) \u2014 nothing to snapshot, proceeding to cold",
10883
- {
10884
- sandboxGroupId: lease.sandboxGroupId,
10885
- backend
10886
- }
10887
- );
10888
- return true;
11879
+ if (isProviderSandboxNotFoundError3(client.backendId, error)) {
11880
+ observability.info("sandbox reaper: box gone during workspace capture", {
11881
+ sandboxGroupId: lease.sandboxGroupId,
11882
+ backend
11883
+ });
11884
+ return { terminated: true, providerMissingBeforeCapture: true };
10889
11885
  }
10890
11886
  observability.warn(
10891
11887
  "sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
@@ -10897,9 +11893,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10897
11893
  );
10898
11894
  throw error;
10899
11895
  }
10900
- if (archiveBytes && archiveBytes.length > 0) {
10901
- const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
10902
- const { wrote, priorArchive, priorArchivePrev } = await persistArchive(archiveBase64);
11896
+ if (verifiedArchive) {
11897
+ const { wrote, priorArchiveForGc } = await persistArchive(
11898
+ verifiedArchive.base64,
11899
+ verifiedArchive.descriptor
11900
+ );
10903
11901
  if (!wrote) {
10904
11902
  observability.info(
10905
11903
  "sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)",
@@ -10908,9 +11906,9 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10908
11906
  backend
10909
11907
  }
10910
11908
  );
10911
- return false;
11909
+ return { terminated: false, providerMissingBeforeCapture: false };
10912
11910
  }
10913
- const deleted = await deletePriorPersistedSnapshot2(session, priorArchive);
11911
+ const deleted = await deletePriorPersistedSnapshot2(session, priorArchiveForGc);
10914
11912
  if (deleted) {
10915
11913
  observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
10916
11914
  sandboxGroupId: lease.sandboxGroupId,
@@ -10918,51 +11916,41 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
10918
11916
  snapshotId: deleted
10919
11917
  });
10920
11918
  }
10921
- const deletedPrev = await deletePriorPersistedSnapshot2(session, priorArchivePrev);
10922
- if (deletedPrev) {
10923
- observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
10924
- sandboxGroupId: lease.sandboxGroupId,
10925
- backend,
10926
- snapshotId: deletedPrev
10927
- });
10928
- }
10929
11919
  } else {
10930
- const { wrote } = await persistArchive(null);
10931
- if (!wrote) {
10932
- observability.info(
10933
- "sandbox reaper: lease re-armed during snapshot window (no-archive path) \u2014 leaving box RUNNING (no terminate)",
10934
- {
10935
- sandboxGroupId: lease.sandboxGroupId,
10936
- backend
10937
- }
10938
- );
10939
- return false;
10940
- }
11920
+ throw new Error(
11921
+ `sandbox backend ${backend} produced no verified workspace archive during drain`
11922
+ );
10941
11923
  }
10942
11924
  try {
10943
11925
  if (client.delete && sessionState !== void 0) {
10944
11926
  await client.delete(sessionState);
10945
- return true;
11927
+ return { terminated: true, providerMissingBeforeCapture: false };
10946
11928
  }
10947
11929
  if (session?.kill) {
10948
11930
  await session.kill();
10949
- return true;
11931
+ return { terminated: true, providerMissingBeforeCapture: false };
10950
11932
  }
10951
11933
  if (session?.terminate) {
10952
11934
  await session.terminate();
10953
- return true;
11935
+ return { terminated: true, providerMissingBeforeCapture: false };
10954
11936
  }
10955
- if (session?.close && !session.closed) {
10956
- await session.close();
11937
+ if (session?.close) {
11938
+ if (!session.closed) {
11939
+ await session.close();
11940
+ }
11941
+ return { terminated: true, providerMissingBeforeCapture: false };
10957
11942
  }
10958
11943
  } catch (error) {
10959
- observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
10960
- sandboxGroupId: lease.sandboxGroupId,
10961
- backend,
10962
- error: error instanceof Error ? error.message : String(error)
10963
- });
11944
+ if (isProviderSandboxNotFoundError3(client.backendId, error)) {
11945
+ observability.info("sandbox reaper: provider already gone during terminate", {
11946
+ sandboxGroupId: lease.sandboxGroupId,
11947
+ backend
11948
+ });
11949
+ return { terminated: true, providerMissingBeforeCapture: false };
11950
+ }
11951
+ throw error;
10964
11952
  }
10965
- return true;
11953
+ throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
10966
11954
  }
10967
11955
 
10968
11956
  // src/activities/scheduled-tasks.ts
@@ -10978,16 +11966,17 @@ import {
10978
11966
  getVariableSet,
10979
11967
  isCodexBilledTurn as isCodexBilledTurn2,
10980
11968
  markScheduledTaskRunFailedIfQueued,
10981
- recordUsageEvent as recordUsageEvent4,
11969
+ recordUsageEvent as recordUsageEvent3,
10982
11970
  requireScheduledTask,
10983
11971
  requireSession as requireSession3,
10984
11972
  setTemporalWorkflowId,
10985
11973
  settleScheduledTaskRunInTransaction,
11974
+ SessionSpawnDeniedDbError,
10986
11975
  sumUsageQuantity as sumUsageQuantity4,
10987
11976
  updateScheduledTask,
10988
11977
  upsertSessionGoal
10989
11978
  } from "@opengeni/db";
10990
- import { appendAndPublishEvents as appendAndPublishEvents2, publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
11979
+ import { appendAndPublishEvents, publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
10991
11980
  import { configuredStaticUsageLimits as configuredStaticUsageLimits4 } from "@opengeni/config";
10992
11981
  function createScheduledTaskActivities(services) {
10993
11982
  return {
@@ -11016,7 +12005,7 @@ function createScheduledTaskActivities(services) {
11016
12005
  producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `scheduled:${crypto.randomUUID()}`,
11017
12006
  scheduledAt: null
11018
12007
  });
11019
- await recordUsageEvent4(db, {
12008
+ await recordUsageEvent3(db, {
11020
12009
  accountId: task.accountId,
11021
12010
  workspaceId: task.workspaceId,
11022
12011
  eventType: "scheduled_task.fired",
@@ -11030,7 +12019,7 @@ function createScheduledTaskActivities(services) {
11030
12019
  idempotencyKey: `usage:scheduled_task.fired:${run.id}`
11031
12020
  });
11032
12021
  if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
11033
- await recordUsageEvent4(db, {
12022
+ await recordUsageEvent3(db, {
11034
12023
  accountId: task.accountId,
11035
12024
  workspaceId: task.workspaceId,
11036
12025
  eventType: "agent_run.created",
@@ -11093,34 +12082,48 @@ function createScheduledTaskActivities(services) {
11093
12082
  frozenRigId = rig.id;
11094
12083
  frozenRigVersionId = rig.activeVersion.id;
11095
12084
  }
11096
- const session = await createSession(db, {
11097
- accountId: task.accountId,
11098
- workspaceId: task.workspaceId,
11099
- initialMessage: task.agentConfig.prompt,
11100
- resources: task.agentConfig.resources,
11101
- tools: taskTools,
11102
- metadata: {
11103
- ...task.agentConfig.metadata,
12085
+ let session;
12086
+ try {
12087
+ session = await createSession(db, {
12088
+ accountId: task.accountId,
12089
+ workspaceId: task.workspaceId,
12090
+ initialMessage: task.agentConfig.prompt,
12091
+ resources: task.agentConfig.resources,
12092
+ tools: taskTools,
12093
+ metadata: {
12094
+ ...task.agentConfig.metadata,
12095
+ model,
12096
+ reasoningEffort,
12097
+ scheduledTaskId: task.id,
12098
+ scheduledTaskRunId: run.id
12099
+ },
12100
+ createdBy: {
12101
+ kind: "service",
12102
+ subjectId: "scheduler",
12103
+ label: "OpenGeni scheduler"
12104
+ },
12105
+ createdByContext: {
12106
+ scheduledTaskId: task.id,
12107
+ scheduledTaskRunId: run.id
12108
+ },
11104
12109
  model,
11105
- reasoningEffort,
11106
- scheduledTaskId: task.id,
11107
- scheduledTaskRunId: run.id
11108
- },
11109
- createdBy: {
11110
- kind: "service",
11111
- subjectId: "scheduler",
11112
- label: "OpenGeni scheduler"
11113
- },
11114
- createdByContext: {
11115
- scheduledTaskId: task.id,
11116
- scheduledTaskRunId: run.id
11117
- },
11118
- model,
11119
- sandboxBackend,
11120
- variableSetId: task.variableSetId ?? null,
11121
- rigId: frozenRigId,
11122
- rigVersionId: frozenRigVersionId
11123
- });
12110
+ sandboxBackend,
12111
+ variableSetId: task.variableSetId ?? null,
12112
+ rigId: frozenRigId,
12113
+ rigVersionId: frozenRigVersionId,
12114
+ maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
12115
+ // The durable agent config was privilege-checked when the task
12116
+ // was created/updated. Preserve that explicit policy if a broader
12117
+ // workspace/deployment limit is narrowed before a later fire.
12118
+ allowNestedAgentDepthIncrease: true,
12119
+ subjectId: `scheduled_task:${task.id}`
12120
+ });
12121
+ } catch (error) {
12122
+ if (error instanceof SessionSpawnDeniedDbError) {
12123
+ throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
12124
+ }
12125
+ throw error;
12126
+ }
11124
12127
  const goal = goalSpec ? await createSessionGoal(db, {
11125
12128
  accountId: task.accountId,
11126
12129
  workspaceId: task.workspaceId,
@@ -11137,7 +12140,7 @@ function createScheduledTaskActivities(services) {
11137
12140
  reusableSessionId: session.id
11138
12141
  });
11139
12142
  }
11140
- await appendAndPublishEvents2(db, bus, task.workspaceId, session.id, [
12143
+ await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
11141
12144
  {
11142
12145
  type: "session.created",
11143
12146
  payload: {
@@ -11314,7 +12317,7 @@ function createScheduledTaskActivities(services) {
11314
12317
  ).catch(() => void 0);
11315
12318
  throw error;
11316
12319
  }
11317
- await recordUsageEvent4(db, {
12320
+ await recordUsageEvent3(db, {
11318
12321
  accountId: task.accountId,
11319
12322
  workspaceId: task.workspaceId,
11320
12323
  eventType: "agent_run.created",
@@ -11635,7 +12638,7 @@ import {
11635
12638
  getRig as getRig2,
11636
12639
  getRigChange,
11637
12640
  getRigVersionById,
11638
- markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
12641
+ markWarmLeaseInstanceLost as markWarmLeaseInstanceLost3,
11639
12642
  recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
11640
12643
  releaseLeaseHolder as releaseLeaseHolder2,
11641
12644
  sanitizeEventPayload,
@@ -11646,11 +12649,11 @@ import {
11646
12649
  } from "@opengeni/db";
11647
12650
  import {
11648
12651
  createTurnToolCancellationController,
11649
- establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
12652
+ establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope3,
11650
12653
  runRigSetupHook,
11651
12654
  sandboxCommandExitCode,
11652
12655
  sandboxCommandOutput,
11653
- serializeEstablishedSandboxEnvelope as serializeEstablishedSandboxEnvelope2,
12656
+ serializeEstablishedSandboxEnvelope,
11654
12657
  tagModalSandbox as tagModalSandbox2
11655
12658
  } from "@opengeni/runtime";
11656
12659
  var OUTPUT_TAIL_LIMIT = 64 * 1024;
@@ -11836,14 +12839,14 @@ async function terminateThrowaway(established) {
11836
12839
  }
11837
12840
  var defaultOwnershipDependencies = {
11838
12841
  acquire: acquireLease2,
11839
- establish: establishSandboxSessionFromEnvelope2,
12842
+ establish: establishSandboxSessionFromEnvelope3,
11840
12843
  recordCreated: recordWarmingSandboxCreated2,
11841
12844
  commitWarm: commitWarmingToWarm2,
11842
12845
  touchHolder: touchLeaseHolder2,
11843
12846
  releaseHolder: releaseLeaseHolder2,
11844
12847
  failWarming: failWarmingToCold2,
11845
- markWarmLost: markWarmLeaseInstanceLost2,
11846
- serialize: serializeEstablishedSandboxEnvelope2,
12848
+ markWarmLost: markWarmLeaseInstanceLost3,
12849
+ serialize: serializeEstablishedSandboxEnvelope,
11847
12850
  tag: tagModalSandbox2,
11848
12851
  terminate: terminateThrowaway,
11849
12852
  createCancellationController: createTurnToolCancellationController,