@opengeni/worker-bundle 0.10.8 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1484 -481
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +5 -4
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +484 -169
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +130 -83
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
package/dist/index.js
CHANGED
|
@@ -39,6 +39,8 @@ import { createObjectStorage as createObjectStorage2 } from "@opengeni/storage";
|
|
|
39
39
|
|
|
40
40
|
// src/activities/agent-turn.ts
|
|
41
41
|
import {
|
|
42
|
+
advanceWorkspaceGeneration as advanceWorkspaceGeneration2,
|
|
43
|
+
verifyWorkspaceMutationSettlement as verifyWorkspaceMutationSettlement2,
|
|
42
44
|
applySessionTurnSettlement,
|
|
43
45
|
requestSessionTurnRecovery,
|
|
44
46
|
claimSessionWorkForAttempt,
|
|
@@ -79,7 +81,7 @@ import {
|
|
|
79
81
|
withCodexCapacityMutation,
|
|
80
82
|
countConsecutiveReactiveRotations,
|
|
81
83
|
requireSession as requireSession2,
|
|
82
|
-
recordUsageEvent
|
|
84
|
+
recordUsageEvent,
|
|
83
85
|
registerPendingSessionToolCall,
|
|
84
86
|
recordPendingSessionToolCallResult,
|
|
85
87
|
clearDurablePendingSessionToolCalls,
|
|
@@ -134,6 +136,7 @@ import {
|
|
|
134
136
|
sandboxFileDownloadFailureNote,
|
|
135
137
|
SUMMARY_BUFFER_TOKENS as SUMMARY_BUFFER_TOKENS2,
|
|
136
138
|
runOwnedSandboxSetup,
|
|
139
|
+
RoutingMutationOutcomeUnknownError,
|
|
137
140
|
swapTargetEstablishability,
|
|
138
141
|
deleteRecordingArtifacts as deleteRecordingArtifacts2,
|
|
139
142
|
stopRecording as stopRecordingOnBox2
|
|
@@ -173,7 +176,7 @@ import {
|
|
|
173
176
|
settingsWithResolvedModelContext,
|
|
174
177
|
resolveTurnExecutionPolicyV1
|
|
175
178
|
} from "@opengeni/config";
|
|
176
|
-
import { CancelledFailure } from "@temporalio/activity";
|
|
179
|
+
import { CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
|
|
177
180
|
|
|
178
181
|
// src/activities/capabilities.ts
|
|
179
182
|
import {
|
|
@@ -674,6 +677,206 @@ function selectCodexCredentialLeaseForTurn(args) {
|
|
|
674
677
|
};
|
|
675
678
|
}
|
|
676
679
|
|
|
680
|
+
// src/activities/codex-fleet-shadow.ts
|
|
681
|
+
import {
|
|
682
|
+
DEFAULT_CODEX_FLEET_POLICY_V1,
|
|
683
|
+
compareCodexFleetCanonicalStringsV1,
|
|
684
|
+
createCodexFleetReplayRecordV1
|
|
685
|
+
} from "@opengeni/contracts";
|
|
686
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
687
|
+
import { createHmac } from "crypto";
|
|
688
|
+
|
|
689
|
+
// src/activities/turn-attempt-fenced.ts
|
|
690
|
+
var TurnAttemptFencedError = class extends Error {
|
|
691
|
+
constructor(message) {
|
|
692
|
+
super(message);
|
|
693
|
+
this.name = "TurnAttemptFencedError";
|
|
694
|
+
}
|
|
695
|
+
};
|
|
696
|
+
|
|
697
|
+
// src/activities/codex-fleet-shadow.ts
|
|
698
|
+
var CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1024;
|
|
699
|
+
var CODEX_FLEET_SHADOW_DECISION_MAX_METRIC_SERIES = 3 * 3 * 4 * 4 * 2;
|
|
700
|
+
var CODEX_FLEET_SHADOW_ERROR_MAX_METRIC_SERIES = 3 * 2;
|
|
701
|
+
function codexFleetShadowDecisionMetricLabelsV1(payload) {
|
|
702
|
+
return {
|
|
703
|
+
actual_outcome: payload.actual.outcome,
|
|
704
|
+
shadow_outcome: payload.replay.decision.outcome,
|
|
705
|
+
comparison: payload.comparison,
|
|
706
|
+
confidence: payload.replay.decision.confidence,
|
|
707
|
+
truncated: payload.replay.truncatedCandidateCount > 0 ? "true" : "false"
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
function codexFleetShadowErrorMetricLabelsV1(result) {
|
|
711
|
+
return { stage: result.stage, reason: result.reason };
|
|
712
|
+
}
|
|
713
|
+
async function publishCodexFleetShadowDecisionV1(input) {
|
|
714
|
+
if (!input.enabled) return { outcome: "disabled" };
|
|
715
|
+
let stage = "build";
|
|
716
|
+
let payloadBytes = null;
|
|
717
|
+
try {
|
|
718
|
+
const payload = buildCodexFleetShadowPayloadV1(input.decision);
|
|
719
|
+
stage = "serialize";
|
|
720
|
+
payloadBytes = new TextEncoder().encode(JSON.stringify(payload)).byteLength;
|
|
721
|
+
if (payloadBytes > CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES) {
|
|
722
|
+
return {
|
|
723
|
+
outcome: "failed",
|
|
724
|
+
stage,
|
|
725
|
+
reason: "payload_too_large",
|
|
726
|
+
errorName: "CodexFleetShadowPayloadTooLargeError",
|
|
727
|
+
payloadBytes
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
stage = "publish";
|
|
731
|
+
await input.publish([{ type: "codex.fleet.decision", payload }]);
|
|
732
|
+
return { outcome: "published", payload, payloadBytes };
|
|
733
|
+
} catch (error) {
|
|
734
|
+
if (error instanceof TurnAttemptFencedError || error instanceof CancelledFailure) throw error;
|
|
735
|
+
return {
|
|
736
|
+
outcome: "failed",
|
|
737
|
+
stage,
|
|
738
|
+
reason: "unexpected",
|
|
739
|
+
errorName: safeErrorName(error),
|
|
740
|
+
payloadBytes
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
function buildCodexFleetShadowPayloadV1(input) {
|
|
745
|
+
const eventAccounts = [...input.accounts].sort(
|
|
746
|
+
(left, right) => compareCodexFleetCanonicalStringsV1(
|
|
747
|
+
aliasSortKey(input.aliasSeed, left.id),
|
|
748
|
+
aliasSortKey(input.aliasSeed, right.id)
|
|
749
|
+
) || compareCodexFleetCanonicalStringsV1(left.id, right.id)
|
|
750
|
+
);
|
|
751
|
+
const aliases = new Map(
|
|
752
|
+
eventAccounts.map((account, index) => [account.id, candidateAlias(index)])
|
|
753
|
+
);
|
|
754
|
+
const actualCandidateKey = input.actualCredentialId ? aliases.get(input.actualCredentialId) ?? null : null;
|
|
755
|
+
const currentCandidateKey = input.affinityCredentialId ? aliases.get(input.affinityCredentialId) ?? null : null;
|
|
756
|
+
const decisionInput = {
|
|
757
|
+
observedAtMs: input.now.getTime(),
|
|
758
|
+
request: {
|
|
759
|
+
placement: input.fencedInFlight ? "fenced_in_flight" : "new",
|
|
760
|
+
priority: "standard",
|
|
761
|
+
currentCandidateKey: input.fencedInFlight ? actualCandidateKey : currentCandidateKey,
|
|
762
|
+
waitAgeMs: 0,
|
|
763
|
+
overlayKey: null,
|
|
764
|
+
overlayMode: "none"
|
|
765
|
+
},
|
|
766
|
+
admission: {
|
|
767
|
+
// The shadow policy does not invent hard capacity from account count. A later typed
|
|
768
|
+
// admission snapshot supplies dynamic capacity; shadow v1 records unknown.
|
|
769
|
+
dynamicCapacityUnits: null,
|
|
770
|
+
inUseUnits: input.accounts.reduce((sum, account) => sum + account.activeLeaseCount, 0),
|
|
771
|
+
queuedManagerCount: 0,
|
|
772
|
+
emergencyFuseActive: false
|
|
773
|
+
},
|
|
774
|
+
candidates: eventAccounts.map((account, index) => ({
|
|
775
|
+
key: candidateAlias(index),
|
|
776
|
+
status: normalizeStatus(account.status),
|
|
777
|
+
allocatorEnabled: account.allocatorEnabled,
|
|
778
|
+
cooldownRemainingMs: remainingMs(account.exhaustedUntil, input.now),
|
|
779
|
+
activeLeaseCount: account.activeLeaseCount,
|
|
780
|
+
quota: {
|
|
781
|
+
primary: {
|
|
782
|
+
usedPercent: account.primaryUsedPercent,
|
|
783
|
+
resetRemainingMs: remainingMs(account.primaryResetAt, input.now)
|
|
784
|
+
},
|
|
785
|
+
secondary: {
|
|
786
|
+
usedPercent: account.secondaryUsedPercent,
|
|
787
|
+
resetRemainingMs: remainingMs(account.secondaryResetAt, input.now)
|
|
788
|
+
},
|
|
789
|
+
checkedAgeMs: ageMs(account.usageCheckedAt, input.now),
|
|
790
|
+
confidence: quotaConfidence(account)
|
|
791
|
+
},
|
|
792
|
+
// Current production cache evidence is aggregate metric + opaque log data,
|
|
793
|
+
// not allocator state. Preserve that gap as unknown instead of fabricating 0.
|
|
794
|
+
cache: {
|
|
795
|
+
hitRatio: null,
|
|
796
|
+
sampledTokens: null,
|
|
797
|
+
checkedAgeMs: null,
|
|
798
|
+
confidence: "unknown",
|
|
799
|
+
state: "unknown",
|
|
800
|
+
thresholdObservedForMs: null
|
|
801
|
+
},
|
|
802
|
+
// The shadow policy has no typed workspace-local burn feed at this boundary yet.
|
|
803
|
+
observedBurn: {
|
|
804
|
+
primaryPercentPerHour: null,
|
|
805
|
+
secondaryPercentPerHour: null,
|
|
806
|
+
confidence: "unknown"
|
|
807
|
+
},
|
|
808
|
+
// Provider accounting has not supplied a typed burn observation yet.
|
|
809
|
+
// Unknown inference is explicit and contributes no fabricated tenant truth.
|
|
810
|
+
inferredUnexplainedBurn: {
|
|
811
|
+
primaryPercentPerHour: null,
|
|
812
|
+
secondaryPercentPerHour: null,
|
|
813
|
+
confidence: "unknown"
|
|
814
|
+
},
|
|
815
|
+
overlayKeys: []
|
|
816
|
+
}))
|
|
817
|
+
};
|
|
818
|
+
const replay = createCodexFleetReplayRecordV1(decisionInput, {
|
|
819
|
+
...DEFAULT_CODEX_FLEET_POLICY_V1,
|
|
820
|
+
placementUsageCeilingPercent: input.nearExhaustionPct
|
|
821
|
+
});
|
|
822
|
+
const actual = {
|
|
823
|
+
outcome: input.actualOutcome,
|
|
824
|
+
candidateKey: actualCandidateKey,
|
|
825
|
+
reason: input.actualReason
|
|
826
|
+
};
|
|
827
|
+
return {
|
|
828
|
+
schemaVersion: 1,
|
|
829
|
+
mode: "shadow",
|
|
830
|
+
actual,
|
|
831
|
+
comparison: compareDecision(actual, replay),
|
|
832
|
+
replay
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
function compareDecision(actual, replay) {
|
|
836
|
+
if (replay.truncatedCandidateCount > 0 && actual.candidateKey !== null && !replay.input.candidates.some((candidate) => candidate.key === actual.candidateKey)) {
|
|
837
|
+
return "not_comparable_truncated";
|
|
838
|
+
}
|
|
839
|
+
const shadowOutcome = replay.decision.outcome === "selected" ? "selected" : "none";
|
|
840
|
+
const actualComparableOutcome = actual.outcome === "selected" ? "selected" : "none";
|
|
841
|
+
if (shadowOutcome !== actualComparableOutcome) return "different_outcome";
|
|
842
|
+
if (shadowOutcome === "selected") {
|
|
843
|
+
return replay.decision.selectedCandidateKey === actual.candidateKey ? "match" : "different_candidate";
|
|
844
|
+
}
|
|
845
|
+
return "match";
|
|
846
|
+
}
|
|
847
|
+
function normalizeStatus(status) {
|
|
848
|
+
if (status === "active" || status === "needs_relogin" || status === "error") return status;
|
|
849
|
+
return "unknown";
|
|
850
|
+
}
|
|
851
|
+
function candidateAlias(index) {
|
|
852
|
+
return `c${index.toString(36).padStart(2, "0")}`;
|
|
853
|
+
}
|
|
854
|
+
function aliasSortKey(seed, credentialId) {
|
|
855
|
+
return createHmac("sha256", seed).update(credentialId).digest("hex");
|
|
856
|
+
}
|
|
857
|
+
function quotaConfidence(account) {
|
|
858
|
+
if (!account.usageCheckedAt) return "unknown";
|
|
859
|
+
const windows = [
|
|
860
|
+
[account.primaryUsedPercent, account.primaryResetAt],
|
|
861
|
+
[account.secondaryUsedPercent, account.secondaryResetAt]
|
|
862
|
+
];
|
|
863
|
+
const complete = windows.filter(([used, reset]) => used !== null && reset !== null).length;
|
|
864
|
+
const partial = windows.some(([used, reset]) => used === null !== (reset === null));
|
|
865
|
+
if (complete === 0) return "unknown";
|
|
866
|
+
if (partial) return "low";
|
|
867
|
+
return complete === windows.length ? "high" : "medium";
|
|
868
|
+
}
|
|
869
|
+
function remainingMs(value, now) {
|
|
870
|
+
return value ? Math.max(0, value.getTime() - now.getTime()) : null;
|
|
871
|
+
}
|
|
872
|
+
function ageMs(value, now) {
|
|
873
|
+
return value ? Math.max(0, now.getTime() - value.getTime()) : null;
|
|
874
|
+
}
|
|
875
|
+
function safeErrorName(error) {
|
|
876
|
+
if (!(error instanceof Error)) return "UnknownError";
|
|
877
|
+
return /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(error.name) ? error.name : "Error";
|
|
878
|
+
}
|
|
879
|
+
|
|
677
880
|
// src/activities/codex-auth.ts
|
|
678
881
|
import { buildCodexTokenResolver } from "@opengeni/db";
|
|
679
882
|
|
|
@@ -880,16 +1083,6 @@ import {
|
|
|
880
1083
|
summarizeForCompaction
|
|
881
1084
|
} from "@opengeni/runtime";
|
|
882
1085
|
import { contextInputBudgetTokens } from "@opengeni/config";
|
|
883
|
-
|
|
884
|
-
// src/activities/turn-attempt-fenced.ts
|
|
885
|
-
var TurnAttemptFencedError = class extends Error {
|
|
886
|
-
constructor(message) {
|
|
887
|
-
super(message);
|
|
888
|
-
this.name = "TurnAttemptFencedError";
|
|
889
|
-
}
|
|
890
|
-
};
|
|
891
|
-
|
|
892
|
-
// src/activities/context-compaction.ts
|
|
893
1086
|
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, {
|
|
894
1087
|
maxOutputTokens: SUMMARY_BUFFER_TOKENS
|
|
895
1088
|
}), options = {}) {
|
|
@@ -1950,19 +2143,16 @@ import {
|
|
|
1950
2143
|
reasoningEffortForMetadata as reasoningEffortForMetadata2
|
|
1951
2144
|
} from "@opengeni/contracts";
|
|
1952
2145
|
import {
|
|
1953
|
-
addSessionSystemUpdate,
|
|
1954
2146
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
1955
|
-
evaluateGoalContinuation,
|
|
1956
2147
|
getBillingBalance,
|
|
1957
2148
|
getLatestStartedSessionTurn,
|
|
1958
2149
|
getWorkspaceModelPolicy,
|
|
1959
2150
|
getSessionGoal,
|
|
1960
2151
|
isCodexBilledTurn,
|
|
1961
|
-
|
|
2152
|
+
materializeGoalContinuation,
|
|
1962
2153
|
requireSession,
|
|
1963
2154
|
sumUsageQuantity
|
|
1964
2155
|
} from "@opengeni/db";
|
|
1965
|
-
import { appendAndPublishEvents } from "@opengeni/events";
|
|
1966
2156
|
function createGoalActivities(services) {
|
|
1967
2157
|
async function enqueueGoalRetryWake2(input) {
|
|
1968
2158
|
const { db } = await services();
|
|
@@ -2019,84 +2209,29 @@ function createGoalActivities(services) {
|
|
|
2019
2209
|
input.workspaceId,
|
|
2020
2210
|
isCodexRun
|
|
2021
2211
|
);
|
|
2022
|
-
const decision = await
|
|
2212
|
+
const decision = await materializeGoalContinuation(db, {
|
|
2213
|
+
accountId: input.accountId,
|
|
2023
2214
|
workspaceId: input.workspaceId,
|
|
2024
2215
|
sessionId: input.sessionId,
|
|
2216
|
+
workflowId: input.workflowId,
|
|
2025
2217
|
defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
|
|
2026
2218
|
noProgressLimit: settings.goalNoProgressLimit,
|
|
2027
2219
|
// A model-policy block takes precedence: it is deterministic (a budget
|
|
2028
2220
|
// pause can clear on its own; a policy pause needs a model/policy change)
|
|
2029
2221
|
// and rides the same visible-pause channel.
|
|
2030
|
-
budgetBlocked: modelPolicyBlocked ?? budgetBlocked
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
2037
|
-
{
|
|
2038
|
-
type: "goal.paused",
|
|
2039
|
-
payload: {
|
|
2040
|
-
goalId: decision.goal.id,
|
|
2041
|
-
actor: "system",
|
|
2042
|
-
reason: decision.reason,
|
|
2043
|
-
...decision.goal.rationale ? { rationale: decision.goal.rationale } : {},
|
|
2044
|
-
autoContinuations: decision.goal.autoContinuations,
|
|
2045
|
-
noProgressStreak: decision.goal.noProgressStreak
|
|
2046
|
-
}
|
|
2047
|
-
}
|
|
2048
|
-
]);
|
|
2049
|
-
return { action: "paused" };
|
|
2050
|
-
}
|
|
2051
|
-
const recheck = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
2052
|
-
if (!recheck || recheck.status !== "active" || recheck.version !== decision.goal.version) {
|
|
2053
|
-
return { action: "none" };
|
|
2054
|
-
}
|
|
2055
|
-
const prompt = goalContinuationPrompt(decision.goal, decision.autoContinuation, decision.cap);
|
|
2056
|
-
const update = await addSessionSystemUpdate(db, {
|
|
2057
|
-
accountId: input.accountId,
|
|
2058
|
-
workspaceId: input.workspaceId,
|
|
2059
|
-
sessionId: input.sessionId,
|
|
2060
|
-
kind: "goal_continuation",
|
|
2061
|
-
classification: "info",
|
|
2062
|
-
sourceId: decision.goal.id,
|
|
2063
|
-
dedupeKey: `goal-continuation:${decision.goal.id}:${decision.goal.version}:${decision.autoContinuation}`,
|
|
2064
|
-
summary: prompt,
|
|
2065
|
-
payload: {
|
|
2066
|
-
type: "goal_continuation",
|
|
2067
|
-
goalId: decision.goal.id,
|
|
2068
|
-
goalVersion: decision.goal.version,
|
|
2069
|
-
autoContinuation: decision.autoContinuation,
|
|
2070
|
-
maxAutoContinuations: decision.cap,
|
|
2071
|
-
prompt,
|
|
2072
|
-
policy: {
|
|
2073
|
-
model: continuationModel,
|
|
2074
|
-
reasoningEffort: continuationReasoningEffort,
|
|
2075
|
-
tools: withFirstPartyTools(settings, session.tools),
|
|
2076
|
-
sandboxBackend: session.sandboxBackend
|
|
2077
|
-
}
|
|
2222
|
+
budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
|
|
2223
|
+
policy: {
|
|
2224
|
+
model: continuationModel,
|
|
2225
|
+
reasoningEffort: continuationReasoningEffort,
|
|
2226
|
+
tools: withFirstPartyTools(settings, session.tools),
|
|
2227
|
+
sandboxBackend: session.sandboxBackend
|
|
2078
2228
|
},
|
|
2079
|
-
|
|
2229
|
+
prompt: goalContinuationPrompt
|
|
2080
2230
|
});
|
|
2081
|
-
if (
|
|
2082
|
-
|
|
2083
|
-
await bus.publish(input.workspaceId, input.sessionId, update.events);
|
|
2231
|
+
if (decision.events.length > 0) {
|
|
2232
|
+
await bus.publish(input.workspaceId, input.sessionId, decision.events);
|
|
2084
2233
|
}
|
|
2085
|
-
|
|
2086
|
-
accountId: input.accountId,
|
|
2087
|
-
workspaceId: input.workspaceId,
|
|
2088
|
-
eventType: "agent_run.created",
|
|
2089
|
-
quantity: 1,
|
|
2090
|
-
unit: "run",
|
|
2091
|
-
sourceResourceType: "session_system_update",
|
|
2092
|
-
sourceResourceId: update.update.id,
|
|
2093
|
-
sessionId: input.sessionId,
|
|
2094
|
-
initiator: { kind: "service", subjectId: "goal-continuation" },
|
|
2095
|
-
initiatorContext: { goalId: decision.goal.id },
|
|
2096
|
-
origin: "goal",
|
|
2097
|
-
idempotencyKey: `agent_run.created:goal:${input.workspaceId}:${update.update.id}`
|
|
2098
|
-
});
|
|
2099
|
-
return { action: "continue" };
|
|
2234
|
+
return { action: decision.action };
|
|
2100
2235
|
}
|
|
2101
2236
|
return {
|
|
2102
2237
|
enqueueGoalRetryWake: enqueueGoalRetryWake2,
|
|
@@ -2580,6 +2715,7 @@ function isPlainObject(value) {
|
|
|
2580
2715
|
}
|
|
2581
2716
|
|
|
2582
2717
|
// src/activities/run-input.ts
|
|
2718
|
+
import { createHash } from "crypto";
|
|
2583
2719
|
import {
|
|
2584
2720
|
getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
|
|
2585
2721
|
getLatestRunState,
|
|
@@ -2624,6 +2760,104 @@ function resumeRunStateForCodexAccount(state, current) {
|
|
|
2624
2760
|
stripReasoningIdentityFromSerializedRunState(state.serializedRunState)
|
|
2625
2761
|
);
|
|
2626
2762
|
}
|
|
2763
|
+
var MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
|
2764
|
+
var MODEL_IMAGE_CONTENT_TYPES = /* @__PURE__ */ new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]);
|
|
2765
|
+
var MODEL_FILE_CONTENT_TYPES = /* @__PURE__ */ new Set([
|
|
2766
|
+
"application/json",
|
|
2767
|
+
"application/pdf",
|
|
2768
|
+
"application/x-yaml",
|
|
2769
|
+
"application/yaml"
|
|
2770
|
+
]);
|
|
2771
|
+
var BLOCKED_TEXT_CONTENT_TYPES = /* @__PURE__ */ new Set([
|
|
2772
|
+
"text/css",
|
|
2773
|
+
"text/html",
|
|
2774
|
+
"text/javascript",
|
|
2775
|
+
"text/xml"
|
|
2776
|
+
]);
|
|
2777
|
+
function safeErrorType(error) {
|
|
2778
|
+
if (!(error instanceof Error)) return typeof error;
|
|
2779
|
+
const name = error.name.trim();
|
|
2780
|
+
return /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(name) ? name : "Error";
|
|
2781
|
+
}
|
|
2782
|
+
function modelAttachmentDescriptor(contentType) {
|
|
2783
|
+
const normalized = contentType.toLowerCase().split(";", 1)[0]?.trim() ?? "";
|
|
2784
|
+
if (MODEL_IMAGE_CONTENT_TYPES.has(normalized)) {
|
|
2785
|
+
return { kind: "image", contentType: normalized };
|
|
2786
|
+
}
|
|
2787
|
+
if (MODEL_FILE_CONTENT_TYPES.has(normalized) || normalized.startsWith("text/") && !BLOCKED_TEXT_CONTENT_TYPES.has(normalized)) {
|
|
2788
|
+
return { kind: "file", contentType: normalized };
|
|
2789
|
+
}
|
|
2790
|
+
return null;
|
|
2791
|
+
}
|
|
2792
|
+
async function modelAttachmentContentForFiles(files, readFileBytes) {
|
|
2793
|
+
const attachments = [];
|
|
2794
|
+
let remainingBytes = MAX_INLINE_MODEL_ATTACHMENT_BYTES;
|
|
2795
|
+
for (const file of files) {
|
|
2796
|
+
const descriptor = modelAttachmentDescriptor(file.contentType);
|
|
2797
|
+
const checksum = file.sha256?.trim().toLowerCase() ?? "";
|
|
2798
|
+
if (file.status !== "ready" || !descriptor || file.sizeBytes > remainingBytes || !/^[a-f0-9]{64}$/.test(checksum)) {
|
|
2799
|
+
continue;
|
|
2800
|
+
}
|
|
2801
|
+
try {
|
|
2802
|
+
const bytes = await readFileBytes(file);
|
|
2803
|
+
if (bytes.byteLength !== file.sizeBytes || bytes.byteLength > remainingBytes) {
|
|
2804
|
+
console.error("model attachment bytes did not match finalized metadata", {
|
|
2805
|
+
fileId: file.id,
|
|
2806
|
+
expectedSizeBytes: file.sizeBytes,
|
|
2807
|
+
actualSizeBytes: bytes.byteLength
|
|
2808
|
+
});
|
|
2809
|
+
continue;
|
|
2810
|
+
}
|
|
2811
|
+
if (createHash("sha256").update(bytes).digest("hex") !== checksum) {
|
|
2812
|
+
console.error("model attachment checksum did not match finalized metadata", {
|
|
2813
|
+
fileId: file.id
|
|
2814
|
+
});
|
|
2815
|
+
continue;
|
|
2816
|
+
}
|
|
2817
|
+
attachments.push({
|
|
2818
|
+
kind: descriptor.kind,
|
|
2819
|
+
fileId: file.id,
|
|
2820
|
+
filename: file.safeFilename,
|
|
2821
|
+
contentType: descriptor.contentType,
|
|
2822
|
+
dataUrl: `data:${descriptor.contentType};base64,${Buffer.from(bytes).toString("base64")}`
|
|
2823
|
+
});
|
|
2824
|
+
remainingBytes -= bytes.byteLength;
|
|
2825
|
+
} catch (error) {
|
|
2826
|
+
console.error("model attachment content read failed; retaining sandbox path fallback", {
|
|
2827
|
+
fileId: file.id,
|
|
2828
|
+
errorType: safeErrorType(error)
|
|
2829
|
+
});
|
|
2830
|
+
}
|
|
2831
|
+
}
|
|
2832
|
+
return attachments;
|
|
2833
|
+
}
|
|
2834
|
+
function withCurrentUserAttachmentContent(historyItems, attachments) {
|
|
2835
|
+
if (attachments.length === 0) return historyItems;
|
|
2836
|
+
let currentUserIndex = -1;
|
|
2837
|
+
for (let index = historyItems.length - 1; index >= 0; index -= 1) {
|
|
2838
|
+
const item = historyItems[index];
|
|
2839
|
+
if (item?.type === "message" && item.role === "user") {
|
|
2840
|
+
currentUserIndex = index;
|
|
2841
|
+
break;
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
if (currentUserIndex < 0) return historyItems;
|
|
2845
|
+
const currentUser = historyItems[currentUserIndex];
|
|
2846
|
+
const existingContent = Array.isArray(currentUser.content) ? [...currentUser.content] : [{ type: "input_text", text: String(currentUser.content ?? "") }];
|
|
2847
|
+
const attachmentContent = attachments.map(
|
|
2848
|
+
(attachment) => attachment.kind === "image" ? { type: "input_image", image: attachment.dataUrl } : {
|
|
2849
|
+
type: "input_file",
|
|
2850
|
+
file: attachment.dataUrl,
|
|
2851
|
+
filename: attachment.filename
|
|
2852
|
+
}
|
|
2853
|
+
);
|
|
2854
|
+
const projected = [...historyItems];
|
|
2855
|
+
projected[currentUserIndex] = {
|
|
2856
|
+
...currentUser,
|
|
2857
|
+
content: [...existingContent, ...attachmentContent]
|
|
2858
|
+
};
|
|
2859
|
+
return projected;
|
|
2860
|
+
}
|
|
2627
2861
|
async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN, options) {
|
|
2628
2862
|
if (!trigger) {
|
|
2629
2863
|
throw new Error("Missing trigger event");
|
|
@@ -2649,11 +2883,17 @@ async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN,
|
|
|
2649
2883
|
if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
|
|
2650
2884
|
throw new Error("user.message payload is missing text");
|
|
2651
2885
|
}
|
|
2652
|
-
const
|
|
2886
|
+
const resources = Array.isArray(payload.resources) ? payload.resources : [];
|
|
2887
|
+
const fileAttachments = await resolveUserMessageFileAttachments(
|
|
2653
2888
|
db,
|
|
2654
2889
|
trigger.workspaceId,
|
|
2655
|
-
|
|
2890
|
+
resources
|
|
2656
2891
|
);
|
|
2892
|
+
const attachmentContext = userMessageAttachmentsContext(fileAttachments);
|
|
2893
|
+
const modelAttachments = options.readFileBytesForModel ? await modelAttachmentContentForFiles(
|
|
2894
|
+
fileAttachments.map((attachment) => attachment.file),
|
|
2895
|
+
options.readFileBytesForModel
|
|
2896
|
+
) : [];
|
|
2657
2897
|
return await messageInput(
|
|
2658
2898
|
db,
|
|
2659
2899
|
runtime,
|
|
@@ -2661,7 +2901,8 @@ async function turnInput(db, runtime, agent, trigger, current = NON_CODEX_TURN,
|
|
|
2661
2901
|
trigger,
|
|
2662
2902
|
void 0,
|
|
2663
2903
|
joinInternalContext(internalContext, attachmentContext),
|
|
2664
|
-
current
|
|
2904
|
+
current,
|
|
2905
|
+
modelAttachments
|
|
2665
2906
|
);
|
|
2666
2907
|
}
|
|
2667
2908
|
if (trigger.type === "system.update.delivered") {
|
|
@@ -2736,10 +2977,13 @@ function systemUpdateContext(updates) {
|
|
|
2736
2977
|
})
|
|
2737
2978
|
].join("\n");
|
|
2738
2979
|
}
|
|
2739
|
-
async function messageInput(db, runtime, agent, trigger, text, internalContext, current = NON_CODEX_TURN) {
|
|
2980
|
+
async function messageInput(db, runtime, agent, trigger, text, internalContext, current = NON_CODEX_TURN, modelAttachments = []) {
|
|
2740
2981
|
const stored = await getActiveSessionHistoryItems2(db, trigger.workspaceId, trigger.sessionId);
|
|
2741
2982
|
const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
|
|
2742
|
-
const historyItems =
|
|
2983
|
+
const historyItems = withCurrentUserAttachmentContent(
|
|
2984
|
+
applyCodexHistoryStrip(stored, current),
|
|
2985
|
+
modelAttachments
|
|
2986
|
+
);
|
|
2743
2987
|
return {
|
|
2744
2988
|
input: await runtime.prepareInput(agent, {
|
|
2745
2989
|
kind: "message",
|
|
@@ -2751,17 +2995,19 @@ async function messageInput(db, runtime, agent, trigger, text, internalContext,
|
|
|
2751
2995
|
modelHistoryFromItems: true
|
|
2752
2996
|
};
|
|
2753
2997
|
}
|
|
2754
|
-
async function
|
|
2755
|
-
const
|
|
2998
|
+
async function resolveUserMessageFileAttachments(db, workspaceId, resources) {
|
|
2999
|
+
const attachments = [];
|
|
2756
3000
|
for (const resource of resources) {
|
|
2757
|
-
if (resource.kind !== "file")
|
|
2758
|
-
continue;
|
|
2759
|
-
}
|
|
3001
|
+
if (resource.kind !== "file") continue;
|
|
2760
3002
|
const file = await requireFile(db, workspaceId, resource.fileId);
|
|
2761
|
-
|
|
2762
|
-
`- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`
|
|
2763
|
-
);
|
|
3003
|
+
attachments.push({ resource, file });
|
|
2764
3004
|
}
|
|
3005
|
+
return attachments;
|
|
3006
|
+
}
|
|
3007
|
+
function userMessageAttachmentsContext(attachments) {
|
|
3008
|
+
const attachedFiles = attachments.map(
|
|
3009
|
+
({ resource, file }) => `- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`
|
|
3010
|
+
);
|
|
2765
3011
|
if (attachedFiles.length === 0) {
|
|
2766
3012
|
return void 0;
|
|
2767
3013
|
}
|
|
@@ -2918,24 +3164,32 @@ async function nextStreamEvent(iterator, context) {
|
|
|
2918
3164
|
// src/sandbox-resume.ts
|
|
2919
3165
|
import {
|
|
2920
3166
|
acquireLease,
|
|
3167
|
+
beginSandboxRematerialization,
|
|
2921
3168
|
commitWarmingToWarm,
|
|
3169
|
+
failSandboxRematerialization,
|
|
2922
3170
|
failWarmingToCold,
|
|
2923
3171
|
getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
|
|
3172
|
+
markSandboxRestoreVerifying,
|
|
2924
3173
|
markWarmLeaseInstanceLost,
|
|
2925
3174
|
persistWarmSnapshot,
|
|
3175
|
+
readWorkspaceArchiveCapturePreflight,
|
|
2926
3176
|
readLease,
|
|
2927
3177
|
recordWarmingSandboxCreated,
|
|
2928
3178
|
releaseLeaseHolder,
|
|
2929
3179
|
touchLeaseHolder,
|
|
3180
|
+
SandboxLeaseRecoveryBlockedError,
|
|
2930
3181
|
SandboxLeaseSupersededError
|
|
2931
3182
|
} from "@opengeni/db";
|
|
2932
3183
|
import {
|
|
3184
|
+
captureVerifiedWorkspaceArchive,
|
|
3185
|
+
WorkspaceArchiveIntegrityError,
|
|
2933
3186
|
establishSandboxSessionFromEnvelope,
|
|
2934
3187
|
isProviderSandboxNotFoundError,
|
|
2935
|
-
|
|
2936
|
-
|
|
3188
|
+
requirePersistableReplacementSandboxEnvelope,
|
|
3189
|
+
serializeReplacementSandboxEnvelope,
|
|
2937
3190
|
deletePriorPersistedSnapshot,
|
|
2938
|
-
tagModalSandbox
|
|
3191
|
+
tagModalSandbox,
|
|
3192
|
+
verifySandboxExecReadiness
|
|
2939
3193
|
} from "@opengeni/runtime";
|
|
2940
3194
|
function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
2941
3195
|
const normalized = attemptId.trim();
|
|
@@ -2968,30 +3222,13 @@ async function sleep(ms) {
|
|
|
2968
3222
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
2969
3223
|
}
|
|
2970
3224
|
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS) {
|
|
2971
|
-
if (established.backendId !== "modal") return;
|
|
2972
|
-
const session = established.session;
|
|
2973
|
-
const run = session.exec ?? session.execCommand;
|
|
2974
|
-
if (!run) {
|
|
2975
|
-
throw new Error("Modal sandbox session does not expose exec");
|
|
2976
|
-
}
|
|
2977
|
-
let timer;
|
|
2978
3225
|
try {
|
|
2979
|
-
await
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
new Promise((_, reject) => {
|
|
2986
|
-
timer = setTimeout(
|
|
2987
|
-
() => reject(new SandboxWarmingTimeoutError(established.backendId, timeoutMs)),
|
|
2988
|
-
timeoutMs
|
|
2989
|
-
);
|
|
2990
|
-
if (timer && "unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
2991
|
-
})
|
|
2992
|
-
]);
|
|
2993
|
-
} finally {
|
|
2994
|
-
if (timer) clearTimeout(timer);
|
|
3226
|
+
await verifySandboxExecReadiness(established, timeoutMs);
|
|
3227
|
+
} catch (error) {
|
|
3228
|
+
if (/sandbox creation timed out/i.test(error instanceof Error ? error.message : "")) {
|
|
3229
|
+
throw new SandboxWarmingTimeoutError(established.backendId, timeoutMs);
|
|
3230
|
+
}
|
|
3231
|
+
throw error;
|
|
2995
3232
|
}
|
|
2996
3233
|
}
|
|
2997
3234
|
var SnapshotTimeoutError = class extends Error {
|
|
@@ -3045,7 +3282,10 @@ async function terminateEstablishedSandbox(established) {
|
|
|
3045
3282
|
try {
|
|
3046
3283
|
await client.delete(established.sessionState);
|
|
3047
3284
|
return true;
|
|
3048
|
-
} catch {
|
|
3285
|
+
} catch (error) {
|
|
3286
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
3287
|
+
return true;
|
|
3288
|
+
}
|
|
3049
3289
|
return false;
|
|
3050
3290
|
}
|
|
3051
3291
|
}
|
|
@@ -3064,7 +3304,10 @@ async function terminateEstablishedSandbox(established) {
|
|
|
3064
3304
|
return true;
|
|
3065
3305
|
}
|
|
3066
3306
|
return false;
|
|
3067
|
-
} catch {
|
|
3307
|
+
} catch (error) {
|
|
3308
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
3309
|
+
return true;
|
|
3310
|
+
}
|
|
3068
3311
|
return false;
|
|
3069
3312
|
}
|
|
3070
3313
|
}
|
|
@@ -3088,27 +3331,23 @@ function workspaceArchiveFieldsFromEnvelope(envelope) {
|
|
|
3088
3331
|
return null;
|
|
3089
3332
|
}
|
|
3090
3333
|
const previous = sessionState?.workspaceArchivePrev;
|
|
3334
|
+
const metadata = sessionState?.workspaceArchiveMeta;
|
|
3335
|
+
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
3091
3336
|
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
3092
3337
|
return {
|
|
3093
3338
|
workspaceArchive: archive,
|
|
3339
|
+
...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
|
|
3094
3340
|
...typeof previous === "string" && previous.length > 0 ? { workspaceArchivePrev: previous } : {},
|
|
3341
|
+
...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
|
|
3095
3342
|
...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
|
|
3096
3343
|
};
|
|
3097
3344
|
}
|
|
3098
|
-
function
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
}
|
|
3103
|
-
|
|
3104
|
-
return {
|
|
3105
|
-
...resumeState ?? {},
|
|
3106
|
-
...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
|
|
3107
|
-
sessionState: {
|
|
3108
|
-
...existingSessionState,
|
|
3109
|
-
...archiveFields
|
|
3110
|
-
}
|
|
3111
|
-
};
|
|
3345
|
+
function withoutProviderIdentity(envelope) {
|
|
3346
|
+
if (!envelope) return null;
|
|
3347
|
+
const sessionState = envelope.sessionState && typeof envelope.sessionState === "object" ? envelope.sessionState : null;
|
|
3348
|
+
if (!sessionState) return envelope;
|
|
3349
|
+
const { providerState: _providerState, ...providerIndependentState } = sessionState;
|
|
3350
|
+
return { ...envelope, sessionState: providerIndependentState };
|
|
3112
3351
|
}
|
|
3113
3352
|
async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal) {
|
|
3114
3353
|
const { db, settings } = services;
|
|
@@ -3125,24 +3364,33 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3125
3364
|
}
|
|
3126
3365
|
try {
|
|
3127
3366
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
3128
|
-
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm") {
|
|
3367
|
+
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm" || lease.instanceId === null) {
|
|
3129
3368
|
return false;
|
|
3130
3369
|
}
|
|
3370
|
+
const preflight = await readWorkspaceArchiveCapturePreflight(db, {
|
|
3371
|
+
accountId: ids.accountId,
|
|
3372
|
+
workspaceId: ids.workspaceId,
|
|
3373
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3374
|
+
expectedEpoch: leaseEpoch,
|
|
3375
|
+
expectedInstanceId: lease.instanceId,
|
|
3376
|
+
liveness: "warm"
|
|
3377
|
+
});
|
|
3378
|
+
if (!preflight) return false;
|
|
3131
3379
|
if (signal?.aborted) {
|
|
3132
3380
|
return false;
|
|
3133
3381
|
}
|
|
3134
3382
|
const sessionState = lease.resumeState && typeof lease.resumeState === "object" ? lease.resumeState.sessionState : void 0;
|
|
3135
3383
|
const priorAtRaw = sessionState && typeof sessionState === "object" ? sessionState.workspaceArchiveAt : void 0;
|
|
3136
3384
|
const priorAtMs = typeof priorAtRaw === "string" ? Date.parse(priorAtRaw) : Number.NaN;
|
|
3137
|
-
if (Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
|
|
3385
|
+
if (preflight.archiveComplete && Number.isFinite(priorAtMs) && Date.now() - priorAtMs < intervalMs) {
|
|
3138
3386
|
return false;
|
|
3139
3387
|
}
|
|
3140
3388
|
let timeout;
|
|
3141
3389
|
const capturedAtMs = Date.now();
|
|
3142
|
-
const capture =
|
|
3390
|
+
const capture = captureVerifiedWorkspaceArchive(session, capturedAtMs);
|
|
3143
3391
|
capture.catch(() => void 0);
|
|
3144
3392
|
let cancelListener;
|
|
3145
|
-
const
|
|
3393
|
+
const archive = await Promise.race([
|
|
3146
3394
|
capture,
|
|
3147
3395
|
...signal ? [
|
|
3148
3396
|
new Promise((resolve) => {
|
|
@@ -3163,7 +3411,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3163
3411
|
}
|
|
3164
3412
|
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
3165
3413
|
});
|
|
3166
|
-
if (!
|
|
3414
|
+
if (!archive) {
|
|
3167
3415
|
return false;
|
|
3168
3416
|
}
|
|
3169
3417
|
if (signal?.aborted) {
|
|
@@ -3177,7 +3425,10 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
3177
3425
|
attemptId: ids.attemptId,
|
|
3178
3426
|
sandboxGroupId: ids.sandboxGroupId,
|
|
3179
3427
|
expectedEpoch: leaseEpoch,
|
|
3180
|
-
|
|
3428
|
+
expectedInstanceId: lease.instanceId,
|
|
3429
|
+
expectedWorkspaceGeneration: preflight.workspaceGeneration,
|
|
3430
|
+
workspaceArchive: archive.base64,
|
|
3431
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
3181
3432
|
minIntervalMs: intervalMs,
|
|
3182
3433
|
capturedAtMs
|
|
3183
3434
|
});
|
|
@@ -3247,6 +3498,15 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3247
3498
|
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
3248
3499
|
holderLivenessTimer.unref();
|
|
3249
3500
|
}
|
|
3501
|
+
if (acquired.role === "blocked") {
|
|
3502
|
+
await release();
|
|
3503
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3504
|
+
ids.sandboxGroupId,
|
|
3505
|
+
acquired.lease.leaseEpoch,
|
|
3506
|
+
acquired.code,
|
|
3507
|
+
acquired.lease.recovery
|
|
3508
|
+
);
|
|
3509
|
+
}
|
|
3250
3510
|
if (acquired.role === "fenced") {
|
|
3251
3511
|
await release();
|
|
3252
3512
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
@@ -3254,9 +3514,52 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3254
3514
|
if (acquired.role === "spawner") {
|
|
3255
3515
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
3256
3516
|
let createdEstablished = null;
|
|
3517
|
+
let rematerialization = null;
|
|
3257
3518
|
try {
|
|
3258
3519
|
const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
3259
|
-
const
|
|
3520
|
+
const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutProviderIdentity(envelope) : null;
|
|
3521
|
+
const spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
3522
|
+
const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
|
|
3523
|
+
if (acquired.lease.recovery.archive.status === "available" || workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
|
|
3524
|
+
const rematerializationId = crypto.randomUUID();
|
|
3525
|
+
const begun = await beginSandboxRematerialization(db, {
|
|
3526
|
+
accountId: ids.accountId,
|
|
3527
|
+
workspaceId: ids.workspaceId,
|
|
3528
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3529
|
+
expectedEpoch,
|
|
3530
|
+
rematerializationId,
|
|
3531
|
+
archiveSource
|
|
3532
|
+
});
|
|
3533
|
+
if (begun.status !== "started") {
|
|
3534
|
+
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
3535
|
+
throw new SandboxLeaseSupersededError(
|
|
3536
|
+
ids.sandboxGroupId,
|
|
3537
|
+
begun.lease?.leaseEpoch ?? expectedEpoch
|
|
3538
|
+
);
|
|
3539
|
+
}
|
|
3540
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3541
|
+
ids.sandboxGroupId,
|
|
3542
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
3543
|
+
begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
|
|
3544
|
+
begun.lease?.recovery ?? acquired.lease.recovery
|
|
3545
|
+
);
|
|
3546
|
+
}
|
|
3547
|
+
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
3548
|
+
if (!selectedRevision) {
|
|
3549
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3550
|
+
"archive_metadata_invalid",
|
|
3551
|
+
"sandbox rematerialization selected no durable archive revision"
|
|
3552
|
+
);
|
|
3553
|
+
}
|
|
3554
|
+
rematerialization = { id: rematerializationId, selectedRevision };
|
|
3555
|
+
} else if (acquired.lease.recovery.archive.status !== "none") {
|
|
3556
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3557
|
+
ids.sandboxGroupId,
|
|
3558
|
+
expectedEpoch,
|
|
3559
|
+
"restore_degraded",
|
|
3560
|
+
acquired.lease.recovery
|
|
3561
|
+
);
|
|
3562
|
+
}
|
|
3260
3563
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
3261
3564
|
sessionId: ids.sessionId,
|
|
3262
3565
|
recovery: "create-or-restore",
|
|
@@ -3265,15 +3568,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3265
3568
|
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
3266
3569
|
onSandboxCreated: async (created) => {
|
|
3267
3570
|
createdEstablished = created;
|
|
3268
|
-
const resumeEnvelope2 =
|
|
3269
|
-
await
|
|
3270
|
-
|
|
3571
|
+
const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
|
|
3572
|
+
await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
|
|
3573
|
+
created.backendId
|
|
3271
3574
|
);
|
|
3272
3575
|
const recorded = await recordWarmingSandboxCreated(db, {
|
|
3273
3576
|
accountId: ids.accountId,
|
|
3274
3577
|
workspaceId: ids.workspaceId,
|
|
3275
3578
|
sandboxGroupId: ids.sandboxGroupId,
|
|
3276
3579
|
expectedEpoch,
|
|
3580
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
3277
3581
|
instanceId: created.instanceId,
|
|
3278
3582
|
resumeBackendId: created.backendId,
|
|
3279
3583
|
resumeState: resumeEnvelope2,
|
|
@@ -3292,12 +3596,38 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3292
3596
|
sandboxGroupId: ids.sandboxGroupId
|
|
3293
3597
|
}).catch(() => void 0);
|
|
3294
3598
|
}
|
|
3599
|
+
},
|
|
3600
|
+
onWorkspaceRestoreVerifying: async (descriptor) => {
|
|
3601
|
+
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
3602
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3603
|
+
"archive_metadata_invalid",
|
|
3604
|
+
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
|
|
3605
|
+
);
|
|
3606
|
+
}
|
|
3607
|
+
const verifying = await markSandboxRestoreVerifying(db, {
|
|
3608
|
+
accountId: ids.accountId,
|
|
3609
|
+
workspaceId: ids.workspaceId,
|
|
3610
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3611
|
+
expectedEpoch,
|
|
3612
|
+
rematerializationId: rematerialization.id
|
|
3613
|
+
});
|
|
3614
|
+
if (!verifying.wrote) {
|
|
3615
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3616
|
+
}
|
|
3295
3617
|
}
|
|
3296
3618
|
});
|
|
3297
3619
|
createdEstablished = established;
|
|
3298
3620
|
await waitForSandboxExecReadiness(established);
|
|
3299
|
-
const
|
|
3300
|
-
|
|
3621
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
3622
|
+
await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
|
|
3623
|
+
established.backendId
|
|
3624
|
+
);
|
|
3625
|
+
if (rematerialization && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
|
|
3626
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3627
|
+
"workspace_fingerprint_mismatch",
|
|
3628
|
+
"sandbox restore completed without the exact selected durable archive revision"
|
|
3629
|
+
);
|
|
3630
|
+
}
|
|
3301
3631
|
const committed = await commitWarmingToWarm(db, {
|
|
3302
3632
|
accountId: ids.accountId,
|
|
3303
3633
|
workspaceId: ids.workspaceId,
|
|
@@ -3309,10 +3639,27 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3309
3639
|
dataPlaneUrl: null,
|
|
3310
3640
|
resumeBackendId: established.backendId,
|
|
3311
3641
|
resumeState: resumeEnvelope,
|
|
3642
|
+
...rematerialization ? {
|
|
3643
|
+
rematerialization: {
|
|
3644
|
+
id: rematerialization.id,
|
|
3645
|
+
verifiedRevision: rematerialization.selectedRevision
|
|
3646
|
+
}
|
|
3647
|
+
} : {},
|
|
3312
3648
|
leaseTtlMs
|
|
3313
3649
|
});
|
|
3314
3650
|
if (!committed.committed || !committed.lease) {
|
|
3315
|
-
await terminateEstablishedSandbox(established);
|
|
3651
|
+
const terminated = await terminateEstablishedSandbox(established);
|
|
3652
|
+
if (terminated && rematerialization) {
|
|
3653
|
+
await failSandboxRematerialization(db, {
|
|
3654
|
+
accountId: ids.accountId,
|
|
3655
|
+
workspaceId: ids.workspaceId,
|
|
3656
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3657
|
+
expectedEpoch,
|
|
3658
|
+
rematerializationId: rematerialization.id,
|
|
3659
|
+
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
3660
|
+
retryable: false
|
|
3661
|
+
});
|
|
3662
|
+
}
|
|
3316
3663
|
await release();
|
|
3317
3664
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3318
3665
|
}
|
|
@@ -3325,12 +3672,24 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3325
3672
|
}
|
|
3326
3673
|
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
3327
3674
|
if (terminated) {
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3675
|
+
if (rematerialization) {
|
|
3676
|
+
await failSandboxRematerialization(db, {
|
|
3677
|
+
accountId: ids.accountId,
|
|
3678
|
+
workspaceId: ids.workspaceId,
|
|
3679
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3680
|
+
expectedEpoch,
|
|
3681
|
+
rematerializationId: rematerialization.id,
|
|
3682
|
+
failureCode: error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
|
|
3683
|
+
retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true
|
|
3684
|
+
});
|
|
3685
|
+
} else {
|
|
3686
|
+
await failWarmingToCold(db, {
|
|
3687
|
+
accountId: ids.accountId,
|
|
3688
|
+
workspaceId: ids.workspaceId,
|
|
3689
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3690
|
+
expectedEpoch
|
|
3691
|
+
});
|
|
3692
|
+
}
|
|
3334
3693
|
}
|
|
3335
3694
|
await release();
|
|
3336
3695
|
const warmingError = asSandboxWarmingError(
|
|
@@ -3361,7 +3720,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
3361
3720
|
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
3362
3721
|
});
|
|
3363
3722
|
} catch (error) {
|
|
3364
|
-
if (!
|
|
3723
|
+
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
3365
3724
|
throw error;
|
|
3366
3725
|
}
|
|
3367
3726
|
const marked = await markWarmLeaseInstanceLost(db, {
|
|
@@ -3430,6 +3789,15 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
|
3430
3789
|
leaseTtlMs,
|
|
3431
3790
|
warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs
|
|
3432
3791
|
});
|
|
3792
|
+
if (acquired.role === "blocked") {
|
|
3793
|
+
await release();
|
|
3794
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3795
|
+
ids.sandboxGroupId,
|
|
3796
|
+
acquired.lease.leaseEpoch,
|
|
3797
|
+
acquired.code,
|
|
3798
|
+
acquired.lease.recovery
|
|
3799
|
+
);
|
|
3800
|
+
}
|
|
3433
3801
|
if (acquired.role === "fenced") {
|
|
3434
3802
|
await release();
|
|
3435
3803
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
@@ -3471,6 +3839,14 @@ async function waitForWarm(services, ids) {
|
|
|
3471
3839
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3472
3840
|
}
|
|
3473
3841
|
if (lease.liveness === "cold") {
|
|
3842
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
|
|
3843
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3844
|
+
ids.sandboxGroupId,
|
|
3845
|
+
lease.leaseEpoch,
|
|
3846
|
+
lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
|
|
3847
|
+
lease.recovery
|
|
3848
|
+
);
|
|
3849
|
+
}
|
|
3474
3850
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3475
3851
|
}
|
|
3476
3852
|
}
|
|
@@ -3478,13 +3854,29 @@ async function waitForWarm(services, ids) {
|
|
|
3478
3854
|
}
|
|
3479
3855
|
|
|
3480
3856
|
// src/sandbox-routing.ts
|
|
3481
|
-
import {
|
|
3857
|
+
import {
|
|
3858
|
+
advanceWorkspaceGenerationForRetainedProcess,
|
|
3859
|
+
advanceWorkspaceGeneration,
|
|
3860
|
+
retainWorkspaceMutationProcess,
|
|
3861
|
+
settleRetainedProcess,
|
|
3862
|
+
verifyRetainedProcessMutationSettlement,
|
|
3863
|
+
verifyWorkspaceMutationSettlement,
|
|
3864
|
+
getSandbox,
|
|
3865
|
+
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
3866
|
+
readLease as readLease2,
|
|
3867
|
+
readActiveSandbox
|
|
3868
|
+
} from "@opengeni/db";
|
|
3482
3869
|
import {
|
|
3483
3870
|
buildSelfhostedBackendSession,
|
|
3871
|
+
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
|
|
3872
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
3873
|
+
isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
|
|
3484
3874
|
makeActiveBackendResolver,
|
|
3485
3875
|
NatsControlRpc,
|
|
3486
3876
|
NatsOpStreamTransport,
|
|
3487
|
-
|
|
3877
|
+
RoutingBackendRecoveryRequiredError,
|
|
3878
|
+
RoutingSandboxSession,
|
|
3879
|
+
verifySandboxExecReadiness as verifySandboxExecReadiness2
|
|
3488
3880
|
} from "@opengeni/runtime";
|
|
3489
3881
|
function relayConfigFromSettings(settings) {
|
|
3490
3882
|
const raw = settings.selfhostedRelayUrl?.trim();
|
|
@@ -3510,6 +3902,231 @@ function selfhostedResolverTimeouts(settings) {
|
|
|
3510
3902
|
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
3511
3903
|
return { selfhostedTimeoutMs: timeoutMs, selfhostedExecTimeoutMs: execTimeoutMs };
|
|
3512
3904
|
}
|
|
3905
|
+
function homeRouteRecoveryDisposition(lease) {
|
|
3906
|
+
if (!lease) return "unrecoverable";
|
|
3907
|
+
if (lease.liveness === "warming") return "pending";
|
|
3908
|
+
if (lease.liveness === "draining") return "superseded";
|
|
3909
|
+
if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
|
|
3910
|
+
return "pending";
|
|
3911
|
+
}
|
|
3912
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
|
|
3913
|
+
return "degraded";
|
|
3914
|
+
}
|
|
3915
|
+
return "unrecoverable";
|
|
3916
|
+
}
|
|
3917
|
+
function homeRouteRecoveryError(lease, fallbackEpoch) {
|
|
3918
|
+
return new RoutingBackendRecoveryRequiredError(
|
|
3919
|
+
"resolve_home_backend",
|
|
3920
|
+
lease?.leaseEpoch ?? fallbackEpoch,
|
|
3921
|
+
homeRouteRecoveryDisposition(lease)
|
|
3922
|
+
);
|
|
3923
|
+
}
|
|
3924
|
+
function providerIdentityFromResumeState(resumeState) {
|
|
3925
|
+
const sessionState = resumeState.sessionState && typeof resumeState.sessionState === "object" ? resumeState.sessionState : null;
|
|
3926
|
+
const providerState = sessionState?.providerState && typeof sessionState.providerState === "object" ? sessionState.providerState : null;
|
|
3927
|
+
for (const field of [
|
|
3928
|
+
"sandboxId",
|
|
3929
|
+
"instanceId",
|
|
3930
|
+
"id",
|
|
3931
|
+
"hostId",
|
|
3932
|
+
"containerId",
|
|
3933
|
+
"workspaceRootPath",
|
|
3934
|
+
"agentId"
|
|
3935
|
+
]) {
|
|
3936
|
+
const value = providerState?.[field];
|
|
3937
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
3938
|
+
}
|
|
3939
|
+
return null;
|
|
3940
|
+
}
|
|
3941
|
+
async function resolveCurrentHomeBackend(services, ids, established) {
|
|
3942
|
+
const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
|
|
3943
|
+
const fallbackEpoch = lease?.leaseEpoch ?? 0;
|
|
3944
|
+
if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
|
|
3945
|
+
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
3946
|
+
}
|
|
3947
|
+
if (lease.instanceId === established.instanceId) {
|
|
3948
|
+
services.onHomeSandboxRebound?.({ established, leaseEpoch: lease.leaseEpoch });
|
|
3949
|
+
return {
|
|
3950
|
+
session: established.session,
|
|
3951
|
+
sandboxId: null,
|
|
3952
|
+
kind: established.backendId,
|
|
3953
|
+
leaseEpoch: lease.leaseEpoch,
|
|
3954
|
+
providerInstanceId: lease.instanceId
|
|
3955
|
+
};
|
|
3956
|
+
}
|
|
3957
|
+
const resumeState = lease.resumeState;
|
|
3958
|
+
const resumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
3959
|
+
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
3960
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
3961
|
+
}
|
|
3962
|
+
let rebound;
|
|
3963
|
+
try {
|
|
3964
|
+
rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
|
|
3965
|
+
sessionId: ids.sessionId,
|
|
3966
|
+
recovery: "resume-only",
|
|
3967
|
+
backendOverride: resumeBackend
|
|
3968
|
+
});
|
|
3969
|
+
await verifySandboxExecReadiness2(rebound);
|
|
3970
|
+
} catch (error) {
|
|
3971
|
+
if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
|
|
3972
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
3973
|
+
"resolve_home_backend",
|
|
3974
|
+
lease.leaseEpoch,
|
|
3975
|
+
"pending"
|
|
3976
|
+
);
|
|
3977
|
+
}
|
|
3978
|
+
const marked = await markWarmLeaseInstanceLost2(services.db, {
|
|
3979
|
+
accountId: ids.accountId,
|
|
3980
|
+
workspaceId: ids.workspaceId,
|
|
3981
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3982
|
+
expectedEpoch: lease.leaseEpoch,
|
|
3983
|
+
expectedInstanceId: lease.instanceId,
|
|
3984
|
+
diagnostic: "provider_not_found_during_home_route_rebind"
|
|
3985
|
+
});
|
|
3986
|
+
if (marked.status === "marked") {
|
|
3987
|
+
await services.onHomeSandboxLost?.({
|
|
3988
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3989
|
+
instanceId: lease.instanceId,
|
|
3990
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
3991
|
+
});
|
|
3992
|
+
}
|
|
3993
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
3994
|
+
"resolve_home_backend",
|
|
3995
|
+
marked.lease?.leaseEpoch ?? lease.leaseEpoch,
|
|
3996
|
+
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
3997
|
+
);
|
|
3998
|
+
}
|
|
3999
|
+
if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
|
|
4000
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4001
|
+
}
|
|
4002
|
+
services.onHomeSandboxRebound?.({ established: rebound, leaseEpoch: lease.leaseEpoch });
|
|
4003
|
+
Object.assign(established, rebound);
|
|
4004
|
+
return {
|
|
4005
|
+
session: rebound.session,
|
|
4006
|
+
sandboxId: null,
|
|
4007
|
+
kind: rebound.backendId,
|
|
4008
|
+
leaseEpoch: lease.leaseEpoch,
|
|
4009
|
+
providerInstanceId: lease.instanceId
|
|
4010
|
+
};
|
|
4011
|
+
}
|
|
4012
|
+
function beforePersistableHomeMutation(services, ids, home) {
|
|
4013
|
+
const fence = ids.workspaceMutationFence;
|
|
4014
|
+
if (!home || !fence) return void 0;
|
|
4015
|
+
return async ({ op, backend }) => {
|
|
4016
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
4017
|
+
return null;
|
|
4018
|
+
}
|
|
4019
|
+
return await advanceWorkspaceGeneration(services.db, {
|
|
4020
|
+
accountId: fence.accountId,
|
|
4021
|
+
workspaceId: ids.workspaceId,
|
|
4022
|
+
sessionId: ids.sessionId,
|
|
4023
|
+
turnId: fence.turnId,
|
|
4024
|
+
executionGeneration: fence.executionGeneration,
|
|
4025
|
+
attemptId: fence.attemptId,
|
|
4026
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
4027
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4028
|
+
expectedEpoch: backend.leaseEpoch,
|
|
4029
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
4030
|
+
operation: op
|
|
4031
|
+
});
|
|
4032
|
+
};
|
|
4033
|
+
}
|
|
4034
|
+
function afterPersistableHomeMutation(services, ids, home) {
|
|
4035
|
+
const fence = ids.workspaceMutationFence;
|
|
4036
|
+
if (!home || !fence) return void 0;
|
|
4037
|
+
return async ({ op, backend, admission, outcome, retainedProcess }) => {
|
|
4038
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
|
|
4039
|
+
return;
|
|
4040
|
+
}
|
|
4041
|
+
const exactAdmission = admission;
|
|
4042
|
+
if (outcome === "resolved" && retainedProcess) {
|
|
4043
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
4044
|
+
accountId: fence.accountId,
|
|
4045
|
+
workspaceId: ids.workspaceId,
|
|
4046
|
+
sessionId: ids.sessionId,
|
|
4047
|
+
processId: retainedProcess.id,
|
|
4048
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
4049
|
+
admissionId: exactAdmission.id,
|
|
4050
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
4051
|
+
operation: op,
|
|
4052
|
+
owner: {
|
|
4053
|
+
kind: "turn",
|
|
4054
|
+
turnId: fence.turnId,
|
|
4055
|
+
executionGeneration: fence.executionGeneration,
|
|
4056
|
+
attemptId: fence.attemptId,
|
|
4057
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
4058
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4059
|
+
expectedEpoch: backend.leaseEpoch,
|
|
4060
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
4061
|
+
routeKind: exactAdmission.routeKind,
|
|
4062
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
4063
|
+
routeEpoch: exactAdmission.routeEpoch
|
|
4064
|
+
}
|
|
4065
|
+
});
|
|
4066
|
+
return;
|
|
4067
|
+
}
|
|
4068
|
+
await verifyWorkspaceMutationSettlement(services.db, {
|
|
4069
|
+
accountId: fence.accountId,
|
|
4070
|
+
workspaceId: ids.workspaceId,
|
|
4071
|
+
sessionId: ids.sessionId,
|
|
4072
|
+
turnId: fence.turnId,
|
|
4073
|
+
executionGeneration: fence.executionGeneration,
|
|
4074
|
+
attemptId: fence.attemptId,
|
|
4075
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
4076
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4077
|
+
expectedEpoch: backend.leaseEpoch,
|
|
4078
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
4079
|
+
admission: exactAdmission,
|
|
4080
|
+
operation: op,
|
|
4081
|
+
outcome
|
|
4082
|
+
});
|
|
4083
|
+
};
|
|
4084
|
+
}
|
|
4085
|
+
function beforeRetainedProcessMutation(services, ids) {
|
|
4086
|
+
const fence = ids.workspaceMutationFence;
|
|
4087
|
+
if (!fence) return void 0;
|
|
4088
|
+
return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
|
|
4089
|
+
accountId: fence.accountId,
|
|
4090
|
+
workspaceId: ids.workspaceId,
|
|
4091
|
+
sessionId: ids.sessionId,
|
|
4092
|
+
processId: process2.id,
|
|
4093
|
+
operation: op
|
|
4094
|
+
});
|
|
4095
|
+
}
|
|
4096
|
+
function afterRetainedProcessMutation(services, ids) {
|
|
4097
|
+
const fence = ids.workspaceMutationFence;
|
|
4098
|
+
if (!fence) return void 0;
|
|
4099
|
+
return async ({ op, process: process2, admission, outcome }) => {
|
|
4100
|
+
if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
|
|
4101
|
+
throw new Error("Retained-process mutation settlement lacked its exact admission");
|
|
4102
|
+
}
|
|
4103
|
+
await verifyRetainedProcessMutationSettlement(services.db, {
|
|
4104
|
+
accountId: fence.accountId,
|
|
4105
|
+
workspaceId: ids.workspaceId,
|
|
4106
|
+
sessionId: ids.sessionId,
|
|
4107
|
+
processId: process2.id,
|
|
4108
|
+
admission,
|
|
4109
|
+
operation: op,
|
|
4110
|
+
outcome
|
|
4111
|
+
});
|
|
4112
|
+
};
|
|
4113
|
+
}
|
|
4114
|
+
function settleRetainedProcessForTurn(services, ids) {
|
|
4115
|
+
const fence = ids.workspaceMutationFence;
|
|
4116
|
+
if (!fence) return void 0;
|
|
4117
|
+
return async ({ process: process2, proof }) => {
|
|
4118
|
+
await settleRetainedProcess(services.db, {
|
|
4119
|
+
accountId: fence.accountId,
|
|
4120
|
+
workspaceId: ids.workspaceId,
|
|
4121
|
+
sessionId: ids.sessionId,
|
|
4122
|
+
processId: process2.id,
|
|
4123
|
+
outcome: proof.outcome,
|
|
4124
|
+
exitCode: proof.exitCode,
|
|
4125
|
+
reason: proof.reason,
|
|
4126
|
+
idleGraceMs: services.settings.sandboxIdleGraceMs
|
|
4127
|
+
});
|
|
4128
|
+
};
|
|
4129
|
+
}
|
|
3513
4130
|
function controlRpcFactory(bus) {
|
|
3514
4131
|
return () => new NatsControlRpc(async () => {
|
|
3515
4132
|
if (!bus) {
|
|
@@ -3520,6 +4137,11 @@ function controlRpcFactory(bus) {
|
|
|
3520
4137
|
}
|
|
3521
4138
|
function wrapTurnBoxWithRouting(services, ids, established) {
|
|
3522
4139
|
const { db, settings, bus, onOp } = services;
|
|
4140
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
|
|
4141
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
|
|
4142
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
4143
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
4144
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
3523
4145
|
const resolver = makeActiveBackendResolver({
|
|
3524
4146
|
workspaceId: ids.workspaceId,
|
|
3525
4147
|
defaultBackend: established.session,
|
|
@@ -3565,7 +4187,20 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
3565
4187
|
// group box established this turn), a mid-turn clear-to-null must NOT fall back to
|
|
3566
4188
|
// the pinned machine — passing defaultIsHome:false makes the null branch throw typed
|
|
3567
4189
|
// `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
|
|
3568
|
-
...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {}
|
|
4190
|
+
...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
|
|
4191
|
+
...ids.homeLease ? {
|
|
4192
|
+
resolveDefaultBackend: () => resolveCurrentHomeBackend(
|
|
4193
|
+
services,
|
|
4194
|
+
{
|
|
4195
|
+
workspaceId: ids.workspaceId,
|
|
4196
|
+
sessionId: ids.sessionId,
|
|
4197
|
+
accountId: ids.homeLease.accountId,
|
|
4198
|
+
sandboxGroupId: ids.homeLease.sandboxGroupId,
|
|
4199
|
+
backend: ids.homeLease.backend
|
|
4200
|
+
},
|
|
4201
|
+
established
|
|
4202
|
+
)
|
|
4203
|
+
} : {}
|
|
3569
4204
|
});
|
|
3570
4205
|
const proxy = new RoutingSandboxSession({
|
|
3571
4206
|
// Seed the DEFAULT backend (the established group box) at construction so
|
|
@@ -3577,18 +4212,67 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
3577
4212
|
defaultResolved: {
|
|
3578
4213
|
session: established.session,
|
|
3579
4214
|
sandboxId: null,
|
|
3580
|
-
kind: established.backendId
|
|
4215
|
+
kind: established.backendId,
|
|
4216
|
+
...ids.homeLease ? {
|
|
4217
|
+
leaseEpoch: ids.homeLease.leaseEpoch,
|
|
4218
|
+
providerInstanceId: ids.homeLease.instanceId
|
|
4219
|
+
} : {}
|
|
3581
4220
|
},
|
|
3582
4221
|
readPointer: async () => {
|
|
4222
|
+
if (!routingEnabled(settings)) {
|
|
4223
|
+
return { activeSandboxId: null, activeEpoch: 0 };
|
|
4224
|
+
}
|
|
3583
4225
|
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
3584
4226
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
3585
4227
|
},
|
|
3586
|
-
resolveActiveBackend: resolver
|
|
4228
|
+
resolveActiveBackend: resolver,
|
|
4229
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
4230
|
+
...afterMutation ? { afterMutation } : {},
|
|
4231
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
4232
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
4233
|
+
...settleProcess ? { settleProcess } : {},
|
|
4234
|
+
...ids.homeLease ? {
|
|
4235
|
+
onDefaultBackendError: async ({
|
|
4236
|
+
error,
|
|
4237
|
+
backend
|
|
4238
|
+
}) => {
|
|
4239
|
+
const home = ids.homeLease;
|
|
4240
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
4241
|
+
const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
|
|
4242
|
+
const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
|
|
4243
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
4244
|
+
accountId: home.accountId,
|
|
4245
|
+
workspaceId: ids.workspaceId,
|
|
4246
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4247
|
+
expectedEpoch,
|
|
4248
|
+
expectedInstanceId,
|
|
4249
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
4250
|
+
});
|
|
4251
|
+
if (marked.status === "marked") {
|
|
4252
|
+
await services.onHomeSandboxLost?.({
|
|
4253
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4254
|
+
instanceId: expectedInstanceId,
|
|
4255
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
4256
|
+
});
|
|
4257
|
+
}
|
|
4258
|
+
const lease = marked.lease;
|
|
4259
|
+
const restore = lease?.recovery.restore.status;
|
|
4260
|
+
return {
|
|
4261
|
+
leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
|
|
4262
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
4263
|
+
};
|
|
4264
|
+
}
|
|
4265
|
+
} : {}
|
|
3587
4266
|
});
|
|
3588
4267
|
return { ...established, session: proxy };
|
|
3589
4268
|
}
|
|
3590
4269
|
function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
3591
4270
|
const { db, settings, bus, onOp } = services;
|
|
4271
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
4272
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
4273
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
4274
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
4275
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
3592
4276
|
const syntheticSession = {
|
|
3593
4277
|
state: { manifest: args.agentDefaultManifest }
|
|
3594
4278
|
};
|
|
@@ -3628,6 +4312,19 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
3628
4312
|
resolveActiveBackend: async (pointer) => {
|
|
3629
4313
|
if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
|
|
3630
4314
|
const provisioned = await args.provisioner.get();
|
|
4315
|
+
if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
|
|
4316
|
+
return resolveCurrentHomeBackend(
|
|
4317
|
+
services,
|
|
4318
|
+
{
|
|
4319
|
+
workspaceId: ids.workspaceId,
|
|
4320
|
+
sessionId: ids.sessionId,
|
|
4321
|
+
accountId: args.homeLeaseIdentity.accountId,
|
|
4322
|
+
sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
|
|
4323
|
+
backend: args.homeLeaseIdentity.backend
|
|
4324
|
+
},
|
|
4325
|
+
provisioned.established
|
|
4326
|
+
);
|
|
4327
|
+
}
|
|
3631
4328
|
return {
|
|
3632
4329
|
session: provisioned.established.session,
|
|
3633
4330
|
sandboxId: null,
|
|
@@ -3635,7 +4332,45 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
3635
4332
|
};
|
|
3636
4333
|
}
|
|
3637
4334
|
return routedResolver(pointer);
|
|
3638
|
-
}
|
|
4335
|
+
},
|
|
4336
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
4337
|
+
...afterMutation ? { afterMutation } : {},
|
|
4338
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
4339
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
4340
|
+
...settleProcess ? { settleProcess } : {},
|
|
4341
|
+
...args.homeLeaseIdentity ? {
|
|
4342
|
+
onDefaultBackendError: async ({
|
|
4343
|
+
error,
|
|
4344
|
+
backend
|
|
4345
|
+
}) => {
|
|
4346
|
+
const home = args.homeLeaseIdentity;
|
|
4347
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
4348
|
+
if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
4349
|
+
return null;
|
|
4350
|
+
}
|
|
4351
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
4352
|
+
accountId: home.accountId,
|
|
4353
|
+
workspaceId: ids.workspaceId,
|
|
4354
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4355
|
+
expectedEpoch: backend.leaseEpoch,
|
|
4356
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
4357
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
4358
|
+
});
|
|
4359
|
+
if (marked.status === "marked") {
|
|
4360
|
+
await services.onHomeSandboxLost?.({
|
|
4361
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
4362
|
+
instanceId: backend.providerInstanceId,
|
|
4363
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
4364
|
+
});
|
|
4365
|
+
}
|
|
4366
|
+
const lease = marked.lease;
|
|
4367
|
+
const restore = lease?.recovery.restore.status;
|
|
4368
|
+
return {
|
|
4369
|
+
leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
|
|
4370
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
4371
|
+
};
|
|
4372
|
+
}
|
|
4373
|
+
} : {}
|
|
3639
4374
|
});
|
|
3640
4375
|
return {
|
|
3641
4376
|
client: args.client,
|
|
@@ -3705,7 +4440,7 @@ function makeTurnOpJournal(context, details) {
|
|
|
3705
4440
|
}
|
|
3706
4441
|
|
|
3707
4442
|
// src/observability-metrics.ts
|
|
3708
|
-
import { createHash } from "crypto";
|
|
4443
|
+
import { createHash as createHash2 } from "crypto";
|
|
3709
4444
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3710
4445
|
import {
|
|
3711
4446
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
@@ -4196,7 +4931,7 @@ function stableAccountHash(credentialId) {
|
|
|
4196
4931
|
if (!credentialId) {
|
|
4197
4932
|
return "none";
|
|
4198
4933
|
}
|
|
4199
|
-
return
|
|
4934
|
+
return createHash2("sha256").update(credentialId).digest("hex").slice(0, 12);
|
|
4200
4935
|
}
|
|
4201
4936
|
function modelCallAccountContext(input) {
|
|
4202
4937
|
const accountChangedFromPrevCall = input.isFirstCallOfTurn && input.servingCredentialId !== null && input.priorSessionCredentialId !== null && input.priorSessionCredentialId !== input.servingCredentialId;
|
|
@@ -4370,7 +5105,7 @@ function scrubFreeText(value) {
|
|
|
4370
5105
|
}
|
|
4371
5106
|
|
|
4372
5107
|
// src/activities/workspace-capture.ts
|
|
4373
|
-
import { createHash as
|
|
5108
|
+
import { createHash as createHash3 } from "crypto";
|
|
4374
5109
|
import {
|
|
4375
5110
|
deleteWorkspaceCaptureRows,
|
|
4376
5111
|
insertFailedWorkspaceCapture,
|
|
@@ -4991,7 +5726,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
|
|
|
4991
5726
|
return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
|
|
4992
5727
|
}
|
|
4993
5728
|
function sha256(bytes) {
|
|
4994
|
-
return
|
|
5729
|
+
return createHash3("sha256").update(bytes).digest("hex");
|
|
4995
5730
|
}
|
|
4996
5731
|
function utf8(s) {
|
|
4997
5732
|
return new TextEncoder().encode(s);
|
|
@@ -5071,7 +5806,7 @@ import {
|
|
|
5071
5806
|
evaluateWorkspaceModelPolicy as evaluateWorkspaceModelPolicy2,
|
|
5072
5807
|
readTurnExecutionPolicyV1
|
|
5073
5808
|
} from "@opengeni/contracts";
|
|
5074
|
-
import { createHash as
|
|
5809
|
+
import { createHash as createHash4, randomUUID } from "crypto";
|
|
5075
5810
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
5076
5811
|
function turnExecutionPolicyBillingIdentity(policy) {
|
|
5077
5812
|
return {
|
|
@@ -5102,17 +5837,17 @@ function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds
|
|
|
5102
5837
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
5103
5838
|
}
|
|
5104
5839
|
function codexWorkspaceMetricKey(workspaceId) {
|
|
5105
|
-
return
|
|
5840
|
+
return createHash4("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
5106
5841
|
}
|
|
5107
5842
|
function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
|
|
5108
|
-
const hex =
|
|
5843
|
+
const hex = createHash4("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
|
|
5109
5844
|
hex[12] = "5";
|
|
5110
5845
|
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
5111
5846
|
const value = hex.join("");
|
|
5112
5847
|
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(12, 16)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
|
5113
5848
|
}
|
|
5114
5849
|
function isWorkerShutdownCancellation(error) {
|
|
5115
|
-
return error instanceof
|
|
5850
|
+
return error instanceof CancelledFailure2 && error.message === "WORKER_SHUTDOWN";
|
|
5116
5851
|
}
|
|
5117
5852
|
function shouldRunTurnEndWorkspacePersistence(input) {
|
|
5118
5853
|
return input.activityStatus !== "cancelled" && !input.cancellationRequested;
|
|
@@ -5179,6 +5914,7 @@ async function drainAttemptOwnedSandboxWriters(input) {
|
|
|
5179
5914
|
);
|
|
5180
5915
|
await input.toolCancellationFence.waitForQuiescence();
|
|
5181
5916
|
}
|
|
5917
|
+
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
5182
5918
|
await input.toolspaceTokenRenewal?.stop();
|
|
5183
5919
|
await input.runCredentialRenewal?.stop();
|
|
5184
5920
|
}
|
|
@@ -5670,6 +6406,9 @@ function computerToolModeForTurn(resolvedModel) {
|
|
|
5670
6406
|
}
|
|
5671
6407
|
return "hosted";
|
|
5672
6408
|
}
|
|
6409
|
+
function modelAcceptsTypedAttachmentContentForTurn(resolvedModel) {
|
|
6410
|
+
return resolvedModel === null || resolvedModel.provider.api === "responses";
|
|
6411
|
+
}
|
|
5673
6412
|
var TurnOperationCancelledError = class extends Error {
|
|
5674
6413
|
constructor(reason) {
|
|
5675
6414
|
super("Turn operation was cancelled with its owning turn", {
|
|
@@ -5680,9 +6419,9 @@ var TurnOperationCancelledError = class extends Error {
|
|
|
5680
6419
|
name = "TurnOperationCancelledError";
|
|
5681
6420
|
};
|
|
5682
6421
|
function turnOperationCancellationFailure(error) {
|
|
5683
|
-
if (error instanceof
|
|
6422
|
+
if (error instanceof CancelledFailure2) return error;
|
|
5684
6423
|
if (!(error instanceof TurnOperationCancelledError)) return null;
|
|
5685
|
-
return error.reason instanceof
|
|
6424
|
+
return error.reason instanceof CancelledFailure2 ? error.reason : new CancelledFailure2("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
|
|
5686
6425
|
}
|
|
5687
6426
|
function throwIfTurnOperationCancelled(signal) {
|
|
5688
6427
|
if (signal?.aborted) {
|
|
@@ -5992,12 +6731,89 @@ function createRunAgentTurnActivity(services) {
|
|
|
5992
6731
|
codexLeaseHeartbeatTimer.unref?.();
|
|
5993
6732
|
};
|
|
5994
6733
|
let resolvedSandbox = null;
|
|
6734
|
+
const requireResolvedSandboxForMutation = (message) => {
|
|
6735
|
+
if (!resolvedSandbox) throw new Error(message);
|
|
6736
|
+
return resolvedSandbox;
|
|
6737
|
+
};
|
|
5995
6738
|
let machinePrimarySession = null;
|
|
5996
6739
|
let lazyOwnedSandbox = null;
|
|
5997
6740
|
let turnSandboxProvisioner = null;
|
|
5998
6741
|
let setupBoxSession = null;
|
|
6742
|
+
const onHomeSandboxRebound = (rebound) => {
|
|
6743
|
+
const current = resolvedSandbox;
|
|
6744
|
+
const previousSession = current?.established.session;
|
|
6745
|
+
const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
|
|
6746
|
+
setupBoxSession = rebound.established.session;
|
|
6747
|
+
if (!current) return;
|
|
6748
|
+
current.leaseEpoch = rebound.leaseEpoch;
|
|
6749
|
+
current.established = preserveRoutingProxy ? {
|
|
6750
|
+
...current.established,
|
|
6751
|
+
client: rebound.established.client,
|
|
6752
|
+
// Keep the stable SDK-facing proxy; only its resolver changes the
|
|
6753
|
+
// underlying backend. The worker's setupBoxSession above is raw.
|
|
6754
|
+
session: previousSession,
|
|
6755
|
+
sessionState: rebound.established.sessionState,
|
|
6756
|
+
instanceId: rebound.established.instanceId,
|
|
6757
|
+
backendId: rebound.established.backendId,
|
|
6758
|
+
...rebound.established.origin ? { origin: rebound.established.origin } : {},
|
|
6759
|
+
...rebound.established.restoredArchive ? { restoredArchive: rebound.established.restoredArchive } : {}
|
|
6760
|
+
} : rebound.established;
|
|
6761
|
+
};
|
|
5999
6762
|
let sandboxHolderId = null;
|
|
6000
6763
|
let sandboxGroupId = null;
|
|
6764
|
+
const runWorkspaceMutationForSandbox = async (sandbox, operation, mutation) => {
|
|
6765
|
+
if (sandbox.established.backendId === "selfhosted") return await mutation();
|
|
6766
|
+
if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
|
|
6767
|
+
throw new Error("Workspace mutation attempted before exact turn sandbox admission");
|
|
6768
|
+
}
|
|
6769
|
+
const identity = {
|
|
6770
|
+
accountId: input.accountId,
|
|
6771
|
+
workspaceId: input.workspaceId,
|
|
6772
|
+
sessionId: input.sessionId,
|
|
6773
|
+
turnId,
|
|
6774
|
+
executionGeneration,
|
|
6775
|
+
attemptId: input.attemptId,
|
|
6776
|
+
holderId: sandboxHolderId,
|
|
6777
|
+
sandboxGroupId,
|
|
6778
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
6779
|
+
expectedInstanceId: sandbox.established.instanceId,
|
|
6780
|
+
operation
|
|
6781
|
+
};
|
|
6782
|
+
const admission = await advanceWorkspaceGeneration2(db, identity);
|
|
6783
|
+
let result;
|
|
6784
|
+
try {
|
|
6785
|
+
result = await mutation();
|
|
6786
|
+
} catch (providerError) {
|
|
6787
|
+
try {
|
|
6788
|
+
await verifyWorkspaceMutationSettlement2(db, {
|
|
6789
|
+
...identity,
|
|
6790
|
+
admission,
|
|
6791
|
+
outcome: "rejected"
|
|
6792
|
+
});
|
|
6793
|
+
} catch (settlementError) {
|
|
6794
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
6795
|
+
operation,
|
|
6796
|
+
`Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
|
|
6797
|
+
{ cause: settlementError }
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
throw providerError;
|
|
6801
|
+
}
|
|
6802
|
+
try {
|
|
6803
|
+
await verifyWorkspaceMutationSettlement2(db, {
|
|
6804
|
+
...identity,
|
|
6805
|
+
admission,
|
|
6806
|
+
outcome: "resolved"
|
|
6807
|
+
});
|
|
6808
|
+
} catch (settlementError) {
|
|
6809
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
6810
|
+
operation,
|
|
6811
|
+
`Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
|
|
6812
|
+
{ cause: settlementError }
|
|
6813
|
+
);
|
|
6814
|
+
}
|
|
6815
|
+
return result;
|
|
6816
|
+
};
|
|
6001
6817
|
let leaseHeartbeatTimer;
|
|
6002
6818
|
let gitCredentialRenewals = [];
|
|
6003
6819
|
let gitCredentialRenewalClosed = false;
|
|
@@ -6715,6 +7531,57 @@ function createRunAgentTurnActivity(services) {
|
|
|
6715
7531
|
codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
|
|
6716
7532
|
codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
|
|
6717
7533
|
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
7534
|
+
const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
|
|
7535
|
+
const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
|
|
7536
|
+
const fencedInFlight = leased.reused || continuationCodexCredentialId !== null && continuationCodexCredentialId === effectiveCodexCredentialId;
|
|
7537
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
7538
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
7539
|
+
decision: {
|
|
7540
|
+
accounts: leased.accounts,
|
|
7541
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
7542
|
+
actualOutcome,
|
|
7543
|
+
actualReason,
|
|
7544
|
+
affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
|
|
7545
|
+
fencedInFlight,
|
|
7546
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
7547
|
+
now: /* @__PURE__ */ new Date(),
|
|
7548
|
+
aliasSeed: randomUUID()
|
|
7549
|
+
},
|
|
7550
|
+
publish
|
|
7551
|
+
});
|
|
7552
|
+
if (shadowResult.outcome === "published") {
|
|
7553
|
+
const shadowPayload = shadowResult.payload;
|
|
7554
|
+
observability.incrementCounter({
|
|
7555
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
7556
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
7557
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
|
|
7558
|
+
});
|
|
7559
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
7560
|
+
workspaceId: input.workspaceId,
|
|
7561
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
7562
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
7563
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
7564
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
7565
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
7566
|
+
comparison: shadowPayload.comparison,
|
|
7567
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
7568
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
7569
|
+
payloadBytes: shadowResult.payloadBytes
|
|
7570
|
+
});
|
|
7571
|
+
} else if (shadowResult.outcome === "failed") {
|
|
7572
|
+
observability.incrementCounter({
|
|
7573
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
7574
|
+
help: "Shadow decision build/publication failures.",
|
|
7575
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
|
|
7576
|
+
});
|
|
7577
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
7578
|
+
workspaceId: input.workspaceId,
|
|
7579
|
+
stage: shadowResult.stage,
|
|
7580
|
+
reason: shadowResult.reason,
|
|
7581
|
+
errorName: shadowResult.errorName,
|
|
7582
|
+
payloadBytes: shadowResult.payloadBytes
|
|
7583
|
+
});
|
|
7584
|
+
}
|
|
6718
7585
|
const eligibleCount = leased.accounts.filter(
|
|
6719
7586
|
(account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
|
|
6720
7587
|
).length;
|
|
@@ -7437,7 +8304,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7437
8304
|
gitTokens: sandboxGitTokens ?? {},
|
|
7438
8305
|
expiresAt: sandboxGitTokenExpiresAt ?? {}
|
|
7439
8306
|
} : void 0;
|
|
7440
|
-
const attachGitCredentialRenewal = async (tokenSession, initial) => {
|
|
8307
|
+
const attachGitCredentialRenewal = async (tokenSession, initial, initialSandbox) => {
|
|
7441
8308
|
if (!initial || initial.bindings.length === 0) return;
|
|
7442
8309
|
const previous = gitCredentialRenewals;
|
|
7443
8310
|
gitCredentialRenewals = [];
|
|
@@ -7476,14 +8343,22 @@ function createRunAgentTurnActivity(services) {
|
|
|
7476
8343
|
throw new Error("credential renewal produced no binding token");
|
|
7477
8344
|
}
|
|
7478
8345
|
const runAs = sandboxRunAs(runSettings);
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
|
|
7486
|
-
|
|
8346
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
8347
|
+
if (!targetSandbox) {
|
|
8348
|
+
throw new Error("Git credential renewal has no exact sandbox lease target");
|
|
8349
|
+
}
|
|
8350
|
+
await runWorkspaceMutationForSandbox(
|
|
8351
|
+
targetSandbox,
|
|
8352
|
+
"gitCredentialRenewal",
|
|
8353
|
+
async () => await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding], {
|
|
8354
|
+
...runAs ? { runAs } : {},
|
|
8355
|
+
...toolCancellationFenceRef.current ? {
|
|
8356
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8357
|
+
toolCancellationFenceRef.current
|
|
8358
|
+
)
|
|
8359
|
+
} : {}
|
|
8360
|
+
})
|
|
8361
|
+
);
|
|
7487
8362
|
},
|
|
7488
8363
|
onSuccess: ({ providers: renewedProviders }) => {
|
|
7489
8364
|
for (const provider of renewedProviders) {
|
|
@@ -7518,7 +8393,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7518
8393
|
}
|
|
7519
8394
|
gitCredentialRenewals = controllers;
|
|
7520
8395
|
};
|
|
7521
|
-
const attachToolspaceTokenRenewal = async (tokenSession, initialExpiresAt = sandboxToolspaceTokenExpiresAt) => {
|
|
8396
|
+
const attachToolspaceTokenRenewal = async (tokenSession, initialExpiresAt = sandboxToolspaceTokenExpiresAt, initialSandbox) => {
|
|
7522
8397
|
if (!sandboxToolspaceToken || !initialExpiresAt) return;
|
|
7523
8398
|
const previous = toolspaceTokenRenewal;
|
|
7524
8399
|
toolspaceTokenRenewal = null;
|
|
@@ -7527,18 +8402,26 @@ function createRunAgentTurnActivity(services) {
|
|
|
7527
8402
|
const mint = async () => await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
7528
8403
|
const write = async (material) => {
|
|
7529
8404
|
const runAs = sandboxRunAs(runSettings);
|
|
7530
|
-
|
|
7531
|
-
|
|
7532
|
-
|
|
7533
|
-
|
|
7534
|
-
|
|
7535
|
-
|
|
7536
|
-
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
8405
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
8406
|
+
if (!targetSandbox) {
|
|
8407
|
+
throw new Error("Toolspace token renewal has no exact sandbox lease target");
|
|
8408
|
+
}
|
|
8409
|
+
await runWorkspaceMutationForSandbox(
|
|
8410
|
+
targetSandbox,
|
|
8411
|
+
"toolspaceTokenRenewal",
|
|
8412
|
+
async () => await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
8413
|
+
...runAs ? { runAs } : {},
|
|
8414
|
+
...sandboxToolspaceTokenFile ? {
|
|
8415
|
+
tokenFile: sandboxToolspaceTokenFile,
|
|
8416
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE
|
|
8417
|
+
} : {},
|
|
8418
|
+
...toolCancellationFenceRef.current ? {
|
|
8419
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8420
|
+
toolCancellationFenceRef.current
|
|
8421
|
+
)
|
|
8422
|
+
} : {}
|
|
8423
|
+
})
|
|
8424
|
+
);
|
|
7542
8425
|
};
|
|
7543
8426
|
let renewalExpiresAt = initialExpiresAt;
|
|
7544
8427
|
if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
|
|
@@ -7580,46 +8463,65 @@ function createRunAgentTurnActivity(services) {
|
|
|
7580
8463
|
}
|
|
7581
8464
|
toolspaceTokenRenewal = controller;
|
|
7582
8465
|
};
|
|
7583
|
-
const attachRunCredentialRenewal = async (credentialSession) => {
|
|
8466
|
+
const attachRunCredentialRenewal = async (credentialSession, initialSandbox) => {
|
|
7584
8467
|
if (!runCredentialResolver) return;
|
|
7585
8468
|
const previous = runCredentialRenewal;
|
|
7586
8469
|
runCredentialRenewal = null;
|
|
7587
8470
|
await previous?.stop();
|
|
7588
8471
|
if (runCredentialRenewalClosed) return;
|
|
7589
8472
|
runCredentialSession = credentialSession;
|
|
8473
|
+
const requireTargetSandbox = () => {
|
|
8474
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
8475
|
+
if (!targetSandbox) {
|
|
8476
|
+
throw new Error("Run credential mutation has no exact sandbox lease target");
|
|
8477
|
+
}
|
|
8478
|
+
return targetSandbox;
|
|
8479
|
+
};
|
|
7590
8480
|
if (!initialRunCredentialMaterial) {
|
|
7591
|
-
await
|
|
7592
|
-
|
|
7593
|
-
|
|
7594
|
-
|
|
7595
|
-
|
|
7596
|
-
|
|
8481
|
+
await runWorkspaceMutationForSandbox(
|
|
8482
|
+
requireTargetSandbox(),
|
|
8483
|
+
"runCredentialClear",
|
|
8484
|
+
async () => await clearRunCredentials(
|
|
8485
|
+
credentialSession,
|
|
8486
|
+
input.sessionId,
|
|
8487
|
+
toolCancellationFenceRef.current ? toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8488
|
+
toolCancellationFenceRef.current
|
|
8489
|
+
) : void 0
|
|
8490
|
+
)
|
|
7597
8491
|
);
|
|
7598
8492
|
return;
|
|
7599
8493
|
}
|
|
7600
8494
|
const write = async (material, pruneOtherAttempts = false) => {
|
|
7601
8495
|
if (!material) {
|
|
7602
|
-
await
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
8496
|
+
await runWorkspaceMutationForSandbox(
|
|
8497
|
+
requireTargetSandbox(),
|
|
8498
|
+
"runCredentialAttemptClear",
|
|
8499
|
+
async () => await clearRunCredentialsForAttempt(credentialSession, {
|
|
8500
|
+
sessionId: input.sessionId,
|
|
8501
|
+
attemptId: input.attemptId,
|
|
8502
|
+
executionGeneration
|
|
8503
|
+
})
|
|
8504
|
+
);
|
|
7607
8505
|
return;
|
|
7608
8506
|
}
|
|
7609
8507
|
registerSecretRedactions(material.redactions);
|
|
7610
|
-
await
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7620
|
-
|
|
7621
|
-
|
|
7622
|
-
|
|
8508
|
+
await runWorkspaceMutationForSandbox(
|
|
8509
|
+
requireTargetSandbox(),
|
|
8510
|
+
"runCredentialMaterialization",
|
|
8511
|
+
async () => await materializeRunCredentials(credentialSession, material, {
|
|
8512
|
+
sessionId: input.sessionId,
|
|
8513
|
+
attemptId: input.attemptId,
|
|
8514
|
+
executionGeneration,
|
|
8515
|
+
...pruneOtherAttempts ? { pruneOtherAttempts: true } : {},
|
|
8516
|
+
...!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {},
|
|
8517
|
+
...material.authNeeded.length > 0 && Object.keys(material.environment).length === 0 && material.files.length === 0 ? { prunePreviousGenerations: true } : {},
|
|
8518
|
+
...toolCancellationFenceRef.current ? {
|
|
8519
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8520
|
+
toolCancellationFenceRef.current
|
|
8521
|
+
)
|
|
8522
|
+
} : {}
|
|
8523
|
+
})
|
|
8524
|
+
);
|
|
7623
8525
|
for (const payload of runCredentialAuthNeededPayloads(material)) {
|
|
7624
8526
|
const key = JSON.stringify(payload);
|
|
7625
8527
|
if (publishedRunCredentialNotices.has(key)) continue;
|
|
@@ -7718,12 +8620,19 @@ function createRunAgentTurnActivity(services) {
|
|
|
7718
8620
|
db,
|
|
7719
8621
|
settings,
|
|
7720
8622
|
bus,
|
|
7721
|
-
onOp: machineOpObserver.observer
|
|
8623
|
+
onOp: machineOpObserver.observer,
|
|
8624
|
+
onHomeSandboxRebound
|
|
7722
8625
|
},
|
|
7723
8626
|
{
|
|
7724
8627
|
workspaceId: input.workspaceId,
|
|
7725
8628
|
sessionId: input.sessionId,
|
|
7726
8629
|
environment: sandboxEnvironment,
|
|
8630
|
+
workspaceMutationFence: {
|
|
8631
|
+
accountId: input.accountId,
|
|
8632
|
+
turnId: turn.id,
|
|
8633
|
+
executionGeneration,
|
|
8634
|
+
attemptId: input.attemptId
|
|
8635
|
+
},
|
|
7727
8636
|
pinnedSelfhosted: {
|
|
7728
8637
|
sandboxId: activeSandboxPointer.activeSandboxId,
|
|
7729
8638
|
epoch: activeSandboxPointer.activeEpoch
|
|
@@ -7789,24 +8698,41 @@ function createRunAgentTurnActivity(services) {
|
|
|
7789
8698
|
);
|
|
7790
8699
|
setupBoxSession = resolvedSandbox.established.session;
|
|
7791
8700
|
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
7792
|
-
|
|
7793
|
-
resolvedSandbox
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
|
|
7804
|
-
|
|
8701
|
+
resolvedSandbox = {
|
|
8702
|
+
...resolvedSandbox,
|
|
8703
|
+
established: wrapTurnBoxWithRouting(
|
|
8704
|
+
{
|
|
8705
|
+
db,
|
|
8706
|
+
settings,
|
|
8707
|
+
bus,
|
|
8708
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
8709
|
+
onHomeSandboxRebound
|
|
8710
|
+
},
|
|
8711
|
+
// Thread the SAME declared environment the group box was created with
|
|
8712
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
8713
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
8714
|
+
// (no "cannot change manifest environment variables" throw).
|
|
8715
|
+
{
|
|
8716
|
+
workspaceId: input.workspaceId,
|
|
8717
|
+
sessionId: input.sessionId,
|
|
8718
|
+
environment: sandboxEnvironment,
|
|
8719
|
+
workspaceMutationFence: {
|
|
8720
|
+
accountId: input.accountId,
|
|
8721
|
+
turnId: turn.id,
|
|
8722
|
+
executionGeneration,
|
|
8723
|
+
attemptId: input.attemptId
|
|
7805
8724
|
},
|
|
7806
|
-
|
|
7807
|
-
|
|
7808
|
-
|
|
7809
|
-
|
|
8725
|
+
homeLease: {
|
|
8726
|
+
accountId: input.accountId,
|
|
8727
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8728
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
8729
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
8730
|
+
backend: groupBoxBackend
|
|
8731
|
+
}
|
|
8732
|
+
},
|
|
8733
|
+
resolvedSandbox.established
|
|
8734
|
+
)
|
|
8735
|
+
};
|
|
7810
8736
|
}
|
|
7811
8737
|
if (resolvedSandbox) {
|
|
7812
8738
|
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
@@ -8047,39 +8973,46 @@ function createRunAgentTurnActivity(services) {
|
|
|
8047
8973
|
);
|
|
8048
8974
|
await publishSandboxLifecycleEvents(provisioned);
|
|
8049
8975
|
await attachRunCredentialRenewal(
|
|
8050
|
-
provisioned.established.session
|
|
8976
|
+
provisioned.established.session,
|
|
8977
|
+
provisioned
|
|
8051
8978
|
);
|
|
8052
8979
|
const provisionedSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(
|
|
8053
8980
|
provisioned.established.session,
|
|
8054
8981
|
input.sessionId
|
|
8055
8982
|
) : provisioned.established.session;
|
|
8056
|
-
await
|
|
8057
|
-
|
|
8058
|
-
|
|
8059
|
-
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
8068
|
-
|
|
8069
|
-
|
|
8070
|
-
|
|
8071
|
-
|
|
8072
|
-
|
|
8073
|
-
|
|
8074
|
-
|
|
8983
|
+
await runWorkspaceMutationForSandbox(
|
|
8984
|
+
provisioned,
|
|
8985
|
+
"lazyOwnedSandboxSetup",
|
|
8986
|
+
async () => await runOwnedSandboxSetup(
|
|
8987
|
+
agent,
|
|
8988
|
+
provisioned.established.session,
|
|
8989
|
+
provisionedSetupSession,
|
|
8990
|
+
{
|
|
8991
|
+
settings: runSettings,
|
|
8992
|
+
environment: sandboxEnvironment,
|
|
8993
|
+
onRuntimeEvent: async (event) => {
|
|
8994
|
+
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
8995
|
+
},
|
|
8996
|
+
...lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {},
|
|
8997
|
+
...lazyGitCredentials?.bindings ? { gitCredentialBindingsOverride: lazyGitCredentials.bindings } : {},
|
|
8998
|
+
...lazyToolspaceToken ? { toolspaceTokenSeedOverride: lazyToolspaceToken.token } : {},
|
|
8999
|
+
...toolCancellationFenceRef.current ? {
|
|
9000
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
9001
|
+
toolCancellationFenceRef.current
|
|
9002
|
+
)
|
|
9003
|
+
} : {}
|
|
9004
|
+
}
|
|
9005
|
+
)
|
|
8075
9006
|
);
|
|
8076
9007
|
await attachToolspaceTokenRenewal(
|
|
8077
9008
|
provisioned.established.session,
|
|
8078
|
-
lazyToolspaceToken?.expiresAt
|
|
9009
|
+
lazyToolspaceToken?.expiresAt,
|
|
9010
|
+
provisioned
|
|
8079
9011
|
);
|
|
8080
9012
|
await attachGitCredentialRenewal(
|
|
8081
9013
|
provisioned.established.session,
|
|
8082
|
-
lazyGitCredentials
|
|
9014
|
+
lazyGitCredentials,
|
|
9015
|
+
provisioned
|
|
8083
9016
|
);
|
|
8084
9017
|
return provisioned;
|
|
8085
9018
|
},
|
|
@@ -8138,18 +9071,31 @@ function createRunAgentTurnActivity(services) {
|
|
|
8138
9071
|
db,
|
|
8139
9072
|
settings,
|
|
8140
9073
|
bus,
|
|
8141
|
-
onOp: machineOpObserver.observer
|
|
9074
|
+
onOp: machineOpObserver.observer,
|
|
9075
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
9076
|
+
onHomeSandboxRebound
|
|
8142
9077
|
},
|
|
8143
9078
|
{
|
|
8144
9079
|
workspaceId: input.workspaceId,
|
|
8145
9080
|
sessionId: input.sessionId,
|
|
8146
|
-
environment: sandboxEnvironment
|
|
9081
|
+
environment: sandboxEnvironment,
|
|
9082
|
+
workspaceMutationFence: {
|
|
9083
|
+
accountId: input.accountId,
|
|
9084
|
+
turnId: turn.id,
|
|
9085
|
+
executionGeneration,
|
|
9086
|
+
attemptId: input.attemptId
|
|
9087
|
+
}
|
|
8147
9088
|
},
|
|
8148
9089
|
{
|
|
8149
9090
|
client: lazyClient,
|
|
8150
9091
|
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
8151
9092
|
agentDefaultManifest,
|
|
8152
|
-
provisioner: turnSandboxProvisioner
|
|
9093
|
+
provisioner: turnSandboxProvisioner,
|
|
9094
|
+
homeLeaseIdentity: {
|
|
9095
|
+
accountId: input.accountId,
|
|
9096
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
9097
|
+
backend: groupBoxBackend
|
|
9098
|
+
}
|
|
8153
9099
|
}
|
|
8154
9100
|
);
|
|
8155
9101
|
}
|
|
@@ -8251,20 +9197,24 @@ function createRunAgentTurnActivity(services) {
|
|
|
8251
9197
|
);
|
|
8252
9198
|
const runAs = sandboxRunAs(runSettings);
|
|
8253
9199
|
if (downloadsToMaterialize.length > 0) {
|
|
8254
|
-
const materialized = await
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
9200
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
9201
|
+
resolvedSandbox,
|
|
9202
|
+
"fileMaterialization",
|
|
9203
|
+
async () => await materializeSandboxFileDownloads(
|
|
9204
|
+
setupBoxSession,
|
|
9205
|
+
downloadsToMaterialize,
|
|
9206
|
+
{
|
|
9207
|
+
onRuntimeEvent: async (event) => {
|
|
9208
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
9209
|
+
},
|
|
9210
|
+
...runAs ? { runAs } : {},
|
|
9211
|
+
...toolCancellationFenceRef.current ? {
|
|
9212
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
9213
|
+
toolCancellationFenceRef.current
|
|
9214
|
+
)
|
|
9215
|
+
} : {}
|
|
9216
|
+
}
|
|
9217
|
+
)
|
|
8268
9218
|
);
|
|
8269
9219
|
fileMaterializationFailures = materialized.failures;
|
|
8270
9220
|
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
@@ -8301,7 +9251,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
8301
9251
|
turnId: activeTurnId,
|
|
8302
9252
|
recovering: turn.executionGeneration > 1,
|
|
8303
9253
|
...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
|
|
8304
|
-
...runCredentialsNote ? { runCredentialsNote } : {}
|
|
9254
|
+
...runCredentialsNote ? { runCredentialsNote } : {},
|
|
9255
|
+
...modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage ? {
|
|
9256
|
+
readFileBytesForModel: (file) => objectStorage.getFileBytes(file)
|
|
9257
|
+
} : {}
|
|
8305
9258
|
}
|
|
8306
9259
|
);
|
|
8307
9260
|
runInput = prepared.input;
|
|
@@ -8375,57 +9328,90 @@ function createRunAgentTurnActivity(services) {
|
|
|
8375
9328
|
throwIfWorkerShuttingDown();
|
|
8376
9329
|
throwIfTurnCancelled();
|
|
8377
9330
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
8378
|
-
const runStreamOnce = () =>
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
8391
|
-
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
|
|
8398
|
-
|
|
8399
|
-
|
|
8400
|
-
|
|
8401
|
-
|
|
8402
|
-
|
|
8403
|
-
|
|
8404
|
-
|
|
8405
|
-
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
|
|
8410
|
-
|
|
8411
|
-
|
|
8412
|
-
|
|
8413
|
-
|
|
8414
|
-
}
|
|
8415
|
-
|
|
8416
|
-
|
|
8417
|
-
|
|
8418
|
-
|
|
8419
|
-
|
|
8420
|
-
const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
|
|
8421
|
-
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
9331
|
+
const runStreamOnce = async () => {
|
|
9332
|
+
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
9333
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
9334
|
+
await runWorkspaceMutationForSandbox(
|
|
9335
|
+
resolvedSandbox,
|
|
9336
|
+
"eagerOwnedSandboxSetup",
|
|
9337
|
+
async () => await runOwnedSandboxSetup(
|
|
9338
|
+
agent,
|
|
9339
|
+
ownedEstablished.session,
|
|
9340
|
+
eagerSetupSession,
|
|
9341
|
+
{
|
|
9342
|
+
settings: runSettings,
|
|
9343
|
+
environment: sandboxEnvironment,
|
|
9344
|
+
preparedInput: runInput,
|
|
9345
|
+
...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
|
|
9346
|
+
onRuntimeEvent: async (event) => {
|
|
9347
|
+
await renewCodexLease("runtime_event");
|
|
9348
|
+
if (codexLeaseLost) {
|
|
9349
|
+
throw new Error("Codex credential lease expired during sandbox setup");
|
|
9350
|
+
}
|
|
9351
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
9352
|
+
},
|
|
9353
|
+
...toolCancellationFenceRef.current ? {
|
|
9354
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
9355
|
+
toolCancellationFenceRef.current
|
|
9356
|
+
)
|
|
9357
|
+
} : {}
|
|
9358
|
+
}
|
|
9359
|
+
)
|
|
9360
|
+
);
|
|
9361
|
+
await attachGitCredentialRenewal(
|
|
9362
|
+
eagerSetupSession,
|
|
9363
|
+
initialGitCredentials
|
|
9364
|
+
);
|
|
9365
|
+
}
|
|
9366
|
+
return await runtime.runStream(agent, runInput, modelRunSettings, {
|
|
9367
|
+
...activityContext ? { signal: activityContext.cancellationSignal } : {},
|
|
9368
|
+
sandboxEnvironment,
|
|
9369
|
+
onRuntimeEvent: async (event) => {
|
|
9370
|
+
await renewCodexLease("runtime_event");
|
|
9371
|
+
if (codexLeaseLost) {
|
|
9372
|
+
throw new Error("Codex credential lease expired during the active turn");
|
|
8422
9373
|
}
|
|
8423
|
-
|
|
8424
|
-
|
|
8425
|
-
|
|
8426
|
-
|
|
8427
|
-
|
|
8428
|
-
|
|
9374
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
9375
|
+
},
|
|
9376
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
9377
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
9378
|
+
...ownedEstablished ? {
|
|
9379
|
+
ownedSandbox: {
|
|
9380
|
+
client: ownedEstablished.client,
|
|
9381
|
+
session: ownedEstablished.session,
|
|
9382
|
+
...resolvedSandbox?.established.sessionState ? {
|
|
9383
|
+
sessionState: resolvedSandbox.established.sessionState
|
|
9384
|
+
} : {},
|
|
9385
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
9386
|
+
// established box — never through the routing proxy, which would
|
|
9387
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
9388
|
+
...setupBoxSession ? { setupSession: setupBoxSession } : {},
|
|
9389
|
+
...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
|
|
9390
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
9391
|
+
// above under its exact admission, lazy in the provisioner.
|
|
9392
|
+
deferredSetup: true
|
|
9393
|
+
}
|
|
9394
|
+
} : {},
|
|
9395
|
+
...sandboxToolspaceToken && sandboxToolspaceTokenExpiresAt && !lazyOwnedSandbox ? {
|
|
9396
|
+
onToolspaceTokenSessionReady: async (tokenSession) => {
|
|
9397
|
+
const renewalSession = activeSandboxBackend === "selfhosted" ? tokenSession : setupBoxSession ?? tokenSession;
|
|
9398
|
+
await attachToolspaceTokenRenewal(renewalSession);
|
|
9399
|
+
}
|
|
9400
|
+
} : {},
|
|
9401
|
+
...runCredentialResolver ? {
|
|
9402
|
+
runCredentialSessionId: input.sessionId,
|
|
9403
|
+
...!lazyOwnedSandbox ? {
|
|
9404
|
+
onRunCredentialSessionReady: async (credentialSession) => {
|
|
9405
|
+
const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
|
|
9406
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
9407
|
+
}
|
|
9408
|
+
} : {}
|
|
9409
|
+
} : {},
|
|
9410
|
+
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
9411
|
+
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
9412
|
+
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
9413
|
+
});
|
|
9414
|
+
};
|
|
8429
9415
|
if (codexLeaseLost) {
|
|
8430
9416
|
throw new Error("Codex credential lease expired before the model run");
|
|
8431
9417
|
}
|
|
@@ -8514,6 +9500,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8514
9500
|
turnId: activeTurnId,
|
|
8515
9501
|
executionGeneration,
|
|
8516
9502
|
attemptId: input.attemptId,
|
|
9503
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
8517
9504
|
...pendingToolCall
|
|
8518
9505
|
});
|
|
8519
9506
|
if (!registered.accepted) {
|
|
@@ -8533,6 +9520,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8533
9520
|
executionGeneration,
|
|
8534
9521
|
attemptId: input.attemptId,
|
|
8535
9522
|
callId: completedToolCall.callId,
|
|
9523
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
8536
9524
|
resultItem: completedToolCall.resultItem
|
|
8537
9525
|
});
|
|
8538
9526
|
if (!recorded.accepted) {
|
|
@@ -8749,7 +9737,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8749
9737
|
return claimedResult({ status: "cancelled" });
|
|
8750
9738
|
}
|
|
8751
9739
|
turnMetricOutcome = "completed";
|
|
8752
|
-
await
|
|
9740
|
+
await recordUsageEvent(db, {
|
|
8753
9741
|
accountId: input.accountId,
|
|
8754
9742
|
workspaceId: input.workspaceId,
|
|
8755
9743
|
eventType: "agent_run.completed",
|
|
@@ -8936,7 +9924,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8936
9924
|
turnFinalizerCancellationSignal(cancellationSignal, activityStatus)
|
|
8937
9925
|
);
|
|
8938
9926
|
turnMetricOutcome = "cancelled";
|
|
8939
|
-
throw new
|
|
9927
|
+
throw new CancelledFailure2("TURN_ATTEMPT_FENCED", [], error);
|
|
8940
9928
|
}
|
|
8941
9929
|
if (cancellationFailure) {
|
|
8942
9930
|
activityStatus = "cancelled";
|
|
@@ -8969,7 +9957,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8969
9957
|
return claimedResult({ status: "cancelled" });
|
|
8970
9958
|
}
|
|
8971
9959
|
turnMetricOutcome = "completed";
|
|
8972
|
-
await
|
|
9960
|
+
await recordUsageEvent(db, {
|
|
8973
9961
|
accountId: input.accountId,
|
|
8974
9962
|
workspaceId: input.workspaceId,
|
|
8975
9963
|
eventType: "agent_run.completed",
|
|
@@ -9456,7 +10444,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9456
10444
|
return claimedResult({ status: "cancelled" });
|
|
9457
10445
|
}
|
|
9458
10446
|
turnMetricOutcome = "completed";
|
|
9459
|
-
await
|
|
10447
|
+
await recordUsageEvent(db, {
|
|
9460
10448
|
accountId: input.accountId,
|
|
9461
10449
|
workspaceId: input.workspaceId,
|
|
9462
10450
|
eventType: "agent_run.completed",
|
|
@@ -9557,15 +10545,34 @@ function createRunAgentTurnActivity(services) {
|
|
|
9557
10545
|
const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
|
|
9558
10546
|
try {
|
|
9559
10547
|
const toolCancellationFence = toolCancellationFenceRef.current;
|
|
10548
|
+
gitCredentialRenewalClosed = true;
|
|
10549
|
+
const gitRenewalsToStop = gitCredentialRenewals;
|
|
10550
|
+
gitCredentialRenewals = [];
|
|
9560
10551
|
toolspaceTokenRenewalClosed = true;
|
|
9561
10552
|
const toolspaceRenewalToStop = toolspaceTokenRenewal;
|
|
9562
10553
|
toolspaceTokenRenewal = null;
|
|
9563
10554
|
runCredentialRenewalClosed = true;
|
|
9564
10555
|
const runRenewalToStop = runCredentialRenewal;
|
|
9565
10556
|
runCredentialRenewal = null;
|
|
10557
|
+
const credentialSessionToClear = runCredentialSession;
|
|
10558
|
+
runCredentialSession = null;
|
|
10559
|
+
if (credentialSessionToClear) {
|
|
10560
|
+
await runWorkspaceMutationForSandbox(
|
|
10561
|
+
requireResolvedSandboxForMutation(
|
|
10562
|
+
"Run credential cleanup has no exact sandbox lease target"
|
|
10563
|
+
),
|
|
10564
|
+
"runCredentialAttemptClear",
|
|
10565
|
+
async () => await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
10566
|
+
sessionId: input.sessionId,
|
|
10567
|
+
attemptId: input.attemptId,
|
|
10568
|
+
executionGeneration
|
|
10569
|
+
})
|
|
10570
|
+
);
|
|
10571
|
+
}
|
|
9566
10572
|
await drainAttemptOwnedSandboxWriters({
|
|
9567
10573
|
toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
|
|
9568
10574
|
cancellationReason: cancellationSignal?.reason,
|
|
10575
|
+
gitCredentialRenewals: gitRenewalsToStop,
|
|
9569
10576
|
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
9570
10577
|
runCredentialRenewal: runRenewalToStop
|
|
9571
10578
|
});
|
|
@@ -9635,37 +10642,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
9635
10642
|
});
|
|
9636
10643
|
}
|
|
9637
10644
|
}
|
|
9638
|
-
const credentialSessionToClear = runCredentialSession;
|
|
9639
|
-
runCredentialSession = null;
|
|
9640
|
-
if (credentialSessionToClear) {
|
|
9641
|
-
const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
9642
|
-
sessionId: input.sessionId,
|
|
9643
|
-
attemptId: input.attemptId,
|
|
9644
|
-
executionGeneration
|
|
9645
|
-
}).catch((error) => {
|
|
9646
|
-
try {
|
|
9647
|
-
observability.incrementCounter({
|
|
9648
|
-
name: "opengeni_run_credential_cleanup_total",
|
|
9649
|
-
help: "Attempt-owned run credential cleanup outcomes.",
|
|
9650
|
-
labels: { outcome: "error" }
|
|
9651
|
-
});
|
|
9652
|
-
observability.warn("Attempt-owned run credential cleanup failed", {
|
|
9653
|
-
sessionId: input.sessionId,
|
|
9654
|
-
turnId,
|
|
9655
|
-
attemptId: input.attemptId,
|
|
9656
|
-
errorClass: error instanceof Error ? error.name : "UnknownError"
|
|
9657
|
-
});
|
|
9658
|
-
} catch {
|
|
9659
|
-
}
|
|
9660
|
-
});
|
|
9661
|
-
await waitForTurnFinalizerStep(cleanup, finalizerSignal);
|
|
9662
|
-
}
|
|
9663
|
-
gitCredentialRenewalClosed = true;
|
|
9664
|
-
const renewalsToStop = gitCredentialRenewals;
|
|
9665
|
-
gitCredentialRenewals = [];
|
|
9666
|
-
for (const renewal of renewalsToStop) {
|
|
9667
|
-
await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
|
|
9668
|
-
}
|
|
9669
10645
|
const machineOpEvents = machineOpObserver.drainEvents();
|
|
9670
10646
|
if (machineOpEvents.length > 0 && turnId && executionGeneration > 0) {
|
|
9671
10647
|
await waitForTurnFinalizerStep(
|
|
@@ -10192,7 +11168,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
10192
11168
|
const outputTokens = sanitizedUsage.outputTokens ?? 0;
|
|
10193
11169
|
const totalTokens = sanitizedUsage.totalTokens ?? 0;
|
|
10194
11170
|
if (input.externallyBilled) {
|
|
10195
|
-
await
|
|
11171
|
+
await recordUsageEvent(db, {
|
|
10196
11172
|
accountId: input.accountId,
|
|
10197
11173
|
workspaceId: input.workspaceId,
|
|
10198
11174
|
eventType: "model.cost",
|
|
@@ -10208,7 +11184,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
10208
11184
|
return;
|
|
10209
11185
|
}
|
|
10210
11186
|
if (totalTokens > 0) {
|
|
10211
|
-
await
|
|
11187
|
+
await recordUsageEvent(db, {
|
|
10212
11188
|
accountId: input.accountId,
|
|
10213
11189
|
workspaceId: input.workspaceId,
|
|
10214
11190
|
eventType: "model.tokens",
|
|
@@ -10230,7 +11206,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
10230
11206
|
throw new Error(`Missing model pricing for ${input.model}`);
|
|
10231
11207
|
}
|
|
10232
11208
|
const costMicros = calculateModelUsageCostMicros(settings, input.model, sanitizedUsage);
|
|
10233
|
-
await
|
|
11209
|
+
await recordUsageEvent(db, {
|
|
10234
11210
|
accountId: input.accountId,
|
|
10235
11211
|
workspaceId: input.workspaceId,
|
|
10236
11212
|
eventType: "model.cost",
|
|
@@ -10334,7 +11310,7 @@ import { indexDocumentNow } from "@opengeni/documents";
|
|
|
10334
11310
|
import { configuredStaticUsageLimits as configuredStaticUsageLimits3 } from "@opengeni/config";
|
|
10335
11311
|
import {
|
|
10336
11312
|
getBillingBalance as getBillingBalance3,
|
|
10337
|
-
recordUsageEvent as
|
|
11313
|
+
recordUsageEvent as recordUsageEvent2,
|
|
10338
11314
|
sumUsageQuantity as sumUsageQuantity3,
|
|
10339
11315
|
withWorkspaceUsageLock
|
|
10340
11316
|
} from "@opengeni/db";
|
|
@@ -10379,7 +11355,7 @@ function createDocumentActivities(services) {
|
|
|
10379
11355
|
}
|
|
10380
11356
|
);
|
|
10381
11357
|
if (document.status === "ready") {
|
|
10382
|
-
await
|
|
11358
|
+
await recordUsageEvent2(lockedDb, {
|
|
10383
11359
|
accountId: input.accountId,
|
|
10384
11360
|
workspaceId: input.workspaceId,
|
|
10385
11361
|
eventType: "document.indexed",
|
|
@@ -10464,17 +11440,19 @@ import {
|
|
|
10464
11440
|
listLiveModalSandboxLeaseAttributions,
|
|
10465
11441
|
listMeterableWarmLeases,
|
|
10466
11442
|
persistDrainSnapshot,
|
|
10467
|
-
|
|
11443
|
+
readWorkspaceArchiveCapturePreflight as readWorkspaceArchiveCapturePreflight2,
|
|
11444
|
+
readLease as readLease3,
|
|
10468
11445
|
reapExpiredSessionListSnapshots,
|
|
10469
11446
|
reapStaleLeaseHoldersGlobal,
|
|
10470
11447
|
rlsContextForWorkspace
|
|
10471
11448
|
} from "@opengeni/db";
|
|
10472
11449
|
import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
|
|
10473
11450
|
import {
|
|
11451
|
+
captureVerifiedWorkspaceArchive as captureVerifiedWorkspaceArchive2,
|
|
10474
11452
|
createSandboxClientForBackend,
|
|
10475
11453
|
deletePriorPersistedSnapshot as deletePriorPersistedSnapshot2,
|
|
10476
11454
|
deserializeSandboxSessionStateEnvelope,
|
|
10477
|
-
isProviderSandboxNotFoundError as
|
|
11455
|
+
isProviderSandboxNotFoundError as isProviderSandboxNotFoundError3,
|
|
10478
11456
|
sweepModalOrphanSandboxes,
|
|
10479
11457
|
terminateModalSandboxById
|
|
10480
11458
|
} from "@opengeni/runtime";
|
|
@@ -10746,28 +11724,45 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
|
|
|
10746
11724
|
}
|
|
10747
11725
|
async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
|
|
10748
11726
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
10749
|
-
const lease = await
|
|
11727
|
+
const lease = await readLease3(db, row.workspaceId, row.sandboxGroupId);
|
|
10750
11728
|
if (!lease) {
|
|
10751
11729
|
return false;
|
|
10752
11730
|
}
|
|
10753
11731
|
if (lease.liveness !== "draining" || lease.refcount !== 0 || lease.leaseEpoch !== row.leaseEpoch) {
|
|
10754
11732
|
return false;
|
|
10755
11733
|
}
|
|
11734
|
+
const capturePreflight = lease.instanceId ? await readWorkspaceArchiveCapturePreflight2(db, {
|
|
11735
|
+
accountId,
|
|
11736
|
+
workspaceId: row.workspaceId,
|
|
11737
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
11738
|
+
expectedEpoch: row.leaseEpoch,
|
|
11739
|
+
expectedInstanceId: lease.instanceId,
|
|
11740
|
+
liveness: "draining"
|
|
11741
|
+
}) : null;
|
|
10756
11742
|
let persisted = false;
|
|
10757
|
-
|
|
11743
|
+
let archiveRevision = null;
|
|
11744
|
+
const persistArchive = async (archiveBase64, archiveMetadata) => {
|
|
11745
|
+
if (!lease.instanceId || !capturePreflight) {
|
|
11746
|
+
return { wrote: false, priorArchiveForGc: null, archiveRevision: null };
|
|
11747
|
+
}
|
|
10758
11748
|
const result = await persistDrainSnapshot(db, {
|
|
10759
11749
|
accountId,
|
|
10760
11750
|
workspaceId: row.workspaceId,
|
|
10761
11751
|
sandboxGroupId: row.sandboxGroupId,
|
|
10762
11752
|
expectedEpoch: row.leaseEpoch,
|
|
10763
|
-
|
|
11753
|
+
expectedInstanceId: lease.instanceId,
|
|
11754
|
+
expectedWorkspaceGeneration: capturePreflight.workspaceGeneration,
|
|
11755
|
+
workspaceArchive: archiveBase64,
|
|
11756
|
+
...archiveMetadata ? { workspaceArchiveMeta: archiveMetadata } : {}
|
|
10764
11757
|
});
|
|
10765
11758
|
if (result.wrote && archiveBase64 !== null) {
|
|
10766
11759
|
persisted = true;
|
|
11760
|
+
archiveRevision = result.archiveRevision;
|
|
10767
11761
|
}
|
|
10768
11762
|
return result;
|
|
10769
11763
|
};
|
|
10770
|
-
const
|
|
11764
|
+
const termination = await terminateBox(settings, lease, observability, persistArchive);
|
|
11765
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
10771
11766
|
if (!terminated) {
|
|
10772
11767
|
return false;
|
|
10773
11768
|
}
|
|
@@ -10775,13 +11770,20 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
10775
11770
|
accountId,
|
|
10776
11771
|
workspaceId: row.workspaceId,
|
|
10777
11772
|
sandboxGroupId: row.sandboxGroupId,
|
|
10778
|
-
expectedEpoch: row.leaseEpoch
|
|
11773
|
+
expectedEpoch: row.leaseEpoch,
|
|
11774
|
+
providerMissingBeforeCapture: typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture
|
|
10779
11775
|
});
|
|
10780
11776
|
if (wentCold) {
|
|
10781
11777
|
try {
|
|
10782
11778
|
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
10783
11779
|
type: "sandbox.box.terminated",
|
|
10784
|
-
payload: {
|
|
11780
|
+
payload: {
|
|
11781
|
+
actor: "reaper",
|
|
11782
|
+
persisted,
|
|
11783
|
+
archiveRevision,
|
|
11784
|
+
instanceId: lease.instanceId,
|
|
11785
|
+
providerMissingBeforeCapture: typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture
|
|
11786
|
+
}
|
|
10785
11787
|
});
|
|
10786
11788
|
} catch (eventError) {
|
|
10787
11789
|
observability.warn(
|
|
@@ -10798,7 +11800,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
10798
11800
|
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
|
|
10799
11801
|
const backend = lease.resumeBackendId ?? lease.backend;
|
|
10800
11802
|
if (!backend || backend === "none") {
|
|
10801
|
-
return true;
|
|
11803
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10802
11804
|
}
|
|
10803
11805
|
if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
|
|
10804
11806
|
observability.info(
|
|
@@ -10808,7 +11810,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10808
11810
|
backend
|
|
10809
11811
|
}
|
|
10810
11812
|
);
|
|
10811
|
-
return true;
|
|
11813
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10812
11814
|
}
|
|
10813
11815
|
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
10814
11816
|
const { wrote } = await persistArchive(null);
|
|
@@ -10821,12 +11823,12 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10821
11823
|
instanceId: lease.instanceId
|
|
10822
11824
|
}
|
|
10823
11825
|
);
|
|
10824
|
-
return false;
|
|
11826
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
10825
11827
|
}
|
|
10826
11828
|
try {
|
|
10827
11829
|
await terminateModalSandboxById(settings, lease.instanceId);
|
|
10828
11830
|
} catch (error) {
|
|
10829
|
-
if (!
|
|
11831
|
+
if (!isProviderSandboxNotFoundError3("modal", error)) {
|
|
10830
11832
|
throw error;
|
|
10831
11833
|
}
|
|
10832
11834
|
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
@@ -10835,20 +11837,20 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10835
11837
|
instanceId: lease.instanceId
|
|
10836
11838
|
});
|
|
10837
11839
|
}
|
|
10838
|
-
return true;
|
|
11840
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10839
11841
|
}
|
|
10840
11842
|
if (!lease.resumeState) {
|
|
10841
|
-
return true;
|
|
11843
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10842
11844
|
}
|
|
10843
11845
|
const client = createClientForBackend(backend, settings);
|
|
10844
11846
|
if (!client) {
|
|
10845
|
-
return true;
|
|
11847
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10846
11848
|
}
|
|
10847
11849
|
let session;
|
|
10848
11850
|
let sessionState;
|
|
10849
11851
|
try {
|
|
10850
11852
|
if (!client.resume || !client.deserializeSessionState) {
|
|
10851
|
-
|
|
11853
|
+
throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
|
|
10852
11854
|
}
|
|
10853
11855
|
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
10854
11856
|
const resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
@@ -10856,36 +11858,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10856
11858
|
envelopeSessionState
|
|
10857
11859
|
);
|
|
10858
11860
|
if (resumedState === void 0) {
|
|
10859
|
-
|
|
11861
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
10860
11862
|
}
|
|
10861
11863
|
session = await client.resume(resumedState);
|
|
10862
11864
|
sessionState = resumedState;
|
|
10863
11865
|
} catch (error) {
|
|
10864
|
-
if (
|
|
10865
|
-
observability.info(
|
|
10866
|
-
|
|
10867
|
-
|
|
10868
|
-
|
|
10869
|
-
|
|
10870
|
-
}
|
|
10871
|
-
);
|
|
10872
|
-
return true;
|
|
11866
|
+
if (isProviderSandboxNotFoundError3(client.backendId, error)) {
|
|
11867
|
+
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
11868
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
11869
|
+
backend
|
|
11870
|
+
});
|
|
11871
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
10873
11872
|
}
|
|
10874
11873
|
throw error;
|
|
10875
11874
|
}
|
|
10876
|
-
let
|
|
11875
|
+
let verifiedArchive;
|
|
10877
11876
|
try {
|
|
10878
|
-
|
|
11877
|
+
verifiedArchive = session?.persistWorkspace ? await captureVerifiedWorkspaceArchive2(session) : void 0;
|
|
10879
11878
|
} catch (error) {
|
|
10880
|
-
if (
|
|
10881
|
-
observability.info(
|
|
10882
|
-
|
|
10883
|
-
|
|
10884
|
-
|
|
10885
|
-
|
|
10886
|
-
}
|
|
10887
|
-
);
|
|
10888
|
-
return true;
|
|
11879
|
+
if (isProviderSandboxNotFoundError3(client.backendId, error)) {
|
|
11880
|
+
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
11881
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
11882
|
+
backend
|
|
11883
|
+
});
|
|
11884
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
10889
11885
|
}
|
|
10890
11886
|
observability.warn(
|
|
10891
11887
|
"sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
|
|
@@ -10897,9 +11893,11 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10897
11893
|
);
|
|
10898
11894
|
throw error;
|
|
10899
11895
|
}
|
|
10900
|
-
if (
|
|
10901
|
-
const
|
|
10902
|
-
|
|
11896
|
+
if (verifiedArchive) {
|
|
11897
|
+
const { wrote, priorArchiveForGc } = await persistArchive(
|
|
11898
|
+
verifiedArchive.base64,
|
|
11899
|
+
verifiedArchive.descriptor
|
|
11900
|
+
);
|
|
10903
11901
|
if (!wrote) {
|
|
10904
11902
|
observability.info(
|
|
10905
11903
|
"sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)",
|
|
@@ -10908,9 +11906,9 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10908
11906
|
backend
|
|
10909
11907
|
}
|
|
10910
11908
|
);
|
|
10911
|
-
return false;
|
|
11909
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
10912
11910
|
}
|
|
10913
|
-
const deleted = await deletePriorPersistedSnapshot2(session,
|
|
11911
|
+
const deleted = await deletePriorPersistedSnapshot2(session, priorArchiveForGc);
|
|
10914
11912
|
if (deleted) {
|
|
10915
11913
|
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
10916
11914
|
sandboxGroupId: lease.sandboxGroupId,
|
|
@@ -10918,51 +11916,41 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
10918
11916
|
snapshotId: deleted
|
|
10919
11917
|
});
|
|
10920
11918
|
}
|
|
10921
|
-
const deletedPrev = await deletePriorPersistedSnapshot2(session, priorArchivePrev);
|
|
10922
|
-
if (deletedPrev) {
|
|
10923
|
-
observability.info("sandbox reaper: GC'd superseded previous workspace snapshot", {
|
|
10924
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
10925
|
-
backend,
|
|
10926
|
-
snapshotId: deletedPrev
|
|
10927
|
-
});
|
|
10928
|
-
}
|
|
10929
11919
|
} else {
|
|
10930
|
-
|
|
10931
|
-
|
|
10932
|
-
|
|
10933
|
-
"sandbox reaper: lease re-armed during snapshot window (no-archive path) \u2014 leaving box RUNNING (no terminate)",
|
|
10934
|
-
{
|
|
10935
|
-
sandboxGroupId: lease.sandboxGroupId,
|
|
10936
|
-
backend
|
|
10937
|
-
}
|
|
10938
|
-
);
|
|
10939
|
-
return false;
|
|
10940
|
-
}
|
|
11920
|
+
throw new Error(
|
|
11921
|
+
`sandbox backend ${backend} produced no verified workspace archive during drain`
|
|
11922
|
+
);
|
|
10941
11923
|
}
|
|
10942
11924
|
try {
|
|
10943
11925
|
if (client.delete && sessionState !== void 0) {
|
|
10944
11926
|
await client.delete(sessionState);
|
|
10945
|
-
return true;
|
|
11927
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10946
11928
|
}
|
|
10947
11929
|
if (session?.kill) {
|
|
10948
11930
|
await session.kill();
|
|
10949
|
-
return true;
|
|
11931
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10950
11932
|
}
|
|
10951
11933
|
if (session?.terminate) {
|
|
10952
11934
|
await session.terminate();
|
|
10953
|
-
return true;
|
|
11935
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10954
11936
|
}
|
|
10955
|
-
if (session?.close
|
|
10956
|
-
|
|
11937
|
+
if (session?.close) {
|
|
11938
|
+
if (!session.closed) {
|
|
11939
|
+
await session.close();
|
|
11940
|
+
}
|
|
11941
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
10957
11942
|
}
|
|
10958
11943
|
} catch (error) {
|
|
10959
|
-
|
|
10960
|
-
|
|
10961
|
-
|
|
10962
|
-
|
|
10963
|
-
|
|
11944
|
+
if (isProviderSandboxNotFoundError3(client.backendId, error)) {
|
|
11945
|
+
observability.info("sandbox reaper: provider already gone during terminate", {
|
|
11946
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
11947
|
+
backend
|
|
11948
|
+
});
|
|
11949
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
11950
|
+
}
|
|
11951
|
+
throw error;
|
|
10964
11952
|
}
|
|
10965
|
-
|
|
11953
|
+
throw new Error(`sandbox backend ${backend} exposes no provider termination method`);
|
|
10966
11954
|
}
|
|
10967
11955
|
|
|
10968
11956
|
// src/activities/scheduled-tasks.ts
|
|
@@ -10978,16 +11966,17 @@ import {
|
|
|
10978
11966
|
getVariableSet,
|
|
10979
11967
|
isCodexBilledTurn as isCodexBilledTurn2,
|
|
10980
11968
|
markScheduledTaskRunFailedIfQueued,
|
|
10981
|
-
recordUsageEvent as
|
|
11969
|
+
recordUsageEvent as recordUsageEvent3,
|
|
10982
11970
|
requireScheduledTask,
|
|
10983
11971
|
requireSession as requireSession3,
|
|
10984
11972
|
setTemporalWorkflowId,
|
|
10985
11973
|
settleScheduledTaskRunInTransaction,
|
|
11974
|
+
SessionSpawnDeniedDbError,
|
|
10986
11975
|
sumUsageQuantity as sumUsageQuantity4,
|
|
10987
11976
|
updateScheduledTask,
|
|
10988
11977
|
upsertSessionGoal
|
|
10989
11978
|
} from "@opengeni/db";
|
|
10990
|
-
import { appendAndPublishEvents
|
|
11979
|
+
import { appendAndPublishEvents, publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
|
|
10991
11980
|
import { configuredStaticUsageLimits as configuredStaticUsageLimits4 } from "@opengeni/config";
|
|
10992
11981
|
function createScheduledTaskActivities(services) {
|
|
10993
11982
|
return {
|
|
@@ -11016,7 +12005,7 @@ function createScheduledTaskActivities(services) {
|
|
|
11016
12005
|
producerKey: input.producerKey ?? input.agentRunUsageIdempotencyKey ?? `scheduled:${crypto.randomUUID()}`,
|
|
11017
12006
|
scheduledAt: null
|
|
11018
12007
|
});
|
|
11019
|
-
await
|
|
12008
|
+
await recordUsageEvent3(db, {
|
|
11020
12009
|
accountId: task.accountId,
|
|
11021
12010
|
workspaceId: task.workspaceId,
|
|
11022
12011
|
eventType: "scheduled_task.fired",
|
|
@@ -11030,7 +12019,7 @@ function createScheduledTaskActivities(services) {
|
|
|
11030
12019
|
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
11031
12020
|
});
|
|
11032
12021
|
if (run.status === "dispatched" && run.sessionId && run.triggerEventId) {
|
|
11033
|
-
await
|
|
12022
|
+
await recordUsageEvent3(db, {
|
|
11034
12023
|
accountId: task.accountId,
|
|
11035
12024
|
workspaceId: task.workspaceId,
|
|
11036
12025
|
eventType: "agent_run.created",
|
|
@@ -11093,34 +12082,48 @@ function createScheduledTaskActivities(services) {
|
|
|
11093
12082
|
frozenRigId = rig.id;
|
|
11094
12083
|
frozenRigVersionId = rig.activeVersion.id;
|
|
11095
12084
|
}
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
11100
|
-
|
|
11101
|
-
|
|
11102
|
-
|
|
11103
|
-
|
|
12085
|
+
let session;
|
|
12086
|
+
try {
|
|
12087
|
+
session = await createSession(db, {
|
|
12088
|
+
accountId: task.accountId,
|
|
12089
|
+
workspaceId: task.workspaceId,
|
|
12090
|
+
initialMessage: task.agentConfig.prompt,
|
|
12091
|
+
resources: task.agentConfig.resources,
|
|
12092
|
+
tools: taskTools,
|
|
12093
|
+
metadata: {
|
|
12094
|
+
...task.agentConfig.metadata,
|
|
12095
|
+
model,
|
|
12096
|
+
reasoningEffort,
|
|
12097
|
+
scheduledTaskId: task.id,
|
|
12098
|
+
scheduledTaskRunId: run.id
|
|
12099
|
+
},
|
|
12100
|
+
createdBy: {
|
|
12101
|
+
kind: "service",
|
|
12102
|
+
subjectId: "scheduler",
|
|
12103
|
+
label: "OpenGeni scheduler"
|
|
12104
|
+
},
|
|
12105
|
+
createdByContext: {
|
|
12106
|
+
scheduledTaskId: task.id,
|
|
12107
|
+
scheduledTaskRunId: run.id
|
|
12108
|
+
},
|
|
11104
12109
|
model,
|
|
11105
|
-
|
|
11106
|
-
|
|
11107
|
-
|
|
11108
|
-
|
|
11109
|
-
|
|
11110
|
-
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
|
|
11119
|
-
|
|
11120
|
-
|
|
11121
|
-
|
|
11122
|
-
rigVersionId: frozenRigVersionId
|
|
11123
|
-
});
|
|
12110
|
+
sandboxBackend,
|
|
12111
|
+
variableSetId: task.variableSetId ?? null,
|
|
12112
|
+
rigId: frozenRigId,
|
|
12113
|
+
rigVersionId: frozenRigVersionId,
|
|
12114
|
+
maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
|
|
12115
|
+
// The durable agent config was privilege-checked when the task
|
|
12116
|
+
// was created/updated. Preserve that explicit policy if a broader
|
|
12117
|
+
// workspace/deployment limit is narrowed before a later fire.
|
|
12118
|
+
allowNestedAgentDepthIncrease: true,
|
|
12119
|
+
subjectId: `scheduled_task:${task.id}`
|
|
12120
|
+
});
|
|
12121
|
+
} catch (error) {
|
|
12122
|
+
if (error instanceof SessionSpawnDeniedDbError) {
|
|
12123
|
+
throw new Error(`${error.denial.code}: denial=${error.denial.id}`, { cause: error });
|
|
12124
|
+
}
|
|
12125
|
+
throw error;
|
|
12126
|
+
}
|
|
11124
12127
|
const goal = goalSpec ? await createSessionGoal(db, {
|
|
11125
12128
|
accountId: task.accountId,
|
|
11126
12129
|
workspaceId: task.workspaceId,
|
|
@@ -11137,7 +12140,7 @@ function createScheduledTaskActivities(services) {
|
|
|
11137
12140
|
reusableSessionId: session.id
|
|
11138
12141
|
});
|
|
11139
12142
|
}
|
|
11140
|
-
await
|
|
12143
|
+
await appendAndPublishEvents(db, bus, task.workspaceId, session.id, [
|
|
11141
12144
|
{
|
|
11142
12145
|
type: "session.created",
|
|
11143
12146
|
payload: {
|
|
@@ -11314,7 +12317,7 @@ function createScheduledTaskActivities(services) {
|
|
|
11314
12317
|
).catch(() => void 0);
|
|
11315
12318
|
throw error;
|
|
11316
12319
|
}
|
|
11317
|
-
await
|
|
12320
|
+
await recordUsageEvent3(db, {
|
|
11318
12321
|
accountId: task.accountId,
|
|
11319
12322
|
workspaceId: task.workspaceId,
|
|
11320
12323
|
eventType: "agent_run.created",
|
|
@@ -11635,7 +12638,7 @@ import {
|
|
|
11635
12638
|
getRig as getRig2,
|
|
11636
12639
|
getRigChange,
|
|
11637
12640
|
getRigVersionById,
|
|
11638
|
-
markWarmLeaseInstanceLost as
|
|
12641
|
+
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost3,
|
|
11639
12642
|
recordWarmingSandboxCreated as recordWarmingSandboxCreated2,
|
|
11640
12643
|
releaseLeaseHolder as releaseLeaseHolder2,
|
|
11641
12644
|
sanitizeEventPayload,
|
|
@@ -11646,11 +12649,11 @@ import {
|
|
|
11646
12649
|
} from "@opengeni/db";
|
|
11647
12650
|
import {
|
|
11648
12651
|
createTurnToolCancellationController,
|
|
11649
|
-
establishSandboxSessionFromEnvelope as
|
|
12652
|
+
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope3,
|
|
11650
12653
|
runRigSetupHook,
|
|
11651
12654
|
sandboxCommandExitCode,
|
|
11652
12655
|
sandboxCommandOutput,
|
|
11653
|
-
serializeEstablishedSandboxEnvelope
|
|
12656
|
+
serializeEstablishedSandboxEnvelope,
|
|
11654
12657
|
tagModalSandbox as tagModalSandbox2
|
|
11655
12658
|
} from "@opengeni/runtime";
|
|
11656
12659
|
var OUTPUT_TAIL_LIMIT = 64 * 1024;
|
|
@@ -11836,14 +12839,14 @@ async function terminateThrowaway(established) {
|
|
|
11836
12839
|
}
|
|
11837
12840
|
var defaultOwnershipDependencies = {
|
|
11838
12841
|
acquire: acquireLease2,
|
|
11839
|
-
establish:
|
|
12842
|
+
establish: establishSandboxSessionFromEnvelope3,
|
|
11840
12843
|
recordCreated: recordWarmingSandboxCreated2,
|
|
11841
12844
|
commitWarm: commitWarmingToWarm2,
|
|
11842
12845
|
touchHolder: touchLeaseHolder2,
|
|
11843
12846
|
releaseHolder: releaseLeaseHolder2,
|
|
11844
12847
|
failWarming: failWarmingToCold2,
|
|
11845
|
-
markWarmLost:
|
|
11846
|
-
serialize:
|
|
12848
|
+
markWarmLost: markWarmLeaseInstanceLost3,
|
|
12849
|
+
serialize: serializeEstablishedSandboxEnvelope,
|
|
11847
12850
|
tag: tagModalSandbox2,
|
|
11848
12851
|
terminate: terminateThrowaway,
|
|
11849
12852
|
createCancellationController: createTurnToolCancellationController,
|