@opengeni/worker-bundle 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +685 -260
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +474 -231
- package/src/activities/capabilities.ts +42 -0
- package/src/activities/context-compaction.ts +51 -58
- package/src/activities/run-input.ts +2 -2
- package/src/activities/sandbox-lease.ts +97 -3
- package/src/sandbox-resume.ts +178 -59
|
@@ -12,9 +12,12 @@ import {
|
|
|
12
12
|
import {
|
|
13
13
|
decryptedCapabilityHeaders,
|
|
14
14
|
listEnabledMcpCapabilityServers,
|
|
15
|
+
listSessionMcpServerMetadata,
|
|
16
|
+
listSessionMcpServersForRun,
|
|
15
17
|
workspaceCodexSubscriptionActive,
|
|
16
18
|
type Database,
|
|
17
19
|
type EnabledMcpCapabilityServer,
|
|
20
|
+
type SessionMcpServerForRun,
|
|
18
21
|
} from "@opengeni/db";
|
|
19
22
|
|
|
20
23
|
export async function settingsWithEnabledCapabilityMcpServers(db: Database, workspaceId: string, settings: Settings): Promise<Settings> {
|
|
@@ -22,6 +25,45 @@ export async function settingsWithEnabledCapabilityMcpServers(db: Database, work
|
|
|
22
25
|
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
23
26
|
}
|
|
24
27
|
|
|
28
|
+
export async function settingsWithSessionMcpServersForRun(
|
|
29
|
+
db: Database,
|
|
30
|
+
workspaceId: string,
|
|
31
|
+
sessionId: string,
|
|
32
|
+
settings: Settings,
|
|
33
|
+
): Promise<Settings> {
|
|
34
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
35
|
+
if (!encryptionKey) {
|
|
36
|
+
const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
|
|
37
|
+
if (metadata.length === 0) {
|
|
38
|
+
return settings;
|
|
39
|
+
}
|
|
40
|
+
throw new Error("session MCP server credentials require OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
41
|
+
}
|
|
42
|
+
return settingsWithSessionMcpServers(settings, await listSessionMcpServersForRun(db, workspaceId, sessionId, encryptionKey));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function settingsWithSessionMcpServers(settings: Settings, servers: SessionMcpServerForRun[]): Settings {
|
|
46
|
+
if (servers.length === 0) {
|
|
47
|
+
return settings;
|
|
48
|
+
}
|
|
49
|
+
const sessionIds = new Set(servers.map((server) => server.id));
|
|
50
|
+
return {
|
|
51
|
+
...settings,
|
|
52
|
+
mcpServers: [
|
|
53
|
+
...settings.mcpServers.filter((server) => !sessionIds.has(server.id)),
|
|
54
|
+
...servers.map((server) => ({
|
|
55
|
+
id: server.id,
|
|
56
|
+
...(server.name ? { name: server.name } : {}),
|
|
57
|
+
url: server.url,
|
|
58
|
+
...(server.allowedTools ? { allowedTools: server.allowedTools } : {}),
|
|
59
|
+
...(server.timeoutMs ? { timeoutMs: server.timeoutMs } : {}),
|
|
60
|
+
cacheToolsList: server.cacheToolsList ?? false,
|
|
61
|
+
headers: server.headers,
|
|
62
|
+
})),
|
|
63
|
+
],
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
25
67
|
/**
|
|
26
68
|
* When the workspace has an active Codex subscription connected and the feature
|
|
27
69
|
* is enabled, inject a synthetic "codex-subscription" registry provider so a
|
|
@@ -1,48 +1,50 @@
|
|
|
1
1
|
import {
|
|
2
2
|
applyContextCompaction,
|
|
3
3
|
getActiveSessionHistoryItems,
|
|
4
|
+
nextSessionHistoryPosition,
|
|
4
5
|
type Database,
|
|
5
6
|
} from "@opengeni/db";
|
|
6
7
|
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
SUMMARY_BUFFER_TOKENS,
|
|
9
|
+
buildCompactionPromptInput,
|
|
10
|
+
buildCompactionReplacementHistory,
|
|
11
|
+
decideClientCompaction,
|
|
12
|
+
sanitizeHistoryItemsForModel,
|
|
10
13
|
summarizeForCompaction,
|
|
11
14
|
type CompactionItem,
|
|
12
15
|
} from "@opengeni/runtime";
|
|
13
|
-
import {
|
|
14
|
-
contextInputBudgetTokens,
|
|
15
|
-
resolveContextCompactionMode,
|
|
16
|
-
type Settings,
|
|
17
|
-
} from "@opengeni/config";
|
|
16
|
+
import { resolveContextCompactionMode, type Settings } from "@opengeni/config";
|
|
18
17
|
|
|
19
18
|
export type MaybeCompactResult =
|
|
20
19
|
| { compacted: false; reason: string }
|
|
21
|
-
| {
|
|
20
|
+
| {
|
|
21
|
+
compacted: true;
|
|
22
|
+
supersededFrom: number;
|
|
23
|
+
summaryPosition: number;
|
|
24
|
+
signalTokens: number;
|
|
25
|
+
thresholdTokens: number;
|
|
26
|
+
};
|
|
22
27
|
|
|
23
28
|
/**
|
|
24
29
|
* Pre-turn client-side context compaction (the Azure path).
|
|
25
30
|
*
|
|
26
31
|
* Runs BEFORE the model call when the resolved compaction mode is "client".
|
|
27
|
-
* Reads the active history rows + the
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
32
|
+
* Reads the active history rows + the most recent provider-reported input
|
|
33
|
+
* tokens, applies the single Codex-parity threshold, and - when it should -
|
|
34
|
+
* summarizes the active history with the Codex checkpoint prompt. The write
|
|
35
|
+
* supersedes the old active rows and inserts replacement active rows:
|
|
36
|
+
* all real user messages plus one summary.
|
|
31
37
|
*
|
|
32
38
|
* Best-effort by design: any failure (planner says no, summarize fails, DB
|
|
33
39
|
* hiccup) returns without compacting and never throws — the turn proceeds with
|
|
34
40
|
* the un-compacted history. The read path's existing sanitizer keeps that safe.
|
|
35
41
|
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
* fractional position a half-step before the kept tail (boundaryPosition - 0.5),
|
|
39
|
-
* which sorts ahead of the tail without overwriting any real prefix row, so the
|
|
40
|
-
* active read path returns [summary, ...recent tail] in order and the full
|
|
41
|
-
* superseded prefix survives untouched as an audit trail.
|
|
42
|
+
* There is no kept assistant/tool tail in client mode now. Assistant messages,
|
|
43
|
+
* tool calls/results, reasoning, and images stay only in inactive audit rows.
|
|
42
44
|
*/
|
|
43
45
|
export type CompactionSummarizer = (
|
|
44
46
|
settings: Settings,
|
|
45
|
-
|
|
47
|
+
input: CompactionItem[],
|
|
46
48
|
) => Promise<string | null>;
|
|
47
49
|
|
|
48
50
|
export async function maybeCompactContext(
|
|
@@ -51,7 +53,7 @@ export async function maybeCompactContext(
|
|
|
51
53
|
scope: { accountId: string; workspaceId: string; sessionId: string; turnId?: string | null },
|
|
52
54
|
lastInputTokens: number | null,
|
|
53
55
|
// Injectable for tests; defaults to the real provider-aware model call.
|
|
54
|
-
summarize: CompactionSummarizer = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens:
|
|
56
|
+
summarize: CompactionSummarizer = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: SUMMARY_BUFFER_TOKENS }),
|
|
55
57
|
// Operator-forced (the /compact command): bypass the budget trigger and
|
|
56
58
|
// compact now if there is anything to summarize. Structural guards still hold.
|
|
57
59
|
options: { force?: boolean } = {},
|
|
@@ -65,61 +67,52 @@ export async function maybeCompactContext(
|
|
|
65
67
|
return { compacted: false, reason: "no_history" };
|
|
66
68
|
}
|
|
67
69
|
|
|
68
|
-
const items = active.map((row) => row.item) as CompactionItem[];
|
|
69
|
-
const
|
|
70
|
+
const items = sanitizeHistoryItemsForModel(active.map((row) => row.item) as CompactionItem[]);
|
|
71
|
+
const decision = decideClientCompaction({
|
|
70
72
|
items,
|
|
71
73
|
lastInputTokens,
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
hardFraction: settings.contextCompactHardFraction,
|
|
75
|
-
keepRecentTokens: settings.contextKeepRecentTokens,
|
|
74
|
+
contextWindowTokens: settings.contextWindowTokens,
|
|
75
|
+
contextReservedOutputTokens: settings.contextReservedOutputTokens,
|
|
76
76
|
...(options.force ? { force: true } : {}),
|
|
77
77
|
});
|
|
78
|
-
if (!
|
|
79
|
-
return { compacted: false, reason:
|
|
80
|
-
}
|
|
81
|
-
if (plan.hardForced) {
|
|
82
|
-
// At/over the hard ceiling (hardFraction*B): the session is past the soft
|
|
83
|
-
// trigger by a wide margin (often because last_input_tokens was stale-low
|
|
84
|
-
// and the actual history overflowed). Logged distinctly so the hard-force
|
|
85
|
-
// backstop firing is observable in production.
|
|
86
|
-
console.warn(
|
|
87
|
-
`context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`,
|
|
88
|
-
);
|
|
78
|
+
if (!decision.shouldCompact) {
|
|
79
|
+
return { compacted: false, reason: decision.reason };
|
|
89
80
|
}
|
|
90
81
|
|
|
91
|
-
const
|
|
92
|
-
const summaryBody = await summarize(settings,
|
|
82
|
+
const promptInput = buildCompactionPromptInput(items);
|
|
83
|
+
const summaryBody = await summarize(settings, promptInput);
|
|
93
84
|
if (!summaryBody) {
|
|
94
85
|
return { compacted: false, reason: "summarize_failed" };
|
|
95
86
|
}
|
|
96
87
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
// the last superseded prefix row while colliding with NO real row — the
|
|
102
|
-
// earlier `boundaryPosition - 1` always landed on (and overwrote) the real
|
|
103
|
-
// prefix row there.
|
|
104
|
-
const boundaryRow = active[plan.boundaryIndex];
|
|
105
|
-
if (!boundaryRow) {
|
|
106
|
-
return { compacted: false, reason: "boundary_out_of_range" };
|
|
88
|
+
const replacementHistory = buildCompactionReplacementHistory(items, summaryBody);
|
|
89
|
+
const summaryItem = replacementHistory.at(-1);
|
|
90
|
+
if (!summaryItem) {
|
|
91
|
+
return { compacted: false, reason: "empty_replacement_history" };
|
|
107
92
|
}
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
93
|
+
const nextPosition = await nextSessionHistoryPosition(db, scope.workspaceId, scope.sessionId);
|
|
94
|
+
const replacementItems = replacementHistory.slice(0, -1).map((item, index) => ({
|
|
95
|
+
position: nextPosition + index,
|
|
96
|
+
item,
|
|
97
|
+
}));
|
|
98
|
+
const summaryPosition = nextPosition + replacementItems.length;
|
|
113
99
|
|
|
114
100
|
await applyContextCompaction(db, {
|
|
115
101
|
accountId: scope.accountId,
|
|
116
102
|
workspaceId: scope.workspaceId,
|
|
117
103
|
sessionId: scope.sessionId,
|
|
118
104
|
turnId: scope.turnId ?? null,
|
|
119
|
-
boundaryPosition,
|
|
105
|
+
boundaryPosition: nextPosition,
|
|
106
|
+
replacementItems,
|
|
120
107
|
summaryPosition,
|
|
121
|
-
summaryItem:
|
|
108
|
+
summaryItem: summaryItem as Record<string, unknown>,
|
|
122
109
|
});
|
|
123
110
|
|
|
124
|
-
return {
|
|
111
|
+
return {
|
|
112
|
+
compacted: true,
|
|
113
|
+
supersededFrom: nextPosition,
|
|
114
|
+
summaryPosition,
|
|
115
|
+
signalTokens: decision.signalTokens,
|
|
116
|
+
thresholdTokens: decision.thresholdTokens,
|
|
117
|
+
};
|
|
125
118
|
}
|
|
@@ -234,8 +234,8 @@ async function messageInput(
|
|
|
234
234
|
const inputBudgetTokens = readPathBudgetTokens(settings);
|
|
235
235
|
if (settings?.sessionHistorySource === "items") {
|
|
236
236
|
// Active rows only: after a client-side context compaction this is
|
|
237
|
-
// [
|
|
238
|
-
//
|
|
237
|
+
// [retained user messages..., active summary]; superseded rows stay in the
|
|
238
|
+
// table as an audit trail but never reach the model.
|
|
239
239
|
const stored = await getActiveSessionHistoryItems(db, trigger.workspaceId, trigger.sessionId);
|
|
240
240
|
if (stored.length > 0) {
|
|
241
241
|
const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
|
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
confirmDrainCold,
|
|
36
36
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
37
|
getBillingBalance,
|
|
38
|
+
listLiveModalSandboxLeaseAttributions,
|
|
38
39
|
listMeterableWarmLeases,
|
|
39
40
|
persistDrainSnapshot,
|
|
40
41
|
readLease,
|
|
@@ -55,6 +56,8 @@ import {
|
|
|
55
56
|
deletePriorPersistedSnapshot,
|
|
56
57
|
deserializeSandboxSessionStateEnvelope,
|
|
57
58
|
isProviderSandboxNotFoundError,
|
|
59
|
+
sweepModalOrphanSandboxes,
|
|
60
|
+
terminateModalSandboxById,
|
|
58
61
|
} from "@opengeni/runtime";
|
|
59
62
|
import type { ActivityServices } from "./types";
|
|
60
63
|
|
|
@@ -73,6 +76,8 @@ export type ReapSandboxLeasesResult = {
|
|
|
73
76
|
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
74
77
|
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
75
78
|
forceDrained: number;
|
|
79
|
+
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
80
|
+
modalOrphansTerminated: number;
|
|
76
81
|
};
|
|
77
82
|
|
|
78
83
|
// The structural slice of a provider SandboxClient we need to resume-by-id and
|
|
@@ -129,10 +134,19 @@ export type TerminateBoxFn = (
|
|
|
129
134
|
persistArchive: PersistArchiveFn,
|
|
130
135
|
) => Promise<boolean>;
|
|
131
136
|
|
|
137
|
+
export type SweepModalOrphansFn = (
|
|
138
|
+
settings: ActivityServices["settings"],
|
|
139
|
+
db: ActivityServices["db"],
|
|
140
|
+
observability: ActivityServices["observability"],
|
|
141
|
+
) => Promise<number>;
|
|
142
|
+
|
|
132
143
|
export type SandboxLeaseActivityOptions = {
|
|
133
144
|
/** Override the provider terminate (tests spy this; defaults to the real
|
|
134
145
|
* resume-by-id + provider stop()). */
|
|
135
146
|
terminateBox?: TerminateBoxFn;
|
|
147
|
+
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
148
|
+
* Modal list+tag comparison when Modal is configured). */
|
|
149
|
+
sweepModalOrphans?: SweepModalOrphansFn;
|
|
136
150
|
};
|
|
137
151
|
|
|
138
152
|
export function createSandboxLeaseActivities(
|
|
@@ -140,6 +154,7 @@ export function createSandboxLeaseActivities(
|
|
|
140
154
|
options: SandboxLeaseActivityOptions = {},
|
|
141
155
|
) {
|
|
142
156
|
const terminateBox: TerminateBoxFn = options.terminateBox ?? terminateProviderBox;
|
|
157
|
+
const sweepModalOrphans: SweepModalOrphansFn = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
143
158
|
/**
|
|
144
159
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
145
160
|
* Gated by the caller (the Schedule is only registered when
|
|
@@ -149,7 +164,7 @@ export function createSandboxLeaseActivities(
|
|
|
149
164
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
150
165
|
const { db, settings, observability } = await services();
|
|
151
166
|
if (!settings.sandboxOwnershipEnabled) {
|
|
152
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
167
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
153
168
|
}
|
|
154
169
|
|
|
155
170
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
@@ -172,6 +187,7 @@ export function createSandboxLeaseActivities(
|
|
|
172
187
|
|
|
173
188
|
let terminated = 0;
|
|
174
189
|
let skipped = 0;
|
|
190
|
+
let modalOrphansTerminated = 0;
|
|
175
191
|
|
|
176
192
|
// (2) Terminate each drainable box, then CAS draining->cold. Per-row failures
|
|
177
193
|
// are isolated: one box's provider error must not abort the whole sweep (the
|
|
@@ -195,17 +211,26 @@ export function createSandboxLeaseActivities(
|
|
|
195
211
|
}
|
|
196
212
|
}
|
|
197
213
|
|
|
198
|
-
|
|
214
|
+
try {
|
|
215
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
216
|
+
} catch (error) {
|
|
217
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
218
|
+
error: error instanceof Error ? error.message : String(error),
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
199
223
|
observability.info("sandbox reaper swept", {
|
|
200
224
|
drainable: drainable.length,
|
|
201
225
|
terminated,
|
|
202
226
|
skipped,
|
|
203
227
|
metered: metered.accrued,
|
|
204
228
|
forceDrained,
|
|
229
|
+
modalOrphansTerminated,
|
|
205
230
|
});
|
|
206
231
|
}
|
|
207
232
|
|
|
208
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
233
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
209
234
|
}
|
|
210
235
|
|
|
211
236
|
return { reapSandboxLeases };
|
|
@@ -310,6 +335,46 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
310
335
|
return forceDrained;
|
|
311
336
|
}
|
|
312
337
|
|
|
338
|
+
async function sweepModalOrphansForConfiguredBackend(
|
|
339
|
+
settings: ActivityServices["settings"],
|
|
340
|
+
db: ActivityServices["db"],
|
|
341
|
+
observability: ActivityServices["observability"],
|
|
342
|
+
): Promise<number> {
|
|
343
|
+
// Keep the provider-side sweep tightly scoped to deployments that have a Modal
|
|
344
|
+
// app path configured. Local/docker-only workers should not attempt Modal API
|
|
345
|
+
// calls just because modalAppName has a default.
|
|
346
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
347
|
+
return 0;
|
|
348
|
+
}
|
|
349
|
+
let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
|
|
350
|
+
try {
|
|
351
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
352
|
+
} catch (error) {
|
|
353
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
354
|
+
error: error instanceof Error ? error.message : String(error),
|
|
355
|
+
});
|
|
356
|
+
return 0;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
360
|
+
for (const terminated of result.terminated) {
|
|
361
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
362
|
+
sandboxId: terminated.sandboxId,
|
|
363
|
+
reason: terminated.reason,
|
|
364
|
+
tags: JSON.stringify(terminated.tags),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
if (result.examined > 0) {
|
|
368
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
369
|
+
examined: result.examined,
|
|
370
|
+
terminated: result.terminated.length,
|
|
371
|
+
skipped: result.skipped,
|
|
372
|
+
appName: settings.modalAppName,
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
return result.terminated.length;
|
|
376
|
+
}
|
|
377
|
+
|
|
313
378
|
/**
|
|
314
379
|
* Terminate one drainable box by id, then CAS its lease draining->cold under the
|
|
315
380
|
* epoch fence. Returns true when the lease went cold (the box is ours to stop
|
|
@@ -458,6 +523,35 @@ export async function terminateProviderBox(
|
|
|
458
523
|
return true;
|
|
459
524
|
}
|
|
460
525
|
|
|
526
|
+
// A warming-death row can have a provider instance id recorded immediately
|
|
527
|
+
// after create() but no resumable envelope yet. For Modal, instance_id is
|
|
528
|
+
// enough to terminate directly; CAS-check first so a re-arm during the sweep
|
|
529
|
+
// leaves the box running.
|
|
530
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
531
|
+
const { wrote } = await persistArchive(null);
|
|
532
|
+
if (!wrote) {
|
|
533
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING", {
|
|
534
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
535
|
+
backend,
|
|
536
|
+
instanceId: lease.instanceId,
|
|
537
|
+
});
|
|
538
|
+
return false;
|
|
539
|
+
}
|
|
540
|
+
try {
|
|
541
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
542
|
+
} catch (error) {
|
|
543
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
544
|
+
throw error;
|
|
545
|
+
}
|
|
546
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
547
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
548
|
+
backend,
|
|
549
|
+
instanceId: lease.instanceId,
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
return true;
|
|
553
|
+
}
|
|
554
|
+
|
|
461
555
|
// resume_state is the folded group box-envelope (the provider sessionState the
|
|
462
556
|
// box was last persisted as). No envelope -> no live box to stop.
|
|
463
557
|
if (!lease.resumeState) {
|