@opengeni/worker-bundle 2.0.0 → 2.0.1-canary.35916963870001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +1 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +1 -1
- package/dist/activities/agent-turn/tool-environment.d.ts +34 -0
- package/dist/activities/codemode-dispatcher.d.ts +3 -1
- package/dist/activities/sandbox-lease.d.ts +3 -4
- package/dist/{activities-control-BOLIY3H4.js → activities-control-UF2CR2LB.js} +199 -23
- package/dist/activities-control-UF2CR2LB.js.map +1 -0
- package/dist/{activities-turn-2B3WMHBB.js → activities-turn-KBLXHBWS.js} +109 -25
- package/dist/activities-turn-KBLXHBWS.js.map +1 -0
- package/dist/{chunk-TVEE2UUC.js → chunk-K3BPAI4R.js} +12 -6
- package/dist/chunk-K3BPAI4R.js.map +1 -0
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +4 -1
- package/package.json +19 -19
- package/src/activities/agent-turn/agent-build.ts +5 -1
- package/src/activities/agent-turn/claim.ts +12 -2
- package/src/activities/agent-turn/errors.ts +1 -0
- package/src/activities/agent-turn/failure-settlement.ts +40 -8
- package/src/activities/agent-turn/run.ts +1 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +25 -9
- package/src/activities/agent-turn/stream-attempt.ts +14 -1
- package/src/activities/agent-turn/tool-environment.ts +5 -0
- package/src/activities/codemode-dispatcher.ts +6 -0
- package/src/activities/knowledge-indexing.ts +124 -6
- package/src/activities/retained-screenshots.ts +42 -4
- package/src/activities/sandbox-lease.ts +141 -21
- package/src/sandbox-resume.ts +6 -0
- package/src/sandbox-routing.ts +7 -3
- package/src/workflows/activities.ts +14 -1
- package/dist/activities-control-BOLIY3H4.js.map +0 -1
- package/dist/activities-turn-2B3WMHBB.js.map +0 -1
- package/dist/chunk-TVEE2UUC.js.map +0 -1
|
@@ -13,6 +13,7 @@ import type { loadWorkspaceEnvironmentForRunWithCredentials, sandboxEnvironmentF
|
|
|
13
13
|
import type { AttemptIdentityState, EventingState, ProviderTurnState, SandboxRuntimeState } from "./turn-context.js";
|
|
14
14
|
export type BuildTurnAgentDeps = {
|
|
15
15
|
skillCatalog: NonNullable<BuildAgentOptions["skillCatalog"]>;
|
|
16
|
+
mcpServers: Settings["mcpServers"];
|
|
16
17
|
input: RunAgentTurnInput;
|
|
17
18
|
db: ActivityServices["db"];
|
|
18
19
|
runtime: ActivityServices["runtime"];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Settings } from "@opengeni/config";
|
|
2
2
|
import type { TurnActivityServices as ActivityServices, RunAgentTurnInput, RunAgentTurnResult } from "../types.js";
|
|
3
3
|
import { createTurnCredentialLeases } from "./credential-leases.js";
|
|
4
4
|
import { createTurnHistorySink } from "./history-sink.js";
|
|
@@ -125,6 +125,40 @@ export declare function prepareTurnToolPolicy(deps: PrepareTurnToolPolicyDeps):
|
|
|
125
125
|
export declare function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps): Promise<{
|
|
126
126
|
attemptConnectorActionBindings: AttemptConnectorActionBinding[];
|
|
127
127
|
connectorActionPolicy: ConnectorActionPolicyHooks;
|
|
128
|
+
mcpServers: {
|
|
129
|
+
id: string;
|
|
130
|
+
name?: string | undefined;
|
|
131
|
+
url: string;
|
|
132
|
+
allowedTools?: string[] | undefined;
|
|
133
|
+
timeoutMs?: number | undefined;
|
|
134
|
+
cacheToolsList: boolean;
|
|
135
|
+
requireApproval?: boolean | string[] | undefined;
|
|
136
|
+
headers?: Record<string, string> | undefined;
|
|
137
|
+
connectionRef?: {
|
|
138
|
+
connectionId?: string | undefined;
|
|
139
|
+
accountSelection?: "all_eligible" | undefined;
|
|
140
|
+
authoritySource?: "host" | undefined;
|
|
141
|
+
hostBinding?: {
|
|
142
|
+
bindingId: string;
|
|
143
|
+
generation: number;
|
|
144
|
+
} | {
|
|
145
|
+
selection: "accepted_turn";
|
|
146
|
+
} | undefined;
|
|
147
|
+
provider?: string | undefined;
|
|
148
|
+
providerDomain: string;
|
|
149
|
+
kind?: "api_key" | "app_install" | "delegated" | "oauth2" | undefined;
|
|
150
|
+
scopes?: string[] | undefined;
|
|
151
|
+
resource?: string | undefined;
|
|
152
|
+
selectedResources?: {
|
|
153
|
+
id: string;
|
|
154
|
+
kind: "repository";
|
|
155
|
+
}[] | undefined;
|
|
156
|
+
subjectScope?: "subject" | "workspace" | undefined;
|
|
157
|
+
} | undefined;
|
|
158
|
+
operationRecovery?: Record<string, {
|
|
159
|
+
observerTool: string;
|
|
160
|
+
}> | undefined;
|
|
161
|
+
}[];
|
|
128
162
|
generateSessionTitleInParallel: boolean;
|
|
129
163
|
postToolPreparationStartedAt: number;
|
|
130
164
|
preparationIndependentToolNames: string[];
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type AttemptToolEnvironment } from "@opengeni/codemode";
|
|
2
|
+
import { type ToolDisplayMetadata } from "@opengeni/contracts";
|
|
2
3
|
import { type Database } from "@opengeni/db";
|
|
3
4
|
import { type EventBus } from "@opengeni/events";
|
|
4
5
|
export type CodemodeDispatcherScope = {
|
|
@@ -25,6 +26,7 @@ export declare class CodemodeAttemptDispatcher {
|
|
|
25
26
|
private readonly scope;
|
|
26
27
|
private readonly turnSignal?;
|
|
27
28
|
private readonly maxConcurrentCalls;
|
|
29
|
+
private readonly toolDisplayMetadata?;
|
|
28
30
|
private readonly stopController;
|
|
29
31
|
private readonly inFlight;
|
|
30
32
|
private pendingClaims;
|
|
@@ -32,7 +34,7 @@ export declare class CodemodeAttemptDispatcher {
|
|
|
32
34
|
private closing;
|
|
33
35
|
private readonly claimLeaseMs;
|
|
34
36
|
private readonly claimHeartbeatMs;
|
|
35
|
-
constructor(db: Database, bus: EventBus, environment: AttemptToolEnvironment, scope: CodemodeDispatcherScope, turnSignal?: AbortSignal | undefined, maxConcurrentCalls?: number, timings?: CodemodeDispatcherTimings);
|
|
37
|
+
constructor(db: Database, bus: EventBus, environment: AttemptToolEnvironment, scope: CodemodeDispatcherScope, turnSignal?: AbortSignal | undefined, maxConcurrentCalls?: number, timings?: CodemodeDispatcherTimings, toolDisplayMetadata?: ((modelName: string) => ToolDisplayMetadata | undefined) | undefined);
|
|
36
38
|
start(): void;
|
|
37
39
|
close(reason?: string): Promise<void>;
|
|
38
40
|
private execute;
|
|
@@ -20,8 +20,7 @@ export type ReapSandboxLeasesResult = {
|
|
|
20
20
|
skipped: number;
|
|
21
21
|
/** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
|
|
22
22
|
metered: number;
|
|
23
|
-
/**
|
|
24
|
-
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
23
|
+
/** Holderless boxes accelerated to draining by the optional warm cap. */
|
|
25
24
|
forceDrained: number;
|
|
26
25
|
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
27
26
|
modalOrphansTerminated: number;
|
|
@@ -60,7 +59,7 @@ export type DrainCaptureDisposition = "capture_required" | "archive_published";
|
|
|
60
59
|
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
61
60
|
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
62
61
|
* provider disappearance before capture. */
|
|
63
|
-
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string) => Promise<boolean | ProviderTerminationOutcome>;
|
|
62
|
+
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string, beforeProviderStop?: () => Promise<void>) => Promise<boolean | ProviderTerminationOutcome>;
|
|
64
63
|
export type SweepModalOrphansFn = (settings: ActivityServices["settings"], db: ActivityServices["db"], observability: ActivityServices["observability"]) => Promise<number>;
|
|
65
64
|
export type InspectOpenSandboxKubernetesInventoryFn = (input: {
|
|
66
65
|
namespace: string;
|
|
@@ -208,4 +207,4 @@ export declare function sandboxDrainReason(lease: Pick<LeaseSnapshot, "rotationR
|
|
|
208
207
|
* draining until its adapter can prove termination. A truly instance-less or
|
|
209
208
|
* `none`-backed row is a no-op.
|
|
210
209
|
*/
|
|
211
|
-
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string): Promise<ProviderTerminationOutcome>;
|
|
210
|
+
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string, beforeProviderStop?: () => Promise<void>): Promise<ProviderTerminationOutcome>;
|
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
warnDrainSnapshotFailure,
|
|
34
34
|
withFirstPartyTools,
|
|
35
35
|
workflowIdForSession
|
|
36
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-K3BPAI4R.js";
|
|
37
37
|
import {
|
|
38
38
|
inspectOpenSandboxKubernetesInventory,
|
|
39
39
|
recordCreditBalanceGauges,
|
|
@@ -58,24 +58,43 @@ import "./chunk-PZ5AY32C.js";
|
|
|
58
58
|
|
|
59
59
|
// src/activities/knowledge-indexing.ts
|
|
60
60
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
61
|
+
import { documentEmbeddingCostMicros, paidDocumentEmbedding } from "@opengeni/core";
|
|
61
62
|
import {
|
|
63
|
+
applyCreditDebitAfterUse,
|
|
62
64
|
appendKnowledgeIndexChunks,
|
|
63
65
|
claimKnowledgeIndexJobs,
|
|
64
66
|
completeKnowledgeIndexJob,
|
|
65
67
|
continueKnowledgeIndexJob,
|
|
66
68
|
deferKnowledgeIndexJob,
|
|
69
|
+
freezeKnowledgeIndexBillingMode,
|
|
67
70
|
getBillingBalance,
|
|
71
|
+
guardPaidKnowledgeIndexPublication,
|
|
72
|
+
knowledgeIndexBillingActivationTime,
|
|
68
73
|
readKnowledgeIndexSource,
|
|
69
74
|
recordUsageEvent,
|
|
70
75
|
sumUsageQuantity,
|
|
71
|
-
withWorkspaceUsageLock
|
|
76
|
+
withWorkspaceUsageLock,
|
|
77
|
+
waitKnowledgeIndexForFunding
|
|
72
78
|
} from "@opengeni/db";
|
|
73
79
|
function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
80
|
+
let activationTime;
|
|
74
81
|
return {
|
|
75
82
|
indexKnowledge: async () => {
|
|
76
83
|
const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
|
|
77
84
|
if (!resolveDocumentServices) return result;
|
|
78
85
|
const { db, settings, observability } = await services();
|
|
86
|
+
let policyActivatedAt;
|
|
87
|
+
if (paidDocumentEmbedding(settings)) {
|
|
88
|
+
if (!settings.documentEmbeddingCreditsActivatedAt)
|
|
89
|
+
throw new Error("paid Knowledge embedding requires an activation cutoff");
|
|
90
|
+
policyActivatedAt = new Date(settings.documentEmbeddingCreditsActivatedAt);
|
|
91
|
+
} else {
|
|
92
|
+
activationTime ??= knowledgeIndexBillingActivationTime(db).catch((error) => {
|
|
93
|
+
activationTime = void 0;
|
|
94
|
+
throw error;
|
|
95
|
+
});
|
|
96
|
+
policyActivatedAt = await activationTime;
|
|
97
|
+
}
|
|
79
98
|
const { embedder } = await resolveDocumentServices();
|
|
80
99
|
const { knowledgeIndexChunks } = await import("@opengeni/documents");
|
|
81
100
|
const claims = await claimKnowledgeIndexJobs(db, {
|
|
@@ -107,9 +126,32 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
107
126
|
chunks.push(chunk);
|
|
108
127
|
}
|
|
109
128
|
if (chunks.length) {
|
|
110
|
-
|
|
129
|
+
const frozenPolicy = await freezeKnowledgeIndexBillingMode(
|
|
130
|
+
lockedDb,
|
|
131
|
+
claim,
|
|
132
|
+
// Deterministic embeddings incur no provider charge; a valid
|
|
133
|
+
// credits-mode config with zero tariff must still index them.
|
|
134
|
+
// OpenAI shadow keeps its price snapshot for internal estimates.
|
|
135
|
+
settings.documentEmbeddingProvider === "openai" ? settings.documentEmbeddingBillingMode ?? "usage_only" : "usage_only",
|
|
136
|
+
policyActivatedAt,
|
|
137
|
+
settings.documentEmbeddingRateMicrosPerMillionBytes ?? 0
|
|
138
|
+
);
|
|
139
|
+
if (frozenPolicy.mode === "awaiting_review") {
|
|
140
|
+
result.deferred++;
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (frozenPolicy.mode === "obsolete") {
|
|
144
|
+
result.unavailable++;
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const paid = frozenPolicy.mode === "credits" && paidDocumentEmbedding(settings);
|
|
148
|
+
if (paid && current.nextIndex === 0) {
|
|
111
149
|
const balance = await getBillingBalance(lockedDb, claim.accountId);
|
|
112
|
-
if (balance.balanceMicros <= 0)
|
|
150
|
+
if (balance.balanceMicros <= 0) {
|
|
151
|
+
await waitKnowledgeIndexForFunding(lockedDb, claim);
|
|
152
|
+
result.deferred++;
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
113
155
|
}
|
|
114
156
|
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
115
157
|
const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
@@ -124,9 +166,22 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
124
166
|
throw new Error("monthly document indexing limit reached");
|
|
125
167
|
}
|
|
126
168
|
}
|
|
127
|
-
const
|
|
169
|
+
const inputs = chunks.map((chunk) => chunk.embeddingInput);
|
|
170
|
+
const bytes = inputs.reduce(
|
|
171
|
+
(sum, input) => sum + Buffer.byteLength(input, "utf8"),
|
|
172
|
+
0
|
|
173
|
+
);
|
|
174
|
+
const vectors = await embedder.embedMany(inputs);
|
|
128
175
|
if (vectors.length !== chunks.length)
|
|
129
176
|
throw new Error("Incomplete Knowledge embeddings");
|
|
177
|
+
if (paid) {
|
|
178
|
+
const publication = await guardPaidKnowledgeIndexPublication(lockedDb, claim);
|
|
179
|
+
if (publication !== "published") {
|
|
180
|
+
if (publication === "obsolete") result.unavailable++;
|
|
181
|
+
else result.deferred++;
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
130
185
|
const appended = await appendKnowledgeIndexChunks(
|
|
131
186
|
lockedDb,
|
|
132
187
|
claim,
|
|
@@ -147,6 +202,61 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
147
202
|
sourceResourceId: claim.revisionId,
|
|
148
203
|
idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
149
204
|
});
|
|
205
|
+
await recordUsageEvent(lockedDb, {
|
|
206
|
+
accountId: claim.accountId,
|
|
207
|
+
workspaceId: current.billingWorkspaceId,
|
|
208
|
+
eventType: "document.embedding_bytes",
|
|
209
|
+
quantity: bytes,
|
|
210
|
+
unit: "byte",
|
|
211
|
+
sourceResourceType: "knowledge_revision",
|
|
212
|
+
sourceResourceId: claim.revisionId,
|
|
213
|
+
idempotencyKey: `knowledge.embedding_bytes:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
214
|
+
});
|
|
215
|
+
if (frozenPolicy.mode === "shadow" && frozenPolicy.rateMicrosPerMillionBytes > 0) {
|
|
216
|
+
const estimate = documentEmbeddingCostMicros(
|
|
217
|
+
{
|
|
218
|
+
...settings,
|
|
219
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
220
|
+
},
|
|
221
|
+
bytes
|
|
222
|
+
);
|
|
223
|
+
if (estimate > 0)
|
|
224
|
+
await recordUsageEvent(lockedDb, {
|
|
225
|
+
accountId: claim.accountId,
|
|
226
|
+
workspaceId: current.billingWorkspaceId,
|
|
227
|
+
eventType: "document.embedding_shadow_estimate",
|
|
228
|
+
quantity: estimate,
|
|
229
|
+
unit: "micro_usd",
|
|
230
|
+
sourceResourceType: "knowledge_revision",
|
|
231
|
+
sourceResourceId: claim.revisionId,
|
|
232
|
+
idempotencyKey: `knowledge.embedding_shadow:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
if (paid) {
|
|
236
|
+
const cost = documentEmbeddingCostMicros(
|
|
237
|
+
{
|
|
238
|
+
...settings,
|
|
239
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
240
|
+
},
|
|
241
|
+
bytes
|
|
242
|
+
);
|
|
243
|
+
if (cost > 0)
|
|
244
|
+
await applyCreditDebitAfterUse(lockedDb, {
|
|
245
|
+
accountId: claim.accountId,
|
|
246
|
+
workspaceId: current.billingWorkspaceId,
|
|
247
|
+
type: "document_embedding_debit",
|
|
248
|
+
amountMicros: cost,
|
|
249
|
+
sourceType: "knowledge_revision",
|
|
250
|
+
sourceId: claim.revisionId,
|
|
251
|
+
idempotencyKey: `knowledge.embedding:${claim.revisionId}:${claim.generation}:${current.nextIndex}`,
|
|
252
|
+
metadata: {
|
|
253
|
+
model: claim.model,
|
|
254
|
+
bytes,
|
|
255
|
+
chunks: chunks.length,
|
|
256
|
+
rateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
}
|
|
150
260
|
}
|
|
151
261
|
if (more) {
|
|
152
262
|
await continueKnowledgeIndexJob(lockedDb, claim);
|
|
@@ -2241,10 +2351,10 @@ import {
|
|
|
2241
2351
|
countSandboxLeasesByLiveness,
|
|
2242
2352
|
deferRetainedProcessReconciliation,
|
|
2243
2353
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
2244
|
-
getBillingBalance as getBillingBalance3,
|
|
2245
2354
|
listCreditBalancesByAccount,
|
|
2246
2355
|
listLegacyModalCheckpointSlots,
|
|
2247
2356
|
listLiveModalSandboxLeaseAttributions,
|
|
2357
|
+
markWarmBillingStopCutoff,
|
|
2248
2358
|
listMeterableWarmLeases,
|
|
2249
2359
|
listSandboxViewerForceDrainWorkspaceIds,
|
|
2250
2360
|
markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
|
|
@@ -2396,7 +2506,10 @@ function sandboxDrainActivityAttempt() {
|
|
|
2396
2506
|
}
|
|
2397
2507
|
}
|
|
2398
2508
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
2399
|
-
const terminateBox = options.terminateBox
|
|
2509
|
+
const terminateBox = (options.terminateBox ? async (...args) => {
|
|
2510
|
+
await args[10]?.();
|
|
2511
|
+
return options.terminateBox(...args);
|
|
2512
|
+
} : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop) => await terminateProviderBox(
|
|
2400
2513
|
settings,
|
|
2401
2514
|
lease,
|
|
2402
2515
|
observability,
|
|
@@ -2408,7 +2521,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2408
2521
|
capturePolicy,
|
|
2409
2522
|
diskBackedArchives,
|
|
2410
2523
|
releaseFailedCapture,
|
|
2411
|
-
workspaceId
|
|
2524
|
+
workspaceId,
|
|
2525
|
+
beforeProviderStop
|
|
2412
2526
|
));
|
|
2413
2527
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
2414
2528
|
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
@@ -2598,7 +2712,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2598
2712
|
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
2599
2713
|
metered = meterResult.accrued;
|
|
2600
2714
|
const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
|
|
2601
|
-
if (settings.
|
|
2715
|
+
if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
|
|
2602
2716
|
for (const workspaceId of meterResult.workspaceIds) {
|
|
2603
2717
|
forceDrainWorkspaceIds.add(workspaceId);
|
|
2604
2718
|
}
|
|
@@ -3209,6 +3323,16 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3209
3323
|
};
|
|
3210
3324
|
}
|
|
3211
3325
|
}
|
|
3326
|
+
if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
|
|
3327
|
+
try {
|
|
3328
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
3329
|
+
} catch (error) {
|
|
3330
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
3331
|
+
processId: process.id,
|
|
3332
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3333
|
+
});
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3212
3336
|
if (observation === null) {
|
|
3213
3337
|
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
3214
3338
|
() => null
|
|
@@ -3263,11 +3387,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3263
3387
|
);
|
|
3264
3388
|
supervisionMetric("cancellation_intent");
|
|
3265
3389
|
}
|
|
3266
|
-
|
|
3390
|
+
const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
|
|
3391
|
+
const probeOutcome = await probe(
|
|
3267
3392
|
settings,
|
|
3268
3393
|
lease,
|
|
3269
3394
|
process,
|
|
3270
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
3395
|
+
claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
|
|
3271
3396
|
async (result, chunkId, stream, streamFidelity) => {
|
|
3272
3397
|
if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
|
|
3273
3398
|
return;
|
|
@@ -3285,7 +3410,14 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3285
3410
|
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
3286
3411
|
},
|
|
3287
3412
|
commandPersistence
|
|
3413
|
+
).then(
|
|
3414
|
+
(value) => ({ ok: true, value }),
|
|
3415
|
+
(error) => ({ ok: false, error })
|
|
3288
3416
|
);
|
|
3417
|
+
if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
3418
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
3419
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
3420
|
+
observation = probeOutcome.value;
|
|
3289
3421
|
if (supervised) {
|
|
3290
3422
|
supervisionMetric(
|
|
3291
3423
|
await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
|
|
@@ -3489,7 +3621,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3489
3621
|
if (pending) {
|
|
3490
3622
|
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
3491
3623
|
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
3492
|
-
|
|
3624
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
3625
|
+
pending.result,
|
|
3626
|
+
process.providerSessionId
|
|
3627
|
+
);
|
|
3628
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
3493
3629
|
}
|
|
3494
3630
|
const envelopeBackend = lease.resumeState.backendId;
|
|
3495
3631
|
if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
|
|
@@ -3570,6 +3706,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3570
3706
|
if (typeof session.writeStdin !== "function") {
|
|
3571
3707
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
3572
3708
|
}
|
|
3709
|
+
let legacyPty = false;
|
|
3573
3710
|
if (process.providerBackend === "modal") {
|
|
3574
3711
|
let liveBinding;
|
|
3575
3712
|
try {
|
|
@@ -3596,6 +3733,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3596
3733
|
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
3597
3734
|
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
3598
3735
|
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
3736
|
+
legacyPty = command.pty === true;
|
|
3599
3737
|
}
|
|
3600
3738
|
const capturePage = async (value) => {
|
|
3601
3739
|
if (!captureOutput) {
|
|
@@ -3627,7 +3765,9 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3627
3765
|
result = await withRetainedProcessProbeTimeout(
|
|
3628
3766
|
session.writeStdin({
|
|
3629
3767
|
sessionId: process.providerSessionId,
|
|
3630
|
-
|
|
3768
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
3769
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
3770
|
+
chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
|
|
3631
3771
|
yieldTimeMs: 1e3,
|
|
3632
3772
|
maxOutputTokens: 2e3
|
|
3633
3773
|
})
|
|
@@ -3653,7 +3793,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3653
3793
|
}
|
|
3654
3794
|
await capturePage(result);
|
|
3655
3795
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
|
|
3656
|
-
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3796
|
+
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3657
3797
|
try {
|
|
3658
3798
|
const interrupted = await withRetainedProcessProbeTimeout(
|
|
3659
3799
|
session.writeStdin({
|
|
@@ -3730,19 +3870,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3730
3870
|
}
|
|
3731
3871
|
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
3732
3872
|
try {
|
|
3733
|
-
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3873
|
+
const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3734
3874
|
const result = await accrueWarmSeconds(db, {
|
|
3735
3875
|
accountId: lease.accountId,
|
|
3736
3876
|
workspaceId: lease.workspaceId,
|
|
3737
3877
|
sandboxGroupId: lease.sandboxGroupId,
|
|
3738
3878
|
expectedEpoch: lease.leaseEpoch,
|
|
3739
3879
|
warmRateMicrosPerSecond: rate,
|
|
3880
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
3740
3881
|
subjectId: lease.sandboxGroupId
|
|
3741
3882
|
});
|
|
3742
3883
|
if (result.accrued) {
|
|
3743
3884
|
accrued += 1;
|
|
3744
3885
|
}
|
|
3745
|
-
|
|
3886
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
3887
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3888
|
+
}
|
|
3746
3889
|
} catch (error) {
|
|
3747
3890
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3748
3891
|
workspaceId: lease.workspaceId,
|
|
@@ -3841,7 +3984,7 @@ async function refreshSandboxInventoryGauge(observability, domain, diagnosticNam
|
|
|
3841
3984
|
}
|
|
3842
3985
|
}
|
|
3843
3986
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3844
|
-
const enforceBalance =
|
|
3987
|
+
const enforceBalance = false;
|
|
3845
3988
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
3846
3989
|
let forceDrained = 0;
|
|
3847
3990
|
await forEachWithConcurrency(
|
|
@@ -3849,11 +3992,8 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3849
3992
|
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
3850
3993
|
async (workspaceId) => {
|
|
3851
3994
|
try {
|
|
3852
|
-
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
3853
|
-
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
3854
3995
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
3855
3996
|
workspaceId,
|
|
3856
|
-
balanceMicros: balance.balanceMicros,
|
|
3857
3997
|
enforceBalance,
|
|
3858
3998
|
maxWarmSecondsPerWorkspace: cap,
|
|
3859
3999
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
@@ -4239,6 +4379,17 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4239
4379
|
})) {
|
|
4240
4380
|
return false;
|
|
4241
4381
|
}
|
|
4382
|
+
let stopCutoffMarked = false;
|
|
4383
|
+
const beforeProviderStop = async () => {
|
|
4384
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
4385
|
+
accountId,
|
|
4386
|
+
workspaceId: row.workspaceId,
|
|
4387
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4388
|
+
expectedEpoch: row.leaseEpoch
|
|
4389
|
+
});
|
|
4390
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
4391
|
+
stopCutoffMarked = true;
|
|
4392
|
+
};
|
|
4242
4393
|
const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
|
|
4243
4394
|
settings,
|
|
4244
4395
|
lease,
|
|
@@ -4259,12 +4410,35 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4259
4410
|
expectedInstanceId: lease.instanceId
|
|
4260
4411
|
});
|
|
4261
4412
|
},
|
|
4262
|
-
row.workspaceId
|
|
4413
|
+
row.workspaceId,
|
|
4414
|
+
beforeProviderStop
|
|
4263
4415
|
);
|
|
4264
4416
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
4265
4417
|
if (!terminated) {
|
|
4266
4418
|
return false;
|
|
4267
4419
|
}
|
|
4420
|
+
const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
|
|
4421
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
4422
|
+
if (managedProvider && lease.instanceId) {
|
|
4423
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
4424
|
+
}
|
|
4425
|
+
await beforeProviderStop();
|
|
4426
|
+
}
|
|
4427
|
+
const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
4428
|
+
if (!providerMissingDuringStop) {
|
|
4429
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
4430
|
+
accountId,
|
|
4431
|
+
workspaceId: row.workspaceId,
|
|
4432
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4433
|
+
expectedEpoch: row.leaseEpoch,
|
|
4434
|
+
warmRateMicrosPerSecond: finalRate,
|
|
4435
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
4436
|
+
finalDrain: true
|
|
4437
|
+
});
|
|
4438
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
4439
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
4268
4442
|
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
4269
4443
|
const { wentCold } = await confirmDrainCold(db, {
|
|
4270
4444
|
accountId,
|
|
@@ -4315,7 +4489,7 @@ function providerDrainLogIdentity(lease, backend) {
|
|
|
4315
4489
|
instanceId: lease.instanceId
|
|
4316
4490
|
};
|
|
4317
4491
|
}
|
|
4318
|
-
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId) {
|
|
4492
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop) {
|
|
4319
4493
|
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
4320
4494
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
4321
4495
|
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
@@ -4379,6 +4553,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4379
4553
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
4380
4554
|
}
|
|
4381
4555
|
try {
|
|
4556
|
+
await beforeProviderStop?.();
|
|
4382
4557
|
await terminateModalById(settings, lease.instanceId);
|
|
4383
4558
|
} catch (error) {
|
|
4384
4559
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
@@ -4548,6 +4723,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4548
4723
|
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
4549
4724
|
}
|
|
4550
4725
|
prepareProviderForTeardownAfterCapture(backend, session);
|
|
4726
|
+
await beforeProviderStop?.();
|
|
4551
4727
|
await terminateManagedSandboxSession2(client, sessionState, session);
|
|
4552
4728
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
4553
4729
|
} catch (error) {
|
|
@@ -7498,4 +7674,4 @@ export {
|
|
|
7498
7674
|
createControlActivities,
|
|
7499
7675
|
createControlActivitiesFromServices
|
|
7500
7676
|
};
|
|
7501
|
-
//# sourceMappingURL=activities-control-
|
|
7677
|
+
//# sourceMappingURL=activities-control-UF2CR2LB.js.map
|