@opengeni/worker-bundle 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/failure-settlement.d.ts +1 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/sandbox-lease.d.ts +3 -4
- package/dist/{activities-control-QA6OLRQY.js → activities-control-IVTL723D.js} +204 -23
- package/dist/activities-control-IVTL723D.js.map +1 -0
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-EZEDZTEZ.js} +108 -33
- package/dist/activities-turn-EZEDZTEZ.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-JMO5ZUW3.js} +14 -7
- package/dist/{chunk-M3NVMF2V.js.map → chunk-JMO5ZUW3.js.map} +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +4 -1
- package/package.json +15 -15
- package/src/activities/agent-turn/errors.ts +8 -3
- package/src/activities/agent-turn/failure-settlement.ts +40 -8
- package/src/activities/agent-turn/history.ts +14 -1
- package/src/activities/agent-turn/media-artifacts.ts +2 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +25 -9
- package/src/activities/agent-turn/stream-attempt.ts +13 -2
- package/src/activities/knowledge-indexing.ts +133 -6
- package/src/activities/retained-screenshots.ts +82 -11
- package/src/activities/sandbox-lease.ts +141 -21
- package/src/sandbox-resume.ts +6 -0
- package/src/workflows/activities.ts +14 -1
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Settings } from "@opengeni/config";
|
|
2
2
|
import type { TurnActivityServices as ActivityServices, RunAgentTurnInput, RunAgentTurnResult } from "../types.js";
|
|
3
3
|
import { createTurnCredentialLeases } from "./credential-leases.js";
|
|
4
4
|
import { createTurnHistorySink } from "./history-sink.js";
|
|
@@ -49,6 +49,8 @@ export declare function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
49
49
|
} | null;
|
|
50
50
|
/** Tools whose intentional visual result must survive inline-media sanitization. */
|
|
51
51
|
export declare function toolCallProducesRetainableSessionImage(name: string | null): boolean;
|
|
52
|
+
/** Browser interactions can carry an image block, including opt-in observe frames. */
|
|
53
|
+
export declare function retainableBrowserScreenshotToolCall(name: string | null): boolean;
|
|
52
54
|
export declare function completedToolCallFromSdkEvent(event: unknown): {
|
|
53
55
|
callId: string;
|
|
54
56
|
resultItem: Record<string, unknown>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type CodemodeTokenWriterSession, type RetainableSessionImageOutputHook, type TurnToolCancellationFence } from "@opengeni/runtime";
|
|
2
2
|
import type { Settings } from "@opengeni/config";
|
|
3
|
+
import type { RetainedSessionScreenshotKind } from "@opengeni/contracts";
|
|
3
4
|
import type { ResumedTurnSandbox } from "../../sandbox-resume.js";
|
|
4
5
|
import type { SharedActivityServices } from "../types.js";
|
|
5
6
|
import { type GeneratedImageOutput, type GeneratedImageReceipt } from "../generated-images.js";
|
|
@@ -35,7 +36,7 @@ export declare class TurnMediaArtifacts {
|
|
|
35
36
|
readonly retainedScreenshotReceiptsByCallId: Map<string, {
|
|
36
37
|
available: true;
|
|
37
38
|
artifactId: string;
|
|
38
|
-
kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
39
|
+
kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
39
40
|
contentType: string;
|
|
40
41
|
originalBytes: number;
|
|
41
42
|
sha256: string;
|
|
@@ -68,7 +69,7 @@ export declare class TurnMediaArtifacts {
|
|
|
68
69
|
artifact: {
|
|
69
70
|
available: true;
|
|
70
71
|
artifactId: string;
|
|
71
|
-
kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
72
|
+
kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
72
73
|
contentType: string;
|
|
73
74
|
originalBytes: number;
|
|
74
75
|
sha256: string;
|
|
@@ -98,7 +99,7 @@ export declare class TurnMediaArtifacts {
|
|
|
98
99
|
artifact: {
|
|
99
100
|
available: true;
|
|
100
101
|
artifactId: string;
|
|
101
|
-
kind: "assistant_completion" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
102
|
+
kind: "assistant_completion" | "browser_screenshot" | "computer_screenshot" | "event_media" | "file" | "generated_image" | "generated_video" | "internal_update" | "tool_result";
|
|
102
103
|
contentType: string;
|
|
103
104
|
originalBytes: number;
|
|
104
105
|
sha256: string;
|
|
@@ -128,6 +129,7 @@ export declare class TurnMediaArtifacts {
|
|
|
128
129
|
sandboxFileDownloadBackend: Settings["sandboxBackend"];
|
|
129
130
|
nativeImageGenerationRetention: NativeImageGenerationRetention | null;
|
|
130
131
|
readonly retainedSessionImageCallIds: Set<string>;
|
|
132
|
+
readonly retainedSessionImageKindsByCallId: Map<string, RetainedSessionScreenshotKind>;
|
|
131
133
|
constructor(deps: TurnMediaArtifactDeps);
|
|
132
134
|
rememberGeneratedImageCreatedThisTurn: (receipt: GeneratedImageReceipt) => void;
|
|
133
135
|
private prepareGeneratedImageDownload;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type RetainedArtifactMetadata, type RetainedOutputUnavailableReason } from "@opengeni/contracts";
|
|
1
|
+
import { type RetainedArtifactMetadata, type RetainedOutputUnavailableReason, type RetainedSessionScreenshotKind } from "@opengeni/contracts";
|
|
2
2
|
import { type Database } from "@opengeni/db";
|
|
3
3
|
import { type ObjectStorage } from "@opengeni/storage";
|
|
4
4
|
export type TypedScreenshotToolOutput = {
|
|
@@ -68,6 +68,7 @@ export declare function retainedScreenshotIdentity(input: {
|
|
|
68
68
|
attemptId: string;
|
|
69
69
|
toolCallId: string;
|
|
70
70
|
toolOutputId: string;
|
|
71
|
+
kind?: RetainedSessionScreenshotKind | undefined;
|
|
71
72
|
}): {
|
|
72
73
|
artifactId: string;
|
|
73
74
|
settlementKey: string;
|
|
@@ -80,8 +81,9 @@ export declare function unavailableRetainedSessionImage(input: {
|
|
|
80
81
|
toolCallId: string;
|
|
81
82
|
toolOutputId: string;
|
|
82
83
|
reason: RetainedOutputUnavailableReason;
|
|
84
|
+
kind?: RetainedSessionScreenshotKind | undefined;
|
|
83
85
|
}): RetainedArtifactMetadata;
|
|
84
|
-
export declare function
|
|
86
|
+
export declare function retainSessionScreenshot(input: {
|
|
85
87
|
db: Database;
|
|
86
88
|
objectStorage: ObjectStorage | null;
|
|
87
89
|
accountId: string;
|
|
@@ -90,10 +92,13 @@ export declare function retainComputerScreenshot(input: {
|
|
|
90
92
|
turnId: string;
|
|
91
93
|
attemptId: string;
|
|
92
94
|
output: TypedScreenshotToolOutput;
|
|
95
|
+
kind?: RetainedSessionScreenshotKind | undefined;
|
|
93
96
|
now?: Date;
|
|
94
97
|
retentionMs?: number;
|
|
95
98
|
workspaceQuotaBytes?: number;
|
|
96
99
|
}): Promise<RetainedArtifactMetadata>;
|
|
100
|
+
/** Existing computer callers keep their stable entry point. */
|
|
101
|
+
export declare const retainComputerScreenshot: typeof retainSessionScreenshot;
|
|
97
102
|
/** Replace only new screenshot bytes with a compact receipt before first persistence. */
|
|
98
103
|
export declare function compactRetainedScreenshotHistory(history: Array<Record<string, unknown>>, receiptsByCallId: ReadonlyMap<string, RetainedArtifactMetadata>): Array<Record<string, unknown>>;
|
|
99
104
|
/** Collect durable screenshot receipts before their provider-only expansion. */
|
|
@@ -20,8 +20,7 @@ export type ReapSandboxLeasesResult = {
|
|
|
20
20
|
skipped: number;
|
|
21
21
|
/** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
|
|
22
22
|
metered: number;
|
|
23
|
-
/**
|
|
24
|
-
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
23
|
+
/** Holderless boxes accelerated to draining by the optional warm cap. */
|
|
25
24
|
forceDrained: number;
|
|
26
25
|
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
27
26
|
modalOrphansTerminated: number;
|
|
@@ -60,7 +59,7 @@ export type DrainCaptureDisposition = "capture_required" | "archive_published";
|
|
|
60
59
|
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
61
60
|
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
62
61
|
* provider disappearance before capture. */
|
|
63
|
-
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string) => Promise<boolean | ProviderTerminationOutcome>;
|
|
62
|
+
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, capturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string, beforeProviderStop?: () => Promise<void>) => Promise<boolean | ProviderTerminationOutcome>;
|
|
64
63
|
export type SweepModalOrphansFn = (settings: ActivityServices["settings"], db: ActivityServices["db"], observability: ActivityServices["observability"]) => Promise<number>;
|
|
65
64
|
export type InspectOpenSandboxKubernetesInventoryFn = (input: {
|
|
66
65
|
namespace: string;
|
|
@@ -208,4 +207,4 @@ export declare function sandboxDrainReason(lease: Pick<LeaseSnapshot, "rotationR
|
|
|
208
207
|
* draining until its adapter can prove termination. A truly instance-less or
|
|
209
208
|
* `none`-backed row is a no-op.
|
|
210
209
|
*/
|
|
211
|
-
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string): Promise<ProviderTerminationOutcome>;
|
|
210
|
+
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn, terminateModalById?: typeof terminateModalSandboxById, providerCaptureRequestId?: string, captureDisposition?: DrainCaptureDisposition, claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null, diskBackedArchives?: boolean, releaseFailedCapture?: () => Promise<void>, workspaceId?: string, beforeProviderStop?: () => Promise<void>): Promise<ProviderTerminationOutcome>;
|
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
warnDrainSnapshotFailure,
|
|
34
34
|
withFirstPartyTools,
|
|
35
35
|
workflowIdForSession
|
|
36
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-JMO5ZUW3.js";
|
|
37
37
|
import {
|
|
38
38
|
inspectOpenSandboxKubernetesInventory,
|
|
39
39
|
recordCreditBalanceGauges,
|
|
@@ -58,24 +58,43 @@ import "./chunk-PZ5AY32C.js";
|
|
|
58
58
|
|
|
59
59
|
// src/activities/knowledge-indexing.ts
|
|
60
60
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
61
|
+
import { documentEmbeddingCostMicros, paidDocumentEmbedding } from "@opengeni/core";
|
|
61
62
|
import {
|
|
63
|
+
applyCreditDebitAfterUse,
|
|
62
64
|
appendKnowledgeIndexChunks,
|
|
63
65
|
claimKnowledgeIndexJobs,
|
|
64
66
|
completeKnowledgeIndexJob,
|
|
65
67
|
continueKnowledgeIndexJob,
|
|
66
68
|
deferKnowledgeIndexJob,
|
|
69
|
+
freezeKnowledgeIndexBillingMode,
|
|
67
70
|
getBillingBalance,
|
|
71
|
+
guardPaidKnowledgeIndexPublication,
|
|
72
|
+
knowledgeIndexBillingActivationTime,
|
|
68
73
|
readKnowledgeIndexSource,
|
|
69
74
|
recordUsageEvent,
|
|
70
75
|
sumUsageQuantity,
|
|
71
|
-
withWorkspaceUsageLock
|
|
76
|
+
withWorkspaceUsageLock,
|
|
77
|
+
waitKnowledgeIndexForFunding
|
|
72
78
|
} from "@opengeni/db";
|
|
73
79
|
function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
80
|
+
let activationTime;
|
|
74
81
|
return {
|
|
75
82
|
indexKnowledge: async () => {
|
|
76
83
|
const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
|
|
77
84
|
if (!resolveDocumentServices) return result;
|
|
78
85
|
const { db, settings, observability } = await services();
|
|
86
|
+
let policyActivatedAt;
|
|
87
|
+
if (paidDocumentEmbedding(settings)) {
|
|
88
|
+
if (!settings.documentEmbeddingCreditsActivatedAt)
|
|
89
|
+
throw new Error("paid Knowledge embedding requires an activation cutoff");
|
|
90
|
+
policyActivatedAt = new Date(settings.documentEmbeddingCreditsActivatedAt);
|
|
91
|
+
} else {
|
|
92
|
+
activationTime ??= knowledgeIndexBillingActivationTime(db).catch((error) => {
|
|
93
|
+
activationTime = void 0;
|
|
94
|
+
throw error;
|
|
95
|
+
});
|
|
96
|
+
policyActivatedAt = await activationTime;
|
|
97
|
+
}
|
|
79
98
|
const { embedder } = await resolveDocumentServices();
|
|
80
99
|
const { knowledgeIndexChunks } = await import("@opengeni/documents");
|
|
81
100
|
const claims = await claimKnowledgeIndexJobs(db, {
|
|
@@ -107,9 +126,37 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
107
126
|
chunks.push(chunk);
|
|
108
127
|
}
|
|
109
128
|
if (chunks.length) {
|
|
110
|
-
|
|
129
|
+
const frozenPolicy = await freezeKnowledgeIndexBillingMode(
|
|
130
|
+
lockedDb,
|
|
131
|
+
claim,
|
|
132
|
+
// Deterministic embeddings incur no provider charge; a valid
|
|
133
|
+
// credits-mode config with zero tariff must still index them.
|
|
134
|
+
// OpenAI shadow keeps its price snapshot for internal estimates.
|
|
135
|
+
settings.documentEmbeddingProvider === "openai" ? settings.documentEmbeddingBillingMode ?? "usage_only" : "usage_only",
|
|
136
|
+
policyActivatedAt,
|
|
137
|
+
settings.documentEmbeddingRateMicrosPerMillionBytes ?? 0
|
|
138
|
+
);
|
|
139
|
+
if (frozenPolicy.mode === "awaiting_review") {
|
|
140
|
+
result.deferred++;
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (frozenPolicy.mode === "obsolete") {
|
|
144
|
+
result.unavailable++;
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (frozenPolicy.mode === "credits" && !paidDocumentEmbedding(settings)) {
|
|
148
|
+
await deferKnowledgeIndexJob(lockedDb, claim);
|
|
149
|
+
result.deferred++;
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
const paid = frozenPolicy.mode === "credits";
|
|
153
|
+
if (paid && current.nextIndex === 0) {
|
|
111
154
|
const balance = await getBillingBalance(lockedDb, claim.accountId);
|
|
112
|
-
if (balance.balanceMicros <= 0)
|
|
155
|
+
if (balance.balanceMicros <= 0) {
|
|
156
|
+
await waitKnowledgeIndexForFunding(lockedDb, claim);
|
|
157
|
+
result.deferred++;
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
113
160
|
}
|
|
114
161
|
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
115
162
|
const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
@@ -124,9 +171,22 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
124
171
|
throw new Error("monthly document indexing limit reached");
|
|
125
172
|
}
|
|
126
173
|
}
|
|
127
|
-
const
|
|
174
|
+
const inputs = chunks.map((chunk) => chunk.embeddingInput);
|
|
175
|
+
const bytes = inputs.reduce(
|
|
176
|
+
(sum, input) => sum + Buffer.byteLength(input, "utf8"),
|
|
177
|
+
0
|
|
178
|
+
);
|
|
179
|
+
const vectors = await embedder.embedMany(inputs);
|
|
128
180
|
if (vectors.length !== chunks.length)
|
|
129
181
|
throw new Error("Incomplete Knowledge embeddings");
|
|
182
|
+
if (paid) {
|
|
183
|
+
const publication = await guardPaidKnowledgeIndexPublication(lockedDb, claim);
|
|
184
|
+
if (publication !== "published") {
|
|
185
|
+
if (publication === "obsolete") result.unavailable++;
|
|
186
|
+
else result.deferred++;
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
130
190
|
const appended = await appendKnowledgeIndexChunks(
|
|
131
191
|
lockedDb,
|
|
132
192
|
claim,
|
|
@@ -147,6 +207,61 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
147
207
|
sourceResourceId: claim.revisionId,
|
|
148
208
|
idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
149
209
|
});
|
|
210
|
+
await recordUsageEvent(lockedDb, {
|
|
211
|
+
accountId: claim.accountId,
|
|
212
|
+
workspaceId: current.billingWorkspaceId,
|
|
213
|
+
eventType: "document.embedding_bytes",
|
|
214
|
+
quantity: bytes,
|
|
215
|
+
unit: "byte",
|
|
216
|
+
sourceResourceType: "knowledge_revision",
|
|
217
|
+
sourceResourceId: claim.revisionId,
|
|
218
|
+
idempotencyKey: `knowledge.embedding_bytes:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
219
|
+
});
|
|
220
|
+
if (frozenPolicy.mode === "shadow" && frozenPolicy.rateMicrosPerMillionBytes > 0) {
|
|
221
|
+
const estimate = documentEmbeddingCostMicros(
|
|
222
|
+
{
|
|
223
|
+
...settings,
|
|
224
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
225
|
+
},
|
|
226
|
+
bytes
|
|
227
|
+
);
|
|
228
|
+
if (estimate > 0)
|
|
229
|
+
await recordUsageEvent(lockedDb, {
|
|
230
|
+
accountId: claim.accountId,
|
|
231
|
+
workspaceId: current.billingWorkspaceId,
|
|
232
|
+
eventType: "document.embedding_shadow_estimate",
|
|
233
|
+
quantity: estimate,
|
|
234
|
+
unit: "micro_usd",
|
|
235
|
+
sourceResourceType: "knowledge_revision",
|
|
236
|
+
sourceResourceId: claim.revisionId,
|
|
237
|
+
idempotencyKey: `knowledge.embedding_shadow:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
if (paid) {
|
|
241
|
+
const cost = documentEmbeddingCostMicros(
|
|
242
|
+
{
|
|
243
|
+
...settings,
|
|
244
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
245
|
+
},
|
|
246
|
+
bytes
|
|
247
|
+
);
|
|
248
|
+
if (cost > 0)
|
|
249
|
+
await applyCreditDebitAfterUse(lockedDb, {
|
|
250
|
+
accountId: claim.accountId,
|
|
251
|
+
workspaceId: current.billingWorkspaceId,
|
|
252
|
+
type: "document_embedding_debit",
|
|
253
|
+
amountMicros: cost,
|
|
254
|
+
sourceType: "knowledge_revision",
|
|
255
|
+
sourceId: claim.revisionId,
|
|
256
|
+
idempotencyKey: `knowledge.embedding:${claim.revisionId}:${claim.generation}:${current.nextIndex}`,
|
|
257
|
+
metadata: {
|
|
258
|
+
model: claim.model,
|
|
259
|
+
bytes,
|
|
260
|
+
chunks: chunks.length,
|
|
261
|
+
rateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
}
|
|
150
265
|
}
|
|
151
266
|
if (more) {
|
|
152
267
|
await continueKnowledgeIndexJob(lockedDb, claim);
|
|
@@ -2241,10 +2356,10 @@ import {
|
|
|
2241
2356
|
countSandboxLeasesByLiveness,
|
|
2242
2357
|
deferRetainedProcessReconciliation,
|
|
2243
2358
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
2244
|
-
getBillingBalance as getBillingBalance3,
|
|
2245
2359
|
listCreditBalancesByAccount,
|
|
2246
2360
|
listLegacyModalCheckpointSlots,
|
|
2247
2361
|
listLiveModalSandboxLeaseAttributions,
|
|
2362
|
+
markWarmBillingStopCutoff,
|
|
2248
2363
|
listMeterableWarmLeases,
|
|
2249
2364
|
listSandboxViewerForceDrainWorkspaceIds,
|
|
2250
2365
|
markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
|
|
@@ -2396,7 +2511,10 @@ function sandboxDrainActivityAttempt() {
|
|
|
2396
2511
|
}
|
|
2397
2512
|
}
|
|
2398
2513
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
2399
|
-
const terminateBox = options.terminateBox
|
|
2514
|
+
const terminateBox = (options.terminateBox ? async (...args) => {
|
|
2515
|
+
await args[10]?.();
|
|
2516
|
+
return options.terminateBox(...args);
|
|
2517
|
+
} : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop) => await terminateProviderBox(
|
|
2400
2518
|
settings,
|
|
2401
2519
|
lease,
|
|
2402
2520
|
observability,
|
|
@@ -2408,7 +2526,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2408
2526
|
capturePolicy,
|
|
2409
2527
|
diskBackedArchives,
|
|
2410
2528
|
releaseFailedCapture,
|
|
2411
|
-
workspaceId
|
|
2529
|
+
workspaceId,
|
|
2530
|
+
beforeProviderStop
|
|
2412
2531
|
));
|
|
2413
2532
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
2414
2533
|
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
@@ -2598,7 +2717,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2598
2717
|
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
2599
2718
|
metered = meterResult.accrued;
|
|
2600
2719
|
const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
|
|
2601
|
-
if (settings.
|
|
2720
|
+
if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
|
|
2602
2721
|
for (const workspaceId of meterResult.workspaceIds) {
|
|
2603
2722
|
forceDrainWorkspaceIds.add(workspaceId);
|
|
2604
2723
|
}
|
|
@@ -3209,6 +3328,16 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3209
3328
|
};
|
|
3210
3329
|
}
|
|
3211
3330
|
}
|
|
3331
|
+
if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
|
|
3332
|
+
try {
|
|
3333
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
3334
|
+
} catch (error) {
|
|
3335
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
3336
|
+
processId: process.id,
|
|
3337
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3338
|
+
});
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3212
3341
|
if (observation === null) {
|
|
3213
3342
|
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
3214
3343
|
() => null
|
|
@@ -3263,11 +3392,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3263
3392
|
);
|
|
3264
3393
|
supervisionMetric("cancellation_intent");
|
|
3265
3394
|
}
|
|
3266
|
-
|
|
3395
|
+
const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
|
|
3396
|
+
const probeOutcome = await probe(
|
|
3267
3397
|
settings,
|
|
3268
3398
|
lease,
|
|
3269
3399
|
process,
|
|
3270
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
3400
|
+
claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
|
|
3271
3401
|
async (result, chunkId, stream, streamFidelity) => {
|
|
3272
3402
|
if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
|
|
3273
3403
|
return;
|
|
@@ -3285,7 +3415,14 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3285
3415
|
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
3286
3416
|
},
|
|
3287
3417
|
commandPersistence
|
|
3418
|
+
).then(
|
|
3419
|
+
(value) => ({ ok: true, value }),
|
|
3420
|
+
(error) => ({ ok: false, error })
|
|
3288
3421
|
);
|
|
3422
|
+
if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
3423
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
3424
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
3425
|
+
observation = probeOutcome.value;
|
|
3289
3426
|
if (supervised) {
|
|
3290
3427
|
supervisionMetric(
|
|
3291
3428
|
await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
|
|
@@ -3489,7 +3626,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3489
3626
|
if (pending) {
|
|
3490
3627
|
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
3491
3628
|
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
3492
|
-
|
|
3629
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
3630
|
+
pending.result,
|
|
3631
|
+
process.providerSessionId
|
|
3632
|
+
);
|
|
3633
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
3493
3634
|
}
|
|
3494
3635
|
const envelopeBackend = lease.resumeState.backendId;
|
|
3495
3636
|
if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
|
|
@@ -3570,6 +3711,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3570
3711
|
if (typeof session.writeStdin !== "function") {
|
|
3571
3712
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
3572
3713
|
}
|
|
3714
|
+
let legacyPty = false;
|
|
3573
3715
|
if (process.providerBackend === "modal") {
|
|
3574
3716
|
let liveBinding;
|
|
3575
3717
|
try {
|
|
@@ -3596,6 +3738,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3596
3738
|
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
3597
3739
|
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
3598
3740
|
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
3741
|
+
legacyPty = command.pty === true;
|
|
3599
3742
|
}
|
|
3600
3743
|
const capturePage = async (value) => {
|
|
3601
3744
|
if (!captureOutput) {
|
|
@@ -3627,7 +3770,9 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3627
3770
|
result = await withRetainedProcessProbeTimeout(
|
|
3628
3771
|
session.writeStdin({
|
|
3629
3772
|
sessionId: process.providerSessionId,
|
|
3630
|
-
|
|
3773
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
3774
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
3775
|
+
chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
|
|
3631
3776
|
yieldTimeMs: 1e3,
|
|
3632
3777
|
maxOutputTokens: 2e3
|
|
3633
3778
|
})
|
|
@@ -3653,7 +3798,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3653
3798
|
}
|
|
3654
3799
|
await capturePage(result);
|
|
3655
3800
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
|
|
3656
|
-
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3801
|
+
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3657
3802
|
try {
|
|
3658
3803
|
const interrupted = await withRetainedProcessProbeTimeout(
|
|
3659
3804
|
session.writeStdin({
|
|
@@ -3730,19 +3875,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3730
3875
|
}
|
|
3731
3876
|
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
3732
3877
|
try {
|
|
3733
|
-
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3878
|
+
const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3734
3879
|
const result = await accrueWarmSeconds(db, {
|
|
3735
3880
|
accountId: lease.accountId,
|
|
3736
3881
|
workspaceId: lease.workspaceId,
|
|
3737
3882
|
sandboxGroupId: lease.sandboxGroupId,
|
|
3738
3883
|
expectedEpoch: lease.leaseEpoch,
|
|
3739
3884
|
warmRateMicrosPerSecond: rate,
|
|
3885
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
3740
3886
|
subjectId: lease.sandboxGroupId
|
|
3741
3887
|
});
|
|
3742
3888
|
if (result.accrued) {
|
|
3743
3889
|
accrued += 1;
|
|
3744
3890
|
}
|
|
3745
|
-
|
|
3891
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
3892
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3893
|
+
}
|
|
3746
3894
|
} catch (error) {
|
|
3747
3895
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3748
3896
|
workspaceId: lease.workspaceId,
|
|
@@ -3841,7 +3989,7 @@ async function refreshSandboxInventoryGauge(observability, domain, diagnosticNam
|
|
|
3841
3989
|
}
|
|
3842
3990
|
}
|
|
3843
3991
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3844
|
-
const enforceBalance =
|
|
3992
|
+
const enforceBalance = false;
|
|
3845
3993
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
3846
3994
|
let forceDrained = 0;
|
|
3847
3995
|
await forEachWithConcurrency(
|
|
@@ -3849,11 +3997,8 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3849
3997
|
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
3850
3998
|
async (workspaceId) => {
|
|
3851
3999
|
try {
|
|
3852
|
-
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
3853
|
-
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
3854
4000
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
3855
4001
|
workspaceId,
|
|
3856
|
-
balanceMicros: balance.balanceMicros,
|
|
3857
4002
|
enforceBalance,
|
|
3858
4003
|
maxWarmSecondsPerWorkspace: cap,
|
|
3859
4004
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
@@ -4239,6 +4384,17 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4239
4384
|
})) {
|
|
4240
4385
|
return false;
|
|
4241
4386
|
}
|
|
4387
|
+
let stopCutoffMarked = false;
|
|
4388
|
+
const beforeProviderStop = async () => {
|
|
4389
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
4390
|
+
accountId,
|
|
4391
|
+
workspaceId: row.workspaceId,
|
|
4392
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4393
|
+
expectedEpoch: row.leaseEpoch
|
|
4394
|
+
});
|
|
4395
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
4396
|
+
stopCutoffMarked = true;
|
|
4397
|
+
};
|
|
4242
4398
|
const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
|
|
4243
4399
|
settings,
|
|
4244
4400
|
lease,
|
|
@@ -4259,12 +4415,35 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4259
4415
|
expectedInstanceId: lease.instanceId
|
|
4260
4416
|
});
|
|
4261
4417
|
},
|
|
4262
|
-
row.workspaceId
|
|
4418
|
+
row.workspaceId,
|
|
4419
|
+
beforeProviderStop
|
|
4263
4420
|
);
|
|
4264
4421
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
4265
4422
|
if (!terminated) {
|
|
4266
4423
|
return false;
|
|
4267
4424
|
}
|
|
4425
|
+
const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
|
|
4426
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
4427
|
+
if (managedProvider && lease.instanceId) {
|
|
4428
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
4429
|
+
}
|
|
4430
|
+
await beforeProviderStop();
|
|
4431
|
+
}
|
|
4432
|
+
const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
4433
|
+
if (!providerMissingDuringStop) {
|
|
4434
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
4435
|
+
accountId,
|
|
4436
|
+
workspaceId: row.workspaceId,
|
|
4437
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4438
|
+
expectedEpoch: row.leaseEpoch,
|
|
4439
|
+
warmRateMicrosPerSecond: finalRate,
|
|
4440
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
4441
|
+
finalDrain: true
|
|
4442
|
+
});
|
|
4443
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
4444
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
4445
|
+
}
|
|
4446
|
+
}
|
|
4268
4447
|
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
4269
4448
|
const { wentCold } = await confirmDrainCold(db, {
|
|
4270
4449
|
accountId,
|
|
@@ -4315,7 +4494,7 @@ function providerDrainLogIdentity(lease, backend) {
|
|
|
4315
4494
|
instanceId: lease.instanceId
|
|
4316
4495
|
};
|
|
4317
4496
|
}
|
|
4318
|
-
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId) {
|
|
4497
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop) {
|
|
4319
4498
|
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
4320
4499
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
4321
4500
|
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
@@ -4379,6 +4558,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4379
4558
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
4380
4559
|
}
|
|
4381
4560
|
try {
|
|
4561
|
+
await beforeProviderStop?.();
|
|
4382
4562
|
await terminateModalById(settings, lease.instanceId);
|
|
4383
4563
|
} catch (error) {
|
|
4384
4564
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
@@ -4548,6 +4728,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4548
4728
|
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
4549
4729
|
}
|
|
4550
4730
|
prepareProviderForTeardownAfterCapture(backend, session);
|
|
4731
|
+
await beforeProviderStop?.();
|
|
4551
4732
|
await terminateManagedSandboxSession2(client, sessionState, session);
|
|
4552
4733
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
4553
4734
|
} catch (error) {
|
|
@@ -7498,4 +7679,4 @@ export {
|
|
|
7498
7679
|
createControlActivities,
|
|
7499
7680
|
createControlActivitiesFromServices
|
|
7500
7681
|
};
|
|
7501
|
-
//# sourceMappingURL=activities-control-
|
|
7682
|
+
//# sourceMappingURL=activities-control-IVTL723D.js.map
|