@opengeni/worker-bundle 2.0.1 → 2.0.3-canary.36199476632001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +2 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +13 -0
- package/dist/activities/agent-turn/errors.d.ts +12 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +1 -1
- package/dist/activities/agent-turn/governance-model.d.ts +2 -0
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/session-title.d.ts +32 -2
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -0
- package/dist/activities/context-compaction.d.ts +10 -2
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/sandbox-lease.d.ts +3 -4
- package/dist/{activities-control-QA6OLRQY.js → activities-control-I72CRMAE.js} +283 -35
- package/dist/activities-control-I72CRMAE.js.map +1 -0
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-CSDKK4N3.js} +531 -77
- package/dist/activities-turn-CSDKK4N3.js.map +1 -0
- package/dist/artifact-outbox-entry.js +3 -0
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/{chunk-M3NVMF2V.js → chunk-7M6G2SCF.js} +172 -17
- package/dist/chunk-7M6G2SCF.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-DSPP6CZL.js} +48 -3
- package/dist/chunk-DSPP6CZL.js.map +1 -0
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +11 -1
- package/dist/sandbox-resume.d.ts +46 -0
- package/dist/workflow-bundle.js +80 -2
- package/dist/workflows/session.d.ts +30 -0
- package/package.json +20 -19
- package/src/activities/agent-turn/agent-build.ts +4 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +37 -2
- package/src/activities/agent-turn/compaction-prep.ts +52 -16
- package/src/activities/agent-turn/errors.ts +67 -4
- package/src/activities/agent-turn/failure-settlement.ts +40 -8
- package/src/activities/agent-turn/governance-model.ts +17 -1
- package/src/activities/agent-turn/history.ts +14 -1
- package/src/activities/agent-turn/media-artifacts.ts +2 -1
- package/src/activities/agent-turn/run.ts +4 -0
- package/src/activities/agent-turn/sandbox-establish.ts +12 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +25 -9
- package/src/activities/agent-turn/session-title.ts +73 -3
- package/src/activities/agent-turn/stream-attempt.ts +27 -15
- package/src/activities/agent-turn/tool-environment.ts +65 -0
- package/src/activities/context-compaction.ts +49 -14
- package/src/activities/knowledge-indexing.ts +240 -15
- package/src/activities/retained-screenshots.ts +82 -11
- package/src/activities/sandbox-lease.ts +141 -21
- package/src/activities/scheduled-tasks.ts +18 -2
- package/src/activity-services.ts +16 -4
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +70 -1
- package/src/personal-github-git-credentials.ts +2 -0
- package/src/sandbox-resume.ts +211 -5
- package/src/workflows/activities.ts +14 -1
- package/src/workflows/session.ts +102 -6
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
warnDrainSnapshotFailure,
|
|
34
34
|
withFirstPartyTools,
|
|
35
35
|
workflowIdForSession
|
|
36
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-7M6G2SCF.js";
|
|
37
37
|
import {
|
|
38
38
|
inspectOpenSandboxKubernetesInventory,
|
|
39
39
|
recordCreditBalanceGauges,
|
|
@@ -53,29 +53,106 @@ import {
|
|
|
53
53
|
recordTurnsQueuedGauge,
|
|
54
54
|
recordWorkerDeathRecoveryMetrics,
|
|
55
55
|
runtimeMetricsHooksForObservability
|
|
56
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-DSPP6CZL.js";
|
|
57
57
|
import "./chunk-PZ5AY32C.js";
|
|
58
58
|
|
|
59
59
|
// src/activities/knowledge-indexing.ts
|
|
60
60
|
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
61
|
+
import { documentEmbeddingCostMicros, paidDocumentEmbedding } from "@opengeni/core";
|
|
61
62
|
import {
|
|
63
|
+
applyCreditDebitAfterUse,
|
|
62
64
|
appendKnowledgeIndexChunks,
|
|
63
65
|
claimKnowledgeIndexJobs,
|
|
64
66
|
completeKnowledgeIndexJob,
|
|
65
67
|
continueKnowledgeIndexJob,
|
|
66
68
|
deferKnowledgeIndexJob,
|
|
69
|
+
freezeKnowledgeIndexBillingMode,
|
|
67
70
|
getBillingBalance,
|
|
71
|
+
guardPaidKnowledgeIndexPublication,
|
|
72
|
+
knowledgeIndexBillingActivationTime,
|
|
68
73
|
readKnowledgeIndexSource,
|
|
69
74
|
recordUsageEvent,
|
|
70
75
|
sumUsageQuantity,
|
|
71
|
-
withWorkspaceUsageLock
|
|
76
|
+
withWorkspaceUsageLock,
|
|
77
|
+
waitKnowledgeIndexForFunding
|
|
72
78
|
} from "@opengeni/db";
|
|
79
|
+
var KnowledgeIndexUsageLimitError = class extends Error {
|
|
80
|
+
constructor() {
|
|
81
|
+
super("monthly document indexing limit reached");
|
|
82
|
+
this.name = "KnowledgeIndexUsageLimitError";
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
function knowledgeIndexFailureDiagnostic(stage, error) {
|
|
86
|
+
if (error instanceof KnowledgeIndexUsageLimitError) {
|
|
87
|
+
return {
|
|
88
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
89
|
+
errorCode: "knowledge_index_usage_limit_reached",
|
|
90
|
+
origin: "worker"
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if (stage === "embedding") {
|
|
94
|
+
const status = ownValue(error, "status");
|
|
95
|
+
return {
|
|
96
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
97
|
+
errorCode: "knowledge_index_embedding_failed",
|
|
98
|
+
origin: "worker",
|
|
99
|
+
...typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? { status } : {}
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
const postgres = postgresErrorState(error);
|
|
103
|
+
if (!postgres) {
|
|
104
|
+
return {
|
|
105
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
106
|
+
errorCode: "knowledge_index_failed",
|
|
107
|
+
origin: "worker"
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
errorClass: "KnowledgeIndexOperationError",
|
|
112
|
+
errorCode: "knowledge_index_persistence_failed",
|
|
113
|
+
origin: "db",
|
|
114
|
+
...postgres.sqlState ? { sqlState: postgres.sqlState } : {}
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
function ownValue(value, key) {
|
|
118
|
+
try {
|
|
119
|
+
if (!value || typeof value !== "object") return void 0;
|
|
120
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
121
|
+
return descriptor && "value" in descriptor ? descriptor.value : void 0;
|
|
122
|
+
} catch {
|
|
123
|
+
return void 0;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function postgresErrorState(error) {
|
|
127
|
+
let current = error;
|
|
128
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
129
|
+
if (ownValue(current, "name") === "PostgresError") {
|
|
130
|
+
const code = ownValue(current, "code");
|
|
131
|
+
return typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? { sqlState: code } : {};
|
|
132
|
+
}
|
|
133
|
+
current = ownValue(current, "cause");
|
|
134
|
+
}
|
|
135
|
+
return void 0;
|
|
136
|
+
}
|
|
73
137
|
function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
138
|
+
let activationTime;
|
|
74
139
|
return {
|
|
75
140
|
indexKnowledge: async () => {
|
|
76
141
|
const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
|
|
77
142
|
if (!resolveDocumentServices) return result;
|
|
78
143
|
const { db, settings, observability } = await services();
|
|
144
|
+
let policyActivatedAt;
|
|
145
|
+
if (paidDocumentEmbedding(settings)) {
|
|
146
|
+
if (!settings.documentEmbeddingCreditsActivatedAt)
|
|
147
|
+
throw new Error("paid Knowledge embedding requires an activation cutoff");
|
|
148
|
+
policyActivatedAt = new Date(settings.documentEmbeddingCreditsActivatedAt);
|
|
149
|
+
} else {
|
|
150
|
+
activationTime ??= knowledgeIndexBillingActivationTime(db).catch((error) => {
|
|
151
|
+
activationTime = void 0;
|
|
152
|
+
throw error;
|
|
153
|
+
});
|
|
154
|
+
policyActivatedAt = await activationTime;
|
|
155
|
+
}
|
|
79
156
|
const { embedder } = await resolveDocumentServices();
|
|
80
157
|
const { knowledgeIndexChunks } = await import("@opengeni/documents");
|
|
81
158
|
const claims = await claimKnowledgeIndexJobs(db, {
|
|
@@ -84,6 +161,7 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
84
161
|
limit: 2
|
|
85
162
|
});
|
|
86
163
|
for (const claim of claims) {
|
|
164
|
+
let stage = "processing";
|
|
87
165
|
try {
|
|
88
166
|
const source = await readKnowledgeIndexSource(db, claim);
|
|
89
167
|
if (!source) {
|
|
@@ -107,9 +185,37 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
107
185
|
chunks.push(chunk);
|
|
108
186
|
}
|
|
109
187
|
if (chunks.length) {
|
|
110
|
-
|
|
188
|
+
const frozenPolicy = await freezeKnowledgeIndexBillingMode(
|
|
189
|
+
lockedDb,
|
|
190
|
+
claim,
|
|
191
|
+
// Deterministic embeddings incur no provider charge; a valid
|
|
192
|
+
// credits-mode config with zero tariff must still index them.
|
|
193
|
+
// OpenAI shadow keeps its price snapshot for internal estimates.
|
|
194
|
+
settings.documentEmbeddingProvider === "openai" ? settings.documentEmbeddingBillingMode ?? "usage_only" : "usage_only",
|
|
195
|
+
policyActivatedAt,
|
|
196
|
+
settings.documentEmbeddingRateMicrosPerMillionBytes ?? 0
|
|
197
|
+
);
|
|
198
|
+
if (frozenPolicy.mode === "awaiting_review") {
|
|
199
|
+
result.deferred++;
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (frozenPolicy.mode === "obsolete") {
|
|
203
|
+
result.unavailable++;
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (frozenPolicy.mode === "credits" && !paidDocumentEmbedding(settings)) {
|
|
207
|
+
await deferKnowledgeIndexJob(lockedDb, claim);
|
|
208
|
+
result.deferred++;
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const paid = frozenPolicy.mode === "credits";
|
|
212
|
+
if (paid && current.nextIndex === 0) {
|
|
111
213
|
const balance = await getBillingBalance(lockedDb, claim.accountId);
|
|
112
|
-
if (balance.balanceMicros <= 0)
|
|
214
|
+
if (balance.balanceMicros <= 0) {
|
|
215
|
+
await waitKnowledgeIndexForFunding(lockedDb, claim);
|
|
216
|
+
result.deferred++;
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
113
219
|
}
|
|
114
220
|
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
115
221
|
const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
@@ -120,13 +226,27 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
120
226
|
eventType: "document.indexed",
|
|
121
227
|
since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1))
|
|
122
228
|
});
|
|
123
|
-
if (used + chunks.length > limit)
|
|
124
|
-
throw new Error("monthly document indexing limit reached");
|
|
229
|
+
if (used + chunks.length > limit) throw new KnowledgeIndexUsageLimitError();
|
|
125
230
|
}
|
|
126
231
|
}
|
|
127
|
-
const
|
|
232
|
+
const inputs = chunks.map((chunk) => chunk.embeddingInput);
|
|
233
|
+
const bytes = inputs.reduce(
|
|
234
|
+
(sum, input) => sum + Buffer.byteLength(input, "utf8"),
|
|
235
|
+
0
|
|
236
|
+
);
|
|
237
|
+
stage = "embedding";
|
|
238
|
+
const vectors = await embedder.embedMany(inputs);
|
|
128
239
|
if (vectors.length !== chunks.length)
|
|
129
240
|
throw new Error("Incomplete Knowledge embeddings");
|
|
241
|
+
stage = "processing";
|
|
242
|
+
if (paid) {
|
|
243
|
+
const publication = await guardPaidKnowledgeIndexPublication(lockedDb, claim);
|
|
244
|
+
if (publication !== "published") {
|
|
245
|
+
if (publication === "obsolete") result.unavailable++;
|
|
246
|
+
else result.deferred++;
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
130
250
|
const appended = await appendKnowledgeIndexChunks(
|
|
131
251
|
lockedDb,
|
|
132
252
|
claim,
|
|
@@ -147,6 +267,61 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
147
267
|
sourceResourceId: claim.revisionId,
|
|
148
268
|
idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
149
269
|
});
|
|
270
|
+
await recordUsageEvent(lockedDb, {
|
|
271
|
+
accountId: claim.accountId,
|
|
272
|
+
workspaceId: current.billingWorkspaceId,
|
|
273
|
+
eventType: "document.embedding_bytes",
|
|
274
|
+
quantity: bytes,
|
|
275
|
+
unit: "byte",
|
|
276
|
+
sourceResourceType: "knowledge_revision",
|
|
277
|
+
sourceResourceId: claim.revisionId,
|
|
278
|
+
idempotencyKey: `knowledge.embedding_bytes:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
279
|
+
});
|
|
280
|
+
if (frozenPolicy.mode === "shadow" && frozenPolicy.rateMicrosPerMillionBytes > 0) {
|
|
281
|
+
const estimate = documentEmbeddingCostMicros(
|
|
282
|
+
{
|
|
283
|
+
...settings,
|
|
284
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
285
|
+
},
|
|
286
|
+
bytes
|
|
287
|
+
);
|
|
288
|
+
if (estimate > 0)
|
|
289
|
+
await recordUsageEvent(lockedDb, {
|
|
290
|
+
accountId: claim.accountId,
|
|
291
|
+
workspaceId: current.billingWorkspaceId,
|
|
292
|
+
eventType: "document.embedding_shadow_estimate",
|
|
293
|
+
quantity: estimate,
|
|
294
|
+
unit: "micro_usd",
|
|
295
|
+
sourceResourceType: "knowledge_revision",
|
|
296
|
+
sourceResourceId: claim.revisionId,
|
|
297
|
+
idempotencyKey: `knowledge.embedding_shadow:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
if (paid) {
|
|
301
|
+
const cost = documentEmbeddingCostMicros(
|
|
302
|
+
{
|
|
303
|
+
...settings,
|
|
304
|
+
documentEmbeddingRateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
305
|
+
},
|
|
306
|
+
bytes
|
|
307
|
+
);
|
|
308
|
+
if (cost > 0)
|
|
309
|
+
await applyCreditDebitAfterUse(lockedDb, {
|
|
310
|
+
accountId: claim.accountId,
|
|
311
|
+
workspaceId: current.billingWorkspaceId,
|
|
312
|
+
type: "document_embedding_debit",
|
|
313
|
+
amountMicros: cost,
|
|
314
|
+
sourceType: "knowledge_revision",
|
|
315
|
+
sourceId: claim.revisionId,
|
|
316
|
+
idempotencyKey: `knowledge.embedding:${claim.revisionId}:${claim.generation}:${current.nextIndex}`,
|
|
317
|
+
metadata: {
|
|
318
|
+
model: claim.model,
|
|
319
|
+
bytes,
|
|
320
|
+
chunks: chunks.length,
|
|
321
|
+
rateMicrosPerMillionBytes: frozenPolicy.rateMicrosPerMillionBytes
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
}
|
|
150
325
|
}
|
|
151
326
|
if (more) {
|
|
152
327
|
await continueKnowledgeIndexJob(lockedDb, claim);
|
|
@@ -161,14 +336,19 @@ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
|
|
|
161
336
|
else result.unavailable++;
|
|
162
337
|
}
|
|
163
338
|
});
|
|
164
|
-
} catch {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
339
|
+
} catch (error) {
|
|
340
|
+
observability.warn(
|
|
341
|
+
"Knowledge indexing batch deferred",
|
|
342
|
+
knowledgeIndexFailureDiagnostic(stage, error)
|
|
343
|
+
);
|
|
344
|
+
await deferKnowledgeIndexJob(db, claim).catch((deferError) => {
|
|
345
|
+
const deferDiagnostic = knowledgeIndexFailureDiagnostic("processing", deferError);
|
|
346
|
+
observability.warn("Knowledge indexing batch deferral failed", {
|
|
347
|
+
...deferDiagnostic,
|
|
348
|
+
errorCode: "knowledge_index_defer_failed"
|
|
349
|
+
});
|
|
171
350
|
});
|
|
351
|
+
result.deferred++;
|
|
172
352
|
}
|
|
173
353
|
}
|
|
174
354
|
return result;
|
|
@@ -2241,10 +2421,10 @@ import {
|
|
|
2241
2421
|
countSandboxLeasesByLiveness,
|
|
2242
2422
|
deferRetainedProcessReconciliation,
|
|
2243
2423
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
2244
|
-
getBillingBalance as getBillingBalance3,
|
|
2245
2424
|
listCreditBalancesByAccount,
|
|
2246
2425
|
listLegacyModalCheckpointSlots,
|
|
2247
2426
|
listLiveModalSandboxLeaseAttributions,
|
|
2427
|
+
markWarmBillingStopCutoff,
|
|
2248
2428
|
listMeterableWarmLeases,
|
|
2249
2429
|
listSandboxViewerForceDrainWorkspaceIds,
|
|
2250
2430
|
markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
|
|
@@ -2396,7 +2576,10 @@ function sandboxDrainActivityAttempt() {
|
|
|
2396
2576
|
}
|
|
2397
2577
|
}
|
|
2398
2578
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
2399
|
-
const terminateBox = options.terminateBox
|
|
2579
|
+
const terminateBox = (options.terminateBox ? async (...args) => {
|
|
2580
|
+
await args[10]?.();
|
|
2581
|
+
return options.terminateBox(...args);
|
|
2582
|
+
} : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop) => await terminateProviderBox(
|
|
2400
2583
|
settings,
|
|
2401
2584
|
lease,
|
|
2402
2585
|
observability,
|
|
@@ -2408,7 +2591,8 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2408
2591
|
capturePolicy,
|
|
2409
2592
|
diskBackedArchives,
|
|
2410
2593
|
releaseFailedCapture,
|
|
2411
|
-
workspaceId
|
|
2594
|
+
workspaceId,
|
|
2595
|
+
beforeProviderStop
|
|
2412
2596
|
));
|
|
2413
2597
|
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
2414
2598
|
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
@@ -2598,7 +2782,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
2598
2782
|
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
2599
2783
|
metered = meterResult.accrued;
|
|
2600
2784
|
const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
|
|
2601
|
-
if (settings.
|
|
2785
|
+
if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
|
|
2602
2786
|
for (const workspaceId of meterResult.workspaceIds) {
|
|
2603
2787
|
forceDrainWorkspaceIds.add(workspaceId);
|
|
2604
2788
|
}
|
|
@@ -3209,6 +3393,16 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3209
3393
|
};
|
|
3210
3394
|
}
|
|
3211
3395
|
}
|
|
3396
|
+
if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
|
|
3397
|
+
try {
|
|
3398
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
3399
|
+
} catch (error) {
|
|
3400
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
3401
|
+
processId: process.id,
|
|
3402
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3403
|
+
});
|
|
3404
|
+
}
|
|
3405
|
+
}
|
|
3212
3406
|
if (observation === null) {
|
|
3213
3407
|
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
3214
3408
|
() => null
|
|
@@ -3263,11 +3457,12 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3263
3457
|
);
|
|
3264
3458
|
supervisionMetric("cancellation_intent");
|
|
3265
3459
|
}
|
|
3266
|
-
|
|
3460
|
+
const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
|
|
3461
|
+
const probeOutcome = await probe(
|
|
3267
3462
|
settings,
|
|
3268
3463
|
lease,
|
|
3269
3464
|
process,
|
|
3270
|
-
claim.ownerState === "background_stopping" ? "cancel" : "observe",
|
|
3465
|
+
claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
|
|
3271
3466
|
async (result, chunkId, stream, streamFidelity) => {
|
|
3272
3467
|
if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
|
|
3273
3468
|
return;
|
|
@@ -3285,7 +3480,14 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
|
|
|
3285
3480
|
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
3286
3481
|
},
|
|
3287
3482
|
commandPersistence
|
|
3483
|
+
).then(
|
|
3484
|
+
(value) => ({ ok: true, value }),
|
|
3485
|
+
(error) => ({ ok: false, error })
|
|
3288
3486
|
);
|
|
3487
|
+
if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
3488
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
3489
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
3490
|
+
observation = probeOutcome.value;
|
|
3289
3491
|
if (supervised) {
|
|
3290
3492
|
supervisionMetric(
|
|
3291
3493
|
await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
|
|
@@ -3489,7 +3691,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3489
3691
|
if (pending) {
|
|
3490
3692
|
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
3491
3693
|
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
3492
|
-
|
|
3694
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
3695
|
+
pending.result,
|
|
3696
|
+
process.providerSessionId
|
|
3697
|
+
);
|
|
3698
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
3493
3699
|
}
|
|
3494
3700
|
const envelopeBackend = lease.resumeState.backendId;
|
|
3495
3701
|
if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
|
|
@@ -3570,6 +3776,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3570
3776
|
if (typeof session.writeStdin !== "function") {
|
|
3571
3777
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
3572
3778
|
}
|
|
3779
|
+
let legacyPty = false;
|
|
3573
3780
|
if (process.providerBackend === "modal") {
|
|
3574
3781
|
let liveBinding;
|
|
3575
3782
|
try {
|
|
@@ -3596,6 +3803,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3596
3803
|
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
3597
3804
|
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
3598
3805
|
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
3806
|
+
legacyPty = command.pty === true;
|
|
3599
3807
|
}
|
|
3600
3808
|
const capturePage = async (value) => {
|
|
3601
3809
|
if (!captureOutput) {
|
|
@@ -3627,7 +3835,9 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3627
3835
|
result = await withRetainedProcessProbeTimeout(
|
|
3628
3836
|
session.writeStdin({
|
|
3629
3837
|
sessionId: process.providerSessionId,
|
|
3630
|
-
|
|
3838
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
3839
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
3840
|
+
chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
|
|
3631
3841
|
yieldTimeMs: 1e3,
|
|
3632
3842
|
maxOutputTokens: 2e3
|
|
3633
3843
|
})
|
|
@@ -3653,7 +3863,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
|
|
|
3653
3863
|
}
|
|
3654
3864
|
await capturePage(result);
|
|
3655
3865
|
const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
|
|
3656
|
-
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3866
|
+
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
3657
3867
|
try {
|
|
3658
3868
|
const interrupted = await withRetainedProcessProbeTimeout(
|
|
3659
3869
|
session.writeStdin({
|
|
@@ -3730,19 +3940,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3730
3940
|
}
|
|
3731
3941
|
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
3732
3942
|
try {
|
|
3733
|
-
const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3943
|
+
const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
3734
3944
|
const result = await accrueWarmSeconds(db, {
|
|
3735
3945
|
accountId: lease.accountId,
|
|
3736
3946
|
workspaceId: lease.workspaceId,
|
|
3737
3947
|
sandboxGroupId: lease.sandboxGroupId,
|
|
3738
3948
|
expectedEpoch: lease.leaseEpoch,
|
|
3739
3949
|
warmRateMicrosPerSecond: rate,
|
|
3950
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
3740
3951
|
subjectId: lease.sandboxGroupId
|
|
3741
3952
|
});
|
|
3742
3953
|
if (result.accrued) {
|
|
3743
3954
|
accrued += 1;
|
|
3744
3955
|
}
|
|
3745
|
-
|
|
3956
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
3957
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3958
|
+
}
|
|
3746
3959
|
} catch (error) {
|
|
3747
3960
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3748
3961
|
workspaceId: lease.workspaceId,
|
|
@@ -3841,7 +4054,7 @@ async function refreshSandboxInventoryGauge(observability, domain, diagnosticNam
|
|
|
3841
4054
|
}
|
|
3842
4055
|
}
|
|
3843
4056
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3844
|
-
const enforceBalance =
|
|
4057
|
+
const enforceBalance = false;
|
|
3845
4058
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
3846
4059
|
let forceDrained = 0;
|
|
3847
4060
|
await forEachWithConcurrency(
|
|
@@ -3849,11 +4062,8 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3849
4062
|
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
3850
4063
|
async (workspaceId) => {
|
|
3851
4064
|
try {
|
|
3852
|
-
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
3853
|
-
const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
|
|
3854
4065
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
3855
4066
|
workspaceId,
|
|
3856
|
-
balanceMicros: balance.balanceMicros,
|
|
3857
4067
|
enforceBalance,
|
|
3858
4068
|
maxWarmSecondsPerWorkspace: cap,
|
|
3859
4069
|
idleGraceMs: settings.sandboxIdleGraceMs
|
|
@@ -4239,6 +4449,17 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4239
4449
|
})) {
|
|
4240
4450
|
return false;
|
|
4241
4451
|
}
|
|
4452
|
+
let stopCutoffMarked = false;
|
|
4453
|
+
const beforeProviderStop = async () => {
|
|
4454
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
4455
|
+
accountId,
|
|
4456
|
+
workspaceId: row.workspaceId,
|
|
4457
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4458
|
+
expectedEpoch: row.leaseEpoch
|
|
4459
|
+
});
|
|
4460
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
4461
|
+
stopCutoffMarked = true;
|
|
4462
|
+
};
|
|
4242
4463
|
const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
|
|
4243
4464
|
settings,
|
|
4244
4465
|
lease,
|
|
@@ -4259,12 +4480,35 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
4259
4480
|
expectedInstanceId: lease.instanceId
|
|
4260
4481
|
});
|
|
4261
4482
|
},
|
|
4262
|
-
row.workspaceId
|
|
4483
|
+
row.workspaceId,
|
|
4484
|
+
beforeProviderStop
|
|
4263
4485
|
);
|
|
4264
4486
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
4265
4487
|
if (!terminated) {
|
|
4266
4488
|
return false;
|
|
4267
4489
|
}
|
|
4490
|
+
const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
|
|
4491
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
4492
|
+
if (managedProvider && lease.instanceId) {
|
|
4493
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
4494
|
+
}
|
|
4495
|
+
await beforeProviderStop();
|
|
4496
|
+
}
|
|
4497
|
+
const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
4498
|
+
if (!providerMissingDuringStop) {
|
|
4499
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
4500
|
+
accountId,
|
|
4501
|
+
workspaceId: row.workspaceId,
|
|
4502
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4503
|
+
expectedEpoch: row.leaseEpoch,
|
|
4504
|
+
warmRateMicrosPerSecond: finalRate,
|
|
4505
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
4506
|
+
finalDrain: true
|
|
4507
|
+
});
|
|
4508
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
4509
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
4510
|
+
}
|
|
4511
|
+
}
|
|
4268
4512
|
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
4269
4513
|
const { wentCold } = await confirmDrainCold(db, {
|
|
4270
4514
|
accountId,
|
|
@@ -4315,7 +4559,7 @@ function providerDrainLogIdentity(lease, backend) {
|
|
|
4315
4559
|
instanceId: lease.instanceId
|
|
4316
4560
|
};
|
|
4317
4561
|
}
|
|
4318
|
-
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId) {
|
|
4562
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop) {
|
|
4319
4563
|
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
4320
4564
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
4321
4565
|
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
@@ -4379,6 +4623,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4379
4623
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
4380
4624
|
}
|
|
4381
4625
|
try {
|
|
4626
|
+
await beforeProviderStop?.();
|
|
4382
4627
|
await terminateModalById(settings, lease.instanceId);
|
|
4383
4628
|
} catch (error) {
|
|
4384
4629
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
@@ -4548,6 +4793,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
4548
4793
|
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
4549
4794
|
}
|
|
4550
4795
|
prepareProviderForTeardownAfterCapture(backend, session);
|
|
4796
|
+
await beforeProviderStop?.();
|
|
4551
4797
|
await terminateManagedSandboxSession2(client, sessionState, session);
|
|
4552
4798
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
4553
4799
|
} catch (error) {
|
|
@@ -4582,6 +4828,7 @@ import {
|
|
|
4582
4828
|
openGeniSlackBotMetadata,
|
|
4583
4829
|
requireOpenGeniSlackBotConnection,
|
|
4584
4830
|
resolveWorkspaceCatalogSettings as resolveWorkspaceCatalogSettings2,
|
|
4831
|
+
resolveScheduledTaskDefaultModel,
|
|
4585
4832
|
resolveSessionToolPolicy,
|
|
4586
4833
|
workspaceCustomModelReference,
|
|
4587
4834
|
lockActiveCustomModelForAdmission,
|
|
@@ -4946,8 +5193,9 @@ function createScheduledTaskActivities(services) {
|
|
|
4946
5193
|
taskAuthoritySubjectId
|
|
4947
5194
|
) : null;
|
|
4948
5195
|
const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
|
|
4949
|
-
const
|
|
4950
|
-
const
|
|
5196
|
+
const resolvedDefault = task.agentConfig.model || targetSessionExecutionBase ? null : await resolveScheduledTaskDefaultModel(db, settings, task);
|
|
5197
|
+
const model = task.agentConfig.model ?? resolvedDefault?.model ?? settings.openaiModel;
|
|
5198
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? resolvedDefault?.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
4951
5199
|
let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
4952
5200
|
let sandboxOs = "linux";
|
|
4953
5201
|
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
@@ -7498,4 +7746,4 @@ export {
|
|
|
7498
7746
|
createControlActivities,
|
|
7499
7747
|
createControlActivitiesFromServices
|
|
7500
7748
|
};
|
|
7501
|
-
//# sourceMappingURL=activities-control-
|
|
7749
|
+
//# sourceMappingURL=activities-control-I72CRMAE.js.map
|