@opengeni/worker-bundle 0.12.10 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2729 -1208
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +873 -131
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
settleSessionAttemptInterruptions,
|
|
3
3
|
applySessionTurnSettlement,
|
|
4
|
+
requestSessionTurnRecovery,
|
|
4
5
|
recoverSessionDispatch,
|
|
5
6
|
reconcileSessionAttemptQuiescence,
|
|
6
7
|
peekSessionWork as peekSessionWorkDb,
|
|
@@ -29,11 +30,14 @@ import type {
|
|
|
29
30
|
ReconcileSessionAttemptQuiescenceResult,
|
|
30
31
|
RecoverDispatchInput,
|
|
31
32
|
RecoverDispatchResult,
|
|
33
|
+
RecoverEscapedMcpTimeoutInput,
|
|
34
|
+
RecoverEscapedMcpTimeoutResult,
|
|
32
35
|
} from "./types";
|
|
33
36
|
|
|
34
37
|
export type SessionStateActivityOverrides = Partial<{
|
|
35
38
|
settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
|
|
36
39
|
applySessionTurnSettlement: typeof applySessionTurnSettlement;
|
|
40
|
+
requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
|
|
37
41
|
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
38
42
|
reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
|
|
39
43
|
peekSessionWork: typeof peekSessionWorkDb;
|
|
@@ -65,6 +69,8 @@ export function createSessionStateActivities(
|
|
|
65
69
|
overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
|
|
66
70
|
const applySessionTurnSettlementFn =
|
|
67
71
|
overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
|
|
72
|
+
const requestSessionTurnRecoveryFn =
|
|
73
|
+
overrides.requestSessionTurnRecovery ?? requestSessionTurnRecovery;
|
|
68
74
|
const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
|
|
69
75
|
const reconcileSessionAttemptQuiescenceFn =
|
|
70
76
|
overrides.reconcileSessionAttemptQuiescence ?? reconcileSessionAttemptQuiescence;
|
|
@@ -271,6 +277,55 @@ export function createSessionStateActivities(
|
|
|
271
277
|
};
|
|
272
278
|
}
|
|
273
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Finish a retryable MCP timeout checkpoint that escaped a recovered turn's
|
|
282
|
+
* activity before any model request. Temporal carries the immutable turn
|
|
283
|
+
* identity in ApplicationFailure details; this activity re-reads and fences
|
|
284
|
+
* every field before mutating the exact still-owned attempt. A checkpoint
|
|
285
|
+
* that committed before event fanout failed is already recovering and
|
|
286
|
+
* therefore returns stale without another event or child callback.
|
|
287
|
+
*/
|
|
288
|
+
async function recoverEscapedMcpTimeout(
|
|
289
|
+
input: RecoverEscapedMcpTimeoutInput,
|
|
290
|
+
): Promise<RecoverEscapedMcpTimeoutResult> {
|
|
291
|
+
const { db, bus, observability } = await services();
|
|
292
|
+
const turn = await getSessionTurnForAttemptFn(
|
|
293
|
+
db,
|
|
294
|
+
input.workspaceId,
|
|
295
|
+
input.sessionId,
|
|
296
|
+
input.attemptId,
|
|
297
|
+
);
|
|
298
|
+
if (!turn) return { action: "stale" };
|
|
299
|
+
if (
|
|
300
|
+
turn.id !== input.turnId ||
|
|
301
|
+
turn.triggerEventId !== input.triggerEventId ||
|
|
302
|
+
turn.executionGeneration !== input.executionGeneration ||
|
|
303
|
+
input.executionGeneration <= 1
|
|
304
|
+
) {
|
|
305
|
+
return { action: "ineligible" };
|
|
306
|
+
}
|
|
307
|
+
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
308
|
+
sessionId: input.sessionId,
|
|
309
|
+
turnId: input.turnId,
|
|
310
|
+
triggerEventId: input.triggerEventId,
|
|
311
|
+
attemptId: input.attemptId,
|
|
312
|
+
reason: "mcp_transport_timeout",
|
|
313
|
+
detail: {
|
|
314
|
+
code: "mcp_transport_timeout",
|
|
315
|
+
retryable: true,
|
|
316
|
+
continueDelayMs: 60_000,
|
|
317
|
+
recoverySource: "workflow_activity_failure",
|
|
318
|
+
},
|
|
319
|
+
fromStatuses: ["running"],
|
|
320
|
+
});
|
|
321
|
+
if (recovery.action !== "recovering") {
|
|
322
|
+
return { action: "stale" };
|
|
323
|
+
}
|
|
324
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
325
|
+
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
326
|
+
return { action: "recovering" };
|
|
327
|
+
}
|
|
328
|
+
|
|
274
329
|
async function peekSessionWork(input: PeekSessionWorkInput) {
|
|
275
330
|
const { db, observability } = await services();
|
|
276
331
|
const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
|
|
@@ -325,6 +380,7 @@ export function createSessionStateActivities(
|
|
|
325
380
|
persistSessionAttemptQuiescence,
|
|
326
381
|
reconcileSessionAttemptQuiescence: reconcileSessionAttemptQuiescenceActivity,
|
|
327
382
|
recoverDispatch,
|
|
383
|
+
recoverEscapedMcpTimeout,
|
|
328
384
|
peekSessionWork,
|
|
329
385
|
expireSessionHumanInput,
|
|
330
386
|
markSessionIdle,
|
package/src/activities/types.ts
CHANGED
|
@@ -202,6 +202,27 @@ export type RecoverDispatchResult =
|
|
|
202
202
|
// ceiling), so the failed attempt was worker death number redispatches + 1.
|
|
203
203
|
| { action: "exceeded"; turnId: string; redispatches: number };
|
|
204
204
|
|
|
205
|
+
export const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
|
|
206
|
+
export const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE =
|
|
207
|
+
"MCP request timeout recovery checkpoint failed before model request";
|
|
208
|
+
|
|
209
|
+
export type EscapedMcpTimeoutRecoveryDetail = {
|
|
210
|
+
turnId: string;
|
|
211
|
+
triggerEventId: string;
|
|
212
|
+
executionGeneration: number;
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
export type RecoverEscapedMcpTimeoutInput = EscapedMcpTimeoutRecoveryDetail & {
|
|
216
|
+
accountId: string;
|
|
217
|
+
workspaceId: string;
|
|
218
|
+
sessionId: string;
|
|
219
|
+
attemptId: string;
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
export type RecoverEscapedMcpTimeoutResult = {
|
|
223
|
+
action: "recovering" | "stale" | "ineligible";
|
|
224
|
+
};
|
|
225
|
+
|
|
205
226
|
export type PeekSessionWorkInput = {
|
|
206
227
|
workspaceId: string;
|
|
207
228
|
sessionId: string;
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// • This module NEVER calls close()/terminate()/kill() on the session handle.
|
|
15
15
|
// session.close() == terminate() on Modal and would kill the user's box.
|
|
16
16
|
// Only the reaper terminates. We drop references, nothing more.
|
|
17
|
-
// • The whole capture is time-capped (
|
|
17
|
+
// • The whole capture is time-capped (120s) and best-effort: ANY failure logs
|
|
18
18
|
// "workspace capture failed — turn outcome unaffected" and returns; nothing
|
|
19
19
|
// throws past captureWorkspaceRevision.
|
|
20
20
|
// • The DB write is fenced on the exact attempt, accepted control requests,
|
|
@@ -62,7 +62,12 @@ import type {
|
|
|
62
62
|
} from "@opengeni/contracts";
|
|
63
63
|
|
|
64
64
|
// ─── Guards & constants for pathological cases; configurable ─────
|
|
65
|
-
|
|
65
|
+
// Production Modal captures of the acceptance workspace normally take ~30s:
|
|
66
|
+
// repository diff framing deliberately trades provider round-trips for bounded
|
|
67
|
+
// output and symlink-safe reads. Sixty seconds gave that healthy path only a 2x
|
|
68
|
+
// margin and repeatedly timed it out under ordinary provider variance. The turn
|
|
69
|
+
// holder keeps the exact lease pinned for this whole window.
|
|
70
|
+
export const CAPTURE_TIMEOUT_MS = 120_000;
|
|
66
71
|
export const PER_FILE_CONTENT_GUARD_BYTES = 5 * 1024 * 1024; // after-image; over → tooLarge marker
|
|
67
72
|
export const PER_FILE_DIFF_GUARD_BYTES = 10 * 1024 * 1024; // per-file diff; over → truncated marker
|
|
68
73
|
export const WHOLE_CAPTURE_GUARD_BYTES = 200 * 1024 * 1024; // total → skip, fall back to live
|
|
@@ -185,6 +190,14 @@ export type CaptureWorkspaceRevisionInput = {
|
|
|
185
190
|
publish: CaptureEventPublisher | null;
|
|
186
191
|
/** The un-proxied setupBoxSession (NOT the routing veneer). */
|
|
187
192
|
session: ChannelASession;
|
|
193
|
+
/**
|
|
194
|
+
* Reopen the exact live provider instance as a fresh, non-owning read handle.
|
|
195
|
+
* Modal sessions keep mutable exec-process bookkeeping; reusing the handle
|
|
196
|
+
* that just drove model tools can strand a later capture command behind stale
|
|
197
|
+
* provider state. The lease still owns the box. Capture drops this handle
|
|
198
|
+
* without calling close(), which would terminate Modal's sandbox.
|
|
199
|
+
*/
|
|
200
|
+
openReadSession?: () => Promise<ChannelASession>;
|
|
188
201
|
leaseEpoch: number;
|
|
189
202
|
sandboxGroupId: string;
|
|
190
203
|
accountId: string;
|
|
@@ -211,6 +224,68 @@ type CaptureRepositoryReadResult =
|
|
|
211
224
|
}
|
|
212
225
|
| { complete: false; degradedReason: "repository_read_unavailable" };
|
|
213
226
|
|
|
227
|
+
type ResumableCaptureClient = {
|
|
228
|
+
resume?: (state: unknown) => Promise<unknown>;
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
type StatefulCaptureSession = ChannelASession & {
|
|
232
|
+
state?: unknown;
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
function captureSessionInstanceId(session: unknown): string | null {
|
|
236
|
+
const state =
|
|
237
|
+
session && typeof session === "object"
|
|
238
|
+
? (session as { state?: Record<string, unknown> }).state
|
|
239
|
+
: undefined;
|
|
240
|
+
if (!state || typeof state !== "object") return null;
|
|
241
|
+
for (const value of [
|
|
242
|
+
state.sandboxId,
|
|
243
|
+
state.instanceId,
|
|
244
|
+
state.id,
|
|
245
|
+
state.hostId,
|
|
246
|
+
state.containerId,
|
|
247
|
+
state.workspaceRootPath,
|
|
248
|
+
]) {
|
|
249
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
250
|
+
}
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Reattach a clean SDK handle to the exact already-live Modal sandbox. Modal's
|
|
256
|
+
* resume contract is a pure from-id reattach. Other providers keep the original
|
|
257
|
+
* session because their generic resume contract may restore or replace a dead
|
|
258
|
+
* instance, which capture does not own.
|
|
259
|
+
*/
|
|
260
|
+
export async function openFreshWorkspaceCaptureSession(input: {
|
|
261
|
+
backendId: string;
|
|
262
|
+
client: unknown;
|
|
263
|
+
session: ChannelASession;
|
|
264
|
+
expectedInstanceId: string;
|
|
265
|
+
}): Promise<ChannelASession> {
|
|
266
|
+
if (input.backendId !== "modal") {
|
|
267
|
+
return input.session;
|
|
268
|
+
}
|
|
269
|
+
const client = input.client as ResumableCaptureClient;
|
|
270
|
+
const state = (input.session as StatefulCaptureSession).state;
|
|
271
|
+
if (typeof client.resume !== "function" || state === undefined) {
|
|
272
|
+
return input.session;
|
|
273
|
+
}
|
|
274
|
+
// The lease—not this temporary reader—owns provider teardown. Preserve the
|
|
275
|
+
// exact state while explicitly removing close/terminate authority from
|
|
276
|
+
// providers (Modal) that encode it in the resumed state.
|
|
277
|
+
const readState =
|
|
278
|
+
state !== null && typeof state === "object" ? { ...state, ownsSandbox: false } : state;
|
|
279
|
+
const reopened = await client.resume(readState);
|
|
280
|
+
const actualInstanceId = captureSessionInstanceId(reopened);
|
|
281
|
+
if (input.expectedInstanceId.length > 0 && actualInstanceId !== input.expectedInstanceId) {
|
|
282
|
+
throw new Error(
|
|
283
|
+
`workspace capture reopened provider instance ${actualInstanceId ?? "unknown"}, expected ${input.expectedInstanceId}`,
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
return reopened as ChannelASession;
|
|
287
|
+
}
|
|
288
|
+
|
|
214
289
|
/** One discovered repository is all-or-nothing capture authority. A provider
|
|
215
290
|
* transport failure must never be represented as an authoritative empty diff. */
|
|
216
291
|
export async function readCaptureRepository(
|
|
@@ -250,7 +325,7 @@ let loggedStorageNullOnce = false;
|
|
|
250
325
|
|
|
251
326
|
/**
|
|
252
327
|
* Turn-end entrypoint. Gates on the flag + configured object storage, races the
|
|
253
|
-
* whole capture against a
|
|
328
|
+
* whole capture against a 120s cap, and swallows every failure ("turn outcome
|
|
254
329
|
* unaffected"). Returns void — the caller awaits it (it self-caps) and moves on
|
|
255
330
|
* to release() regardless.
|
|
256
331
|
*/
|
|
@@ -258,6 +333,7 @@ export async function captureWorkspaceRevision(
|
|
|
258
333
|
input: CaptureWorkspaceRevisionInput,
|
|
259
334
|
): Promise<void> {
|
|
260
335
|
const { observability } = input;
|
|
336
|
+
const stage = { value: "preflight" };
|
|
261
337
|
if (input.signal?.aborted) {
|
|
262
338
|
observability.incrementCounter({
|
|
263
339
|
name: "opengeni_workspace_capture_total",
|
|
@@ -305,6 +381,7 @@ export async function captureWorkspaceRevision(
|
|
|
305
381
|
{ ...input, objectStorage: input.objectStorage },
|
|
306
382
|
startedAt,
|
|
307
383
|
controller.signal,
|
|
384
|
+
stage,
|
|
308
385
|
).finally(() => {
|
|
309
386
|
observability.incrementGauge({
|
|
310
387
|
name: "opengeni_workspace_captures_inflight",
|
|
@@ -339,6 +416,7 @@ export async function captureWorkspaceRevision(
|
|
|
339
416
|
"opengeni.turn_id": input.turnId ?? "",
|
|
340
417
|
"error.message": error instanceof Error ? error.message : String(error),
|
|
341
418
|
"workspace_capture.duration_ms": Date.now() - startedAt,
|
|
419
|
+
"workspace_capture.stage": stage.value,
|
|
342
420
|
});
|
|
343
421
|
observability.incrementCounter({
|
|
344
422
|
name: "opengeni_workspace_capture_total",
|
|
@@ -357,22 +435,28 @@ async function runCapture(
|
|
|
357
435
|
ctx: CaptureWorkspaceRevisionInput & { objectStorage: ObjectStorage },
|
|
358
436
|
startedAt: number,
|
|
359
437
|
signal: AbortSignal,
|
|
438
|
+
stage: { value: string },
|
|
360
439
|
): Promise<void> {
|
|
361
440
|
const { observability } = input;
|
|
362
441
|
const storage = ctx.objectStorage;
|
|
363
442
|
const keepN = input.keepLatest ?? KEEP_LATEST_REVISIONS;
|
|
443
|
+
stage.value = "open_read_session";
|
|
444
|
+
const readSession = input.openReadSession ? await input.openReadSession() : input.session;
|
|
445
|
+
throwIfCaptureAborted(signal);
|
|
364
446
|
// Reads only — no emitter (we publish the announce ourselves after commit).
|
|
365
|
-
const svc = new SandboxChannelAService({ session:
|
|
447
|
+
const svc = new SandboxChannelAService({ session: readSession, leaseEpoch: input.leaseEpoch });
|
|
366
448
|
|
|
367
449
|
// Resolve the previous revision before discovery so even a failed discovery
|
|
368
450
|
// can commit a monotonic, explicit degraded marker. That marker becomes the
|
|
369
451
|
// newest read result and prevents an older successful capture from being
|
|
370
452
|
// mistaken for the current turn's authoritative workspace state.
|
|
453
|
+
stage.value = "load_previous_capture";
|
|
371
454
|
const prev = await latestWorkspaceCapture(input.db, input.workspaceId, input.sessionId);
|
|
372
455
|
throwIfCaptureAborted(signal);
|
|
373
456
|
const revision = (prev?.revision ?? -1) + 1;
|
|
374
457
|
|
|
375
458
|
// ── 1. per-repo status + diff, union the touched set ──────────────────────
|
|
459
|
+
stage.value = "repository_discovery";
|
|
376
460
|
const discovery = await svc.detectReposDetailed();
|
|
377
461
|
throwIfCaptureAborted(signal);
|
|
378
462
|
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
@@ -400,6 +484,7 @@ async function runCapture(
|
|
|
400
484
|
let deletions = 0;
|
|
401
485
|
|
|
402
486
|
for (const root of repoRoots) {
|
|
487
|
+
stage.value = "repository_read";
|
|
403
488
|
throwIfCaptureAborted(signal);
|
|
404
489
|
// Repository discovery defines the authoritative set. Once a root is in
|
|
405
490
|
// that set, status/diff failure must degrade the entire revision rather than
|
|
@@ -475,6 +560,7 @@ async function runCapture(
|
|
|
475
560
|
let binaryCount = 0;
|
|
476
561
|
|
|
477
562
|
for (const [wsPath, info] of touched) {
|
|
563
|
+
stage.value = "file_read";
|
|
478
564
|
throwIfCaptureAborted(signal);
|
|
479
565
|
if (info.deleted) {
|
|
480
566
|
files.push({
|
|
@@ -580,6 +666,7 @@ async function runCapture(
|
|
|
580
666
|
}
|
|
581
667
|
|
|
582
668
|
// ── 5. tree index (one bounded listing; residue dirs pruned at source) ─────
|
|
669
|
+
stage.value = "tree_index";
|
|
583
670
|
const tree = await buildTreeIndex(svc, startedAt, signal);
|
|
584
671
|
throwIfCaptureAborted(signal);
|
|
585
672
|
|
|
@@ -602,6 +689,7 @@ async function runCapture(
|
|
|
602
689
|
let refreshedFiles = 0;
|
|
603
690
|
const omittedFiles = new Set<string>();
|
|
604
691
|
for (const file of files) {
|
|
692
|
+
stage.value = "content_upload";
|
|
605
693
|
throwIfCaptureAborted(signal);
|
|
606
694
|
if (file.deleted || file.tooLarge || !file.contentRef || !file.hash) {
|
|
607
695
|
continue;
|
|
@@ -730,10 +818,12 @@ async function runCapture(
|
|
|
730
818
|
entryCount: tree.entryCount,
|
|
731
819
|
}),
|
|
732
820
|
);
|
|
821
|
+
stage.value = "tree_upload";
|
|
733
822
|
await storage.putObject({ key: treeKey, contentType: "application/json", body: treeBytes });
|
|
734
823
|
throwIfCaptureAborted(signal);
|
|
735
824
|
stats.durationMs = Date.now() - startedAt;
|
|
736
825
|
const manifestBytes = utf8(JSON.stringify(manifest));
|
|
826
|
+
stage.value = "manifest_upload";
|
|
737
827
|
await storage.putObject({
|
|
738
828
|
key: manifestKey,
|
|
739
829
|
contentType: "application/json",
|
|
@@ -743,6 +833,7 @@ async function runCapture(
|
|
|
743
833
|
const sizeBytes = totalBytes + treeBytes.byteLength + manifestBytes.byteLength;
|
|
744
834
|
|
|
745
835
|
// ── 8. epoch-fenced insert (superseded lease → zero rows) ─────────────────
|
|
836
|
+
stage.value = "database_commit";
|
|
746
837
|
const inserted = await insertWorkspaceCapture(input.db, {
|
|
747
838
|
accountId: input.accountId,
|
|
748
839
|
workspaceId: input.workspaceId,
|
package/src/activities.ts
CHANGED
|
@@ -36,6 +36,8 @@ export type {
|
|
|
36
36
|
ReconcileCodexCapacityWaitResult,
|
|
37
37
|
RecoverDispatchInput,
|
|
38
38
|
RecoverDispatchResult,
|
|
39
|
+
RecoverEscapedMcpTimeoutInput,
|
|
40
|
+
RecoverEscapedMcpTimeoutResult,
|
|
39
41
|
PersistSessionAttemptQuiescenceInput,
|
|
40
42
|
ReconcileSessionAttemptQuiescenceInput,
|
|
41
43
|
ReconcileSessionAttemptQuiescenceResult,
|
|
@@ -167,6 +169,7 @@ export const persistSessionAttemptQuiescence =
|
|
|
167
169
|
export const reconcileSessionAttemptQuiescence =
|
|
168
170
|
defaultControlActivities.reconcileSessionAttemptQuiescence;
|
|
169
171
|
export const recoverDispatch = defaultControlActivities.recoverDispatch;
|
|
172
|
+
export const recoverEscapedMcpTimeout = defaultControlActivities.recoverEscapedMcpTimeout;
|
|
170
173
|
export const peekSessionWork = defaultControlActivities.peekSessionWork;
|
|
171
174
|
export const expireSessionHumanInput = defaultControlActivities.expireSessionHumanInput;
|
|
172
175
|
export const markSessionIdle = defaultControlActivities.markSessionIdle;
|
|
@@ -384,6 +384,97 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
|
|
|
384
384
|
});
|
|
385
385
|
}
|
|
386
386
|
|
|
387
|
+
export type SandboxCheckpointArtifactMetricState =
|
|
388
|
+
| "candidate"
|
|
389
|
+
| "current"
|
|
390
|
+
| "previous"
|
|
391
|
+
| "delete_pending"
|
|
392
|
+
| "deleting"
|
|
393
|
+
| "delete_failed"
|
|
394
|
+
| "deleted";
|
|
395
|
+
|
|
396
|
+
export function recordSandboxCheckpointArtifactGauges(
|
|
397
|
+
observability: Observability,
|
|
398
|
+
counts: Record<SandboxCheckpointArtifactMetricState, number>,
|
|
399
|
+
): void {
|
|
400
|
+
for (const state of [
|
|
401
|
+
"candidate",
|
|
402
|
+
"current",
|
|
403
|
+
"previous",
|
|
404
|
+
"delete_pending",
|
|
405
|
+
"deleting",
|
|
406
|
+
"delete_failed",
|
|
407
|
+
"deleted",
|
|
408
|
+
] as const) {
|
|
409
|
+
observability.setGauge({
|
|
410
|
+
name: "opengeni_sandbox_checkpoint_artifacts",
|
|
411
|
+
help: "Current durable provider checkpoint artifacts by lifecycle state.",
|
|
412
|
+
labels: { state },
|
|
413
|
+
value: counts[state],
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
export type SandboxCheckpointArtifactOutcome =
|
|
419
|
+
| "legacy_adopted"
|
|
420
|
+
| "claimed"
|
|
421
|
+
| "deleted"
|
|
422
|
+
| "delete_failed"
|
|
423
|
+
| "tombstone_pruned";
|
|
424
|
+
|
|
425
|
+
export function recordSandboxCheckpointArtifactOutcome(
|
|
426
|
+
observability: Observability,
|
|
427
|
+
outcome: SandboxCheckpointArtifactOutcome,
|
|
428
|
+
count = 1,
|
|
429
|
+
): void {
|
|
430
|
+
if (count <= 0) return;
|
|
431
|
+
observability.incrementCounter({
|
|
432
|
+
name: "opengeni_sandbox_checkpoint_artifact_operations_total",
|
|
433
|
+
help: "Durable provider checkpoint artifact operations by fixed outcome.",
|
|
434
|
+
labels: { outcome },
|
|
435
|
+
amount: count,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
export function recordSandboxDeadlineRotationsRequested(
|
|
440
|
+
observability: Observability,
|
|
441
|
+
count: number,
|
|
442
|
+
): void {
|
|
443
|
+
if (count <= 0) return;
|
|
444
|
+
observability.incrementCounter({
|
|
445
|
+
name: "opengeni_sandbox_deadline_rotations_requested_total",
|
|
446
|
+
help: "Total finite-lifetime sandbox rotations requested before provider deadline.",
|
|
447
|
+
amount: count,
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
export function recordSandboxRotationBacklogGauges(
|
|
452
|
+
observability: Observability,
|
|
453
|
+
backlog: {
|
|
454
|
+
requested: number;
|
|
455
|
+
overdue: number;
|
|
456
|
+
turnBlocked: number;
|
|
457
|
+
directBlocked: number;
|
|
458
|
+
processBlocked: number;
|
|
459
|
+
},
|
|
460
|
+
): void {
|
|
461
|
+
const values = {
|
|
462
|
+
requested: backlog.requested,
|
|
463
|
+
overdue: backlog.overdue,
|
|
464
|
+
turn_blocked: backlog.turnBlocked,
|
|
465
|
+
direct_blocked: backlog.directBlocked,
|
|
466
|
+
process_blocked: backlog.processBlocked,
|
|
467
|
+
} as const;
|
|
468
|
+
for (const [kind, value] of Object.entries(values)) {
|
|
469
|
+
observability.setGauge({
|
|
470
|
+
name: "opengeni_sandbox_rotation_backlog",
|
|
471
|
+
help: "Current finite-lifetime sandbox rotation backlog by fixed condition.",
|
|
472
|
+
labels: { kind },
|
|
473
|
+
value,
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
387
478
|
const RETAINED_PROCESS_OWNER_STATES = [
|
|
388
479
|
"direct",
|
|
389
480
|
"queued",
|
|
@@ -503,6 +594,9 @@ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
|
|
|
503
594
|
"provider_unknown",
|
|
504
595
|
"provider_timeout",
|
|
505
596
|
"provider_error",
|
|
597
|
+
"provider_binding_missing",
|
|
598
|
+
"provider_binding_mismatch",
|
|
599
|
+
"provider_binding_adopted",
|
|
506
600
|
"defer_failed",
|
|
507
601
|
] as const;
|
|
508
602
|
|