@opengeni/worker-bundle 0.7.5 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2812 -813
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2045 -545
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -6,6 +6,8 @@ import {
|
|
|
6
6
|
countQueuedTurns,
|
|
7
7
|
getSessionEvent,
|
|
8
8
|
getSessionTurnForAttempt,
|
|
9
|
+
expireSessionHumanInputRequest,
|
|
10
|
+
markSessionAttemptQuiesced,
|
|
9
11
|
requireSession,
|
|
10
12
|
settleSessionIdleWithParentOutbox,
|
|
11
13
|
} from "@opengeni/db";
|
|
@@ -14,10 +16,13 @@ import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./paren
|
|
|
14
16
|
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
15
17
|
import type {
|
|
16
18
|
ActivityServices,
|
|
19
|
+
ExpireSessionHumanInputInput,
|
|
20
|
+
ExpireSessionHumanInputResult,
|
|
17
21
|
PeekSessionWorkInput,
|
|
18
22
|
FailSessionAttemptInput,
|
|
19
23
|
SettleSessionInterruptionsInput,
|
|
20
24
|
MarkSessionIdleInput,
|
|
25
|
+
PersistSessionAttemptQuiescenceInput,
|
|
21
26
|
RecoverDispatchInput,
|
|
22
27
|
RecoverDispatchResult,
|
|
23
28
|
} from "./types";
|
|
@@ -30,8 +35,10 @@ export type SessionStateActivityOverrides = Partial<{
|
|
|
30
35
|
countQueuedTurns: typeof countQueuedTurns;
|
|
31
36
|
getSessionEvent: typeof getSessionEvent;
|
|
32
37
|
getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
|
|
38
|
+
expireSessionHumanInputRequest: typeof expireSessionHumanInputRequest;
|
|
33
39
|
requireSession: typeof requireSession;
|
|
34
40
|
settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
|
|
41
|
+
markSessionAttemptQuiesced: typeof markSessionAttemptQuiesced;
|
|
35
42
|
publishDurableSessionEvents: typeof publishDurableSessionEvents;
|
|
36
43
|
deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
|
|
37
44
|
notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
|
|
@@ -57,9 +64,13 @@ export function createSessionStateActivities(
|
|
|
57
64
|
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
|
|
58
65
|
const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
|
|
59
66
|
const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
|
|
67
|
+
const expireSessionHumanInputRequestFn =
|
|
68
|
+
overrides.expireSessionHumanInputRequest ?? expireSessionHumanInputRequest;
|
|
60
69
|
const requireSessionFn = overrides.requireSession ?? requireSession;
|
|
61
70
|
const settleSessionIdleWithParentOutboxFn =
|
|
62
71
|
overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
|
|
72
|
+
const markSessionAttemptQuiescedFn =
|
|
73
|
+
overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
|
|
63
74
|
const publishDurableSessionEventsFn =
|
|
64
75
|
overrides.publishDurableSessionEvents ?? publishDurableSessionEvents;
|
|
65
76
|
const deliverFailedChildTurnToParentFn =
|
|
@@ -117,6 +128,19 @@ export function createSessionStateActivities(
|
|
|
117
128
|
input: SettleSessionInterruptionsInput,
|
|
118
129
|
): Promise<{ action: "paused" | "continue" | "stale" }> {
|
|
119
130
|
const { db, bus, observability } = await services();
|
|
131
|
+
if (input.phase === "attempt_quiesced") {
|
|
132
|
+
// Replay compatibility only: v1 histories scheduled this idempotent
|
|
133
|
+
// fallback after WAIT_CANCELLATION_COMPLETED. Receipt-gated v2 workflows
|
|
134
|
+
// never call it; runAgentTurn writes immediately after its hard fence.
|
|
135
|
+
const events = await markSessionAttemptQuiescedFn(db, {
|
|
136
|
+
workspaceId: input.workspaceId,
|
|
137
|
+
sessionId: input.sessionId,
|
|
138
|
+
attemptId: input.attemptId,
|
|
139
|
+
temporalWorkflowId: input.workflowId,
|
|
140
|
+
});
|
|
141
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
|
|
142
|
+
return { action: "stale" };
|
|
143
|
+
}
|
|
120
144
|
const applied = await settleSessionAttemptInterruptionsFn(
|
|
121
145
|
db,
|
|
122
146
|
input.workspaceId,
|
|
@@ -130,6 +154,38 @@ export function createSessionStateActivities(
|
|
|
130
154
|
return { action: applied.action };
|
|
131
155
|
}
|
|
132
156
|
|
|
157
|
+
/** Persist an exact activity-owned physical-quiescence proof through the
|
|
158
|
+
* workflow control-activity retry policy. The DB transaction remains the
|
|
159
|
+
* sole receipt/wake authority; duplicate signals and activity retries reuse
|
|
160
|
+
* its attempt-scoped idempotency key. */
|
|
161
|
+
async function persistSessionAttemptQuiescence(
|
|
162
|
+
input: PersistSessionAttemptQuiescenceInput,
|
|
163
|
+
): Promise<void> {
|
|
164
|
+
const { db, bus, observability } = await services();
|
|
165
|
+
const events = await markSessionAttemptQuiescedFn(db, {
|
|
166
|
+
accountId: input.accountId,
|
|
167
|
+
workspaceId: input.workspaceId,
|
|
168
|
+
sessionId: input.sessionId,
|
|
169
|
+
attemptId: input.attemptId,
|
|
170
|
+
temporalWorkflowId: input.workflowId,
|
|
171
|
+
temporalWorkflowRunId: input.workflowRunId,
|
|
172
|
+
temporalActivityId: input.activityId,
|
|
173
|
+
allowUninterrupted: true,
|
|
174
|
+
});
|
|
175
|
+
try {
|
|
176
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, events);
|
|
177
|
+
} catch (error) {
|
|
178
|
+
// The receipt and exact workflow wake already committed atomically in
|
|
179
|
+
// Postgres. NATS is best-effort live fanout and must not keep this
|
|
180
|
+
// control activity retrying or delay receipt-gated admission.
|
|
181
|
+
observability.error("session-attempt quiescence event fanout failed", {
|
|
182
|
+
"opengeni.session_id": input.sessionId,
|
|
183
|
+
"opengeni.attempt_id": input.attemptId,
|
|
184
|
+
error: error instanceof Error ? error.message : String(error),
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
133
189
|
/**
|
|
134
190
|
* Recover the same current inference when its worker dies without completing
|
|
135
191
|
* a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
|
|
@@ -177,6 +233,20 @@ export function createSessionStateActivities(
|
|
|
177
233
|
return peek;
|
|
178
234
|
}
|
|
179
235
|
|
|
236
|
+
async function expireSessionHumanInput(
|
|
237
|
+
input: ExpireSessionHumanInputInput,
|
|
238
|
+
): Promise<ExpireSessionHumanInputResult> {
|
|
239
|
+
const { db, bus } = await services();
|
|
240
|
+
const result = await expireSessionHumanInputRequestFn(db, input);
|
|
241
|
+
if (result.action === "not_found") return { action: "not_found" };
|
|
242
|
+
if (result.events.length > 0) {
|
|
243
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
action: result.request.status === "expired" ? "expired" : "stale",
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
180
250
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
181
251
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
182
252
|
const settled = await settleSessionIdleWithParentOutboxFn(
|
|
@@ -207,8 +277,10 @@ export function createSessionStateActivities(
|
|
|
207
277
|
return {
|
|
208
278
|
failSessionAttempt,
|
|
209
279
|
settleSessionInterruptions,
|
|
280
|
+
persistSessionAttemptQuiescence,
|
|
210
281
|
recoverDispatch,
|
|
211
282
|
peekSessionWork,
|
|
283
|
+
expireSessionHumanInput,
|
|
212
284
|
markSessionIdle,
|
|
213
285
|
};
|
|
214
286
|
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import {
|
|
2
|
+
startExpiringMaterialRenewalLoop,
|
|
3
|
+
type ExpiringMaterialRenewalController,
|
|
4
|
+
} from "./expiring-material-renewal";
|
|
5
|
+
import type { MintedSandboxToolspaceToken } from "./environment";
|
|
6
|
+
|
|
7
|
+
export const TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS = 30 * 60_000;
|
|
8
|
+
export const TOOLSPACE_TOKEN_EXPIRY_LEAD_MS = 5 * 60_000;
|
|
9
|
+
export const TOOLSPACE_TOKEN_MIN_REFRESH_MS = 5_000;
|
|
10
|
+
export const TOOLSPACE_TOKEN_MAX_RETRY_MS = 5 * 60_000;
|
|
11
|
+
|
|
12
|
+
export type ToolspaceTokenRenewalController = ExpiringMaterialRenewalController;
|
|
13
|
+
|
|
14
|
+
export type ToolspaceTokenRenewalOptions = {
|
|
15
|
+
initialExpiresAt: Date;
|
|
16
|
+
mint: () => Promise<MintedSandboxToolspaceToken | undefined>;
|
|
17
|
+
write: (material: MintedSandboxToolspaceToken) => Promise<void>;
|
|
18
|
+
now?: () => number;
|
|
19
|
+
schedule?: (callback: () => void, delayMs: number) => unknown;
|
|
20
|
+
clearSchedule?: (timer: unknown) => void;
|
|
21
|
+
onSuccess?: (result: { nextDelayMs: number }) => void;
|
|
22
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
class ToolspaceTokenRenewalError extends Error {
|
|
26
|
+
override readonly name = "ToolspaceTokenRenewalError";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function startToolspaceTokenRenewalLoop(
|
|
30
|
+
options: ToolspaceTokenRenewalOptions,
|
|
31
|
+
): ToolspaceTokenRenewalController {
|
|
32
|
+
return startExpiringMaterialRenewalLoop<MintedSandboxToolspaceToken>({
|
|
33
|
+
initialExpiresAt: options.initialExpiresAt,
|
|
34
|
+
resolve: async () => {
|
|
35
|
+
const minted = await options.mint();
|
|
36
|
+
if (!minted) {
|
|
37
|
+
throw new ToolspaceTokenRenewalError("Toolspace token mint is no longer available");
|
|
38
|
+
}
|
|
39
|
+
return minted;
|
|
40
|
+
},
|
|
41
|
+
write: options.write,
|
|
42
|
+
expiresAt: (material) => material.expiresAt,
|
|
43
|
+
policy: {
|
|
44
|
+
defaultRefreshMs: TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS,
|
|
45
|
+
expiryLeadMs: TOOLSPACE_TOKEN_EXPIRY_LEAD_MS,
|
|
46
|
+
minRefreshMs: TOOLSPACE_TOKEN_MIN_REFRESH_MS,
|
|
47
|
+
maxRetryMs: TOOLSPACE_TOKEN_MAX_RETRY_MS,
|
|
48
|
+
},
|
|
49
|
+
...(options.now ? { now: options.now } : {}),
|
|
50
|
+
...(options.schedule ? { schedule: options.schedule } : {}),
|
|
51
|
+
...(options.clearSchedule ? { clearSchedule: options.clearSchedule } : {}),
|
|
52
|
+
...(options.onSuccess
|
|
53
|
+
? { onSuccess: ({ nextDelayMs }) => options.onSuccess?.({ nextDelayMs }) }
|
|
54
|
+
: {}),
|
|
55
|
+
...(options.onFailure ? { onFailure: options.onFailure } : {}),
|
|
56
|
+
});
|
|
57
|
+
}
|
package/src/activities/types.ts
CHANGED
|
@@ -36,6 +36,21 @@ export type SignalCodexCapacityWorkflow = (input: {
|
|
|
36
36
|
wakeRevision: number;
|
|
37
37
|
}) => Promise<void>;
|
|
38
38
|
|
|
39
|
+
/** Exact activity-owned proof that the hard sandbox/tool fence physically
|
|
40
|
+
* drained. This is delivery evidence only: the workflow still validates the
|
|
41
|
+
* persisted attempt dispatch and commits the authoritative Postgres receipt. */
|
|
42
|
+
export type SessionAttemptQuiescenceProof = {
|
|
43
|
+
accountId: string;
|
|
44
|
+
workspaceId: string;
|
|
45
|
+
sessionId: string;
|
|
46
|
+
attemptId: string;
|
|
47
|
+
workflowId: string;
|
|
48
|
+
workflowRunId: string;
|
|
49
|
+
activityId: string;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export type SignalSessionAttemptQuiesced = (input: SessionAttemptQuiescenceProof) => Promise<void>;
|
|
53
|
+
|
|
39
54
|
export type ActivityServices = {
|
|
40
55
|
settings: Settings;
|
|
41
56
|
db: Database;
|
|
@@ -45,6 +60,9 @@ export type ActivityServices = {
|
|
|
45
60
|
documentServices: DocumentServices;
|
|
46
61
|
observability: Observability;
|
|
47
62
|
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
63
|
+
/** Durable signalWithStart fallback used only after the activity's direct
|
|
64
|
+
* physical-quiescence receipt write exhausts its bounded DB retries. */
|
|
65
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced | null;
|
|
48
66
|
/** Revision-carrying capacity nudge; generic outbox repair is also sufficient. */
|
|
49
67
|
signalCodexCapacityWorkflow?: SignalCodexCapacityWorkflow | null;
|
|
50
68
|
// §7.5 P3 — host-entitlements port, the WORKER half of the same seam the API
|
|
@@ -61,7 +79,7 @@ export type ActivityServices = {
|
|
|
61
79
|
// admission and metering are separate operations, and only metering carries
|
|
62
80
|
// the idempotency key.
|
|
63
81
|
entitlements?: EntitlementsPort | null;
|
|
64
|
-
// §7.6
|
|
82
|
+
// §7.6 connection-credential provider — host connection-credential provider, the WORKER half of the
|
|
65
83
|
// federated-connection boundary. When set, the run's per-run credential mint
|
|
66
84
|
// delegates to the host instead of self-minting from `settings`:
|
|
67
85
|
// - `gitCredentials` REPLACES `createGitHubAppInstallationToken(settings,…)`
|
|
@@ -72,7 +90,7 @@ export type ActivityServices = {
|
|
|
72
90
|
// self-mint for THAT leg. null/undefined (standalone default) → both legs
|
|
73
91
|
// self-mint byte-for-byte as today.
|
|
74
92
|
//
|
|
75
|
-
//
|
|
93
|
+
// workspace-scope cross-check CROSS-CHECK: a provider echoes the `workspaceId` it scoped the
|
|
76
94
|
// credential to; the consuming activity ASSERTS agreement with the run's
|
|
77
95
|
// workspace BEFORE injecting `GH_TOKEN` (or applying decrypted values). A host
|
|
78
96
|
// mapping bug returning tenant B's creds for a tenant-A run is caught here.
|
|
@@ -123,8 +141,16 @@ export type SettleSessionInterruptionsInput = {
|
|
|
123
141
|
sessionId: string;
|
|
124
142
|
attemptId: string;
|
|
125
143
|
workflowId: string;
|
|
144
|
+
/**
|
|
145
|
+
* Replay-only compatibility for session workflow histories created before
|
|
146
|
+
* the receipt-gated cancellation v2 patch. New histories never send this
|
|
147
|
+
* phase; the exact activity writes the authoritative receipt itself.
|
|
148
|
+
*/
|
|
149
|
+
phase?: "logical" | "attempt_quiesced";
|
|
126
150
|
};
|
|
127
151
|
|
|
152
|
+
export type PersistSessionAttemptQuiescenceInput = SessionAttemptQuiescenceProof;
|
|
153
|
+
|
|
128
154
|
export type FailSessionAttemptInput = {
|
|
129
155
|
accountId: string;
|
|
130
156
|
workspaceId: string;
|
|
@@ -160,6 +186,17 @@ export type PeekSessionWorkInput = {
|
|
|
160
186
|
sessionId: string;
|
|
161
187
|
};
|
|
162
188
|
|
|
189
|
+
export type ExpireSessionHumanInputInput = {
|
|
190
|
+
accountId: string;
|
|
191
|
+
workspaceId: string;
|
|
192
|
+
sessionId: string;
|
|
193
|
+
requestId: string;
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
export type ExpireSessionHumanInputResult = {
|
|
197
|
+
action: "expired" | "stale" | "not_found";
|
|
198
|
+
};
|
|
199
|
+
|
|
163
200
|
export type MarkSessionIdleInput = {
|
|
164
201
|
workspaceId: string;
|
|
165
202
|
sessionId: string;
|
|
@@ -220,10 +257,11 @@ type ClaimedRunAgentTurnResult = {
|
|
|
220
257
|
// persisted in Postgres and reconstructed after workflow/worker restart.
|
|
221
258
|
// The workflow must not call maybeContinueGoal while this waiter is active.
|
|
222
259
|
capacityWait?: CodexCapacityWaitRef;
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
226
|
-
// prompt
|
|
260
|
+
// This execution reached a durable terminal-for-now boundary (for example,
|
|
261
|
+
// maintenance could not run or same-turn context recovery failed). End this
|
|
262
|
+
// workflow run without synthesizing another goal continuation from unchanged
|
|
263
|
+
// state. A later prompt/control/new-update wake may retry through normal claim
|
|
264
|
+
// ordering.
|
|
227
265
|
deferredUntilWake?: boolean;
|
|
228
266
|
};
|
|
229
267
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Workbench v2 — turn-end workspace capture
|
|
1
|
+
// Workbench v2 — turn-end workspace capture.
|
|
2
2
|
//
|
|
3
3
|
// The universal spine that makes the session workspace paint instantly on cold /
|
|
4
4
|
// offline reads: at TURN END, while the box is still guaranteed live and the
|
|
@@ -10,24 +10,25 @@
|
|
|
10
10
|
// this when the machine is cold/offline; a warm box always wins (capture is a
|
|
11
11
|
// labelled cache, never a replacement).
|
|
12
12
|
//
|
|
13
|
-
// SAFETY INVARIANTS (non-negotiable — see
|
|
13
|
+
// SAFETY INVARIANTS (non-negotiable — see):
|
|
14
14
|
// • This module NEVER calls close()/terminate()/kill() on the session handle.
|
|
15
15
|
// session.close() == terminate() on Modal and would kill the user's box.
|
|
16
16
|
// Only the reaper terminates. We drop references, nothing more.
|
|
17
17
|
// • The whole capture is time-capped (60s) and best-effort: ANY failure logs
|
|
18
18
|
// "workspace capture failed — turn outcome unaffected" and returns; nothing
|
|
19
19
|
// throws past captureWorkspaceRevision.
|
|
20
|
-
// • The DB write is fenced on the
|
|
21
|
-
// rows
|
|
20
|
+
// • The DB write is fenced on the exact attempt, accepted control requests,
|
|
21
|
+
// and lease epoch — a cancelled attempt or superseded lease writes zero rows
|
|
22
|
+
// (insertWorkspaceCapture).
|
|
22
23
|
// • F9 storage ordering: blobs + manifest are PUT and the row is committed
|
|
23
24
|
// BEFORE any GC delete runs.
|
|
24
25
|
//
|
|
25
26
|
// It is deliberately an EXTERNAL module with a SINGLE call-site line in
|
|
26
|
-
// agent-turn.ts's finally (
|
|
27
|
+
// agent-turn.ts's finally (the accepted design tradeoff — do not grow the turn
|
|
27
28
|
// activity). The emitted `workspace.revision.captured` event is ANNOUNCE-ONLY
|
|
28
29
|
// (metadata, never content) and must never gain a rendered timeline item.
|
|
29
30
|
|
|
30
|
-
import { createHash
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
31
32
|
import type { Settings } from "@opengeni/config";
|
|
32
33
|
import type { Database } from "@opengeni/db";
|
|
33
34
|
import {
|
|
@@ -42,7 +43,7 @@ import type { Observability } from "@opengeni/observability";
|
|
|
42
43
|
// The un-agent-loop leaf. Constructing the Channel-A service over the un-proxied
|
|
43
44
|
// setupBoxSession here (NOT the routing veneer) guarantees a mid-turn
|
|
44
45
|
// sandbox_swap can never re-route capture execs onto a user's machine — the same
|
|
45
|
-
// reason the snapshot uses setupBoxSession
|
|
46
|
+
// reason the snapshot uses setupBoxSession.
|
|
46
47
|
import {
|
|
47
48
|
SandboxChannelAService,
|
|
48
49
|
type ChannelASession,
|
|
@@ -60,7 +61,7 @@ import type {
|
|
|
60
61
|
WorkspaceCaptureStats,
|
|
61
62
|
} from "@opengeni/contracts";
|
|
62
63
|
|
|
63
|
-
// ─── Guards & constants
|
|
64
|
+
// ─── Guards & constants for pathological cases; configurable ─────
|
|
64
65
|
export const CAPTURE_TIMEOUT_MS = 60_000;
|
|
65
66
|
export const PER_FILE_CONTENT_GUARD_BYTES = 5 * 1024 * 1024; // after-image; over → tooLarge marker
|
|
66
67
|
export const PER_FILE_DIFF_GUARD_BYTES = 10 * 1024 * 1024; // per-file diff; over → truncated marker
|
|
@@ -171,7 +172,7 @@ function classifyCaptureEntryError(error: unknown): BoxExitingError | null {
|
|
|
171
172
|
return null;
|
|
172
173
|
}
|
|
173
174
|
const TREE_MAX_ENTRIES = 20_000; // whole-tree node cap (truncate beyond)
|
|
174
|
-
const
|
|
175
|
+
const TREE_MAX_DEPTH = 600; // pathological nesting cap; still one remote round-trip
|
|
175
176
|
|
|
176
177
|
// The capture row and announcement are committed together by @opengeni/db. This
|
|
177
178
|
// callback performs only best-effort live fanout of those already-durable events.
|
|
@@ -189,8 +190,15 @@ export type CaptureWorkspaceRevisionInput = {
|
|
|
189
190
|
accountId: string;
|
|
190
191
|
workspaceId: string;
|
|
191
192
|
sessionId: string;
|
|
192
|
-
turnId: string
|
|
193
|
+
turnId: string;
|
|
194
|
+
attemptId: string;
|
|
193
195
|
observability: Observability;
|
|
196
|
+
/**
|
|
197
|
+
* The owning turn activity's cancellation signal. Pause/Steer must preempt
|
|
198
|
+
* review-cache housekeeping immediately; correctness still comes from the
|
|
199
|
+
* activity plus the capture's transaction-level attempt/control/lease fences.
|
|
200
|
+
*/
|
|
201
|
+
signal?: AbortSignal;
|
|
194
202
|
/** Test-only: override keep-latest-N GC threshold (default 10). */
|
|
195
203
|
keepLatest?: number;
|
|
196
204
|
};
|
|
@@ -207,6 +215,13 @@ export async function captureWorkspaceRevision(
|
|
|
207
215
|
input: CaptureWorkspaceRevisionInput,
|
|
208
216
|
): Promise<void> {
|
|
209
217
|
const { observability } = input;
|
|
218
|
+
if (input.signal?.aborted) {
|
|
219
|
+
observability.incrementCounter({
|
|
220
|
+
name: "opengeni_workspace_capture_total",
|
|
221
|
+
labels: { result: "cancelled" },
|
|
222
|
+
});
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
210
225
|
if (!input.settings.workspaceCaptureEnabled) {
|
|
211
226
|
return; // flag off → capture skipped; reads fall back to live/wake (status quo)
|
|
212
227
|
}
|
|
@@ -224,6 +239,20 @@ export async function captureWorkspaceRevision(
|
|
|
224
239
|
|
|
225
240
|
const startedAt = Date.now();
|
|
226
241
|
const controller = new AbortController();
|
|
242
|
+
let rejectOwnerCancellation: ((reason?: unknown) => void) | undefined;
|
|
243
|
+
const ownerCancelled = input.signal
|
|
244
|
+
? new Promise<never>((_resolve, reject) => {
|
|
245
|
+
rejectOwnerCancellation = reject;
|
|
246
|
+
})
|
|
247
|
+
: new Promise<never>(() => undefined);
|
|
248
|
+
const cancelFromOwner = (): void => {
|
|
249
|
+
controller.abort(input.signal?.reason);
|
|
250
|
+
rejectOwnerCancellation?.(
|
|
251
|
+
input.signal?.reason ?? new Error("workspace capture cancelled by its owning activity"),
|
|
252
|
+
);
|
|
253
|
+
};
|
|
254
|
+
input.signal?.addEventListener("abort", cancelFromOwner, { once: true });
|
|
255
|
+
if (input.signal?.aborted) cancelFromOwner();
|
|
227
256
|
observability.incrementGauge({
|
|
228
257
|
name: "opengeni_workspace_captures_inflight",
|
|
229
258
|
help: "Current workspace-capture operations still resident in this worker process.",
|
|
@@ -244,6 +273,7 @@ export async function captureWorkspaceRevision(
|
|
|
244
273
|
try {
|
|
245
274
|
await Promise.race([
|
|
246
275
|
capture,
|
|
276
|
+
ownerCancelled,
|
|
247
277
|
new Promise<never>((_, reject) => {
|
|
248
278
|
timer = setTimeout(() => {
|
|
249
279
|
controller.abort();
|
|
@@ -252,6 +282,13 @@ export async function captureWorkspaceRevision(
|
|
|
252
282
|
}),
|
|
253
283
|
]);
|
|
254
284
|
} catch (error) {
|
|
285
|
+
if (input.signal?.aborted) {
|
|
286
|
+
observability.incrementCounter({
|
|
287
|
+
name: "opengeni_workspace_capture_total",
|
|
288
|
+
labels: { result: "cancelled" },
|
|
289
|
+
});
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
255
292
|
// The one and only place a capture failure surfaces — as a log line, never
|
|
256
293
|
// an exception past this boundary (the turn already completed).
|
|
257
294
|
observability.warn("workspace capture failed — turn outcome unaffected", {
|
|
@@ -266,6 +303,7 @@ export async function captureWorkspaceRevision(
|
|
|
266
303
|
});
|
|
267
304
|
} finally {
|
|
268
305
|
if (timer) clearTimeout(timer);
|
|
306
|
+
input.signal?.removeEventListener("abort", cancelFromOwner);
|
|
269
307
|
controller.abort();
|
|
270
308
|
}
|
|
271
309
|
}
|
|
@@ -294,6 +332,11 @@ async function runCapture(
|
|
|
294
332
|
// ── 1. per-repo status + diff, union the touched set ──────────────────────
|
|
295
333
|
const discovery = await svc.detectReposDetailed();
|
|
296
334
|
throwIfCaptureAborted(signal);
|
|
335
|
+
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
336
|
+
console.log(
|
|
337
|
+
`[sandbox-e2e] capture discovery complete=${discovery.complete} repos=${JSON.stringify(discovery.repos)} degraded=${discovery.degradedReason ?? "none"}`,
|
|
338
|
+
);
|
|
339
|
+
}
|
|
297
340
|
if (!discovery.complete) {
|
|
298
341
|
const capturedAt = new Date();
|
|
299
342
|
const reason = captureDegradedReason(discovery.degradedReason);
|
|
@@ -307,6 +350,7 @@ async function runCapture(
|
|
|
307
350
|
workspaceId: input.workspaceId,
|
|
308
351
|
sessionId: input.sessionId,
|
|
309
352
|
turnId: input.turnId,
|
|
353
|
+
attemptId: input.attemptId,
|
|
310
354
|
sandboxGroupId: input.sandboxGroupId,
|
|
311
355
|
expectedEpoch: input.leaseEpoch,
|
|
312
356
|
revision,
|
|
@@ -368,6 +412,7 @@ async function runCapture(
|
|
|
368
412
|
diff = await svc.gitDiff({
|
|
369
413
|
path: root,
|
|
370
414
|
staged: false,
|
|
415
|
+
includeUntracked: true,
|
|
371
416
|
fromRef: "HEAD",
|
|
372
417
|
pathspec: [],
|
|
373
418
|
contextLines: 3,
|
|
@@ -425,8 +470,8 @@ async function runCapture(
|
|
|
425
470
|
// dirty tree on every read-only turn. Instead we skip when the change surface
|
|
426
471
|
// is byte-identical to the previous revision — "no new revision when nothing
|
|
427
472
|
// changed" holds even with a persistently dirty tree, and content-addressed
|
|
428
|
-
// blobs already dedupe storage.
|
|
429
|
-
//
|
|
473
|
+
// blobs already dedupe storage. This preserves the intended empty-turn behavior
|
|
474
|
+
// while correctly handling a persistently dirty tree.
|
|
430
475
|
// ── 3. after-images of touched files (size-gated), content-addressed ───────
|
|
431
476
|
const files: WorkspaceCaptureFile[] = [];
|
|
432
477
|
const blobKeys = new Set<string>();
|
|
@@ -525,7 +570,7 @@ async function runCapture(
|
|
|
525
570
|
});
|
|
526
571
|
}
|
|
527
572
|
|
|
528
|
-
// ── 4. empty-turn gate (
|
|
573
|
+
// ── 4. empty-turn gate (B3) ───────────────────────────────
|
|
529
574
|
// Skip when the change surface is byte-identical to the previous revision.
|
|
530
575
|
// Fires BEFORE the tree BFS + storage PUTs (the expensive parts) so a no-op
|
|
531
576
|
// read-only turn on a persistently dirty tree costs only the (small) status/
|
|
@@ -539,14 +584,14 @@ async function runCapture(
|
|
|
539
584
|
return;
|
|
540
585
|
}
|
|
541
586
|
|
|
542
|
-
// ── 5. tree index (bounded
|
|
587
|
+
// ── 5. tree index (one bounded listing; residue dirs pruned at source) ─────
|
|
543
588
|
const tree = await buildTreeIndex(svc, startedAt, signal);
|
|
544
589
|
throwIfCaptureAborted(signal);
|
|
545
590
|
|
|
546
591
|
// ── 6. PUT blobs + tree + manifest (F9: all writes BEFORE any delete) ──────
|
|
547
592
|
// Key manifest/tree by the turn (one capture per turn) so the key is known
|
|
548
593
|
// before the revision is committed; content blobs are content-addressed.
|
|
549
|
-
const turnKey = input.turnId
|
|
594
|
+
const turnKey = input.turnId;
|
|
550
595
|
const treeKey = `workspace-captures/${input.workspaceId}/${input.sessionId}/trees/${turnKey}.json`;
|
|
551
596
|
const manifestKey = `workspace-captures/${input.workspaceId}/${input.sessionId}/manifests/${turnKey}.json`;
|
|
552
597
|
|
|
@@ -708,6 +753,7 @@ async function runCapture(
|
|
|
708
753
|
workspaceId: input.workspaceId,
|
|
709
754
|
sessionId: input.sessionId,
|
|
710
755
|
turnId: input.turnId,
|
|
756
|
+
attemptId: input.attemptId,
|
|
711
757
|
sandboxGroupId: input.sandboxGroupId,
|
|
712
758
|
expectedEpoch: input.leaseEpoch,
|
|
713
759
|
revision,
|
|
@@ -718,6 +764,11 @@ async function runCapture(
|
|
|
718
764
|
stats,
|
|
719
765
|
capturedAt,
|
|
720
766
|
});
|
|
767
|
+
if (process.env.OPENGENI_TEST_SCENARIO === "sandbox") {
|
|
768
|
+
console.log(
|
|
769
|
+
`[sandbox-e2e] capture commit inserted=${Boolean(inserted)} revision=${revision} epoch=${input.leaseEpoch} group=${input.sandboxGroupId}`,
|
|
770
|
+
);
|
|
771
|
+
}
|
|
721
772
|
throwIfCaptureAborted(signal);
|
|
722
773
|
if (!inserted) {
|
|
723
774
|
// Lease superseded/released between capture and commit. Best-effort clean up
|
|
@@ -871,94 +922,63 @@ async function safeDelete(
|
|
|
871
922
|
}
|
|
872
923
|
|
|
873
924
|
/**
|
|
874
|
-
* Build the workspace tree index
|
|
875
|
-
*
|
|
876
|
-
*
|
|
877
|
-
*
|
|
878
|
-
*
|
|
925
|
+
* Build the workspace tree index in ONE remote filesystem round-trip. Channel-A
|
|
926
|
+
* prunes residue directories inside `find` while still emitting their collapsed
|
|
927
|
+
* node, so node_modules cannot consume the authored-file budget. The former
|
|
928
|
+
* serial per-directory BFS took 55.8 seconds for a 4,522-entry production tree
|
|
929
|
+
* and blocked Steer cancellation. This path is bounded by entry count, nesting
|
|
930
|
+
* depth, the outer capture deadline, and the owning activity's cancellation.
|
|
879
931
|
*/
|
|
880
932
|
async function buildTreeIndex(
|
|
881
933
|
svc: SandboxChannelAService,
|
|
882
934
|
startedAt: number,
|
|
883
935
|
signal: AbortSignal,
|
|
884
936
|
): Promise<{ root: FsTreeNode; entryCount: number; truncated: boolean }> {
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
while (queue.length > 0) {
|
|
937
|
+
throwIfCaptureAborted(signal);
|
|
938
|
+
if (Date.now() - startedAt > CAPTURE_TIMEOUT_MS - 5_000) {
|
|
939
|
+
throw new Error("workspace capture tree index reached its deadline before listing");
|
|
940
|
+
}
|
|
941
|
+
let listing: Awaited<ReturnType<typeof svc.fsListPruned>>;
|
|
942
|
+
try {
|
|
943
|
+
listing = await svc.fsListPruned(
|
|
944
|
+
{
|
|
945
|
+
path: "",
|
|
946
|
+
depth: TREE_MAX_DEPTH,
|
|
947
|
+
maxEntries: TREE_MAX_ENTRIES,
|
|
948
|
+
includeHidden: true,
|
|
949
|
+
},
|
|
950
|
+
RESIDUE_DIRS,
|
|
951
|
+
);
|
|
902
952
|
throwIfCaptureAborted(signal);
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
953
|
+
} catch (error) {
|
|
954
|
+
if (isBoxExitingError(error)) {
|
|
955
|
+
throw new BoxExitingError(error instanceof Error ? error.message : String(error));
|
|
906
956
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
throwIfCaptureAborted(signal);
|
|
921
|
-
} catch (error) {
|
|
922
|
-
if (isBoxExitingError(error)) {
|
|
923
|
-
throw new BoxExitingError(error instanceof Error ? error.message : String(error));
|
|
924
|
-
}
|
|
925
|
-
truncated = true; // this subtree is incomplete; continue with the rest
|
|
957
|
+
throw error;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
let entryCount = 0;
|
|
961
|
+
let depthTruncated = false;
|
|
962
|
+
const stack = [...(listing.root.children ?? [])];
|
|
963
|
+
while (stack.length > 0) {
|
|
964
|
+
const node = stack.pop()!;
|
|
965
|
+
entryCount += 1;
|
|
966
|
+
if (node.type !== "dir") continue;
|
|
967
|
+
if (RESIDUE_DIR_SET.has(node.name)) {
|
|
968
|
+
node.truncated = true;
|
|
969
|
+
node.children = [];
|
|
926
970
|
continue;
|
|
927
971
|
}
|
|
928
|
-
if (
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
if (entryCount >= TREE_MAX_ENTRIES) {
|
|
933
|
-
truncated = true;
|
|
934
|
-
break;
|
|
935
|
-
}
|
|
936
|
-
const node: FsTreeNode = {
|
|
937
|
-
name: child.name,
|
|
938
|
-
path: child.path,
|
|
939
|
-
type: child.type,
|
|
940
|
-
sizeBytes: child.sizeBytes,
|
|
941
|
-
mtimeMs: child.mtimeMs,
|
|
942
|
-
mode: child.mode,
|
|
943
|
-
truncated: false,
|
|
944
|
-
...(child.type === "dir" ? { children: [] as FsTreeNode[] } : {}),
|
|
945
|
-
};
|
|
946
|
-
(parent.children ??= []).push(node);
|
|
947
|
-
byPath.set(node.path, node);
|
|
948
|
-
entryCount += 1;
|
|
949
|
-
if (node.type === "dir") {
|
|
950
|
-
if (RESIDUE_DIR_SET.has(node.name)) {
|
|
951
|
-
node.truncated = true; // collapsed residue dir — contents on the machine
|
|
952
|
-
} else {
|
|
953
|
-
queue.push(node.path);
|
|
954
|
-
}
|
|
955
|
-
}
|
|
972
|
+
if (node.path.split("/").length >= TREE_MAX_DEPTH) {
|
|
973
|
+
node.truncated = true;
|
|
974
|
+
depthTruncated = true;
|
|
975
|
+
continue;
|
|
956
976
|
}
|
|
977
|
+
stack.push(...(node.children ?? []));
|
|
957
978
|
}
|
|
958
|
-
return {
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
return node.children ?? [];
|
|
979
|
+
return {
|
|
980
|
+
root: listing.root,
|
|
981
|
+
entryCount,
|
|
982
|
+
truncated: listing.truncated || depthTruncated,
|
|
983
|
+
};
|
|
964
984
|
}
|