@opengeni/worker-bundle 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +184 -68
- package/dist/index.js +7099 -1643
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4332 -1068
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +14 -2
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +216 -42
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +165 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +106 -36
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflows/activities.ts +65 -8
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -192
- package/src/workflows/workflow-wake.ts +13 -0
- package/src/workflows.ts +9 -7
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import type { Session, SessionGoal } from "@opengeni/contracts";
|
|
3
|
-
import {
|
|
2
|
+
import type { Session, SessionGoal, SessionSystemUpdatePayload } from "@opengeni/contracts";
|
|
3
|
+
import {
|
|
4
|
+
getSession,
|
|
5
|
+
getSessionGoal,
|
|
6
|
+
addSessionSystemUpdateWithSourceMutation,
|
|
7
|
+
claimPendingSessionSystemUpdateOutbox,
|
|
8
|
+
claimPendingSessionWorkflowWakes,
|
|
9
|
+
getOrCreateSessionSystemUpdateOutbox,
|
|
10
|
+
markSessionWorkflowWakeFailed,
|
|
11
|
+
markSessionSystemUpdateOutboxDeliveredInTransaction,
|
|
12
|
+
markSessionSystemUpdateOutboxFailed,
|
|
13
|
+
type Database,
|
|
14
|
+
type SessionSystemUpdateOutboxDelivery,
|
|
15
|
+
} from "@opengeni/db";
|
|
4
16
|
import type { EventBus } from "@opengeni/events";
|
|
5
17
|
import type { ActivityServices, WakeSessionWorkflowSignal } from "./types";
|
|
6
18
|
|
|
@@ -12,6 +24,14 @@ export type NotifyServices = {
|
|
|
12
24
|
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
13
25
|
};
|
|
14
26
|
|
|
27
|
+
export type ReconcileParentSystemUpdateOverrides = Partial<{
|
|
28
|
+
claimPendingSessionSystemUpdateOutbox: typeof claimPendingSessionSystemUpdateOutbox;
|
|
29
|
+
}>;
|
|
30
|
+
|
|
31
|
+
export type ReconcileSessionWorkflowWakeOverrides = Partial<{
|
|
32
|
+
claimPendingSessionWorkflowWakes: typeof claimPendingSessionWorkflowWakes;
|
|
33
|
+
}>;
|
|
34
|
+
|
|
15
35
|
/**
|
|
16
36
|
* Deliver exactly one completion wake to a spawned worker's parent (manager)
|
|
17
37
|
* session when the worker reaches a terminal-for-now state. No-op for a
|
|
@@ -20,11 +40,10 @@ export type NotifyServices = {
|
|
|
20
40
|
* which advances every time the child does work, so a retry of the same
|
|
21
41
|
* terminal transition (activity retry, the workflow's idle re-check, the
|
|
22
42
|
* runAgentTurn-failed path overlapping a workflow-level wake) is deduped while
|
|
23
|
-
* a genuinely new idle-after-work episode notifies again. The parent
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* best-effort nudge layered on durable DB state.
|
|
43
|
+
* a genuinely new idle-after-work episode notifies again. The parent receives
|
|
44
|
+
* one typed internal update; signalling its workflow ensures an idle parent can
|
|
45
|
+
* coalesce and process all pending updates in one inference. Failures here never
|
|
46
|
+
* fail the child because the durable outbox remains retryable.
|
|
28
47
|
*
|
|
29
48
|
* Lives in its own module so both the session-state terminal activities
|
|
30
49
|
* (markSessionIdle / failSession) and runAgentTurn's in-turn failure path can
|
|
@@ -51,69 +70,214 @@ export async function notifyParentOfChildTerminal(
|
|
|
51
70
|
return;
|
|
52
71
|
}
|
|
53
72
|
const goal = await getSessionGoal(svc.db, workspaceId, childSessionId);
|
|
73
|
+
// Sacred user pause: if the MANAGER's own goal was paused by the user, the
|
|
74
|
+
// wake must not tell it to "resume it now" — that instruction is exactly
|
|
75
|
+
// what re-arms the loop the user just stopped. Suppress the resume nudge and
|
|
76
|
+
// tell the agent to stay paused. Paired with the goal_set reactivation
|
|
77
|
+
// guard so a nudge that slips through still cannot revive the goal.
|
|
54
78
|
const clientEventId = `child-completion:${childSessionId}:${episodeKey ?? child.lastSequence}`;
|
|
55
|
-
const
|
|
79
|
+
const payload = childCompletionPayload(child, goal, terminalStatus);
|
|
80
|
+
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
81
|
+
accountId: child.accountId,
|
|
56
82
|
workspaceId,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
83
|
+
sourceSessionId: child.id,
|
|
84
|
+
targetSessionId: child.parentSessionId,
|
|
85
|
+
kind: "child_terminal_result",
|
|
86
|
+
classification:
|
|
87
|
+
terminalStatus === "failed"
|
|
88
|
+
? "failure"
|
|
89
|
+
: goal?.status === "paused"
|
|
90
|
+
? "action_required"
|
|
91
|
+
: "success",
|
|
92
|
+
sourceId: child.id,
|
|
93
|
+
dedupeKey: clientEventId,
|
|
94
|
+
summary: childCompletionSummary(child, goal, terminalStatus),
|
|
95
|
+
payload,
|
|
96
|
+
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
62
97
|
});
|
|
63
|
-
if (
|
|
98
|
+
if (outbox.status === "delivered") {
|
|
64
99
|
return;
|
|
65
100
|
}
|
|
66
|
-
await svc
|
|
67
|
-
|
|
101
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
// A durable pending outbox row survives this boundary. The global worker
|
|
104
|
+
// reaper retries it; child terminal settlement never depends on this turn.
|
|
105
|
+
svc.observability.error("Failed to wake parent session on worker completion", {
|
|
106
|
+
childSessionId,
|
|
107
|
+
error: error instanceof Error ? error.message : String(error),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function deliverParentSystemUpdateOutbox(
|
|
113
|
+
svc: NotifyServices,
|
|
114
|
+
outbox: SessionSystemUpdateOutboxDelivery,
|
|
115
|
+
): Promise<void> {
|
|
116
|
+
try {
|
|
117
|
+
const result = await addSessionSystemUpdateWithSourceMutation(
|
|
118
|
+
svc.db,
|
|
119
|
+
{
|
|
120
|
+
accountId: outbox.accountId,
|
|
121
|
+
workspaceId: outbox.workspaceId,
|
|
122
|
+
sessionId: outbox.targetSessionId,
|
|
123
|
+
kind: outbox.kind,
|
|
124
|
+
classification: outbox.classification,
|
|
125
|
+
sourceId: outbox.sourceId,
|
|
126
|
+
dedupeKey: outbox.dedupeKey,
|
|
127
|
+
summary: outbox.summary,
|
|
128
|
+
payload: outbox.payload,
|
|
129
|
+
lineage: outbox.lineage,
|
|
130
|
+
},
|
|
131
|
+
async (tx) => {
|
|
132
|
+
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
133
|
+
},
|
|
134
|
+
);
|
|
135
|
+
if (result.reason === "session_cancelled") {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (result.added && result.events.length > 0) {
|
|
139
|
+
await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
|
|
140
|
+
}
|
|
141
|
+
if (result.shouldWake && svc.wakeSessionWorkflow) {
|
|
142
|
+
if (result.workflowWakeRevision === null) {
|
|
143
|
+
throw new Error("Runnable system update has no workflow wake revision");
|
|
144
|
+
}
|
|
68
145
|
await svc.wakeSessionWorkflow({
|
|
69
|
-
accountId:
|
|
70
|
-
workspaceId,
|
|
71
|
-
sessionId:
|
|
72
|
-
workflowId: result.temporalWorkflowId
|
|
146
|
+
accountId: outbox.accountId,
|
|
147
|
+
workspaceId: outbox.workspaceId,
|
|
148
|
+
sessionId: outbox.targetSessionId,
|
|
149
|
+
workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
|
|
150
|
+
wakeRevision: result.workflowWakeRevision,
|
|
73
151
|
});
|
|
74
152
|
}
|
|
75
153
|
svc.observability.info("Woke parent session on worker completion", {
|
|
76
|
-
childSessionId,
|
|
77
|
-
parentSessionId:
|
|
78
|
-
|
|
154
|
+
childSessionId: outbox.sourceSessionId,
|
|
155
|
+
parentSessionId: outbox.targetSessionId,
|
|
156
|
+
dedupeKey: outbox.dedupeKey,
|
|
79
157
|
});
|
|
80
158
|
} catch (error) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
error
|
|
85
|
-
|
|
159
|
+
await markSessionSystemUpdateOutboxFailed(
|
|
160
|
+
svc.db,
|
|
161
|
+
outbox,
|
|
162
|
+
error instanceof Error ? error.message : String(error),
|
|
163
|
+
).catch(() => undefined);
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export async function reconcilePendingParentSystemUpdates(
|
|
169
|
+
svc: NotifyServices,
|
|
170
|
+
limit = 100,
|
|
171
|
+
overrides: ReconcileParentSystemUpdateOverrides = {},
|
|
172
|
+
): Promise<{ claimed: number; delivered: number; failed: number }> {
|
|
173
|
+
const claimPendingSessionSystemUpdateOutboxFn =
|
|
174
|
+
overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
|
|
175
|
+
const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
|
|
176
|
+
let delivered = 0;
|
|
177
|
+
let failed = 0;
|
|
178
|
+
for (const row of rows) {
|
|
179
|
+
try {
|
|
180
|
+
await deliverParentSystemUpdateOutbox(svc, row);
|
|
181
|
+
delivered += 1;
|
|
182
|
+
} catch {
|
|
183
|
+
failed += 1;
|
|
184
|
+
}
|
|
86
185
|
}
|
|
186
|
+
return { claimed: rows.length, delivered, failed };
|
|
87
187
|
}
|
|
88
188
|
|
|
89
|
-
function
|
|
189
|
+
export async function reconcilePendingSessionWorkflowWakes(
|
|
190
|
+
svc: NotifyServices,
|
|
191
|
+
limit = 1_000,
|
|
192
|
+
overrides: ReconcileSessionWorkflowWakeOverrides = {},
|
|
193
|
+
): Promise<{ claimed: number; delivered: number; failed: number }> {
|
|
194
|
+
const claimPendingSessionWorkflowWakesFn =
|
|
195
|
+
overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
|
|
196
|
+
if (!svc.wakeSessionWorkflow) {
|
|
197
|
+
return { claimed: 0, delivered: 0, failed: 0 };
|
|
198
|
+
}
|
|
199
|
+
const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
|
|
200
|
+
let delivered = 0;
|
|
201
|
+
let failed = 0;
|
|
202
|
+
const queue = [...repairs];
|
|
203
|
+
const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
|
|
204
|
+
for (;;) {
|
|
205
|
+
const repair = queue.shift();
|
|
206
|
+
if (!repair) return;
|
|
207
|
+
try {
|
|
208
|
+
await svc.wakeSessionWorkflow!({
|
|
209
|
+
accountId: repair.accountId,
|
|
210
|
+
workspaceId: repair.workspaceId,
|
|
211
|
+
sessionId: repair.sessionId,
|
|
212
|
+
workflowId: repair.temporalWorkflowId,
|
|
213
|
+
wakeRevision: repair.wakeRevision,
|
|
214
|
+
...(repair.interruptionRequested ? { interruptionRequested: true } : {}),
|
|
215
|
+
});
|
|
216
|
+
delivered += 1;
|
|
217
|
+
} catch (error) {
|
|
218
|
+
failed += 1;
|
|
219
|
+
await markSessionWorkflowWakeFailed(
|
|
220
|
+
svc.db,
|
|
221
|
+
repair,
|
|
222
|
+
error instanceof Error ? error.message : String(error),
|
|
223
|
+
).catch(() => undefined);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
await Promise.all(workers);
|
|
228
|
+
return { claimed: repairs.length, delivered, failed };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function childCompletionPayload(
|
|
232
|
+
child: Session,
|
|
233
|
+
goal: SessionGoal | null,
|
|
234
|
+
terminalStatus: "idle" | "failed",
|
|
235
|
+
): Extract<SessionSystemUpdatePayload, { type: "child_terminal_result" }> {
|
|
90
236
|
return {
|
|
237
|
+
type: "child_terminal_result",
|
|
91
238
|
childSessionId: child.id,
|
|
92
239
|
status: terminalStatus,
|
|
93
240
|
...(goal
|
|
94
241
|
? {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
242
|
+
goal: {
|
|
243
|
+
status: goal.status,
|
|
244
|
+
text: goal.text,
|
|
245
|
+
...(goal.evidence ? { evidence: goal.evidence } : {}),
|
|
246
|
+
...(goal.rationale ? { rationale: goal.rationale } : {}),
|
|
247
|
+
...(goal.pausedReason ? { pausedReason: goal.pausedReason } : {}),
|
|
248
|
+
},
|
|
249
|
+
}
|
|
103
250
|
: {}),
|
|
104
251
|
};
|
|
105
252
|
}
|
|
106
253
|
|
|
107
|
-
|
|
254
|
+
/**
|
|
255
|
+
* The worker-specific lines for one child (what happened + its goal), WITHOUT
|
|
256
|
+
* the trailing "what to do next" instruction. Kept separate so N child
|
|
257
|
+
* completions can be coalesced into one internal-update inference: the DB layer
|
|
258
|
+
* stores each child's summary and rebuilds a single numbered digest with one
|
|
259
|
+
* shared trailing instruction instead of running N model calls.
|
|
260
|
+
*/
|
|
261
|
+
export function childCompletionSummary(
|
|
262
|
+
child: Session,
|
|
263
|
+
goal: SessionGoal | null,
|
|
264
|
+
terminalStatus: "idle" | "failed",
|
|
265
|
+
): string {
|
|
108
266
|
const lines: string[] = [];
|
|
109
267
|
if (terminalStatus === "failed") {
|
|
110
268
|
lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
|
|
111
269
|
} else if (goal?.status === "completed") {
|
|
112
|
-
lines.push(
|
|
270
|
+
lines.push(
|
|
271
|
+
`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`,
|
|
272
|
+
);
|
|
113
273
|
} else if (goal?.status === "paused") {
|
|
114
|
-
lines.push(
|
|
274
|
+
lines.push(
|
|
275
|
+
`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`,
|
|
276
|
+
);
|
|
115
277
|
} else {
|
|
116
|
-
lines.push(
|
|
278
|
+
lines.push(
|
|
279
|
+
`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`,
|
|
280
|
+
);
|
|
117
281
|
}
|
|
118
282
|
if (goal) {
|
|
119
283
|
lines.push(`Worker goal: ${goal.text}`);
|
|
@@ -124,6 +288,16 @@ function childCompletionWakeText(child: Session, goal: SessionGoal | null, termi
|
|
|
124
288
|
lines.push(`Pause rationale: ${goal.rationale}`);
|
|
125
289
|
}
|
|
126
290
|
}
|
|
127
|
-
lines.push("Read the worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting this worker, resume it now.");
|
|
128
291
|
return lines.join("\n");
|
|
129
292
|
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* The single trailing instruction appended to a child-completion (or digest)
|
|
296
|
+
* inference. Suppresses the "resume it now" nudge when the manager's own goal
|
|
297
|
+
* was paused by the user (paired with the goal_set reactivation guard).
|
|
298
|
+
*/
|
|
299
|
+
export function childCompletionTrailing(parentGoalUserPaused: boolean): string {
|
|
300
|
+
return parentGoalUserPaused
|
|
301
|
+
? "Read each worker's session events/notebook output for its result. This session was paused by the user — do NOT resume or replace your goal; summarize the result for the user and remain paused."
|
|
302
|
+
: "Read each worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting these workers, resume it now.";
|
|
303
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// apps/worker/src/activities/recording.ts — the recording finalize helper (P4.3).
|
|
2
2
|
//
|
|
3
3
|
// The "agent films itself proving the fix" finalize loop, run IN the process that
|
|
4
|
-
// already holds the resumed-by-id box (the agent turn's own activity).
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// already holds the resumed-by-id box (the agent turn's own activity). A scoped
|
|
5
|
+
// upload URL sends bytes box → object storage; the worker never buffers the
|
|
6
|
+
// recording and no bytes enter a Temporal payload.
|
|
7
7
|
//
|
|
8
8
|
// Ordering invariant (F9): the box file is deleted ONLY after the storage PUT
|
|
9
9
|
// confirms and the `available` row commits — so a failed upload leaves the bytes
|
|
@@ -11,9 +11,8 @@
|
|
|
11
11
|
|
|
12
12
|
import type { Settings } from "@opengeni/config";
|
|
13
13
|
import type { Database } from "@opengeni/db";
|
|
14
|
-
import { deleteRecording
|
|
14
|
+
import { deleteRecording } from "@opengeni/db";
|
|
15
15
|
import type {
|
|
16
|
-
RecordingAvailablePayload,
|
|
17
16
|
RecordingCodec,
|
|
18
17
|
RecordingFailedReason,
|
|
19
18
|
RecordingStartedPayload,
|
|
@@ -21,11 +20,11 @@ import type {
|
|
|
21
20
|
import {
|
|
22
21
|
contentTypeForCodec,
|
|
23
22
|
deleteRecordingArtifacts,
|
|
24
|
-
readRecordingBytes,
|
|
25
23
|
RecordingError,
|
|
26
24
|
recordingStorageKey,
|
|
27
25
|
startRecording as startRecordingOnBox,
|
|
28
26
|
stopRecording as stopRecordingOnBox,
|
|
27
|
+
uploadRecordingArtifact,
|
|
29
28
|
type RecordingProcess,
|
|
30
29
|
} from "@opengeni/runtime";
|
|
31
30
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
@@ -42,6 +41,27 @@ export type ActiveRecording = {
|
|
|
42
41
|
framerate: number;
|
|
43
42
|
};
|
|
44
43
|
|
|
44
|
+
export const RECORDING_SETTLEMENT_PREP_TIMEOUT_MS = 100_000;
|
|
45
|
+
const RECORDING_SETTLEMENT_UPLOAD_MAX_SECONDS = 45;
|
|
46
|
+
|
|
47
|
+
export async function withRecordingPreparationTimeout(
|
|
48
|
+
preparation: Promise<PreparedRecordingSettlement>,
|
|
49
|
+
timeoutResult: PreparedRecordingSettlement,
|
|
50
|
+
timeoutMs = RECORDING_SETTLEMENT_PREP_TIMEOUT_MS,
|
|
51
|
+
): Promise<PreparedRecordingSettlement> {
|
|
52
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
53
|
+
try {
|
|
54
|
+
return await Promise.race([
|
|
55
|
+
preparation,
|
|
56
|
+
new Promise<PreparedRecordingSettlement>((resolve) => {
|
|
57
|
+
timer = setTimeout(() => resolve(timeoutResult), timeoutMs);
|
|
58
|
+
}),
|
|
59
|
+
]);
|
|
60
|
+
} finally {
|
|
61
|
+
if (timer) clearTimeout(timer);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
45
65
|
/**
|
|
46
66
|
* Insert the recording row, launch ffmpeg on the box, and return the live handle.
|
|
47
67
|
* Emits the `recording.started` payload (the caller publishes it on the spine).
|
|
@@ -61,28 +81,43 @@ export async function beginRecording(args: {
|
|
|
61
81
|
}): Promise<{ active: ActiveRecording; started: RecordingStartedPayload }> {
|
|
62
82
|
const { settings, db } = args;
|
|
63
83
|
const codec = settings.recordingDefaultCodec as RecordingCodec;
|
|
64
|
-
const dimensions: [number, number] = [
|
|
84
|
+
const dimensions: [number, number] = [
|
|
85
|
+
settings.streamResolutionWidth,
|
|
86
|
+
settings.streamResolutionHeight,
|
|
87
|
+
];
|
|
65
88
|
const framerate = settings.recordingFramerate;
|
|
66
|
-
await import("@opengeni/db").then(({ insertRecording }) =>
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
89
|
+
await import("@opengeni/db").then(({ insertRecording }) =>
|
|
90
|
+
insertRecording(db, {
|
|
91
|
+
id: args.recordingId,
|
|
92
|
+
accountId: args.accountId,
|
|
93
|
+
workspaceId: args.workspaceId,
|
|
94
|
+
sessionId: args.sessionId,
|
|
95
|
+
turnId: args.turnId,
|
|
96
|
+
mode: args.mode,
|
|
97
|
+
codec,
|
|
98
|
+
width: dimensions[0],
|
|
99
|
+
height: dimensions[1],
|
|
100
|
+
reason: scrubFreeText(args.reason),
|
|
101
|
+
}),
|
|
102
|
+
);
|
|
103
|
+
let proc: RecordingProcess;
|
|
104
|
+
try {
|
|
105
|
+
proc = await startRecordingOnBox(args.session, {
|
|
106
|
+
recordingId: args.recordingId,
|
|
107
|
+
codec,
|
|
108
|
+
framerate,
|
|
109
|
+
maxSeconds: settings.recordingMaxSeconds,
|
|
110
|
+
dimensions,
|
|
111
|
+
...(args.runAs ? { runAs: args.runAs } : {}),
|
|
112
|
+
});
|
|
113
|
+
} catch (error) {
|
|
114
|
+
await deleteRecording(db, {
|
|
115
|
+
accountId: args.accountId,
|
|
116
|
+
workspaceId: args.workspaceId,
|
|
117
|
+
recordingId: args.recordingId,
|
|
118
|
+
}).catch(() => undefined);
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
86
121
|
const active: ActiveRecording = {
|
|
87
122
|
recordingId: args.recordingId,
|
|
88
123
|
turnId: args.turnId,
|
|
@@ -104,114 +139,116 @@ export async function beginRecording(args: {
|
|
|
104
139
|
return { active, started };
|
|
105
140
|
}
|
|
106
141
|
|
|
107
|
-
export type
|
|
108
|
-
| {
|
|
109
|
-
|
|
142
|
+
export type PreparedRecordingSettlement =
|
|
143
|
+
| {
|
|
144
|
+
mutation: {
|
|
145
|
+
action: "available";
|
|
146
|
+
recordingId: string;
|
|
147
|
+
storageKey: string;
|
|
148
|
+
sizeBytes: number;
|
|
149
|
+
durationSeconds: number;
|
|
150
|
+
};
|
|
151
|
+
deleteArtifactsAfterCommit: true;
|
|
152
|
+
}
|
|
153
|
+
| {
|
|
154
|
+
mutation: {
|
|
155
|
+
action: "failed";
|
|
156
|
+
recordingId: string;
|
|
157
|
+
reason: RecordingFailedReason;
|
|
158
|
+
detail: string | null;
|
|
159
|
+
};
|
|
160
|
+
deleteArtifactsAfterCommit: false;
|
|
161
|
+
}
|
|
162
|
+
| {
|
|
163
|
+
mutation: { action: "discard"; recordingId: string };
|
|
164
|
+
deleteArtifactsAfterCommit: true;
|
|
165
|
+
};
|
|
110
166
|
|
|
111
167
|
/**
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
168
|
+
* Prepare an on-turn recording for the attempt-closing settlement. This performs
|
|
169
|
+
* only bounded box/storage work; it does not mutate canonical database state and
|
|
170
|
+
* never deletes the box artifact. The caller folds the returned mutation into the
|
|
171
|
+
* exact attempt-fenced turn settlement, then deletes artifacts only after commit.
|
|
172
|
+
* NEVER throws and never prevents turn truth from settling.
|
|
116
173
|
*/
|
|
117
|
-
export async function
|
|
174
|
+
export async function prepareRecordingForSettlement(args: {
|
|
118
175
|
settings: Settings;
|
|
119
|
-
db: Database;
|
|
120
176
|
objectStorage: ObjectStorage | null;
|
|
121
|
-
accountId: string;
|
|
122
177
|
workspaceId: string;
|
|
123
178
|
sessionId: string;
|
|
124
179
|
active: ActiveRecording;
|
|
125
180
|
session: unknown;
|
|
126
|
-
|
|
127
|
-
}): Promise<
|
|
128
|
-
const { settings,
|
|
129
|
-
const codec =
|
|
130
|
-
const fail =
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
181
|
+
didComputerUse: boolean;
|
|
182
|
+
}): Promise<PreparedRecordingSettlement> {
|
|
183
|
+
const { settings, objectStorage, active } = args;
|
|
184
|
+
const codec = active.proc.codec;
|
|
185
|
+
const fail = (
|
|
186
|
+
reason: RecordingFailedReason,
|
|
187
|
+
detail: string | null,
|
|
188
|
+
): PreparedRecordingSettlement => {
|
|
189
|
+
return {
|
|
190
|
+
mutation: {
|
|
191
|
+
action: "failed",
|
|
192
|
+
recordingId: active.recordingId,
|
|
193
|
+
reason,
|
|
194
|
+
detail: scrubFreeText(detail),
|
|
195
|
+
},
|
|
196
|
+
deleteArtifactsAfterCommit: false,
|
|
197
|
+
};
|
|
139
198
|
};
|
|
140
199
|
|
|
141
|
-
|
|
142
|
-
await updateRecording(db, {
|
|
143
|
-
accountId: args.accountId,
|
|
144
|
-
workspaceId: args.workspaceId,
|
|
145
|
-
recordingId: active.recordingId,
|
|
146
|
-
state: "finalizing",
|
|
147
|
-
}).catch(() => undefined);
|
|
148
|
-
|
|
149
|
-
// 1. SIGINT ffmpeg and wait for the clean trailer.
|
|
150
|
-
await stopRecordingOnBox(args.session, active.proc);
|
|
151
|
-
|
|
152
|
-
// 2. Read bytes off the box (size-gated; no eager delete — F8/F9).
|
|
153
|
-
const finalized = await readRecordingBytes(args.session, active.proc, settings.recordingMaxBytes);
|
|
154
|
-
|
|
155
|
-
if (!objectStorage) {
|
|
156
|
-
return await fail("upload-failed", "object storage is not configured");
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// 3. PUT to storage (the byte transfer stays IN this process — F10).
|
|
160
|
-
//
|
|
161
|
-
// SERVER-SIDE authenticated direct PUT, NOT presign+fetch. The worker holds the
|
|
162
|
-
// storage creds, so a presigned URL buys nothing — and on a split public/
|
|
163
|
-
// internal endpoint topology (a PUBLIC `objectStorageEndpoint` with no in-cluster
|
|
164
|
-
// route, the preview case) the presigned URL points at the public host and the
|
|
165
|
-
// fetch PUT 401s. `putObject` sends the bytes straight to the backend over the
|
|
166
|
-
// configured (in-cluster) endpoint with the in-process SDK client. Browser
|
|
167
|
-
// uploads keep using `createPutUrl`; this is the trusted-server twin.
|
|
168
|
-
const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
|
|
200
|
+
const prepare = async (): Promise<PreparedRecordingSettlement> => {
|
|
169
201
|
try {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
202
|
+
await stopRecordingOnBox(args.session, active.proc);
|
|
203
|
+
if (!args.didComputerUse) {
|
|
204
|
+
return {
|
|
205
|
+
mutation: { action: "discard", recordingId: active.recordingId },
|
|
206
|
+
deleteArtifactsAfterCommit: true,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
if (!objectStorage) {
|
|
210
|
+
return fail("upload-failed", "object storage is not configured");
|
|
211
|
+
}
|
|
212
|
+
const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
|
|
213
|
+
const upload = await objectStorage.createPutUrl({
|
|
214
|
+
key,
|
|
215
|
+
contentType: contentTypeForCodec(codec),
|
|
216
|
+
});
|
|
217
|
+
const finalized = await uploadRecordingArtifact(args.session, active.proc, {
|
|
218
|
+
url: upload.url,
|
|
219
|
+
requiredHeaders: upload.requiredHeaders,
|
|
220
|
+
maxBytes: settings.recordingMaxBytes,
|
|
221
|
+
maxSeconds: RECORDING_SETTLEMENT_UPLOAD_MAX_SECONDS,
|
|
222
|
+
});
|
|
223
|
+
return {
|
|
224
|
+
mutation: {
|
|
225
|
+
action: "available",
|
|
226
|
+
recordingId: active.recordingId,
|
|
227
|
+
storageKey: key,
|
|
228
|
+
sizeBytes: finalized.sizeBytes,
|
|
229
|
+
durationSeconds: finalized.durationSeconds,
|
|
230
|
+
},
|
|
231
|
+
deleteArtifactsAfterCommit: true,
|
|
232
|
+
};
|
|
233
|
+
} catch (error) {
|
|
234
|
+
const reason: RecordingFailedReason =
|
|
235
|
+
error instanceof RecordingError ? error.reason : "upload-failed";
|
|
236
|
+
return fail(reason, error instanceof Error ? error.message : String(error));
|
|
174
237
|
}
|
|
238
|
+
};
|
|
175
239
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
recordingId: active.recordingId,
|
|
181
|
-
state: "available",
|
|
182
|
-
storageKey: key,
|
|
183
|
-
sizeBytes: finalized.sizeBytes,
|
|
184
|
-
durationSeconds: finalized.durationSeconds,
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
// 5. ONLY NOW delete the box artifacts (F9 — never before a confirmed PUT).
|
|
188
|
-
await deleteRecordingArtifacts(args.session, active.proc);
|
|
189
|
-
|
|
190
|
-
const available: RecordingAvailablePayload = {
|
|
191
|
-
recordingId: active.recordingId,
|
|
192
|
-
turnId: active.turnId,
|
|
193
|
-
codec,
|
|
194
|
-
contentType: contentTypeForCodec(codec),
|
|
195
|
-
storageKey: key,
|
|
196
|
-
durationSeconds: finalized.durationSeconds,
|
|
197
|
-
sizeBytes: finalized.sizeBytes,
|
|
198
|
-
dimensions: active.dimensions,
|
|
199
|
-
};
|
|
200
|
-
return { ok: true, available };
|
|
201
|
-
} catch (error) {
|
|
202
|
-
const reason: RecordingFailedReason = error instanceof RecordingError ? error.reason : "ffmpeg-error";
|
|
203
|
-
return await fail(reason, error instanceof Error ? error.message : String(error));
|
|
204
|
-
}
|
|
240
|
+
return await withRecordingPreparationTimeout(
|
|
241
|
+
prepare(),
|
|
242
|
+
fail("upload-failed", "recording settlement preparation timed out"),
|
|
243
|
+
);
|
|
205
244
|
}
|
|
206
245
|
|
|
207
246
|
/**
|
|
208
|
-
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
* no-op. NEVER throws (it runs in the turn `finally` and must not mask the turn
|
|
212
|
-
* outcome). Returns nothing; the caller emits no recording event for a discard.
|
|
247
|
+
* Compensate a start whose fenced `recording.started` event was rejected. Since
|
|
248
|
+
* no canonical event was admitted, this exact recording id must leave no row,
|
|
249
|
+
* process, or box artifact behind.
|
|
213
250
|
*/
|
|
214
|
-
export async function
|
|
251
|
+
export async function discardUnpublishedRecording(args: {
|
|
215
252
|
db: Database;
|
|
216
253
|
accountId: string;
|
|
217
254
|
workspaceId: string;
|
|
@@ -220,14 +257,9 @@ export async function discardRecording(args: {
|
|
|
220
257
|
}): Promise<void> {
|
|
221
258
|
const { db, active } = args;
|
|
222
259
|
try {
|
|
223
|
-
// Stop ffmpeg cleanly, then delete the partial artifact off the box. Both are
|
|
224
|
-
// best-effort: a stuck box must not strand the discard (the box rides the
|
|
225
|
-
// provider idle-timeout regardless).
|
|
226
260
|
await stopRecordingOnBox(args.session, active.proc).catch(() => undefined);
|
|
227
261
|
await deleteRecordingArtifacts(args.session, active.proc).catch(() => undefined);
|
|
228
262
|
} finally {
|
|
229
|
-
// Remove the phantom row inserted at beginRecording so a no-activity turn
|
|
230
|
-
// leaves no recording trace at all.
|
|
231
263
|
await deleteRecording(db, {
|
|
232
264
|
accountId: args.accountId,
|
|
233
265
|
workspaceId: args.workspaceId,
|