@opengeni/worker-bundle 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7113 -1652
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4337 -1052
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -1,9 +1,9 @@
1
1
  // apps/worker/src/activities/recording.ts — the recording finalize helper (P4.3).
2
2
  //
3
3
  // The "agent films itself proving the fix" finalize loop, run IN the process that
4
- // already holds the resumed-by-id box (the agent turn's own activity). The bytes
5
- // go box → process memory → object-storage PUT and are NEVER serialized as a
6
- // Temporal activity result (F10): read + PUT happen in ONE invocation here.
4
+ // already holds the resumed-by-id box (the agent turn's own activity). A scoped
5
+ // upload URL sends bytes box → object storage; the worker never buffers the
6
+ // recording and no bytes enter a Temporal payload.
7
7
  //
8
8
  // Ordering invariant (F9): the box file is deleted ONLY after the storage PUT
9
9
  // confirms and the `available` row commits — so a failed upload leaves the bytes
@@ -11,9 +11,8 @@
11
11
 
12
12
  import type { Settings } from "@opengeni/config";
13
13
  import type { Database } from "@opengeni/db";
14
- import { deleteRecording, updateRecording } from "@opengeni/db";
14
+ import { deleteRecording } from "@opengeni/db";
15
15
  import type {
16
- RecordingAvailablePayload,
17
16
  RecordingCodec,
18
17
  RecordingFailedReason,
19
18
  RecordingStartedPayload,
@@ -21,11 +20,11 @@ import type {
21
20
  import {
22
21
  contentTypeForCodec,
23
22
  deleteRecordingArtifacts,
24
- readRecordingBytes,
25
23
  RecordingError,
26
24
  recordingStorageKey,
27
25
  startRecording as startRecordingOnBox,
28
26
  stopRecording as stopRecordingOnBox,
27
+ uploadRecordingArtifact,
29
28
  type RecordingProcess,
30
29
  } from "@opengeni/runtime";
31
30
  import type { ObjectStorage } from "@opengeni/storage";
@@ -42,6 +41,27 @@ export type ActiveRecording = {
42
41
  framerate: number;
43
42
  };
44
43
 
44
+ export const RECORDING_SETTLEMENT_PREP_TIMEOUT_MS = 100_000;
45
+ const RECORDING_SETTLEMENT_UPLOAD_MAX_SECONDS = 45;
46
+
47
+ export async function withRecordingPreparationTimeout(
48
+ preparation: Promise<PreparedRecordingSettlement>,
49
+ timeoutResult: PreparedRecordingSettlement,
50
+ timeoutMs = RECORDING_SETTLEMENT_PREP_TIMEOUT_MS,
51
+ ): Promise<PreparedRecordingSettlement> {
52
+ let timer: ReturnType<typeof setTimeout> | undefined;
53
+ try {
54
+ return await Promise.race([
55
+ preparation,
56
+ new Promise<PreparedRecordingSettlement>((resolve) => {
57
+ timer = setTimeout(() => resolve(timeoutResult), timeoutMs);
58
+ }),
59
+ ]);
60
+ } finally {
61
+ if (timer) clearTimeout(timer);
62
+ }
63
+ }
64
+
45
65
  /**
46
66
  * Insert the recording row, launch ffmpeg on the box, and return the live handle.
47
67
  * Emits the `recording.started` payload (the caller publishes it on the spine).
@@ -61,28 +81,43 @@ export async function beginRecording(args: {
61
81
  }): Promise<{ active: ActiveRecording; started: RecordingStartedPayload }> {
62
82
  const { settings, db } = args;
63
83
  const codec = settings.recordingDefaultCodec as RecordingCodec;
64
- const dimensions: [number, number] = [settings.streamResolutionWidth, settings.streamResolutionHeight];
84
+ const dimensions: [number, number] = [
85
+ settings.streamResolutionWidth,
86
+ settings.streamResolutionHeight,
87
+ ];
65
88
  const framerate = settings.recordingFramerate;
66
- await import("@opengeni/db").then(({ insertRecording }) => insertRecording(db, {
67
- id: args.recordingId,
68
- accountId: args.accountId,
69
- workspaceId: args.workspaceId,
70
- sessionId: args.sessionId,
71
- turnId: args.turnId,
72
- mode: args.mode,
73
- codec,
74
- width: dimensions[0],
75
- height: dimensions[1],
76
- reason: scrubFreeText(args.reason),
77
- }));
78
- const proc = await startRecordingOnBox(args.session, {
79
- recordingId: args.recordingId,
80
- codec,
81
- framerate,
82
- maxSeconds: settings.recordingMaxSeconds,
83
- dimensions,
84
- ...(args.runAs ? { runAs: args.runAs } : {}),
85
- });
89
+ await import("@opengeni/db").then(({ insertRecording }) =>
90
+ insertRecording(db, {
91
+ id: args.recordingId,
92
+ accountId: args.accountId,
93
+ workspaceId: args.workspaceId,
94
+ sessionId: args.sessionId,
95
+ turnId: args.turnId,
96
+ mode: args.mode,
97
+ codec,
98
+ width: dimensions[0],
99
+ height: dimensions[1],
100
+ reason: scrubFreeText(args.reason),
101
+ }),
102
+ );
103
+ let proc: RecordingProcess;
104
+ try {
105
+ proc = await startRecordingOnBox(args.session, {
106
+ recordingId: args.recordingId,
107
+ codec,
108
+ framerate,
109
+ maxSeconds: settings.recordingMaxSeconds,
110
+ dimensions,
111
+ ...(args.runAs ? { runAs: args.runAs } : {}),
112
+ });
113
+ } catch (error) {
114
+ await deleteRecording(db, {
115
+ accountId: args.accountId,
116
+ workspaceId: args.workspaceId,
117
+ recordingId: args.recordingId,
118
+ }).catch(() => undefined);
119
+ throw error;
120
+ }
86
121
  const active: ActiveRecording = {
87
122
  recordingId: args.recordingId,
88
123
  turnId: args.turnId,
@@ -104,114 +139,116 @@ export async function beginRecording(args: {
104
139
  return { active, started };
105
140
  }
106
141
 
107
- export type FinalizeOutcome =
108
- | { ok: true; available: RecordingAvailablePayload }
109
- | { ok: false; reason: RecordingFailedReason; detail: string | null };
142
+ export type PreparedRecordingSettlement =
143
+ | {
144
+ mutation: {
145
+ action: "available";
146
+ recordingId: string;
147
+ storageKey: string;
148
+ sizeBytes: number;
149
+ durationSeconds: number;
150
+ };
151
+ deleteArtifactsAfterCommit: true;
152
+ }
153
+ | {
154
+ mutation: {
155
+ action: "failed";
156
+ recordingId: string;
157
+ reason: RecordingFailedReason;
158
+ detail: string | null;
159
+ };
160
+ deleteArtifactsAfterCommit: false;
161
+ }
162
+ | {
163
+ mutation: { action: "discard"; recordingId: string };
164
+ deleteArtifactsAfterCommit: true;
165
+ };
110
166
 
111
167
  /**
112
- * Stop ffmpeg, read the bytes off the box, PUT them to storage, commit `available`,
113
- * and (only then) delete the box file (F9). Returns the available payload, or a
114
- * failure reason. NEVER throws — finalize runs in a turn `finally` and must not
115
- * mask the turn outcome.
168
+ * Prepare an on-turn recording for the attempt-closing settlement. This performs
169
+ * only bounded box/storage work; it does not mutate canonical database state and
170
+ * never deletes the box artifact. The caller folds the returned mutation into the
171
+ * exact attempt-fenced turn settlement, then deletes artifacts only after commit.
172
+ * NEVER throws and never prevents turn truth from settling.
116
173
  */
117
- export async function finalizeRecording(args: {
174
+ export async function prepareRecordingForSettlement(args: {
118
175
  settings: Settings;
119
- db: Database;
120
176
  objectStorage: ObjectStorage | null;
121
- accountId: string;
122
177
  workspaceId: string;
123
178
  sessionId: string;
124
179
  active: ActiveRecording;
125
180
  session: unknown;
126
- runAs?: string | undefined;
127
- }): Promise<FinalizeOutcome> {
128
- const { settings, db, objectStorage, active } = args;
129
- const codec = settings.recordingDefaultCodec as RecordingCodec;
130
- const fail = async (reason: RecordingFailedReason, detail: string | null): Promise<FinalizeOutcome> => {
131
- await updateRecording(db, {
132
- accountId: args.accountId,
133
- workspaceId: args.workspaceId,
134
- recordingId: active.recordingId,
135
- state: "failed",
136
- reason: scrubFreeText(detail),
137
- }).catch(() => undefined);
138
- return { ok: false, reason, detail: scrubFreeText(detail) };
181
+ didComputerUse: boolean;
182
+ }): Promise<PreparedRecordingSettlement> {
183
+ const { settings, objectStorage, active } = args;
184
+ const codec = active.proc.codec;
185
+ const fail = (
186
+ reason: RecordingFailedReason,
187
+ detail: string | null,
188
+ ): PreparedRecordingSettlement => {
189
+ return {
190
+ mutation: {
191
+ action: "failed",
192
+ recordingId: active.recordingId,
193
+ reason,
194
+ detail: scrubFreeText(detail),
195
+ },
196
+ deleteArtifactsAfterCommit: false,
197
+ };
139
198
  };
140
199
 
141
- try {
142
- await updateRecording(db, {
143
- accountId: args.accountId,
144
- workspaceId: args.workspaceId,
145
- recordingId: active.recordingId,
146
- state: "finalizing",
147
- }).catch(() => undefined);
148
-
149
- // 1. SIGINT ffmpeg and wait for the clean trailer.
150
- await stopRecordingOnBox(args.session, active.proc);
151
-
152
- // 2. Read bytes off the box (size-gated; no eager delete — F8/F9).
153
- const finalized = await readRecordingBytes(args.session, active.proc, settings.recordingMaxBytes);
154
-
155
- if (!objectStorage) {
156
- return await fail("upload-failed", "object storage is not configured");
157
- }
158
-
159
- // 3. PUT to storage (the byte transfer stays IN this process — F10).
160
- //
161
- // SERVER-SIDE authenticated direct PUT, NOT presign+fetch. The worker holds the
162
- // storage creds, so a presigned URL buys nothing — and on a split public/
163
- // internal endpoint topology (a PUBLIC `objectStorageEndpoint` with no in-cluster
164
- // route, the preview case) the presigned URL points at the public host and the
165
- // fetch PUT 401s. `putObject` sends the bytes straight to the backend over the
166
- // configured (in-cluster) endpoint with the in-process SDK client. Browser
167
- // uploads keep using `createPutUrl`; this is the trusted-server twin.
168
- const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
200
+ const prepare = async (): Promise<PreparedRecordingSettlement> => {
169
201
  try {
170
- // `.slice()` detaches a fresh ArrayBuffer from the box read.
171
- await objectStorage.putObject({ key, contentType: finalized.contentType, body: finalized.bytes.slice() });
172
- } catch (uploadError) {
173
- return await fail("upload-failed", uploadError instanceof Error ? uploadError.message : String(uploadError));
202
+ await stopRecordingOnBox(args.session, active.proc);
203
+ if (!args.didComputerUse) {
204
+ return {
205
+ mutation: { action: "discard", recordingId: active.recordingId },
206
+ deleteArtifactsAfterCommit: true,
207
+ };
208
+ }
209
+ if (!objectStorage) {
210
+ return fail("upload-failed", "object storage is not configured");
211
+ }
212
+ const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
213
+ const upload = await objectStorage.createPutUrl({
214
+ key,
215
+ contentType: contentTypeForCodec(codec),
216
+ });
217
+ const finalized = await uploadRecordingArtifact(args.session, active.proc, {
218
+ url: upload.url,
219
+ requiredHeaders: upload.requiredHeaders,
220
+ maxBytes: settings.recordingMaxBytes,
221
+ maxSeconds: RECORDING_SETTLEMENT_UPLOAD_MAX_SECONDS,
222
+ });
223
+ return {
224
+ mutation: {
225
+ action: "available",
226
+ recordingId: active.recordingId,
227
+ storageKey: key,
228
+ sizeBytes: finalized.sizeBytes,
229
+ durationSeconds: finalized.durationSeconds,
230
+ },
231
+ deleteArtifactsAfterCommit: true,
232
+ };
233
+ } catch (error) {
234
+ const reason: RecordingFailedReason =
235
+ error instanceof RecordingError ? error.reason : "upload-failed";
236
+ return fail(reason, error instanceof Error ? error.message : String(error));
174
237
  }
238
+ };
175
239
 
176
- // 4. Commit `available` with the artifact ref.
177
- await updateRecording(db, {
178
- accountId: args.accountId,
179
- workspaceId: args.workspaceId,
180
- recordingId: active.recordingId,
181
- state: "available",
182
- storageKey: key,
183
- sizeBytes: finalized.sizeBytes,
184
- durationSeconds: finalized.durationSeconds,
185
- });
186
-
187
- // 5. ONLY NOW delete the box artifacts (F9 — never before a confirmed PUT).
188
- await deleteRecordingArtifacts(args.session, active.proc);
189
-
190
- const available: RecordingAvailablePayload = {
191
- recordingId: active.recordingId,
192
- turnId: active.turnId,
193
- codec,
194
- contentType: contentTypeForCodec(codec),
195
- storageKey: key,
196
- durationSeconds: finalized.durationSeconds,
197
- sizeBytes: finalized.sizeBytes,
198
- dimensions: active.dimensions,
199
- };
200
- return { ok: true, available };
201
- } catch (error) {
202
- const reason: RecordingFailedReason = error instanceof RecordingError ? error.reason : "ffmpeg-error";
203
- return await fail(reason, error instanceof Error ? error.message : String(error));
204
- }
240
+ return await withRecordingPreparationTimeout(
241
+ prepare(),
242
+ fail("upload-failed", "recording settlement preparation timed out"),
243
+ );
205
244
  }
206
245
 
207
246
  /**
208
- * Discard an on-turn recording that captured NO computer-use activity (a plain
209
- * text turn): stop ffmpeg on the box, delete the box artifacts, and remove the
210
- * recording row entirely. NO storage PUT and NO `recording.failed` — a clean
211
- * no-op. NEVER throws (it runs in the turn `finally` and must not mask the turn
212
- * outcome). Returns nothing; the caller emits no recording event for a discard.
247
+ * Compensate a start whose fenced `recording.started` event was rejected. Since
248
+ * no canonical event was admitted, this exact recording id must leave no row,
249
+ * process, or box artifact behind.
213
250
  */
214
- export async function discardRecording(args: {
251
+ export async function discardUnpublishedRecording(args: {
215
252
  db: Database;
216
253
  accountId: string;
217
254
  workspaceId: string;
@@ -220,14 +257,9 @@ export async function discardRecording(args: {
220
257
  }): Promise<void> {
221
258
  const { db, active } = args;
222
259
  try {
223
- // Stop ffmpeg cleanly, then delete the partial artifact off the box. Both are
224
- // best-effort: a stuck box must not strand the discard (the box rides the
225
- // provider idle-timeout regardless).
226
260
  await stopRecordingOnBox(args.session, active.proc).catch(() => undefined);
227
261
  await deleteRecordingArtifacts(args.session, active.proc).catch(() => undefined);
228
262
  } finally {
229
- // Remove the phantom row inserted at beginRecording so a no-activity turn
230
- // leaves no recording trace at all.
231
263
  await deleteRecording(db, {
232
264
  accountId: args.accountId,
233
265
  workspaceId: args.workspaceId,
@@ -34,7 +34,9 @@ export function createSecretRedactor(secrets: SecretForRedaction[]): (payload: u
34
34
  return payload.map(redact);
35
35
  }
36
36
  if (isPlainObject(payload)) {
37
- return Object.fromEntries(Object.entries(payload).map(([key, value]) => [key, redact(value)]));
37
+ return Object.fromEntries(
38
+ Object.entries(payload).map(([key, value]) => [key, redact(value)]),
39
+ );
38
40
  }
39
41
  return payload;
40
42
  };