@opengeni/worker-bundle 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7127 -1654
- package/dist/index.js.map +1 -1
- package/package.json +19 -19
- package/src/activities/agent-turn.ts +4405 -1089
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
2
|
import type { Session, SessionGoal } from "@opengeni/contracts";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
getSession,
|
|
5
|
+
getSessionGoal,
|
|
6
|
+
addSessionSystemUpdateWithSourceMutation,
|
|
7
|
+
claimPendingSessionSystemUpdateOutbox,
|
|
8
|
+
claimPendingSessionWorkflowWakes,
|
|
9
|
+
getOrCreateSessionSystemUpdateOutbox,
|
|
10
|
+
markSessionWorkflowWakeFailed,
|
|
11
|
+
markSessionSystemUpdateOutboxDeliveredInTransaction,
|
|
12
|
+
markSessionSystemUpdateOutboxFailed,
|
|
13
|
+
type Database,
|
|
14
|
+
type SessionSystemUpdateOutboxDelivery,
|
|
15
|
+
} from "@opengeni/db";
|
|
4
16
|
import type { EventBus } from "@opengeni/events";
|
|
5
17
|
import type { ActivityServices, WakeSessionWorkflowSignal } from "./types";
|
|
6
18
|
|
|
@@ -12,6 +24,14 @@ export type NotifyServices = {
|
|
|
12
24
|
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
13
25
|
};
|
|
14
26
|
|
|
27
|
+
export type ReconcileParentSystemUpdateOverrides = Partial<{
|
|
28
|
+
claimPendingSessionSystemUpdateOutbox: typeof claimPendingSessionSystemUpdateOutbox;
|
|
29
|
+
}>;
|
|
30
|
+
|
|
31
|
+
export type ReconcileSessionWorkflowWakeOverrides = Partial<{
|
|
32
|
+
claimPendingSessionWorkflowWakes: typeof claimPendingSessionWorkflowWakes;
|
|
33
|
+
}>;
|
|
34
|
+
|
|
15
35
|
/**
|
|
16
36
|
* Deliver exactly one completion wake to a spawned worker's parent (manager)
|
|
17
37
|
* session when the worker reaches a terminal-for-now state. No-op for a
|
|
@@ -20,11 +40,10 @@ export type NotifyServices = {
|
|
|
20
40
|
* which advances every time the child does work, so a retry of the same
|
|
21
41
|
* terminal transition (activity retry, the workflow's idle re-check, the
|
|
22
42
|
* runAgentTurn-failed path overlapping a workflow-level wake) is deduped while
|
|
23
|
-
* a genuinely new idle-after-work episode notifies again. The parent
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* best-effort nudge layered on durable DB state.
|
|
43
|
+
* a genuinely new idle-after-work episode notifies again. The parent receives
|
|
44
|
+
* one typed internal update; signalling its workflow ensures an idle parent can
|
|
45
|
+
* coalesce and process all pending updates in one inference. Failures here never
|
|
46
|
+
* fail the child because the durable outbox remains retryable.
|
|
28
47
|
*
|
|
29
48
|
* Lives in its own module so both the session-state terminal activities
|
|
30
49
|
* (markSessionIdle / failSession) and runAgentTurn's in-turn failure path can
|
|
@@ -45,75 +64,233 @@ export async function notifyParentOfChildTerminal(
|
|
|
45
64
|
// per work batch and is stable across retries of that same idle transition.
|
|
46
65
|
episodeKey?: string | null,
|
|
47
66
|
): Promise<void> {
|
|
67
|
+
// Temporarily disabled by default: child completion remains durable on the
|
|
68
|
+
// child session, but must not manufacture parent system-update/inference
|
|
69
|
+
// work. Keep this before every DB read, event publish, and workflow signal so
|
|
70
|
+
// the disabled path cannot mutate or wake the parent.
|
|
71
|
+
if (!svc.settings.childCompletionParentWakeEnabled) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
48
74
|
try {
|
|
49
75
|
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
50
76
|
if (!child || !child.parentSessionId) {
|
|
51
77
|
return;
|
|
52
78
|
}
|
|
53
79
|
const goal = await getSessionGoal(svc.db, workspaceId, childSessionId);
|
|
80
|
+
// Sacred user pause: if the MANAGER's own goal was paused by the user, the
|
|
81
|
+
// wake must not tell it to "resume it now" — that instruction is exactly
|
|
82
|
+
// what re-arms the loop the user just stopped. Suppress the resume nudge and
|
|
83
|
+
// tell the agent to stay paused. Paired with the goal_set reactivation
|
|
84
|
+
// guard so a nudge that slips through still cannot revive the goal.
|
|
54
85
|
const clientEventId = `child-completion:${childSessionId}:${episodeKey ?? child.lastSequence}`;
|
|
55
|
-
const
|
|
86
|
+
const payload = childCompletionPayload(child, goal, terminalStatus);
|
|
87
|
+
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
88
|
+
accountId: child.accountId,
|
|
56
89
|
workspaceId,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
90
|
+
sourceSessionId: child.id,
|
|
91
|
+
targetSessionId: child.parentSessionId,
|
|
92
|
+
kind: "child_session_update",
|
|
93
|
+
classification:
|
|
94
|
+
terminalStatus === "failed"
|
|
95
|
+
? "failure"
|
|
96
|
+
: goal?.status === "paused"
|
|
97
|
+
? "action_required"
|
|
98
|
+
: "success",
|
|
99
|
+
sourceId: child.id,
|
|
100
|
+
dedupeKey: clientEventId,
|
|
101
|
+
summary: childCompletionSummary(child, goal, terminalStatus),
|
|
102
|
+
payload,
|
|
103
|
+
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
62
104
|
});
|
|
63
|
-
if (
|
|
105
|
+
if (outbox.status === "delivered") {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
109
|
+
} catch (error) {
|
|
110
|
+
// A durable pending outbox row survives this boundary. The global worker
|
|
111
|
+
// reaper retries it; child terminal settlement never depends on this turn.
|
|
112
|
+
svc.observability.error("Failed to wake parent session on worker completion", {
|
|
113
|
+
childSessionId,
|
|
114
|
+
error: error instanceof Error ? error.message : String(error),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function deliverParentSystemUpdateOutbox(
|
|
120
|
+
svc: NotifyServices,
|
|
121
|
+
outbox: SessionSystemUpdateOutboxDelivery,
|
|
122
|
+
): Promise<void> {
|
|
123
|
+
try {
|
|
124
|
+
const result = await addSessionSystemUpdateWithSourceMutation(
|
|
125
|
+
svc.db,
|
|
126
|
+
{
|
|
127
|
+
accountId: outbox.accountId,
|
|
128
|
+
workspaceId: outbox.workspaceId,
|
|
129
|
+
sessionId: outbox.targetSessionId,
|
|
130
|
+
kind: outbox.kind,
|
|
131
|
+
classification: outbox.classification,
|
|
132
|
+
sourceId: outbox.sourceId,
|
|
133
|
+
dedupeKey: outbox.dedupeKey,
|
|
134
|
+
summary: outbox.summary,
|
|
135
|
+
payload: outbox.payload,
|
|
136
|
+
lineage: outbox.lineage,
|
|
137
|
+
},
|
|
138
|
+
async (tx) => {
|
|
139
|
+
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
140
|
+
},
|
|
141
|
+
);
|
|
142
|
+
if (result.reason === "session_cancelled") {
|
|
64
143
|
return;
|
|
65
144
|
}
|
|
66
|
-
|
|
67
|
-
|
|
145
|
+
if (result.added && result.events.length > 0) {
|
|
146
|
+
await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
|
|
147
|
+
}
|
|
148
|
+
if (result.shouldWake && svc.wakeSessionWorkflow) {
|
|
149
|
+
if (result.workflowWakeRevision === null) {
|
|
150
|
+
throw new Error("Runnable system update has no workflow wake revision");
|
|
151
|
+
}
|
|
68
152
|
await svc.wakeSessionWorkflow({
|
|
69
|
-
accountId:
|
|
70
|
-
workspaceId,
|
|
71
|
-
sessionId:
|
|
72
|
-
workflowId: result.temporalWorkflowId
|
|
153
|
+
accountId: outbox.accountId,
|
|
154
|
+
workspaceId: outbox.workspaceId,
|
|
155
|
+
sessionId: outbox.targetSessionId,
|
|
156
|
+
workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
|
|
157
|
+
wakeRevision: result.workflowWakeRevision,
|
|
73
158
|
});
|
|
74
159
|
}
|
|
75
160
|
svc.observability.info("Woke parent session on worker completion", {
|
|
76
|
-
childSessionId,
|
|
77
|
-
parentSessionId:
|
|
78
|
-
|
|
161
|
+
childSessionId: outbox.sourceSessionId,
|
|
162
|
+
parentSessionId: outbox.targetSessionId,
|
|
163
|
+
dedupeKey: outbox.dedupeKey,
|
|
79
164
|
});
|
|
80
165
|
} catch (error) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
error
|
|
85
|
-
|
|
166
|
+
await markSessionSystemUpdateOutboxFailed(
|
|
167
|
+
svc.db,
|
|
168
|
+
outbox,
|
|
169
|
+
error instanceof Error ? error.message : String(error),
|
|
170
|
+
).catch(() => undefined);
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export async function reconcilePendingParentSystemUpdates(
|
|
176
|
+
svc: NotifyServices,
|
|
177
|
+
limit = 100,
|
|
178
|
+
overrides: ReconcileParentSystemUpdateOverrides = {},
|
|
179
|
+
): Promise<{ claimed: number; delivered: number; failed: number }> {
|
|
180
|
+
const claimPendingSessionSystemUpdateOutboxFn =
|
|
181
|
+
overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
|
|
182
|
+
// Do not claim or deliver the child-terminal outbox while completion wakes
|
|
183
|
+
// are disabled. Atomic terminal producers use the same setting, so no new
|
|
184
|
+
// backlog is manufactured either. Session-workflow wake repair is owned by
|
|
185
|
+
// its dedicated dispatcher and never depends on sandbox or child settings.
|
|
186
|
+
if (!svc.settings.childCompletionParentWakeEnabled) {
|
|
187
|
+
return { claimed: 0, delivered: 0, failed: 0 };
|
|
188
|
+
}
|
|
189
|
+
const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
|
|
190
|
+
let delivered = 0;
|
|
191
|
+
let failed = 0;
|
|
192
|
+
for (const row of rows) {
|
|
193
|
+
try {
|
|
194
|
+
await deliverParentSystemUpdateOutbox(svc, row);
|
|
195
|
+
delivered += 1;
|
|
196
|
+
} catch {
|
|
197
|
+
failed += 1;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return { claimed: rows.length, delivered, failed };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export async function reconcilePendingSessionWorkflowWakes(
|
|
204
|
+
svc: NotifyServices,
|
|
205
|
+
limit = 1_000,
|
|
206
|
+
overrides: ReconcileSessionWorkflowWakeOverrides = {},
|
|
207
|
+
): Promise<{ claimed: number; delivered: number; failed: number }> {
|
|
208
|
+
const claimPendingSessionWorkflowWakesFn =
|
|
209
|
+
overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
|
|
210
|
+
if (!svc.wakeSessionWorkflow) {
|
|
211
|
+
return { claimed: 0, delivered: 0, failed: 0 };
|
|
86
212
|
}
|
|
213
|
+
const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
|
|
214
|
+
let delivered = 0;
|
|
215
|
+
let failed = 0;
|
|
216
|
+
const queue = [...repairs];
|
|
217
|
+
const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
|
|
218
|
+
for (;;) {
|
|
219
|
+
const repair = queue.shift();
|
|
220
|
+
if (!repair) return;
|
|
221
|
+
try {
|
|
222
|
+
await svc.wakeSessionWorkflow!({
|
|
223
|
+
accountId: repair.accountId,
|
|
224
|
+
workspaceId: repair.workspaceId,
|
|
225
|
+
sessionId: repair.sessionId,
|
|
226
|
+
workflowId: repair.temporalWorkflowId,
|
|
227
|
+
wakeRevision: repair.wakeRevision,
|
|
228
|
+
...(repair.controlEventId ? { controlEventId: repair.controlEventId } : {}),
|
|
229
|
+
});
|
|
230
|
+
delivered += 1;
|
|
231
|
+
} catch (error) {
|
|
232
|
+
failed += 1;
|
|
233
|
+
await markSessionWorkflowWakeFailed(
|
|
234
|
+
svc.db,
|
|
235
|
+
repair,
|
|
236
|
+
error instanceof Error ? error.message : String(error),
|
|
237
|
+
).catch(() => undefined);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
await Promise.all(workers);
|
|
242
|
+
return { claimed: repairs.length, delivered, failed };
|
|
87
243
|
}
|
|
88
244
|
|
|
89
|
-
function childCompletionPayload(
|
|
245
|
+
function childCompletionPayload(
|
|
246
|
+
child: Session,
|
|
247
|
+
goal: SessionGoal | null,
|
|
248
|
+
terminalStatus: "idle" | "failed",
|
|
249
|
+
): Record<string, unknown> {
|
|
90
250
|
return {
|
|
91
251
|
childSessionId: child.id,
|
|
92
252
|
status: terminalStatus,
|
|
93
253
|
...(goal
|
|
94
254
|
? {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
255
|
+
goal: {
|
|
256
|
+
status: goal.status,
|
|
257
|
+
text: goal.text,
|
|
258
|
+
...(goal.evidence ? { evidence: goal.evidence } : {}),
|
|
259
|
+
...(goal.rationale ? { rationale: goal.rationale } : {}),
|
|
260
|
+
...(goal.pausedReason ? { pausedReason: goal.pausedReason } : {}),
|
|
261
|
+
},
|
|
262
|
+
}
|
|
103
263
|
: {}),
|
|
104
264
|
};
|
|
105
265
|
}
|
|
106
266
|
|
|
107
|
-
|
|
267
|
+
/**
|
|
268
|
+
* The worker-specific lines for one child (what happened + its goal), WITHOUT
|
|
269
|
+
* the trailing "what to do next" instruction. Kept separate so N child
|
|
270
|
+
* completions can be coalesced into one internal-update inference: the DB layer
|
|
271
|
+
* stores each child's summary and rebuilds a single numbered digest with one
|
|
272
|
+
* shared trailing instruction instead of running N model calls.
|
|
273
|
+
*/
|
|
274
|
+
export function childCompletionSummary(
|
|
275
|
+
child: Session,
|
|
276
|
+
goal: SessionGoal | null,
|
|
277
|
+
terminalStatus: "idle" | "failed",
|
|
278
|
+
): string {
|
|
108
279
|
const lines: string[] = [];
|
|
109
280
|
if (terminalStatus === "failed") {
|
|
110
281
|
lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
|
|
111
282
|
} else if (goal?.status === "completed") {
|
|
112
|
-
lines.push(
|
|
283
|
+
lines.push(
|
|
284
|
+
`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`,
|
|
285
|
+
);
|
|
113
286
|
} else if (goal?.status === "paused") {
|
|
114
|
-
lines.push(
|
|
287
|
+
lines.push(
|
|
288
|
+
`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`,
|
|
289
|
+
);
|
|
115
290
|
} else {
|
|
116
|
-
lines.push(
|
|
291
|
+
lines.push(
|
|
292
|
+
`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`,
|
|
293
|
+
);
|
|
117
294
|
}
|
|
118
295
|
if (goal) {
|
|
119
296
|
lines.push(`Worker goal: ${goal.text}`);
|
|
@@ -124,6 +301,16 @@ function childCompletionWakeText(child: Session, goal: SessionGoal | null, termi
|
|
|
124
301
|
lines.push(`Pause rationale: ${goal.rationale}`);
|
|
125
302
|
}
|
|
126
303
|
}
|
|
127
|
-
lines.push("Read the worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting this worker, resume it now.");
|
|
128
304
|
return lines.join("\n");
|
|
129
305
|
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* The single trailing instruction appended to a child-completion (or digest)
|
|
309
|
+
* inference. Suppresses the "resume it now" nudge when the manager's own goal
|
|
310
|
+
* was paused by the user (paired with the goal_set reactivation guard).
|
|
311
|
+
*/
|
|
312
|
+
export function childCompletionTrailing(parentGoalUserPaused: boolean): string {
|
|
313
|
+
return parentGoalUserPaused
|
|
314
|
+
? "Read each worker's session events/notebook output for its result. This session was paused by the user — do NOT resume or replace your goal; summarize the result for the user and remain paused."
|
|
315
|
+
: "Read each worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting these workers, resume it now.";
|
|
316
|
+
}
|