@opengeni/worker-bundle 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +184 -68
- package/dist/index.js +7099 -1643
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4332 -1068
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +14 -2
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +216 -42
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +165 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +106 -36
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflows/activities.ts +65 -8
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -192
- package/src/workflows/workflow-wake.ts +13 -0
- package/src/workflows.ts +9 -7
|
@@ -1,208 +1,214 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
settleSessionAttemptInterruptions,
|
|
3
|
+
applySessionTurnSettlement,
|
|
4
|
+
recoverSessionDispatch,
|
|
5
|
+
peekSessionWork as peekSessionWorkDb,
|
|
3
6
|
countQueuedTurns,
|
|
4
|
-
countTurnSessionHistoryItems,
|
|
5
|
-
finishTurn,
|
|
6
7
|
getSessionEvent,
|
|
7
|
-
|
|
8
|
-
incrementTurnWorkerDeathRedispatches,
|
|
9
|
-
requeuePreemptedTurn,
|
|
8
|
+
getSessionTurnForAttempt,
|
|
10
9
|
requireSession,
|
|
11
|
-
|
|
10
|
+
settleSessionIdleWithParentOutbox,
|
|
12
11
|
} from "@opengeni/db";
|
|
13
|
-
import {
|
|
14
|
-
import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
|
|
15
|
-
import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
|
|
12
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
16
13
|
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
17
14
|
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
18
15
|
import type {
|
|
19
16
|
ActivityServices,
|
|
20
|
-
|
|
17
|
+
PeekSessionWorkInput,
|
|
18
|
+
FailSessionAttemptInput,
|
|
19
|
+
SettleSessionInterruptionsInput,
|
|
21
20
|
MarkSessionIdleInput,
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
RunAgentTurnInput,
|
|
21
|
+
RecoverDispatchInput,
|
|
22
|
+
RecoverDispatchResult,
|
|
25
23
|
} from "./types";
|
|
26
24
|
|
|
25
|
+
export type SessionStateActivityOverrides = Partial<{
|
|
26
|
+
settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
|
|
27
|
+
applySessionTurnSettlement: typeof applySessionTurnSettlement;
|
|
28
|
+
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
29
|
+
peekSessionWork: typeof peekSessionWorkDb;
|
|
30
|
+
countQueuedTurns: typeof countQueuedTurns;
|
|
31
|
+
getSessionEvent: typeof getSessionEvent;
|
|
32
|
+
getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
|
|
33
|
+
requireSession: typeof requireSession;
|
|
34
|
+
settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
|
|
35
|
+
publishDurableSessionEvents: typeof publishDurableSessionEvents;
|
|
36
|
+
notifyParentOfChildTerminal: typeof notifyParentOfChildTerminal;
|
|
37
|
+
recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
|
|
38
|
+
}>;
|
|
39
|
+
|
|
27
40
|
// Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
|
|
28
41
|
// down this many times in a row is assumed to be the cause, not the victim,
|
|
29
42
|
// and the session fails for real on the next death. A plain constant by
|
|
30
43
|
// design — this is a pathology bound, not a run-length limit.
|
|
31
44
|
export const WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
32
45
|
|
|
33
|
-
export function createSessionStateActivities(
|
|
34
|
-
|
|
46
|
+
export function createSessionStateActivities(
|
|
47
|
+
services: () => Promise<ActivityServices>,
|
|
48
|
+
overrides: SessionStateActivityOverrides = {},
|
|
49
|
+
) {
|
|
50
|
+
const settleSessionAttemptInterruptionsFn =
|
|
51
|
+
overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
|
|
52
|
+
const applySessionTurnSettlementFn =
|
|
53
|
+
overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
|
|
54
|
+
const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
|
|
55
|
+
const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
|
|
56
|
+
const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
|
|
57
|
+
const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
|
|
58
|
+
const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
|
|
59
|
+
const requireSessionFn = overrides.requireSession ?? requireSession;
|
|
60
|
+
const settleSessionIdleWithParentOutboxFn =
|
|
61
|
+
overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
|
|
62
|
+
const publishDurableSessionEventsFn =
|
|
63
|
+
overrides.publishDurableSessionEvents ?? publishDurableSessionEvents;
|
|
64
|
+
const notifyParentOfChildTerminalFn =
|
|
65
|
+
overrides.notifyParentOfChildTerminal ?? notifyParentOfChildTerminal;
|
|
66
|
+
const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
|
|
67
|
+
|
|
68
|
+
async function failSessionAttempt(input: FailSessionAttemptInput): Promise<void> {
|
|
35
69
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
36
|
-
const session = await
|
|
37
|
-
// Already terminal: runAgentTurn settled this turn as failed (status failed,
|
|
38
|
-
// activeTurnId cleared, turn finished) and already woke any parent, then the
|
|
39
|
-
// workflow still routed here because the activity's finally threw after the
|
|
40
|
-
// failed return. Re-failing would append a second turn.failed/status.changed
|
|
41
|
-
// and a second parent wake (a different lastSequence dodges the idle dedupe).
|
|
42
|
-
// The session is already where this activity would put it, so stop.
|
|
70
|
+
const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
|
|
43
71
|
if (session.status === "failed") {
|
|
44
72
|
return;
|
|
45
73
|
}
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
74
|
+
const turn = await getSessionTurnForAttemptFn(
|
|
75
|
+
db,
|
|
76
|
+
input.workspaceId,
|
|
77
|
+
input.sessionId,
|
|
78
|
+
input.attemptId,
|
|
79
|
+
);
|
|
80
|
+
if (!turn) return;
|
|
81
|
+
const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
|
|
82
|
+
const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
|
|
83
|
+
sessionId: input.sessionId,
|
|
84
|
+
turnId: turn.id,
|
|
85
|
+
triggerEventId: turn.triggerEventId,
|
|
86
|
+
attemptId: input.attemptId,
|
|
87
|
+
turnStatus: "failed",
|
|
88
|
+
sessionStatus: "failed",
|
|
89
|
+
activeTurnId: null,
|
|
90
|
+
events: [
|
|
91
|
+
{
|
|
92
|
+
type: "turn.failed",
|
|
93
|
+
payload: {
|
|
94
|
+
triggerEventId: turn.triggerEventId,
|
|
95
|
+
trigger: trigger?.payload ?? null,
|
|
96
|
+
error: input.error ?? "Agent activity failed before it could report a terminal state.",
|
|
97
|
+
},
|
|
56
98
|
},
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
},
|
|
63
|
-
]);
|
|
64
|
-
if (turnId) {
|
|
65
|
-
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
99
|
+
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
100
|
+
],
|
|
101
|
+
});
|
|
102
|
+
if (result.action === "stale") {
|
|
103
|
+
return;
|
|
66
104
|
}
|
|
67
|
-
await
|
|
68
|
-
await
|
|
105
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
106
|
+
await notifyParentOfChildTerminalFn(
|
|
107
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
108
|
+
input.workspaceId,
|
|
109
|
+
input.sessionId,
|
|
110
|
+
"failed",
|
|
111
|
+
);
|
|
69
112
|
}
|
|
70
113
|
|
|
71
|
-
async function
|
|
114
|
+
async function settleSessionInterruptions(
|
|
115
|
+
input: SettleSessionInterruptionsInput,
|
|
116
|
+
): Promise<{ action: "paused" | "continue" | "stale" }> {
|
|
72
117
|
const { db, bus, observability } = await services();
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
if (!isSteerInterrupt(trigger)) {
|
|
82
|
-
await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
|
|
83
|
-
}
|
|
84
|
-
if (!session.activeTurnId) {
|
|
85
|
-
return;
|
|
118
|
+
const applied = await settleSessionAttemptInterruptionsFn(
|
|
119
|
+
db,
|
|
120
|
+
input.workspaceId,
|
|
121
|
+
input.sessionId,
|
|
122
|
+
input.attemptId,
|
|
123
|
+
);
|
|
124
|
+
if (applied.events.length > 0) {
|
|
125
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, applied.events);
|
|
86
126
|
}
|
|
87
|
-
await
|
|
88
|
-
|
|
89
|
-
turnId: session.activeTurnId,
|
|
90
|
-
type: "turn.cancelled",
|
|
91
|
-
payload: { triggerEventId: input.triggerEventId, reason: trigger?.payload ?? null },
|
|
92
|
-
},
|
|
93
|
-
{
|
|
94
|
-
turnId: session.activeTurnId,
|
|
95
|
-
type: "session.status.changed",
|
|
96
|
-
payload: { status: "queued" },
|
|
97
|
-
},
|
|
98
|
-
]);
|
|
99
|
-
await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
100
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
101
|
-
await refreshQueuedTurnsGauge(db, observability);
|
|
127
|
+
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
128
|
+
return { action: applied.action };
|
|
102
129
|
}
|
|
103
130
|
|
|
104
131
|
/**
|
|
105
|
-
*
|
|
106
|
-
* checkpoint (heartbeat
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
* from the trigger event plus stored items loses at most the work since the
|
|
111
|
-
* last reconcile. When the dead attempt persisted items for this turn, the
|
|
112
|
-
* rerun enters through a synthesized `turn.preempted` resume notice (its
|
|
113
|
-
* partial progress is already conversation truth; replaying the original
|
|
114
|
-
* trigger would duplicate input the model has seen). When nothing was
|
|
115
|
-
* persisted — checkpoint absent — the original trigger replays cleanly.
|
|
116
|
-
* Approval reruns always replay their original trigger: the decision is
|
|
117
|
-
* applied through the RunState resume path and a swapped trigger could
|
|
118
|
-
* drop it. Re-dispatches are bounded per turn by
|
|
119
|
-
* WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
|
|
132
|
+
* Recover the same current inference when its worker dies without completing
|
|
133
|
+
* a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
|
|
134
|
+
* Durable conversation truth and the original trigger stay attached to the
|
|
135
|
+
* same turn; recovery creates a new fenced attempt, never a prompt-queue row
|
|
136
|
+
* or a synthetic resume message. Repeated worker deaths are bounded per turn.
|
|
120
137
|
*/
|
|
121
|
-
async function
|
|
122
|
-
const { settings, db, bus, observability } = await services();
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
async function recoverDispatch(input: RecoverDispatchInput): Promise<RecoverDispatchResult> {
|
|
139
|
+
const { settings, db, bus, observability, wakeSessionWorkflow } = await services();
|
|
140
|
+
const result = await recoverSessionDispatchFn(db, input.workspaceId, {
|
|
141
|
+
sessionId: input.sessionId,
|
|
142
|
+
attemptId: input.attemptId,
|
|
143
|
+
timeoutType: input.timeoutType,
|
|
144
|
+
maxRedispatches: WORKER_DEATH_MAX_REDISPATCHES,
|
|
145
|
+
});
|
|
146
|
+
if (result.action === "stale" || result.action === "unclaimed") {
|
|
147
|
+
return { action: result.action };
|
|
129
148
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
149
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
|
|
150
|
+
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
151
|
+
if (result.action === "exceeded") {
|
|
152
|
+
await notifyParentOfChildTerminalFn(
|
|
153
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
154
|
+
input.workspaceId,
|
|
155
|
+
input.sessionId,
|
|
156
|
+
"failed",
|
|
157
|
+
`turn:${result.turnId}`,
|
|
158
|
+
);
|
|
159
|
+
return {
|
|
160
|
+
action: "exceeded",
|
|
161
|
+
turnId: result.turnId,
|
|
162
|
+
redispatches: result.redispatches,
|
|
163
|
+
};
|
|
133
164
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
const resumeWithNotice = !approvalRerun
|
|
140
|
-
&& settings.sessionHistorySource === "items"
|
|
141
|
-
&& await countTurnSessionHistoryItems(db, input.workspaceId, input.turnId) > 0;
|
|
142
|
-
const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
|
|
143
|
-
{
|
|
144
|
-
turnId: turn.id,
|
|
145
|
-
type: "turn.preempted",
|
|
146
|
-
payload: {
|
|
147
|
-
triggerEventId: input.triggerEventId,
|
|
148
|
-
reason: "worker_death",
|
|
149
|
-
redispatches,
|
|
150
|
-
resumeWithNotice,
|
|
151
|
-
...(resumeWithNotice ? { text: WORKER_DEATH_RESUME_TEXT } : {}),
|
|
152
|
-
},
|
|
153
|
-
},
|
|
154
|
-
{
|
|
155
|
-
turnId: turn.id,
|
|
156
|
-
type: "session.status.changed",
|
|
157
|
-
payload: { status: "queued" },
|
|
158
|
-
},
|
|
159
|
-
]);
|
|
160
|
-
try {
|
|
161
|
-
await requeuePreemptedTurn(db, input.workspaceId, turn.id, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
162
|
-
} catch (requeueError) {
|
|
163
|
-
// The zombie attempt can settle the turn between the status check above
|
|
164
|
-
// and this requeue (it keeps executing until it notices the timeout).
|
|
165
|
-
// A settled turn means its recorded outcome is the truth: report stale
|
|
166
|
-
// so the workflow continues instead of failing the session over a lost
|
|
167
|
-
// race. Anything else is a real persistence failure — rethrow.
|
|
168
|
-
const current = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
169
|
-
if (current && current.status !== "running" && current.status !== "requires_action") {
|
|
170
|
-
return { action: "stale" };
|
|
171
|
-
}
|
|
172
|
-
throw requeueError;
|
|
173
|
-
}
|
|
174
|
-
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
175
|
-
await refreshQueuedTurnsGauge(db, observability);
|
|
176
|
-
return { action: "requeued", redispatches };
|
|
165
|
+
return {
|
|
166
|
+
action: "recovering",
|
|
167
|
+
turnId: result.turnId,
|
|
168
|
+
redispatches: result.redispatches,
|
|
169
|
+
};
|
|
177
170
|
}
|
|
178
171
|
|
|
179
|
-
async function
|
|
172
|
+
async function peekSessionWork(input: PeekSessionWorkInput) {
|
|
180
173
|
const { db, observability } = await services();
|
|
181
|
-
const
|
|
182
|
-
await refreshQueuedTurnsGauge(db, observability);
|
|
183
|
-
return
|
|
174
|
+
const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
|
|
175
|
+
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
176
|
+
return peek;
|
|
184
177
|
}
|
|
185
178
|
|
|
186
179
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
187
180
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
181
|
+
const settled = await settleSessionIdleWithParentOutboxFn(
|
|
182
|
+
db,
|
|
183
|
+
input.workspaceId,
|
|
184
|
+
input.sessionId,
|
|
185
|
+
);
|
|
186
|
+
if (settled.events.length > 0) {
|
|
187
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, settled.events);
|
|
188
|
+
}
|
|
189
|
+
await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
|
|
190
|
+
if (settled.action === "stale") {
|
|
191
|
+
return;
|
|
191
192
|
}
|
|
192
|
-
await refreshQueuedTurnsGauge(db, observability);
|
|
193
193
|
// The workflow reaches markSessionIdle exactly when it has decided to stop
|
|
194
194
|
// for now (no queued turn, no goal continuation): the terminal-for-now
|
|
195
195
|
// point for a spawned worker, whatever the cause (goal completed, agent or
|
|
196
|
-
// system paused goal, goalless work finished, idle
|
|
196
|
+
// system paused goal, goalless work finished, idle control settlement). Wake
|
|
197
197
|
// the parent here, deduped per idle episode so the manager is nudged once.
|
|
198
|
-
await
|
|
198
|
+
await notifyParentOfChildTerminalFn(
|
|
199
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
200
|
+
input.workspaceId,
|
|
201
|
+
input.sessionId,
|
|
202
|
+
"idle",
|
|
203
|
+
settled.episodeKey,
|
|
204
|
+
);
|
|
199
205
|
}
|
|
200
206
|
|
|
201
207
|
return {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
failSessionAttempt,
|
|
209
|
+
settleSessionInterruptions,
|
|
210
|
+
recoverDispatch,
|
|
211
|
+
peekSessionWork,
|
|
206
212
|
markSessionIdle,
|
|
207
213
|
};
|
|
208
214
|
}
|
|
@@ -210,9 +216,11 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
210
216
|
async function refreshQueuedTurnsGauge(
|
|
211
217
|
db: ActivityServices["db"],
|
|
212
218
|
observability: ActivityServices["observability"],
|
|
219
|
+
countQueuedTurnsFn: typeof countQueuedTurns,
|
|
220
|
+
recordTurnsQueuedGaugeFn: typeof recordTurnsQueuedGauge,
|
|
213
221
|
): Promise<void> {
|
|
214
222
|
try {
|
|
215
|
-
|
|
223
|
+
recordTurnsQueuedGaugeFn(observability, await countQueuedTurnsFn(db));
|
|
216
224
|
} catch {
|
|
217
225
|
// Best-effort telemetry; session state transitions remain authoritative.
|
|
218
226
|
}
|
|
@@ -2,15 +2,52 @@ import type { SessionEventType } from "@opengeni/contracts";
|
|
|
2
2
|
import type { AppendEventInput } from "@opengeni/db";
|
|
3
3
|
import { Context } from "@temporalio/activity";
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
// Trailing-flush window. A burst of coalesced deltas followed by model silence
|
|
6
|
+
// must not sit in `pending` unbounded — without a timer, flush fires only on
|
|
7
|
+
// the NEXT push. Matched to the 33ms coalesce window so the client-visible
|
|
8
|
+
// latency added by batching is bounded to ~one window (imperceptible, and far
|
|
9
|
+
// cheaper than the per-delta DB round-trip it replaces).
|
|
10
|
+
const TRAILING_FLUSH_MS = 33;
|
|
11
|
+
const MAX_BATCH_EVENTS = 50;
|
|
12
|
+
// Keep model ingestion independent from a slow durable append, but never let a
|
|
13
|
+
// provider outrun storage without bound. At the high-water mark `push` waits for
|
|
14
|
+
// the serialized drain before consuming more provider output.
|
|
15
|
+
const MAX_BUFFERED_EVENTS = 1_000;
|
|
16
|
+
|
|
17
|
+
/** Optional metrics seam for the batcher. `onFlush` fires once per flush (on both
|
|
18
|
+
* success and failure) with the coalesced event count and the flush duration, so
|
|
19
|
+
* the worker can meter batch shape without the batcher knowing about Observability.
|
|
20
|
+
* `now` is injectable for deterministic tests. Both are optional; absent ⇒ no-op. */
|
|
21
|
+
export type RuntimeBatcherOptions = {
|
|
22
|
+
onFlush?: (info: { events: number; durationSeconds: number }) => void;
|
|
23
|
+
now?: () => number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export function createRuntimeBatcher(
|
|
27
|
+
flushEvents: (events: AppendEventInput[]) => Promise<void>,
|
|
28
|
+
options: RuntimeBatcherOptions = {},
|
|
29
|
+
) {
|
|
30
|
+
const now = options.now ?? (() => performance.now());
|
|
6
31
|
let pending: AppendEventInput[] = [];
|
|
7
32
|
let lastFlush = Date.now();
|
|
33
|
+
let trailingTimer: ReturnType<typeof setTimeout> | null = null;
|
|
34
|
+
let activeBatchSize = 0;
|
|
35
|
+
// One drain loop owns `flushEvents`, so DB-assigned sequence order always
|
|
36
|
+
// matches push order. Ordinary deltas do not await this promise; structural
|
|
37
|
+
// boundaries, explicit flushes, and the bounded high-water mark do.
|
|
38
|
+
let drainPromise: Promise<void> | null = null;
|
|
39
|
+
// A timer has no direct awaiter. Retain its failure so the next push or final
|
|
40
|
+
// flush terminates the turn instead of silently losing streamed evidence.
|
|
41
|
+
let flushFailed = false;
|
|
42
|
+
let flushFailure: unknown;
|
|
43
|
+
// The high-volume token deltas (agent.message.delta, agent.reasoning.delta,
|
|
44
|
+
// sandbox.command.output.delta) are DELIBERATELY NOT here: they coalesce
|
|
45
|
+
// under the 50-event / 33ms policy into one appendSessionEvents txn + one
|
|
46
|
+
// publish per flush, instead of one DB round-trip per token. Only events that
|
|
47
|
+
// must be delivered promptly stay structural (flush-immediately). PTY bytes
|
|
48
|
+
// stay structural on purpose — an interactive terminal is useless batched
|
|
49
|
+
// 33ms behind (P4.4).
|
|
8
50
|
const structural = new Set<SessionEventType>([
|
|
9
|
-
"agent.message.delta",
|
|
10
|
-
"agent.reasoning.delta",
|
|
11
|
-
"sandbox.command.output.delta",
|
|
12
|
-
// Channel-A PTY bytes flush promptly (P4.4) — a terminal stream is useless
|
|
13
|
-
// batched 33ms behind; the structural set bypasses the batcher's coalesce.
|
|
14
51
|
"terminal.pty.output.delta",
|
|
15
52
|
"agent.toolCall.created",
|
|
16
53
|
"agent.toolCall.output",
|
|
@@ -23,23 +60,107 @@ export function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) =
|
|
|
23
60
|
]);
|
|
24
61
|
return {
|
|
25
62
|
push: async (event: { type: SessionEventType; payload: unknown }) => {
|
|
63
|
+
throwIfFlushFailed();
|
|
64
|
+
// Append BEFORE the structural check so a structural event's flush always
|
|
65
|
+
// carries any pending deltas in order (same flush, order preserved).
|
|
26
66
|
pending.push({ type: event.type, payload: event.payload });
|
|
27
67
|
const elapsed = Date.now() - lastFlush;
|
|
28
|
-
if (
|
|
68
|
+
if (structural.has(event.type)) {
|
|
69
|
+
await flush();
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
|
|
73
|
+
// Start the serialized drain, but keep consuming provider deltas while
|
|
74
|
+
// the append/publish round-trip is in flight. The active drain records
|
|
75
|
+
// any failure for the next awaited boundary.
|
|
76
|
+
void startDrain().catch(() => undefined);
|
|
77
|
+
}
|
|
78
|
+
if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
|
|
29
79
|
await flush();
|
|
80
|
+
return;
|
|
30
81
|
}
|
|
82
|
+
armTrailingTimer();
|
|
31
83
|
},
|
|
32
84
|
flush,
|
|
33
85
|
};
|
|
34
86
|
|
|
35
|
-
|
|
36
|
-
if (
|
|
87
|
+
function armTrailingTimer() {
|
|
88
|
+
if (trailingTimer !== null) {
|
|
37
89
|
return;
|
|
38
90
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
91
|
+
trailingTimer = setTimeout(() => {
|
|
92
|
+
trailingTimer = null;
|
|
93
|
+
// A timer has no awaiter, so prevent an unhandled rejection. startDrain
|
|
94
|
+
// retains the exact error and the next push/final flush rethrows it.
|
|
95
|
+
void flush().catch(() => undefined);
|
|
96
|
+
}, TRAILING_FLUSH_MS);
|
|
97
|
+
if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
|
|
98
|
+
trailingTimer.unref();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function clearTrailingTimer() {
|
|
103
|
+
if (trailingTimer !== null) {
|
|
104
|
+
clearTimeout(trailingTimer);
|
|
105
|
+
trailingTimer = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function flush(): Promise<void> {
|
|
110
|
+
throwIfFlushFailed();
|
|
111
|
+
clearTrailingTimer();
|
|
112
|
+
await startDrain();
|
|
113
|
+
throwIfFlushFailed();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function startDrain(): Promise<void> {
|
|
117
|
+
if (drainPromise) {
|
|
118
|
+
return drainPromise;
|
|
119
|
+
}
|
|
120
|
+
if (flushFailed) {
|
|
121
|
+
return Promise.reject(flushFailure);
|
|
122
|
+
}
|
|
123
|
+
if (pending.length === 0) {
|
|
124
|
+
return Promise.resolve();
|
|
125
|
+
}
|
|
126
|
+
clearTrailingTimer();
|
|
127
|
+
const running = (async () => {
|
|
128
|
+
while (pending.length > 0) {
|
|
129
|
+
const events = pending.splice(0, MAX_BATCH_EVENTS);
|
|
130
|
+
activeBatchSize = events.length;
|
|
131
|
+
lastFlush = Date.now();
|
|
132
|
+
const startedAt = now();
|
|
133
|
+
try {
|
|
134
|
+
await flushEvents(events);
|
|
135
|
+
} catch (error) {
|
|
136
|
+
flushFailed = true;
|
|
137
|
+
flushFailure = error;
|
|
138
|
+
throw error;
|
|
139
|
+
} finally {
|
|
140
|
+
activeBatchSize = 0;
|
|
141
|
+
if (options.onFlush) {
|
|
142
|
+
try {
|
|
143
|
+
options.onFlush({
|
|
144
|
+
events: events.length,
|
|
145
|
+
durationSeconds: Math.max(0, (now() - startedAt) / 1000),
|
|
146
|
+
});
|
|
147
|
+
} catch {
|
|
148
|
+
// Metrics emission must never affect a flush.
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
})();
|
|
154
|
+
drainPromise = running.finally(() => {
|
|
155
|
+
drainPromise = null;
|
|
156
|
+
});
|
|
157
|
+
return drainPromise;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function throwIfFlushFailed(): void {
|
|
161
|
+
if (flushFailed) {
|
|
162
|
+
throw flushFailure;
|
|
163
|
+
}
|
|
43
164
|
}
|
|
44
165
|
}
|
|
45
166
|
|
|
@@ -51,7 +172,10 @@ export function currentActivityContext(): Context | null {
|
|
|
51
172
|
}
|
|
52
173
|
}
|
|
53
174
|
|
|
54
|
-
export function startActivityHeartbeat(
|
|
175
|
+
export function startActivityHeartbeat(
|
|
176
|
+
context: Context | null,
|
|
177
|
+
details: Record<string, unknown>,
|
|
178
|
+
): ReturnType<typeof setInterval> | null {
|
|
55
179
|
if (!context) {
|
|
56
180
|
return null;
|
|
57
181
|
}
|
|
@@ -64,12 +188,12 @@ export function startActivityHeartbeat(context: Context | null, details: Record<
|
|
|
64
188
|
return timer;
|
|
65
189
|
}
|
|
66
190
|
|
|
67
|
-
export async function nextStreamEvent<T>(
|
|
191
|
+
export async function nextStreamEvent<T>(
|
|
192
|
+
iterator: AsyncIterator<T>,
|
|
193
|
+
context: Context | null,
|
|
194
|
+
): Promise<IteratorResult<T>> {
|
|
68
195
|
if (!context) {
|
|
69
196
|
return await iterator.next();
|
|
70
197
|
}
|
|
71
|
-
return await Promise.race([
|
|
72
|
-
iterator.next(),
|
|
73
|
-
context.cancelled,
|
|
74
|
-
]);
|
|
198
|
+
return await Promise.race([iterator.next(), context.cancelled]);
|
|
75
199
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The activity lost ownership of its exact turn generation/attempt.
|
|
3
|
+
*
|
|
4
|
+
* This is an ordinary stale-writer outcome, not a Temporal cancellation. The
|
|
5
|
+
* current owner or control transaction already holds authoritative state, so
|
|
6
|
+
* the losing activity must stop locally without asking the workflow to settle
|
|
7
|
+
* or fail the turn again.
|
|
8
|
+
*/
|
|
9
|
+
export class TurnAttemptFencedError extends Error {
|
|
10
|
+
constructor(message: string) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.name = "TurnAttemptFencedError";
|
|
13
|
+
}
|
|
14
|
+
}
|