@opengeni/worker-bundle 0.7.1 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/index.d.ts +184 -68
  2. package/dist/index.js +7099 -1643
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4332 -1068
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +14 -2
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +216 -42
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +165 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +106 -36
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflows/activities.ts +65 -8
  39. package/src/workflows/file-upload-reaper.ts +12 -0
  40. package/src/workflows/rig-verification.ts +28 -0
  41. package/src/workflows/scheduled-tasks.ts +12 -26
  42. package/src/workflows/session.ts +330 -192
  43. package/src/workflows/workflow-wake.ts +13 -0
  44. package/src/workflows.ts +9 -7
@@ -1,208 +1,214 @@
1
1
  import {
2
- claimNextQueuedTurn as claimNextQueuedTurnDb,
2
+ settleSessionAttemptInterruptions,
3
+ applySessionTurnSettlement,
4
+ recoverSessionDispatch,
5
+ peekSessionWork as peekSessionWorkDb,
3
6
  countQueuedTurns,
4
- countTurnSessionHistoryItems,
5
- finishTurn,
6
7
  getSessionEvent,
7
- getSessionTurn,
8
- incrementTurnWorkerDeathRedispatches,
9
- requeuePreemptedTurn,
8
+ getSessionTurnForAttempt,
10
9
  requireSession,
11
- setSessionStatus,
10
+ settleSessionIdleWithParentOutbox,
12
11
  } from "@opengeni/db";
13
- import { appendAndPublishEvents } from "@opengeni/events";
14
- import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
15
- import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
12
+ import { publishDurableSessionEvents } from "@opengeni/events";
16
13
  import { notifyParentOfChildTerminal } from "./parent-wake";
17
14
  import { recordTurnsQueuedGauge } from "../observability-metrics";
18
15
  import type {
19
16
  ActivityServices,
20
- ClaimNextQueuedTurnInput,
17
+ PeekSessionWorkInput,
18
+ FailSessionAttemptInput,
19
+ SettleSessionInterruptionsInput,
21
20
  MarkSessionIdleInput,
22
- RequeueTurnAfterWorkerDeathInput,
23
- RequeueTurnAfterWorkerDeathResult,
24
- RunAgentTurnInput,
21
+ RecoverDispatchInput,
22
+ RecoverDispatchResult,
25
23
  } from "./types";
26
24
 
25
+ export type SessionStateActivityOverrides = Partial<{
26
+ settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
27
+ applySessionTurnSettlement: typeof applySessionTurnSettlement;
28
+ recoverSessionDispatch: typeof recoverSessionDispatch;
29
+ peekSessionWork: typeof peekSessionWorkDb;
30
+ countQueuedTurns: typeof countQueuedTurns;
31
+ getSessionEvent: typeof getSessionEvent;
32
+ getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
33
+ requireSession: typeof requireSession;
34
+ settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
35
+ publishDurableSessionEvents: typeof publishDurableSessionEvents;
36
+ notifyParentOfChildTerminal: typeof notifyParentOfChildTerminal;
37
+ recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
38
+ }>;
39
+
27
40
  // Crash-loop guard for worker-death re-dispatch: a turn that takes a worker
28
41
  // down this many times in a row is assumed to be the cause, not the victim,
29
42
  // and the session fails for real on the next death. A plain constant by
30
43
  // design — this is a pathology bound, not a run-length limit.
31
44
  export const WORKER_DEATH_MAX_REDISPATCHES = 3;
32
45
 
33
- export function createSessionStateActivities(services: () => Promise<ActivityServices>) {
34
- async function failSession(input: RunAgentTurnInput & { error?: string }): Promise<void> {
46
+ export function createSessionStateActivities(
47
+ services: () => Promise<ActivityServices>,
48
+ overrides: SessionStateActivityOverrides = {},
49
+ ) {
50
+ const settleSessionAttemptInterruptionsFn =
51
+ overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
52
+ const applySessionTurnSettlementFn =
53
+ overrides.applySessionTurnSettlement ?? applySessionTurnSettlement;
54
+ const recoverSessionDispatchFn = overrides.recoverSessionDispatch ?? recoverSessionDispatch;
55
+ const peekSessionWorkFn = overrides.peekSessionWork ?? peekSessionWorkDb;
56
+ const countQueuedTurnsFn = overrides.countQueuedTurns ?? countQueuedTurns;
57
+ const getSessionEventFn = overrides.getSessionEvent ?? getSessionEvent;
58
+ const getSessionTurnForAttemptFn = overrides.getSessionTurnForAttempt ?? getSessionTurnForAttempt;
59
+ const requireSessionFn = overrides.requireSession ?? requireSession;
60
+ const settleSessionIdleWithParentOutboxFn =
61
+ overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
62
+ const publishDurableSessionEventsFn =
63
+ overrides.publishDurableSessionEvents ?? publishDurableSessionEvents;
64
+ const notifyParentOfChildTerminalFn =
65
+ overrides.notifyParentOfChildTerminal ?? notifyParentOfChildTerminal;
66
+ const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
67
+
68
+ async function failSessionAttempt(input: FailSessionAttemptInput): Promise<void> {
35
69
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
36
- const session = await requireSession(db, input.workspaceId, input.sessionId);
37
- // Already terminal: runAgentTurn settled this turn as failed (status failed,
38
- // activeTurnId cleared, turn finished) and already woke any parent, then the
39
- // workflow still routed here because the activity's finally threw after the
40
- // failed return. Re-failing would append a second turn.failed/status.changed
41
- // and a second parent wake (a different lastSequence dodges the idle dedupe).
42
- // The session is already where this activity would put it, so stop.
70
+ const session = await requireSessionFn(db, input.workspaceId, input.sessionId);
43
71
  if (session.status === "failed") {
44
72
  return;
45
73
  }
46
- const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
47
- const turnId = session.activeTurnId ?? null;
48
- await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
49
- {
50
- type: "turn.failed",
51
- turnId,
52
- payload: {
53
- triggerEventId: input.triggerEventId,
54
- trigger: trigger?.payload ?? null,
55
- error: input.error ?? "Agent activity failed before it could report a terminal state.",
74
+ const turn = await getSessionTurnForAttemptFn(
75
+ db,
76
+ input.workspaceId,
77
+ input.sessionId,
78
+ input.attemptId,
79
+ );
80
+ if (!turn) return;
81
+ const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
82
+ const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
83
+ sessionId: input.sessionId,
84
+ turnId: turn.id,
85
+ triggerEventId: turn.triggerEventId,
86
+ attemptId: input.attemptId,
87
+ turnStatus: "failed",
88
+ sessionStatus: "failed",
89
+ activeTurnId: null,
90
+ events: [
91
+ {
92
+ type: "turn.failed",
93
+ payload: {
94
+ triggerEventId: turn.triggerEventId,
95
+ trigger: trigger?.payload ?? null,
96
+ error: input.error ?? "Agent activity failed before it could report a terminal state.",
97
+ },
56
98
  },
57
- },
58
- {
59
- type: "session.status.changed",
60
- turnId,
61
- payload: { status: "failed" },
62
- },
63
- ]);
64
- if (turnId) {
65
- await finishTurn(db, input.workspaceId, turnId, "failed");
99
+ { type: "session.status.changed", payload: { status: "failed" } },
100
+ ],
101
+ });
102
+ if (result.action === "stale") {
103
+ return;
66
104
  }
67
- await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
68
- await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
105
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
106
+ await notifyParentOfChildTerminalFn(
107
+ { db, bus, settings, observability, wakeSessionWorkflow },
108
+ input.workspaceId,
109
+ input.sessionId,
110
+ "failed",
111
+ );
69
112
  }
70
113
 
71
- async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
114
+ async function settleSessionInterruptions(
115
+ input: SettleSessionInterruptionsInput,
116
+ ): Promise<{ action: "paused" | "continue" | "stale" }> {
72
117
  const { db, bus, observability } = await services();
73
- const session = await requireSession(db, input.workspaceId, input.sessionId);
74
- const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
75
- // Pause an active goal before the early return below: an interrupt can
76
- // land after the turn already cleared activeTurnId, and skipping the pause
77
- // there would let the loop auto-continue the goal the user just stopped.
78
- // Steer interrupts are the exception: steering cancels the running turn
79
- // only to deliver the steered message next — redirection, not a stop —
80
- // so the goal loop stays active.
81
- if (!isSteerInterrupt(trigger)) {
82
- await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
83
- }
84
- if (!session.activeTurnId) {
85
- return;
118
+ const applied = await settleSessionAttemptInterruptionsFn(
119
+ db,
120
+ input.workspaceId,
121
+ input.sessionId,
122
+ input.attemptId,
123
+ );
124
+ if (applied.events.length > 0) {
125
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, applied.events);
86
126
  }
87
- await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
88
- {
89
- turnId: session.activeTurnId,
90
- type: "turn.cancelled",
91
- payload: { triggerEventId: input.triggerEventId, reason: trigger?.payload ?? null },
92
- },
93
- {
94
- turnId: session.activeTurnId,
95
- type: "session.status.changed",
96
- payload: { status: "queued" },
97
- },
98
- ]);
99
- await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
100
- await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
101
- await refreshQueuedTurnsGauge(db, observability);
127
+ await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
128
+ return { action: applied.action };
102
129
  }
103
130
 
104
131
  /**
105
- * Put a turn whose hosting worker died WITHOUT the graceful-preempt
106
- * checkpoint (heartbeat-timeout activity failure: SIGKILL, OOM, node loss)
107
- * back on the session queue, so the workflow re-dispatches it instead of
108
- * failing the session. The degraded-resume contract: conversation truth is
109
- * dual-written after every model response during the turn, so re-running
110
- * from the trigger event plus stored items loses at most the work since the
111
- * last reconcile. When the dead attempt persisted items for this turn, the
112
- * rerun enters through a synthesized `turn.preempted` resume notice (its
113
- * partial progress is already conversation truth; replaying the original
114
- * trigger would duplicate input the model has seen). When nothing was
115
- * persisted — checkpoint absent — the original trigger replays cleanly.
116
- * Approval reruns always replay their original trigger: the decision is
117
- * applied through the RunState resume path and a swapped trigger could
118
- * drop it. Re-dispatches are bounded per turn by
119
- * WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
132
+ * Recover the same current inference when its worker dies without completing
133
+ * a graceful checkpoint (heartbeat timeout, SIGKILL, OOM, or node loss).
134
+ * Durable conversation truth and the original trigger stay attached to the
135
+ * same turn; recovery creates a new fenced attempt, never a prompt-queue row
136
+ * or a synthetic resume message. Repeated worker deaths are bounded per turn.
120
137
  */
121
- async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
122
- const { settings, db, bus, observability } = await services();
123
- const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
124
- if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
125
- // The timed-out attempt was a zombie that actually settled the turn
126
- // (completed/failed/cancelled it) after the server gave up on its
127
- // heartbeats. Whatever it recorded is the truth; nothing to redo.
128
- return { action: "stale" };
138
+ async function recoverDispatch(input: RecoverDispatchInput): Promise<RecoverDispatchResult> {
139
+ const { settings, db, bus, observability, wakeSessionWorkflow } = await services();
140
+ const result = await recoverSessionDispatchFn(db, input.workspaceId, {
141
+ sessionId: input.sessionId,
142
+ attemptId: input.attemptId,
143
+ timeoutType: input.timeoutType,
144
+ maxRedispatches: WORKER_DEATH_MAX_REDISPATCHES,
145
+ });
146
+ if (result.action === "stale" || result.action === "unclaimed") {
147
+ return { action: result.action };
129
148
  }
130
- const redispatches = await incrementTurnWorkerDeathRedispatches(db, input.workspaceId, input.turnId);
131
- if (redispatches > WORKER_DEATH_MAX_REDISPATCHES) {
132
- return { action: "exceeded", redispatches: redispatches - 1 };
149
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
150
+ await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
151
+ if (result.action === "exceeded") {
152
+ await notifyParentOfChildTerminalFn(
153
+ { db, bus, settings, observability, wakeSessionWorkflow },
154
+ input.workspaceId,
155
+ input.sessionId,
156
+ "failed",
157
+ `turn:${result.turnId}`,
158
+ );
159
+ return {
160
+ action: "exceeded",
161
+ turnId: result.turnId,
162
+ redispatches: result.redispatches,
163
+ };
133
164
  }
134
- const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
135
- const approvalRerun = trigger?.type === "user.approvalDecision";
136
- // Legacy run-state mode has no crash checkpoint (the dying worker never
137
- // captured the blob), so it always replays the original trigger against
138
- // the previous RunState snapshot — the documented degraded resume.
139
- const resumeWithNotice = !approvalRerun
140
- && settings.sessionHistorySource === "items"
141
- && await countTurnSessionHistoryItems(db, input.workspaceId, input.turnId) > 0;
142
- const [preemptedEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [
143
- {
144
- turnId: turn.id,
145
- type: "turn.preempted",
146
- payload: {
147
- triggerEventId: input.triggerEventId,
148
- reason: "worker_death",
149
- redispatches,
150
- resumeWithNotice,
151
- ...(resumeWithNotice ? { text: WORKER_DEATH_RESUME_TEXT } : {}),
152
- },
153
- },
154
- {
155
- turnId: turn.id,
156
- type: "session.status.changed",
157
- payload: { status: "queued" },
158
- },
159
- ]);
160
- try {
161
- await requeuePreemptedTurn(db, input.workspaceId, turn.id, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
162
- } catch (requeueError) {
163
- // The zombie attempt can settle the turn between the status check above
164
- // and this requeue (it keeps executing until it notices the timeout).
165
- // A settled turn means its recorded outcome is the truth: report stale
166
- // so the workflow continues instead of failing the session over a lost
167
- // race. Anything else is a real persistence failure — rethrow.
168
- const current = await getSessionTurn(db, input.workspaceId, input.turnId);
169
- if (current && current.status !== "running" && current.status !== "requires_action") {
170
- return { action: "stale" };
171
- }
172
- throw requeueError;
173
- }
174
- await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
175
- await refreshQueuedTurnsGauge(db, observability);
176
- return { action: "requeued", redispatches };
165
+ return {
166
+ action: "recovering",
167
+ turnId: result.turnId,
168
+ redispatches: result.redispatches,
169
+ };
177
170
  }
178
171
 
179
- async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
172
+ async function peekSessionWork(input: PeekSessionWorkInput) {
180
173
  const { db, observability } = await services();
181
- const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
182
- await refreshQueuedTurnsGauge(db, observability);
183
- return turn;
174
+ const peek = await peekSessionWorkFn(db, input.workspaceId, input.sessionId);
175
+ await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
176
+ return peek;
184
177
  }
185
178
 
186
179
  async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
187
180
  const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
188
- const session = await requireSession(db, input.workspaceId, input.sessionId);
189
- if (session.status === "queued" || session.status === "running") {
190
- await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
181
+ const settled = await settleSessionIdleWithParentOutboxFn(
182
+ db,
183
+ input.workspaceId,
184
+ input.sessionId,
185
+ );
186
+ if (settled.events.length > 0) {
187
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, settled.events);
188
+ }
189
+ await refreshQueuedTurnsGauge(db, observability, countQueuedTurnsFn, recordTurnsQueuedGaugeFn);
190
+ if (settled.action === "stale") {
191
+ return;
191
192
  }
192
- await refreshQueuedTurnsGauge(db, observability);
193
193
  // The workflow reaches markSessionIdle exactly when it has decided to stop
194
194
  // for now (no queued turn, no goal continuation): the terminal-for-now
195
195
  // point for a spawned worker, whatever the cause (goal completed, agent or
196
- // system paused goal, goalless work finished, idle-interrupt stop). Wake
196
+ // system paused goal, goalless work finished, idle control settlement). Wake
197
197
  // the parent here, deduped per idle episode so the manager is nudged once.
198
- await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
198
+ await notifyParentOfChildTerminalFn(
199
+ { db, bus, settings, observability, wakeSessionWorkflow },
200
+ input.workspaceId,
201
+ input.sessionId,
202
+ "idle",
203
+ settled.episodeKey,
204
+ );
199
205
  }
200
206
 
201
207
  return {
202
- failSession,
203
- interruptActiveTurn,
204
- requeueTurnAfterWorkerDeath,
205
- claimNextQueuedTurn,
208
+ failSessionAttempt,
209
+ settleSessionInterruptions,
210
+ recoverDispatch,
211
+ peekSessionWork,
206
212
  markSessionIdle,
207
213
  };
208
214
  }
@@ -210,9 +216,11 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
210
216
  async function refreshQueuedTurnsGauge(
211
217
  db: ActivityServices["db"],
212
218
  observability: ActivityServices["observability"],
219
+ countQueuedTurnsFn: typeof countQueuedTurns,
220
+ recordTurnsQueuedGaugeFn: typeof recordTurnsQueuedGauge,
213
221
  ): Promise<void> {
214
222
  try {
215
- recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
223
+ recordTurnsQueuedGaugeFn(observability, await countQueuedTurnsFn(db));
216
224
  } catch {
217
225
  // Best-effort telemetry; session state transitions remain authoritative.
218
226
  }
@@ -2,15 +2,52 @@ import type { SessionEventType } from "@opengeni/contracts";
2
2
  import type { AppendEventInput } from "@opengeni/db";
3
3
  import { Context } from "@temporalio/activity";
4
4
 
5
- export function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) => Promise<void>) {
5
+ // Trailing-flush window. A burst of coalesced deltas followed by model silence
6
+ // must not sit in `pending` unbounded — without a timer, flush fires only on
7
+ // the NEXT push. Matched to the 33ms coalesce window so the client-visible
8
+ // latency added by batching is bounded to ~one window (imperceptible, and far
9
+ // cheaper than the per-delta DB round-trip it replaces).
10
+ const TRAILING_FLUSH_MS = 33;
11
+ const MAX_BATCH_EVENTS = 50;
12
+ // Keep model ingestion independent from a slow durable append, but never let a
13
+ // provider outrun storage without bound. At the high-water mark `push` waits for
14
+ // the serialized drain before consuming more provider output.
15
+ const MAX_BUFFERED_EVENTS = 1_000;
16
+
17
+ /** Optional metrics seam for the batcher. `onFlush` fires once per flush (on both
18
+ * success and failure) with the coalesced event count and the flush duration, so
19
+ * the worker can meter batch shape without the batcher knowing about Observability.
20
+ * `now` is injectable for deterministic tests. Both are optional; absent ⇒ no-op. */
21
+ export type RuntimeBatcherOptions = {
22
+ onFlush?: (info: { events: number; durationSeconds: number }) => void;
23
+ now?: () => number;
24
+ };
25
+
26
+ export function createRuntimeBatcher(
27
+ flushEvents: (events: AppendEventInput[]) => Promise<void>,
28
+ options: RuntimeBatcherOptions = {},
29
+ ) {
30
+ const now = options.now ?? (() => performance.now());
6
31
  let pending: AppendEventInput[] = [];
7
32
  let lastFlush = Date.now();
33
+ let trailingTimer: ReturnType<typeof setTimeout> | null = null;
34
+ let activeBatchSize = 0;
35
+ // One drain loop owns `flushEvents`, so DB-assigned sequence order always
36
+ // matches push order. Ordinary deltas do not await this promise; structural
37
+ // boundaries, explicit flushes, and the bounded high-water mark do.
38
+ let drainPromise: Promise<void> | null = null;
39
+ // A timer has no direct awaiter. Retain its failure so the next push or final
40
+ // flush terminates the turn instead of silently losing streamed evidence.
41
+ let flushFailed = false;
42
+ let flushFailure: unknown;
43
+ // The high-volume token deltas (agent.message.delta, agent.reasoning.delta,
44
+ // sandbox.command.output.delta) are DELIBERATELY NOT here: they coalesce
45
+ // under the 50-event / 33ms policy into one appendSessionEvents txn + one
46
+ // publish per flush, instead of one DB round-trip per token. Only events that
47
+ // must be delivered promptly stay structural (flush-immediately). PTY bytes
48
+ // stay structural on purpose — an interactive terminal is useless batched
49
+ // 33ms behind (P4.4).
8
50
  const structural = new Set<SessionEventType>([
9
- "agent.message.delta",
10
- "agent.reasoning.delta",
11
- "sandbox.command.output.delta",
12
- // Channel-A PTY bytes flush promptly (P4.4) — a terminal stream is useless
13
- // batched 33ms behind; the structural set bypasses the batcher's coalesce.
14
51
  "terminal.pty.output.delta",
15
52
  "agent.toolCall.created",
16
53
  "agent.toolCall.output",
@@ -23,23 +60,107 @@ export function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) =
23
60
  ]);
24
61
  return {
25
62
  push: async (event: { type: SessionEventType; payload: unknown }) => {
63
+ throwIfFlushFailed();
64
+ // Append BEFORE the structural check so a structural event's flush always
65
+ // carries any pending deltas in order (same flush, order preserved).
26
66
  pending.push({ type: event.type, payload: event.payload });
27
67
  const elapsed = Date.now() - lastFlush;
28
- if (pending.length >= 50 || elapsed >= 33 || structural.has(event.type)) {
68
+ if (structural.has(event.type)) {
69
+ await flush();
70
+ return;
71
+ }
72
+ if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
73
+ // Start the serialized drain, but keep consuming provider deltas while
74
+ // the append/publish round-trip is in flight. The active drain records
75
+ // any failure for the next awaited boundary.
76
+ void startDrain().catch(() => undefined);
77
+ }
78
+ if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
29
79
  await flush();
80
+ return;
30
81
  }
82
+ armTrailingTimer();
31
83
  },
32
84
  flush,
33
85
  };
34
86
 
35
- async function flush() {
36
- if (pending.length === 0) {
87
+ function armTrailingTimer() {
88
+ if (trailingTimer !== null) {
37
89
  return;
38
90
  }
39
- const events = pending;
40
- pending = [];
41
- lastFlush = Date.now();
42
- await flushEvents(events);
91
+ trailingTimer = setTimeout(() => {
92
+ trailingTimer = null;
93
+ // A timer has no awaiter, so prevent an unhandled rejection. startDrain
94
+ // retains the exact error and the next push/final flush rethrows it.
95
+ void flush().catch(() => undefined);
96
+ }, TRAILING_FLUSH_MS);
97
+ if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
98
+ trailingTimer.unref();
99
+ }
100
+ }
101
+
102
+ function clearTrailingTimer() {
103
+ if (trailingTimer !== null) {
104
+ clearTimeout(trailingTimer);
105
+ trailingTimer = null;
106
+ }
107
+ }
108
+
109
+ async function flush(): Promise<void> {
110
+ throwIfFlushFailed();
111
+ clearTrailingTimer();
112
+ await startDrain();
113
+ throwIfFlushFailed();
114
+ }
115
+
116
+ function startDrain(): Promise<void> {
117
+ if (drainPromise) {
118
+ return drainPromise;
119
+ }
120
+ if (flushFailed) {
121
+ return Promise.reject(flushFailure);
122
+ }
123
+ if (pending.length === 0) {
124
+ return Promise.resolve();
125
+ }
126
+ clearTrailingTimer();
127
+ const running = (async () => {
128
+ while (pending.length > 0) {
129
+ const events = pending.splice(0, MAX_BATCH_EVENTS);
130
+ activeBatchSize = events.length;
131
+ lastFlush = Date.now();
132
+ const startedAt = now();
133
+ try {
134
+ await flushEvents(events);
135
+ } catch (error) {
136
+ flushFailed = true;
137
+ flushFailure = error;
138
+ throw error;
139
+ } finally {
140
+ activeBatchSize = 0;
141
+ if (options.onFlush) {
142
+ try {
143
+ options.onFlush({
144
+ events: events.length,
145
+ durationSeconds: Math.max(0, (now() - startedAt) / 1000),
146
+ });
147
+ } catch {
148
+ // Metrics emission must never affect a flush.
149
+ }
150
+ }
151
+ }
152
+ }
153
+ })();
154
+ drainPromise = running.finally(() => {
155
+ drainPromise = null;
156
+ });
157
+ return drainPromise;
158
+ }
159
+
160
+ function throwIfFlushFailed(): void {
161
+ if (flushFailed) {
162
+ throw flushFailure;
163
+ }
43
164
  }
44
165
  }
45
166
 
@@ -51,7 +172,10 @@ export function currentActivityContext(): Context | null {
51
172
  }
52
173
  }
53
174
 
54
- export function startActivityHeartbeat(context: Context | null, details: Record<string, unknown>): ReturnType<typeof setInterval> | null {
175
+ export function startActivityHeartbeat(
176
+ context: Context | null,
177
+ details: Record<string, unknown>,
178
+ ): ReturnType<typeof setInterval> | null {
55
179
  if (!context) {
56
180
  return null;
57
181
  }
@@ -64,12 +188,12 @@ export function startActivityHeartbeat(context: Context | null, details: Record<
64
188
  return timer;
65
189
  }
66
190
 
67
- export async function nextStreamEvent<T>(iterator: AsyncIterator<T>, context: Context | null): Promise<IteratorResult<T>> {
191
+ export async function nextStreamEvent<T>(
192
+ iterator: AsyncIterator<T>,
193
+ context: Context | null,
194
+ ): Promise<IteratorResult<T>> {
68
195
  if (!context) {
69
196
  return await iterator.next();
70
197
  }
71
- return await Promise.race([
72
- iterator.next(),
73
- context.cancelled,
74
- ]);
198
+ return await Promise.race([iterator.next(), context.cancelled]);
75
199
  }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * The activity lost ownership of its exact turn generation/attempt.
3
+ *
4
+ * This is an ordinary stale-writer outcome, not a Temporal cancellation. The
5
+ * current owner or control transaction already holds authoritative state, so
6
+ * the losing activity must stop locally without asking the workflow to settle
7
+ * or fail the turn again.
8
+ */
9
+ export class TurnAttemptFencedError extends Error {
10
+ constructor(message: string) {
11
+ super(message);
12
+ this.name = "TurnAttemptFencedError";
13
+ }
14
+ }