@ixo/oracle-runtime-workers 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +5 -2
  2. package/package.json +1 -1
  3. package/src/core/context-budget.ts +201 -0
  4. package/src/core/context-window.ts +289 -0
  5. package/src/core/env.ts +29 -0
  6. package/src/core/main-agent-types.ts +46 -0
  7. package/src/core/main-agent.ts +86 -9
  8. package/src/core/middlewares/context-guard.ts +270 -0
  9. package/src/core/middlewares/result-cap.ts +180 -0
  10. package/src/core/middlewares/summarization.ts +87 -6
  11. package/src/core/middlewares/tool-marks.ts +209 -0
  12. package/src/core/openrouter-pricing.ts +42 -1
  13. package/src/core/read-result-tool.ts +77 -0
  14. package/src/core/runtime-context.ts +7 -0
  15. package/src/core/sub-agent-fallback.ts +20 -3
  16. package/src/core/wrap-plugin-tool.ts +24 -3
  17. package/src/do/ambient.ts +17 -0
  18. package/src/do/contracts.ts +227 -0
  19. package/src/do/result-store-test-do.ts +88 -0
  20. package/src/do/result-store.ts +339 -0
  21. package/src/do/run-buffer.ts +204 -0
  22. package/src/do/run-coordinator.ts +572 -0
  23. package/src/do/run-request.ts +52 -0
  24. package/src/do/run-store-test-do.ts +130 -0
  25. package/src/do/run-store.ts +604 -0
  26. package/src/do/session-id-migration.ts +85 -0
  27. package/src/do/speaker-prefix.ts +37 -0
  28. package/src/do/sse-stream.ts +567 -400
  29. package/src/do/transcript.ts +238 -0
  30. package/src/do/turn-body.ts +10 -0
  31. package/src/do/user-oracle-do.ts +875 -146
  32. package/src/matrix/gateway-do.ts +230 -29
  33. package/src/matrix/group-chat-summarizer.ts +107 -0
  34. package/src/matrix/group-chat.ts +1163 -0
  35. package/src/matrix/inbox-store.ts +23 -3
  36. package/src/matrix/ingest.ts +54 -11
  37. package/src/matrix/reply-chain.ts +34 -25
  38. package/src/plugin-api/tool-helper.ts +9 -1
  39. package/src/plugin-api/types.ts +86 -0
  40. package/src/plugins/index.ts +3 -0
  41. package/src/plugins/matrix-group-chats/README.md +70 -0
  42. package/src/plugins/matrix-group-chats/index.ts +132 -0
  43. package/src/plugins/matrix-group-chats/tools.ts +172 -0
  44. package/src/plugins/tasks/tasks-tools.ts +4 -0
  45. package/src/shell/app.ts +66 -0
  46. package/src/sqlite/sqlite-saver.ts +81 -0
  47. package/src/tasks/scheduler.ts +134 -3
  48. package/src/tasks/store.ts +22 -0
@@ -20,6 +20,7 @@ import { DurableObject } from 'cloudflare:workers';
20
20
  import {
21
21
  AIMessage,
22
22
  HumanMessage,
23
+ isBaseMessage,
23
24
  type BaseMessage,
24
25
  } from '@langchain/core/messages';
25
26
  import {
@@ -50,6 +51,7 @@ import {
50
51
  llmEnvFromWorkerEnv,
51
52
  resolveLangsmithTracing,
52
53
  type OpenRouterLlmAdapter,
54
+ DEFAULT_MODEL_ID,
53
55
  } from '../core/llm';
54
56
  import { createMainAgent } from '../core/main-agent';
55
57
  import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
@@ -140,16 +142,27 @@ import {
140
142
  type OracleWorkerEnv,
141
143
  type SessionSummary,
142
144
  type StorageStatus,
145
+ type RunsStatus,
146
+ type ContextStatus,
147
+ type SessionContextStatus,
148
+ type RunSummary,
143
149
  type TurnIdentity,
144
150
  type TurnRequest,
145
151
  type TurnResult,
146
152
  type MemorySchemaDebug,
147
153
  TURN_INTERRUPTED_MARKER,
154
+ type JsonString,
148
155
  } from './contracts';
149
156
  import { turnRecursionLimit } from './turn-config';
150
157
  import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
151
158
  import { decideBootDirty } from './boot-dirty';
152
159
  import { mirrorTxnId, RoomMirror } from './room-mirror';
160
+ import { prefixSpeaker } from './speaker-prefix';
161
+ import {
162
+ type ObservedMessage,
163
+ summarizeObservedMessages,
164
+ } from '../matrix/group-chat-summarizer';
165
+ import { migrateThreadSessionIds } from './session-id-migration';
153
166
  import { ReauthPrompter } from './reauth-prompt';
154
167
  import {
155
168
  createTaskScheduler,
@@ -167,8 +180,46 @@ import {
167
180
  type HistoryMessage,
168
181
  type IndexableSession,
169
182
  } from '../memory/session-history-indexer';
170
- import { createSseTurnStream, formatSSE, SSE_HEADERS } from './sse-stream';
183
+ import {
184
+ createSseSubscriberStream,
185
+ runTurnFrames,
186
+ SSE_HEADERS,
187
+ } from './sse-stream';
171
188
  import { parseTurnBody, type TurnBody } from './turn-body';
189
+ import {
190
+ RunCoordinator,
191
+ type LiveRun,
192
+ type RunOutcome,
193
+ } from './run-coordinator';
194
+ import {
195
+ RunStore,
196
+ runDurabilityConfig,
197
+ type RunDurabilityConfig,
198
+ type RunRecord,
199
+ } from './run-store';
200
+ import {
201
+ runSummaryOf,
202
+ storedRunRequest,
203
+ type StoredRunRequest,
204
+ } from './run-request';
205
+ import {
206
+ createToolMarksMiddleware,
207
+ toolEffectOf,
208
+ type ToolEffect,
209
+ } from '../core/middlewares/tool-marks';
210
+ import { createResultCapMiddleware } from '../core/middlewares/result-cap';
211
+ import { READ_RESULT_TOOL_NAME } from '../core/read-result-tool';
212
+ import {
213
+ ContextWindowResolver,
214
+ contextWindowConfig,
215
+ } from '../core/context-window';
216
+ import { contextBudgetFor, contextKnobs } from '../core/context-budget';
217
+ import {
218
+ estimateRequestTokens,
219
+ type ContextGuardEvent,
220
+ } from '../core/middlewares/context-guard';
221
+ import { fetchOpenRouterContextLengths } from '../core/openrouter-pricing';
222
+ import { ResultStore, resultStoreKnobs } from './result-store';
172
223
  import { formatReplay } from '../matrix/replay-format';
173
224
  import { retryGateway } from './gateway-retry';
174
225
  import {
@@ -178,7 +229,10 @@ import {
178
229
  import {
179
230
  contentToText,
180
231
  isSummarizationMessage,
232
+ pageThreadTranscript,
233
+ TranscriptCursorError,
181
234
  transformTranscript,
235
+ type TranscriptPageOptions,
182
236
  } from './transcript';
183
237
  import { WorkersUcanService } from './ucan-service';
184
238
 
@@ -208,6 +262,27 @@ const IDLE_EVICT_MS = 5 * 24 * 60 * 60 * 1000;
208
262
  /** Set once the legacy Matrix copy has been removed (or found absent): no more lookups. */
209
263
  const META_LEGACY_CLEARED = 'meta:legacyCleared';
210
264
  const META_USER_DID = 'meta:userDid';
265
+
266
+ /** KV key of a session's context-guard counters (`GET /debug/context?session=`). */
267
+ const contextStatsKey = (sessionId: string): string => `ctxstats:${sessionId}`;
268
+
269
+ type SessionContextCounters = Pick<
270
+ SessionContextStatus,
271
+ | 'prunes'
272
+ | 'hardPrunes'
273
+ | 'prunedResults'
274
+ | 'overflowRetries'
275
+ | 'refusals'
276
+ | 'lastEventAt'
277
+ >;
278
+
279
+ const EMPTY_CONTEXT_COUNTERS: SessionContextCounters = {
280
+ prunes: 0,
281
+ hardPrunes: 0,
282
+ prunedResults: 0,
283
+ overflowRetries: 0,
284
+ refusals: 0,
285
+ };
211
286
  /** VFS write generation of the file at the last successful upload (see `DoVfs.writeGeneration`). */
212
287
  const META_UPLOADED_GEN = 'meta:uploadedGen';
213
288
  /**
@@ -424,6 +499,38 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
424
499
  if (this.ctx.getWebSockets().length > 0) this.realtime.nextPingAt();
425
500
  }
426
501
  private readonly aborts = new Map<string, AbortController>();
502
+ /** Durable runs (docs/plans/durable-runs.md): the rows, and the live side. */
503
+ private runStore: RunStore | null = null;
504
+ private runs: RunCoordinator | null = null;
505
+ private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
506
+ this.env,
507
+ );
508
+ /** Whole tool results the result cap saved (docs/plans/context-budgets.md). */
509
+ private resultStore: ResultStore | null = null;
510
+ /**
511
+ * Per-model context windows; learned limits persist in this object's KV
512
+ * storage (`ctxwin:<model>`), so a provider's rejection is remembered
513
+ * across boots.
514
+ */
515
+ private readonly contextWindows = new ContextWindowResolver({
516
+ config: contextWindowConfig(this.env, console),
517
+ catalog: () =>
518
+ fetchOpenRouterContextLengths({
519
+ apiKey: this.env.OPEN_ROUTER_API_KEY,
520
+ logger: console,
521
+ }),
522
+ learned: {
523
+ get: (model) => this.ctx.storage.get<number>(`ctxwin:${model}`),
524
+ set: (model, tokens) => this.ctx.storage.put(`ctxwin:${model}`, tokens),
525
+ },
526
+ logger: console,
527
+ });
528
+ /**
529
+ * Per-session context-guard counters (`ctxstats:<session>` in KV
530
+ * storage; `GET /debug/context?session=`). Writes are chained so two
531
+ * events of one request (a soft then a hard prune) never race.
532
+ */
533
+ private contextStatsWrites: Promise<void> = Promise.resolve();
427
534
  private readonly delegations = new Map<string, { raw: string }>();
428
535
  /** Epoch-ms until which a room-state delegation lookup is not retried. */
429
536
  private delegationMissUntil = 0;
@@ -456,6 +563,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
456
563
  private mirror: RoomMirror | null = null;
457
564
  /** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
458
565
  private readonly sessionRooms = new Map<string, string>();
566
+ /** The user's main oracle room, once resolved (the session list is scoped to it). */
567
+ private mainRoomId: string | null = null;
459
568
  /** The throttled `delegation_required` prompt (see reauth-prompt.ts). */
460
569
  private reauthPrompter: ReauthPrompter | null = null;
461
570
  /** Room turns running right now, by Matrix event id: a gateway that asks again attaches instead of re-running. */
@@ -1015,6 +1124,29 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1015
1124
  await this.sessions.setup();
1016
1125
  this.matrixLedger = new MatrixTurnLedger(liveDb);
1017
1126
  await this.matrixLedger.setup();
1127
+ this.runStore = new RunStore(liveDb);
1128
+ await this.runStore.setup();
1129
+ this.resultStore = new ResultStore(liveDb, {
1130
+ ...(this.env.TIER_BUCKET ? { bucket: this.env.TIER_BUCKET } : {}),
1131
+ prefix: this.ctx.id.toString(),
1132
+ ...resultStoreKnobs(this.env),
1133
+ logger: console,
1134
+ });
1135
+ await this.resultStore.setup();
1136
+ // Threads opened before the runtime adopted Node's session rule
1137
+ // (session id = thread root event id) are renamed to it, once.
1138
+ const renamed = await migrateThreadSessionIds(liveDb, console);
1139
+ if (renamed.renamed > 0) this.markDirty();
1140
+ this.runs = new RunCoordinator({
1141
+ store: this.runStore,
1142
+ config: this.runConfig,
1143
+ instanceId: this.instanceId,
1144
+ log: console,
1145
+ requestAlarm: (at) => this.requestAlarm(at),
1146
+ runAttempt: (live, resumed) => this.runAttempt(live, resumed),
1147
+ checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
1148
+ onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
1149
+ });
1018
1150
 
1019
1151
  const core = this.core;
1020
1152
  // Boot-time plugin hooks + collision checks, once per object (memoised).
@@ -1031,6 +1163,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1031
1163
  gateway: this.gateway,
1032
1164
  runTurn: (req) => this.runTurn(req),
1033
1165
  requestAlarm: (at) => this.requestAlarm(at),
1166
+ turnRunLive: (taskRunId) => Boolean(this.runs?.byTaskRunId(taskRunId)),
1034
1167
  log: console,
1035
1168
  ...(Number.isFinite(maxTasks) ? { maxTasksPerUser: maxTasks } : {}),
1036
1169
  ...(Number.isFinite(minCron) ? { minCronIntervalSec: minCron } : {}),
@@ -1040,6 +1173,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1040
1173
  if (next !== null) this.requestAlarm(Math.max(next, Date.now() + 1000));
1041
1174
  }
1042
1175
 
1176
+ // Runs a previous incarnation left in flight: schedule their recovery
1177
+ // (attempts run from the alarm, never inside this boot).
1178
+ try {
1179
+ await this.runs.recoverOrphans();
1180
+ } catch (err) {
1181
+ console.error(
1182
+ `[user-do] run recovery scan failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
1183
+ );
1184
+ }
1185
+
1043
1186
  // Resume (or start) legacy-blob compaction in the background: files
1044
1187
  // imported from the Node runtime arrive with uncompressed blobs.
1045
1188
  this.compactDone ??=
@@ -1268,12 +1411,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1268
1411
  this.ctx.getWebSockets().length > 0 ? this.realtime.pingTick() : null;
1269
1412
  const housekeepingAt =
1270
1413
  await this.ctx.storage.get<number>(META_HOUSEKEEPING_AT);
1414
+ // Runs in flight in THIS instance need the keep-alive re-armed; a
1415
+ // recovery attempt that came due needs starting. Both are cheap and
1416
+ // do not open the database on a bare object (the coordinator only
1417
+ // exists once the object booted).
1418
+ const keepAliveAt = this.runs?.keepAliveDeadline(wakeAt) ?? null;
1419
+ const recoveryAt = this.runs?.nextRecoveryAt() ?? null;
1420
+ if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000)
1421
+ await this.runs.resumeDue(wakeAt);
1271
1422
  if (
1272
1423
  nextPingAt !== null &&
1273
1424
  housekeepingAt !== undefined &&
1274
- housekeepingAt > wakeAt + 1000
1425
+ housekeepingAt > wakeAt + 1000 &&
1426
+ keepAliveAt === null &&
1427
+ (recoveryAt === null || recoveryAt > wakeAt + 1000)
1275
1428
  ) {
1276
- await this.ctx.storage.setAlarm(Math.min(nextPingAt, housekeepingAt));
1429
+ await this.ctx.storage.setAlarm(
1430
+ Math.min(nextPingAt, housekeepingAt, recoveryAt ?? Infinity),
1431
+ );
1277
1432
  return;
1278
1433
  }
1279
1434
 
@@ -1308,6 +1463,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1308
1463
  const now = Date.now();
1309
1464
  const deadlines: number[] = [];
1310
1465
 
1466
+ // Durable runs: start the recovery attempts that are due (the boot
1467
+ // above may have just scheduled them), then keep the alarm inside the
1468
+ // keep-alive horizon while anything executes.
1469
+ if (this.runs) {
1470
+ try {
1471
+ await this.runs.resumeDue(now);
1472
+ } catch (err) {
1473
+ console.error(
1474
+ `[user-do] run recovery tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
1475
+ );
1476
+ }
1477
+ const keepAlive = this.runs.keepAliveDeadline(now);
1478
+ if (keepAlive !== null) deadlines.push(keepAlive);
1479
+ const nextRecovery = this.runs.nextRecoveryAt();
1480
+ if (nextRecovery !== null)
1481
+ deadlines.push(Math.max(nextRecovery, now + 1000));
1482
+ }
1483
+
1311
1484
  if (this.dirty && this.db) {
1312
1485
  try {
1313
1486
  await this.flushToOwnerStore();
@@ -1364,7 +1537,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1364
1537
  // of DO storage (see sqlite/page-tier.ts). After the flush above (the
1365
1538
  // upload reads hot rows, not R2) and never over an in-flight one (it
1366
1539
  // holds a snapshot). A pass is capped; more work re-arms soon.
1367
- if (this.db?.tierEnabled && !this.flushInFlight) {
1540
+ if (
1541
+ this.db?.tierEnabled &&
1542
+ !this.flushInFlight &&
1543
+ (this.runs?.activeCount ?? 0) === 0
1544
+ ) {
1368
1545
  const next = await this.tierTick(this.db, now);
1369
1546
  if (next !== null) deadlines.push(next);
1370
1547
  }
@@ -1375,7 +1552,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1375
1552
  // above ran first; the wipe additionally verifies the upstream copy
1376
1553
  // matches the working copy, else the copy stays until it does.
1377
1554
  const idle = now - lastAccess > IDLE_EVICT_MS;
1378
- if (idle && !this.dirty && this.db && deadlines.length === 0) {
1555
+ if (
1556
+ idle &&
1557
+ !this.dirty &&
1558
+ this.db &&
1559
+ deadlines.length === 0 &&
1560
+ (this.runs?.size ?? 0) === 0
1561
+ ) {
1379
1562
  if (await this.ownerCopyIsCurrent(this.db)) {
1380
1563
  await this.wipeWorkingCopy(this.db);
1381
1564
  await this.ctx.storage.delete(META_HOUSEKEEPING_AT);
@@ -1541,6 +1724,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1541
1724
  );
1542
1725
  if (room) {
1543
1726
  roomId = room.roomId;
1727
+ if (!o.roomId) this.mainRoomId = room.roomId;
1544
1728
  // The marker is a waited, non-durable send: it must never be
1545
1729
  // replayed after a gateway restart (its event id IS the session id,
1546
1730
  // an orphan marker would be a ghost session). Instead it retries
@@ -1583,11 +1767,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1583
1767
  o: { limit?: number; offset?: number } = {},
1584
1768
  ): Promise<{ sessions: SessionSummary[]; total: number }> {
1585
1769
  await this.ready(identity);
1586
- // Task runs re-enter the agent as `task:<id>` sessions; like the Node
1587
- // runtime (which lists only the user's main room) keep them out of
1588
- // the user's session list.
1770
+ // Node parity (`SessionsService.listSessions`): the list is the user's
1771
+ // MAIN oracle room only — Portal sessions and the threads opened by
1772
+ // room messages there. Threads in dedicated task rooms (or any other
1773
+ // room) stay out of it, and task runs (`task:<id>`) are hidden
1774
+ // whichever room they deliver to. A user without an oracle room sees
1775
+ // every session, as on Node.
1776
+ const mainRoomId = await this.resolveMainRoomId(identity.userDid);
1589
1777
  const { sessions, total } = await this.sessions!.listSessions(
1590
- undefined,
1778
+ mainRoomId ?? undefined,
1591
1779
  o.limit ?? 20,
1592
1780
  o.offset ?? 0,
1593
1781
  TASK_SESSION_PREFIX,
@@ -1595,6 +1783,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1595
1783
  return { sessions: sessions.map(toSummary), total };
1596
1784
  }
1597
1785
 
1786
+ /**
1787
+ * The user↔oracle room, resolved once per instance. `null` is the
1788
+ * definitive "no oracle room"; a transient gateway failure retries and
1789
+ * then fails the request rather than answering with the wrong list.
1790
+ */
1791
+ private async resolveMainRoomId(userDid: string): Promise<string | null> {
1792
+ if (this.mainRoomId) return this.mainRoomId;
1793
+ const room = await retryGateway(() =>
1794
+ this.gateway.resolveUserRoom(userDid),
1795
+ );
1796
+ if (room) this.mainRoomId = room.roomId;
1797
+ return room?.roomId ?? null;
1798
+ }
1799
+
1598
1800
  async deleteSession(
1599
1801
  identity: TurnIdentity,
1600
1802
  sessionId: string,
@@ -1615,6 +1817,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1615
1817
  const deleted = await this.sessions!.deleteSession(sessionId);
1616
1818
  if (deleted) {
1617
1819
  await this.saver!.deleteThread(sessionId);
1820
+ await this.resultStore
1821
+ ?.deleteForSession(sessionId)
1822
+ .catch((err: unknown) => {
1823
+ console.warn(
1824
+ `[user-do] could not delete the saved tool results of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
1825
+ );
1826
+ });
1827
+ await this.ctx.storage.delete(contextStatsKey(sessionId));
1618
1828
  this.markDirty();
1619
1829
  }
1620
1830
  return deleted;
@@ -1723,6 +1933,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1723
1933
  return JSON.stringify(dtos);
1724
1934
  }
1725
1935
 
1936
+ async listMessagesPage(
1937
+ identity: TurnIdentity,
1938
+ sessionId: string,
1939
+ options: TranscriptPageOptions,
1940
+ ): Promise<
1941
+ { ok: true; json: string } | { ok: false; status: 400; message: string }
1942
+ > {
1943
+ await this.ready(identity);
1944
+ try {
1945
+ const page = await pageThreadTranscript(
1946
+ this.saver!,
1947
+ sessionId,
1948
+ options,
1949
+ );
1950
+ return { ok: true, json: JSON.stringify(page) };
1951
+ } catch (err) {
1952
+ if (err instanceof TranscriptCursorError)
1953
+ return { ok: false, status: 400, message: err.message };
1954
+ throw err;
1955
+ }
1956
+ }
1957
+
1726
1958
  /**
1727
1959
  * The shell deposited a (new) delegation for this user: adopt it now so
1728
1960
  * header-less turns mint from it immediately. Does not boot the object.
@@ -1861,7 +2093,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1861
2093
  return { present: false, source: 'none' };
1862
2094
  }
1863
2095
 
2096
+ /**
2097
+ * `POST /messages/abort`: stop the session's active run. The run is
2098
+ * closed as `aborted` with the text streamed so far kept on its row;
2099
+ * a queued run is dropped. (A client that merely disconnects does NOT
2100
+ * reach here — runs outlive their connections.)
2101
+ */
1864
2102
  async abortTurn(sessionId: string): Promise<boolean> {
2103
+ if (this.runs?.abortSession(sessionId)) return true;
1865
2104
  const controller = this.aborts.get(sessionId);
1866
2105
  if (!controller) return false;
1867
2106
  controller.abort();
@@ -1904,8 +2143,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1904
2143
  toolCalls: [],
1905
2144
  replayed: true,
1906
2145
  };
1907
- case 'interrupted':
2146
+ case 'interrupted': {
2147
+ // The object reset mid-turn. The run may be recovering (durable
2148
+ // runs): attach to it and answer with its reply; only a run that
2149
+ // is over without a reply is refused.
2150
+ const attached = existing?.requestId
2151
+ ? await this.attachToRun(existing.requestId, req)
2152
+ : null;
2153
+ if (attached) return attached;
1908
2154
  throw new Error(`${TURN_INTERRUPTED_MARKER} (${eventId})`);
2155
+ }
1909
2156
  case 'run':
1910
2157
  break;
1911
2158
  }
@@ -1952,43 +2199,78 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1952
2199
  });
1953
2200
  card.emit(req.requestId, 'routing');
1954
2201
  }
1955
- const { agent, stateInput, config, abortController, turnDisposables } =
1956
- await this.prepareTurn(req, {
1957
- message: req.message,
1958
- timezone: req.identity.timezone,
1959
- model: req.model,
1960
- attachments: req.attachments,
1961
- });
1962
- try {
1963
- const result = (await agent.invoke(stateInput, config)) as {
1964
- messages?: BaseMessage[];
2202
+ // Every turn is a durable run (docs/plans/durable-runs.md): recorded
2203
+ // before the first model call, kept alive by the alarm, resumed after
2204
+ // a reset. Headless turns (Matrix, tasks) simply await the outcome.
2205
+ const { live } = await this.runs!.begin({
2206
+ runId: crypto.randomUUID(),
2207
+ sessionId: req.sessionId,
2208
+ requestId: req.requestId,
2209
+ client: req.client,
2210
+ request: JSON.stringify(
2211
+ storedRunRequest(req, { timezone: req.identity.timezone }),
2212
+ ),
2213
+ multitask: req.multitask ?? this.runConfig.multitaskDefault,
2214
+ ...(req.taskRunId ? { taskRunId: req.taskRunId } : {}),
2215
+ });
2216
+ const outcome = await live.done;
2217
+ if (outcome.status !== 'finished') {
2218
+ const reason =
2219
+ outcome.error instanceof Error
2220
+ ? outcome.error.message
2221
+ : outcome.error !== undefined
2222
+ ? String(outcome.error)
2223
+ : `turn ${outcome.status}`;
2224
+ throw new Error(reason);
2225
+ }
2226
+ card?.emit(req.requestId, 'delivering');
2227
+ return {
2228
+ sessionId: req.sessionId,
2229
+ requestId: req.requestId,
2230
+ text: outcome.text,
2231
+ ...(outcome.messageId !== undefined
2232
+ ? { messageId: outcome.messageId }
2233
+ : {}),
2234
+ toolCalls: outcome.toolCalls ?? [],
2235
+ };
2236
+ }
2237
+
2238
+ /**
2239
+ * A Matrix turn asked about again after this object reset: hand back the
2240
+ * reply of its (recovering or finished) run instead of refusing it.
2241
+ */
2242
+ private async attachToRun(
2243
+ requestId: string,
2244
+ req: TurnRequest,
2245
+ ): Promise<TurnResult | null> {
2246
+ const live = this.runs?.byRequestId(requestId);
2247
+ if (live) {
2248
+ console.log(
2249
+ `[user-do] turn ${requestId} for ${req.eventId} is being recovered; the gateway attaches to it`,
2250
+ );
2251
+ const outcome = await live.done;
2252
+ if (outcome.status !== 'finished') return null;
2253
+ return {
2254
+ sessionId: req.sessionId,
2255
+ requestId: req.requestId,
2256
+ text: outcome.text,
2257
+ ...(outcome.messageId ? { messageId: outcome.messageId } : {}),
2258
+ toolCalls: outcome.toolCalls ?? [],
2259
+ replayed: true,
1965
2260
  };
1966
- const messages = result.messages ?? [];
1967
- const text = lastAiText(messages);
1968
- await this.afterTurn(req.sessionId, messages);
1969
- await this.runTurnDisposables(turnDisposables);
1970
- card?.emit(req.requestId, 'delivering');
1971
- const toolCalls = messages
1972
- .filter((m): m is AIMessage => m.type === 'ai')
1973
- .flatMap((m) =>
1974
- (m.tool_calls ?? []).map((t) => ({
1975
- name: t.name,
1976
- status: 'done' as const,
1977
- })),
1978
- );
2261
+ }
2262
+ const record = await this.runStore?.getByRequestId(requestId);
2263
+ if (record?.status === 'finished') {
1979
2264
  return {
1980
2265
  sessionId: req.sessionId,
1981
2266
  requestId: req.requestId,
1982
- text,
1983
- ...(lastAiMessageId(messages) !== undefined
1984
- ? { messageId: lastAiMessageId(messages) }
1985
- : {}),
1986
- toolCalls,
2267
+ text: record.partialText ?? '',
2268
+ ...(record.messageId ? { messageId: record.messageId } : {}),
2269
+ toolCalls: [],
2270
+ replayed: true,
1987
2271
  };
1988
- } finally {
1989
- if (this.aborts.get(req.sessionId) === abortController)
1990
- this.aborts.delete(req.sessionId);
1991
2272
  }
2273
+ return null;
1992
2274
  }
1993
2275
 
1994
2276
  /**
@@ -2261,7 +2543,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2261
2543
  this.saver = null;
2262
2544
  this.sessions = null;
2263
2545
  this.matrixLedger = null;
2546
+ this.runStore = null;
2547
+ this.runs = null;
2548
+ this.resultStore = null;
2264
2549
  this.sessionRooms.clear();
2550
+ this.mainRoomId = null;
2265
2551
  this.ambient = null;
2266
2552
  this.secretsService = null;
2267
2553
  this.byo = null;
@@ -2574,6 +2860,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2574
2860
  console,
2575
2861
  );
2576
2862
  }
2863
+ const joinMatch = /^\/runs\/([^/]+)$/.exec(url.pathname);
2864
+ if (joinMatch && request.method === 'GET')
2865
+ return this.joinRun(request, decodeURIComponent(joinMatch[1]!), url);
2577
2866
  const match = /^\/turn\/([^/]+)$/.exec(url.pathname);
2578
2867
  if (!match || request.method !== 'POST')
2579
2868
  return new Response('Not found', { status: 404 });
@@ -2661,75 +2950,428 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2661
2950
  }
2662
2951
  }
2663
2952
 
2953
+ // A streamed turn is a durable run: recorded, kept alive and resumable
2954
+ // (docs/plans/durable-runs.md). The response is one subscriber of the
2955
+ // run's buffer; the run itself outlives it.
2956
+ this.replayToRoom(req, body.message, 'user');
2957
+ const agActionNames = (body.agActions ?? []).map((a) => a.name);
2958
+ const { live, queued } = await this.runs!.begin({
2959
+ runId: crypto.randomUUID(),
2960
+ sessionId,
2961
+ requestId,
2962
+ client: 'portal',
2963
+ request: JSON.stringify(
2964
+ storedRunRequest(req, {
2965
+ timezone: body.timezone,
2966
+ agActionNames,
2967
+ stream: true,
2968
+ }),
2969
+ ),
2970
+ multitask: body.multitask ?? this.runConfig.multitaskDefault,
2971
+ });
2972
+ if (queued)
2973
+ live.buffer.push('router.update', {
2974
+ step: 'Queued…',
2975
+ sessionId,
2976
+ requestId,
2977
+ runId: live.runId,
2978
+ queued: true,
2979
+ });
2980
+ const responseStream = createSseSubscriberStream({
2981
+ replay: live.buffer.tailAfter(0),
2982
+ buffer: live.buffer,
2983
+ onCancel: () =>
2984
+ console.log(
2985
+ `[user-do] client left run ${live.runId} (${sessionId}); the run continues`,
2986
+ ),
2987
+ });
2988
+ return new Response(responseStream, {
2989
+ headers: {
2990
+ ...SSE_HEADERS,
2991
+ 'x-request-id': requestId,
2992
+ 'x-run-id': live.runId,
2993
+ },
2994
+ });
2995
+ }
2996
+
2997
+ /**
2998
+ * `GET /runs/:id?after=<seq>` — re-join a run: replay the frames after
2999
+ * the cursor (packed segments, then the live tail) and stay attached
3000
+ * until its `done` frame. A run that already ended replays what is left
3001
+ * and closes with a synthetic `done` carrying its final state.
3002
+ */
3003
+ private async joinRun(
3004
+ request: Request,
3005
+ runId: string,
3006
+ url: URL,
3007
+ ): Promise<Response> {
3008
+ const identity = JSON.parse(
3009
+ request.headers.get('x-identity') ?? '{}',
3010
+ ) as TurnIdentity;
3011
+ try {
3012
+ await this.ready(identity);
3013
+ } catch (err) {
3014
+ return readyFailureResponse(err);
3015
+ }
3016
+ const afterRaw = Number(url.searchParams.get('after') ?? '0');
3017
+ const after =
3018
+ Number.isFinite(afterRaw) && afterRaw > 0 ? Math.floor(afterRaw) : 0;
3019
+ const joined = await this.runs!.join(runId, after);
3020
+ if (!joined)
3021
+ return Response.json(
3022
+ { statusCode: 404, message: `Run ${runId} not found` },
3023
+ { status: 404 },
3024
+ );
3025
+ const { record, replay, buffer } = joined;
3026
+ const ended = !buffer || buffer.isClosed;
3027
+ const stream = createSseSubscriberStream({
3028
+ replay,
3029
+ ...(buffer && !buffer.isClosed ? { buffer } : {}),
3030
+ ...(ended
3031
+ ? {
3032
+ trailer: {
3033
+ event: 'done',
3034
+ data: {
3035
+ runId,
3036
+ status: record.status,
3037
+ ...(record.messageId ? { messageId: record.messageId } : {}),
3038
+ ...(record.partialText && record.status !== 'finished'
3039
+ ? { partialText: record.partialText }
3040
+ : {}),
3041
+ replayed: true,
3042
+ },
3043
+ },
3044
+ }
3045
+ : {}),
3046
+ });
3047
+ return new Response(stream, {
3048
+ headers: {
3049
+ ...SSE_HEADERS,
3050
+ 'x-request-id': record.requestId,
3051
+ 'x-run-id': runId,
3052
+ 'x-run-status': record.status,
3053
+ },
3054
+ });
3055
+ }
3056
+
3057
+ /**
3058
+ * One attempt of a run (the coordinator's `runAttempt` host hook): build
3059
+ * the agent for the stored request, stream the turn's frames into the
3060
+ * run's buffer, and report the outcome. `resumed` attempts hand the
3061
+ * graph no new input, so LangGraph continues from the checkpoint's
3062
+ * pending node; the tool-mark middleware keeps that safe.
3063
+ */
3064
+ private async runAttempt(
3065
+ live: LiveRun,
3066
+ resumed: boolean,
3067
+ ): Promise<RunOutcome> {
3068
+ const stored = JSON.parse(live.record.request) as StoredRunRequest;
3069
+ const req = stored.turn;
2664
3070
  const {
2665
3071
  agent,
2666
3072
  stateInput,
2667
3073
  config,
2668
- abortController,
3074
+ turnDisposables,
2669
3075
  byoNotice,
2670
3076
  byoProvider,
2671
- turnDisposables,
2672
- } = await this.prepareTurn(req, body);
2673
- this.replayToRoom(req, body.message, 'user');
2674
- const agActionNames = new Set((body.agActions ?? []).map((a) => a.name));
3077
+ toolOutputCapChars,
3078
+ } = await this.prepareTurn(
3079
+ req,
3080
+ {
3081
+ message: req.message,
3082
+ timezone: stored.timezone ?? req.identity.timezone,
3083
+ model: req.model,
3084
+ ...(resumed ? {} : { attachments: req.attachments }),
3085
+ },
3086
+ {
3087
+ runId: live.runId,
3088
+ abortController: live.abort,
3089
+ resumed,
3090
+ continuation: resumed ? live.continuation : null,
3091
+ },
3092
+ );
3093
+ const sessionId = req.sessionId;
2675
3094
  const sink = {
2676
3095
  emit: (eventName: string, payload: Record<string, unknown>) => {
2677
- void payload;
2678
- void eventName;
3096
+ if (live.buffer.isClosed) return;
3097
+ live.buffer.push(eventName, payload);
2679
3098
  },
2680
3099
  };
2681
- let write: ((name: string, payload: unknown) => void) | null = null;
2682
- sink.emit = (eventName, payload) => write?.(eventName, payload);
2683
3100
  this.events.register(sessionId, sink);
2684
- const events = agent.streamEvents(stateInput, {
3101
+ if (byoNotice)
3102
+ sink.emit('error', {
3103
+ ...byoNotice,
3104
+ sessionId,
3105
+ requestId: req.requestId,
3106
+ });
3107
+ const events = agent.streamEvents(resumed ? null : stateInput, {
2685
3108
  ...config,
2686
3109
  version: 'v2',
2687
3110
  });
2688
3111
  const capture: BaseMessage[] = [];
2689
- const sse = createSseTurnStream({
2690
- events: tapMessages(events, capture),
2691
- sessionId,
2692
- requestId,
2693
- abortController,
2694
- mirror: (eventName, payload) =>
2695
- this.events.emitToTaps(eventName, payload),
2696
- agActionNames,
2697
- byoProvider,
2698
- onComplete: async () => {
2699
- this.replayToRoom(req, lastAiText(capture), 'oracle');
2700
- await this.afterTurn(sessionId, capture);
2701
- await this.runTurnDisposables(turnDisposables);
2702
- },
2703
- onError: (err) => {
2704
- console.error(`[user-do] turn ${requestId} failed:`, err);
2705
- void this.runTurnDisposables(turnDisposables);
2706
- },
2707
- log: (m) => console.log(`[user-do] ${m}`),
2708
- });
2709
- // Let plugin-emitted events (ctx.emit.*) ride the same SSE stream.
2710
- const [clientBranch, emitterBranch] = sse.tee();
2711
- void emitterBranch.cancel();
2712
- write = null;
2713
- const merged = mergeWithEmitter(clientBranch, (register) => {
2714
- write = register;
2715
- });
2716
- // A BYO turn that degraded to the platform model tells the user so —
2717
- // the Node runtime emits the same notice on the SSE `error` channel.
2718
- if (byoNotice) sink.emit('error', { ...byoNotice, sessionId, requestId });
2719
- const cleanup = () => {
3112
+ let completed = false;
3113
+ try {
3114
+ const outcome = await runTurnFrames({
3115
+ events: tapMessages(events, capture),
3116
+ sink: live.buffer,
3117
+ sessionId,
3118
+ requestId: req.requestId,
3119
+ runId: live.runId,
3120
+ resumed,
3121
+ abortController: live.abort,
3122
+ mirror: (eventName, payload) =>
3123
+ this.events.emitToTaps(eventName, payload),
3124
+ agActionNames: new Set(stored.agActionNames ?? []),
3125
+ byoProvider,
3126
+ toolOutputCapChars,
3127
+ messageIdOf: () => lastAiMessageId(capture),
3128
+ onFrame: () => this.runs?.touchKeepAlive(),
3129
+ onComplete: async () => {
3130
+ completed = true;
3131
+ this.replayToRoom(req, lastAiText(capture), 'oracle');
3132
+ await this.afterTurn(sessionId, capture);
3133
+ },
3134
+ onError: (err) => {
3135
+ console.error(`[user-do] turn ${req.requestId} failed:`, err);
3136
+ },
3137
+ log: (m) => console.log(`[user-do] ${m}`),
3138
+ });
3139
+ const toolCalls = capture
3140
+ .filter((m): m is AIMessage => m.type === 'ai')
3141
+ .flatMap((m) =>
3142
+ (m.tool_calls ?? []).map((t) => ({
3143
+ name: t.name,
3144
+ status: 'done' as const,
3145
+ })),
3146
+ );
3147
+ const text = completed ? lastAiText(capture) : outcome.fullText;
3148
+ const messageId = completed ? lastAiMessageId(capture) : undefined;
3149
+ return {
3150
+ status:
3151
+ outcome.status === 'completed'
3152
+ ? 'finished'
3153
+ : outcome.status === 'aborted'
3154
+ ? 'aborted'
3155
+ : 'failed',
3156
+ text: (live.continuation ?? '') + text,
3157
+ ...(messageId ? { messageId } : {}),
3158
+ toolCalls,
3159
+ ...(outcome.status === 'failed' ? { error: outcome.error } : {}),
3160
+ };
3161
+ } finally {
2720
3162
  this.events.unregister(sessionId, sink);
2721
- if (this.aborts.get(sessionId) === abortController)
3163
+ await this.runTurnDisposables(turnDisposables);
3164
+ if (this.aborts.get(sessionId) === live.abort)
2722
3165
  this.aborts.delete(sessionId);
3166
+ }
3167
+ }
3168
+
3169
+ /** Latest checkpoint id of a session — recovery's progress marker. */
3170
+ private async checkpointIdOf(sessionId: string): Promise<string | null> {
3171
+ const tuple = await this.saver
3172
+ ?.getTupleWithoutMessages({ configurable: { thread_id: sessionId } })
3173
+ .catch(() => undefined);
3174
+ return tuple?.checkpoint.id ?? null;
3175
+ }
3176
+
3177
+ /**
3178
+ * A run reached a terminal state. A recovered task run has no scheduler
3179
+ * continuation waiting on it any more: hand the result (or the failure)
3180
+ * to the scheduler so it is delivered / recorded as it would have been.
3181
+ */
3182
+ private async onRunEnded(
3183
+ record: RunRecord,
3184
+ outcome: RunOutcome,
3185
+ ): Promise<void> {
3186
+ if (!record.taskRunId || !this.taskScheduler) return;
3187
+ if (this.taskScheduler.isRunActive(record.taskRunId)) return;
3188
+ if (outcome.status === 'finished')
3189
+ await this.taskScheduler.completeRecoveredRun(
3190
+ record.taskRunId,
3191
+ outcome.text,
3192
+ );
3193
+ else
3194
+ await this.taskScheduler.failRecoveredRun(
3195
+ record.taskRunId,
3196
+ outcome.status === 'interrupted'
3197
+ ? 'interrupted: the object was reset while the run was in progress and recovery did not complete'
3198
+ : `${outcome.status}: ${outcome.error instanceof Error ? outcome.error.message : String(outcome.error ?? outcome.status)}`,
3199
+ );
3200
+ }
3201
+
3202
+ /** `GET /sessions/:id/run` — the session's active run, if any. */
3203
+ async sessionRun(
3204
+ identity: TurnIdentity,
3205
+ sessionId: string,
3206
+ ): Promise<RunSummary | null> {
3207
+ await this.ready(identity);
3208
+ const live = this.runs?.activeForSession(sessionId);
3209
+ const record =
3210
+ live?.record ?? (await this.runStore?.activeForSession(sessionId));
3211
+ return record ? runSummaryOf(record) : null;
3212
+ }
3213
+
3214
+ /** `GET /debug/context?model=` — the context budget a model gets here. */
3215
+ async contextStatus(
3216
+ identity: TurnIdentity,
3217
+ model?: string,
3218
+ byoProvider?: string,
3219
+ sessionId?: string,
3220
+ ): Promise<ContextStatus> {
3221
+ await this.ready(identity);
3222
+ const llm = this.ambient?.llm;
3223
+ const modelId =
3224
+ model ??
3225
+ (llm && isProviderAdapter(llm)
3226
+ ? llm.modelForRole('main')
3227
+ : DEFAULT_MODEL_ID);
3228
+ const window = await this.contextWindows.resolve(modelId, {
3229
+ ...(byoProvider ? { byoProvider } : {}),
3230
+ });
3231
+ const budget = contextBudgetFor(window, contextKnobs(this.env, console));
3232
+ const stats = (await this.resultStore?.stats()) ?? {
3233
+ rows: 0,
3234
+ sqliteBytes: 0,
3235
+ r2Rows: 0,
3236
+ r2Bytes: 0,
2723
3237
  };
2724
- const finalStream = merged.pipeThrough(
2725
- new TransformStream<Uint8Array, Uint8Array>({
2726
- transform: (chunk, controller) => controller.enqueue(chunk),
2727
- flush: cleanup,
2728
- }),
2729
- );
2730
- return new Response(finalStream, {
2731
- headers: { ...SSE_HEADERS, 'x-request-id': requestId },
3238
+ return {
3239
+ model: modelId,
3240
+ ...(byoProvider ? { byoProvider } : {}),
3241
+ window: {
3242
+ tokens: window.tokens,
3243
+ origin: window.origin,
3244
+ ...(window.catalogId ? { catalogId: window.catalogId } : {}),
3245
+ },
3246
+ budget: {
3247
+ summarizeAtTokens: budget.summarizeAtTokens,
3248
+ pruneAtTokens: budget.pruneAtTokens,
3249
+ resultCapChars: budget.resultCapChars,
3250
+ requestCapTokens: budget.requestCapTokens,
3251
+ outputReserveTokens: budget.outputReserveTokens,
3252
+ keepMessages: budget.keepMessages,
3253
+ ...(budget.summarizeTriggerMessages !== undefined
3254
+ ? { summarizeTriggerMessages: budget.summarizeTriggerMessages }
3255
+ : {}),
3256
+ },
3257
+ results: {
3258
+ ...stats,
3259
+ tier: this.env.TIER_BUCKET ? 'sqlite+r2' : 'sqlite',
3260
+ },
3261
+ ...(sessionId
3262
+ ? { session: await this.sessionContextStatus(sessionId) }
3263
+ : {}),
3264
+ };
3265
+ }
3266
+
3267
+ /**
3268
+ * One session's working context plus the guard's counters: what the
3269
+ * next request starts from (the messages the latest checkpoint carries,
3270
+ * a summary plus the kept tail once the history was condensed — the
3271
+ * transcript rows are never condensed and are listed separately) and
3272
+ * what earlier requests needed (pruning, a provider overflow, a
3273
+ * refusal).
3274
+ */
3275
+ private async sessionContextStatus(
3276
+ sessionId: string,
3277
+ ): Promise<SessionContextStatus | null> {
3278
+ if (!(await this.sessions!.getSession(sessionId))) return null;
3279
+ const tuple = await this.saver!.getTuple({
3280
+ configurable: { thread_id: sessionId },
2732
3281
  });
3282
+ const channel: unknown = tuple?.checkpoint.channel_values.messages;
3283
+ const context: BaseMessage[] = Array.isArray(channel)
3284
+ ? channel.filter(isBaseMessage)
3285
+ : [];
3286
+ const thread = await this.saver!.listThreadMessages(sessionId);
3287
+ const stats = await this.ctx.storage.get<SessionContextCounters>(
3288
+ contextStatsKey(sessionId),
3289
+ );
3290
+ return {
3291
+ id: sessionId,
3292
+ threadMessages: thread.length,
3293
+ contextMessages: context.length,
3294
+ contextSummaries: context.filter(isSummarizationMessage).length,
3295
+ contextToolMessages: context.filter((m) => m.type === 'tool').length,
3296
+ contextTokens: estimateRequestTokens({
3297
+ systemTokens: 0,
3298
+ schemaTokens: 0,
3299
+ messages: context,
3300
+ }),
3301
+ ...(stats ?? EMPTY_CONTEXT_COUNTERS),
3302
+ };
3303
+ }
3304
+
3305
+ /** Fold a guard event into the session's stored counters (ordered, never throws). */
3306
+ private recordContextEvent(
3307
+ sessionId: string,
3308
+ event: ContextGuardEvent,
3309
+ ): void {
3310
+ const key = contextStatsKey(sessionId);
3311
+ this.contextStatsWrites = this.contextStatsWrites
3312
+ .then(async () => {
3313
+ const prev =
3314
+ (await this.ctx.storage.get<SessionContextCounters>(key)) ??
3315
+ EMPTY_CONTEXT_COUNTERS;
3316
+ const next: SessionContextCounters = {
3317
+ ...prev,
3318
+ lastEventAt: new Date().toISOString(),
3319
+ };
3320
+ switch (event.kind) {
3321
+ case 'prune':
3322
+ next.prunes += 1;
3323
+ if (event.stage !== 'soft') next.hardPrunes += 1;
3324
+ next.prunedResults += event.pruned;
3325
+ break;
3326
+ case 'overflow-retry':
3327
+ next.overflowRetries += 1;
3328
+ break;
3329
+ case 'refused':
3330
+ next.refusals += 1;
3331
+ break;
3332
+ }
3333
+ await this.ctx.storage.put(key, next);
3334
+ })
3335
+ .catch((err: unknown) => {
3336
+ console.warn(
3337
+ `[context] could not record a ${event.kind} event for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
3338
+ );
3339
+ });
3340
+ }
3341
+
3342
+ /** `GET /debug/runs` — recent runs with their marks and live state. */
3343
+ async runsStatus(): Promise<RunsStatus> {
3344
+ const userDid =
3345
+ this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
3346
+ if (!this.db && userDid) await this.ready({ userDid });
3347
+ const store = this.runStore;
3348
+ const records = store ? await store.listRecent(50) : [];
3349
+ const runs: RunsStatus['runs'] = [];
3350
+ for (const record of records) {
3351
+ const marks = store ? await store.listMarks(record.runId) : [];
3352
+ runs.push({
3353
+ ...runSummaryOf(record),
3354
+ marks: marks.map((m) => ({
3355
+ toolCallId: m.toolCallId,
3356
+ toolName: m.toolName,
3357
+ effect: m.effect,
3358
+ startedAt: m.startedAt,
3359
+ doneAt: m.doneAt,
3360
+ outcome: m.outcome,
3361
+ attempts: m.attempts,
3362
+ })),
3363
+ segments: store ? await store.countSegments(record.runId) : 0,
3364
+ attempts: record.attempts,
3365
+ nextAttemptAt: record.nextAttemptAt,
3366
+ error: record.error,
3367
+ taskRunId: record.taskRunId,
3368
+ });
3369
+ }
3370
+ return {
3371
+ config: { ...this.runConfig },
3372
+ live: this.runs?.snapshot() ?? [],
3373
+ runs,
3374
+ };
2733
3375
  }
2734
3376
 
2735
3377
  // ── turn plumbing ──────────────────────────────────────────────────────
@@ -2930,6 +3572,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2930
3572
  TurnBody,
2931
3573
  'message' | 'timezone' | 'model' | 'tools' | 'agActions' | 'attachments'
2932
3574
  >,
3575
+ run: {
3576
+ runId: string;
3577
+ abortController: AbortController;
3578
+ /** A recovery attempt: no new input, the graph continues from the checkpoint. */
3579
+ resumed: boolean;
3580
+ /** The reply text the user already received (resumed attempts). */
3581
+ continuation: string | null;
3582
+ },
2933
3583
  ) {
2934
3584
  const core = this.core;
2935
3585
  const baseAmbient = this.ambient!;
@@ -3004,6 +3654,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3004
3654
  }
3005
3655
  : baseAmbient;
3006
3656
  const effectiveModel = byoTurn ? byoTurn.byoModelId : requestedModel;
3657
+ // The turn's context budget, from the main model's window: the model
3658
+ // id the turn will actually run on (platform default, a per-request
3659
+ // choice, or the BYO credential's model) — see docs/plans/context-budgets.md.
3660
+ const mainModelId = byoTurn
3661
+ ? byoTurn.mainModelId
3662
+ : (requestedModel ??
3663
+ (isProviderAdapter(baseAmbient.llm)
3664
+ ? baseAmbient.llm.modelForRole('main')
3665
+ : DEFAULT_MODEL_ID));
3666
+ const windowResolution = await this.contextWindows.resolve(mainModelId, {
3667
+ ...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
3668
+ });
3669
+ const contextBudget = contextBudgetFor(
3670
+ windowResolution,
3671
+ contextKnobs(this.env, console),
3672
+ );
3007
3673
  if (
3008
3674
  req.client === 'matrix' &&
3009
3675
  !(await sessions.getSession(req.sessionId))
@@ -3016,8 +3682,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3016
3682
  oracleEntityDid: core.identity.entityDid,
3017
3683
  });
3018
3684
  }
3019
- this.aborts.get(req.sessionId)?.abort();
3020
- const abortController = new AbortController();
3685
+ // The run coordinator already applied the session's multitask rule
3686
+ // (aborted or queued behind the previous run); this attempt's signal
3687
+ // is the run's own.
3688
+ const abortController = run.abortController;
3021
3689
  this.aborts.set(req.sessionId, abortController);
3022
3690
 
3023
3691
  // Every turn runs inside the user's oracle room, like the Node
@@ -3133,12 +3801,55 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3133
3801
  const hostRoomTitle = opts.hooks?.getRoomTitle;
3134
3802
  const hostSafetyModel = opts.hooks?.safetyModel;
3135
3803
 
3136
- const { agent, context } = await createMainAgent({
3804
+ // A resumed attempt reads the client's tool surface back from the
3805
+ // checkpointed state (the run row does not store the catalogue).
3806
+ const priorSurface = priorState as {
3807
+ browserTools?: TurnBody['tools'];
3808
+ agActions?: TurnBody['agActions'];
3809
+ };
3810
+ const browserTools =
3811
+ body.tools ??
3812
+ (run.resumed ? priorSurface.browserTools : undefined) ??
3813
+ [];
3814
+ const agActions =
3815
+ body.agActions ??
3816
+ (run.resumed ? priorSurface.agActions : undefined) ??
3817
+ [];
3818
+
3819
+ // Write-ahead tool marks + the resume policy (tool-marks.ts). The
3820
+ // effect map is filled from the build below; the closure reads it at
3821
+ // call time, so the middleware can be created before the agent.
3822
+ const toolEffects = new Map<string, ToolEffect>();
3823
+ const marksMiddleware = createToolMarksMiddleware({
3824
+ runId: run.runId,
3825
+ store: this.runStore!,
3826
+ effectOf: (name) => toolEffects.get(name) ?? toolEffectOf({ name }),
3827
+ continuation: run.continuation,
3828
+ logger: console,
3829
+ });
3830
+ // Every tool result above the budget's cap is saved whole and handed
3831
+ // to the model as head + tail + handle (result-cap.ts); applied to
3832
+ // sub-agents too through `toolMiddlewares`.
3833
+ const resultStore = this.resultStore;
3834
+ const resultCap = {
3835
+ capChars: contextBudget.resultCapChars,
3836
+ sessionId: req.sessionId,
3837
+ ...(resultStore ? { store: resultStore } : {}),
3838
+ exempt: new Set([READ_RESULT_TOOL_NAME]),
3839
+ logger: console,
3840
+ };
3841
+ // The middleware covers what the wrapper cannot: a sub-agent's reply,
3842
+ // which is a tool result to the main agent. Wrapped tools are capped
3843
+ // inside the wrapper (already under the cap by the time it runs).
3844
+ const capMiddleware = createResultCapMiddleware(resultCap);
3845
+
3846
+ const built = await createMainAgent({
3137
3847
  registries: core.registries,
3138
3848
  identity: core.identity,
3139
3849
  config: core.validatedEnv,
3140
3850
  availablePlugins: core.availablePlugins,
3141
3851
  byoProvider: byoTurn?.provider,
3852
+ contextBudget,
3142
3853
  ambient: {
3143
3854
  ...ambient,
3144
3855
  attachments: attachmentAccess,
@@ -3150,6 +3861,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3150
3861
  middlewares: this.workStatus
3151
3862
  ? [createWorkStatusMiddleware({ producer: this.workStatus })]
3152
3863
  : [],
3864
+ // Applied to the main agent AND every sub-agent: a sub-agent's
3865
+ // own tool calls are marked and capped too.
3866
+ toolMiddlewares: [marksMiddleware, capMiddleware],
3867
+ resultCap,
3868
+ onContextOverflow: (error) =>
3869
+ this.contextWindows.learnFromError(mainModelId, error, {
3870
+ ...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
3871
+ }),
3872
+ onContextEvent: (event) =>
3873
+ this.recordContextEvent(req.sessionId, event),
3874
+ ...(resultStore
3875
+ ? {
3876
+ readResult: (id: string, offset: number, length: number) =>
3877
+ resultStore.read(id, offset, length),
3878
+ }
3879
+ : {}),
3153
3880
  ...(hostRoomTitle
3154
3881
  ? {
3155
3882
  getRoomTitle: (roomId: string) =>
@@ -3185,25 +3912,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3185
3912
  // AG-UI plugins turn these into tools at build time
3186
3913
  // (`getRequestTools` reads `history.state`), so they must be here
3187
3914
  // and not only in the graph input.
3188
- browserTools: body.tools ?? [],
3189
- agActions: body.agActions ?? [],
3915
+ browserTools,
3916
+ agActions,
3190
3917
  },
3191
3918
  checkpointer: saver,
3192
3919
  abortSignal: abortController.signal,
3193
3920
  });
3921
+ const { agent, context } = built;
3922
+ for (const [name, effect] of built.toolEffects)
3923
+ toolEffects.set(name, effect);
3194
3924
 
3195
3925
  const attachmentKwargs =
3196
3926
  prepared && prepared.metas.length > 0
3197
3927
  ? { attachment: prepared.metas[0], attachments: prepared.metas }
3198
3928
  : {};
3929
+ // Node's `MessagesService` metadata for room messages: who spoke
3930
+ // (the group-chat tools and prompts read it), and in a group room the
3931
+ // `[DisplayName]: ` prefix on the message itself.
3932
+ const speakerName =
3933
+ req.senderDisplayName ??
3934
+ req.identity.matrixUserId ??
3935
+ req.identity.userDid;
3936
+ const speakerKwargs =
3937
+ req.client === 'matrix'
3938
+ ? {
3939
+ senderDid: req.identity.userDid,
3940
+ ...(req.identity.matrixUserId
3941
+ ? { senderMatrixUserId: req.identity.matrixUserId }
3942
+ : {}),
3943
+ senderDisplayName: speakerName,
3944
+ threadId: req.sessionId,
3945
+ ...(req.eventId ? { eventId: req.eventId } : {}),
3946
+ }
3947
+ : {};
3948
+ const rawContent = prepared ? prepared.content : body.message;
3949
+ const content =
3950
+ req.client === 'matrix' && req.roomKind === 'group'
3951
+ ? prefixSpeaker(rawContent, speakerName)
3952
+ : rawContent;
3199
3953
  const stateInput = {
3200
3954
  messages: [
3201
3955
  new HumanMessage({
3202
- content: prepared ? prepared.content : body.message,
3956
+ content,
3203
3957
  additional_kwargs: {
3204
3958
  timestamp: new Date().toISOString(),
3205
3959
  oracleName: core.identity.name,
3206
3960
  msgFromMatrixRoom: req.client === 'matrix',
3961
+ ...speakerKwargs,
3207
3962
  ...attachmentKwargs,
3208
3963
  },
3209
3964
  }),
@@ -3224,8 +3979,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3224
3979
  config: { did: req.identity.userDid },
3225
3980
  client: req.client,
3226
3981
  ...metadataGraphInput(meta, priorMeta),
3227
- browserTools: body.tools ?? [],
3228
- agActions: body.agActions ?? [],
3982
+ browserTools,
3983
+ agActions,
3229
3984
  };
3230
3985
  // LangSmith: metadata is attached unconditionally (inert without a
3231
3986
  // tracer); the explicit tracer only when this turn is traced (global
@@ -3252,6 +4007,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3252
4007
  byoNotice,
3253
4008
  byoProvider: byoTurn?.provider,
3254
4009
  turnDisposables,
4010
+ toolOutputCapChars: contextBudget.resultCapChars,
3255
4011
  };
3256
4012
  }
3257
4013
 
@@ -3358,6 +4114,26 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3358
4114
  }
3359
4115
  }
3360
4116
 
4117
+ /**
4118
+ * Channel-memory compaction for the gateway's group rooms
4119
+ * (`src/matrix/group-chat.ts`): the platform's small model distils a
4120
+ * batch of room messages with the Node summarizer's prompt. Needs no
4121
+ * user file — only this object's model adapter.
4122
+ */
4123
+ async summarizeGroupMessages(messages: JsonString): Promise<string | null> {
4124
+ const parsed: unknown = JSON.parse(messages);
4125
+ if (!Array.isArray(parsed) || parsed.length === 0) return null;
4126
+ const model = (this.ambient?.llm ?? this.core.llm).get('session-title', {
4127
+ temperature: 0.2,
4128
+ maxTokens: 800,
4129
+ });
4130
+ return summarizeObservedMessages(
4131
+ model,
4132
+ parsed as ObservedMessage[],
4133
+ console,
4134
+ );
4135
+ }
4136
+
3361
4137
  private async generateTitle(
3362
4138
  messages: BaseMessage[],
3363
4139
  ): Promise<string | null> {
@@ -3428,50 +4204,3 @@ async function* tapMessages(
3428
4204
  yield evt;
3429
4205
  }
3430
4206
  }
3431
-
3432
- /**
3433
- * Merge the turn's SSE stream with plugin-emitted events. Plugin events are
3434
- * written straight into the same byte stream as they happen.
3435
- */
3436
- function mergeWithEmitter(
3437
- base: ReadableStream<Uint8Array>,
3438
- onRegister: (write: (eventName: string, payload: unknown) => void) => void,
3439
- ): ReadableStream<Uint8Array> {
3440
- const encoder = new TextEncoder();
3441
- let closed = false;
3442
- return new ReadableStream<Uint8Array>({
3443
- start(controller) {
3444
- onRegister((eventName, payload) => {
3445
- if (closed) return;
3446
- try {
3447
- controller.enqueue(encoder.encode(formatSSE(eventName, payload)));
3448
- } catch {
3449
- closed = true;
3450
- }
3451
- });
3452
- const reader = base.getReader();
3453
- void (async () => {
3454
- try {
3455
- for (;;) {
3456
- const { done, value } = await reader.read();
3457
- if (done) break;
3458
- if (!closed) controller.enqueue(value);
3459
- }
3460
- } catch (err) {
3461
- if (!closed) controller.error(err);
3462
- } finally {
3463
- closed = true;
3464
- try {
3465
- controller.close();
3466
- } catch {
3467
- /* already closed */
3468
- }
3469
- }
3470
- })();
3471
- },
3472
- cancel() {
3473
- closed = true;
3474
- void base.cancel();
3475
- },
3476
- });
3477
- }