@ixo/oracle-runtime-workers 0.10.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +5 -2
  2. package/package.json +1 -1
  3. package/src/core/context-budget.ts +201 -0
  4. package/src/core/context-window.ts +289 -0
  5. package/src/core/env.ts +29 -0
  6. package/src/core/main-agent-types.ts +46 -0
  7. package/src/core/main-agent.ts +86 -9
  8. package/src/core/middlewares/context-guard.ts +270 -0
  9. package/src/core/middlewares/result-cap.ts +180 -0
  10. package/src/core/middlewares/summarization.ts +87 -6
  11. package/src/core/middlewares/tool-marks.ts +209 -0
  12. package/src/core/openrouter-pricing.ts +42 -1
  13. package/src/core/read-result-tool.ts +77 -0
  14. package/src/core/runtime-context.ts +7 -0
  15. package/src/core/sub-agent-fallback.ts +20 -3
  16. package/src/core/wrap-plugin-tool.ts +24 -3
  17. package/src/do/ambient.ts +17 -0
  18. package/src/do/contracts.ts +229 -0
  19. package/src/do/flush-schedule.ts +40 -0
  20. package/src/do/result-store-test-do.ts +88 -0
  21. package/src/do/result-store.ts +339 -0
  22. package/src/do/run-buffer.ts +204 -0
  23. package/src/do/run-coordinator.ts +572 -0
  24. package/src/do/run-request.ts +52 -0
  25. package/src/do/run-store-test-do.ts +130 -0
  26. package/src/do/run-store.ts +604 -0
  27. package/src/do/session-id-migration.ts +85 -0
  28. package/src/do/speaker-prefix.ts +37 -0
  29. package/src/do/sse-stream.ts +567 -400
  30. package/src/do/transcript.ts +238 -0
  31. package/src/do/turn-body.ts +10 -0
  32. package/src/do/user-oracle-do.ts +948 -162
  33. package/src/matrix/gateway-do.ts +230 -29
  34. package/src/matrix/group-chat-summarizer.ts +107 -0
  35. package/src/matrix/group-chat.ts +1163 -0
  36. package/src/matrix/inbox-store.ts +23 -3
  37. package/src/matrix/ingest.ts +54 -11
  38. package/src/matrix/reply-chain.ts +34 -25
  39. package/src/plugin-api/tool-helper.ts +9 -1
  40. package/src/plugin-api/types.ts +86 -0
  41. package/src/plugins/index.ts +3 -0
  42. package/src/plugins/matrix-group-chats/README.md +70 -0
  43. package/src/plugins/matrix-group-chats/index.ts +132 -0
  44. package/src/plugins/matrix-group-chats/tools.ts +172 -0
  45. package/src/plugins/tasks/tasks-tools.ts +4 -0
  46. package/src/shell/app.ts +66 -0
  47. package/src/sqlite/sqlite-saver.ts +81 -0
  48. package/src/tasks/scheduler.ts +134 -3
  49. package/src/tasks/store.ts +22 -0
@@ -20,6 +20,7 @@ import { DurableObject } from 'cloudflare:workers';
20
20
  import {
21
21
  AIMessage,
22
22
  HumanMessage,
23
+ isBaseMessage,
23
24
  type BaseMessage,
24
25
  } from '@langchain/core/messages';
25
26
  import {
@@ -50,6 +51,7 @@ import {
50
51
  llmEnvFromWorkerEnv,
51
52
  resolveLangsmithTracing,
52
53
  type OpenRouterLlmAdapter,
54
+ DEFAULT_MODEL_ID,
53
55
  } from '../core/llm';
54
56
  import { createMainAgent } from '../core/main-agent';
55
57
  import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
@@ -140,16 +142,28 @@ import {
140
142
  type OracleWorkerEnv,
141
143
  type SessionSummary,
142
144
  type StorageStatus,
145
+ type RunsStatus,
146
+ type ContextStatus,
147
+ type SessionContextStatus,
148
+ type RunSummary,
143
149
  type TurnIdentity,
144
150
  type TurnRequest,
145
151
  type TurnResult,
146
152
  type MemorySchemaDebug,
147
153
  TURN_INTERRUPTED_MARKER,
154
+ type JsonString,
148
155
  } from './contracts';
149
156
  import { turnRecursionLimit } from './turn-config';
150
157
  import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
151
158
  import { decideBootDirty } from './boot-dirty';
159
+ import { decideFlush } from './flush-schedule';
152
160
  import { mirrorTxnId, RoomMirror } from './room-mirror';
161
+ import { prefixSpeaker } from './speaker-prefix';
162
+ import {
163
+ type ObservedMessage,
164
+ summarizeObservedMessages,
165
+ } from '../matrix/group-chat-summarizer';
166
+ import { migrateThreadSessionIds } from './session-id-migration';
153
167
  import { ReauthPrompter } from './reauth-prompt';
154
168
  import {
155
169
  createTaskScheduler,
@@ -167,8 +181,46 @@ import {
167
181
  type HistoryMessage,
168
182
  type IndexableSession,
169
183
  } from '../memory/session-history-indexer';
170
- import { createSseTurnStream, formatSSE, SSE_HEADERS } from './sse-stream';
184
+ import {
185
+ createSseSubscriberStream,
186
+ runTurnFrames,
187
+ SSE_HEADERS,
188
+ } from './sse-stream';
171
189
  import { parseTurnBody, type TurnBody } from './turn-body';
190
+ import {
191
+ RunCoordinator,
192
+ type LiveRun,
193
+ type RunOutcome,
194
+ } from './run-coordinator';
195
+ import {
196
+ RunStore,
197
+ runDurabilityConfig,
198
+ type RunDurabilityConfig,
199
+ type RunRecord,
200
+ } from './run-store';
201
+ import {
202
+ runSummaryOf,
203
+ storedRunRequest,
204
+ type StoredRunRequest,
205
+ } from './run-request';
206
+ import {
207
+ createToolMarksMiddleware,
208
+ toolEffectOf,
209
+ type ToolEffect,
210
+ } from '../core/middlewares/tool-marks';
211
+ import { createResultCapMiddleware } from '../core/middlewares/result-cap';
212
+ import { READ_RESULT_TOOL_NAME } from '../core/read-result-tool';
213
+ import {
214
+ ContextWindowResolver,
215
+ contextWindowConfig,
216
+ } from '../core/context-window';
217
+ import { contextBudgetFor, contextKnobs } from '../core/context-budget';
218
+ import {
219
+ estimateRequestTokens,
220
+ type ContextGuardEvent,
221
+ } from '../core/middlewares/context-guard';
222
+ import { fetchOpenRouterContextLengths } from '../core/openrouter-pricing';
223
+ import { ResultStore, resultStoreKnobs } from './result-store';
172
224
  import { formatReplay } from '../matrix/replay-format';
173
225
  import { retryGateway } from './gateway-retry';
174
226
  import {
@@ -178,7 +230,10 @@ import {
178
230
  import {
179
231
  contentToText,
180
232
  isSummarizationMessage,
233
+ pageThreadTranscript,
234
+ TranscriptCursorError,
181
235
  transformTranscript,
236
+ type TranscriptPageOptions,
182
237
  } from './transcript';
183
238
  import { WorkersUcanService } from './ucan-service';
184
239
 
@@ -208,6 +263,27 @@ const IDLE_EVICT_MS = 5 * 24 * 60 * 60 * 1000;
208
263
  /** Set once the legacy Matrix copy has been removed (or found absent): no more lookups. */
209
264
  const META_LEGACY_CLEARED = 'meta:legacyCleared';
210
265
  const META_USER_DID = 'meta:userDid';
266
+
267
+ /** KV key of a session's context-guard counters (`GET /debug/context?session=`). */
268
+ const contextStatsKey = (sessionId: string): string => `ctxstats:${sessionId}`;
269
+
270
+ type SessionContextCounters = Pick<
271
+ SessionContextStatus,
272
+ | 'prunes'
273
+ | 'hardPrunes'
274
+ | 'prunedResults'
275
+ | 'overflowRetries'
276
+ | 'refusals'
277
+ | 'lastEventAt'
278
+ >;
279
+
280
+ const EMPTY_CONTEXT_COUNTERS: SessionContextCounters = {
281
+ prunes: 0,
282
+ hardPrunes: 0,
283
+ prunedResults: 0,
284
+ overflowRetries: 0,
285
+ refusals: 0,
286
+ };
211
287
  /** VFS write generation of the file at the last successful upload (see `DoVfs.writeGeneration`). */
212
288
  const META_UPLOADED_GEN = 'meta:uploadedGen';
213
289
  /**
@@ -245,6 +321,15 @@ const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
245
321
  * silently stale.
246
322
  */
247
323
  const META_DIRTY = 'meta:dirty';
324
+ /**
325
+ * When the dirty working copy is due to be uploaded. Armed by the first
326
+ * write after an upload (`FLUSH_DEBOUNCE_MS` later), replaced by the retry
327
+ * delay after a failed upload, cleared with the dirty mark. The alarm is
328
+ * shared with the realtime heartbeat, the durable-run keep-alive, task runs
329
+ * and compaction, so a wake alone never means "upload now" — the tick
330
+ * consults this deadline (`do/flush-schedule.ts`).
331
+ */
332
+ const META_FLUSH_AT = 'meta:flushAt';
248
333
  const META_OWNER_ETAG = 'meta:ownerEtag';
249
334
  const META_LAST_FLUSH = 'meta:lastFlushAt';
250
335
  const META_LAST_CHECKSUM = 'meta:lastChecksum';
@@ -424,6 +509,38 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
424
509
  if (this.ctx.getWebSockets().length > 0) this.realtime.nextPingAt();
425
510
  }
426
511
  private readonly aborts = new Map<string, AbortController>();
512
+ /** Durable runs (docs/plans/durable-runs.md): the rows, and the live side. */
513
+ private runStore: RunStore | null = null;
514
+ private runs: RunCoordinator | null = null;
515
+ private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
516
+ this.env,
517
+ );
518
+ /** Whole tool results the result cap saved (docs/plans/context-budgets.md). */
519
+ private resultStore: ResultStore | null = null;
520
+ /**
521
+ * Per-model context windows; learned limits persist in this object's KV
522
+ * storage (`ctxwin:<model>`), so a provider's rejection is remembered
523
+ * across boots.
524
+ */
525
+ private readonly contextWindows = new ContextWindowResolver({
526
+ config: contextWindowConfig(this.env, console),
527
+ catalog: () =>
528
+ fetchOpenRouterContextLengths({
529
+ apiKey: this.env.OPEN_ROUTER_API_KEY,
530
+ logger: console,
531
+ }),
532
+ learned: {
533
+ get: (model) => this.ctx.storage.get<number>(`ctxwin:${model}`),
534
+ set: (model, tokens) => this.ctx.storage.put(`ctxwin:${model}`, tokens),
535
+ },
536
+ logger: console,
537
+ });
538
+ /**
539
+ * Per-session context-guard counters (`ctxstats:<session>` in KV
540
+ * storage; `GET /debug/context?session=`). Writes are chained so two
541
+ * events of one request (a soft then a hard prune) never race.
542
+ */
543
+ private contextStatsWrites: Promise<void> = Promise.resolve();
427
544
  private readonly delegations = new Map<string, { raw: string }>();
428
545
  /** Epoch-ms until which a room-state delegation lookup is not retried. */
429
546
  private delegationMissUntil = 0;
@@ -456,6 +573,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
456
573
  private mirror: RoomMirror | null = null;
457
574
  /** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
458
575
  private readonly sessionRooms = new Map<string, string>();
576
+ /** The user's main oracle room, once resolved (the session list is scoped to it). */
577
+ private mainRoomId: string | null = null;
459
578
  /** The throttled `delegation_required` prompt (see reauth-prompt.ts). */
460
579
  private reauthPrompter: ReauthPrompter | null = null;
461
580
  /** Room turns running right now, by Matrix event id: a gateway that asks again attaches instead of re-running. */
@@ -974,9 +1093,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
974
1093
  console.warn(
975
1094
  `[user-do] VFS migration write failed for ${userDid} — serving the legacy Matrix copy for now: ${err instanceof Error ? err.message : String(err)}`,
976
1095
  );
977
- // Sooner than the 24 h debounce `markDirty` armed: the system of
978
- // record is still empty for this user.
979
- this.requestAlarm(Date.now() + FLUSH_RETRY_DELAY_MS);
1096
+ // The failed upload armed the 10-minute retry (`flushOnce`), well
1097
+ // before the 24 h debounce `markDirty` set: the system of record
1098
+ // is still empty for this user.
980
1099
  }
981
1100
  }
982
1101
  await this.reconcileDirtyOnBoot(liveDb, userDid);
@@ -1015,6 +1134,29 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1015
1134
  await this.sessions.setup();
1016
1135
  this.matrixLedger = new MatrixTurnLedger(liveDb);
1017
1136
  await this.matrixLedger.setup();
1137
+ this.runStore = new RunStore(liveDb);
1138
+ await this.runStore.setup();
1139
+ this.resultStore = new ResultStore(liveDb, {
1140
+ ...(this.env.TIER_BUCKET ? { bucket: this.env.TIER_BUCKET } : {}),
1141
+ prefix: this.ctx.id.toString(),
1142
+ ...resultStoreKnobs(this.env),
1143
+ logger: console,
1144
+ });
1145
+ await this.resultStore.setup();
1146
+ // Threads opened before the runtime adopted Node's session rule
1147
+ // (session id = thread root event id) are renamed to it, once.
1148
+ const renamed = await migrateThreadSessionIds(liveDb, console);
1149
+ if (renamed.renamed > 0) this.markDirty();
1150
+ this.runs = new RunCoordinator({
1151
+ store: this.runStore,
1152
+ config: this.runConfig,
1153
+ instanceId: this.instanceId,
1154
+ log: console,
1155
+ requestAlarm: (at) => this.requestAlarm(at),
1156
+ runAttempt: (live, resumed) => this.runAttempt(live, resumed),
1157
+ checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
1158
+ onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
1159
+ });
1018
1160
 
1019
1161
  const core = this.core;
1020
1162
  // Boot-time plugin hooks + collision checks, once per object (memoised).
@@ -1031,6 +1173,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1031
1173
  gateway: this.gateway,
1032
1174
  runTurn: (req) => this.runTurn(req),
1033
1175
  requestAlarm: (at) => this.requestAlarm(at),
1176
+ turnRunLive: (taskRunId) => Boolean(this.runs?.byTaskRunId(taskRunId)),
1034
1177
  log: console,
1035
1178
  ...(Number.isFinite(maxTasks) ? { maxTasksPerUser: maxTasks } : {}),
1036
1179
  ...(Number.isFinite(minCron) ? { minCronIntervalSec: minCron } : {}),
@@ -1040,6 +1183,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1040
1183
  if (next !== null) this.requestAlarm(Math.max(next, Date.now() + 1000));
1041
1184
  }
1042
1185
 
1186
+ // Runs a previous incarnation left in flight: schedule their recovery
1187
+ // (attempts run from the alarm, never inside this boot).
1188
+ try {
1189
+ await this.runs.recoverOrphans();
1190
+ } catch (err) {
1191
+ console.error(
1192
+ `[user-do] run recovery scan failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
1193
+ );
1194
+ }
1195
+
1043
1196
  // Resume (or start) legacy-blob compaction in the background: files
1044
1197
  // imported from the Node runtime arrive with uncompressed blobs.
1045
1198
  this.compactDone ??=
@@ -1135,6 +1288,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1135
1288
  META_OWNER_ETAG,
1136
1289
  META_LAST_CHECKSUM,
1137
1290
  META_LAST_FLUSH,
1291
+ META_FLUSH_AT,
1138
1292
  META_UPLOADED_GEN,
1139
1293
  META_FLUSH_FAILURES,
1140
1294
  META_LAST_VACUUM,
@@ -1240,7 +1394,30 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1240
1394
  private markDirty(): void {
1241
1395
  this.dirty = true;
1242
1396
  void this.ctx.storage.put(META_DIRTY, true);
1243
- this.requestAlarm(Date.now() + FLUSH_DEBOUNCE_MS);
1397
+ void this.armFlush(Date.now() + FLUSH_DEBOUNCE_MS);
1398
+ }
1399
+
1400
+ /**
1401
+ * Record the flush deadline and arm the alarm for it. An earlier
1402
+ * deadline already on disk wins, so a burst of writes uploads once,
1403
+ * `FLUSH_DEBOUNCE_MS` after the first of them, never later.
1404
+ */
1405
+ private async armFlush(at: number): Promise<void> {
1406
+ const existing = await this.ctx.storage.get<number>(META_FLUSH_AT);
1407
+ const deadline =
1408
+ typeof existing === 'number' && existing <= at ? existing : at;
1409
+ if (deadline !== existing)
1410
+ await this.ctx.storage.put(META_FLUSH_AT, deadline);
1411
+ this.requestAlarm(deadline);
1412
+ }
1413
+
1414
+ /**
1415
+ * After a failed upload: the next attempt is `at`, replacing the deadline
1416
+ * that just came due (which would otherwise make every wake retry).
1417
+ */
1418
+ private async scheduleFlushRetry(at: number): Promise<void> {
1419
+ await this.ctx.storage.put(META_FLUSH_AT, at);
1420
+ this.requestAlarm(at);
1244
1421
  }
1245
1422
 
1246
1423
  /** Arm the object's single alarm no later than `at` (multiplexed). */
@@ -1268,12 +1445,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1268
1445
  this.ctx.getWebSockets().length > 0 ? this.realtime.pingTick() : null;
1269
1446
  const housekeepingAt =
1270
1447
  await this.ctx.storage.get<number>(META_HOUSEKEEPING_AT);
1448
+ // Runs in flight in THIS instance need the keep-alive re-armed; a
1449
+ // recovery attempt that came due needs starting. Both are cheap and
1450
+ // do not open the database on a bare object (the coordinator only
1451
+ // exists once the object booted).
1452
+ const keepAliveAt = this.runs?.keepAliveDeadline(wakeAt) ?? null;
1453
+ const recoveryAt = this.runs?.nextRecoveryAt() ?? null;
1454
+ if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000)
1455
+ await this.runs.resumeDue(wakeAt);
1271
1456
  if (
1272
1457
  nextPingAt !== null &&
1273
1458
  housekeepingAt !== undefined &&
1274
- housekeepingAt > wakeAt + 1000
1459
+ housekeepingAt > wakeAt + 1000 &&
1460
+ keepAliveAt === null &&
1461
+ (recoveryAt === null || recoveryAt > wakeAt + 1000)
1275
1462
  ) {
1276
- await this.ctx.storage.setAlarm(Math.min(nextPingAt, housekeepingAt));
1463
+ await this.ctx.storage.setAlarm(
1464
+ Math.min(nextPingAt, housekeepingAt, recoveryAt ?? Infinity),
1465
+ );
1277
1466
  return;
1278
1467
  }
1279
1468
 
@@ -1308,15 +1497,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1308
1497
  const now = Date.now();
1309
1498
  const deadlines: number[] = [];
1310
1499
 
1311
- if (this.dirty && this.db) {
1500
+ // Durable runs: start the recovery attempts that are due (the boot
1501
+ // above may have just scheduled them), then keep the alarm inside the
1502
+ // keep-alive horizon while anything executes.
1503
+ if (this.runs) {
1504
+ try {
1505
+ await this.runs.resumeDue(now);
1506
+ } catch (err) {
1507
+ console.error(
1508
+ `[user-do] run recovery tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
1509
+ );
1510
+ }
1511
+ const keepAlive = this.runs.keepAliveDeadline(now);
1512
+ if (keepAlive !== null) deadlines.push(keepAlive);
1513
+ const nextRecovery = this.runs.nextRecoveryAt();
1514
+ if (nextRecovery !== null)
1515
+ deadlines.push(Math.max(nextRecovery, now + 1000));
1516
+ }
1517
+
1518
+ const lastAccess =
1519
+ (await this.ctx.storage.get<number>(META_LAST_ACCESS)) ?? now;
1520
+ const idle = now - lastAccess > IDLE_EVICT_MS;
1521
+
1522
+ // Owner-store flush: only when its own deadline is due, or the copy
1523
+ // is about to be evicted below. A wake for anything else (a run's
1524
+ // keep-alive, a heartbeat round that fell through, a compaction or
1525
+ // tier re-arm) leaves a dirty copy alone — the upload is a full-file
1526
+ // transfer and is meant to happen once a day, not once a turn.
1527
+ const flush = decideFlush({
1528
+ dirty: this.dirty && this.db !== null,
1529
+ flushAt: await this.ctx.storage.get<number>(META_FLUSH_AT),
1530
+ now,
1531
+ idle,
1532
+ });
1533
+ if (flush.action === 'flush') {
1312
1534
  try {
1313
1535
  await this.flushToOwnerStore();
1314
1536
  } catch (err) {
1315
1537
  console.error(
1316
1538
  `[user-do] flush failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
1317
1539
  );
1540
+ // `flushOnce` arms the retry on an upload failure; anything that
1541
+ // failed before the upload gets the same retry window here.
1542
+ await this.scheduleFlushRetry(now + FLUSH_RETRY_DELAY_MS);
1318
1543
  deadlines.push(now + FLUSH_RETRY_DELAY_MS);
1319
1544
  }
1545
+ } else if (flush.action === 'wait') {
1546
+ deadlines.push(Math.max(flush.at, now + 1000));
1320
1547
  }
1321
1548
 
1322
1549
  if (this.taskScheduler) {
@@ -1349,9 +1576,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1349
1576
  }
1350
1577
  }
1351
1578
 
1352
- const lastAccess =
1353
- (await this.ctx.storage.get<number>(META_LAST_ACCESS)) ?? now;
1354
-
1355
1579
  // Free-page reclaim on a quiet object (policy in sqlite/vacuum-policy.ts):
1356
1580
  // runs after the flush above so the upload holds the pre-rebuild file,
1357
1581
  // and marks dirty so the smaller file goes up on the next tick.
@@ -1364,7 +1588,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1364
1588
  // of DO storage (see sqlite/page-tier.ts). After the flush above (the
1365
1589
  // upload reads hot rows, not R2) and never over an in-flight one (it
1366
1590
  // holds a snapshot). A pass is capped; more work re-arms soon.
1367
- if (this.db?.tierEnabled && !this.flushInFlight) {
1591
+ if (
1592
+ this.db?.tierEnabled &&
1593
+ !this.flushInFlight &&
1594
+ (this.runs?.activeCount ?? 0) === 0
1595
+ ) {
1368
1596
  const next = await this.tierTick(this.db, now);
1369
1597
  if (next !== null) deadlines.push(next);
1370
1598
  }
@@ -1374,8 +1602,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1374
1602
  // evict while tasks are scheduled: their runs ARE activity. The flush
1375
1603
  // above ran first; the wipe additionally verifies the upstream copy
1376
1604
  // matches the working copy, else the copy stays until it does.
1377
- const idle = now - lastAccess > IDLE_EVICT_MS;
1378
- if (idle && !this.dirty && this.db && deadlines.length === 0) {
1605
+ if (
1606
+ idle &&
1607
+ !this.dirty &&
1608
+ this.db &&
1609
+ deadlines.length === 0 &&
1610
+ (this.runs?.size ?? 0) === 0
1611
+ ) {
1379
1612
  if (await this.ownerCopyIsCurrent(this.db)) {
1380
1613
  await this.wipeWorkingCopy(this.db);
1381
1614
  await this.ctx.storage.delete(META_HOUSEKEEPING_AT);
@@ -1541,6 +1774,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1541
1774
  );
1542
1775
  if (room) {
1543
1776
  roomId = room.roomId;
1777
+ if (!o.roomId) this.mainRoomId = room.roomId;
1544
1778
  // The marker is a waited, non-durable send: it must never be
1545
1779
  // replayed after a gateway restart (its event id IS the session id,
1546
1780
  // an orphan marker would be a ghost session). Instead it retries
@@ -1583,11 +1817,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1583
1817
  o: { limit?: number; offset?: number } = {},
1584
1818
  ): Promise<{ sessions: SessionSummary[]; total: number }> {
1585
1819
  await this.ready(identity);
1586
- // Task runs re-enter the agent as `task:<id>` sessions; like the Node
1587
- // runtime (which lists only the user's main room) keep them out of
1588
- // the user's session list.
1820
+ // Node parity (`SessionsService.listSessions`): the list is the user's
1821
+ // MAIN oracle room only — Portal sessions and the threads opened by
1822
+ // room messages there. Threads in dedicated task rooms (or any other
1823
+ // room) stay out of it, and task runs (`task:<id>`) are hidden
1824
+ // whichever room they deliver to. A user without an oracle room sees
1825
+ // every session, as on Node.
1826
+ const mainRoomId = await this.resolveMainRoomId(identity.userDid);
1589
1827
  const { sessions, total } = await this.sessions!.listSessions(
1590
- undefined,
1828
+ mainRoomId ?? undefined,
1591
1829
  o.limit ?? 20,
1592
1830
  o.offset ?? 0,
1593
1831
  TASK_SESSION_PREFIX,
@@ -1595,6 +1833,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1595
1833
  return { sessions: sessions.map(toSummary), total };
1596
1834
  }
1597
1835
 
1836
+ /**
1837
+ * The user↔oracle room, resolved once per instance. `null` is the
1838
+ * definitive "no oracle room"; a transient gateway failure retries and
1839
+ * then fails the request rather than answering with the wrong list.
1840
+ */
1841
+ private async resolveMainRoomId(userDid: string): Promise<string | null> {
1842
+ if (this.mainRoomId) return this.mainRoomId;
1843
+ const room = await retryGateway(() =>
1844
+ this.gateway.resolveUserRoom(userDid),
1845
+ );
1846
+ if (room) this.mainRoomId = room.roomId;
1847
+ return room?.roomId ?? null;
1848
+ }
1849
+
1598
1850
  async deleteSession(
1599
1851
  identity: TurnIdentity,
1600
1852
  sessionId: string,
@@ -1615,6 +1867,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1615
1867
  const deleted = await this.sessions!.deleteSession(sessionId);
1616
1868
  if (deleted) {
1617
1869
  await this.saver!.deleteThread(sessionId);
1870
+ await this.resultStore
1871
+ ?.deleteForSession(sessionId)
1872
+ .catch((err: unknown) => {
1873
+ console.warn(
1874
+ `[user-do] could not delete the saved tool results of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
1875
+ );
1876
+ });
1877
+ await this.ctx.storage.delete(contextStatsKey(sessionId));
1618
1878
  this.markDirty();
1619
1879
  }
1620
1880
  return deleted;
@@ -1723,6 +1983,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1723
1983
  return JSON.stringify(dtos);
1724
1984
  }
1725
1985
 
1986
+ async listMessagesPage(
1987
+ identity: TurnIdentity,
1988
+ sessionId: string,
1989
+ options: TranscriptPageOptions,
1990
+ ): Promise<
1991
+ { ok: true; json: string } | { ok: false; status: 400; message: string }
1992
+ > {
1993
+ await this.ready(identity);
1994
+ try {
1995
+ const page = await pageThreadTranscript(
1996
+ this.saver!,
1997
+ sessionId,
1998
+ options,
1999
+ );
2000
+ return { ok: true, json: JSON.stringify(page) };
2001
+ } catch (err) {
2002
+ if (err instanceof TranscriptCursorError)
2003
+ return { ok: false, status: 400, message: err.message };
2004
+ throw err;
2005
+ }
2006
+ }
2007
+
1726
2008
  /**
1727
2009
  * The shell deposited a (new) delegation for this user: adopt it now so
1728
2010
  * header-less turns mint from it immediately. Does not boot the object.
@@ -1820,9 +2102,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1820
2102
  writeGeneration: db.writeGeneration,
1821
2103
  });
1822
2104
  if (decision === 'flagged') {
1823
- // The mark is on disk already (its flush alarm too); just adopt it.
2105
+ // The mark is on disk already (its deadline too); adopt it and make
2106
+ // sure the alarm is armed for that deadline.
1824
2107
  this.dirty = true;
1825
- this.requestAlarm(Date.now() + FLUSH_DEBOUNCE_MS);
2108
+ void this.armFlush(Date.now() + FLUSH_DEBOUNCE_MS);
1826
2109
  } else if (decision === 'behind-upload') {
1827
2110
  console.log(
1828
2111
  `[user-do] working copy of ${userDid} is at generation ${db.writeGeneration}, last upload at ${String(got.get(META_UPLOADED_GEN))} — a turn ended without marking it; flushing`,
@@ -1861,7 +2144,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1861
2144
  return { present: false, source: 'none' };
1862
2145
  }
1863
2146
 
2147
+ /**
2148
+ * `POST /messages/abort`: stop the session's active run. The run is
2149
+ * closed as `aborted` with the text streamed so far kept on its row;
2150
+ * a queued run is dropped. (A client that merely disconnects does NOT
2151
+ * reach here — runs outlive their connections.)
2152
+ */
1864
2153
  async abortTurn(sessionId: string): Promise<boolean> {
2154
+ if (this.runs?.abortSession(sessionId)) return true;
1865
2155
  const controller = this.aborts.get(sessionId);
1866
2156
  if (!controller) return false;
1867
2157
  controller.abort();
@@ -1904,8 +2194,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1904
2194
  toolCalls: [],
1905
2195
  replayed: true,
1906
2196
  };
1907
- case 'interrupted':
2197
+ case 'interrupted': {
2198
+ // The object reset mid-turn. The run may be recovering (durable
2199
+ // runs): attach to it and answer with its reply; only a run that
2200
+ // is over without a reply is refused.
2201
+ const attached = existing?.requestId
2202
+ ? await this.attachToRun(existing.requestId, req)
2203
+ : null;
2204
+ if (attached) return attached;
1908
2205
  throw new Error(`${TURN_INTERRUPTED_MARKER} (${eventId})`);
2206
+ }
1909
2207
  case 'run':
1910
2208
  break;
1911
2209
  }
@@ -1952,43 +2250,78 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1952
2250
  });
1953
2251
  card.emit(req.requestId, 'routing');
1954
2252
  }
1955
- const { agent, stateInput, config, abortController, turnDisposables } =
1956
- await this.prepareTurn(req, {
1957
- message: req.message,
1958
- timezone: req.identity.timezone,
1959
- model: req.model,
1960
- attachments: req.attachments,
1961
- });
1962
- try {
1963
- const result = (await agent.invoke(stateInput, config)) as {
1964
- messages?: BaseMessage[];
2253
+ // Every turn is a durable run (docs/plans/durable-runs.md): recorded
2254
+ // before the first model call, kept alive by the alarm, resumed after
2255
+ // a reset. Headless turns (Matrix, tasks) simply await the outcome.
2256
+ const { live } = await this.runs!.begin({
2257
+ runId: crypto.randomUUID(),
2258
+ sessionId: req.sessionId,
2259
+ requestId: req.requestId,
2260
+ client: req.client,
2261
+ request: JSON.stringify(
2262
+ storedRunRequest(req, { timezone: req.identity.timezone }),
2263
+ ),
2264
+ multitask: req.multitask ?? this.runConfig.multitaskDefault,
2265
+ ...(req.taskRunId ? { taskRunId: req.taskRunId } : {}),
2266
+ });
2267
+ const outcome = await live.done;
2268
+ if (outcome.status !== 'finished') {
2269
+ const reason =
2270
+ outcome.error instanceof Error
2271
+ ? outcome.error.message
2272
+ : outcome.error !== undefined
2273
+ ? String(outcome.error)
2274
+ : `turn ${outcome.status}`;
2275
+ throw new Error(reason);
2276
+ }
2277
+ card?.emit(req.requestId, 'delivering');
2278
+ return {
2279
+ sessionId: req.sessionId,
2280
+ requestId: req.requestId,
2281
+ text: outcome.text,
2282
+ ...(outcome.messageId !== undefined
2283
+ ? { messageId: outcome.messageId }
2284
+ : {}),
2285
+ toolCalls: outcome.toolCalls ?? [],
2286
+ };
2287
+ }
2288
+
2289
+ /**
2290
+ * A Matrix turn asked about again after this object reset: hand back the
2291
+ * reply of its (recovering or finished) run instead of refusing it.
2292
+ */
2293
+ private async attachToRun(
2294
+ requestId: string,
2295
+ req: TurnRequest,
2296
+ ): Promise<TurnResult | null> {
2297
+ const live = this.runs?.byRequestId(requestId);
2298
+ if (live) {
2299
+ console.log(
2300
+ `[user-do] turn ${requestId} for ${req.eventId} is being recovered; the gateway attaches to it`,
2301
+ );
2302
+ const outcome = await live.done;
2303
+ if (outcome.status !== 'finished') return null;
2304
+ return {
2305
+ sessionId: req.sessionId,
2306
+ requestId: req.requestId,
2307
+ text: outcome.text,
2308
+ ...(outcome.messageId ? { messageId: outcome.messageId } : {}),
2309
+ toolCalls: outcome.toolCalls ?? [],
2310
+ replayed: true,
1965
2311
  };
1966
- const messages = result.messages ?? [];
1967
- const text = lastAiText(messages);
1968
- await this.afterTurn(req.sessionId, messages);
1969
- await this.runTurnDisposables(turnDisposables);
1970
- card?.emit(req.requestId, 'delivering');
1971
- const toolCalls = messages
1972
- .filter((m): m is AIMessage => m.type === 'ai')
1973
- .flatMap((m) =>
1974
- (m.tool_calls ?? []).map((t) => ({
1975
- name: t.name,
1976
- status: 'done' as const,
1977
- })),
1978
- );
2312
+ }
2313
+ const record = await this.runStore?.getByRequestId(requestId);
2314
+ if (record?.status === 'finished') {
1979
2315
  return {
1980
2316
  sessionId: req.sessionId,
1981
2317
  requestId: req.requestId,
1982
- text,
1983
- ...(lastAiMessageId(messages) !== undefined
1984
- ? { messageId: lastAiMessageId(messages) }
1985
- : {}),
1986
- toolCalls,
2318
+ text: record.partialText ?? '',
2319
+ ...(record.messageId ? { messageId: record.messageId } : {}),
2320
+ toolCalls: [],
2321
+ replayed: true,
1987
2322
  };
1988
- } finally {
1989
- if (this.aborts.get(req.sessionId) === abortController)
1990
- this.aborts.delete(req.sessionId);
1991
2323
  }
2324
+ return null;
1992
2325
  }
1993
2326
 
1994
2327
  /**
@@ -2149,6 +2482,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2149
2482
  const line = `[user-do] owner-store flush failed (${failures} in a row) for ${this.userDid} — working copy kept dirty, retrying in ${FLUSH_RETRY_DELAY_MS / 60_000} min: ${detail}`;
2150
2483
  if (failures >= FLUSH_FAILURES_ERROR_THRESHOLD) console.error(line);
2151
2484
  else console.warn(line);
2485
+ await this.scheduleFlushRetry(Date.now() + FLUSH_RETRY_DELAY_MS);
2152
2486
  throw err;
2153
2487
  }
2154
2488
  await this.ctx.storage.put(META_OWNER_ETAG, saved.etag);
@@ -2228,18 +2562,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2228
2562
  }
2229
2563
 
2230
2564
  private async clearDirty(): Promise<void> {
2231
- // A write that landed while the snapshot was being uploaded already
2232
- // re-armed the alarm through `markDirty`; only clear when the file's
2233
- // generation still matches what was just recorded.
2565
+ // A write that landed while the snapshot was being uploaded keeps the
2566
+ // copy dirty; its upload is a full debounce away (the deadline the
2567
+ // write found on disk was the one that just ran).
2234
2568
  const uploadedGen = await this.ctx.storage.get<number>(META_UPLOADED_GEN);
2235
2569
  if (
2236
2570
  this.db &&
2237
2571
  uploadedGen !== undefined &&
2238
2572
  uploadedGen !== this.db.writeGeneration
2239
- )
2573
+ ) {
2574
+ const at = Date.now() + FLUSH_DEBOUNCE_MS;
2575
+ await this.ctx.storage.put(META_FLUSH_AT, at);
2576
+ this.requestAlarm(at);
2240
2577
  return;
2578
+ }
2241
2579
  this.dirty = false;
2242
- await this.ctx.storage.delete(META_DIRTY);
2580
+ await this.ctx.storage.delete([META_DIRTY, META_FLUSH_AT]);
2243
2581
  }
2244
2582
 
2245
2583
  async resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }> {
@@ -2261,7 +2599,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2261
2599
  this.saver = null;
2262
2600
  this.sessions = null;
2263
2601
  this.matrixLedger = null;
2602
+ this.runStore = null;
2603
+ this.runs = null;
2604
+ this.resultStore = null;
2264
2605
  this.sessionRooms.clear();
2606
+ this.mainRoomId = null;
2265
2607
  this.ambient = null;
2266
2608
  this.secretsService = null;
2267
2609
  this.byo = null;
@@ -2364,6 +2706,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2364
2706
  pages: Math.ceil(fileBytes / 4096),
2365
2707
  ownerEtag: await this.ctx.storage.get<string>(META_OWNER_ETAG),
2366
2708
  lastFlushAt: await this.ctx.storage.get<number>(META_LAST_FLUSH),
2709
+ nextFlushAt: await this.ctx.storage.get<number>(META_FLUSH_AT),
2367
2710
  dirty: this.dirty,
2368
2711
  flushInFlight: this.flushInFlight !== null,
2369
2712
  writeGeneration: db?.writeGeneration,
@@ -2574,6 +2917,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2574
2917
  console,
2575
2918
  );
2576
2919
  }
2920
+ const joinMatch = /^\/runs\/([^/]+)$/.exec(url.pathname);
2921
+ if (joinMatch && request.method === 'GET')
2922
+ return this.joinRun(request, decodeURIComponent(joinMatch[1]!), url);
2577
2923
  const match = /^\/turn\/([^/]+)$/.exec(url.pathname);
2578
2924
  if (!match || request.method !== 'POST')
2579
2925
  return new Response('Not found', { status: 404 });
@@ -2661,75 +3007,428 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2661
3007
  }
2662
3008
  }
2663
3009
 
3010
+ // A streamed turn is a durable run: recorded, kept alive and resumable
3011
+ // (docs/plans/durable-runs.md). The response is one subscriber of the
3012
+ // run's buffer; the run itself outlives it.
3013
+ this.replayToRoom(req, body.message, 'user');
3014
+ const agActionNames = (body.agActions ?? []).map((a) => a.name);
3015
+ const { live, queued } = await this.runs!.begin({
3016
+ runId: crypto.randomUUID(),
3017
+ sessionId,
3018
+ requestId,
3019
+ client: 'portal',
3020
+ request: JSON.stringify(
3021
+ storedRunRequest(req, {
3022
+ timezone: body.timezone,
3023
+ agActionNames,
3024
+ stream: true,
3025
+ }),
3026
+ ),
3027
+ multitask: body.multitask ?? this.runConfig.multitaskDefault,
3028
+ });
3029
+ if (queued)
3030
+ live.buffer.push('router.update', {
3031
+ step: 'Queued…',
3032
+ sessionId,
3033
+ requestId,
3034
+ runId: live.runId,
3035
+ queued: true,
3036
+ });
3037
+ const responseStream = createSseSubscriberStream({
3038
+ replay: live.buffer.tailAfter(0),
3039
+ buffer: live.buffer,
3040
+ onCancel: () =>
3041
+ console.log(
3042
+ `[user-do] client left run ${live.runId} (${sessionId}); the run continues`,
3043
+ ),
3044
+ });
3045
+ return new Response(responseStream, {
3046
+ headers: {
3047
+ ...SSE_HEADERS,
3048
+ 'x-request-id': requestId,
3049
+ 'x-run-id': live.runId,
3050
+ },
3051
+ });
3052
+ }
3053
+
3054
+ /**
3055
+ * `GET /runs/:id?after=<seq>` — re-join a run: replay the frames after
3056
+ * the cursor (packed segments, then the live tail) and stay attached
3057
+ * until its `done` frame. A run that already ended replays what is left
3058
+ * and closes with a synthetic `done` carrying its final state.
3059
+ */
3060
+ private async joinRun(
3061
+ request: Request,
3062
+ runId: string,
3063
+ url: URL,
3064
+ ): Promise<Response> {
3065
+ const identity = JSON.parse(
3066
+ request.headers.get('x-identity') ?? '{}',
3067
+ ) as TurnIdentity;
3068
+ try {
3069
+ await this.ready(identity);
3070
+ } catch (err) {
3071
+ return readyFailureResponse(err);
3072
+ }
3073
+ const afterRaw = Number(url.searchParams.get('after') ?? '0');
3074
+ const after =
3075
+ Number.isFinite(afterRaw) && afterRaw > 0 ? Math.floor(afterRaw) : 0;
3076
+ const joined = await this.runs!.join(runId, after);
3077
+ if (!joined)
3078
+ return Response.json(
3079
+ { statusCode: 404, message: `Run ${runId} not found` },
3080
+ { status: 404 },
3081
+ );
3082
+ const { record, replay, buffer } = joined;
3083
+ const ended = !buffer || buffer.isClosed;
3084
+ const stream = createSseSubscriberStream({
3085
+ replay,
3086
+ ...(buffer && !buffer.isClosed ? { buffer } : {}),
3087
+ ...(ended
3088
+ ? {
3089
+ trailer: {
3090
+ event: 'done',
3091
+ data: {
3092
+ runId,
3093
+ status: record.status,
3094
+ ...(record.messageId ? { messageId: record.messageId } : {}),
3095
+ ...(record.partialText && record.status !== 'finished'
3096
+ ? { partialText: record.partialText }
3097
+ : {}),
3098
+ replayed: true,
3099
+ },
3100
+ },
3101
+ }
3102
+ : {}),
3103
+ });
3104
+ return new Response(stream, {
3105
+ headers: {
3106
+ ...SSE_HEADERS,
3107
+ 'x-request-id': record.requestId,
3108
+ 'x-run-id': runId,
3109
+ 'x-run-status': record.status,
3110
+ },
3111
+ });
3112
+ }
3113
+
3114
+ /**
3115
+ * One attempt of a run (the coordinator's `runAttempt` host hook): build
3116
+ * the agent for the stored request, stream the turn's frames into the
3117
+ * run's buffer, and report the outcome. `resumed` attempts hand the
3118
+ * graph no new input, so LangGraph continues from the checkpoint's
3119
+ * pending node; the tool-mark middleware keeps that safe.
3120
+ */
3121
+ private async runAttempt(
3122
+ live: LiveRun,
3123
+ resumed: boolean,
3124
+ ): Promise<RunOutcome> {
3125
+ const stored = JSON.parse(live.record.request) as StoredRunRequest;
3126
+ const req = stored.turn;
2664
3127
  const {
2665
3128
  agent,
2666
3129
  stateInput,
2667
3130
  config,
2668
- abortController,
3131
+ turnDisposables,
2669
3132
  byoNotice,
2670
3133
  byoProvider,
2671
- turnDisposables,
2672
- } = await this.prepareTurn(req, body);
2673
- this.replayToRoom(req, body.message, 'user');
2674
- const agActionNames = new Set((body.agActions ?? []).map((a) => a.name));
3134
+ toolOutputCapChars,
3135
+ } = await this.prepareTurn(
3136
+ req,
3137
+ {
3138
+ message: req.message,
3139
+ timezone: stored.timezone ?? req.identity.timezone,
3140
+ model: req.model,
3141
+ ...(resumed ? {} : { attachments: req.attachments }),
3142
+ },
3143
+ {
3144
+ runId: live.runId,
3145
+ abortController: live.abort,
3146
+ resumed,
3147
+ continuation: resumed ? live.continuation : null,
3148
+ },
3149
+ );
3150
+ const sessionId = req.sessionId;
2675
3151
  const sink = {
2676
3152
  emit: (eventName: string, payload: Record<string, unknown>) => {
2677
- void payload;
2678
- void eventName;
3153
+ if (live.buffer.isClosed) return;
3154
+ live.buffer.push(eventName, payload);
2679
3155
  },
2680
3156
  };
2681
- let write: ((name: string, payload: unknown) => void) | null = null;
2682
- sink.emit = (eventName, payload) => write?.(eventName, payload);
2683
3157
  this.events.register(sessionId, sink);
2684
- const events = agent.streamEvents(stateInput, {
3158
+ if (byoNotice)
3159
+ sink.emit('error', {
3160
+ ...byoNotice,
3161
+ sessionId,
3162
+ requestId: req.requestId,
3163
+ });
3164
+ const events = agent.streamEvents(resumed ? null : stateInput, {
2685
3165
  ...config,
2686
3166
  version: 'v2',
2687
3167
  });
2688
3168
  const capture: BaseMessage[] = [];
2689
- const sse = createSseTurnStream({
2690
- events: tapMessages(events, capture),
2691
- sessionId,
2692
- requestId,
2693
- abortController,
2694
- mirror: (eventName, payload) =>
2695
- this.events.emitToTaps(eventName, payload),
2696
- agActionNames,
2697
- byoProvider,
2698
- onComplete: async () => {
2699
- this.replayToRoom(req, lastAiText(capture), 'oracle');
2700
- await this.afterTurn(sessionId, capture);
2701
- await this.runTurnDisposables(turnDisposables);
2702
- },
2703
- onError: (err) => {
2704
- console.error(`[user-do] turn ${requestId} failed:`, err);
2705
- void this.runTurnDisposables(turnDisposables);
2706
- },
2707
- log: (m) => console.log(`[user-do] ${m}`),
2708
- });
2709
- // Let plugin-emitted events (ctx.emit.*) ride the same SSE stream.
2710
- const [clientBranch, emitterBranch] = sse.tee();
2711
- void emitterBranch.cancel();
2712
- write = null;
2713
- const merged = mergeWithEmitter(clientBranch, (register) => {
2714
- write = register;
2715
- });
2716
- // A BYO turn that degraded to the platform model tells the user so —
2717
- // the Node runtime emits the same notice on the SSE `error` channel.
2718
- if (byoNotice) sink.emit('error', { ...byoNotice, sessionId, requestId });
2719
- const cleanup = () => {
3169
+ let completed = false;
3170
+ try {
3171
+ const outcome = await runTurnFrames({
3172
+ events: tapMessages(events, capture),
3173
+ sink: live.buffer,
3174
+ sessionId,
3175
+ requestId: req.requestId,
3176
+ runId: live.runId,
3177
+ resumed,
3178
+ abortController: live.abort,
3179
+ mirror: (eventName, payload) =>
3180
+ this.events.emitToTaps(eventName, payload),
3181
+ agActionNames: new Set(stored.agActionNames ?? []),
3182
+ byoProvider,
3183
+ toolOutputCapChars,
3184
+ messageIdOf: () => lastAiMessageId(capture),
3185
+ onFrame: () => this.runs?.touchKeepAlive(),
3186
+ onComplete: async () => {
3187
+ completed = true;
3188
+ this.replayToRoom(req, lastAiText(capture), 'oracle');
3189
+ await this.afterTurn(sessionId, capture);
3190
+ },
3191
+ onError: (err) => {
3192
+ console.error(`[user-do] turn ${req.requestId} failed:`, err);
3193
+ },
3194
+ log: (m) => console.log(`[user-do] ${m}`),
3195
+ });
3196
+ const toolCalls = capture
3197
+ .filter((m): m is AIMessage => m.type === 'ai')
3198
+ .flatMap((m) =>
3199
+ (m.tool_calls ?? []).map((t) => ({
3200
+ name: t.name,
3201
+ status: 'done' as const,
3202
+ })),
3203
+ );
3204
+ const text = completed ? lastAiText(capture) : outcome.fullText;
3205
+ const messageId = completed ? lastAiMessageId(capture) : undefined;
3206
+ return {
3207
+ status:
3208
+ outcome.status === 'completed'
3209
+ ? 'finished'
3210
+ : outcome.status === 'aborted'
3211
+ ? 'aborted'
3212
+ : 'failed',
3213
+ text: (live.continuation ?? '') + text,
3214
+ ...(messageId ? { messageId } : {}),
3215
+ toolCalls,
3216
+ ...(outcome.status === 'failed' ? { error: outcome.error } : {}),
3217
+ };
3218
+ } finally {
2720
3219
  this.events.unregister(sessionId, sink);
2721
- if (this.aborts.get(sessionId) === abortController)
3220
+ await this.runTurnDisposables(turnDisposables);
3221
+ if (this.aborts.get(sessionId) === live.abort)
2722
3222
  this.aborts.delete(sessionId);
3223
+ }
3224
+ }
3225
+
3226
+ /** Latest checkpoint id of a session — recovery's progress marker. */
3227
+ private async checkpointIdOf(sessionId: string): Promise<string | null> {
3228
+ const tuple = await this.saver
3229
+ ?.getTupleWithoutMessages({ configurable: { thread_id: sessionId } })
3230
+ .catch(() => undefined);
3231
+ return tuple?.checkpoint.id ?? null;
3232
+ }
3233
+
3234
+ /**
3235
+ * A run reached a terminal state. A recovered task run has no scheduler
3236
+ * continuation waiting on it any more: hand the result (or the failure)
3237
+ * to the scheduler so it is delivered / recorded as it would have been.
3238
+ */
3239
+ private async onRunEnded(
3240
+ record: RunRecord,
3241
+ outcome: RunOutcome,
3242
+ ): Promise<void> {
3243
+ if (!record.taskRunId || !this.taskScheduler) return;
3244
+ if (this.taskScheduler.isRunActive(record.taskRunId)) return;
3245
+ if (outcome.status === 'finished')
3246
+ await this.taskScheduler.completeRecoveredRun(
3247
+ record.taskRunId,
3248
+ outcome.text,
3249
+ );
3250
+ else
3251
+ await this.taskScheduler.failRecoveredRun(
3252
+ record.taskRunId,
3253
+ outcome.status === 'interrupted'
3254
+ ? 'interrupted: the object was reset while the run was in progress and recovery did not complete'
3255
+ : `${outcome.status}: ${outcome.error instanceof Error ? outcome.error.message : String(outcome.error ?? outcome.status)}`,
3256
+ );
3257
+ }
3258
+
3259
+ /** `GET /sessions/:id/run` — the session's active run, if any. */
3260
+ async sessionRun(
3261
+ identity: TurnIdentity,
3262
+ sessionId: string,
3263
+ ): Promise<RunSummary | null> {
3264
+ await this.ready(identity);
3265
+ const live = this.runs?.activeForSession(sessionId);
3266
+ const record =
3267
+ live?.record ?? (await this.runStore?.activeForSession(sessionId));
3268
+ return record ? runSummaryOf(record) : null;
3269
+ }
3270
+
3271
+ /** `GET /debug/context?model=` — the context budget a model gets here. */
3272
+ async contextStatus(
3273
+ identity: TurnIdentity,
3274
+ model?: string,
3275
+ byoProvider?: string,
3276
+ sessionId?: string,
3277
+ ): Promise<ContextStatus> {
3278
+ await this.ready(identity);
3279
+ const llm = this.ambient?.llm;
3280
+ const modelId =
3281
+ model ??
3282
+ (llm && isProviderAdapter(llm)
3283
+ ? llm.modelForRole('main')
3284
+ : DEFAULT_MODEL_ID);
3285
+ const window = await this.contextWindows.resolve(modelId, {
3286
+ ...(byoProvider ? { byoProvider } : {}),
3287
+ });
3288
+ const budget = contextBudgetFor(window, contextKnobs(this.env, console));
3289
+ const stats = (await this.resultStore?.stats()) ?? {
3290
+ rows: 0,
3291
+ sqliteBytes: 0,
3292
+ r2Rows: 0,
3293
+ r2Bytes: 0,
2723
3294
  };
2724
- const finalStream = merged.pipeThrough(
2725
- new TransformStream<Uint8Array, Uint8Array>({
2726
- transform: (chunk, controller) => controller.enqueue(chunk),
2727
- flush: cleanup,
2728
- }),
2729
- );
2730
- return new Response(finalStream, {
2731
- headers: { ...SSE_HEADERS, 'x-request-id': requestId },
3295
+ return {
3296
+ model: modelId,
3297
+ ...(byoProvider ? { byoProvider } : {}),
3298
+ window: {
3299
+ tokens: window.tokens,
3300
+ origin: window.origin,
3301
+ ...(window.catalogId ? { catalogId: window.catalogId } : {}),
3302
+ },
3303
+ budget: {
3304
+ summarizeAtTokens: budget.summarizeAtTokens,
3305
+ pruneAtTokens: budget.pruneAtTokens,
3306
+ resultCapChars: budget.resultCapChars,
3307
+ requestCapTokens: budget.requestCapTokens,
3308
+ outputReserveTokens: budget.outputReserveTokens,
3309
+ keepMessages: budget.keepMessages,
3310
+ ...(budget.summarizeTriggerMessages !== undefined
3311
+ ? { summarizeTriggerMessages: budget.summarizeTriggerMessages }
3312
+ : {}),
3313
+ },
3314
+ results: {
3315
+ ...stats,
3316
+ tier: this.env.TIER_BUCKET ? 'sqlite+r2' : 'sqlite',
3317
+ },
3318
+ ...(sessionId
3319
+ ? { session: await this.sessionContextStatus(sessionId) }
3320
+ : {}),
3321
+ };
3322
+ }
3323
+
3324
+ /**
3325
+ * One session's working context plus the guard's counters: what the
3326
+ * next request starts from (the messages the latest checkpoint carries,
3327
+ * a summary plus the kept tail once the history was condensed — the
3328
+ * transcript rows are never condensed and are listed separately) and
3329
+ * what earlier requests needed (pruning, a provider overflow, a
3330
+ * refusal).
3331
+ */
3332
+ private async sessionContextStatus(
3333
+ sessionId: string,
3334
+ ): Promise<SessionContextStatus | null> {
3335
+ if (!(await this.sessions!.getSession(sessionId))) return null;
3336
+ const tuple = await this.saver!.getTuple({
3337
+ configurable: { thread_id: sessionId },
2732
3338
  });
3339
+ const channel: unknown = tuple?.checkpoint.channel_values.messages;
3340
+ const context: BaseMessage[] = Array.isArray(channel)
3341
+ ? channel.filter(isBaseMessage)
3342
+ : [];
3343
+ const thread = await this.saver!.listThreadMessages(sessionId);
3344
+ const stats = await this.ctx.storage.get<SessionContextCounters>(
3345
+ contextStatsKey(sessionId),
3346
+ );
3347
+ return {
3348
+ id: sessionId,
3349
+ threadMessages: thread.length,
3350
+ contextMessages: context.length,
3351
+ contextSummaries: context.filter(isSummarizationMessage).length,
3352
+ contextToolMessages: context.filter((m) => m.type === 'tool').length,
3353
+ contextTokens: estimateRequestTokens({
3354
+ systemTokens: 0,
3355
+ schemaTokens: 0,
3356
+ messages: context,
3357
+ }),
3358
+ ...(stats ?? EMPTY_CONTEXT_COUNTERS),
3359
+ };
3360
+ }
3361
+
3362
+ /** Fold a guard event into the session's stored counters (ordered, never throws). */
3363
+ private recordContextEvent(
3364
+ sessionId: string,
3365
+ event: ContextGuardEvent,
3366
+ ): void {
3367
+ const key = contextStatsKey(sessionId);
3368
+ this.contextStatsWrites = this.contextStatsWrites
3369
+ .then(async () => {
3370
+ const prev =
3371
+ (await this.ctx.storage.get<SessionContextCounters>(key)) ??
3372
+ EMPTY_CONTEXT_COUNTERS;
3373
+ const next: SessionContextCounters = {
3374
+ ...prev,
3375
+ lastEventAt: new Date().toISOString(),
3376
+ };
3377
+ switch (event.kind) {
3378
+ case 'prune':
3379
+ next.prunes += 1;
3380
+ if (event.stage !== 'soft') next.hardPrunes += 1;
3381
+ next.prunedResults += event.pruned;
3382
+ break;
3383
+ case 'overflow-retry':
3384
+ next.overflowRetries += 1;
3385
+ break;
3386
+ case 'refused':
3387
+ next.refusals += 1;
3388
+ break;
3389
+ }
3390
+ await this.ctx.storage.put(key, next);
3391
+ })
3392
+ .catch((err: unknown) => {
3393
+ console.warn(
3394
+ `[context] could not record a ${event.kind} event for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
3395
+ );
3396
+ });
3397
+ }
3398
+
3399
+ /** `GET /debug/runs` — recent runs with their marks and live state. */
3400
+ async runsStatus(): Promise<RunsStatus> {
3401
+ const userDid =
3402
+ this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
3403
+ if (!this.db && userDid) await this.ready({ userDid });
3404
+ const store = this.runStore;
3405
+ const records = store ? await store.listRecent(50) : [];
3406
+ const runs: RunsStatus['runs'] = [];
3407
+ for (const record of records) {
3408
+ const marks = store ? await store.listMarks(record.runId) : [];
3409
+ runs.push({
3410
+ ...runSummaryOf(record),
3411
+ marks: marks.map((m) => ({
3412
+ toolCallId: m.toolCallId,
3413
+ toolName: m.toolName,
3414
+ effect: m.effect,
3415
+ startedAt: m.startedAt,
3416
+ doneAt: m.doneAt,
3417
+ outcome: m.outcome,
3418
+ attempts: m.attempts,
3419
+ })),
3420
+ segments: store ? await store.countSegments(record.runId) : 0,
3421
+ attempts: record.attempts,
3422
+ nextAttemptAt: record.nextAttemptAt,
3423
+ error: record.error,
3424
+ taskRunId: record.taskRunId,
3425
+ });
3426
+ }
3427
+ return {
3428
+ config: { ...this.runConfig },
3429
+ live: this.runs?.snapshot() ?? [],
3430
+ runs,
3431
+ };
2733
3432
  }
2734
3433
 
2735
3434
  // ── turn plumbing ──────────────────────────────────────────────────────
@@ -2930,6 +3629,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
2930
3629
  TurnBody,
2931
3630
  'message' | 'timezone' | 'model' | 'tools' | 'agActions' | 'attachments'
2932
3631
  >,
3632
+ run: {
3633
+ runId: string;
3634
+ abortController: AbortController;
3635
+ /** A recovery attempt: no new input, the graph continues from the checkpoint. */
3636
+ resumed: boolean;
3637
+ /** The reply text the user already received (resumed attempts). */
3638
+ continuation: string | null;
3639
+ },
2933
3640
  ) {
2934
3641
  const core = this.core;
2935
3642
  const baseAmbient = this.ambient!;
@@ -3004,6 +3711,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3004
3711
  }
3005
3712
  : baseAmbient;
3006
3713
  const effectiveModel = byoTurn ? byoTurn.byoModelId : requestedModel;
3714
+ // The turn's context budget, from the main model's window: the model
3715
+ // id the turn will actually run on (platform default, a per-request
3716
+ // choice, or the BYO credential's model) — see docs/plans/context-budgets.md.
3717
+ const mainModelId = byoTurn
3718
+ ? byoTurn.mainModelId
3719
+ : (requestedModel ??
3720
+ (isProviderAdapter(baseAmbient.llm)
3721
+ ? baseAmbient.llm.modelForRole('main')
3722
+ : DEFAULT_MODEL_ID));
3723
+ const windowResolution = await this.contextWindows.resolve(mainModelId, {
3724
+ ...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
3725
+ });
3726
+ const contextBudget = contextBudgetFor(
3727
+ windowResolution,
3728
+ contextKnobs(this.env, console),
3729
+ );
3007
3730
  if (
3008
3731
  req.client === 'matrix' &&
3009
3732
  !(await sessions.getSession(req.sessionId))
@@ -3016,8 +3739,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3016
3739
  oracleEntityDid: core.identity.entityDid,
3017
3740
  });
3018
3741
  }
3019
- this.aborts.get(req.sessionId)?.abort();
3020
- const abortController = new AbortController();
3742
+ // The run coordinator already applied the session's multitask rule
3743
+ // (aborted or queued behind the previous run); this attempt's signal
3744
+ // is the run's own.
3745
+ const abortController = run.abortController;
3021
3746
  this.aborts.set(req.sessionId, abortController);
3022
3747
 
3023
3748
  // Every turn runs inside the user's oracle room, like the Node
@@ -3133,12 +3858,55 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3133
3858
  const hostRoomTitle = opts.hooks?.getRoomTitle;
3134
3859
  const hostSafetyModel = opts.hooks?.safetyModel;
3135
3860
 
3136
- const { agent, context } = await createMainAgent({
3861
+ // A resumed attempt reads the client's tool surface back from the
3862
+ // checkpointed state (the run row does not store the catalogue).
3863
+ const priorSurface = priorState as {
3864
+ browserTools?: TurnBody['tools'];
3865
+ agActions?: TurnBody['agActions'];
3866
+ };
3867
+ const browserTools =
3868
+ body.tools ??
3869
+ (run.resumed ? priorSurface.browserTools : undefined) ??
3870
+ [];
3871
+ const agActions =
3872
+ body.agActions ??
3873
+ (run.resumed ? priorSurface.agActions : undefined) ??
3874
+ [];
3875
+
3876
+ // Write-ahead tool marks + the resume policy (tool-marks.ts). The
3877
+ // effect map is filled from the build below; the closure reads it at
3878
+ // call time, so the middleware can be created before the agent.
3879
+ const toolEffects = new Map<string, ToolEffect>();
3880
+ const marksMiddleware = createToolMarksMiddleware({
3881
+ runId: run.runId,
3882
+ store: this.runStore!,
3883
+ effectOf: (name) => toolEffects.get(name) ?? toolEffectOf({ name }),
3884
+ continuation: run.continuation,
3885
+ logger: console,
3886
+ });
3887
+ // Every tool result above the budget's cap is saved whole and handed
3888
+ // to the model as head + tail + handle (result-cap.ts); applied to
3889
+ // sub-agents too through `toolMiddlewares`.
3890
+ const resultStore = this.resultStore;
3891
+ const resultCap = {
3892
+ capChars: contextBudget.resultCapChars,
3893
+ sessionId: req.sessionId,
3894
+ ...(resultStore ? { store: resultStore } : {}),
3895
+ exempt: new Set([READ_RESULT_TOOL_NAME]),
3896
+ logger: console,
3897
+ };
3898
+ // The middleware covers what the wrapper cannot: a sub-agent's reply,
3899
+ // which is a tool result to the main agent. Wrapped tools are capped
3900
+ // inside the wrapper (already under the cap by the time it runs).
3901
+ const capMiddleware = createResultCapMiddleware(resultCap);
3902
+
3903
+ const built = await createMainAgent({
3137
3904
  registries: core.registries,
3138
3905
  identity: core.identity,
3139
3906
  config: core.validatedEnv,
3140
3907
  availablePlugins: core.availablePlugins,
3141
3908
  byoProvider: byoTurn?.provider,
3909
+ contextBudget,
3142
3910
  ambient: {
3143
3911
  ...ambient,
3144
3912
  attachments: attachmentAccess,
@@ -3150,6 +3918,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3150
3918
  middlewares: this.workStatus
3151
3919
  ? [createWorkStatusMiddleware({ producer: this.workStatus })]
3152
3920
  : [],
3921
+ // Applied to the main agent AND every sub-agent: a sub-agent's
3922
+ // own tool calls are marked and capped too.
3923
+ toolMiddlewares: [marksMiddleware, capMiddleware],
3924
+ resultCap,
3925
+ onContextOverflow: (error) =>
3926
+ this.contextWindows.learnFromError(mainModelId, error, {
3927
+ ...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
3928
+ }),
3929
+ onContextEvent: (event) =>
3930
+ this.recordContextEvent(req.sessionId, event),
3931
+ ...(resultStore
3932
+ ? {
3933
+ readResult: (id: string, offset: number, length: number) =>
3934
+ resultStore.read(id, offset, length),
3935
+ }
3936
+ : {}),
3153
3937
  ...(hostRoomTitle
3154
3938
  ? {
3155
3939
  getRoomTitle: (roomId: string) =>
@@ -3185,25 +3969,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3185
3969
  // AG-UI plugins turn these into tools at build time
3186
3970
  // (`getRequestTools` reads `history.state`), so they must be here
3187
3971
  // and not only in the graph input.
3188
- browserTools: body.tools ?? [],
3189
- agActions: body.agActions ?? [],
3972
+ browserTools,
3973
+ agActions,
3190
3974
  },
3191
3975
  checkpointer: saver,
3192
3976
  abortSignal: abortController.signal,
3193
3977
  });
3978
+ const { agent, context } = built;
3979
+ for (const [name, effect] of built.toolEffects)
3980
+ toolEffects.set(name, effect);
3194
3981
 
3195
3982
  const attachmentKwargs =
3196
3983
  prepared && prepared.metas.length > 0
3197
3984
  ? { attachment: prepared.metas[0], attachments: prepared.metas }
3198
3985
  : {};
3986
+ // Node's `MessagesService` metadata for room messages: who spoke
3987
+ // (the group-chat tools and prompts read it), and in a group room the
3988
+ // `[DisplayName]: ` prefix on the message itself.
3989
+ const speakerName =
3990
+ req.senderDisplayName ??
3991
+ req.identity.matrixUserId ??
3992
+ req.identity.userDid;
3993
+ const speakerKwargs =
3994
+ req.client === 'matrix'
3995
+ ? {
3996
+ senderDid: req.identity.userDid,
3997
+ ...(req.identity.matrixUserId
3998
+ ? { senderMatrixUserId: req.identity.matrixUserId }
3999
+ : {}),
4000
+ senderDisplayName: speakerName,
4001
+ threadId: req.sessionId,
4002
+ ...(req.eventId ? { eventId: req.eventId } : {}),
4003
+ }
4004
+ : {};
4005
+ const rawContent = prepared ? prepared.content : body.message;
4006
+ const content =
4007
+ req.client === 'matrix' && req.roomKind === 'group'
4008
+ ? prefixSpeaker(rawContent, speakerName)
4009
+ : rawContent;
3199
4010
  const stateInput = {
3200
4011
  messages: [
3201
4012
  new HumanMessage({
3202
- content: prepared ? prepared.content : body.message,
4013
+ content,
3203
4014
  additional_kwargs: {
3204
4015
  timestamp: new Date().toISOString(),
3205
4016
  oracleName: core.identity.name,
3206
4017
  msgFromMatrixRoom: req.client === 'matrix',
4018
+ ...speakerKwargs,
3207
4019
  ...attachmentKwargs,
3208
4020
  },
3209
4021
  }),
@@ -3224,8 +4036,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3224
4036
  config: { did: req.identity.userDid },
3225
4037
  client: req.client,
3226
4038
  ...metadataGraphInput(meta, priorMeta),
3227
- browserTools: body.tools ?? [],
3228
- agActions: body.agActions ?? [],
4039
+ browserTools,
4040
+ agActions,
3229
4041
  };
3230
4042
  // LangSmith: metadata is attached unconditionally (inert without a
3231
4043
  // tracer); the explicit tracer only when this turn is traced (global
@@ -3252,6 +4064,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3252
4064
  byoNotice,
3253
4065
  byoProvider: byoTurn?.provider,
3254
4066
  turnDisposables,
4067
+ toolOutputCapChars: contextBudget.resultCapChars,
3255
4068
  };
3256
4069
  }
3257
4070
 
@@ -3358,6 +4171,26 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
3358
4171
  }
3359
4172
  }
3360
4173
 
4174
+ /**
4175
+ * Channel-memory compaction for the gateway's group rooms
4176
+ * (`src/matrix/group-chat.ts`): the platform's small model distils a
4177
+ * batch of room messages with the Node summarizer's prompt. Needs no
4178
+ * user file — only this object's model adapter.
4179
+ */
4180
+ async summarizeGroupMessages(messages: JsonString): Promise<string | null> {
4181
+ const parsed: unknown = JSON.parse(messages);
4182
+ if (!Array.isArray(parsed) || parsed.length === 0) return null;
4183
+ const model = (this.ambient?.llm ?? this.core.llm).get('session-title', {
4184
+ temperature: 0.2,
4185
+ maxTokens: 800,
4186
+ });
4187
+ return summarizeObservedMessages(
4188
+ model,
4189
+ parsed as ObservedMessage[],
4190
+ console,
4191
+ );
4192
+ }
4193
+
3361
4194
  private async generateTitle(
3362
4195
  messages: BaseMessage[],
3363
4196
  ): Promise<string | null> {
@@ -3428,50 +4261,3 @@ async function* tapMessages(
3428
4261
  yield evt;
3429
4262
  }
3430
4263
  }
3431
-
3432
- /**
3433
- * Merge the turn's SSE stream with plugin-emitted events. Plugin events are
3434
- * written straight into the same byte stream as they happen.
3435
- */
3436
- function mergeWithEmitter(
3437
- base: ReadableStream<Uint8Array>,
3438
- onRegister: (write: (eventName: string, payload: unknown) => void) => void,
3439
- ): ReadableStream<Uint8Array> {
3440
- const encoder = new TextEncoder();
3441
- let closed = false;
3442
- return new ReadableStream<Uint8Array>({
3443
- start(controller) {
3444
- onRegister((eventName, payload) => {
3445
- if (closed) return;
3446
- try {
3447
- controller.enqueue(encoder.encode(formatSSE(eventName, payload)));
3448
- } catch {
3449
- closed = true;
3450
- }
3451
- });
3452
- const reader = base.getReader();
3453
- void (async () => {
3454
- try {
3455
- for (;;) {
3456
- const { done, value } = await reader.read();
3457
- if (done) break;
3458
- if (!closed) controller.enqueue(value);
3459
- }
3460
- } catch (err) {
3461
- if (!closed) controller.error(err);
3462
- } finally {
3463
- closed = true;
3464
- try {
3465
- controller.close();
3466
- } catch {
3467
- /* already closed */
3468
- }
3469
- }
3470
- })();
3471
- },
3472
- cancel() {
3473
- closed = true;
3474
- void base.cancel();
3475
- },
3476
- });
3477
- }