@ixo/oracle-runtime-workers 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/package.json +1 -1
- package/src/core/context-budget.ts +201 -0
- package/src/core/context-window.ts +289 -0
- package/src/core/env.ts +29 -0
- package/src/core/main-agent-types.ts +46 -0
- package/src/core/main-agent.ts +86 -9
- package/src/core/middlewares/context-guard.ts +270 -0
- package/src/core/middlewares/result-cap.ts +180 -0
- package/src/core/middlewares/summarization.ts +87 -6
- package/src/core/middlewares/tool-marks.ts +209 -0
- package/src/core/openrouter-pricing.ts +42 -1
- package/src/core/read-result-tool.ts +77 -0
- package/src/core/runtime-context.ts +7 -0
- package/src/core/sub-agent-fallback.ts +20 -3
- package/src/core/wrap-plugin-tool.ts +24 -3
- package/src/do/ambient.ts +17 -0
- package/src/do/contracts.ts +229 -0
- package/src/do/flush-schedule.ts +40 -0
- package/src/do/result-store-test-do.ts +88 -0
- package/src/do/result-store.ts +339 -0
- package/src/do/run-buffer.ts +204 -0
- package/src/do/run-coordinator.ts +572 -0
- package/src/do/run-request.ts +52 -0
- package/src/do/run-store-test-do.ts +130 -0
- package/src/do/run-store.ts +604 -0
- package/src/do/session-id-migration.ts +85 -0
- package/src/do/speaker-prefix.ts +37 -0
- package/src/do/sse-stream.ts +567 -400
- package/src/do/transcript.ts +238 -0
- package/src/do/turn-body.ts +10 -0
- package/src/do/user-oracle-do.ts +948 -162
- package/src/matrix/gateway-do.ts +230 -29
- package/src/matrix/group-chat-summarizer.ts +107 -0
- package/src/matrix/group-chat.ts +1163 -0
- package/src/matrix/inbox-store.ts +23 -3
- package/src/matrix/ingest.ts +54 -11
- package/src/matrix/reply-chain.ts +34 -25
- package/src/plugin-api/tool-helper.ts +9 -1
- package/src/plugin-api/types.ts +86 -0
- package/src/plugins/index.ts +3 -0
- package/src/plugins/matrix-group-chats/README.md +70 -0
- package/src/plugins/matrix-group-chats/index.ts +132 -0
- package/src/plugins/matrix-group-chats/tools.ts +172 -0
- package/src/plugins/tasks/tasks-tools.ts +4 -0
- package/src/shell/app.ts +66 -0
- package/src/sqlite/sqlite-saver.ts +81 -0
- package/src/tasks/scheduler.ts +134 -3
- package/src/tasks/store.ts +22 -0
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -20,6 +20,7 @@ import { DurableObject } from 'cloudflare:workers';
|
|
|
20
20
|
import {
|
|
21
21
|
AIMessage,
|
|
22
22
|
HumanMessage,
|
|
23
|
+
isBaseMessage,
|
|
23
24
|
type BaseMessage,
|
|
24
25
|
} from '@langchain/core/messages';
|
|
25
26
|
import {
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
llmEnvFromWorkerEnv,
|
|
51
52
|
resolveLangsmithTracing,
|
|
52
53
|
type OpenRouterLlmAdapter,
|
|
54
|
+
DEFAULT_MODEL_ID,
|
|
53
55
|
} from '../core/llm';
|
|
54
56
|
import { createMainAgent } from '../core/main-agent';
|
|
55
57
|
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
|
@@ -140,16 +142,28 @@ import {
|
|
|
140
142
|
type OracleWorkerEnv,
|
|
141
143
|
type SessionSummary,
|
|
142
144
|
type StorageStatus,
|
|
145
|
+
type RunsStatus,
|
|
146
|
+
type ContextStatus,
|
|
147
|
+
type SessionContextStatus,
|
|
148
|
+
type RunSummary,
|
|
143
149
|
type TurnIdentity,
|
|
144
150
|
type TurnRequest,
|
|
145
151
|
type TurnResult,
|
|
146
152
|
type MemorySchemaDebug,
|
|
147
153
|
TURN_INTERRUPTED_MARKER,
|
|
154
|
+
type JsonString,
|
|
148
155
|
} from './contracts';
|
|
149
156
|
import { turnRecursionLimit } from './turn-config';
|
|
150
157
|
import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
|
|
151
158
|
import { decideBootDirty } from './boot-dirty';
|
|
159
|
+
import { decideFlush } from './flush-schedule';
|
|
152
160
|
import { mirrorTxnId, RoomMirror } from './room-mirror';
|
|
161
|
+
import { prefixSpeaker } from './speaker-prefix';
|
|
162
|
+
import {
|
|
163
|
+
type ObservedMessage,
|
|
164
|
+
summarizeObservedMessages,
|
|
165
|
+
} from '../matrix/group-chat-summarizer';
|
|
166
|
+
import { migrateThreadSessionIds } from './session-id-migration';
|
|
153
167
|
import { ReauthPrompter } from './reauth-prompt';
|
|
154
168
|
import {
|
|
155
169
|
createTaskScheduler,
|
|
@@ -167,8 +181,46 @@ import {
|
|
|
167
181
|
type HistoryMessage,
|
|
168
182
|
type IndexableSession,
|
|
169
183
|
} from '../memory/session-history-indexer';
|
|
170
|
-
import {
|
|
184
|
+
import {
|
|
185
|
+
createSseSubscriberStream,
|
|
186
|
+
runTurnFrames,
|
|
187
|
+
SSE_HEADERS,
|
|
188
|
+
} from './sse-stream';
|
|
171
189
|
import { parseTurnBody, type TurnBody } from './turn-body';
|
|
190
|
+
import {
|
|
191
|
+
RunCoordinator,
|
|
192
|
+
type LiveRun,
|
|
193
|
+
type RunOutcome,
|
|
194
|
+
} from './run-coordinator';
|
|
195
|
+
import {
|
|
196
|
+
RunStore,
|
|
197
|
+
runDurabilityConfig,
|
|
198
|
+
type RunDurabilityConfig,
|
|
199
|
+
type RunRecord,
|
|
200
|
+
} from './run-store';
|
|
201
|
+
import {
|
|
202
|
+
runSummaryOf,
|
|
203
|
+
storedRunRequest,
|
|
204
|
+
type StoredRunRequest,
|
|
205
|
+
} from './run-request';
|
|
206
|
+
import {
|
|
207
|
+
createToolMarksMiddleware,
|
|
208
|
+
toolEffectOf,
|
|
209
|
+
type ToolEffect,
|
|
210
|
+
} from '../core/middlewares/tool-marks';
|
|
211
|
+
import { createResultCapMiddleware } from '../core/middlewares/result-cap';
|
|
212
|
+
import { READ_RESULT_TOOL_NAME } from '../core/read-result-tool';
|
|
213
|
+
import {
|
|
214
|
+
ContextWindowResolver,
|
|
215
|
+
contextWindowConfig,
|
|
216
|
+
} from '../core/context-window';
|
|
217
|
+
import { contextBudgetFor, contextKnobs } from '../core/context-budget';
|
|
218
|
+
import {
|
|
219
|
+
estimateRequestTokens,
|
|
220
|
+
type ContextGuardEvent,
|
|
221
|
+
} from '../core/middlewares/context-guard';
|
|
222
|
+
import { fetchOpenRouterContextLengths } from '../core/openrouter-pricing';
|
|
223
|
+
import { ResultStore, resultStoreKnobs } from './result-store';
|
|
172
224
|
import { formatReplay } from '../matrix/replay-format';
|
|
173
225
|
import { retryGateway } from './gateway-retry';
|
|
174
226
|
import {
|
|
@@ -178,7 +230,10 @@ import {
|
|
|
178
230
|
import {
|
|
179
231
|
contentToText,
|
|
180
232
|
isSummarizationMessage,
|
|
233
|
+
pageThreadTranscript,
|
|
234
|
+
TranscriptCursorError,
|
|
181
235
|
transformTranscript,
|
|
236
|
+
type TranscriptPageOptions,
|
|
182
237
|
} from './transcript';
|
|
183
238
|
import { WorkersUcanService } from './ucan-service';
|
|
184
239
|
|
|
@@ -208,6 +263,27 @@ const IDLE_EVICT_MS = 5 * 24 * 60 * 60 * 1000;
|
|
|
208
263
|
/** Set once the legacy Matrix copy has been removed (or found absent): no more lookups. */
|
|
209
264
|
const META_LEGACY_CLEARED = 'meta:legacyCleared';
|
|
210
265
|
const META_USER_DID = 'meta:userDid';
|
|
266
|
+
|
|
267
|
+
/** KV key of a session's context-guard counters (`GET /debug/context?session=`). */
|
|
268
|
+
const contextStatsKey = (sessionId: string): string => `ctxstats:${sessionId}`;
|
|
269
|
+
|
|
270
|
+
type SessionContextCounters = Pick<
|
|
271
|
+
SessionContextStatus,
|
|
272
|
+
| 'prunes'
|
|
273
|
+
| 'hardPrunes'
|
|
274
|
+
| 'prunedResults'
|
|
275
|
+
| 'overflowRetries'
|
|
276
|
+
| 'refusals'
|
|
277
|
+
| 'lastEventAt'
|
|
278
|
+
>;
|
|
279
|
+
|
|
280
|
+
const EMPTY_CONTEXT_COUNTERS: SessionContextCounters = {
|
|
281
|
+
prunes: 0,
|
|
282
|
+
hardPrunes: 0,
|
|
283
|
+
prunedResults: 0,
|
|
284
|
+
overflowRetries: 0,
|
|
285
|
+
refusals: 0,
|
|
286
|
+
};
|
|
211
287
|
/** VFS write generation of the file at the last successful upload (see `DoVfs.writeGeneration`). */
|
|
212
288
|
const META_UPLOADED_GEN = 'meta:uploadedGen';
|
|
213
289
|
/**
|
|
@@ -245,6 +321,15 @@ const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
|
|
|
245
321
|
* silently stale.
|
|
246
322
|
*/
|
|
247
323
|
const META_DIRTY = 'meta:dirty';
|
|
324
|
+
/**
|
|
325
|
+
* When the dirty working copy is due to be uploaded. Armed by the first
|
|
326
|
+
* write after an upload (`FLUSH_DEBOUNCE_MS` later), replaced by the retry
|
|
327
|
+
* delay after a failed upload, cleared with the dirty mark. The alarm is
|
|
328
|
+
* shared with the realtime heartbeat, the durable-run keep-alive, task runs
|
|
329
|
+
* and compaction, so a wake alone never means "upload now" — the tick
|
|
330
|
+
* consults this deadline (`do/flush-schedule.ts`).
|
|
331
|
+
*/
|
|
332
|
+
const META_FLUSH_AT = 'meta:flushAt';
|
|
248
333
|
const META_OWNER_ETAG = 'meta:ownerEtag';
|
|
249
334
|
const META_LAST_FLUSH = 'meta:lastFlushAt';
|
|
250
335
|
const META_LAST_CHECKSUM = 'meta:lastChecksum';
|
|
@@ -424,6 +509,38 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
424
509
|
if (this.ctx.getWebSockets().length > 0) this.realtime.nextPingAt();
|
|
425
510
|
}
|
|
426
511
|
private readonly aborts = new Map<string, AbortController>();
|
|
512
|
+
/** Durable runs (docs/plans/durable-runs.md): the rows, and the live side. */
|
|
513
|
+
private runStore: RunStore | null = null;
|
|
514
|
+
private runs: RunCoordinator | null = null;
|
|
515
|
+
private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
|
|
516
|
+
this.env,
|
|
517
|
+
);
|
|
518
|
+
/** Whole tool results the result cap saved (docs/plans/context-budgets.md). */
|
|
519
|
+
private resultStore: ResultStore | null = null;
|
|
520
|
+
/**
|
|
521
|
+
* Per-model context windows; learned limits persist in this object's KV
|
|
522
|
+
* storage (`ctxwin:<model>`), so a provider's rejection is remembered
|
|
523
|
+
* across boots.
|
|
524
|
+
*/
|
|
525
|
+
private readonly contextWindows = new ContextWindowResolver({
|
|
526
|
+
config: contextWindowConfig(this.env, console),
|
|
527
|
+
catalog: () =>
|
|
528
|
+
fetchOpenRouterContextLengths({
|
|
529
|
+
apiKey: this.env.OPEN_ROUTER_API_KEY,
|
|
530
|
+
logger: console,
|
|
531
|
+
}),
|
|
532
|
+
learned: {
|
|
533
|
+
get: (model) => this.ctx.storage.get<number>(`ctxwin:${model}`),
|
|
534
|
+
set: (model, tokens) => this.ctx.storage.put(`ctxwin:${model}`, tokens),
|
|
535
|
+
},
|
|
536
|
+
logger: console,
|
|
537
|
+
});
|
|
538
|
+
/**
|
|
539
|
+
* Per-session context-guard counters (`ctxstats:<session>` in KV
|
|
540
|
+
* storage; `GET /debug/context?session=`). Writes are chained so two
|
|
541
|
+
* events of one request (a soft then a hard prune) never race.
|
|
542
|
+
*/
|
|
543
|
+
private contextStatsWrites: Promise<void> = Promise.resolve();
|
|
427
544
|
private readonly delegations = new Map<string, { raw: string }>();
|
|
428
545
|
/** Epoch-ms until which a room-state delegation lookup is not retried. */
|
|
429
546
|
private delegationMissUntil = 0;
|
|
@@ -456,6 +573,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
456
573
|
private mirror: RoomMirror | null = null;
|
|
457
574
|
/** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
|
|
458
575
|
private readonly sessionRooms = new Map<string, string>();
|
|
576
|
+
/** The user's main oracle room, once resolved (the session list is scoped to it). */
|
|
577
|
+
private mainRoomId: string | null = null;
|
|
459
578
|
/** The throttled `delegation_required` prompt (see reauth-prompt.ts). */
|
|
460
579
|
private reauthPrompter: ReauthPrompter | null = null;
|
|
461
580
|
/** Room turns running right now, by Matrix event id: a gateway that asks again attaches instead of re-running. */
|
|
@@ -974,9 +1093,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
974
1093
|
console.warn(
|
|
975
1094
|
`[user-do] VFS migration write failed for ${userDid} — serving the legacy Matrix copy for now: ${err instanceof Error ? err.message : String(err)}`,
|
|
976
1095
|
);
|
|
977
|
-
//
|
|
978
|
-
//
|
|
979
|
-
this.
|
|
1096
|
+
// The failed upload armed the 10-minute retry (`flushOnce`), well
|
|
1097
|
+
// before the 24 h debounce `markDirty` set: the system of record
|
|
1098
|
+
// is still empty for this user.
|
|
980
1099
|
}
|
|
981
1100
|
}
|
|
982
1101
|
await this.reconcileDirtyOnBoot(liveDb, userDid);
|
|
@@ -1015,6 +1134,29 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1015
1134
|
await this.sessions.setup();
|
|
1016
1135
|
this.matrixLedger = new MatrixTurnLedger(liveDb);
|
|
1017
1136
|
await this.matrixLedger.setup();
|
|
1137
|
+
this.runStore = new RunStore(liveDb);
|
|
1138
|
+
await this.runStore.setup();
|
|
1139
|
+
this.resultStore = new ResultStore(liveDb, {
|
|
1140
|
+
...(this.env.TIER_BUCKET ? { bucket: this.env.TIER_BUCKET } : {}),
|
|
1141
|
+
prefix: this.ctx.id.toString(),
|
|
1142
|
+
...resultStoreKnobs(this.env),
|
|
1143
|
+
logger: console,
|
|
1144
|
+
});
|
|
1145
|
+
await this.resultStore.setup();
|
|
1146
|
+
// Threads opened before the runtime adopted Node's session rule
|
|
1147
|
+
// (session id = thread root event id) are renamed to it, once.
|
|
1148
|
+
const renamed = await migrateThreadSessionIds(liveDb, console);
|
|
1149
|
+
if (renamed.renamed > 0) this.markDirty();
|
|
1150
|
+
this.runs = new RunCoordinator({
|
|
1151
|
+
store: this.runStore,
|
|
1152
|
+
config: this.runConfig,
|
|
1153
|
+
instanceId: this.instanceId,
|
|
1154
|
+
log: console,
|
|
1155
|
+
requestAlarm: (at) => this.requestAlarm(at),
|
|
1156
|
+
runAttempt: (live, resumed) => this.runAttempt(live, resumed),
|
|
1157
|
+
checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
|
|
1158
|
+
onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
|
|
1159
|
+
});
|
|
1018
1160
|
|
|
1019
1161
|
const core = this.core;
|
|
1020
1162
|
// Boot-time plugin hooks + collision checks, once per object (memoised).
|
|
@@ -1031,6 +1173,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1031
1173
|
gateway: this.gateway,
|
|
1032
1174
|
runTurn: (req) => this.runTurn(req),
|
|
1033
1175
|
requestAlarm: (at) => this.requestAlarm(at),
|
|
1176
|
+
turnRunLive: (taskRunId) => Boolean(this.runs?.byTaskRunId(taskRunId)),
|
|
1034
1177
|
log: console,
|
|
1035
1178
|
...(Number.isFinite(maxTasks) ? { maxTasksPerUser: maxTasks } : {}),
|
|
1036
1179
|
...(Number.isFinite(minCron) ? { minCronIntervalSec: minCron } : {}),
|
|
@@ -1040,6 +1183,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1040
1183
|
if (next !== null) this.requestAlarm(Math.max(next, Date.now() + 1000));
|
|
1041
1184
|
}
|
|
1042
1185
|
|
|
1186
|
+
// Runs a previous incarnation left in flight: schedule their recovery
|
|
1187
|
+
// (attempts run from the alarm, never inside this boot).
|
|
1188
|
+
try {
|
|
1189
|
+
await this.runs.recoverOrphans();
|
|
1190
|
+
} catch (err) {
|
|
1191
|
+
console.error(
|
|
1192
|
+
`[user-do] run recovery scan failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1043
1196
|
// Resume (or start) legacy-blob compaction in the background: files
|
|
1044
1197
|
// imported from the Node runtime arrive with uncompressed blobs.
|
|
1045
1198
|
this.compactDone ??=
|
|
@@ -1135,6 +1288,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1135
1288
|
META_OWNER_ETAG,
|
|
1136
1289
|
META_LAST_CHECKSUM,
|
|
1137
1290
|
META_LAST_FLUSH,
|
|
1291
|
+
META_FLUSH_AT,
|
|
1138
1292
|
META_UPLOADED_GEN,
|
|
1139
1293
|
META_FLUSH_FAILURES,
|
|
1140
1294
|
META_LAST_VACUUM,
|
|
@@ -1240,7 +1394,30 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1240
1394
|
private markDirty(): void {
|
|
1241
1395
|
this.dirty = true;
|
|
1242
1396
|
void this.ctx.storage.put(META_DIRTY, true);
|
|
1243
|
-
this.
|
|
1397
|
+
void this.armFlush(Date.now() + FLUSH_DEBOUNCE_MS);
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
/**
|
|
1401
|
+
* Record the flush deadline and arm the alarm for it. An earlier
|
|
1402
|
+
* deadline already on disk wins, so a burst of writes uploads once,
|
|
1403
|
+
* `FLUSH_DEBOUNCE_MS` after the first of them, never later.
|
|
1404
|
+
*/
|
|
1405
|
+
private async armFlush(at: number): Promise<void> {
|
|
1406
|
+
const existing = await this.ctx.storage.get<number>(META_FLUSH_AT);
|
|
1407
|
+
const deadline =
|
|
1408
|
+
typeof existing === 'number' && existing <= at ? existing : at;
|
|
1409
|
+
if (deadline !== existing)
|
|
1410
|
+
await this.ctx.storage.put(META_FLUSH_AT, deadline);
|
|
1411
|
+
this.requestAlarm(deadline);
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
/**
|
|
1415
|
+
* After a failed upload: the next attempt is `at`, replacing the deadline
|
|
1416
|
+
* that just came due (which would otherwise make every wake retry).
|
|
1417
|
+
*/
|
|
1418
|
+
private async scheduleFlushRetry(at: number): Promise<void> {
|
|
1419
|
+
await this.ctx.storage.put(META_FLUSH_AT, at);
|
|
1420
|
+
this.requestAlarm(at);
|
|
1244
1421
|
}
|
|
1245
1422
|
|
|
1246
1423
|
/** Arm the object's single alarm no later than `at` (multiplexed). */
|
|
@@ -1268,12 +1445,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1268
1445
|
this.ctx.getWebSockets().length > 0 ? this.realtime.pingTick() : null;
|
|
1269
1446
|
const housekeepingAt =
|
|
1270
1447
|
await this.ctx.storage.get<number>(META_HOUSEKEEPING_AT);
|
|
1448
|
+
// Runs in flight in THIS instance need the keep-alive re-armed; a
|
|
1449
|
+
// recovery attempt that came due needs starting. Both are cheap and
|
|
1450
|
+
// do not open the database on a bare object (the coordinator only
|
|
1451
|
+
// exists once the object booted).
|
|
1452
|
+
const keepAliveAt = this.runs?.keepAliveDeadline(wakeAt) ?? null;
|
|
1453
|
+
const recoveryAt = this.runs?.nextRecoveryAt() ?? null;
|
|
1454
|
+
if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000)
|
|
1455
|
+
await this.runs.resumeDue(wakeAt);
|
|
1271
1456
|
if (
|
|
1272
1457
|
nextPingAt !== null &&
|
|
1273
1458
|
housekeepingAt !== undefined &&
|
|
1274
|
-
housekeepingAt > wakeAt + 1000
|
|
1459
|
+
housekeepingAt > wakeAt + 1000 &&
|
|
1460
|
+
keepAliveAt === null &&
|
|
1461
|
+
(recoveryAt === null || recoveryAt > wakeAt + 1000)
|
|
1275
1462
|
) {
|
|
1276
|
-
await this.ctx.storage.setAlarm(
|
|
1463
|
+
await this.ctx.storage.setAlarm(
|
|
1464
|
+
Math.min(nextPingAt, housekeepingAt, recoveryAt ?? Infinity),
|
|
1465
|
+
);
|
|
1277
1466
|
return;
|
|
1278
1467
|
}
|
|
1279
1468
|
|
|
@@ -1308,15 +1497,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1308
1497
|
const now = Date.now();
|
|
1309
1498
|
const deadlines: number[] = [];
|
|
1310
1499
|
|
|
1311
|
-
|
|
1500
|
+
// Durable runs: start the recovery attempts that are due (the boot
|
|
1501
|
+
// above may have just scheduled them), then keep the alarm inside the
|
|
1502
|
+
// keep-alive horizon while anything executes.
|
|
1503
|
+
if (this.runs) {
|
|
1504
|
+
try {
|
|
1505
|
+
await this.runs.resumeDue(now);
|
|
1506
|
+
} catch (err) {
|
|
1507
|
+
console.error(
|
|
1508
|
+
`[user-do] run recovery tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1509
|
+
);
|
|
1510
|
+
}
|
|
1511
|
+
const keepAlive = this.runs.keepAliveDeadline(now);
|
|
1512
|
+
if (keepAlive !== null) deadlines.push(keepAlive);
|
|
1513
|
+
const nextRecovery = this.runs.nextRecoveryAt();
|
|
1514
|
+
if (nextRecovery !== null)
|
|
1515
|
+
deadlines.push(Math.max(nextRecovery, now + 1000));
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
const lastAccess =
|
|
1519
|
+
(await this.ctx.storage.get<number>(META_LAST_ACCESS)) ?? now;
|
|
1520
|
+
const idle = now - lastAccess > IDLE_EVICT_MS;
|
|
1521
|
+
|
|
1522
|
+
// Owner-store flush: only when its own deadline is due, or the copy
|
|
1523
|
+
// is about to be evicted below. A wake for anything else (a run's
|
|
1524
|
+
// keep-alive, a heartbeat round that fell through, a compaction or
|
|
1525
|
+
// tier re-arm) leaves a dirty copy alone — the upload is a full-file
|
|
1526
|
+
// transfer and is meant to happen once a day, not once a turn.
|
|
1527
|
+
const flush = decideFlush({
|
|
1528
|
+
dirty: this.dirty && this.db !== null,
|
|
1529
|
+
flushAt: await this.ctx.storage.get<number>(META_FLUSH_AT),
|
|
1530
|
+
now,
|
|
1531
|
+
idle,
|
|
1532
|
+
});
|
|
1533
|
+
if (flush.action === 'flush') {
|
|
1312
1534
|
try {
|
|
1313
1535
|
await this.flushToOwnerStore();
|
|
1314
1536
|
} catch (err) {
|
|
1315
1537
|
console.error(
|
|
1316
1538
|
`[user-do] flush failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1317
1539
|
);
|
|
1540
|
+
// `flushOnce` arms the retry on an upload failure; anything that
|
|
1541
|
+
// failed before the upload gets the same retry window here.
|
|
1542
|
+
await this.scheduleFlushRetry(now + FLUSH_RETRY_DELAY_MS);
|
|
1318
1543
|
deadlines.push(now + FLUSH_RETRY_DELAY_MS);
|
|
1319
1544
|
}
|
|
1545
|
+
} else if (flush.action === 'wait') {
|
|
1546
|
+
deadlines.push(Math.max(flush.at, now + 1000));
|
|
1320
1547
|
}
|
|
1321
1548
|
|
|
1322
1549
|
if (this.taskScheduler) {
|
|
@@ -1349,9 +1576,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1349
1576
|
}
|
|
1350
1577
|
}
|
|
1351
1578
|
|
|
1352
|
-
const lastAccess =
|
|
1353
|
-
(await this.ctx.storage.get<number>(META_LAST_ACCESS)) ?? now;
|
|
1354
|
-
|
|
1355
1579
|
// Free-page reclaim on a quiet object (policy in sqlite/vacuum-policy.ts):
|
|
1356
1580
|
// runs after the flush above so the upload holds the pre-rebuild file,
|
|
1357
1581
|
// and marks dirty so the smaller file goes up on the next tick.
|
|
@@ -1364,7 +1588,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1364
1588
|
// of DO storage (see sqlite/page-tier.ts). After the flush above (the
|
|
1365
1589
|
// upload reads hot rows, not R2) and never over an in-flight one (it
|
|
1366
1590
|
// holds a snapshot). A pass is capped; more work re-arms soon.
|
|
1367
|
-
if (
|
|
1591
|
+
if (
|
|
1592
|
+
this.db?.tierEnabled &&
|
|
1593
|
+
!this.flushInFlight &&
|
|
1594
|
+
(this.runs?.activeCount ?? 0) === 0
|
|
1595
|
+
) {
|
|
1368
1596
|
const next = await this.tierTick(this.db, now);
|
|
1369
1597
|
if (next !== null) deadlines.push(next);
|
|
1370
1598
|
}
|
|
@@ -1374,8 +1602,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1374
1602
|
// evict while tasks are scheduled: their runs ARE activity. The flush
|
|
1375
1603
|
// above ran first; the wipe additionally verifies the upstream copy
|
|
1376
1604
|
// matches the working copy, else the copy stays until it does.
|
|
1377
|
-
|
|
1378
|
-
|
|
1605
|
+
if (
|
|
1606
|
+
idle &&
|
|
1607
|
+
!this.dirty &&
|
|
1608
|
+
this.db &&
|
|
1609
|
+
deadlines.length === 0 &&
|
|
1610
|
+
(this.runs?.size ?? 0) === 0
|
|
1611
|
+
) {
|
|
1379
1612
|
if (await this.ownerCopyIsCurrent(this.db)) {
|
|
1380
1613
|
await this.wipeWorkingCopy(this.db);
|
|
1381
1614
|
await this.ctx.storage.delete(META_HOUSEKEEPING_AT);
|
|
@@ -1541,6 +1774,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1541
1774
|
);
|
|
1542
1775
|
if (room) {
|
|
1543
1776
|
roomId = room.roomId;
|
|
1777
|
+
if (!o.roomId) this.mainRoomId = room.roomId;
|
|
1544
1778
|
// The marker is a waited, non-durable send: it must never be
|
|
1545
1779
|
// replayed after a gateway restart (its event id IS the session id,
|
|
1546
1780
|
// an orphan marker would be a ghost session). Instead it retries
|
|
@@ -1583,11 +1817,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1583
1817
|
o: { limit?: number; offset?: number } = {},
|
|
1584
1818
|
): Promise<{ sessions: SessionSummary[]; total: number }> {
|
|
1585
1819
|
await this.ready(identity);
|
|
1586
|
-
//
|
|
1587
|
-
//
|
|
1588
|
-
//
|
|
1820
|
+
// Node parity (`SessionsService.listSessions`): the list is the user's
|
|
1821
|
+
// MAIN oracle room only — Portal sessions and the threads opened by
|
|
1822
|
+
// room messages there. Threads in dedicated task rooms (or any other
|
|
1823
|
+
// room) stay out of it, and task runs (`task:<id>`) are hidden
|
|
1824
|
+
// whichever room they deliver to. A user without an oracle room sees
|
|
1825
|
+
// every session, as on Node.
|
|
1826
|
+
const mainRoomId = await this.resolveMainRoomId(identity.userDid);
|
|
1589
1827
|
const { sessions, total } = await this.sessions!.listSessions(
|
|
1590
|
-
undefined,
|
|
1828
|
+
mainRoomId ?? undefined,
|
|
1591
1829
|
o.limit ?? 20,
|
|
1592
1830
|
o.offset ?? 0,
|
|
1593
1831
|
TASK_SESSION_PREFIX,
|
|
@@ -1595,6 +1833,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1595
1833
|
return { sessions: sessions.map(toSummary), total };
|
|
1596
1834
|
}
|
|
1597
1835
|
|
|
1836
|
+
/**
|
|
1837
|
+
* The user↔oracle room, resolved once per instance. `null` is the
|
|
1838
|
+
* definitive "no oracle room"; a transient gateway failure retries and
|
|
1839
|
+
* then fails the request rather than answering with the wrong list.
|
|
1840
|
+
*/
|
|
1841
|
+
private async resolveMainRoomId(userDid: string): Promise<string | null> {
|
|
1842
|
+
if (this.mainRoomId) return this.mainRoomId;
|
|
1843
|
+
const room = await retryGateway(() =>
|
|
1844
|
+
this.gateway.resolveUserRoom(userDid),
|
|
1845
|
+
);
|
|
1846
|
+
if (room) this.mainRoomId = room.roomId;
|
|
1847
|
+
return room?.roomId ?? null;
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1598
1850
|
async deleteSession(
|
|
1599
1851
|
identity: TurnIdentity,
|
|
1600
1852
|
sessionId: string,
|
|
@@ -1615,6 +1867,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1615
1867
|
const deleted = await this.sessions!.deleteSession(sessionId);
|
|
1616
1868
|
if (deleted) {
|
|
1617
1869
|
await this.saver!.deleteThread(sessionId);
|
|
1870
|
+
await this.resultStore
|
|
1871
|
+
?.deleteForSession(sessionId)
|
|
1872
|
+
.catch((err: unknown) => {
|
|
1873
|
+
console.warn(
|
|
1874
|
+
`[user-do] could not delete the saved tool results of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1875
|
+
);
|
|
1876
|
+
});
|
|
1877
|
+
await this.ctx.storage.delete(contextStatsKey(sessionId));
|
|
1618
1878
|
this.markDirty();
|
|
1619
1879
|
}
|
|
1620
1880
|
return deleted;
|
|
@@ -1723,6 +1983,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1723
1983
|
return JSON.stringify(dtos);
|
|
1724
1984
|
}
|
|
1725
1985
|
|
|
1986
|
+
async listMessagesPage(
|
|
1987
|
+
identity: TurnIdentity,
|
|
1988
|
+
sessionId: string,
|
|
1989
|
+
options: TranscriptPageOptions,
|
|
1990
|
+
): Promise<
|
|
1991
|
+
{ ok: true; json: string } | { ok: false; status: 400; message: string }
|
|
1992
|
+
> {
|
|
1993
|
+
await this.ready(identity);
|
|
1994
|
+
try {
|
|
1995
|
+
const page = await pageThreadTranscript(
|
|
1996
|
+
this.saver!,
|
|
1997
|
+
sessionId,
|
|
1998
|
+
options,
|
|
1999
|
+
);
|
|
2000
|
+
return { ok: true, json: JSON.stringify(page) };
|
|
2001
|
+
} catch (err) {
|
|
2002
|
+
if (err instanceof TranscriptCursorError)
|
|
2003
|
+
return { ok: false, status: 400, message: err.message };
|
|
2004
|
+
throw err;
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
|
|
1726
2008
|
/**
|
|
1727
2009
|
* The shell deposited a (new) delegation for this user: adopt it now so
|
|
1728
2010
|
* header-less turns mint from it immediately. Does not boot the object.
|
|
@@ -1820,9 +2102,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1820
2102
|
writeGeneration: db.writeGeneration,
|
|
1821
2103
|
});
|
|
1822
2104
|
if (decision === 'flagged') {
|
|
1823
|
-
// The mark is on disk already (its
|
|
2105
|
+
// The mark is on disk already (its deadline too); adopt it and make
|
|
2106
|
+
// sure the alarm is armed for that deadline.
|
|
1824
2107
|
this.dirty = true;
|
|
1825
|
-
this.
|
|
2108
|
+
void this.armFlush(Date.now() + FLUSH_DEBOUNCE_MS);
|
|
1826
2109
|
} else if (decision === 'behind-upload') {
|
|
1827
2110
|
console.log(
|
|
1828
2111
|
`[user-do] working copy of ${userDid} is at generation ${db.writeGeneration}, last upload at ${String(got.get(META_UPLOADED_GEN))} — a turn ended without marking it; flushing`,
|
|
@@ -1861,7 +2144,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1861
2144
|
return { present: false, source: 'none' };
|
|
1862
2145
|
}
|
|
1863
2146
|
|
|
2147
|
+
/**
|
|
2148
|
+
* `POST /messages/abort`: stop the session's active run. The run is
|
|
2149
|
+
* closed as `aborted` with the text streamed so far kept on its row;
|
|
2150
|
+
* a queued run is dropped. (A client that merely disconnects does NOT
|
|
2151
|
+
* reach here — runs outlive their connections.)
|
|
2152
|
+
*/
|
|
1864
2153
|
async abortTurn(sessionId: string): Promise<boolean> {
|
|
2154
|
+
if (this.runs?.abortSession(sessionId)) return true;
|
|
1865
2155
|
const controller = this.aborts.get(sessionId);
|
|
1866
2156
|
if (!controller) return false;
|
|
1867
2157
|
controller.abort();
|
|
@@ -1904,8 +2194,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1904
2194
|
toolCalls: [],
|
|
1905
2195
|
replayed: true,
|
|
1906
2196
|
};
|
|
1907
|
-
case 'interrupted':
|
|
2197
|
+
case 'interrupted': {
|
|
2198
|
+
// The object reset mid-turn. The run may be recovering (durable
|
|
2199
|
+
// runs): attach to it and answer with its reply; only a run that
|
|
2200
|
+
// is over without a reply is refused.
|
|
2201
|
+
const attached = existing?.requestId
|
|
2202
|
+
? await this.attachToRun(existing.requestId, req)
|
|
2203
|
+
: null;
|
|
2204
|
+
if (attached) return attached;
|
|
1908
2205
|
throw new Error(`${TURN_INTERRUPTED_MARKER} (${eventId})`);
|
|
2206
|
+
}
|
|
1909
2207
|
case 'run':
|
|
1910
2208
|
break;
|
|
1911
2209
|
}
|
|
@@ -1952,43 +2250,78 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1952
2250
|
});
|
|
1953
2251
|
card.emit(req.requestId, 'routing');
|
|
1954
2252
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
2253
|
+
// Every turn is a durable run (docs/plans/durable-runs.md): recorded
|
|
2254
|
+
// before the first model call, kept alive by the alarm, resumed after
|
|
2255
|
+
// a reset. Headless turns (Matrix, tasks) simply await the outcome.
|
|
2256
|
+
const { live } = await this.runs!.begin({
|
|
2257
|
+
runId: crypto.randomUUID(),
|
|
2258
|
+
sessionId: req.sessionId,
|
|
2259
|
+
requestId: req.requestId,
|
|
2260
|
+
client: req.client,
|
|
2261
|
+
request: JSON.stringify(
|
|
2262
|
+
storedRunRequest(req, { timezone: req.identity.timezone }),
|
|
2263
|
+
),
|
|
2264
|
+
multitask: req.multitask ?? this.runConfig.multitaskDefault,
|
|
2265
|
+
...(req.taskRunId ? { taskRunId: req.taskRunId } : {}),
|
|
2266
|
+
});
|
|
2267
|
+
const outcome = await live.done;
|
|
2268
|
+
if (outcome.status !== 'finished') {
|
|
2269
|
+
const reason =
|
|
2270
|
+
outcome.error instanceof Error
|
|
2271
|
+
? outcome.error.message
|
|
2272
|
+
: outcome.error !== undefined
|
|
2273
|
+
? String(outcome.error)
|
|
2274
|
+
: `turn ${outcome.status}`;
|
|
2275
|
+
throw new Error(reason);
|
|
2276
|
+
}
|
|
2277
|
+
card?.emit(req.requestId, 'delivering');
|
|
2278
|
+
return {
|
|
2279
|
+
sessionId: req.sessionId,
|
|
2280
|
+
requestId: req.requestId,
|
|
2281
|
+
text: outcome.text,
|
|
2282
|
+
...(outcome.messageId !== undefined
|
|
2283
|
+
? { messageId: outcome.messageId }
|
|
2284
|
+
: {}),
|
|
2285
|
+
toolCalls: outcome.toolCalls ?? [],
|
|
2286
|
+
};
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
/**
|
|
2290
|
+
* A Matrix turn asked about again after this object reset: hand back the
|
|
2291
|
+
* reply of its (recovering or finished) run instead of refusing it.
|
|
2292
|
+
*/
|
|
2293
|
+
private async attachToRun(
|
|
2294
|
+
requestId: string,
|
|
2295
|
+
req: TurnRequest,
|
|
2296
|
+
): Promise<TurnResult | null> {
|
|
2297
|
+
const live = this.runs?.byRequestId(requestId);
|
|
2298
|
+
if (live) {
|
|
2299
|
+
console.log(
|
|
2300
|
+
`[user-do] turn ${requestId} for ${req.eventId} is being recovered; the gateway attaches to it`,
|
|
2301
|
+
);
|
|
2302
|
+
const outcome = await live.done;
|
|
2303
|
+
if (outcome.status !== 'finished') return null;
|
|
2304
|
+
return {
|
|
2305
|
+
sessionId: req.sessionId,
|
|
2306
|
+
requestId: req.requestId,
|
|
2307
|
+
text: outcome.text,
|
|
2308
|
+
...(outcome.messageId ? { messageId: outcome.messageId } : {}),
|
|
2309
|
+
toolCalls: outcome.toolCalls ?? [],
|
|
2310
|
+
replayed: true,
|
|
1965
2311
|
};
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
await this.runTurnDisposables(turnDisposables);
|
|
1970
|
-
card?.emit(req.requestId, 'delivering');
|
|
1971
|
-
const toolCalls = messages
|
|
1972
|
-
.filter((m): m is AIMessage => m.type === 'ai')
|
|
1973
|
-
.flatMap((m) =>
|
|
1974
|
-
(m.tool_calls ?? []).map((t) => ({
|
|
1975
|
-
name: t.name,
|
|
1976
|
-
status: 'done' as const,
|
|
1977
|
-
})),
|
|
1978
|
-
);
|
|
2312
|
+
}
|
|
2313
|
+
const record = await this.runStore?.getByRequestId(requestId);
|
|
2314
|
+
if (record?.status === 'finished') {
|
|
1979
2315
|
return {
|
|
1980
2316
|
sessionId: req.sessionId,
|
|
1981
2317
|
requestId: req.requestId,
|
|
1982
|
-
text,
|
|
1983
|
-
...(
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
toolCalls,
|
|
2318
|
+
text: record.partialText ?? '',
|
|
2319
|
+
...(record.messageId ? { messageId: record.messageId } : {}),
|
|
2320
|
+
toolCalls: [],
|
|
2321
|
+
replayed: true,
|
|
1987
2322
|
};
|
|
1988
|
-
} finally {
|
|
1989
|
-
if (this.aborts.get(req.sessionId) === abortController)
|
|
1990
|
-
this.aborts.delete(req.sessionId);
|
|
1991
2323
|
}
|
|
2324
|
+
return null;
|
|
1992
2325
|
}
|
|
1993
2326
|
|
|
1994
2327
|
/**
|
|
@@ -2149,6 +2482,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2149
2482
|
const line = `[user-do] owner-store flush failed (${failures} in a row) for ${this.userDid} — working copy kept dirty, retrying in ${FLUSH_RETRY_DELAY_MS / 60_000} min: ${detail}`;
|
|
2150
2483
|
if (failures >= FLUSH_FAILURES_ERROR_THRESHOLD) console.error(line);
|
|
2151
2484
|
else console.warn(line);
|
|
2485
|
+
await this.scheduleFlushRetry(Date.now() + FLUSH_RETRY_DELAY_MS);
|
|
2152
2486
|
throw err;
|
|
2153
2487
|
}
|
|
2154
2488
|
await this.ctx.storage.put(META_OWNER_ETAG, saved.etag);
|
|
@@ -2228,18 +2562,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2228
2562
|
}
|
|
2229
2563
|
|
|
2230
2564
|
private async clearDirty(): Promise<void> {
|
|
2231
|
-
// A write that landed while the snapshot was being uploaded
|
|
2232
|
-
//
|
|
2233
|
-
//
|
|
2565
|
+
// A write that landed while the snapshot was being uploaded keeps the
|
|
2566
|
+
// copy dirty; its upload is a full debounce away (the deadline the
|
|
2567
|
+
// write found on disk was the one that just ran).
|
|
2234
2568
|
const uploadedGen = await this.ctx.storage.get<number>(META_UPLOADED_GEN);
|
|
2235
2569
|
if (
|
|
2236
2570
|
this.db &&
|
|
2237
2571
|
uploadedGen !== undefined &&
|
|
2238
2572
|
uploadedGen !== this.db.writeGeneration
|
|
2239
|
-
)
|
|
2573
|
+
) {
|
|
2574
|
+
const at = Date.now() + FLUSH_DEBOUNCE_MS;
|
|
2575
|
+
await this.ctx.storage.put(META_FLUSH_AT, at);
|
|
2576
|
+
this.requestAlarm(at);
|
|
2240
2577
|
return;
|
|
2578
|
+
}
|
|
2241
2579
|
this.dirty = false;
|
|
2242
|
-
await this.ctx.storage.delete(META_DIRTY);
|
|
2580
|
+
await this.ctx.storage.delete([META_DIRTY, META_FLUSH_AT]);
|
|
2243
2581
|
}
|
|
2244
2582
|
|
|
2245
2583
|
async resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }> {
|
|
@@ -2261,7 +2599,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2261
2599
|
this.saver = null;
|
|
2262
2600
|
this.sessions = null;
|
|
2263
2601
|
this.matrixLedger = null;
|
|
2602
|
+
this.runStore = null;
|
|
2603
|
+
this.runs = null;
|
|
2604
|
+
this.resultStore = null;
|
|
2264
2605
|
this.sessionRooms.clear();
|
|
2606
|
+
this.mainRoomId = null;
|
|
2265
2607
|
this.ambient = null;
|
|
2266
2608
|
this.secretsService = null;
|
|
2267
2609
|
this.byo = null;
|
|
@@ -2364,6 +2706,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2364
2706
|
pages: Math.ceil(fileBytes / 4096),
|
|
2365
2707
|
ownerEtag: await this.ctx.storage.get<string>(META_OWNER_ETAG),
|
|
2366
2708
|
lastFlushAt: await this.ctx.storage.get<number>(META_LAST_FLUSH),
|
|
2709
|
+
nextFlushAt: await this.ctx.storage.get<number>(META_FLUSH_AT),
|
|
2367
2710
|
dirty: this.dirty,
|
|
2368
2711
|
flushInFlight: this.flushInFlight !== null,
|
|
2369
2712
|
writeGeneration: db?.writeGeneration,
|
|
@@ -2574,6 +2917,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2574
2917
|
console,
|
|
2575
2918
|
);
|
|
2576
2919
|
}
|
|
2920
|
+
const joinMatch = /^\/runs\/([^/]+)$/.exec(url.pathname);
|
|
2921
|
+
if (joinMatch && request.method === 'GET')
|
|
2922
|
+
return this.joinRun(request, decodeURIComponent(joinMatch[1]!), url);
|
|
2577
2923
|
const match = /^\/turn\/([^/]+)$/.exec(url.pathname);
|
|
2578
2924
|
if (!match || request.method !== 'POST')
|
|
2579
2925
|
return new Response('Not found', { status: 404 });
|
|
@@ -2661,75 +3007,428 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2661
3007
|
}
|
|
2662
3008
|
}
|
|
2663
3009
|
|
|
3010
|
+
// A streamed turn is a durable run: recorded, kept alive and resumable
|
|
3011
|
+
// (docs/plans/durable-runs.md). The response is one subscriber of the
|
|
3012
|
+
// run's buffer; the run itself outlives it.
|
|
3013
|
+
this.replayToRoom(req, body.message, 'user');
|
|
3014
|
+
const agActionNames = (body.agActions ?? []).map((a) => a.name);
|
|
3015
|
+
const { live, queued } = await this.runs!.begin({
|
|
3016
|
+
runId: crypto.randomUUID(),
|
|
3017
|
+
sessionId,
|
|
3018
|
+
requestId,
|
|
3019
|
+
client: 'portal',
|
|
3020
|
+
request: JSON.stringify(
|
|
3021
|
+
storedRunRequest(req, {
|
|
3022
|
+
timezone: body.timezone,
|
|
3023
|
+
agActionNames,
|
|
3024
|
+
stream: true,
|
|
3025
|
+
}),
|
|
3026
|
+
),
|
|
3027
|
+
multitask: body.multitask ?? this.runConfig.multitaskDefault,
|
|
3028
|
+
});
|
|
3029
|
+
if (queued)
|
|
3030
|
+
live.buffer.push('router.update', {
|
|
3031
|
+
step: 'Queued…',
|
|
3032
|
+
sessionId,
|
|
3033
|
+
requestId,
|
|
3034
|
+
runId: live.runId,
|
|
3035
|
+
queued: true,
|
|
3036
|
+
});
|
|
3037
|
+
const responseStream = createSseSubscriberStream({
|
|
3038
|
+
replay: live.buffer.tailAfter(0),
|
|
3039
|
+
buffer: live.buffer,
|
|
3040
|
+
onCancel: () =>
|
|
3041
|
+
console.log(
|
|
3042
|
+
`[user-do] client left run ${live.runId} (${sessionId}); the run continues`,
|
|
3043
|
+
),
|
|
3044
|
+
});
|
|
3045
|
+
return new Response(responseStream, {
|
|
3046
|
+
headers: {
|
|
3047
|
+
...SSE_HEADERS,
|
|
3048
|
+
'x-request-id': requestId,
|
|
3049
|
+
'x-run-id': live.runId,
|
|
3050
|
+
},
|
|
3051
|
+
});
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
/**
|
|
3055
|
+
* `GET /runs/:id?after=<seq>` — re-join a run: replay the frames after
|
|
3056
|
+
* the cursor (packed segments, then the live tail) and stay attached
|
|
3057
|
+
* until its `done` frame. A run that already ended replays what is left
|
|
3058
|
+
* and closes with a synthetic `done` carrying its final state.
|
|
3059
|
+
*/
|
|
3060
|
+
private async joinRun(
|
|
3061
|
+
request: Request,
|
|
3062
|
+
runId: string,
|
|
3063
|
+
url: URL,
|
|
3064
|
+
): Promise<Response> {
|
|
3065
|
+
const identity = JSON.parse(
|
|
3066
|
+
request.headers.get('x-identity') ?? '{}',
|
|
3067
|
+
) as TurnIdentity;
|
|
3068
|
+
try {
|
|
3069
|
+
await this.ready(identity);
|
|
3070
|
+
} catch (err) {
|
|
3071
|
+
return readyFailureResponse(err);
|
|
3072
|
+
}
|
|
3073
|
+
const afterRaw = Number(url.searchParams.get('after') ?? '0');
|
|
3074
|
+
const after =
|
|
3075
|
+
Number.isFinite(afterRaw) && afterRaw > 0 ? Math.floor(afterRaw) : 0;
|
|
3076
|
+
const joined = await this.runs!.join(runId, after);
|
|
3077
|
+
if (!joined)
|
|
3078
|
+
return Response.json(
|
|
3079
|
+
{ statusCode: 404, message: `Run ${runId} not found` },
|
|
3080
|
+
{ status: 404 },
|
|
3081
|
+
);
|
|
3082
|
+
const { record, replay, buffer } = joined;
|
|
3083
|
+
const ended = !buffer || buffer.isClosed;
|
|
3084
|
+
const stream = createSseSubscriberStream({
|
|
3085
|
+
replay,
|
|
3086
|
+
...(buffer && !buffer.isClosed ? { buffer } : {}),
|
|
3087
|
+
...(ended
|
|
3088
|
+
? {
|
|
3089
|
+
trailer: {
|
|
3090
|
+
event: 'done',
|
|
3091
|
+
data: {
|
|
3092
|
+
runId,
|
|
3093
|
+
status: record.status,
|
|
3094
|
+
...(record.messageId ? { messageId: record.messageId } : {}),
|
|
3095
|
+
...(record.partialText && record.status !== 'finished'
|
|
3096
|
+
? { partialText: record.partialText }
|
|
3097
|
+
: {}),
|
|
3098
|
+
replayed: true,
|
|
3099
|
+
},
|
|
3100
|
+
},
|
|
3101
|
+
}
|
|
3102
|
+
: {}),
|
|
3103
|
+
});
|
|
3104
|
+
return new Response(stream, {
|
|
3105
|
+
headers: {
|
|
3106
|
+
...SSE_HEADERS,
|
|
3107
|
+
'x-request-id': record.requestId,
|
|
3108
|
+
'x-run-id': runId,
|
|
3109
|
+
'x-run-status': record.status,
|
|
3110
|
+
},
|
|
3111
|
+
});
|
|
3112
|
+
}
|
|
3113
|
+
|
|
3114
|
+
/**
|
|
3115
|
+
* One attempt of a run (the coordinator's `runAttempt` host hook): build
|
|
3116
|
+
* the agent for the stored request, stream the turn's frames into the
|
|
3117
|
+
* run's buffer, and report the outcome. `resumed` attempts hand the
|
|
3118
|
+
* graph no new input, so LangGraph continues from the checkpoint's
|
|
3119
|
+
* pending node; the tool-mark middleware keeps that safe.
|
|
3120
|
+
*/
|
|
3121
|
+
private async runAttempt(
|
|
3122
|
+
live: LiveRun,
|
|
3123
|
+
resumed: boolean,
|
|
3124
|
+
): Promise<RunOutcome> {
|
|
3125
|
+
const stored = JSON.parse(live.record.request) as StoredRunRequest;
|
|
3126
|
+
const req = stored.turn;
|
|
2664
3127
|
const {
|
|
2665
3128
|
agent,
|
|
2666
3129
|
stateInput,
|
|
2667
3130
|
config,
|
|
2668
|
-
|
|
3131
|
+
turnDisposables,
|
|
2669
3132
|
byoNotice,
|
|
2670
3133
|
byoProvider,
|
|
2671
|
-
|
|
2672
|
-
} = await this.prepareTurn(
|
|
2673
|
-
|
|
2674
|
-
|
|
3134
|
+
toolOutputCapChars,
|
|
3135
|
+
} = await this.prepareTurn(
|
|
3136
|
+
req,
|
|
3137
|
+
{
|
|
3138
|
+
message: req.message,
|
|
3139
|
+
timezone: stored.timezone ?? req.identity.timezone,
|
|
3140
|
+
model: req.model,
|
|
3141
|
+
...(resumed ? {} : { attachments: req.attachments }),
|
|
3142
|
+
},
|
|
3143
|
+
{
|
|
3144
|
+
runId: live.runId,
|
|
3145
|
+
abortController: live.abort,
|
|
3146
|
+
resumed,
|
|
3147
|
+
continuation: resumed ? live.continuation : null,
|
|
3148
|
+
},
|
|
3149
|
+
);
|
|
3150
|
+
const sessionId = req.sessionId;
|
|
2675
3151
|
const sink = {
|
|
2676
3152
|
emit: (eventName: string, payload: Record<string, unknown>) => {
|
|
2677
|
-
|
|
2678
|
-
|
|
3153
|
+
if (live.buffer.isClosed) return;
|
|
3154
|
+
live.buffer.push(eventName, payload);
|
|
2679
3155
|
},
|
|
2680
3156
|
};
|
|
2681
|
-
let write: ((name: string, payload: unknown) => void) | null = null;
|
|
2682
|
-
sink.emit = (eventName, payload) => write?.(eventName, payload);
|
|
2683
3157
|
this.events.register(sessionId, sink);
|
|
2684
|
-
|
|
3158
|
+
if (byoNotice)
|
|
3159
|
+
sink.emit('error', {
|
|
3160
|
+
...byoNotice,
|
|
3161
|
+
sessionId,
|
|
3162
|
+
requestId: req.requestId,
|
|
3163
|
+
});
|
|
3164
|
+
const events = agent.streamEvents(resumed ? null : stateInput, {
|
|
2685
3165
|
...config,
|
|
2686
3166
|
version: 'v2',
|
|
2687
3167
|
});
|
|
2688
3168
|
const capture: BaseMessage[] = [];
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
3169
|
+
let completed = false;
|
|
3170
|
+
try {
|
|
3171
|
+
const outcome = await runTurnFrames({
|
|
3172
|
+
events: tapMessages(events, capture),
|
|
3173
|
+
sink: live.buffer,
|
|
3174
|
+
sessionId,
|
|
3175
|
+
requestId: req.requestId,
|
|
3176
|
+
runId: live.runId,
|
|
3177
|
+
resumed,
|
|
3178
|
+
abortController: live.abort,
|
|
3179
|
+
mirror: (eventName, payload) =>
|
|
3180
|
+
this.events.emitToTaps(eventName, payload),
|
|
3181
|
+
agActionNames: new Set(stored.agActionNames ?? []),
|
|
3182
|
+
byoProvider,
|
|
3183
|
+
toolOutputCapChars,
|
|
3184
|
+
messageIdOf: () => lastAiMessageId(capture),
|
|
3185
|
+
onFrame: () => this.runs?.touchKeepAlive(),
|
|
3186
|
+
onComplete: async () => {
|
|
3187
|
+
completed = true;
|
|
3188
|
+
this.replayToRoom(req, lastAiText(capture), 'oracle');
|
|
3189
|
+
await this.afterTurn(sessionId, capture);
|
|
3190
|
+
},
|
|
3191
|
+
onError: (err) => {
|
|
3192
|
+
console.error(`[user-do] turn ${req.requestId} failed:`, err);
|
|
3193
|
+
},
|
|
3194
|
+
log: (m) => console.log(`[user-do] ${m}`),
|
|
3195
|
+
});
|
|
3196
|
+
const toolCalls = capture
|
|
3197
|
+
.filter((m): m is AIMessage => m.type === 'ai')
|
|
3198
|
+
.flatMap((m) =>
|
|
3199
|
+
(m.tool_calls ?? []).map((t) => ({
|
|
3200
|
+
name: t.name,
|
|
3201
|
+
status: 'done' as const,
|
|
3202
|
+
})),
|
|
3203
|
+
);
|
|
3204
|
+
const text = completed ? lastAiText(capture) : outcome.fullText;
|
|
3205
|
+
const messageId = completed ? lastAiMessageId(capture) : undefined;
|
|
3206
|
+
return {
|
|
3207
|
+
status:
|
|
3208
|
+
outcome.status === 'completed'
|
|
3209
|
+
? 'finished'
|
|
3210
|
+
: outcome.status === 'aborted'
|
|
3211
|
+
? 'aborted'
|
|
3212
|
+
: 'failed',
|
|
3213
|
+
text: (live.continuation ?? '') + text,
|
|
3214
|
+
...(messageId ? { messageId } : {}),
|
|
3215
|
+
toolCalls,
|
|
3216
|
+
...(outcome.status === 'failed' ? { error: outcome.error } : {}),
|
|
3217
|
+
};
|
|
3218
|
+
} finally {
|
|
2720
3219
|
this.events.unregister(sessionId, sink);
|
|
2721
|
-
|
|
3220
|
+
await this.runTurnDisposables(turnDisposables);
|
|
3221
|
+
if (this.aborts.get(sessionId) === live.abort)
|
|
2722
3222
|
this.aborts.delete(sessionId);
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
|
|
3226
|
+
/** Latest checkpoint id of a session — recovery's progress marker. */
|
|
3227
|
+
private async checkpointIdOf(sessionId: string): Promise<string | null> {
|
|
3228
|
+
const tuple = await this.saver
|
|
3229
|
+
?.getTupleWithoutMessages({ configurable: { thread_id: sessionId } })
|
|
3230
|
+
.catch(() => undefined);
|
|
3231
|
+
return tuple?.checkpoint.id ?? null;
|
|
3232
|
+
}
|
|
3233
|
+
|
|
3234
|
+
/**
|
|
3235
|
+
* A run reached a terminal state. A recovered task run has no scheduler
|
|
3236
|
+
* continuation waiting on it any more: hand the result (or the failure)
|
|
3237
|
+
* to the scheduler so it is delivered / recorded as it would have been.
|
|
3238
|
+
*/
|
|
3239
|
+
private async onRunEnded(
|
|
3240
|
+
record: RunRecord,
|
|
3241
|
+
outcome: RunOutcome,
|
|
3242
|
+
): Promise<void> {
|
|
3243
|
+
if (!record.taskRunId || !this.taskScheduler) return;
|
|
3244
|
+
if (this.taskScheduler.isRunActive(record.taskRunId)) return;
|
|
3245
|
+
if (outcome.status === 'finished')
|
|
3246
|
+
await this.taskScheduler.completeRecoveredRun(
|
|
3247
|
+
record.taskRunId,
|
|
3248
|
+
outcome.text,
|
|
3249
|
+
);
|
|
3250
|
+
else
|
|
3251
|
+
await this.taskScheduler.failRecoveredRun(
|
|
3252
|
+
record.taskRunId,
|
|
3253
|
+
outcome.status === 'interrupted'
|
|
3254
|
+
? 'interrupted: the object was reset while the run was in progress and recovery did not complete'
|
|
3255
|
+
: `${outcome.status}: ${outcome.error instanceof Error ? outcome.error.message : String(outcome.error ?? outcome.status)}`,
|
|
3256
|
+
);
|
|
3257
|
+
}
|
|
3258
|
+
|
|
3259
|
+
/** `GET /sessions/:id/run` — the session's active run, if any. */
|
|
3260
|
+
async sessionRun(
|
|
3261
|
+
identity: TurnIdentity,
|
|
3262
|
+
sessionId: string,
|
|
3263
|
+
): Promise<RunSummary | null> {
|
|
3264
|
+
await this.ready(identity);
|
|
3265
|
+
const live = this.runs?.activeForSession(sessionId);
|
|
3266
|
+
const record =
|
|
3267
|
+
live?.record ?? (await this.runStore?.activeForSession(sessionId));
|
|
3268
|
+
return record ? runSummaryOf(record) : null;
|
|
3269
|
+
}
|
|
3270
|
+
|
|
3271
|
+
/** `GET /debug/context?model=` — the context budget a model gets here. */
|
|
3272
|
+
async contextStatus(
|
|
3273
|
+
identity: TurnIdentity,
|
|
3274
|
+
model?: string,
|
|
3275
|
+
byoProvider?: string,
|
|
3276
|
+
sessionId?: string,
|
|
3277
|
+
): Promise<ContextStatus> {
|
|
3278
|
+
await this.ready(identity);
|
|
3279
|
+
const llm = this.ambient?.llm;
|
|
3280
|
+
const modelId =
|
|
3281
|
+
model ??
|
|
3282
|
+
(llm && isProviderAdapter(llm)
|
|
3283
|
+
? llm.modelForRole('main')
|
|
3284
|
+
: DEFAULT_MODEL_ID);
|
|
3285
|
+
const window = await this.contextWindows.resolve(modelId, {
|
|
3286
|
+
...(byoProvider ? { byoProvider } : {}),
|
|
3287
|
+
});
|
|
3288
|
+
const budget = contextBudgetFor(window, contextKnobs(this.env, console));
|
|
3289
|
+
const stats = (await this.resultStore?.stats()) ?? {
|
|
3290
|
+
rows: 0,
|
|
3291
|
+
sqliteBytes: 0,
|
|
3292
|
+
r2Rows: 0,
|
|
3293
|
+
r2Bytes: 0,
|
|
2723
3294
|
};
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
3295
|
+
return {
|
|
3296
|
+
model: modelId,
|
|
3297
|
+
...(byoProvider ? { byoProvider } : {}),
|
|
3298
|
+
window: {
|
|
3299
|
+
tokens: window.tokens,
|
|
3300
|
+
origin: window.origin,
|
|
3301
|
+
...(window.catalogId ? { catalogId: window.catalogId } : {}),
|
|
3302
|
+
},
|
|
3303
|
+
budget: {
|
|
3304
|
+
summarizeAtTokens: budget.summarizeAtTokens,
|
|
3305
|
+
pruneAtTokens: budget.pruneAtTokens,
|
|
3306
|
+
resultCapChars: budget.resultCapChars,
|
|
3307
|
+
requestCapTokens: budget.requestCapTokens,
|
|
3308
|
+
outputReserveTokens: budget.outputReserveTokens,
|
|
3309
|
+
keepMessages: budget.keepMessages,
|
|
3310
|
+
...(budget.summarizeTriggerMessages !== undefined
|
|
3311
|
+
? { summarizeTriggerMessages: budget.summarizeTriggerMessages }
|
|
3312
|
+
: {}),
|
|
3313
|
+
},
|
|
3314
|
+
results: {
|
|
3315
|
+
...stats,
|
|
3316
|
+
tier: this.env.TIER_BUCKET ? 'sqlite+r2' : 'sqlite',
|
|
3317
|
+
},
|
|
3318
|
+
...(sessionId
|
|
3319
|
+
? { session: await this.sessionContextStatus(sessionId) }
|
|
3320
|
+
: {}),
|
|
3321
|
+
};
|
|
3322
|
+
}
|
|
3323
|
+
|
|
3324
|
+
/**
|
|
3325
|
+
* One session's working context plus the guard's counters: what the
|
|
3326
|
+
* next request starts from (the messages the latest checkpoint carries,
|
|
3327
|
+
* a summary plus the kept tail once the history was condensed — the
|
|
3328
|
+
* transcript rows are never condensed and are listed separately) and
|
|
3329
|
+
* what earlier requests needed (pruning, a provider overflow, a
|
|
3330
|
+
* refusal).
|
|
3331
|
+
*/
|
|
3332
|
+
private async sessionContextStatus(
|
|
3333
|
+
sessionId: string,
|
|
3334
|
+
): Promise<SessionContextStatus | null> {
|
|
3335
|
+
if (!(await this.sessions!.getSession(sessionId))) return null;
|
|
3336
|
+
const tuple = await this.saver!.getTuple({
|
|
3337
|
+
configurable: { thread_id: sessionId },
|
|
2732
3338
|
});
|
|
3339
|
+
const channel: unknown = tuple?.checkpoint.channel_values.messages;
|
|
3340
|
+
const context: BaseMessage[] = Array.isArray(channel)
|
|
3341
|
+
? channel.filter(isBaseMessage)
|
|
3342
|
+
: [];
|
|
3343
|
+
const thread = await this.saver!.listThreadMessages(sessionId);
|
|
3344
|
+
const stats = await this.ctx.storage.get<SessionContextCounters>(
|
|
3345
|
+
contextStatsKey(sessionId),
|
|
3346
|
+
);
|
|
3347
|
+
return {
|
|
3348
|
+
id: sessionId,
|
|
3349
|
+
threadMessages: thread.length,
|
|
3350
|
+
contextMessages: context.length,
|
|
3351
|
+
contextSummaries: context.filter(isSummarizationMessage).length,
|
|
3352
|
+
contextToolMessages: context.filter((m) => m.type === 'tool').length,
|
|
3353
|
+
contextTokens: estimateRequestTokens({
|
|
3354
|
+
systemTokens: 0,
|
|
3355
|
+
schemaTokens: 0,
|
|
3356
|
+
messages: context,
|
|
3357
|
+
}),
|
|
3358
|
+
...(stats ?? EMPTY_CONTEXT_COUNTERS),
|
|
3359
|
+
};
|
|
3360
|
+
}
|
|
3361
|
+
|
|
3362
|
+
/** Fold a guard event into the session's stored counters (ordered, never throws). */
|
|
3363
|
+
private recordContextEvent(
|
|
3364
|
+
sessionId: string,
|
|
3365
|
+
event: ContextGuardEvent,
|
|
3366
|
+
): void {
|
|
3367
|
+
const key = contextStatsKey(sessionId);
|
|
3368
|
+
this.contextStatsWrites = this.contextStatsWrites
|
|
3369
|
+
.then(async () => {
|
|
3370
|
+
const prev =
|
|
3371
|
+
(await this.ctx.storage.get<SessionContextCounters>(key)) ??
|
|
3372
|
+
EMPTY_CONTEXT_COUNTERS;
|
|
3373
|
+
const next: SessionContextCounters = {
|
|
3374
|
+
...prev,
|
|
3375
|
+
lastEventAt: new Date().toISOString(),
|
|
3376
|
+
};
|
|
3377
|
+
switch (event.kind) {
|
|
3378
|
+
case 'prune':
|
|
3379
|
+
next.prunes += 1;
|
|
3380
|
+
if (event.stage !== 'soft') next.hardPrunes += 1;
|
|
3381
|
+
next.prunedResults += event.pruned;
|
|
3382
|
+
break;
|
|
3383
|
+
case 'overflow-retry':
|
|
3384
|
+
next.overflowRetries += 1;
|
|
3385
|
+
break;
|
|
3386
|
+
case 'refused':
|
|
3387
|
+
next.refusals += 1;
|
|
3388
|
+
break;
|
|
3389
|
+
}
|
|
3390
|
+
await this.ctx.storage.put(key, next);
|
|
3391
|
+
})
|
|
3392
|
+
.catch((err: unknown) => {
|
|
3393
|
+
console.warn(
|
|
3394
|
+
`[context] could not record a ${event.kind} event for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
3395
|
+
);
|
|
3396
|
+
});
|
|
3397
|
+
}
|
|
3398
|
+
|
|
3399
|
+
/** `GET /debug/runs` — recent runs with their marks and live state. */
|
|
3400
|
+
async runsStatus(): Promise<RunsStatus> {
|
|
3401
|
+
const userDid =
|
|
3402
|
+
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
3403
|
+
if (!this.db && userDid) await this.ready({ userDid });
|
|
3404
|
+
const store = this.runStore;
|
|
3405
|
+
const records = store ? await store.listRecent(50) : [];
|
|
3406
|
+
const runs: RunsStatus['runs'] = [];
|
|
3407
|
+
for (const record of records) {
|
|
3408
|
+
const marks = store ? await store.listMarks(record.runId) : [];
|
|
3409
|
+
runs.push({
|
|
3410
|
+
...runSummaryOf(record),
|
|
3411
|
+
marks: marks.map((m) => ({
|
|
3412
|
+
toolCallId: m.toolCallId,
|
|
3413
|
+
toolName: m.toolName,
|
|
3414
|
+
effect: m.effect,
|
|
3415
|
+
startedAt: m.startedAt,
|
|
3416
|
+
doneAt: m.doneAt,
|
|
3417
|
+
outcome: m.outcome,
|
|
3418
|
+
attempts: m.attempts,
|
|
3419
|
+
})),
|
|
3420
|
+
segments: store ? await store.countSegments(record.runId) : 0,
|
|
3421
|
+
attempts: record.attempts,
|
|
3422
|
+
nextAttemptAt: record.nextAttemptAt,
|
|
3423
|
+
error: record.error,
|
|
3424
|
+
taskRunId: record.taskRunId,
|
|
3425
|
+
});
|
|
3426
|
+
}
|
|
3427
|
+
return {
|
|
3428
|
+
config: { ...this.runConfig },
|
|
3429
|
+
live: this.runs?.snapshot() ?? [],
|
|
3430
|
+
runs,
|
|
3431
|
+
};
|
|
2733
3432
|
}
|
|
2734
3433
|
|
|
2735
3434
|
// ── turn plumbing ──────────────────────────────────────────────────────
|
|
@@ -2930,6 +3629,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2930
3629
|
TurnBody,
|
|
2931
3630
|
'message' | 'timezone' | 'model' | 'tools' | 'agActions' | 'attachments'
|
|
2932
3631
|
>,
|
|
3632
|
+
run: {
|
|
3633
|
+
runId: string;
|
|
3634
|
+
abortController: AbortController;
|
|
3635
|
+
/** A recovery attempt: no new input, the graph continues from the checkpoint. */
|
|
3636
|
+
resumed: boolean;
|
|
3637
|
+
/** The reply text the user already received (resumed attempts). */
|
|
3638
|
+
continuation: string | null;
|
|
3639
|
+
},
|
|
2933
3640
|
) {
|
|
2934
3641
|
const core = this.core;
|
|
2935
3642
|
const baseAmbient = this.ambient!;
|
|
@@ -3004,6 +3711,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3004
3711
|
}
|
|
3005
3712
|
: baseAmbient;
|
|
3006
3713
|
const effectiveModel = byoTurn ? byoTurn.byoModelId : requestedModel;
|
|
3714
|
+
// The turn's context budget, from the main model's window: the model
|
|
3715
|
+
// id the turn will actually run on (platform default, a per-request
|
|
3716
|
+
// choice, or the BYO credential's model) — see docs/plans/context-budgets.md.
|
|
3717
|
+
const mainModelId = byoTurn
|
|
3718
|
+
? byoTurn.mainModelId
|
|
3719
|
+
: (requestedModel ??
|
|
3720
|
+
(isProviderAdapter(baseAmbient.llm)
|
|
3721
|
+
? baseAmbient.llm.modelForRole('main')
|
|
3722
|
+
: DEFAULT_MODEL_ID));
|
|
3723
|
+
const windowResolution = await this.contextWindows.resolve(mainModelId, {
|
|
3724
|
+
...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
|
|
3725
|
+
});
|
|
3726
|
+
const contextBudget = contextBudgetFor(
|
|
3727
|
+
windowResolution,
|
|
3728
|
+
contextKnobs(this.env, console),
|
|
3729
|
+
);
|
|
3007
3730
|
if (
|
|
3008
3731
|
req.client === 'matrix' &&
|
|
3009
3732
|
!(await sessions.getSession(req.sessionId))
|
|
@@ -3016,8 +3739,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3016
3739
|
oracleEntityDid: core.identity.entityDid,
|
|
3017
3740
|
});
|
|
3018
3741
|
}
|
|
3019
|
-
|
|
3020
|
-
|
|
3742
|
+
// The run coordinator already applied the session's multitask rule
|
|
3743
|
+
// (aborted or queued behind the previous run); this attempt's signal
|
|
3744
|
+
// is the run's own.
|
|
3745
|
+
const abortController = run.abortController;
|
|
3021
3746
|
this.aborts.set(req.sessionId, abortController);
|
|
3022
3747
|
|
|
3023
3748
|
// Every turn runs inside the user's oracle room, like the Node
|
|
@@ -3133,12 +3858,55 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3133
3858
|
const hostRoomTitle = opts.hooks?.getRoomTitle;
|
|
3134
3859
|
const hostSafetyModel = opts.hooks?.safetyModel;
|
|
3135
3860
|
|
|
3136
|
-
|
|
3861
|
+
// A resumed attempt reads the client's tool surface back from the
|
|
3862
|
+
// checkpointed state (the run row does not store the catalogue).
|
|
3863
|
+
const priorSurface = priorState as {
|
|
3864
|
+
browserTools?: TurnBody['tools'];
|
|
3865
|
+
agActions?: TurnBody['agActions'];
|
|
3866
|
+
};
|
|
3867
|
+
const browserTools =
|
|
3868
|
+
body.tools ??
|
|
3869
|
+
(run.resumed ? priorSurface.browserTools : undefined) ??
|
|
3870
|
+
[];
|
|
3871
|
+
const agActions =
|
|
3872
|
+
body.agActions ??
|
|
3873
|
+
(run.resumed ? priorSurface.agActions : undefined) ??
|
|
3874
|
+
[];
|
|
3875
|
+
|
|
3876
|
+
// Write-ahead tool marks + the resume policy (tool-marks.ts). The
|
|
3877
|
+
// effect map is filled from the build below; the closure reads it at
|
|
3878
|
+
// call time, so the middleware can be created before the agent.
|
|
3879
|
+
const toolEffects = new Map<string, ToolEffect>();
|
|
3880
|
+
const marksMiddleware = createToolMarksMiddleware({
|
|
3881
|
+
runId: run.runId,
|
|
3882
|
+
store: this.runStore!,
|
|
3883
|
+
effectOf: (name) => toolEffects.get(name) ?? toolEffectOf({ name }),
|
|
3884
|
+
continuation: run.continuation,
|
|
3885
|
+
logger: console,
|
|
3886
|
+
});
|
|
3887
|
+
// Every tool result above the budget's cap is saved whole and handed
|
|
3888
|
+
// to the model as head + tail + handle (result-cap.ts); applied to
|
|
3889
|
+
// sub-agents too through `toolMiddlewares`.
|
|
3890
|
+
const resultStore = this.resultStore;
|
|
3891
|
+
const resultCap = {
|
|
3892
|
+
capChars: contextBudget.resultCapChars,
|
|
3893
|
+
sessionId: req.sessionId,
|
|
3894
|
+
...(resultStore ? { store: resultStore } : {}),
|
|
3895
|
+
exempt: new Set([READ_RESULT_TOOL_NAME]),
|
|
3896
|
+
logger: console,
|
|
3897
|
+
};
|
|
3898
|
+
// The middleware covers what the wrapper cannot: a sub-agent's reply,
|
|
3899
|
+
// which is a tool result to the main agent. Wrapped tools are capped
|
|
3900
|
+
// inside the wrapper (already under the cap by the time it runs).
|
|
3901
|
+
const capMiddleware = createResultCapMiddleware(resultCap);
|
|
3902
|
+
|
|
3903
|
+
const built = await createMainAgent({
|
|
3137
3904
|
registries: core.registries,
|
|
3138
3905
|
identity: core.identity,
|
|
3139
3906
|
config: core.validatedEnv,
|
|
3140
3907
|
availablePlugins: core.availablePlugins,
|
|
3141
3908
|
byoProvider: byoTurn?.provider,
|
|
3909
|
+
contextBudget,
|
|
3142
3910
|
ambient: {
|
|
3143
3911
|
...ambient,
|
|
3144
3912
|
attachments: attachmentAccess,
|
|
@@ -3150,6 +3918,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3150
3918
|
middlewares: this.workStatus
|
|
3151
3919
|
? [createWorkStatusMiddleware({ producer: this.workStatus })]
|
|
3152
3920
|
: [],
|
|
3921
|
+
// Applied to the main agent AND every sub-agent: a sub-agent's
|
|
3922
|
+
// own tool calls are marked and capped too.
|
|
3923
|
+
toolMiddlewares: [marksMiddleware, capMiddleware],
|
|
3924
|
+
resultCap,
|
|
3925
|
+
onContextOverflow: (error) =>
|
|
3926
|
+
this.contextWindows.learnFromError(mainModelId, error, {
|
|
3927
|
+
...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
|
|
3928
|
+
}),
|
|
3929
|
+
onContextEvent: (event) =>
|
|
3930
|
+
this.recordContextEvent(req.sessionId, event),
|
|
3931
|
+
...(resultStore
|
|
3932
|
+
? {
|
|
3933
|
+
readResult: (id: string, offset: number, length: number) =>
|
|
3934
|
+
resultStore.read(id, offset, length),
|
|
3935
|
+
}
|
|
3936
|
+
: {}),
|
|
3153
3937
|
...(hostRoomTitle
|
|
3154
3938
|
? {
|
|
3155
3939
|
getRoomTitle: (roomId: string) =>
|
|
@@ -3185,25 +3969,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3185
3969
|
// AG-UI plugins turn these into tools at build time
|
|
3186
3970
|
// (`getRequestTools` reads `history.state`), so they must be here
|
|
3187
3971
|
// and not only in the graph input.
|
|
3188
|
-
browserTools
|
|
3189
|
-
agActions
|
|
3972
|
+
browserTools,
|
|
3973
|
+
agActions,
|
|
3190
3974
|
},
|
|
3191
3975
|
checkpointer: saver,
|
|
3192
3976
|
abortSignal: abortController.signal,
|
|
3193
3977
|
});
|
|
3978
|
+
const { agent, context } = built;
|
|
3979
|
+
for (const [name, effect] of built.toolEffects)
|
|
3980
|
+
toolEffects.set(name, effect);
|
|
3194
3981
|
|
|
3195
3982
|
const attachmentKwargs =
|
|
3196
3983
|
prepared && prepared.metas.length > 0
|
|
3197
3984
|
? { attachment: prepared.metas[0], attachments: prepared.metas }
|
|
3198
3985
|
: {};
|
|
3986
|
+
// Node's `MessagesService` metadata for room messages: who spoke
|
|
3987
|
+
// (the group-chat tools and prompts read it), and in a group room the
|
|
3988
|
+
// `[DisplayName]: ` prefix on the message itself.
|
|
3989
|
+
const speakerName =
|
|
3990
|
+
req.senderDisplayName ??
|
|
3991
|
+
req.identity.matrixUserId ??
|
|
3992
|
+
req.identity.userDid;
|
|
3993
|
+
const speakerKwargs =
|
|
3994
|
+
req.client === 'matrix'
|
|
3995
|
+
? {
|
|
3996
|
+
senderDid: req.identity.userDid,
|
|
3997
|
+
...(req.identity.matrixUserId
|
|
3998
|
+
? { senderMatrixUserId: req.identity.matrixUserId }
|
|
3999
|
+
: {}),
|
|
4000
|
+
senderDisplayName: speakerName,
|
|
4001
|
+
threadId: req.sessionId,
|
|
4002
|
+
...(req.eventId ? { eventId: req.eventId } : {}),
|
|
4003
|
+
}
|
|
4004
|
+
: {};
|
|
4005
|
+
const rawContent = prepared ? prepared.content : body.message;
|
|
4006
|
+
const content =
|
|
4007
|
+
req.client === 'matrix' && req.roomKind === 'group'
|
|
4008
|
+
? prefixSpeaker(rawContent, speakerName)
|
|
4009
|
+
: rawContent;
|
|
3199
4010
|
const stateInput = {
|
|
3200
4011
|
messages: [
|
|
3201
4012
|
new HumanMessage({
|
|
3202
|
-
content
|
|
4013
|
+
content,
|
|
3203
4014
|
additional_kwargs: {
|
|
3204
4015
|
timestamp: new Date().toISOString(),
|
|
3205
4016
|
oracleName: core.identity.name,
|
|
3206
4017
|
msgFromMatrixRoom: req.client === 'matrix',
|
|
4018
|
+
...speakerKwargs,
|
|
3207
4019
|
...attachmentKwargs,
|
|
3208
4020
|
},
|
|
3209
4021
|
}),
|
|
@@ -3224,8 +4036,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3224
4036
|
config: { did: req.identity.userDid },
|
|
3225
4037
|
client: req.client,
|
|
3226
4038
|
...metadataGraphInput(meta, priorMeta),
|
|
3227
|
-
browserTools
|
|
3228
|
-
agActions
|
|
4039
|
+
browserTools,
|
|
4040
|
+
agActions,
|
|
3229
4041
|
};
|
|
3230
4042
|
// LangSmith: metadata is attached unconditionally (inert without a
|
|
3231
4043
|
// tracer); the explicit tracer only when this turn is traced (global
|
|
@@ -3252,6 +4064,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3252
4064
|
byoNotice,
|
|
3253
4065
|
byoProvider: byoTurn?.provider,
|
|
3254
4066
|
turnDisposables,
|
|
4067
|
+
toolOutputCapChars: contextBudget.resultCapChars,
|
|
3255
4068
|
};
|
|
3256
4069
|
}
|
|
3257
4070
|
|
|
@@ -3358,6 +4171,26 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3358
4171
|
}
|
|
3359
4172
|
}
|
|
3360
4173
|
|
|
4174
|
+
/**
|
|
4175
|
+
* Channel-memory compaction for the gateway's group rooms
|
|
4176
|
+
* (`src/matrix/group-chat.ts`): the platform's small model distils a
|
|
4177
|
+
* batch of room messages with the Node summarizer's prompt. Needs no
|
|
4178
|
+
* user file — only this object's model adapter.
|
|
4179
|
+
*/
|
|
4180
|
+
async summarizeGroupMessages(messages: JsonString): Promise<string | null> {
|
|
4181
|
+
const parsed: unknown = JSON.parse(messages);
|
|
4182
|
+
if (!Array.isArray(parsed) || parsed.length === 0) return null;
|
|
4183
|
+
const model = (this.ambient?.llm ?? this.core.llm).get('session-title', {
|
|
4184
|
+
temperature: 0.2,
|
|
4185
|
+
maxTokens: 800,
|
|
4186
|
+
});
|
|
4187
|
+
return summarizeObservedMessages(
|
|
4188
|
+
model,
|
|
4189
|
+
parsed as ObservedMessage[],
|
|
4190
|
+
console,
|
|
4191
|
+
);
|
|
4192
|
+
}
|
|
4193
|
+
|
|
3361
4194
|
private async generateTitle(
|
|
3362
4195
|
messages: BaseMessage[],
|
|
3363
4196
|
): Promise<string | null> {
|
|
@@ -3428,50 +4261,3 @@ async function* tapMessages(
|
|
|
3428
4261
|
yield evt;
|
|
3429
4262
|
}
|
|
3430
4263
|
}
|
|
3431
|
-
|
|
3432
|
-
/**
|
|
3433
|
-
* Merge the turn's SSE stream with plugin-emitted events. Plugin events are
|
|
3434
|
-
* written straight into the same byte stream as they happen.
|
|
3435
|
-
*/
|
|
3436
|
-
function mergeWithEmitter(
|
|
3437
|
-
base: ReadableStream<Uint8Array>,
|
|
3438
|
-
onRegister: (write: (eventName: string, payload: unknown) => void) => void,
|
|
3439
|
-
): ReadableStream<Uint8Array> {
|
|
3440
|
-
const encoder = new TextEncoder();
|
|
3441
|
-
let closed = false;
|
|
3442
|
-
return new ReadableStream<Uint8Array>({
|
|
3443
|
-
start(controller) {
|
|
3444
|
-
onRegister((eventName, payload) => {
|
|
3445
|
-
if (closed) return;
|
|
3446
|
-
try {
|
|
3447
|
-
controller.enqueue(encoder.encode(formatSSE(eventName, payload)));
|
|
3448
|
-
} catch {
|
|
3449
|
-
closed = true;
|
|
3450
|
-
}
|
|
3451
|
-
});
|
|
3452
|
-
const reader = base.getReader();
|
|
3453
|
-
void (async () => {
|
|
3454
|
-
try {
|
|
3455
|
-
for (;;) {
|
|
3456
|
-
const { done, value } = await reader.read();
|
|
3457
|
-
if (done) break;
|
|
3458
|
-
if (!closed) controller.enqueue(value);
|
|
3459
|
-
}
|
|
3460
|
-
} catch (err) {
|
|
3461
|
-
if (!closed) controller.error(err);
|
|
3462
|
-
} finally {
|
|
3463
|
-
closed = true;
|
|
3464
|
-
try {
|
|
3465
|
-
controller.close();
|
|
3466
|
-
} catch {
|
|
3467
|
-
/* already closed */
|
|
3468
|
-
}
|
|
3469
|
-
}
|
|
3470
|
-
})();
|
|
3471
|
-
},
|
|
3472
|
-
cancel() {
|
|
3473
|
-
closed = true;
|
|
3474
|
-
void base.cancel();
|
|
3475
|
-
},
|
|
3476
|
-
});
|
|
3477
|
-
}
|