@ixo/oracle-runtime-workers 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/package.json +1 -1
- package/src/core/context-budget.ts +201 -0
- package/src/core/context-window.ts +289 -0
- package/src/core/env.ts +38 -0
- package/src/core/main-agent-types.ts +46 -0
- package/src/core/main-agent.ts +86 -9
- package/src/core/middlewares/context-guard.ts +270 -0
- package/src/core/middlewares/result-cap.ts +180 -0
- package/src/core/middlewares/summarization.ts +87 -6
- package/src/core/middlewares/tool-marks.ts +209 -0
- package/src/core/openrouter-pricing.ts +42 -1
- package/src/core/read-result-tool.ts +77 -0
- package/src/core/runtime-context.ts +7 -0
- package/src/core/sub-agent-fallback.ts +20 -3
- package/src/core/wrap-plugin-tool.ts +24 -3
- package/src/do/ambient.ts +17 -0
- package/src/do/contracts.ts +249 -0
- package/src/do/result-store-test-do.ts +88 -0
- package/src/do/result-store.ts +339 -0
- package/src/do/run-buffer.ts +204 -0
- package/src/do/run-coordinator.ts +572 -0
- package/src/do/run-request.ts +52 -0
- package/src/do/run-store-test-do.ts +130 -0
- package/src/do/run-store.ts +604 -0
- package/src/do/session-id-migration.ts +85 -0
- package/src/do/speaker-prefix.ts +37 -0
- package/src/do/sse-stream.ts +567 -400
- package/src/do/transcript.ts +238 -0
- package/src/do/turn-body.ts +10 -0
- package/src/do/user-oracle-do.ts +1019 -169
- package/src/matrix/gateway-do.ts +230 -29
- package/src/matrix/group-chat-summarizer.ts +107 -0
- package/src/matrix/group-chat.ts +1163 -0
- package/src/matrix/inbox-store.ts +23 -3
- package/src/matrix/ingest.ts +54 -11
- package/src/matrix/reply-chain.ts +34 -25
- package/src/owner-store/ixo-vfs-store.ts +12 -5
- package/src/owner-store/matrix-media-store.ts +12 -7
- package/src/owner-store/measure.ts +39 -0
- package/src/owner-store/migrating-store.ts +9 -3
- package/src/owner-store/types.ts +14 -4
- package/src/plugin-api/tool-helper.ts +9 -1
- package/src/plugin-api/types.ts +86 -0
- package/src/plugins/index.ts +3 -0
- package/src/plugins/matrix-group-chats/README.md +70 -0
- package/src/plugins/matrix-group-chats/index.ts +132 -0
- package/src/plugins/matrix-group-chats/tools.ts +172 -0
- package/src/plugins/tasks/tasks-tools.ts +4 -0
- package/src/shell/app.ts +78 -0
- package/src/sqlite/cache-config.ts +27 -6
- package/src/sqlite/chunk-layout.ts +14 -0
- package/src/sqlite/database.ts +188 -19
- package/src/sqlite/do-vfs.ts +963 -99
- package/src/sqlite/index.ts +13 -0
- package/src/sqlite/page-tier.ts +668 -0
- package/src/sqlite/sqlite-saver.ts +81 -0
- package/src/tasks/scheduler.ts +134 -3
- package/src/tasks/store.ts +22 -0
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -20,6 +20,7 @@ import { DurableObject } from 'cloudflare:workers';
|
|
|
20
20
|
import {
|
|
21
21
|
AIMessage,
|
|
22
22
|
HumanMessage,
|
|
23
|
+
isBaseMessage,
|
|
23
24
|
type BaseMessage,
|
|
24
25
|
} from '@langchain/core/messages';
|
|
25
26
|
import {
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
llmEnvFromWorkerEnv,
|
|
51
52
|
resolveLangsmithTracing,
|
|
52
53
|
type OpenRouterLlmAdapter,
|
|
54
|
+
DEFAULT_MODEL_ID,
|
|
53
55
|
} from '../core/llm';
|
|
54
56
|
import { createMainAgent } from '../core/main-agent';
|
|
55
57
|
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
|
@@ -83,12 +85,17 @@ import {
|
|
|
83
85
|
finishCompaction,
|
|
84
86
|
type CompactCursors,
|
|
85
87
|
} from '../sqlite/blob-compactor';
|
|
86
|
-
import { createHash } from 'node:crypto';
|
|
87
88
|
import {
|
|
88
89
|
cachePagesForBytes,
|
|
90
|
+
parseByteSize,
|
|
89
91
|
parseChunkCacheBytes,
|
|
90
92
|
} from '../sqlite/cache-config';
|
|
91
|
-
import { DoSqliteDatabase } from '../sqlite/database';
|
|
93
|
+
import { DoSqliteDatabase, type DoSqliteOpenOptions } from '../sqlite/database';
|
|
94
|
+
import type { TierFlushResult } from '../sqlite/do-vfs';
|
|
95
|
+
import {
|
|
96
|
+
DEFAULT_TIER_HOT_BUDGET_BYTES,
|
|
97
|
+
type PageTierOptions,
|
|
98
|
+
} from '../sqlite/page-tier';
|
|
92
99
|
import {
|
|
93
100
|
shouldVacuum,
|
|
94
101
|
vacuumWanted,
|
|
@@ -107,6 +114,7 @@ import {
|
|
|
107
114
|
} from '../sqlite/sessions-store';
|
|
108
115
|
import { createAttachmentDecryptor } from '@ixo/matrix-bot-workers-sdk';
|
|
109
116
|
import { MatrixMediaOwnerStore } from '../owner-store/matrix-media-store';
|
|
117
|
+
import { measureForSave } from '../owner-store/measure';
|
|
110
118
|
import { MigratingOwnerStore } from '../owner-store/migrating-store';
|
|
111
119
|
import type { OwnerCopy } from '../owner-store/types';
|
|
112
120
|
import {
|
|
@@ -134,16 +142,27 @@ import {
|
|
|
134
142
|
type OracleWorkerEnv,
|
|
135
143
|
type SessionSummary,
|
|
136
144
|
type StorageStatus,
|
|
145
|
+
type RunsStatus,
|
|
146
|
+
type ContextStatus,
|
|
147
|
+
type SessionContextStatus,
|
|
148
|
+
type RunSummary,
|
|
137
149
|
type TurnIdentity,
|
|
138
150
|
type TurnRequest,
|
|
139
151
|
type TurnResult,
|
|
140
152
|
type MemorySchemaDebug,
|
|
141
153
|
TURN_INTERRUPTED_MARKER,
|
|
154
|
+
type JsonString,
|
|
142
155
|
} from './contracts';
|
|
143
156
|
import { turnRecursionLimit } from './turn-config';
|
|
144
157
|
import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
|
|
145
158
|
import { decideBootDirty } from './boot-dirty';
|
|
146
159
|
import { mirrorTxnId, RoomMirror } from './room-mirror';
|
|
160
|
+
import { prefixSpeaker } from './speaker-prefix';
|
|
161
|
+
import {
|
|
162
|
+
type ObservedMessage,
|
|
163
|
+
summarizeObservedMessages,
|
|
164
|
+
} from '../matrix/group-chat-summarizer';
|
|
165
|
+
import { migrateThreadSessionIds } from './session-id-migration';
|
|
147
166
|
import { ReauthPrompter } from './reauth-prompt';
|
|
148
167
|
import {
|
|
149
168
|
createTaskScheduler,
|
|
@@ -161,8 +180,46 @@ import {
|
|
|
161
180
|
type HistoryMessage,
|
|
162
181
|
type IndexableSession,
|
|
163
182
|
} from '../memory/session-history-indexer';
|
|
164
|
-
import {
|
|
183
|
+
import {
|
|
184
|
+
createSseSubscriberStream,
|
|
185
|
+
runTurnFrames,
|
|
186
|
+
SSE_HEADERS,
|
|
187
|
+
} from './sse-stream';
|
|
165
188
|
import { parseTurnBody, type TurnBody } from './turn-body';
|
|
189
|
+
import {
|
|
190
|
+
RunCoordinator,
|
|
191
|
+
type LiveRun,
|
|
192
|
+
type RunOutcome,
|
|
193
|
+
} from './run-coordinator';
|
|
194
|
+
import {
|
|
195
|
+
RunStore,
|
|
196
|
+
runDurabilityConfig,
|
|
197
|
+
type RunDurabilityConfig,
|
|
198
|
+
type RunRecord,
|
|
199
|
+
} from './run-store';
|
|
200
|
+
import {
|
|
201
|
+
runSummaryOf,
|
|
202
|
+
storedRunRequest,
|
|
203
|
+
type StoredRunRequest,
|
|
204
|
+
} from './run-request';
|
|
205
|
+
import {
|
|
206
|
+
createToolMarksMiddleware,
|
|
207
|
+
toolEffectOf,
|
|
208
|
+
type ToolEffect,
|
|
209
|
+
} from '../core/middlewares/tool-marks';
|
|
210
|
+
import { createResultCapMiddleware } from '../core/middlewares/result-cap';
|
|
211
|
+
import { READ_RESULT_TOOL_NAME } from '../core/read-result-tool';
|
|
212
|
+
import {
|
|
213
|
+
ContextWindowResolver,
|
|
214
|
+
contextWindowConfig,
|
|
215
|
+
} from '../core/context-window';
|
|
216
|
+
import { contextBudgetFor, contextKnobs } from '../core/context-budget';
|
|
217
|
+
import {
|
|
218
|
+
estimateRequestTokens,
|
|
219
|
+
type ContextGuardEvent,
|
|
220
|
+
} from '../core/middlewares/context-guard';
|
|
221
|
+
import { fetchOpenRouterContextLengths } from '../core/openrouter-pricing';
|
|
222
|
+
import { ResultStore, resultStoreKnobs } from './result-store';
|
|
166
223
|
import { formatReplay } from '../matrix/replay-format';
|
|
167
224
|
import { retryGateway } from './gateway-retry';
|
|
168
225
|
import {
|
|
@@ -172,7 +229,10 @@ import {
|
|
|
172
229
|
import {
|
|
173
230
|
contentToText,
|
|
174
231
|
isSummarizationMessage,
|
|
232
|
+
pageThreadTranscript,
|
|
233
|
+
TranscriptCursorError,
|
|
175
234
|
transformTranscript,
|
|
235
|
+
type TranscriptPageOptions,
|
|
176
236
|
} from './transcript';
|
|
177
237
|
import { WorkersUcanService } from './ucan-service';
|
|
178
238
|
|
|
@@ -202,6 +262,27 @@ const IDLE_EVICT_MS = 5 * 24 * 60 * 60 * 1000;
|
|
|
202
262
|
/** Set once the legacy Matrix copy has been removed (or found absent): no more lookups. */
|
|
203
263
|
const META_LEGACY_CLEARED = 'meta:legacyCleared';
|
|
204
264
|
const META_USER_DID = 'meta:userDid';
|
|
265
|
+
|
|
266
|
+
/** KV key of a session's context-guard counters (`GET /debug/context?session=`). */
|
|
267
|
+
const contextStatsKey = (sessionId: string): string => `ctxstats:${sessionId}`;
|
|
268
|
+
|
|
269
|
+
type SessionContextCounters = Pick<
|
|
270
|
+
SessionContextStatus,
|
|
271
|
+
| 'prunes'
|
|
272
|
+
| 'hardPrunes'
|
|
273
|
+
| 'prunedResults'
|
|
274
|
+
| 'overflowRetries'
|
|
275
|
+
| 'refusals'
|
|
276
|
+
| 'lastEventAt'
|
|
277
|
+
>;
|
|
278
|
+
|
|
279
|
+
const EMPTY_CONTEXT_COUNTERS: SessionContextCounters = {
|
|
280
|
+
prunes: 0,
|
|
281
|
+
hardPrunes: 0,
|
|
282
|
+
prunedResults: 0,
|
|
283
|
+
overflowRetries: 0,
|
|
284
|
+
refusals: 0,
|
|
285
|
+
};
|
|
205
286
|
/** VFS write generation of the file at the last successful upload (see `DoVfs.writeGeneration`). */
|
|
206
287
|
const META_UPLOADED_GEN = 'meta:uploadedGen';
|
|
207
288
|
/**
|
|
@@ -224,6 +305,14 @@ interface MatrixUserIdCache {
|
|
|
224
305
|
/** Consecutive flush failures (cleared on success). */
|
|
225
306
|
const META_FLUSH_FAILURES = 'meta:flushFailures';
|
|
226
307
|
const META_LAST_VACUUM = 'meta:lastVacuumAt';
|
|
308
|
+
/** When the last R2 page-tier eviction pass ran (see `tierTick`). */
|
|
309
|
+
const META_LAST_TIER_PASS = 'meta:lastTierPassAt';
|
|
310
|
+
/**
|
|
311
|
+
* Eviction passes at most this often per object. The horizon is two daily
|
|
312
|
+
* periods, so four checks a day catch chunks the moment they age out
|
|
313
|
+
* without re-reading the hot set more than that.
|
|
314
|
+
*/
|
|
315
|
+
const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
|
|
227
316
|
/**
|
|
228
317
|
* Persisted twin of the in-memory `dirty` flag. The flag alone dies with an
|
|
229
318
|
* eviction, and evictions are routine — without the marker an alarm waking a
|
|
@@ -314,22 +403,6 @@ interface FlushResult {
|
|
|
314
403
|
}
|
|
315
404
|
|
|
316
405
|
/** Hex SHA-256 of a byte stream, hashed as it flows. */
|
|
317
|
-
async function sha256OfStream(
|
|
318
|
-
stream: ReadableStream<Uint8Array>,
|
|
319
|
-
): Promise<string> {
|
|
320
|
-
const hash = createHash('sha256');
|
|
321
|
-
const reader = stream.getReader();
|
|
322
|
-
try {
|
|
323
|
-
for (;;) {
|
|
324
|
-
const { done, value } = await reader.read();
|
|
325
|
-
if (done) break;
|
|
326
|
-
hash.update(value);
|
|
327
|
-
}
|
|
328
|
-
} finally {
|
|
329
|
-
reader.releaseLock();
|
|
330
|
-
}
|
|
331
|
-
return hash.digest('hex');
|
|
332
|
-
}
|
|
333
406
|
|
|
334
407
|
/** Does this adapter expose provider config + role resolution (platform adapters do)? */
|
|
335
408
|
function isProviderAdapter(llm: LlmAdapter): llm is OpenRouterLlmAdapter {
|
|
@@ -426,6 +499,38 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
426
499
|
if (this.ctx.getWebSockets().length > 0) this.realtime.nextPingAt();
|
|
427
500
|
}
|
|
428
501
|
private readonly aborts = new Map<string, AbortController>();
|
|
502
|
+
/** Durable runs (docs/plans/durable-runs.md): the rows, and the live side. */
|
|
503
|
+
private runStore: RunStore | null = null;
|
|
504
|
+
private runs: RunCoordinator | null = null;
|
|
505
|
+
private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
|
|
506
|
+
this.env,
|
|
507
|
+
);
|
|
508
|
+
/** Whole tool results the result cap saved (docs/plans/context-budgets.md). */
|
|
509
|
+
private resultStore: ResultStore | null = null;
|
|
510
|
+
/**
|
|
511
|
+
* Per-model context windows; learned limits persist in this object's KV
|
|
512
|
+
* storage (`ctxwin:<model>`), so a provider's rejection is remembered
|
|
513
|
+
* across boots.
|
|
514
|
+
*/
|
|
515
|
+
private readonly contextWindows = new ContextWindowResolver({
|
|
516
|
+
config: contextWindowConfig(this.env, console),
|
|
517
|
+
catalog: () =>
|
|
518
|
+
fetchOpenRouterContextLengths({
|
|
519
|
+
apiKey: this.env.OPEN_ROUTER_API_KEY,
|
|
520
|
+
logger: console,
|
|
521
|
+
}),
|
|
522
|
+
learned: {
|
|
523
|
+
get: (model) => this.ctx.storage.get<number>(`ctxwin:${model}`),
|
|
524
|
+
set: (model, tokens) => this.ctx.storage.put(`ctxwin:${model}`, tokens),
|
|
525
|
+
},
|
|
526
|
+
logger: console,
|
|
527
|
+
});
|
|
528
|
+
/**
|
|
529
|
+
* Per-session context-guard counters (`ctxstats:<session>` in KV
|
|
530
|
+
* storage; `GET /debug/context?session=`). Writes are chained so two
|
|
531
|
+
* events of one request (a soft then a hard prune) never race.
|
|
532
|
+
*/
|
|
533
|
+
private contextStatsWrites: Promise<void> = Promise.resolve();
|
|
429
534
|
private readonly delegations = new Map<string, { raw: string }>();
|
|
430
535
|
/** Epoch-ms until which a room-state delegation lookup is not retried. */
|
|
431
536
|
private delegationMissUntil = 0;
|
|
@@ -458,6 +563,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
458
563
|
private mirror: RoomMirror | null = null;
|
|
459
564
|
/** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
|
|
460
565
|
private readonly sessionRooms = new Map<string, string>();
|
|
566
|
+
/** The user's main oracle room, once resolved (the session list is scoped to it). */
|
|
567
|
+
private mainRoomId: string | null = null;
|
|
461
568
|
/** The throttled `delegation_required` prompt (see reauth-prompt.ts). */
|
|
462
569
|
private reauthPrompter: ReauthPrompter | null = null;
|
|
463
570
|
/** Room turns running right now, by Matrix event id: a gateway that asks again attaches instead of re-running. */
|
|
@@ -1017,6 +1124,29 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1017
1124
|
await this.sessions.setup();
|
|
1018
1125
|
this.matrixLedger = new MatrixTurnLedger(liveDb);
|
|
1019
1126
|
await this.matrixLedger.setup();
|
|
1127
|
+
this.runStore = new RunStore(liveDb);
|
|
1128
|
+
await this.runStore.setup();
|
|
1129
|
+
this.resultStore = new ResultStore(liveDb, {
|
|
1130
|
+
...(this.env.TIER_BUCKET ? { bucket: this.env.TIER_BUCKET } : {}),
|
|
1131
|
+
prefix: this.ctx.id.toString(),
|
|
1132
|
+
...resultStoreKnobs(this.env),
|
|
1133
|
+
logger: console,
|
|
1134
|
+
});
|
|
1135
|
+
await this.resultStore.setup();
|
|
1136
|
+
// Threads opened before the runtime adopted Node's session rule
|
|
1137
|
+
// (session id = thread root event id) are renamed to it, once.
|
|
1138
|
+
const renamed = await migrateThreadSessionIds(liveDb, console);
|
|
1139
|
+
if (renamed.renamed > 0) this.markDirty();
|
|
1140
|
+
this.runs = new RunCoordinator({
|
|
1141
|
+
store: this.runStore,
|
|
1142
|
+
config: this.runConfig,
|
|
1143
|
+
instanceId: this.instanceId,
|
|
1144
|
+
log: console,
|
|
1145
|
+
requestAlarm: (at) => this.requestAlarm(at),
|
|
1146
|
+
runAttempt: (live, resumed) => this.runAttempt(live, resumed),
|
|
1147
|
+
checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
|
|
1148
|
+
onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
|
|
1149
|
+
});
|
|
1020
1150
|
|
|
1021
1151
|
const core = this.core;
|
|
1022
1152
|
// Boot-time plugin hooks + collision checks, once per object (memoised).
|
|
@@ -1033,6 +1163,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1033
1163
|
gateway: this.gateway,
|
|
1034
1164
|
runTurn: (req) => this.runTurn(req),
|
|
1035
1165
|
requestAlarm: (at) => this.requestAlarm(at),
|
|
1166
|
+
turnRunLive: (taskRunId) => Boolean(this.runs?.byTaskRunId(taskRunId)),
|
|
1036
1167
|
log: console,
|
|
1037
1168
|
...(Number.isFinite(maxTasks) ? { maxTasksPerUser: maxTasks } : {}),
|
|
1038
1169
|
...(Number.isFinite(minCron) ? { minCronIntervalSec: minCron } : {}),
|
|
@@ -1042,6 +1173,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1042
1173
|
if (next !== null) this.requestAlarm(Math.max(next, Date.now() + 1000));
|
|
1043
1174
|
}
|
|
1044
1175
|
|
|
1176
|
+
// Runs a previous incarnation left in flight: schedule their recovery
|
|
1177
|
+
// (attempts run from the alarm, never inside this boot).
|
|
1178
|
+
try {
|
|
1179
|
+
await this.runs.recoverOrphans();
|
|
1180
|
+
} catch (err) {
|
|
1181
|
+
console.error(
|
|
1182
|
+
`[user-do] run recovery scan failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1045
1186
|
// Resume (or start) legacy-blob compaction in the background: files
|
|
1046
1187
|
// imported from the Node runtime arrive with uncompressed blobs.
|
|
1047
1188
|
this.compactDone ??=
|
|
@@ -1131,7 +1272,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1131
1272
|
if (db) {
|
|
1132
1273
|
await db.close().catch(() => undefined);
|
|
1133
1274
|
}
|
|
1134
|
-
|
|
1275
|
+
const tier = this.tierOptions();
|
|
1276
|
+
await DoSqliteDatabase.wipe(this.ctx, DB_FILE, tier ? { tier } : {});
|
|
1135
1277
|
await this.ctx.storage.delete([
|
|
1136
1278
|
META_OWNER_ETAG,
|
|
1137
1279
|
META_LAST_CHECKSUM,
|
|
@@ -1142,12 +1284,40 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1142
1284
|
]);
|
|
1143
1285
|
}
|
|
1144
1286
|
|
|
1145
|
-
/**
|
|
1146
|
-
|
|
1287
|
+
/**
|
|
1288
|
+
* `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object, and
|
|
1289
|
+
* the R2 page tier when the script binds `TIER_BUCKET` (the object's id
|
|
1290
|
+
* is its key prefix; the knobs are documented in `contracts.ts`).
|
|
1291
|
+
*/
|
|
1292
|
+
private sqliteOpenOptions(): DoSqliteOpenOptions {
|
|
1147
1293
|
const bytes = parseChunkCacheBytes(this.env.CHUNK_CACHE_BYTES, (m) =>
|
|
1148
1294
|
console.warn(`[user-do] ${m}`),
|
|
1149
1295
|
);
|
|
1150
|
-
|
|
1296
|
+
const tier = this.tierOptions();
|
|
1297
|
+
return { cachePages: cachePagesForBytes(bytes), ...(tier && { tier }) };
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
private tierOptions(): PageTierOptions | undefined {
|
|
1301
|
+
const bucket = this.env.TIER_BUCKET;
|
|
1302
|
+
if (!bucket) return undefined;
|
|
1303
|
+
const warn = (m: string): void => console.warn(`[user-do] ${m}`);
|
|
1304
|
+
const hotBudgetBytes = parseByteSize(this.env.TIER_HOT_BUDGET_BYTES, {
|
|
1305
|
+
name: 'TIER_HOT_BUDGET_BYTES',
|
|
1306
|
+
fallback: DEFAULT_TIER_HOT_BUDGET_BYTES,
|
|
1307
|
+
min: 1024 * 1024,
|
|
1308
|
+
max: 1024 * 1024 * 1024,
|
|
1309
|
+
warn,
|
|
1310
|
+
});
|
|
1311
|
+
const periods = Number(this.env.TIER_EVICT_AFTER_PERIODS);
|
|
1312
|
+
const periodMs = Number(this.env.TIER_PERIOD_MS);
|
|
1313
|
+
return {
|
|
1314
|
+
bucket,
|
|
1315
|
+
prefix: this.ctx.id.toString(),
|
|
1316
|
+
hotBudgetBytes,
|
|
1317
|
+
...(Number.isInteger(periods) &&
|
|
1318
|
+
periods >= 1 && { evictAfterPeriods: periods }),
|
|
1319
|
+
...(Number.isFinite(periodMs) && periodMs >= 1000 && { periodMs }),
|
|
1320
|
+
};
|
|
1151
1321
|
}
|
|
1152
1322
|
|
|
1153
1323
|
/**
|
|
@@ -1241,12 +1411,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1241
1411
|
this.ctx.getWebSockets().length > 0 ? this.realtime.pingTick() : null;
|
|
1242
1412
|
const housekeepingAt =
|
|
1243
1413
|
await this.ctx.storage.get<number>(META_HOUSEKEEPING_AT);
|
|
1414
|
+
// Runs in flight in THIS instance need the keep-alive re-armed; a
|
|
1415
|
+
// recovery attempt that came due needs starting. Both are cheap and
|
|
1416
|
+
// do not open the database on a bare object (the coordinator only
|
|
1417
|
+
// exists once the object booted).
|
|
1418
|
+
const keepAliveAt = this.runs?.keepAliveDeadline(wakeAt) ?? null;
|
|
1419
|
+
const recoveryAt = this.runs?.nextRecoveryAt() ?? null;
|
|
1420
|
+
if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000)
|
|
1421
|
+
await this.runs.resumeDue(wakeAt);
|
|
1244
1422
|
if (
|
|
1245
1423
|
nextPingAt !== null &&
|
|
1246
1424
|
housekeepingAt !== undefined &&
|
|
1247
|
-
housekeepingAt > wakeAt + 1000
|
|
1425
|
+
housekeepingAt > wakeAt + 1000 &&
|
|
1426
|
+
keepAliveAt === null &&
|
|
1427
|
+
(recoveryAt === null || recoveryAt > wakeAt + 1000)
|
|
1248
1428
|
) {
|
|
1249
|
-
await this.ctx.storage.setAlarm(
|
|
1429
|
+
await this.ctx.storage.setAlarm(
|
|
1430
|
+
Math.min(nextPingAt, housekeepingAt, recoveryAt ?? Infinity),
|
|
1431
|
+
);
|
|
1250
1432
|
return;
|
|
1251
1433
|
}
|
|
1252
1434
|
|
|
@@ -1281,6 +1463,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1281
1463
|
const now = Date.now();
|
|
1282
1464
|
const deadlines: number[] = [];
|
|
1283
1465
|
|
|
1466
|
+
// Durable runs: start the recovery attempts that are due (the boot
|
|
1467
|
+
// above may have just scheduled them), then keep the alarm inside the
|
|
1468
|
+
// keep-alive horizon while anything executes.
|
|
1469
|
+
if (this.runs) {
|
|
1470
|
+
try {
|
|
1471
|
+
await this.runs.resumeDue(now);
|
|
1472
|
+
} catch (err) {
|
|
1473
|
+
console.error(
|
|
1474
|
+
`[user-do] run recovery tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1475
|
+
);
|
|
1476
|
+
}
|
|
1477
|
+
const keepAlive = this.runs.keepAliveDeadline(now);
|
|
1478
|
+
if (keepAlive !== null) deadlines.push(keepAlive);
|
|
1479
|
+
const nextRecovery = this.runs.nextRecoveryAt();
|
|
1480
|
+
if (nextRecovery !== null)
|
|
1481
|
+
deadlines.push(Math.max(nextRecovery, now + 1000));
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1284
1484
|
if (this.dirty && this.db) {
|
|
1285
1485
|
try {
|
|
1286
1486
|
await this.flushToOwnerStore();
|
|
@@ -1333,13 +1533,32 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1333
1533
|
if (next !== null) deadlines.push(next);
|
|
1334
1534
|
}
|
|
1335
1535
|
|
|
1536
|
+
// R2 page tier: move chunks no turn touched for a couple of days out
|
|
1537
|
+
// of DO storage (see sqlite/page-tier.ts). After the flush above (the
|
|
1538
|
+
// upload reads hot rows, not R2) and never over an in-flight one (it
|
|
1539
|
+
// holds a snapshot). A pass is capped; more work re-arms soon.
|
|
1540
|
+
if (
|
|
1541
|
+
this.db?.tierEnabled &&
|
|
1542
|
+
!this.flushInFlight &&
|
|
1543
|
+
(this.runs?.activeCount ?? 0) === 0
|
|
1544
|
+
) {
|
|
1545
|
+
const next = await this.tierTick(this.db, now);
|
|
1546
|
+
if (next !== null) deadlines.push(next);
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1336
1549
|
// Idle housekeeping: a user silent for a day loses the cached pages —
|
|
1337
1550
|
// their file is safe upstream, and we stop paying to store a copy. Never
|
|
1338
1551
|
// evict while tasks are scheduled: their runs ARE activity. The flush
|
|
1339
1552
|
// above ran first; the wipe additionally verifies the upstream copy
|
|
1340
1553
|
// matches the working copy, else the copy stays until it does.
|
|
1341
1554
|
const idle = now - lastAccess > IDLE_EVICT_MS;
|
|
1342
|
-
if (
|
|
1555
|
+
if (
|
|
1556
|
+
idle &&
|
|
1557
|
+
!this.dirty &&
|
|
1558
|
+
this.db &&
|
|
1559
|
+
deadlines.length === 0 &&
|
|
1560
|
+
(this.runs?.size ?? 0) === 0
|
|
1561
|
+
) {
|
|
1343
1562
|
if (await this.ownerCopyIsCurrent(this.db)) {
|
|
1344
1563
|
await this.wipeWorkingCopy(this.db);
|
|
1345
1564
|
await this.ctx.storage.delete(META_HOUSEKEEPING_AT);
|
|
@@ -1361,6 +1580,71 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1361
1580
|
);
|
|
1362
1581
|
}
|
|
1363
1582
|
|
|
1583
|
+
/**
|
|
1584
|
+
* One page-tier eviction pass, at most once per `TIER_PASS_INTERVAL_MS`
|
|
1585
|
+
* (a pass that hit its segment cap re-arms in a minute). Returns the next
|
|
1586
|
+
* time a pass is due, or null when nothing is pending.
|
|
1587
|
+
*/
|
|
1588
|
+
private async tierTick(
|
|
1589
|
+
db: DoSqliteDatabase,
|
|
1590
|
+
now: number,
|
|
1591
|
+
): Promise<number | null> {
|
|
1592
|
+
const last =
|
|
1593
|
+
(await this.ctx.storage.get<number>(META_LAST_TIER_PASS)) ?? 0;
|
|
1594
|
+
if (now - last < TIER_PASS_INTERVAL_MS)
|
|
1595
|
+
return last + TIER_PASS_INTERVAL_MS;
|
|
1596
|
+
try {
|
|
1597
|
+
const pass = await db.tierFlush();
|
|
1598
|
+
await this.ctx.storage.put(META_LAST_TIER_PASS, now);
|
|
1599
|
+
if (pass.skipped) {
|
|
1600
|
+
if (pass.skipped !== 'no-tier')
|
|
1601
|
+
console.log(
|
|
1602
|
+
`[user-do] tier pass skipped for ${this.userDid}: ${pass.skipped}`,
|
|
1603
|
+
);
|
|
1604
|
+
return now + 60_000;
|
|
1605
|
+
}
|
|
1606
|
+
if (pass.evictedChunks > 0 || pass.remaining > 0) {
|
|
1607
|
+
console.log(
|
|
1608
|
+
`[user-do] tier pass for ${this.userDid}: ${pass.evictedChunks} chunks → R2 in ${pass.segmentsRewritten} segment(s), ${pass.hotRows} hot rows (${Math.round(pass.hotBytes / 1024 / 1024)} MB) kept, ${pass.remaining} segment(s) pending`,
|
|
1609
|
+
);
|
|
1610
|
+
}
|
|
1611
|
+
const budget = this.tierOptions()?.hotBudgetBytes;
|
|
1612
|
+
if (
|
|
1613
|
+
budget !== undefined &&
|
|
1614
|
+
pass.remaining === 0 &&
|
|
1615
|
+
pass.hotBytes > budget
|
|
1616
|
+
) {
|
|
1617
|
+
// Everything left is recent: the user's real working set exceeds
|
|
1618
|
+
// the target — a sizing signal, not a failure.
|
|
1619
|
+
console.warn(
|
|
1620
|
+
`[user-do] tier hot set of ${this.userDid} is ${Math.round(pass.hotBytes / 1024 / 1024)} MB, over the ${Math.round(budget / 1024 / 1024)} MB target (all of it touched within the eviction horizon)`,
|
|
1621
|
+
);
|
|
1622
|
+
}
|
|
1623
|
+
return pass.remaining > 0 ? now + 60_000 : now + TIER_PASS_INTERVAL_MS;
|
|
1624
|
+
} catch (err) {
|
|
1625
|
+
console.error(
|
|
1626
|
+
`[user-do] tier pass failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1627
|
+
);
|
|
1628
|
+
return now + FLUSH_RETRY_DELAY_MS;
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/** Operator/test entry: run an eviction pass now (debug routes). */
|
|
1633
|
+
async tierFlush(
|
|
1634
|
+
opts: { force?: boolean; maxSegments?: number } = {},
|
|
1635
|
+
): Promise<TierFlushResult> {
|
|
1636
|
+
if (!this.db) {
|
|
1637
|
+
const userDid =
|
|
1638
|
+
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
1639
|
+
if (userDid) await this.ready({ userDid });
|
|
1640
|
+
}
|
|
1641
|
+
if (!this.db) throw new Error('no working copy');
|
|
1642
|
+
if (this.flushInFlight) await this.flushInFlight.catch(() => undefined);
|
|
1643
|
+
const pass = await this.db.tierFlush(opts);
|
|
1644
|
+
await this.ctx.storage.put(META_LAST_TIER_PASS, Date.now());
|
|
1645
|
+
return pass;
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1364
1648
|
/**
|
|
1365
1649
|
* One VACUUM policy check. Returns the time of a follow-up check when a
|
|
1366
1650
|
* rebuild is wanted but the object is not quiet yet, else null.
|
|
@@ -1440,6 +1724,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1440
1724
|
);
|
|
1441
1725
|
if (room) {
|
|
1442
1726
|
roomId = room.roomId;
|
|
1727
|
+
if (!o.roomId) this.mainRoomId = room.roomId;
|
|
1443
1728
|
// The marker is a waited, non-durable send: it must never be
|
|
1444
1729
|
// replayed after a gateway restart (its event id IS the session id,
|
|
1445
1730
|
// an orphan marker would be a ghost session). Instead it retries
|
|
@@ -1482,11 +1767,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1482
1767
|
o: { limit?: number; offset?: number } = {},
|
|
1483
1768
|
): Promise<{ sessions: SessionSummary[]; total: number }> {
|
|
1484
1769
|
await this.ready(identity);
|
|
1485
|
-
//
|
|
1486
|
-
//
|
|
1487
|
-
//
|
|
1770
|
+
// Node parity (`SessionsService.listSessions`): the list is the user's
|
|
1771
|
+
// MAIN oracle room only — Portal sessions and the threads opened by
|
|
1772
|
+
// room messages there. Threads in dedicated task rooms (or any other
|
|
1773
|
+
// room) stay out of it, and task runs (`task:<id>`) are hidden
|
|
1774
|
+
// whichever room they deliver to. A user without an oracle room sees
|
|
1775
|
+
// every session, as on Node.
|
|
1776
|
+
const mainRoomId = await this.resolveMainRoomId(identity.userDid);
|
|
1488
1777
|
const { sessions, total } = await this.sessions!.listSessions(
|
|
1489
|
-
undefined,
|
|
1778
|
+
mainRoomId ?? undefined,
|
|
1490
1779
|
o.limit ?? 20,
|
|
1491
1780
|
o.offset ?? 0,
|
|
1492
1781
|
TASK_SESSION_PREFIX,
|
|
@@ -1494,6 +1783,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1494
1783
|
return { sessions: sessions.map(toSummary), total };
|
|
1495
1784
|
}
|
|
1496
1785
|
|
|
1786
|
+
/**
|
|
1787
|
+
* The user↔oracle room, resolved once per instance. `null` is the
|
|
1788
|
+
* definitive "no oracle room"; a transient gateway failure retries and
|
|
1789
|
+
* then fails the request rather than answering with the wrong list.
|
|
1790
|
+
*/
|
|
1791
|
+
private async resolveMainRoomId(userDid: string): Promise<string | null> {
|
|
1792
|
+
if (this.mainRoomId) return this.mainRoomId;
|
|
1793
|
+
const room = await retryGateway(() =>
|
|
1794
|
+
this.gateway.resolveUserRoom(userDid),
|
|
1795
|
+
);
|
|
1796
|
+
if (room) this.mainRoomId = room.roomId;
|
|
1797
|
+
return room?.roomId ?? null;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1497
1800
|
async deleteSession(
|
|
1498
1801
|
identity: TurnIdentity,
|
|
1499
1802
|
sessionId: string,
|
|
@@ -1514,6 +1817,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1514
1817
|
const deleted = await this.sessions!.deleteSession(sessionId);
|
|
1515
1818
|
if (deleted) {
|
|
1516
1819
|
await this.saver!.deleteThread(sessionId);
|
|
1820
|
+
await this.resultStore
|
|
1821
|
+
?.deleteForSession(sessionId)
|
|
1822
|
+
.catch((err: unknown) => {
|
|
1823
|
+
console.warn(
|
|
1824
|
+
`[user-do] could not delete the saved tool results of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1825
|
+
);
|
|
1826
|
+
});
|
|
1827
|
+
await this.ctx.storage.delete(contextStatsKey(sessionId));
|
|
1517
1828
|
this.markDirty();
|
|
1518
1829
|
}
|
|
1519
1830
|
return deleted;
|
|
@@ -1622,6 +1933,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1622
1933
|
return JSON.stringify(dtos);
|
|
1623
1934
|
}
|
|
1624
1935
|
|
|
1936
|
+
async listMessagesPage(
|
|
1937
|
+
identity: TurnIdentity,
|
|
1938
|
+
sessionId: string,
|
|
1939
|
+
options: TranscriptPageOptions,
|
|
1940
|
+
): Promise<
|
|
1941
|
+
{ ok: true; json: string } | { ok: false; status: 400; message: string }
|
|
1942
|
+
> {
|
|
1943
|
+
await this.ready(identity);
|
|
1944
|
+
try {
|
|
1945
|
+
const page = await pageThreadTranscript(
|
|
1946
|
+
this.saver!,
|
|
1947
|
+
sessionId,
|
|
1948
|
+
options,
|
|
1949
|
+
);
|
|
1950
|
+
return { ok: true, json: JSON.stringify(page) };
|
|
1951
|
+
} catch (err) {
|
|
1952
|
+
if (err instanceof TranscriptCursorError)
|
|
1953
|
+
return { ok: false, status: 400, message: err.message };
|
|
1954
|
+
throw err;
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1625
1958
|
/**
|
|
1626
1959
|
* The shell deposited a (new) delegation for this user: adopt it now so
|
|
1627
1960
|
* header-less turns mint from it immediately. Does not boot the object.
|
|
@@ -1760,7 +2093,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1760
2093
|
return { present: false, source: 'none' };
|
|
1761
2094
|
}
|
|
1762
2095
|
|
|
2096
|
+
/**
|
|
2097
|
+
* `POST /messages/abort`: stop the session's active run. The run is
|
|
2098
|
+
* closed as `aborted` with the text streamed so far kept on its row;
|
|
2099
|
+
* a queued run is dropped. (A client that merely disconnects does NOT
|
|
2100
|
+
* reach here — runs outlive their connections.)
|
|
2101
|
+
*/
|
|
1763
2102
|
async abortTurn(sessionId: string): Promise<boolean> {
|
|
2103
|
+
if (this.runs?.abortSession(sessionId)) return true;
|
|
1764
2104
|
const controller = this.aborts.get(sessionId);
|
|
1765
2105
|
if (!controller) return false;
|
|
1766
2106
|
controller.abort();
|
|
@@ -1803,8 +2143,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1803
2143
|
toolCalls: [],
|
|
1804
2144
|
replayed: true,
|
|
1805
2145
|
};
|
|
1806
|
-
case 'interrupted':
|
|
2146
|
+
case 'interrupted': {
|
|
2147
|
+
// The object reset mid-turn. The run may be recovering (durable
|
|
2148
|
+
// runs): attach to it and answer with its reply; only a run that
|
|
2149
|
+
// is over without a reply is refused.
|
|
2150
|
+
const attached = existing?.requestId
|
|
2151
|
+
? await this.attachToRun(existing.requestId, req)
|
|
2152
|
+
: null;
|
|
2153
|
+
if (attached) return attached;
|
|
1807
2154
|
throw new Error(`${TURN_INTERRUPTED_MARKER} (${eventId})`);
|
|
2155
|
+
}
|
|
1808
2156
|
case 'run':
|
|
1809
2157
|
break;
|
|
1810
2158
|
}
|
|
@@ -1851,43 +2199,78 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1851
2199
|
});
|
|
1852
2200
|
card.emit(req.requestId, 'routing');
|
|
1853
2201
|
}
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
2202
|
+
// Every turn is a durable run (docs/plans/durable-runs.md): recorded
|
|
2203
|
+
// before the first model call, kept alive by the alarm, resumed after
|
|
2204
|
+
// a reset. Headless turns (Matrix, tasks) simply await the outcome.
|
|
2205
|
+
const { live } = await this.runs!.begin({
|
|
2206
|
+
runId: crypto.randomUUID(),
|
|
2207
|
+
sessionId: req.sessionId,
|
|
2208
|
+
requestId: req.requestId,
|
|
2209
|
+
client: req.client,
|
|
2210
|
+
request: JSON.stringify(
|
|
2211
|
+
storedRunRequest(req, { timezone: req.identity.timezone }),
|
|
2212
|
+
),
|
|
2213
|
+
multitask: req.multitask ?? this.runConfig.multitaskDefault,
|
|
2214
|
+
...(req.taskRunId ? { taskRunId: req.taskRunId } : {}),
|
|
2215
|
+
});
|
|
2216
|
+
const outcome = await live.done;
|
|
2217
|
+
if (outcome.status !== 'finished') {
|
|
2218
|
+
const reason =
|
|
2219
|
+
outcome.error instanceof Error
|
|
2220
|
+
? outcome.error.message
|
|
2221
|
+
: outcome.error !== undefined
|
|
2222
|
+
? String(outcome.error)
|
|
2223
|
+
: `turn ${outcome.status}`;
|
|
2224
|
+
throw new Error(reason);
|
|
2225
|
+
}
|
|
2226
|
+
card?.emit(req.requestId, 'delivering');
|
|
2227
|
+
return {
|
|
2228
|
+
sessionId: req.sessionId,
|
|
2229
|
+
requestId: req.requestId,
|
|
2230
|
+
text: outcome.text,
|
|
2231
|
+
...(outcome.messageId !== undefined
|
|
2232
|
+
? { messageId: outcome.messageId }
|
|
2233
|
+
: {}),
|
|
2234
|
+
toolCalls: outcome.toolCalls ?? [],
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
|
|
2238
|
+
/**
|
|
2239
|
+
* A Matrix turn asked about again after this object reset: hand back the
|
|
2240
|
+
* reply of its (recovering or finished) run instead of refusing it.
|
|
2241
|
+
*/
|
|
2242
|
+
private async attachToRun(
|
|
2243
|
+
requestId: string,
|
|
2244
|
+
req: TurnRequest,
|
|
2245
|
+
): Promise<TurnResult | null> {
|
|
2246
|
+
const live = this.runs?.byRequestId(requestId);
|
|
2247
|
+
if (live) {
|
|
2248
|
+
console.log(
|
|
2249
|
+
`[user-do] turn ${requestId} for ${req.eventId} is being recovered; the gateway attaches to it`,
|
|
2250
|
+
);
|
|
2251
|
+
const outcome = await live.done;
|
|
2252
|
+
if (outcome.status !== 'finished') return null;
|
|
2253
|
+
return {
|
|
2254
|
+
sessionId: req.sessionId,
|
|
2255
|
+
requestId: req.requestId,
|
|
2256
|
+
text: outcome.text,
|
|
2257
|
+
...(outcome.messageId ? { messageId: outcome.messageId } : {}),
|
|
2258
|
+
toolCalls: outcome.toolCalls ?? [],
|
|
2259
|
+
replayed: true,
|
|
1864
2260
|
};
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
await this.runTurnDisposables(turnDisposables);
|
|
1869
|
-
card?.emit(req.requestId, 'delivering');
|
|
1870
|
-
const toolCalls = messages
|
|
1871
|
-
.filter((m): m is AIMessage => m.type === 'ai')
|
|
1872
|
-
.flatMap((m) =>
|
|
1873
|
-
(m.tool_calls ?? []).map((t) => ({
|
|
1874
|
-
name: t.name,
|
|
1875
|
-
status: 'done' as const,
|
|
1876
|
-
})),
|
|
1877
|
-
);
|
|
2261
|
+
}
|
|
2262
|
+
const record = await this.runStore?.getByRequestId(requestId);
|
|
2263
|
+
if (record?.status === 'finished') {
|
|
1878
2264
|
return {
|
|
1879
2265
|
sessionId: req.sessionId,
|
|
1880
2266
|
requestId: req.requestId,
|
|
1881
|
-
text,
|
|
1882
|
-
...(
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
toolCalls,
|
|
2267
|
+
text: record.partialText ?? '',
|
|
2268
|
+
...(record.messageId ? { messageId: record.messageId } : {}),
|
|
2269
|
+
toolCalls: [],
|
|
2270
|
+
replayed: true,
|
|
1886
2271
|
};
|
|
1887
|
-
} finally {
|
|
1888
|
-
if (this.aborts.get(req.sessionId) === abortController)
|
|
1889
|
-
this.aborts.delete(req.sessionId);
|
|
1890
2272
|
}
|
|
2273
|
+
return null;
|
|
1891
2274
|
}
|
|
1892
2275
|
|
|
1893
2276
|
/**
|
|
@@ -2015,7 +2398,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2015
2398
|
|
|
2016
2399
|
const snapshot = await db.snapshot();
|
|
2017
2400
|
try {
|
|
2018
|
-
|
|
2401
|
+
// One pass yields the change gate (hash) AND the length the store
|
|
2402
|
+
// needs to upload; the upload is the only other read of the file.
|
|
2403
|
+
const { sha256Hex: checksum, gzippedLength } = await measureForSave(
|
|
2404
|
+
snapshot.open(),
|
|
2405
|
+
);
|
|
2019
2406
|
const last = await this.ctx.storage.get<string>(META_LAST_CHECKSUM);
|
|
2020
2407
|
if (last === checksum) {
|
|
2021
2408
|
await this.ctx.storage.put(META_UPLOADED_GEN, snapshot.generation);
|
|
@@ -2030,7 +2417,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2030
2417
|
|
|
2031
2418
|
let saved: { etag: string; bytes: number };
|
|
2032
2419
|
try {
|
|
2033
|
-
saved = await store.save(snapshot);
|
|
2420
|
+
saved = await store.save(snapshot, { gzippedLength });
|
|
2034
2421
|
} catch (err) {
|
|
2035
2422
|
// Never diverted anywhere else: the working copy in Durable Object
|
|
2036
2423
|
// storage is durable, stays dirty, and is retried. Loud from the
|
|
@@ -2156,7 +2543,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2156
2543
|
this.saver = null;
|
|
2157
2544
|
this.sessions = null;
|
|
2158
2545
|
this.matrixLedger = null;
|
|
2546
|
+
this.runStore = null;
|
|
2547
|
+
this.runs = null;
|
|
2548
|
+
this.resultStore = null;
|
|
2159
2549
|
this.sessionRooms.clear();
|
|
2550
|
+
this.mainRoomId = null;
|
|
2160
2551
|
this.ambient = null;
|
|
2161
2552
|
this.secretsService = null;
|
|
2162
2553
|
this.byo = null;
|
|
@@ -2295,6 +2686,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2295
2686
|
rowsWritten: vfs.rowsWritten,
|
|
2296
2687
|
}
|
|
2297
2688
|
: undefined,
|
|
2689
|
+
tier: db
|
|
2690
|
+
? {
|
|
2691
|
+
...db.tierStatus(),
|
|
2692
|
+
lastPassAt:
|
|
2693
|
+
await this.ctx.storage.get<number>(META_LAST_TIER_PASS),
|
|
2694
|
+
}
|
|
2695
|
+
: undefined,
|
|
2298
2696
|
};
|
|
2299
2697
|
}
|
|
2300
2698
|
|
|
@@ -2462,6 +2860,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2462
2860
|
console,
|
|
2463
2861
|
);
|
|
2464
2862
|
}
|
|
2863
|
+
const joinMatch = /^\/runs\/([^/]+)$/.exec(url.pathname);
|
|
2864
|
+
if (joinMatch && request.method === 'GET')
|
|
2865
|
+
return this.joinRun(request, decodeURIComponent(joinMatch[1]!), url);
|
|
2465
2866
|
const match = /^\/turn\/([^/]+)$/.exec(url.pathname);
|
|
2466
2867
|
if (!match || request.method !== 'POST')
|
|
2467
2868
|
return new Response('Not found', { status: 404 });
|
|
@@ -2549,75 +2950,428 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2549
2950
|
}
|
|
2550
2951
|
}
|
|
2551
2952
|
|
|
2953
|
+
// A streamed turn is a durable run: recorded, kept alive and resumable
|
|
2954
|
+
// (docs/plans/durable-runs.md). The response is one subscriber of the
|
|
2955
|
+
// run's buffer; the run itself outlives it.
|
|
2956
|
+
this.replayToRoom(req, body.message, 'user');
|
|
2957
|
+
const agActionNames = (body.agActions ?? []).map((a) => a.name);
|
|
2958
|
+
const { live, queued } = await this.runs!.begin({
|
|
2959
|
+
runId: crypto.randomUUID(),
|
|
2960
|
+
sessionId,
|
|
2961
|
+
requestId,
|
|
2962
|
+
client: 'portal',
|
|
2963
|
+
request: JSON.stringify(
|
|
2964
|
+
storedRunRequest(req, {
|
|
2965
|
+
timezone: body.timezone,
|
|
2966
|
+
agActionNames,
|
|
2967
|
+
stream: true,
|
|
2968
|
+
}),
|
|
2969
|
+
),
|
|
2970
|
+
multitask: body.multitask ?? this.runConfig.multitaskDefault,
|
|
2971
|
+
});
|
|
2972
|
+
if (queued)
|
|
2973
|
+
live.buffer.push('router.update', {
|
|
2974
|
+
step: 'Queued…',
|
|
2975
|
+
sessionId,
|
|
2976
|
+
requestId,
|
|
2977
|
+
runId: live.runId,
|
|
2978
|
+
queued: true,
|
|
2979
|
+
});
|
|
2980
|
+
const responseStream = createSseSubscriberStream({
|
|
2981
|
+
replay: live.buffer.tailAfter(0),
|
|
2982
|
+
buffer: live.buffer,
|
|
2983
|
+
onCancel: () =>
|
|
2984
|
+
console.log(
|
|
2985
|
+
`[user-do] client left run ${live.runId} (${sessionId}); the run continues`,
|
|
2986
|
+
),
|
|
2987
|
+
});
|
|
2988
|
+
return new Response(responseStream, {
|
|
2989
|
+
headers: {
|
|
2990
|
+
...SSE_HEADERS,
|
|
2991
|
+
'x-request-id': requestId,
|
|
2992
|
+
'x-run-id': live.runId,
|
|
2993
|
+
},
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
|
|
2997
|
+
/**
|
|
2998
|
+
* `GET /runs/:id?after=<seq>` — re-join a run: replay the frames after
|
|
2999
|
+
* the cursor (packed segments, then the live tail) and stay attached
|
|
3000
|
+
* until its `done` frame. A run that already ended replays what is left
|
|
3001
|
+
* and closes with a synthetic `done` carrying its final state.
|
|
3002
|
+
*/
|
|
3003
|
+
private async joinRun(
|
|
3004
|
+
request: Request,
|
|
3005
|
+
runId: string,
|
|
3006
|
+
url: URL,
|
|
3007
|
+
): Promise<Response> {
|
|
3008
|
+
const identity = JSON.parse(
|
|
3009
|
+
request.headers.get('x-identity') ?? '{}',
|
|
3010
|
+
) as TurnIdentity;
|
|
3011
|
+
try {
|
|
3012
|
+
await this.ready(identity);
|
|
3013
|
+
} catch (err) {
|
|
3014
|
+
return readyFailureResponse(err);
|
|
3015
|
+
}
|
|
3016
|
+
const afterRaw = Number(url.searchParams.get('after') ?? '0');
|
|
3017
|
+
const after =
|
|
3018
|
+
Number.isFinite(afterRaw) && afterRaw > 0 ? Math.floor(afterRaw) : 0;
|
|
3019
|
+
const joined = await this.runs!.join(runId, after);
|
|
3020
|
+
if (!joined)
|
|
3021
|
+
return Response.json(
|
|
3022
|
+
{ statusCode: 404, message: `Run ${runId} not found` },
|
|
3023
|
+
{ status: 404 },
|
|
3024
|
+
);
|
|
3025
|
+
const { record, replay, buffer } = joined;
|
|
3026
|
+
const ended = !buffer || buffer.isClosed;
|
|
3027
|
+
const stream = createSseSubscriberStream({
|
|
3028
|
+
replay,
|
|
3029
|
+
...(buffer && !buffer.isClosed ? { buffer } : {}),
|
|
3030
|
+
...(ended
|
|
3031
|
+
? {
|
|
3032
|
+
trailer: {
|
|
3033
|
+
event: 'done',
|
|
3034
|
+
data: {
|
|
3035
|
+
runId,
|
|
3036
|
+
status: record.status,
|
|
3037
|
+
...(record.messageId ? { messageId: record.messageId } : {}),
|
|
3038
|
+
...(record.partialText && record.status !== 'finished'
|
|
3039
|
+
? { partialText: record.partialText }
|
|
3040
|
+
: {}),
|
|
3041
|
+
replayed: true,
|
|
3042
|
+
},
|
|
3043
|
+
},
|
|
3044
|
+
}
|
|
3045
|
+
: {}),
|
|
3046
|
+
});
|
|
3047
|
+
return new Response(stream, {
|
|
3048
|
+
headers: {
|
|
3049
|
+
...SSE_HEADERS,
|
|
3050
|
+
'x-request-id': record.requestId,
|
|
3051
|
+
'x-run-id': runId,
|
|
3052
|
+
'x-run-status': record.status,
|
|
3053
|
+
},
|
|
3054
|
+
});
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
/**
|
|
3058
|
+
* One attempt of a run (the coordinator's `runAttempt` host hook): build
|
|
3059
|
+
* the agent for the stored request, stream the turn's frames into the
|
|
3060
|
+
* run's buffer, and report the outcome. `resumed` attempts hand the
|
|
3061
|
+
* graph no new input, so LangGraph continues from the checkpoint's
|
|
3062
|
+
* pending node; the tool-mark middleware keeps that safe.
|
|
3063
|
+
*/
|
|
3064
|
+
private async runAttempt(
|
|
3065
|
+
live: LiveRun,
|
|
3066
|
+
resumed: boolean,
|
|
3067
|
+
): Promise<RunOutcome> {
|
|
3068
|
+
const stored = JSON.parse(live.record.request) as StoredRunRequest;
|
|
3069
|
+
const req = stored.turn;
|
|
2552
3070
|
const {
|
|
2553
3071
|
agent,
|
|
2554
3072
|
stateInput,
|
|
2555
3073
|
config,
|
|
2556
|
-
|
|
3074
|
+
turnDisposables,
|
|
2557
3075
|
byoNotice,
|
|
2558
3076
|
byoProvider,
|
|
2559
|
-
|
|
2560
|
-
} = await this.prepareTurn(
|
|
2561
|
-
|
|
2562
|
-
|
|
3077
|
+
toolOutputCapChars,
|
|
3078
|
+
} = await this.prepareTurn(
|
|
3079
|
+
req,
|
|
3080
|
+
{
|
|
3081
|
+
message: req.message,
|
|
3082
|
+
timezone: stored.timezone ?? req.identity.timezone,
|
|
3083
|
+
model: req.model,
|
|
3084
|
+
...(resumed ? {} : { attachments: req.attachments }),
|
|
3085
|
+
},
|
|
3086
|
+
{
|
|
3087
|
+
runId: live.runId,
|
|
3088
|
+
abortController: live.abort,
|
|
3089
|
+
resumed,
|
|
3090
|
+
continuation: resumed ? live.continuation : null,
|
|
3091
|
+
},
|
|
3092
|
+
);
|
|
3093
|
+
const sessionId = req.sessionId;
|
|
2563
3094
|
const sink = {
|
|
2564
3095
|
emit: (eventName: string, payload: Record<string, unknown>) => {
|
|
2565
|
-
|
|
2566
|
-
|
|
3096
|
+
if (live.buffer.isClosed) return;
|
|
3097
|
+
live.buffer.push(eventName, payload);
|
|
2567
3098
|
},
|
|
2568
3099
|
};
|
|
2569
|
-
let write: ((name: string, payload: unknown) => void) | null = null;
|
|
2570
|
-
sink.emit = (eventName, payload) => write?.(eventName, payload);
|
|
2571
3100
|
this.events.register(sessionId, sink);
|
|
2572
|
-
|
|
3101
|
+
if (byoNotice)
|
|
3102
|
+
sink.emit('error', {
|
|
3103
|
+
...byoNotice,
|
|
3104
|
+
sessionId,
|
|
3105
|
+
requestId: req.requestId,
|
|
3106
|
+
});
|
|
3107
|
+
const events = agent.streamEvents(resumed ? null : stateInput, {
|
|
2573
3108
|
...config,
|
|
2574
3109
|
version: 'v2',
|
|
2575
3110
|
});
|
|
2576
3111
|
const capture: BaseMessage[] = [];
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
3112
|
+
let completed = false;
|
|
3113
|
+
try {
|
|
3114
|
+
const outcome = await runTurnFrames({
|
|
3115
|
+
events: tapMessages(events, capture),
|
|
3116
|
+
sink: live.buffer,
|
|
3117
|
+
sessionId,
|
|
3118
|
+
requestId: req.requestId,
|
|
3119
|
+
runId: live.runId,
|
|
3120
|
+
resumed,
|
|
3121
|
+
abortController: live.abort,
|
|
3122
|
+
mirror: (eventName, payload) =>
|
|
3123
|
+
this.events.emitToTaps(eventName, payload),
|
|
3124
|
+
agActionNames: new Set(stored.agActionNames ?? []),
|
|
3125
|
+
byoProvider,
|
|
3126
|
+
toolOutputCapChars,
|
|
3127
|
+
messageIdOf: () => lastAiMessageId(capture),
|
|
3128
|
+
onFrame: () => this.runs?.touchKeepAlive(),
|
|
3129
|
+
onComplete: async () => {
|
|
3130
|
+
completed = true;
|
|
3131
|
+
this.replayToRoom(req, lastAiText(capture), 'oracle');
|
|
3132
|
+
await this.afterTurn(sessionId, capture);
|
|
3133
|
+
},
|
|
3134
|
+
onError: (err) => {
|
|
3135
|
+
console.error(`[user-do] turn ${req.requestId} failed:`, err);
|
|
3136
|
+
},
|
|
3137
|
+
log: (m) => console.log(`[user-do] ${m}`),
|
|
3138
|
+
});
|
|
3139
|
+
const toolCalls = capture
|
|
3140
|
+
.filter((m): m is AIMessage => m.type === 'ai')
|
|
3141
|
+
.flatMap((m) =>
|
|
3142
|
+
(m.tool_calls ?? []).map((t) => ({
|
|
3143
|
+
name: t.name,
|
|
3144
|
+
status: 'done' as const,
|
|
3145
|
+
})),
|
|
3146
|
+
);
|
|
3147
|
+
const text = completed ? lastAiText(capture) : outcome.fullText;
|
|
3148
|
+
const messageId = completed ? lastAiMessageId(capture) : undefined;
|
|
3149
|
+
return {
|
|
3150
|
+
status:
|
|
3151
|
+
outcome.status === 'completed'
|
|
3152
|
+
? 'finished'
|
|
3153
|
+
: outcome.status === 'aborted'
|
|
3154
|
+
? 'aborted'
|
|
3155
|
+
: 'failed',
|
|
3156
|
+
text: (live.continuation ?? '') + text,
|
|
3157
|
+
...(messageId ? { messageId } : {}),
|
|
3158
|
+
toolCalls,
|
|
3159
|
+
...(outcome.status === 'failed' ? { error: outcome.error } : {}),
|
|
3160
|
+
};
|
|
3161
|
+
} finally {
|
|
2608
3162
|
this.events.unregister(sessionId, sink);
|
|
2609
|
-
|
|
3163
|
+
await this.runTurnDisposables(turnDisposables);
|
|
3164
|
+
if (this.aborts.get(sessionId) === live.abort)
|
|
2610
3165
|
this.aborts.delete(sessionId);
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
/** Latest checkpoint id of a session — recovery's progress marker. */
|
|
3170
|
+
private async checkpointIdOf(sessionId: string): Promise<string | null> {
|
|
3171
|
+
const tuple = await this.saver
|
|
3172
|
+
?.getTupleWithoutMessages({ configurable: { thread_id: sessionId } })
|
|
3173
|
+
.catch(() => undefined);
|
|
3174
|
+
return tuple?.checkpoint.id ?? null;
|
|
3175
|
+
}
|
|
3176
|
+
|
|
3177
|
+
/**
|
|
3178
|
+
* A run reached a terminal state. A recovered task run has no scheduler
|
|
3179
|
+
* continuation waiting on it any more: hand the result (or the failure)
|
|
3180
|
+
* to the scheduler so it is delivered / recorded as it would have been.
|
|
3181
|
+
*/
|
|
3182
|
+
private async onRunEnded(
|
|
3183
|
+
record: RunRecord,
|
|
3184
|
+
outcome: RunOutcome,
|
|
3185
|
+
): Promise<void> {
|
|
3186
|
+
if (!record.taskRunId || !this.taskScheduler) return;
|
|
3187
|
+
if (this.taskScheduler.isRunActive(record.taskRunId)) return;
|
|
3188
|
+
if (outcome.status === 'finished')
|
|
3189
|
+
await this.taskScheduler.completeRecoveredRun(
|
|
3190
|
+
record.taskRunId,
|
|
3191
|
+
outcome.text,
|
|
3192
|
+
);
|
|
3193
|
+
else
|
|
3194
|
+
await this.taskScheduler.failRecoveredRun(
|
|
3195
|
+
record.taskRunId,
|
|
3196
|
+
outcome.status === 'interrupted'
|
|
3197
|
+
? 'interrupted: the object was reset while the run was in progress and recovery did not complete'
|
|
3198
|
+
: `${outcome.status}: ${outcome.error instanceof Error ? outcome.error.message : String(outcome.error ?? outcome.status)}`,
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
|
|
3202
|
+
/** `GET /sessions/:id/run` — the session's active run, if any. */
|
|
3203
|
+
async sessionRun(
|
|
3204
|
+
identity: TurnIdentity,
|
|
3205
|
+
sessionId: string,
|
|
3206
|
+
): Promise<RunSummary | null> {
|
|
3207
|
+
await this.ready(identity);
|
|
3208
|
+
const live = this.runs?.activeForSession(sessionId);
|
|
3209
|
+
const record =
|
|
3210
|
+
live?.record ?? (await this.runStore?.activeForSession(sessionId));
|
|
3211
|
+
return record ? runSummaryOf(record) : null;
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
/** `GET /debug/context?model=` — the context budget a model gets here. */
|
|
3215
|
+
async contextStatus(
|
|
3216
|
+
identity: TurnIdentity,
|
|
3217
|
+
model?: string,
|
|
3218
|
+
byoProvider?: string,
|
|
3219
|
+
sessionId?: string,
|
|
3220
|
+
): Promise<ContextStatus> {
|
|
3221
|
+
await this.ready(identity);
|
|
3222
|
+
const llm = this.ambient?.llm;
|
|
3223
|
+
const modelId =
|
|
3224
|
+
model ??
|
|
3225
|
+
(llm && isProviderAdapter(llm)
|
|
3226
|
+
? llm.modelForRole('main')
|
|
3227
|
+
: DEFAULT_MODEL_ID);
|
|
3228
|
+
const window = await this.contextWindows.resolve(modelId, {
|
|
3229
|
+
...(byoProvider ? { byoProvider } : {}),
|
|
3230
|
+
});
|
|
3231
|
+
const budget = contextBudgetFor(window, contextKnobs(this.env, console));
|
|
3232
|
+
const stats = (await this.resultStore?.stats()) ?? {
|
|
3233
|
+
rows: 0,
|
|
3234
|
+
sqliteBytes: 0,
|
|
3235
|
+
r2Rows: 0,
|
|
3236
|
+
r2Bytes: 0,
|
|
2611
3237
|
};
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
3238
|
+
return {
|
|
3239
|
+
model: modelId,
|
|
3240
|
+
...(byoProvider ? { byoProvider } : {}),
|
|
3241
|
+
window: {
|
|
3242
|
+
tokens: window.tokens,
|
|
3243
|
+
origin: window.origin,
|
|
3244
|
+
...(window.catalogId ? { catalogId: window.catalogId } : {}),
|
|
3245
|
+
},
|
|
3246
|
+
budget: {
|
|
3247
|
+
summarizeAtTokens: budget.summarizeAtTokens,
|
|
3248
|
+
pruneAtTokens: budget.pruneAtTokens,
|
|
3249
|
+
resultCapChars: budget.resultCapChars,
|
|
3250
|
+
requestCapTokens: budget.requestCapTokens,
|
|
3251
|
+
outputReserveTokens: budget.outputReserveTokens,
|
|
3252
|
+
keepMessages: budget.keepMessages,
|
|
3253
|
+
...(budget.summarizeTriggerMessages !== undefined
|
|
3254
|
+
? { summarizeTriggerMessages: budget.summarizeTriggerMessages }
|
|
3255
|
+
: {}),
|
|
3256
|
+
},
|
|
3257
|
+
results: {
|
|
3258
|
+
...stats,
|
|
3259
|
+
tier: this.env.TIER_BUCKET ? 'sqlite+r2' : 'sqlite',
|
|
3260
|
+
},
|
|
3261
|
+
...(sessionId
|
|
3262
|
+
? { session: await this.sessionContextStatus(sessionId) }
|
|
3263
|
+
: {}),
|
|
3264
|
+
};
|
|
3265
|
+
}
|
|
3266
|
+
|
|
3267
|
+
/**
|
|
3268
|
+
* One session's working context plus the guard's counters: what the
|
|
3269
|
+
* next request starts from (the messages the latest checkpoint carries,
|
|
3270
|
+
* a summary plus the kept tail once the history was condensed — the
|
|
3271
|
+
* transcript rows are never condensed and are listed separately) and
|
|
3272
|
+
* what earlier requests needed (pruning, a provider overflow, a
|
|
3273
|
+
* refusal).
|
|
3274
|
+
*/
|
|
3275
|
+
private async sessionContextStatus(
|
|
3276
|
+
sessionId: string,
|
|
3277
|
+
): Promise<SessionContextStatus | null> {
|
|
3278
|
+
if (!(await this.sessions!.getSession(sessionId))) return null;
|
|
3279
|
+
const tuple = await this.saver!.getTuple({
|
|
3280
|
+
configurable: { thread_id: sessionId },
|
|
2620
3281
|
});
|
|
3282
|
+
const channel: unknown = tuple?.checkpoint.channel_values.messages;
|
|
3283
|
+
const context: BaseMessage[] = Array.isArray(channel)
|
|
3284
|
+
? channel.filter(isBaseMessage)
|
|
3285
|
+
: [];
|
|
3286
|
+
const thread = await this.saver!.listThreadMessages(sessionId);
|
|
3287
|
+
const stats = await this.ctx.storage.get<SessionContextCounters>(
|
|
3288
|
+
contextStatsKey(sessionId),
|
|
3289
|
+
);
|
|
3290
|
+
return {
|
|
3291
|
+
id: sessionId,
|
|
3292
|
+
threadMessages: thread.length,
|
|
3293
|
+
contextMessages: context.length,
|
|
3294
|
+
contextSummaries: context.filter(isSummarizationMessage).length,
|
|
3295
|
+
contextToolMessages: context.filter((m) => m.type === 'tool').length,
|
|
3296
|
+
contextTokens: estimateRequestTokens({
|
|
3297
|
+
systemTokens: 0,
|
|
3298
|
+
schemaTokens: 0,
|
|
3299
|
+
messages: context,
|
|
3300
|
+
}),
|
|
3301
|
+
...(stats ?? EMPTY_CONTEXT_COUNTERS),
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
|
|
3305
|
+
/** Fold a guard event into the session's stored counters (ordered, never throws). */
|
|
3306
|
+
private recordContextEvent(
|
|
3307
|
+
sessionId: string,
|
|
3308
|
+
event: ContextGuardEvent,
|
|
3309
|
+
): void {
|
|
3310
|
+
const key = contextStatsKey(sessionId);
|
|
3311
|
+
this.contextStatsWrites = this.contextStatsWrites
|
|
3312
|
+
.then(async () => {
|
|
3313
|
+
const prev =
|
|
3314
|
+
(await this.ctx.storage.get<SessionContextCounters>(key)) ??
|
|
3315
|
+
EMPTY_CONTEXT_COUNTERS;
|
|
3316
|
+
const next: SessionContextCounters = {
|
|
3317
|
+
...prev,
|
|
3318
|
+
lastEventAt: new Date().toISOString(),
|
|
3319
|
+
};
|
|
3320
|
+
switch (event.kind) {
|
|
3321
|
+
case 'prune':
|
|
3322
|
+
next.prunes += 1;
|
|
3323
|
+
if (event.stage !== 'soft') next.hardPrunes += 1;
|
|
3324
|
+
next.prunedResults += event.pruned;
|
|
3325
|
+
break;
|
|
3326
|
+
case 'overflow-retry':
|
|
3327
|
+
next.overflowRetries += 1;
|
|
3328
|
+
break;
|
|
3329
|
+
case 'refused':
|
|
3330
|
+
next.refusals += 1;
|
|
3331
|
+
break;
|
|
3332
|
+
}
|
|
3333
|
+
await this.ctx.storage.put(key, next);
|
|
3334
|
+
})
|
|
3335
|
+
.catch((err: unknown) => {
|
|
3336
|
+
console.warn(
|
|
3337
|
+
`[context] could not record a ${event.kind} event for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
3338
|
+
);
|
|
3339
|
+
});
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
/** `GET /debug/runs` — recent runs with their marks and live state. */
|
|
3343
|
+
async runsStatus(): Promise<RunsStatus> {
|
|
3344
|
+
const userDid =
|
|
3345
|
+
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
3346
|
+
if (!this.db && userDid) await this.ready({ userDid });
|
|
3347
|
+
const store = this.runStore;
|
|
3348
|
+
const records = store ? await store.listRecent(50) : [];
|
|
3349
|
+
const runs: RunsStatus['runs'] = [];
|
|
3350
|
+
for (const record of records) {
|
|
3351
|
+
const marks = store ? await store.listMarks(record.runId) : [];
|
|
3352
|
+
runs.push({
|
|
3353
|
+
...runSummaryOf(record),
|
|
3354
|
+
marks: marks.map((m) => ({
|
|
3355
|
+
toolCallId: m.toolCallId,
|
|
3356
|
+
toolName: m.toolName,
|
|
3357
|
+
effect: m.effect,
|
|
3358
|
+
startedAt: m.startedAt,
|
|
3359
|
+
doneAt: m.doneAt,
|
|
3360
|
+
outcome: m.outcome,
|
|
3361
|
+
attempts: m.attempts,
|
|
3362
|
+
})),
|
|
3363
|
+
segments: store ? await store.countSegments(record.runId) : 0,
|
|
3364
|
+
attempts: record.attempts,
|
|
3365
|
+
nextAttemptAt: record.nextAttemptAt,
|
|
3366
|
+
error: record.error,
|
|
3367
|
+
taskRunId: record.taskRunId,
|
|
3368
|
+
});
|
|
3369
|
+
}
|
|
3370
|
+
return {
|
|
3371
|
+
config: { ...this.runConfig },
|
|
3372
|
+
live: this.runs?.snapshot() ?? [],
|
|
3373
|
+
runs,
|
|
3374
|
+
};
|
|
2621
3375
|
}
|
|
2622
3376
|
|
|
2623
3377
|
// ── turn plumbing ──────────────────────────────────────────────────────
|
|
@@ -2818,6 +3572,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2818
3572
|
TurnBody,
|
|
2819
3573
|
'message' | 'timezone' | 'model' | 'tools' | 'agActions' | 'attachments'
|
|
2820
3574
|
>,
|
|
3575
|
+
run: {
|
|
3576
|
+
runId: string;
|
|
3577
|
+
abortController: AbortController;
|
|
3578
|
+
/** A recovery attempt: no new input, the graph continues from the checkpoint. */
|
|
3579
|
+
resumed: boolean;
|
|
3580
|
+
/** The reply text the user already received (resumed attempts). */
|
|
3581
|
+
continuation: string | null;
|
|
3582
|
+
},
|
|
2821
3583
|
) {
|
|
2822
3584
|
const core = this.core;
|
|
2823
3585
|
const baseAmbient = this.ambient!;
|
|
@@ -2892,6 +3654,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2892
3654
|
}
|
|
2893
3655
|
: baseAmbient;
|
|
2894
3656
|
const effectiveModel = byoTurn ? byoTurn.byoModelId : requestedModel;
|
|
3657
|
+
// The turn's context budget, from the main model's window: the model
|
|
3658
|
+
// id the turn will actually run on (platform default, a per-request
|
|
3659
|
+
// choice, or the BYO credential's model) — see docs/plans/context-budgets.md.
|
|
3660
|
+
const mainModelId = byoTurn
|
|
3661
|
+
? byoTurn.mainModelId
|
|
3662
|
+
: (requestedModel ??
|
|
3663
|
+
(isProviderAdapter(baseAmbient.llm)
|
|
3664
|
+
? baseAmbient.llm.modelForRole('main')
|
|
3665
|
+
: DEFAULT_MODEL_ID));
|
|
3666
|
+
const windowResolution = await this.contextWindows.resolve(mainModelId, {
|
|
3667
|
+
...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
|
|
3668
|
+
});
|
|
3669
|
+
const contextBudget = contextBudgetFor(
|
|
3670
|
+
windowResolution,
|
|
3671
|
+
contextKnobs(this.env, console),
|
|
3672
|
+
);
|
|
2895
3673
|
if (
|
|
2896
3674
|
req.client === 'matrix' &&
|
|
2897
3675
|
!(await sessions.getSession(req.sessionId))
|
|
@@ -2904,8 +3682,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2904
3682
|
oracleEntityDid: core.identity.entityDid,
|
|
2905
3683
|
});
|
|
2906
3684
|
}
|
|
2907
|
-
|
|
2908
|
-
|
|
3685
|
+
// The run coordinator already applied the session's multitask rule
|
|
3686
|
+
// (aborted or queued behind the previous run); this attempt's signal
|
|
3687
|
+
// is the run's own.
|
|
3688
|
+
const abortController = run.abortController;
|
|
2909
3689
|
this.aborts.set(req.sessionId, abortController);
|
|
2910
3690
|
|
|
2911
3691
|
// Every turn runs inside the user's oracle room, like the Node
|
|
@@ -3021,12 +3801,55 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3021
3801
|
const hostRoomTitle = opts.hooks?.getRoomTitle;
|
|
3022
3802
|
const hostSafetyModel = opts.hooks?.safetyModel;
|
|
3023
3803
|
|
|
3024
|
-
|
|
3804
|
+
// A resumed attempt reads the client's tool surface back from the
|
|
3805
|
+
// checkpointed state (the run row does not store the catalogue).
|
|
3806
|
+
const priorSurface = priorState as {
|
|
3807
|
+
browserTools?: TurnBody['tools'];
|
|
3808
|
+
agActions?: TurnBody['agActions'];
|
|
3809
|
+
};
|
|
3810
|
+
const browserTools =
|
|
3811
|
+
body.tools ??
|
|
3812
|
+
(run.resumed ? priorSurface.browserTools : undefined) ??
|
|
3813
|
+
[];
|
|
3814
|
+
const agActions =
|
|
3815
|
+
body.agActions ??
|
|
3816
|
+
(run.resumed ? priorSurface.agActions : undefined) ??
|
|
3817
|
+
[];
|
|
3818
|
+
|
|
3819
|
+
// Write-ahead tool marks + the resume policy (tool-marks.ts). The
|
|
3820
|
+
// effect map is filled from the build below; the closure reads it at
|
|
3821
|
+
// call time, so the middleware can be created before the agent.
|
|
3822
|
+
const toolEffects = new Map<string, ToolEffect>();
|
|
3823
|
+
const marksMiddleware = createToolMarksMiddleware({
|
|
3824
|
+
runId: run.runId,
|
|
3825
|
+
store: this.runStore!,
|
|
3826
|
+
effectOf: (name) => toolEffects.get(name) ?? toolEffectOf({ name }),
|
|
3827
|
+
continuation: run.continuation,
|
|
3828
|
+
logger: console,
|
|
3829
|
+
});
|
|
3830
|
+
// Every tool result above the budget's cap is saved whole and handed
|
|
3831
|
+
// to the model as head + tail + handle (result-cap.ts); applied to
|
|
3832
|
+
// sub-agents too through `toolMiddlewares`.
|
|
3833
|
+
const resultStore = this.resultStore;
|
|
3834
|
+
const resultCap = {
|
|
3835
|
+
capChars: contextBudget.resultCapChars,
|
|
3836
|
+
sessionId: req.sessionId,
|
|
3837
|
+
...(resultStore ? { store: resultStore } : {}),
|
|
3838
|
+
exempt: new Set([READ_RESULT_TOOL_NAME]),
|
|
3839
|
+
logger: console,
|
|
3840
|
+
};
|
|
3841
|
+
// The middleware covers what the wrapper cannot: a sub-agent's reply,
|
|
3842
|
+
// which is a tool result to the main agent. Wrapped tools are capped
|
|
3843
|
+
// inside the wrapper (already under the cap by the time it runs).
|
|
3844
|
+
const capMiddleware = createResultCapMiddleware(resultCap);
|
|
3845
|
+
|
|
3846
|
+
const built = await createMainAgent({
|
|
3025
3847
|
registries: core.registries,
|
|
3026
3848
|
identity: core.identity,
|
|
3027
3849
|
config: core.validatedEnv,
|
|
3028
3850
|
availablePlugins: core.availablePlugins,
|
|
3029
3851
|
byoProvider: byoTurn?.provider,
|
|
3852
|
+
contextBudget,
|
|
3030
3853
|
ambient: {
|
|
3031
3854
|
...ambient,
|
|
3032
3855
|
attachments: attachmentAccess,
|
|
@@ -3038,6 +3861,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3038
3861
|
middlewares: this.workStatus
|
|
3039
3862
|
? [createWorkStatusMiddleware({ producer: this.workStatus })]
|
|
3040
3863
|
: [],
|
|
3864
|
+
// Applied to the main agent AND every sub-agent: a sub-agent's
|
|
3865
|
+
// own tool calls are marked and capped too.
|
|
3866
|
+
toolMiddlewares: [marksMiddleware, capMiddleware],
|
|
3867
|
+
resultCap,
|
|
3868
|
+
onContextOverflow: (error) =>
|
|
3869
|
+
this.contextWindows.learnFromError(mainModelId, error, {
|
|
3870
|
+
...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
|
|
3871
|
+
}),
|
|
3872
|
+
onContextEvent: (event) =>
|
|
3873
|
+
this.recordContextEvent(req.sessionId, event),
|
|
3874
|
+
...(resultStore
|
|
3875
|
+
? {
|
|
3876
|
+
readResult: (id: string, offset: number, length: number) =>
|
|
3877
|
+
resultStore.read(id, offset, length),
|
|
3878
|
+
}
|
|
3879
|
+
: {}),
|
|
3041
3880
|
...(hostRoomTitle
|
|
3042
3881
|
? {
|
|
3043
3882
|
getRoomTitle: (roomId: string) =>
|
|
@@ -3073,25 +3912,53 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3073
3912
|
// AG-UI plugins turn these into tools at build time
|
|
3074
3913
|
// (`getRequestTools` reads `history.state`), so they must be here
|
|
3075
3914
|
// and not only in the graph input.
|
|
3076
|
-
browserTools
|
|
3077
|
-
agActions
|
|
3915
|
+
browserTools,
|
|
3916
|
+
agActions,
|
|
3078
3917
|
},
|
|
3079
3918
|
checkpointer: saver,
|
|
3080
3919
|
abortSignal: abortController.signal,
|
|
3081
3920
|
});
|
|
3921
|
+
const { agent, context } = built;
|
|
3922
|
+
for (const [name, effect] of built.toolEffects)
|
|
3923
|
+
toolEffects.set(name, effect);
|
|
3082
3924
|
|
|
3083
3925
|
const attachmentKwargs =
|
|
3084
3926
|
prepared && prepared.metas.length > 0
|
|
3085
3927
|
? { attachment: prepared.metas[0], attachments: prepared.metas }
|
|
3086
3928
|
: {};
|
|
3929
|
+
// Node's `MessagesService` metadata for room messages: who spoke
|
|
3930
|
+
// (the group-chat tools and prompts read it), and in a group room the
|
|
3931
|
+
// `[DisplayName]: ` prefix on the message itself.
|
|
3932
|
+
const speakerName =
|
|
3933
|
+
req.senderDisplayName ??
|
|
3934
|
+
req.identity.matrixUserId ??
|
|
3935
|
+
req.identity.userDid;
|
|
3936
|
+
const speakerKwargs =
|
|
3937
|
+
req.client === 'matrix'
|
|
3938
|
+
? {
|
|
3939
|
+
senderDid: req.identity.userDid,
|
|
3940
|
+
...(req.identity.matrixUserId
|
|
3941
|
+
? { senderMatrixUserId: req.identity.matrixUserId }
|
|
3942
|
+
: {}),
|
|
3943
|
+
senderDisplayName: speakerName,
|
|
3944
|
+
threadId: req.sessionId,
|
|
3945
|
+
...(req.eventId ? { eventId: req.eventId } : {}),
|
|
3946
|
+
}
|
|
3947
|
+
: {};
|
|
3948
|
+
const rawContent = prepared ? prepared.content : body.message;
|
|
3949
|
+
const content =
|
|
3950
|
+
req.client === 'matrix' && req.roomKind === 'group'
|
|
3951
|
+
? prefixSpeaker(rawContent, speakerName)
|
|
3952
|
+
: rawContent;
|
|
3087
3953
|
const stateInput = {
|
|
3088
3954
|
messages: [
|
|
3089
3955
|
new HumanMessage({
|
|
3090
|
-
content
|
|
3956
|
+
content,
|
|
3091
3957
|
additional_kwargs: {
|
|
3092
3958
|
timestamp: new Date().toISOString(),
|
|
3093
3959
|
oracleName: core.identity.name,
|
|
3094
3960
|
msgFromMatrixRoom: req.client === 'matrix',
|
|
3961
|
+
...speakerKwargs,
|
|
3095
3962
|
...attachmentKwargs,
|
|
3096
3963
|
},
|
|
3097
3964
|
}),
|
|
@@ -3112,8 +3979,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3112
3979
|
config: { did: req.identity.userDid },
|
|
3113
3980
|
client: req.client,
|
|
3114
3981
|
...metadataGraphInput(meta, priorMeta),
|
|
3115
|
-
browserTools
|
|
3116
|
-
agActions
|
|
3982
|
+
browserTools,
|
|
3983
|
+
agActions,
|
|
3117
3984
|
};
|
|
3118
3985
|
// LangSmith: metadata is attached unconditionally (inert without a
|
|
3119
3986
|
// tracer); the explicit tracer only when this turn is traced (global
|
|
@@ -3140,6 +4007,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3140
4007
|
byoNotice,
|
|
3141
4008
|
byoProvider: byoTurn?.provider,
|
|
3142
4009
|
turnDisposables,
|
|
4010
|
+
toolOutputCapChars: contextBudget.resultCapChars,
|
|
3143
4011
|
};
|
|
3144
4012
|
}
|
|
3145
4013
|
|
|
@@ -3235,6 +4103,35 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3235
4103
|
await sessions.setTitle(sessionId, title, { onlyIfUntitled: true });
|
|
3236
4104
|
}
|
|
3237
4105
|
this.markDirty();
|
|
4106
|
+
// Which chunks this turn touched, for the page tier's eviction policy
|
|
4107
|
+
// (one row write; a no-op without a tier).
|
|
4108
|
+
try {
|
|
4109
|
+
this.db?.recordAccess();
|
|
4110
|
+
} catch (err) {
|
|
4111
|
+
console.warn(
|
|
4112
|
+
`[user-do] tier access record failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
4113
|
+
);
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
|
|
4117
|
+
/**
|
|
4118
|
+
* Channel-memory compaction for the gateway's group rooms
|
|
4119
|
+
* (`src/matrix/group-chat.ts`): the platform's small model distils a
|
|
4120
|
+
* batch of room messages with the Node summarizer's prompt. Needs no
|
|
4121
|
+
* user file — only this object's model adapter.
|
|
4122
|
+
*/
|
|
4123
|
+
async summarizeGroupMessages(messages: JsonString): Promise<string | null> {
|
|
4124
|
+
const parsed: unknown = JSON.parse(messages);
|
|
4125
|
+
if (!Array.isArray(parsed) || parsed.length === 0) return null;
|
|
4126
|
+
const model = (this.ambient?.llm ?? this.core.llm).get('session-title', {
|
|
4127
|
+
temperature: 0.2,
|
|
4128
|
+
maxTokens: 800,
|
|
4129
|
+
});
|
|
4130
|
+
return summarizeObservedMessages(
|
|
4131
|
+
model,
|
|
4132
|
+
parsed as ObservedMessage[],
|
|
4133
|
+
console,
|
|
4134
|
+
);
|
|
3238
4135
|
}
|
|
3239
4136
|
|
|
3240
4137
|
private async generateTitle(
|
|
@@ -3307,50 +4204,3 @@ async function* tapMessages(
|
|
|
3307
4204
|
yield evt;
|
|
3308
4205
|
}
|
|
3309
4206
|
}
|
|
3310
|
-
|
|
3311
|
-
/**
|
|
3312
|
-
* Merge the turn's SSE stream with plugin-emitted events. Plugin events are
|
|
3313
|
-
* written straight into the same byte stream as they happen.
|
|
3314
|
-
*/
|
|
3315
|
-
function mergeWithEmitter(
|
|
3316
|
-
base: ReadableStream<Uint8Array>,
|
|
3317
|
-
onRegister: (write: (eventName: string, payload: unknown) => void) => void,
|
|
3318
|
-
): ReadableStream<Uint8Array> {
|
|
3319
|
-
const encoder = new TextEncoder();
|
|
3320
|
-
let closed = false;
|
|
3321
|
-
return new ReadableStream<Uint8Array>({
|
|
3322
|
-
start(controller) {
|
|
3323
|
-
onRegister((eventName, payload) => {
|
|
3324
|
-
if (closed) return;
|
|
3325
|
-
try {
|
|
3326
|
-
controller.enqueue(encoder.encode(formatSSE(eventName, payload)));
|
|
3327
|
-
} catch {
|
|
3328
|
-
closed = true;
|
|
3329
|
-
}
|
|
3330
|
-
});
|
|
3331
|
-
const reader = base.getReader();
|
|
3332
|
-
void (async () => {
|
|
3333
|
-
try {
|
|
3334
|
-
for (;;) {
|
|
3335
|
-
const { done, value } = await reader.read();
|
|
3336
|
-
if (done) break;
|
|
3337
|
-
if (!closed) controller.enqueue(value);
|
|
3338
|
-
}
|
|
3339
|
-
} catch (err) {
|
|
3340
|
-
if (!closed) controller.error(err);
|
|
3341
|
-
} finally {
|
|
3342
|
-
closed = true;
|
|
3343
|
-
try {
|
|
3344
|
-
controller.close();
|
|
3345
|
-
} catch {
|
|
3346
|
-
/* already closed */
|
|
3347
|
-
}
|
|
3348
|
-
}
|
|
3349
|
-
})();
|
|
3350
|
-
},
|
|
3351
|
-
cancel() {
|
|
3352
|
-
closed = true;
|
|
3353
|
-
void base.cancel();
|
|
3354
|
-
},
|
|
3355
|
-
});
|
|
3356
|
-
}
|