@ixo/oracle-runtime-workers 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -10
- package/package.json +13 -8
- package/src/artifacts/config.ts +158 -0
- package/src/artifacts/crypto.ts +94 -0
- package/src/artifacts/routes.ts +73 -0
- package/src/artifacts/store.ts +238 -0
- package/src/artifacts/sweep.ts +200 -0
- package/src/artifacts/tool.ts +103 -0
- package/src/artifacts/viewer-page.ts +614 -0
- package/src/attachments/download.ts +168 -45
- package/src/attachments/index.ts +9 -0
- package/src/attachments/magic.ts +33 -0
- package/src/attachments/pipeline.ts +105 -49
- package/src/attachments/sandbox-archive.ts +9 -8
- package/src/attachments/view-cache.ts +142 -0
- package/src/attachments/view.ts +53 -12
- package/src/channels/auth.ts +267 -0
- package/src/channels/contract.ts +137 -0
- package/src/channels/turns.ts +270 -0
- package/src/core/budgeted-llm.ts +223 -0
- package/src/core/capability-router.ts +273 -0
- package/src/core/context-budget.ts +97 -10
- package/src/core/env.ts +111 -0
- package/src/core/execution-profile.ts +17 -0
- package/src/core/index.ts +267 -9
- package/src/core/llm.ts +30 -9
- package/src/core/main-agent-types.ts +41 -1
- package/src/core/main-agent.ts +367 -65
- package/src/core/manifest.ts +34 -0
- package/src/core/meta-tools.ts +128 -18
- package/src/core/middlewares/capability-gate.ts +130 -12
- package/src/core/middlewares/context-guard.ts +55 -24
- package/src/core/middlewares/index.ts +20 -2
- package/src/core/middlewares/page-context.ts +46 -2
- package/src/core/middlewares/return-direct-first.ts +47 -0
- package/src/core/middlewares/summarization.ts +152 -10
- package/src/core/middlewares/tool-execution.ts +323 -0
- package/src/core/middlewares/tool-repetition-guard.ts +202 -35
- package/src/core/middlewares/turn-boundary.ts +162 -0
- package/src/core/openrouter-pricing.ts +155 -22
- package/src/core/prompt-composer.ts +112 -7
- package/src/core/registries.ts +253 -48
- package/src/core/request-admission.ts +159 -0
- package/src/core/runtime-context.ts +107 -8
- package/src/core/state.ts +1 -1
- package/src/core/sub-agent-fallback.ts +21 -2
- package/src/core/subagent-as-tool.ts +78 -43
- package/src/core/tool-access.ts +189 -0
- package/src/core/tool-scheduler.ts +125 -0
- package/src/core/turn-budget.ts +205 -0
- package/src/core/turn-time-note.ts +43 -0
- package/src/core/user-kv.ts +446 -0
- package/src/core/wrap-plugin-tool.ts +20 -0
- package/src/delivery/plan.ts +278 -0
- package/src/delivery/profile.ts +161 -0
- package/src/delivery/prompt.ts +27 -0
- package/src/delivery/schema.ts +61 -0
- package/src/delivery/shaper.ts +389 -0
- package/src/delivery/types.ts +84 -0
- package/src/do/ambient.ts +51 -15
- package/src/do/contracts.ts +170 -6
- package/src/do/idle-eviction.ts +38 -0
- package/src/do/matrix-turn-ledger.ts +24 -5
- package/src/do/result-store-test-do.ts +137 -3
- package/src/do/result-store.ts +239 -58
- package/src/do/room-mirror.ts +29 -0
- package/src/do/run-buffer.ts +64 -9
- package/src/do/run-coordinator.ts +451 -76
- package/src/do/run-request.ts +2 -0
- package/src/do/run-store-test-do.ts +301 -1
- package/src/do/run-store.ts +385 -43
- package/src/do/secrets-adapter.ts +22 -3
- package/src/do/sse-stream.ts +60 -9
- package/src/do/transcript.ts +48 -5
- package/src/do/turn-body.ts +105 -5
- package/src/do/ucan-service.ts +204 -34
- package/src/do/user-oracle-do.ts +1883 -264
- package/src/feedback/config.ts +127 -0
- package/src/feedback/contract.ts +148 -0
- package/src/feedback/linear-sink.ts +285 -0
- package/src/feedback/privacy.ts +150 -0
- package/src/feedback/reservation.ts +298 -0
- package/src/feedback/submit.ts +247 -0
- package/src/index.ts +138 -6
- package/src/llm/byo-adapter.ts +27 -6
- package/src/llm/byo-catalog.ts +13 -0
- package/src/llm/byo-model-fallback.ts +248 -0
- package/src/llm/byo-service.ts +45 -8
- package/src/llm/provider-error.ts +139 -2
- package/src/matrix/gateway-do.ts +504 -108
- package/src/matrix/group-chat.ts +66 -23
- package/src/matrix/inbox-store.ts +16 -1
- package/src/matrix/ingest.ts +79 -14
- package/src/matrix/media-deadline.ts +130 -0
- package/src/matrix/replay-format.ts +5 -3
- package/src/matrix/reply-parts.ts +92 -0
- package/src/matrix/room-alias.ts +96 -0
- package/src/matrix/user-homeserver.ts +166 -7
- package/src/owner-store/ixo-vfs-store.ts +170 -43
- package/src/owner-store/migrating-store.ts +31 -15
- package/src/owner-store/types.ts +18 -3
- package/src/plugin-api/index.ts +43 -2
- package/src/plugin-api/oracle-plugin.ts +35 -0
- package/src/plugin-api/request-admission.ts +41 -0
- package/src/plugin-api/tool-helper.ts +25 -1
- package/src/plugin-api/tool-plane.ts +92 -0
- package/src/plugin-api/types.ts +194 -6
- package/src/plugins/agui/agui.plugin.ts +31 -16
- package/src/plugins/composio/composio-tools.ts +57 -6
- package/src/plugins/composio/composio-ucan.ts +3 -2
- package/src/plugins/composio/composio.plugin.ts +2 -0
- package/src/plugins/delegated-capabilities.ts +25 -0
- package/src/plugins/editor/content-session.ts +173 -83
- package/src/plugins/editor/content-tools.ts +76 -17
- package/src/plugins/editor/editor-mx.ts +28 -9
- package/src/plugins/editor/editor.plugin.ts +3 -1
- package/src/plugins/editor/failures.ts +28 -7
- package/src/plugins/editor/prompts.ts +2 -1
- package/src/plugins/editor/provider.ts +553 -40
- package/src/plugins/editor/standalone-editor-tool.ts +61 -18
- package/src/plugins/firecrawl/firecrawl-tools.ts +13 -5
- package/src/plugins/flows/action-metadata.ts +111 -11
- package/src/plugins/flows/actions.ts +18 -0
- package/src/plugins/flows/decision-context.ts +31 -0
- package/src/plugins/flows/edit.ts +223 -52
- package/src/plugins/flows/errors.ts +22 -1
- package/src/plugins/flows/flow-doc.ts +92 -9
- package/src/plugins/flows/flows.plugin.ts +11 -5
- package/src/plugins/flows/forms.ts +34 -0
- package/src/plugins/flows/index.ts +2 -0
- package/src/plugins/flows/input-policy.ts +140 -0
- package/src/plugins/flows/prompts.ts +14 -11
- package/src/plugins/flows/read.ts +49 -9
- package/src/plugins/flows/ref-syntax.ts +129 -0
- package/src/plugins/flows/test-homeserver.ts +245 -0
- package/src/plugins/flows/tools/authoring.ts +216 -112
- package/src/plugins/flows/tools/discovery.ts +4 -1
- package/src/plugins/flows/tools/forms.ts +21 -2
- package/src/plugins/flows/tools/settings.ts +34 -13
- package/src/plugins/flows/translator.ts +13 -10
- package/src/plugins/flows/types.ts +167 -42
- package/src/plugins/index.ts +11 -1
- package/src/plugins/ixo-transaction/chain-ids.ts +43 -0
- package/src/plugins/ixo-transaction/index.ts +2 -0
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +119 -0
- package/src/plugins/ixo-transaction/receipts.ts +99 -0
- package/src/plugins/ixo-transaction/tools.ts +329 -0
- package/src/plugins/ixo-transaction/wallet-signing.ts +202 -0
- package/src/plugins/mcp-tool-adapter.ts +23 -2
- package/src/plugins/memory/memory-tools.ts +7 -2
- package/src/plugins/memory/memory-ucan.ts +3 -2
- package/src/plugins/memory/memory.plugin.ts +2 -0
- package/src/plugins/pod-creator/blueprint-store.ts +163 -0
- package/src/plugins/pod-creator/blueprint-types.ts +61 -0
- package/src/plugins/pod-creator/bounded-map.ts +87 -0
- package/src/plugins/pod-creator/capsule-content-client.ts +285 -0
- package/src/plugins/pod-creator/chain-gateway.ts +110 -0
- package/src/plugins/pod-creator/config.ts +85 -0
- package/src/plugins/pod-creator/create-session-store.ts +180 -0
- package/src/plugins/pod-creator/create-tools.ts +538 -0
- package/src/plugins/pod-creator/design-pod-roles.ts +111 -0
- package/src/plugins/pod-creator/index.ts +65 -0
- package/src/plugins/pod-creator/orchestration-tools.ts +172 -0
- package/src/plugins/pod-creator/pod-creator.plugin.ts +190 -0
- package/src/plugins/pod-creator/stage.ts +201 -0
- package/src/plugins/pod-creator/sub-agents.ts +189 -0
- package/src/plugins/portal/action-log.ts +21 -1
- package/src/plugins/portal/declared-tools.ts +90 -0
- package/src/plugins/portal/portal.plugin.ts +42 -16
- package/src/plugins/sandbox/sandbox-bridge.ts +79 -10
- package/src/plugins/sandbox/sandbox-mcp.ts +3 -2
- package/src/plugins/sandbox/sandbox-write-blob.ts +11 -1
- package/src/plugins/sandbox/sandbox.plugin.ts +8 -16
- package/src/plugins/tasks/manifest.ts +1 -1
- package/src/plugins/tasks/middleware.ts +20 -7
- package/src/plugins/tasks/tasks-tools.ts +30 -3
- package/src/plugins/tasks/tasks.plugin.ts +64 -11
- package/src/plugins/user-preferences/schema.ts +12 -1
- package/src/plugins/user-preferences/user-preferences-store.ts +38 -13
- package/src/plugins/user-preferences/user-preferences-tool.ts +3 -2
- package/src/plugins/vfs/vfs-client.ts +250 -71
- package/src/plugins/vfs/vfs-content.ts +30 -11
- package/src/plugins/vfs/vfs-errors.ts +22 -0
- package/src/plugins/vfs/vfs-sandbox-tools.ts +12 -5
- package/src/plugins/vfs/vfs-tools.ts +60 -20
- package/src/realtime/frontend-call-registry.ts +228 -81
- package/src/realtime/realtime-endpoint.ts +215 -44
- package/src/realtime/session-socket-hub.ts +47 -2
- package/src/secrets/secrets-service.ts +9 -3
- package/src/shell/app.ts +529 -82
- package/src/shell/auth.ts +131 -37
- package/src/shell/turn-body-cap.ts +11 -13
- package/src/sqlite/blob-compactor.ts +61 -28
- package/src/sqlite/database.ts +156 -90
- package/src/sqlite/do-vfs.ts +416 -129
- package/src/sqlite/page-tier.ts +133 -40
- package/src/sqlite/serialization.ts +40 -0
- package/src/sqlite/sessions-store.ts +10 -2
- package/src/sqlite/sqlite-saver.ts +139 -19
- package/src/sqlite/user-kv-store.ts +317 -0
- package/src/tasks/schedule.ts +201 -12
- package/src/tasks/scheduler.ts +586 -98
- package/src/tasks/spec.ts +67 -3
- package/src/tasks/store.ts +292 -27
- package/src/tasks/topic-deliverables.ts +70 -0
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
import { ChannelTurns } from '../channels/turns';
|
|
2
|
+
import { assertChannelAttemptAllowed } from '../channels/auth';
|
|
3
|
+
import {
|
|
4
|
+
ChannelError,
|
|
5
|
+
channelRequestHash,
|
|
6
|
+
channelOrigin,
|
|
7
|
+
requireChannelDelegation,
|
|
8
|
+
type ChannelTurnInput,
|
|
9
|
+
type ChannelTurnOutcome,
|
|
10
|
+
} from '../channels/contract';
|
|
1
11
|
/* eslint-disable no-console -- console IS the logger on Workers (Logs/observability). */
|
|
2
12
|
/**
|
|
3
13
|
* `UserOracleDO` — one Durable Object per (user DID, oracle).
|
|
@@ -16,6 +26,12 @@
|
|
|
16
26
|
* A Durable Object is single-threaded, which replaces the Node runtime's
|
|
17
27
|
* per-user ref-counting, busy timeouts and cron locks outright.
|
|
18
28
|
*/
|
|
29
|
+
import {
|
|
30
|
+
TopicOperationId,
|
|
31
|
+
TopicDeliverableRequestSchema,
|
|
32
|
+
type TopicDeliverableCommand,
|
|
33
|
+
type TopicDeliverableResult,
|
|
34
|
+
} from '../tasks/topic-deliverables';
|
|
19
35
|
import { DurableObject } from 'cloudflare:workers';
|
|
20
36
|
import {
|
|
21
37
|
AIMessage,
|
|
@@ -38,8 +54,10 @@ import {
|
|
|
38
54
|
type AttachmentViewSurface,
|
|
39
55
|
type MatrixMediaSource,
|
|
40
56
|
type SandboxUploadConfig,
|
|
57
|
+
DOWNLOAD_TIMEOUT_MS,
|
|
41
58
|
MAX_FILE_SIZE,
|
|
42
59
|
readBytesCapped,
|
|
60
|
+
AttachmentTextCacheStore,
|
|
43
61
|
} from '../attachments';
|
|
44
62
|
import type { RuntimeCore } from '../core';
|
|
45
63
|
import {
|
|
@@ -50,15 +68,35 @@ import {
|
|
|
50
68
|
langsmithEnvFromWorkerEnv,
|
|
51
69
|
llmEnvFromWorkerEnv,
|
|
52
70
|
resolveLangsmithTracing,
|
|
71
|
+
type LangsmithTracingDecision,
|
|
53
72
|
type OpenRouterLlmAdapter,
|
|
54
73
|
DEFAULT_MODEL_ID,
|
|
74
|
+
OPENROUTER_MODEL_MAP,
|
|
55
75
|
} from '../core/llm';
|
|
76
|
+
import { renderTurnTimeNote } from '../core/prompt-composer';
|
|
77
|
+
import { TURN_TIME_NOTE_KWARG, withTurnTimeNote } from '../core/turn-time-note';
|
|
78
|
+
import {
|
|
79
|
+
capabilityRouterMode,
|
|
80
|
+
createCapabilityRouter,
|
|
81
|
+
shadowAgreement,
|
|
82
|
+
type CapabilityRouter,
|
|
83
|
+
} from '../core/capability-router';
|
|
56
84
|
import { createMainAgent } from '../core/main-agent';
|
|
85
|
+
import {
|
|
86
|
+
REQUEST_ADMISSION_TIMEOUT_MS_DEFAULT,
|
|
87
|
+
admitRequest,
|
|
88
|
+
admissionMetadata,
|
|
89
|
+
} from '../core/request-admission';
|
|
90
|
+
import type {
|
|
91
|
+
RequestAdmissionResult,
|
|
92
|
+
RequestDisposition,
|
|
93
|
+
} from '../plugin-api/request-admission';
|
|
94
|
+
import { bootHiddenPlugins } from '../core/tool-access';
|
|
57
95
|
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
|
58
96
|
import type { AmbientServices, LlmAdapter } from '../core/runtime-context';
|
|
59
97
|
import { chatGptBackendFromEnv } from '../llm/byo-client';
|
|
60
98
|
import { createByoLlmAdapter } from '../llm/byo-adapter';
|
|
61
|
-
import { isByoModelId } from '../llm/byo-catalog';
|
|
99
|
+
import { byoModelForRole, isByoModelId } from '../llm/byo-catalog';
|
|
62
100
|
import { handleByoRequest } from '../llm/byo-routes';
|
|
63
101
|
import {
|
|
64
102
|
WorkersByoService,
|
|
@@ -79,7 +117,7 @@ import {
|
|
|
79
117
|
RealtimeEndpoint,
|
|
80
118
|
type RealtimeStatus,
|
|
81
119
|
} from '../realtime/realtime-endpoint';
|
|
82
|
-
import { authenticate } from '../shell/auth';
|
|
120
|
+
import { authConfigFromEnv, authenticate } from '../shell/auth';
|
|
83
121
|
import {
|
|
84
122
|
compactStep,
|
|
85
123
|
finishCompaction,
|
|
@@ -135,7 +173,11 @@ import {
|
|
|
135
173
|
VFS_DEFAULT_BASE_URLS,
|
|
136
174
|
} from '../owner-store/ixo-vfs-store';
|
|
137
175
|
import type { OwnerStore } from '../owner-store/types';
|
|
138
|
-
import {
|
|
176
|
+
import {
|
|
177
|
+
createAmbientServices,
|
|
178
|
+
SessionEventRouter,
|
|
179
|
+
sweepExpiredBlobs,
|
|
180
|
+
} from './ambient';
|
|
139
181
|
import {
|
|
140
182
|
checkpointStorageKey,
|
|
141
183
|
type MatrixGatewayObject,
|
|
@@ -157,7 +199,7 @@ import { turnRecursionLimit } from './turn-config';
|
|
|
157
199
|
import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
|
|
158
200
|
import { decideBootDirty } from './boot-dirty';
|
|
159
201
|
import { decideFlush } from './flush-schedule';
|
|
160
|
-
import { mirrorTxnId, RoomMirror } from './room-mirror';
|
|
202
|
+
import { mirrorTxnId, mirrorEventContent, RoomMirror } from './room-mirror';
|
|
161
203
|
import { prefixSpeaker } from './speaker-prefix';
|
|
162
204
|
import {
|
|
163
205
|
type ObservedMessage,
|
|
@@ -176,6 +218,7 @@ import { WorkStatusProducer } from '../matrix/work-status';
|
|
|
176
218
|
import { fetchUserMatrixServerName } from '../matrix/user-homeserver';
|
|
177
219
|
import { createWorkStatusMiddleware } from '../core/middlewares/work-status';
|
|
178
220
|
import { UserPreferencesStore } from '../plugins/user-preferences/user-preferences-store';
|
|
221
|
+
import { SqliteUserKv } from '../sqlite/user-kv-store';
|
|
179
222
|
import {
|
|
180
223
|
SessionHistoryIndexer,
|
|
181
224
|
type HistoryMessage,
|
|
@@ -210,6 +253,15 @@ import {
|
|
|
210
253
|
type ToolEffect,
|
|
211
254
|
} from '../core/middlewares/tool-marks';
|
|
212
255
|
import { createResultCapMiddleware } from '../core/middlewares/result-cap';
|
|
256
|
+
import { createToolExecutionMiddleware } from '../core/middlewares/tool-execution';
|
|
257
|
+
import { budgetedLlm } from '../core/budgeted-llm';
|
|
258
|
+
import { ToolScheduler, type ToolLane } from '../core/tool-scheduler';
|
|
259
|
+
import {
|
|
260
|
+
HarnessLimitError,
|
|
261
|
+
TurnBudget,
|
|
262
|
+
turnLimitsFromEnv,
|
|
263
|
+
type TurnUsage,
|
|
264
|
+
} from '../core/turn-budget';
|
|
213
265
|
import { READ_RESULT_TOOL_NAME } from '../core/read-result-tool';
|
|
214
266
|
import {
|
|
215
267
|
ContextWindowResolver,
|
|
@@ -222,6 +274,21 @@ import {
|
|
|
222
274
|
} from '../core/middlewares/context-guard';
|
|
223
275
|
import { fetchOpenRouterContextLengths } from '../core/openrouter-pricing';
|
|
224
276
|
import { ResultStore, resultStoreKnobs } from './result-store';
|
|
277
|
+
import { artifactStorageConfig } from '../artifacts/config';
|
|
278
|
+
import {
|
|
279
|
+
ArtifactStore,
|
|
280
|
+
artifactIdFor,
|
|
281
|
+
type OwnedArtifact,
|
|
282
|
+
} from '../artifacts/store';
|
|
283
|
+
import { buildCreateArtifactTool } from '../artifacts/tool';
|
|
284
|
+
import { resolveDeliveryProfile } from '../delivery/profile';
|
|
285
|
+
import {
|
|
286
|
+
draftReplyPlan,
|
|
287
|
+
materializeReplyPlan,
|
|
288
|
+
turnSteps,
|
|
289
|
+
} from '../delivery/plan';
|
|
290
|
+
import { parseReplyPlan } from '../delivery/schema';
|
|
291
|
+
import type { DeliveryProfile, ReplyPlan } from '../delivery/types';
|
|
225
292
|
import { formatReplay } from '../matrix/replay-format';
|
|
226
293
|
import { retryGateway } from './gateway-retry';
|
|
227
294
|
import {
|
|
@@ -235,8 +302,55 @@ import {
|
|
|
235
302
|
TranscriptCursorError,
|
|
236
303
|
transformTranscript,
|
|
237
304
|
type TranscriptPageOptions,
|
|
305
|
+
visibleText,
|
|
238
306
|
} from './transcript';
|
|
239
|
-
import {
|
|
307
|
+
import { evictIdleWorkingCopy } from './idle-eviction';
|
|
308
|
+
import { FeedbackMarkers, reserveFeedback } from '../feedback/reservation';
|
|
309
|
+
import type {
|
|
310
|
+
FeedbackReservation,
|
|
311
|
+
FeedbackSettlement,
|
|
312
|
+
FeedbackTarget,
|
|
313
|
+
} from '../feedback/contract';
|
|
314
|
+
import { resolveTurnDelegation, WorkersUcanService } from './ucan-service';
|
|
315
|
+
import type { UcanDelegation } from '../plugin-api/types';
|
|
316
|
+
|
|
317
|
+
/** What one attempt's agent build reads from the request body. */
|
|
318
|
+
type TurnBuildBody = Pick<
|
|
319
|
+
TurnBody,
|
|
320
|
+
'message' | 'timezone' | 'model' | 'tools' | 'agActions' | 'attachments'
|
|
321
|
+
>;
|
|
322
|
+
|
|
323
|
+
/** The run an attempt's agent build belongs to. */
|
|
324
|
+
interface TurnBuildRun {
|
|
325
|
+
runId: string;
|
|
326
|
+
abortController: AbortController;
|
|
327
|
+
/** A recovery attempt: no new input, the graph continues from the checkpoint. */
|
|
328
|
+
resumed: boolean;
|
|
329
|
+
/** The reply text the user already received (resumed attempts). */
|
|
330
|
+
continuation: string | null;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Whether a turn is offered to the plugins' admission handlers. Not for a
|
|
335
|
+
* turn with attachments, a scheduled task run, or a Matrix group room: a
|
|
336
|
+
* direct read is posted into the room, so one user's authorized read would
|
|
337
|
+
* be shown to every member.
|
|
338
|
+
*/
|
|
339
|
+
function admissionApplies(req: TurnRequest): boolean {
|
|
340
|
+
return (
|
|
341
|
+
!req.attachments?.length &&
|
|
342
|
+
!req.taskRunId &&
|
|
343
|
+
!req.sessionId.startsWith(TASK_SESSION_PREFIX) &&
|
|
344
|
+
req.roomKind !== 'group'
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function admissionTimeoutMs(env: Record<string, unknown>): number {
|
|
349
|
+
const ms = env.REQUEST_ADMISSION_TIMEOUT_MS;
|
|
350
|
+
return typeof ms === 'number' && Number.isFinite(ms) && ms > 0
|
|
351
|
+
? ms
|
|
352
|
+
: REQUEST_ADMISSION_TIMEOUT_MS_DEFAULT;
|
|
353
|
+
}
|
|
240
354
|
|
|
241
355
|
const DB_FILE = 'oracle.db';
|
|
242
356
|
/**
|
|
@@ -261,8 +375,31 @@ const FLUSH_FAILURES_ERROR_THRESHOLD = 3;
|
|
|
261
375
|
* more for anyone who comes back within that window.
|
|
262
376
|
*/
|
|
263
377
|
const IDLE_EVICT_MS = 5 * 24 * 60 * 60 * 1000;
|
|
378
|
+
/**
|
|
379
|
+
* A stream of requests to a loaded object persists its last access at most
|
|
380
|
+
* this often. The idle checks read the in-memory value as well, so what an
|
|
381
|
+
* instance saw is never lost to the throttle while it lives; an instance
|
|
382
|
+
* that unloads takes at most this much of its last access with it, against
|
|
383
|
+
* the five-day idle horizon.
|
|
384
|
+
*/
|
|
385
|
+
const ACCESS_PERSIST_INTERVAL_MS = 60 * 60_000;
|
|
386
|
+
/**
|
|
387
|
+
* The session title is a separate model call after the reply; it runs in
|
|
388
|
+
* the background and is abandoned after this long.
|
|
389
|
+
*/
|
|
390
|
+
const TITLE_TIMEOUT_MS = 15_000;
|
|
391
|
+
/** A new sweep of expired ambient blobs starts at most this often per instance. */
|
|
392
|
+
const BLOB_SWEEP_INTERVAL_MS = 60 * 60_000;
|
|
393
|
+
/** The Blocksync lookup of the user's homeserver is abandoned after this long (the fallback applies). */
|
|
394
|
+
const HOMESERVER_LOOKUP_TIMEOUT_MS = 3_000;
|
|
264
395
|
/** Set once the legacy Matrix copy has been removed (or found absent): no more lookups. */
|
|
265
396
|
const META_LEGACY_CLEARED = 'meta:legacyCleared';
|
|
397
|
+
/**
|
|
398
|
+
* Set once a zero-turn working copy looked for a legacy Matrix copy to adopt
|
|
399
|
+
* and found none, or one without turns: later boots skip the gateway round
|
|
400
|
+
* trips. Cleared when a legacy copy is imported after all.
|
|
401
|
+
*/
|
|
402
|
+
const META_LEGACY_UNUSABLE = 'meta:legacyUnusable';
|
|
266
403
|
const META_USER_DID = 'meta:userDid';
|
|
267
404
|
|
|
268
405
|
/** KV key of a session's context-guard counters (`GET /debug/context?session=`). */
|
|
@@ -466,6 +603,62 @@ function lastAiText(messages: BaseMessage[]): string {
|
|
|
466
603
|
}
|
|
467
604
|
|
|
468
605
|
/** Id of the final assistant message (what `POST /messages` reports as `message.id`). */
|
|
606
|
+
/** `turn_runs.usage` for `/debug/runs`; a row written before the column existed, or an unreadable one, is null. */
|
|
607
|
+
function parseUsage(raw: string | null): TurnUsage | null {
|
|
608
|
+
if (!raw) return null;
|
|
609
|
+
try {
|
|
610
|
+
const parsed: unknown = JSON.parse(raw);
|
|
611
|
+
if (!parsed || typeof parsed !== 'object') return null;
|
|
612
|
+
const num = (key: string): number | undefined => {
|
|
613
|
+
const value: unknown = Reflect.get(parsed, key);
|
|
614
|
+
return typeof value === 'number' ? value : undefined;
|
|
615
|
+
};
|
|
616
|
+
const limits: unknown = Reflect.get(parsed, 'limits');
|
|
617
|
+
const limit = (key: string): number | undefined => {
|
|
618
|
+
if (!limits || typeof limits !== 'object') return undefined;
|
|
619
|
+
const value: unknown = Reflect.get(limits, key);
|
|
620
|
+
return typeof value === 'number' ? value : undefined;
|
|
621
|
+
};
|
|
622
|
+
const usage = {
|
|
623
|
+
tokens: num('tokens'),
|
|
624
|
+
reportedTokens: num('reportedTokens'),
|
|
625
|
+
modelCalls: num('modelCalls'),
|
|
626
|
+
toolAttempts: num('toolAttempts'),
|
|
627
|
+
elapsedMs: num('elapsedMs'),
|
|
628
|
+
limits: {
|
|
629
|
+
tokens: limit('tokens'),
|
|
630
|
+
tools: limit('tools'),
|
|
631
|
+
durationMs: limit('durationMs'),
|
|
632
|
+
},
|
|
633
|
+
};
|
|
634
|
+
if (
|
|
635
|
+
usage.tokens === undefined ||
|
|
636
|
+
usage.reportedTokens === undefined ||
|
|
637
|
+
usage.modelCalls === undefined ||
|
|
638
|
+
usage.toolAttempts === undefined ||
|
|
639
|
+
usage.elapsedMs === undefined ||
|
|
640
|
+
usage.limits.tokens === undefined ||
|
|
641
|
+
usage.limits.tools === undefined ||
|
|
642
|
+
usage.limits.durationMs === undefined
|
|
643
|
+
)
|
|
644
|
+
return null;
|
|
645
|
+
return {
|
|
646
|
+
tokens: usage.tokens,
|
|
647
|
+
reportedTokens: usage.reportedTokens,
|
|
648
|
+
modelCalls: usage.modelCalls,
|
|
649
|
+
toolAttempts: usage.toolAttempts,
|
|
650
|
+
elapsedMs: usage.elapsedMs,
|
|
651
|
+
limits: {
|
|
652
|
+
tokens: usage.limits.tokens,
|
|
653
|
+
tools: usage.limits.tools,
|
|
654
|
+
durationMs: usage.limits.durationMs,
|
|
655
|
+
},
|
|
656
|
+
};
|
|
657
|
+
} catch {
|
|
658
|
+
return null;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
469
662
|
function lastAiMessageId(messages: BaseMessage[]): string | undefined {
|
|
470
663
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
471
664
|
const m = messages[i];
|
|
@@ -510,14 +703,38 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
510
703
|
if (this.ctx.getWebSockets().length > 0) this.realtime.nextPingAt();
|
|
511
704
|
}
|
|
512
705
|
private readonly aborts = new Map<string, AbortController>();
|
|
706
|
+
/** Capability router (core/capability-router.ts); built with the ambient services. */
|
|
707
|
+
private capabilityRouter: CapabilityRouter | null = null;
|
|
708
|
+
/**
|
|
709
|
+
* Shadow-mode predictions awaiting the turn's end, by session: what the
|
|
710
|
+
* router would have preloaded, against the plugins loaded before the turn.
|
|
711
|
+
*/
|
|
712
|
+
private readonly shadowRoutes = new Map<
|
|
713
|
+
string,
|
|
714
|
+
{
|
|
715
|
+
requestId: string;
|
|
716
|
+
priorLoaded: ReadonlySet<string>;
|
|
717
|
+
wouldPreload: readonly string[];
|
|
718
|
+
}
|
|
719
|
+
>();
|
|
513
720
|
/** Durable runs (docs/plans/durable-runs.md): the rows, and the live side. */
|
|
514
721
|
private runStore: RunStore | null = null;
|
|
722
|
+
/**
|
|
723
|
+
* Tool concurrency of this user object (tool-scheduler.ts): writes one
|
|
724
|
+
* at a time across every session and turn, reads and sub-agents in
|
|
725
|
+
* bounded lanes. Lives as long as the object.
|
|
726
|
+
*/
|
|
727
|
+
private readonly toolScheduler = new ToolScheduler();
|
|
515
728
|
private runs: RunCoordinator | null = null;
|
|
516
729
|
private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
|
|
517
730
|
this.env,
|
|
518
731
|
);
|
|
519
732
|
/** Whole tool results the result cap saved (docs/plans/context-budgets.md). */
|
|
520
733
|
private resultStore: ResultStore | null = null;
|
|
734
|
+
/** Chat delivery's artefacts; null when no bucket is bound (see `canShare` for new ones). */
|
|
735
|
+
private artifacts: ArtifactStore | null = null;
|
|
736
|
+
/** Text `view_attachment` extracted, per session (`attachments/view-cache.ts`). */
|
|
737
|
+
private attachmentTextCache: AttachmentTextCacheStore | null = null;
|
|
521
738
|
/**
|
|
522
739
|
* Per-model context windows; learned limits persist in this object's KV
|
|
523
740
|
* storage (`ctxwin:<model>`), so a provider's rejection is remembered
|
|
@@ -529,6 +746,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
529
746
|
fetchOpenRouterContextLengths({
|
|
530
747
|
apiKey: this.env.OPEN_ROUTER_API_KEY,
|
|
531
748
|
logger: console,
|
|
749
|
+
// A background refresh of the catalogue outlives the request.
|
|
750
|
+
waitUntil: (work) => this.ctx.waitUntil(work),
|
|
532
751
|
}),
|
|
533
752
|
learned: {
|
|
534
753
|
get: (model) => this.ctx.storage.get<number>(`ctxwin:${model}`),
|
|
@@ -546,6 +765,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
546
765
|
/** Epoch-ms until which a room-state delegation lookup is not retried. */
|
|
547
766
|
private delegationMissUntil = 0;
|
|
548
767
|
private dirty = false;
|
|
768
|
+
/**
|
|
769
|
+
* `META_FLUSH_AT` as this instance last wrote or read it (null =
|
|
770
|
+
* unknown). With `dirty`, it tells `markDirty` that the upload is
|
|
771
|
+
* already scheduled, so a turn's mark costs no storage operation.
|
|
772
|
+
*/
|
|
773
|
+
private flushAt: number | null = null;
|
|
774
|
+
/** The newest real request this instance served (null = none yet). */
|
|
775
|
+
private lastAccessAt: number | null = null;
|
|
776
|
+
/** The `META_LAST_ACCESS` this instance last wrote (null = none yet). */
|
|
777
|
+
private persistedAccessAt: number | null = null;
|
|
778
|
+
/** The user's Matrix id as resolved (or read from storage) by this instance. */
|
|
779
|
+
private matrixUserIdCache: MatrixUserIdCache | null = null;
|
|
780
|
+
/**
|
|
781
|
+
* The earliest housekeeping deadline armed since the current full alarm
|
|
782
|
+
* tick began: the tick's own `META_HOUSEKEEPING_AT` write never rises
|
|
783
|
+
* above a deadline armed while it ran.
|
|
784
|
+
*/
|
|
785
|
+
private housekeepingFloor = Infinity;
|
|
786
|
+
/** Where the expired-blob sweep continues on the next tick (null = from the start). */
|
|
787
|
+
private blobSweepCursor: string | null = null;
|
|
788
|
+
/** When this instance last swept from the start (the sweep reads whole values). */
|
|
789
|
+
private lastBlobSweepAt: number | null = null;
|
|
549
790
|
/** The flush in progress, if any (single-flight; a VFS snapshot is open while it runs). */
|
|
550
791
|
private flushInFlight: Promise<FlushResult> | null = null;
|
|
551
792
|
|
|
@@ -572,6 +813,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
572
813
|
private matrixLedger: MatrixTurnLedger | null = null;
|
|
573
814
|
/** Room mirrors of HTTP turns, serialised per session and retried across gateway restarts (see room-mirror.ts). */
|
|
574
815
|
private mirror: RoomMirror | null = null;
|
|
816
|
+
private channelTurns: ChannelTurns | null = null;
|
|
817
|
+
/** Anonymous-feedback idempotency markers and the user's submission limit. */
|
|
818
|
+
private feedbackMarkers: FeedbackMarkers | null = null;
|
|
575
819
|
/** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
|
|
576
820
|
private readonly sessionRooms = new Map<string, string>();
|
|
577
821
|
/** The user's main oracle room, once resolved (the session list is scoped to it). */
|
|
@@ -593,6 +837,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
593
837
|
{ session: IndexableSession; messages: HistoryMessage[] }
|
|
594
838
|
>();
|
|
595
839
|
private initPromise: Promise<void> | null = null;
|
|
840
|
+
/** An idle eviction deleting the working copy; a boot waits for it. */
|
|
841
|
+
private evicting: Promise<void> | null = null;
|
|
596
842
|
private reloadedFromOwnerStore = false;
|
|
597
843
|
/** Set by `adoptOwnerCopy` for a legacy copy: `ready()` flushes it to the system of record right after. */
|
|
598
844
|
private pendingLegacyMigration: { bytes: number } | null = null;
|
|
@@ -617,22 +863,44 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
617
863
|
userDid: string,
|
|
618
864
|
): Promise<string | undefined> {
|
|
619
865
|
if (!userDid.startsWith('did:ixo:')) return undefined;
|
|
866
|
+
const fresh = (
|
|
867
|
+
entry: MatrixUserIdCache | null | undefined,
|
|
868
|
+
): entry is MatrixUserIdCache =>
|
|
869
|
+
!!entry &&
|
|
870
|
+
entry.userDid === userDid &&
|
|
871
|
+
(entry.resolved ||
|
|
872
|
+
Date.now() - entry.at < MATRIX_USER_ID_FALLBACK_TTL_MS);
|
|
873
|
+
if (fresh(this.matrixUserIdCache))
|
|
874
|
+
return this.matrixUserIdCache.matrixUserId;
|
|
620
875
|
const cached =
|
|
621
876
|
await this.ctx.storage.get<MatrixUserIdCache>(META_MATRIX_USER_ID);
|
|
622
|
-
if (
|
|
623
|
-
cached
|
|
624
|
-
cached.userDid === userDid &&
|
|
625
|
-
(cached.resolved ||
|
|
626
|
-
Date.now() - cached.at < MATRIX_USER_ID_FALLBACK_TTL_MS)
|
|
627
|
-
) {
|
|
877
|
+
if (fresh(cached)) {
|
|
878
|
+
this.matrixUserIdCache = cached;
|
|
628
879
|
return cached.matrixUserId;
|
|
629
880
|
}
|
|
630
881
|
let server: string | undefined;
|
|
631
882
|
let resolved = false;
|
|
632
883
|
const blocksync = this.env.BLOCKSYNC_GRAPHQL_URL;
|
|
633
884
|
if (blocksync) {
|
|
885
|
+
// Bounded: this lookup sits on the boot path and on every turn
|
|
886
|
+
// start. The timer is cleared however the lookup ends.
|
|
887
|
+
const lookup = new AbortController();
|
|
888
|
+
const timer = setTimeout(
|
|
889
|
+
() =>
|
|
890
|
+
lookup.abort(
|
|
891
|
+
new Error(
|
|
892
|
+
`Blocksync lookup timed out after ${HOMESERVER_LOOKUP_TIMEOUT_MS} ms`,
|
|
893
|
+
),
|
|
894
|
+
),
|
|
895
|
+
HOMESERVER_LOOKUP_TIMEOUT_MS,
|
|
896
|
+
);
|
|
634
897
|
try {
|
|
635
|
-
const fromDid = await fetchUserMatrixServerName(
|
|
898
|
+
const fromDid = await fetchUserMatrixServerName(
|
|
899
|
+
blocksync,
|
|
900
|
+
userDid,
|
|
901
|
+
(input, init) =>
|
|
902
|
+
globalThis.fetch(input, { ...init, signal: lookup.signal }),
|
|
903
|
+
);
|
|
636
904
|
if (fromDid) {
|
|
637
905
|
server = fromDid;
|
|
638
906
|
resolved = true;
|
|
@@ -643,6 +911,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
643
911
|
err instanceof Error ? err.message : String(err)
|
|
644
912
|
}`,
|
|
645
913
|
);
|
|
914
|
+
} finally {
|
|
915
|
+
clearTimeout(timer);
|
|
646
916
|
}
|
|
647
917
|
}
|
|
648
918
|
server ??=
|
|
@@ -651,12 +921,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
651
921
|
: undefined;
|
|
652
922
|
if (!server) return undefined;
|
|
653
923
|
const matrixUserId = `@${userDid.replace(/:/g, '-')}:${server}`;
|
|
654
|
-
|
|
924
|
+
const entry: MatrixUserIdCache = {
|
|
655
925
|
userDid,
|
|
656
926
|
matrixUserId,
|
|
657
927
|
resolved,
|
|
658
928
|
at: Date.now(),
|
|
659
|
-
}
|
|
929
|
+
};
|
|
930
|
+
this.matrixUserIdCache = entry;
|
|
931
|
+
await this.ctx.storage.put<MatrixUserIdCache>(META_MATRIX_USER_ID, entry);
|
|
660
932
|
return matrixUserId;
|
|
661
933
|
}
|
|
662
934
|
|
|
@@ -673,8 +945,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
673
945
|
);
|
|
674
946
|
}
|
|
675
947
|
|
|
676
|
-
/**
|
|
677
|
-
private async
|
|
948
|
+
/** The delegation a turn runs under (`resolveTurnDelegation`). */
|
|
949
|
+
private async turnDelegation(
|
|
950
|
+
identity: TurnIdentity,
|
|
951
|
+
): Promise<UcanDelegation> {
|
|
952
|
+
return resolveTurnDelegation(
|
|
953
|
+
identity.ucanDelegation,
|
|
954
|
+
this.delegations.get(identity.userDid)?.raw,
|
|
955
|
+
this.ucan,
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
/**
|
|
960
|
+
* Idempotent boot: bind user, open SQLite (importing the owner copy on a
|
|
961
|
+
* cold object). A request from the user records the access the idle
|
|
962
|
+
* eviction reads; the object's own wakes (the alarm, a flush, debug and
|
|
963
|
+
* operator calls) pass `recordAccess: false`, or an idle object would
|
|
964
|
+
* keep itself alive.
|
|
965
|
+
*/
|
|
966
|
+
private async ready(
|
|
967
|
+
identity: TurnIdentity,
|
|
968
|
+
options: { recordAccess?: boolean } = {},
|
|
969
|
+
): Promise<void> {
|
|
678
970
|
this.installDebugTimerTracker();
|
|
679
971
|
this.adoptHibernatedSockets();
|
|
680
972
|
let delegationReplaced = false;
|
|
@@ -688,17 +980,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
688
980
|
this.delegations.set(identity.userDid, {
|
|
689
981
|
raw: identity.ucanDelegation,
|
|
690
982
|
});
|
|
691
|
-
// Remember
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
983
|
+
// Remember a new one for turns that arrive without the header.
|
|
984
|
+
if (delegationReplaced)
|
|
985
|
+
await this.ctx.storage.put(META_DELEGATION, {
|
|
986
|
+
raw: identity.ucanDelegation,
|
|
987
|
+
at: Date.now(),
|
|
988
|
+
...(typeof identity.ucanDelegationExpiration === 'number'
|
|
989
|
+
? { expiration: identity.ucanDelegationExpiration }
|
|
990
|
+
: {}),
|
|
991
|
+
} satisfies StoredDelegation);
|
|
699
992
|
} else if (!this.delegations.has(identity.userDid)) {
|
|
700
993
|
await this.hydrateDelegation(identity.userDid);
|
|
701
994
|
}
|
|
995
|
+
// A request that arrived while an idle eviction deletes the working
|
|
996
|
+
// copy boots afresh from the owner copy once it is gone.
|
|
997
|
+
while (this.evicting) await this.evicting;
|
|
702
998
|
if (!this.initPromise) {
|
|
703
999
|
this.initPromise = this.boot(identity.userDid).catch((err) => {
|
|
704
1000
|
this.initPromise = null;
|
|
@@ -711,10 +1007,59 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
711
1007
|
`UserOracleDO bound to ${this.userDid} received a request for ${identity.userDid}`,
|
|
712
1008
|
);
|
|
713
1009
|
}
|
|
714
|
-
await this.
|
|
1010
|
+
if (options.recordAccess !== false) await this.recordAccess();
|
|
715
1011
|
if (delegationReplaced) await this.onDelegationReplaced();
|
|
716
1012
|
}
|
|
717
1013
|
|
|
1014
|
+
/**
|
|
1015
|
+
* A request from the user: remember it in memory, and in storage on this
|
|
1016
|
+
* instance's first request and then at most every
|
|
1017
|
+
* `ACCESS_PERSIST_INTERVAL_MS`.
|
|
1018
|
+
*/
|
|
1019
|
+
private async recordAccess(): Promise<void> {
|
|
1020
|
+
const now = Date.now();
|
|
1021
|
+
this.lastAccessAt = now;
|
|
1022
|
+
if (
|
|
1023
|
+
typeof this.persistedAccessAt === 'number' &&
|
|
1024
|
+
now - this.persistedAccessAt < ACCESS_PERSIST_INTERVAL_MS
|
|
1025
|
+
)
|
|
1026
|
+
return;
|
|
1027
|
+
this.persistedAccessAt = now;
|
|
1028
|
+
await this.ctx.storage.put(META_LAST_ACCESS, now);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
/** The in-memory last access, when it is newer than the stored one. */
|
|
1032
|
+
private async persistAccess(): Promise<void> {
|
|
1033
|
+
const inMemory = this.lastAccessAt;
|
|
1034
|
+
if (
|
|
1035
|
+
typeof inMemory !== 'number' ||
|
|
1036
|
+
(typeof this.persistedAccessAt === 'number' &&
|
|
1037
|
+
this.persistedAccessAt >= inMemory)
|
|
1038
|
+
)
|
|
1039
|
+
return;
|
|
1040
|
+
this.persistedAccessAt = inMemory;
|
|
1041
|
+
await this.ctx.storage.put(META_LAST_ACCESS, inMemory);
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
/**
|
|
1045
|
+
* The user's last request: the newer of what storage holds and what
|
|
1046
|
+
* this instance saw (undefined when neither knows of one).
|
|
1047
|
+
*/
|
|
1048
|
+
private async knownLastAccess(): Promise<number | undefined> {
|
|
1049
|
+
const stored = await this.ctx.storage.get<number>(META_LAST_ACCESS);
|
|
1050
|
+
const inMemory = this.lastAccessAt;
|
|
1051
|
+
const known = Math.max(
|
|
1052
|
+
typeof stored === 'number' ? stored : -Infinity,
|
|
1053
|
+
typeof inMemory === 'number' ? inMemory : -Infinity,
|
|
1054
|
+
);
|
|
1055
|
+
return Number.isFinite(known) ? known : undefined;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/** `knownLastAccess`, or `now` when there is none (never idle). */
|
|
1059
|
+
private async lastAccess(now: number): Promise<number> {
|
|
1060
|
+
return (await this.knownLastAccess()) ?? now;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
718
1063
|
/**
|
|
719
1064
|
* The mnemonic the oracle signs downstream UCAN invocations with:
|
|
720
1065
|
* `ORACLE_SIGNING_MNEMONIC` when set, otherwise the one the Node runtime
|
|
@@ -745,10 +1090,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
745
1090
|
* The object holds a delegation it had not seen before (a header on this
|
|
746
1091
|
* request, or a deposit through the shell). A fresh delegation is the one
|
|
747
1092
|
* thing that turns a "no file-storage grant" flush failure into a
|
|
748
|
-
* success: forget the
|
|
749
|
-
* instead of at the next 10-minute retry.
|
|
1093
|
+
* success: after a recorded failure, forget the streak and, with unsaved
|
|
1094
|
+
* turns, flush now instead of at the next 10-minute retry. Without one
|
|
1095
|
+
* the debounce stands: a user on two clients alternates two delegations,
|
|
1096
|
+
* and an upload per request would replace the daily one.
|
|
750
1097
|
*/
|
|
751
1098
|
private async onDelegationReplaced(): Promise<void> {
|
|
1099
|
+
const failures =
|
|
1100
|
+
(await this.ctx.storage.get<number>(META_FLUSH_FAILURES)) ?? 0;
|
|
1101
|
+
if (failures === 0) return;
|
|
752
1102
|
await this.ctx.storage.delete(META_FLUSH_FAILURES);
|
|
753
1103
|
if (!this.db) return;
|
|
754
1104
|
if (
|
|
@@ -900,6 +1250,45 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
900
1250
|
DB_FILE,
|
|
901
1251
|
this.sqliteOpenOptions(),
|
|
902
1252
|
);
|
|
1253
|
+
try {
|
|
1254
|
+
await this.bootWorkingCopy(
|
|
1255
|
+
db,
|
|
1256
|
+
userDid,
|
|
1257
|
+
this.ownerStore,
|
|
1258
|
+
this.ucan,
|
|
1259
|
+
secretsService,
|
|
1260
|
+
);
|
|
1261
|
+
} catch (err) {
|
|
1262
|
+
// Nothing of a failed boot stays behind: the connection (and the
|
|
1263
|
+
// one a dropped zero-turn copy was reopened into) is closed and the
|
|
1264
|
+
// handles forgotten, so the next request or alarm boots afresh on
|
|
1265
|
+
// one connection instead of finding a half-built object.
|
|
1266
|
+
const reopened = this.db;
|
|
1267
|
+
this.dropInMemoryState();
|
|
1268
|
+
await db.close().catch(() => undefined);
|
|
1269
|
+
if (reopened && reopened !== db)
|
|
1270
|
+
await reopened.close().catch(() => undefined);
|
|
1271
|
+
throw err;
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
/**
|
|
1276
|
+
* The fallible part of `boot()`, from the open database on: reconcile it
|
|
1277
|
+
* with the owner copy, then build the stores and services over it.
|
|
1278
|
+
*/
|
|
1279
|
+
private async bootWorkingCopy(
|
|
1280
|
+
db: DoSqliteDatabase,
|
|
1281
|
+
userDid: string,
|
|
1282
|
+
ownerStore: OwnerStore,
|
|
1283
|
+
ucan: WorkersUcanService,
|
|
1284
|
+
secretsService: WorkersSecretsService,
|
|
1285
|
+
): Promise<void> {
|
|
1286
|
+
const env = this.env;
|
|
1287
|
+
// Whether the working copy holds a turn: one existence query, asked
|
|
1288
|
+
// again only after an import or a wipe changed the file.
|
|
1289
|
+
let turns: boolean | undefined;
|
|
1290
|
+
const hasTurns = async (target: DoSqliteDatabase): Promise<boolean> =>
|
|
1291
|
+
(turns ??= await this.hasLocalTurns(target));
|
|
903
1292
|
// Cold object (no pages yet) → pull the user's file if they have one.
|
|
904
1293
|
// `null` means the owner store PROVABLY holds no file (a new user): an
|
|
905
1294
|
// empty working copy is right. A load that FAILS is different: there is
|
|
@@ -908,7 +1297,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
908
1297
|
// real copy. So the request fails instead; the boot is not memoised on
|
|
909
1298
|
// failure, and the next request simply retries the load.
|
|
910
1299
|
if (db.fileSize === 0) {
|
|
911
|
-
const store =
|
|
1300
|
+
const store = ownerStore;
|
|
912
1301
|
let loaded: OwnerCopy | null;
|
|
913
1302
|
let attempts = 0;
|
|
914
1303
|
try {
|
|
@@ -938,24 +1327,35 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
938
1327
|
throw toRpcError(failure);
|
|
939
1328
|
}
|
|
940
1329
|
if (loaded) {
|
|
941
|
-
|
|
1330
|
+
let imported: number;
|
|
1331
|
+
try {
|
|
1332
|
+
imported = await this.adoptOwnerCopy(db, loaded);
|
|
1333
|
+
} catch (err) {
|
|
1334
|
+
// The object was empty, so nothing is lost by emptying it
|
|
1335
|
+
// again: a copy that is not a database (a short or truncated
|
|
1336
|
+
// file passes the import's header check) must not stay behind,
|
|
1337
|
+
// or every later boot would open it and fail.
|
|
1338
|
+
await this.wipeWorkingCopy(db).catch(() => undefined);
|
|
1339
|
+
throw err;
|
|
1340
|
+
}
|
|
942
1341
|
this.reloadedFromOwnerStore = true;
|
|
943
1342
|
console.log(
|
|
944
|
-
`[user-do] imported ${imported} bytes from ${loaded.fromLegacy ? 'legacy Matrix media' :
|
|
1343
|
+
`[user-do] imported ${imported} bytes from ${loaded.fromLegacy ? 'legacy Matrix media' : ownerStore.kind} for ${userDid}`,
|
|
945
1344
|
);
|
|
946
1345
|
}
|
|
947
1346
|
} else {
|
|
948
1347
|
// Warm object: notice if the user replaced or deleted their file
|
|
949
1348
|
// upstream. A `head()` that FAILS (a VFS error/timeout, or our own
|
|
950
|
-
//
|
|
951
|
-
//
|
|
1349
|
+
// replace window between moving the old file aside and moving the
|
|
1350
|
+
// new one in, where the real path momentarily does not exist) must
|
|
1351
|
+
// NEVER be read as "the file is gone" — swallowing
|
|
952
1352
|
// the error to null and wiping the working copy on it would destroy
|
|
953
1353
|
// the durable local history over a transient blip. So distinguish a
|
|
954
1354
|
// proven absence (null) from an unknown (the call threw).
|
|
955
1355
|
const knownEtag = await this.ctx.storage.get<string>(META_OWNER_ETAG);
|
|
956
1356
|
let head: { etag: string } | null | undefined;
|
|
957
1357
|
try {
|
|
958
|
-
head = await
|
|
1358
|
+
head = await ownerStore.head();
|
|
959
1359
|
} catch (err) {
|
|
960
1360
|
head = undefined; // unknown — leave the working copy untouched
|
|
961
1361
|
console.warn(
|
|
@@ -971,11 +1371,12 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
971
1371
|
// re-flushed, re-establishing the owner copy; a genuine "forget me"
|
|
972
1372
|
// goes through the explicit `remove()` path, never through an
|
|
973
1373
|
// auto-wipe that could fire on a mid-replace crash or a stale list.
|
|
974
|
-
if ((await
|
|
1374
|
+
if (!(await hasTurns(db))) {
|
|
975
1375
|
console.warn(
|
|
976
1376
|
`[user-do] owner copy for ${userDid} is gone upstream and the working copy is empty — dropping it`,
|
|
977
1377
|
);
|
|
978
1378
|
await this.wipeWorkingCopy(db);
|
|
1379
|
+
turns = undefined;
|
|
979
1380
|
this.db = await DoSqliteDatabase.open(
|
|
980
1381
|
this.ctx,
|
|
981
1382
|
DB_FILE,
|
|
@@ -988,13 +1389,27 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
988
1389
|
this.markDirty();
|
|
989
1390
|
}
|
|
990
1391
|
} else if (head && knownEtag && head.etag !== knownEtag) {
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1392
|
+
// The file changed upstream. Re-import it only when every local
|
|
1393
|
+
// write already went up: up to a day of turns waits for the
|
|
1394
|
+
// debounced flush, and an import would drop them. With unflushed
|
|
1395
|
+
// turns neither side can win automatically, so keep local (the
|
|
1396
|
+
// next flush replaces the upstream file) and say so loudly.
|
|
1397
|
+
const unflushed = await this.unflushedLocalWrites(db);
|
|
1398
|
+
if (unflushed) {
|
|
1399
|
+
console.warn(
|
|
1400
|
+
`[user-do] owner copy for ${userDid} changed upstream (${knownEtag} → ${head.etag}) but the working copy holds writes not uploaded yet — keeping the local copy`,
|
|
997
1401
|
);
|
|
1402
|
+
this.markDirty();
|
|
1403
|
+
} else {
|
|
1404
|
+
const loaded = await ownerStore.load().catch(() => null);
|
|
1405
|
+
if (loaded) {
|
|
1406
|
+
await this.adoptOwnerCopy(db, loaded);
|
|
1407
|
+
turns = undefined;
|
|
1408
|
+
this.reloadedFromOwnerStore = true;
|
|
1409
|
+
console.log(
|
|
1410
|
+
`[user-do] owner copy changed upstream (${knownEtag} → ${head.etag}); re-imported`,
|
|
1411
|
+
);
|
|
1412
|
+
}
|
|
998
1413
|
}
|
|
999
1414
|
} else if (head && !knownEtag) {
|
|
1000
1415
|
// This object first booted before the user's owner file became
|
|
@@ -1005,9 +1420,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1005
1420
|
// empty session, but no turn ever ran → no checkpoints); once local
|
|
1006
1421
|
// turns exist neither side can win automatically, so keep local
|
|
1007
1422
|
// and say so loudly (operators resolve via /debug/storage/reset).
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
const loaded = await this.ownerStore.load().catch((err) => {
|
|
1423
|
+
if (!(await hasTurns(db))) {
|
|
1424
|
+
const loaded = await ownerStore.load().catch((err) => {
|
|
1011
1425
|
console.warn(
|
|
1012
1426
|
`[user-do] late owner copy load failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1013
1427
|
);
|
|
@@ -1015,14 +1429,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1015
1429
|
});
|
|
1016
1430
|
if (loaded) {
|
|
1017
1431
|
const imported = await this.adoptOwnerCopy(db, loaded);
|
|
1432
|
+
turns = undefined;
|
|
1018
1433
|
this.reloadedFromOwnerStore = true;
|
|
1019
1434
|
console.log(
|
|
1020
|
-
`[user-do] owner copy appeared upstream after first boot; imported ${imported} bytes from ${
|
|
1435
|
+
`[user-do] owner copy appeared upstream after first boot; imported ${imported} bytes from ${ownerStore.kind} for ${userDid}`,
|
|
1021
1436
|
);
|
|
1022
1437
|
}
|
|
1023
1438
|
} else {
|
|
1024
1439
|
console.warn(
|
|
1025
|
-
`[user-do] owner copy exists upstream (${head.etag}) but was never imported and
|
|
1440
|
+
`[user-do] owner copy exists upstream (${head.etag}) but was never imported and local checkpoints already exist for ${userDid} — keeping the local copy`,
|
|
1026
1441
|
);
|
|
1027
1442
|
}
|
|
1028
1443
|
}
|
|
@@ -1036,21 +1451,39 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1036
1451
|
// the working copy has zero turns and a legacy (Matrix) copy WITH turns
|
|
1037
1452
|
// exists, adopt it and flush it over the empty file. Guard on the legacy
|
|
1038
1453
|
// copy actually having turns so a genuinely new user (empty everywhere)
|
|
1039
|
-
// is left empty, not churned.
|
|
1454
|
+
// is left empty, not churned. A probe that proved there is nothing
|
|
1455
|
+
// usable is remembered (`META_LEGACY_UNUSABLE`), as is a legacy copy
|
|
1456
|
+
// already removed: later boots of a still zero-turn copy skip the
|
|
1457
|
+
// gateway. A probe that failed proves nothing and runs again on the
|
|
1458
|
+
// next boot.
|
|
1040
1459
|
const activeDb = this.db ?? db;
|
|
1041
|
-
if (
|
|
1042
|
-
this.
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1460
|
+
if (ownerStore.loadLegacy && !(await hasTurns(activeDb))) {
|
|
1461
|
+
const markers = await this.ctx.storage.get<boolean>([
|
|
1462
|
+
META_LEGACY_CLEARED,
|
|
1463
|
+
META_LEGACY_UNUSABLE,
|
|
1464
|
+
]);
|
|
1465
|
+
const settled =
|
|
1466
|
+
markers.get(META_LEGACY_CLEARED) === true ||
|
|
1467
|
+
markers.get(META_LEGACY_UNUSABLE) === true;
|
|
1468
|
+
let legacy: OwnerCopy | null = null;
|
|
1469
|
+
if (!settled) {
|
|
1470
|
+
try {
|
|
1471
|
+
legacy = await ownerStore.loadLegacy();
|
|
1472
|
+
if (!legacy) await this.ctx.storage.put(META_LEGACY_UNUSABLE, true);
|
|
1473
|
+
} catch (err) {
|
|
1474
|
+
console.warn(
|
|
1475
|
+
`[user-do] legacy Matrix copy of ${userDid} could not be read; checking again on the next boot: ${err instanceof Error ? err.message : String(err)}`,
|
|
1476
|
+
);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1046
1479
|
if (legacy) {
|
|
1047
|
-
// Trial import: only the
|
|
1048
|
-
//
|
|
1049
|
-
//
|
|
1480
|
+
// Trial import: only the checkpoints tell us whether the legacy
|
|
1481
|
+
// copy holds history, so keep the (tiny, zero-turn) working copy
|
|
1482
|
+
// to put back if it does not.
|
|
1050
1483
|
const before = await activeDb.export();
|
|
1051
1484
|
const adoptedBytes = await activeDb.importFromStream(legacy.stream);
|
|
1052
|
-
|
|
1053
|
-
if (
|
|
1485
|
+
turns = undefined;
|
|
1486
|
+
if (await hasTurns(activeDb)) {
|
|
1054
1487
|
// The adopted bytes are NOT what the system of record holds:
|
|
1055
1488
|
// forget the last upload so the flush below actually sends them.
|
|
1056
1489
|
await this.ctx.storage.delete([
|
|
@@ -1062,10 +1495,12 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1062
1495
|
// record so the next cold boot imports it directly.
|
|
1063
1496
|
this.markDirty();
|
|
1064
1497
|
console.log(
|
|
1065
|
-
`[user-do] working copy had no turns; adopted the legacy Matrix copy (${adoptedBytes} bytes
|
|
1498
|
+
`[user-do] working copy had no turns; adopted the legacy Matrix copy (${adoptedBytes} bytes) for ${userDid} — flushing it to the system of record`,
|
|
1066
1499
|
);
|
|
1067
1500
|
} else {
|
|
1068
1501
|
await activeDb.import(before);
|
|
1502
|
+
turns = undefined;
|
|
1503
|
+
await this.ctx.storage.put(META_LEGACY_UNUSABLE, true);
|
|
1069
1504
|
console.log(
|
|
1070
1505
|
`[user-do] legacy Matrix copy for ${userDid} holds no turns either; working copy left as is`,
|
|
1071
1506
|
);
|
|
@@ -1088,7 +1523,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1088
1523
|
try {
|
|
1089
1524
|
const flushed = await this.flushToOwnerStore();
|
|
1090
1525
|
console.log(
|
|
1091
|
-
`[user-do] migrated ${bytes} bytes from legacy Matrix media to ${
|
|
1526
|
+
`[user-do] migrated ${bytes} bytes from legacy Matrix media to ${ownerStore.kind} for ${userDid} (${flushed.etag ?? '-'})`,
|
|
1092
1527
|
);
|
|
1093
1528
|
} catch (err) {
|
|
1094
1529
|
console.warn(
|
|
@@ -1115,16 +1550,19 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1115
1550
|
// Redacting on etag alone would drop the last backstop while the VFS
|
|
1116
1551
|
// copy could still be a smaller or older export. A dirty or
|
|
1117
1552
|
// not-yet-verified copy waits: the flush that lands it does the
|
|
1118
|
-
// removal itself.
|
|
1553
|
+
// removal itself. The verification hashes the whole file, so it
|
|
1554
|
+
// only runs while a legacy copy may still exist.
|
|
1555
|
+
const ownerEtag = await this.ctx.storage.get<string>(META_OWNER_ETAG);
|
|
1119
1556
|
if (
|
|
1120
|
-
|
|
1557
|
+
ownerEtag &&
|
|
1558
|
+
ownerStore.removeLegacyCopy &&
|
|
1559
|
+
(await this.ctx.storage.get<boolean>(META_LEGACY_CLEARED)) !== true &&
|
|
1560
|
+
(await hasTurns(liveDb)) &&
|
|
1121
1561
|
(await this.ownerCopyIsCurrent(liveDb))
|
|
1122
|
-
)
|
|
1123
|
-
|
|
1124
|
-
if (ownerEtag) await this.removeLegacyCopyIfAny(ownerEtag);
|
|
1125
|
-
}
|
|
1562
|
+
)
|
|
1563
|
+
await this.removeLegacyCopyIfAny(ownerEtag);
|
|
1126
1564
|
console.log(
|
|
1127
|
-
`[user-do] ready ${userDid}: store=${
|
|
1565
|
+
`[user-do] ready ${userDid}: store=${ownerStore.kind} legacy=${ownerStore.loadLegacy ? 'yes' : 'no'} file=${liveDb.fileSize}B turns=${(await hasTurns(liveDb)) ? 'yes' : 'none'} sessions=${sessionRow?.n ?? 0} ownerEtag=${ownerEtag ?? '-'} reloaded=${this.reloadedFromOwnerStore}`,
|
|
1128
1566
|
);
|
|
1129
1567
|
}
|
|
1130
1568
|
this.saver = new SqliteSaver(liveDb, undefined, {
|
|
@@ -1137,6 +1575,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1137
1575
|
await this.matrixLedger.setup();
|
|
1138
1576
|
this.runStore = new RunStore(liveDb);
|
|
1139
1577
|
await this.runStore.setup();
|
|
1578
|
+
this.feedbackMarkers = new FeedbackMarkers(liveDb);
|
|
1579
|
+
await this.feedbackMarkers.setup();
|
|
1140
1580
|
this.resultStore = new ResultStore(liveDb, {
|
|
1141
1581
|
...(this.env.TIER_BUCKET ? { bucket: this.env.TIER_BUCKET } : {}),
|
|
1142
1582
|
prefix: this.ctx.id.toString(),
|
|
@@ -1144,6 +1584,17 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1144
1584
|
logger: console,
|
|
1145
1585
|
});
|
|
1146
1586
|
await this.resultStore.setup();
|
|
1587
|
+
this.attachmentTextCache = new AttachmentTextCacheStore(liveDb, console);
|
|
1588
|
+
// The store exists whenever the bucket is bound, so artefacts made
|
|
1589
|
+
// earlier stay readable, revocable and deleted with their session after
|
|
1590
|
+
// the public origin is removed; only new ones need it.
|
|
1591
|
+
const artifactConfig = artifactStorageConfig(this.env, (message) =>
|
|
1592
|
+
console.warn(message),
|
|
1593
|
+
);
|
|
1594
|
+
if (artifactConfig) {
|
|
1595
|
+
this.artifacts = new ArtifactStore(liveDb, artifactConfig);
|
|
1596
|
+
await this.artifacts.setup();
|
|
1597
|
+
}
|
|
1147
1598
|
// Threads opened before the runtime adopted Node's session rule
|
|
1148
1599
|
// (session id = thread root event id) are renamed to it, once.
|
|
1149
1600
|
const renamed = await migrateThreadSessionIds(liveDb, console);
|
|
@@ -1153,12 +1604,81 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1153
1604
|
config: this.runConfig,
|
|
1154
1605
|
instanceId: this.instanceId,
|
|
1155
1606
|
log: console,
|
|
1156
|
-
|
|
1607
|
+
// Keep-alive and recovery deadlines alike: after a reset, the
|
|
1608
|
+
// keep-alive deadline is the wake that boots the object and
|
|
1609
|
+
// recovers its runs, so the heartbeat fast path must not skip it.
|
|
1610
|
+
requestAlarm: (at) => this.ctx.waitUntil(this.requestHousekeeping(at)),
|
|
1157
1611
|
runAttempt: (live, resumed) => this.runAttempt(live, resumed),
|
|
1158
1612
|
checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
|
|
1159
1613
|
onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
|
|
1160
1614
|
});
|
|
1161
1615
|
|
|
1616
|
+
this.channelTurns = new ChannelTurns(liveDb, {
|
|
1617
|
+
createSession: async (identity, markerTxnId) => {
|
|
1618
|
+
const room = await this.gateway.resolveUserRoom(identity.userDid);
|
|
1619
|
+
if (!room) throw new ChannelError(409, 'Companion room is not ready');
|
|
1620
|
+
return (
|
|
1621
|
+
await this.createSession(identity, {
|
|
1622
|
+
roomId: room.roomId,
|
|
1623
|
+
markerTxnId,
|
|
1624
|
+
})
|
|
1625
|
+
).sessionId;
|
|
1626
|
+
},
|
|
1627
|
+
assertSession: async (identity, sessionId) => {
|
|
1628
|
+
const session = await this.sessions!.getSession(sessionId);
|
|
1629
|
+
const room = await this.gateway.resolveUserRoom(identity.userDid);
|
|
1630
|
+
if (
|
|
1631
|
+
!session ||
|
|
1632
|
+
!sessionId.startsWith('$') ||
|
|
1633
|
+
!room ||
|
|
1634
|
+
session.roomId !== room.roomId
|
|
1635
|
+
)
|
|
1636
|
+
throw new ChannelError(404, 'Companion session not found');
|
|
1637
|
+
const encryption = await this.gateway.getRoomStateEvent(
|
|
1638
|
+
room.roomId,
|
|
1639
|
+
'm.room.encryption',
|
|
1640
|
+
);
|
|
1641
|
+
const parsed: unknown = encryption ? JSON.parse(encryption) : null;
|
|
1642
|
+
if (
|
|
1643
|
+
!parsed ||
|
|
1644
|
+
typeof parsed !== 'object' ||
|
|
1645
|
+
!('algorithm' in parsed) ||
|
|
1646
|
+
parsed.algorithm !== 'm.megolm.v1.aes-sha2'
|
|
1647
|
+
)
|
|
1648
|
+
throw new ChannelError(409, 'Companion room must be encrypted');
|
|
1649
|
+
},
|
|
1650
|
+
requireDelegation: async (identity) =>
|
|
1651
|
+
requireChannelDelegation(
|
|
1652
|
+
!!this.delegations.get(identity.userDid)?.raw,
|
|
1653
|
+
),
|
|
1654
|
+
getRun: (runId) => this.runStore!.get(runId),
|
|
1655
|
+
getPlan: (runId) => this.runStore!.getPlan(runId),
|
|
1656
|
+
wasPruned: (runId) => this.runStore!.wasChannelRunPruned(runId),
|
|
1657
|
+
begin: async (runId, request) => {
|
|
1658
|
+
const { live } = await this.runs!.begin({
|
|
1659
|
+
runId,
|
|
1660
|
+
sessionId: request.sessionId,
|
|
1661
|
+
requestId: request.requestId,
|
|
1662
|
+
client: 'channel',
|
|
1663
|
+
request: this.runRequestJson(request, storedRunRequest(request)),
|
|
1664
|
+
multitask: 'enqueue',
|
|
1665
|
+
});
|
|
1666
|
+
this.markDirty();
|
|
1667
|
+
await this.requestHousekeeping(
|
|
1668
|
+
Date.now() + this.runConfig.keepAliveMs,
|
|
1669
|
+
);
|
|
1670
|
+
await this.ctx.storage.sync();
|
|
1671
|
+
this.ctx.waitUntil(live.done.catch(() => undefined));
|
|
1672
|
+
return live.record;
|
|
1673
|
+
},
|
|
1674
|
+
mirror: (request, text, author) =>
|
|
1675
|
+
this.replayToRoom(request, text, author),
|
|
1676
|
+
changed: async () => {
|
|
1677
|
+
this.markDirty();
|
|
1678
|
+
await this.ctx.storage.sync();
|
|
1679
|
+
},
|
|
1680
|
+
});
|
|
1681
|
+
|
|
1162
1682
|
const core = this.core;
|
|
1163
1683
|
// Boot-time plugin hooks + collision checks, once per object (memoised).
|
|
1164
1684
|
await core.warm();
|
|
@@ -1173,7 +1693,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1173
1693
|
matrixUserId: await this.resolveMatrixUserId(userDid),
|
|
1174
1694
|
gateway: this.gateway,
|
|
1175
1695
|
runTurn: (req) => this.runTurn(req),
|
|
1176
|
-
|
|
1696
|
+
abortTurn: (sessionId) => this.abortTurn(sessionId),
|
|
1697
|
+
requestAlarm: (at) => this.requestHousekeeping(at),
|
|
1177
1698
|
turnRunLive: (taskRunId) => Boolean(this.runs?.byTaskRunId(taskRunId)),
|
|
1178
1699
|
log: console,
|
|
1179
1700
|
...(Number.isFinite(maxTasks) ? { maxTasksPerUser: maxTasks } : {}),
|
|
@@ -1181,7 +1702,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1181
1702
|
});
|
|
1182
1703
|
if (this.taskScheduler) {
|
|
1183
1704
|
const next = await this.taskScheduler.nextWakeAt().catch(() => null);
|
|
1184
|
-
if (next !== null)
|
|
1705
|
+
if (next !== null)
|
|
1706
|
+
await this.requestHousekeeping(Math.max(next, Date.now() + 1000));
|
|
1185
1707
|
}
|
|
1186
1708
|
|
|
1187
1709
|
// Runs a previous incarnation left in flight: schedule their recovery
|
|
@@ -1198,7 +1720,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1198
1720
|
// imported from the Node runtime arrive with uncompressed blobs.
|
|
1199
1721
|
this.compactDone ??=
|
|
1200
1722
|
(await this.ctx.storage.get<boolean>(META_COMPACT_DONE)) ?? false;
|
|
1201
|
-
if (!this.compactDone) this.
|
|
1723
|
+
if (!this.compactDone) await this.requestHousekeeping(Date.now() + 3000);
|
|
1202
1724
|
|
|
1203
1725
|
// Provider selection: `core.llm` is the OpenRouter adapter built from
|
|
1204
1726
|
// the validated base env; `LLM_PROVIDER=nebius` (read from the raw
|
|
@@ -1225,20 +1747,27 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1225
1747
|
this.ambient = createAmbientServices({
|
|
1226
1748
|
...(this.taskScheduler ? { tasks: this.taskScheduler.surface } : {}),
|
|
1227
1749
|
preferences: this.preferences,
|
|
1750
|
+
kv: new SqliteUserKv(liveDb),
|
|
1228
1751
|
frontend: this.realtime.frontend,
|
|
1229
1752
|
config: core.validatedEnv,
|
|
1230
1753
|
identity: core.identity,
|
|
1231
1754
|
availablePlugins: core.availablePlugins,
|
|
1232
1755
|
llm,
|
|
1756
|
+
decisions: core.decisions,
|
|
1233
1757
|
logger: console,
|
|
1234
1758
|
storage: this.ctx.storage,
|
|
1235
1759
|
gateway: this.gateway,
|
|
1236
|
-
ucan
|
|
1760
|
+
ucan,
|
|
1237
1761
|
delegationFor: (did) => this.delegations.get(did),
|
|
1238
1762
|
events: this.events,
|
|
1239
1763
|
secrets: createSecretsAdapter(secretsService),
|
|
1240
1764
|
background: (work) => this.ctx.waitUntil(work),
|
|
1241
1765
|
});
|
|
1766
|
+
this.capabilityRouter = createCapabilityRouter({
|
|
1767
|
+
evaluator: core.decisions,
|
|
1768
|
+
logger: console,
|
|
1769
|
+
background: (work) => this.ctx.waitUntil(work),
|
|
1770
|
+
});
|
|
1242
1771
|
}
|
|
1243
1772
|
|
|
1244
1773
|
/**
|
|
@@ -1278,6 +1807,50 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1278
1807
|
});
|
|
1279
1808
|
}
|
|
1280
1809
|
|
|
1810
|
+
/**
|
|
1811
|
+
* Forget every handle on the working copy, so the next request boots
|
|
1812
|
+
* afresh (and imports the owner copy into an empty file).
|
|
1813
|
+
*/
|
|
1814
|
+
private dropInMemoryState(): void {
|
|
1815
|
+
this.db = null;
|
|
1816
|
+
this.saver = null;
|
|
1817
|
+
this.sessions = null;
|
|
1818
|
+
this.matrixLedger = null;
|
|
1819
|
+
this.runStore = null;
|
|
1820
|
+
this.channelTurns = null;
|
|
1821
|
+
this.feedbackMarkers = null;
|
|
1822
|
+
this.runs = null;
|
|
1823
|
+
this.resultStore = null;
|
|
1824
|
+
this.artifacts = null;
|
|
1825
|
+
this.attachmentTextCache = null;
|
|
1826
|
+
// Its store is bound to the database just dropped: a caller that sees
|
|
1827
|
+
// no scheduler boots again instead of reading a closed connection.
|
|
1828
|
+
this.taskScheduler = null;
|
|
1829
|
+
this.sessionRooms.clear();
|
|
1830
|
+
this.mainRoomId = null;
|
|
1831
|
+
this.ambient = null;
|
|
1832
|
+
this.secretsService = null;
|
|
1833
|
+
this.byo = null;
|
|
1834
|
+
this.initPromise = null;
|
|
1835
|
+
this.reloadedFromOwnerStore = false;
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
/**
|
|
1839
|
+
* Still idle after the idle tick's awaits: the same working copy, nothing
|
|
1840
|
+
* written or uploading, no run, and no request since (every request
|
|
1841
|
+
* records its access in `ready()`, in memory at once).
|
|
1842
|
+
*/
|
|
1843
|
+
private async stillIdle(db: DoSqliteDatabase): Promise<boolean> {
|
|
1844
|
+
const lastAccess = await this.lastAccess(Date.now());
|
|
1845
|
+
return (
|
|
1846
|
+
this.db === db &&
|
|
1847
|
+
!this.dirty &&
|
|
1848
|
+
!this.flushInFlight &&
|
|
1849
|
+
(this.runs?.size ?? 0) === 0 &&
|
|
1850
|
+
Date.now() - lastAccess > IDLE_EVICT_MS
|
|
1851
|
+
);
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1281
1854
|
/** Drop the working copy entirely; the caller reopens/reboots afterwards. */
|
|
1282
1855
|
private async wipeWorkingCopy(db: DoSqliteDatabase | null): Promise<void> {
|
|
1283
1856
|
if (db) {
|
|
@@ -1294,6 +1867,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1294
1867
|
META_FLUSH_FAILURES,
|
|
1295
1868
|
META_LAST_VACUUM,
|
|
1296
1869
|
]);
|
|
1870
|
+
this.flushAt = null;
|
|
1297
1871
|
}
|
|
1298
1872
|
|
|
1299
1873
|
/**
|
|
@@ -1354,6 +1928,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1354
1928
|
META_LAST_CHECKSUM,
|
|
1355
1929
|
META_UPLOADED_GEN,
|
|
1356
1930
|
META_LEGACY_CLEARED,
|
|
1931
|
+
META_LEGACY_UNUSABLE,
|
|
1357
1932
|
]);
|
|
1358
1933
|
this.markDirty();
|
|
1359
1934
|
this.pendingLegacyMigration = { bytes };
|
|
@@ -1375,8 +1950,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1375
1950
|
const uploadedGen = await this.ctx.storage.get<number>(META_UPLOADED_GEN);
|
|
1376
1951
|
const lastChecksum =
|
|
1377
1952
|
await this.ctx.storage.get<string>(META_LAST_CHECKSUM);
|
|
1378
|
-
if (lastChecksum === undefined)
|
|
1379
|
-
return (await this.localTurnCount(db)) === 0;
|
|
1953
|
+
if (lastChecksum === undefined) return !(await this.hasLocalTurns(db));
|
|
1380
1954
|
// Objects that last flushed before generations were recorded verify
|
|
1381
1955
|
// by hash alone; everyone else gets the cheap generation gate first.
|
|
1382
1956
|
if (uploadedGen !== undefined && uploadedGen !== db.writeGeneration)
|
|
@@ -1384,17 +1958,47 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1384
1958
|
return (await db.checksum()) === lastChecksum;
|
|
1385
1959
|
}
|
|
1386
1960
|
|
|
1387
|
-
/**
|
|
1388
|
-
private async
|
|
1961
|
+
/** Whether the working copy holds a checkpoint — none means no turn ever ran here. */
|
|
1962
|
+
private async hasLocalTurns(db: DoSqliteDatabase): Promise<boolean> {
|
|
1389
1963
|
const row = await db
|
|
1390
|
-
.get<{
|
|
1964
|
+
.get<{ one: number }>('SELECT 1 AS one FROM checkpoints LIMIT 1')
|
|
1391
1965
|
.catch(() => undefined);
|
|
1392
|
-
return row
|
|
1966
|
+
return row !== undefined;
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
/**
|
|
1970
|
+
* Whether the working copy holds writes the owner store has not seen:
|
|
1971
|
+
* marked dirty (in memory or on disk), or written past the generation
|
|
1972
|
+
* of the last upload (`decideBootDirty`).
|
|
1973
|
+
*/
|
|
1974
|
+
private async unflushedLocalWrites(db: DoSqliteDatabase): Promise<boolean> {
|
|
1975
|
+
const got = await this.ctx.storage.get<unknown>([
|
|
1976
|
+
META_DIRTY,
|
|
1977
|
+
META_UPLOADED_GEN,
|
|
1978
|
+
]);
|
|
1979
|
+
const decision = decideBootDirty({
|
|
1980
|
+
dirtyInMemory: this.dirty,
|
|
1981
|
+
dirtyFlag: got.get(META_DIRTY),
|
|
1982
|
+
uploadedGen: got.get(META_UPLOADED_GEN),
|
|
1983
|
+
writeGeneration: db.writeGeneration,
|
|
1984
|
+
});
|
|
1985
|
+
return (
|
|
1986
|
+
decision === 'already-dirty' ||
|
|
1987
|
+
decision === 'flagged' ||
|
|
1988
|
+
decision === 'behind-upload'
|
|
1989
|
+
);
|
|
1393
1990
|
}
|
|
1394
1991
|
|
|
1992
|
+
/**
|
|
1993
|
+
* The working copy holds writes the owner store has not seen: persist the
|
|
1994
|
+
* mark and make sure the upload is scheduled. Free once the copy is dirty
|
|
1995
|
+
* with a known deadline — the alarm armed for it only ever moves earlier,
|
|
1996
|
+
* and the tick that serves it re-arms whatever it leaves pending.
|
|
1997
|
+
*/
|
|
1395
1998
|
private markDirty(): void {
|
|
1999
|
+
if (this.dirty && typeof this.flushAt === 'number') return;
|
|
2000
|
+
if (!this.dirty) void this.ctx.storage.put(META_DIRTY, true);
|
|
1396
2001
|
this.dirty = true;
|
|
1397
|
-
void this.ctx.storage.put(META_DIRTY, true);
|
|
1398
2002
|
void this.armFlush(Date.now() + FLUSH_DEBOUNCE_MS);
|
|
1399
2003
|
}
|
|
1400
2004
|
|
|
@@ -1404,12 +2008,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1404
2008
|
* `FLUSH_DEBOUNCE_MS` after the first of them, never later.
|
|
1405
2009
|
*/
|
|
1406
2010
|
private async armFlush(at: number): Promise<void> {
|
|
1407
|
-
const existing =
|
|
2011
|
+
const existing =
|
|
2012
|
+
typeof this.flushAt === 'number'
|
|
2013
|
+
? this.flushAt
|
|
2014
|
+
: await this.ctx.storage.get<number>(META_FLUSH_AT);
|
|
1408
2015
|
const deadline =
|
|
1409
2016
|
typeof existing === 'number' && existing <= at ? existing : at;
|
|
2017
|
+
this.flushAt = deadline;
|
|
1410
2018
|
if (deadline !== existing)
|
|
1411
2019
|
await this.ctx.storage.put(META_FLUSH_AT, deadline);
|
|
1412
|
-
this.
|
|
2020
|
+
await this.requestHousekeeping(deadline);
|
|
1413
2021
|
}
|
|
1414
2022
|
|
|
1415
2023
|
/**
|
|
@@ -1417,17 +2025,47 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1417
2025
|
* that just came due (which would otherwise make every wake retry).
|
|
1418
2026
|
*/
|
|
1419
2027
|
private async scheduleFlushRetry(at: number): Promise<void> {
|
|
2028
|
+
this.flushAt = at;
|
|
1420
2029
|
await this.ctx.storage.put(META_FLUSH_AT, at);
|
|
1421
|
-
this.
|
|
2030
|
+
await this.requestHousekeeping(at);
|
|
2031
|
+
}
|
|
2032
|
+
|
|
2033
|
+
private alarmArm: Promise<void> = Promise.resolve();
|
|
2034
|
+
|
|
2035
|
+
/**
|
|
2036
|
+
* Arm the alarm for a deadline the full housekeeping tick serves (a
|
|
2037
|
+
* flush, its retry, a task, compaction, a run's keep-alive or recovery),
|
|
2038
|
+
* and lower the stored housekeeping deadline to it: the heartbeat fast
|
|
2039
|
+
* path trusts that value, and a deadline it does not know of would wait
|
|
2040
|
+
* for as long as a socket stays attached. Serialised with every other
|
|
2041
|
+
* arm.
|
|
2042
|
+
*/
|
|
2043
|
+
private requestHousekeeping(at: number): Promise<void> {
|
|
2044
|
+
this.housekeepingFloor = Math.min(this.housekeepingFloor, at);
|
|
2045
|
+
this.alarmArm = this.alarmArm
|
|
2046
|
+
.catch(() => {})
|
|
2047
|
+
.then(async () => {
|
|
2048
|
+
const stored =
|
|
2049
|
+
await this.ctx.storage.get<number>(META_HOUSEKEEPING_AT);
|
|
2050
|
+
if (typeof stored === 'number' && stored > at)
|
|
2051
|
+
await this.ctx.storage.put(META_HOUSEKEEPING_AT, at);
|
|
2052
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
2053
|
+
if (existing === null || existing > at)
|
|
2054
|
+
await this.ctx.storage.setAlarm(at);
|
|
2055
|
+
});
|
|
2056
|
+
return this.alarmArm;
|
|
1422
2057
|
}
|
|
1423
2058
|
|
|
1424
2059
|
/** Arm the object's single alarm no later than `at` (multiplexed). */
|
|
1425
|
-
private requestAlarm(at: number): void {
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
2060
|
+
private requestAlarm(at: number): Promise<void> {
|
|
2061
|
+
this.alarmArm = this.alarmArm
|
|
2062
|
+
.catch(() => {})
|
|
2063
|
+
.then(async () => {
|
|
2064
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
2065
|
+
if (existing === null || existing > at)
|
|
2066
|
+
await this.ctx.storage.setAlarm(at);
|
|
2067
|
+
});
|
|
2068
|
+
return this.alarmArm;
|
|
1431
2069
|
}
|
|
1432
2070
|
|
|
1433
2071
|
/**
|
|
@@ -1452,8 +2090,17 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1452
2090
|
// exists once the object booted).
|
|
1453
2091
|
const keepAliveAt = this.runs?.keepAliveDeadline(wakeAt) ?? null;
|
|
1454
2092
|
const recoveryAt = this.runs?.nextRecoveryAt() ?? null;
|
|
1455
|
-
if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000)
|
|
1456
|
-
|
|
2093
|
+
if (this.runs && recoveryAt !== null && recoveryAt <= wakeAt + 1000) {
|
|
2094
|
+
// A store error here must not reject the alarm: the tick below (or
|
|
2095
|
+
// the re-arm) still has to run.
|
|
2096
|
+
try {
|
|
2097
|
+
await this.runs.resumeDue(wakeAt);
|
|
2098
|
+
} catch (err) {
|
|
2099
|
+
console.error(
|
|
2100
|
+
`[user-do] run recovery tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2101
|
+
);
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
1457
2104
|
if (
|
|
1458
2105
|
nextPingAt !== null &&
|
|
1459
2106
|
housekeepingAt !== undefined &&
|
|
@@ -1461,28 +2108,33 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1461
2108
|
keepAliveAt === null &&
|
|
1462
2109
|
(recoveryAt === null || recoveryAt > wakeAt + 1000)
|
|
1463
2110
|
) {
|
|
1464
|
-
await this.
|
|
2111
|
+
await this.requestAlarm(
|
|
1465
2112
|
Math.min(nextPingAt, housekeepingAt, recoveryAt ?? Infinity),
|
|
1466
2113
|
);
|
|
1467
2114
|
return;
|
|
1468
2115
|
}
|
|
1469
2116
|
|
|
2117
|
+
// A full tick from here on. Deadlines armed while it runs (the boot
|
|
2118
|
+
// below included) are kept by the housekeeping write at its end.
|
|
2119
|
+
this.housekeepingFloor = Infinity;
|
|
2120
|
+
|
|
1470
2121
|
// Alarms routinely wake an EVICTED object: no db, no task scheduler,
|
|
1471
2122
|
// dirty flag gone. Boot from the persisted identity first — a tick on
|
|
1472
2123
|
// the un-booted object would skip every client and re-arm at the idle
|
|
1473
|
-
// horizon, silently dropping pending task runs and flushes.
|
|
2124
|
+
// horizon, silently dropping pending task runs and flushes. The boot
|
|
2125
|
+
// is not a request: it records no access.
|
|
1474
2126
|
if (!this.db) {
|
|
1475
2127
|
const storedDid = await this.ctx.storage.get<string>(META_USER_DID);
|
|
1476
2128
|
if (storedDid) {
|
|
1477
2129
|
try {
|
|
1478
|
-
await this.ready({ userDid: storedDid });
|
|
2130
|
+
await this.ready({ userDid: storedDid }, { recordAccess: false });
|
|
1479
2131
|
} catch (err) {
|
|
1480
2132
|
console.error(
|
|
1481
2133
|
`[user-do] alarm boot failed for ${storedDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1482
2134
|
);
|
|
1483
2135
|
// State exists but could not be opened — retry soon instead of
|
|
1484
2136
|
// falling through to a schedulerless re-arm.
|
|
1485
|
-
await this.
|
|
2137
|
+
await this.requestHousekeeping(Date.now() + 60_000);
|
|
1486
2138
|
return;
|
|
1487
2139
|
}
|
|
1488
2140
|
}
|
|
@@ -1516,8 +2168,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1516
2168
|
deadlines.push(Math.max(nextRecovery, now + 1000));
|
|
1517
2169
|
}
|
|
1518
2170
|
|
|
1519
|
-
|
|
1520
|
-
|
|
2171
|
+
// The last real request, stored for the next instance before anything
|
|
2172
|
+
// is decided on it.
|
|
2173
|
+
await this.persistAccess();
|
|
2174
|
+
const lastAccess = await this.lastAccess(now);
|
|
1521
2175
|
const idle = now - lastAccess > IDLE_EVICT_MS;
|
|
1522
2176
|
|
|
1523
2177
|
// Owner-store flush: only when its own deadline is due, or the copy
|
|
@@ -1548,14 +2202,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1548
2202
|
}
|
|
1549
2203
|
|
|
1550
2204
|
if (this.taskScheduler) {
|
|
2205
|
+
let next: number | null;
|
|
1551
2206
|
try {
|
|
1552
|
-
await this.taskScheduler.onAlarm(now);
|
|
2207
|
+
next = await this.taskScheduler.onAlarm(now);
|
|
1553
2208
|
} catch (err) {
|
|
1554
2209
|
console.error(
|
|
1555
2210
|
`[user-do] task scheduler tick failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1556
2211
|
);
|
|
2212
|
+
next = await this.taskScheduler.nextWakeAt().catch(() => null);
|
|
1557
2213
|
}
|
|
1558
|
-
const next = await this.taskScheduler.nextWakeAt().catch(() => null);
|
|
1559
2214
|
if (next !== null) deadlines.push(Math.max(next, now + 1000));
|
|
1560
2215
|
}
|
|
1561
2216
|
|
|
@@ -1585,24 +2240,39 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1585
2240
|
if (next !== null) deadlines.push(next);
|
|
1586
2241
|
}
|
|
1587
2242
|
|
|
1588
|
-
//
|
|
1589
|
-
//
|
|
1590
|
-
//
|
|
1591
|
-
//
|
|
2243
|
+
// Expired ambient blobs (`blob:` keys), one page per tick while a
|
|
2244
|
+
// sweep is under way, a new sweep at most hourly (a page read loads
|
|
2245
|
+
// the whole values, and blobs live a day at most). Its follow-up is
|
|
2246
|
+
// no reason to keep an idle copy: it is armed after the eviction
|
|
2247
|
+
// decision below.
|
|
2248
|
+
let blobSweepPending = false;
|
|
1592
2249
|
if (
|
|
1593
|
-
this.
|
|
1594
|
-
|
|
1595
|
-
|
|
2250
|
+
this.blobSweepCursor !== null ||
|
|
2251
|
+
this.lastBlobSweepAt === null ||
|
|
2252
|
+
now - this.lastBlobSweepAt >= BLOB_SWEEP_INTERVAL_MS
|
|
1596
2253
|
) {
|
|
1597
|
-
|
|
1598
|
-
|
|
2254
|
+
if (this.blobSweepCursor === null) this.lastBlobSweepAt = now;
|
|
2255
|
+
try {
|
|
2256
|
+
const swept = await sweepExpiredBlobs(
|
|
2257
|
+
this.ctx.storage,
|
|
2258
|
+
this.blobSweepCursor ? { startAfter: this.blobSweepCursor } : {},
|
|
2259
|
+
);
|
|
2260
|
+
this.blobSweepCursor = swept.next;
|
|
2261
|
+
blobSweepPending = swept.next !== null;
|
|
2262
|
+
} catch (err) {
|
|
2263
|
+
console.warn(
|
|
2264
|
+
`[user-do] blob sweep failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2265
|
+
);
|
|
2266
|
+
}
|
|
1599
2267
|
}
|
|
1600
2268
|
|
|
1601
|
-
// Idle housekeeping: a user silent for
|
|
1602
|
-
// their file is safe upstream, and we stop paying to store a
|
|
1603
|
-
// evict while tasks are scheduled: their runs ARE
|
|
1604
|
-
// above ran first; the wipe additionally verifies
|
|
1605
|
-
// matches the working copy, else the copy stays
|
|
2269
|
+
// Idle housekeeping: a user silent for five days loses the cached
|
|
2270
|
+
// pages — their file is safe upstream, and we stop paying to store a
|
|
2271
|
+
// copy. Never evict while tasks are scheduled: their runs ARE
|
|
2272
|
+
// activity. The flush above ran first; the wipe additionally verifies
|
|
2273
|
+
// the upstream copy matches the working copy, else the copy stays
|
|
2274
|
+
// until it does. The R2 tier's next pass is no reason to stay: the
|
|
2275
|
+
// wipe deletes the object's R2 prefix with the working copy.
|
|
1606
2276
|
if (
|
|
1607
2277
|
idle &&
|
|
1608
2278
|
!this.dirty &&
|
|
@@ -1610,27 +2280,82 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1610
2280
|
deadlines.length === 0 &&
|
|
1611
2281
|
(this.runs?.size ?? 0) === 0
|
|
1612
2282
|
) {
|
|
1613
|
-
|
|
1614
|
-
|
|
2283
|
+
const db = this.db;
|
|
2284
|
+
const outcome = await evictIdleWorkingCopy({
|
|
2285
|
+
ownerCopyIsCurrent: () => this.ownerCopyIsCurrent(db),
|
|
2286
|
+
stillIdle: () => this.stillIdle(db),
|
|
2287
|
+
dropHandles: () => this.dropInMemoryState(),
|
|
2288
|
+
wipe: () => {
|
|
2289
|
+
const wipe = this.wipeWorkingCopy(db);
|
|
2290
|
+
this.evicting = wipe
|
|
2291
|
+
.catch(() => undefined)
|
|
2292
|
+
.finally(() => {
|
|
2293
|
+
this.evicting = null;
|
|
2294
|
+
});
|
|
2295
|
+
return wipe;
|
|
2296
|
+
},
|
|
2297
|
+
});
|
|
2298
|
+
if (outcome === 'evicted') {
|
|
1615
2299
|
await this.ctx.storage.delete(META_HOUSEKEEPING_AT);
|
|
1616
2300
|
// An attached (idle) socket still needs its heartbeat rounds.
|
|
1617
|
-
if (nextPingAt !== null) await this.
|
|
1618
|
-
|
|
2301
|
+
if (nextPingAt !== null) await this.requestAlarm(nextPingAt);
|
|
2302
|
+
// A request that waited on the wipe has booted the object afresh
|
|
2303
|
+
// and armed its own deadlines (a task Start's run): keep them.
|
|
2304
|
+
else if (this.initPromise === null)
|
|
2305
|
+
await this.ctx.storage.deleteAlarm();
|
|
1619
2306
|
return;
|
|
1620
2307
|
}
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
2308
|
+
if (outcome === 'not-current') {
|
|
2309
|
+
console.warn(
|
|
2310
|
+
`[user-do] ${this.userDid} is idle but the owner copy is not verified current — keeping the working copy`,
|
|
2311
|
+
);
|
|
2312
|
+
deadlines.push(now + FLUSH_RETRY_DELAY_MS);
|
|
2313
|
+
} else {
|
|
2314
|
+
console.log(
|
|
2315
|
+
`[user-do] ${this.userDid} became active during the idle check — keeping the working copy`,
|
|
2316
|
+
);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
// R2 page tier: move chunks no turn touched for a couple of days out
|
|
2321
|
+
// of DO storage (see sqlite/page-tier.ts). After the flush above (the
|
|
2322
|
+
// upload reads hot rows, not R2) and never over an in-flight one (it
|
|
2323
|
+
// holds a snapshot). A pass is capped; more work re-arms soon.
|
|
2324
|
+
if (
|
|
2325
|
+
this.db?.tierEnabled &&
|
|
2326
|
+
!this.flushInFlight &&
|
|
2327
|
+
(this.runs?.activeCount ?? 0) === 0
|
|
2328
|
+
) {
|
|
2329
|
+
const next = await this.tierTick(this.db, now);
|
|
2330
|
+
if (next !== null) deadlines.push(next);
|
|
1625
2331
|
}
|
|
2332
|
+
if (blobSweepPending) deadlines.push(now + 60_000);
|
|
2333
|
+
|
|
1626
2334
|
deadlines.push(now + IDLE_EVICT_MS);
|
|
1627
2335
|
const housekeeping = Math.min(...deadlines);
|
|
1628
|
-
await this.
|
|
1629
|
-
await this.
|
|
2336
|
+
await this.storeHousekeeping(housekeeping);
|
|
2337
|
+
await this.requestAlarm(
|
|
1630
2338
|
nextPingAt === null ? housekeeping : Math.min(housekeeping, nextPingAt),
|
|
1631
2339
|
);
|
|
1632
2340
|
}
|
|
1633
2341
|
|
|
2342
|
+
/**
|
|
2343
|
+
* Record the next housekeeping deadline at the end of a full tick: the
|
|
2344
|
+
* earliest the tick found, lowered by any deadline armed while it ran
|
|
2345
|
+
* (an arm that lands after this write lowers it itself). Serialised
|
|
2346
|
+
* with the arms, so neither overwrites the other.
|
|
2347
|
+
*/
|
|
2348
|
+
private storeHousekeeping(at: number): Promise<void> {
|
|
2349
|
+
this.alarmArm = this.alarmArm
|
|
2350
|
+
.catch(() => {})
|
|
2351
|
+
.then(async () => {
|
|
2352
|
+
const deadline = Math.min(at, this.housekeepingFloor);
|
|
2353
|
+
this.housekeepingFloor = Infinity;
|
|
2354
|
+
await this.ctx.storage.put(META_HOUSEKEEPING_AT, deadline);
|
|
2355
|
+
});
|
|
2356
|
+
return this.alarmArm;
|
|
2357
|
+
}
|
|
2358
|
+
|
|
1634
2359
|
/**
|
|
1635
2360
|
* One page-tier eviction pass, at most once per `TIER_PASS_INTERVAL_MS`
|
|
1636
2361
|
* (a pass that hit its segment cap re-arms in a minute). Returns the next
|
|
@@ -1687,7 +2412,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1687
2412
|
if (!this.db) {
|
|
1688
2413
|
const userDid =
|
|
1689
2414
|
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
1690
|
-
if (userDid) await this.ready({ userDid });
|
|
2415
|
+
if (userDid) await this.ready({ userDid }, { recordAccess: false });
|
|
1691
2416
|
}
|
|
1692
2417
|
if (!this.db) throw new Error('no working copy');
|
|
1693
2418
|
if (this.flushInFlight) await this.flushInFlight.catch(() => undefined);
|
|
@@ -1747,14 +2472,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1747
2472
|
|
|
1748
2473
|
async createSession(
|
|
1749
2474
|
identity: TurnIdentity,
|
|
1750
|
-
o: { roomId?: string; sessionId?: string } = {},
|
|
2475
|
+
o: { roomId?: string; sessionId?: string; markerTxnId?: string } = {},
|
|
1751
2476
|
): Promise<SessionSummary> {
|
|
1752
2477
|
await this.ready(identity);
|
|
1753
2478
|
const sessions = this.sessions!;
|
|
1754
2479
|
// Like the Node runtime: creating a session sends the MOST RECENT
|
|
1755
2480
|
// previous session's transcript to the memory engine, in the
|
|
1756
|
-
// background — the new session never waits on it.
|
|
1757
|
-
|
|
2481
|
+
// background — the new session never waits on it. Task runs are not
|
|
2482
|
+
// conversations and are never indexed (`scheduleHistoryIndexing`).
|
|
2483
|
+
const { sessions: recent } = await sessions.listSessions(
|
|
2484
|
+
undefined,
|
|
2485
|
+
1,
|
|
2486
|
+
0,
|
|
2487
|
+
TASK_SESSION_PREFIX,
|
|
2488
|
+
);
|
|
1758
2489
|
const previous = recent[0];
|
|
1759
2490
|
if (previous) this.scheduleHistoryIndexing(previous.sessionId);
|
|
1760
2491
|
let sessionId = o.sessionId;
|
|
@@ -1782,7 +2513,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1782
2513
|
// the same transaction id across the restart window, which the
|
|
1783
2514
|
// homeserver deduplicates. Node fails the request on send failure;
|
|
1784
2515
|
// so do we — no silent local id when the user has an oracle room.
|
|
1785
|
-
const txnId = `marker-${crypto.randomUUID()}`;
|
|
2516
|
+
const txnId = o.markerTxnId ?? `marker-${crypto.randomUUID()}`;
|
|
1786
2517
|
sessionId = await retryGateway(
|
|
1787
2518
|
() =>
|
|
1788
2519
|
this.gateway.sendText(room.roomId, NEW_CONVERSATION_TEXT, {
|
|
@@ -1802,6 +2533,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1802
2533
|
}
|
|
1803
2534
|
}
|
|
1804
2535
|
sessionId ??= `s_${crypto.randomUUID()}`;
|
|
2536
|
+
if (o.markerTxnId) {
|
|
2537
|
+
const existing = await sessions.getSession(sessionId);
|
|
2538
|
+
if (existing) return toSummary(existing);
|
|
2539
|
+
}
|
|
1805
2540
|
const row = await sessions.createSession({
|
|
1806
2541
|
sessionId,
|
|
1807
2542
|
roomId,
|
|
@@ -1813,6 +2548,30 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1813
2548
|
return toSummary(row);
|
|
1814
2549
|
}
|
|
1815
2550
|
|
|
2551
|
+
async channelTurn(
|
|
2552
|
+
identity: TurnIdentity,
|
|
2553
|
+
input: ChannelTurnInput,
|
|
2554
|
+
requestHash: string,
|
|
2555
|
+
): Promise<ChannelTurnOutcome> {
|
|
2556
|
+
try {
|
|
2557
|
+
if (identity.ucanDelegation)
|
|
2558
|
+
throw new ChannelError(
|
|
2559
|
+
403,
|
|
2560
|
+
'Channel grants cannot become tool authority',
|
|
2561
|
+
);
|
|
2562
|
+
// The shell (the only caller) checked the binding just before.
|
|
2563
|
+
await this.ready(identity);
|
|
2564
|
+
return {
|
|
2565
|
+
ok: true,
|
|
2566
|
+
result: await this.channelTurns!.submit(identity, input, requestHash),
|
|
2567
|
+
};
|
|
2568
|
+
} catch (error) {
|
|
2569
|
+
if (error instanceof ChannelError)
|
|
2570
|
+
return { ok: false, status: error.status, message: error.message };
|
|
2571
|
+
throw error;
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
|
|
1816
2575
|
async listSessions(
|
|
1817
2576
|
identity: TurnIdentity,
|
|
1818
2577
|
o: { limit?: number; offset?: number } = {},
|
|
@@ -1853,12 +2612,17 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1853
2612
|
sessionId: string,
|
|
1854
2613
|
): Promise<boolean> {
|
|
1855
2614
|
await this.ready(identity);
|
|
2615
|
+
// End every run of the session first: the executing one is aborted
|
|
2616
|
+
// and waited for (up to the supersede grace), queued and recovering
|
|
2617
|
+
// ones are closed — none may start a turn on the deleted thread, or
|
|
2618
|
+
// recreate its session row, after the rows below are gone.
|
|
2619
|
+
await this.runs?.abortAllForSession(sessionId);
|
|
1856
2620
|
this.aborts.get(sessionId)?.abort();
|
|
1857
2621
|
// Node indexes a session into the memory engine as it is deleted.
|
|
1858
2622
|
// The rows are gone once this RPC returns, so capture the transcript
|
|
1859
2623
|
// now and let the background indexer read the snapshot.
|
|
1860
2624
|
const doomed = await this.sessions!.getSession(sessionId);
|
|
1861
|
-
if (doomed) {
|
|
2625
|
+
if (doomed && !sessionId.startsWith(TASK_SESSION_PREFIX)) {
|
|
1862
2626
|
this.historySnapshots.set(sessionId, {
|
|
1863
2627
|
session: doomed,
|
|
1864
2628
|
messages: await this.historyMessages(sessionId),
|
|
@@ -1868,6 +2632,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1868
2632
|
const deleted = await this.sessions!.deleteSession(sessionId);
|
|
1869
2633
|
if (deleted) {
|
|
1870
2634
|
await this.saver!.deleteThread(sessionId);
|
|
2635
|
+
await this.attachmentTextCache
|
|
2636
|
+
?.forgetSession(sessionId)
|
|
2637
|
+
.catch((err: unknown) => {
|
|
2638
|
+
console.warn(
|
|
2639
|
+
`[user-do] could not delete the cached attachment text of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2640
|
+
);
|
|
2641
|
+
});
|
|
1871
2642
|
await this.resultStore
|
|
1872
2643
|
?.deleteForSession(sessionId)
|
|
1873
2644
|
.catch((err: unknown) => {
|
|
@@ -1875,12 +2646,60 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1875
2646
|
`[user-do] could not delete the saved tool results of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1876
2647
|
);
|
|
1877
2648
|
});
|
|
2649
|
+
await this.artifacts
|
|
2650
|
+
?.deleteForSession(sessionId)
|
|
2651
|
+
.catch((err: unknown) => {
|
|
2652
|
+
console.warn(
|
|
2653
|
+
`[user-do] could not delete the artefacts of ${sessionId}; their links expire on schedule: ${err instanceof Error ? err.message : String(err)}`,
|
|
2654
|
+
);
|
|
2655
|
+
});
|
|
1878
2656
|
await this.ctx.storage.delete(contextStatsKey(sessionId));
|
|
2657
|
+
await this.feedbackMarkers!.forgetSession(sessionId);
|
|
1879
2658
|
this.markDirty();
|
|
1880
2659
|
}
|
|
2660
|
+
// A channel binding that pointed at the session opens a new one on
|
|
2661
|
+
// its next message instead of failing the session check for ever.
|
|
2662
|
+
if (await this.channelTurns!.forgetSession(sessionId)) this.markDirty();
|
|
1881
2663
|
return deleted;
|
|
1882
2664
|
}
|
|
1883
2665
|
|
|
2666
|
+
/**
|
|
2667
|
+
* `GET /artifacts/:id` — the canonical copy, for its owner. `url` is
|
|
2668
|
+
* present only while the share link still works.
|
|
2669
|
+
*/
|
|
2670
|
+
async artifact(
|
|
2671
|
+
identity: TurnIdentity,
|
|
2672
|
+
artifactId: string,
|
|
2673
|
+
): Promise<OwnedArtifact | null> {
|
|
2674
|
+
await this.ready(identity);
|
|
2675
|
+
const stored = await this.artifacts?.get(artifactId);
|
|
2676
|
+
if (!stored) return null;
|
|
2677
|
+
const { url, ...ref } = stored.ref;
|
|
2678
|
+
const live =
|
|
2679
|
+
url !== undefined &&
|
|
2680
|
+
!stored.revoked &&
|
|
2681
|
+
Date.parse(ref.expiresAt) > Date.now();
|
|
2682
|
+
return {
|
|
2683
|
+
...ref,
|
|
2684
|
+
...(live ? { url } : {}),
|
|
2685
|
+
sessionId: stored.sessionId,
|
|
2686
|
+
createdAt: stored.createdAt,
|
|
2687
|
+
revoked: stored.revoked,
|
|
2688
|
+
content: stored.content,
|
|
2689
|
+
};
|
|
2690
|
+
}
|
|
2691
|
+
|
|
2692
|
+
/** `DELETE /artifacts/:id` — revoke the share link; the canonical copy stays. */
|
|
2693
|
+
async revokeArtifact(
|
|
2694
|
+
identity: TurnIdentity,
|
|
2695
|
+
artifactId: string,
|
|
2696
|
+
): Promise<boolean> {
|
|
2697
|
+
await this.ready(identity);
|
|
2698
|
+
const revoked = (await this.artifacts?.revoke(artifactId)) ?? false;
|
|
2699
|
+
if (revoked) this.markDirty();
|
|
2700
|
+
return revoked;
|
|
2701
|
+
}
|
|
2702
|
+
|
|
1884
2703
|
/** The session transcript as the indexer wants it (summarisation bookkeeping removed). */
|
|
1885
2704
|
private async historyMessages(
|
|
1886
2705
|
sessionId: string,
|
|
@@ -1890,7 +2709,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1890
2709
|
.filter(
|
|
1891
2710
|
(m) => !isSummarizationMessage(m) && !isAttachmentViewMessage(m),
|
|
1892
2711
|
)
|
|
1893
|
-
.map((m) => ({ type: m.type, content:
|
|
2712
|
+
.map((m) => ({ type: m.type, content: visibleText(m) }));
|
|
1894
2713
|
}
|
|
1895
2714
|
|
|
1896
2715
|
/**
|
|
@@ -1899,8 +2718,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1899
2718
|
* seeded from the row's `lastProcessedCount` for files migrated from Node;
|
|
1900
2719
|
* `touchSession` is not used because it bumps `last_updated_at` and
|
|
1901
2720
|
* would reorder the session list.
|
|
2721
|
+
*
|
|
2722
|
+
* Task-run sessions (`task:<id>`) are never indexed: a run's transcript
|
|
2723
|
+
* is the task's instructions and its result, not something the user
|
|
2724
|
+
* said, and a supplied-context task's source text must not become
|
|
2725
|
+
* long-term memory that ordinary, tool-enabled turns can retrieve.
|
|
1902
2726
|
*/
|
|
1903
2727
|
private scheduleHistoryIndexing(sessionId: string): void {
|
|
2728
|
+
if (sessionId.startsWith(TASK_SESSION_PREFIX)) return;
|
|
1904
2729
|
const ambient = this.ambient;
|
|
1905
2730
|
const userDid = this.userDid;
|
|
1906
2731
|
if (!ambient || !this.sessions || !userDid) return;
|
|
@@ -1984,6 +2809,57 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1984
2809
|
return JSON.stringify(dtos);
|
|
1985
2810
|
}
|
|
1986
2811
|
|
|
2812
|
+
/**
|
|
2813
|
+
* Anonymous feedback, step one (src/feedback/submit.ts): the message must
|
|
2814
|
+
* be a completed Agent reply in this user's own session; then the
|
|
2815
|
+
* idempotency marker and the user's submission limit decide. The shell
|
|
2816
|
+
* holds the feedback text; it never reaches this object.
|
|
2817
|
+
*/
|
|
2818
|
+
async reserveMessageFeedback(
|
|
2819
|
+
identity: TurnIdentity,
|
|
2820
|
+
target: FeedbackTarget,
|
|
2821
|
+
): Promise<FeedbackReservation> {
|
|
2822
|
+
await this.ready(identity);
|
|
2823
|
+
const reservation = await reserveFeedback(
|
|
2824
|
+
{
|
|
2825
|
+
sessionExists: async (sessionId) =>
|
|
2826
|
+
(await this.sessions!.getSession(sessionId)) !== undefined,
|
|
2827
|
+
transcript: async (sessionId) =>
|
|
2828
|
+
(
|
|
2829
|
+
await transformTranscript(
|
|
2830
|
+
(await this.saver!.listThreadMessages(sessionId)).filter(
|
|
2831
|
+
(m) => !isSummarizationMessage(m),
|
|
2832
|
+
),
|
|
2833
|
+
)
|
|
2834
|
+
).messages,
|
|
2835
|
+
// A queued run has not written its user message yet, so it does
|
|
2836
|
+
// not make the latest reply incomplete; a running one has.
|
|
2837
|
+
runActive: async (sessionId) => {
|
|
2838
|
+
const status = (
|
|
2839
|
+
this.runs?.activeForSession(sessionId)?.record ??
|
|
2840
|
+
(await this.runStore?.activeForSession(sessionId))
|
|
2841
|
+
)?.status;
|
|
2842
|
+
return status === 'running' || status === 'recovering';
|
|
2843
|
+
},
|
|
2844
|
+
markers: this.feedbackMarkers!,
|
|
2845
|
+
},
|
|
2846
|
+
target,
|
|
2847
|
+
);
|
|
2848
|
+
if (reservation.kind === 'reserved') this.markDirty();
|
|
2849
|
+
return reservation;
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
/** Anonymous feedback, step two: settle the reservation (`FeedbackMarkers.settle`). */
|
|
2853
|
+
async settleMessageFeedback(
|
|
2854
|
+
identity: TurnIdentity,
|
|
2855
|
+
target: FeedbackTarget,
|
|
2856
|
+
outcome: FeedbackSettlement,
|
|
2857
|
+
): Promise<void> {
|
|
2858
|
+
await this.ready(identity);
|
|
2859
|
+
await this.feedbackMarkers!.settle(target, outcome);
|
|
2860
|
+
this.markDirty();
|
|
2861
|
+
}
|
|
2862
|
+
|
|
1987
2863
|
async listMessagesPage(
|
|
1988
2864
|
identity: TurnIdentity,
|
|
1989
2865
|
sessionId: string,
|
|
@@ -2192,6 +3068,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2192
3068
|
sessionId: req.sessionId,
|
|
2193
3069
|
requestId: req.requestId,
|
|
2194
3070
|
text: existing?.replyText ?? '',
|
|
3071
|
+
...(existing?.replyPlan ? { plan: existing.replyPlan } : {}),
|
|
2195
3072
|
toolCalls: [],
|
|
2196
3073
|
replayed: true,
|
|
2197
3074
|
};
|
|
@@ -2211,7 +3088,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2211
3088
|
await ledger.start(eventId, req.sessionId, req.requestId);
|
|
2212
3089
|
const run = this.runTurnOnce(req)
|
|
2213
3090
|
.then(async (result) => {
|
|
2214
|
-
await ledger.answer(eventId, result.text);
|
|
3091
|
+
await ledger.answer(eventId, result.text, result.plan);
|
|
2215
3092
|
return result;
|
|
2216
3093
|
})
|
|
2217
3094
|
.finally(() => {
|
|
@@ -2259,7 +3136,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2259
3136
|
sessionId: req.sessionId,
|
|
2260
3137
|
requestId: req.requestId,
|
|
2261
3138
|
client: req.client,
|
|
2262
|
-
request:
|
|
3139
|
+
request: this.runRequestJson(
|
|
3140
|
+
req,
|
|
2263
3141
|
storedRunRequest(req, { timezone: req.identity.timezone }),
|
|
2264
3142
|
),
|
|
2265
3143
|
multitask: req.multitask ?? this.runConfig.multitaskDefault,
|
|
@@ -2280,6 +3158,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2280
3158
|
sessionId: req.sessionId,
|
|
2281
3159
|
requestId: req.requestId,
|
|
2282
3160
|
text: outcome.text,
|
|
3161
|
+
...(outcome.plan ? { plan: JSON.stringify(outcome.plan) } : {}),
|
|
2283
3162
|
...(outcome.messageId !== undefined
|
|
2284
3163
|
? { messageId: outcome.messageId }
|
|
2285
3164
|
: {}),
|
|
@@ -2306,6 +3185,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2306
3185
|
sessionId: req.sessionId,
|
|
2307
3186
|
requestId: req.requestId,
|
|
2308
3187
|
text: outcome.text,
|
|
3188
|
+
...(outcome.plan ? { plan: JSON.stringify(outcome.plan) } : {}),
|
|
2309
3189
|
...(outcome.messageId ? { messageId: outcome.messageId } : {}),
|
|
2310
3190
|
toolCalls: outcome.toolCalls ?? [],
|
|
2311
3191
|
replayed: true,
|
|
@@ -2313,10 +3193,12 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2313
3193
|
}
|
|
2314
3194
|
const record = await this.runStore?.getByRequestId(requestId);
|
|
2315
3195
|
if (record?.status === 'finished') {
|
|
3196
|
+
const plan = parseReplyPlan(await this.runStore!.getPlan(record.runId));
|
|
2316
3197
|
return {
|
|
2317
3198
|
sessionId: req.sessionId,
|
|
2318
3199
|
requestId: req.requestId,
|
|
2319
3200
|
text: record.partialText ?? '',
|
|
3201
|
+
...(plan ? { plan: JSON.stringify(plan) } : {}),
|
|
2320
3202
|
...(record.messageId ? { messageId: record.messageId } : {}),
|
|
2321
3203
|
toolCalls: [],
|
|
2322
3204
|
replayed: true,
|
|
@@ -2359,6 +3241,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2359
3241
|
// One export at the end, not per batch — flushing a multi-GB file
|
|
2360
3242
|
// after every 300 rows would dwarf the savings.
|
|
2361
3243
|
this.markDirty();
|
|
3244
|
+
} else if (await this.unflushedLocalWrites(db)) {
|
|
3245
|
+
// Nothing saved, but the closing vacuum rewrote the file: an
|
|
3246
|
+
// unmarked write would keep the owner copy unverifiable, and the
|
|
3247
|
+
// idle wipe waiting on it, until some later boot noticed.
|
|
3248
|
+
this.markDirty();
|
|
2362
3249
|
}
|
|
2363
3250
|
return false;
|
|
2364
3251
|
}
|
|
@@ -2421,7 +3308,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2421
3308
|
// an operator's `POST /debug/storage/flush` is never a silent no-op.
|
|
2422
3309
|
const userDid =
|
|
2423
3310
|
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
2424
|
-
if (userDid) await this.ready({ userDid });
|
|
3311
|
+
if (userDid) await this.ready({ userDid }, { recordAccess: false });
|
|
2425
3312
|
}
|
|
2426
3313
|
if (this.flushInFlight) return this.flushInFlight;
|
|
2427
3314
|
const run = this.flushOnce().finally(() => {
|
|
@@ -2521,7 +3408,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2521
3408
|
// Defence in depth: never drop the backstop for an empty working copy —
|
|
2522
3409
|
// a zero-turn file is not worth confirming against, and redacting here
|
|
2523
3410
|
// would strip a migrated user's only history if VFS held a stub.
|
|
2524
|
-
if (this.db && (await this.
|
|
3411
|
+
if (this.db && !(await this.hasLocalTurns(this.db))) return false;
|
|
2525
3412
|
try {
|
|
2526
3413
|
const removed = await store.removeLegacyCopy(etag);
|
|
2527
3414
|
await this.ctx.storage.put(META_LEGACY_CLEARED, true);
|
|
@@ -2573,21 +3460,36 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2573
3460
|
uploadedGen !== this.db.writeGeneration
|
|
2574
3461
|
) {
|
|
2575
3462
|
const at = Date.now() + FLUSH_DEBOUNCE_MS;
|
|
3463
|
+
this.flushAt = at;
|
|
2576
3464
|
await this.ctx.storage.put(META_FLUSH_AT, at);
|
|
2577
|
-
this.
|
|
3465
|
+
await this.requestHousekeeping(at);
|
|
2578
3466
|
return;
|
|
2579
3467
|
}
|
|
2580
3468
|
this.dirty = false;
|
|
3469
|
+
this.flushAt = null;
|
|
2581
3470
|
await this.ctx.storage.delete([META_DIRTY, META_FLUSH_AT]);
|
|
2582
3471
|
}
|
|
2583
3472
|
|
|
2584
|
-
|
|
3473
|
+
/**
|
|
3474
|
+
* `POST /debug/storage/reset`: upload what is unsaved, drop the working
|
|
3475
|
+
* copy and boot again from the owner copy. `identity` is the caller the
|
|
3476
|
+
* shell authenticated, with the delegation the request carried: an
|
|
3477
|
+
* object that never booted boots for it (the owner store needs that
|
|
3478
|
+
* delegation), and an object bound to another DID refuses before
|
|
3479
|
+
* anything is dropped.
|
|
3480
|
+
*/
|
|
3481
|
+
async resetWorkingCopy(
|
|
3482
|
+
identity?: TurnIdentity,
|
|
3483
|
+
): Promise<{ reloadedFromOwnerStore: boolean }> {
|
|
2585
3484
|
const userDid =
|
|
2586
|
-
|
|
3485
|
+
identity?.userDid ??
|
|
3486
|
+
this.userDid ??
|
|
3487
|
+
(await this.ctx.storage.get<string>(META_USER_DID));
|
|
3488
|
+
const caller = identity ?? (userDid ? { userDid } : undefined);
|
|
2587
3489
|
// On a cold (evicted) object the in-memory dirty flag is gone — boot and
|
|
2588
3490
|
// consult the persisted marker, or the wipe below discards changes the
|
|
2589
|
-
// owner store never received.
|
|
2590
|
-
if (
|
|
3491
|
+
// owner store never received. A booted object only checks the caller.
|
|
3492
|
+
if (caller) await this.ready(caller, { recordAccess: false });
|
|
2591
3493
|
if (
|
|
2592
3494
|
this.dirty ||
|
|
2593
3495
|
(await this.ctx.storage.get<boolean>(META_DIRTY)) === true
|
|
@@ -2596,21 +3498,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2596
3498
|
await this.flushToOwnerStore();
|
|
2597
3499
|
}
|
|
2598
3500
|
await this.wipeWorkingCopy(this.db);
|
|
2599
|
-
this.
|
|
2600
|
-
this.
|
|
2601
|
-
this.sessions = null;
|
|
2602
|
-
this.matrixLedger = null;
|
|
2603
|
-
this.runStore = null;
|
|
2604
|
-
this.runs = null;
|
|
2605
|
-
this.resultStore = null;
|
|
2606
|
-
this.sessionRooms.clear();
|
|
2607
|
-
this.mainRoomId = null;
|
|
2608
|
-
this.ambient = null;
|
|
2609
|
-
this.secretsService = null;
|
|
2610
|
-
this.byo = null;
|
|
2611
|
-
this.initPromise = null;
|
|
2612
|
-
this.reloadedFromOwnerStore = false;
|
|
2613
|
-
if (userDid) await this.ready({ userDid });
|
|
3501
|
+
this.dropInMemoryState();
|
|
3502
|
+
if (caller) await this.ready(caller, { recordAccess: false });
|
|
2614
3503
|
return { reloadedFromOwnerStore: this.reloadedFromOwnerStore };
|
|
2615
3504
|
}
|
|
2616
3505
|
|
|
@@ -2624,7 +3513,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2624
3513
|
userDid: string,
|
|
2625
3514
|
sessionId: string,
|
|
2626
3515
|
): Promise<Record<string, unknown> | null> {
|
|
2627
|
-
await this.ready({ userDid });
|
|
3516
|
+
await this.ready({ userDid }, { recordAccess: false });
|
|
2628
3517
|
const session = await this.sessions!.getSession(sessionId);
|
|
2629
3518
|
if (!session) return null;
|
|
2630
3519
|
// Whether the thread's agent context was condensed: the summarization
|
|
@@ -2656,7 +3545,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2656
3545
|
}
|
|
2657
3546
|
|
|
2658
3547
|
async debugMemorySchema(userDid: string): Promise<MemorySchemaDebug> {
|
|
2659
|
-
await this.ready({ userDid });
|
|
3548
|
+
await this.ready({ userDid }, { recordAccess: false });
|
|
2660
3549
|
const ambient = this.ambient;
|
|
2661
3550
|
const url = this.core.validatedEnv['MEMORY_MCP_URL'];
|
|
2662
3551
|
if (!ambient || typeof url !== 'string' || url.length === 0) {
|
|
@@ -2719,7 +3608,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2719
3608
|
lastVacuumAt: await this.ctx.storage.get<number>(META_LAST_VACUUM),
|
|
2720
3609
|
legacyCleared:
|
|
2721
3610
|
(await this.ctx.storage.get<boolean>(META_LEGACY_CLEARED)) === true,
|
|
2722
|
-
lastAccessAt: await this.
|
|
3611
|
+
lastAccessAt: await this.knownLastAccess(),
|
|
2723
3612
|
indexingInFlight: this.historyIndexer?.inFlightCount ?? 0,
|
|
2724
3613
|
activeTurns: this.aborts.size,
|
|
2725
3614
|
pendingTimers: timerTrackerInstalled()
|
|
@@ -2754,6 +3643,49 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2754
3643
|
};
|
|
2755
3644
|
}
|
|
2756
3645
|
|
|
3646
|
+
async topicDeliverable(
|
|
3647
|
+
identity: TurnIdentity,
|
|
3648
|
+
operationId: string,
|
|
3649
|
+
command: TopicDeliverableCommand,
|
|
3650
|
+
): Promise<TopicDeliverableResult> {
|
|
3651
|
+
if (this.env.TOPIC_DELIVERABLES_ENABLED !== 'true')
|
|
3652
|
+
return { ok: false, status: 404, message: 'Not found.' };
|
|
3653
|
+
TopicOperationId.parse(operationId);
|
|
3654
|
+
// Like every request: waits out an idle eviction in progress and
|
|
3655
|
+
// records the access, so the idle tick re-reads this as activity.
|
|
3656
|
+
await this.ready(identity);
|
|
3657
|
+
const scheduler = this.taskScheduler;
|
|
3658
|
+
const db = this.db;
|
|
3659
|
+
if (!scheduler || !db) throw new Error('Task scheduler unavailable');
|
|
3660
|
+
// Start and cancel write the task row outside any turn; nothing else
|
|
3661
|
+
// would schedule their upload (and an unmarked write keeps the owner
|
|
3662
|
+
// copy "not current", which blocks eviction for good). A write that
|
|
3663
|
+
// committed before a later step threw (the alarm arm) counts too.
|
|
3664
|
+
const execute = (): Promise<TopicDeliverableResult> => {
|
|
3665
|
+
switch (command.action) {
|
|
3666
|
+
case 'start':
|
|
3667
|
+
return scheduler.startTopicDeliverable(
|
|
3668
|
+
operationId,
|
|
3669
|
+
TopicDeliverableRequestSchema.parse(command.request),
|
|
3670
|
+
);
|
|
3671
|
+
case 'read':
|
|
3672
|
+
return scheduler.readTopicDeliverable(operationId);
|
|
3673
|
+
case 'cancel':
|
|
3674
|
+
return scheduler.cancelTopicDeliverable(
|
|
3675
|
+
operationId,
|
|
3676
|
+
TopicDeliverableRequestSchema.parse(command.request),
|
|
3677
|
+
);
|
|
3678
|
+
}
|
|
3679
|
+
};
|
|
3680
|
+
const generation = db.writeGeneration;
|
|
3681
|
+
try {
|
|
3682
|
+
return await execute();
|
|
3683
|
+
} finally {
|
|
3684
|
+
if (this.db === db && db.writeGeneration !== generation)
|
|
3685
|
+
this.markDirty();
|
|
3686
|
+
}
|
|
3687
|
+
}
|
|
3688
|
+
|
|
2757
3689
|
async tasksStatus(userDid?: string): Promise<{
|
|
2758
3690
|
now: number;
|
|
2759
3691
|
alarm: number | null;
|
|
@@ -2781,7 +3713,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2781
3713
|
if (!this.taskScheduler) {
|
|
2782
3714
|
const did =
|
|
2783
3715
|
userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
2784
|
-
if (did) await this.ready({ userDid: did });
|
|
3716
|
+
if (did) await this.ready({ userDid: did }, { recordAccess: false });
|
|
2785
3717
|
}
|
|
2786
3718
|
const records = this.taskScheduler
|
|
2787
3719
|
? await this.taskScheduler.surface.list()
|
|
@@ -2831,13 +3763,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2831
3763
|
}
|
|
2832
3764
|
if (auth.ucanDelegation)
|
|
2833
3765
|
headers.set('x-ucan-delegation', auth.ucanDelegation);
|
|
2834
|
-
return authenticate(headers,
|
|
2835
|
-
oracleDid: this.env.ORACLE_DID,
|
|
2836
|
-
blocksyncUri: this.env.BLOCKSYNC_GRAPHQL_URL,
|
|
2837
|
-
maxTtlSeconds: this.env.UCAN_AUTH_MAX_TTL_SECONDS
|
|
2838
|
-
? Number(this.env.UCAN_AUTH_MAX_TTL_SECONDS)
|
|
2839
|
-
: undefined,
|
|
2840
|
-
});
|
|
3766
|
+
return authenticate(headers, authConfigFromEnv(this.env));
|
|
2841
3767
|
},
|
|
2842
3768
|
sessionExists: async (userDid, sessionId) => {
|
|
2843
3769
|
await this.ready({ userDid });
|
|
@@ -2845,7 +3771,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2845
3771
|
},
|
|
2846
3772
|
router: this.events,
|
|
2847
3773
|
logger: console,
|
|
2848
|
-
requestAlarm: (at) => this.requestAlarm(at),
|
|
3774
|
+
requestAlarm: (at) => this.ctx.waitUntil(this.requestAlarm(at)),
|
|
2849
3775
|
// Node indexes a session into the memory engine when its last socket
|
|
2850
3776
|
// disconnects (`WsService.removeClientConnection`); same trigger here.
|
|
2851
3777
|
onSessionDrained: (sessionId) =>
|
|
@@ -2975,10 +3901,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2975
3901
|
};
|
|
2976
3902
|
const stream = body.stream !== false;
|
|
2977
3903
|
if (!stream) {
|
|
2978
|
-
this.replayToRoom(req, body.message, 'user');
|
|
3904
|
+
void this.replayToRoom(req, body.message, 'user');
|
|
2979
3905
|
try {
|
|
2980
3906
|
const result = await this.runTurn(req);
|
|
2981
|
-
this.replayToRoom(req, result.text, 'oracle');
|
|
3907
|
+
void this.replayToRoom(req, result.text, 'oracle');
|
|
2982
3908
|
// Node's `SendMessageResponse.message` is `{ type, content, id }`.
|
|
2983
3909
|
const payload: Record<string, unknown> = {
|
|
2984
3910
|
message: {
|
|
@@ -3011,13 +3937,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3011
3937
|
// A streamed turn is a durable run: recorded, kept alive and resumable
|
|
3012
3938
|
// (docs/plans/durable-runs.md). The response is one subscriber of the
|
|
3013
3939
|
// run's buffer; the run itself outlives it.
|
|
3014
|
-
this.replayToRoom(req, body.message, 'user');
|
|
3940
|
+
void this.replayToRoom(req, body.message, 'user');
|
|
3015
3941
|
const { live, queued } = await this.runs!.begin({
|
|
3016
3942
|
runId: crypto.randomUUID(),
|
|
3017
3943
|
sessionId,
|
|
3018
3944
|
requestId,
|
|
3019
3945
|
client: 'portal',
|
|
3020
|
-
request:
|
|
3946
|
+
request: this.runRequestJson(
|
|
3947
|
+
req,
|
|
3021
3948
|
storedRunRequest(req, {
|
|
3022
3949
|
timezone: body.timezone,
|
|
3023
3950
|
tools: body.tools,
|
|
@@ -3112,6 +4039,23 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3112
4039
|
});
|
|
3113
4040
|
}
|
|
3114
4041
|
|
|
4042
|
+
/**
|
|
4043
|
+
* The stored request of a new run. When no admission handler can see the
|
|
4044
|
+
* turn (no plugin declares one, or the turn is out of admission's scope)
|
|
4045
|
+
* it is recorded as an agent turn from the start, which spares the first
|
|
4046
|
+
* attempt both rewrites of the row (`admitting`, then `agent`). Recovery
|
|
4047
|
+
* stays correct: a run the graph never checkpointed is retried with its
|
|
4048
|
+
* input whatever its disposition.
|
|
4049
|
+
*/
|
|
4050
|
+
private runRequestJson(req: TurnRequest, stored: StoredRunRequest): string {
|
|
4051
|
+
const admissible =
|
|
4052
|
+
admissionApplies(req) &&
|
|
4053
|
+
this.core.plugins.some((plugin) => plugin.getRequestAdmission);
|
|
4054
|
+
return JSON.stringify(
|
|
4055
|
+
admissible ? stored : { ...stored, disposition: { kind: 'agent' } },
|
|
4056
|
+
);
|
|
4057
|
+
}
|
|
4058
|
+
|
|
3115
4059
|
/**
|
|
3116
4060
|
* One attempt of a run (the coordinator's `runAttempt` host hook): build
|
|
3117
4061
|
* the agent for the stored request, stream the turn's frames into the
|
|
@@ -3125,6 +4069,100 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3125
4069
|
): Promise<RunOutcome> {
|
|
3126
4070
|
const stored = JSON.parse(live.record.request) as StoredRunRequest;
|
|
3127
4071
|
const req = stored.turn;
|
|
4072
|
+
if (req.client === 'channel') {
|
|
4073
|
+
await this.ctx.storage.sync();
|
|
4074
|
+
// The binding was checked as the request was admitted; a queued or
|
|
4075
|
+
// recovered attempt starts later and checks it again.
|
|
4076
|
+
if (live.attemptSource !== 'begin')
|
|
4077
|
+
await assertChannelAttemptAllowed(req.identity, this.env);
|
|
4078
|
+
requireChannelDelegation(
|
|
4079
|
+
!!this.delegations.get(req.identity.userDid)?.raw,
|
|
4080
|
+
);
|
|
4081
|
+
}
|
|
4082
|
+
// An attempt that starts from `admitting` never reached the agent (the
|
|
4083
|
+
// previous incarnation stopped during admission), so the user's message
|
|
4084
|
+
// is not in the graph: the agent runs with its input, not from the
|
|
4085
|
+
// checkpoint.
|
|
4086
|
+
const freshInput = !resumed || stored.disposition?.kind === 'admitting';
|
|
4087
|
+
let announced = resumed;
|
|
4088
|
+
const announce = (): void => {
|
|
4089
|
+
if (announced) return;
|
|
4090
|
+
announced = true;
|
|
4091
|
+
live.buffer.push('run', {
|
|
4092
|
+
runId: live.runId,
|
|
4093
|
+
sessionId: req.sessionId,
|
|
4094
|
+
requestId: req.requestId,
|
|
4095
|
+
});
|
|
4096
|
+
};
|
|
4097
|
+
if (!stored.disposition && !resumed) {
|
|
4098
|
+
stored.disposition = { kind: 'admitting' };
|
|
4099
|
+
const request = JSON.stringify(stored);
|
|
4100
|
+
await this.runStore!.update(live.runId, { request });
|
|
4101
|
+
live.record.request = request;
|
|
4102
|
+
}
|
|
4103
|
+
if (!stored.disposition || stored.disposition.kind === 'admitting') {
|
|
4104
|
+
let admission: RequestAdmissionResult = { kind: 'pass' };
|
|
4105
|
+
try {
|
|
4106
|
+
if (stored.disposition?.kind === 'admitting' && admissionApplies(req))
|
|
4107
|
+
admission = await admitRequest(
|
|
4108
|
+
this.core.plugins,
|
|
4109
|
+
{
|
|
4110
|
+
user: {
|
|
4111
|
+
did: req.identity.userDid,
|
|
4112
|
+
matrixUserId: req.identity.matrixUserId ?? '',
|
|
4113
|
+
ucanDelegation: { raw: req.identity.ucanDelegation ?? '' },
|
|
4114
|
+
timezone: req.identity.timezone,
|
|
4115
|
+
currentTime: new Date().toISOString(),
|
|
4116
|
+
},
|
|
4117
|
+
session: {
|
|
4118
|
+
id: req.sessionId,
|
|
4119
|
+
client: req.client,
|
|
4120
|
+
requestId: req.requestId,
|
|
4121
|
+
roomId:
|
|
4122
|
+
req.roomId ??
|
|
4123
|
+
(await this.sessions!.getSession(req.sessionId))?.roomId,
|
|
4124
|
+
...(req.roomKind ? { roomKind: req.roomKind } : {}),
|
|
4125
|
+
...(req.eventId ? { eventId: req.eventId } : {}),
|
|
4126
|
+
...(req.threadId ? { threadId: req.threadId } : {}),
|
|
4127
|
+
},
|
|
4128
|
+
message: req.message,
|
|
4129
|
+
metadata: admissionMetadata(req.metadata),
|
|
4130
|
+
signal: live.abort.signal,
|
|
4131
|
+
},
|
|
4132
|
+
{
|
|
4133
|
+
env: this.core.validatedEnv,
|
|
4134
|
+
timeoutMs: admissionTimeoutMs(this.core.validatedEnv),
|
|
4135
|
+
warn: (m) => console.warn(`[user-do] ${m}`),
|
|
4136
|
+
},
|
|
4137
|
+
);
|
|
4138
|
+
} catch (error) {
|
|
4139
|
+
return this.refuseTurn(live, req, error, announce, 'admission');
|
|
4140
|
+
}
|
|
4141
|
+
stored.disposition =
|
|
4142
|
+
admission.kind === 'handled'
|
|
4143
|
+
? {
|
|
4144
|
+
kind: 'direct-read',
|
|
4145
|
+
text: admission.text,
|
|
4146
|
+
title: admission.title,
|
|
4147
|
+
messageId: `direct:${req.sessionId}:${req.requestId}:ai`,
|
|
4148
|
+
}
|
|
4149
|
+
: { kind: 'agent' };
|
|
4150
|
+
const request = JSON.stringify(stored);
|
|
4151
|
+
await this.runStore!.update(live.runId, { request });
|
|
4152
|
+
live.record.request = request;
|
|
4153
|
+
}
|
|
4154
|
+
if (stored.disposition.kind === 'direct-read') {
|
|
4155
|
+
try {
|
|
4156
|
+
return await this.runDirectRead(
|
|
4157
|
+
live,
|
|
4158
|
+
req,
|
|
4159
|
+
stored.disposition,
|
|
4160
|
+
announce,
|
|
4161
|
+
);
|
|
4162
|
+
} catch (error) {
|
|
4163
|
+
return this.refuseTurn(live, req, error, announce, 'direct read');
|
|
4164
|
+
}
|
|
4165
|
+
}
|
|
3128
4166
|
const {
|
|
3129
4167
|
agent,
|
|
3130
4168
|
stateInput,
|
|
@@ -3133,6 +4171,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3133
4171
|
byoNotice,
|
|
3134
4172
|
byoProvider,
|
|
3135
4173
|
toolOutputCapChars,
|
|
4174
|
+
delivery,
|
|
4175
|
+
usage,
|
|
3136
4176
|
} = await this.prepareTurn(
|
|
3137
4177
|
req,
|
|
3138
4178
|
{
|
|
@@ -3141,13 +4181,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3141
4181
|
model: req.model,
|
|
3142
4182
|
tools: stored.tools,
|
|
3143
4183
|
agActions: stored.agActions,
|
|
3144
|
-
...(
|
|
4184
|
+
...(freshInput ? { attachments: req.attachments } : {}),
|
|
3145
4185
|
},
|
|
3146
4186
|
{
|
|
3147
4187
|
runId: live.runId,
|
|
3148
4188
|
abortController: live.abort,
|
|
3149
|
-
resumed,
|
|
3150
|
-
continuation:
|
|
4189
|
+
resumed: !freshInput,
|
|
4190
|
+
continuation: freshInput ? null : live.continuation,
|
|
3151
4191
|
},
|
|
3152
4192
|
);
|
|
3153
4193
|
const sessionId = req.sessionId;
|
|
@@ -3157,20 +4197,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3157
4197
|
live.buffer.push(eventName, payload);
|
|
3158
4198
|
},
|
|
3159
4199
|
};
|
|
3160
|
-
this.events.register(sessionId, sink);
|
|
3161
|
-
if (byoNotice)
|
|
3162
|
-
sink.emit('error', {
|
|
3163
|
-
...byoNotice,
|
|
3164
|
-
sessionId,
|
|
3165
|
-
requestId: req.requestId,
|
|
3166
|
-
});
|
|
3167
|
-
const events = agent.streamEvents(resumed ? null : stateInput, {
|
|
3168
|
-
...config,
|
|
3169
|
-
version: 'v2',
|
|
3170
|
-
});
|
|
3171
4200
|
const capture: BaseMessage[] = [];
|
|
3172
4201
|
let completed = false;
|
|
4202
|
+
// Everything after `prepareTurn` runs inside the try: its cleanups
|
|
4203
|
+
// (the turn's deadline timer among them) run however the attempt ends.
|
|
3173
4204
|
try {
|
|
4205
|
+
this.events.register(sessionId, sink);
|
|
4206
|
+
if (byoNotice)
|
|
4207
|
+
sink.emit('error', {
|
|
4208
|
+
...byoNotice,
|
|
4209
|
+
sessionId,
|
|
4210
|
+
requestId: req.requestId,
|
|
4211
|
+
});
|
|
4212
|
+
const events = agent.streamEvents(freshInput ? stateInput : null, {
|
|
4213
|
+
...config,
|
|
4214
|
+
version: 'v2',
|
|
4215
|
+
});
|
|
3174
4216
|
const outcome = await runTurnFrames({
|
|
3175
4217
|
events: tapMessages(events, capture),
|
|
3176
4218
|
sink: live.buffer,
|
|
@@ -3188,8 +4230,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3188
4230
|
onFrame: () => this.runs?.touchKeepAlive(),
|
|
3189
4231
|
onComplete: async () => {
|
|
3190
4232
|
completed = true;
|
|
3191
|
-
|
|
3192
|
-
|
|
4233
|
+
if (req.client !== 'channel')
|
|
4234
|
+
void this.replayToRoom(req, lastAiText(capture), 'oracle');
|
|
4235
|
+
await this.afterTurn(
|
|
4236
|
+
sessionId,
|
|
4237
|
+
capture,
|
|
4238
|
+
{ kind: 'agent' },
|
|
4239
|
+
live.abort.signal,
|
|
4240
|
+
);
|
|
3193
4241
|
},
|
|
3194
4242
|
onError: (err) => {
|
|
3195
4243
|
console.error(`[user-do] turn ${req.requestId} failed:`, err);
|
|
@@ -3204,8 +4252,18 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3204
4252
|
status: 'done' as const,
|
|
3205
4253
|
})),
|
|
3206
4254
|
);
|
|
3207
|
-
const text = completed ? lastAiText(capture) : outcome.fullText;
|
|
3208
4255
|
const messageId = completed ? lastAiMessageId(capture) : undefined;
|
|
4256
|
+
// A chat surface also gets its reply as a plan. `text` stays the
|
|
4257
|
+
// model's own text on every surface: stored results (task runs, the
|
|
4258
|
+
// run record, the Matrix ledger) never hold a chat rendering whose
|
|
4259
|
+
// links expire.
|
|
4260
|
+
const plan =
|
|
4261
|
+
completed && delivery.kind === 'chat'
|
|
4262
|
+
? await this.replyPlanOf(live, req, capture, delivery)
|
|
4263
|
+
: undefined;
|
|
4264
|
+
const text =
|
|
4265
|
+
(live.continuation ?? '') +
|
|
4266
|
+
(completed ? lastAiText(capture) : outcome.fullText);
|
|
3209
4267
|
return {
|
|
3210
4268
|
status:
|
|
3211
4269
|
outcome.status === 'completed'
|
|
@@ -3213,10 +4271,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3213
4271
|
: outcome.status === 'aborted'
|
|
3214
4272
|
? 'aborted'
|
|
3215
4273
|
: 'failed',
|
|
3216
|
-
text
|
|
4274
|
+
text,
|
|
4275
|
+
...(plan ? { plan } : {}),
|
|
3217
4276
|
...(messageId ? { messageId } : {}),
|
|
3218
4277
|
toolCalls,
|
|
3219
4278
|
...(outcome.status === 'failed' ? { error: outcome.error } : {}),
|
|
4279
|
+
// Written with the terminal status: one row update per run.
|
|
4280
|
+
usage: usage(),
|
|
3220
4281
|
};
|
|
3221
4282
|
} finally {
|
|
3222
4283
|
this.events.unregister(sessionId, sink);
|
|
@@ -3226,6 +4287,178 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3226
4287
|
}
|
|
3227
4288
|
}
|
|
3228
4289
|
|
|
4290
|
+
/**
|
|
4291
|
+
* The Reply Plan of a finished chat-surface run, built once from the
|
|
4292
|
+
* turn's messages and kept with the run (`/channels/turn` polls read it
|
|
4293
|
+
* back). A long step becomes an artefact when storage is configured; an
|
|
4294
|
+
* artefact that fails falls back to plain messages.
|
|
4295
|
+
*/
|
|
4296
|
+
private async replyPlanOf(
|
|
4297
|
+
live: LiveRun,
|
|
4298
|
+
req: TurnRequest,
|
|
4299
|
+
capture: BaseMessage[],
|
|
4300
|
+
delivery: Extract<DeliveryProfile, { kind: 'chat' }>,
|
|
4301
|
+
): Promise<ReplyPlan> {
|
|
4302
|
+
const artifacts = this.artifacts?.canShare ? this.artifacts : null;
|
|
4303
|
+
const { steps, toolResults } = turnSteps(capture);
|
|
4304
|
+
const draft = draftReplyPlan({
|
|
4305
|
+
steps,
|
|
4306
|
+
toolResults,
|
|
4307
|
+
continuation: live.continuation,
|
|
4308
|
+
limits: delivery.limits,
|
|
4309
|
+
canSpill: artifacts !== null,
|
|
4310
|
+
});
|
|
4311
|
+
const plan = await materializeReplyPlan(draft, {
|
|
4312
|
+
limits: delivery.limits,
|
|
4313
|
+
createSpill: async (key, spill) => {
|
|
4314
|
+
if (!artifacts)
|
|
4315
|
+
throw new Error('artifacts: storage is not configured');
|
|
4316
|
+
return artifacts.create({
|
|
4317
|
+
artifactId: await artifactIdFor(live.runId, `spill:${key}`),
|
|
4318
|
+
sessionId: req.sessionId,
|
|
4319
|
+
runId: live.runId,
|
|
4320
|
+
title: spill.title,
|
|
4321
|
+
content: spill.markdown,
|
|
4322
|
+
});
|
|
4323
|
+
},
|
|
4324
|
+
onSpillError: (error) =>
|
|
4325
|
+
console.warn(
|
|
4326
|
+
`[delivery] run ${live.runId}: could not store a long reply as an artefact; sending it as messages: ${error instanceof Error ? error.message : String(error)}`,
|
|
4327
|
+
),
|
|
4328
|
+
});
|
|
4329
|
+
await this.runStore!.setPlan(live.runId, JSON.stringify(plan));
|
|
4330
|
+
console.log(
|
|
4331
|
+
`[delivery] run ${live.runId}: ${delivery.surface} reply in ${plan.parts.length} parts (${plan.parts.filter((p) => p.kind === 'artifact').length} artefacts)`,
|
|
4332
|
+
);
|
|
4333
|
+
return plan;
|
|
4334
|
+
}
|
|
4335
|
+
|
|
4336
|
+
private async runDirectRead(
|
|
4337
|
+
live: LiveRun,
|
|
4338
|
+
req: TurnRequest,
|
|
4339
|
+
disposition: Extract<RequestDisposition, { kind: 'direct-read' }>,
|
|
4340
|
+
announce: () => void,
|
|
4341
|
+
): Promise<RunOutcome> {
|
|
4342
|
+
live.abort.signal.throwIfAborted();
|
|
4343
|
+
announce();
|
|
4344
|
+
const sessions = this.sessions!;
|
|
4345
|
+
if (
|
|
4346
|
+
req.client === 'matrix' &&
|
|
4347
|
+
!(await sessions.getSession(req.sessionId))
|
|
4348
|
+
) {
|
|
4349
|
+
await sessions.createSession({
|
|
4350
|
+
sessionId: req.sessionId,
|
|
4351
|
+
roomId: req.roomId,
|
|
4352
|
+
oracleName: this.core.identity.name,
|
|
4353
|
+
oracleDid: this.env.ORACLE_DID,
|
|
4354
|
+
oracleEntityDid: this.core.identity.entityDid,
|
|
4355
|
+
});
|
|
4356
|
+
}
|
|
4357
|
+
const kwargs = {
|
|
4358
|
+
timestamp: new Date().toISOString(),
|
|
4359
|
+
oracleName: this.core.identity.name,
|
|
4360
|
+
msgFromMatrixRoom: req.client === 'matrix',
|
|
4361
|
+
...(req.client === 'matrix'
|
|
4362
|
+
? {
|
|
4363
|
+
senderDid: req.identity.userDid,
|
|
4364
|
+
senderMatrixUserId: req.identity.matrixUserId,
|
|
4365
|
+
senderDisplayName: req.senderDisplayName ?? req.identity.userDid,
|
|
4366
|
+
threadId: req.sessionId,
|
|
4367
|
+
eventId: req.eventId,
|
|
4368
|
+
}
|
|
4369
|
+
: {}),
|
|
4370
|
+
};
|
|
4371
|
+
const messages = [
|
|
4372
|
+
new HumanMessage({
|
|
4373
|
+
id: `direct:${req.sessionId}:${req.requestId}:human`,
|
|
4374
|
+
content:
|
|
4375
|
+
req.client === 'matrix' && req.roomKind === 'group'
|
|
4376
|
+
? prefixSpeaker(
|
|
4377
|
+
req.message,
|
|
4378
|
+
req.senderDisplayName ??
|
|
4379
|
+
req.identity.matrixUserId ??
|
|
4380
|
+
req.identity.userDid,
|
|
4381
|
+
)
|
|
4382
|
+
: req.message,
|
|
4383
|
+
additional_kwargs: kwargs,
|
|
4384
|
+
}),
|
|
4385
|
+
new AIMessage({
|
|
4386
|
+
id: disposition.messageId,
|
|
4387
|
+
content: disposition.text,
|
|
4388
|
+
additional_kwargs: {
|
|
4389
|
+
timestamp: kwargs.timestamp,
|
|
4390
|
+
oracleName: kwargs.oracleName,
|
|
4391
|
+
msgFromMatrixRoom: kwargs.msgFromMatrixRoom,
|
|
4392
|
+
},
|
|
4393
|
+
}),
|
|
4394
|
+
];
|
|
4395
|
+
await this.saver!.appendTurnMessages(req.sessionId, messages);
|
|
4396
|
+
// An abort that lands now ends the run as `aborted`; the reply stays
|
|
4397
|
+
// in the transcript (the Matrix mirror and the title are skipped).
|
|
4398
|
+
live.abort.signal.throwIfAborted();
|
|
4399
|
+
await this.afterTurn(req.sessionId, messages, disposition);
|
|
4400
|
+
// A channel's reply reaches the room through the channel's own
|
|
4401
|
+
// delivery (`onRunEnded`), as on the agent path; its mirror is the
|
|
4402
|
+
// required kind, whose failure would surface here unhandled.
|
|
4403
|
+
if (req.client !== 'channel')
|
|
4404
|
+
void this.replayToRoom(req, disposition.text, 'oracle');
|
|
4405
|
+
const delivered = live.continuation ?? '';
|
|
4406
|
+
const content = disposition.text.startsWith(delivered)
|
|
4407
|
+
? disposition.text.slice(delivered.length)
|
|
4408
|
+
: disposition.text;
|
|
4409
|
+
if (content)
|
|
4410
|
+
live.buffer.push('message', { content, timestamp: kwargs.timestamp });
|
|
4411
|
+
live.buffer.push('done', {
|
|
4412
|
+
runId: live.runId,
|
|
4413
|
+
messageId: disposition.messageId,
|
|
4414
|
+
});
|
|
4415
|
+
return {
|
|
4416
|
+
status: 'finished',
|
|
4417
|
+
text: disposition.text,
|
|
4418
|
+
messageId: disposition.messageId,
|
|
4419
|
+
toolCalls: [],
|
|
4420
|
+
};
|
|
4421
|
+
}
|
|
4422
|
+
|
|
4423
|
+
/**
|
|
4424
|
+
* End a turn whose admission or direct read did not complete. The client
|
|
4425
|
+
* gets the same terminal frames as a failed agent turn (`error`, then
|
|
4426
|
+
* `done`) or, after a user abort or a superseding message, `done` with
|
|
4427
|
+
* `aborted`. The handler's own message goes to the log only.
|
|
4428
|
+
*/
|
|
4429
|
+
private refuseTurn(
|
|
4430
|
+
live: LiveRun,
|
|
4431
|
+
req: TurnRequest,
|
|
4432
|
+
error: unknown,
|
|
4433
|
+
announce: () => void,
|
|
4434
|
+
stage: 'admission' | 'direct read',
|
|
4435
|
+
): RunOutcome {
|
|
4436
|
+
announce();
|
|
4437
|
+
if (live.abort.signal.aborted) {
|
|
4438
|
+
live.buffer.push('done', { runId: live.runId, aborted: true });
|
|
4439
|
+
return { status: 'aborted', text: live.continuation ?? '' };
|
|
4440
|
+
}
|
|
4441
|
+
console.error(
|
|
4442
|
+
`[user-do] turn ${req.requestId}: ${stage} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
4443
|
+
);
|
|
4444
|
+
live.buffer.push('error', {
|
|
4445
|
+
error: 'This request could not be answered. Please try again.',
|
|
4446
|
+
kind: 'request_admission',
|
|
4447
|
+
source: 'platform',
|
|
4448
|
+
retryable: true,
|
|
4449
|
+
sessionId: req.sessionId,
|
|
4450
|
+
requestId: req.requestId,
|
|
4451
|
+
runId: live.runId,
|
|
4452
|
+
timestamp: new Date().toISOString(),
|
|
4453
|
+
});
|
|
4454
|
+
live.buffer.push('done', { runId: live.runId, failed: true });
|
|
4455
|
+
return {
|
|
4456
|
+
status: 'failed',
|
|
4457
|
+
text: live.continuation ?? '',
|
|
4458
|
+
error: new Error(`request ${stage} failed`),
|
|
4459
|
+
};
|
|
4460
|
+
}
|
|
4461
|
+
|
|
3229
4462
|
/** Latest checkpoint id of a session — recovery's progress marker. */
|
|
3230
4463
|
private async checkpointIdOf(sessionId: string): Promise<string | null> {
|
|
3231
4464
|
const tuple = await this.saver
|
|
@@ -3243,6 +4476,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3243
4476
|
record: RunRecord,
|
|
3244
4477
|
outcome: RunOutcome,
|
|
3245
4478
|
): Promise<void> {
|
|
4479
|
+
if (record.client === 'channel' && outcome.status === 'finished') {
|
|
4480
|
+
const stored = JSON.parse(record.request) as StoredRunRequest;
|
|
4481
|
+
// Without the handle (a reset dropped it mid-run) the next poll of
|
|
4482
|
+
// the request delivers the reply instead.
|
|
4483
|
+
await this.channelTurns?.deliverReply(
|
|
4484
|
+
stored.turn,
|
|
4485
|
+
record.runId,
|
|
4486
|
+
outcome.text,
|
|
4487
|
+
);
|
|
4488
|
+
}
|
|
3246
4489
|
if (!record.taskRunId || !this.taskScheduler) return;
|
|
3247
4490
|
if (this.taskScheduler.isRunActive(record.taskRunId)) return;
|
|
3248
4491
|
if (outcome.status === 'finished')
|
|
@@ -3288,7 +4531,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3288
4531
|
const window = await this.contextWindows.resolve(modelId, {
|
|
3289
4532
|
...(byoProvider ? { byoProvider } : {}),
|
|
3290
4533
|
});
|
|
3291
|
-
const budget = contextBudgetFor(
|
|
4534
|
+
const budget = contextBudgetFor(
|
|
4535
|
+
window,
|
|
4536
|
+
contextKnobs(this.env, console),
|
|
4537
|
+
await this.contextWindows.resolve(
|
|
4538
|
+
llm && isProviderAdapter(llm)
|
|
4539
|
+
? llm.modelForRole('routing')
|
|
4540
|
+
: OPENROUTER_MODEL_MAP.routing,
|
|
4541
|
+
),
|
|
4542
|
+
);
|
|
3292
4543
|
const stats = (await this.resultStore?.stats()) ?? {
|
|
3293
4544
|
rows: 0,
|
|
3294
4545
|
sqliteBytes: 0,
|
|
@@ -3403,7 +4654,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3403
4654
|
async runsStatus(): Promise<RunsStatus> {
|
|
3404
4655
|
const userDid =
|
|
3405
4656
|
this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
|
|
3406
|
-
if (!this.db && userDid)
|
|
4657
|
+
if (!this.db && userDid)
|
|
4658
|
+
await this.ready({ userDid }, { recordAccess: false });
|
|
3407
4659
|
const store = this.runStore;
|
|
3408
4660
|
const records = store ? await store.listRecent(50) : [];
|
|
3409
4661
|
const runs: RunsStatus['runs'] = [];
|
|
@@ -3425,12 +4677,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3425
4677
|
nextAttemptAt: record.nextAttemptAt,
|
|
3426
4678
|
error: record.error,
|
|
3427
4679
|
taskRunId: record.taskRunId,
|
|
4680
|
+
usage: parseUsage(record.usage),
|
|
3428
4681
|
});
|
|
3429
4682
|
}
|
|
3430
4683
|
return {
|
|
3431
4684
|
config: { ...this.runConfig },
|
|
3432
4685
|
live: this.runs?.snapshot() ?? [],
|
|
3433
4686
|
runs,
|
|
4687
|
+
claims: store ? await store.listClaims() : [],
|
|
3434
4688
|
};
|
|
3435
4689
|
}
|
|
3436
4690
|
|
|
@@ -3443,25 +4697,39 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3443
4697
|
* the gateway (its whole-buffer download grew the crypto WASM heap for
|
|
3444
4698
|
* good, and a hash mismatch on the far side of the RPC would only reach
|
|
3445
4699
|
* this object as a disconnect).
|
|
4700
|
+
*
|
|
4701
|
+
* The gateway bounds each transfer to the attachment download deadline,
|
|
4702
|
+
* from opening it to the last byte. An `AbortSignal` cannot cross the
|
|
4703
|
+
* Durable Object RPC, so the turn's abort acts on this side instead:
|
|
4704
|
+
* `readBytesCapped` cancels the stream it reads, and the cancel travels
|
|
4705
|
+
* back over the RPC stream to stop the gateway's download.
|
|
3446
4706
|
*/
|
|
3447
4707
|
private matrixMediaSource(): MatrixMediaSource {
|
|
4708
|
+
const deadline = { timeoutMs: DOWNLOAD_TIMEOUT_MS };
|
|
3448
4709
|
return {
|
|
3449
|
-
downloadMxc: async (mxc, maxBytes = MAX_FILE_SIZE) =>
|
|
4710
|
+
downloadMxc: async (mxc, maxBytes = MAX_FILE_SIZE, signal) =>
|
|
3450
4711
|
readBytesCapped(
|
|
3451
|
-
await this.gateway.downloadMxcMediaStream(mxc),
|
|
4712
|
+
await this.gateway.downloadMxcMediaStream(mxc, deadline),
|
|
3452
4713
|
maxBytes,
|
|
4714
|
+
signal,
|
|
3453
4715
|
),
|
|
3454
|
-
downloadEvent: async (
|
|
4716
|
+
downloadEvent: async (
|
|
4717
|
+
roomId,
|
|
4718
|
+
eventId,
|
|
4719
|
+
maxBytes = MAX_FILE_SIZE,
|
|
4720
|
+
signal,
|
|
4721
|
+
) => {
|
|
3455
4722
|
const media = await this.gateway.downloadEventMediaStream(
|
|
3456
4723
|
roomId,
|
|
3457
4724
|
eventId,
|
|
4725
|
+
deadline,
|
|
3458
4726
|
);
|
|
3459
4727
|
if (!media) return null;
|
|
3460
4728
|
const plain = media.file
|
|
3461
4729
|
? media.stream.pipeThrough(createAttachmentDecryptor(media.file))
|
|
3462
4730
|
: media.stream;
|
|
3463
4731
|
return {
|
|
3464
|
-
bytes: await readBytesCapped(plain, maxBytes),
|
|
4732
|
+
bytes: await readBytesCapped(plain, maxBytes, signal),
|
|
3465
4733
|
...(media.mimetype ? { mimetype: media.mimetype } : {}),
|
|
3466
4734
|
...(media.filename ? { filename: media.filename } : {}),
|
|
3467
4735
|
};
|
|
@@ -3523,6 +4791,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3523
4791
|
roomId: input.roomId,
|
|
3524
4792
|
signal: input.signal,
|
|
3525
4793
|
logger: console,
|
|
4794
|
+
...(this.attachmentTextCache
|
|
4795
|
+
? { cache: this.attachmentTextCache.forSession(input.sessionId) }
|
|
4796
|
+
: {}),
|
|
3526
4797
|
});
|
|
3527
4798
|
return { meta, view };
|
|
3528
4799
|
},
|
|
@@ -3626,23 +4897,73 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3626
4897
|
};
|
|
3627
4898
|
}
|
|
3628
4899
|
|
|
4900
|
+
/**
|
|
4901
|
+
* The model that summarises the turn's history (the `routing` role): its
|
|
4902
|
+
* window bounds what the summariser is handed.
|
|
4903
|
+
*/
|
|
4904
|
+
private summaryModelId(
|
|
4905
|
+
platform: LlmAdapter,
|
|
4906
|
+
byoTurn: ByoTurnState | null,
|
|
4907
|
+
): string {
|
|
4908
|
+
if (byoTurn)
|
|
4909
|
+
return (
|
|
4910
|
+
byoModelForRole(
|
|
4911
|
+
byoTurn.credential.provider,
|
|
4912
|
+
'routing',
|
|
4913
|
+
byoTurn.mainModelId,
|
|
4914
|
+
) ?? OPENROUTER_MODEL_MAP.routing
|
|
4915
|
+
);
|
|
4916
|
+
return isProviderAdapter(platform)
|
|
4917
|
+
? platform.modelForRole('routing')
|
|
4918
|
+
: OPENROUTER_MODEL_MAP.routing;
|
|
4919
|
+
}
|
|
4920
|
+
|
|
4921
|
+
/**
|
|
4922
|
+
* Build one attempt's agent and input (`buildTurn`). A build that fails
|
|
4923
|
+
* part-way leaves nothing behind: the cleanups it registered run (the
|
|
4924
|
+
* turn's deadline timer would otherwise keep the object resident for
|
|
4925
|
+
* the whole turn limit and abort a finished run later) and the session's
|
|
4926
|
+
* abort entry is released.
|
|
4927
|
+
*/
|
|
3629
4928
|
private async prepareTurn(
|
|
3630
4929
|
req: TurnRequest,
|
|
3631
|
-
body:
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
}
|
|
4930
|
+
body: TurnBuildBody,
|
|
4931
|
+
run: TurnBuildRun,
|
|
4932
|
+
) {
|
|
4933
|
+
const turnDisposables = new Set<() => void | Promise<void>>();
|
|
4934
|
+
try {
|
|
4935
|
+
return await this.buildTurn(req, body, run, turnDisposables);
|
|
4936
|
+
} catch (err) {
|
|
4937
|
+
await this.runTurnDisposables(turnDisposables);
|
|
4938
|
+
if (this.aborts.get(req.sessionId) === run.abortController)
|
|
4939
|
+
this.aborts.delete(req.sessionId);
|
|
4940
|
+
throw err;
|
|
4941
|
+
}
|
|
4942
|
+
}
|
|
4943
|
+
|
|
4944
|
+
private async buildTurn(
|
|
4945
|
+
req: TurnRequest,
|
|
4946
|
+
body: TurnBuildBody,
|
|
4947
|
+
run: TurnBuildRun,
|
|
4948
|
+
turnDisposables: Set<() => void | Promise<void>>,
|
|
3643
4949
|
) {
|
|
4950
|
+
if (req.executionProfile && !this.taskScheduler)
|
|
4951
|
+
throw new Error('Task scheduler unavailable');
|
|
4952
|
+
await this.taskScheduler?.assertTurnProfile(req);
|
|
4953
|
+
const suppliedContextOnly =
|
|
4954
|
+
req.executionProfile === 'supplied-context-markdown';
|
|
4955
|
+
if (
|
|
4956
|
+
suppliedContextOnly &&
|
|
4957
|
+
(body.attachments?.length ||
|
|
4958
|
+
body.tools?.length ||
|
|
4959
|
+
body.agActions?.length)
|
|
4960
|
+
) {
|
|
4961
|
+
throw new Error('Supplied-context task accepts plain text only');
|
|
4962
|
+
}
|
|
3644
4963
|
const core = this.core;
|
|
3645
4964
|
const baseAmbient = this.ambient!;
|
|
4965
|
+
// One instant per turn: the prompt's date and the message's time note.
|
|
4966
|
+
const turnAt = new Date();
|
|
3646
4967
|
const saver = this.saver!;
|
|
3647
4968
|
const sessions = this.sessions!;
|
|
3648
4969
|
|
|
@@ -3729,6 +5050,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3729
5050
|
const contextBudget = contextBudgetFor(
|
|
3730
5051
|
windowResolution,
|
|
3731
5052
|
contextKnobs(this.env, console),
|
|
5053
|
+
await this.contextWindows.resolve(
|
|
5054
|
+
this.summaryModelId(baseAmbient.llm, byoTurn),
|
|
5055
|
+
byoTurn ? { byoProvider: byoTurn.provider } : {},
|
|
5056
|
+
),
|
|
3732
5057
|
);
|
|
3733
5058
|
if (
|
|
3734
5059
|
req.client === 'matrix' &&
|
|
@@ -3753,15 +5078,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3753
5078
|
// memory engine rejects calls that carry no `x-room-id` (as a generic
|
|
3754
5079
|
// "invalid token"). Matrix-ingress turns bring the room; HTTP turns
|
|
3755
5080
|
// take it from the session row or resolve the user↔oracle room alias.
|
|
3756
|
-
const sessionRoomId =
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
.
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
5081
|
+
const sessionRoomId = suppliedContextOnly
|
|
5082
|
+
? undefined
|
|
5083
|
+
: (req.roomId ??
|
|
5084
|
+
(await sessions.getSession(req.sessionId))?.roomId ??
|
|
5085
|
+
(
|
|
5086
|
+
await this.gateway
|
|
5087
|
+
.resolveUserRoom(req.identity.userDid)
|
|
5088
|
+
.catch(() => null)
|
|
5089
|
+
)?.roomId);
|
|
5090
|
+
if (!suppliedContextOnly && !sessionRoomId) {
|
|
3765
5091
|
console.warn(
|
|
3766
5092
|
`[user-do] no oracle room resolved for ${req.identity.userDid}; room-scoped plugins (memory) will be unavailable this turn`,
|
|
3767
5093
|
);
|
|
@@ -3784,7 +5110,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3784
5110
|
// on turn 1, exactly like the Node AgentBuilder. Best-effort: a
|
|
3785
5111
|
// preferences read failure never fails the turn.
|
|
3786
5112
|
const userPreferences =
|
|
3787
|
-
sessionRoomId && this.preferences
|
|
5113
|
+
!suppliedContextOnly && sessionRoomId && this.preferences
|
|
3788
5114
|
? await this.preferences.get(sessionRoomId).catch((err: unknown) => {
|
|
3789
5115
|
console.warn(
|
|
3790
5116
|
`[user-do] could not load user preferences for ${sessionRoomId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -3834,18 +5160,19 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3834
5160
|
// `retainAttachmentPayloads`); the tool fetches one again on demand.
|
|
3835
5161
|
// Cleanups registered by tools for THIS turn (MCP clients …): run once
|
|
3836
5162
|
// the turn ends, however it ends — see `RuntimeContext.onTurnEnd`.
|
|
3837
|
-
const turnDisposables = new Set<() => void | Promise<void>>();
|
|
3838
5163
|
abortController.signal.addEventListener('abort', () => {
|
|
3839
5164
|
void this.runTurnDisposables(turnDisposables);
|
|
3840
5165
|
});
|
|
3841
|
-
const attachmentAccess =
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
5166
|
+
const attachmentAccess = suppliedContextOnly
|
|
5167
|
+
? undefined
|
|
5168
|
+
: await this.attachmentViewSurface({
|
|
5169
|
+
sessionId: req.sessionId,
|
|
5170
|
+
roomId: sessionRoomId,
|
|
5171
|
+
model: attachmentModel,
|
|
5172
|
+
ambientLlm: baseAmbient.llm,
|
|
5173
|
+
platform: core.llm,
|
|
5174
|
+
signal: abortController.signal,
|
|
5175
|
+
});
|
|
3849
5176
|
|
|
3850
5177
|
const existing = await saver.getTupleWithoutMessages({
|
|
3851
5178
|
configurable: { thread_id: req.sessionId },
|
|
@@ -3853,29 +5180,153 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3853
5180
|
const priorState = existing?.checkpoint.channel_values ?? {};
|
|
3854
5181
|
// Request metadata (editor room, space, session run, entity) → state,
|
|
3855
5182
|
// by the Node agent-builder's rules (see turn-metadata.ts).
|
|
3856
|
-
const meta = parseTurnMetadata(
|
|
3857
|
-
|
|
5183
|
+
const meta = parseTurnMetadata(
|
|
5184
|
+
suppliedContextOnly ? undefined : req.metadata,
|
|
5185
|
+
);
|
|
5186
|
+
const priorMeta = priorMetadataState(
|
|
5187
|
+
suppliedContextOnly ? {} : priorState,
|
|
5188
|
+
);
|
|
5189
|
+
|
|
5190
|
+
// Capability router: predict the on-demand plugin this message needs
|
|
5191
|
+
// and preload it for THIS turn only. `on` is awaited here, ahead of the
|
|
5192
|
+
// first model call; `shadow` returns at once and logs its verdict when
|
|
5193
|
+
// it lands. The preload never enters `stateInput` — the checkpointed
|
|
5194
|
+
// `loadedPlugins` channel is written by `load_capability` alone.
|
|
5195
|
+
const priorLoaded: ReadonlySet<string> = new Set(
|
|
5196
|
+
priorMeta.loadedPlugins ?? [],
|
|
5197
|
+
);
|
|
5198
|
+
this.shadowRoutes.delete(req.sessionId);
|
|
5199
|
+
// LangSmith: resolved before the router so its Decision, which runs
|
|
5200
|
+
// ahead of the graph, lands on the turn's tracer under the same
|
|
5201
|
+
// allowlist gate. Metadata is attached unconditionally (inert without a
|
|
5202
|
+
// tracer); the explicit tracer only when this turn is traced (global
|
|
5203
|
+
// switch or per-DID allowlist — see `resolveLangsmithTracing`).
|
|
5204
|
+
// A supplied-context turn is never traced: its source and output stay
|
|
5205
|
+
// off third-party services whatever the tracing switches say.
|
|
5206
|
+
const tracing: LangsmithTracingDecision = suppliedContextOnly
|
|
5207
|
+
? { metadata: {} }
|
|
5208
|
+
: resolveLangsmithTracing({
|
|
5209
|
+
userDid: req.identity.userDid,
|
|
5210
|
+
client: req.client,
|
|
5211
|
+
env: langsmithEnvFromWorkerEnv(this.env),
|
|
5212
|
+
});
|
|
5213
|
+
// The delegation the whole turn reads: the router's candidates here,
|
|
5214
|
+
// every capability check of the build and its tools below.
|
|
5215
|
+
const ucanDelegation = await this.turnDelegation(req.identity);
|
|
5216
|
+
const preloadedPlugins =
|
|
5217
|
+
!suppliedContextOnly && this.capabilityRouter
|
|
5218
|
+
? await this.capabilityRouter({
|
|
5219
|
+
mode: capabilityRouterMode(core.validatedEnv.CAPABILITY_ROUTER),
|
|
5220
|
+
manifests: core.registries.manifests.collect(),
|
|
5221
|
+
loaded: priorLoaded,
|
|
5222
|
+
hidden: () =>
|
|
5223
|
+
bootHiddenPlugins({
|
|
5224
|
+
registries: core.registries,
|
|
5225
|
+
buildCtx: core.buildCtx(),
|
|
5226
|
+
has: (resource, action) =>
|
|
5227
|
+
ambient.ucan.hasCapability(
|
|
5228
|
+
ucanDelegation,
|
|
5229
|
+
resource,
|
|
5230
|
+
action,
|
|
5231
|
+
),
|
|
5232
|
+
}),
|
|
5233
|
+
hasCapability: (resource, action) =>
|
|
5234
|
+
ambient.ucan.hasCapability(ucanDelegation, resource, action),
|
|
5235
|
+
text: body.message,
|
|
5236
|
+
requestId: req.requestId,
|
|
5237
|
+
signal: abortController.signal,
|
|
5238
|
+
trace: {
|
|
5239
|
+
...(tracing.callbacks && { callbacks: tracing.callbacks }),
|
|
5240
|
+
// `thread_id` matches the graph run's, so LangSmith's thread
|
|
5241
|
+
// view groups the router span with the turn.
|
|
5242
|
+
metadata: {
|
|
5243
|
+
...tracing.metadata,
|
|
5244
|
+
thread_id: req.sessionId,
|
|
5245
|
+
request_id: req.requestId,
|
|
5246
|
+
},
|
|
5247
|
+
},
|
|
5248
|
+
onShadowVerdict: (wouldPreload) =>
|
|
5249
|
+
this.shadowRoutes.set(req.sessionId, {
|
|
5250
|
+
requestId: req.requestId,
|
|
5251
|
+
priorLoaded,
|
|
5252
|
+
wouldPreload,
|
|
5253
|
+
}),
|
|
5254
|
+
})
|
|
5255
|
+
: undefined;
|
|
3858
5256
|
|
|
3859
5257
|
// Host page-context / safety-guardrail hooks, resolved against this
|
|
3860
5258
|
// object's ambient services (see `OracleWorkerHooks`).
|
|
3861
|
-
const hostRoomTitle =
|
|
3862
|
-
|
|
5259
|
+
const hostRoomTitle = suppliedContextOnly
|
|
5260
|
+
? undefined
|
|
5261
|
+
: opts.hooks?.getRoomTitle;
|
|
5262
|
+
const hostSafetyModel = suppliedContextOnly
|
|
5263
|
+
? undefined
|
|
5264
|
+
: opts.hooks?.safetyModel;
|
|
3863
5265
|
|
|
3864
5266
|
// The client-declared tool surface: this body's, else the thread's
|
|
3865
5267
|
// checkpointed one (run-request.ts).
|
|
3866
|
-
const surface =
|
|
5268
|
+
const surface = suppliedContextOnly
|
|
5269
|
+
? clientSurfaceFor({}, {})
|
|
5270
|
+
: clientSurfaceFor(body, priorState);
|
|
3867
5271
|
|
|
3868
5272
|
// Write-ahead tool marks + the resume policy (tool-marks.ts). The
|
|
3869
5273
|
// effect map is filled from the build below; the closure reads it at
|
|
3870
5274
|
// call time, so the middleware can be created before the agent.
|
|
3871
5275
|
const toolEffects = new Map<string, ToolEffect>();
|
|
5276
|
+
const subAgentToolNames = new Set<string>();
|
|
5277
|
+
const effectOf = (name: string): ToolEffect =>
|
|
5278
|
+
toolEffects.get(name) ?? toolEffectOf({ name });
|
|
3872
5279
|
const marksMiddleware = createToolMarksMiddleware({
|
|
3873
5280
|
runId: run.runId,
|
|
3874
5281
|
store: this.runStore!,
|
|
3875
|
-
effectOf
|
|
5282
|
+
effectOf,
|
|
3876
5283
|
continuation: run.continuation,
|
|
3877
5284
|
logger: console,
|
|
3878
5285
|
});
|
|
5286
|
+
// The turn's budget (turn-budget.ts): model calls are charged by the
|
|
5287
|
+
// metered adapter every model of this turn comes from, tool calls by
|
|
5288
|
+
// the execution middleware below, and the deadline aborts the run
|
|
5289
|
+
// with the limit as the reason (the stream reports it as a terminal
|
|
5290
|
+
// error, not as a user abort).
|
|
5291
|
+
const turnLimits = turnLimitsFromEnv(core.validatedEnv);
|
|
5292
|
+
const budget = new TurnBudget(turnLimits);
|
|
5293
|
+
const meteredLlm = budgetedLlm(ambient.llm, {
|
|
5294
|
+
budget,
|
|
5295
|
+
outputReserveTokens: contextBudget.outputReserveTokens,
|
|
5296
|
+
signal: abortController.signal,
|
|
5297
|
+
});
|
|
5298
|
+
const deadline = setTimeout(() => {
|
|
5299
|
+
if (abortController.signal.aborted) return;
|
|
5300
|
+
console.warn(
|
|
5301
|
+
`[harness] turn ${req.requestId} reached its deadline (${turnLimits.durationMs} ms); aborting`,
|
|
5302
|
+
);
|
|
5303
|
+
abortController.abort(
|
|
5304
|
+
new HarnessLimitError(
|
|
5305
|
+
'budget_exhausted',
|
|
5306
|
+
'time',
|
|
5307
|
+
'The turn reached its time limit. Completed work is preserved; no further work was started.',
|
|
5308
|
+
),
|
|
5309
|
+
);
|
|
5310
|
+
}, turnLimits.durationMs);
|
|
5311
|
+
turnDisposables.add(async () => {
|
|
5312
|
+
clearTimeout(deadline);
|
|
5313
|
+
const usage = budget.snapshot();
|
|
5314
|
+
console.log(
|
|
5315
|
+
`[harness] turn ${req.requestId} usage: ~${usage.tokens} tokens (${usage.reportedTokens} reported over ${usage.modelCalls} model calls), ${usage.toolAttempts} tool attempts, ${usage.elapsedMs} ms`,
|
|
5316
|
+
);
|
|
5317
|
+
});
|
|
5318
|
+
const laneOf = (name: string): ToolLane =>
|
|
5319
|
+
subAgentToolNames.has(name) ? 'subagent' : effectOf(name);
|
|
5320
|
+
const executionMiddleware = createToolExecutionMiddleware({
|
|
5321
|
+
budget,
|
|
5322
|
+
scheduler: this.toolScheduler,
|
|
5323
|
+
laneOf,
|
|
5324
|
+
runId: run.runId,
|
|
5325
|
+
sessionId: req.sessionId,
|
|
5326
|
+
claims: this.runStore!,
|
|
5327
|
+
signal: abortController.signal,
|
|
5328
|
+
logger: console,
|
|
5329
|
+
});
|
|
3879
5330
|
// Every tool result above the budget's cap is saved whole and handed
|
|
3880
5331
|
// to the model as head + tail + handle (result-cap.ts); applied to
|
|
3881
5332
|
// sub-agents too through `toolMiddlewares`.
|
|
@@ -3892,15 +5343,45 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3892
5343
|
// inside the wrapper (already under the cap by the time it runs).
|
|
3893
5344
|
const capMiddleware = createResultCapMiddleware(resultCap);
|
|
3894
5345
|
|
|
5346
|
+
// Chat delivery: a channel or Matrix turn replies as short messages and
|
|
5347
|
+
// may hand long content to `create_artifact` (bound only when artefact
|
|
5348
|
+
// storage is configured). The Portal streams as it always has.
|
|
5349
|
+
const delivery = resolveDeliveryProfile(req, core.delivery);
|
|
5350
|
+
const artifacts =
|
|
5351
|
+
delivery.kind === 'chat' && this.artifacts?.canShare
|
|
5352
|
+
? this.artifacts
|
|
5353
|
+
: null;
|
|
5354
|
+
const turnTools = artifacts
|
|
5355
|
+
? [
|
|
5356
|
+
{
|
|
5357
|
+
tool: buildCreateArtifactTool(async (input) =>
|
|
5358
|
+
artifacts.create({
|
|
5359
|
+
artifactId: await artifactIdFor(run.runId, input.source),
|
|
5360
|
+
sessionId: req.sessionId,
|
|
5361
|
+
runId: run.runId,
|
|
5362
|
+
title: input.title,
|
|
5363
|
+
content: input.content,
|
|
5364
|
+
}),
|
|
5365
|
+
),
|
|
5366
|
+
returnDirect: true,
|
|
5367
|
+
},
|
|
5368
|
+
]
|
|
5369
|
+
: [];
|
|
5370
|
+
|
|
3895
5371
|
const built = await createMainAgent({
|
|
5372
|
+
executionProfile: req.executionProfile,
|
|
3896
5373
|
registries: core.registries,
|
|
3897
5374
|
identity: core.identity,
|
|
3898
5375
|
config: core.validatedEnv,
|
|
3899
5376
|
availablePlugins: core.availablePlugins,
|
|
5377
|
+
preloadedPlugins,
|
|
3900
5378
|
byoProvider: byoTurn?.provider,
|
|
3901
5379
|
contextBudget,
|
|
5380
|
+
turnBudget: budget,
|
|
5381
|
+
delivery,
|
|
3902
5382
|
ambient: {
|
|
3903
5383
|
...ambient,
|
|
5384
|
+
llm: meteredLlm,
|
|
3904
5385
|
attachments: attachmentAccess,
|
|
3905
5386
|
onTurnEnd: (dispose) => turnDisposables.add(dispose),
|
|
3906
5387
|
},
|
|
@@ -3913,7 +5394,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3913
5394
|
// Applied to the main agent AND every sub-agent: a sub-agent's
|
|
3914
5395
|
// own tool calls are marked and capped too.
|
|
3915
5396
|
toolMiddlewares: [marksMiddleware, capMiddleware],
|
|
5397
|
+
// Budget, scheduling and write claims, innermost around each tool
|
|
5398
|
+
// call (main agent and sub-agents alike).
|
|
5399
|
+
toolExecution: executionMiddleware,
|
|
3916
5400
|
resultCap,
|
|
5401
|
+
turnTools,
|
|
3917
5402
|
onContextOverflow: (error) =>
|
|
3918
5403
|
this.contextWindows.learnFromError(mainModelId, error, {
|
|
3919
5404
|
...(byoTurn ? { byoProvider: byoTurn.provider } : {}),
|
|
@@ -3941,9 +5426,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3941
5426
|
req.identity.matrixUserId ??
|
|
3942
5427
|
(await this.resolveMatrixUserId(req.identity.userDid)) ??
|
|
3943
5428
|
'',
|
|
3944
|
-
ucanDelegation
|
|
5429
|
+
ucanDelegation,
|
|
3945
5430
|
timezone: req.identity.timezone,
|
|
3946
|
-
currentTime:
|
|
5431
|
+
currentTime: turnAt.toISOString(),
|
|
3947
5432
|
},
|
|
3948
5433
|
session: {
|
|
3949
5434
|
id: req.sessionId,
|
|
@@ -3954,7 +5439,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3954
5439
|
model: effectiveModel,
|
|
3955
5440
|
},
|
|
3956
5441
|
state: {
|
|
3957
|
-
...priorState,
|
|
5442
|
+
...(suppliedContextOnly ? {} : priorState),
|
|
3958
5443
|
userPreferences,
|
|
3959
5444
|
...metadataBuildState(meta, priorMeta),
|
|
3960
5445
|
// The client-declared surface: the portal and AG-UI plugins turn
|
|
@@ -3969,6 +5454,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3969
5454
|
const { agent, context } = built;
|
|
3970
5455
|
for (const [name, effect] of built.toolEffects)
|
|
3971
5456
|
toolEffects.set(name, effect);
|
|
5457
|
+
for (const name of built.subAgentToolNames) subAgentToolNames.add(name);
|
|
3972
5458
|
|
|
3973
5459
|
const attachmentKwargs =
|
|
3974
5460
|
prepared && prepared.metas.length > 0
|
|
@@ -3994,19 +5480,31 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3994
5480
|
}
|
|
3995
5481
|
: {};
|
|
3996
5482
|
const rawContent = prepared ? prepared.content : body.message;
|
|
3997
|
-
const
|
|
5483
|
+
const spoken =
|
|
3998
5484
|
req.client === 'matrix' && req.roomKind === 'group'
|
|
3999
5485
|
? prefixSpeaker(rawContent, speakerName)
|
|
4000
5486
|
: rawContent;
|
|
5487
|
+
// The exact time rides on the message (outermost, after the speaker
|
|
5488
|
+
// prefix and the attachment blocks), so the system prompt carries the
|
|
5489
|
+
// date only and stays identical across the session's turns. A
|
|
5490
|
+
// supplied-context task's input stays exactly what was supplied.
|
|
5491
|
+
const timeNote = suppliedContextOnly
|
|
5492
|
+
? undefined
|
|
5493
|
+
: renderTurnTimeNote(turnAt, req.identity.timezone);
|
|
5494
|
+
const content = timeNote ? withTurnTimeNote(spoken, timeNote) : spoken;
|
|
4001
5495
|
const stateInput = {
|
|
4002
5496
|
messages: [
|
|
4003
5497
|
new HumanMessage({
|
|
4004
5498
|
content,
|
|
4005
5499
|
additional_kwargs: {
|
|
4006
|
-
timestamp:
|
|
5500
|
+
timestamp: turnAt.toISOString(),
|
|
5501
|
+
...(timeNote ? { [TURN_TIME_NOTE_KWARG]: timeNote } : {}),
|
|
4007
5502
|
oracleName: core.identity.name,
|
|
4008
5503
|
msgFromMatrixRoom: req.client === 'matrix',
|
|
4009
5504
|
...speakerKwargs,
|
|
5505
|
+
...(req.channel
|
|
5506
|
+
? { 'org.ixo.qi.origin': channelOrigin(req.channel) }
|
|
5507
|
+
: {}),
|
|
4010
5508
|
...attachmentKwargs,
|
|
4011
5509
|
},
|
|
4012
5510
|
}),
|
|
@@ -4029,22 +5527,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4029
5527
|
...metadataGraphInput(meta, priorMeta),
|
|
4030
5528
|
...surface.input,
|
|
4031
5529
|
};
|
|
4032
|
-
// LangSmith: metadata is attached unconditionally (inert without a
|
|
4033
|
-
// tracer); the explicit tracer only when this turn is traced (global
|
|
4034
|
-
// switch or per-DID allowlist — see `resolveLangsmithTracing`).
|
|
4035
|
-
const tracing = resolveLangsmithTracing({
|
|
4036
|
-
userDid: req.identity.userDid,
|
|
4037
|
-
client: req.client,
|
|
4038
|
-
env: langsmithEnvFromWorkerEnv(this.env),
|
|
4039
|
-
});
|
|
4040
|
-
|
|
4041
5530
|
const config = {
|
|
4042
5531
|
configurable: { thread_id: req.sessionId },
|
|
4043
5532
|
context,
|
|
4044
5533
|
signal: abortController.signal,
|
|
4045
5534
|
recursionLimit: turnRecursionLimit(this.env),
|
|
4046
5535
|
metadata: tracing.metadata,
|
|
4047
|
-
|
|
5536
|
+
callbacks: [meteredLlm.callback, ...(tracing.callbacks ?? [])],
|
|
4048
5537
|
};
|
|
4049
5538
|
return {
|
|
4050
5539
|
agent,
|
|
@@ -4055,6 +5544,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4055
5544
|
byoProvider: byoTurn?.provider,
|
|
4056
5545
|
turnDisposables,
|
|
4057
5546
|
toolOutputCapChars: contextBudget.resultCapChars,
|
|
5547
|
+
/** The budget's usage so far (JSON), stored with the run's end. */
|
|
5548
|
+
usage: (): string => JSON.stringify(budget.snapshot()),
|
|
5549
|
+
delivery,
|
|
4058
5550
|
};
|
|
4059
5551
|
}
|
|
4060
5552
|
|
|
@@ -4070,18 +5562,19 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4070
5562
|
req: TurnRequest,
|
|
4071
5563
|
text: string,
|
|
4072
5564
|
who: 'user' | 'oracle',
|
|
4073
|
-
): void {
|
|
4074
|
-
if (req.client === 'matrix') return;
|
|
4075
|
-
if (req.sessionId.startsWith(SYNTHETIC_SESSION_PREFIX))
|
|
5565
|
+
): Promise<void> {
|
|
5566
|
+
if (req.client === 'matrix') return Promise.resolve();
|
|
5567
|
+
if (req.sessionId.startsWith(SYNTHETIC_SESSION_PREFIX))
|
|
5568
|
+
return Promise.resolve();
|
|
4076
5569
|
// A local session id has no marker event to thread on: the user had
|
|
4077
5570
|
// no oracle room when the session was created, so there is no room
|
|
4078
5571
|
// transcript to mirror either.
|
|
4079
|
-
if (!req.sessionId.startsWith('$')) return;
|
|
4080
|
-
if (!text.trim()) return;
|
|
5572
|
+
if (!req.sessionId.startsWith('$')) return Promise.resolve();
|
|
5573
|
+
if (!text.trim()) return Promise.resolve();
|
|
4081
5574
|
const label = who === 'user' ? 'user message' : 'AI response';
|
|
4082
5575
|
// Serialised per session, retried across a gateway restart with a fixed
|
|
4083
5576
|
// transaction id, kept alive past the request (room-mirror.ts).
|
|
4084
|
-
|
|
5577
|
+
return this.roomMirror().enqueue(
|
|
4085
5578
|
req.sessionId,
|
|
4086
5579
|
async () => {
|
|
4087
5580
|
const roomId =
|
|
@@ -4105,24 +5598,35 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4105
5598
|
body,
|
|
4106
5599
|
...(formattedBody ? { formattedBody } : {}),
|
|
4107
5600
|
threadId: req.sessionId,
|
|
4108
|
-
txnId:
|
|
5601
|
+
txnId: req.channel
|
|
5602
|
+
? `channel-${await channelRequestHash(JSON.stringify([req.identity.userDid, req.channel.bindingId, req.requestId, who]))}`
|
|
5603
|
+
: mirrorTxnId(req.sessionId, req.requestId, who),
|
|
5604
|
+
...(req.channel ? { origin: req.channel } : {}),
|
|
4109
5605
|
};
|
|
4110
5606
|
},
|
|
4111
5607
|
label,
|
|
5608
|
+
req.client === 'channel',
|
|
4112
5609
|
);
|
|
4113
5610
|
}
|
|
4114
5611
|
|
|
4115
5612
|
private roomMirror(): RoomMirror {
|
|
4116
5613
|
this.mirror ??= new RoomMirror({
|
|
4117
5614
|
sendText: (send) =>
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
5615
|
+
send.origin
|
|
5616
|
+
? this.gateway.sendEvent(
|
|
5617
|
+
send.roomId,
|
|
5618
|
+
'm.room.message',
|
|
5619
|
+
JSON.stringify(mirrorEventContent(send)),
|
|
5620
|
+
{ txnId: send.txnId },
|
|
5621
|
+
)
|
|
5622
|
+
: this.gateway.sendText(send.roomId, send.body, {
|
|
5623
|
+
threadId: send.threadId,
|
|
5624
|
+
...(send.formattedBody
|
|
5625
|
+
? { formattedBody: send.formattedBody }
|
|
5626
|
+
: {}),
|
|
5627
|
+
priority: 'background',
|
|
5628
|
+
txnId: send.txnId,
|
|
5629
|
+
}),
|
|
4126
5630
|
keepAlive: (work) => this.ctx.waitUntil(work),
|
|
4127
5631
|
log: (message) => console.log(message),
|
|
4128
5632
|
warn: (message) => console.warn(message),
|
|
@@ -4133,6 +5637,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4133
5637
|
private async afterTurn(
|
|
4134
5638
|
sessionId: string,
|
|
4135
5639
|
messages: BaseMessage[],
|
|
5640
|
+
disposition: RequestDisposition = { kind: 'agent' },
|
|
5641
|
+
signal?: AbortSignal,
|
|
4136
5642
|
): Promise<void> {
|
|
4137
5643
|
const sessions = this.sessions!;
|
|
4138
5644
|
await sessions.touchSession(sessionId);
|
|
@@ -4143,11 +5649,25 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4143
5649
|
);
|
|
4144
5650
|
},
|
|
4145
5651
|
);
|
|
5652
|
+
if (disposition.kind === 'agent')
|
|
5653
|
+
await this.compareShadowRoute(sessionId);
|
|
4146
5654
|
const row = await sessions.getSession(sessionId);
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
5655
|
+
// The title model is the platform adapter, outside the turn's metered
|
|
5656
|
+
// model and budget: a supplied-context task's source and result never
|
|
5657
|
+
// reach it. A direct read brings its own deterministic title.
|
|
5658
|
+
// A generated title is a model call of its own: it runs in the
|
|
5659
|
+
// background, so the reply's `done` (and a Matrix answer) never waits
|
|
5660
|
+
// for it.
|
|
5661
|
+
if (
|
|
5662
|
+
row &&
|
|
5663
|
+
(!row.title || row.title === UNTITLED_SESSION) &&
|
|
5664
|
+
!(await this.isRestrictedTaskSession(sessionId))
|
|
5665
|
+
) {
|
|
5666
|
+
if (disposition.kind === 'direct-read')
|
|
5667
|
+
await sessions.setTitle(sessionId, disposition.title, {
|
|
5668
|
+
onlyIfUntitled: true,
|
|
5669
|
+
});
|
|
5670
|
+
else this.ctx.waitUntil(this.titleSession(sessionId, messages, signal));
|
|
4151
5671
|
}
|
|
4152
5672
|
this.markDirty();
|
|
4153
5673
|
// Which chunks this turn touched, for the page tier's eviction policy
|
|
@@ -4161,6 +5681,51 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4161
5681
|
}
|
|
4162
5682
|
}
|
|
4163
5683
|
|
|
5684
|
+
/**
|
|
5685
|
+
* Whether a session belongs to a supplied-context task, read from the
|
|
5686
|
+
* task row (never from the request). A task session with no scheduler to
|
|
5687
|
+
* read the row from is treated as restricted.
|
|
5688
|
+
*/
|
|
5689
|
+
private async isRestrictedTaskSession(sessionId: string): Promise<boolean> {
|
|
5690
|
+
if (!sessionId.startsWith(TASK_SESSION_PREFIX)) return false;
|
|
5691
|
+
return this.taskScheduler
|
|
5692
|
+
? this.taskScheduler.isRestrictedSession(sessionId)
|
|
5693
|
+
: true;
|
|
5694
|
+
}
|
|
5695
|
+
|
|
5696
|
+
/**
|
|
5697
|
+
* Shadow-mode follow-up: once the turn has ended, compare what the router
|
|
5698
|
+
* would have preloaded with what `load_capability` actually added to the
|
|
5699
|
+
* thread. Best effort — one checkpoint read, and it never fails the turn.
|
|
5700
|
+
* Nothing to compare when the shadow verdict has not landed yet.
|
|
5701
|
+
*/
|
|
5702
|
+
private async compareShadowRoute(sessionId: string): Promise<void> {
|
|
5703
|
+
const shadow = this.shadowRoutes.get(sessionId);
|
|
5704
|
+
if (!shadow) return;
|
|
5705
|
+
this.shadowRoutes.delete(sessionId);
|
|
5706
|
+
try {
|
|
5707
|
+
const tuple = await this.saver?.getTupleWithoutMessages({
|
|
5708
|
+
configurable: { thread_id: sessionId },
|
|
5709
|
+
});
|
|
5710
|
+
const loadedAfter = new Set(
|
|
5711
|
+
priorMetadataState(tuple?.checkpoint.channel_values ?? {})
|
|
5712
|
+
.loadedPlugins ?? [],
|
|
5713
|
+
);
|
|
5714
|
+
const { loadedDuringTurn, agree } = shadowAgreement({
|
|
5715
|
+
priorLoaded: shadow.priorLoaded,
|
|
5716
|
+
loadedAfter,
|
|
5717
|
+
wouldPreload: shadow.wouldPreload,
|
|
5718
|
+
});
|
|
5719
|
+
console.log(
|
|
5720
|
+
`[capability-router-shadow] request=${shadow.requestId} loadedDuringTurn=[${loadedDuringTurn.join(', ')}] agree=${agree}`,
|
|
5721
|
+
);
|
|
5722
|
+
} catch (err: unknown) {
|
|
5723
|
+
console.warn(
|
|
5724
|
+
`[capability-router-shadow] request=${shadow.requestId} comparison failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
5725
|
+
);
|
|
5726
|
+
}
|
|
5727
|
+
}
|
|
5728
|
+
|
|
4164
5729
|
/**
|
|
4165
5730
|
* Channel-memory compaction for the gateway's group rooms
|
|
4166
5731
|
* (`src/matrix/group-chat.ts`): the platform's small model distils a
|
|
@@ -4181,8 +5746,47 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4181
5746
|
);
|
|
4182
5747
|
}
|
|
4183
5748
|
|
|
5749
|
+
/**
|
|
5750
|
+
* Title an untitled session from its first exchange, in the background:
|
|
5751
|
+
* bounded by `TITLE_TIMEOUT_MS` and by the turn's abort, never throws.
|
|
5752
|
+
* The timer is cleared however the call ends.
|
|
5753
|
+
*/
|
|
5754
|
+
private async titleSession(
|
|
5755
|
+
sessionId: string,
|
|
5756
|
+
messages: BaseMessage[],
|
|
5757
|
+
turnSignal: AbortSignal | undefined,
|
|
5758
|
+
): Promise<void> {
|
|
5759
|
+
const bound = new AbortController();
|
|
5760
|
+
const onAbort = (): void => bound.abort(turnSignal?.reason);
|
|
5761
|
+
if (turnSignal?.aborted) return;
|
|
5762
|
+
turnSignal?.addEventListener('abort', onAbort, { once: true });
|
|
5763
|
+
const timer = setTimeout(
|
|
5764
|
+
() =>
|
|
5765
|
+
bound.abort(
|
|
5766
|
+
new Error(`session title timed out after ${TITLE_TIMEOUT_MS} ms`),
|
|
5767
|
+
),
|
|
5768
|
+
TITLE_TIMEOUT_MS,
|
|
5769
|
+
);
|
|
5770
|
+
try {
|
|
5771
|
+
const title = await this.generateTitle(messages, bound.signal);
|
|
5772
|
+
// A session deleted (or reset away) meanwhile is not titled.
|
|
5773
|
+
const sessions = this.sessions;
|
|
5774
|
+
if (!title || !sessions || bound.signal.aborted) return;
|
|
5775
|
+
if (await sessions.setTitle(sessionId, title, { onlyIfUntitled: true }))
|
|
5776
|
+
this.markDirty();
|
|
5777
|
+
} catch (err) {
|
|
5778
|
+
console.warn(
|
|
5779
|
+
`[user-do] no title for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
5780
|
+
);
|
|
5781
|
+
} finally {
|
|
5782
|
+
clearTimeout(timer);
|
|
5783
|
+
turnSignal?.removeEventListener('abort', onAbort);
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
|
|
4184
5787
|
private async generateTitle(
|
|
4185
5788
|
messages: BaseMessage[],
|
|
5789
|
+
signal: AbortSignal,
|
|
4186
5790
|
): Promise<string | null> {
|
|
4187
5791
|
const human = messages.find((m) => m.type === 'human');
|
|
4188
5792
|
const ai = [...messages].reverse().find((m) => m.type === 'ai');
|
|
@@ -4192,9 +5796,24 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4192
5796
|
const model = (this.ambient?.llm ?? this.core.llm).get('session-title');
|
|
4193
5797
|
const prompt =
|
|
4194
5798
|
'Write a 3–6 word title for this conversation. Reply with the title only, no quotes.\n\n' +
|
|
4195
|
-
`User: ${
|
|
5799
|
+
`User: ${visibleText(human).slice(0, 500)}\n` +
|
|
4196
5800
|
(ai ? `Assistant: ${contentToText(ai.content).slice(0, 500)}` : '');
|
|
4197
|
-
|
|
5801
|
+
// Raced as well as signalled: a provider that ignores the signal
|
|
5802
|
+
// still cannot hold the call past it.
|
|
5803
|
+
const aborted = new Promise<never>((_resolve, reject) => {
|
|
5804
|
+
const fail = (): void =>
|
|
5805
|
+
reject(
|
|
5806
|
+
signal.reason instanceof Error
|
|
5807
|
+
? signal.reason
|
|
5808
|
+
: new Error('session title aborted'),
|
|
5809
|
+
);
|
|
5810
|
+
if (signal.aborted) fail();
|
|
5811
|
+
else signal.addEventListener('abort', fail, { once: true });
|
|
5812
|
+
});
|
|
5813
|
+
const res = await Promise.race([
|
|
5814
|
+
model.invoke(prompt, { signal }),
|
|
5815
|
+
aborted,
|
|
5816
|
+
]);
|
|
4198
5817
|
const title = contentToText(res.content)
|
|
4199
5818
|
.trim()
|
|
4200
5819
|
.replace(/^["']|["']$/g, '')
|