@ixo/oracle-runtime-workers 0.37.2 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/attachments/channel-links.ts +74 -7
- package/src/attachments/channel.ts +15 -4
- package/src/attachments/content-blocks.ts +19 -16
- package/src/attachments/download.ts +173 -50
- package/src/attachments/index.ts +6 -5
- package/src/attachments/pipeline.ts +117 -49
- package/src/attachments/resize.ts +142 -0
- package/src/attachments/retention.ts +26 -70
- package/src/attachments/room-media.ts +643 -0
- package/src/attachments/route.ts +47 -4
- package/src/attachments/sandbox-archive.ts +15 -4
- package/src/attachments/sanitize.ts +24 -13
- package/src/attachments/view.ts +82 -38
- package/src/core/main-agent-types.ts +8 -0
- package/src/core/main-agent.ts +18 -3
- package/src/core/middlewares/attachment-recall.ts +502 -0
- package/src/core/middlewares/index.ts +5 -0
- package/src/do/contracts.ts +51 -23
- package/src/do/room-mirror.ts +144 -20
- package/src/do/transcript.ts +1 -1
- package/src/do/turn-attachments.ts +45 -6
- package/src/do/user-oracle-do.ts +507 -93
- package/src/llm/provider-error.ts +5 -2
- package/src/llm/stream-liveness.ts +3 -2
- package/src/matrix/gateway-do.ts +164 -102
- package/src/matrix/ingest.ts +12 -3
- package/src/matrix/media-deadline.ts +5 -47
- package/src/plugins/attachments/attachments.plugin.ts +3 -3
- package/src/plugins/attachments/view-attachment-tool.ts +6 -4
- package/src/sqlite/inline-payloads.ts +121 -0
- package/src/sqlite/sqlite-saver.ts +241 -42
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -123,19 +123,20 @@ import {
|
|
|
123
123
|
type BaseMessage,
|
|
124
124
|
} from '@langchain/core/messages';
|
|
125
125
|
import {
|
|
126
|
-
applyAttachmentRetention,
|
|
127
|
-
ATTACHMENT_PAYLOAD_TURNS,
|
|
128
126
|
ATTACHMENT_PLACEHOLDER_PREFIX,
|
|
127
|
+
classifyAttachment,
|
|
129
128
|
createAttachmentViewSurface,
|
|
130
129
|
findPersistedAttachmentMeta,
|
|
131
130
|
IngestTextLru,
|
|
132
|
-
INLINE_PAYLOAD_NEEDLE,
|
|
133
131
|
isAttachmentViewMessage,
|
|
134
|
-
|
|
132
|
+
loadAttachmentBytes,
|
|
135
133
|
parseAttachmentInputs,
|
|
136
134
|
prepareAttachments,
|
|
135
|
+
resizeForModel,
|
|
136
|
+
sanitizeAttachmentFilename,
|
|
137
137
|
ChannelAttachmentLinkStore,
|
|
138
138
|
type AttachmentViewSurface,
|
|
139
|
+
type AttachmentInput,
|
|
139
140
|
type MatrixMediaSource,
|
|
140
141
|
type ViewAttachmentDeps,
|
|
141
142
|
type SandboxUploadConfig,
|
|
@@ -145,6 +146,26 @@ import {
|
|
|
145
146
|
AttachmentTextCacheStore,
|
|
146
147
|
} from '../attachments';
|
|
147
148
|
import {
|
|
149
|
+
AttachmentTooLargeError,
|
|
150
|
+
boundedSignal,
|
|
151
|
+
HomeserverUnauthorizedError,
|
|
152
|
+
openHomeserverMedia,
|
|
153
|
+
untilAborted,
|
|
154
|
+
} from '../attachments/download';
|
|
155
|
+
// Not in the attachments index: it loads the bot SDK, which only workerd runs.
|
|
156
|
+
import {
|
|
157
|
+
CHANNEL_MEDIA_SEND_TIMEOUT_MS,
|
|
158
|
+
CHANNEL_MEDIA_TEXT_WAIT_MS,
|
|
159
|
+
CHANNEL_MEDIA_TURN_TIMEOUT_MS,
|
|
160
|
+
channelFilesToKeep,
|
|
161
|
+
channelMediaTxnId,
|
|
162
|
+
startChannelFileUploads,
|
|
163
|
+
type ChannelFileUpload,
|
|
164
|
+
type ThreadFileCopy,
|
|
165
|
+
} from '../attachments/room-media';
|
|
166
|
+
import {
|
|
167
|
+
channelGatewayOrigins,
|
|
168
|
+
channelThreadFiles,
|
|
148
169
|
channelTurnLinks,
|
|
149
170
|
prepareTurnAttachments,
|
|
150
171
|
turnAttachmentInputs,
|
|
@@ -274,6 +295,7 @@ import {
|
|
|
274
295
|
} from './ambient';
|
|
275
296
|
import {
|
|
276
297
|
checkpointStorageKey,
|
|
298
|
+
type BotCredentials,
|
|
277
299
|
type MatrixGatewayObject,
|
|
278
300
|
type OracleWorkerEnv,
|
|
279
301
|
type SessionSummary,
|
|
@@ -314,6 +336,11 @@ import { freshStub } from './fresh-stub';
|
|
|
314
336
|
import { WorkStatusProducer } from '../matrix/work-status';
|
|
315
337
|
import { fetchUserMatrixServerName } from '../matrix/user-homeserver';
|
|
316
338
|
import { createWorkStatusMiddleware } from '../core/middlewares/work-status';
|
|
339
|
+
import {
|
|
340
|
+
RecalledImageCache,
|
|
341
|
+
type RecallImageLoader,
|
|
342
|
+
} from '../core/middlewares/attachment-recall';
|
|
343
|
+
import { bytesToBase64 } from '../plugins/base64';
|
|
317
344
|
import { UserPreferencesStore } from '../plugins/user-preferences/user-preferences-store';
|
|
318
345
|
import { SqliteUserKv } from '../sqlite/user-kv-store';
|
|
319
346
|
import {
|
|
@@ -1183,6 +1210,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1183
1210
|
private matrixLedger: MatrixTurnLedger | null = null;
|
|
1184
1211
|
/** Room mirrors of HTTP turns, serialised per session and retried across gateway restarts (see room-mirror.ts). */
|
|
1185
1212
|
private mirror: RoomMirror | null = null;
|
|
1213
|
+
/** The oracle's plugin device, once known (see `pluginDevice`). */
|
|
1214
|
+
private pluginCredentials: BotCredentials | null = null;
|
|
1215
|
+
/** Its lookup in flight: concurrent callers share one gateway RPC. */
|
|
1216
|
+
private pluginCredentialsLoad: Promise<BotCredentials> | null = null;
|
|
1217
|
+
/**
|
|
1218
|
+
* Channel requests with files whose person's text the mirror pass has
|
|
1219
|
+
* queued on the session chain (value without `waiter`), or whose file
|
|
1220
|
+
* events wait for that (`waiter`); see `afterChannelText`.
|
|
1221
|
+
*/
|
|
1222
|
+
private readonly channelTextQueued = new Map<
|
|
1223
|
+
string,
|
|
1224
|
+
{ at: number; waiter?: () => void }
|
|
1225
|
+
>();
|
|
1226
|
+
/** Images the recall middleware re-attached lately, kept across this instance's turns. */
|
|
1227
|
+
private readonly recallCache = new RecalledImageCache();
|
|
1186
1228
|
private channelTurns: ChannelTurns | null = null;
|
|
1187
1229
|
/** Anonymous-feedback idempotency markers and the user's submission limit. */
|
|
1188
1230
|
private feedbackMarkers: FeedbackMarkers | null = null;
|
|
@@ -7241,52 +7283,81 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7241
7283
|
// ── turn plumbing ──────────────────────────────────────────────────────
|
|
7242
7284
|
|
|
7243
7285
|
/**
|
|
7244
|
-
* Matrix media access for the attachments pipeline
|
|
7245
|
-
*
|
|
7246
|
-
* the
|
|
7247
|
-
*
|
|
7248
|
-
*
|
|
7249
|
-
*
|
|
7250
|
-
*
|
|
7251
|
-
*
|
|
7252
|
-
* from opening it to the last byte. An `AbortSignal` cannot cross the
|
|
7253
|
-
* Durable Object RPC, so the turn's abort acts on this side instead:
|
|
7254
|
-
* `readBytesCapped` cancels the stream it reads, and the cancel travels
|
|
7255
|
-
* back over the RPC stream to stop the gateway's download.
|
|
7286
|
+
* Matrix media access for the attachments pipeline. The gateway is asked
|
|
7287
|
+
* only for the media event's small JSON (`eventMediaInfo`: it alone holds
|
|
7288
|
+
* the room keys); the bytes come straight from the homeserver on the
|
|
7289
|
+
* plugin device's token (`openMatrixMedia`) and an encrypted attachment
|
|
7290
|
+
* is decrypted HERE, so no media byte crosses the shared gateway object.
|
|
7291
|
+
* Each transfer is bounded by the attachment download deadline (and the
|
|
7292
|
+
* caller's signal), from opening it to the last byte, and cut off at
|
|
7293
|
+
* `maxBytes`.
|
|
7256
7294
|
*/
|
|
7257
7295
|
private matrixMediaSource(): MatrixMediaSource {
|
|
7258
|
-
const
|
|
7296
|
+
const bounded = async <T>(
|
|
7297
|
+
signal: AbortSignal | undefined,
|
|
7298
|
+
work: (signal: AbortSignal) => Promise<T>,
|
|
7299
|
+
): Promise<T> => {
|
|
7300
|
+
const bound = boundedSignal(
|
|
7301
|
+
signal,
|
|
7302
|
+
DOWNLOAD_TIMEOUT_MS,
|
|
7303
|
+
'Matrix media download',
|
|
7304
|
+
);
|
|
7305
|
+
try {
|
|
7306
|
+
return await untilAborted(work(bound.signal), bound.signal);
|
|
7307
|
+
} finally {
|
|
7308
|
+
bound.release();
|
|
7309
|
+
}
|
|
7310
|
+
};
|
|
7259
7311
|
return {
|
|
7260
|
-
downloadMxc:
|
|
7261
|
-
|
|
7262
|
-
|
|
7263
|
-
|
|
7264
|
-
|
|
7312
|
+
downloadMxc: (mxc, maxBytes = MAX_FILE_SIZE, signal) =>
|
|
7313
|
+
bounded(signal, async (bound) =>
|
|
7314
|
+
readBytesCapped(
|
|
7315
|
+
await this.openMatrixMedia(mxc, bound),
|
|
7316
|
+
maxBytes,
|
|
7317
|
+
bound,
|
|
7318
|
+
),
|
|
7265
7319
|
),
|
|
7266
|
-
downloadEvent:
|
|
7267
|
-
|
|
7268
|
-
|
|
7269
|
-
|
|
7270
|
-
|
|
7271
|
-
|
|
7272
|
-
|
|
7273
|
-
|
|
7274
|
-
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
bytes: await readBytesCapped(plain, maxBytes, signal),
|
|
7283
|
-
...(media.mimetype ? { mimetype: media.mimetype } : {}),
|
|
7284
|
-
...(media.filename ? { filename: media.filename } : {}),
|
|
7285
|
-
};
|
|
7286
|
-
},
|
|
7320
|
+
downloadEvent: (roomId, eventId, maxBytes = MAX_FILE_SIZE, signal) =>
|
|
7321
|
+
bounded(signal, async (bound) => {
|
|
7322
|
+
const info = await this.gateway.eventMediaInfo(roomId, eventId, {
|
|
7323
|
+
timeoutMs: DOWNLOAD_TIMEOUT_MS,
|
|
7324
|
+
});
|
|
7325
|
+
if (!info) return null;
|
|
7326
|
+
const stream = await this.openMatrixMedia(info.mxc, bound);
|
|
7327
|
+
const plain = info.file
|
|
7328
|
+
? stream.pipeThrough(createAttachmentDecryptor(info.file))
|
|
7329
|
+
: stream;
|
|
7330
|
+
return {
|
|
7331
|
+
bytes: await readBytesCapped(plain, maxBytes, bound),
|
|
7332
|
+
...(info.mimetype ? { mimetype: info.mimetype } : {}),
|
|
7333
|
+
...(info.filename ? { filename: info.filename } : {}),
|
|
7334
|
+
};
|
|
7335
|
+
}),
|
|
7287
7336
|
};
|
|
7288
7337
|
}
|
|
7289
7338
|
|
|
7339
|
+
/**
|
|
7340
|
+
* The media behind `mxc`, opened on the homeserver with the plugin
|
|
7341
|
+
* device's token; a 401 asks the gateway for fresh credentials
|
|
7342
|
+
* (`botCredentials({ rejected })`) and tries once more.
|
|
7343
|
+
*/
|
|
7344
|
+
private async openMatrixMedia(
|
|
7345
|
+
mxc: string,
|
|
7346
|
+
signal: AbortSignal,
|
|
7347
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
7348
|
+
const device = await untilAborted(this.pluginDevice(), signal);
|
|
7349
|
+
try {
|
|
7350
|
+
return await openHomeserverMedia(mxc, device, { signal });
|
|
7351
|
+
} catch (error) {
|
|
7352
|
+
if (!(error instanceof HomeserverUnauthorizedError)) throw error;
|
|
7353
|
+
const fresh = await untilAborted(
|
|
7354
|
+
this.pluginDevice({ rejected: device.accessToken }),
|
|
7355
|
+
signal,
|
|
7356
|
+
);
|
|
7357
|
+
return openHomeserverMedia(mxc, fresh, { signal });
|
|
7358
|
+
}
|
|
7359
|
+
}
|
|
7360
|
+
|
|
7290
7361
|
/**
|
|
7291
7362
|
* Session-scoped attachment access for the `view_attachment` tool: a
|
|
7292
7363
|
* reference is resolved against the attachments recorded on this
|
|
@@ -7314,6 +7385,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7314
7385
|
findPersistedAttachmentMeta(saver, input.sessionId, ref);
|
|
7315
7386
|
const viewDeps = (): ViewAttachmentDeps => ({
|
|
7316
7387
|
source: this.matrixMediaSource(),
|
|
7388
|
+
...(this.env.IMAGES ? { images: this.env.IMAGES } : {}),
|
|
7317
7389
|
channel: turnChannelMediaSource({
|
|
7318
7390
|
sessionId: input.sessionId,
|
|
7319
7391
|
turnLinks: input.channelLinks,
|
|
@@ -7339,45 +7411,76 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7339
7411
|
}
|
|
7340
7412
|
|
|
7341
7413
|
/**
|
|
7342
|
-
*
|
|
7343
|
-
*
|
|
7344
|
-
*
|
|
7345
|
-
*
|
|
7346
|
-
*
|
|
7347
|
-
*
|
|
7348
|
-
*
|
|
7349
|
-
*
|
|
7414
|
+
* Image fetch for the attachment recall middleware
|
|
7415
|
+
* (`core/middlewares/attachment-recall.ts`): one image of this session,
|
|
7416
|
+
* read through the same sources as the turn pipeline (Matrix media from
|
|
7417
|
+
* the homeserver, the session's channel links or their room copies,
|
|
7418
|
+
* http(s)), cut off at the middleware's `maxBytes` (null then), refused
|
|
7419
|
+
* unless the bytes are an image, downscaled for the model when the
|
|
7420
|
+
* oracle binds Cloudflare Images, as base64. An image the resize step
|
|
7421
|
+
* drops resolves null: its original never goes to the model.
|
|
7350
7422
|
*/
|
|
7351
|
-
private
|
|
7352
|
-
sessionId: string
|
|
7353
|
-
|
|
7354
|
-
|
|
7355
|
-
|
|
7356
|
-
|
|
7357
|
-
|
|
7358
|
-
|
|
7359
|
-
|
|
7360
|
-
|
|
7361
|
-
|
|
7362
|
-
|
|
7363
|
-
|
|
7364
|
-
|
|
7365
|
-
|
|
7366
|
-
|
|
7367
|
-
|
|
7368
|
-
|
|
7369
|
-
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7379
|
-
|
|
7380
|
-
|
|
7423
|
+
private recallImageLoader(input: {
|
|
7424
|
+
sessionId: string;
|
|
7425
|
+
roomId: string | undefined;
|
|
7426
|
+
/** The links of this turn's channel files (earlier ones come from the store). */
|
|
7427
|
+
channelLinks: ReturnType<typeof channelTurnLinks>;
|
|
7428
|
+
}): RecallImageLoader {
|
|
7429
|
+
return async (meta, { signal, maxBytes }) => {
|
|
7430
|
+
const attachment: AttachmentInput = {
|
|
7431
|
+
filename: meta.filename,
|
|
7432
|
+
mimetype: meta.mimetype,
|
|
7433
|
+
...(meta.size !== undefined ? { size: meta.size } : {}),
|
|
7434
|
+
...(meta.mxcUri ? { mxcUri: meta.mxcUri } : {}),
|
|
7435
|
+
...(meta.eventId ? { eventId: meta.eventId } : {}),
|
|
7436
|
+
...(meta.channelRef ? { channelRef: meta.channelRef } : {}),
|
|
7437
|
+
};
|
|
7438
|
+
const name = sanitizeAttachmentFilename(attachment.filename);
|
|
7439
|
+
const channel = turnChannelMediaSource({
|
|
7440
|
+
sessionId: input.sessionId,
|
|
7441
|
+
turnLinks: input.channelLinks,
|
|
7442
|
+
store: this.channelAttachmentLinks,
|
|
7443
|
+
channelServiceDid: this.env.CHANNEL_SERVICE_DID,
|
|
7444
|
+
});
|
|
7445
|
+
let loaded: Awaited<ReturnType<typeof loadAttachmentBytes>>;
|
|
7446
|
+
try {
|
|
7447
|
+
loaded = await loadAttachmentBytes(
|
|
7448
|
+
attachment,
|
|
7449
|
+
input.roomId,
|
|
7450
|
+
this.matrixMediaSource(),
|
|
7451
|
+
{ signal, maxBytes, channel },
|
|
7452
|
+
);
|
|
7453
|
+
} catch (error) {
|
|
7454
|
+
if (!(error instanceof AttachmentTooLargeError)) throw error;
|
|
7455
|
+
console.warn(
|
|
7456
|
+
`[attachments] recall: "${name}" is over the ${maxBytes}-byte recall limit`,
|
|
7457
|
+
);
|
|
7458
|
+
return null;
|
|
7459
|
+
}
|
|
7460
|
+
const { bytes, mimetype, sniffed } = loaded;
|
|
7461
|
+
// Bytes that are not an image never go back as an image block.
|
|
7462
|
+
const kind = classifyAttachment({
|
|
7463
|
+
mimetype,
|
|
7464
|
+
filename: sniffed ? '' : attachment.filename,
|
|
7465
|
+
});
|
|
7466
|
+
if (kind !== 'image')
|
|
7467
|
+
throw new Error(`content is ${mimetype}, not an image`);
|
|
7468
|
+
const sent = await resizeForModel(bytes, mimetype, this.env.IMAGES, {
|
|
7469
|
+
logger: console,
|
|
7470
|
+
});
|
|
7471
|
+
if (sent.dropped) {
|
|
7472
|
+
console.warn(
|
|
7473
|
+
`[attachments] recall: "${name}" not re-attached: ${sent.dropped}`,
|
|
7474
|
+
);
|
|
7475
|
+
return null;
|
|
7476
|
+
}
|
|
7477
|
+
return {
|
|
7478
|
+
kind: 'image',
|
|
7479
|
+
mimeType: sent.mimetype,
|
|
7480
|
+
base64: bytesToBase64(sent.bytes),
|
|
7481
|
+
filename: name,
|
|
7482
|
+
};
|
|
7483
|
+
};
|
|
7381
7484
|
}
|
|
7382
7485
|
|
|
7383
7486
|
/**
|
|
@@ -7665,7 +7768,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7665
7768
|
// A channel turn's files are read by their `channel:` reference; their
|
|
7666
7769
|
// links stay on the server (the run request and the session's link
|
|
7667
7770
|
// store), out of every message and note.
|
|
7668
|
-
const attachmentInputs = turnAttachmentInputs(req, body.attachments);
|
|
7669
7771
|
const attachmentModel =
|
|
7670
7772
|
effectiveModel && isAllowedModel(effectiveModel)
|
|
7671
7773
|
? effectiveModel
|
|
@@ -7681,6 +7783,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7681
7783
|
: undefined;
|
|
7682
7784
|
if (turnLinks.length && !run.resumed)
|
|
7683
7785
|
await this.channelAttachmentLinks?.put(req.sessionId, turnLinks);
|
|
7786
|
+
// A channel turn's files are also posted to the session thread as
|
|
7787
|
+
// encrypted media events (attachments/room-media.ts), in the
|
|
7788
|
+
// background: the run names each by its `channel:` reference, and the
|
|
7789
|
+
// copies read on the way reach the pipeline instead of a second
|
|
7790
|
+
// download. A resumed attempt posts nothing.
|
|
7791
|
+
const threadCopies =
|
|
7792
|
+
turnLinks.length && sessionRoomId
|
|
7793
|
+
? this.postChannelFilesToThread(
|
|
7794
|
+
req,
|
|
7795
|
+
sessionRoomId,
|
|
7796
|
+
abortController.signal,
|
|
7797
|
+
run.resumed,
|
|
7798
|
+
)
|
|
7799
|
+
: null;
|
|
7800
|
+
const attachmentInputs = turnAttachmentInputs(req, body.attachments);
|
|
7684
7801
|
const { prepared, metas: turnMetas } = await prepareTurnAttachments({
|
|
7685
7802
|
inputs: attachmentInputs,
|
|
7686
7803
|
resumed: run.resumed,
|
|
@@ -7696,22 +7813,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
7696
7813
|
{
|
|
7697
7814
|
source: this.matrixMediaSource(),
|
|
7698
7815
|
...(channelMedia ? { channel: channelMedia } : {}),
|
|
7816
|
+
...(threadCopies ? { preloaded: threadCopies } : {}),
|
|
7699
7817
|
extraction: this.extractionProvider(baseAmbient.llm, core.llm),
|
|
7700
7818
|
sandbox: await this.sandboxArchiveConfig(req.identity.userDid),
|
|
7819
|
+
...(this.env.IMAGES ? { images: this.env.IMAGES } : {}),
|
|
7701
7820
|
signal: abortController.signal,
|
|
7702
7821
|
logger: console,
|
|
7703
7822
|
},
|
|
7704
7823
|
),
|
|
7705
7824
|
});
|
|
7825
|
+
// Copies the pipeline did not take (a file it refused) go now, not
|
|
7826
|
+
// with the turn.
|
|
7827
|
+
threadCopies?.clear();
|
|
7706
7828
|
if (prepared) {
|
|
7707
7829
|
console.log(
|
|
7708
7830
|
`[user-do] attachments: ${attachmentInputs.length} attached, ${prepared.extracted.length} extracted, ${Array.isArray(prepared.content) ? prepared.content.length - 1 : 0} native`,
|
|
7709
7831
|
);
|
|
7710
7832
|
}
|
|
7711
7833
|
|
|
7712
|
-
// `view_attachment` for this session:
|
|
7713
|
-
//
|
|
7714
|
-
// `
|
|
7834
|
+
// `view_attachment` for this session: attachment bytes are never
|
|
7835
|
+
// stored with the conversation (the checkpointer keeps a placeholder,
|
|
7836
|
+
// `src/sqlite/inline-payloads.ts`); the tool fetches one again on
|
|
7837
|
+
// demand.
|
|
7715
7838
|
// Cleanups registered by tools for THIS turn (MCP clients …): run once
|
|
7716
7839
|
// the turn ends, however it ends — see `RuntimeContext.onTurnEnd`.
|
|
7717
7840
|
abortController.signal.addEventListener('abort', () => {
|
|
@@ -8038,6 +8161,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8038
8161
|
}
|
|
8039
8162
|
: {}),
|
|
8040
8163
|
...(hostSafetyModel ? { safetyModel: hostSafetyModel(ambient) } : {}),
|
|
8164
|
+
// The images of the newest user turns, back in each model request
|
|
8165
|
+
// (their bytes are not stored with the conversation).
|
|
8166
|
+
...(suppliedContextOnly
|
|
8167
|
+
? {}
|
|
8168
|
+
: {
|
|
8169
|
+
attachmentRecall: {
|
|
8170
|
+
loadImage: this.recallImageLoader({
|
|
8171
|
+
sessionId: req.sessionId,
|
|
8172
|
+
roomId: sessionRoomId,
|
|
8173
|
+
channelLinks: turnLinks,
|
|
8174
|
+
}),
|
|
8175
|
+
imageInput: getModelCapabilities(attachmentModel).image,
|
|
8176
|
+
signal: abortController.signal,
|
|
8177
|
+
cache: this.recallCache,
|
|
8178
|
+
},
|
|
8179
|
+
}),
|
|
8041
8180
|
},
|
|
8042
8181
|
requestCtx: {
|
|
8043
8182
|
user: {
|
|
@@ -8241,6 +8380,18 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8241
8380
|
text: string,
|
|
8242
8381
|
who: 'user' | 'oracle',
|
|
8243
8382
|
receipt = false,
|
|
8383
|
+
): Promise<void> {
|
|
8384
|
+
const queued = this.queueReplay(req, text, who, receipt);
|
|
8385
|
+
// The turn's file events follow the person's text on the chain.
|
|
8386
|
+
if (who === 'user') this.noteChannelTextQueued(req);
|
|
8387
|
+
return queued;
|
|
8388
|
+
}
|
|
8389
|
+
|
|
8390
|
+
private queueReplay(
|
|
8391
|
+
req: TurnRequest,
|
|
8392
|
+
text: string,
|
|
8393
|
+
who: 'user' | 'oracle',
|
|
8394
|
+
receipt: boolean,
|
|
8244
8395
|
): Promise<void> {
|
|
8245
8396
|
if (req.client === 'matrix' && !receipt) return Promise.resolve();
|
|
8246
8397
|
if (req.sessionId.startsWith(SYNTHETIC_SESSION_PREFIX))
|
|
@@ -8308,7 +8459,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8308
8459
|
send.roomId,
|
|
8309
8460
|
'm.room.message',
|
|
8310
8461
|
JSON.stringify(mirrorEventContent(send)),
|
|
8311
|
-
{ txnId: send.txnId },
|
|
8462
|
+
{ txnId: send.txnId, priority: 'background' },
|
|
8312
8463
|
)
|
|
8313
8464
|
: this.gateway.sendText(send.roomId, send.body, {
|
|
8314
8465
|
threadId: send.threadId,
|
|
@@ -8318,6 +8469,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8318
8469
|
priority: 'background',
|
|
8319
8470
|
txnId: send.txnId,
|
|
8320
8471
|
}),
|
|
8472
|
+
sendEvent: (roomId, content, txnId) =>
|
|
8473
|
+
this.gateway.sendEvent(roomId, 'm.room.message', content, {
|
|
8474
|
+
txnId,
|
|
8475
|
+
priority: 'background',
|
|
8476
|
+
}),
|
|
8321
8477
|
keepAlive: (work) => this.ctx.waitUntil(work),
|
|
8322
8478
|
log: (message) => console.log(message),
|
|
8323
8479
|
warn: (message) => console.warn(message),
|
|
@@ -8325,6 +8481,271 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8325
8481
|
return this.mirror;
|
|
8326
8482
|
}
|
|
8327
8483
|
|
|
8484
|
+
/**
|
|
8485
|
+
* The oracle's plugin device (`botCredentials`): what channel files are
|
|
8486
|
+
* uploaded and Matrix media downloaded with. One gateway RPC per
|
|
8487
|
+
* instance, shared by concurrent callers. A caller the homeserver
|
|
8488
|
+
* answered 401 passes the token it used as `rejected`: the cache is
|
|
8489
|
+
* dropped only when it still holds that token, and the gateway
|
|
8490
|
+
* re-validates its device (or logs it in again) once for everyone.
|
|
8491
|
+
*/
|
|
8492
|
+
private pluginDevice(
|
|
8493
|
+
opts: { rejected?: string } = {},
|
|
8494
|
+
): Promise<BotCredentials> {
|
|
8495
|
+
if (
|
|
8496
|
+
opts.rejected &&
|
|
8497
|
+
this.pluginCredentials?.accessToken === opts.rejected
|
|
8498
|
+
)
|
|
8499
|
+
this.pluginCredentials = null;
|
|
8500
|
+
if (this.pluginCredentials)
|
|
8501
|
+
return Promise.resolve(this.pluginCredentials);
|
|
8502
|
+
this.pluginCredentialsLoad ??= this.gateway
|
|
8503
|
+
.botCredentials(opts.rejected ? { rejected: opts.rejected } : {})
|
|
8504
|
+
.then((credentials) => {
|
|
8505
|
+
this.pluginCredentials = credentials;
|
|
8506
|
+
return credentials;
|
|
8507
|
+
})
|
|
8508
|
+
.finally(() => {
|
|
8509
|
+
this.pluginCredentialsLoad = null;
|
|
8510
|
+
});
|
|
8511
|
+
return this.pluginCredentialsLoad;
|
|
8512
|
+
}
|
|
8513
|
+
|
|
8514
|
+
/**
|
|
8515
|
+
* The mirror pass has queued the person's text of channel request
|
|
8516
|
+
* `req` on the session chain (or found none to queue): release the
|
|
8517
|
+
* request's file events waiting for it, or remember it for a turn that
|
|
8518
|
+
* has not asked yet. Only requests with files are tracked; entries older
|
|
8519
|
+
* than the wait are dropped.
|
|
8520
|
+
*/
|
|
8521
|
+
private noteChannelTextQueued(req: TurnRequest): void {
|
|
8522
|
+
if (req.client !== 'channel' || !req.channel?.attachments?.length) return;
|
|
8523
|
+
const entry = this.channelTextQueued.get(req.requestId);
|
|
8524
|
+
if (entry?.waiter) {
|
|
8525
|
+
this.channelTextQueued.delete(req.requestId);
|
|
8526
|
+
entry.waiter();
|
|
8527
|
+
return;
|
|
8528
|
+
}
|
|
8529
|
+
const now = Date.now();
|
|
8530
|
+
for (const [requestId, noted] of this.channelTextQueued)
|
|
8531
|
+
if (!noted.waiter && now - noted.at > CHANNEL_MEDIA_TEXT_WAIT_MS)
|
|
8532
|
+
this.channelTextQueued.delete(requestId);
|
|
8533
|
+
this.channelTextQueued.set(req.requestId, { at: now });
|
|
8534
|
+
}
|
|
8535
|
+
|
|
8536
|
+
/**
|
|
8537
|
+
* `queued` resolves once the person's text of `requestId` is queued on
|
|
8538
|
+
* the session chain (true), after `CHANNEL_MEDIA_TEXT_WAIT_MS` (false),
|
|
8539
|
+
* or with null when `cancel` is called first (nothing left to post).
|
|
8540
|
+
*/
|
|
8541
|
+
private afterChannelText(requestId: string): {
|
|
8542
|
+
queued: Promise<boolean | null>;
|
|
8543
|
+
cancel: () => void;
|
|
8544
|
+
} {
|
|
8545
|
+
const noted = this.channelTextQueued.get(requestId);
|
|
8546
|
+
if (noted && !noted.waiter) {
|
|
8547
|
+
this.channelTextQueued.delete(requestId);
|
|
8548
|
+
return { queued: Promise.resolve(true), cancel: () => undefined };
|
|
8549
|
+
}
|
|
8550
|
+
let settle: (value: boolean | null) => void = () => undefined;
|
|
8551
|
+
const queued = new Promise<boolean | null>((resolve) => {
|
|
8552
|
+
settle = resolve;
|
|
8553
|
+
});
|
|
8554
|
+
const finish = (value: boolean | null): void => {
|
|
8555
|
+
clearTimeout(timer);
|
|
8556
|
+
if (this.channelTextQueued.get(requestId)?.waiter === waiter)
|
|
8557
|
+
this.channelTextQueued.delete(requestId);
|
|
8558
|
+
settle(value);
|
|
8559
|
+
};
|
|
8560
|
+
const waiter = (): void => finish(true);
|
|
8561
|
+
const timer = setTimeout(() => finish(false), CHANNEL_MEDIA_TEXT_WAIT_MS);
|
|
8562
|
+
this.channelTextQueued.set(requestId, { at: Date.now(), waiter });
|
|
8563
|
+
return { queued, cancel: () => finish(null) };
|
|
8564
|
+
}
|
|
8565
|
+
|
|
8566
|
+
/**
|
|
8567
|
+
* Post a channel turn's files as encrypted media events in the session
|
|
8568
|
+
* thread (`attachments/room-media.ts`), off the turn's critical path.
|
|
8569
|
+
* Returns at once: per file the turn may read directly
|
|
8570
|
+
* (`channelFilesToKeep`), a promise of its verified plaintext, resolved
|
|
8571
|
+
* as soon as the read passes its check, or null when no copy comes (the
|
|
8572
|
+
* pipeline then reads the gateway as before). In the background:
|
|
8573
|
+
*
|
|
8574
|
+
* - the encryption check and the plugin device, bounded with the uploads'
|
|
8575
|
+
* reads by `CHANNEL_MEDIA_TURN_TIMEOUT_MS` from now and by the run's
|
|
8576
|
+
* signal; on a failure (or a room that is not Megolm-encrypted) one
|
|
8577
|
+
* line is logged and nothing is posted;
|
|
8578
|
+
* - the uploads (`startChannelFileUploads`), kept alive by `waitUntil`;
|
|
8579
|
+
* - one event per file on the session's mirror chain, queued in message
|
|
8580
|
+
* order right after the person's text (`afterChannelText`) at
|
|
8581
|
+
* background priority, each within `CHANNEL_MEDIA_SEND_TIMEOUT_MS`;
|
|
8582
|
+
* their event ids then go to the link store (`setEventIds`), whose room
|
|
8583
|
+
* copies serve the files once their links expire.
|
|
8584
|
+
*
|
|
8585
|
+
* A resumed attempt posts nothing: its first attempt did, or lost the
|
|
8586
|
+
* uploads with the object, and the person's message is in the
|
|
8587
|
+
* checkpoint already.
|
|
8588
|
+
*/
|
|
8589
|
+
private postChannelFilesToThread(
|
|
8590
|
+
req: TurnRequest,
|
|
8591
|
+
roomId: string,
|
|
8592
|
+
runSignal: AbortSignal,
|
|
8593
|
+
resumed: boolean,
|
|
8594
|
+
): Map<string, Promise<ThreadFileCopy | null>> {
|
|
8595
|
+
const copies = new Map<string, Promise<ThreadFileCopy | null>>();
|
|
8596
|
+
const files = channelThreadFiles(req);
|
|
8597
|
+
const channel = req.channel;
|
|
8598
|
+
// A channel session is always a thread under its marker event.
|
|
8599
|
+
if (
|
|
8600
|
+
resumed ||
|
|
8601
|
+
!files.length ||
|
|
8602
|
+
!channel ||
|
|
8603
|
+
!req.sessionId.startsWith('$')
|
|
8604
|
+
)
|
|
8605
|
+
return copies;
|
|
8606
|
+
const keep = channelFilesToKeep(files);
|
|
8607
|
+
const pending = new Map<
|
|
8608
|
+
string,
|
|
8609
|
+
(copy: Promise<ThreadFileCopy | null> | null) => void
|
|
8610
|
+
>();
|
|
8611
|
+
for (const ref of keep)
|
|
8612
|
+
copies.set(
|
|
8613
|
+
ref,
|
|
8614
|
+
new Promise((resolve) => {
|
|
8615
|
+
pending.set(ref, resolve);
|
|
8616
|
+
}),
|
|
8617
|
+
);
|
|
8618
|
+
const textOrder = this.afterChannelText(req.requestId);
|
|
8619
|
+
const deadline = Date.now() + CHANNEL_MEDIA_TURN_TIMEOUT_MS;
|
|
8620
|
+
|
|
8621
|
+
const started = (async (): Promise<ChannelFileUpload[]> => {
|
|
8622
|
+
const setup = boundedSignal(
|
|
8623
|
+
runSignal,
|
|
8624
|
+
CHANNEL_MEDIA_TURN_TIMEOUT_MS,
|
|
8625
|
+
'Preparing the channel file posts',
|
|
8626
|
+
);
|
|
8627
|
+
try {
|
|
8628
|
+
const [encrypted, device] = await untilAborted(
|
|
8629
|
+
Promise.all([this.roomIsEncrypted(roomId), this.pluginDevice()]),
|
|
8630
|
+
setup.signal,
|
|
8631
|
+
);
|
|
8632
|
+
if (!encrypted) {
|
|
8633
|
+
console.warn(
|
|
8634
|
+
`[attachments] channel files of ${req.sessionId} not posted to its thread: the room is not end-to-end encrypted`,
|
|
8635
|
+
);
|
|
8636
|
+
return [];
|
|
8637
|
+
}
|
|
8638
|
+
const run = startChannelFileUploads({
|
|
8639
|
+
files,
|
|
8640
|
+
sessionId: req.sessionId,
|
|
8641
|
+
deps: {
|
|
8642
|
+
origins: channelGatewayOrigins(this.env.CHANNEL_SERVICE_DID),
|
|
8643
|
+
homeserver: {
|
|
8644
|
+
baseUrl: device.baseUrl,
|
|
8645
|
+
accessToken: device.accessToken,
|
|
8646
|
+
},
|
|
8647
|
+
keep,
|
|
8648
|
+
turnTimeoutMs: Math.max(0, deadline - Date.now()),
|
|
8649
|
+
onUnauthorized: (token) => {
|
|
8650
|
+
void this.pluginDevice({ rejected: token }).catch(
|
|
8651
|
+
() => undefined,
|
|
8652
|
+
);
|
|
8653
|
+
},
|
|
8654
|
+
log: console,
|
|
8655
|
+
},
|
|
8656
|
+
});
|
|
8657
|
+
this.ctx.waitUntil(run.done);
|
|
8658
|
+
for (const upload of run.uploads)
|
|
8659
|
+
pending.get(upload.file.ref)?.(upload.copy);
|
|
8660
|
+
return run.uploads;
|
|
8661
|
+
} catch (error) {
|
|
8662
|
+
console.warn(
|
|
8663
|
+
`[attachments] channel files of ${req.sessionId} not posted to its thread; the turn reads them from the channel gateway: ${error instanceof Error ? error.message : String(error)}`,
|
|
8664
|
+
);
|
|
8665
|
+
return [];
|
|
8666
|
+
} finally {
|
|
8667
|
+
setup.release();
|
|
8668
|
+
// A copy not handed over by now never comes (a resolved one stays).
|
|
8669
|
+
for (const resolve of pending.values()) resolve(null);
|
|
8670
|
+
}
|
|
8671
|
+
})();
|
|
8672
|
+
|
|
8673
|
+
const origin = {
|
|
8674
|
+
provider: channel.provider,
|
|
8675
|
+
bindingId: channel.bindingId,
|
|
8676
|
+
remoteMessageRef: channel.remoteMessageRef,
|
|
8677
|
+
};
|
|
8678
|
+
// Nothing to post: the events need not wait for the text.
|
|
8679
|
+
void started.then((uploads) => {
|
|
8680
|
+
if (uploads.length === 0) textOrder.cancel();
|
|
8681
|
+
});
|
|
8682
|
+
const posted = textOrder.queued.then(async (afterText) => {
|
|
8683
|
+
if (afterText === null) return;
|
|
8684
|
+
if (!afterText)
|
|
8685
|
+
console.warn(
|
|
8686
|
+
`[attachments] the text of ${req.requestId} was not queued within ${CHANNEL_MEDIA_TEXT_WAIT_MS} ms; its files are queued without it`,
|
|
8687
|
+
);
|
|
8688
|
+
// Queued synchronously, one after the other: the chain keeps the
|
|
8689
|
+
// message order whichever upload ends first.
|
|
8690
|
+
const sent = await Promise.all(
|
|
8691
|
+
files.map(async (file): Promise<[string, string] | null> => {
|
|
8692
|
+
const bound = boundedSignal(
|
|
8693
|
+
undefined,
|
|
8694
|
+
CHANNEL_MEDIA_SEND_TIMEOUT_MS,
|
|
8695
|
+
'Posting the channel file',
|
|
8696
|
+
);
|
|
8697
|
+
try {
|
|
8698
|
+
const eventId = await this.roomMirror().sendMedia(
|
|
8699
|
+
req.sessionId,
|
|
8700
|
+
async () => {
|
|
8701
|
+
const upload = (await started).find(
|
|
8702
|
+
(entry) => entry.file.ref === file.ref,
|
|
8703
|
+
);
|
|
8704
|
+
// A failed upload is logged by `startChannelFileUploads`.
|
|
8705
|
+
const media = await upload?.uploaded.catch(() => null);
|
|
8706
|
+
if (!media) return null;
|
|
8707
|
+
return {
|
|
8708
|
+
roomId,
|
|
8709
|
+
threadId: req.sessionId,
|
|
8710
|
+
txnId: await channelMediaTxnId({
|
|
8711
|
+
userDid: req.identity.userDid,
|
|
8712
|
+
bindingId: channel.bindingId,
|
|
8713
|
+
requestId: req.requestId,
|
|
8714
|
+
index: file.index,
|
|
8715
|
+
}),
|
|
8716
|
+
origin,
|
|
8717
|
+
media,
|
|
8718
|
+
};
|
|
8719
|
+
},
|
|
8720
|
+
'channel file',
|
|
8721
|
+
bound.signal,
|
|
8722
|
+
);
|
|
8723
|
+
return eventId ? [file.ref, eventId] : null;
|
|
8724
|
+
} catch (error) {
|
|
8725
|
+
console.warn(
|
|
8726
|
+
`[attachments] channel file ${file.ref} not posted to thread ${req.sessionId}: ${error instanceof Error ? error.message : String(error)}`,
|
|
8727
|
+
);
|
|
8728
|
+
return null;
|
|
8729
|
+
} finally {
|
|
8730
|
+
bound.release();
|
|
8731
|
+
}
|
|
8732
|
+
}),
|
|
8733
|
+
);
|
|
8734
|
+
const eventIds = new Map(
|
|
8735
|
+
sent.filter((entry): entry is [string, string] => entry !== null),
|
|
8736
|
+
);
|
|
8737
|
+
await this.channelAttachmentLinks?.setEventIds(req.sessionId, eventIds);
|
|
8738
|
+
});
|
|
8739
|
+
this.ctx.waitUntil(
|
|
8740
|
+
posted.catch((error: unknown) => {
|
|
8741
|
+
console.warn(
|
|
8742
|
+
`[attachments] channel file posts of ${req.sessionId} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
8743
|
+
);
|
|
8744
|
+
}),
|
|
8745
|
+
);
|
|
8746
|
+
return copies;
|
|
8747
|
+
}
|
|
8748
|
+
|
|
8328
8749
|
private async afterTurn(
|
|
8329
8750
|
sessionId: string,
|
|
8330
8751
|
messages: BaseMessage[],
|
|
@@ -8333,13 +8754,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
8333
8754
|
): Promise<void> {
|
|
8334
8755
|
const sessions = this.sessions!;
|
|
8335
8756
|
await sessions.touchSession(sessionId);
|
|
8336
|
-
await this.retainAttachmentPayloads(sessionId, messages).catch(
|
|
8337
|
-
(err: unknown) => {
|
|
8338
|
-
console.warn(
|
|
8339
|
-
`[user-do] attachment payload retention failed for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
8340
|
-
);
|
|
8341
|
-
},
|
|
8342
|
-
);
|
|
8343
8757
|
if (disposition.kind === 'agent')
|
|
8344
8758
|
await this.compareShadowRoute(sessionId);
|
|
8345
8759
|
const row = await sessions.getSession(sessionId);
|