@ixo/oracle-runtime-workers 0.37.2 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -123,19 +123,20 @@ import {
123
123
  type BaseMessage,
124
124
  } from '@langchain/core/messages';
125
125
  import {
126
- applyAttachmentRetention,
127
- ATTACHMENT_PAYLOAD_TURNS,
128
126
  ATTACHMENT_PLACEHOLDER_PREFIX,
127
+ classifyAttachment,
129
128
  createAttachmentViewSurface,
130
129
  findPersistedAttachmentMeta,
131
130
  IngestTextLru,
132
- INLINE_PAYLOAD_NEEDLE,
133
131
  isAttachmentViewMessage,
134
- offloadInlinePayloads,
132
+ loadAttachmentBytes,
135
133
  parseAttachmentInputs,
136
134
  prepareAttachments,
135
+ resizeForModel,
136
+ sanitizeAttachmentFilename,
137
137
  ChannelAttachmentLinkStore,
138
138
  type AttachmentViewSurface,
139
+ type AttachmentInput,
139
140
  type MatrixMediaSource,
140
141
  type ViewAttachmentDeps,
141
142
  type SandboxUploadConfig,
@@ -145,6 +146,26 @@ import {
145
146
  AttachmentTextCacheStore,
146
147
  } from '../attachments';
147
148
  import {
149
+ AttachmentTooLargeError,
150
+ boundedSignal,
151
+ HomeserverUnauthorizedError,
152
+ openHomeserverMedia,
153
+ untilAborted,
154
+ } from '../attachments/download';
155
+ // Not in the attachments index: it loads the bot SDK, which only workerd runs.
156
+ import {
157
+ CHANNEL_MEDIA_SEND_TIMEOUT_MS,
158
+ CHANNEL_MEDIA_TEXT_WAIT_MS,
159
+ CHANNEL_MEDIA_TURN_TIMEOUT_MS,
160
+ channelFilesToKeep,
161
+ channelMediaTxnId,
162
+ startChannelFileUploads,
163
+ type ChannelFileUpload,
164
+ type ThreadFileCopy,
165
+ } from '../attachments/room-media';
166
+ import {
167
+ channelGatewayOrigins,
168
+ channelThreadFiles,
148
169
  channelTurnLinks,
149
170
  prepareTurnAttachments,
150
171
  turnAttachmentInputs,
@@ -274,6 +295,7 @@ import {
274
295
  } from './ambient';
275
296
  import {
276
297
  checkpointStorageKey,
298
+ type BotCredentials,
277
299
  type MatrixGatewayObject,
278
300
  type OracleWorkerEnv,
279
301
  type SessionSummary,
@@ -314,6 +336,11 @@ import { freshStub } from './fresh-stub';
314
336
  import { WorkStatusProducer } from '../matrix/work-status';
315
337
  import { fetchUserMatrixServerName } from '../matrix/user-homeserver';
316
338
  import { createWorkStatusMiddleware } from '../core/middlewares/work-status';
339
+ import {
340
+ RecalledImageCache,
341
+ type RecallImageLoader,
342
+ } from '../core/middlewares/attachment-recall';
343
+ import { bytesToBase64 } from '../plugins/base64';
317
344
  import { UserPreferencesStore } from '../plugins/user-preferences/user-preferences-store';
318
345
  import { SqliteUserKv } from '../sqlite/user-kv-store';
319
346
  import {
@@ -1183,6 +1210,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
1183
1210
  private matrixLedger: MatrixTurnLedger | null = null;
1184
1211
  /** Room mirrors of HTTP turns, serialised per session and retried across gateway restarts (see room-mirror.ts). */
1185
1212
  private mirror: RoomMirror | null = null;
1213
+ /** The oracle's plugin device, once known (see `pluginDevice`). */
1214
+ private pluginCredentials: BotCredentials | null = null;
1215
+ /** Its lookup in flight: concurrent callers share one gateway RPC. */
1216
+ private pluginCredentialsLoad: Promise<BotCredentials> | null = null;
1217
+ /**
1218
+ * Channel requests with files whose person's text the mirror pass has
1219
+ * queued on the session chain (value without `waiter`), or whose file
1220
+ * events wait for that (`waiter`); see `afterChannelText`.
1221
+ */
1222
+ private readonly channelTextQueued = new Map<
1223
+ string,
1224
+ { at: number; waiter?: () => void }
1225
+ >();
1226
+ /** Images the recall middleware re-attached lately, kept across this instance's turns. */
1227
+ private readonly recallCache = new RecalledImageCache();
1186
1228
  private channelTurns: ChannelTurns | null = null;
1187
1229
  /** Anonymous-feedback idempotency markers and the user's submission limit. */
1188
1230
  private feedbackMarkers: FeedbackMarkers | null = null;
@@ -7241,52 +7283,81 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7241
7283
  // ── turn plumbing ──────────────────────────────────────────────────────
7242
7284
 
7243
7285
  /**
7244
- * Matrix media access for the attachments pipeline, through the gateway.
7245
- * Both transfers stream across the object boundary and are cut off at
7246
- * the attachment cap; an encrypted attachment is decrypted HERE, not in
7247
- * the gateway (its whole-buffer download grew the crypto WASM heap for
7248
- * good, and a hash mismatch on the far side of the RPC would only reach
7249
- * this object as a disconnect).
7250
- *
7251
- * The gateway bounds each transfer to the attachment download deadline,
7252
- * from opening it to the last byte. An `AbortSignal` cannot cross the
7253
- * Durable Object RPC, so the turn's abort acts on this side instead:
7254
- * `readBytesCapped` cancels the stream it reads, and the cancel travels
7255
- * back over the RPC stream to stop the gateway's download.
7286
+ * Matrix media access for the attachments pipeline. The gateway is asked
7287
+ * only for the media event's small JSON (`eventMediaInfo`: it alone holds
7288
+ * the room keys); the bytes come straight from the homeserver on the
7289
+ * plugin device's token (`openMatrixMedia`) and an encrypted attachment
7290
+ * is decrypted HERE, so no media byte crosses the shared gateway object.
7291
+ * Each transfer is bounded by the attachment download deadline (and the
7292
+ * caller's signal), from opening it to the last byte, and cut off at
7293
+ * `maxBytes`.
7256
7294
  */
7257
7295
  private matrixMediaSource(): MatrixMediaSource {
7258
- const deadline = { timeoutMs: DOWNLOAD_TIMEOUT_MS };
7296
+ const bounded = async <T>(
7297
+ signal: AbortSignal | undefined,
7298
+ work: (signal: AbortSignal) => Promise<T>,
7299
+ ): Promise<T> => {
7300
+ const bound = boundedSignal(
7301
+ signal,
7302
+ DOWNLOAD_TIMEOUT_MS,
7303
+ 'Matrix media download',
7304
+ );
7305
+ try {
7306
+ return await untilAborted(work(bound.signal), bound.signal);
7307
+ } finally {
7308
+ bound.release();
7309
+ }
7310
+ };
7259
7311
  return {
7260
- downloadMxc: async (mxc, maxBytes = MAX_FILE_SIZE, signal) =>
7261
- readBytesCapped(
7262
- await this.gateway.downloadMxcMediaStream(mxc, deadline),
7263
- maxBytes,
7264
- signal,
7312
+ downloadMxc: (mxc, maxBytes = MAX_FILE_SIZE, signal) =>
7313
+ bounded(signal, async (bound) =>
7314
+ readBytesCapped(
7315
+ await this.openMatrixMedia(mxc, bound),
7316
+ maxBytes,
7317
+ bound,
7318
+ ),
7265
7319
  ),
7266
- downloadEvent: async (
7267
- roomId,
7268
- eventId,
7269
- maxBytes = MAX_FILE_SIZE,
7270
- signal,
7271
- ) => {
7272
- const media = await this.gateway.downloadEventMediaStream(
7273
- roomId,
7274
- eventId,
7275
- deadline,
7276
- );
7277
- if (!media) return null;
7278
- const plain = media.file
7279
- ? media.stream.pipeThrough(createAttachmentDecryptor(media.file))
7280
- : media.stream;
7281
- return {
7282
- bytes: await readBytesCapped(plain, maxBytes, signal),
7283
- ...(media.mimetype ? { mimetype: media.mimetype } : {}),
7284
- ...(media.filename ? { filename: media.filename } : {}),
7285
- };
7286
- },
7320
+ downloadEvent: (roomId, eventId, maxBytes = MAX_FILE_SIZE, signal) =>
7321
+ bounded(signal, async (bound) => {
7322
+ const info = await this.gateway.eventMediaInfo(roomId, eventId, {
7323
+ timeoutMs: DOWNLOAD_TIMEOUT_MS,
7324
+ });
7325
+ if (!info) return null;
7326
+ const stream = await this.openMatrixMedia(info.mxc, bound);
7327
+ const plain = info.file
7328
+ ? stream.pipeThrough(createAttachmentDecryptor(info.file))
7329
+ : stream;
7330
+ return {
7331
+ bytes: await readBytesCapped(plain, maxBytes, bound),
7332
+ ...(info.mimetype ? { mimetype: info.mimetype } : {}),
7333
+ ...(info.filename ? { filename: info.filename } : {}),
7334
+ };
7335
+ }),
7287
7336
  };
7288
7337
  }
7289
7338
 
7339
+ /**
7340
+ * The media behind `mxc`, opened on the homeserver with the plugin
7341
+ * device's token; a 401 asks the gateway for fresh credentials
7342
+ * (`botCredentials({ rejected })`) and tries once more.
7343
+ */
7344
+ private async openMatrixMedia(
7345
+ mxc: string,
7346
+ signal: AbortSignal,
7347
+ ): Promise<ReadableStream<Uint8Array>> {
7348
+ const device = await untilAborted(this.pluginDevice(), signal);
7349
+ try {
7350
+ return await openHomeserverMedia(mxc, device, { signal });
7351
+ } catch (error) {
7352
+ if (!(error instanceof HomeserverUnauthorizedError)) throw error;
7353
+ const fresh = await untilAborted(
7354
+ this.pluginDevice({ rejected: device.accessToken }),
7355
+ signal,
7356
+ );
7357
+ return openHomeserverMedia(mxc, fresh, { signal });
7358
+ }
7359
+ }
7360
+
7290
7361
  /**
7291
7362
  * Session-scoped attachment access for the `view_attachment` tool: a
7292
7363
  * reference is resolved against the attachments recorded on this
@@ -7314,6 +7385,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7314
7385
  findPersistedAttachmentMeta(saver, input.sessionId, ref);
7315
7386
  const viewDeps = (): ViewAttachmentDeps => ({
7316
7387
  source: this.matrixMediaSource(),
7388
+ ...(this.env.IMAGES ? { images: this.env.IMAGES } : {}),
7317
7389
  channel: turnChannelMediaSource({
7318
7390
  sessionId: input.sessionId,
7319
7391
  turnLinks: input.channelLinks,
@@ -7339,45 +7411,76 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7339
7411
  }
7340
7412
 
7341
7413
  /**
7342
- * Attachment payload retention (`src/attachments/retention.ts`), run
7343
- * once a turn has completed and its checkpoints are written: inline
7344
- * payloads older than the newest `ATTACHMENT_PAYLOAD_TURNS` user turns
7345
- * are rewritten in place in the messages table — the rows the next turn
7346
- * loads, so the model sees the placeholder and the bytes leave the
7347
- * database. Rows the graph state no longer references (summarised away)
7348
- * are older than any retained turn and are rewritten too. Idempotent:
7349
- * a rewritten row holds no inline block and is never selected again.
7414
+ * Image fetch for the attachment recall middleware
7415
+ * (`core/middlewares/attachment-recall.ts`): one image of this session,
7416
+ * read through the same sources as the turn pipeline (Matrix media from
7417
+ * the homeserver, the session's channel links or their room copies,
7418
+ * http(s)), cut off at the middleware's `maxBytes` (null then), refused
7419
+ * unless the bytes are an image, downscaled for the model when the
7420
+ * oracle binds Cloudflare Images, as base64. An image the resize step
7421
+ * drops resolves null: its original never goes to the model.
7350
7422
  */
7351
- private async retainAttachmentPayloads(
7352
- sessionId: string,
7353
- stateMessages: BaseMessage[],
7354
- ): Promise<void> {
7355
- // Nothing captured (an interrupted stream): never offload blindly.
7356
- if (stateMessages.length === 0) return;
7357
- const saver = this.saver!;
7358
- const { retainedIds, rewrites } = applyAttachmentRetention(
7359
- stateMessages,
7360
- ATTACHMENT_PAYLOAD_TURNS,
7361
- );
7362
- const pending = new Set(rewrites.map((m) => m.id));
7363
- const stale = await saver.listThreadMessagesMatching(sessionId, {
7364
- messageType: 'human',
7365
- contentContains: INLINE_PAYLOAD_NEEDLE,
7366
- });
7367
- for (const row of stale) {
7368
- if (!row.id || retainedIds.has(row.id) || pending.has(row.id)) continue;
7369
- const rewritten = offloadInlinePayloads(row);
7370
- if (rewritten) rewrites.push(rewritten);
7371
- }
7372
- let replaced = 0;
7373
- for (const message of rewrites) {
7374
- if (await saver.replaceThreadMessage(sessionId, message)) replaced += 1;
7375
- }
7376
- if (replaced > 0) {
7377
- console.log(
7378
- `[user-do] attachments: offloaded the inline payload of ${replaced} message(s) in ${sessionId}`,
7379
- );
7380
- }
7423
+ private recallImageLoader(input: {
7424
+ sessionId: string;
7425
+ roomId: string | undefined;
7426
+ /** The links of this turn's channel files (earlier ones come from the store). */
7427
+ channelLinks: ReturnType<typeof channelTurnLinks>;
7428
+ }): RecallImageLoader {
7429
+ return async (meta, { signal, maxBytes }) => {
7430
+ const attachment: AttachmentInput = {
7431
+ filename: meta.filename,
7432
+ mimetype: meta.mimetype,
7433
+ ...(meta.size !== undefined ? { size: meta.size } : {}),
7434
+ ...(meta.mxcUri ? { mxcUri: meta.mxcUri } : {}),
7435
+ ...(meta.eventId ? { eventId: meta.eventId } : {}),
7436
+ ...(meta.channelRef ? { channelRef: meta.channelRef } : {}),
7437
+ };
7438
+ const name = sanitizeAttachmentFilename(attachment.filename);
7439
+ const channel = turnChannelMediaSource({
7440
+ sessionId: input.sessionId,
7441
+ turnLinks: input.channelLinks,
7442
+ store: this.channelAttachmentLinks,
7443
+ channelServiceDid: this.env.CHANNEL_SERVICE_DID,
7444
+ });
7445
+ let loaded: Awaited<ReturnType<typeof loadAttachmentBytes>>;
7446
+ try {
7447
+ loaded = await loadAttachmentBytes(
7448
+ attachment,
7449
+ input.roomId,
7450
+ this.matrixMediaSource(),
7451
+ { signal, maxBytes, channel },
7452
+ );
7453
+ } catch (error) {
7454
+ if (!(error instanceof AttachmentTooLargeError)) throw error;
7455
+ console.warn(
7456
+ `[attachments] recall: "${name}" is over the ${maxBytes}-byte recall limit`,
7457
+ );
7458
+ return null;
7459
+ }
7460
+ const { bytes, mimetype, sniffed } = loaded;
7461
+ // Bytes that are not an image never go back as an image block.
7462
+ const kind = classifyAttachment({
7463
+ mimetype,
7464
+ filename: sniffed ? '' : attachment.filename,
7465
+ });
7466
+ if (kind !== 'image')
7467
+ throw new Error(`content is ${mimetype}, not an image`);
7468
+ const sent = await resizeForModel(bytes, mimetype, this.env.IMAGES, {
7469
+ logger: console,
7470
+ });
7471
+ if (sent.dropped) {
7472
+ console.warn(
7473
+ `[attachments] recall: "${name}" not re-attached: ${sent.dropped}`,
7474
+ );
7475
+ return null;
7476
+ }
7477
+ return {
7478
+ kind: 'image',
7479
+ mimeType: sent.mimetype,
7480
+ base64: bytesToBase64(sent.bytes),
7481
+ filename: name,
7482
+ };
7483
+ };
7381
7484
  }
7382
7485
 
7383
7486
  /**
@@ -7665,7 +7768,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7665
7768
  // A channel turn's files are read by their `channel:` reference; their
7666
7769
  // links stay on the server (the run request and the session's link
7667
7770
  // store), out of every message and note.
7668
- const attachmentInputs = turnAttachmentInputs(req, body.attachments);
7669
7771
  const attachmentModel =
7670
7772
  effectiveModel && isAllowedModel(effectiveModel)
7671
7773
  ? effectiveModel
@@ -7681,6 +7783,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7681
7783
  : undefined;
7682
7784
  if (turnLinks.length && !run.resumed)
7683
7785
  await this.channelAttachmentLinks?.put(req.sessionId, turnLinks);
7786
+ // A channel turn's files are also posted to the session thread as
7787
+ // encrypted media events (attachments/room-media.ts), in the
7788
+ // background: the run names each by its `channel:` reference, and the
7789
+ // copies read on the way reach the pipeline instead of a second
7790
+ // download. A resumed attempt posts nothing.
7791
+ const threadCopies =
7792
+ turnLinks.length && sessionRoomId
7793
+ ? this.postChannelFilesToThread(
7794
+ req,
7795
+ sessionRoomId,
7796
+ abortController.signal,
7797
+ run.resumed,
7798
+ )
7799
+ : null;
7800
+ const attachmentInputs = turnAttachmentInputs(req, body.attachments);
7684
7801
  const { prepared, metas: turnMetas } = await prepareTurnAttachments({
7685
7802
  inputs: attachmentInputs,
7686
7803
  resumed: run.resumed,
@@ -7696,22 +7813,28 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
7696
7813
  {
7697
7814
  source: this.matrixMediaSource(),
7698
7815
  ...(channelMedia ? { channel: channelMedia } : {}),
7816
+ ...(threadCopies ? { preloaded: threadCopies } : {}),
7699
7817
  extraction: this.extractionProvider(baseAmbient.llm, core.llm),
7700
7818
  sandbox: await this.sandboxArchiveConfig(req.identity.userDid),
7819
+ ...(this.env.IMAGES ? { images: this.env.IMAGES } : {}),
7701
7820
  signal: abortController.signal,
7702
7821
  logger: console,
7703
7822
  },
7704
7823
  ),
7705
7824
  });
7825
+ // Copies the pipeline did not take (a file it refused) go now, not
7826
+ // with the turn.
7827
+ threadCopies?.clear();
7706
7828
  if (prepared) {
7707
7829
  console.log(
7708
7830
  `[user-do] attachments: ${attachmentInputs.length} attached, ${prepared.extracted.length} extracted, ${Array.isArray(prepared.content) ? prepared.content.length - 1 : 0} native`,
7709
7831
  );
7710
7832
  }
7711
7833
 
7712
- // `view_attachment` for this session: payloads older than the newest
7713
- // `ATTACHMENT_PAYLOAD_TURNS` user turns are placeholders by now (see
7714
- // `retainAttachmentPayloads`); the tool fetches one again on demand.
7834
+ // `view_attachment` for this session: attachment bytes are never
7835
+ // stored with the conversation (the checkpointer keeps a placeholder,
7836
+ // `src/sqlite/inline-payloads.ts`); the tool fetches one again on
7837
+ // demand.
7715
7838
  // Cleanups registered by tools for THIS turn (MCP clients …): run once
7716
7839
  // the turn ends, however it ends — see `RuntimeContext.onTurnEnd`.
7717
7840
  abortController.signal.addEventListener('abort', () => {
@@ -8038,6 +8161,22 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8038
8161
  }
8039
8162
  : {}),
8040
8163
  ...(hostSafetyModel ? { safetyModel: hostSafetyModel(ambient) } : {}),
8164
+ // The images of the newest user turns, back in each model request
8165
+ // (their bytes are not stored with the conversation).
8166
+ ...(suppliedContextOnly
8167
+ ? {}
8168
+ : {
8169
+ attachmentRecall: {
8170
+ loadImage: this.recallImageLoader({
8171
+ sessionId: req.sessionId,
8172
+ roomId: sessionRoomId,
8173
+ channelLinks: turnLinks,
8174
+ }),
8175
+ imageInput: getModelCapabilities(attachmentModel).image,
8176
+ signal: abortController.signal,
8177
+ cache: this.recallCache,
8178
+ },
8179
+ }),
8041
8180
  },
8042
8181
  requestCtx: {
8043
8182
  user: {
@@ -8241,6 +8380,18 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8241
8380
  text: string,
8242
8381
  who: 'user' | 'oracle',
8243
8382
  receipt = false,
8383
+ ): Promise<void> {
8384
+ const queued = this.queueReplay(req, text, who, receipt);
8385
+ // The turn's file events follow the person's text on the chain.
8386
+ if (who === 'user') this.noteChannelTextQueued(req);
8387
+ return queued;
8388
+ }
8389
+
8390
+ private queueReplay(
8391
+ req: TurnRequest,
8392
+ text: string,
8393
+ who: 'user' | 'oracle',
8394
+ receipt: boolean,
8244
8395
  ): Promise<void> {
8245
8396
  if (req.client === 'matrix' && !receipt) return Promise.resolve();
8246
8397
  if (req.sessionId.startsWith(SYNTHETIC_SESSION_PREFIX))
@@ -8308,7 +8459,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8308
8459
  send.roomId,
8309
8460
  'm.room.message',
8310
8461
  JSON.stringify(mirrorEventContent(send)),
8311
- { txnId: send.txnId },
8462
+ { txnId: send.txnId, priority: 'background' },
8312
8463
  )
8313
8464
  : this.gateway.sendText(send.roomId, send.body, {
8314
8465
  threadId: send.threadId,
@@ -8318,6 +8469,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8318
8469
  priority: 'background',
8319
8470
  txnId: send.txnId,
8320
8471
  }),
8472
+ sendEvent: (roomId, content, txnId) =>
8473
+ this.gateway.sendEvent(roomId, 'm.room.message', content, {
8474
+ txnId,
8475
+ priority: 'background',
8476
+ }),
8321
8477
  keepAlive: (work) => this.ctx.waitUntil(work),
8322
8478
  log: (message) => console.log(message),
8323
8479
  warn: (message) => console.warn(message),
@@ -8325,6 +8481,271 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8325
8481
  return this.mirror;
8326
8482
  }
8327
8483
 
8484
+ /**
8485
+ * The oracle's plugin device (`botCredentials`): what channel files are
8486
+ * uploaded and Matrix media downloaded with. One gateway RPC per
8487
+ * instance, shared by concurrent callers. A caller the homeserver
8488
+ * answered 401 passes the token it used as `rejected`: the cache is
8489
+ * dropped only when it still holds that token, and the gateway
8490
+ * re-validates its device (or logs it in again) once for everyone.
8491
+ */
8492
+ private pluginDevice(
8493
+ opts: { rejected?: string } = {},
8494
+ ): Promise<BotCredentials> {
8495
+ if (
8496
+ opts.rejected &&
8497
+ this.pluginCredentials?.accessToken === opts.rejected
8498
+ )
8499
+ this.pluginCredentials = null;
8500
+ if (this.pluginCredentials)
8501
+ return Promise.resolve(this.pluginCredentials);
8502
+ this.pluginCredentialsLoad ??= this.gateway
8503
+ .botCredentials(opts.rejected ? { rejected: opts.rejected } : {})
8504
+ .then((credentials) => {
8505
+ this.pluginCredentials = credentials;
8506
+ return credentials;
8507
+ })
8508
+ .finally(() => {
8509
+ this.pluginCredentialsLoad = null;
8510
+ });
8511
+ return this.pluginCredentialsLoad;
8512
+ }
8513
+
8514
+ /**
8515
+ * The mirror pass has queued the person's text of channel request
8516
+ * `req` on the session chain (or found none to queue): release the
8517
+ * request's file events waiting for it, or remember it for a turn that
8518
+ * has not asked yet. Only requests with files are tracked; entries older
8519
+ * than the wait are dropped.
8520
+ */
8521
+ private noteChannelTextQueued(req: TurnRequest): void {
8522
+ if (req.client !== 'channel' || !req.channel?.attachments?.length) return;
8523
+ const entry = this.channelTextQueued.get(req.requestId);
8524
+ if (entry?.waiter) {
8525
+ this.channelTextQueued.delete(req.requestId);
8526
+ entry.waiter();
8527
+ return;
8528
+ }
8529
+ const now = Date.now();
8530
+ for (const [requestId, noted] of this.channelTextQueued)
8531
+ if (!noted.waiter && now - noted.at > CHANNEL_MEDIA_TEXT_WAIT_MS)
8532
+ this.channelTextQueued.delete(requestId);
8533
+ this.channelTextQueued.set(req.requestId, { at: now });
8534
+ }
8535
+
8536
+ /**
8537
+ * `queued` resolves once the person's text of `requestId` is queued on
8538
+ * the session chain (true), after `CHANNEL_MEDIA_TEXT_WAIT_MS` (false),
8539
+ * or with null when `cancel` is called first (nothing left to post).
8540
+ */
8541
+ private afterChannelText(requestId: string): {
8542
+ queued: Promise<boolean | null>;
8543
+ cancel: () => void;
8544
+ } {
8545
+ const noted = this.channelTextQueued.get(requestId);
8546
+ if (noted && !noted.waiter) {
8547
+ this.channelTextQueued.delete(requestId);
8548
+ return { queued: Promise.resolve(true), cancel: () => undefined };
8549
+ }
8550
+ let settle: (value: boolean | null) => void = () => undefined;
8551
+ const queued = new Promise<boolean | null>((resolve) => {
8552
+ settle = resolve;
8553
+ });
8554
+ const finish = (value: boolean | null): void => {
8555
+ clearTimeout(timer);
8556
+ if (this.channelTextQueued.get(requestId)?.waiter === waiter)
8557
+ this.channelTextQueued.delete(requestId);
8558
+ settle(value);
8559
+ };
8560
+ const waiter = (): void => finish(true);
8561
+ const timer = setTimeout(() => finish(false), CHANNEL_MEDIA_TEXT_WAIT_MS);
8562
+ this.channelTextQueued.set(requestId, { at: Date.now(), waiter });
8563
+ return { queued, cancel: () => finish(null) };
8564
+ }
8565
+
8566
+ /**
8567
+ * Post a channel turn's files as encrypted media events in the session
8568
+ * thread (`attachments/room-media.ts`), off the turn's critical path.
8569
+ * Returns at once: per file the turn may read directly
8570
+ * (`channelFilesToKeep`), a promise of its verified plaintext, resolved
8571
+ * as soon as the read passes its check, or null when no copy comes (the
8572
+ * pipeline then reads the gateway as before). In the background:
8573
+ *
8574
+ * - the encryption check and the plugin device, bounded with the uploads'
8575
+ * reads by `CHANNEL_MEDIA_TURN_TIMEOUT_MS` from now and by the run's
8576
+ * signal; on a failure (or a room that is not Megolm-encrypted) one
8577
+ * line is logged and nothing is posted;
8578
+ * - the uploads (`startChannelFileUploads`), kept alive by `waitUntil`;
8579
+ * - one event per file on the session's mirror chain, queued in message
8580
+ * order right after the person's text (`afterChannelText`) at
8581
+ * background priority, each within `CHANNEL_MEDIA_SEND_TIMEOUT_MS`;
8582
+ * their event ids then go to the link store (`setEventIds`), whose room
8583
+ * copies serve the files once their links expire.
8584
+ *
8585
+ * A resumed attempt posts nothing: its first attempt did, or lost the
8586
+ * uploads with the object, and the person's message is in the
8587
+ * checkpoint already.
8588
+ */
8589
+ private postChannelFilesToThread(
8590
+ req: TurnRequest,
8591
+ roomId: string,
8592
+ runSignal: AbortSignal,
8593
+ resumed: boolean,
8594
+ ): Map<string, Promise<ThreadFileCopy | null>> {
8595
+ const copies = new Map<string, Promise<ThreadFileCopy | null>>();
8596
+ const files = channelThreadFiles(req);
8597
+ const channel = req.channel;
8598
+ // A channel session is always a thread under its marker event.
8599
+ if (
8600
+ resumed ||
8601
+ !files.length ||
8602
+ !channel ||
8603
+ !req.sessionId.startsWith('$')
8604
+ )
8605
+ return copies;
8606
+ const keep = channelFilesToKeep(files);
8607
+ const pending = new Map<
8608
+ string,
8609
+ (copy: Promise<ThreadFileCopy | null> | null) => void
8610
+ >();
8611
+ for (const ref of keep)
8612
+ copies.set(
8613
+ ref,
8614
+ new Promise((resolve) => {
8615
+ pending.set(ref, resolve);
8616
+ }),
8617
+ );
8618
+ const textOrder = this.afterChannelText(req.requestId);
8619
+ const deadline = Date.now() + CHANNEL_MEDIA_TURN_TIMEOUT_MS;
8620
+
8621
+ const started = (async (): Promise<ChannelFileUpload[]> => {
8622
+ const setup = boundedSignal(
8623
+ runSignal,
8624
+ CHANNEL_MEDIA_TURN_TIMEOUT_MS,
8625
+ 'Preparing the channel file posts',
8626
+ );
8627
+ try {
8628
+ const [encrypted, device] = await untilAborted(
8629
+ Promise.all([this.roomIsEncrypted(roomId), this.pluginDevice()]),
8630
+ setup.signal,
8631
+ );
8632
+ if (!encrypted) {
8633
+ console.warn(
8634
+ `[attachments] channel files of ${req.sessionId} not posted to its thread: the room is not end-to-end encrypted`,
8635
+ );
8636
+ return [];
8637
+ }
8638
+ const run = startChannelFileUploads({
8639
+ files,
8640
+ sessionId: req.sessionId,
8641
+ deps: {
8642
+ origins: channelGatewayOrigins(this.env.CHANNEL_SERVICE_DID),
8643
+ homeserver: {
8644
+ baseUrl: device.baseUrl,
8645
+ accessToken: device.accessToken,
8646
+ },
8647
+ keep,
8648
+ turnTimeoutMs: Math.max(0, deadline - Date.now()),
8649
+ onUnauthorized: (token) => {
8650
+ void this.pluginDevice({ rejected: token }).catch(
8651
+ () => undefined,
8652
+ );
8653
+ },
8654
+ log: console,
8655
+ },
8656
+ });
8657
+ this.ctx.waitUntil(run.done);
8658
+ for (const upload of run.uploads)
8659
+ pending.get(upload.file.ref)?.(upload.copy);
8660
+ return run.uploads;
8661
+ } catch (error) {
8662
+ console.warn(
8663
+ `[attachments] channel files of ${req.sessionId} not posted to its thread; the turn reads them from the channel gateway: ${error instanceof Error ? error.message : String(error)}`,
8664
+ );
8665
+ return [];
8666
+ } finally {
8667
+ setup.release();
8668
+ // A copy not handed over by now never comes (a resolved one stays).
8669
+ for (const resolve of pending.values()) resolve(null);
8670
+ }
8671
+ })();
8672
+
8673
+ const origin = {
8674
+ provider: channel.provider,
8675
+ bindingId: channel.bindingId,
8676
+ remoteMessageRef: channel.remoteMessageRef,
8677
+ };
8678
+ // Nothing to post: the events need not wait for the text.
8679
+ void started.then((uploads) => {
8680
+ if (uploads.length === 0) textOrder.cancel();
8681
+ });
8682
+ const posted = textOrder.queued.then(async (afterText) => {
8683
+ if (afterText === null) return;
8684
+ if (!afterText)
8685
+ console.warn(
8686
+ `[attachments] the text of ${req.requestId} was not queued within ${CHANNEL_MEDIA_TEXT_WAIT_MS} ms; its files are queued without it`,
8687
+ );
8688
+ // Queued synchronously, one after the other: the chain keeps the
8689
+ // message order whichever upload ends first.
8690
+ const sent = await Promise.all(
8691
+ files.map(async (file): Promise<[string, string] | null> => {
8692
+ const bound = boundedSignal(
8693
+ undefined,
8694
+ CHANNEL_MEDIA_SEND_TIMEOUT_MS,
8695
+ 'Posting the channel file',
8696
+ );
8697
+ try {
8698
+ const eventId = await this.roomMirror().sendMedia(
8699
+ req.sessionId,
8700
+ async () => {
8701
+ const upload = (await started).find(
8702
+ (entry) => entry.file.ref === file.ref,
8703
+ );
8704
+ // A failed upload is logged by `startChannelFileUploads`.
8705
+ const media = await upload?.uploaded.catch(() => null);
8706
+ if (!media) return null;
8707
+ return {
8708
+ roomId,
8709
+ threadId: req.sessionId,
8710
+ txnId: await channelMediaTxnId({
8711
+ userDid: req.identity.userDid,
8712
+ bindingId: channel.bindingId,
8713
+ requestId: req.requestId,
8714
+ index: file.index,
8715
+ }),
8716
+ origin,
8717
+ media,
8718
+ };
8719
+ },
8720
+ 'channel file',
8721
+ bound.signal,
8722
+ );
8723
+ return eventId ? [file.ref, eventId] : null;
8724
+ } catch (error) {
8725
+ console.warn(
8726
+ `[attachments] channel file ${file.ref} not posted to thread ${req.sessionId}: ${error instanceof Error ? error.message : String(error)}`,
8727
+ );
8728
+ return null;
8729
+ } finally {
8730
+ bound.release();
8731
+ }
8732
+ }),
8733
+ );
8734
+ const eventIds = new Map(
8735
+ sent.filter((entry): entry is [string, string] => entry !== null),
8736
+ );
8737
+ await this.channelAttachmentLinks?.setEventIds(req.sessionId, eventIds);
8738
+ });
8739
+ this.ctx.waitUntil(
8740
+ posted.catch((error: unknown) => {
8741
+ console.warn(
8742
+ `[attachments] channel file posts of ${req.sessionId} failed: ${error instanceof Error ? error.message : String(error)}`,
8743
+ );
8744
+ }),
8745
+ );
8746
+ return copies;
8747
+ }
8748
+
8328
8749
  private async afterTurn(
8329
8750
  sessionId: string,
8330
8751
  messages: BaseMessage[],
@@ -8333,13 +8754,6 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
8333
8754
  ): Promise<void> {
8334
8755
  const sessions = this.sessions!;
8335
8756
  await sessions.touchSession(sessionId);
8336
- await this.retainAttachmentPayloads(sessionId, messages).catch(
8337
- (err: unknown) => {
8338
- console.warn(
8339
- `[user-do] attachment payload retention failed for ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
8340
- );
8341
- },
8342
- );
8343
8757
  if (disposition.kind === 'agent')
8344
8758
  await this.compareShadowRoute(sessionId);
8345
8759
  const row = await sessions.getSession(sessionId);