@ixo/oracle-runtime-workers 0.37.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/attachments/channel-links.ts +74 -7
- package/src/attachments/channel.ts +15 -4
- package/src/attachments/content-blocks.ts +19 -16
- package/src/attachments/download.ts +173 -50
- package/src/attachments/index.ts +6 -5
- package/src/attachments/pipeline.ts +117 -49
- package/src/attachments/resize.ts +142 -0
- package/src/attachments/retention.ts +26 -70
- package/src/attachments/room-media.ts +643 -0
- package/src/attachments/route.ts +47 -4
- package/src/attachments/sandbox-archive.ts +15 -4
- package/src/attachments/sanitize.ts +24 -13
- package/src/attachments/view.ts +82 -38
- package/src/channels/replies.ts +8 -4
- package/src/core/capability-expiry.ts +62 -0
- package/src/core/capability-router.ts +126 -95
- package/src/core/context-budget.ts +116 -4
- package/src/core/domain-context/document.ts +74 -2
- package/src/core/domain-context/index.ts +1 -0
- package/src/core/domain-context/resolver.ts +9 -6
- package/src/core/domain-context/turn-context.ts +62 -13
- package/src/core/domain-context/types.ts +17 -0
- package/src/core/env.ts +21 -2
- package/src/core/index.ts +4 -0
- package/src/core/llm.ts +29 -0
- package/src/core/main-agent-types.ts +53 -4
- package/src/core/main-agent.ts +228 -125
- package/src/core/manifest.ts +109 -24
- package/src/core/mcp-result.ts +56 -0
- package/src/core/meta-tools.ts +258 -110
- package/src/core/middlewares/attachment-recall.ts +502 -0
- package/src/core/middlewares/capability-gate.ts +77 -28
- package/src/core/middlewares/context-guard.ts +154 -21
- package/src/core/middlewares/index.ts +5 -0
- package/src/core/middlewares/result-cap.ts +1 -1
- package/src/core/middlewares/summarization.ts +9 -0
- package/src/core/middlewares/tool-marks.ts +11 -2
- package/src/core/middlewares/turn-boundary.ts +22 -12
- package/src/core/operating-guides.ts +126 -0
- package/src/core/plugins/skills/skills-tools.ts +6 -31
- package/src/core/plugins/skills/skills.plugin.ts +4 -3
- package/src/core/plugins/weather/weather.plugin.ts +1 -0
- package/src/core/prompt-composer.ts +188 -111
- package/src/core/read-result-tool.ts +48 -14
- package/src/core/registries.ts +13 -5
- package/src/core/state.ts +4 -3
- package/src/core/sub-agent-fallback.ts +1 -0
- package/src/core/subagent-as-tool.ts +4 -1
- package/src/core/tool-access.ts +3 -32
- package/src/core/wrap-plugin-tool.ts +2 -1
- package/src/do/contracts.ts +64 -24
- package/src/do/room-mirror.ts +144 -20
- package/src/do/transcript.ts +41 -12
- package/src/do/turn-attachments.ts +45 -6
- package/src/do/turn-metadata.ts +16 -4
- package/src/do/user-oracle-do.ts +639 -152
- package/src/index.ts +1 -0
- package/src/llm/provider-error.ts +5 -2
- package/src/llm/stream-liveness.ts +3 -2
- package/src/matrix/gateway-do.ts +164 -102
- package/src/matrix/ingest.ts +12 -3
- package/src/matrix/media-deadline.ts +5 -47
- package/src/memory/session-history-indexer.ts +60 -14
- package/src/plugin-api/oracle-plugin.ts +4 -5
- package/src/plugin-api/types.ts +32 -3
- package/src/plugins/action-proposals/action-proposals.plugin.ts +7 -2
- package/src/plugins/agui/agui.plugin.ts +1 -0
- package/src/plugins/attachments/attachments.plugin.ts +3 -3
- package/src/plugins/attachments/view-attachment-tool.ts +6 -4
- package/src/plugins/compact-schema.ts +80 -0
- package/src/plugins/composio/composio.plugin.ts +1 -0
- package/src/plugins/domain-indexer/domain-indexer.plugin.ts +4 -3
- package/src/plugins/editor/editor.plugin.ts +4 -0
- package/src/plugins/firecrawl/firecrawl-agent.ts +2 -0
- package/src/plugins/firecrawl/firecrawl.plugin.ts +1 -0
- package/src/plugins/flows/flows.plugin.ts +37 -6
- package/src/plugins/flows/prompts.ts +1 -1
- package/src/plugins/flows/tools/authoring.ts +5 -7
- package/src/plugins/flows/tools/discovery.ts +1 -4
- package/src/plugins/flows/tools/forms.ts +2 -8
- package/src/plugins/flows/tools/linkage.ts +1 -4
- package/src/plugins/flows/tools/settings.ts +1 -4
- package/src/plugins/flows/translator.ts +16 -5
- package/src/plugins/flows/types.ts +27 -48
- package/src/plugins/hub-actions/hub-actions.plugin.ts +30 -4
- package/src/plugins/index.ts +2 -2
- package/src/plugins/ixo-chain/ixo-chain.plugin.ts +29 -5
- package/src/plugins/ixo-chain/kinds/core.ts +2 -2
- package/src/plugins/ixo-chain/kinds/types.ts +34 -5
- package/src/plugins/ixo-chain/queries.ts +10 -1
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +1 -0
- package/src/plugins/matrix-group-chats/index.ts +1 -0
- package/src/plugins/memory/memory-tools.ts +4 -4
- package/src/plugins/memory/memory.plugin.ts +27 -1
- package/src/plugins/memory/oracle-document-tools.ts +24 -49
- package/src/plugins/pod-creator/pod-creator.plugin.ts +1 -0
- package/src/plugins/portal/portal.plugin.ts +1 -0
- package/src/plugins/sandbox/sandbox-write-blob.ts +3 -15
- package/src/plugins/sandbox/sandbox.plugin.ts +19 -2
- package/src/plugins/tasks/manifest.ts +1 -0
- package/src/plugins/tasks/tasks-tools.ts +14 -19
- package/src/plugins/user-preferences/user-preferences-tool.ts +11 -37
- package/src/plugins/user-preferences/user-preferences.plugin.ts +4 -0
- package/src/plugins/vfs/vfs-tools.ts +6 -6
- package/src/plugins/vfs/vfs.plugin.ts +4 -5
- package/src/plugins/yoma/yoma.plugin.ts +1 -0
- package/src/sqlite/inline-payloads.ts +121 -0
- package/src/sqlite/serialization.ts +12 -0
- package/src/sqlite/sqlite-saver.ts +258 -42
- package/src/tasks/schedule-input.ts +5 -22
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Files the person sent on an IXO Channels turn, posted as encrypted Matrix
|
|
3
|
+
* media events in the session's thread of their oracle room — so the room
|
|
4
|
+
* (Qi.Space) shows the photo or document next to the mirrored text, and the
|
|
5
|
+
* file outlives the gateway's 30-day link (`channel-links.ts` keeps the
|
|
6
|
+
* event; `download.ts` reads the room copy once the link fails).
|
|
7
|
+
*
|
|
8
|
+
* Everything heavy runs in the user's own object; the shared Matrix gateway
|
|
9
|
+
* only carries the small event send. One pass over the gateway's link:
|
|
10
|
+
*
|
|
11
|
+
* gateway link ─▶ check (size, SHA-256, head, plaintext copy) ─▶ AES-CTR ─▶ FixedLengthStream ─▶ POST /_matrix/media/v3/upload
|
|
12
|
+
*
|
|
13
|
+
* The check is a transform in that single pipeline rather than a `tee()`: a
|
|
14
|
+
* tee buffers for its slower branch, so a fast hash branch would queue the
|
|
15
|
+
* whole file for a slow upload branch. Inline, memory stays O(chunk) plus
|
|
16
|
+
* the plaintext copy, kept only for a file the model may read directly (an
|
|
17
|
+
* image, or a document of at most `NATIVE_DOCUMENT_MAX_BYTES`) and only
|
|
18
|
+
* while the turn's copies stay within `CHANNEL_MEDIA_KEEP_TOTAL_BYTES`. The
|
|
19
|
+
* check holds the latest chunk back until the stream ends, then releases it
|
|
20
|
+
* only when the byte count and the SHA-256 are the gateway's: a file that
|
|
21
|
+
* is not what the gateway announced never completes its upload.
|
|
22
|
+
*
|
|
23
|
+
* The turn never waits for the homeserver or the room. The copy of each kept
|
|
24
|
+
* file resolves the moment the check passes (before the upload's answer);
|
|
25
|
+
* the turn's wait for it is bounded by the per-file read deadline and the
|
|
26
|
+
* turn's read deadline, and a copy that does not come leaves the file to be
|
|
27
|
+
* read from the gateway as before. Once the read has passed the check, the
|
|
28
|
+
* upload has its own deadline (`CHANNEL_MEDIA_UPLOAD_TIMEOUT_MS`) and the
|
|
29
|
+
* event is sent later by the owner object, in file order on the session's
|
|
30
|
+
* mirror chain (`RoomMirror.sendMedia`).
|
|
31
|
+
*
|
|
32
|
+
* Only a Megolm room gets a file (the owner object checks; the gateway
|
|
33
|
+
* refuses to send an attachment key into a room it does not see as
|
|
34
|
+
* encrypted). The upload carries no file name and an
|
|
35
|
+
* `application/octet-stream` type, so the homeserver operator learns only
|
|
36
|
+
* the size; the name and the type live in the encrypted event, the type
|
|
37
|
+
* taken from the file's own bytes (`roomMediaType`). The upload uses the
|
|
38
|
+
* oracle's crypto-less plugin device (`MatrixGatewayObject.botCredentials`).
|
|
39
|
+
*/
|
|
40
|
+
import { createAttachmentEncryptor } from '@ixo/matrix-bot-workers-sdk';
|
|
41
|
+
import { channelRequestHash } from '../channels/contract';
|
|
42
|
+
import type { MediaMessage } from '../do/room-mirror';
|
|
43
|
+
import type { Logger } from '../plugin-api/types';
|
|
44
|
+
import {
|
|
45
|
+
CHANNEL_MAX_FILE_BYTES,
|
|
46
|
+
ChannelFileUnavailableError,
|
|
47
|
+
hex,
|
|
48
|
+
isGatewayMediaUrl,
|
|
49
|
+
type ChannelMediaLink,
|
|
50
|
+
} from './channel';
|
|
51
|
+
import {
|
|
52
|
+
abortError,
|
|
53
|
+
boundedSignal,
|
|
54
|
+
DOWNLOAD_TIMEOUT_MS,
|
|
55
|
+
MAX_FILE_SIZE,
|
|
56
|
+
untilAborted,
|
|
57
|
+
} from './download';
|
|
58
|
+
import { containerMatchesClaim, detectMimeFromMagicBytes } from './magic';
|
|
59
|
+
import { NATIVE_DOCUMENT_MAX_BYTES } from './route';
|
|
60
|
+
import { sanitizeAttachmentFilename } from './sanitize';
|
|
61
|
+
|
|
62
|
+
/** Upper bound on reading all of one turn's files; a file not read by then is read from the gateway as before. */
|
|
63
|
+
export const CHANNEL_MEDIA_TURN_TIMEOUT_MS = 120_000;
|
|
64
|
+
/** Upper bound on the homeserver's answer once a file has been read and checked. */
|
|
65
|
+
export const CHANNEL_MEDIA_UPLOAD_TIMEOUT_MS = 60_000;
|
|
66
|
+
/**
|
|
67
|
+
* How long a turn's file events wait for the mirror pass to queue the
|
|
68
|
+
* person's text ahead of them on the session chain; past it they are queued
|
|
69
|
+
* anyway (and the room may show them before the text).
|
|
70
|
+
*/
|
|
71
|
+
export const CHANNEL_MEDIA_TEXT_WAIT_MS = 60_000;
|
|
72
|
+
/**
|
|
73
|
+
* Upper bound on one file event in the background, from being queued on the
|
|
74
|
+
* session chain to its event id (the wait for the text, the upload, earlier
|
|
75
|
+
* mirrors, the send and its retries); past it nothing more is attempted.
|
|
76
|
+
*/
|
|
77
|
+
export const CHANNEL_MEDIA_SEND_TIMEOUT_MS = 300_000;
|
|
78
|
+
/** Files of one turn uploaded at the same time. */
|
|
79
|
+
export const CHANNEL_MEDIA_CONCURRENCY = 3;
|
|
80
|
+
/** The most plaintext one turn keeps for the model across its files (16 MiB). */
|
|
81
|
+
export const CHANNEL_MEDIA_KEEP_TOTAL_BYTES = 16 * 1024 * 1024;
|
|
82
|
+
|
|
83
|
+
export type ChannelFileKind = 'image' | 'video' | 'document' | 'audio';
|
|
84
|
+
|
|
85
|
+
/** One channel file as the gateway announced it. */
|
|
86
|
+
export interface ChannelThreadFile {
|
|
87
|
+
/** Its `channel:` reference. */
|
|
88
|
+
ref: string;
|
|
89
|
+
/** Its position in the person's message (the event's transaction id). */
|
|
90
|
+
index: number;
|
|
91
|
+
link: ChannelMediaLink;
|
|
92
|
+
filename: string;
|
|
93
|
+
mimetype: string;
|
|
94
|
+
/** Exact byte length the gateway serves. */
|
|
95
|
+
size: number;
|
|
96
|
+
kind: ChannelFileKind;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The plugin device's homeserver and access token. */
|
|
100
|
+
export interface MediaHomeserver {
|
|
101
|
+
baseUrl: string;
|
|
102
|
+
accessToken: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A file's verified plaintext, for the turn's model lane. */
|
|
106
|
+
export interface ThreadFileCopy {
|
|
107
|
+
bytes: Uint8Array;
|
|
108
|
+
/** The gateway's content type, else the announced one. */
|
|
109
|
+
mimetype: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** One file in flight: what the turn may read, and what the room will show. */
|
|
113
|
+
export interface ChannelFileUpload {
|
|
114
|
+
file: ChannelThreadFile;
|
|
115
|
+
/**
|
|
116
|
+
* The verified plaintext, resolved as soon as the check passes (before the
|
|
117
|
+
* upload's answer) for a kept file; null for any other, or when the read
|
|
118
|
+
* failed or ran out of time. Never rejects, and stays resolved when the
|
|
119
|
+
* upload fails afterwards.
|
|
120
|
+
*/
|
|
121
|
+
copy: Promise<ThreadFileCopy | null>;
|
|
122
|
+
/** The event content once the homeserver holds the file; rejects (without the link) when it does not. */
|
|
123
|
+
uploaded: Promise<MediaMessage>;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface ChannelUploadDeps {
|
|
127
|
+
/** The https origins of this deployment's channel gateways. */
|
|
128
|
+
origins: readonly string[];
|
|
129
|
+
homeserver: MediaHomeserver;
|
|
130
|
+
fetchImpl?: typeof fetch;
|
|
131
|
+
/** Per-file deadline on the read, from opening the link to the check passing (default `DOWNLOAD_TIMEOUT_MS`). */
|
|
132
|
+
readTimeoutMs?: number;
|
|
133
|
+
/** Deadline on the homeserver's answer once the read has passed (default `CHANNEL_MEDIA_UPLOAD_TIMEOUT_MS`). */
|
|
134
|
+
uploadTimeoutMs?: number;
|
|
135
|
+
/** The homeserver rejected `accessToken` (401): the caller asks for fresh credentials. */
|
|
136
|
+
onUnauthorized?: (accessToken: string) => void;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** The homeserver refused or failed the upload (`status` when it answered). */
|
|
140
|
+
export class ChannelMediaUploadError extends Error {
|
|
141
|
+
override readonly name: string = 'ChannelMediaUploadError';
|
|
142
|
+
|
|
143
|
+
constructor(
|
|
144
|
+
message: string,
|
|
145
|
+
readonly status?: number,
|
|
146
|
+
) {
|
|
147
|
+
super(message);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const errorText = (error: unknown): string =>
|
|
152
|
+
error instanceof Error ? error.message : String(error);
|
|
153
|
+
|
|
154
|
+
/** A promise with its settle functions (ES2024 `Promise.withResolvers`). */
|
|
155
|
+
function deferred<T>(): {
|
|
156
|
+
promise: Promise<T>;
|
|
157
|
+
resolve: (value: T | PromiseLike<T>) => void;
|
|
158
|
+
reject: (reason: unknown) => void;
|
|
159
|
+
} {
|
|
160
|
+
let resolve: (value: T | PromiseLike<T>) => void = () => undefined;
|
|
161
|
+
let reject: (reason: unknown) => void = () => undefined;
|
|
162
|
+
const promise = new Promise<T>((res, rej) => {
|
|
163
|
+
resolve = res;
|
|
164
|
+
reject = rej;
|
|
165
|
+
});
|
|
166
|
+
return { promise, resolve, reject };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Enough leading bytes for every signature `detectMimeFromMagicBytes` knows. */
|
|
170
|
+
const HEAD_BYTES = 16;
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The size and digest check, inline in the upload pipeline (see the module
|
|
174
|
+
* comment): counts and hashes every chunk, keeps the first bytes for the
|
|
175
|
+
* type, copies the file when `keep`, and withholds the latest chunk until
|
|
176
|
+
* the end proves the file is the announced one. `onVerified` runs then,
|
|
177
|
+
* before that chunk goes on (so before the upload can be answered), with
|
|
178
|
+
* the copy when one was kept. `failure()` is the first refusal, for a
|
|
179
|
+
* caller whose upload failed because of it.
|
|
180
|
+
*/
|
|
181
|
+
function verifiedPassThrough(expected: {
|
|
182
|
+
size: number;
|
|
183
|
+
sha256: string;
|
|
184
|
+
keep: boolean;
|
|
185
|
+
onVerified: (copy: Uint8Array | null, head: Uint8Array) => void;
|
|
186
|
+
}): {
|
|
187
|
+
transform: TransformStream<Uint8Array, Uint8Array>;
|
|
188
|
+
failure: () => Error | null;
|
|
189
|
+
} {
|
|
190
|
+
const digest = new crypto.DigestStream('SHA-256');
|
|
191
|
+
const hash = digest.getWriter();
|
|
192
|
+
let copy: Uint8Array | null = expected.keep
|
|
193
|
+
? new Uint8Array(expected.size)
|
|
194
|
+
: null;
|
|
195
|
+
const head = new Uint8Array(Math.min(HEAD_BYTES, expected.size));
|
|
196
|
+
let seen = 0;
|
|
197
|
+
let held: Uint8Array | null = null;
|
|
198
|
+
let failure: Error | null = null;
|
|
199
|
+
const refuse = (reason: string): Error => {
|
|
200
|
+
failure ??= new ChannelFileUnavailableError(reason);
|
|
201
|
+
copy = null;
|
|
202
|
+
return failure;
|
|
203
|
+
};
|
|
204
|
+
const transform = new TransformStream<Uint8Array, Uint8Array>({
|
|
205
|
+
async transform(chunk, controller) {
|
|
206
|
+
// An empty chunk carries nothing, and held back it would release the
|
|
207
|
+
// real last chunk before the check.
|
|
208
|
+
if (chunk.byteLength === 0) return;
|
|
209
|
+
if (seen + chunk.byteLength > expected.size)
|
|
210
|
+
throw refuse('it is larger than the channel announced');
|
|
211
|
+
if (seen < head.length)
|
|
212
|
+
head.set(chunk.subarray(0, head.length - seen), seen);
|
|
213
|
+
copy?.set(chunk, seen);
|
|
214
|
+
seen += chunk.byteLength;
|
|
215
|
+
await hash.write(chunk);
|
|
216
|
+
if (held) controller.enqueue(held);
|
|
217
|
+
held = chunk;
|
|
218
|
+
},
|
|
219
|
+
async flush(controller) {
|
|
220
|
+
if (seen !== expected.size)
|
|
221
|
+
throw refuse('it is smaller than the channel announced');
|
|
222
|
+
await hash.close();
|
|
223
|
+
if (hex(await digest.digest) !== expected.sha256)
|
|
224
|
+
throw refuse('its bytes do not match the digest the channel sent');
|
|
225
|
+
const kept = copy;
|
|
226
|
+
// The turn owns the copy from here; the pipeline lets go of it.
|
|
227
|
+
copy = null;
|
|
228
|
+
expected.onVerified(kept, head);
|
|
229
|
+
if (held) controller.enqueue(held);
|
|
230
|
+
held = null;
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
return { transform, failure: () => failure };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Raster images a Matrix client shows inline (`m.image`). */
|
|
237
|
+
const RASTER_TYPES = new Set([
|
|
238
|
+
'image/png',
|
|
239
|
+
'image/jpeg',
|
|
240
|
+
'image/gif',
|
|
241
|
+
'image/webp',
|
|
242
|
+
]);
|
|
243
|
+
|
|
244
|
+
/** Types a file event may name; anything else is `application/octet-stream`. */
|
|
245
|
+
const ROOM_TYPES = new Set([
|
|
246
|
+
...RASTER_TYPES,
|
|
247
|
+
'video/mp4',
|
|
248
|
+
'video/webm',
|
|
249
|
+
'video/quicktime',
|
|
250
|
+
'video/3gpp',
|
|
251
|
+
'audio/mpeg',
|
|
252
|
+
'audio/ogg',
|
|
253
|
+
'audio/mp4',
|
|
254
|
+
'audio/x-m4a',
|
|
255
|
+
'audio/flac',
|
|
256
|
+
'audio/webm',
|
|
257
|
+
'audio/wav',
|
|
258
|
+
'audio/x-wav',
|
|
259
|
+
'application/pdf',
|
|
260
|
+
]);
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* The type and msgtype of the file event, from the file's own first bytes
|
|
264
|
+
* (`detectMimeFromMagicBytes`): the claim is kept only when it is the more
|
|
265
|
+
* precise name of the sniffed container (`containerMatchesClaim`: an
|
|
266
|
+
* `audio/mp4` voice note in an ISO-BMFF box); bytes no signature matches,
|
|
267
|
+
* and any type outside the allowlist, are `application/octet-stream`.
|
|
268
|
+
* `m.image` only for a raster image; video and audio by their type; every
|
|
269
|
+
* other file is `m.file`.
|
|
270
|
+
*/
|
|
271
|
+
export function roomMediaType(
|
|
272
|
+
head: Uint8Array,
|
|
273
|
+
claimed: string,
|
|
274
|
+
): { mimetype: string; msgtype: MediaMessage['msgtype'] } {
|
|
275
|
+
const claim = (claimed.split(';')[0] ?? '').trim().toLowerCase();
|
|
276
|
+
const sniffed = detectMimeFromMagicBytes(head);
|
|
277
|
+
const named = sniffed
|
|
278
|
+
? containerMatchesClaim(sniffed, claim)
|
|
279
|
+
? claim
|
|
280
|
+
: sniffed
|
|
281
|
+
: null;
|
|
282
|
+
const mimetype =
|
|
283
|
+
named && ROOM_TYPES.has(named) ? named : 'application/octet-stream';
|
|
284
|
+
const msgtype: MediaMessage['msgtype'] = RASTER_TYPES.has(mimetype)
|
|
285
|
+
? 'm.image'
|
|
286
|
+
: mimetype.startsWith('video/')
|
|
287
|
+
? 'm.video'
|
|
288
|
+
: mimetype.startsWith('audio/')
|
|
289
|
+
? 'm.audio'
|
|
290
|
+
: 'm.file';
|
|
291
|
+
return { mimetype, msgtype };
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** The file's name in the room: the cleaned name, or its kind when nothing is left. */
|
|
295
|
+
function roomFilename(file: ChannelThreadFile): string {
|
|
296
|
+
const cleaned = sanitizeAttachmentFilename(file.filename).trim();
|
|
297
|
+
return cleaned || (file.kind === 'image' ? 'photo' : file.kind);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Stream one channel file from the gateway's link to the homeserver,
|
|
302
|
+
* encrypted for a Megolm room (`EncryptedFile` v2), and build its event
|
|
303
|
+
* content. `keep` asks for the verified plaintext (`copy`). `readSignal`
|
|
304
|
+
* (the turn's read deadline) stops a read that has not passed its check;
|
|
305
|
+
* after the check only the upload deadline applies. Nothing is sent here.
|
|
306
|
+
*/
|
|
307
|
+
export function uploadChannelFile(
|
|
308
|
+
file: ChannelThreadFile,
|
|
309
|
+
deps: ChannelUploadDeps & { keep: boolean; readSignal?: AbortSignal },
|
|
310
|
+
): ChannelFileUpload {
|
|
311
|
+
const copy = deferred<ThreadFileCopy | null>();
|
|
312
|
+
const uploaded = streamUpload(file, deps, (kept) => copy.resolve(kept));
|
|
313
|
+
// Whatever happened, a copy that has not come by the end never will.
|
|
314
|
+
void uploaded.then(
|
|
315
|
+
() => copy.resolve(null),
|
|
316
|
+
() => copy.resolve(null),
|
|
317
|
+
);
|
|
318
|
+
return { file, copy: copy.promise, uploaded };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function streamUpload(
|
|
322
|
+
file: ChannelThreadFile,
|
|
323
|
+
deps: ChannelUploadDeps & { keep: boolean; readSignal?: AbortSignal },
|
|
324
|
+
onCopy: (copy: ThreadFileCopy | null) => void,
|
|
325
|
+
): Promise<MediaMessage> {
|
|
326
|
+
if (!isGatewayMediaUrl(file.link, deps.origins))
|
|
327
|
+
throw new ChannelFileUnavailableError(
|
|
328
|
+
'its link is not one this channel gateway serves',
|
|
329
|
+
);
|
|
330
|
+
if (
|
|
331
|
+
!Number.isSafeInteger(file.size) ||
|
|
332
|
+
file.size <= 0 ||
|
|
333
|
+
file.size > CHANNEL_MAX_FILE_BYTES
|
|
334
|
+
)
|
|
335
|
+
throw new ChannelFileUnavailableError(
|
|
336
|
+
`its announced size (${file.size} bytes) is outside 1..${CHANNEL_MAX_FILE_BYTES}`,
|
|
337
|
+
);
|
|
338
|
+
const fetchImpl = deps.fetchImpl ?? fetch;
|
|
339
|
+
// One controller stops the gateway read, the pipe and the upload: the
|
|
340
|
+
// read's deadlines until the check passes, the upload's after it.
|
|
341
|
+
const stop = new AbortController();
|
|
342
|
+
const read = boundedSignal(
|
|
343
|
+
deps.readSignal,
|
|
344
|
+
deps.readTimeoutMs ?? DOWNLOAD_TIMEOUT_MS,
|
|
345
|
+
'Reading the channel file',
|
|
346
|
+
);
|
|
347
|
+
let uploadTimer: ReturnType<typeof setTimeout> | undefined;
|
|
348
|
+
let verified = false;
|
|
349
|
+
const onReadAbort = (): void => {
|
|
350
|
+
if (!verified) stop.abort(read.signal.reason);
|
|
351
|
+
};
|
|
352
|
+
read.signal.addEventListener('abort', onReadAbort, { once: true });
|
|
353
|
+
let head: Uint8Array = new Uint8Array(0);
|
|
354
|
+
try {
|
|
355
|
+
let source: Response;
|
|
356
|
+
try {
|
|
357
|
+
source = await fetchImpl(file.link.url, {
|
|
358
|
+
signal: stop.signal,
|
|
359
|
+
redirect: 'manual',
|
|
360
|
+
});
|
|
361
|
+
} catch {
|
|
362
|
+
if (stop.signal.aborted) throw abortError(stop.signal);
|
|
363
|
+
throw new ChannelFileUnavailableError(
|
|
364
|
+
'the channel gateway could not be reached',
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
if (!source.ok || !source.body) {
|
|
368
|
+
await source.body?.cancel().catch(() => undefined);
|
|
369
|
+
throw new ChannelFileUnavailableError(
|
|
370
|
+
`the channel gateway answered ${source.status}`,
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
const contentType = source.headers
|
|
374
|
+
.get('content-type')
|
|
375
|
+
?.split(';')[0]
|
|
376
|
+
?.trim();
|
|
377
|
+
const mimetype =
|
|
378
|
+
contentType && contentType !== 'application/octet-stream'
|
|
379
|
+
? contentType
|
|
380
|
+
: file.mimetype;
|
|
381
|
+
|
|
382
|
+
const check = verifiedPassThrough({
|
|
383
|
+
size: file.size,
|
|
384
|
+
sha256: file.link.sha256,
|
|
385
|
+
keep: deps.keep,
|
|
386
|
+
onVerified: (kept, first) => {
|
|
387
|
+
verified = true;
|
|
388
|
+
head = first;
|
|
389
|
+
read.signal.removeEventListener('abort', onReadAbort);
|
|
390
|
+
read.release();
|
|
391
|
+
const uploadMs =
|
|
392
|
+
deps.uploadTimeoutMs ?? CHANNEL_MEDIA_UPLOAD_TIMEOUT_MS;
|
|
393
|
+
uploadTimer = setTimeout(
|
|
394
|
+
() =>
|
|
395
|
+
stop.abort(
|
|
396
|
+
new Error(
|
|
397
|
+
`Uploading the file to the homeserver timed out after ${Math.round(uploadMs / 1000)} s`,
|
|
398
|
+
),
|
|
399
|
+
),
|
|
400
|
+
uploadMs,
|
|
401
|
+
);
|
|
402
|
+
onCopy(kept ? { bytes: kept, mimetype } : null);
|
|
403
|
+
},
|
|
404
|
+
});
|
|
405
|
+
const encryptor = createAttachmentEncryptor();
|
|
406
|
+
// CTR keeps the length, so the ciphertext is `size` bytes too; the
|
|
407
|
+
// fixed-length stream gives the upload its Content-Length and fails it
|
|
408
|
+
// if the stream ends short or runs long.
|
|
409
|
+
const fixed = new FixedLengthStream(file.size);
|
|
410
|
+
const piped = source.body
|
|
411
|
+
.pipeThrough(check.transform)
|
|
412
|
+
.pipeThrough(encryptor.transform)
|
|
413
|
+
.pipeTo(fixed.writable, { signal: stop.signal })
|
|
414
|
+
.then(
|
|
415
|
+
() => null,
|
|
416
|
+
(error: unknown) => error,
|
|
417
|
+
);
|
|
418
|
+
|
|
419
|
+
let answer: Response;
|
|
420
|
+
try {
|
|
421
|
+
// No `filename` and no type: the homeserver sees only ciphertext.
|
|
422
|
+
answer = await fetchImpl(
|
|
423
|
+
new URL('/_matrix/media/v3/upload', deps.homeserver.baseUrl),
|
|
424
|
+
{
|
|
425
|
+
method: 'POST',
|
|
426
|
+
headers: {
|
|
427
|
+
authorization: `Bearer ${deps.homeserver.accessToken}`,
|
|
428
|
+
'content-type': 'application/octet-stream',
|
|
429
|
+
},
|
|
430
|
+
body: fixed.readable,
|
|
431
|
+
signal: stop.signal,
|
|
432
|
+
},
|
|
433
|
+
);
|
|
434
|
+
} catch (error) {
|
|
435
|
+
const timedOut = stop.signal.aborted;
|
|
436
|
+
stop.abort(error);
|
|
437
|
+
// A refusal of the check fails the upload request (it errors the body).
|
|
438
|
+
const refused = check.failure();
|
|
439
|
+
if (refused) throw refused;
|
|
440
|
+
if (timedOut) throw abortError(stop.signal);
|
|
441
|
+
if ((await piped) && !verified)
|
|
442
|
+
throw new ChannelFileUnavailableError(
|
|
443
|
+
'the channel gateway stopped sending it',
|
|
444
|
+
);
|
|
445
|
+
throw new ChannelMediaUploadError(
|
|
446
|
+
`the homeserver could not be reached: ${errorText(error)}`,
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
if (!answer.ok) {
|
|
450
|
+
if (answer.status === 401)
|
|
451
|
+
deps.onUnauthorized?.(deps.homeserver.accessToken);
|
|
452
|
+
stop.abort(new Error('upload refused'));
|
|
453
|
+
await piped;
|
|
454
|
+
const refused = check.failure();
|
|
455
|
+
if (refused) throw refused;
|
|
456
|
+
const detail = (await answer.text().catch(() => ''))
|
|
457
|
+
.replace(/\s+/g, ' ')
|
|
458
|
+
.slice(0, 200);
|
|
459
|
+
throw new ChannelMediaUploadError(
|
|
460
|
+
`the homeserver refused the upload: ${answer.status}${detail ? ` ${detail}` : ''}`,
|
|
461
|
+
answer.status,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
const json: unknown = await answer.json().catch(() => null);
|
|
465
|
+
const mxc =
|
|
466
|
+
typeof json === 'object' &&
|
|
467
|
+
json !== null &&
|
|
468
|
+
'content_uri' in json &&
|
|
469
|
+
typeof json.content_uri === 'string' &&
|
|
470
|
+
json.content_uri.startsWith('mxc://')
|
|
471
|
+
? json.content_uri
|
|
472
|
+
: null;
|
|
473
|
+
if (!mxc)
|
|
474
|
+
throw new ChannelMediaUploadError(
|
|
475
|
+
'the homeserver answered the upload without a content_uri',
|
|
476
|
+
);
|
|
477
|
+
// The homeserver answered after the last byte: the pipe has ended and
|
|
478
|
+
// the check has passed (it withholds that byte otherwise).
|
|
479
|
+
const pipeError = await untilAborted(piped, stop.signal);
|
|
480
|
+
const refused = check.failure();
|
|
481
|
+
if (refused) throw refused;
|
|
482
|
+
if (pipeError || !verified)
|
|
483
|
+
throw new ChannelFileUnavailableError(
|
|
484
|
+
'the channel gateway stopped sending it',
|
|
485
|
+
);
|
|
486
|
+
const name = roomFilename(file);
|
|
487
|
+
const type = roomMediaType(head, file.mimetype);
|
|
488
|
+
return {
|
|
489
|
+
msgtype: type.msgtype,
|
|
490
|
+
body: name,
|
|
491
|
+
filename: name,
|
|
492
|
+
info: { mimetype: type.mimetype, size: file.size },
|
|
493
|
+
file: { url: mxc, ...(await encryptor.info) },
|
|
494
|
+
};
|
|
495
|
+
} finally {
|
|
496
|
+
read.signal.removeEventListener('abort', onReadAbort);
|
|
497
|
+
read.release();
|
|
498
|
+
clearTimeout(uploadTimer);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* The transaction id of a turn's `index`-th file event: fixed, so a retried
|
|
504
|
+
* send is deduplicated, and — like the channel text mirrors' — a hash, so
|
|
505
|
+
* the request id never reaches the homeserver.
|
|
506
|
+
*/
|
|
507
|
+
export async function channelMediaTxnId(input: {
|
|
508
|
+
userDid: string;
|
|
509
|
+
bindingId: string;
|
|
510
|
+
requestId: string;
|
|
511
|
+
index: number;
|
|
512
|
+
}): Promise<string> {
|
|
513
|
+
return `channel-${await channelRequestHash(
|
|
514
|
+
JSON.stringify([
|
|
515
|
+
input.userDid,
|
|
516
|
+
input.bindingId,
|
|
517
|
+
input.requestId,
|
|
518
|
+
'file',
|
|
519
|
+
input.index,
|
|
520
|
+
]),
|
|
521
|
+
)}`;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Whether the turn keeps `file`'s plaintext: one the model may read
|
|
526
|
+
* directly (an image of at most `MAX_FILE_SIZE`, or a document of at most
|
|
527
|
+
* `NATIVE_DOCUMENT_MAX_BYTES`), in turn order while the copies stay within
|
|
528
|
+
* `keepTotalBytes`. Every other file is read by the extract lane when its
|
|
529
|
+
* turn comes, as without a room post.
|
|
530
|
+
*/
|
|
531
|
+
export function channelFilesToKeep(
|
|
532
|
+
files: readonly ChannelThreadFile[],
|
|
533
|
+
keepTotalBytes = CHANNEL_MEDIA_KEEP_TOTAL_BYTES,
|
|
534
|
+
): Set<string> {
|
|
535
|
+
const keep = new Set<string>();
|
|
536
|
+
let total = 0;
|
|
537
|
+
for (const file of files) {
|
|
538
|
+
const direct =
|
|
539
|
+
(file.kind === 'image' && file.size <= MAX_FILE_SIZE) ||
|
|
540
|
+
(file.kind === 'document' && file.size <= NATIVE_DOCUMENT_MAX_BYTES);
|
|
541
|
+
if (!direct || total + file.size > keepTotalBytes) continue;
|
|
542
|
+
keep.add(file.ref);
|
|
543
|
+
total += file.size;
|
|
544
|
+
}
|
|
545
|
+
return keep;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Start the uploads of a turn's (distinct) files, `CHANNEL_MEDIA_CONCURRENCY`
|
|
550
|
+
* at a time, every read within `turnTimeoutMs` (default
|
|
551
|
+
* `CHANNEL_MEDIA_TURN_TIMEOUT_MS`) of the start and the per-file deadline;
|
|
552
|
+
* the files in `keep` (`channelFilesToKeep`) hand their plaintext to the
|
|
553
|
+
* turn. Returns at once, one entry per file in message order; `done`
|
|
554
|
+
* settles when every upload has. A file that cannot be uploaded is logged
|
|
555
|
+
* once, without its link. A 429 from the homeserver skips the files not yet
|
|
556
|
+
* started.
|
|
557
|
+
*/
|
|
558
|
+
export function startChannelFileUploads(input: {
|
|
559
|
+
files: readonly ChannelThreadFile[];
|
|
560
|
+
sessionId: string;
|
|
561
|
+
deps: ChannelUploadDeps & {
|
|
562
|
+
keep: ReadonlySet<string>;
|
|
563
|
+
turnTimeoutMs?: number;
|
|
564
|
+
concurrency?: number;
|
|
565
|
+
log: Pick<Logger, 'log' | 'warn'>;
|
|
566
|
+
};
|
|
567
|
+
}): { uploads: ChannelFileUpload[]; done: Promise<void> } {
|
|
568
|
+
const { deps, files } = input;
|
|
569
|
+
const keep = deps.keep;
|
|
570
|
+
const slots = files.map((file) => ({
|
|
571
|
+
file,
|
|
572
|
+
copy: deferred<ThreadFileCopy | null>(),
|
|
573
|
+
uploaded: deferred<MediaMessage>(),
|
|
574
|
+
}));
|
|
575
|
+
for (const slot of slots) slot.uploaded.promise.catch(() => undefined);
|
|
576
|
+
const uploads: ChannelFileUpload[] = slots.map((slot) => ({
|
|
577
|
+
file: slot.file,
|
|
578
|
+
copy: slot.copy.promise,
|
|
579
|
+
uploaded: slot.uploaded.promise,
|
|
580
|
+
}));
|
|
581
|
+
if (slots.length === 0) return { uploads, done: Promise.resolve() };
|
|
582
|
+
|
|
583
|
+
const startedAt = Date.now();
|
|
584
|
+
const turn = boundedSignal(
|
|
585
|
+
undefined,
|
|
586
|
+
deps.turnTimeoutMs ?? CHANNEL_MEDIA_TURN_TIMEOUT_MS,
|
|
587
|
+
"Reading the turn's channel files",
|
|
588
|
+
);
|
|
589
|
+
const queue = [...slots];
|
|
590
|
+
let rateLimited = false;
|
|
591
|
+
let posted = 0;
|
|
592
|
+
const worker = async (): Promise<void> => {
|
|
593
|
+
for (let slot = queue.shift(); slot; slot = queue.shift()) {
|
|
594
|
+
const { file } = slot;
|
|
595
|
+
if (rateLimited || turn.signal.aborted) {
|
|
596
|
+
const reason = rateLimited
|
|
597
|
+
? 'the homeserver rate-limited this turn'
|
|
598
|
+
: errorText(abortError(turn.signal));
|
|
599
|
+
slot.copy.resolve(null);
|
|
600
|
+
slot.uploaded.reject(new ChannelMediaUploadError(reason));
|
|
601
|
+
deps.log.warn(
|
|
602
|
+
`[attachments] channel file ${file.ref} (${file.kind}, ${file.size} bytes) not posted to thread ${input.sessionId}: ${reason}`,
|
|
603
|
+
);
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
const upload = uploadChannelFile(file, {
|
|
607
|
+
...deps,
|
|
608
|
+
keep: keep.has(file.ref),
|
|
609
|
+
readSignal: turn.signal,
|
|
610
|
+
});
|
|
611
|
+
slot.copy.resolve(upload.copy);
|
|
612
|
+
try {
|
|
613
|
+
slot.uploaded.resolve(await upload.uploaded);
|
|
614
|
+
posted += 1;
|
|
615
|
+
} catch (error) {
|
|
616
|
+
if (error instanceof ChannelMediaUploadError && error.status === 429)
|
|
617
|
+
rateLimited = true;
|
|
618
|
+
slot.uploaded.reject(error);
|
|
619
|
+
deps.log.warn(
|
|
620
|
+
`[attachments] channel file ${file.ref} (${file.kind}, ${file.size} bytes) not posted to thread ${input.sessionId}: ${errorText(error)}`,
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
};
|
|
625
|
+
const done = Promise.all(
|
|
626
|
+
Array.from(
|
|
627
|
+
{
|
|
628
|
+
length: Math.min(
|
|
629
|
+
deps.concurrency ?? CHANNEL_MEDIA_CONCURRENCY,
|
|
630
|
+
slots.length,
|
|
631
|
+
),
|
|
632
|
+
},
|
|
633
|
+
worker,
|
|
634
|
+
),
|
|
635
|
+
)
|
|
636
|
+
.then(() => {
|
|
637
|
+
deps.log.log(
|
|
638
|
+
`[attachments] uploaded ${posted}/${slots.length} channel file(s) of thread ${input.sessionId} in ${Date.now() - startedAt} ms`,
|
|
639
|
+
);
|
|
640
|
+
})
|
|
641
|
+
.finally(() => turn.release());
|
|
642
|
+
return { uploads, done };
|
|
643
|
+
}
|