@ixo/oracle-runtime-workers 0.29.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -5
- package/src/attachments/channel-links.ts +135 -0
- package/src/attachments/channel.ts +203 -0
- package/src/attachments/download.ts +23 -6
- package/src/attachments/index.ts +23 -0
- package/src/attachments/pipeline.ts +120 -42
- package/src/attachments/retention.ts +11 -3
- package/src/attachments/sanitize.ts +20 -8
- package/src/attachments/types.ts +7 -0
- package/src/attachments/view.ts +24 -7
- package/src/channels/contract.ts +69 -0
- package/src/channels/turns.ts +23 -1
- package/src/core/main-agent-types.ts +2 -0
- package/src/core/runtime-context.ts +3 -0
- package/src/delivery/plan.ts +3 -1
- package/src/do/contracts.ts +3 -0
- package/src/do/turn-attachments.ts +103 -0
- package/src/do/user-oracle-do.ts +142 -42
- package/src/hub-actions/client.ts +87 -7
- package/src/hub-actions/hub-text.ts +82 -0
- package/src/hub-actions/request.ts +7 -4
- package/src/hub-actions/store.ts +5 -1
- package/src/hub-actions/transfer.ts +23 -6
- package/src/index.ts +1 -0
- package/src/plugin-api/index.ts +1 -0
- package/src/plugin-api/types.ts +43 -2
- package/src/plugins/attachments/view-attachment-tool.ts +7 -3
- package/src/plugins/hub-actions/hub-actions.plugin.ts +96 -17
- package/src/plugins/ixo-chain/balances.ts +141 -0
- package/src/plugins/ixo-chain/blocksync.ts +17 -8
- package/src/plugins/ixo-chain/denoms.ts +84 -36
- package/src/plugins/ixo-chain/http.ts +315 -70
- package/src/plugins/ixo-chain/ixo-chain.plugin.ts +182 -301
- package/src/plugins/ixo-chain/kinds/chain/account.ts +516 -0
- package/src/plugins/ixo-chain/kinds/chain/claims.ts +287 -0
- package/src/plugins/ixo-chain/kinds/chain/economics.ts +237 -0
- package/src/plugins/ixo-chain/kinds/chain/governance.ts +455 -0
- package/src/plugins/ixo-chain/kinds/chain/grants.ts +682 -0
- package/src/plugins/ixo-chain/kinds/chain/index.ts +23 -0
- package/src/plugins/ixo-chain/kinds/chain/liquid-staking.ts +279 -0
- package/src/plugins/ixo-chain/kinds/chain/shared.ts +286 -0
- package/src/plugins/ixo-chain/kinds/chain/staking.ts +220 -0
- package/src/plugins/ixo-chain/kinds/coins.ts +112 -0
- package/src/plugins/ixo-chain/kinds/core.ts +246 -0
- package/src/plugins/ixo-chain/kinds/indexed/agent-deposits.ts +124 -0
- package/src/plugins/ixo-chain/kinds/indexed/dao-proposals.ts +379 -0
- package/src/plugins/ixo-chain/kinds/indexed/daos.ts +319 -0
- package/src/plugins/ixo-chain/kinds/indexed/disputes.ts +159 -0
- package/src/plugins/ixo-chain/kinds/indexed/evaluations.ts +172 -0
- package/src/plugins/ixo-chain/kinds/indexed/index.ts +21 -0
- package/src/plugins/ixo-chain/kinds/indexed/liquid-stake-history.ts +160 -0
- package/src/plugins/ixo-chain/kinds/indexed/member-budgets.ts +175 -0
- package/src/plugins/ixo-chain/kinds/indexed/shared.ts +211 -0
- package/src/plugins/ixo-chain/kinds/indexed/token-history.ts +205 -0
- package/src/plugins/ixo-chain/kinds/registry.ts +43 -0
- package/src/plugins/ixo-chain/kinds/types.ts +156 -0
- package/src/plugins/ixo-chain/lookup.ts +470 -0
- package/src/plugins/ixo-chain/overview.ts +1060 -0
- package/src/plugins/ixo-chain/queries.ts +170 -138
- package/src/plugins/ixo-chain/rest.ts +135 -54
- package/src/plugins/ixo-chain/summaries.ts +3 -2
- package/src/plugins/yoma/catalog.ts +780 -0
- package/src/plugins/yoma/clef.ts +98 -0
- package/src/plugins/yoma/drafts.ts +339 -0
- package/src/plugins/yoma/engine.ts +926 -0
- package/src/plugins/yoma/index.ts +13 -0
- package/src/plugins/yoma/judge.ts +358 -0
- package/src/plugins/yoma/permission.ts +62 -0
- package/src/plugins/yoma/reads.ts +195 -0
- package/src/plugins/yoma/submission.ts +534 -0
- package/src/plugins/yoma/text.ts +16 -0
- package/src/plugins/yoma/work.ts +190 -0
- package/src/plugins/yoma/yoma.plugin.ts +2162 -0
- package/src/sqlite/serialization.ts +2 -0
|
@@ -22,7 +22,15 @@
|
|
|
22
22
|
*
|
|
23
23
|
* Both lanes share one `MAX_TOTAL_SIZE` download budget (`DownloadBudget`):
|
|
24
24
|
* declared sizes are checked against what is left before a download, and
|
|
25
|
-
* each download is cut off at what is left.
|
|
25
|
+
* each download is cut off at what is left. A file the budget cannot take
|
|
26
|
+
* is noted; the turn always runs.
|
|
27
|
+
*
|
|
28
|
+
* A channel file (`channelRef`, see `channel.ts`) is private: it is read
|
|
29
|
+
* through the owner object's channel source and always goes to a model as
|
|
30
|
+
* the bytes already loaded, never by URL; the model, the notes and the
|
|
31
|
+
* stored metadata know it only by its `channel:` reference. It may be up to
|
|
32
|
+
* the gateway's 50 MiB: one above `MAX_FILE_SIZE` is archived to the
|
|
33
|
+
* sandbox and noted, never handed to a model.
|
|
26
34
|
*/
|
|
27
35
|
import type { MessageContent } from '@langchain/core/messages';
|
|
28
36
|
import type { ModelInputCapabilities } from '../core/llm';
|
|
@@ -34,6 +42,12 @@ import {
|
|
|
34
42
|
buildUserMessageContent,
|
|
35
43
|
type NativeAttachment,
|
|
36
44
|
} from './content-blocks';
|
|
45
|
+
import {
|
|
46
|
+
CHANNEL_MAX_FILE_BYTES,
|
|
47
|
+
channelArchiveName,
|
|
48
|
+
ChannelFileUnavailableError,
|
|
49
|
+
type ChannelMediaSource,
|
|
50
|
+
} from './channel';
|
|
37
51
|
import {
|
|
38
52
|
ALLOWED_URI_SCHEMES,
|
|
39
53
|
AttachmentRejectedError,
|
|
@@ -59,6 +73,7 @@ import {
|
|
|
59
73
|
} from './sandbox-archive';
|
|
60
74
|
import {
|
|
61
75
|
NOTE_EVENT_ID_MAX,
|
|
76
|
+
NOTE_FILENAME_MAX,
|
|
62
77
|
NOTE_MIMETYPE_MAX,
|
|
63
78
|
NOTE_URI_MAX,
|
|
64
79
|
sanitizeAttachmentFilename,
|
|
@@ -73,6 +88,8 @@ export const SANDBOX_TRUNCATE_LIMIT = 500;
|
|
|
73
88
|
|
|
74
89
|
export interface AttachmentPipelineDeps {
|
|
75
90
|
source: MatrixMediaSource;
|
|
91
|
+
/** Where `channelRef` files are read; absent = none can be read (noted). */
|
|
92
|
+
channel?: ChannelMediaSource;
|
|
76
93
|
/** Helper-model provider for the extract lane; null → media cannot be extracted here. */
|
|
77
94
|
extraction: ExtractionProvider | null;
|
|
78
95
|
/** Sandbox archive target; null → originals are not archived (logged once per turn). */
|
|
@@ -115,21 +132,34 @@ export interface PrepareAttachmentsParams {
|
|
|
115
132
|
caps: ModelInputCapabilities;
|
|
116
133
|
}
|
|
117
134
|
|
|
118
|
-
|
|
135
|
+
/** The metadata stored on the turn's messages (a channel file: its reference, never a link). */
|
|
136
|
+
export function attachmentMetaFor(attachment: AttachmentInput): AttachmentMeta {
|
|
119
137
|
return {
|
|
120
138
|
filename: attachment.filename,
|
|
121
139
|
mimetype: attachment.mimetype,
|
|
122
140
|
...(attachment.size !== undefined ? { size: attachment.size } : {}),
|
|
123
141
|
...(attachment.mxcUri ? { mxcUri: attachment.mxcUri } : {}),
|
|
124
142
|
...(attachment.eventId ? { eventId: attachment.eventId } : {}),
|
|
143
|
+
...(attachment.channelRef ? { channelRef: attachment.channelRef } : {}),
|
|
125
144
|
category: classifyAttachment(attachment),
|
|
126
145
|
};
|
|
127
146
|
}
|
|
128
147
|
|
|
148
|
+
/** A filename as the hidden notes quote it (no controls, brackets or quotes). */
|
|
149
|
+
function noteName(attachment: AttachmentInput): string {
|
|
150
|
+
return sanitizeNoteField(attachment.filename, NOTE_FILENAME_MAX);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** A mimetype as the hidden notes show it. */
|
|
154
|
+
function noteType(attachment: AttachmentInput): string {
|
|
155
|
+
return sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX);
|
|
156
|
+
}
|
|
157
|
+
|
|
129
158
|
/**
|
|
130
159
|
* The line that names an attachment's reference to the model, for both lanes:
|
|
131
|
-
* `[source: eventId="…"]` (the media event id)
|
|
132
|
-
* mxc / http URI), the
|
|
160
|
+
* `[source: eventId="…"]` (the media event id), `[source: ref="channel:…"]`
|
|
161
|
+
* (a channel file) or `[source: url="…"]` (the mxc / http URI), the values
|
|
162
|
+
* `attachmentRef` accepts.
|
|
133
163
|
*/
|
|
134
164
|
function sourceRef(attachment: AttachmentInput): string {
|
|
135
165
|
// A reference the cleaner would alter is not the reference tools match on,
|
|
@@ -142,6 +172,10 @@ function sourceRef(attachment: AttachmentInput): string {
|
|
|
142
172
|
? clean(attachment.eventId, NOTE_EVENT_ID_MAX)
|
|
143
173
|
: '';
|
|
144
174
|
if (eventId) return `[source: eventId="${eventId}"]`;
|
|
175
|
+
const channelRef = attachment.channelRef
|
|
176
|
+
? clean(attachment.channelRef, NOTE_EVENT_ID_MAX)
|
|
177
|
+
: '';
|
|
178
|
+
if (channelRef) return `[source: ref="${channelRef}"]`;
|
|
145
179
|
const url = attachment.mxcUri ? clean(attachment.mxcUri, NOTE_URI_MAX) : '';
|
|
146
180
|
if (url) return `[source: url="${url}"]`;
|
|
147
181
|
return '';
|
|
@@ -151,7 +185,24 @@ function sourceRef(attachment: AttachmentInput): string {
|
|
|
151
185
|
export function nativeAttachmentNote(attachment: AttachmentInput): string {
|
|
152
186
|
const ref = sourceRef(attachment);
|
|
153
187
|
if (!ref) return '';
|
|
154
|
-
return `${ref} "${
|
|
188
|
+
return `${ref} "${noteName(attachment)}" (${noteType(attachment)}) is attached to this message.`;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** The sandbox archive name: a channel file's carries its digest (`photo-<12 hex>.jpg`). */
|
|
192
|
+
function archiveName(attachment: AttachmentInput): string {
|
|
193
|
+
const safeName = sanitizeAttachmentFilename(attachment.filename);
|
|
194
|
+
return attachment.channelRef
|
|
195
|
+
? channelArchiveName(safeName, attachment.channelRef)
|
|
196
|
+
: safeName;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** The largest file of this source the turn downloads. */
|
|
200
|
+
function fileCap(attachment: AttachmentInput): number {
|
|
201
|
+
return attachment.channelRef ? CHANNEL_MAX_FILE_BYTES : MAX_FILE_SIZE;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function megabytes(bytes: number): string {
|
|
205
|
+
return `${Math.round(bytes / 1024 / 1024)} MB`;
|
|
155
206
|
}
|
|
156
207
|
|
|
157
208
|
function errorMessage(error: unknown): string {
|
|
@@ -179,24 +230,30 @@ class DownloadBudget {
|
|
|
179
230
|
params: PrepareAttachmentsParams,
|
|
180
231
|
deps: AttachmentPipelineDeps,
|
|
181
232
|
): Promise<{ bytes: Uint8Array; mimetype: string; sniffed: boolean }> {
|
|
182
|
-
|
|
233
|
+
const perFile = fileCap(attachment);
|
|
234
|
+
if (attachment.size && attachment.size > perFile)
|
|
183
235
|
throw new AttachmentTooLargeError('File exceeds maximum size');
|
|
184
236
|
const remaining = this.remaining;
|
|
185
237
|
if (remaining === 0 || (attachment.size && attachment.size > remaining))
|
|
186
238
|
throw new AttachmentRejectedError(BUDGET_EXCEEDED);
|
|
187
|
-
const cap = Math.min(
|
|
239
|
+
const cap = Math.min(perFile, remaining);
|
|
188
240
|
try {
|
|
189
241
|
const loaded = await loadAttachmentBytes(
|
|
190
242
|
attachment,
|
|
191
243
|
params.roomId,
|
|
192
244
|
deps.source,
|
|
193
|
-
{
|
|
245
|
+
{
|
|
246
|
+
fetchImpl: deps.fetchImpl,
|
|
247
|
+
signal: deps.signal,
|
|
248
|
+
maxBytes: cap,
|
|
249
|
+
...(deps.channel ? { channel: deps.channel } : {}),
|
|
250
|
+
},
|
|
194
251
|
);
|
|
195
252
|
this.used += loaded.bytes.length;
|
|
196
253
|
return loaded;
|
|
197
254
|
} catch (error) {
|
|
198
255
|
// Cut off by the budget rather than by the per-file cap.
|
|
199
|
-
if (error instanceof AttachmentTooLargeError && cap <
|
|
256
|
+
if (error instanceof AttachmentTooLargeError && cap < perFile)
|
|
200
257
|
throw new AttachmentRejectedError(BUDGET_EXCEEDED);
|
|
201
258
|
throw error;
|
|
202
259
|
}
|
|
@@ -215,7 +272,7 @@ async function extractOne(
|
|
|
215
272
|
preloaded?: Uint8Array,
|
|
216
273
|
): Promise<{ text: string; usage?: ExtractionUsage }> {
|
|
217
274
|
const safeName = sanitizeAttachmentFilename(attachment.filename);
|
|
218
|
-
if (!attachment.eventId && !attachment.mxcUri)
|
|
275
|
+
if (!attachment.eventId && !attachment.mxcUri && !attachment.channelRef)
|
|
219
276
|
throw new Error('Either mxcUri or eventId must be provided');
|
|
220
277
|
if (attachment.mxcUri && !ALLOWED_URI_SCHEMES.test(attachment.mxcUri))
|
|
221
278
|
throw new Error('Invalid URI scheme');
|
|
@@ -226,7 +283,7 @@ async function extractOne(
|
|
|
226
283
|
`[attachments] unsupported file type ${attachment.mimetype} for ${attachment.filename}`,
|
|
227
284
|
);
|
|
228
285
|
return {
|
|
229
|
-
text: `[File "${
|
|
286
|
+
text: `[File "${noteName(attachment)}" (${noteType(attachment)}) is not a supported file type and could not be processed]`,
|
|
230
287
|
};
|
|
231
288
|
}
|
|
232
289
|
|
|
@@ -236,14 +293,19 @@ async function extractOne(
|
|
|
236
293
|
deps.logger.warn(`[attachments] ${m}`),
|
|
237
294
|
);
|
|
238
295
|
|
|
239
|
-
// Public http(s) images/videos: let the provider fetch the URL itself.
|
|
296
|
+
// Public http(s) images/videos: let the provider fetch the URL itself. A
|
|
297
|
+
// channel file has no URL here (only its reference): it goes as bytes.
|
|
240
298
|
const isHttpUrl =
|
|
241
299
|
Boolean(attachment.mxcUri) &&
|
|
242
300
|
!attachment.eventId &&
|
|
301
|
+
!attachment.channelRef &&
|
|
243
302
|
/^https?:\/\//i.test(attachment.mxcUri ?? '');
|
|
244
303
|
let text: string | undefined;
|
|
245
304
|
let usage: ExtractionUsage | undefined;
|
|
246
|
-
if (
|
|
305
|
+
if (bytes.length > MAX_FILE_SIZE) {
|
|
306
|
+
// A channel file larger than any model is handed here: kept, not read.
|
|
307
|
+
text = `[File "${noteName(attachment)}" (${noteType(attachment)}, ${megabytes(bytes.length)}) is larger than the ${megabytes(MAX_FILE_SIZE)} a model reads here, so its content was not read]`;
|
|
308
|
+
} else if (
|
|
247
309
|
isHttpUrl &&
|
|
248
310
|
deps.extraction &&
|
|
249
311
|
(category === 'image' || category === 'video')
|
|
@@ -280,23 +342,28 @@ async function extractOne(
|
|
|
280
342
|
if (!deps.sandbox) {
|
|
281
343
|
return { text, ...(usage ? { usage } : {}) };
|
|
282
344
|
}
|
|
283
|
-
const
|
|
345
|
+
const storedName = archiveName(attachment);
|
|
346
|
+
const destPath = `${SANDBOX_OUTPUT_PREFIX}/${storedName}`;
|
|
284
347
|
try {
|
|
285
|
-
|
|
348
|
+
// The path the sandbox reports is where the file is (it may differ).
|
|
349
|
+
const archived = await uploadToSandbox(
|
|
286
350
|
bytes,
|
|
287
|
-
|
|
351
|
+
storedName,
|
|
288
352
|
destPath,
|
|
289
353
|
deps.sandbox,
|
|
290
354
|
attachment.mimetype,
|
|
291
355
|
deps.fetchImpl,
|
|
292
356
|
);
|
|
293
|
-
const actualPath = sanitizeSandboxPath(
|
|
357
|
+
const actualPath = sanitizeSandboxPath(archived.path);
|
|
294
358
|
let analysisPath: string | undefined;
|
|
295
|
-
if (
|
|
296
|
-
|
|
359
|
+
if (
|
|
360
|
+
bytes.length <= MAX_FILE_SIZE &&
|
|
361
|
+
(category === 'image' || category === 'video' || category === 'audio')
|
|
362
|
+
) {
|
|
363
|
+
const analysisFilename = `${storedName.replace(/\.[^.]+$/, '')}-analysis.md`;
|
|
297
364
|
const analysisDest = `${SANDBOX_OUTPUT_PREFIX}/${analysisFilename}`;
|
|
298
365
|
try {
|
|
299
|
-
await uploadToSandbox(
|
|
366
|
+
const analysis = await uploadToSandbox(
|
|
300
367
|
new TextEncoder().encode(
|
|
301
368
|
buildAnalysisMarkdown(
|
|
302
369
|
safeName,
|
|
@@ -312,7 +379,7 @@ async function extractOne(
|
|
|
312
379
|
'text/markdown',
|
|
313
380
|
deps.fetchImpl,
|
|
314
381
|
);
|
|
315
|
-
analysisPath = sanitizeSandboxPath(
|
|
382
|
+
analysisPath = sanitizeSandboxPath(analysis.path);
|
|
316
383
|
} catch (error) {
|
|
317
384
|
deps.logger.warn(
|
|
318
385
|
`[attachments] analysis .md upload failed for "${attachment.filename}": ${errorMessage(error)}`,
|
|
@@ -350,7 +417,7 @@ export async function prepareAttachments(
|
|
|
350
417
|
params: PrepareAttachmentsParams,
|
|
351
418
|
deps: AttachmentPipelineDeps,
|
|
352
419
|
): Promise<PreparedAttachments> {
|
|
353
|
-
const metas = params.attachments.map(
|
|
420
|
+
const metas = params.attachments.map((a) => attachmentMetaFor(a));
|
|
354
421
|
const usage = { cost: 0, promptTokens: 0, completionTokens: 0 };
|
|
355
422
|
if (params.attachments.length === 0) {
|
|
356
423
|
return {
|
|
@@ -362,16 +429,9 @@ export async function prepareAttachments(
|
|
|
362
429
|
};
|
|
363
430
|
}
|
|
364
431
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
);
|
|
369
|
-
if (reportedTotal > MAX_TOTAL_SIZE) {
|
|
370
|
-
throw new Error(
|
|
371
|
-
`Total attachment size (${Math.round(reportedTotal / 1024 / 1024)} MB) exceeds budget (${Math.round(MAX_TOTAL_SIZE / 1024 / 1024)} MB)`,
|
|
372
|
-
);
|
|
373
|
-
}
|
|
374
|
-
|
|
432
|
+
// Declared sizes over the budget are not refused here as a whole: each
|
|
433
|
+
// file is checked against what is left (`DownloadBudget`) and one that
|
|
434
|
+
// does not fit is noted, so the turn always runs.
|
|
375
435
|
const nativeCandidates: AttachmentInput[] = [];
|
|
376
436
|
// A file the native lane already refused (empty, over budget) carries the
|
|
377
437
|
// reason and is noted; one it downloaded but will not send carries its
|
|
@@ -387,7 +447,12 @@ export async function prepareAttachments(
|
|
|
387
447
|
deps.logger.log(
|
|
388
448
|
`[attachments] "${attachment.filename}" (${attachment.mimetype}) kind=${kind} model=${params.model} → ${strategy}`,
|
|
389
449
|
);
|
|
390
|
-
|
|
450
|
+
// A file larger than a model is handed is never sent as a block.
|
|
451
|
+
if (
|
|
452
|
+
strategy === 'send-native' &&
|
|
453
|
+
!(attachment.size && attachment.size > MAX_FILE_SIZE)
|
|
454
|
+
)
|
|
455
|
+
nativeCandidates.push(attachment);
|
|
391
456
|
else extractQueue.push({ attachment });
|
|
392
457
|
}
|
|
393
458
|
|
|
@@ -409,6 +474,10 @@ export async function prepareAttachments(
|
|
|
409
474
|
mimetype,
|
|
410
475
|
filename: sniffed ? '' : attachment.filename,
|
|
411
476
|
});
|
|
477
|
+
if (bytes.length > MAX_FILE_SIZE) {
|
|
478
|
+
extractQueue.push({ attachment, preloaded: bytes });
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
412
481
|
if (kind !== classifyAttachment(attachment)) {
|
|
413
482
|
deps.logger.warn(
|
|
414
483
|
`[attachments] "${attachment.filename}" claims ${attachment.mimetype} but is ${mimetype} — not sent natively`,
|
|
@@ -420,20 +489,22 @@ export async function prepareAttachments(
|
|
|
420
489
|
kind: kind === 'image' ? 'image' : 'file',
|
|
421
490
|
mimeType: mimetype,
|
|
422
491
|
base64: bytesToBase64(bytes),
|
|
423
|
-
filename: attachment.filename,
|
|
492
|
+
filename: sanitizeAttachmentFilename(attachment.filename),
|
|
424
493
|
});
|
|
425
494
|
const note = nativeAttachmentNote(attachment);
|
|
426
|
-
if (note)
|
|
495
|
+
if (note)
|
|
496
|
+
nativeNotes.push({ text: note, meta: attachmentMetaFor(attachment) });
|
|
427
497
|
deps.logger.log(
|
|
428
498
|
`[attachments] NATIVE → "${attachment.filename}" sent directly to ${params.model} (${bytes.length} bytes, ${kind})`,
|
|
429
499
|
);
|
|
430
500
|
if (deps.sandbox) {
|
|
431
|
-
const
|
|
432
|
-
// Off the hot path — the agent's file tools can reach the original
|
|
501
|
+
const storedName = archiveName(attachment);
|
|
502
|
+
// Off the hot path — the agent's file tools can reach the original
|
|
503
|
+
// later. The upload holds the bytes until it ends; nothing else does.
|
|
433
504
|
void uploadToSandbox(
|
|
434
505
|
bytes,
|
|
435
|
-
|
|
436
|
-
`${SANDBOX_OUTPUT_PREFIX}/${
|
|
506
|
+
storedName,
|
|
507
|
+
`${SANDBOX_OUTPUT_PREFIX}/${storedName}`,
|
|
437
508
|
deps.sandbox,
|
|
438
509
|
mimetype,
|
|
439
510
|
deps.fetchImpl,
|
|
@@ -468,8 +539,11 @@ export async function prepareAttachments(
|
|
|
468
539
|
'[attachments] sandbox archive unavailable for this turn — originals are not archived',
|
|
469
540
|
);
|
|
470
541
|
}
|
|
471
|
-
for (const
|
|
472
|
-
const
|
|
542
|
+
for (const entry of extractQueue) {
|
|
543
|
+
const { attachment, refused, preloaded } = entry;
|
|
544
|
+
// The queue lets go of a file's bytes as soon as its turn comes.
|
|
545
|
+
delete entry.preloaded;
|
|
546
|
+
const meta = attachmentMetaFor(attachment);
|
|
473
547
|
let text: string;
|
|
474
548
|
try {
|
|
475
549
|
if (refused) throw refused;
|
|
@@ -491,7 +565,11 @@ export async function prepareAttachments(
|
|
|
491
565
|
deps.logger.error(
|
|
492
566
|
`[attachments] failed to process ${attachment.filename}: ${errorMessage(error)}`,
|
|
493
567
|
);
|
|
494
|
-
|
|
568
|
+
// A channel file's error never names its link (`channel.ts`).
|
|
569
|
+
text =
|
|
570
|
+
error instanceof ChannelFileUnavailableError
|
|
571
|
+
? `[File "${noteName(attachment)}" (${noteType(attachment)}) could not be downloaded: ${errorMessage(error)}]`
|
|
572
|
+
: `[File "${noteName(attachment)}" (${noteType(attachment)}) failed to process: ${errorMessage(error)}]`;
|
|
495
573
|
}
|
|
496
574
|
const ref = sourceRef(attachment);
|
|
497
575
|
extracted.push({ text: ref ? `${ref}\n${text}` : text, meta });
|
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { HumanMessage, type BaseMessage } from '@langchain/core/messages';
|
|
18
18
|
import type { AttachmentMeta } from '../sqlite/serialization';
|
|
19
|
+
import {
|
|
20
|
+
NOTE_FILENAME_MAX,
|
|
21
|
+
NOTE_MIMETYPE_MAX,
|
|
22
|
+
sanitizeNoteField,
|
|
23
|
+
} from './sanitize';
|
|
19
24
|
|
|
20
25
|
/** Newest user turns whose inline attachment payloads stay in the messages. */
|
|
21
26
|
export const ATTACHMENT_PAYLOAD_TURNS = 2;
|
|
@@ -90,9 +95,12 @@ export function isAttachmentViewMessage(message: BaseMessage): boolean {
|
|
|
90
95
|
);
|
|
91
96
|
}
|
|
92
97
|
|
|
93
|
-
/**
|
|
98
|
+
/**
|
|
99
|
+
* The reference `view_attachment` takes: the media event id, else a channel
|
|
100
|
+
* file's opaque reference, else the URI.
|
|
101
|
+
*/
|
|
94
102
|
export function attachmentRef(meta: AttachmentMeta): string | undefined {
|
|
95
|
-
return meta.eventId ?? meta.mxcUri;
|
|
103
|
+
return meta.eventId ?? meta.channelRef ?? meta.mxcUri;
|
|
96
104
|
}
|
|
97
105
|
|
|
98
106
|
/** The attachment metadata a message carries (`attachments`, else `attachment`). */
|
|
@@ -115,7 +123,7 @@ function formatSize(size: number | undefined): string {
|
|
|
115
123
|
export function buildPlaceholderText(metas: AttachmentMeta[]): string {
|
|
116
124
|
const lines = metas.map((meta, i) => {
|
|
117
125
|
const ref = attachmentRef(meta);
|
|
118
|
-
return `${i + 1}. "${meta.filename}" (${meta.mimetype}${formatSize(meta.size)})${
|
|
126
|
+
return `${i + 1}. "${sanitizeNoteField(meta.filename, NOTE_FILENAME_MAX)}" (${sanitizeNoteField(meta.mimetype, NOTE_MIMETYPE_MAX)}${formatSize(meta.size)})${
|
|
119
127
|
ref ? ` — ref: ${ref}` : ' — no reference; it cannot be fetched again'
|
|
120
128
|
}`;
|
|
121
129
|
});
|
|
@@ -6,33 +6,45 @@
|
|
|
6
6
|
/** Where every attachment a user sends is archived in their sandbox. */
|
|
7
7
|
export const SANDBOX_OUTPUT_PREFIX = '/workspace/output';
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* `value` in at most `max` UTF-16 units, cut before a surrogate pair rather
|
|
11
|
+
* than through it (a multi-byte filename never ends in half a character).
|
|
12
|
+
*/
|
|
13
|
+
function cutUnits(value: string, max: number): string {
|
|
14
|
+
if (value.length <= max) return value;
|
|
15
|
+
const end = /[\ud800-\udbff]/.test(value[max - 1] ?? '') ? max - 1 : max;
|
|
16
|
+
return value.slice(0, end);
|
|
17
|
+
}
|
|
18
|
+
|
|
9
19
|
/**
|
|
10
20
|
* Strip control characters and bracket sequences from a filename to prevent
|
|
11
21
|
* prompt injection when interpolated into LLM context. The archive writes
|
|
12
22
|
* under this exact name.
|
|
13
23
|
*/
|
|
14
24
|
export function sanitizeAttachmentFilename(filename: string): string {
|
|
15
|
-
return (
|
|
25
|
+
return cutUnits(
|
|
16
26
|
filename
|
|
17
27
|
// eslint-disable-next-line no-control-regex
|
|
18
28
|
.replace(/[\x00-\x1f\x7f]/g, '')
|
|
19
|
-
.replace(/[[\]]/g, '')
|
|
20
|
-
|
|
29
|
+
.replace(/[[\]]/g, ''),
|
|
30
|
+
255,
|
|
21
31
|
);
|
|
22
32
|
}
|
|
23
33
|
|
|
24
|
-
/** Longest mimetype / event id / mxc URI written into a hidden note. */
|
|
34
|
+
/** Longest filename / mimetype / event id / mxc URI written into a hidden note. */
|
|
35
|
+
export const NOTE_FILENAME_MAX = 255;
|
|
25
36
|
export const NOTE_MIMETYPE_MAX = 100;
|
|
26
37
|
export const NOTE_EVENT_ID_MAX = 255;
|
|
27
38
|
export const NOTE_URI_MAX = 512;
|
|
28
39
|
|
|
29
40
|
/**
|
|
30
|
-
* Clean a client-supplied value (mimetype, media event id, mxc
|
|
31
|
-
* hidden notes the model reads: the filename cleaner plus
|
|
32
|
-
* (the notes wrap refs in `"…"`), cut to `max`
|
|
41
|
+
* Clean a client-supplied value (filename, mimetype, media event id, mxc
|
|
42
|
+
* URI) for the hidden notes the model reads: the filename cleaner plus
|
|
43
|
+
* quotes removed (the notes wrap names and refs in `"…"`), cut to `max`
|
|
44
|
+
* UTF-16 units without splitting a surrogate pair.
|
|
33
45
|
*/
|
|
34
46
|
export function sanitizeNoteField(value: string, max: number): string {
|
|
35
|
-
return sanitizeAttachmentFilename(value).replace(/"/g, '')
|
|
47
|
+
return cutUnits(sanitizeAttachmentFilename(value).replace(/"/g, ''), max);
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
/**
|
package/src/attachments/types.ts
CHANGED
|
@@ -11,6 +11,13 @@ export interface AttachmentInput {
|
|
|
11
11
|
mxcUri?: string;
|
|
12
12
|
/** Matrix event id (`$…`) — encrypted media is fetched and decrypted via the event. */
|
|
13
13
|
eventId?: string;
|
|
14
|
+
/**
|
|
15
|
+
* A file the person sent on a channel (`channel:<16 hex>`, see
|
|
16
|
+
* `channel.ts`): private, read only through the owner object's channel
|
|
17
|
+
* source, never by URL. Set by the runtime for channel turns, never taken
|
|
18
|
+
* from a request body.
|
|
19
|
+
*/
|
|
20
|
+
channelRef?: string;
|
|
14
21
|
filename: string;
|
|
15
22
|
mimetype: string;
|
|
16
23
|
size?: number;
|
package/src/attachments/view.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* are cached per session (`deps.cache`); native views are not (they are the
|
|
8
8
|
* bytes themselves).
|
|
9
9
|
*/
|
|
10
|
+
import type { ChannelMediaSource } from './channel';
|
|
10
11
|
import { classifyAttachment } from './classify';
|
|
11
12
|
import type { NativeAttachment } from './content-blocks';
|
|
12
13
|
import { loadAttachmentBytes, type MatrixMediaSource } from './download';
|
|
@@ -20,7 +21,11 @@ import { categorizeFile, isPlainTextType, verifyMagicBytes } from './magic';
|
|
|
20
21
|
import { extractPdfText } from './pdf-text';
|
|
21
22
|
import { attachmentMetas, attachmentRef } from './retention';
|
|
22
23
|
import { routeAttachment } from './route';
|
|
23
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
NOTE_FILENAME_MAX,
|
|
26
|
+
NOTE_MIMETYPE_MAX,
|
|
27
|
+
sanitizeNoteField,
|
|
28
|
+
} from './sanitize';
|
|
24
29
|
import {
|
|
25
30
|
LOCAL_TEXT_EXTRACTION,
|
|
26
31
|
type AttachmentTextCache,
|
|
@@ -39,6 +44,8 @@ export type AttachmentView =
|
|
|
39
44
|
|
|
40
45
|
export interface ViewAttachmentDeps {
|
|
41
46
|
source: MatrixMediaSource;
|
|
47
|
+
/** Where `channel:` references are read (the owner object's links). */
|
|
48
|
+
channel?: ChannelMediaSource;
|
|
42
49
|
extraction: ExtractionProvider | null;
|
|
43
50
|
/** Capabilities of the model that will read the result. */
|
|
44
51
|
caps: ModelInputCapabilities;
|
|
@@ -103,6 +110,16 @@ function toInput(meta: AttachmentMeta): AttachmentInput {
|
|
|
103
110
|
...(meta.size !== undefined ? { size: meta.size } : {}),
|
|
104
111
|
...(meta.mxcUri ? { mxcUri: meta.mxcUri } : {}),
|
|
105
112
|
...(meta.eventId ? { eventId: meta.eventId } : {}),
|
|
113
|
+
...(meta.channelRef ? { channelRef: meta.channelRef } : {}),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** How `deps` reads bytes: the turn's signal and fetch, and its channel source. */
|
|
118
|
+
function downloadOptions(deps: ViewAttachmentDeps) {
|
|
119
|
+
return {
|
|
120
|
+
fetchImpl: deps.fetchImpl,
|
|
121
|
+
signal: deps.signal,
|
|
122
|
+
...(deps.channel ? { channel: deps.channel } : {}),
|
|
106
123
|
};
|
|
107
124
|
}
|
|
108
125
|
|
|
@@ -118,7 +135,7 @@ export async function viewAttachment(
|
|
|
118
135
|
attachment,
|
|
119
136
|
deps.roomId,
|
|
120
137
|
deps.source,
|
|
121
|
-
|
|
138
|
+
downloadOptions(deps),
|
|
122
139
|
);
|
|
123
140
|
deps.logger.log(
|
|
124
141
|
`[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
|
|
@@ -147,7 +164,7 @@ export async function viewAttachment(
|
|
|
147
164
|
if (category === 'unsupported') {
|
|
148
165
|
return {
|
|
149
166
|
kind: 'text',
|
|
150
|
-
text: `[File "${
|
|
167
|
+
text: `[File "${sanitizeNoteField(attachment.filename, NOTE_FILENAME_MAX)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) is not a supported file type and could not be processed]`,
|
|
151
168
|
};
|
|
152
169
|
}
|
|
153
170
|
// Text extracted earlier in this session by the same model is final.
|
|
@@ -170,7 +187,7 @@ export async function viewAttachment(
|
|
|
170
187
|
attachment,
|
|
171
188
|
deps.roomId,
|
|
172
189
|
deps.source,
|
|
173
|
-
|
|
190
|
+
downloadOptions(deps),
|
|
174
191
|
);
|
|
175
192
|
deps.logger.log(
|
|
176
193
|
`[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
|
|
@@ -202,7 +219,7 @@ function megabytes(bytes: number): string {
|
|
|
202
219
|
/** The refusal for a scanned or binary file over the helper-model limit. */
|
|
203
220
|
function modelLaneTooLarge(filename: string, bytes: number): Error {
|
|
204
221
|
return new Error(
|
|
205
|
-
`"${
|
|
222
|
+
`"${sanitizeNoteField(filename, NOTE_FILENAME_MAX)}" cannot be read: scanned or binary files up to ${megabytes(MAX_MODEL_LANE_BYTES)} MB can be processed; this one is ${megabytes(bytes)} MB. PDFs with a text layer and text files can be larger.`,
|
|
206
223
|
);
|
|
207
224
|
}
|
|
208
225
|
|
|
@@ -223,7 +240,7 @@ export async function extractAttachmentText(
|
|
|
223
240
|
const category = categorizeFile(attachment.mimetype);
|
|
224
241
|
if (category === 'unsupported') {
|
|
225
242
|
throw new Error(
|
|
226
|
-
`"${
|
|
243
|
+
`"${sanitizeNoteField(attachment.filename, NOTE_FILENAME_MAX)}" (${sanitizeNoteField(attachment.mimetype, NOTE_MIMETYPE_MAX)}) is not a supported file type`,
|
|
227
244
|
);
|
|
228
245
|
}
|
|
229
246
|
const plain = category === 'document' && isPlainTextType(attachment.mimetype);
|
|
@@ -306,7 +323,7 @@ export async function extractAttachmentText(
|
|
|
306
323
|
attachment,
|
|
307
324
|
deps.roomId,
|
|
308
325
|
deps.source,
|
|
309
|
-
|
|
326
|
+
downloadOptions(deps),
|
|
310
327
|
);
|
|
311
328
|
verifyMagicBytes(bytes, category, attachment, (m) =>
|
|
312
329
|
deps.logger.warn(`[attachments] ${m}`),
|
package/src/channels/contract.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { ActionOutcomeWire } from '@ixo/ixo-transaction/topic-proposals';
|
|
2
2
|
import type { ActionOutcomeV1 } from '@ixo/action-protocol';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
+
import {
|
|
5
|
+
CHANNEL_MAX_FILE_BYTES,
|
|
6
|
+
channelAttachmentRef,
|
|
7
|
+
} from '../attachments/channel';
|
|
8
|
+
import type { AttachmentInput } from '../attachments/types';
|
|
4
9
|
import type { ReplyPlan } from '../delivery/types';
|
|
5
10
|
|
|
6
11
|
/**
|
|
@@ -109,6 +114,64 @@ export type ChannelControlOutcome =
|
|
|
109
114
|
| { ok: true; result: ChannelControlResult }
|
|
110
115
|
| Exclude<ChannelTurnOutcome, { ok: true }>;
|
|
111
116
|
|
|
117
|
+
/** The longest filename a channel file keeps, in code points (the gateway's own bound). */
|
|
118
|
+
export const CHANNEL_FILENAME_MAX = 200;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* A filename as the runtime keeps it: at most `CHANNEL_FILENAME_MAX` code
|
|
122
|
+
* points (a longer one is cut, never refused), and the file's kind when the
|
|
123
|
+
* name is empty, so a name alone never refuses a turn.
|
|
124
|
+
*/
|
|
125
|
+
export function channelFilename(filename: string, kind: string): string {
|
|
126
|
+
const clipped = Array.from(filename).slice(0, CHANNEL_FILENAME_MAX).join('');
|
|
127
|
+
return clipped || (kind === 'image' ? 'photo' : kind);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A file the person sent on the channel (a WhatsApp photo, video, document
|
|
132
|
+
* or audio file). The gateway downloads it from the provider and serves it
|
|
133
|
+
* from a capability link of its own; the bytes never ride in the body.
|
|
134
|
+
* `size` is bounded by the gateway's own per-file limit (50 MiB).
|
|
135
|
+
*/
|
|
136
|
+
export const ChannelAttachment = z
|
|
137
|
+
.strictObject({
|
|
138
|
+
url: z
|
|
139
|
+
.string()
|
|
140
|
+
.max(2000)
|
|
141
|
+
.url()
|
|
142
|
+
.refine((value) => value.startsWith('https://'), 'must be an https URL'),
|
|
143
|
+
filename: z.string(),
|
|
144
|
+
mimetype: z.string().min(1).max(100),
|
|
145
|
+
size: z.number().int().nonnegative().max(CHANNEL_MAX_FILE_BYTES),
|
|
146
|
+
sha256: z.string().regex(/^[0-9a-f]{64}$/),
|
|
147
|
+
kind: z.enum(['image', 'video', 'document', 'audio']),
|
|
148
|
+
})
|
|
149
|
+
.transform((attachment) => ({
|
|
150
|
+
...attachment,
|
|
151
|
+
filename: channelFilename(attachment.filename, attachment.kind),
|
|
152
|
+
}));
|
|
153
|
+
export type ChannelAttachment = z.infer<typeof ChannelAttachment>;
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* The run's attachment inputs for channel files: each names its file by its
|
|
157
|
+
* opaque `channel:` reference, never by its link, so the model, the hidden
|
|
158
|
+
* notes and the stored metadata never see a link. The owner object reads
|
|
159
|
+
* the bytes through its channel source (`attachments/channel.ts`).
|
|
160
|
+
*/
|
|
161
|
+
export function channelAttachmentInputs(
|
|
162
|
+
attachments: readonly ChannelAttachment[],
|
|
163
|
+
): AttachmentInput[] {
|
|
164
|
+
return attachments.map((attachment) => ({
|
|
165
|
+
channelRef: channelAttachmentRef(attachment.sha256),
|
|
166
|
+
filename: attachment.filename,
|
|
167
|
+
mimetype: attachment.mimetype,
|
|
168
|
+
size: attachment.size,
|
|
169
|
+
}));
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Same cap as the attachment pipeline (`MAX_ATTACHMENTS`). */
|
|
173
|
+
export const MAX_CHANNEL_ATTACHMENTS = 10;
|
|
174
|
+
|
|
112
175
|
export const ChannelTurnBody = z.strictObject({
|
|
113
176
|
provider: ChannelProviderId,
|
|
114
177
|
bindingId: z.string().regex(/^chb_[A-Za-z0-9_-]{1,120}$/),
|
|
@@ -119,6 +182,12 @@ export const ChannelTurnBody = z.strictObject({
|
|
|
119
182
|
/** An explicit correction stays in its current session, even with routing enabled. */
|
|
120
183
|
sessionPolicy: z.literal('continue').optional(),
|
|
121
184
|
message: z.string().min(1).max(16_000),
|
|
185
|
+
/** Files sent with the message; `message` carries the caption or a placeholder. */
|
|
186
|
+
attachments: z
|
|
187
|
+
.array(ChannelAttachment)
|
|
188
|
+
.min(1)
|
|
189
|
+
.max(MAX_CHANNEL_ATTACHMENTS)
|
|
190
|
+
.optional(),
|
|
122
191
|
replyTo: ChannelReplyReference.optional(),
|
|
123
192
|
/**
|
|
124
193
|
* Which kind of oracle the gateway believes it addresses. `companion` is
|