@ixo/oracle-runtime-workers 0.37.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/attachments/channel-links.ts +74 -7
- package/src/attachments/channel.ts +15 -4
- package/src/attachments/content-blocks.ts +19 -16
- package/src/attachments/download.ts +173 -50
- package/src/attachments/index.ts +6 -5
- package/src/attachments/pipeline.ts +117 -49
- package/src/attachments/resize.ts +142 -0
- package/src/attachments/retention.ts +26 -70
- package/src/attachments/room-media.ts +643 -0
- package/src/attachments/route.ts +47 -4
- package/src/attachments/sandbox-archive.ts +15 -4
- package/src/attachments/sanitize.ts +24 -13
- package/src/attachments/view.ts +82 -38
- package/src/channels/replies.ts +8 -4
- package/src/core/capability-expiry.ts +62 -0
- package/src/core/capability-router.ts +126 -95
- package/src/core/context-budget.ts +116 -4
- package/src/core/domain-context/document.ts +74 -2
- package/src/core/domain-context/index.ts +1 -0
- package/src/core/domain-context/resolver.ts +9 -6
- package/src/core/domain-context/turn-context.ts +62 -13
- package/src/core/domain-context/types.ts +17 -0
- package/src/core/env.ts +21 -2
- package/src/core/index.ts +4 -0
- package/src/core/llm.ts +29 -0
- package/src/core/main-agent-types.ts +53 -4
- package/src/core/main-agent.ts +228 -125
- package/src/core/manifest.ts +109 -24
- package/src/core/mcp-result.ts +56 -0
- package/src/core/meta-tools.ts +258 -110
- package/src/core/middlewares/attachment-recall.ts +502 -0
- package/src/core/middlewares/capability-gate.ts +77 -28
- package/src/core/middlewares/context-guard.ts +154 -21
- package/src/core/middlewares/index.ts +5 -0
- package/src/core/middlewares/result-cap.ts +1 -1
- package/src/core/middlewares/summarization.ts +9 -0
- package/src/core/middlewares/tool-marks.ts +11 -2
- package/src/core/middlewares/turn-boundary.ts +22 -12
- package/src/core/operating-guides.ts +126 -0
- package/src/core/plugins/skills/skills-tools.ts +6 -31
- package/src/core/plugins/skills/skills.plugin.ts +4 -3
- package/src/core/plugins/weather/weather.plugin.ts +1 -0
- package/src/core/prompt-composer.ts +188 -111
- package/src/core/read-result-tool.ts +48 -14
- package/src/core/registries.ts +13 -5
- package/src/core/state.ts +4 -3
- package/src/core/sub-agent-fallback.ts +1 -0
- package/src/core/subagent-as-tool.ts +4 -1
- package/src/core/tool-access.ts +3 -32
- package/src/core/wrap-plugin-tool.ts +2 -1
- package/src/do/contracts.ts +64 -24
- package/src/do/room-mirror.ts +144 -20
- package/src/do/transcript.ts +41 -12
- package/src/do/turn-attachments.ts +45 -6
- package/src/do/turn-metadata.ts +16 -4
- package/src/do/user-oracle-do.ts +639 -152
- package/src/index.ts +1 -0
- package/src/llm/provider-error.ts +5 -2
- package/src/llm/stream-liveness.ts +3 -2
- package/src/matrix/gateway-do.ts +164 -102
- package/src/matrix/ingest.ts +12 -3
- package/src/matrix/media-deadline.ts +5 -47
- package/src/memory/session-history-indexer.ts +60 -14
- package/src/plugin-api/oracle-plugin.ts +4 -5
- package/src/plugin-api/types.ts +32 -3
- package/src/plugins/action-proposals/action-proposals.plugin.ts +7 -2
- package/src/plugins/agui/agui.plugin.ts +1 -0
- package/src/plugins/attachments/attachments.plugin.ts +3 -3
- package/src/plugins/attachments/view-attachment-tool.ts +6 -4
- package/src/plugins/compact-schema.ts +80 -0
- package/src/plugins/composio/composio.plugin.ts +1 -0
- package/src/plugins/domain-indexer/domain-indexer.plugin.ts +4 -3
- package/src/plugins/editor/editor.plugin.ts +4 -0
- package/src/plugins/firecrawl/firecrawl-agent.ts +2 -0
- package/src/plugins/firecrawl/firecrawl.plugin.ts +1 -0
- package/src/plugins/flows/flows.plugin.ts +37 -6
- package/src/plugins/flows/prompts.ts +1 -1
- package/src/plugins/flows/tools/authoring.ts +5 -7
- package/src/plugins/flows/tools/discovery.ts +1 -4
- package/src/plugins/flows/tools/forms.ts +2 -8
- package/src/plugins/flows/tools/linkage.ts +1 -4
- package/src/plugins/flows/tools/settings.ts +1 -4
- package/src/plugins/flows/translator.ts +16 -5
- package/src/plugins/flows/types.ts +27 -48
- package/src/plugins/hub-actions/hub-actions.plugin.ts +30 -4
- package/src/plugins/index.ts +2 -2
- package/src/plugins/ixo-chain/ixo-chain.plugin.ts +29 -5
- package/src/plugins/ixo-chain/kinds/core.ts +2 -2
- package/src/plugins/ixo-chain/kinds/types.ts +34 -5
- package/src/plugins/ixo-chain/queries.ts +10 -1
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +1 -0
- package/src/plugins/matrix-group-chats/index.ts +1 -0
- package/src/plugins/memory/memory-tools.ts +4 -4
- package/src/plugins/memory/memory.plugin.ts +27 -1
- package/src/plugins/memory/oracle-document-tools.ts +24 -49
- package/src/plugins/pod-creator/pod-creator.plugin.ts +1 -0
- package/src/plugins/portal/portal.plugin.ts +1 -0
- package/src/plugins/sandbox/sandbox-write-blob.ts +3 -15
- package/src/plugins/sandbox/sandbox.plugin.ts +19 -2
- package/src/plugins/tasks/manifest.ts +1 -0
- package/src/plugins/tasks/tasks-tools.ts +14 -19
- package/src/plugins/user-preferences/user-preferences-tool.ts +11 -37
- package/src/plugins/user-preferences/user-preferences.plugin.ts +4 -0
- package/src/plugins/vfs/vfs-tools.ts +6 -6
- package/src/plugins/vfs/vfs.plugin.ts +4 -5
- package/src/plugins/yoma/yoma.plugin.ts +1 -0
- package/src/sqlite/inline-payloads.ts +121 -0
- package/src/sqlite/serialization.ts +12 -0
- package/src/sqlite/sqlite-saver.ts +258 -42
- package/src/tasks/schedule-input.ts +5 -22
package/src/index.ts
CHANGED
|
@@ -268,10 +268,13 @@ export function markPlatformFallbackFailure(error: unknown): void {
|
|
|
268
268
|
if (error && typeof error === 'object') PLATFORM_FALLBACK_FAILURES.add(error);
|
|
269
269
|
}
|
|
270
270
|
|
|
271
|
-
/**
|
|
271
|
+
/**
|
|
272
|
+
* The error, or one it wraps (`cause`, as LangChain's MiddlewareError does,
|
|
273
|
+
* once per middleware layer).
|
|
274
|
+
*/
|
|
272
275
|
function isPlatformFallbackFailure(error: unknown): boolean {
|
|
273
276
|
let current: unknown = error;
|
|
274
|
-
for (let depth = 0; depth <
|
|
277
|
+
for (let depth = 0; depth < 32; depth += 1) {
|
|
275
278
|
if (!current || typeof current !== 'object') return false;
|
|
276
279
|
if (PLATFORM_FALLBACK_FAILURES.has(current)) return true;
|
|
277
280
|
current = 'cause' in current ? current.cause : undefined;
|
|
@@ -83,11 +83,12 @@ export class ProviderStallError extends Error {
|
|
|
83
83
|
/**
|
|
84
84
|
* The `ProviderStallError` in `error` or its `cause` chain — the OpenAI SDK
|
|
85
85
|
* wraps a failed `fetch` in `APIConnectionError` with the original as
|
|
86
|
-
* `cause`, LangChain's middleware errors wrap again
|
|
86
|
+
* `cause`, LangChain's middleware errors wrap again, once per middleware
|
|
87
|
+
* layer (a dozen on the main agent), so the walk goes deep.
|
|
87
88
|
*/
|
|
88
89
|
export function findProviderStall(error: unknown): ProviderStallError | null {
|
|
89
90
|
let current: unknown = error;
|
|
90
|
-
for (let depth = 0; depth <
|
|
91
|
+
for (let depth = 0; depth < 32; depth += 1) {
|
|
91
92
|
if (current instanceof ProviderStallError) return current;
|
|
92
93
|
if (!current || typeof current !== 'object' || !('cause' in current))
|
|
93
94
|
return null;
|
package/src/matrix/gateway-do.ts
CHANGED
|
@@ -24,9 +24,11 @@ import {
|
|
|
24
24
|
listPendingSends,
|
|
25
25
|
MatrixBotDO,
|
|
26
26
|
optionsFromEnv,
|
|
27
|
+
parseEncryptedFileInfo,
|
|
27
28
|
Semaphore,
|
|
28
29
|
withRateLimitRetry,
|
|
29
30
|
type BotMessage,
|
|
31
|
+
type EncryptedFileInfo,
|
|
30
32
|
type BotTimelineEvent,
|
|
31
33
|
type MatrixBotOptions,
|
|
32
34
|
type StartResult,
|
|
@@ -37,6 +39,7 @@ import {
|
|
|
37
39
|
userOracleRoomAlias,
|
|
38
40
|
type BotCredentials,
|
|
39
41
|
type CreateRoomOptions,
|
|
42
|
+
type EventMediaInfo,
|
|
40
43
|
type GatewayStatus,
|
|
41
44
|
type GroupRoomInfo,
|
|
42
45
|
type JsonString,
|
|
@@ -99,11 +102,7 @@ import {
|
|
|
99
102
|
} from './user-homeserver';
|
|
100
103
|
import { replyPartTxnId, roomReplyMessages } from './reply-parts';
|
|
101
104
|
import { VerifiedRoomAliases } from './room-alias';
|
|
102
|
-
import {
|
|
103
|
-
boundedSignal,
|
|
104
|
-
raceSignal,
|
|
105
|
-
streamWithDeadline,
|
|
106
|
-
} from './media-deadline';
|
|
105
|
+
import { boundedSignal, raceSignal } from './media-deadline';
|
|
107
106
|
|
|
108
107
|
const TYPING_REFRESH_MS = 20_000;
|
|
109
108
|
const TYPING_TIMEOUT_MS = 30_000;
|
|
@@ -122,11 +121,19 @@ const MEMO_MAX_ENTRIES = 5_000;
|
|
|
122
121
|
*/
|
|
123
122
|
const STATUS_CACHE_MS = 5_000;
|
|
124
123
|
/**
|
|
125
|
-
* Upper bound on one media
|
|
126
|
-
* asks for less. A stalled homeserver
|
|
127
|
-
*
|
|
124
|
+
* Upper bound on one media event lookup (`eventMediaInfo`), unless the caller
|
|
125
|
+
* asks for less. A stalled homeserver must not hold the user's turn open
|
|
126
|
+
* indefinitely.
|
|
128
127
|
*/
|
|
129
128
|
export const MEDIA_DOWNLOAD_TIMEOUT_MS = 60_000;
|
|
129
|
+
/**
|
|
130
|
+
* `sendEvent` calls with `priority: 'background'` (channel text mirrors and
|
|
131
|
+
* file events) in the SDK's send path at once. The SDK 0.7.0 `sendEvent`
|
|
132
|
+
* dispatches every event `interactive`, so this lane is what keeps a burst
|
|
133
|
+
* of them from queueing ahead of replies in the rate-limited sender: at most
|
|
134
|
+
* this many sit there, the rest wait here.
|
|
135
|
+
*/
|
|
136
|
+
export const BACKGROUND_SEND_CONCURRENCY = 2;
|
|
130
137
|
/**
|
|
131
138
|
* A live message whose sender's homeserver could not be looked up stays in
|
|
132
139
|
* the inbox and is replayed on its own after this delay (each replay is
|
|
@@ -184,6 +191,90 @@ interface OracleGatewayConfig {
|
|
|
184
191
|
oracleRoomDid: string;
|
|
185
192
|
}
|
|
186
193
|
|
|
194
|
+
/** A JSON object's own property, or undefined. */
|
|
195
|
+
function prop(value: unknown, name: string): unknown {
|
|
196
|
+
return typeof value === 'object' && value !== null && name in value
|
|
197
|
+
? Reflect.get(value, name)
|
|
198
|
+
: undefined;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Whether event content (JSON) carries an attachment key (`file.key`, `EncryptedFile`). */
|
|
202
|
+
function carriesAttachmentKey(content: JsonString): boolean {
|
|
203
|
+
let parsed: unknown;
|
|
204
|
+
try {
|
|
205
|
+
parsed = JSON.parse(content);
|
|
206
|
+
} catch {
|
|
207
|
+
// The SDK refuses content that is not a JSON object.
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
return prop(prop(parsed, 'file'), 'key') !== undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The `EncryptedFile` fields of an event's `file`, validated
|
|
215
|
+
* (`parseEncryptedFileInfo` throws on a malformed one) and rebuilt from the
|
|
216
|
+
* checked values, without `url`, `mimetype` and `size`.
|
|
217
|
+
*/
|
|
218
|
+
function encryptedFileOf(file: unknown): EncryptedFileInfo {
|
|
219
|
+
parseEncryptedFileInfo(file);
|
|
220
|
+
const key = prop(file, 'key');
|
|
221
|
+
const k = prop(key, 'k');
|
|
222
|
+
const iv = prop(file, 'iv');
|
|
223
|
+
const sha256 = prop(prop(file, 'hashes'), 'sha256');
|
|
224
|
+
if (
|
|
225
|
+
typeof k !== 'string' ||
|
|
226
|
+
typeof iv !== 'string' ||
|
|
227
|
+
typeof sha256 !== 'string'
|
|
228
|
+
)
|
|
229
|
+
throw new Error('encrypted attachment: malformed EncryptedFile');
|
|
230
|
+
const ops = prop(key, 'key_ops');
|
|
231
|
+
return {
|
|
232
|
+
v: 'v2',
|
|
233
|
+
key: {
|
|
234
|
+
kty: 'oct',
|
|
235
|
+
key_ops: Array.isArray(ops)
|
|
236
|
+
? ops.filter((op): op is string => typeof op === 'string')
|
|
237
|
+
: ['encrypt', 'decrypt'],
|
|
238
|
+
alg: 'A256CTR',
|
|
239
|
+
k,
|
|
240
|
+
ext: true,
|
|
241
|
+
},
|
|
242
|
+
iv,
|
|
243
|
+
hashes: { sha256 },
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Where a decrypted media event's file lives (`eventMediaInfo`): null for a
|
|
249
|
+
* redaction, a redacted (empty) event or one without an `mxc://` URL.
|
|
250
|
+
*/
|
|
251
|
+
export function mediaInfoOfEvent(
|
|
252
|
+
event: unknown,
|
|
253
|
+
eventId: string,
|
|
254
|
+
): EventMediaInfo | null {
|
|
255
|
+
if (prop(event, 'type') === 'm.room.redaction') return null;
|
|
256
|
+
const content = prop(event, 'content');
|
|
257
|
+
if (typeof content !== 'object' || content === null) return null;
|
|
258
|
+
if (Object.keys(content).length === 0) return null;
|
|
259
|
+
const file = prop(content, 'file');
|
|
260
|
+
const mxc = file !== undefined ? prop(file, 'url') : prop(content, 'url');
|
|
261
|
+
if (typeof mxc !== 'string' || !mxc.startsWith('mxc://'))
|
|
262
|
+
throw new Error(
|
|
263
|
+
`MatrixGatewayDO: media event ${eventId} carries no mxc url`,
|
|
264
|
+
);
|
|
265
|
+
const info = prop(content, 'info');
|
|
266
|
+
const mimetype = prop(info, 'mimetype');
|
|
267
|
+
const size = prop(info, 'size');
|
|
268
|
+
const filename = prop(content, 'filename') ?? prop(content, 'body');
|
|
269
|
+
return {
|
|
270
|
+
mxc,
|
|
271
|
+
...(file !== undefined ? { file: encryptedFileOf(file) } : {}),
|
|
272
|
+
...(typeof mimetype === 'string' ? { mimetype } : {}),
|
|
273
|
+
...(typeof filename === 'string' ? { filename } : {}),
|
|
274
|
+
...(typeof size === 'number' ? { size } : {}),
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
187
278
|
/** The plugins' device credentials as stored (`identity:bot-client`). */
|
|
188
279
|
interface StoredDevice {
|
|
189
280
|
userId: string;
|
|
@@ -197,9 +288,9 @@ interface AliasCacheEntry {
|
|
|
197
288
|
at: number;
|
|
198
289
|
}
|
|
199
290
|
|
|
200
|
-
/** A media
|
|
291
|
+
/** A media event lookup's bound. `signal` only works for a caller in this isolate: an `AbortSignal` cannot cross an RPC boundary. */
|
|
201
292
|
export interface MediaDownloadOptions {
|
|
202
|
-
/** Upper bound on the whole
|
|
293
|
+
/** Upper bound on the whole lookup (default `MEDIA_DOWNLOAD_TIMEOUT_MS`). */
|
|
203
294
|
timeoutMs?: number;
|
|
204
295
|
signal?: AbortSignal;
|
|
205
296
|
}
|
|
@@ -371,6 +462,9 @@ export class MatrixGatewayDO
|
|
|
371
462
|
gateSizeFromEnv(this.env.MATRIX_TURN_CONCURRENCY, DEFAULT_TURN_CONCURRENCY),
|
|
372
463
|
);
|
|
373
464
|
private pluginDevice: BotCredentials | null = null;
|
|
465
|
+
/** The plugin device lookup in flight: concurrent first callers share one `whoami` / login. */
|
|
466
|
+
private pluginDeviceLoad: Promise<BotCredentials> | null = null;
|
|
467
|
+
private readonly backgroundSends = new Semaphore(BACKGROUND_SEND_CONCURRENCY);
|
|
374
468
|
/** Memoised `getOracleSecretsKey` result (private JWK JSON). */
|
|
375
469
|
private oracleSecretsKeyJson: string | null = null;
|
|
376
470
|
private oracleSigningMnemonic: string | null = null;
|
|
@@ -1535,7 +1629,7 @@ export class MatrixGatewayDO
|
|
|
1535
1629
|
}
|
|
1536
1630
|
|
|
1537
1631
|
// -------------------------------------------------------------------------
|
|
1538
|
-
// User snapshots (SQLite files as room media) and media
|
|
1632
|
+
// User snapshots (SQLite files as room media) and media event lookups
|
|
1539
1633
|
// -------------------------------------------------------------------------
|
|
1540
1634
|
|
|
1541
1635
|
async uploadUserSnapshotStream(
|
|
@@ -1648,105 +1742,28 @@ export class MatrixGatewayDO
|
|
|
1648
1742
|
}
|
|
1649
1743
|
|
|
1650
1744
|
/**
|
|
1651
|
-
*
|
|
1652
|
-
*
|
|
1745
|
+
* See `MatrixGatewayObject.eventMediaInfo`: the media event fetched and
|
|
1746
|
+
* decrypted here, its file left on the homeserver for the caller to read.
|
|
1653
1747
|
*/
|
|
1654
|
-
async
|
|
1748
|
+
async eventMediaInfo(
|
|
1655
1749
|
roomId: string,
|
|
1656
1750
|
eventId: string,
|
|
1657
1751
|
opts: MediaDownloadOptions = {},
|
|
1658
|
-
): Promise<
|
|
1752
|
+
): Promise<EventMediaInfo | null> {
|
|
1659
1753
|
const bound = boundedSignal(
|
|
1660
1754
|
opts.timeoutMs ?? MEDIA_DOWNLOAD_TIMEOUT_MS,
|
|
1661
1755
|
opts.signal,
|
|
1662
1756
|
);
|
|
1663
|
-
let media: MediaStream | null;
|
|
1664
1757
|
try {
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
media = await raceSignal(
|
|
1668
|
-
this.mediaStream(roomId, eventId),
|
|
1758
|
+
const raw = await raceSignal(
|
|
1759
|
+
this.getEvent(roomId, eventId),
|
|
1669
1760
|
bound.signal,
|
|
1670
|
-
(
|
|
1761
|
+
() => undefined,
|
|
1671
1762
|
);
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
throw err;
|
|
1675
|
-
}
|
|
1676
|
-
if (!media) {
|
|
1677
|
-
bound.clear();
|
|
1678
|
-
return null;
|
|
1679
|
-
}
|
|
1680
|
-
return { ...media, stream: streamWithDeadline(media.stream, bound) };
|
|
1681
|
-
}
|
|
1682
|
-
|
|
1683
|
-
/**
|
|
1684
|
-
* Authenticated download of an `mxc://` URI as a stream
|
|
1685
|
-
* (`/_matrix/client/v1/media/download`, legacy fallback on 404), bounded
|
|
1686
|
-
* by `opts.timeoutMs` (default `MEDIA_DOWNLOAD_TIMEOUT_MS`) from the
|
|
1687
|
-
* request to the last byte.
|
|
1688
|
-
*/
|
|
1689
|
-
async downloadMxcMediaStream(
|
|
1690
|
-
mxc: string,
|
|
1691
|
-
opts: MediaDownloadOptions = {},
|
|
1692
|
-
): Promise<ReadableStream<Uint8Array>> {
|
|
1693
|
-
const bound = boundedSignal(
|
|
1694
|
-
opts.timeoutMs ?? MEDIA_DOWNLOAD_TIMEOUT_MS,
|
|
1695
|
-
opts.signal,
|
|
1696
|
-
);
|
|
1697
|
-
try {
|
|
1698
|
-
const body = await this.fetchMxc(mxc, bound.signal);
|
|
1699
|
-
return streamWithDeadline(body, bound);
|
|
1700
|
-
} catch (err) {
|
|
1763
|
+
return raw ? mediaInfoOfEvent(JSON.parse(raw), eventId) : null;
|
|
1764
|
+
} finally {
|
|
1701
1765
|
bound.clear();
|
|
1702
|
-
throw err;
|
|
1703
|
-
}
|
|
1704
|
-
}
|
|
1705
|
-
|
|
1706
|
-
private async fetchMxc(
|
|
1707
|
-
mxc: string,
|
|
1708
|
-
signal: AbortSignal,
|
|
1709
|
-
): Promise<ReadableStream<Uint8Array>> {
|
|
1710
|
-
const client = await this.startedClient();
|
|
1711
|
-
const token = client.getAccessToken();
|
|
1712
|
-
const headers: Record<string, string> = token
|
|
1713
|
-
? { Authorization: `Bearer ${token}` }
|
|
1714
|
-
: {};
|
|
1715
|
-
const authed = client.mxcUrlToHttp(
|
|
1716
|
-
mxc,
|
|
1717
|
-
undefined,
|
|
1718
|
-
undefined,
|
|
1719
|
-
undefined,
|
|
1720
|
-
false,
|
|
1721
|
-
true,
|
|
1722
|
-
true,
|
|
1723
|
-
);
|
|
1724
|
-
if (!authed) throw new Error(`MatrixGatewayDO: invalid mxc url ${mxc}`);
|
|
1725
|
-
let res = await fetch(authed, { headers, signal });
|
|
1726
|
-
if (res.status === 404) {
|
|
1727
|
-
const legacy = client.mxcUrlToHttp(
|
|
1728
|
-
mxc,
|
|
1729
|
-
undefined,
|
|
1730
|
-
undefined,
|
|
1731
|
-
undefined,
|
|
1732
|
-
false,
|
|
1733
|
-
true,
|
|
1734
|
-
false,
|
|
1735
|
-
);
|
|
1736
|
-
if (legacy) {
|
|
1737
|
-
await res.body?.cancel().catch(() => undefined);
|
|
1738
|
-
res = await fetch(legacy, { headers, signal });
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
if (!res.ok) {
|
|
1742
|
-
await res.body?.cancel().catch(() => undefined);
|
|
1743
|
-
throw new Error(
|
|
1744
|
-
`MatrixGatewayDO: media download failed: ${res.status} ${res.statusText}`,
|
|
1745
|
-
);
|
|
1746
1766
|
}
|
|
1747
|
-
if (!res.body)
|
|
1748
|
-
throw new Error('MatrixGatewayDO: media download has no body');
|
|
1749
|
-
return res.body;
|
|
1750
1767
|
}
|
|
1751
1768
|
|
|
1752
1769
|
// -------------------------------------------------------------------------
|
|
@@ -1801,13 +1818,31 @@ export class MatrixGatewayDO
|
|
|
1801
1818
|
/**
|
|
1802
1819
|
* See `MatrixGatewayObject.botCredentials`: a second password device with
|
|
1803
1820
|
* NO crypto store, for plugins that drive a raw matrix-js-sdk client
|
|
1804
|
-
* (editor, flows)
|
|
1805
|
-
* one-time keys and store are the
|
|
1806
|
-
* re-login
|
|
1807
|
-
* after one `whoami
|
|
1821
|
+
* (editor, flows) and the user objects' media transfers — it must never
|
|
1822
|
+
* be the gateway's own E2EE device, whose one-time keys and store are the
|
|
1823
|
+
* SDK's. Its id is pinned in storage so a re-login keeps the same device.
|
|
1824
|
+
* Cached per instance after one `whoami`; concurrent callers share one
|
|
1825
|
+
* lookup (a login rotates the token, so a herd would invalidate each
|
|
1826
|
+
* other's). `rejected` drops the cache only when it is the cached token.
|
|
1808
1827
|
*/
|
|
1809
|
-
async botCredentials(
|
|
1828
|
+
async botCredentials(
|
|
1829
|
+
opts: { rejected?: string } = {},
|
|
1830
|
+
): Promise<BotCredentials> {
|
|
1831
|
+
if (opts.rejected && this.pluginDevice?.accessToken === opts.rejected) {
|
|
1832
|
+
this.log(
|
|
1833
|
+
'warn',
|
|
1834
|
+
'plugin device token rejected by a caller; re-validating',
|
|
1835
|
+
);
|
|
1836
|
+
this.pluginDevice = null;
|
|
1837
|
+
}
|
|
1810
1838
|
if (this.pluginDevice) return this.pluginDevice;
|
|
1839
|
+
this.pluginDeviceLoad ??= this.loadPluginDevice().finally(() => {
|
|
1840
|
+
this.pluginDeviceLoad = null;
|
|
1841
|
+
});
|
|
1842
|
+
return this.pluginDeviceLoad;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
private async loadPluginDevice(): Promise<BotCredentials> {
|
|
1811
1846
|
const cfg = this.cfg();
|
|
1812
1847
|
const stored = await this.ctx.storage.get<StoredDevice>(
|
|
1813
1848
|
BOT_CLIENT_IDENTITY_KEY,
|
|
@@ -1841,6 +1876,33 @@ export class MatrixGatewayDO
|
|
|
1841
1876
|
return this.pluginDevice;
|
|
1842
1877
|
}
|
|
1843
1878
|
|
|
1879
|
+
/**
|
|
1880
|
+
* See `MatrixGatewayObject.sendEvent`: the SDK's send, behind the
|
|
1881
|
+
* background lane for `priority: 'background'`, and never an attachment
|
|
1882
|
+
* key into a room this gateway does not see as end-to-end encrypted.
|
|
1883
|
+
*/
|
|
1884
|
+
override async sendEvent(
|
|
1885
|
+
roomId: string,
|
|
1886
|
+
type: string,
|
|
1887
|
+
content: JsonString,
|
|
1888
|
+
opts: { txnId?: string; priority?: 'interactive' | 'background' } = {},
|
|
1889
|
+
): Promise<string> {
|
|
1890
|
+
if (carriesAttachmentKey(content) && !(await this.isRoomEncrypted(roomId)))
|
|
1891
|
+
throw new Error(
|
|
1892
|
+
`MatrixGatewayDO: refusing to send an encrypted attachment's key into ${roomId}, which is not end-to-end encrypted`,
|
|
1893
|
+
);
|
|
1894
|
+
const send = (): Promise<string> =>
|
|
1895
|
+
super.sendEvent(
|
|
1896
|
+
roomId,
|
|
1897
|
+
type,
|
|
1898
|
+
content,
|
|
1899
|
+
opts.txnId ? { txnId: opts.txnId } : {},
|
|
1900
|
+
);
|
|
1901
|
+
return opts.priority === 'background'
|
|
1902
|
+
? this.backgroundSends.run(send)
|
|
1903
|
+
: send();
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1844
1906
|
// -------------------------------------------------------------------------
|
|
1845
1907
|
// Operator surface
|
|
1846
1908
|
// -------------------------------------------------------------------------
|
package/src/matrix/ingest.ts
CHANGED
|
@@ -85,9 +85,10 @@ export interface IngestDeps {
|
|
|
85
85
|
*/
|
|
86
86
|
userServerName(userDid: string): string | null;
|
|
87
87
|
/**
|
|
88
|
-
* The bot's Matrix user id. Its
|
|
89
|
-
*
|
|
90
|
-
* bridge does; omit to
|
|
88
|
+
* The bot's Matrix user id. Its own messages are dropped (`'own'`), and its
|
|
89
|
+
* first occurrence in a message is rewritten to `(USER MENTIONED YOU
|
|
90
|
+
* @AI_AGENT)` before the agent sees it, as the Node bridge does; omit to
|
|
91
|
+
* leave messages as sent.
|
|
91
92
|
*/
|
|
92
93
|
botUserId?: string;
|
|
93
94
|
debounceMs?: number;
|
|
@@ -210,6 +211,7 @@ export type OfferOutcome =
|
|
|
210
211
|
| 'queued'
|
|
211
212
|
| 'unverified'
|
|
212
213
|
| 'empty'
|
|
214
|
+
| 'own'
|
|
213
215
|
| 'unmapped'
|
|
214
216
|
| 'foreign';
|
|
215
217
|
|
|
@@ -229,6 +231,13 @@ export class IngestPipeline {
|
|
|
229
231
|
* for none); omitted, `canonicalAlias` is asked.
|
|
230
232
|
*/
|
|
231
233
|
offer(msg: InboundMessage, alias?: string | null): OfferOutcome {
|
|
234
|
+
// The oracle's own events — its room mirrors and the channel files it
|
|
235
|
+
// posts in a session thread — are never a turn. The bot SDK already
|
|
236
|
+
// drops them before `onMessage`; this holds on every path that offers a
|
|
237
|
+
// message here (inbox replays included), and keeps a DID-shaped bot
|
|
238
|
+
// account from being attributed to itself.
|
|
239
|
+
if (this.deps.botUserId !== undefined && msg.sender === this.deps.botUserId)
|
|
240
|
+
return 'own';
|
|
232
241
|
if (!msg.body.trim() && !msg.attachment) return 'empty';
|
|
233
242
|
const attribution = attributeUser({
|
|
234
243
|
alias: alias === undefined ? this.deps.canonicalAlias(msg.roomId) : alias,
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Deadlines for the gateway's media
|
|
3
|
-
*
|
|
4
|
-
* assembling an attachment for a turn — waiting for as long as
|
|
5
|
-
* connection stays open.
|
|
6
|
-
*
|
|
7
|
-
* The bound covers the whole transfer: opening the download and every byte
|
|
8
|
-
* of the body. When it passes (or the caller's own signal aborts), the
|
|
9
|
-
* source is cancelled and the stream handed out errors with the reason.
|
|
2
|
+
* Deadlines for the gateway's media event lookups (`eventMediaInfo`). A
|
|
3
|
+
* homeserver that never answers would otherwise keep the asking side — a
|
|
4
|
+
* user object assembling an attachment for a turn — waiting for as long as
|
|
5
|
+
* the connection stays open. When the bound passes (or the caller's own
|
|
6
|
+
* signal aborts), the lookup rejects with the reason.
|
|
10
7
|
*/
|
|
11
8
|
|
|
12
9
|
export interface BoundedSignal {
|
|
@@ -89,42 +86,3 @@ export function raceSignal<T>(
|
|
|
89
86
|
);
|
|
90
87
|
});
|
|
91
88
|
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* `source` behind the deadline: the returned stream carries the same bytes,
|
|
95
|
-
* and errors with the signal's reason (the source cancelled) when the bound
|
|
96
|
-
* passes before the last byte. The timer is cleared once the transfer ends
|
|
97
|
-
* either way, including when the reader cancels.
|
|
98
|
-
*/
|
|
99
|
-
export function streamWithDeadline(
|
|
100
|
-
source: ReadableStream<Uint8Array>,
|
|
101
|
-
bound: BoundedSignal,
|
|
102
|
-
): ReadableStream<Uint8Array> {
|
|
103
|
-
const { readable, writable } = new TransformStream<Uint8Array, Uint8Array>();
|
|
104
|
-
const writer = writable.getWriter();
|
|
105
|
-
const reader = source.getReader();
|
|
106
|
-
const stop = (reason: unknown): void => {
|
|
107
|
-
void writer.abort(reason).catch(() => undefined);
|
|
108
|
-
void reader.cancel(reason).catch(() => undefined);
|
|
109
|
-
};
|
|
110
|
-
const onAbort = (): void => stop(bound.signal.reason);
|
|
111
|
-
if (bound.signal.aborted) onAbort();
|
|
112
|
-
else bound.signal.addEventListener('abort', onAbort, { once: true });
|
|
113
|
-
void (async () => {
|
|
114
|
-
try {
|
|
115
|
-
for (;;) {
|
|
116
|
-
const { done, value } = await reader.read();
|
|
117
|
-
if (done || bound.signal.aborted) break;
|
|
118
|
-
await writer.write(value);
|
|
119
|
-
}
|
|
120
|
-
if (!bound.signal.aborted) await writer.close();
|
|
121
|
-
} catch (err) {
|
|
122
|
-
// The reader went away (cancelled) or the source failed: end both.
|
|
123
|
-
stop(err);
|
|
124
|
-
} finally {
|
|
125
|
-
bound.signal.removeEventListener('abort', onAbort);
|
|
126
|
-
bound.clear();
|
|
127
|
-
}
|
|
128
|
-
})();
|
|
129
|
-
return readable;
|
|
130
|
-
}
|
|
@@ -27,8 +27,13 @@
|
|
|
27
27
|
* transcript's last chance).
|
|
28
28
|
* - Speakers get real identities (the user's preferred name / the oracle's
|
|
29
29
|
* preferred or configured name) so Graphiti's extractor does not pin
|
|
30
|
-
* facts to literal "user"/"assistant" nodes.
|
|
31
|
-
*
|
|
30
|
+
* facts to literal "user"/"assistant" nodes.
|
|
31
|
+
* - Only the conversation is sent: what the user said and the oracle's
|
|
32
|
+
* replies (`conversationReader`). The engine makes every message an episode
|
|
33
|
+
* of evidence, so a tool result, a status line before a tool call, or a
|
|
34
|
+
* reply that repeats what a memory search just returned would come back
|
|
35
|
+
* as a fact, and push real ones out of the capped user-context block. Those
|
|
36
|
+
* messages still count towards the watermark.
|
|
32
37
|
* - Auth is the oracle's two-hop UCAN invocation minted from the user's
|
|
33
38
|
* delegation (`ixo:memory` / `memory/*`) — no delegation, no indexing.
|
|
34
39
|
* - Endpoint: `POST <MEMORY_ENGINE_URL>/messages` with `x-room-id`, as
|
|
@@ -45,13 +50,18 @@
|
|
|
45
50
|
* Not ported: Node also asks Matrix for the user's display name when no
|
|
46
51
|
* preferred name is set; here the fallback is the literal "Me" straight away.
|
|
47
52
|
*/
|
|
53
|
+
import { AIMessage, type BaseMessage } from '@langchain/core/messages';
|
|
54
|
+
import { visibleText, visibleToUser } from '../do/transcript';
|
|
48
55
|
import type { Logger } from '../plugin-api/types';
|
|
49
56
|
import { MEMORY_ENTITY_HEADER } from '../plugins/memory/memory-entity';
|
|
57
|
+
import { MEMORY_SEARCH_MCP_NAME } from '../plugins/memory/memory-tools';
|
|
50
58
|
|
|
51
59
|
export interface HistoryMessage {
|
|
52
60
|
/** LangChain message type: `human` | `ai` | `tool` | `system`. */
|
|
53
61
|
type: string;
|
|
54
62
|
content: string;
|
|
63
|
+
/** Counted by the watermark but not sent (`conversationReader`). */
|
|
64
|
+
notConversation?: boolean;
|
|
55
65
|
}
|
|
56
66
|
|
|
57
67
|
export interface MemoryEngineMessage {
|
|
@@ -230,6 +240,47 @@ interface QuietWait {
|
|
|
230
240
|
resolve: () => void;
|
|
231
241
|
}
|
|
232
242
|
|
|
243
|
+
/** Text that parses as a JSON object or list: a tool's output, not a reply. */
|
|
244
|
+
function isJsonDocument(text: string): boolean {
|
|
245
|
+
const start = text.trimStart()[0];
|
|
246
|
+
if (start !== '{' && start !== '[') return false;
|
|
247
|
+
try {
|
|
248
|
+
JSON.parse(text);
|
|
249
|
+
return true;
|
|
250
|
+
} catch {
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Reads a transcript in order, one reader per read, and turns each message
|
|
257
|
+
* into what the indexer takes. A turn's memory search is seen before its reply,
|
|
258
|
+
* so that reply is known to repeat what memory returned, which must not go
|
|
259
|
+
* back in as new evidence. A read that starts in the middle of a turn has not
|
|
260
|
+
* seen the search; a pass normally starts at a turn's end.
|
|
261
|
+
*/
|
|
262
|
+
export function conversationReader(): (message: BaseMessage) => HistoryMessage {
|
|
263
|
+
let readMemory = false;
|
|
264
|
+
return (message) => {
|
|
265
|
+
const content = visibleText(message);
|
|
266
|
+
const calls = AIMessage.isInstance(message)
|
|
267
|
+
? (message.tool_calls ?? [])
|
|
268
|
+
: [];
|
|
269
|
+
if (message.type === 'human') readMemory = false;
|
|
270
|
+
if (calls.some((call) => call.name === MEMORY_SEARCH_MCP_NAME))
|
|
271
|
+
readMemory = true;
|
|
272
|
+
const conversation =
|
|
273
|
+
visibleToUser(message) &&
|
|
274
|
+
(message.type === 'human' ||
|
|
275
|
+
(calls.length === 0 && !readMemory && !isJsonDocument(content)));
|
|
276
|
+
return {
|
|
277
|
+
type: message.type,
|
|
278
|
+
content,
|
|
279
|
+
...(conversation ? {} : { notConversation: true }),
|
|
280
|
+
};
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
233
284
|
/** Node's `transformMessagesToMemoryEngineFormat`, verbatim in mapping. */
|
|
234
285
|
export function transformMessagesToMemoryEngineFormat(
|
|
235
286
|
messages: readonly HistoryMessage[],
|
|
@@ -237,7 +288,12 @@ export function transformMessagesToMemoryEngineFormat(
|
|
|
237
288
|
userSpeakerLabel: string,
|
|
238
289
|
oracleSpeakerLabel: string,
|
|
239
290
|
): MemoryEngineMessage[] {
|
|
240
|
-
|
|
291
|
+
// The engine rejects empty episodes, and has no tool role.
|
|
292
|
+
const conversation = messages.filter(
|
|
293
|
+
(m) =>
|
|
294
|
+
m.type !== 'tool' && !m.notConversation && m.content.trim().length > 0,
|
|
295
|
+
);
|
|
296
|
+
return conversation.map((message) => {
|
|
241
297
|
let role_type: MemoryEngineMessage['role_type'];
|
|
242
298
|
let label: string;
|
|
243
299
|
switch (message.type) {
|
|
@@ -253,12 +309,6 @@ export function transformMessagesToMemoryEngineFormat(
|
|
|
253
309
|
role_type = 'system';
|
|
254
310
|
label = 'System';
|
|
255
311
|
break;
|
|
256
|
-
case 'tool':
|
|
257
|
-
// The engine has no tool role — tool replies are the oracle
|
|
258
|
-
// reporting back, so they share the oracle speaker label.
|
|
259
|
-
role_type = 'assistant';
|
|
260
|
-
label = oracleSpeakerLabel;
|
|
261
|
-
break;
|
|
262
312
|
default:
|
|
263
313
|
role_type = 'user';
|
|
264
314
|
label = userSpeakerLabel;
|
|
@@ -564,11 +614,7 @@ export class SessionHistoryIndexer {
|
|
|
564
614
|
try {
|
|
565
615
|
while (chunk.length > 0) {
|
|
566
616
|
const transformed = transformMessagesToMemoryEngineFormat(
|
|
567
|
-
|
|
568
|
-
// extract from them and rejects empty episodes.
|
|
569
|
-
chunk
|
|
570
|
-
.map((entry) => entry.message)
|
|
571
|
-
.filter((m) => m.content.trim().length > 0),
|
|
617
|
+
chunk.map((entry) => entry.message),
|
|
572
618
|
session.title ?? '',
|
|
573
619
|
labels.user,
|
|
574
620
|
labels.oracle,
|
|
@@ -63,11 +63,10 @@ export abstract class OraclePlugin {
|
|
|
63
63
|
* plugin's working rules, as plain text (markdown, starting at a `###`
|
|
64
64
|
* heading). Never part of the manifest, the Tier-1 capability block or
|
|
65
65
|
* `list_capabilities`, so it costs nothing on turns that do not use the
|
|
66
|
-
* plugin.
|
|
67
|
-
*
|
|
68
|
-
* system prompt
|
|
69
|
-
*
|
|
70
|
-
* delegation.
|
|
66
|
+
* plugin. The model has it when it first uses the plugin's tools: in the
|
|
67
|
+
* load result or on the turn's message when the plugin loads on a warm
|
|
68
|
+
* turn, in the system prompt from the next cold turn on, and always there
|
|
69
|
+
* for an `always` plugin the user's delegation can use.
|
|
71
70
|
*/
|
|
72
71
|
readonly operatingGuide?: string;
|
|
73
72
|
|