@zooid/transport-matrix 0.7.4 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +116 -1
- package/dist/index.js +387 -51
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/src/attachments.test.ts +58 -0
- package/src/attachments.ts +30 -0
- package/src/bot-pool.ts +1 -1
- package/src/context-provider.test.ts +33 -0
- package/src/context-provider.ts +22 -4
- package/src/event-encoders.test.ts +22 -0
- package/src/event-encoders.ts +13 -0
- package/src/index.ts +8 -2
- package/src/matrix-client.test.ts +35 -2
- package/src/matrix-client.ts +19 -0
- package/src/media-client.test.ts +102 -0
- package/src/media-client.ts +69 -0
- package/src/pending-media.test.ts +51 -0
- package/src/pending-media.ts +37 -0
- package/src/router.test.ts +22 -1
- package/src/router.ts +11 -0
- package/src/space-provisioner.test.ts +26 -1
- package/src/space-provisioner.ts +15 -4
- package/src/transport.test.ts +401 -30
- package/src/transport.ts +402 -70
- package/src/workforce-publisher.test.ts +2 -2
- package/src/workforce-publisher.ts +1 -1
package/src/transport.ts
CHANGED
|
@@ -1,14 +1,38 @@
|
|
|
1
1
|
import { Hono } from 'hono'
|
|
2
2
|
import { timingSafeEqual } from 'node:crypto'
|
|
3
3
|
import type { AcpRegistry, ApprovalCorrelator, RegisteredApproval } from '@zooid/core'
|
|
4
|
-
import type { AgentEvent } from '@zooid/acp-client'
|
|
4
|
+
import type { AgentEvent, ContentBlock } from '@zooid/acp-client'
|
|
5
5
|
import { MatrixClient } from './matrix-client.js'
|
|
6
6
|
import { BotPool } from './bot-pool.js'
|
|
7
|
-
import { route, type AgentBinding, type ThreadState } from './router.js'
|
|
7
|
+
import { route, isMediaMsgtype, type AgentBinding, type ThreadState } from './router.js'
|
|
8
8
|
import { stripMention, extractMentions } from './mentions.js'
|
|
9
|
-
import { toToolCallBody, toUpdateBody, toPlanBody, toErrorBody } from './event-encoders.js'
|
|
9
|
+
import { toToolCallBody, toUpdateBody, toPlanBody, toAvailableCommandsBody, toErrorBody } from './event-encoders.js'
|
|
10
10
|
import { classify } from '@zooid/acp-client'
|
|
11
11
|
import { toMatrixHtml } from './markdown-to-matrix-html.js'
|
|
12
|
+
import {
|
|
13
|
+
PendingMediaStore,
|
|
14
|
+
type PendingMediaItem,
|
|
15
|
+
} from './pending-media.js'
|
|
16
|
+
import {
|
|
17
|
+
MediaClient,
|
|
18
|
+
MAX_INLINE_IMAGE_BYTES,
|
|
19
|
+
INLINE_IMAGE_MIMES,
|
|
20
|
+
} from './media-client.js'
|
|
21
|
+
import { writeAttachment } from './attachments.js'
|
|
22
|
+
|
|
23
|
+
export interface MediaClientLike {
|
|
24
|
+
download(input: {
|
|
25
|
+
mxcUri: string
|
|
26
|
+
asUserId: string
|
|
27
|
+
maxBytes?: number
|
|
28
|
+
}): Promise<{ data: Uint8Array; contentType: string }>
|
|
29
|
+
upload(input: {
|
|
30
|
+
data: Uint8Array
|
|
31
|
+
contentType: string
|
|
32
|
+
filename?: string
|
|
33
|
+
asUserId: string
|
|
34
|
+
}): Promise<{ content_uri: string }>
|
|
35
|
+
}
|
|
12
36
|
|
|
13
37
|
export interface CreateMatrixTransportOptions {
|
|
14
38
|
agents: AcpRegistry
|
|
@@ -24,6 +48,14 @@ export interface CreateMatrixTransportOptions {
|
|
|
24
48
|
drainQuietMs?: number
|
|
25
49
|
/** Hard cap on the post-turn drain. Defaults to `DRAIN_MAX_MS`. */
|
|
26
50
|
drainMaxMs?: number
|
|
51
|
+
/** Injected media client for downloading/uploading Matrix media. */
|
|
52
|
+
media?: MediaClientLike
|
|
53
|
+
/** Injected attachment writer (defaults to the real writeAttachment). */
|
|
54
|
+
writeAttachmentFn?: typeof writeAttachment
|
|
55
|
+
/** AS sender-bot MXID (@<sender_localpart>:<server>). Together with the agent
|
|
56
|
+
* bindings this forms the set of "our bot users" whose ad-hoc invites are
|
|
57
|
+
* declined. */
|
|
58
|
+
botUserId?: string
|
|
27
59
|
}
|
|
28
60
|
|
|
29
61
|
interface SessionContext {
|
|
@@ -39,14 +71,131 @@ interface MatrixEvent {
|
|
|
39
71
|
origin_server_ts?: number
|
|
40
72
|
room_id?: string
|
|
41
73
|
sender?: string
|
|
74
|
+
/** Present on state events (m.room.member → the affected user). */
|
|
75
|
+
state_key?: string
|
|
42
76
|
content?: Record<string, unknown> & {
|
|
43
77
|
msgtype?: string
|
|
44
78
|
body?: string
|
|
79
|
+
membership?: string
|
|
45
80
|
'm.relates_to'?: { rel_type?: string; event_id?: string }
|
|
46
81
|
}
|
|
47
82
|
}
|
|
48
83
|
|
|
49
84
|
const STARTUP_GRACE_MS = 5_000
|
|
85
|
+
|
|
86
|
+
interface MediaBlocksResult {
|
|
87
|
+
blocks: ContentBlock[]
|
|
88
|
+
pathLines: string[]
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function buildMediaBlocks(
|
|
92
|
+
items: PendingMediaItem[],
|
|
93
|
+
opts: {
|
|
94
|
+
agent: AgentBinding
|
|
95
|
+
media: MediaClientLike | undefined
|
|
96
|
+
writeAttachmentFn: typeof writeAttachment
|
|
97
|
+
onError: (item: PendingMediaItem, err: unknown) => void
|
|
98
|
+
},
|
|
99
|
+
): Promise<MediaBlocksResult> {
|
|
100
|
+
const blocks: ContentBlock[] = []
|
|
101
|
+
const pathLines: string[] = []
|
|
102
|
+
|
|
103
|
+
if (!opts.media || items.length === 0) return { blocks, pathLines }
|
|
104
|
+
|
|
105
|
+
for (const item of items) {
|
|
106
|
+
try {
|
|
107
|
+
const isInlineCandidate =
|
|
108
|
+
item.msgtype === 'm.image' &&
|
|
109
|
+
INLINE_IMAGE_MIMES.includes(item.info?.mimetype ?? '') &&
|
|
110
|
+
(item.info?.size === undefined || item.info.size <= MAX_INLINE_IMAGE_BYTES)
|
|
111
|
+
|
|
112
|
+
if (isInlineCandidate) {
|
|
113
|
+
const { data, contentType } = await opts.media.download({
|
|
114
|
+
mxcUri: item.url,
|
|
115
|
+
asUserId: opts.agent.userId,
|
|
116
|
+
})
|
|
117
|
+
// Double-check actual size (info can lie)
|
|
118
|
+
if (data.byteLength <= MAX_INLINE_IMAGE_BYTES) {
|
|
119
|
+
blocks.push({
|
|
120
|
+
type: 'image',
|
|
121
|
+
data: Buffer.from(data).toString('base64'),
|
|
122
|
+
mimeType: contentType,
|
|
123
|
+
})
|
|
124
|
+
continue
|
|
125
|
+
}
|
|
126
|
+
// Actual size exceeded cap — fall through to file route with the already-downloaded bytes
|
|
127
|
+
if (opts.agent.workspaceDir) {
|
|
128
|
+
const paths = opts.writeAttachmentFn({
|
|
129
|
+
workspaceDir: opts.agent.workspaceDir,
|
|
130
|
+
agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
|
|
131
|
+
eventId: item.eventId,
|
|
132
|
+
filename: item.filename ?? item.body,
|
|
133
|
+
data,
|
|
134
|
+
})
|
|
135
|
+
blocks.push({
|
|
136
|
+
type: 'resource_link',
|
|
137
|
+
uri: `file://${paths.agentPath}`,
|
|
138
|
+
name: item.filename ?? item.body,
|
|
139
|
+
})
|
|
140
|
+
pathLines.push(`Attached file: ${paths.agentPath}`)
|
|
141
|
+
}
|
|
142
|
+
} else {
|
|
143
|
+
// File route (m.file, m.video, m.audio, or oversized image)
|
|
144
|
+
if (!opts.agent.workspaceDir) continue
|
|
145
|
+
const { data } = await opts.media.download({
|
|
146
|
+
mxcUri: item.url,
|
|
147
|
+
asUserId: opts.agent.userId,
|
|
148
|
+
})
|
|
149
|
+
const paths = opts.writeAttachmentFn({
|
|
150
|
+
workspaceDir: opts.agent.workspaceDir,
|
|
151
|
+
agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
|
|
152
|
+
eventId: item.eventId,
|
|
153
|
+
filename: item.filename ?? item.body,
|
|
154
|
+
data,
|
|
155
|
+
})
|
|
156
|
+
blocks.push({
|
|
157
|
+
type: 'resource_link',
|
|
158
|
+
uri: `file://${paths.agentPath}`,
|
|
159
|
+
name: item.filename ?? item.body,
|
|
160
|
+
mimeType: item.info?.mimetype,
|
|
161
|
+
size: item.info?.size,
|
|
162
|
+
})
|
|
163
|
+
pathLines.push(`Attached file: ${paths.agentPath}`)
|
|
164
|
+
}
|
|
165
|
+
} catch (err) {
|
|
166
|
+
opts.onError(item, err)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return { blocks, pathLines }
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function sendMediaError(
|
|
174
|
+
ctx: { agent: AgentBinding; roomId: string; threadRoot: string },
|
|
175
|
+
_err: unknown,
|
|
176
|
+
message: string,
|
|
177
|
+
client: MatrixClient,
|
|
178
|
+
): Promise<void> {
|
|
179
|
+
await client
|
|
180
|
+
.sendCustomEvent({
|
|
181
|
+
roomId: ctx.roomId,
|
|
182
|
+
asUserId: ctx.agent.userId,
|
|
183
|
+
eventType: 'dev.zooid.error',
|
|
184
|
+
content: toErrorBody(
|
|
185
|
+
{
|
|
186
|
+
kind: 'error' as const,
|
|
187
|
+
agentId: ctx.agent.name,
|
|
188
|
+
sessionId: null,
|
|
189
|
+
turnId: null,
|
|
190
|
+
code: 'media_failed',
|
|
191
|
+
message: message.slice(0, 250),
|
|
192
|
+
transient: false,
|
|
193
|
+
},
|
|
194
|
+
ctx.threadRoot,
|
|
195
|
+
),
|
|
196
|
+
})
|
|
197
|
+
.catch((e) => console.warn(`[matrix:${ctx.agent.name}] dev.zooid.error send failed:`, e))
|
|
198
|
+
}
|
|
50
199
|
const SEEN_EVENT_CAP = 5_000
|
|
51
200
|
|
|
52
201
|
// ACP only guarantees that an agent flushes pending `session/update`
|
|
@@ -74,10 +223,20 @@ function inboundThreadRoot(evt: MatrixEvent): string | undefined {
|
|
|
74
223
|
}
|
|
75
224
|
|
|
76
225
|
export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
77
|
-
const { agents, approvals, client, bindings, hsToken, adminUserId } = opts
|
|
226
|
+
const { agents, approvals, client, bindings, hsToken, adminUserId, botUserId } = opts
|
|
78
227
|
const drainQuietMs = opts.drainQuietMs ?? DRAIN_QUIET_MS
|
|
79
228
|
const drainMaxMs = opts.drainMaxMs ?? DRAIN_MAX_MS
|
|
229
|
+
const mediaClient = opts.media
|
|
230
|
+
const writeAttachmentFn = opts.writeAttachmentFn ?? writeAttachment
|
|
231
|
+
const pendingMedia = new PendingMediaStore()
|
|
80
232
|
const pool = new BotPool(client, bindings)
|
|
233
|
+
const ourBotUserIds = new Set<string>([
|
|
234
|
+
...(botUserId ? [botUserId] : []),
|
|
235
|
+
...bindings.map((b) => b.userId),
|
|
236
|
+
])
|
|
237
|
+
const DECLINE_REASON =
|
|
238
|
+
'Bots are placed in rooms only by the zooid daemon (workforce-as-code). ' +
|
|
239
|
+
'Ad-hoc invites are declined — add the bot to the room in zooid.yaml.'
|
|
81
240
|
const sessions = new Map<string, SessionContext>()
|
|
82
241
|
const buffers = new Map<string, string>()
|
|
83
242
|
// Last messageId seen per session's buffer. opencode streams each assistant
|
|
@@ -95,56 +254,172 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
95
254
|
// Idempotency: appservice transactions are retried on 4xx/5xx/timeout, and
|
|
96
255
|
// the same event_id can arrive twice. Skip ones we've already taken.
|
|
97
256
|
const seenEventIds = new Set<string>()
|
|
257
|
+
// Messages flushed per session this turn. Lets the drain loop tell "stream
|
|
258
|
+
// not started yet" (0 flushes, empty buffer → keep waiting) from "turn done,
|
|
259
|
+
// last message already flushed mid-stream" (>0 flushes, empty buffer → stop).
|
|
260
|
+
const flushedCounts = new Map<string, number>()
|
|
261
|
+
// Commands a shim advertises during session load/new — i.e. before runTurn
|
|
262
|
+
// registers the session ctx (sessions.set). Stashed here keyed by sessionId
|
|
263
|
+
// and replayed once the ctx exists, so `available_commands_update` (which is
|
|
264
|
+
// only ever emitted at session establishment, never mid-turn) isn't dropped.
|
|
265
|
+
const pendingCommands = new Map<string, AgentEvent>()
|
|
266
|
+
|
|
267
|
+
// Build the m.text content for a chunk of assistant prose, attaching a
|
|
268
|
+
// formatted_body only when the HTML render adds rich text the plain body
|
|
269
|
+
// can't carry (marked wraps plain prose in <p>…</p>; skip that — most
|
|
270
|
+
// clients render `body` better than a stripped re-encode).
|
|
271
|
+
const buildTextContent = (
|
|
272
|
+
text: string,
|
|
273
|
+
): { msgtype: string; body: string; [k: string]: unknown } => {
|
|
274
|
+
const content: { msgtype: string; body: string; [k: string]: unknown } = {
|
|
275
|
+
msgtype: 'm.text',
|
|
276
|
+
body: text,
|
|
277
|
+
}
|
|
278
|
+
const html = toMatrixHtml(text)
|
|
279
|
+
if (html) {
|
|
280
|
+
const escapedPlain =
|
|
281
|
+
'<p>' +
|
|
282
|
+
text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') +
|
|
283
|
+
'</p>'
|
|
284
|
+
const norm = (s: string) => s.replace(/\s+/g, ' ').trim()
|
|
285
|
+
if (norm(html) !== norm(escapedPlain)) {
|
|
286
|
+
content.format = 'org.matrix.custom.html'
|
|
287
|
+
content.formatted_body = html
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return content
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// Flush a session's buffered assistant text as its own Matrix message and
|
|
294
|
+
// clear the buffer. No-op on an empty buffer. The send is chained onto
|
|
295
|
+
// sendQueue so it orders correctly against tool_call/plan events from the
|
|
296
|
+
// same turn. The buffer is cleared synchronously (before the first await),
|
|
297
|
+
// so a chunk for the *next* message that arrives during the send starts
|
|
298
|
+
// fresh. Returns true when a message was enqueued.
|
|
299
|
+
const flushBuffer = (sessionId: string): boolean => {
|
|
300
|
+
const ctx = sessions.get(sessionId)
|
|
301
|
+
const text = buffers.get(sessionId) ?? ''
|
|
302
|
+
if (!ctx || text.length === 0) return false
|
|
303
|
+
buffers.set(sessionId, '')
|
|
304
|
+
flushedCounts.set(sessionId, (flushedCounts.get(sessionId) ?? 0) + 1)
|
|
305
|
+
const content = buildTextContent(text)
|
|
306
|
+
const tail = (sendQueue.get(sessionId) ?? Promise.resolve()).then(async () => {
|
|
307
|
+
try {
|
|
308
|
+
await client.sendMessage({
|
|
309
|
+
roomId: ctx.roomId,
|
|
310
|
+
asUserId: ctx.agent.userId,
|
|
311
|
+
content,
|
|
312
|
+
threadRoot: ctx.threadRoot,
|
|
313
|
+
})
|
|
314
|
+
} catch (err) {
|
|
315
|
+
console.warn(`[matrix:${ctx.agent.name}] sendMessage flush failed:`, err)
|
|
316
|
+
}
|
|
317
|
+
})
|
|
318
|
+
sendQueue.set(sessionId, tail)
|
|
319
|
+
return true
|
|
320
|
+
}
|
|
98
321
|
|
|
99
322
|
agents.onEvent = async (name, event: AgentEvent) => {
|
|
100
323
|
const ctx = sessions.get(event.sessionId)
|
|
101
324
|
if (!ctx) {
|
|
102
|
-
|
|
325
|
+
// available_commands_update is advertised during ensureSession (session
|
|
326
|
+
// load/new), before runTurn calls sessions.set — so the ctx isn't there
|
|
327
|
+
// yet. Stash the latest roster and replay it once runTurn registers the
|
|
328
|
+
// ctx. Other event types arriving without a ctx are genuinely orphaned
|
|
329
|
+
// (e.g. replayed history for a thread we're not handling) — drop them.
|
|
330
|
+
if (event.type === 'available_commands') {
|
|
331
|
+
pendingCommands.set(event.sessionId, event)
|
|
332
|
+
} else {
|
|
333
|
+
console.warn(`[matrix:${name}] no session ctx for ${event.sessionId}`)
|
|
334
|
+
}
|
|
103
335
|
return
|
|
104
336
|
}
|
|
105
337
|
|
|
106
338
|
if (event.type === 'agent_message_chunk') {
|
|
107
|
-
const block = event.content as { type?: string; text?: string }
|
|
339
|
+
const block = event.content as { type?: string; text?: string; data?: string; mimeType?: string }
|
|
108
340
|
if (block.type === 'text' && typeof block.text === 'string') {
|
|
109
|
-
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
// - a change in messageId — opencode streams each assistant message
|
|
116
|
-
// under its own id and emits no delimiter chunk between them, and the
|
|
117
|
-
// first token of the new message has no leading space, so without
|
|
118
|
-
// this they weld together ("…one.🅿️").
|
|
341
|
+
// A change in ACP messageId marks the previous assistant message as
|
|
342
|
+
// complete. opencode streams each assistant message under its own id
|
|
343
|
+
// with no delimiter chunk between them, so a change here is the only
|
|
344
|
+
// boundary signal. Flush the previous message as its own Matrix
|
|
345
|
+
// message — each ACP message lands separately (and interleaves with
|
|
346
|
+
// tool_call/plan events) instead of welding into one turn-end blob.
|
|
119
347
|
const prevMessageId = bufferMessageIds.get(event.sessionId)
|
|
120
348
|
const messageChanged =
|
|
121
349
|
event.messageId !== undefined &&
|
|
122
350
|
prevMessageId !== undefined &&
|
|
123
351
|
event.messageId !== prevMessageId
|
|
124
|
-
const needsBreak =
|
|
125
|
-
current.length > 0 && (block.text === '' || messageChanged)
|
|
126
|
-
const prefix = needsBreak ? '\n\n' : ''
|
|
127
|
-
buffers.set(event.sessionId, current + prefix + block.text)
|
|
128
352
|
if (event.messageId !== undefined)
|
|
129
353
|
bufferMessageIds.set(event.sessionId, event.messageId)
|
|
354
|
+
// flushBuffer clears the buffer synchronously, so the new message's
|
|
355
|
+
// text below starts fresh.
|
|
356
|
+
if (messageChanged) flushBuffer(event.sessionId)
|
|
357
|
+
// Within a single message, tokens carry their own leading spaces, so we
|
|
358
|
+
// concatenate raw. An empty chunk (some agents emit one between blocks,
|
|
359
|
+
// e.g. after a tool call within the same message) is a paragraph break.
|
|
360
|
+
const current = buffers.get(event.sessionId) ?? ''
|
|
361
|
+
const needsBreak = current.length > 0 && block.text === ''
|
|
362
|
+
const prefix = needsBreak ? '\n\n' : ''
|
|
363
|
+
buffers.set(event.sessionId, current + prefix + block.text)
|
|
364
|
+
} else if (
|
|
365
|
+
block.type === 'image' &&
|
|
366
|
+
typeof block.data === 'string' &&
|
|
367
|
+
typeof block.mimeType === 'string' &&
|
|
368
|
+
mediaClient
|
|
369
|
+
) {
|
|
370
|
+
// Outbound agent image: upload immediately and send as a threaded m.image.
|
|
371
|
+
const ctx = sessions.get(event.sessionId)
|
|
372
|
+
if (ctx) {
|
|
373
|
+
const bytes = Buffer.from(block.data, 'base64')
|
|
374
|
+
const ext = (block.mimeType.split('/')[1] ?? 'png').replace(/[^a-z0-9]/gi, '')
|
|
375
|
+
const filename = `image.${ext}`
|
|
376
|
+
void mediaClient
|
|
377
|
+
.upload({ data: bytes, contentType: block.mimeType, filename, asUserId: ctx.agent.userId })
|
|
378
|
+
.then(({ content_uri }) =>
|
|
379
|
+
client.sendMessage({
|
|
380
|
+
roomId: ctx.roomId,
|
|
381
|
+
asUserId: ctx.agent.userId,
|
|
382
|
+
threadRoot: ctx.threadRoot,
|
|
383
|
+
content: {
|
|
384
|
+
msgtype: 'm.image',
|
|
385
|
+
body: filename,
|
|
386
|
+
url: content_uri,
|
|
387
|
+
info: { mimetype: block.mimeType, size: bytes.length },
|
|
388
|
+
},
|
|
389
|
+
}),
|
|
390
|
+
)
|
|
391
|
+
.catch((err) => {
|
|
392
|
+
console.warn(`[matrix:${name}] outbound image upload failed:`, err)
|
|
393
|
+
void sendMediaError(ctx, err, 'agent image upload failed', client)
|
|
394
|
+
})
|
|
395
|
+
}
|
|
130
396
|
} else {
|
|
131
397
|
console.warn(`[matrix:${name}] dropped chunk block type=${block.type}`, block)
|
|
132
398
|
}
|
|
133
399
|
return
|
|
134
400
|
}
|
|
135
401
|
|
|
402
|
+
// An out-of-band event (tool_call / tool_call_update / plan) after some
|
|
403
|
+
// buffered text means that assistant message is complete — flush it first
|
|
404
|
+
// so it lands before this event on the wire, preserving interleaving.
|
|
405
|
+
flushBuffer(event.sessionId)
|
|
406
|
+
|
|
136
407
|
const eventType =
|
|
137
408
|
event.type === 'tool_call'
|
|
138
|
-
? '
|
|
409
|
+
? 'dev.zooid.tool_call'
|
|
139
410
|
: event.type === 'tool_call_update'
|
|
140
|
-
? '
|
|
141
|
-
:
|
|
411
|
+
? 'dev.zooid.tool_call_update'
|
|
412
|
+
: event.type === 'available_commands'
|
|
413
|
+
? 'dev.zooid.available_commands_update'
|
|
414
|
+
: 'dev.zooid.plan'
|
|
142
415
|
const body =
|
|
143
416
|
event.type === 'tool_call'
|
|
144
417
|
? toToolCallBody(event)
|
|
145
418
|
: event.type === 'tool_call_update'
|
|
146
419
|
? toUpdateBody(event)
|
|
147
|
-
:
|
|
420
|
+
: event.type === 'available_commands'
|
|
421
|
+
? toAvailableCommandsBody(event)
|
|
422
|
+
: toPlanBody(event)
|
|
148
423
|
body['m.relates_to'] = { rel_type: 'm.thread', event_id: ctx.threadRoot }
|
|
149
424
|
const tail = (sendQueue.get(event.sessionId) ?? Promise.resolve()).then(async () => {
|
|
150
425
|
try {
|
|
@@ -185,7 +460,7 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
185
460
|
void client.sendCustomEvent({
|
|
186
461
|
roomId: ctx.roomId,
|
|
187
462
|
asUserId: ctx.agent.userId,
|
|
188
|
-
eventType: '
|
|
463
|
+
eventType: 'dev.zooid.approval_request',
|
|
189
464
|
content,
|
|
190
465
|
})
|
|
191
466
|
})
|
|
@@ -227,7 +502,28 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
227
502
|
)
|
|
228
503
|
continue
|
|
229
504
|
}
|
|
230
|
-
if (evt.type === '
|
|
505
|
+
if (evt.type === 'm.room.member' && evt.content?.membership === 'invite') {
|
|
506
|
+
const target = evt.state_key
|
|
507
|
+
const inviter = evt.sender
|
|
508
|
+
if (
|
|
509
|
+
target &&
|
|
510
|
+
evt.room_id &&
|
|
511
|
+
ourBotUserIds.has(target) &&
|
|
512
|
+
(!inviter || !ourBotUserIds.has(inviter))
|
|
513
|
+
) {
|
|
514
|
+
console.log(
|
|
515
|
+
`[matrix] declining ad-hoc invite for ${target} in ${evt.room_id} ` +
|
|
516
|
+
`from ${inviter ?? 'unknown'}`,
|
|
517
|
+
)
|
|
518
|
+
await client
|
|
519
|
+
.leaveRoom(evt.room_id, target, { reason: DECLINE_REASON })
|
|
520
|
+
.catch((err) =>
|
|
521
|
+
console.warn(`[matrix] leaveRoom(${evt.room_id}, ${target}) failed:`, err),
|
|
522
|
+
)
|
|
523
|
+
}
|
|
524
|
+
continue
|
|
525
|
+
}
|
|
526
|
+
if (evt.type === 'dev.zooid.session_reset') {
|
|
231
527
|
// Spec § /clear: room-scope reset is unsupported. Only thread-scoped
|
|
232
528
|
// resets carry a thread relation; drop bare room-level resets silently.
|
|
233
529
|
const relates = evt.content?.['m.relates_to'] as
|
|
@@ -236,10 +532,10 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
236
532
|
const threadRoot =
|
|
237
533
|
relates?.rel_type === 'm.thread' && relates.event_id ? relates.event_id : undefined
|
|
238
534
|
if (!threadRoot) {
|
|
239
|
-
console.log('[matrix] dropping
|
|
535
|
+
console.log('[matrix] dropping dev.zooid.session_reset without thread relation')
|
|
240
536
|
continue
|
|
241
537
|
}
|
|
242
|
-
console.log(`[matrix] inbound
|
|
538
|
+
console.log(`[matrix] inbound dev.zooid.session_reset in ${evt.room_id} thread=${threadRoot}`)
|
|
243
539
|
for (const a of bindings) {
|
|
244
540
|
agents.endSession(a.name, threadRoot)
|
|
245
541
|
}
|
|
@@ -249,7 +545,7 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
249
545
|
// the most-recently-posting agent under the same sessionKey.
|
|
250
546
|
continue
|
|
251
547
|
}
|
|
252
|
-
if (evt.type === '
|
|
548
|
+
if (evt.type === 'dev.zooid.interrupt') {
|
|
253
549
|
const content = (evt.content ?? {}) as { session_id?: string; reason?: string }
|
|
254
550
|
// Thread-relation form (client-friendly): /interrupt in a thread sends
|
|
255
551
|
// an empty event with `m.relates_to: thread/<root>`. Cancel every
|
|
@@ -279,7 +575,7 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
279
575
|
}
|
|
280
576
|
// Legacy form: explicit session_id in content.
|
|
281
577
|
if (!content.session_id) {
|
|
282
|
-
console.warn(`[matrix]
|
|
578
|
+
console.warn(`[matrix] dev.zooid.interrupt missing session_id (event_id=${evt.event_id})`)
|
|
283
579
|
continue
|
|
284
580
|
}
|
|
285
581
|
const ctx = sessions.get(content.session_id)
|
|
@@ -295,7 +591,7 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
295
591
|
})
|
|
296
592
|
continue
|
|
297
593
|
}
|
|
298
|
-
if (evt.type === '
|
|
594
|
+
if (evt.type === 'dev.zooid.approval_response') {
|
|
299
595
|
const content = (evt.content ?? {}) as {
|
|
300
596
|
approval_id?: string
|
|
301
597
|
session_id?: string
|
|
@@ -315,6 +611,29 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
315
611
|
continue
|
|
316
612
|
}
|
|
317
613
|
logInbound(evt)
|
|
614
|
+
|
|
615
|
+
// Capture media events in the pending store; never route them to agents.
|
|
616
|
+
if (
|
|
617
|
+
evt.type === 'm.room.message' &&
|
|
618
|
+
isMediaMsgtype(evt.content?.msgtype) &&
|
|
619
|
+
evt.room_id &&
|
|
620
|
+
evt.event_id &&
|
|
621
|
+
evt.sender &&
|
|
622
|
+
evt.content?.url &&
|
|
623
|
+
!bindings.some((b) => b.userId === evt.sender)
|
|
624
|
+
) {
|
|
625
|
+
pendingMedia.add(evt.room_id, inboundThreadRoot(evt), {
|
|
626
|
+
eventId: evt.event_id,
|
|
627
|
+
sender: evt.sender,
|
|
628
|
+
msgtype: evt.content.msgtype as string,
|
|
629
|
+
body: (evt.content.body as string | undefined) ?? '',
|
|
630
|
+
filename: evt.content.filename as string | undefined,
|
|
631
|
+
url: evt.content.url as string,
|
|
632
|
+
info: evt.content.info as PendingMediaItem['info'],
|
|
633
|
+
})
|
|
634
|
+
continue
|
|
635
|
+
}
|
|
636
|
+
|
|
318
637
|
// Agent-promotion: top-level inbound event becomes the thread root.
|
|
319
638
|
// For in-thread messages the existing root is preserved.
|
|
320
639
|
const promotedRoot = inboundThreadRoot(evt) ?? evt.event_id
|
|
@@ -396,10 +715,10 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
396
715
|
.sendCustomEvent({
|
|
397
716
|
roomId: evt.room_id,
|
|
398
717
|
asUserId: a.userId,
|
|
399
|
-
eventType: '
|
|
718
|
+
eventType: 'dev.zooid.error',
|
|
400
719
|
content: body,
|
|
401
720
|
})
|
|
402
|
-
.catch((e) => console.warn(`[matrix:${a.name}]
|
|
721
|
+
.catch((e) => console.warn(`[matrix:${a.name}] dev.zooid.error send failed:`, e))
|
|
403
722
|
})
|
|
404
723
|
}
|
|
405
724
|
}
|
|
@@ -437,6 +756,15 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
437
756
|
sessions.set(sessionId, { agent, roomId: evt.room_id, threadRoot })
|
|
438
757
|
buffers.set(sessionId, '')
|
|
439
758
|
bufferMessageIds.delete(sessionId)
|
|
759
|
+
flushedCounts.set(sessionId, 0)
|
|
760
|
+
// Commands the shim advertised during ensureSession (session load/new)
|
|
761
|
+
// arrived before the ctx above existed and were stashed — replay the latest
|
|
762
|
+
// now that the session is fully registered, so the palette actually fills.
|
|
763
|
+
const stashedCommands = pendingCommands.get(sessionId)
|
|
764
|
+
if (stashedCommands) {
|
|
765
|
+
pendingCommands.delete(sessionId)
|
|
766
|
+
void agents.onEvent?.(agent.name, stashedCommands)
|
|
767
|
+
}
|
|
440
768
|
|
|
441
769
|
const roomId = evt.room_id
|
|
442
770
|
const TYPING_TTL_MS = 30_000
|
|
@@ -461,10 +789,33 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
461
789
|
try {
|
|
462
790
|
const rawBody = evt.content?.body ?? ''
|
|
463
791
|
const promptText = stripMention(rawBody, agent.userId)
|
|
792
|
+
|
|
793
|
+
// Drain pending media for this sender+thread and prepend as ACP content blocks.
|
|
794
|
+
const pendingItems = pendingMedia.drain(
|
|
795
|
+
evt.room_id,
|
|
796
|
+
inboundThreadRoot(evt),
|
|
797
|
+
evt.sender ?? '',
|
|
798
|
+
)
|
|
799
|
+
const { blocks, pathLines } = await buildMediaBlocks(pendingItems, {
|
|
800
|
+
agent,
|
|
801
|
+
media: mediaClient,
|
|
802
|
+
writeAttachmentFn,
|
|
803
|
+
onError: (item, err) => {
|
|
804
|
+
console.warn(`[matrix:${agent.name}] media_failed for ${item.body}:`, err)
|
|
805
|
+
void sendMediaError(
|
|
806
|
+
{ agent, roomId: evt.room_id!, threadRoot },
|
|
807
|
+
err,
|
|
808
|
+
`Could not process attachment: ${item.body}`,
|
|
809
|
+
client,
|
|
810
|
+
)
|
|
811
|
+
},
|
|
812
|
+
})
|
|
813
|
+
|
|
814
|
+
const fullPromptText = [promptText, ...pathLines].filter(Boolean).join('\n')
|
|
464
815
|
await agents.prompt(agent.name, {
|
|
465
816
|
threadId: sessionKey,
|
|
466
817
|
channelId: evt.room_id,
|
|
467
|
-
content: [{ type: 'text', text:
|
|
818
|
+
content: [...blocks, { type: 'text', text: fullPromptText }],
|
|
468
819
|
})
|
|
469
820
|
// Drain: the prompt promise resolves on the stopReason response, but
|
|
470
821
|
// trailing chunks may still arrive (see DRAIN_* above). Wait until the
|
|
@@ -481,44 +832,23 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
481
832
|
while (drainQuietMs > 0 && Date.now() - drainStart < drainMaxMs) {
|
|
482
833
|
await delay(drainQuietMs)
|
|
483
834
|
const next = buffers.get(sessionId) ?? ''
|
|
484
|
-
// Stop
|
|
485
|
-
//
|
|
486
|
-
// empty buffer means the
|
|
487
|
-
|
|
835
|
+
// Stop when the buffer is quiet (unchanged) and either it holds the
|
|
836
|
+
// final message to flush, or we already flushed a message this turn
|
|
837
|
+
// (so an empty, quiet buffer means the turn is genuinely done — the
|
|
838
|
+
// last message was flushed mid-stream). An unchanged *empty* buffer
|
|
839
|
+
// with nothing flushed yet means the stream hasn't started; keep
|
|
840
|
+
// waiting up to drainMaxMs.
|
|
841
|
+
if (next === drained && (next.length > 0 || (flushedCounts.get(sessionId) ?? 0) > 0))
|
|
842
|
+
break
|
|
488
843
|
drained = next
|
|
489
844
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
// Only attach formatted_body when it adds rich-text the plain body
|
|
498
|
-
// can't carry. marked wraps plain prose in <p>…</p>; if that's all
|
|
499
|
-
// we'd add, skip — most clients render `body` better than a stripped
|
|
500
|
-
// re-encode.
|
|
501
|
-
if (html) {
|
|
502
|
-
const escapedPlain =
|
|
503
|
-
'<p>' +
|
|
504
|
-
text
|
|
505
|
-
.replace(/&/g, '&')
|
|
506
|
-
.replace(/</g, '<')
|
|
507
|
-
.replace(/>/g, '>') +
|
|
508
|
-
'</p>'
|
|
509
|
-
const norm = (s: string) => s.replace(/\s+/g, ' ').trim()
|
|
510
|
-
if (norm(html) !== norm(escapedPlain)) {
|
|
511
|
-
content.format = 'org.matrix.custom.html'
|
|
512
|
-
content.formatted_body = html
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
await client.sendMessage({
|
|
516
|
-
roomId: evt.room_id,
|
|
517
|
-
asUserId: agent.userId,
|
|
518
|
-
content,
|
|
519
|
-
threadRoot, // every reply threads, full stop
|
|
520
|
-
})
|
|
521
|
-
} else {
|
|
845
|
+
// Flush the final assistant message — the one with no following messageId
|
|
846
|
+
// change or out-of-band event to have triggered an earlier flush.
|
|
847
|
+
flushBuffer(sessionId)
|
|
848
|
+
// Wait for every queued send (mid-turn flushes, tool/plan events, final
|
|
849
|
+
// flush) to settle before tearing the session down.
|
|
850
|
+
await (sendQueue.get(sessionId) ?? Promise.resolve())
|
|
851
|
+
if ((flushedCounts.get(sessionId) ?? 0) === 0) {
|
|
522
852
|
console.warn(
|
|
523
853
|
`[matrix:${agent.name}] turn finished with empty buffer (session=${sessionId}); nothing sent to ${evt.room_id}`,
|
|
524
854
|
)
|
|
@@ -529,6 +859,8 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
|
|
|
529
859
|
await safePresence('online')
|
|
530
860
|
buffers.delete(sessionId)
|
|
531
861
|
bufferMessageIds.delete(sessionId)
|
|
862
|
+
flushedCounts.delete(sessionId)
|
|
863
|
+
sendQueue.delete(sessionId)
|
|
532
864
|
}
|
|
533
865
|
}
|
|
534
866
|
|
|
@@ -36,7 +36,7 @@ describe('buildWorkforceRoster', () => {
|
|
|
36
36
|
})
|
|
37
37
|
|
|
38
38
|
describe('publishWorkforce', () => {
|
|
39
|
-
it('PUTs
|
|
39
|
+
it('PUTs dev.zooid.workforce state event on the configured space', async () => {
|
|
40
40
|
const fetch = vi.fn(async () => new Response('{}', { status: 200 }))
|
|
41
41
|
const client = new MatrixClient({
|
|
42
42
|
homeserver: 'https://hs.zoon.local',
|
|
@@ -54,7 +54,7 @@ describe('publishWorkforce', () => {
|
|
|
54
54
|
expect(fetch).toHaveBeenCalledTimes(1)
|
|
55
55
|
const [url, init] = fetch.mock.calls[0]!
|
|
56
56
|
expect(url).toBe(
|
|
57
|
-
'https://hs.zoon.local/_matrix/client/v3/rooms/!space%3Azoon.local/state/
|
|
57
|
+
'https://hs.zoon.local/_matrix/client/v3/rooms/!space%3Azoon.local/state/dev.zooid.workforce/?user_id=%40zooid%3Azoon.local',
|
|
58
58
|
)
|
|
59
59
|
expect(init?.method).toBe('PUT')
|
|
60
60
|
expect(init?.headers).toMatchObject({ Authorization: 'Bearer as-tok' })
|
|
@@ -28,7 +28,7 @@ export async function publishWorkforce(opts: PublishOpts): Promise<void> {
|
|
|
28
28
|
await opts.client.sendStateEvent({
|
|
29
29
|
roomId: opts.spaceRoomId,
|
|
30
30
|
asUserId: opts.asUserId,
|
|
31
|
-
eventType: '
|
|
31
|
+
eventType: 'dev.zooid.workforce',
|
|
32
32
|
stateKey: '',
|
|
33
33
|
content: buildWorkforceRoster(opts.agents) as unknown as Record<string, unknown>,
|
|
34
34
|
})
|