@zooid/transport-matrix 0.7.4 → 0.8.0

package/src/transport.test.ts

@@ -1312,3 +1312,245 @@ describe('full loop integration', () => {
     )
   })
 })
+
+// ─── Media pipeline tests ────────────────────────────────────────────────────
+
+const TINY_PNG_B64 =
+  'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='
+const TINY_PNG = Buffer.from(TINY_PNG_B64, 'base64')
+
+function fakeMedia() {
+  return {
+    download: vi.fn(async () => ({ data: new Uint8Array(TINY_PNG), contentType: 'image/png' })),
+    upload: vi.fn(async () => ({ content_uri: 'mxc://localhost/up1' })),
+  }
+}
+
+const workspaceBinding = {
+  ...baseAgents[0],
+  workspaceDir: '/tmp/ws',
+  agentWorkspacePath: '/workspace',
+}
+
+function makeMediaTransport(opts: {
+  media?: ReturnType<typeof fakeMedia>
+  writeAttachmentFn?: unknown
+} = {}) {
+  const { reg, finishPrompt } = fakeRegistry()
+  const approvals = fakeApprovals()
+  const client = fakeClient()
+  const transport = createMatrixTransport({
+    agents: reg as never,
+    approvals: approvals as never,
+    client: client as never,
+    bindings: [workspaceBinding],
+    hsToken: 'hs-secret',
+    drainQuietMs: 0,
+    media: opts.media as never,
+    writeAttachmentFn: opts.writeAttachmentFn as never,
+  })
+  return { transport, agents: reg, client, finishPrompt }
+}
+
+function imageEvent(over: {
+  size?: number
+  mimetype?: string
+  msgtype?: string
+  body?: string
+  eventId?: string
+} = {}) {
+  return {
+    type: 'm.room.message',
+    event_id: over.eventId ?? '$media1',
+    room_id: '!r:example.com',
+    sender: '@alice:example.com',
+    content: {
+      msgtype: over.msgtype ?? 'm.image',
+      body: over.body ?? 'dog.png',
+      url: 'mxc://localhost/abc',
+      info: { mimetype: over.mimetype ?? 'image/png', size: over.size ?? 67 },
+    },
+  }
+}
+
+function mentionMsg(body: string, eventId = '$text1') {
+  return {
+    type: 'm.room.message',
+    event_id: eventId,
+    room_id: '!r:example.com',
+    sender: '@alice:example.com',
+    content: {
+      msgtype: 'm.text',
+      body: `@architect ${body}`,
+      'm.mentions': { user_ids: ['@architect:example.com'] },
+    },
+  }
+}
+
+describe('inbound media', () => {
+  it('media events do not trigger a turn; m.text from the same sender drains them inline', async () => {
+    const media = fakeMedia()
+    const { transport, agents } = makeMediaTransport({ media })
+
+    // Image event: no turn fired
+    await postTxn(transport.app, { events: [imageEvent()] })
+    await settleTurn()
+    expect(agents.prompt).not.toHaveBeenCalled()
+
+    // m.text mention from same sender: turn fires, image block prepended
+    agents.prompt.mockImplementation(async (_name: string, p: { content: unknown[] }) => {
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'text', text: 'got it' },
+      })
+      return { stopReason: 'end_turn' as const }
+    })
+    await postTxn(transport.app, { events: [mentionMsg('look at this')] })
+    await settleTurn()
+
+    expect(agents.prompt).toHaveBeenCalledOnce()
+    const content = (agents.prompt.mock.calls[0][1] as { content: unknown[] }).content
+    expect(content[0]).toMatchObject({ type: 'image', data: TINY_PNG_B64, mimeType: 'image/png' })
+    expect((content[1] as { type: string; text: string }).type).toBe('text')
+    expect(media.download).toHaveBeenCalledOnce()
+  })
+
+  it('routes an oversized image to the file path with a resource_link block and prose line', async () => {
+    const media = fakeMedia()
+    const writeAttachmentFn = vi.fn(() => ({
+      hostPath: '/tmp/ws/.zooid/attachments/media1/dog.png',
+      agentPath: '/workspace/.zooid/attachments/media1/dog.png',
+    }))
+    const { transport, agents } = makeMediaTransport({ media, writeAttachmentFn })
+
+    agents.prompt.mockResolvedValue({ stopReason: 'end_turn' as const })
+    await postTxn(transport.app, { events: [imageEvent({ size: 600_000 })] }) // > MAX_INLINE_IMAGE_BYTES
+    await postTxn(transport.app, { events: [mentionMsg('summarize')] })
+    await settleTurn()
+
+    const content = (agents.prompt.mock.calls[0][1] as { content: unknown[] }).content
+    expect(content[0]).toMatchObject({
+      type: 'resource_link',
+      uri: 'file:///workspace/.zooid/attachments/media1/dog.png',
+      name: 'dog.png',
+    })
+    expect((content[1] as { text: string }).text).toContain(
+      '/workspace/.zooid/attachments/media1/dog.png',
+    )
+  })
+
+  it('routes m.file to the workspace regardless of size', async () => {
+    const media = fakeMedia()
+    const writeAttachmentFn = vi.fn(() => ({
+      hostPath: '/tmp/ws/.zooid/attachments/media1/report.pdf',
+      agentPath: '/workspace/.zooid/attachments/media1/report.pdf',
+    }))
+    const { transport, agents } = makeMediaTransport({ media, writeAttachmentFn })
+
+    agents.prompt.mockResolvedValue({ stopReason: 'end_turn' as const })
+    await postTxn(transport.app, {
+      events: [imageEvent({ msgtype: 'm.file', body: 'report.pdf', mimetype: 'application/pdf' })],
+    })
+    await postTxn(transport.app, { events: [mentionMsg('read it')] })
+    await settleTurn()
+
+    expect(writeAttachmentFn).toHaveBeenCalledOnce()
+    const content = (agents.prompt.mock.calls[0][1] as { content: unknown[] }).content
+    expect((content[0] as { type: string }).type).toBe('resource_link')
+  })
+
+  it('emits eco.zoon.error (code media_failed) when download fails, still runs the turn text-only', async () => {
+    const media = fakeMedia()
+    media.download.mockRejectedValueOnce(new Error('download boom'))
+    const { transport, agents, client } = makeMediaTransport({ media })
+
+    agents.prompt.mockImplementation(async () => {
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'text', text: 'ok' },
+      })
+      return { stopReason: 'end_turn' as const }
+    })
+    await postTxn(transport.app, { events: [imageEvent()] })
+    await postTxn(transport.app, { events: [mentionMsg('look')] })
+    await settleTurn()
+
+    expect(client.sendCustomEvent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        eventType: 'eco.zoon.error',
+        content: expect.objectContaining({ code: 'media_failed' }),
+      }),
+    )
+    const content = (agents.prompt.mock.calls[0][1] as { content: unknown[] }).content
+    expect(content).toHaveLength(1)
+    expect((content[0] as { type: string }).type).toBe('text')
+  })
+})
+
+describe('outbound agent images', () => {
+  it('uploads an image chunk and sends a threaded m.image as the agent user', async () => {
+    const media = fakeMedia()
+    const { transport, agents, client } = makeMediaTransport({ media })
+
+    agents.prompt.mockImplementation(async () => {
+      // Emit image block during prompt
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'image', data: TINY_PNG_B64, mimeType: 'image/png' },
+      })
+      // Also emit text block so the turn isn't empty
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'text', text: 'here is the image' },
+      })
+      return { stopReason: 'end_turn' as const }
+    })
+
+    await postTxn(transport.app, { events: [mentionMsg('show me an image')] })
+    await settleTurn()
+    // Give async upload/send a moment to settle
+    await new Promise((r) => setTimeout(r, 10))
+
+    expect(media.upload).toHaveBeenCalledWith(
+      expect.objectContaining({ contentType: 'image/png', asUserId: '@architect:example.com' }),
+    )
+    expect(client.sendMessage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        asUserId: '@architect:example.com',
+        content: expect.objectContaining({
+          msgtype: 'm.image',
+          url: 'mxc://localhost/up1',
+          info: expect.objectContaining({ mimetype: 'image/png', size: TINY_PNG.length }),
+        }),
+      }),
+    )
+  })
+
+  it('does not throw when an audio block arrives (non-goal — warn and drop)', async () => {
+    const media = fakeMedia()
+    const { transport, agents } = makeMediaTransport({ media })
+
+    agents.prompt.mockImplementation(async () => {
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'audio', data: 'AAAA', mimeType: 'audio/wav' },
+      })
+      agents.onEvent('architect', {
+        type: 'agent_message_chunk',
+        sessionId: 'sess-$text1',
+        content: { type: 'text', text: 'ok' },
+      })
+      return { stopReason: 'end_turn' as const }
+    })
+
+    await expect(
+      postTxn(transport.app, { events: [mentionMsg('test audio')] }),
+    ).resolves.not.toThrow()
+    await settleTurn()
+  })
+})

package/src/transport.ts

@@ -1,14 +1,38 @@
 import { Hono } from 'hono'
 import { timingSafeEqual } from 'node:crypto'
 import type { AcpRegistry, ApprovalCorrelator, RegisteredApproval } from '@zooid/core'
-import type { AgentEvent } from '@zooid/acp-client'
+import type { AgentEvent, ContentBlock } from '@zooid/acp-client'
 import { MatrixClient } from './matrix-client.js'
 import { BotPool } from './bot-pool.js'
-import { route, type AgentBinding, type ThreadState } from './router.js'
+import { route, isMediaMsgtype, type AgentBinding, type ThreadState } from './router.js'
 import { stripMention, extractMentions } from './mentions.js'
 import { toToolCallBody, toUpdateBody, toPlanBody, toErrorBody } from './event-encoders.js'
 import { classify } from '@zooid/acp-client'
 import { toMatrixHtml } from './markdown-to-matrix-html.js'
+import {
+  PendingMediaStore,
+  type PendingMediaItem,
+} from './pending-media.js'
+import {
+  MediaClient,
+  MAX_INLINE_IMAGE_BYTES,
+  INLINE_IMAGE_MIMES,
+} from './media-client.js'
+import { writeAttachment } from './attachments.js'
+
+export interface MediaClientLike {
+  download(input: {
+    mxcUri: string
+    asUserId: string
+    maxBytes?: number
+  }): Promise<{ data: Uint8Array; contentType: string }>
+  upload(input: {
+    data: Uint8Array
+    contentType: string
+    filename?: string
+    asUserId: string
+  }): Promise<{ content_uri: string }>
+}
 
 export interface CreateMatrixTransportOptions {
   agents: AcpRegistry
@@ -24,6 +48,10 @@ export interface CreateMatrixTransportOptions {
   drainQuietMs?: number
   /** Hard cap on the post-turn drain. Defaults to `DRAIN_MAX_MS`. */
   drainMaxMs?: number
+  /** Injected media client for downloading/uploading Matrix media. */
+  media?: MediaClientLike
+  /** Injected attachment writer (defaults to the real writeAttachment). */
+  writeAttachmentFn?: typeof writeAttachment
 }
 
 interface SessionContext {
@@ -47,6 +75,120 @@ interface MatrixEvent {
 }
 
 const STARTUP_GRACE_MS = 5_000
+
+interface MediaBlocksResult {
+  blocks: ContentBlock[]
+  pathLines: string[]
+}
+
+async function buildMediaBlocks(
+  items: PendingMediaItem[],
+  opts: {
+    agent: AgentBinding
+    media: MediaClientLike | undefined
+    writeAttachmentFn: typeof writeAttachment
+    onError: (item: PendingMediaItem, err: unknown) => void
+  },
+): Promise<MediaBlocksResult> {
+  const blocks: ContentBlock[] = []
+  const pathLines: string[] = []
+
+  if (!opts.media || items.length === 0) return { blocks, pathLines }
+
+  for (const item of items) {
+    try {
+      const isInlineCandidate =
+        item.msgtype === 'm.image' &&
+        INLINE_IMAGE_MIMES.includes(item.info?.mimetype ?? '') &&
+        (item.info?.size === undefined || item.info.size <= MAX_INLINE_IMAGE_BYTES)
+
+      if (isInlineCandidate) {
+        const { data, contentType } = await opts.media.download({
+          mxcUri: item.url,
+          asUserId: opts.agent.userId,
+        })
+        // Double-check actual size (info can lie)
+        if (data.byteLength <= MAX_INLINE_IMAGE_BYTES) {
+          blocks.push({
+            type: 'image',
+            data: Buffer.from(data).toString('base64'),
+            mimeType: contentType,
+          })
+          continue
+        }
+        // Actual size exceeded cap — fall through to file route with the already-downloaded bytes
+        if (opts.agent.workspaceDir) {
+          const paths = opts.writeAttachmentFn({
+            workspaceDir: opts.agent.workspaceDir,
+            agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
+            eventId: item.eventId,
+            filename: item.filename ?? item.body,
+            data,
+          })
+          blocks.push({
+            type: 'resource_link',
+            uri: `file://${paths.agentPath}`,
+            name: item.filename ?? item.body,
+          })
+          pathLines.push(`Attached file: ${paths.agentPath}`)
+        }
+      } else {
+        // File route (m.file, m.video, m.audio, or oversized image)
+        if (!opts.agent.workspaceDir) continue
+        const { data } = await opts.media.download({
+          mxcUri: item.url,
+          asUserId: opts.agent.userId,
+        })
+        const paths = opts.writeAttachmentFn({
+          workspaceDir: opts.agent.workspaceDir,
+          agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
+          eventId: item.eventId,
+          filename: item.filename ?? item.body,
+          data,
+        })
+        blocks.push({
+          type: 'resource_link',
+          uri: `file://${paths.agentPath}`,
+          name: item.filename ?? item.body,
+          mimeType: item.info?.mimetype,
+          size: item.info?.size,
+        })
+        pathLines.push(`Attached file: ${paths.agentPath}`)
+      }
+    } catch (err) {
+      opts.onError(item, err)
+    }
+  }
+
+  return { blocks, pathLines }
+}
+
+async function sendMediaError(
+  ctx: { agent: AgentBinding; roomId: string; threadRoot: string },
+  _err: unknown,
+  message: string,
+  client: MatrixClient,
+): Promise<void> {
+  await client
+    .sendCustomEvent({
+      roomId: ctx.roomId,
+      asUserId: ctx.agent.userId,
+      eventType: 'eco.zoon.error',
+      content: toErrorBody(
+        {
+          kind: 'error' as const,
+          agentId: ctx.agent.name,
+          sessionId: null,
+          turnId: null,
+          code: 'media_failed',
+          message: message.slice(0, 250),
+          transient: false,
+        },
+        ctx.threadRoot,
+      ),
+    })
+    .catch((e) => console.warn(`[matrix:${ctx.agent.name}] eco.zoon.error send failed:`, e))
+}
 const SEEN_EVENT_CAP = 5_000
 
 // ACP only guarantees that an agent flushes pending `session/update`
@@ -77,6 +219,9 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
   const { agents, approvals, client, bindings, hsToken, adminUserId } = opts
   const drainQuietMs = opts.drainQuietMs ?? DRAIN_QUIET_MS
   const drainMaxMs = opts.drainMaxMs ?? DRAIN_MAX_MS
+  const mediaClient = opts.media
+  const writeAttachmentFn = opts.writeAttachmentFn ?? writeAttachment
+  const pendingMedia = new PendingMediaStore()
   const pool = new BotPool(client, bindings)
   const sessions = new Map<string, SessionContext>()
   const buffers = new Map<string, string>()
@@ -104,7 +249,7 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
     }
 
     if (event.type === 'agent_message_chunk') {
-      const block = event.content as { type?: string; text?: string }
+      const block = event.content as { type?: string; text?: string; data?: string; mimeType?: string }
       if (block.type === 'text' && typeof block.text === 'string') {
         const current = buffers.get(event.sessionId) ?? ''
         // Within a message, tokens carry their own leading spaces, so we
@@ -127,6 +272,38 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
         buffers.set(event.sessionId, current + prefix + block.text)
         if (event.messageId !== undefined)
           bufferMessageIds.set(event.sessionId, event.messageId)
+      } else if (
+        block.type === 'image' &&
+        typeof block.data === 'string' &&
+        typeof block.mimeType === 'string' &&
+        mediaClient
+      ) {
+        // Outbound agent image: upload immediately and send as a threaded m.image.
+        const ctx = sessions.get(event.sessionId)
+        if (ctx) {
+          const bytes = Buffer.from(block.data, 'base64')
+          const ext = (block.mimeType.split('/')[1] ?? 'png').replace(/[^a-z0-9]/gi, '')
+          const filename = `image.${ext}`
+          void mediaClient
+            .upload({ data: bytes, contentType: block.mimeType, filename, asUserId: ctx.agent.userId })
+            .then(({ content_uri }) =>
+              client.sendMessage({
+                roomId: ctx.roomId,
+                asUserId: ctx.agent.userId,
+                threadRoot: ctx.threadRoot,
+                content: {
+                  msgtype: 'm.image',
+                  body: filename,
+                  url: content_uri,
+                  info: { mimetype: block.mimeType, size: bytes.length },
+                },
+              }),
+            )
+            .catch((err) => {
+              console.warn(`[matrix:${name}] outbound image upload failed:`, err)
+              void sendMediaError(ctx, err, 'agent image upload failed', client)
+            })
+        }
       } else {
         console.warn(`[matrix:${name}] dropped chunk block type=${block.type}`, block)
       }
@@ -315,6 +492,29 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
         continue
       }
       logInbound(evt)
+
+      // Capture media events in the pending store; never route them to agents.
+      if (
+        evt.type === 'm.room.message' &&
+        isMediaMsgtype(evt.content?.msgtype) &&
+        evt.room_id &&
+        evt.event_id &&
+        evt.sender &&
+        evt.content?.url &&
+        !bindings.some((b) => b.userId === evt.sender)
+      ) {
+        pendingMedia.add(evt.room_id, inboundThreadRoot(evt), {
+          eventId: evt.event_id,
+          sender: evt.sender,
+          msgtype: evt.content.msgtype as string,
+          body: (evt.content.body as string | undefined) ?? '',
+          filename: evt.content.filename as string | undefined,
+          url: evt.content.url as string,
+          info: evt.content.info as PendingMediaItem['info'],
+        })
+        continue
+      }
+
       // Agent-promotion: top-level inbound event becomes the thread root.
       // For in-thread messages the existing root is preserved.
       const promotedRoot = inboundThreadRoot(evt) ?? evt.event_id
@@ -461,10 +661,33 @@ export function createMatrixTransport(opts: CreateMatrixTransportOptions) {
     try {
       const rawBody = evt.content?.body ?? ''
       const promptText = stripMention(rawBody, agent.userId)
+
+      // Drain pending media for this sender+thread and prepend as ACP content blocks.
+      const pendingItems = pendingMedia.drain(
+        evt.room_id,
+        inboundThreadRoot(evt),
+        evt.sender ?? '',
+      )
+      const { blocks, pathLines } = await buildMediaBlocks(pendingItems, {
+        agent,
+        media: mediaClient,
+        writeAttachmentFn,
+        onError: (item, err) => {
+          console.warn(`[matrix:${agent.name}] media_failed for ${item.body}:`, err)
+          void sendMediaError(
+            { agent, roomId: evt.room_id!, threadRoot },
+            err,
+            `Could not process attachment: ${item.body}`,
+            client,
+          )
+        },
+      })
+
+      const fullPromptText = [promptText, ...pathLines].filter(Boolean).join('\n')
       await agents.prompt(agent.name, {
         threadId: sessionKey,
         channelId: evt.room_id,
-        content: [{ type: 'text', text: promptText }],
+        content: [...blocks, { type: 'text', text: fullPromptText }],
       })
       // Drain: the prompt promise resolves on the stopReason response, but
       // trailing chunks may still arrive (see DRAIN_* above). Wait until the