@zooid/transport-matrix 0.7.3 → 0.8.0

package/dist/index.d.ts

@@ -221,7 +221,13 @@ interface AgentBinding {
      */
     rooms: RoomBinding[];
     trigger: 'mention' | 'any';
+    /** Host path of the agent's workspace (resolved agent.workdir). Media files land here. */
+    workspaceDir?: string;
+    /** Path prefix as the agent sees it: '/workspace' for containers, = workspaceDir for local. */
+    agentWorkspacePath?: string;
 }
+declare const MEDIA_MSGTYPES: Set<string>;
+declare function isMediaMsgtype(t: string | undefined): boolean;
 interface ThreadState {
     /** Agent names that have posted in this thread, in order. */
     participants: string[];
@@ -266,6 +272,36 @@ declare class BotPool {
     findByName(name: string): AgentBinding | undefined;
 }
 
+interface WriteAttachmentInput {
+    workspaceDir: string;
+    agentWorkspacePath: string;
+    eventId: string;
+    filename: string;
+    data: Uint8Array;
+}
+declare function writeAttachment(input: WriteAttachmentInput): {
+    hostPath: string;
+    agentPath: string;
+};
+
+interface MediaClientLike {
+    download(input: {
+        mxcUri: string;
+        asUserId: string;
+        maxBytes?: number;
+    }): Promise<{
+        data: Uint8Array;
+        contentType: string;
+    }>;
+    upload(input: {
+        data: Uint8Array;
+        contentType: string;
+        filename?: string;
+        asUserId: string;
+    }): Promise<{
+        content_uri: string;
+    }>;
+}
 interface CreateMatrixTransportOptions {
     agents: AcpRegistry;
     approvals: ApprovalCorrelator;
@@ -280,6 +316,10 @@ interface CreateMatrixTransportOptions {
     drainQuietMs?: number;
     /** Hard cap on the post-turn drain. Defaults to `DRAIN_MAX_MS`. */
     drainMaxMs?: number;
+    /** Injected media client for downloading/uploading Matrix media. */
+    media?: MediaClientLike;
+    /** Injected attachment writer (defaults to the real writeAttachment). */
+    writeAttachmentFn?: typeof writeAttachment;
 }
 declare function createMatrixTransport(opts: CreateMatrixTransportOptions): {
     app: Hono<hono_types.BlankEnv, hono_types.BlankSchema, "/">;
@@ -304,6 +344,15 @@ interface EnsureSpaceOpts {
      * if the alias already resolves we return the existing room untouched.
      */
     admins?: string[];
+    /**
+     * Join rule pinned on the space at creation. Defaults to `invite`: a
+     * workspace is joined by invitation, not self-service, so it can't be walked
+     * into (which would otherwise satisfy every restricted child room's `allow`).
+     * `zooid dev` passes `public` so a self-service-registered local account can
+     * join `#<space>` straight from the web client without an invite — acceptable
+     * because the dev homeserver is local-only and never deployed.
+     */
+    joinRule?: 'invite' | 'public';
 }
 declare function ensureWorkforceSpace(opts: EnsureSpaceOpts): Promise<string>;
 interface EnsureDefaultChannelOpts {
@@ -359,4 +408,63 @@ interface StartOpts {
 }
 declare function startWorkforcePublisher(opts: StartOpts): Promise<PublisherHandle>;
 
-export { type AgentBinding, BotPool, type CreateMatrixTransportOptions, type EnsureDefaultChannelOpts, type EnsureSpaceOpts, MatrixClient, type MatrixClientOptions, MatrixContextProvider, type MatrixContextProviderOpts, type MatrixTransportConfig, type MaybeMessage, type PublishOpts, type PublisherHandle, type RouteMatch, type SendCustomEventInput, type SendMessageInput, type StartOpts as StartWorkforcePublisherOpts, type WorkforceRoster, buildWorkforceRoster, createMatrixTransport, ensureDefaultChannel, ensureWorkforceSpace, extractMentions, publishWorkforce, renderRegistration, route, serverNameFromMxid, startWorkforcePublisher };
+/** Limits are routing policy, not enforcement — see ZOD057 (enforcement lives
+ *  in Tuwunel config + the Zoon composer). */
+declare const MAX_INLINE_IMAGE_BYTES = 524288;
+declare const INLINE_IMAGE_MIMES: string[];
+declare const MAX_DOWNLOAD_BYTES = 33554432;
+interface MediaClientOptions {
+    homeserver: string;
+    asToken: string;
+    fetch?: typeof globalThis.fetch;
+}
+declare function parseMxcUri(uri: string): {
+    serverName: string;
+    mediaId: string;
+} | null;
+declare class MediaClient {
+    private readonly homeserver;
+    private readonly asToken;
+    private readonly fetch;
+    constructor(opts: MediaClientOptions);
+    download(input: {
+        mxcUri: string;
+        asUserId: string;
+        maxBytes?: number;
+    }): Promise<{
+        data: Uint8Array;
+        contentType: string;
+    }>;
+    upload(input: {
+        data: Uint8Array;
+        contentType: string;
+        filename?: string;
+        asUserId: string;
+    }): Promise<{
+        content_uri: string;
+    }>;
+}
+
+declare const MAX_MEDIA_PER_TURN = 8;
+interface PendingMediaItem {
+    eventId: string;
+    sender: string;
+    msgtype: string;
+    body: string;
+    filename?: string;
+    url: string;
+    info?: {
+        mimetype?: string;
+        size?: number;
+        w?: number;
+        h?: number;
+    };
+}
+declare class PendingMediaStore {
+    private readonly queues;
+    private key;
+    add(roomId: string, threadKey: string | undefined, item: PendingMediaItem): void;
+    drain(roomId: string, threadKey: string | undefined, sender: string): PendingMediaItem[];
+}
+
+export { type AgentBinding, BotPool, type CreateMatrixTransportOptions, type EnsureDefaultChannelOpts, type EnsureSpaceOpts, INLINE_IMAGE_MIMES, MAX_DOWNLOAD_BYTES, MAX_INLINE_IMAGE_BYTES, MAX_MEDIA_PER_TURN, MEDIA_MSGTYPES, MatrixClient, type MatrixClientOptions, MatrixContextProvider, type MatrixContextProviderOpts, type MatrixTransportConfig, type MaybeMessage, MediaClient, type MediaClientLike, type MediaClientOptions, type PendingMediaItem, PendingMediaStore, type PublishOpts, type PublisherHandle, type RouteMatch, type SendCustomEventInput, type SendMessageInput, type StartOpts as StartWorkforcePublisherOpts, type WorkforceRoster, type WriteAttachmentInput, buildWorkforceRoster, createMatrixTransport, ensureDefaultChannel, ensureWorkforceSpace, extractMentions, isMediaMsgtype, parseMxcUri, publishWorkforce, renderRegistration, route, serverNameFromMxid, startWorkforcePublisher, writeAttachment };

package/dist/index.js

@@ -359,14 +359,29 @@ var MatrixContextProvider = class {
   }
   toMessage(ev) {
     if (ev.type !== "m.room.message") return null;
-    if (ev.content?.msgtype !== "m.text" || typeof ev.content.body !== "string") return null;
+    const msgtype = ev.content?.msgtype;
+    const body = ev.content?.body;
     const agent = this.opts.agentBots.get(ev.sender);
-    const relatesTo = ev.content["m.relates_to"];
+    const relatesTo = ev.content?.["m.relates_to"];
     const threadId = relatesTo?.rel_type === "m.thread" && relatesTo.event_id ? relatesTo.event_id : void 0;
+    if (msgtype === "m.image" || msgtype === "m.file" || msgtype === "m.video" || msgtype === "m.audio") {
+      const kind = msgtype.slice(2);
+      const name = typeof body === "string" && body ? body : "untitled";
+      return {
+        id: ev.event_id,
+        sender: ev.sender,
+        text: `[${kind}: ${name}]`,
+        timestamp: new Date(ev.origin_server_ts).toISOString(),
+        is_agent: agent !== void 0,
+        ...agent !== void 0 ? { agent_name: agent } : {},
+        ...threadId !== void 0 ? { thread_id: threadId } : {}
+      };
+    }
+    if (msgtype !== "m.text" || typeof body !== "string") return null;
     return {
       id: ev.event_id,
       sender: ev.sender,
-      text: ev.content.body,
+      text: body,
       timestamp: new Date(ev.origin_server_ts).toISOString(),
       is_agent: agent !== void 0,
       ...agent !== void 0 ? { agent_name: agent } : {},
@@ -440,6 +455,10 @@ function stripMention(body, userId) {
 }
 
 // src/router.ts
+var MEDIA_MSGTYPES = /* @__PURE__ */ new Set(["m.image", "m.file", "m.video", "m.audio"]);
+function isMediaMsgtype(t) {
+  return t !== void 0 && MEDIA_MSGTYPES.has(t);
+}
 function inboundThreadRoot(event) {
   const r = event.content?.["m.relates_to"];
   return r?.rel_type === "m.thread" && r.event_id ? r.event_id : void 0;
@@ -447,6 +466,7 @@ function inboundThreadRoot(event) {
 function route(event, agents, threadStates) {
   if (event.type !== "m.room.message") return [];
   if (!event.content?.msgtype) return [];
+  if (isMediaMsgtype(event.content.msgtype)) return [];
   const mentions = new Set(extractMentions(event));
   const matches = [];
   const threadRoot = inboundThreadRoot(event);
@@ -485,10 +505,12 @@ async function ensureWorkforceSpace(opts) {
     name: display,
     preset: opts.preset,
     creation_content: { type: "m.space" },
-    // A workspace is joined by invitation, not self-service. Pin the space's
-    // join rule to invite regardless of preset so it can't be walked into
-    // (which would otherwise satisfy every restricted child room's allow).
-    initial_state: [{ type: "m.room.join_rules", state_key: "", content: { join_rule: "invite" } }]
+    // Pin the join rule regardless of preset. Defaults to invite so the space
+    // can't be walked into (which would otherwise satisfy every restricted
+    // child room's allow); `zooid dev` opts into `public` for local-only use.
+    initial_state: [
+      { type: "m.room.join_rules", state_key: "", content: { join_rule: opts.joinRule ?? "invite" } }
+    ]
   };
   if (opts.admins && opts.admins.length > 0) {
     body.invite = opts.admins;
@@ -804,8 +826,180 @@ function toMatrixHtml(markdown) {
   });
 }
 
+// src/pending-media.ts
+var MAX_MEDIA_PER_TURN = 8;
+var PendingMediaStore = class {
+  queues = /* @__PURE__ */ new Map();
+  key(roomId, threadKey) {
+    return `${roomId} ${threadKey ?? "main"}`;
+  }
+  add(roomId, threadKey, item) {
+    const k = this.key(roomId, threadKey);
+    const q = this.queues.get(k) ?? [];
+    q.push(item);
+    while (q.length > MAX_MEDIA_PER_TURN) q.shift();
+    this.queues.set(k, q);
+  }
+  drain(roomId, threadKey, sender) {
+    const k = this.key(roomId, threadKey);
+    const q = this.queues.get(k) ?? [];
+    const mine = q.filter((i) => i.sender === sender);
+    const rest = q.filter((i) => i.sender !== sender);
+    if (rest.length) this.queues.set(k, rest);
+    else this.queues.delete(k);
+    return mine;
+  }
+};
+
+// src/media-client.ts
+var MAX_INLINE_IMAGE_BYTES = 524288;
+var INLINE_IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"];
+var MAX_DOWNLOAD_BYTES = 33554432;
+function parseMxcUri(uri) {
+  const m = /^mxc:\/\/([^/]+)\/(.+)$/.exec(uri);
+  return m ? { serverName: m[1], mediaId: m[2] } : null;
+}
+var MediaClient = class {
+  homeserver;
+  asToken;
+  fetch;
+  constructor(opts) {
+    this.homeserver = opts.homeserver.replace(/\/$/, "");
+    this.asToken = opts.asToken;
+    this.fetch = opts.fetch ?? globalThis.fetch;
+  }
+  async download(input) {
+    const parsed = parseMxcUri(input.mxcUri);
+    if (!parsed) throw new Error(`not an mxc uri: ${input.mxcUri}`);
+    const url = `${this.homeserver}/_matrix/client/v1/media/download/${encodeURIComponent(parsed.serverName)}/${encodeURIComponent(parsed.mediaId)}?user_id=${encodeURIComponent(input.asUserId)}`;
+    const r = await this.fetch(url, {
+      headers: { Authorization: `Bearer ${this.asToken}` }
+    });
+    if (!r.ok) throw new Error(`media download failed: ${r.status}`);
+    const buf = new Uint8Array(await r.arrayBuffer());
+    const max = input.maxBytes ?? MAX_DOWNLOAD_BYTES;
+    if (buf.byteLength > max) {
+      throw new Error(`media too large: ${buf.byteLength} > ${max}`);
+    }
+    return { data: buf, contentType: r.headers.get("content-type") ?? "application/octet-stream" };
+  }
+  async upload(input) {
+    const params = new URLSearchParams();
+    if (input.filename) params.set("filename", input.filename);
+    params.set("user_id", input.asUserId);
+    const r = await this.fetch(`${this.homeserver}/_matrix/media/v3/upload?${params}`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${this.asToken}`, "Content-Type": input.contentType },
+      body: input.data
+    });
+    if (!r.ok) throw new Error(`media upload failed: ${r.status}`);
+    return await r.json();
+  }
+};
+
+// src/attachments.ts
+import { mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+import { posix } from "path";
+function sanitize(s, fallback) {
+  const cleaned = s.replace(/[^A-Za-z0-9._-]/g, "").replace(/^\.+/, "");
+  return cleaned || fallback;
+}
+function writeAttachment(input) {
+  const dir = sanitize(input.eventId, "event");
+  const name = sanitize(input.filename, "file");
+  const hostDir = join(input.workspaceDir, ".zooid", "attachments", dir);
+  mkdirSync(hostDir, { recursive: true });
+  const hostPath = join(hostDir, name);
+  writeFileSync(hostPath, input.data);
+  const agentPath = posix.join(input.agentWorkspacePath, ".zooid", "attachments", dir, name);
+  return { hostPath, agentPath };
+}
+
 // src/transport.ts
 var STARTUP_GRACE_MS = 5e3;
+async function buildMediaBlocks(items, opts) {
+  const blocks = [];
+  const pathLines = [];
+  if (!opts.media || items.length === 0) return { blocks, pathLines };
+  for (const item of items) {
+    try {
+      const isInlineCandidate = item.msgtype === "m.image" && INLINE_IMAGE_MIMES.includes(item.info?.mimetype ?? "") && (item.info?.size === void 0 || item.info.size <= MAX_INLINE_IMAGE_BYTES);
+      if (isInlineCandidate) {
+        const { data, contentType } = await opts.media.download({
+          mxcUri: item.url,
+          asUserId: opts.agent.userId
+        });
+        if (data.byteLength <= MAX_INLINE_IMAGE_BYTES) {
+          blocks.push({
+            type: "image",
+            data: Buffer.from(data).toString("base64"),
+            mimeType: contentType
+          });
+          continue;
+        }
+        if (opts.agent.workspaceDir) {
+          const paths = opts.writeAttachmentFn({
+            workspaceDir: opts.agent.workspaceDir,
+            agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
+            eventId: item.eventId,
+            filename: item.filename ?? item.body,
+            data
+          });
+          blocks.push({
+            type: "resource_link",
+            uri: `file://${paths.agentPath}`,
+            name: item.filename ?? item.body
+          });
+          pathLines.push(`Attached file: ${paths.agentPath}`);
+        }
+      } else {
+        if (!opts.agent.workspaceDir) continue;
+        const { data } = await opts.media.download({
+          mxcUri: item.url,
+          asUserId: opts.agent.userId
+        });
+        const paths = opts.writeAttachmentFn({
+          workspaceDir: opts.agent.workspaceDir,
+          agentWorkspacePath: opts.agent.agentWorkspacePath ?? opts.agent.workspaceDir,
+          eventId: item.eventId,
+          filename: item.filename ?? item.body,
+          data
+        });
+        blocks.push({
+          type: "resource_link",
+          uri: `file://${paths.agentPath}`,
+          name: item.filename ?? item.body,
+          mimeType: item.info?.mimetype,
+          size: item.info?.size
+        });
+        pathLines.push(`Attached file: ${paths.agentPath}`);
+      }
+    } catch (err) {
+      opts.onError(item, err);
+    }
+  }
+  return { blocks, pathLines };
+}
+async function sendMediaError(ctx, _err, message, client) {
+  await client.sendCustomEvent({
+    roomId: ctx.roomId,
+    asUserId: ctx.agent.userId,
+    eventType: "eco.zoon.error",
+    content: toErrorBody(
+      {
+        kind: "error",
+        agentId: ctx.agent.name,
+        sessionId: null,
+        turnId: null,
+        code: "media_failed",
+        message: message.slice(0, 250),
+        transient: false
+      },
+      ctx.threadRoot
+    )
+  }).catch((e) => console.warn(`[matrix:${ctx.agent.name}] eco.zoon.error send failed:`, e));
+}
 var SEEN_EVENT_CAP = 5e3;
 var DRAIN_QUIET_MS = 300;
 var DRAIN_MAX_MS = 3e4;
@@ -818,9 +1012,13 @@ function createMatrixTransport(opts) {
   const { agents, approvals, client, bindings, hsToken, adminUserId } = opts;
   const drainQuietMs = opts.drainQuietMs ?? DRAIN_QUIET_MS;
   const drainMaxMs = opts.drainMaxMs ?? DRAIN_MAX_MS;
+  const mediaClient = opts.media;
+  const writeAttachmentFn = opts.writeAttachmentFn ?? writeAttachment;
+  const pendingMedia = new PendingMediaStore();
   const pool = new BotPool(client, bindings);
   const sessions = /* @__PURE__ */ new Map();
   const buffers = /* @__PURE__ */ new Map();
+  const bufferMessageIds = /* @__PURE__ */ new Map();
   const sendQueue = /* @__PURE__ */ new Map();
   const threadStates = /* @__PURE__ */ new Map();
   const cutoffTs = Date.now() - STARTUP_GRACE_MS;
@@ -835,8 +1033,36 @@ function createMatrixTransport(opts) {
       const block = event.content;
       if (block.type === "text" && typeof block.text === "string") {
         const current = buffers.get(event.sessionId) ?? "";
-        const prefix = block.text === "" && current.length > 0 ? "\n\n" : "";
+        const prevMessageId = bufferMessageIds.get(event.sessionId);
+        const messageChanged = event.messageId !== void 0 && prevMessageId !== void 0 && event.messageId !== prevMessageId;
+        const needsBreak = current.length > 0 && (block.text === "" || messageChanged);
+        const prefix = needsBreak ? "\n\n" : "";
         buffers.set(event.sessionId, current + prefix + block.text);
+        if (event.messageId !== void 0)
+          bufferMessageIds.set(event.sessionId, event.messageId);
+      } else if (block.type === "image" && typeof block.data === "string" && typeof block.mimeType === "string" && mediaClient) {
+        const ctx2 = sessions.get(event.sessionId);
+        if (ctx2) {
+          const bytes = Buffer.from(block.data, "base64");
+          const ext = (block.mimeType.split("/")[1] ?? "png").replace(/[^a-z0-9]/gi, "");
+          const filename = `image.${ext}`;
+          void mediaClient.upload({ data: bytes, contentType: block.mimeType, filename, asUserId: ctx2.agent.userId }).then(
+            ({ content_uri }) => client.sendMessage({
+              roomId: ctx2.roomId,
+              asUserId: ctx2.agent.userId,
+              threadRoot: ctx2.threadRoot,
+              content: {
+                msgtype: "m.image",
+                body: filename,
+                url: content_uri,
+                info: { mimetype: block.mimeType, size: bytes.length }
+              }
+            })
+          ).catch((err) => {
+            console.warn(`[matrix:${name}] outbound image upload failed:`, err);
+            void sendMediaError(ctx2, err, "agent image upload failed", client);
+          });
+        }
       } else {
         console.warn(`[matrix:${name}] dropped chunk block type=${block.type}`, block);
       }
@@ -979,6 +1205,18 @@ function createMatrixTransport(opts) {
         continue;
       }
       logInbound(evt);
+      if (evt.type === "m.room.message" && isMediaMsgtype(evt.content?.msgtype) && evt.room_id && evt.event_id && evt.sender && evt.content?.url && !bindings.some((b) => b.userId === evt.sender)) {
+        pendingMedia.add(evt.room_id, inboundThreadRoot2(evt), {
+          eventId: evt.event_id,
+          sender: evt.sender,
+          msgtype: evt.content.msgtype,
+          body: evt.content.body ?? "",
+          filename: evt.content.filename,
+          url: evt.content.url,
+          info: evt.content.info
+        });
+        continue;
+      }
       const promotedRoot = inboundThreadRoot2(evt) ?? evt.event_id;
       const inboundRel = inboundThreadRoot2(evt);
       if (evt.type === "m.room.message" && inboundRel && !threadStates.has(inboundRel) && evt.room_id) {
@@ -1079,6 +1317,7 @@ function createMatrixTransport(opts) {
     const sessionId = await agents.ensureSession(agent.name, sessionKey, evt.room_id);
     sessions.set(sessionId, { agent, roomId: evt.room_id, threadRoot });
     buffers.set(sessionId, "");
+    bufferMessageIds.delete(sessionId);
     const roomId = evt.room_id;
     const TYPING_TTL_MS = 3e4;
     const TYPING_REFRESH_MS = 25e3;
@@ -1094,10 +1333,30 @@ function createMatrixTransport(opts) {
     try {
       const rawBody = evt.content?.body ?? "";
       const promptText = stripMention(rawBody, agent.userId);
+      const pendingItems = pendingMedia.drain(
+        evt.room_id,
+        inboundThreadRoot2(evt),
+        evt.sender ?? ""
+      );
+      const { blocks, pathLines } = await buildMediaBlocks(pendingItems, {
+        agent,
+        media: mediaClient,
+        writeAttachmentFn,
+        onError: (item, err) => {
+          console.warn(`[matrix:${agent.name}] media_failed for ${item.body}:`, err);
+          void sendMediaError(
+            { agent, roomId: evt.room_id, threadRoot },
+            err,
+            `Could not process attachment: ${item.body}`,
+            client
+          );
+        }
+      });
+      const fullPromptText = [promptText, ...pathLines].filter(Boolean).join("\n");
       await agents.prompt(agent.name, {
         threadId: sessionKey,
         channelId: evt.room_id,
-        content: [{ type: "text", text: promptText }]
+        content: [...blocks, { type: "text", text: fullPromptText }]
       });
       const drainStart = Date.now();
       let drained = buffers.get(sessionId) ?? "";
@@ -1139,6 +1398,7 @@ function createMatrixTransport(opts) {
       await safeTyping(false);
       await safePresence("online");
       buffers.delete(sessionId);
+      bufferMessageIds.delete(sessionId);
     }
   }
   return {
@@ -1241,17 +1501,27 @@ async function startWorkforcePublisher(opts) {
 }
 export {
   BotPool,
+  INLINE_IMAGE_MIMES,
+  MAX_DOWNLOAD_BYTES,
+  MAX_INLINE_IMAGE_BYTES,
+  MAX_MEDIA_PER_TURN,
+  MEDIA_MSGTYPES,
   MatrixClient,
   MatrixContextProvider,
+  MediaClient,
+  PendingMediaStore,
   buildWorkforceRoster,
   createMatrixTransport,
   ensureDefaultChannel,
   ensureWorkforceSpace,
   extractMentions,
+  isMediaMsgtype,
+  parseMxcUri,
   publishWorkforce,
   renderRegistration,
   route,
   serverNameFromMxid,
-  startWorkforcePublisher
+  startWorkforcePublisher,
+  writeAttachment
 };
 //# sourceMappingURL=index.js.map