@openclaw/matrix 2026.1.29

package/src/matrix/send/media.ts

@@ -0,0 +1,220 @@
+import type {
+  DimensionalFileInfo,
+  EncryptedFile,
+  FileWithThumbnailInfo,
+  MatrixClient,
+  TimedFileInfo,
+  VideoFileInfo,
+} from "@vector-im/matrix-bot-sdk";
+import { parseBuffer, type IFileInfo } from "music-metadata";
+
+import { getMatrixRuntime } from "../../runtime.js";
+import {
+  type MatrixMediaContent,
+  type MatrixMediaInfo,
+  type MatrixMediaMsgType,
+  type MatrixRelation,
+  type MediaKind,
+} from "./types.js";
+import { applyMatrixFormatting } from "./formatting.js";
+
+const getCore = () => getMatrixRuntime();
+
+export function buildMatrixMediaInfo(params: {
+  size: number;
+  mimetype?: string;
+  durationMs?: number;
+  imageInfo?: DimensionalFileInfo;
+}): MatrixMediaInfo | undefined {
+  const base: FileWithThumbnailInfo = {};
+  if (Number.isFinite(params.size)) {
+    base.size = params.size;
+  }
+  if (params.mimetype) {
+    base.mimetype = params.mimetype;
+  }
+  if (params.imageInfo) {
+    const dimensional: DimensionalFileInfo = {
+      ...base,
+      ...params.imageInfo,
+    };
+    if (typeof params.durationMs === "number") {
+      const videoInfo: VideoFileInfo = {
+        ...dimensional,
+        duration: params.durationMs,
+      };
+      return videoInfo;
+    }
+    return dimensional;
+  }
+  if (typeof params.durationMs === "number") {
+    const timedInfo: TimedFileInfo = {
+      ...base,
+      duration: params.durationMs,
+    };
+    return timedInfo;
+  }
+  if (Object.keys(base).length === 0) return undefined;
+  return base;
+}
+
+export function buildMediaContent(params: {
+  msgtype: MatrixMediaMsgType;
+  body: string;
+  url?: string;
+  filename?: string;
+  mimetype?: string;
+  size: number;
+  relation?: MatrixRelation;
+  isVoice?: boolean;
+  durationMs?: number;
+  imageInfo?: DimensionalFileInfo;
+  file?: EncryptedFile;
+}): MatrixMediaContent {
+  const info = buildMatrixMediaInfo({
+    size: params.size,
+    mimetype: params.mimetype,
+    durationMs: params.durationMs,
+    imageInfo: params.imageInfo,
+  });
+  const base: MatrixMediaContent = {
+    msgtype: params.msgtype,
+    body: params.body,
+    filename: params.filename,
+    info: info ?? undefined,
+  };
+  // Encrypted media should only include the "file" payload, not top-level "url".
+  if (!params.file && params.url) {
+    base.url = params.url;
+  }
+  // For encrypted files, add the file object
+  if (params.file) {
+    base.file = params.file;
+  }
+  if (params.isVoice) {
+    base["org.matrix.msc3245.voice"] = {};
+    if (typeof params.durationMs === "number") {
+      base["org.matrix.msc1767.audio"] = {
+        duration: params.durationMs,
+      };
+    }
+  }
+  if (params.relation) {
+    base["m.relates_to"] = params.relation;
+  }
+  applyMatrixFormatting(base, params.body);
+  return base;
+}
+
+const THUMBNAIL_MAX_SIDE = 800;
+const THUMBNAIL_QUALITY = 80;
+
+export async function prepareImageInfo(params: {
+  buffer: Buffer;
+  client: MatrixClient;
+}): Promise<DimensionalFileInfo | undefined> {
+  const meta = await getCore().media.getImageMetadata(params.buffer).catch(() => null);
+  if (!meta) return undefined;
+  const imageInfo: DimensionalFileInfo = { w: meta.width, h: meta.height };
+  const maxDim = Math.max(meta.width, meta.height);
+  if (maxDim > THUMBNAIL_MAX_SIDE) {
+    try {
+      const thumbBuffer = await getCore().media.resizeToJpeg({
+        buffer: params.buffer,
+        maxSide: THUMBNAIL_MAX_SIDE,
+        quality: THUMBNAIL_QUALITY,
+        withoutEnlargement: true,
+      });
+      const thumbMeta = await getCore().media.getImageMetadata(thumbBuffer).catch(() => null);
+      const thumbUri = await params.client.uploadContent(
+        thumbBuffer,
+        "image/jpeg",
+        "thumbnail.jpg",
+      );
+      imageInfo.thumbnail_url = thumbUri;
+      if (thumbMeta) {
+        imageInfo.thumbnail_info = {
+          w: thumbMeta.width,
+          h: thumbMeta.height,
+          mimetype: "image/jpeg",
+          size: thumbBuffer.byteLength,
+        };
+      }
+    } catch {
+      // Thumbnail generation failed, continue without it
+    }
+  }
+  return imageInfo;
+}
+
+export async function resolveMediaDurationMs(params: {
+  buffer: Buffer;
+  contentType?: string;
+  fileName?: string;
+  kind: MediaKind;
+}): Promise<number | undefined> {
+  if (params.kind !== "audio" && params.kind !== "video") return undefined;
+  try {
+    const fileInfo: IFileInfo | string | undefined =
+      params.contentType || params.fileName
+        ? {
+            mimeType: params.contentType,
+            size: params.buffer.byteLength,
+            path: params.fileName,
+          }
+        : undefined;
+    const metadata = await parseBuffer(params.buffer, fileInfo, {
+      duration: true,
+      skipCovers: true,
+    });
+    const durationSeconds = metadata.format.duration;
+    if (typeof durationSeconds === "number" && Number.isFinite(durationSeconds)) {
+      return Math.max(0, Math.round(durationSeconds * 1000));
+    }
+  } catch {
+    // Duration is optional; ignore parse failures.
+  }
+  return undefined;
+}
+
+async function uploadFile(
+  client: MatrixClient,
+  file: Buffer,
+  params: {
+    contentType?: string;
+    filename?: string;
+  },
+): Promise<string> {
+  return await client.uploadContent(file, params.contentType, params.filename);
+}
+
+/**
+ * Upload media with optional encryption for E2EE rooms.
+ */
+export async function uploadMediaMaybeEncrypted(
+  client: MatrixClient,
+  roomId: string,
+  buffer: Buffer,
+  params: {
+    contentType?: string;
+    filename?: string;
+  },
+): Promise<{ url: string; file?: EncryptedFile }> {
+  // Check if room is encrypted and crypto is available
+  const isEncrypted = client.crypto && await client.crypto.isRoomEncrypted(roomId);
+
+  if (isEncrypted && client.crypto) {
+    // Encrypt the media before uploading
+    const encrypted = await client.crypto.encryptMedia(buffer);
+    const mxc = await client.uploadContent(encrypted.buffer, params.contentType, params.filename);
+    const file: EncryptedFile = { url: mxc, ...encrypted.file };
+    return {
+      url: mxc,
+      file,
+    };
+  }
+
+  // Upload unencrypted
+  const mxc = await uploadFile(client, buffer, params);
+  return { url: mxc };
+}

package/src/matrix/send/targets.test.ts

@@ -0,0 +1,102 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { MatrixClient } from "@vector-im/matrix-bot-sdk";
+import { EventType } from "./types.js";
+
+let resolveMatrixRoomId: typeof import("./targets.js").resolveMatrixRoomId;
+let normalizeThreadId: typeof import("./targets.js").normalizeThreadId;
+
+beforeEach(async () => {
+  vi.resetModules();
+  ({ resolveMatrixRoomId, normalizeThreadId } = await import("./targets.js"));
+});
+
+describe("resolveMatrixRoomId", () => {
+  it("uses m.direct when available", async () => {
+    const userId = "@user:example.org";
+    const client = {
+      getAccountData: vi.fn().mockResolvedValue({
+        [userId]: ["!room:example.org"],
+      }),
+      getJoinedRooms: vi.fn(),
+      getJoinedRoomMembers: vi.fn(),
+      setAccountData: vi.fn(),
+    } as unknown as MatrixClient;
+
+    const roomId = await resolveMatrixRoomId(client, userId);
+
+    expect(roomId).toBe("!room:example.org");
+    expect(client.getJoinedRooms).not.toHaveBeenCalled();
+    expect(client.setAccountData).not.toHaveBeenCalled();
+  });
+
+  it("falls back to joined rooms and persists m.direct", async () => {
+    const userId = "@fallback:example.org";
+    const roomId = "!room:example.org";
+    const setAccountData = vi.fn().mockResolvedValue(undefined);
+    const client = {
+      getAccountData: vi.fn().mockRejectedValue(new Error("nope")),
+      getJoinedRooms: vi.fn().mockResolvedValue([roomId]),
+      getJoinedRoomMembers: vi.fn().mockResolvedValue([
+        "@bot:example.org",
+        userId,
+      ]),
+      setAccountData,
+    } as unknown as MatrixClient;
+
+    const resolved = await resolveMatrixRoomId(client, userId);
+
+    expect(resolved).toBe(roomId);
+    expect(setAccountData).toHaveBeenCalledWith(
+      EventType.Direct,
+      expect.objectContaining({ [userId]: [roomId] }),
+    );
+  });
+
+  it("continues when a room member lookup fails", async () => {
+    const userId = "@continue:example.org";
+    const roomId = "!good:example.org";
+    const setAccountData = vi.fn().mockResolvedValue(undefined);
+    const getJoinedRoomMembers = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("boom"))
+      .mockResolvedValueOnce(["@bot:example.org", userId]);
+    const client = {
+      getAccountData: vi.fn().mockRejectedValue(new Error("nope")),
+      getJoinedRooms: vi.fn().mockResolvedValue(["!bad:example.org", roomId]),
+      getJoinedRoomMembers,
+      setAccountData,
+    } as unknown as MatrixClient;
+
+    const resolved = await resolveMatrixRoomId(client, userId);
+
+    expect(resolved).toBe(roomId);
+    expect(setAccountData).toHaveBeenCalled();
+  });
+
+  it("allows larger rooms when no 1:1 match exists", async () => {
+    const userId = "@group:example.org";
+    const roomId = "!group:example.org";
+    const client = {
+      getAccountData: vi.fn().mockRejectedValue(new Error("nope")),
+      getJoinedRooms: vi.fn().mockResolvedValue([roomId]),
+      getJoinedRoomMembers: vi.fn().mockResolvedValue([
+        "@bot:example.org",
+        userId,
+        "@extra:example.org",
+      ]),
+      setAccountData: vi.fn().mockResolvedValue(undefined),
+    } as unknown as MatrixClient;
+
+    const resolved = await resolveMatrixRoomId(client, userId);
+
+    expect(resolved).toBe(roomId);
+  });
+});
+
+describe("normalizeThreadId", () => {
+  it("returns null for empty thread ids", () => {
+    expect(normalizeThreadId("   ")).toBeNull();
+    expect(normalizeThreadId("$thread")).toBe("$thread");
+  });
+});

package/src/matrix/send/targets.ts

@@ -0,0 +1,144 @@
+import type { MatrixClient } from "@vector-im/matrix-bot-sdk";
+
+import { EventType, type MatrixDirectAccountData } from "./types.js";
+
+function normalizeTarget(raw: string): string {
+  const trimmed = raw.trim();
+  if (!trimmed) {
+    throw new Error("Matrix target is required (room:<id> or #alias)");
+  }
+  return trimmed;
+}
+
+export function normalizeThreadId(raw?: string | number | null): string | null {
+  if (raw === undefined || raw === null) return null;
+  const trimmed = String(raw).trim();
+  return trimmed ? trimmed : null;
+}
+
+const directRoomCache = new Map<string, string>();
+
+async function persistDirectRoom(
+  client: MatrixClient,
+  userId: string,
+  roomId: string,
+): Promise<void> {
+  let directContent: MatrixDirectAccountData | null = null;
+  try {
+    directContent = (await client.getAccountData(
+      EventType.Direct,
+    )) as MatrixDirectAccountData | null;
+  } catch {
+    // Ignore fetch errors and fall back to an empty map.
+  }
+  const existing =
+    directContent && !Array.isArray(directContent) ? directContent : {};
+  const current = Array.isArray(existing[userId]) ? existing[userId] : [];
+  if (current[0] === roomId) return;
+  const next = [roomId, ...current.filter((id) => id !== roomId)];
+  try {
+    await client.setAccountData(EventType.Direct, {
+      ...existing,
+      [userId]: next,
+    });
+  } catch {
+    // Ignore persistence errors.
+  }
+}
+
+async function resolveDirectRoomId(
+  client: MatrixClient,
+  userId: string,
+): Promise<string> {
+  const trimmed = userId.trim();
+  if (!trimmed.startsWith("@")) {
+    throw new Error(
+      `Matrix user IDs must be fully qualified (got "${trimmed}")`,
+    );
+  }
+
+  const cached = directRoomCache.get(trimmed);
+  if (cached) return cached;
+
+  // 1) Fast path: use account data (m.direct) for *this* logged-in user (the bot).
+  try {
+    const directContent = (await client.getAccountData(
+      EventType.Direct,
+    )) as MatrixDirectAccountData | null;
+    const list = Array.isArray(directContent?.[trimmed])
+      ? directContent[trimmed]
+      : [];
+    if (list.length > 0) {
+      directRoomCache.set(trimmed, list[0]);
+      return list[0];
+    }
+  } catch {
+    // Ignore and fall back.
+  }
+
+  // 2) Fallback: look for an existing joined room that looks like a 1:1 with the user.
+  // Many clients only maintain m.direct for *their own* account data, so relying on it is brittle.
+  let fallbackRoom: string | null = null;
+  try {
+    const rooms = await client.getJoinedRooms();
+    for (const roomId of rooms) {
+      let members: string[];
+      try {
+        members = await client.getJoinedRoomMembers(roomId);
+      } catch {
+        continue;
+      }
+      if (!members.includes(trimmed)) continue;
+      // Prefer classic 1:1 rooms, but allow larger rooms if requested.
+      if (members.length === 2) {
+        directRoomCache.set(trimmed, roomId);
+        await persistDirectRoom(client, trimmed, roomId);
+        return roomId;
+      }
+      if (!fallbackRoom) {
+        fallbackRoom = roomId;
+      }
+    }
+  } catch {
+    // Ignore and fall back.
+  }
+
+  if (fallbackRoom) {
+    directRoomCache.set(trimmed, fallbackRoom);
+    await persistDirectRoom(client, trimmed, fallbackRoom);
+    return fallbackRoom;
+  }
+
+  throw new Error(`No direct room found for ${trimmed} (m.direct missing)`);
+}
+
+export async function resolveMatrixRoomId(
+  client: MatrixClient,
+  raw: string,
+): Promise<string> {
+  const target = normalizeTarget(raw);
+  const lowered = target.toLowerCase();
+  if (lowered.startsWith("matrix:")) {
+    return await resolveMatrixRoomId(client, target.slice("matrix:".length));
+  }
+  if (lowered.startsWith("room:")) {
+    return await resolveMatrixRoomId(client, target.slice("room:".length));
+  }
+  if (lowered.startsWith("channel:")) {
+    return await resolveMatrixRoomId(client, target.slice("channel:".length));
+  }
+  if (lowered.startsWith("user:")) {
+    return await resolveDirectRoomId(client, target.slice("user:".length));
+  }
+  if (target.startsWith("@")) {
+    return await resolveDirectRoomId(client, target);
+  }
+  if (target.startsWith("#")) {
+    const resolved = await client.resolveRoom(target);
+    if (!resolved) {
+      throw new Error(`Matrix alias ${target} could not be resolved`);
+    }
+    return resolved;
+  }
+  return target;
+}

package/src/matrix/send/types.ts

@@ -0,0 +1,109 @@
+import type {
+  DimensionalFileInfo,
+  EncryptedFile,
+  FileWithThumbnailInfo,
+  MessageEventContent,
+  TextualMessageEventContent,
+  TimedFileInfo,
+  VideoFileInfo,
+} from "@vector-im/matrix-bot-sdk";
+
+// Message types
+export const MsgType = {
+  Text: "m.text",
+  Image: "m.image",
+  Audio: "m.audio",
+  Video: "m.video",
+  File: "m.file",
+  Notice: "m.notice",
+} as const;
+
+// Relation types
+export const RelationType = {
+  Annotation: "m.annotation",
+  Replace: "m.replace",
+  Thread: "m.thread",
+} as const;
+
+// Event types
+export const EventType = {
+  Direct: "m.direct",
+  Reaction: "m.reaction",
+  RoomMessage: "m.room.message",
+} as const;
+
+export type MatrixDirectAccountData = Record<string, string[]>;
+
+export type MatrixReplyRelation = {
+  "m.in_reply_to": { event_id: string };
+};
+
+export type MatrixThreadRelation = {
+  rel_type: typeof RelationType.Thread;
+  event_id: string;
+  is_falling_back?: boolean;
+  "m.in_reply_to"?: { event_id: string };
+};
+
+export type MatrixRelation = MatrixReplyRelation | MatrixThreadRelation;
+
+export type MatrixReplyMeta = {
+  "m.relates_to"?: MatrixRelation;
+};
+
+export type MatrixMediaInfo =
+  | FileWithThumbnailInfo
+  | DimensionalFileInfo
+  | TimedFileInfo
+  | VideoFileInfo;
+
+export type MatrixTextContent = TextualMessageEventContent & MatrixReplyMeta;
+
+export type MatrixMediaContent = MessageEventContent &
+  MatrixReplyMeta & {
+    info?: MatrixMediaInfo;
+    url?: string;
+    file?: EncryptedFile;
+    filename?: string;
+    "org.matrix.msc3245.voice"?: Record<string, never>;
+    "org.matrix.msc1767.audio"?: { duration: number };
+  };
+
+export type MatrixOutboundContent = MatrixTextContent | MatrixMediaContent;
+
+export type ReactionEventContent = {
+  "m.relates_to": {
+    rel_type: typeof RelationType.Annotation;
+    event_id: string;
+    key: string;
+  };
+};
+
+export type MatrixSendResult = {
+  messageId: string;
+  roomId: string;
+};
+
+export type MatrixSendOpts = {
+  client?: import("@vector-im/matrix-bot-sdk").MatrixClient;
+  mediaUrl?: string;
+  accountId?: string;
+  replyToId?: string;
+  threadId?: string | number | null;
+  timeoutMs?: number;
+  /** Send audio as voice message (voice bubble) instead of audio file. Defaults to false. */
+  audioAsVoice?: boolean;
+};
+
+export type MatrixMediaMsgType =
+  | typeof MsgType.Image
+  | typeof MsgType.Audio
+  | typeof MsgType.Video
+  | typeof MsgType.File;
+
+export type MediaKind = "image" | "audio" | "video" | "document" | "unknown";
+
+export type MatrixFormattedContent = MessageEventContent & {
+  format?: string;
+  formatted_body?: string;
+};