@openclaw/crabline 0.1.8 → 0.1.10

package/dist/src/servers/mattermost.js

@@ -1,14 +1,29 @@
 import { createHash, randomBytes } from "node:crypto";
-import fs from "node:fs/promises";
 import path from "node:path";
 import { WebSocket, WebSocketServer } from "ws";
-import { adminAuthError, hasAdminToken, jsonResponse, parseRequestBody, queryRecord, readTrimmedString, startHttpJsonServer, } from "./http.js";
+import { adminAuthError, drainRequestBody, hasAdminToken, InvalidJsonBodyError, isJsonObject, isLoopbackHost, jsonResponse, parseUnknownRequestBody, queryRecord, readTrimmedString, RequestBodyTooLargeError, startHttpJsonServer, } from "./http.js";
+import { recordCommittedServerEvent, recordServerEvent, } from "./recorder.js";
+import { resolveMaxPendingInboundEvents } from "./pending-events.js";
+import { closeWebSocketServer } from "./websocket.js";
+const DEFAULT_WEBSOCKET_AUTHENTICATION_TIMEOUT_MS = 5_000;
+const DEFAULT_MAX_WEBSOCKET_BUFFERED_BYTES = 1024 * 1024;
+const DEFAULT_MAX_WEBSOCKET_MESSAGE_BYTES = 64 * 1024;
+const DEFAULT_MAX_UNAUTHENTICATED_WEBSOCKET_CLIENTS = 32;
+function resolvePositiveLimit(value, name, fallback) {
+    if (value === undefined) {
+        return fallback;
+    }
+    if (!Number.isSafeInteger(value) || value < 1) {
+        throw new Error(`${name} must be a positive safe integer.`);
+    }
+    return value;
+}
 export function mattermostId(value) {
     return createHash("sha256").update(value).digest("hex").slice(0, 26);
 }
-async function appendEvent(state, event) {
-    await fs.mkdir(path.dirname(state.recorderPath), { recursive: true });
-    await fs.appendFile(state.recorderPath, `${JSON.stringify(event)}\n`, "utf8");
+async function appendEvent(state, event, committed = false) {
+    const params = { event, onEvent: state.onEvent, recorderPath: state.recorderPath };
+    await (committed ? recordCommittedServerEvent(params) : recordServerEvent(params));
 }
 function authorized(request, token) {
     const [scheme, value] = request.headers.authorization?.trim().split(/\s+/, 2) ?? [];
@@ -22,6 +37,14 @@ function mattermostError(message, status) {
         status_code: status,
     }, status);
 }
+function decodeMattermostPathSegment(value) {
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return mattermostError("Invalid path parameter", 400);
+    }
+}
 function eventBroadcast(params) {
     return {
         channel_id: params.channelId ?? "",
@@ -44,22 +67,87 @@ function postEvent(event, post, senderName, channelType) {
         broadcast: eventBroadcast({ channelId: post.channel_id, userId: post.user_id }),
     };
 }
+function webSocketEventBytes(event) {
+    return Buffer.byteLength(JSON.stringify({ ...event, seq: 0 }), "utf8");
+}
 function sendEvent(state, client, event) {
     const seq = state.websocketClients.get(client);
     if (seq === undefined || client.readyState !== WebSocket.OPEN) {
-        return;
+        state.websocketClients.delete(client);
+        return false;
+    }
+    const payload = JSON.stringify({ ...event, seq });
+    if (client.bufferedAmount + Buffer.byteLength(payload, "utf8") >
+        state.maxWebSocketBufferedBytes) {
+        state.websocketClients.delete(client);
+        client.close(1013, "client too slow");
+        return false;
+    }
+    try {
+        client.send(payload);
+    }
+    catch {
+        state.websocketClients.delete(client);
+        client.terminate();
+        return false;
     }
-    client.send(JSON.stringify({ ...event, seq }));
     state.websocketClients.set(client, seq + 1);
+    return true;
+}
+function sendControlMessage(state, client, value) {
+    if (client.readyState !== WebSocket.OPEN) {
+        state.websocketClients.delete(client);
+        return false;
+    }
+    const payload = JSON.stringify(value);
+    if (client.bufferedAmount + Buffer.byteLength(payload, "utf8") >
+        state.maxWebSocketBufferedBytes) {
+        state.websocketClients.delete(client);
+        client.close(1013, "client too slow");
+        return false;
+    }
+    try {
+        client.send(payload);
+        return true;
+    }
+    catch {
+        state.websocketClients.delete(client);
+        client.terminate();
+        return false;
+    }
 }
-function broadcast(state, event) {
-    if (state.websocketClients.size === 0) {
-        state.pendingEvents.push(event);
-        return;
+function broadcast(state, event, queueIfUndelivered = true) {
+    if (event.broadcast.omit_users?.[state.botUserId]) {
+        return true;
+    }
+    if (webSocketEventBytes(event) > state.maxWebSocketBufferedBytes) {
+        for (const client of state.websocketClients.keys()) {
+            state.websocketClients.delete(client);
+            client.close(1013, "client too slow");
+        }
+        return false;
     }
+    let delivered = false;
     for (const client of state.websocketClients.keys()) {
-        sendEvent(state, client, event);
+        delivered = sendEvent(state, client, event) || delivered;
     }
+    if (delivered) {
+        return true;
+    }
+    if (!queueIfUndelivered) {
+        return false;
+    }
+    if (state.pendingEvents.length >= state.maxPendingInboundEvents) {
+        return false;
+    }
+    state.pendingEvents.push(event);
+    return true;
+}
+function pendingQueueFullResponse(state) {
+    return jsonResponse({
+        error: `Pending inbound queue is full (${state.maxPendingInboundEvents} events)`,
+        ok: false,
+    }, 503);
 }
 function createPost(params) {
     const id = mattermostId(`post-${params.state.nextPost++}`);
@@ -75,20 +163,27 @@ function createPost(params) {
     params.state.posts.set(id, post);
     return post;
 }
-async function handleAdminInbound(params) {
+function handleAdminInbound(params) {
     const channelId = readTrimmedString(params.body.channelId ?? params.body.channel_id);
     const senderId = readTrimmedString(params.body.senderId ?? params.body.user_id);
     const text = readTrimmedString(params.body.text ?? params.body.message);
     if (!channelId || !senderId || !text) {
         return jsonResponse({ error: "channelId, senderId, and text are required", ok: false }, 400);
     }
-    const senderName = readTrimmedString(params.body.senderName) ?? senderId;
-    const channelType = readTrimmedString(params.body.channelType) ?? "D";
+    if (params.state.websocketClients.size === 0 &&
+        params.state.pendingEvents.length >= params.state.maxPendingInboundEvents) {
+        return pendingQueueFullResponse(params.state);
+    }
+    const previousUser = params.state.users.get(senderId);
+    const previousChannel = params.state.channels.get(channelId);
+    const senderName = readTrimmedString(params.body.senderName) ?? previousUser?.username ?? senderId;
+    const channelType = readTrimmedString(params.body.channelType) ?? previousChannel?.type ?? "D";
+    const previousNextPost = params.state.nextPost;
     params.state.users.set(senderId, { id: senderId, update_at: Date.now(), username: senderName });
     params.state.channels.set(channelId, {
-        display_name: channelId,
+        display_name: previousChannel?.display_name ?? channelId,
         id: channelId,
-        name: channelId,
+        name: previousChannel?.name ?? channelId,
         type: channelType,
     });
     const post = createPost({
@@ -98,7 +193,31 @@ async function handleAdminInbound(params) {
         state: params.state,
         userId: senderId,
     });
-    broadcast(params.state, postEvent("posted", post, senderName, channelType));
+    const rollback = () => {
+        params.state.posts.delete(post.id);
+        params.state.nextPost = previousNextPost;
+        if (previousUser) {
+            params.state.users.set(senderId, previousUser);
+        }
+        else {
+            params.state.users.delete(senderId);
+        }
+        if (previousChannel) {
+            params.state.channels.set(channelId, previousChannel);
+        }
+        else {
+            params.state.channels.delete(channelId);
+        }
+    };
+    const event = postEvent("posted", post, senderName, channelType);
+    if (webSocketEventBytes(event) > params.state.maxWebSocketBufferedBytes) {
+        rollback();
+        return jsonResponse({ error: "Inbound event is too large", ok: false }, 413);
+    }
+    if (!broadcast(params.state, event)) {
+        rollback();
+        return pendingQueueFullResponse(params.state);
+    }
     return jsonResponse({ ok: true, post });
 }
 async function handleApi(params) {
@@ -108,7 +227,10 @@ async function handleApi(params) {
     }
     const usernameMatch = /^\/users\/username\/([^/]+)$/u.exec(apiPath);
     if (method === "GET" && usernameMatch?.[1]) {
-        const username = decodeURIComponent(usernameMatch[1]);
+        const username = decodeMattermostPathSegment(usernameMatch[1]);
+        if (username instanceof Response) {
+            return username;
+        }
         const user = [...state.users.values()].find((entry) => entry.username === username);
         return user ? jsonResponse(user) : mattermostError("User not found", 404);
     }
@@ -123,20 +245,36 @@ async function handleApi(params) {
         return channel ? jsonResponse(channel) : mattermostError("Channel not found", 404);
     }
     if (method === "POST" && apiPath === "/channels/direct") {
-        const userIds = Array.isArray(body) ? body : [];
-        if (userIds.length !== 2 || userIds.some((value) => typeof value !== "string" || !value)) {
+        const userIds = Array.isArray(body) ? body.map(readTrimmedString) : [];
+        if (userIds.length !== 2 || userIds.some((value) => !value)) {
             return mattermostError("Two user IDs are required", 400);
         }
-        const channelId = mattermostId(`dm:${userIds.map(String).sort().join(":")}`);
+        const [firstUserId, secondUserId] = userIds;
+        if (firstUserId === secondUserId) {
+            return mattermostError("Direct channel users must be distinct", 400);
+        }
+        if (!state.users.has(firstUserId) || !state.users.has(secondUserId)) {
+            return mattermostError("User not found", 404);
+        }
+        if (firstUserId !== state.botUserId && secondUserId !== state.botUserId) {
+            return mattermostError("Authenticated user must belong to the direct channel", 403);
+        }
+        const channelId = mattermostId(`dm:${userIds.sort().join(":")}`);
         const channel = { display_name: "", id: channelId, name: channelId, type: "D" };
         state.channels.set(channelId, channel);
         return jsonResponse(channel, 201);
     }
+    if (!isJsonObject(body)) {
+        return mattermostError("Request body must be a JSON object", 400);
+    }
     if (method === "POST" && apiPath === "/users/me/typing") {
         const channelId = readTrimmedString(body.channel_id);
         if (!channelId) {
             return mattermostError("channel_id is required", 400);
         }
+        if (!state.channels.has(channelId)) {
+            return mattermostError("Channel not found", 404);
+        }
         broadcast(state, {
             broadcast: eventBroadcast({
                 channelId,
@@ -150,8 +288,8 @@ async function handleApi(params) {
                 user_id: state.botUserId,
             },
             event: "typing",
-        });
-        return jsonResponse({});
+        }, false);
+        return jsonResponse({ status: "OK" });
     }
     if (method === "POST" && apiPath === "/posts") {
         const channelId = readTrimmedString(body.channel_id);
@@ -159,15 +297,28 @@ async function handleApi(params) {
         if (!channelId || !message) {
             return mattermostError("channel_id and message are required", 400);
         }
+        const channel = state.channels.get(channelId);
+        if (!channel) {
+            return mattermostError("Channel not found", 404);
+        }
+        const rootId = readTrimmedString(body.root_id);
+        if (rootId) {
+            const rootPost = state.posts.get(rootId);
+            if (!rootPost) {
+                return mattermostError("Root post not found", 404);
+            }
+            if (rootPost.channel_id !== channelId) {
+                return mattermostError("Root post belongs to another channel", 400);
+            }
+        }
         const post = createPost({
             channelId,
             message,
-            rootId: readTrimmedString(body.root_id),
+            rootId,
             state,
             userId: state.botUserId,
         });
-        const channelType = state.channels.get(channelId)?.type ?? "O";
-        broadcast(state, postEvent("posted", post, state.botUsername, channelType));
+        broadcast(state, postEvent("posted", post, state.botUsername, channel.type), false);
         return jsonResponse(post, 201);
     }
     const postMatch = /^\/posts\/([^/]+)$/u.exec(apiPath);
@@ -176,9 +327,12 @@ async function handleApi(params) {
         if (!post) {
             return mattermostError("Post not found", 404);
         }
+        if (post.user_id !== state.botUserId) {
+            return mattermostError("You do not have permission to edit this post", 403);
+        }
         const updated = { ...post, message: readTrimmedString(body.message) ?? post.message };
         state.posts.set(updated.id, updated);
-        broadcast(state, postEvent("post_edited", updated, state.botUsername, state.channels.get(updated.channel_id)?.type ?? "O"));
+        broadcast(state, postEvent("post_edited", updated, state.botUsername, state.channels.get(updated.channel_id)?.type ?? "O"), false);
         return jsonResponse(updated);
     }
     if (postMatch?.[1] && method === "DELETE") {
@@ -186,9 +340,12 @@ async function handleApi(params) {
         if (!post) {
             return mattermostError("Post not found", 404);
         }
+        if (post.user_id !== state.botUserId) {
+            return mattermostError("You do not have permission to delete this post", 403);
+        }
         state.posts.delete(post.id);
-        broadcast(state, postEvent("post_deleted", post, state.botUsername, state.channels.get(post.channel_id)?.type ?? "O"));
-        return new Response(null, { status: 204 });
+        broadcast(state, postEvent("post_deleted", post, state.botUsername, state.channels.get(post.channel_id)?.type ?? "O"), false);
+        return jsonResponse({ status: "OK" });
     }
     return mattermostError("Not found", 404);
 }
@@ -197,9 +354,13 @@ async function handleRequest(request, state) {
     const method = request.method ?? "GET";
     if (url.pathname === "/crabline/mattermost/inbound" && method === "POST") {
         if (!hasAdminToken(request, state.adminToken)) {
+            drainRequestBody(request);
             return adminAuthError();
         }
-        const body = await parseRequestBody(request);
+        const body = await parseUnknownRequestBody(request);
+        if (!isJsonObject(body)) {
+            return jsonResponse({ error: "Request body must be a JSON object", ok: false }, 400);
+        }
         await appendEvent(state, {
             at: new Date().toISOString(),
             body,
@@ -208,27 +369,42 @@ async function handleRequest(request, state) {
             query: queryRecord(url),
             type: "admin",
         });
-        return await handleAdminInbound({ body, state });
+        return handleAdminInbound({ body, state });
     }
     if (!url.pathname.startsWith("/api/v4/")) {
         return new Response("not found", { status: 404 });
     }
     if (!authorized(request, state.botToken)) {
+        drainRequestBody(request);
         return mattermostError("Invalid or missing token", 401);
     }
-    const body = method === "GET" || method === "DELETE" ? {} : await parseRequestBody(request);
-    await appendEvent(state, {
+    const body = method === "GET" || method === "DELETE" ? {} : await parseUnknownRequestBody(request);
+    const event = {
         at: new Date().toISOString(),
-        ...(Object.keys(body).length > 0 ? { body } : {}),
+        ...(typeof body === "object" && body !== null && Object.keys(body).length > 0 ? { body } : {}),
         method,
         path: url.pathname,
         query: queryRecord(url),
         type: "api",
+    };
+    const response = await handleApi({
+        body,
+        method,
+        path: url.pathname.slice("/api/v4".length),
+        state,
     });
-    return await handleApi({ body, method, path: url.pathname.slice("/api/v4".length), state });
+    if (method === "POST" && url.pathname === "/api/v4/posts") {
+        event.accepted = response.status === 201;
+    }
+    await appendEvent(state, event, response.ok && method !== "GET");
+    return response;
 }
 function attachWebSocketServer(params) {
-    const websocketServer = new WebSocketServer({ noServer: true });
+    const websocketServer = new WebSocketServer({
+        maxPayload: params.maxMessageBytes,
+        noServer: true,
+    });
+    const unauthenticatedClients = new Set();
     const onUpgrade = (request, socket, head) => {
         if (new URL(request.url ?? "/", "http://localhost").pathname !== "/api/v4/websocket") {
             socket.destroy();
@@ -240,6 +416,21 @@ function attachWebSocketServer(params) {
     };
     params.server.on("upgrade", onUpgrade);
     websocketServer.on("connection", (client) => {
+        client.on("error", () => {
+            unauthenticatedClients.delete(client);
+            params.state.websocketClients.delete(client);
+        });
+        if (unauthenticatedClients.size >= params.maxUnauthenticatedClients) {
+            client.terminate();
+            return;
+        }
+        unauthenticatedClients.add(client);
+        let authenticationOpen = true;
+        const authenticationTimeout = setTimeout(() => {
+            authenticationOpen = false;
+            client.close(4001, "authentication timeout");
+        }, params.authenticationTimeoutMs);
+        authenticationTimeout.unref();
         client.on("message", (raw) => {
             let message;
             try {
@@ -249,19 +440,31 @@ function attachWebSocketServer(params) {
                 client.close(1003, "invalid json");
                 return;
             }
+            if (!isJsonObject(message)) {
+                client.close(1003, "invalid json");
+                return;
+            }
             const seq = message.seq ?? 0;
             if (!params.state.websocketClients.has(client)) {
+                if (!authenticationOpen || client.readyState !== WebSocket.OPEN) {
+                    return;
+                }
                 if (message.action !== "authentication_challenge" ||
                     message.data?.token !== params.state.botToken) {
-                    client.send(JSON.stringify({
+                    sendControlMessage(params.state, client, {
                         error: { id: "api.context.unauthorized", message: "Authentication failed" },
                         seq_reply: seq,
                         status: "FAIL",
-                    }));
+                    });
+                    authenticationOpen = false;
+                    client.close(4001, "authentication failed");
                     return;
                 }
+                clearTimeout(authenticationTimeout);
+                authenticationOpen = false;
+                unauthenticatedClients.delete(client);
                 params.state.websocketClients.set(client, 0);
-                client.send(JSON.stringify({ seq_reply: seq, status: "OK" }));
+                sendControlMessage(params.state, client, { seq_reply: seq, status: "OK" });
                 sendEvent(params.state, client, {
                     broadcast: eventBroadcast({ userId: params.state.botUserId }),
                     data: {
@@ -270,56 +473,86 @@ function attachWebSocketServer(params) {
                     },
                     event: "hello",
                 });
-                for (const event of params.state.pendingEvents.splice(0)) {
-                    sendEvent(params.state, client, event);
+                const pending = params.state.pendingEvents.splice(0);
+                for (const [index, event] of pending.entries()) {
+                    if (!sendEvent(params.state, client, event)) {
+                        params.state.pendingEvents.unshift(...pending.slice(index));
+                        break;
+                    }
                 }
                 return;
             }
-            if (message.action === "user_typing" && message.data?.channel_id) {
+            if (message.action === "user_typing") {
+                const channelId = readTrimmedString(message.data?.channel_id);
+                if (!channelId || !params.state.channels.has(channelId)) {
+                    sendControlMessage(params.state, client, {
+                        error: {
+                            id: "api.channel.get.find.app_error",
+                            message: "Channel not found",
+                        },
+                        seq_reply: seq,
+                        status: "FAIL",
+                    });
+                    return;
+                }
                 broadcast(params.state, {
                     broadcast: eventBroadcast({
-                        channelId: message.data.channel_id,
+                        channelId,
                         omitUsers: { [params.state.botUserId]: true },
                     }),
                     data: {
-                        parent_id: message.data.parent_id ?? "",
+                        channel_id: channelId,
+                        parent_id: readTrimmedString(message.data?.parent_id) ?? "",
                         user_id: params.state.botUserId,
                     },
                     event: "typing",
-                });
-                client.send(JSON.stringify({ seq_reply: seq, status: "OK" }));
+                }, false);
+                sendControlMessage(params.state, client, { seq_reply: seq, status: "OK" });
                 return;
             }
             if (message.action === "ping") {
-                client.send(JSON.stringify({ data: { text: "pong" }, seq_reply: seq, status: "OK" }));
+                sendControlMessage(params.state, client, {
+                    data: { text: "pong" },
+                    seq_reply: seq,
+                    status: "OK",
+                });
                 return;
             }
-            client.send(JSON.stringify({
+            sendControlMessage(params.state, client, {
                 error: { id: "api.websocket.invalid_action", message: "Unsupported action" },
                 seq_reply: seq,
                 status: "FAIL",
-            }));
+            });
+        });
+        client.once("close", () => {
+            authenticationOpen = false;
+            clearTimeout(authenticationTimeout);
+            unauthenticatedClients.delete(client);
+            params.state.websocketClients.delete(client);
         });
-        client.once("close", () => params.state.websocketClients.delete(client));
     });
     return async () => {
         params.server.off("upgrade", onUpgrade);
-        for (const client of websocketServer.clients) {
-            client.close();
-        }
-        await new Promise((resolve, reject) => websocketServer.close((error) => (error ? reject(error) : resolve())));
+        await closeWebSocketServer(websocketServer);
     };
 }
 export async function startMattermostServer(params = {}) {
+    const host = params.host ?? "127.0.0.1";
     const botUserId = params.botUserId ?? mattermostId("crabline-mattermost-bot");
     const botUsername = params.botUsername ?? "crabline_bot";
+    const maxWebSocketMessageBytes = resolvePositiveLimit(params.maxWebSocketMessageBytes, "maxWebSocketMessageBytes", DEFAULT_MAX_WEBSOCKET_MESSAGE_BYTES);
+    const maxUnauthenticatedWebSocketClients = resolvePositiveLimit(params.maxUnauthenticatedWebSocketClients, "maxUnauthenticatedWebSocketClients", DEFAULT_MAX_UNAUTHENTICATED_WEBSOCKET_CLIENTS);
     const state = {
         adminToken: params.adminToken ?? randomBytes(24).toString("base64url"),
-        botToken: params.botToken ?? "crabline-mattermost-token",
+        botToken: params.botToken ??
+            (isLoopbackHost(host) ? "crabline-mattermost-token" : randomBytes(13).toString("hex")),
         botUserId,
         botUsername,
         channels: new Map(),
+        maxPendingInboundEvents: resolveMaxPendingInboundEvents(params.maxPendingInboundEvents),
+        maxWebSocketBufferedBytes: resolvePositiveLimit(params.maxWebSocketBufferedBytes, "maxWebSocketBufferedBytes", DEFAULT_MAX_WEBSOCKET_BUFFERED_BYTES),
         nextPost: 1,
+        onEvent: params.onEvent,
         pendingEvents: [],
         posts: new Map(),
         recorderPath: params.recorderPath ?? path.resolve(".crabline", "servers", "mattermost.jsonl"),
@@ -328,14 +561,29 @@ export async function startMattermostServer(params = {}) {
     };
     const httpServer = await startHttpJsonServer({
         handle: (request) => handleRequest(request, state),
-        host: params.host ?? "127.0.0.1",
+        handleError: (error) => {
+            if (error instanceof InvalidJsonBodyError) {
+                return mattermostError("Request body is not valid JSON", 400);
+            }
+            if (error instanceof RequestBodyTooLargeError) {
+                return mattermostError("Request body is too large", 413);
+            }
+            return undefined;
+        },
+        host,
         port: params.port ?? 0,
         serverName: "Mattermost",
     });
-    const closeWebSocketServer = attachWebSocketServer({ server: httpServer.server, state });
+    const closeMattermostWebSocketServer = attachWebSocketServer({
+        authenticationTimeoutMs: params.websocketAuthenticationTimeoutMs ?? DEFAULT_WEBSOCKET_AUTHENTICATION_TIMEOUT_MS,
+        maxMessageBytes: maxWebSocketMessageBytes,
+        maxUnauthenticatedClients: maxUnauthenticatedWebSocketClients,
+        server: httpServer.server,
+        state,
+    });
     return {
         async close() {
-            await closeWebSocketServer();
+            await closeMattermostWebSocketServer();
             await httpServer.close();
         },
         manifest: {