@openclaw/crabline 0.1.9 → 0.1.11

package/dist/src/servers/signal.js

@@ -1,37 +1,242 @@
 import { randomBytes } from "node:crypto";
 import { createServer } from "node:http";
 import path from "node:path";
-import { adminAuthError, closeServer, hasAdminToken, jsonResponse, parseRequestBody, queryRecord, readInteger, readTrimmedString, writeResponse, } from "./http.js";
-import { recordServerEvent } from "./recorder.js";
+import { adminAuthError, closeServer, drainRequestBody, hasAdminToken, InvalidJsonBodyError, isJsonObject, jsonResponse, parseUnknownRequestBody, queryRecord, readInteger, readTrimmedString, RequestBodyTooLargeError, writeResponse, } from "./http.js";
+import { recordCommittedServerEvent, recordServerEvent, } from "./recorder.js";
+import { resolveMaxPendingInboundEvents } from "./pending-events.js";
 const SIGNAL_CLI_SSE_KEEPALIVE_MS = 15_000;
-async function appendEvent(state, event) {
-    await recordServerEvent({ event, onEvent: state.onEvent, recorderPath: state.recorderPath });
+const DEFAULT_MAX_SIGNAL_SSE_CLIENTS = 32;
+const MAX_SIGNAL_SSE_BUFFER_BYTES = 2 * 1024 * 1024;
+async function appendEvent(state, event, committed = false) {
+    const params = { event, onEvent: state.onEvent, recorderPath: state.recorderPath };
+    await (committed ? recordCommittedServerEvent(params) : recordServerEvent(params));
 }
 function rpcResponse(id, result) {
     return jsonResponse({ id: id ?? null, jsonrpc: "2.0", result });
 }
-function emitSignalEvent(state, payload) {
-    const event = `event:receive\ndata:${JSON.stringify(payload)}\n\n`;
-    if (state.clients.size === 0) {
-        state.pendingEvents.push(event);
+function rpcError(code, message, id = null, status = 400) {
+    return jsonResponse({
+        error: { code, message },
+        id: id ?? null,
+        jsonrpc: "2.0",
+    }, status);
+}
+function removeSignalClient(state, client, destroy) {
+    const buffer = state.clientBuffers.get(client);
+    state.clients.delete(client);
+    state.clientBuffers.delete(client);
+    if (buffer) {
+        const undeliveredEvents = [...buffer.inFlight, ...buffer.events].filter((event) => {
+            event.recipients?.delete(client);
+            return event.sequence !== undefined && event.recipients?.size === 0;
+        });
+        const restoredEvents = undeliveredEvents.filter((event) => !state.pendingEvents.includes(event));
+        if (restoredEvents.length > 0) {
+            state.pendingEvents.push(...restoredEvents);
+            state.pendingEvents.sort((left, right) => (left.sequence ?? Number.MAX_SAFE_INTEGER) - (right.sequence ?? 0));
+            state.pendingEventBytes += restoredEvents.reduce((total, event) => total + Buffer.byteLength(event.data), 0);
+        }
+    }
+    if (destroy) {
+        client.destroy();
+    }
+    if (state.pendingEvents.length > 0 && state.clients.size > 0) {
+        queueMicrotask(() => flushPendingSignalEvents(state));
+    }
+}
+function evictSignalClient(state, client) {
+    removeSignalClient(state, client, true);
+}
+function scheduleSignalDrain(state, client) {
+    const buffer = state.clientBuffers.get(client);
+    if (!buffer || buffer.draining || client.destroyed || client.writableEnded) {
         return;
     }
+    buffer.draining = true;
+    client.once("drain", () => {
+        const current = state.clientBuffers.get(client);
+        if (current) {
+            current.bytes -= current.inFlight.reduce((total, event) => total + Buffer.byteLength(event.data), 0);
+            current.inFlight.length = 0;
+            current.draining = false;
+            flushSignalClientEvents(state, client);
+            const remaining = state.clientBuffers.get(client);
+            if (remaining && remaining.events.length === 0 && !client.writableNeedDrain) {
+                flushPendingSignalEvents(state);
+            }
+        }
+    });
+}
+function queueSignalClientEvent(state, client, event) {
+    const buffer = state.clientBuffers.get(client);
+    const eventBytes = Buffer.byteLength(event.data);
+    const alreadyBuffered = isSignalEventBuffered(state, event);
+    const pendingEventBytes = state.pendingEvents.includes(event) ? 0 : eventBytes;
+    if (!buffer ||
+        (!alreadyBuffered && signalBufferedEventCount(state) >= state.maxPendingInboundEvents) ||
+        state.pendingEventBytes + buffer.bytes + pendingEventBytes > MAX_SIGNAL_SSE_BUFFER_BYTES) {
+        evictSignalClient(state, client);
+        return "rejected";
+    }
+    buffer.events.push(event);
+    buffer.bytes += eventBytes;
+    event.recipients?.add(client);
+    scheduleSignalDrain(state, client);
+    return "queued";
+}
+function writeSignalSse(state, client, event) {
+    if (client.destroyed) {
+        removeSignalClient(state, client, false);
+        return "rejected";
+    }
+    if (client.writableEnded) {
+        evictSignalClient(state, client);
+        return "rejected";
+    }
+    const eventBytes = Buffer.byteLength(event.data);
+    if (eventBytes > MAX_SIGNAL_SSE_BUFFER_BYTES) {
+        return "rejected";
+    }
+    const buffer = state.clientBuffers.get(client);
+    if (client.writableNeedDrain || buffer?.draining || buffer?.events.length) {
+        return queueSignalClientEvent(state, client, event);
+    }
+    if (client.writableLength + eventBytes > MAX_SIGNAL_SSE_BUFFER_BYTES) {
+        evictSignalClient(state, client);
+        return "rejected";
+    }
+    const accepted = client.write(event.data);
+    event.recipients?.add(client);
+    if (!accepted) {
+        if (buffer) {
+            buffer.inFlight.push(event);
+            buffer.bytes += eventBytes;
+        }
+        scheduleSignalDrain(state, client);
+    }
+    return "accepted";
+}
+function maxSignalClientBufferBytes(state) {
+    let maxBytes = 0;
+    for (const buffer of state.clientBuffers.values()) {
+        maxBytes = Math.max(maxBytes, buffer.bytes);
+    }
+    return maxBytes;
+}
+function isSignalEventBuffered(state, event) {
+    return (state.pendingEvents.includes(event) ||
+        [...state.clientBuffers.values()].some((buffer) => buffer.inFlight.includes(event) || buffer.events.includes(event)));
+}
+function signalBufferedEventCount(state) {
+    const sequences = new Set();
+    for (const event of state.pendingEvents) {
+        if (event.sequence !== undefined) {
+            sequences.add(event.sequence);
+        }
+    }
+    for (const buffer of state.clientBuffers.values()) {
+        for (const event of [...buffer.inFlight, ...buffer.events]) {
+            if (event.sequence !== undefined) {
+                sequences.add(event.sequence);
+            }
+        }
+    }
+    return sequences.size;
+}
+function queueSignalEvent(state, event) {
+    const eventBytes = Buffer.byteLength(event.data);
+    if (eventBytes > MAX_SIGNAL_SSE_BUFFER_BYTES ||
+        state.pendingEventBytes + maxSignalClientBufferBytes(state) + eventBytes >
+            MAX_SIGNAL_SSE_BUFFER_BYTES ||
+        (!isSignalEventBuffered(state, event) &&
+            signalBufferedEventCount(state) >= state.maxPendingInboundEvents)) {
+        return false;
+    }
+    state.pendingEvents.push(event);
+    state.pendingEventBytes += eventBytes;
+    return true;
+}
+function replayExclusiveSignalEvents(state, client) {
+    const events = new Map();
+    for (const [owner, buffer] of state.clientBuffers) {
+        if (owner === client) {
+            continue;
+        }
+        for (const event of [...buffer.inFlight, ...buffer.events]) {
+            if (event.sequence !== undefined &&
+                event.recipients?.size === 1 &&
+                event.recipients.has(owner)) {
+                events.set(event.sequence, event);
+            }
+        }
+    }
+    for (const event of [...events.values()].sort((left, right) => left.sequence - right.sequence)) {
+        if (writeSignalSse(state, client, event) === "rejected") {
+            break;
+        }
+    }
+}
+function flushSignalClientEvents(state, client) {
+    while (!client.writableNeedDrain) {
+        const buffer = state.clientBuffers.get(client);
+        if (!buffer || buffer.events.length === 0) {
+            break;
+        }
+        const event = buffer.events.shift();
+        if (!client.write(event.data)) {
+            buffer.inFlight.push(event);
+            scheduleSignalDrain(state, client);
+            break;
+        }
+        buffer.bytes -= Buffer.byteLength(event.data);
+    }
+}
+function flushPendingSignalEvents(state) {
+    while (state.pendingEvents.length > 0 && state.clients.size > 0) {
+        const event = state.pendingEvents[0];
+        for (const client of [...state.clients]) {
+            if (!event.recipients?.has(client)) {
+                writeSignalSse(state, client, event);
+            }
+        }
+        if (state.clients.size === 0 ||
+            [...state.clients].some((client) => !event.recipients?.has(client))) {
+            break;
+        }
+        state.pendingEvents.shift();
+        state.pendingEventBytes -= Buffer.byteLength(event.data);
+    }
+}
+function emitSignalEvent(state, payload) {
+    const event = {
+        data: `event:receive\ndata:${JSON.stringify(payload)}\n\n`,
+        recipients: new Set(),
+        sequence: state.nextEventSequence++,
+    };
+    if (state.clients.size === 0 || state.pendingEvents.length > 0) {
+        return queueSignalEvent(state, event);
+    }
+    let delivered = false;
     for (const client of state.clients) {
-        client.write(event);
+        const result = writeSignalSse(state, client, event);
+        delivered = result === "accepted" || result === "queued" || delivered;
     }
+    return delivered || queueSignalEvent(state, event);
 }
 async function handleAdminInbound(params) {
-    const text = readTrimmedString(params.body.text);
+    const text = typeof params.body.text === "string" ? params.body.text : undefined;
     const sourceNumber = readTrimmedString(params.body.sourceNumber ?? params.body.senderId);
-    if (!text || !sourceNumber) {
-        return jsonResponse({ error: "sourceNumber and text are required", ok: false }, 400);
+    const sourceUuid = readTrimmedString(params.body.sourceUuid);
+    if (!text || text.trim().length === 0 || (!sourceNumber && !sourceUuid)) {
+        return jsonResponse({ error: "text and at least one source identity are required", ok: false }, 400);
     }
     const timestamp = readInteger(params.body.timestamp) ?? params.state.nextTimestamp++;
     const groupId = readTrimmedString(params.body.groupId);
     const payload = {
         envelope: {
             sourceName: readTrimmedString(params.body.sourceName ?? params.body.senderName),
-            sourceNumber,
+            ...(sourceNumber ? { sourceNumber } : {}),
+            ...(sourceUuid ? { sourceUuid } : {}),
             timestamp,
             dataMessage: {
                 message: text,
@@ -40,61 +245,207 @@ async function handleAdminInbound(params) {
             },
         },
     };
-    emitSignalEvent(params.state, payload);
+    if (!emitSignalEvent(params.state, payload)) {
+        return jsonResponse({
+            error: `Pending inbound queue is full (${params.state.maxPendingInboundEvents} events)`,
+            ok: false,
+        }, 503);
+    }
     return jsonResponse({ event: payload, ok: true });
 }
 async function handleRpc(params) {
-    const method = readTrimmedString(params.body.method);
-    if (!method) {
-        return jsonResponse({
-            error: { code: -32600, message: "Invalid Request" },
-            id: params.body.id ?? null,
-            jsonrpc: "2.0",
-        }, 400);
+    const hasId = Object.hasOwn(params.body, "id");
+    const id = params.body.id;
+    const validId = !hasId ||
+        id === null ||
+        typeof id === "string" ||
+        (typeof id === "number" && Number.isFinite(id));
+    const method = params.body.method;
+    if (typeof method !== "string" || !validId) {
+        return {
+            accepted: false,
+            record: false,
+            response: rpcError(-32600, "Invalid Request", validId ? id : null),
+        };
+    }
+    if (params.body.jsonrpc !== "2.0") {
+        return {
+            accepted: false,
+            record: false,
+            response: rpcError(-32600, "Invalid Request", id),
+        };
+    }
+    const notification = !hasId;
+    if (params.body.params !== undefined &&
+        !Array.isArray(params.body.params) &&
+        !isJsonObject(params.body.params)) {
+        return {
+            accepted: false,
+            record: false,
+            response: rpcError(-32600, "Invalid Request", id),
+        };
     }
     if (method === "version") {
-        return rpcResponse(params.body.id, { version: "crabline-signal-1" });
+        return {
+            accepted: false,
+            record: true,
+            response: notification
+                ? new Response(null, { status: 204 })
+                : rpcResponse(id, { version: "crabline-signal-1" }),
+        };
     }
     if (["send", "sendReaction", "sendReceipt", "sendTyping"].includes(method)) {
+        if (!validSignalRpcParams(method, params.body.params)) {
+            return {
+                accepted: false,
+                record: false,
+                response: notification
+                    ? new Response(null, { status: 204 })
+                    : rpcError(-32602, "Invalid params", id),
+            };
+        }
         const timestamp = params.state.nextTimestamp++;
-        return rpcResponse(params.body.id, method === "sendTyping" ? {} : { timestamp });
+        return {
+            accepted: true,
+            record: true,
+            response: notification
+                ? new Response(null, { status: 204 })
+                : rpcResponse(id, method === "sendTyping" ? {} : { timestamp }),
+        };
     }
-    return jsonResponse({
-        error: { code: -32601, message: `Method not found: ${method}` },
-        id: params.body.id ?? null,
-        jsonrpc: "2.0",
-    });
+    return {
+        accepted: false,
+        record: false,
+        response: notification
+            ? new Response(null, { status: 204 })
+            : jsonResponse({
+                error: { code: -32601, message: `Method not found: ${method}` },
+                id: id ?? null,
+                jsonrpc: "2.0",
+            }),
+    };
+}
+function hasRecipients(params) {
+    const values = [
+        params.recipient,
+        params.recipients,
+        params.groupId,
+        params.groupIds,
+        params.username,
+        params.usernames,
+    ];
+    return (params.noteToSelf === true ||
+        values.some((value) => (typeof value === "string" && value.trim().length > 0) ||
+            (Array.isArray(value) &&
+                value.length > 0 &&
+                value.every((entry) => typeof entry === "string" && entry.trim().length > 0))));
+}
+function hasTypingRecipients(params) {
+    if (params.noteToSelf !== undefined ||
+        params.username !== undefined ||
+        params.usernames !== undefined ||
+        (params.stop !== undefined && typeof params.stop !== "boolean")) {
+        return false;
+    }
+    return [params.recipient, params.recipients, params.groupId, params.groupIds].some((value) => (typeof value === "string" && value.trim().length > 0) ||
+        (Array.isArray(value) &&
+            value.length > 0 &&
+            value.every((entry) => typeof entry === "string" && entry.trim().length > 0)));
+}
+function validTimestamp(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function hasStringArray(value) {
+    return (Array.isArray(value) &&
+        value.length > 0 &&
+        value.every((entry) => typeof entry === "string" && entry.trim().length > 0));
+}
+function validSignalRpcParams(method, value) {
+    if (!isJsonObject(value)) {
+        return false;
+    }
+    if (method === "send") {
+        return (hasRecipients(value) &&
+            (readTrimmedString(value.message) !== undefined ||
+                readTrimmedString(value.attachment) !== undefined ||
+                hasStringArray(value.attachments)));
+    }
+    if (method === "sendReaction") {
+        return (hasRecipients(value) &&
+            readTrimmedString(value.emoji) !== undefined &&
+            readTrimmedString(value.targetAuthor) !== undefined &&
+            validTimestamp(value.targetTimestamp));
+    }
+    if (method === "sendReceipt") {
+        const targetTimestamps = value.targetTimestamps ?? value.targetTimestamp;
+        const timestamps = Array.isArray(targetTimestamps) ? targetTimestamps : [targetTimestamps];
+        return ((readTrimmedString(value.recipient) !== undefined ||
+            readTrimmedString(value.username) !== undefined ||
+            hasStringArray(value.usernames)) &&
+            timestamps.length > 0 &&
+            timestamps.every(validTimestamp) &&
+            (value.type === undefined || value.type === "read" || value.type === "viewed"));
+    }
+    return method === "sendTyping" && hasTypingRecipients(value);
 }
 async function handleRequest(params) {
     const url = new URL(params.request.url ?? "/", "http://localhost");
     if (url.pathname === "/api/v1/events" && params.request.method === "GET") {
-        await appendEvent(params.state, {
-            at: new Date().toISOString(),
-            method: "GET",
-            path: url.pathname,
-            query: queryRecord(url),
-            type: "api",
-        });
+        if (params.state.clients.size + params.state.pendingSseClients >= params.state.maxSseClients) {
+            await writeResponse(params.response, jsonResponse({ error: "Too many event stream clients", ok: false }, 503));
+            return;
+        }
+        params.state.pendingSseClients += 1;
+        try {
+            await appendEvent(params.state, {
+                at: new Date().toISOString(),
+                method: "GET",
+                path: url.pathname,
+                query: queryRecord(url),
+                type: "api",
+            });
+        }
+        finally {
+            params.state.pendingSseClients -= 1;
+        }
+        if (params.request.destroyed || params.response.destroyed) {
+            return;
+        }
         params.response.writeHead(200, {
             "cache-control": "no-cache",
             connection: "keep-alive",
             "content-type": "text/event-stream",
         });
         params.response.flushHeaders();
+        params.response.once("close", () => {
+            removeSignalClient(params.state, params.response, false);
+        });
         params.state.clients.add(params.response);
-        for (const event of params.state.pendingEvents.splice(0)) {
-            params.response.write(event);
+        params.state.clientBuffers.set(params.response, {
+            bytes: 0,
+            draining: false,
+            events: [],
+            inFlight: [],
+        });
+        replayExclusiveSignalEvents(params.state, params.response);
+        if (params.state.clients.has(params.response)) {
+            flushPendingSignalEvents(params.state);
         }
-        params.response.once("close", () => params.state.clients.delete(params.response));
         return;
     }
     let fetchResponse;
     if (url.pathname === "/crabline/signal/inbound" && params.request.method === "POST") {
         if (!hasAdminToken(params.request, params.state.adminToken)) {
+            drainRequestBody(params.request);
             fetchResponse = adminAuthError();
         }
         else {
-            const body = await parseRequestBody(params.request);
+            const body = await parseUnknownRequestBody(params.request);
+            if (!isJsonObject(body)) {
+                fetchResponse = jsonResponse({ error: "Request body must be a JSON object", ok: false }, 400);
+                await writeResponse(params.response, fetchResponse);
+                return;
+            }
             await appendEvent(params.state, {
                 at: new Date().toISOString(),
                 body,
@@ -117,16 +468,25 @@ async function handleRequest(params) {
         fetchResponse = new Response(null, { status: 200 });
     }
     else if (url.pathname === "/api/v1/rpc" && params.request.method === "POST") {
-        const body = await parseRequestBody(params.request);
-        await appendEvent(params.state, {
-            at: new Date().toISOString(),
-            body,
-            method: "POST",
-            path: url.pathname,
-            query: queryRecord(url),
-            type: "api",
-        });
-        fetchResponse = await handleRpc({ body, state: params.state });
+        const body = await parseUnknownRequestBody(params.request);
+        if (!isJsonObject(body)) {
+            await writeResponse(params.response, rpcError(-32600, "Invalid Request"));
+            return;
+        }
+        const rpc = await handleRpc({ body, state: params.state });
+        if (rpc.record) {
+            const event = {
+                accepted: rpc.accepted,
+                at: new Date().toISOString(),
+                body,
+                method: "POST",
+                path: url.pathname,
+                query: queryRecord(url),
+                type: "api",
+            };
+            await appendEvent(params.state, event, rpc.accepted || rpc.response.status === 204);
+        }
+        fetchResponse = rpc.response;
     }
     else {
         fetchResponse = new Response("not found", { status: 404 });
@@ -138,28 +498,50 @@ export async function startSignalServer(params = {}) {
         account: params.account ?? "+15550000000",
         adminToken: params.adminToken ?? randomBytes(24).toString("base64url"),
         clients: new Set(),
+        clientBuffers: new Map(),
+        maxPendingInboundEvents: resolveMaxPendingInboundEvents(params.maxPendingInboundEvents),
+        maxSseClients: Number.isSafeInteger(params.maxSseClients) && (params.maxSseClients ?? 0) > 0
+            ? params.maxSseClients
+            : DEFAULT_MAX_SIGNAL_SSE_CLIENTS,
+        nextEventSequence: 1,
         nextTimestamp: Date.now(),
         onEvent: params.onEvent,
+        pendingEventBytes: 0,
         pendingEvents: [],
+        pendingSseClients: 0,
         recorderPath: params.recorderPath ?? path.resolve(".crabline", "servers", "signal.jsonl"),
     };
     const host = params.host ?? "127.0.0.1";
     const server = createServer((request, response) => {
         void handleRequest({ request, response, state }).catch(async (error) => {
             if (!response.headersSent) {
-                await writeResponse(response, jsonResponse({ error: error instanceof Error ? error.message : String(error), ok: false }, 500));
+                let errorResponse;
+                const isAdminRequest = new URL(request.url ?? "/", "http://localhost").pathname === "/crabline/signal/inbound";
+                if (error instanceof InvalidJsonBodyError) {
+                    errorResponse = isAdminRequest
+                        ? jsonResponse({ error: "Request body is not valid JSON", ok: false }, 400)
+                        : rpcError(-32700, "Parse error");
+                }
+                else if (error instanceof RequestBodyTooLargeError) {
+                    errorResponse = isAdminRequest
+                        ? jsonResponse({ error: "Request body is too large", ok: false }, 413)
+                        : rpcError(-32600, "Request body is too large", null, 413);
+                }
+                else {
+                    errorResponse = jsonResponse({ error: "internal server error", ok: false }, 500);
+                }
+                try {
+                    await writeResponse(response, errorResponse);
+                }
+                catch (writeError) {
+                    response.destroy(writeError instanceof Error ? writeError : new Error(String(writeError)));
+                }
             }
             else {
                 response.destroy(error instanceof Error ? error : new Error(String(error)));
             }
         });
     });
-    const keepalive = setInterval(() => {
-        for (const client of state.clients) {
-            client.write(":\n");
-        }
-    }, SIGNAL_CLI_SSE_KEEPALIVE_MS);
-    keepalive.unref();
     await new Promise((resolve, reject) => {
         server.once("error", reject);
         server.listen(params.port ?? 0, host, () => {
@@ -172,6 +554,12 @@ export async function startSignalServer(params = {}) {
         await closeServer(server);
         throw new Error("Unable to resolve Signal local server address.");
     }
+    const keepalive = setInterval(() => {
+        for (const client of state.clients) {
+            writeSignalSse(state, client, { data: ":\n" });
+        }
+    }, SIGNAL_CLI_SSE_KEEPALIVE_MS);
+    keepalive.unref();
     const baseUrl = `http://${host.includes(":") ? `[${host}]` : host}:${address.port}`;
     return {
         async close() {