@newbase-clawchat/openclaw-clawchat 2026.4.15

package/src/outbound.ts

@@ -0,0 +1,141 @@
+import type { ClawlingChatClient, Envelope, Fragment, MessageAckPayload } from "@newbase-clawchat/sdk";
+import type { ChatType } from "./client.ts";
+import type { ResolvedOpenclawClawlingAccount } from "./config.ts";
+import type { ClawlingMediaFragment } from "./media-runtime.ts";
+import { textToFragments } from "./message-mapper.ts";
+
+export interface OutboundTarget {
+  chatId: string;
+  chatType: ChatType;
+}
+
+export interface OutboundReplyCtx {
+  replyToMessageId: string;
+  replyPreviewSenderId: string;
+  replyPreviewNickName: string;
+  replyPreviewText: string;
+}
+
+export interface LogSink {
+  info?: (m: string) => void;
+  error?: (m: string) => void;
+}
+
+export interface SendParams {
+  client: ClawlingChatClient;
+  account: ResolvedOpenclawClawlingAccount;
+  to: OutboundTarget;
+  text: string;
+  replyCtx?: OutboundReplyCtx;
+  mediaFragments?: ClawlingMediaFragment[];
+  mentions?: string[];
+  log?: LogSink;
+}
+
+export interface SendResult {
+  messageId: string;
+  acceptedAt: number;
+}
+
+export async function sendOpenclawClawlingText(params: SendParams): Promise<SendResult | null> {
+  const text = (params.text ?? "").trim();
+  const mediaFragments = params.mediaFragments ?? [];
+  if (!text && mediaFragments.length === 0) {
+    params.log?.info?.(
+      `[${params.account.accountId}] openclaw-clawchat outbound suppressed: empty text and no media`,
+    );
+    return null;
+  }
+
+  const mentions = params.mentions ?? [];
+  const textFragments = text ? textToFragments(text) : [];
+  // Cast at the SDK boundary: each MediaItem object is structurally compatible
+  // with one of the SDK's narrow Fragment members (ImageFragment / FileFragment /
+  // AudioFragment / VideoFragment) based on its runtime `kind`. The wide local
+  // shape lets us build a single uniform array without a per-kind switch.
+  const fragments = [...textFragments, ...mediaFragments] as Fragment[];
+
+  const useReply = params.replyCtx && mediaFragments.length === 0;
+  if (params.replyCtx && mediaFragments.length > 0) {
+    params.log?.info?.(
+      `[${params.account.accountId}] openclaw-clawchat replyCtx + media: downgraded to sendMessage`,
+    );
+  }
+
+  let ack: Envelope<MessageAckPayload>;
+  let mode: "send" | "reply";
+  if (useReply && params.replyCtx) {
+    mode = "reply";
+    ack = await params.client.replyMessage({
+      chat_id: params.to.chatId,
+      chat_type: params.to.chatType,
+      mode: "normal",
+      replyTo: {
+        msgId: params.replyCtx.replyToMessageId,
+        senderId: params.replyCtx.replyPreviewSenderId,
+        nickName: params.replyCtx.replyPreviewNickName,
+        fragments: [{ kind: "text", text: params.replyCtx.replyPreviewText }],
+      },
+      body: { fragments },
+      context: { mentions },
+    });
+  } else {
+    mode = "send";
+    ack = await params.client.sendMessage({
+      chat_id: params.to.chatId,
+      chat_type: params.to.chatType,
+      mode: "normal",
+      body: { fragments },
+      context: { mentions, reply: null },
+    });
+  }
+  params.log?.info?.(
+    `[${params.account.accountId}] openclaw-clawchat outbound mode=${mode} msg=${ack.payload.message_id} text_len=${text.length} media=${mediaFragments.length} trace=${ack.trace_id}`,
+  );
+  return {
+    messageId: ack.payload.message_id,
+    acceptedAt: ack.payload.accepted_at,
+  };
+}
+
+export interface SendMediaParams {
+  client: ClawlingChatClient;
+  account: ResolvedOpenclawClawlingAccount;
+  to: OutboundTarget;
+  mediaFragments: ClawlingMediaFragment[];
+  /** Optional caption alongside the media. */
+  text?: string;
+  replyCtx?: OutboundReplyCtx;
+  mentions?: string[];
+  log?: LogSink;
+}
+
+/**
+ * Send one or more media fragments (image / file / audio / video) to the
+ * given target, with an optional text caption.
+ *
+ * Validates that mediaFragments is non-empty (returns null + info log
+ * otherwise) and delegates to {@link sendOpenclawClawlingText} for the
+ * actual envelope construction. Reuses the existing replyCtx-downgrade,
+ * ack backfill, and log shape.
+ */
+export async function sendOpenclawClawlingMedia(
+  params: SendMediaParams,
+): Promise<SendResult | null> {
+  if (params.mediaFragments.length === 0) {
+    params.log?.info?.(
+      `[${params.account.accountId}] openclaw-clawchat sendMedia called with empty mediaFragments; suppressed`,
+    );
+    return null;
+  }
+  return await sendOpenclawClawlingText({
+    client: params.client,
+    account: params.account,
+    to: params.to,
+    text: params.text ?? "",
+    mediaFragments: params.mediaFragments,
+    ...(params.replyCtx ? { replyCtx: params.replyCtx } : {}),
+    ...(params.mentions ? { mentions: params.mentions } : {}),
+    ...(params.log ? { log: params.log } : {}),
+  });
+}

package/src/protocol.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { isInboundMessagePayload, isGroupSender, hasRenderableText } from "./protocol.ts";
+
+describe("openclaw-clawchat protocol guards", () => {
+  it("accepts a well-formed downlink message payload", () => {
+    const payload = {
+      message_id: "m1",
+      message_mode: "normal",
+      message: {
+        body: { fragments: [{ kind: "text", text: "hi" }] },
+        context: { mentions: [], reply: null },
+        streaming: {
+          status: "static",
+          sequence: 0,
+          mutation_policy: "sealed",
+          started_at: null,
+          completed_at: null,
+        },
+      },
+    };
+    expect(isInboundMessagePayload(payload)).toBe(true);
+  });
+
+  it("rejects payload missing message_id", () => {
+    expect(isInboundMessagePayload({ message: { body: { fragments: [] } } })).toBe(false);
+  });
+
+  it("rejects payload missing message", () => {
+    expect(isInboundMessagePayload({ message_id: "m1" })).toBe(false);
+  });
+
+  it("detects group sender", () => {
+    expect(isGroupSender({ sender_id: "g", type: "group", display_name: "Group" })).toBe(true);
+    expect(isGroupSender({ sender_id: "u", type: "direct", display_name: "User" })).toBe(false);
+  });
+
+  it("detects renderable text", () => {
+    expect(hasRenderableText({ body: { fragments: [{ kind: "text", text: "hi" }] } })).toBe(true);
+    expect(hasRenderableText({ body: { fragments: [{ kind: "image", url: "x" }] } })).toBe(false);
+    expect(hasRenderableText({ body: { fragments: [{ kind: "text", text: "  " }] } })).toBe(false);
+  });
+});

package/src/protocol.ts

@@ -0,0 +1,38 @@
+/**
+ * Local narrow guards for inbound protocol envelopes.
+ *
+ * The SDK's `message` event hands us `Envelope<unknown>`. Before casting the
+ * payload to `DownlinkMessageSendPayload` we run these cheap structural checks
+ * so runtime errors surface as skipped messages, not crashes.
+ */
+
+export function isInboundMessagePayload(payload: unknown): payload is {
+  message_id: string;
+  message_mode: string;
+  message: {
+    body: { fragments: Array<Record<string, unknown>> };
+    context: { mentions: string[]; reply: unknown };
+  };
+} {
+  if (!payload || typeof payload !== "object") return false;
+  const p = payload as Record<string, unknown>;
+  if (typeof p.message_id !== "string" || !p.message_id) return false;
+  if (typeof p.message !== "object" || p.message === null) return false;
+  const m = p.message as Record<string, unknown>;
+  if (typeof m.body !== "object" || m.body === null) return false;
+  const body = m.body as Record<string, unknown>;
+  if (!Array.isArray(body.fragments)) return false;
+  return true;
+}
+
+export function hasRenderableText(message: {
+  body?: { fragments?: Array<Record<string, unknown>> };
+}): boolean {
+  const fragments = message?.body?.fragments ?? [];
+  return fragments.some(
+    (f) =>
+      (f as { kind?: string }).kind === "text" &&
+      typeof (f as { text?: unknown }).text === "string" &&
+      (f as { text: string }).text.trim().length > 0,
+  );
+}

package/src/reply-dispatcher.ts

@@ -0,0 +1,387 @@
+import type { ClawlingChatClient, Fragment } from "@newbase-clawchat/sdk";
+import type { PluginRuntime } from "openclaw/plugin-sdk/core";
+import type { OpenClawConfig } from "openclaw/plugin-sdk/core";
+import type { ReplyPayload } from "openclaw/plugin-sdk/reply-runtime";
+import { createOpenclawClawlingApiClient } from "./api-client.ts";
+import {
+  openBufferedStreamingSession,
+  mergeStreamingText,
+  type BufferedStreamSession,
+} from "./buffered-stream.ts";
+import { emitFinalStreamReply, type ChatType, type EnvelopeRouting } from "./client.ts";
+import type { ResolvedOpenclawClawlingAccount } from "./config.ts";
+import { textToFragments } from "./message-mapper.ts";
+import { uploadOutboundMedia, type ClawlingMediaFragment } from "./media-runtime.ts";
+import { sendOpenclawClawlingText, type OutboundReplyCtx } from "./outbound.ts";
+
+export interface ReplyDispatcherOptions {
+  cfg: OpenClawConfig;
+  runtime: PluginRuntime;
+  account: ResolvedOpenclawClawlingAccount;
+  client: ClawlingChatClient;
+  /**
+   * New-protocol routing: `chatId` is the subject id, `chatType` is `"direct"`
+   * or `"group"`. Every outbound frame carries these as `chat_id` + `chat_type`.
+   */
+  target: { chatId: string; chatType: ChatType };
+  replyCtx?: OutboundReplyCtx;
+  /**
+   * The `message_id` from the inbound `message.send` envelope that triggered
+   * this reply run. When provided, streaming frames (`message.created`,
+   * `message.add`, `message.done`) reference this id so the client can
+   * correlate the streamed reply with the original user message. When
+   * omitted, a locally-generated id is used as a fallback.
+   */
+  inboundMessageId?: string;
+  /**
+   * Describes the inbound user message, used as the `replyTo` preview on
+   * the consolidated `message.reply` that closes a streaming run.
+   */
+  inboundForFinalReply?: {
+    senderId: string;
+    senderNickName: string;
+    bodyText: string;
+  };
+  log?: { info?: (m: string) => void; error?: (m: string) => void };
+}
+
+type TypedReplyDispatcherResult = ReturnType<
+  PluginRuntime["channel"]["reply"]["createReplyDispatcherWithTyping"]
+>;
+
+type StreamingReplyHooks = {
+  onPartialReply?: (payload: ReplyPayload) => Promise<void>;
+  onReasoningStream?: (payload: ReplyPayload) => Promise<void>;
+};
+
+/**
+ * Reply dispatcher for openclaw-clawchat.
+ *
+ * Streaming mode (`account.replyMode === "stream"`, no replyCtx):
+ *   1. `onReplyStart` opens a buffered streaming session — `message.created`
+ *      fires immediately.
+ *   2. As the agent runs, `onPartialReply` / `onReasoningStream` snapshots
+ *      and `deliver(block|tool)` deltas feed into the session; each flush
+ *      emits `message.add` with the new text delta (chunked by
+ *      `stream.flushIntervalMs` + `stream.minChunkChars`).
+ *   3. On run end (`onIdle`), the session flushes pending buffer, emits
+ *      `message.done`, then the merged full text plus any accumulated
+ *      media is emitted as a separate `message.send` / `message.reply` —
+ *      mirroring the clawling-channel v1 pattern where streaming `agent`
+ *      events are followed by a consolidated `chat` final.
+ *
+ * Static mode or replyCtx: bypass streaming and emit one `message.send` /
+ * `message.reply` per deliver with text + media.
+ */
+export function createOpenclawClawlingReplyDispatcher(options: ReplyDispatcherOptions): {
+  dispatcher: TypedReplyDispatcherResult["dispatcher"];
+  replyOptions: TypedReplyDispatcherResult["replyOptions"] & StreamingReplyHooks;
+  markDispatchIdle: TypedReplyDispatcherResult["markDispatchIdle"];
+} {
+  const {
+    cfg,
+    runtime,
+    account,
+    client,
+    target,
+    replyCtx,
+    inboundMessageId,
+    inboundForFinalReply,
+    log,
+  } = options;
+  const routing: EnvelopeRouting = { chatId: target.chatId, chatType: target.chatType };
+  const humanDelay = runtime.channel.reply.resolveHumanDelayConfig(cfg, account.userId);
+  const streamingEnabled = account.replyMode === "stream" && !replyCtx;
+
+  const buildApiClient = () => {
+    if (!account.baseUrl || !account.token) return null;
+    return createOpenclawClawlingApiClient({
+      baseUrl: account.baseUrl,
+      token: account.token,
+      userId: account.userId,
+    });
+  };
+
+  async function uploadMediaUrls(urls: string[]): Promise<ClawlingMediaFragment[]> {
+    if (urls.length === 0) return [];
+    const apiClient = buildApiClient();
+    if (!apiClient) {
+      log?.info?.(
+        `[${account.accountId}] openclaw-clawchat outbound media skipped: baseUrl not configured`,
+      );
+      return [];
+    }
+    return await uploadOutboundMedia(urls, { apiClient, runtime, log });
+  }
+
+  // ----- Streaming session state -----------------------------------------
+
+  let streamingSession: BufferedStreamSession | null = null;
+  let streamingMessageId = "";
+  let streamText = "";
+  let reasoningText = "";
+  const accumulatedMediaUrls: string[] = [];
+  let finalEmitted = false;
+  let streamingClosed = false;
+  let runDone = false;
+  // `streamCreatedEmitted` is the authoritative guard: once a `message.created`
+  // has been emitted for this dispatcher instance, never emit another — even
+  // if `onReplyStart` fires again or a pre-onReplyStart `onPartialReply`
+  // raced the lazy open path.
+  let streamCreatedEmitted = false;
+
+  const openSessionIfNeeded = () => {
+    if (!streamingEnabled || streamingSession || streamCreatedEmitted) return;
+    streamCreatedEmitted = true;
+    // Use the inbound message_id when available so the streaming frames
+    // correlate with the user message that triggered this reply; fall back
+    // to a generated id only when not provided (e.g. tests).
+    streamingMessageId = inboundMessageId ?? `${account.userId}-${Date.now()}`;
+    streamingSession = openBufferedStreamingSession({
+      client,
+      routing,
+      sender: {
+        id: account.userId,
+        type: target.chatType,
+        nick_name: account.userId,
+      },
+      messageId: streamingMessageId,
+      flushIntervalMs: account.stream.flushIntervalMs,
+      minChunkChars: account.stream.minChunkChars,
+      maxBufferChars: account.stream.maxBufferChars,
+    });
+    log?.info?.(
+      `[${account.accountId}] openclaw-clawchat streaming opened msg=${streamingMessageId}`,
+    );
+  };
+
+  const buildCombinedStreamSnapshot = (): string => {
+    if (!reasoningText && !streamText) return "";
+    if (!reasoningText) return streamText;
+    if (!streamText) return reasoningText;
+    return `${reasoningText}\n\n${streamText}`;
+  };
+
+  const queueStreamSnapshot = async () => {
+    openSessionIfNeeded();
+    if (!streamingSession) return;
+    const combined = buildCombinedStreamSnapshot();
+    if (!combined) return;
+    await streamingSession.queueSnapshot(combined);
+  };
+
+  const closeStreamingSession = async (reason?: "done" | "fail", failReason?: string) => {
+    if (!streamingSession || streamingClosed) return;
+    streamingClosed = true;
+    if (reason === "fail") {
+      await streamingSession.fail(failReason);
+    } else {
+      await streamingSession.done();
+    }
+    log?.info?.(
+      `[${account.accountId}] openclaw-clawchat streaming closed msg=${streamingMessageId} reason=${reason ?? "done"}`,
+    );
+  };
+
+  // ----- Static send ------------------------------------------------------
+
+  const sendStatic = async (text: string, mediaFragments: ClawlingMediaFragment[] = []) => {
+    if (!text.trim() && mediaFragments.length === 0) return;
+    log?.info?.(
+      `[${account.accountId}] openclaw-clawchat sending static text_len=${text.length} media=${mediaFragments.length} to=${target.chatId}`,
+    );
+    await sendOpenclawClawlingText({
+      client,
+      account,
+      to: target,
+      text,
+      ...(replyCtx ? { replyCtx } : {}),
+      ...(mediaFragments.length > 0 ? { mediaFragments } : {}),
+      log,
+    });
+    log?.info?.(
+      `[${account.accountId}] openclaw-clawchat send complete to=${target.chatId}`,
+    );
+  };
+
+  const emitFinalConsolidatedMessage = async () => {
+    if (finalEmitted) return;
+    finalEmitted = true;
+    const mergedMedia = await uploadMediaUrls(accumulatedMediaUrls.slice());
+    const mergedText = streamText.trim();
+    if (!mergedText && mergedMedia.length === 0) {
+      log?.info?.(
+        `[${account.accountId}] openclaw-clawchat no merged final content; skip consolidated reply`,
+      );
+      return;
+    }
+    log?.info?.(
+      `[${account.accountId}] openclaw-clawchat emitting consolidated final (message.reply) msg=${streamingMessageId} text_len=${mergedText.length} media=${mergedMedia.length}`,
+    );
+    const bodyFragments: Fragment[] = [
+      ...(mergedText ? textToFragments(mergedText) : []),
+      // mediaFragments is the local wide shape; cast at SDK boundary as
+      // we do in outbound.ts.
+      ...(mergedMedia as Fragment[]),
+    ];
+    // Streaming message_id must match the created/add/done frames so the
+    // backend can correlate the consolidated reply with the stream.
+    emitFinalStreamReply(client, {
+      messageId: streamingMessageId,
+      routing,
+      replyTo: {
+        msgId: inboundMessageId ?? streamingMessageId,
+        senderId: inboundForFinalReply?.senderId ?? target.chatId,
+        nickName:
+          inboundForFinalReply?.senderNickName ?? inboundForFinalReply?.senderId ?? target.chatId,
+        fragments: inboundForFinalReply?.bodyText
+          ? [{ kind: "text", text: inboundForFinalReply.bodyText }]
+          : [],
+      },
+      body: { fragments: bodyFragments },
+    });
+  };
+
+  const ingestFinalPayload = (payload: ReplyPayload) => {
+    const text = payload.text ?? "";
+    if (text) streamText = mergeStreamingText(streamText, text);
+    const urls = [
+      ...(payload.mediaUrl ? [payload.mediaUrl] : []),
+      ...(payload.mediaUrls ?? []),
+    ].filter((u): u is string => Boolean(u));
+    for (const url of urls) {
+      if (!accumulatedMediaUrls.includes(url)) accumulatedMediaUrls.push(url);
+    }
+  };
+
+  const ingestBlockText = async (text: string) => {
+    if (!text) return;
+    streamText = `${streamText}${text}`;
+    await queueStreamSnapshot();
+  };
+
+  // ----- Dispatcher -------------------------------------------------------
+
+  const base = runtime.channel.reply.createReplyDispatcherWithTyping({
+    humanDelay,
+    onReplyStart: async () => {
+      // Only clear transient accumulators the first time the run starts.
+      // If `onReplyStart` fires again during the same dispatcher instance
+      // (e.g. typing controller re-entry), we must NOT tear down the stream
+      // session — that would cause a second `message.created`.
+      if (!streamCreatedEmitted) {
+        streamText = "";
+        reasoningText = "";
+        accumulatedMediaUrls.length = 0;
+        finalEmitted = false;
+        streamingClosed = false;
+        runDone = false;
+      }
+      if (streamingEnabled) openSessionIfNeeded();
+    },
+    deliver: async (payload: ReplyPayload, info?: { kind: "tool" | "block" | "final" }) => {
+      const text = payload.text ?? "";
+      const urls = [
+        ...(payload.mediaUrl ? [payload.mediaUrl] : []),
+        ...(payload.mediaUrls ?? []),
+      ].filter((u): u is string => Boolean(u));
+      log?.info?.(
+        `[${account.accountId}] openclaw-clawchat deliver kind=${info?.kind ?? "unknown"} text_len=${text.length} media_urls=${urls.length} reasoning=${payload.isReasoning === true}`,
+      );
+
+      if (payload.isReasoning) {
+        if (!account.forwardThinking) return;
+        if (streamingEnabled) {
+          reasoningText = mergeStreamingText(reasoningText, text);
+          await queueStreamSnapshot();
+        } else {
+          await sendStatic(text);
+        }
+        return;
+      }
+
+      if (info?.kind === "tool" && !account.forwardToolCalls) return;
+
+      if (info?.kind === "final") {
+        ingestFinalPayload(payload);
+        // For streaming: consolidated final is emitted in onIdle after done.
+        // For static: emit immediately.
+        if (!streamingEnabled) {
+          const mediaFragments = await uploadMediaUrls(urls);
+          await sendStatic(text, mediaFragments);
+        }
+        return;
+      }
+
+      // kind === "block" or "tool" — text during the run
+      if (streamingEnabled) {
+        if (text) await ingestBlockText(text);
+        if (urls.length > 0) {
+          const mediaFragments = await uploadMediaUrls(urls);
+          if (mediaFragments.length > 0) {
+            log?.info?.(
+              `[${account.accountId}] openclaw-clawchat mid-stream media emitted as separate message (count=${mediaFragments.length})`,
+            );
+            await sendOpenclawClawlingText({
+              client,
+              account,
+              to: target,
+              text: "",
+              mediaFragments,
+              log,
+            });
+          }
+        }
+      } else {
+        const mediaFragments = await uploadMediaUrls(urls);
+        if (text.trim() || mediaFragments.length > 0) {
+          await sendStatic(text, mediaFragments);
+        }
+      }
+    },
+    onError: (error: unknown, info: { kind: string }) => {
+      log?.error?.(
+        `[${account.accountId}] openclaw-clawchat ${info.kind} reply failed: ${String(error)}`,
+      );
+      void closeStreamingSession("fail", String(error));
+    },
+    onIdle: async () => {
+      if (runDone) return;
+      runDone = true;
+      if (!streamingEnabled) return;
+      await closeStreamingSession("done");
+      await emitFinalConsolidatedMessage();
+    },
+  });
+
+  const streamingHooks: StreamingReplyHooks = streamingEnabled
+    ? {
+        onPartialReply: async (payload: ReplyPayload) => {
+          if (!payload.text || payload.isReasoning) return;
+          // onPartialReply gives progressive snapshots of the current assistant
+          // message text (not deltas). Use mergeStreamingText so overlapping
+          // prefixes collapse into a single growing snapshot.
+          streamText = mergeStreamingText(streamText, payload.text);
+          await queueStreamSnapshot();
+        },
+        ...(account.forwardThinking
+          ? {
+              onReasoningStream: async (payload: ReplyPayload) => {
+                if (!payload.text) return;
+                reasoningText = mergeStreamingText(reasoningText, payload.text);
+                await queueStreamSnapshot();
+              },
+            }
+          : {}),
+      }
+    : {};
+
+  return {
+    dispatcher: base.dispatcher,
+    replyOptions: {
+      ...base.replyOptions,
+      ...streamingHooks,
+    },
+    markDispatchIdle: base.markDispatchIdle,
+  };
+}