openclaw-elys 1.7.7 → 1.8.1

package/README.md

@@ -52,6 +52,120 @@ One command to revoke device, clean up config, and remove the plugin:
 npx openclaw-elys@latest uninstall
 ```
 
+## Streaming / 流式输出
+
+OpenClaw 支持 block streaming，需要在 `~/.openclaw/openclaw.json` 中配置：
+
+```json
+{
+  "agents": {
+    "defaults": {
+      "blockStreamingDefault": "on"
+    }
+  },
+  "channels": {
+    "elys": {
+      "blockStreaming": true
+    }
+  }
+}
+```
+
+配置后需要**完全重启 OpenClaw**（非热重载）才能生效。
+
+发送消息时设置 `stream: true` 可以获取流式输出（多个 chunk），否则只返回最终结果。
+
+## MQTT Protocol / 通信协议
+
+### Topics
+
+| 方向 | Topic | 说明 |
+|------|-------|------|
+| 下行（命令） | `elys/down/{device_id}` | Gateway → Plugin |
+| 上行（响应） | `elys/up/{device_id}` | Plugin → Gateway |
+
+### Message Types
+
+#### Command（下行）
+
+```json
+{
+  "id": "cmd_xxx",
+  "type": "command",
+  "timestamp": 1709827200,
+  "command": "chat",
+  "args": { "text": "你好" },
+  "stream": true,
+  "media_url": "https://example.com/photo.jpg",
+  "media_urls": ["https://example.com/1.jpg", "https://example.com/2.jpg"],
+  "media_type": "image/jpeg"
+}
+```
+
+- `stream`: 是否请求流式响应（可选，默认 false）
+- `media_url`: 单个媒体 URL（可选）
+- `media_urls`: 多个媒体 URL（可选）
+- `media_type`: 媒体 MIME 类型（可选，默认 `application/octet-stream`）
+- `args.text` 和 `media_url` 可以同时存在（文字 + 附件）
+
+#### Ack（上行）
+
+```json
+{ "id": "cmd_xxx", "type": "ack", "timestamp": 1709827200 }
+```
+
+#### Stream（上行，流式响应）
+
+```json
+{
+  "id": "cmd_xxx",
+  "type": "stream",
+  "timestamp": 1709827200,
+  "chunk": "这是一段回复文字",
+  "seq": 1,
+  "done": false,
+  "media_url": "https://example.com/generated.png",
+  "media_urls": ["https://example.com/1.png", "https://example.com/2.png"]
+}
+```
+
+- `seq`: 序号，从 1 开始递增
+- `done`: 最后一个 chunk 设为 true
+- `media_url`/`media_urls`: AI 生成的媒体（可选）
+
+#### Result（上行，最终结果）
+
+```json
+{
+  "id": "cmd_xxx",
+  "type": "result",
+  "timestamp": 1709827200,
+  "status": "success",
+  "result": { "text": "完整回复内容" },
+  "media_url": "https://example.com/output.png",
+  "media_urls": ["https://example.com/1.png"]
+}
+```
+
+### Supported Media Types / 支持的媒体类型
+
+| 类型 | MIME type | 说明 |
+|------|-----------|------|
+| 图片 | `image/jpeg`, `image/png`, `image/webp` | 照片、截图 |
+| GIF | `image/gif` | 动图 |
+| 视频 | `video/mp4`, `video/quicktime` | 视频 |
+| 音频 | `audio/mpeg`, `audio/ogg`, `audio/wav` | 语音/音频 |
+| 文件 | `application/pdf`, `application/octet-stream` | 通用文件 |
+
+不传 `media_type` 时默认为 `application/octet-stream`。
+
+### Behavior / 行为
+
+- **Debounce**：500ms 内的多条消息会合并为一条发送给 AI
+- **Sequential Queue**：推理中收到新消息会排队等待当前推理完成后再处理（与 Discord/Slack 一致）
+- **Dedup**：QoS 1 重传的消息会自动去重（基于 command ID）
+- **Auto ID**：Gateway 会自动为空 ID 的命令生成 `cmd_` 前缀的唯一 ID
+
 ## License
 
 MIT

package/dist/src/monitor.js

@@ -1,3 +1,8 @@
+import { createWriteStream } from "node:fs";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pipeline } from "node:stream/promises";
 import { loadCredentials } from "./config.js";
 import { registerDevice } from "./register.js";
 import { ElysDeviceMQTTClient } from "./mqtt-client.js";
@@ -39,6 +44,25 @@ export async function monitorElysProvider(opts) {
             try {
                 let seq = 0;
                 let fullText = "";
+                // Download inbound media (user-sent) to local temp files
+                // OpenClaw expects local file paths in MediaPath/MediaUrl, not remote URLs
+                const rawMediaUrls = cmd.mediaUrls?.length
+                    ? cmd.mediaUrls
+                    : cmd.mediaUrl
+                        ? [cmd.mediaUrl]
+                        : [];
+                const downloadedPaths = [];
+                const downloadedTypes = [];
+                for (const url of rawMediaUrls) {
+                    try {
+                        const localPath = await downloadToTemp(url, log);
+                        downloadedPaths.push(localPath);
+                        downloadedTypes.push(cmd.mediaType ?? guessMediaType(url));
+                    }
+                    catch (err) {
+                        log(`[elys] failed to download media ${url}:`, err);
+                    }
+                }
                 const inboundCtx = finalizeCtx({
                     Body: formatCommandAsText(cmd),
                     BodyForAgent: formatCommandAsText(cmd),
@@ -54,19 +78,36 @@ export async function monitorElysProvider(opts) {
                     CommandAuthorized: true,
                     OriginatingChannel: "elys",
                     OriginatingTo: credentials.deviceId,
+                    // Inbound media as local file paths
+                    ...(downloadedPaths.length > 0 && {
+                        MediaPath: downloadedPaths[0],
+                        MediaUrl: downloadedPaths[0],
+                        MediaPaths: downloadedPaths,
+                        MediaUrls: downloadedPaths,
+                        MediaType: downloadedTypes[0],
+                        MediaTypes: downloadedTypes,
+                    }),
                 });
                 // Deliver callback: stream chunks back via MQTT
                 const wantStream = cmd.stream === true;
                 const deliver = async (payload, info) => {
-                    if (payload.text) {
-                        fullText += payload.text;
+                    const mediaUrl = payload.mediaUrl?.trim();
+                    const mediaUrls = payload.mediaUrls?.filter((u) => u?.trim());
+                    const hasMedia = Boolean(mediaUrl || mediaUrls?.length);
+                    const media = hasMedia ? { mediaUrl, mediaUrls } : undefined;
+                    if (payload.text || hasMedia) {
+                        if (payload.text)
+                            fullText += payload.text;
                         if (wantStream || info.kind === "final") {
                             seq++;
                             const done = info.kind === "final";
-                            mqttClient.publishStreamChunk(cmd.id, payload.text, seq, done);
+                            mqttClient.publishStreamChunk(cmd.id, payload.text ?? "", seq, done, media);
+                        }
+                        if (hasMedia) {
+                            log(`[elys] media: ${mediaUrl ?? mediaUrls?.join(", ")}`);
                         }
                         if (info.kind === "block") {
-                            log(`[elys] stream chunk #${seq}: ${payload.text.slice(0, 80)}...`);
+                            log(`[elys] stream chunk #${seq}: ${(payload.text ?? "").slice(0, 80)}...`);
                         }
                         else if (info.kind === "final") {
                             log(`[elys] final reply delivered`);
@@ -150,3 +191,52 @@ function formatCommandAsText(cmd) {
     }
     return parts.join(" ");
 }
+const MIME_BY_EXT = {
+    ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png",
+    ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
+    ".mp4": "video/mp4", ".mov": "video/quicktime", ".avi": "video/x-msvideo",
+    ".mp3": "audio/mpeg", ".ogg": "audio/ogg", ".wav": "audio/wav",
+    ".pdf": "application/pdf",
+};
+function guessMediaType(url) {
+    const pathname = new URL(url).pathname.toLowerCase();
+    for (const [ext, mime] of Object.entries(MIME_BY_EXT)) {
+        if (pathname.includes(ext))
+            return mime;
+    }
+    return "application/octet-stream";
+}
+function extFromMimeOrUrl(url, mime) {
+    for (const [ext, m] of Object.entries(MIME_BY_EXT)) {
+        if (m === mime)
+            return ext;
+    }
+    const pathname = new URL(url).pathname;
+    const dot = pathname.lastIndexOf(".");
+    if (dot >= 0) {
+        const ext = pathname.slice(dot).split(/[?#!/]/)[0];
+        if (ext.length <= 6)
+            return ext;
+    }
+    return ".bin";
+}
+let tempDir = null;
+async function downloadToTemp(url, log) {
+    if (!tempDir) {
+        tempDir = await mkdtemp(join(tmpdir(), "elys-media-"));
+    }
+    const mime = guessMediaType(url);
+    const ext = extFromMimeOrUrl(url, mime);
+    const filename = `media_${Date.now()}${ext}`;
+    const filePath = join(tempDir, filename);
+    log(`[elys] downloading media: ${url} → ${filePath}`);
+    const resp = await fetch(url);
+    if (!resp.ok || !resp.body) {
+        throw new Error(`HTTP ${resp.status} downloading ${url}`);
+    }
+    const { Readable } = await import("node:stream");
+    const nodeStream = Readable.fromWeb(resp.body);
+    await pipeline(nodeStream, createWriteStream(filePath));
+    log(`[elys] downloaded media: ${filePath}`);
+    return filePath;
+}

package/dist/src/mqtt-client.d.ts

@@ -37,7 +37,10 @@ export declare class ElysDeviceMQTTClient {
     connect(abortSignal?: AbortSignal): Promise<void>;
     disconnect(): void;
     /** Send a stream chunk (for streaming AI responses) */
-    publishStreamChunk(commandId: string, chunk: string, seq: number, done: boolean): void;
+    publishStreamChunk(commandId: string, chunk: string, seq: number, done: boolean, media?: {
+        mediaUrl?: string;
+        mediaUrls?: string[];
+    }): void;
     private onMessage;
     private flushDebounce;
     private startCommand;

package/dist/src/mqtt-client.js

@@ -123,7 +123,7 @@ export class ElysDeviceMQTTClient {
         }
     }
     /** Send a stream chunk (for streaming AI responses) */
-    publishStreamChunk(commandId, chunk, seq, done) {
+    publishStreamChunk(commandId, chunk, seq, done, media) {
         const msg = {
             id: commandId,
             type: "stream",
@@ -132,6 +132,10 @@ export class ElysDeviceMQTTClient {
             seq,
             done,
         };
+        if (media?.mediaUrl)
+            msg.mediaUrl = media.mediaUrl;
+        if (media?.mediaUrls?.length)
+            msg.mediaUrls = media.mediaUrls;
         this.publish(msg);
     }
     // ─── Inbound message pipeline: dedup → ack → debounce → abort → execute ───

package/dist/src/types.d.ts

@@ -25,6 +25,9 @@ export interface CommandMessage extends MQTTBaseMessage {
     command: string;
     args?: Record<string, unknown>;
     stream?: boolean;
+    mediaUrl?: string;
+    mediaUrls?: string[];
+    mediaType?: string;
 }
 export interface AckMessage extends MQTTBaseMessage {
     type: "ack";
@@ -34,10 +37,14 @@ export interface StreamMessage extends MQTTBaseMessage {
     chunk: string;
     done: boolean;
     seq: number;
+    mediaUrl?: string;
+    mediaUrls?: string[];
 }
 export interface ResultMessage extends MQTTBaseMessage {
     type: "result";
     status: "success" | "error";
     result?: Record<string, unknown>;
     error?: string;
+    mediaUrl?: string;
+    mediaUrls?: string[];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openclaw-elys",
-  "version": "1.7.7",
+  "version": "1.8.1",
   "description": "OpenClaw Elys channel plugin — connects to Elys App",
   "type": "module",
   "main": "dist/index.js",