@elizaos/plugin-streaming 2.0.0-beta.1

package/dist/api/stream-routes.js

@@ -0,0 +1,609 @@
+import fs from "node:fs";
+import {
+  formatError,
+  logger,
+  readRequestBody,
+  readRequestBodyBuffer,
+  sendJson,
+  sendJsonError
+} from "@elizaos/core";
+import {
+  getHeadlessCaptureConfig,
+  readStreamSettings,
+  seedOverlayDefaults,
+  validateStreamSettings,
+  writeStreamSettings
+} from "./stream-persistence.js";
+const PLUGIN_BROWSER_PACKAGE = "@elizaos/plugin-browser";
+async function loadBrowserCapture() {
+  return await import(PLUGIN_BROWSER_PACKAGE);
+}
+const MJPEG_BOUNDARY = "elizaframe";
+const mjpegSubscribers = /* @__PURE__ */ new Set();
+let latestFrame = null;
+function pushFrameToSubscribers(frame) {
+  latestFrame = frame;
+  if (mjpegSubscribers.size === 0) return;
+  const header = `--${MJPEG_BOUNDARY}\r
+Content-Type: image/jpeg\r
+Content-Length: ${frame.length}\r
+\r
+`;
+  const headerBuf = Buffer.from(header, "ascii");
+  const trailer = Buffer.from("\r\n", "ascii");
+  const chunk = Buffer.concat([headerBuf, frame, trailer]);
+  const failed = [];
+  for (const sub of mjpegSubscribers) {
+    try {
+      sub.write(chunk);
+    } catch {
+      failed.push(sub);
+    }
+  }
+  for (const sub of failed) {
+    mjpegSubscribers.delete(sub);
+  }
+}
+function getActiveDestination(state) {
+  if (state.activeDestinationId) {
+    return state.destinations.get(state.activeDestinationId);
+  }
+  const first = state.destinations.values().next();
+  return first.done ? void 0 : first.value;
+}
+function json(res, data, status = 200) {
+  sendJson(res, data, status);
+}
+function error(res, message, status) {
+  sendJsonError(res, message, status);
+}
+function detectCaptureMode() {
+  const explicit = process.env.STREAM_MODE;
+  if (explicit === "ui" || explicit === "pipe") return "pipe";
+  if (explicit === "x11grab") return "x11grab";
+  if (explicit === "avfoundation" || explicit === "screen")
+    return "avfoundation";
+  if (explicit === "file") return "file";
+  if ("__elizaScreenCapture" in globalThis) {
+    return "pipe";
+  }
+  if (process.platform === "linux" && process.env.DISPLAY) return "x11grab";
+  if (process.platform === "darwin") return "avfoundation";
+  return "file";
+}
+async function ensureXvfb(display, resolution) {
+  if (process.platform !== "linux") return false;
+  if (!/^:\d+$/.test(display)) {
+    logger.warn(
+      `[stream] Invalid display format: ${display} (expected :<number>)`
+    );
+    return false;
+  }
+  const [w, h] = resolution.split("x");
+  if (!w || !h || !/^\d+$/.test(w) || !/^\d+$/.test(h)) {
+    logger.warn(`[stream] Invalid resolution for Xvfb: ${resolution}`);
+    return false;
+  }
+  if (process.env.DISPLAY === display) return true;
+  try {
+    const { execSync } = await import("node:child_process");
+    try {
+      execSync(`xdpyinfo -display ${display}`, {
+        stdio: "ignore",
+        timeout: 3e3
+      });
+      logger.info(`[stream] Xvfb already running on display ${display}`);
+      return true;
+    } catch {
+    }
+    const { spawn: spawnProc } = await import("node:child_process");
+    const xvfb = spawnProc(
+      "Xvfb",
+      [display, "-screen", "0", `${w}x${h}x24`, "-ac"],
+      {
+        stdio: "ignore",
+        detached: true
+      }
+    );
+    xvfb.unref();
+    await new Promise((r) => setTimeout(r, 1e3));
+    logger.info(`[stream] Started Xvfb on display ${display} (${resolution})`);
+    process.env.DISPLAY = display;
+    return true;
+  } catch (err) {
+    logger.warn(`[stream] Failed to start Xvfb: ${err}`);
+    return false;
+  }
+}
+async function startStreamPipeline(state, rtmpUrl, rtmpKey) {
+  if (!/^rtmps?:\/\//i.test(rtmpUrl)) {
+    throw new Error("RTMP URL must use rtmp:// or rtmps:// scheme");
+  }
+  const activeDest = getActiveDestination(state);
+  if (activeDest) {
+    seedOverlayDefaults(activeDest);
+  }
+  const destId = activeDest?.id ?? null;
+  const mode = detectCaptureMode();
+  const audioSource = process.env.STREAM_AUDIO_SOURCE ?? "silent";
+  const audioDevice = process.env.STREAM_AUDIO_DEVICE;
+  const volume = parseInt(process.env.STREAM_VOLUME ?? "80", 10);
+  const resolution = "1280x720";
+  const baseConfig = {
+    rtmpUrl,
+    rtmpKey,
+    resolution,
+    bitrate: "1500k",
+    audioSource,
+    audioDevice,
+    volume
+  };
+  switch (mode) {
+    case "pipe": {
+      logger.info("[stream] Capture mode: pipe (desktop UI)");
+      await state.streamManager.start({
+        ...baseConfig,
+        inputMode: "pipe",
+        framerate: 15
+      });
+      if (state.screenCapture && !state.screenCapture.isFrameCaptureActive()) {
+        try {
+          const captureOpts = {
+            fps: 15,
+            quality: 70,
+            endpoint: "/api/stream/frame"
+          };
+          if (state.activeStreamSource.type !== "stream-tab" && state.activeStreamSource.url) {
+            captureOpts.gameUrl = state.activeStreamSource.url;
+          }
+          await state.screenCapture.startFrameCapture(captureOpts);
+          logger.info("[stream] Auto-started desktop frame capture");
+        } catch (err) {
+          logger.warn(`[stream] Failed to auto-start frame capture: ${err}`);
+        }
+      } else if (!state.screenCapture) {
+        logger.warn(
+          "[stream] ScreenCaptureManager not available -- frame capture must be started manually"
+        );
+      }
+      break;
+    }
+    case "x11grab": {
+      const display = process.env.STREAM_DISPLAY ?? ":99";
+      logger.info(`[stream] Capture mode: x11grab (display ${display})`);
+      await ensureXvfb(display, resolution);
+      const captureUrl = state.captureUrl ?? process.env.STREAM_CAPTURE_URL ?? `http://127.0.0.1:${state.port ?? 2138}`;
+      try {
+        const { startBrowserCapture } = await loadBrowserCapture();
+        await startBrowserCapture({
+          url: captureUrl,
+          width: 1280,
+          height: 720,
+          quality: 70,
+          ...getHeadlessCaptureConfig(destId)
+        });
+      } catch (err) {
+        logger.warn(`[stream] Browser launch on ${display} failed: ${err}`);
+      }
+      await state.streamManager.start({
+        ...baseConfig,
+        inputMode: "x11grab",
+        display,
+        framerate: 30
+      });
+      break;
+    }
+    case "avfoundation": {
+      const videoDevice = process.env.STREAM_VIDEO_DEVICE ?? "3";
+      logger.info(
+        `[stream] Capture mode: avfoundation (device ${videoDevice})`
+      );
+      await state.streamManager.start({
+        ...baseConfig,
+        inputMode: "avfoundation",
+        videoDevice,
+        framerate: 30
+      });
+      break;
+    }
+    default: {
+      const captureUrl = state.captureUrl ?? process.env.STREAM_CAPTURE_URL ?? `http://127.0.0.1:${state.port ?? 2138}`;
+      logger.info(
+        `[stream] Capture mode: file (browser capture -> ${captureUrl})`
+      );
+      const { startBrowserCapture, FRAME_FILE } = await loadBrowserCapture();
+      try {
+        await startBrowserCapture({
+          url: captureUrl,
+          width: 1280,
+          height: 720,
+          quality: 70,
+          ...getHeadlessCaptureConfig(destId)
+        });
+        await new Promise((resolve) => {
+          const check = setInterval(() => {
+            try {
+              if (fs.existsSync(FRAME_FILE) && fs.statSync(FRAME_FILE).size > 0) {
+                clearInterval(check);
+                resolve(true);
+              }
+            } catch {
+            }
+          }, 200);
+          setTimeout(() => {
+            clearInterval(check);
+            resolve(false);
+          }, 1e4);
+        });
+      } catch (captureErr) {
+        logger.warn(`[stream] Browser capture failed: ${captureErr}`);
+      }
+      await state.streamManager.start({
+        ...baseConfig,
+        inputMode: "file",
+        frameFile: FRAME_FILE,
+        framerate: 30
+      });
+      break;
+    }
+  }
+  return { inputMode: mode || "file", audioSource };
+}
+async function handleStreamRoute(req, res, pathname, method, state) {
+  if (!pathname.startsWith("/api/stream/") && !pathname.startsWith("/api/streaming/")) {
+    return false;
+  }
+  if (method === "POST" && pathname === "/api/stream/frame") {
+    try {
+      const buf = await readRequestBodyBuffer(req, {
+        maxBytes: 2 * 1024 * 1024
+      });
+      if (!buf || buf.length === 0) {
+        error(res, "Empty frame", 400);
+        return true;
+      }
+      pushFrameToSubscribers(buf);
+      if (state.streamManager.isRunning()) {
+        state.streamManager.writeFrame(buf);
+      }
+      res.writeHead(200);
+      res.end();
+    } catch {
+      error(res, "Frame write failed", 500);
+    }
+    return true;
+  }
+  if (method === "GET" && pathname === "/api/stream/screen") {
+    res.writeHead(200, {
+      "Content-Type": `multipart/x-mixed-replace; boundary=${MJPEG_BOUNDARY}`,
+      "Cache-Control": "no-store, no-cache",
+      Connection: "close",
+      "Access-Control-Allow-Origin": "*"
+    });
+    mjpegSubscribers.add(res);
+    if (latestFrame) {
+      const header = `--${MJPEG_BOUNDARY}\r
+Content-Type: image/jpeg\r
+Content-Length: ${latestFrame.length}\r
+\r
+`;
+      res.write(
+        Buffer.concat([
+          Buffer.from(header, "ascii"),
+          latestFrame,
+          Buffer.from("\r\n", "ascii")
+        ])
+      );
+    }
+    const cleanup = () => {
+      mjpegSubscribers.delete(res);
+    };
+    req.on("close", cleanup);
+    req.on("error", cleanup);
+    res.on("close", cleanup);
+    res.on("error", cleanup);
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/live") {
+    if (state.streamManager.isRunning()) {
+      const health = state.streamManager.getHealth();
+      json(res, {
+        ok: true,
+        live: true,
+        message: "Already streaming",
+        ...health
+      });
+      return true;
+    }
+    const dest = getActiveDestination(state);
+    if (!dest) {
+      error(res, "No streaming destination configured", 400);
+      return true;
+    }
+    try {
+      const { rtmpUrl, rtmpKey } = await dest.getCredentials();
+      const { inputMode, audioSource } = await startStreamPipeline(
+        state,
+        rtmpUrl,
+        rtmpKey
+      );
+      await dest.onStreamStart?.();
+      json(res, {
+        ok: true,
+        live: true,
+        rtmpUrl,
+        inputMode,
+        audioSource,
+        destination: dest.id
+      });
+    } catch (err) {
+      error(res, formatError(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/offline") {
+    try {
+      try {
+        const { stopBrowserCapture } = await loadBrowserCapture();
+        await stopBrowserCapture();
+      } catch {
+      }
+      if (state.streamManager.isRunning()) {
+        await state.streamManager.stop();
+      }
+      try {
+        await getActiveDestination(state)?.onStreamStop?.();
+      } catch {
+      }
+      json(res, { ok: true, live: false });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/start") {
+    try {
+      const bodyStr = await readRequestBody(req);
+      const body = typeof bodyStr === "string" ? JSON.parse(bodyStr) : bodyStr;
+      const rtmpUrl = body?.rtmpUrl;
+      const rtmpKey = body?.rtmpKey;
+      if (!rtmpUrl || !rtmpKey) {
+        error(res, "rtmpUrl and rtmpKey are required", 400);
+        return true;
+      }
+      if (!/^rtmps?:\/\//i.test(rtmpUrl)) {
+        error(res, "rtmpUrl must use rtmp:// or rtmps:// scheme", 400);
+        return true;
+      }
+      const VALID_INPUT_MODES = ["testsrc", "avfoundation", "pipe"];
+      const inputMode = body?.inputMode ?? "testsrc";
+      if (!VALID_INPUT_MODES.includes(inputMode)) {
+        error(
+          res,
+          `inputMode must be one of: ${VALID_INPUT_MODES.join(", ")}`,
+          400
+        );
+        return true;
+      }
+      const resolution = body?.resolution || "1280x720";
+      if (!/^\d{3,4}x\d{3,4}$/.test(resolution)) {
+        error(res, "resolution must match WIDTHxHEIGHT (e.g. 1280x720)", 400);
+        return true;
+      }
+      const bitrate = body?.bitrate || "2500k";
+      if (!/^\d+k$/.test(bitrate)) {
+        error(res, "bitrate must match NUMBERk (e.g. 2500k)", 400);
+        return true;
+      }
+      const framerate = body?.framerate ?? 30;
+      if (typeof framerate !== "number" || !Number.isInteger(framerate) || framerate < 1 || framerate > 60) {
+        error(res, "framerate must be an integer between 1 and 60", 400);
+        return true;
+      }
+      await state.streamManager.start({
+        rtmpUrl,
+        rtmpKey,
+        inputMode,
+        resolution,
+        bitrate,
+        framerate
+      });
+      json(res, { ok: true, message: "Stream started" });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/stop") {
+    try {
+      const result = await state.streamManager.stop();
+      json(res, { ok: true, ...result });
+    } catch (err) {
+      error(res, formatError(err), 500);
+    }
+    return true;
+  }
+  if (method === "GET" && pathname === "/api/stream/status") {
+    const health = state.streamManager.getHealth();
+    const activeDest = getActiveDestination(state);
+    const destInfo = activeDest ? { id: activeDest.id, name: activeDest.name } : null;
+    json(res, { ok: true, ...health, destination: destInfo });
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/volume") {
+    try {
+      const body = await readRequestBody(req);
+      const parsed = typeof body === "string" ? JSON.parse(body) : body;
+      const level = parsed?.volume;
+      if (typeof level !== "number" || !Number.isFinite(level) || level < 0 || level > 100) {
+        error(res, "volume must be a number between 0 and 100", 400);
+        return true;
+      }
+      await state.streamManager.setVolume(level);
+      json(res, {
+        ok: true,
+        volume: state.streamManager.getVolume(),
+        muted: state.streamManager.isMuted()
+      });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/mute") {
+    try {
+      await state.streamManager.mute();
+      json(res, {
+        ok: true,
+        muted: true,
+        volume: state.streamManager.getVolume()
+      });
+    } catch (err) {
+      error(res, formatError(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/unmute") {
+    try {
+      await state.streamManager.unmute();
+      json(res, {
+        ok: true,
+        muted: false,
+        volume: state.streamManager.getVolume()
+      });
+    } catch (err) {
+      error(res, formatError(err), 500);
+    }
+    return true;
+  }
+  if (method === "GET" && pathname === "/api/streaming/destinations") {
+    const destinations = Array.from(state.destinations.values()).map((d) => ({
+      id: d.id,
+      name: d.name,
+      active: d.id === (state.activeDestinationId ?? state.destinations.keys().next().value)
+    }));
+    json(res, { ok: true, destinations });
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/streaming/destination") {
+    try {
+      const body = await readRequestBody(req);
+      const parsed = typeof body === "string" ? JSON.parse(body) : body;
+      const destinationId = parsed?.destinationId;
+      if (!destinationId) {
+        error(res, "destinationId is required", 400);
+        return true;
+      }
+      const target = state.destinations.get(destinationId);
+      if (target) {
+        state.activeDestinationId = destinationId;
+        json(res, {
+          ok: true,
+          destination: { id: target.id, name: target.name }
+        });
+      } else {
+        error(res, `Unknown destination: ${destinationId}`, 404);
+      }
+    } catch (err) {
+      error(res, formatError(err), 500);
+    }
+    return true;
+  }
+  if (method === "GET" && pathname === "/api/stream/settings") {
+    try {
+      const settings = readStreamSettings();
+      json(res, { ok: true, settings });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/settings") {
+    try {
+      const body = await readRequestBody(req);
+      const parsed = typeof body === "string" ? JSON.parse(body) : body;
+      const result = validateStreamSettings(parsed?.settings);
+      if (result.error || !result.settings) {
+        error(res, result.error ?? "Invalid settings", 400);
+        return true;
+      }
+      const existing = readStreamSettings();
+      const merged = { ...existing, ...result.settings };
+      writeStreamSettings(merged);
+      if (typeof merged.avatarIndex === "number" && Number.isFinite(merged.avatarIndex)) {
+        try {
+          state.mirrorStreamAvatarToElizaConfig?.(merged.avatarIndex);
+        } catch (err) {
+          logger.warn(
+            `[stream] mirrorStreamAvatarToElizaConfig failed (stream settings still saved): ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+      json(res, { ok: true, settings: merged });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  if (method === "GET" && pathname === "/api/stream/source") {
+    json(res, { source: state.activeStreamSource });
+    return true;
+  }
+  if (method === "POST" && pathname === "/api/stream/source") {
+    try {
+      const body = await readRequestBody(req);
+      const { sourceType, customUrl } = JSON.parse(
+        typeof body === "string" ? body : JSON.stringify(body)
+      );
+      if (!["stream-tab", "game", "custom-url"].includes(sourceType)) {
+        error(res, "Invalid sourceType", 400);
+        return true;
+      }
+      if (sourceType === "custom-url" && !customUrl) {
+        error(res, "customUrl required for custom-url source", 400);
+        return true;
+      }
+      if (sourceType === "game" && !customUrl) {
+        error(res, "customUrl required for game source", 400);
+        return true;
+      }
+      if ((sourceType === "game" || sourceType === "custom-url") && customUrl && !/^https?:\/\//i.test(customUrl)) {
+        error(res, "customUrl must use http:// or https:// scheme", 400);
+        return true;
+      }
+      if (state.screenCapture?.isFrameCaptureActive()) {
+        state.screenCapture.stopFrameCapture?.();
+      }
+      const captureOpts = {
+        fps: 15,
+        quality: 70,
+        endpoint: "/api/stream/frame"
+      };
+      if (sourceType === "game" || sourceType === "custom-url") {
+        captureOpts.gameUrl = customUrl;
+      }
+      state.activeStreamSource = { type: sourceType, url: customUrl };
+      if (state.streamManager.isRunning() && state.screenCapture) {
+        try {
+          await state.screenCapture.startFrameCapture(captureOpts);
+        } catch (err) {
+          logger.warn(
+            `[stream] Failed to restart frame capture after source switch: ${err}`
+          );
+        }
+      }
+      json(res, { ok: true, source: state.activeStreamSource });
+    } catch (err) {
+      error(res, String(err), 500);
+    }
+    return true;
+  }
+  return false;
+}
+export {
+  detectCaptureMode,
+  ensureXvfb,
+  getActiveDestination,
+  handleStreamRoute
+};
+//# sourceMappingURL=stream-routes.js.map