@elizaos/app-core 2.0.0-alpha.37

package/src/components/stream/StreamVoiceConfig.tsx

@@ -0,0 +1,160 @@
+/**
+ * StreamVoiceConfig — compact voice config panel for StreamView.
+ *
+ * Shows a toggle to enable/disable TTS on the RTMP stream, detected
+ * provider + API key status, and a test button to speak sample text.
+ */
+
+import { client } from "@elizaos/app-core/api";
+import { useTimeout } from "@elizaos/app-core/hooks";
+import { useApp } from "@elizaos/app-core/state";
+import { Button } from "@elizaos/ui";
+import { Volume2, VolumeX } from "lucide-react";
+import { useCallback, useEffect, useState } from "react";
+
+interface VoiceStatus {
+  enabled: boolean;
+  autoSpeak: boolean;
+  provider: string | null;
+  configuredProvider: string | null;
+  hasApiKey: boolean;
+  isSpeaking: boolean;
+  isAttached: boolean;
+}
+
+export function StreamVoiceConfig({ streamLive }: { streamLive: boolean }) {
+  const { setTimeout } = useTimeout();
+
+  const { t } = useApp();
+  const [status, setStatus] = useState<VoiceStatus | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [speaking, setSpeaking] = useState(false);
+
+  // Poll voice status
+  useEffect(() => {
+    let mounted = true;
+    const poll = async () => {
+      try {
+        const res = await client.getStreamVoice();
+        if (mounted && res.ok) {
+          setStatus(res);
+          setSpeaking(res.isSpeaking);
+        }
+      } catch {
+        // API may not be available yet
+      }
+    };
+    poll();
+    const id = setInterval(poll, 5_000);
+    return () => {
+      mounted = false;
+      clearInterval(id);
+    };
+  }, []);
+
+  const toggleEnabled = useCallback(async () => {
+    if (!status || loading) return;
+    setLoading(true);
+    try {
+      const next = !status.enabled;
+      const res = await client.saveStreamVoice({ enabled: next });
+      if (res.ok) {
+        setStatus((prev) =>
+          prev ? { ...prev, enabled: res.voice.enabled } : prev,
+        );
+      }
+    } catch {
+      // Ignore
+    } finally {
+      setLoading(false);
+    }
+  }, [status, loading]);
+
+  const testSpeak = useCallback(async () => {
+    if (speaking) return;
+    setSpeaking(true);
+    try {
+      await client.streamVoiceSpeak("Hello, I am now speaking on the stream.");
+    } catch {
+      // Ignore
+    } finally {
+      // Poll will update speaking state
+      setTimeout(() => setSpeaking(false), 3000);
+    }
+  }, [
+    speaking, // Poll will update speaking state
+    setTimeout,
+  ]);
+
+  if (!status) return null;
+
+  const providerLabel = status.provider
+    ? status.provider.charAt(0).toUpperCase() + status.provider.slice(1)
+    : "None";
+
+  return (
+    <div className="flex items-center gap-2 text-xs">
+      {/* Toggle */}
+      <Button
+        variant="ghost"
+        size="sm"
+        onClick={toggleEnabled}
+        disabled={loading}
+        className={`flex items-center gap-1.5 px-2 py-1 h-7 rounded text-xs font-medium transition-colors ${
+          status.enabled
+            ? "bg-accent/20 text-txt hover:bg-accent/30"
+            : "bg-surface text-muted hover:bg-surface-hover"
+        }`}
+        title={
+          status.enabled
+            ? "Voice on stream: ON (click to disable)"
+            : "Voice on stream: OFF (click to enable)"
+        }
+      >
+        {status.enabled ? (
+          <Volume2 className="w-3.5 h-3.5" />
+        ) : (
+          <VolumeX className="w-3.5 h-3.5" />
+        )}
+        <span>{t("streamvoiceconfig.Voice")}</span>
+      </Button>
+
+      {/* Status indicators (when enabled) */}
+      {status.enabled && (
+        <>
+          <span className="text-muted">
+            {providerLabel}
+            {!status.hasApiKey && status.provider !== "edge" && (
+              <span
+                className="text-warning ml-1"
+                title={t("streamvoiceconfig.NoAPIKeyConfigure")}
+              >
+                {t("streamvoiceconfig.NoKey")}
+              </span>
+            )}
+          </span>
+
+          {speaking && (
+            <span className="text-txt animate-pulse">
+              {t("streamvoiceconfig.Speaking")}
+            </span>
+          )}
+
+          {/* Test button — only when stream is live and bridge attached */}
+          {streamLive && status.isAttached && (
+            <Button
+              variant="ghost"
+              size="sm"
+              onClick={testSpeak}
+              disabled={speaking}
+              className="px-1.5 py-0.5 h-6 rounded bg-surface text-muted hover:bg-surface-hover hover:text-txt text-[10px] transition-colors"
+              title={t("streamvoiceconfig.TestTTSOnStream")}
+            >
+              {t("streamvoiceconfig.Test")}
+            </Button>
+          )}
+        </>
+      )}
+    </div>
+  );
+}

package/src/components/stream/helpers.ts

@@ -0,0 +1,134 @@
+/**
+ * Shared helpers, constants, and types for StreamView sub-components.
+ */
+
+import type { StreamEventEnvelope } from "@elizaos/app-core/api";
+import { invokeDesktopBridgeRequest } from "@elizaos/app-core/bridge";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+export const CHAT_ACTIVE_WINDOW_MS = 30_000;
+export const TERMINAL_ACTIVE_WINDOW_MS = 15_000;
+
+/** PIP window dimensions (640x360 → captures at 1280x720 on Retina 2x displays). */
+export const PIP_SIZE = { width: 640, height: 360 };
+export const FULL_SIZE = { width: 1280, height: 720 };
+
+export const CHANNEL_COLORS: Record<
+  string,
+  { border: string; bg: string; text: string }
+> = {
+  retake: {
+    border: "border-fuchsia-500/30",
+    bg: "bg-fuchsia-500/5",
+    text: "text-fuchsia-400",
+  },
+  discord: {
+    border: "border-indigo-500/30",
+    bg: "bg-indigo-500/5",
+    text: "text-indigo-400",
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Event helpers
+// ---------------------------------------------------------------------------
+
+export function getEventFrom(event: StreamEventEnvelope): string | undefined {
+  const payload = event.payload as Record<string, unknown>;
+  if (typeof payload.displayName === "string" && payload.displayName.trim())
+    return payload.displayName.trim();
+  if (typeof payload.from === "string" && payload.from.trim())
+    return payload.from.trim();
+  return undefined;
+}
+
+export function getEventText(event: StreamEventEnvelope): string {
+  const payload = event.payload as Record<string, unknown>;
+  const text = payload.text;
+  if (typeof text === "string" && text.trim()) return text.trim();
+  const preview = payload.preview;
+  if (typeof preview === "string" && preview.trim()) return preview.trim();
+  const reason = payload.reason;
+  if (typeof reason === "string" && reason.trim()) return reason.trim();
+  return event.stream ? `${event.stream} event` : event.type;
+}
+
+export function getEventSource(event: StreamEventEnvelope): string {
+  const payload = event.payload as Record<string, unknown>;
+  if (typeof payload.source === "string") return payload.source;
+  if (typeof payload.channel === "string") return payload.channel;
+  return event.stream ?? "agent";
+}
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type AgentMode = "gaming" | "terminal" | "chatting" | "idle";
+
+export type StreamSourceType = "stream-tab" | "game" | "custom-url";
+
+export const STREAM_SOURCE_LABELS: Record<StreamSourceType, string> = {
+  "stream-tab": "Stream Tab",
+  game: "Game",
+  "custom-url": "Custom URL",
+};
+
+export function isSupportedStreamUrl(value: string): boolean {
+  const trimmed = value.trim();
+  if (!trimmed) return false;
+
+  try {
+    const url = new URL(trimmed);
+    return url.protocol === "http:" || url.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Popout / Always-on-top
+// ---------------------------------------------------------------------------
+
+/** Detect popout mode from URL. */
+export const IS_POPOUT = (() => {
+  if (typeof window === "undefined") return false;
+  const params = new URLSearchParams(
+    window.location.search || window.location.hash.split("?")[1] || "",
+  );
+  return params.has("popout");
+})();
+
+/** Toggle always-on-top for the current window (Electron only). */
+export async function toggleAlwaysOnTop(pinned: boolean): Promise<boolean> {
+  try {
+    // Try Capacitor Desktop plugin
+    const cap = (window as unknown as Record<string, unknown>).Capacitor as
+      | Record<string, unknown>
+      | undefined;
+    if (cap?.Plugins) {
+      const plugins = cap.Plugins as Record<string, unknown>;
+      const desktop = plugins.Desktop as
+        | { setAlwaysOnTop?: (opts: { flag: boolean }) => Promise<void> }
+        | undefined;
+      if (desktop?.setAlwaysOnTop) {
+        await desktop.setAlwaysOnTop({ flag: pinned });
+        return pinned;
+      }
+    }
+    const result = await invokeDesktopBridgeRequest({
+      rpcMethod: "desktopSetAlwaysOnTop",
+      ipcChannel: "desktop:setAlwaysOnTop",
+      params: { flag: pinned },
+    });
+    if (result !== null) {
+      return pinned;
+    }
+  } catch {
+    // Non-fatal — may not be in Electron
+  }
+  return false;
+}

package/src/components/stream/overlays/OverlayLayer.tsx

@@ -0,0 +1,75 @@
+/**
+ * OverlayLayer — Renders all enabled widget instances as absolute-positioned
+ * DOM elements inside the StreamView content area.
+ *
+ * Z-index strategy:
+ *   Content 0 | Widgets 10-39 | Alerts 40-49 | GameViewOverlay 50+
+ */
+
+import type { StreamEventEnvelope } from "@elizaos/app-core/api";
+import { useMemo } from "react";
+import type { AgentMode } from "../helpers";
+import { getWidget } from "./registry";
+import type { OverlayLayout } from "./types";
+
+// Ensure all built-in widgets are registered
+import "./built-in";
+
+interface OverlayLayerProps {
+  layout: OverlayLayout;
+  events: StreamEventEnvelope[];
+  agentMode: AgentMode;
+  agentName: string;
+}
+
+export function OverlayLayer({
+  layout,
+  events,
+  agentMode,
+  agentName,
+}: OverlayLayerProps) {
+  const enabledWidgets = useMemo(
+    () => layout.widgets.filter((w) => w.enabled),
+    [layout.widgets],
+  );
+
+  if (enabledWidgets.length === 0) return null;
+
+  return (
+    <div
+      className="absolute inset-0 pointer-events-none"
+      style={{ zIndex: 10 }}
+    >
+      {enabledWidgets.map((instance) => {
+        const def = getWidget(instance.type);
+        if (!def) return null;
+
+        const filtered = events.filter(
+          (e) => e.stream != null && def.subscribesTo.includes(e.stream),
+        );
+        const Widget = def.render;
+
+        return (
+          <div
+            key={instance.id}
+            className="absolute pointer-events-auto"
+            style={{
+              left: `${instance.position.x}%`,
+              top: `${instance.position.y}%`,
+              width: `${instance.position.width}%`,
+              height: `${instance.position.height}%`,
+              zIndex: instance.zIndex,
+            }}
+          >
+            <Widget
+              instance={instance}
+              events={filtered}
+              agentMode={agentMode}
+              agentName={agentName}
+            />
+          </div>
+        );
+      })}
+    </div>
+  );
+}

package/src/components/stream/overlays/built-in/ActionTickerWidget.tsx

@@ -0,0 +1,64 @@
+/**
+ * ActionTicker — Horizontal scrolling strip of recent agent actions/tool calls.
+ * Follows the ChatTicker pattern.
+ */
+
+import { useApp } from "@elizaos/app-core/state";
+import { useMemo } from "react";
+import { getEventText } from "../../helpers";
+import { registerWidget } from "../registry";
+import type { WidgetDefinition, WidgetRenderProps } from "../types";
+
+function ActionTicker({ instance, events }: WidgetRenderProps) {
+  const { t } = useApp();
+  const maxItems = (instance.config.maxItems as number) ?? 8;
+
+  const items = useMemo(() => {
+    return events.slice(-maxItems).map((e) => ({
+      id: e.eventId,
+      stream: e.stream ?? "action",
+      text: getEventText(e),
+    }));
+  }, [events, maxItems]);
+
+  if (items.length === 0) return null;
+
+  return (
+    <div className="h-full flex items-center px-3 bg-bg/70 border-b border-border/30 backdrop-blur-sm overflow-hidden">
+      <div className="flex items-center gap-4 text-[11px] text-muted overflow-x-auto whitespace-nowrap scrollbar-hide">
+        <span className="text-[9px] uppercase tracking-wider text-muted/60 shrink-0 font-medium">
+          {t("actiontickerwidget.actions")}
+        </span>
+        {items.map((item) => (
+          <span key={item.id} className="shrink-0">
+            <span className="text-txt/70">[{item.stream}]</span>{" "}
+            <span className="text-txt/80">{item.text.slice(0, 60)}</span>
+          </span>
+        ))}
+      </div>
+    </div>
+  );
+}
+
+const definition: WidgetDefinition = {
+  type: "action-ticker",
+  name: "Action Ticker",
+  description: "Scrolling strip of recent agent actions and tool calls",
+  subscribesTo: ["action", "tool"],
+  defaultPosition: { x: 0, y: 0, width: 100, height: 4 },
+  defaultZIndex: 12,
+  configSchema: {
+    maxItems: {
+      type: "number",
+      label: "Max visible items",
+      default: 8,
+      min: 3,
+      max: 20,
+    },
+  },
+  defaultConfig: { maxItems: 8 },
+  render: ActionTicker,
+};
+
+registerWidget(definition);
+export default definition;

package/src/components/stream/overlays/built-in/AlertPopupWidget.tsx

@@ -0,0 +1,87 @@
+/**
+ * AlertPopup — Animated fade-in/out card for new viewer joins and chat triggers.
+ */
+
+import { useEffect, useMemo, useState } from "react";
+import { getEventFrom, getEventText } from "../../helpers";
+import { registerWidget } from "../registry";
+import type { WidgetDefinition, WidgetRenderProps } from "../types";
+
+interface AlertEntry {
+  id: string;
+  title: string;
+  body: string;
+}
+
+function AlertPopup({ instance, events }: WidgetRenderProps) {
+  const displayMs = (instance.config.displayDuration as number) ?? 5000;
+
+  const latest = useMemo(() => {
+    if (events.length === 0) return null;
+    const e = events[events.length - 1];
+    const from = getEventFrom(e);
+    const isViewer = e.stream === "new_viewer";
+    return {
+      id: e.eventId,
+      title: isViewer ? "New Viewer!" : (from ?? "Alert"),
+      body: isViewer
+        ? `${from ?? "Someone"} joined the stream`
+        : getEventText(e),
+    } satisfies AlertEntry;
+  }, [events]);
+
+  const [shown, setShown] = useState<AlertEntry | null>(null);
+  const [fading, setFading] = useState(false);
+
+  useEffect(() => {
+    if (!latest) return;
+    setShown(latest);
+    setFading(false);
+
+    const fadeTimer = setTimeout(() => setFading(true), displayMs - 500);
+    const hideTimer = setTimeout(() => setShown(null), displayMs);
+
+    return () => {
+      clearTimeout(fadeTimer);
+      clearTimeout(hideTimer);
+    };
+  }, [latest, displayMs]);
+
+  if (!shown) return null;
+
+  return (
+    <div className="h-full flex items-start justify-end p-2">
+      <div
+        className={`bg-bg/90 border border-accent/40 rounded-lg px-4 py-3 backdrop-blur-sm shadow-lg max-w-[280px] transition-opacity duration-500 ${fading ? "opacity-0" : "opacity-100"}`}
+      >
+        <p className="text-xs font-medium text-txt">{shown.title}</p>
+        <p className="text-[11px] text-txt/80 mt-0.5 leading-snug">
+          {shown.body}
+        </p>
+      </div>
+    </div>
+  );
+}
+
+const definition: WidgetDefinition = {
+  type: "alert-popup",
+  name: "Alert Popup",
+  description: "Animated alerts for new viewers and chat messages",
+  subscribesTo: ["new_viewer", "message"],
+  defaultPosition: { x: 65, y: 5, width: 33, height: 15 },
+  defaultZIndex: 40,
+  configSchema: {
+    displayDuration: {
+      type: "number",
+      label: "Display duration (ms)",
+      default: 5000,
+      min: 2000,
+      max: 15000,
+    },
+  },
+  defaultConfig: { displayDuration: 5000 },
+  render: AlertPopup,
+};
+
+registerWidget(definition);
+export default definition;

package/src/components/stream/overlays/built-in/BrandingWidget.tsx

@@ -0,0 +1,51 @@
+/**
+ * Branding — Static agent name / logo watermark in the corner.
+ */
+
+import { registerWidget } from "../registry";
+import type { WidgetDefinition, WidgetRenderProps } from "../types";
+
+function Branding({ instance, agentName }: WidgetRenderProps) {
+  const opacity = (instance.config.opacity as number) ?? 0.6;
+  const showLabel = (instance.config.showLabel as boolean) ?? true;
+
+  return (
+    <div className="h-full flex items-end justify-end p-2" style={{ opacity }}>
+      <div className="bg-bg/60 border border-border/30 rounded-md px-3 py-1.5 backdrop-blur-sm">
+        {showLabel && (
+          <p className="text-[10px] uppercase tracking-widest text-muted/80 font-medium">
+            {agentName}
+          </p>
+        )}
+      </div>
+    </div>
+  );
+}
+
+const definition: WidgetDefinition = {
+  type: "branding",
+  name: "Branding",
+  description: "Agent name/logo watermark",
+  subscribesTo: [],
+  defaultPosition: { x: 78, y: 90, width: 20, height: 8 },
+  defaultZIndex: 11,
+  configSchema: {
+    opacity: {
+      type: "number",
+      label: "Opacity",
+      default: 0.6,
+      min: 0.1,
+      max: 1,
+    },
+    showLabel: {
+      type: "boolean",
+      label: "Show agent name",
+      default: true,
+    },
+  },
+  defaultConfig: { opacity: 0.6, showLabel: true },
+  render: Branding,
+};
+
+registerWidget(definition);
+export default definition;

package/src/components/stream/overlays/built-in/CustomHtmlWidget.tsx

@@ -0,0 +1,105 @@
+/**
+ * CustomHtml — User-provided HTML/CSS/JS or external URL rendered in a
+ * sandboxed iframe.
+ *
+ * ## Security model
+ *
+ * Both modes use `sandbox="allow-scripts"` WITHOUT `allow-same-origin`.
+ * This means the iframe:
+ *   - Can execute JavaScript but is treated as a unique opaque origin
+ *   - Cannot access the parent frame's DOM, cookies, or localStorage
+ *   - Cannot make same-origin requests to the host application
+ *   - Cannot navigate the parent frame
+ *
+ * Inline mode adds a strict CSP meta tag that blocks all network requests
+ * (no fetch, XHR, image loads, or script imports from external origins).
+ * This confines user-provided JS to pure computation and DOM manipulation.
+ *
+ * URL mode loads external content with `allow-scripts` only. The content
+ * can make network requests to its own origin but cannot interact with
+ * the host application. Only use trusted URLs — the iframe will execute
+ * whatever JavaScript the remote page serves.
+ */
+
+import { useApp } from "@elizaos/app-core/state";
+import { useRef } from "react";
+import { registerWidget } from "../registry";
+import type { WidgetDefinition, WidgetRenderProps } from "../types";
+
+/** CSP that blocks all network access — inline scripts/styles only. */
+const INLINE_CSP =
+  "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline';";
+
+function CustomHtml({ instance }: WidgetRenderProps) {
+  const { t } = useApp();
+  const mode = (instance.config.mode as string) ?? "inline";
+  const htmlContent = (instance.config.html as string) ?? "";
+  const cssContent = (instance.config.css as string) ?? "";
+  const jsContent = (instance.config.js as string) ?? "";
+  const url = (instance.config.url as string) ?? "";
+
+  const iframeRef = useRef<HTMLIFrameElement>(null);
+
+  if (mode === "url" && url) {
+    return (
+      <iframe
+        ref={iframeRef}
+        src={url}
+        sandbox="allow-scripts"
+        className="w-full h-full border-0 rounded"
+        title={t("customhtmlwidget.CustomWidgetExter")}
+      />
+    );
+  }
+
+  // Inline mode: render HTML/CSS/JS via srcdoc with strict CSP
+  const srcdoc = `<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta http-equiv="Content-Security-Policy" content="${INLINE_CSP}">
+<style>${cssContent}</style>
+</head>
+<body>${htmlContent}<script>${jsContent}</script></body>
+</html>`;
+
+  return (
+    <iframe
+      ref={iframeRef}
+      srcDoc={srcdoc}
+      sandbox="allow-scripts"
+      className="w-full h-full border-0 rounded"
+      title={t("customhtmlwidget.CustomWidgetInlin")}
+    />
+  );
+}
+
+const definition: WidgetDefinition = {
+  type: "custom-html",
+  name: "Custom HTML",
+  description:
+    "User-provided HTML/CSS/JS or external URL in a sandboxed iframe",
+  subscribesTo: [],
+  defaultPosition: { x: 10, y: 10, width: 30, height: 30 },
+  defaultZIndex: 20,
+  configSchema: {
+    mode: {
+      type: "select",
+      label: "Mode",
+      default: "inline",
+      options: [
+        { label: "Inline HTML", value: "inline" },
+        { label: "External URL", value: "url" },
+      ],
+    },
+    html: { type: "string", label: "HTML", default: "" },
+    css: { type: "string", label: "CSS", default: "" },
+    js: { type: "string", label: "JavaScript", default: "" },
+    url: { type: "string", label: "URL", default: "" },
+  },
+  defaultConfig: { mode: "inline", html: "", css: "", js: "", url: "" },
+  render: CustomHtml,
+};
+
+registerWidget(definition);
+export default definition;