@elizaos/agent 0.25.8 → 2.0.0-alpha.83

package/packages/agent/src/services/app-manager.d.ts

@@ -0,0 +1,34 @@
+/**
+ * App Manager — manages app lifecycle: discover, install plugin, show viewer.
+ *
+ * Apps are hosted services. The manager's job is:
+ * 1. List/search apps from the registry
+ * 2. Install the game's plugin onto the agent (triggers restart)
+ * 3. Return the viewer URL so the UI can embed the game client in an iframe
+ *
+ * @module services/app-manager
+ */
+import type { IAgentRuntime } from "@elizaos/core";
+import type { AppLaunchResult, AppStopResult, InstalledAppInfo } from "../contracts/apps";
+import type { InstallProgressLike, PluginManagerLike, RegistryPluginInfo, RegistrySearchResult } from "./plugin-manager-types";
+export type { AppLaunchResult, AppStopResult, AppViewerAuthMessage, InstalledAppInfo, } from "../contracts/apps";
+export declare class AppManager {
+    private readonly activeSessions;
+    listAvailable(pluginManager: PluginManagerLike): Promise<RegistryPluginInfo[]>;
+    search(pluginManager: PluginManagerLike, query: string, limit?: number): Promise<RegistrySearchResult[]>;
+    getInfo(pluginManager: PluginManagerLike, name: string): Promise<RegistryPluginInfo | null>;
+    /**
+     * Launch an app: install its plugin (if needed) and return the viewer URL.
+     *
+     * The plugin connects the agent to the game server. The viewer URL is what
+     * the UI shows in an iframe so the user can watch the agent play.
+     *
+     * After installing a new plugin, the agent needs to restart. The UI should
+     * handle this by showing "connecting..." while the runtime restarts.
+     */
+    launch(pluginManager: PluginManagerLike, name: string, onProgress?: (progress: InstallProgressLike) => void, _runtime?: IAgentRuntime | null): Promise<AppLaunchResult>;
+    stop(pluginManager: PluginManagerLike, name: string): Promise<AppStopResult>;
+    /** List apps whose plugins are currently installed on the agent. */
+    listInstalled(pluginManager: PluginManagerLike): Promise<InstalledAppInfo[]>;
+}
+//# sourceMappingURL=app-manager.d.ts.map

package/packages/agent/src/services/app-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"app-manager.d.ts","sourceRoot":"","sources":["../../../../../src/services/app-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EAEb,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,wBAAwB,CAAC;AAKhC,YAAY,EACV,eAAe,EACf,aAAa,EACb,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAmR3B,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAuC;IAEhE,aAAa,CACjB,aAAa,EAAE,iBAAiB,GAC/B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAwC1B,MAAM,CACV,aAAa,EAAE,iBAAiB,EAChC,KAAK,EAAE,MAAM,EACb,KAAK,SAAK,GACT,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAU5B,OAAO,CACX,aAAa,EAAE,iBAAiB,EAChC,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAIrC;;;;;;;;OAQG;IACG,MAAM,CACV,aAAa,EAAE,iBAAiB,EAChC,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,mBAAmB,KAAK,IAAI,EACpD,QAAQ,CAAC,EAAE,aAAa,GAAG,IAAI,GAC9B,OAAO,CAAC,eAAe,CAAC;IAuGrB,IAAI,CACR,aAAa,EAAE,iBAAiB,EAChC,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,CAAC;IAyDzB,oEAAoE;IAC9D,aAAa,CACjB,aAAa,EAAE,iBAAiB,GAC/B,OAAO,CAAC,gBAAgB,EAAE,CAAC;CAkB/B"}

package/packages/agent/src/services/app-manager.js

@@ -0,0 +1,425 @@
+/**
+ * App Manager — manages app lifecycle: discover, install plugin, show viewer.
+ *
+ * Apps are hosted services. The manager's job is:
+ * 1. List/search apps from the registry
+ * 2. Install the game's plugin onto the agent (triggers restart)
+ * 3. Return the viewer URL so the UI can embed the game client in an iframe
+ *
+ * @module services/app-manager
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { logger } from "@elizaos/core";
+import { deriveEvmAddress, deriveSolanaAddress } from "../api/wallet";
+import { getPluginInfo, getRegistryPlugins } from "./registry-client";
+const LOCAL_PLUGINS_DIR = "plugins";
+const DEFAULT_VIEWER_SANDBOX = "allow-scripts allow-same-origin allow-popups";
+const RS_2004SCAPE_APP_NAME = "@elizaos/app-2004scape";
+const RS_2004SCAPE_AUTH_MESSAGE_TYPE = "RS_2004SCAPE_AUTH";
+const SAFE_APP_URL_PROTOCOLS = new Set(["http:", "https:"]);
+const ALLOWED_APP_URL_TEMPLATE_KEYS = new Set([
+    // Public display identity only.
+    "BOT_NAME",
+    "RS_SDK_BOT_NAME",
+]);
+function resolvePluginPackageName(appInfo) {
+    const npmPackage = appInfo.npm.package.trim();
+    return npmPackage && npmPackage.length > 0 ? npmPackage : appInfo.name;
+}
+function mergeAppMeta(appInfo, meta) {
+    if (!meta)
+        return;
+    appInfo.viewer = meta.viewer ?? appInfo.viewer;
+    appInfo.launchUrl = meta.launchUrl ?? appInfo.launchUrl;
+    appInfo.launchType = meta.launchType ?? appInfo.launchType;
+    appInfo.displayName = meta.displayName ?? appInfo.displayName;
+    appInfo.category = meta.category ?? appInfo.category;
+    appInfo.capabilities = meta.capabilities ?? appInfo.capabilities;
+    appInfo.icon = meta.icon ?? appInfo.icon;
+}
+function isAutoInstallable(appInfo) {
+    const supportsRuntime = appInfo.supports.v0 || appInfo.supports.v1 || appInfo.supports.v2;
+    const hasVersion = Boolean(appInfo.npm.v0Version || appInfo.npm.v1Version || appInfo.npm.v2Version);
+    return supportsRuntime && hasVersion;
+}
+/**
+ * Check if a plugin exists locally in the plugins/ directory.
+ * Local plugins don't need to be installed - they're already available.
+ */
+function isLocalPlugin(appInfo) {
+    const pluginsDir = path.resolve(process.cwd(), LOCAL_PLUGINS_DIR);
+    if (!fs.existsSync(pluginsDir)) {
+        return false;
+    }
+    // Check for directory names that match the app
+    // E.g., @elizaos/app-babylon -> app-babylon
+    const bareName = appInfo.name.replace(/^@[^/]+\//, "");
+    const possibleDirs = [bareName, appInfo.name.replace("/", "-")];
+    for (const dirName of possibleDirs) {
+        const pluginPath = path.join(pluginsDir, dirName);
+        const pluginJsonPath = path.join(pluginPath, "elizaos.plugin.json");
+        if (fs.existsSync(pluginJsonPath)) {
+            return true;
+        }
+    }
+    return false;
+}
+function getTemplateFallbackValue(key) {
+    if (key === "RS_SDK_BOT_NAME") {
+        const runtimeBotName = process.env.BOT_NAME?.trim();
+        if (runtimeBotName && runtimeBotName.length > 0) {
+            return runtimeBotName;
+        }
+        return "testbot";
+    }
+    return undefined;
+}
+function substituteTemplateVars(raw) {
+    return raw.replace(/\{([A-Z0-9_]+)\}/g, (_full, key) => {
+        if (!ALLOWED_APP_URL_TEMPLATE_KEYS.has(key)) {
+            return getTemplateFallbackValue(key) ?? "";
+        }
+        const value = process.env[key];
+        if (value && value.trim().length > 0) {
+            return value.trim();
+        }
+        return getTemplateFallbackValue(key) ?? "";
+    });
+}
+function buildViewerUrl(baseUrl, embedParams) {
+    if (!embedParams || Object.keys(embedParams).length === 0) {
+        return substituteTemplateVars(baseUrl);
+    }
+    const resolvedBaseUrl = substituteTemplateVars(baseUrl);
+    const [beforeHash, hashPartRaw] = resolvedBaseUrl.split("#", 2);
+    const [pathPart, queryPartRaw] = beforeHash.split("?", 2);
+    const queryParams = new URLSearchParams(queryPartRaw ?? "");
+    for (const [key, rawValue] of Object.entries(embedParams)) {
+        queryParams.set(key, substituteTemplateVars(rawValue));
+    }
+    const query = queryParams.toString();
+    const hash = hashPartRaw ? `#${hashPartRaw}` : "";
+    return `${pathPart}${query.length > 0 ? `?${query}` : ""}${hash}`;
+}
+function normalizeSafeAppUrl(url) {
+    const trimmed = url.trim();
+    if (!trimmed)
+        return null;
+    if (trimmed.startsWith("/")) {
+        // Disallow protocol-relative form (`//evil.test`) which escapes same-origin.
+        return trimmed.startsWith("//") ? null : trimmed;
+    }
+    try {
+        const parsed = new URL(trimmed);
+        if (!SAFE_APP_URL_PROTOCOLS.has(parsed.protocol)) {
+            return null;
+        }
+        return trimmed;
+    }
+    catch {
+        return null;
+    }
+}
+function buildViewerAuthMessage(appName, postMessageAuth) {
+    if (!postMessageAuth)
+        return undefined;
+    // 2004scape auth - uses bot name and password from environment
+    if (appName === RS_2004SCAPE_APP_NAME) {
+        // Get username from RS_SDK_BOT_NAME or BOT_NAME, fallback to testbot
+        const username = process.env.RS_SDK_BOT_NAME?.trim() ||
+            process.env.BOT_NAME?.trim() ||
+            "testbot";
+        // Get password from RS_SDK_BOT_PASSWORD or BOT_PASSWORD
+        const password = process.env.RS_SDK_BOT_PASSWORD?.trim() ||
+            process.env.BOT_PASSWORD?.trim() ||
+            "";
+        return {
+            type: RS_2004SCAPE_AUTH_MESSAGE_TYPE,
+            authToken: username, // Using authToken field for username
+            sessionToken: password, // Using sessionToken field for password
+        };
+    }
+    return undefined;
+}
+function buildViewerConfig(appInfo, launchUrl) {
+    const viewerInfo = appInfo.viewer;
+    if (viewerInfo) {
+        const requestedPostMessageAuth = Boolean(viewerInfo.postMessageAuth);
+        const authMessage = buildViewerAuthMessage(appInfo.name, requestedPostMessageAuth);
+        const postMessageAuth = requestedPostMessageAuth && Boolean(authMessage);
+        if (requestedPostMessageAuth && !authMessage) {
+            logger.warn(`[app-manager] ${appInfo.name} requires postMessage auth but no auth payload was generated.`);
+        }
+        const viewerUrl = normalizeSafeAppUrl(buildViewerUrl(viewerInfo.url, viewerInfo.embedParams));
+        if (!viewerUrl) {
+            throw new Error(`Refusing to launch app "${appInfo.name}": unsafe viewer URL`);
+        }
+        return {
+            url: viewerUrl,
+            embedParams: viewerInfo.embedParams,
+            postMessageAuth,
+            sandbox: viewerInfo.sandbox ?? DEFAULT_VIEWER_SANDBOX,
+            authMessage,
+        };
+    }
+    if ((appInfo.launchType === "connect" || appInfo.launchType === "local") &&
+        launchUrl) {
+        const viewerUrl = normalizeSafeAppUrl(launchUrl);
+        if (!viewerUrl) {
+            throw new Error(`Refusing to launch app "${appInfo.name}": unsafe launch URL`);
+        }
+        return {
+            url: viewerUrl,
+            sandbox: DEFAULT_VIEWER_SANDBOX,
+        };
+    }
+    return null;
+}
+/**
+ * Get wallet addresses from agent runtime settings (character secrets).
+ * Falls back to process.env if runtime is not available.
+ */
+function _getWalletAddressesFromRuntime(runtime) {
+    const PLACEHOLDER_RE = /^\[?\s*(REDACTED|PLACEHOLDER|TODO|CHANGEME|EMPTY)\s*]?$/i;
+    let evmAddress = null;
+    let solanaAddress = null;
+    // Try runtime settings first (character secrets), fall back to process.env
+    const evmKey = runtime?.getSetting?.("EVM_PRIVATE_KEY")?.trim() ||
+        process.env.EVM_PRIVATE_KEY?.trim();
+    if (evmKey && !PLACEHOLDER_RE.test(evmKey)) {
+        try {
+            evmAddress = deriveEvmAddress(evmKey);
+        }
+        catch (e) {
+            logger.warn(`[app-manager] Bad EVM key: ${e}`);
+        }
+    }
+    const solKey = runtime?.getSetting?.("SOLANA_PRIVATE_KEY")?.trim() || process.env.SOLANA_PRIVATE_KEY?.trim();
+    if (solKey && !PLACEHOLDER_RE.test(solKey)) {
+        try {
+            solanaAddress = deriveSolanaAddress(solKey);
+        }
+        catch (e) {
+            logger.warn(`[app-manager] Bad SOL key: ${e}`);
+        }
+    }
+    return { evmAddress, solanaAddress };
+}
+export class AppManager {
+    activeSessions = new Map();
+    async listAvailable(pluginManager) {
+        const registry = await pluginManager.refreshRegistry();
+        // Merge in local workspace app entries that are discovered by our
+        // registry-client but not by the elizaos
+        // plugin-manager service.
+        try {
+            const localRegistry = await getRegistryPlugins();
+            for (const [name, info] of localRegistry) {
+                if (!registry.has(name) && info.kind === "app") {
+                    registry.set(name, info);
+                }
+            }
+        }
+        catch {
+            // local discovery is best-effort
+        }
+        // Include app packages: those with "/app-" in the name OR kind === "app"
+        const apps = Array.from(registry.values()).filter((plugin) => {
+            if (plugin.kind === "app")
+                return true;
+            const name = plugin.name.toLowerCase();
+            const npmPackage = plugin.npm.package.toLowerCase();
+            return name.includes("/app-") || npmPackage.includes("/app-");
+        });
+        // Flatten appMeta into top-level fields for the frontend
+        return apps.map((p) => {
+            const meta = p.appMeta;
+            if (!meta)
+                return p;
+            return {
+                ...p,
+                displayName: meta.displayName,
+                launchType: meta.launchType,
+                launchUrl: meta.launchUrl,
+                icon: meta.icon,
+                category: meta.category,
+                capabilities: meta.capabilities,
+                uiExtension: meta.uiExtension,
+                viewer: meta.viewer,
+            };
+        });
+    }
+    async search(pluginManager, query, limit = 15) {
+        const results = await pluginManager.searchRegistry(query, limit);
+        // Filter to only include app packages
+        return results.filter((result) => {
+            const name = result.name.toLowerCase();
+            const npmPackage = result.npmPackage.toLowerCase();
+            return name.includes("/app-") || npmPackage.includes("/app-");
+        });
+    }
+    async getInfo(pluginManager, name) {
+        return pluginManager.getRegistryPlugin(name);
+    }
+    /**
+     * Launch an app: install its plugin (if needed) and return the viewer URL.
+     *
+     * The plugin connects the agent to the game server. The viewer URL is what
+     * the UI shows in an iframe so the user can watch the agent play.
+     *
+     * After installing a new plugin, the agent needs to restart. The UI should
+     * handle this by showing "connecting..." while the runtime restarts.
+     */
+    async launch(pluginManager, name, onProgress, _runtime) {
+        let appInfo = (await pluginManager.getRegistryPlugin(name));
+        // Supplement with local registry metadata since the elizaos plugin-manager
+        // service doesn't include our local workspace app discovery.
+        try {
+            const localInfo = await getPluginInfo(name);
+            if (localInfo) {
+                const meta = localInfo.appMeta;
+                if (!appInfo) {
+                    appInfo = { ...localInfo };
+                    mergeAppMeta(appInfo, meta);
+                }
+                else if (meta && !appInfo.viewer) {
+                    // Merge local metadata into existing registry entry
+                    mergeAppMeta(appInfo, meta);
+                    appInfo.kind = localInfo.kind ?? appInfo.kind;
+                }
+            }
+        }
+        catch {
+            // local lookup is best-effort
+        }
+        if (!appInfo) {
+            throw new Error(`App "${name}" not found in the registry.`);
+        }
+        // The app's plugin is what the agent needs to play the game.
+        // It's the same npm package name as the app, or a separate plugin ref.
+        const pluginName = resolvePluginPackageName(appInfo);
+        // Check if this is a local plugin (already present in plugins/ directory)
+        const isLocal = isLocalPlugin(appInfo);
+        // Check if the plugin is already installed
+        const installed = await pluginManager.listInstalledPlugins();
+        const alreadyInstalled = installed.some((p) => p.name === pluginName);
+        let pluginInstalled = alreadyInstalled || isLocal;
+        let needsRestart = false;
+        if (isLocal) {
+            // Local plugins are already available, no installation needed
+            logger.info(`[app-manager] Using local plugin for ${name}: ${pluginName}`);
+        }
+        else if (!alreadyInstalled) {
+            if (isAutoInstallable(appInfo)) {
+                logger.info(`[app-manager] Installing plugin for app: ${pluginName}`);
+                const result = await pluginManager.installPlugin(pluginName, onProgress);
+                if (!result.success) {
+                    throw new Error(`Failed to install plugin "${pluginName}": ${result.error}`);
+                }
+                pluginInstalled = true;
+                needsRestart = result.requiresRestart;
+                logger.info(`[app-manager] Plugin installed: ${pluginName} v${result.version}`);
+            }
+            else {
+                logger.info(`[app-manager] Skipping plugin install for ${name}: no installable runtime package/version in registry metadata.`);
+            }
+        }
+        else {
+            logger.info(`[app-manager] Plugin already installed: ${pluginName}`);
+        }
+        // Build viewer config from registry app metadata
+        const resolvedLaunchUrl = appInfo.launchUrl
+            ? substituteTemplateVars(appInfo.launchUrl)
+            : null;
+        const launchUrl = resolvedLaunchUrl
+            ? normalizeSafeAppUrl(resolvedLaunchUrl)
+            : null;
+        if (resolvedLaunchUrl && !launchUrl) {
+            throw new Error(`Refusing to launch app "${appInfo.name}": unsafe launch URL`);
+        }
+        const viewer = buildViewerConfig(appInfo, launchUrl);
+        this.activeSessions.set(name, {
+            appName: name,
+            pluginName,
+            launchType: appInfo.launchType ?? "connect",
+            launchUrl,
+            viewerUrl: viewer?.url ?? null,
+            startedAt: new Date().toISOString(),
+        });
+        return {
+            pluginInstalled,
+            needsRestart,
+            displayName: appInfo.displayName ?? appInfo.name,
+            launchType: appInfo.launchType ?? "connect",
+            launchUrl,
+            viewer,
+        };
+    }
+    async stop(pluginManager, name) {
+        const appInfo = (await pluginManager.getRegistryPlugin(name));
+        if (!appInfo) {
+            throw new Error(`App "${name}" not found in the registry.`);
+        }
+        const hadSession = this.activeSessions.delete(name);
+        const pluginName = resolvePluginPackageName(appInfo);
+        const installed = await pluginManager.listInstalledPlugins();
+        const isPluginInstalled = installed.some((plugin) => plugin.name === pluginName);
+        if (!hadSession && !isPluginInstalled) {
+            return {
+                success: false,
+                appName: name,
+                stoppedAt: new Date().toISOString(),
+                pluginUninstalled: false,
+                needsRestart: false,
+                stopScope: "no-op",
+                message: `No active session or installed plugin found for "${name}".`,
+            };
+        }
+        if (isPluginInstalled) {
+            const uninstallResult = await pluginManager.uninstallPlugin(pluginName);
+            if (!uninstallResult.success) {
+                throw new Error(`Failed to stop "${name}": ${uninstallResult.error ?? "plugin uninstall failed"}`);
+            }
+            return {
+                success: true,
+                appName: name,
+                stoppedAt: new Date().toISOString(),
+                pluginUninstalled: true,
+                needsRestart: uninstallResult.requiresRestart,
+                stopScope: "plugin-uninstalled",
+                message: uninstallResult.requiresRestart
+                    ? `${name} disconnected and plugin uninstalled. Agent restart required.`
+                    : `${name} disconnected and plugin uninstalled.`,
+            };
+        }
+        return {
+            success: true,
+            appName: name,
+            stoppedAt: new Date().toISOString(),
+            pluginUninstalled: false,
+            needsRestart: false,
+            stopScope: "viewer-session",
+            message: `${name} viewer session stopped.`,
+        };
+    }
+    /** List apps whose plugins are currently installed on the agent. */
+    async listInstalled(pluginManager) {
+        const installed = await pluginManager.listInstalledPlugins();
+        // Filter to only include app plugins (by name convention or known game plugins)
+        const appPlugins = installed.filter((p) => {
+            const name = p.name.toLowerCase();
+            return name.includes("/app-");
+        });
+        return appPlugins.map((p) => ({
+            name: p.name,
+            displayName: p.name
+                .replace(/^@elizaos\/(app-|plugin-)/, "")
+                .replace(/-/g, " ")
+                .replace(/\b\w/g, (c) => c.toUpperCase()),
+            pluginName: p.name,
+            version: p.version ?? "unknown",
+            installedAt: new Date().toISOString(), // Ejected plugins don't track install time yet
+        }));
+    }
+}

package/packages/agent/src/services/browser-capture.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Headless browser capture — opens the StreamView in headless Chrome and
+ * saves screenshots to a temp file. FFmpeg reads the temp file using
+ * -loop 1 to continuously re-read the latest frame.
+ *
+ * This approach avoids the pipe bottleneck — FFmpeg reads at its own
+ * pace while the browser updates the file independently.
+ *
+ * Visual parity with the desktop shell:
+ * - Appends `?popout` to the URL so the app renders StreamView directly
+ *   (skips onboarding, auth gates, navigation chrome).
+ * - Enables SwiftShader for WebGL so VRM avatar renders identically.
+ * - Seeds localStorage with overlay layout, theme, and avatar index so
+ *   the first rendered frame matches the configured appearance.
+ * - Uses `waitUntil: "networkidle0"` to ensure all assets load before capture.
+ * - Keeps CSS animations/transitions enabled for visual parity.
+ */
+/** Path to the temp frame file that FFmpeg reads */
+export declare const FRAME_FILE: string;
+export interface BrowserCaptureConfig {
+    url: string;
+    width?: number;
+    height?: number;
+    fps?: number;
+    quality?: number;
+    /** Optional overlay layout JSON to seed into localStorage before page load. */
+    overlayLayout?: string;
+    /** Theme name to apply (e.g. "eliza", "haxor", "psycho"). */
+    theme?: string;
+    /** Avatar VRM index (1–8). */
+    avatarIndex?: number;
+    /** Destination ID — seeds the destination-specific localStorage key. */
+    destinationId?: string;
+}
+export declare function startBrowserCapture(config: BrowserCaptureConfig): Promise<void>;
+export declare function stopBrowserCapture(): Promise<void>;
+export declare function isBrowserCaptureRunning(): boolean;
+export declare function hasFrameFile(): boolean;
+//# sourceMappingURL=browser-capture.d.ts.map

package/packages/agent/src/services/browser-capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-capture.d.ts","sourceRoot":"","sources":["../../../../../src/services/browser-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiBH,oDAAoD;AACpD,eAAO,MAAM,UAAU,QAA2C,CAAC;AAEnE,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8BAA8B;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAgCD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,oBAAoB,iBAiHrE;AAED,wBAAsB,kBAAkB,kBASvC;AAED,wBAAgB,uBAAuB,IAAI,OAAO,CAEjD;AAED,wBAAgB,YAAY,IAAI,OAAO,CAEtC"}

package/packages/agent/src/services/browser-capture.js

@@ -0,0 +1,162 @@
+/**
+ * Headless browser capture — opens the StreamView in headless Chrome and
+ * saves screenshots to a temp file. FFmpeg reads the temp file using
+ * -loop 1 to continuously re-read the latest frame.
+ *
+ * This approach avoids the pipe bottleneck — FFmpeg reads at its own
+ * pace while the browser updates the file independently.
+ *
+ * Visual parity with the desktop shell:
+ * - Appends `?popout` to the URL so the app renders StreamView directly
+ *   (skips onboarding, auth gates, navigation chrome).
+ * - Enables SwiftShader for WebGL so VRM avatar renders identically.
+ * - Seeds localStorage with overlay layout, theme, and avatar index so
+ *   the first rendered frame matches the configured appearance.
+ * - Uses `waitUntil: "networkidle0"` to ensure all assets load before capture.
+ * - Keeps CSS animations/transitions enabled for visual parity.
+ */
+import { existsSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import puppeteer from "puppeteer-core";
+const CHROME_PATH = process.platform === "darwin"
+    ? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+    : process.platform === "win32"
+        ? "C:\\Program Files\\Google Chrome\\Application\\chrome.exe"
+        : "/usr/bin/google-chrome-stable";
+let activeBrowser = null;
+let stopSignal = false;
+/** Path to the temp frame file that FFmpeg reads */
+export const FRAME_FILE = join(tmpdir(), "eliza-stream-frame.jpg");
+/**
+ * Ensure the URL includes the `?popout` parameter so the app renders only
+ * StreamView, skipping startup gates and navigation chrome.
+ */
+function ensurePopoutUrl(raw) {
+    try {
+        const u = new URL(raw);
+        // Handle both query and hash-based routing
+        if (u.hash?.includes("?")) {
+            if (!u.hash.includes("popout")) {
+                u.hash = `${u.hash}&popout`;
+            }
+        }
+        else if (u.hash) {
+            u.hash = `${u.hash}?popout`;
+        }
+        else if (!u.searchParams.has("popout")) {
+            u.searchParams.set("popout", "");
+        }
+        return u.toString();
+    }
+    catch {
+        // Fallback: just append
+        const sep = raw.includes("?") ? "&" : "?";
+        return `${raw}${sep}popout`;
+    }
+}
+export async function startBrowserCapture(config) {
+    if (activeBrowser) {
+        console.log("[browser-capture] Already running");
+        return;
+    }
+    const { url, width = 1280, height = 720, quality = 70 } = config;
+    const captureUrl = ensurePopoutUrl(url);
+    stopSignal = false;
+    console.log(`[browser-capture] Launching headless Chrome → ${captureUrl}`);
+    const browser = await puppeteer.launch({
+        executablePath: CHROME_PATH,
+        headless: true,
+        args: [
+            `--window-size=${width},${height}`,
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--disable-extensions",
+            "--mute-audio",
+            // WebGL / SwiftShader — required for VRM avatar rendering parity
+            "--use-gl=swiftshader",
+            "--enable-webgl",
+            "--ignore-gpu-blocklist",
+        ],
+    });
+    activeBrowser = browser;
+    const page = await browser.newPage();
+    await page.setViewport({ width, height, deviceScaleFactor: 1 });
+    // Seed localStorage before navigation so the first render matches the desktop shell.
+    // Keys must match exactly what the React app reads:
+    //   - "eliza:theme"                        → ThemeName
+    //   - "eliza_avatar_index"                 → VRM index (1–8)
+    //   - "eliza.stream.overlay-layout.v1[.destId]" → OverlayLayout JSON
+    await page.evaluateOnNewDocument((overlayLayout, theme, avatarIndex, destinationId) => {
+        if (overlayLayout) {
+            // Seed both global and destination-specific keys so the hook
+            // resolves correctly regardless of when activeDestination loads.
+            localStorage.setItem("eliza.stream.overlay-layout.v1", overlayLayout);
+            if (destinationId) {
+                localStorage.setItem(`eliza.stream.overlay-layout.v1.${destinationId}`, overlayLayout);
+            }
+        }
+        if (theme) {
+            localStorage.setItem("eliza:theme", theme);
+        }
+        if (avatarIndex != null) {
+            localStorage.setItem("eliza_avatar_index", String(avatarIndex));
+        }
+    }, config.overlayLayout, config.theme, config.avatarIndex, config.destinationId);
+    // Use networkidle0 so fonts, VRM models, and preview images finish loading
+    await page.goto(captureUrl, {
+        waitUntil: "networkidle0",
+        timeout: 60_000,
+    });
+    console.log(`[browser-capture] Page loaded, writing frames to ${FRAME_FILE}`);
+    // Use CDP screencast for efficient frame delivery
+    const cdp = await page.createCDPSession();
+    let frameCount = 0;
+    cdp.on("Page.screencastFrame", async (params) => {
+        if (stopSignal)
+            return;
+        try {
+            const buf = Buffer.from(params.data, "base64");
+            if (buf.length > 0) {
+                writeFileSync(FRAME_FILE, buf);
+                frameCount++;
+                if (frameCount % 100 === 0) {
+                    console.log(`[browser-capture] ${frameCount} frames written`);
+                }
+            }
+            await cdp.send("Page.screencastFrameAck", {
+                sessionId: params.sessionId,
+            });
+        }
+        catch {
+            // Ignore
+        }
+    });
+    // Capture every frame from Chrome's compositor (~60fps internally,
+    // limited by everyNthFrame to ~15-30fps actual delivery)
+    await cdp.send("Page.startScreencast", {
+        format: "jpeg",
+        quality,
+        maxWidth: width,
+        maxHeight: height,
+        everyNthFrame: 2, // ~30fps from 60fps compositor
+    });
+    console.log(`[browser-capture] CDP screencast active, saving to ${FRAME_FILE}`);
+}
+export async function stopBrowserCapture() {
+    stopSignal = true;
+    if (activeBrowser) {
+        try {
+            await activeBrowser.close();
+        }
+        catch { }
+        activeBrowser = null;
+    }
+    console.log("[browser-capture] Stopped");
+}
+export function isBrowserCaptureRunning() {
+    return activeBrowser !== null;
+}
+export function hasFrameFile() {
+    return existsSync(FRAME_FILE);
+}