@elizaos/app-core 2.0.0-alpha.37

package/src/bridge/capacitor-bridge.ts

@@ -0,0 +1,295 @@
+/**
+ * Capacitor Bridge
+ *
+ * This module provides a bridge between the Milady web UI and native
+ * Capacitor plugins. It exposes a global API that the UI can use to
+ * access native capabilities like camera, microphone, file system, etc.
+ *
+ * The bridge is designed to be progressively enhanced - features are
+ * only available when running on platforms that support them.
+ */
+
+import { Capacitor } from "@capacitor/core";
+import { Haptics, ImpactStyle, NotificationType } from "@capacitor/haptics";
+import { BRIDGE_READY_EVENT, dispatchMiladyEvent } from "../events";
+import { isElectrobunRuntime } from "./electrobun-runtime";
+
+// Import the plugin bridge
+import {
+  getPluginCapabilities,
+  getPlugins,
+  isFeatureAvailable,
+  type MiladyPlugins,
+  type PluginCapabilities,
+} from "./plugin-bridge";
+
+// Platform detection
+const platform = Capacitor.getPlatform();
+const isNative = Capacitor.isNativePlatform();
+const isIOS = platform === "ios";
+const isAndroid = platform === "android";
+
+function isElectronPlatform(): boolean {
+  return platform === "electron" || isElectrobunRuntime();
+}
+
+function isWebPlatform(): boolean {
+  return platform === "web" && !isElectrobunRuntime();
+}
+
+/**
+ * Capability flags indicating what features are available
+ */
+export interface CapacitorCapabilities {
+  /** Whether we're running in a native container */
+  native: boolean;
+  /** Platform identifier */
+  platform: "ios" | "android" | "electron" | "web";
+  /** Haptic feedback support */
+  haptics: boolean;
+  /** Camera capture support */
+  camera: boolean;
+  /** Microphone/audio capture support */
+  microphone: boolean;
+  /** Screen recording support */
+  screenCapture: boolean;
+  /** File system access */
+  fileSystem: boolean;
+  /** Push notifications */
+  notifications: boolean;
+  /** Geolocation */
+  geolocation: boolean;
+  /** Background execution */
+  background: boolean;
+  /** Voice wake/always-on listening */
+  voiceWake: boolean;
+}
+
+/**
+ * Get the current platform capabilities
+ */
+export function getCapabilities(): CapacitorCapabilities {
+  const isElectron = isElectronPlatform();
+  return {
+    native: isNative,
+    platform: platform as CapacitorCapabilities["platform"],
+    haptics: isNative && (isIOS || isAndroid),
+    camera: isNative,
+    microphone: isNative,
+    screenCapture: isNative && !isElectron, // Electron needs different handling
+    fileSystem: isNative,
+    notifications: isNative,
+    geolocation: true, // Available on web too via browser API
+    background: isNative && !isElectron,
+    voiceWake: isNative && (isIOS || isAndroid), // macOS via Swabble handled separately
+  };
+}
+
+/**
+ * Haptic feedback wrapper
+ */
+export const haptics = {
+  /**
+   * Trigger a light impact haptic (for UI interactions)
+   */
+  async light(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.impact({ style: ImpactStyle.Light });
+  },
+
+  /**
+   * Trigger a medium impact haptic (for confirmations)
+   */
+  async medium(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.impact({ style: ImpactStyle.Medium });
+  },
+
+  /**
+   * Trigger a heavy impact haptic (for important actions)
+   */
+  async heavy(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.impact({ style: ImpactStyle.Heavy });
+  },
+
+  /**
+   * Trigger a success notification haptic
+   */
+  async success(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.notification({ type: NotificationType.Success });
+  },
+
+  /**
+   * Trigger a warning notification haptic
+   */
+  async warning(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.notification({ type: NotificationType.Warning });
+  },
+
+  /**
+   * Trigger an error notification haptic
+   */
+  async error(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.notification({ type: NotificationType.Error });
+  },
+
+  /**
+   * Start a selection change haptic (for pickers)
+   */
+  async selectionStart(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.selectionStart();
+  },
+
+  /**
+   * Trigger selection changed haptic
+   */
+  async selectionChanged(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.selectionChanged();
+  },
+
+  /**
+   * End selection change haptic
+   */
+  async selectionEnd(): Promise<void> {
+    if (!isNative) return;
+    await Haptics.selectionEnd();
+  },
+};
+
+/**
+ * Plugin registry for custom Milady plugins
+ *
+ * Custom plugins (Gateway, Swabble, Canvas, etc.) will register themselves here
+ * when they're loaded. This allows the UI to check for plugin availability
+ * and access them in a type-safe way.
+ */
+type PluginInstance = Record<string, unknown>;
+const pluginRegistry = new Map<string, PluginInstance>();
+
+/**
+ * Register a custom plugin
+ */
+export function registerPlugin(name: string, plugin: PluginInstance): void {
+  pluginRegistry.set(name, plugin);
+  console.log(`[Capacitor Bridge] Registered plugin: ${name}`);
+}
+
+/**
+ * Get a registered plugin
+ */
+export function getPlugin<T extends PluginInstance>(
+  name: string,
+): T | undefined {
+  return pluginRegistry.get(name) as T | undefined;
+}
+
+/**
+ * Check if a plugin is registered
+ */
+export function hasPlugin(name: string): boolean {
+  return pluginRegistry.has(name);
+}
+
+/**
+ * The global Milady bridge object exposed to the UI
+ */
+export interface MiladyBridge {
+  /** Platform capabilities */
+  capabilities: CapacitorCapabilities;
+  /** Plugin-specific capabilities */
+  pluginCapabilities: PluginCapabilities;
+  /** Haptic feedback */
+  haptics: typeof haptics;
+  /** Get a registered plugin */
+  getPlugin: typeof getPlugin;
+  /** Check if a plugin exists */
+  hasPlugin: typeof hasPlugin;
+  /** Register a new plugin */
+  registerPlugin: typeof registerPlugin;
+  /** Get all Milady plugins with fallback support */
+  plugins: MiladyPlugins;
+  /** Check if a specific feature is available */
+  isFeatureAvailable: typeof isFeatureAvailable;
+  /** Platform info */
+  platform: {
+    name: string;
+    isNative: boolean;
+    isIOS: boolean;
+    isAndroid: boolean;
+    isElectron: boolean;
+    isWeb: boolean;
+    isMacOS: boolean;
+  };
+}
+
+/**
+ * Create the global bridge object
+ */
+function createBridge(): MiladyBridge {
+  const isElectron = isElectronPlatform();
+  return {
+    capabilities: getCapabilities(),
+    pluginCapabilities: getPluginCapabilities(),
+    haptics,
+    getPlugin,
+    hasPlugin,
+    registerPlugin,
+    plugins: getPlugins(),
+    isFeatureAvailable,
+    platform: {
+      name: platform,
+      isNative,
+      isIOS,
+      isAndroid,
+      isElectron,
+      isWeb: isWebPlatform(),
+      isMacOS: isElectron, // Electron is used for macOS/desktop
+    },
+  };
+}
+
+// Extend the Window interface to include our bridge
+declare global {
+  interface Window {
+    Milady: MiladyBridge;
+  }
+}
+
+/**
+ * Initialize the Capacitor bridge
+ *
+ * This exposes the bridge object on window.Milady for use by the UI.
+ */
+export function initializeCapacitorBridge(): void {
+  window.Milady = createBridge();
+
+  // Dispatch an event to notify that the bridge is ready
+  dispatchMiladyEvent(BRIDGE_READY_EVENT, window.Milady);
+}
+
+/**
+ * Wait for the bridge to be ready
+ *
+ * Returns immediately if already initialized, otherwise waits for the event.
+ */
+export function waitForBridge(): Promise<MiladyBridge> {
+  if (window.Milady) {
+    return Promise.resolve(window.Milady);
+  }
+
+  return new Promise((resolve) => {
+    document.addEventListener(
+      BRIDGE_READY_EVENT,
+      (event) => {
+        resolve((event as CustomEvent<MiladyBridge>).detail);
+      },
+      { once: true },
+    );
+  });
+}

package/src/bridge/electrobun-rpc.ts

@@ -0,0 +1,58 @@
+export type ElectrobunRequestHandler = (params?: unknown) => Promise<unknown>;
+
+export type ElectrobunMessageListener = (payload: unknown) => void;
+
+export interface ElectrobunRendererRpc {
+  request: Record<string, ElectrobunRequestHandler>;
+  onMessage: (messageName: string, listener: ElectrobunMessageListener) => void;
+  offMessage: (
+    messageName: string,
+    listener: ElectrobunMessageListener,
+  ) => void;
+}
+
+interface DesktopBridgeWindow extends Window {
+  __MILADY_ELECTROBUN_RPC__?: ElectrobunRendererRpc;
+}
+
+function getDesktopBridgeWindow(): DesktopBridgeWindow | null {
+  if (typeof window === "undefined") {
+    return null;
+  }
+
+  return window as DesktopBridgeWindow;
+}
+
+export function getElectrobunRendererRpc(): ElectrobunRendererRpc | undefined {
+  return getDesktopBridgeWindow()?.__MILADY_ELECTROBUN_RPC__;
+}
+
+export async function invokeDesktopBridgeRequest<T>(options: {
+  rpcMethod: string;
+  ipcChannel: string;
+  params?: unknown;
+}): Promise<T | null> {
+  const rpc = getElectrobunRendererRpc();
+  const request = rpc?.request?.[options.rpcMethod];
+  if (request) {
+    return (await request(options.params)) as T;
+  }
+
+  return null;
+}
+
+export function subscribeDesktopBridgeEvent(options: {
+  rpcMessage: string;
+  ipcChannel: string;
+  listener: ElectrobunMessageListener;
+}): () => void {
+  const rpc = getElectrobunRendererRpc();
+  if (rpc) {
+    rpc.onMessage(options.rpcMessage, options.listener);
+    return () => {
+      rpc.offMessage(options.rpcMessage, options.listener);
+    };
+  }
+
+  return () => {};
+}

package/src/bridge/electrobun-runtime.ts

@@ -0,0 +1,28 @@
+interface ElectrobunBrowserWindow extends Window {
+  __electrobunWindowId?: number;
+  __electrobunWebviewId?: number;
+}
+
+function getRuntimeWindow(): ElectrobunBrowserWindow | null {
+  if (typeof window === "undefined") {
+    return null;
+  }
+
+  return window as ElectrobunBrowserWindow;
+}
+
+export function isElectrobunRuntime(): boolean {
+  const runtimeWindow = getRuntimeWindow();
+  if (!runtimeWindow) {
+    return false;
+  }
+
+  return (
+    typeof runtimeWindow.__electrobunWindowId === "number" ||
+    typeof runtimeWindow.__electrobunWebviewId === "number"
+  );
+}
+
+export function getBackendStartupTimeoutMs(): number {
+  return isElectrobunRuntime() ? 180_000 : 30_000;
+}

package/src/bridge/index.ts

@@ -0,0 +1,5 @@
+export * from "./capacitor-bridge";
+export * from "./electrobun-rpc";
+export * from "./electrobun-runtime";
+export * from "./plugin-bridge";
+export * from "./storage-bridge";

package/src/bridge/native-plugins.ts

@@ -0,0 +1,134 @@
+import { Capacitor, type PluginListenerHandle } from "@capacitor/core";
+
+type NativePlugin = Record<string, unknown>;
+
+function getCapacitorPlugins(): Record<string, unknown> {
+  const capacitor = Capacitor as unknown as {
+    Plugins?: Record<string, unknown>;
+  };
+  if (capacitor.Plugins) {
+    return capacitor.Plugins;
+  }
+  if (typeof window !== "undefined") {
+    const windowCapacitor = (
+      window as unknown as {
+        Capacitor?: { Plugins?: Record<string, unknown> };
+      }
+    ).Capacitor;
+    return windowCapacitor?.Plugins ?? {};
+  }
+  return {};
+}
+
+export function getNativePlugin<T extends NativePlugin>(name: string): T {
+  return (getCapacitorPlugins()[name] ?? {}) as T;
+}
+
+export interface SwabbleConfig {
+  triggers: string[];
+  minPostTriggerGap?: number;
+  minCommandLength?: number;
+  locale?: string;
+  sampleRate?: number;
+  modelSize?: "tiny" | "base" | "small" | "medium" | "large";
+}
+
+export interface SwabbleAudioLevelEvent {
+  level: number;
+  peak?: number;
+}
+
+export interface SwabblePluginLike extends NativePlugin {
+  getConfig(): Promise<{ config: SwabbleConfig | null }>;
+  isListening(): Promise<{ listening: boolean }>;
+  updateConfig(options: { config: Partial<SwabbleConfig> }): Promise<void>;
+  start(options: { config: SwabbleConfig }): Promise<{ started: boolean }>;
+  stop(): Promise<void>;
+  addListener(
+    eventName: "audioLevel",
+    listenerFunc: (event: SwabbleAudioLevelEvent) => void,
+  ): Promise<PluginListenerHandle>;
+}
+
+export interface TalkModeTranscriptEvent {
+  transcript?: string;
+  isFinal?: boolean;
+}
+
+export interface TalkModeErrorEvent {
+  code?: string;
+  message?: string;
+}
+
+export interface TalkModeStateEvent {
+  state?: string;
+}
+
+export interface TalkModePermissionStatus {
+  microphone?: "granted" | "denied" | "prompt";
+  speechRecognition?: "granted" | "denied" | "prompt" | "not_supported";
+}
+
+export interface TalkModePluginLike extends NativePlugin {
+  addListener(
+    eventName: "transcript",
+    listenerFunc: (event: TalkModeTranscriptEvent) => void,
+  ): Promise<PluginListenerHandle>;
+  addListener(
+    eventName: "error",
+    listenerFunc: (event: TalkModeErrorEvent) => void,
+  ): Promise<PluginListenerHandle>;
+  addListener(
+    eventName: "stateChange",
+    listenerFunc: (event: TalkModeStateEvent) => void,
+  ): Promise<PluginListenerHandle>;
+  checkPermissions(): Promise<TalkModePermissionStatus>;
+  requestPermissions(): Promise<TalkModePermissionStatus>;
+  start(options?: {
+    config?: {
+      stt?: {
+        engine?: "whisper" | "web";
+        modelSize?: "tiny" | "base" | "small" | "medium" | "large";
+        language?: string;
+        sampleRate?: number;
+      };
+      silenceWindowMs?: number;
+      interruptOnSpeech?: boolean;
+    };
+  }): Promise<{ started: boolean; error?: string }>;
+  stop(): Promise<void>;
+}
+
+export type GenericNativePlugin = NativePlugin;
+
+export function getGatewayPlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("Gateway");
+}
+
+export function getSwabblePlugin(): SwabblePluginLike {
+  return getNativePlugin<SwabblePluginLike>("Swabble");
+}
+
+export function getTalkModePlugin(): TalkModePluginLike {
+  return getNativePlugin<TalkModePluginLike>("TalkMode");
+}
+
+export function getCameraPlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("Camera");
+}
+
+export function getLocationPlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("Location");
+}
+
+export function getScreenCapturePlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("ScreenCapture");
+}
+
+export function getCanvasPlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("Canvas");
+}
+
+export function getDesktopPlugin(): GenericNativePlugin {
+  return getNativePlugin<GenericNativePlugin>("Desktop");
+}