@ateam-ai-mobile/plugin-runtime 0.1.20 → 0.1.21

package/dist/index.d.mts

@@ -1245,6 +1245,74 @@ interface VoiceAPI {
     transcript(): VoiceTranscriptEntry[];
 }
 
+type AiLiveCamera = 'front' | 'back';
+/** What the AI saw over time, as the live session's episode events. */
+interface AiLiveEpisode {
+    /** episode.started / .interrupted / .recovered / .ongoing / .ended */
+    type: string;
+    /** The task class id from the chosen setup, e.g. "drinking". */
+    class: string;
+    episodeId: string;
+    /** When (ms since the epoch, the session's clock). */
+    at: number;
+    [key: string]: unknown;
+}
+type AiLiveState = 
+/** Nothing running for this screen. */
+'idle'
+/** Asking for the live session, opening the camera and the connection. */
+ | 'starting'
+/** Pictures are going to the AI ("Sending to AI" is on screen). */
+ | 'sending'
+/** Stopped by stop() or by the screen closing. */
+ | 'stopped'
+/** It could not start or it broke; `lastError` says why in plain words. */
+ | 'error';
+interface AiLiveSnapshot {
+    state: AiLiveState;
+    camera: AiLiveCamera | null;
+    /** Pictures sent since start. */
+    sent: number;
+    /** Picture slots that sent nothing (no fresh camera frame, or the connection was busy). */
+    skipped: number;
+    lastEpisode: AiLiveEpisode | null;
+    lastError: string | null;
+}
+interface AiLiveStartOptions {
+    /** Which camera: 'back' (default) or 'front', the selfie camera. */
+    camera?: AiLiveCamera;
+}
+interface AiLiveAPI {
+    /** Start sending this phone's camera to the chosen AI. Refuses by name when either
+     *  side has not said yes, or when another screen is sending already. */
+    start(opts?: AiLiveStartOptions): Promise<AiLiveSnapshot>;
+    /** Stop sending. Closing the screen does the same. */
+    stop(): Promise<void>;
+    getSnapshot(): AiLiveSnapshot;
+    /** Called with every episode event the AI makes while this screen sends. */
+    onEpisode(cb: (ev: AiLiveEpisode) => void): Subscription;
+    /** Called whenever the snapshot changes. */
+    subscribe(cb: (snap: AiLiveSnapshot) => void): Subscription;
+}
+/** What the host supplies (setAiLiveProvider). `owner` is the screen's holder id,
+ *  captured by the gate — a plugin never supplies its own. */
+interface AiLiveProvider {
+    start(opts: AiLiveStartOptions, owner: string, ctx: {
+        visual: DeviceState['visual'];
+    }): Promise<AiLiveSnapshot>;
+    stop(owner: string): Promise<void>;
+    getSnapshot(owner: string): AiLiveSnapshot;
+    onEpisode(cb: (ev: AiLiveEpisode) => void, owner: string): Subscription;
+    subscribe(cb: (snap: AiLiveSnapshot) => void, owner: string): Subscription;
+    /** The screen is gone: drop its callbacks and its last snapshot (after stop). */
+    forget(owner: string): void;
+    /** Whether ANY screen is sending right now — what the platform's indicator shows. */
+    isSending(): boolean;
+    /** Any change of isSending(). */
+    subscribeSending(cb: () => void): Subscription;
+}
+declare const AI_LIVE_IDLE: AiLiveSnapshot;
+
 /**
  * @adas/plugin-sdk — Type definitions
  *
@@ -1821,6 +1889,11 @@ interface NativeCapabilities$1 {
     nfc?: boolean;
     /** Real biometric auth (Face ID / Touch ID / fingerprint). */
     biometricsV2?: boolean;
+    /** The developer's half of "camera pictures may go to an AI service off the
+     *  phone". Declaring it grants nothing by itself: the person on the phone must
+     *  also allow it in the host's settings. Both are checked together by
+     *  `mayPicturesLeavePhone()`. Not declared = pictures never leave. */
+    picturesToAi?: boolean;
 }
 interface PluginHooks$1 {
     onAppStart?: boolean;
@@ -1912,6 +1985,14 @@ interface PluginProps {
      * plugin gets a refusal (reads say so, actions throw) — never a fake `idle`.
      */
     voice: VoiceAPI;
+    /**
+     * The workspace's CHOSEN AI watching the phone camera, live — gated to plugins
+     * that declared `capabilities: { picturesToAi: true }` AND used only when the
+     * person on the phone allowed "Pictures to AI". The screen gets episodes
+     * ("drinking" started), never pictures; "Sending to AI" is platform-owned.
+     * REQUIRED like `voice`: an undeclared plugin gets a refusal, never a fake idle.
+     */
+    aiLive: AiLiveAPI;
     theme: ThemeTokens;
     context: PluginContext;
     /** Engine-supplied props from `sys.focusUiPlugin(pluginId, props)`.
@@ -2287,6 +2368,24 @@ declare function implementedVoice(): boolean;
  */
 declare function gateVoice(declared: boolean, pluginId: string): VoiceAPI;
 
+/** Host DI: the app supplies the live provider once at startup; null detaches it. */
+declare function setAiLiveProvider(provider: AiLiveProvider | null): void;
+/**
+ * The `aiLive` a plugin receives. `capabilities` is its TRUSTED declaration
+ * (pluginRuntime.ts); `holder` is the mounted screen it is bound to; `camera()` gives
+ * the visual session the SDK's sender reads, held as aiLiveCameraHolder(holder). It
+ * is built by resolvePluginRuntimeProps (the one place device gates are called) and
+ * never reaches the plugin: the plugin gets episodes, the sender gets the camera.
+ */
+declare function gateAiLive(capabilities: NativeCapabilities$1 | undefined, pluginId: string, holder: string, camera: () => DeviceState['visual']): AiLiveAPI;
+/** The screen closed: stop its live session and release its camera. */
+declare function releaseAiLive(holder: string): Promise<void>;
+/** For the platform's "Sending to AI" indicator: is any screen sending, and when that changes. */
+declare const aiLiveIndicator: {
+    isSending: () => boolean;
+    subscribe: (cb: () => void) => Subscription;
+};
+
 /**
  * Host integration points — the DI seams that keep this package free of
  * Expo imports. The host app (or @ateam-ai-mobile/expo) plugs in concrete providers
@@ -2324,6 +2423,12 @@ declare function setCameraPreviewProvider(Component: ((props: CameraPreviewImplP
 /** Null when the host installed none — the SDK then renders a stated reason,
  *  never a black rectangle that looks like a dark room. */
 declare function getCameraPreviewProvider(): ((props: CameraPreviewImplProps) => any) | null;
+/** Install the host's answer to "did the person on this phone allow camera
+ *  pictures to go to AI?". Pass null to remove it (consent then reads as no). */
+declare function setPicturesToAiConsentProvider(provider: (() => boolean) | null): void;
+/** The one check: true only when the plugin's trusted declaration includes
+ *  `picturesToAi` AND the person on the phone allowed it right now. */
+declare function mayPicturesLeavePhone(declared?: NativeCapabilities$1 | null): boolean;
 type PluginResolver = RegisteredPlugin | (() => Promise<RegisteredPlugin | null>);
 declare function registerNativePlugin(id: string, plugin: PluginResolver): void;
 declare function unregisterNativePlugin(id: string): void;
@@ -2352,6 +2457,8 @@ interface PluginRuntimeProps {
     native: PluginNative;
     deviceState: DeviceState;
     voice: VoiceAPI;
+    /** The workspace's chosen AI watching the phone camera, live (`picturesToAi`). */
+    aiLive: AiLiveAPI;
 }
 /** One declaration source — either the server manifest or the registry entry. */
 interface PluginDeclaration {
@@ -3347,4 +3454,4 @@ declare function isInlineSurface(m: SurfaceManifest): m is InlineSurfaceManifest
 
 declare function SurfaceDispatcher(): react_jsx_runtime.JSX.Element;
 
-export { ActionButton, type ActivitySnapshot, AdaOrb, type AdaOrbState, Badge, type BiometricsAPI, type BiometricsAPIv2, type BluetoothAPI, type BluetoothDevice, type Bridge, type BundleCacheEntry, type BundleStorage, type CameraAPI, CameraPreview, type CameraPreviewProps, type CapabilitySnapshot, type CapabilityStatus, Card, type Confidence, type ConnectivityAPI, type ConnectivitySnapshot, type CookiesAPI, type DeviceBridgeDomain, type DeviceContextSnapshot, type DeviceState, type DeviceToolContext, type DeviceToolDefinition, type DeviceToolFailureReason, type DeviceToolFn, type DeviceToolOutcome, type DeviceToolSummary, type DeviceToolsAction, type DeviceToolsSessionDeps, EmptyState, type EnvironmentSnapshot, ErrorState, type FileSystemAPI, type Freshness, type HapticsAPI, type InlineSurfaceManifest, type InteractionSnapshot, ListItem, type LoadedBundleInfo, type LoadedBundleSource, LoadingState, type LocationAPI, type MapsAPI, type MapsAddress, type MapsCoordinate, type MapsPlace, type MapsRoute, type MapsRouteStep, MemoryBundleStorage, type MountSurfaceManifest, type NativeCapabilities$1 as NativeCapabilities, type NfcAPI, type NotificationsAPI, PLATFORM_VOICE_CALLER, type PluginManifest as PlatformPluginManifest, type PluginBridge, type PluginContext, type PluginGrants, type PluginHooks$1 as PluginHooks, type PluginManifest$1 as PluginManifest, type PluginNative, type PluginProps, type PluginRuntimeProps, type PluginRuntimeSource, PluginSDK, type PluginSummary, type PluginSurfaceMeta, type RegisteredPlugin, type RegisteredServicePlugin, type RegisteredToolsPlugin, type RegisteredUIPlugin, type RemotePlugin, type RenderMode, type ScannerAPI, SearchBar, type SessionSnapshot, type SharingAPI, type SpatialEvent, type SpatialIntent, type SpatialPoint, type SpatialSnapshot, type SpatialStateEvent, type StateEvent, type StateListener, type StateQuality, type StorageAPI, type Subscription, SurfaceCard, type SurfaceCardProps, SurfaceDispatcher, SurfaceDrawer, type SurfaceDrawerProps, SurfaceFullscreen, type SurfaceFullscreenProps, type SurfaceManifest, type SurfacePlacement, type SurfaceType, type SurfaceVisibility, TabBar, type ThemeTokens, type TrustedToolDeclaration, type VisionAPI, type VisionFrameQuality, type VisionLabel, type VisionObject, type VisionText, type VisualSnapshot, type VisualStartOptions, type VoiceAPI, type VoiceCaller, type VoiceDeviceToolCall, type VoiceDeviceToolReason, type VoiceEvent, type VoicePersona, type VoiceSessionProvider, type VoiceSnapshot, type VoiceStartOptions, type VoiceState, type VoiceSubscription, type VoiceTranscriptEntry, type WebViewCookie, __resetSessionSyncForTest, bundleManager, callDeviceTool, clearDeviceTools, createNativeBridge, deviceStateHolders, deviceState as deviceStateV2, deviceToolsActionForSession, ensureDeviceToolsLoaded, explainCapability, gateDeviceState, gatePluginNative, gateVoice, getCameraPreviewProvider, getHaptics, getNativeBundleHash, getNativeBundleUrl, getNativeComponentId, getPlugin, getPluginNative, handleServerDeviceToolCall, hasDeviceTool, implementedStateDomains, implementedVoice, isActionable, isDeviceToolCommand, isInlineSurface, isMountSurface, listDeviceTools, listPlugins, nativePluginRegistry, neverObserved, pluginBundleUrl, register, registerBundleModules, registerDeviceTools, registerNativePlugin, registerPlatformDeviceTools, registerService, registerTools, releaseDeviceStateHolds, rendersNative, resetDeviceToolLoader, resolvePluginGrants, resolvePluginRuntimeProps, resolvePluginUrl, setBundleStorage, setCameraPreviewProvider, setDeviceStateProvider, setDeviceToolContextProvider, setDeviceToolResultPoster, setHapticsProvider, setPluginNativeProvider, setPluginThemeColors, setVoiceProvider, supportsNative, supportsWebView, syncDeviceToolsForSession, unregisterDeviceTools, unregisterNativePlugin, useApi, useNativeBridge, usePlugin, usePluginRuntimeProps, usePluginTheme, useRemotePlugins, useSurfaceStore };
+export { AI_LIVE_IDLE, ActionButton, type ActivitySnapshot, AdaOrb, type AdaOrbState, type AiLiveAPI, type AiLiveCamera, type AiLiveEpisode, type AiLiveProvider, type AiLiveSnapshot, type AiLiveStartOptions, type AiLiveState, Badge, type BiometricsAPI, type BiometricsAPIv2, type BluetoothAPI, type BluetoothDevice, type Bridge, type BundleCacheEntry, type BundleStorage, type CameraAPI, CameraPreview, type CameraPreviewProps, type CapabilitySnapshot, type CapabilityStatus, Card, type Confidence, type ConnectivityAPI, type ConnectivitySnapshot, type CookiesAPI, type DeviceBridgeDomain, type DeviceContextSnapshot, type DeviceState, type DeviceToolContext, type DeviceToolDefinition, type DeviceToolFailureReason, type DeviceToolFn, type DeviceToolOutcome, type DeviceToolSummary, type DeviceToolsAction, type DeviceToolsSessionDeps, EmptyState, type EnvironmentSnapshot, ErrorState, type FileSystemAPI, type Freshness, type HapticsAPI, type InlineSurfaceManifest, type InteractionSnapshot, ListItem, type LoadedBundleInfo, type LoadedBundleSource, LoadingState, type LocationAPI, type MapsAPI, type MapsAddress, type MapsCoordinate, type MapsPlace, type MapsRoute, type MapsRouteStep, MemoryBundleStorage, type MountSurfaceManifest, type NativeCapabilities$1 as NativeCapabilities, type NfcAPI, type NotificationsAPI, PLATFORM_VOICE_CALLER, type PluginManifest as PlatformPluginManifest, type PluginBridge, type PluginContext, type PluginGrants, type PluginHooks$1 as PluginHooks, type PluginManifest$1 as PluginManifest, type PluginNative, type PluginProps, type PluginRuntimeProps, type PluginRuntimeSource, PluginSDK, type PluginSummary, type PluginSurfaceMeta, type RegisteredPlugin, type RegisteredServicePlugin, type RegisteredToolsPlugin, type RegisteredUIPlugin, type RemotePlugin, type RenderMode, type ScannerAPI, SearchBar, type SessionSnapshot, type SharingAPI, type SpatialEvent, type SpatialIntent, type SpatialPoint, type SpatialSnapshot, type SpatialStateEvent, type StateEvent, type StateListener, type StateQuality, type StorageAPI, type Subscription, SurfaceCard, type SurfaceCardProps, SurfaceDispatcher, SurfaceDrawer, type SurfaceDrawerProps, SurfaceFullscreen, type SurfaceFullscreenProps, type SurfaceManifest, type SurfacePlacement, type SurfaceType, type SurfaceVisibility, TabBar, type ThemeTokens, type TrustedToolDeclaration, type VisionAPI, type VisionFrameQuality, type VisionLabel, type VisionObject, type VisionText, type VisualSnapshot, type VisualStartOptions, type VoiceAPI, type VoiceCaller, type VoiceDeviceToolCall, type VoiceDeviceToolReason, type VoiceEvent, type VoicePersona, type VoiceSessionProvider, type VoiceSnapshot, type VoiceStartOptions, type VoiceState, type VoiceSubscription, type VoiceTranscriptEntry, type WebViewCookie, __resetSessionSyncForTest, aiLiveIndicator, bundleManager, callDeviceTool, clearDeviceTools, createNativeBridge, deviceStateHolders, deviceState as deviceStateV2, deviceToolsActionForSession, ensureDeviceToolsLoaded, explainCapability, gateAiLive, gateDeviceState, gatePluginNative, gateVoice, getCameraPreviewProvider, getHaptics, getNativeBundleHash, getNativeBundleUrl, getNativeComponentId, getPlugin, getPluginNative, handleServerDeviceToolCall, hasDeviceTool, implementedStateDomains, implementedVoice, isActionable, isDeviceToolCommand, isInlineSurface, isMountSurface, listDeviceTools, listPlugins, mayPicturesLeavePhone, nativePluginRegistry, neverObserved, pluginBundleUrl, register, registerBundleModules, registerDeviceTools, registerNativePlugin, registerPlatformDeviceTools, registerService, registerTools, releaseAiLive, releaseDeviceStateHolds, rendersNative, resetDeviceToolLoader, resolvePluginGrants, resolvePluginRuntimeProps, resolvePluginUrl, setAiLiveProvider, setBundleStorage, setCameraPreviewProvider, setDeviceStateProvider, setDeviceToolContextProvider, setDeviceToolResultPoster, setHapticsProvider, setPicturesToAiConsentProvider, setPluginNativeProvider, setPluginThemeColors, setVoiceProvider, supportsNative, supportsWebView, syncDeviceToolsForSession, unregisterDeviceTools, unregisterNativePlugin, useApi, useNativeBridge, usePlugin, usePluginRuntimeProps, usePluginTheme, useRemotePlugins, useSurfaceStore };

package/dist/index.d.ts

@@ -1245,6 +1245,74 @@ interface VoiceAPI {
     transcript(): VoiceTranscriptEntry[];
 }
 
+type AiLiveCamera = 'front' | 'back';
+/** What the AI saw over time, as the live session's episode events. */
+interface AiLiveEpisode {
+    /** episode.started / .interrupted / .recovered / .ongoing / .ended */
+    type: string;
+    /** The task class id from the chosen setup, e.g. "drinking". */
+    class: string;
+    episodeId: string;
+    /** When (ms since the epoch, the session's clock). */
+    at: number;
+    [key: string]: unknown;
+}
+type AiLiveState = 
+/** Nothing running for this screen. */
+'idle'
+/** Asking for the live session, opening the camera and the connection. */
+ | 'starting'
+/** Pictures are going to the AI ("Sending to AI" is on screen). */
+ | 'sending'
+/** Stopped by stop() or by the screen closing. */
+ | 'stopped'
+/** It could not start or it broke; `lastError` says why in plain words. */
+ | 'error';
+interface AiLiveSnapshot {
+    state: AiLiveState;
+    camera: AiLiveCamera | null;
+    /** Pictures sent since start. */
+    sent: number;
+    /** Picture slots that sent nothing (no fresh camera frame, or the connection was busy). */
+    skipped: number;
+    lastEpisode: AiLiveEpisode | null;
+    lastError: string | null;
+}
+interface AiLiveStartOptions {
+    /** Which camera: 'back' (default) or 'front', the selfie camera. */
+    camera?: AiLiveCamera;
+}
+interface AiLiveAPI {
+    /** Start sending this phone's camera to the chosen AI. Refuses by name when either
+     *  side has not said yes, or when another screen is sending already. */
+    start(opts?: AiLiveStartOptions): Promise<AiLiveSnapshot>;
+    /** Stop sending. Closing the screen does the same. */
+    stop(): Promise<void>;
+    getSnapshot(): AiLiveSnapshot;
+    /** Called with every episode event the AI makes while this screen sends. */
+    onEpisode(cb: (ev: AiLiveEpisode) => void): Subscription;
+    /** Called whenever the snapshot changes. */
+    subscribe(cb: (snap: AiLiveSnapshot) => void): Subscription;
+}
+/** What the host supplies (setAiLiveProvider). `owner` is the screen's holder id,
+ *  captured by the gate — a plugin never supplies its own. */
+interface AiLiveProvider {
+    start(opts: AiLiveStartOptions, owner: string, ctx: {
+        visual: DeviceState['visual'];
+    }): Promise<AiLiveSnapshot>;
+    stop(owner: string): Promise<void>;
+    getSnapshot(owner: string): AiLiveSnapshot;
+    onEpisode(cb: (ev: AiLiveEpisode) => void, owner: string): Subscription;
+    subscribe(cb: (snap: AiLiveSnapshot) => void, owner: string): Subscription;
+    /** The screen is gone: drop its callbacks and its last snapshot (after stop). */
+    forget(owner: string): void;
+    /** Whether ANY screen is sending right now — what the platform's indicator shows. */
+    isSending(): boolean;
+    /** Any change of isSending(). */
+    subscribeSending(cb: () => void): Subscription;
+}
+declare const AI_LIVE_IDLE: AiLiveSnapshot;
+
 /**
  * @adas/plugin-sdk — Type definitions
  *
@@ -1821,6 +1889,11 @@ interface NativeCapabilities$1 {
     nfc?: boolean;
     /** Real biometric auth (Face ID / Touch ID / fingerprint). */
     biometricsV2?: boolean;
+    /** The developer's half of "camera pictures may go to an AI service off the
+     *  phone". Declaring it grants nothing by itself: the person on the phone must
+     *  also allow it in the host's settings. Both are checked together by
+     *  `mayPicturesLeavePhone()`. Not declared = pictures never leave. */
+    picturesToAi?: boolean;
 }
 interface PluginHooks$1 {
     onAppStart?: boolean;
@@ -1912,6 +1985,14 @@ interface PluginProps {
      * plugin gets a refusal (reads say so, actions throw) — never a fake `idle`.
      */
     voice: VoiceAPI;
+    /**
+     * The workspace's CHOSEN AI watching the phone camera, live — gated to plugins
+     * that declared `capabilities: { picturesToAi: true }` AND used only when the
+     * person on the phone allowed "Pictures to AI". The screen gets episodes
+     * ("drinking" started), never pictures; "Sending to AI" is platform-owned.
+     * REQUIRED like `voice`: an undeclared plugin gets a refusal, never a fake idle.
+     */
+    aiLive: AiLiveAPI;
     theme: ThemeTokens;
     context: PluginContext;
     /** Engine-supplied props from `sys.focusUiPlugin(pluginId, props)`.
@@ -2287,6 +2368,24 @@ declare function implementedVoice(): boolean;
  */
 declare function gateVoice(declared: boolean, pluginId: string): VoiceAPI;
 
+/** Host DI: the app supplies the live provider once at startup; null detaches it. */
+declare function setAiLiveProvider(provider: AiLiveProvider | null): void;
+/**
+ * The `aiLive` a plugin receives. `capabilities` is its TRUSTED declaration
+ * (pluginRuntime.ts); `holder` is the mounted screen it is bound to; `camera()` gives
+ * the visual session the SDK's sender reads, held as aiLiveCameraHolder(holder). It
+ * is built by resolvePluginRuntimeProps (the one place device gates are called) and
+ * never reaches the plugin: the plugin gets episodes, the sender gets the camera.
+ */
+declare function gateAiLive(capabilities: NativeCapabilities$1 | undefined, pluginId: string, holder: string, camera: () => DeviceState['visual']): AiLiveAPI;
+/** The screen closed: stop its live session and release its camera. */
+declare function releaseAiLive(holder: string): Promise<void>;
+/** For the platform's "Sending to AI" indicator: is any screen sending, and when that changes. */
+declare const aiLiveIndicator: {
+    isSending: () => boolean;
+    subscribe: (cb: () => void) => Subscription;
+};
+
 /**
  * Host integration points — the DI seams that keep this package free of
  * Expo imports. The host app (or @ateam-ai-mobile/expo) plugs in concrete providers
@@ -2324,6 +2423,12 @@ declare function setCameraPreviewProvider(Component: ((props: CameraPreviewImplP
 /** Null when the host installed none — the SDK then renders a stated reason,
  *  never a black rectangle that looks like a dark room. */
 declare function getCameraPreviewProvider(): ((props: CameraPreviewImplProps) => any) | null;
+/** Install the host's answer to "did the person on this phone allow camera
+ *  pictures to go to AI?". Pass null to remove it (consent then reads as no). */
+declare function setPicturesToAiConsentProvider(provider: (() => boolean) | null): void;
+/** The one check: true only when the plugin's trusted declaration includes
+ *  `picturesToAi` AND the person on the phone allowed it right now. */
+declare function mayPicturesLeavePhone(declared?: NativeCapabilities$1 | null): boolean;
 type PluginResolver = RegisteredPlugin | (() => Promise<RegisteredPlugin | null>);
 declare function registerNativePlugin(id: string, plugin: PluginResolver): void;
 declare function unregisterNativePlugin(id: string): void;
@@ -2352,6 +2457,8 @@ interface PluginRuntimeProps {
     native: PluginNative;
     deviceState: DeviceState;
     voice: VoiceAPI;
+    /** The workspace's chosen AI watching the phone camera, live (`picturesToAi`). */
+    aiLive: AiLiveAPI;
 }
 /** One declaration source — either the server manifest or the registry entry. */
 interface PluginDeclaration {
@@ -3347,4 +3454,4 @@ declare function isInlineSurface(m: SurfaceManifest): m is InlineSurfaceManifest
 
 declare function SurfaceDispatcher(): react_jsx_runtime.JSX.Element;
 
-export { ActionButton, type ActivitySnapshot, AdaOrb, type AdaOrbState, Badge, type BiometricsAPI, type BiometricsAPIv2, type BluetoothAPI, type BluetoothDevice, type Bridge, type BundleCacheEntry, type BundleStorage, type CameraAPI, CameraPreview, type CameraPreviewProps, type CapabilitySnapshot, type CapabilityStatus, Card, type Confidence, type ConnectivityAPI, type ConnectivitySnapshot, type CookiesAPI, type DeviceBridgeDomain, type DeviceContextSnapshot, type DeviceState, type DeviceToolContext, type DeviceToolDefinition, type DeviceToolFailureReason, type DeviceToolFn, type DeviceToolOutcome, type DeviceToolSummary, type DeviceToolsAction, type DeviceToolsSessionDeps, EmptyState, type EnvironmentSnapshot, ErrorState, type FileSystemAPI, type Freshness, type HapticsAPI, type InlineSurfaceManifest, type InteractionSnapshot, ListItem, type LoadedBundleInfo, type LoadedBundleSource, LoadingState, type LocationAPI, type MapsAPI, type MapsAddress, type MapsCoordinate, type MapsPlace, type MapsRoute, type MapsRouteStep, MemoryBundleStorage, type MountSurfaceManifest, type NativeCapabilities$1 as NativeCapabilities, type NfcAPI, type NotificationsAPI, PLATFORM_VOICE_CALLER, type PluginManifest as PlatformPluginManifest, type PluginBridge, type PluginContext, type PluginGrants, type PluginHooks$1 as PluginHooks, type PluginManifest$1 as PluginManifest, type PluginNative, type PluginProps, type PluginRuntimeProps, type PluginRuntimeSource, PluginSDK, type PluginSummary, type PluginSurfaceMeta, type RegisteredPlugin, type RegisteredServicePlugin, type RegisteredToolsPlugin, type RegisteredUIPlugin, type RemotePlugin, type RenderMode, type ScannerAPI, SearchBar, type SessionSnapshot, type SharingAPI, type SpatialEvent, type SpatialIntent, type SpatialPoint, type SpatialSnapshot, type SpatialStateEvent, type StateEvent, type StateListener, type StateQuality, type StorageAPI, type Subscription, SurfaceCard, type SurfaceCardProps, SurfaceDispatcher, SurfaceDrawer, type SurfaceDrawerProps, SurfaceFullscreen, type SurfaceFullscreenProps, type SurfaceManifest, type SurfacePlacement, type SurfaceType, type SurfaceVisibility, TabBar, type ThemeTokens, type TrustedToolDeclaration, type VisionAPI, type VisionFrameQuality, type VisionLabel, type VisionObject, type VisionText, type VisualSnapshot, type VisualStartOptions, type VoiceAPI, type VoiceCaller, type VoiceDeviceToolCall, type VoiceDeviceToolReason, type VoiceEvent, type VoicePersona, type VoiceSessionProvider, type VoiceSnapshot, type VoiceStartOptions, type VoiceState, type VoiceSubscription, type VoiceTranscriptEntry, type WebViewCookie, __resetSessionSyncForTest, bundleManager, callDeviceTool, clearDeviceTools, createNativeBridge, deviceStateHolders, deviceState as deviceStateV2, deviceToolsActionForSession, ensureDeviceToolsLoaded, explainCapability, gateDeviceState, gatePluginNative, gateVoice, getCameraPreviewProvider, getHaptics, getNativeBundleHash, getNativeBundleUrl, getNativeComponentId, getPlugin, getPluginNative, handleServerDeviceToolCall, hasDeviceTool, implementedStateDomains, implementedVoice, isActionable, isDeviceToolCommand, isInlineSurface, isMountSurface, listDeviceTools, listPlugins, nativePluginRegistry, neverObserved, pluginBundleUrl, register, registerBundleModules, registerDeviceTools, registerNativePlugin, registerPlatformDeviceTools, registerService, registerTools, releaseDeviceStateHolds, rendersNative, resetDeviceToolLoader, resolvePluginGrants, resolvePluginRuntimeProps, resolvePluginUrl, setBundleStorage, setCameraPreviewProvider, setDeviceStateProvider, setDeviceToolContextProvider, setDeviceToolResultPoster, setHapticsProvider, setPluginNativeProvider, setPluginThemeColors, setVoiceProvider, supportsNative, supportsWebView, syncDeviceToolsForSession, unregisterDeviceTools, unregisterNativePlugin, useApi, useNativeBridge, usePlugin, usePluginRuntimeProps, usePluginTheme, useRemotePlugins, useSurfaceStore };
+export { AI_LIVE_IDLE, ActionButton, type ActivitySnapshot, AdaOrb, type AdaOrbState, type AiLiveAPI, type AiLiveCamera, type AiLiveEpisode, type AiLiveProvider, type AiLiveSnapshot, type AiLiveStartOptions, type AiLiveState, Badge, type BiometricsAPI, type BiometricsAPIv2, type BluetoothAPI, type BluetoothDevice, type Bridge, type BundleCacheEntry, type BundleStorage, type CameraAPI, CameraPreview, type CameraPreviewProps, type CapabilitySnapshot, type CapabilityStatus, Card, type Confidence, type ConnectivityAPI, type ConnectivitySnapshot, type CookiesAPI, type DeviceBridgeDomain, type DeviceContextSnapshot, type DeviceState, type DeviceToolContext, type DeviceToolDefinition, type DeviceToolFailureReason, type DeviceToolFn, type DeviceToolOutcome, type DeviceToolSummary, type DeviceToolsAction, type DeviceToolsSessionDeps, EmptyState, type EnvironmentSnapshot, ErrorState, type FileSystemAPI, type Freshness, type HapticsAPI, type InlineSurfaceManifest, type InteractionSnapshot, ListItem, type LoadedBundleInfo, type LoadedBundleSource, LoadingState, type LocationAPI, type MapsAPI, type MapsAddress, type MapsCoordinate, type MapsPlace, type MapsRoute, type MapsRouteStep, MemoryBundleStorage, type MountSurfaceManifest, type NativeCapabilities$1 as NativeCapabilities, type NfcAPI, type NotificationsAPI, PLATFORM_VOICE_CALLER, type PluginManifest as PlatformPluginManifest, type PluginBridge, type PluginContext, type PluginGrants, type PluginHooks$1 as PluginHooks, type PluginManifest$1 as PluginManifest, type PluginNative, type PluginProps, type PluginRuntimeProps, type PluginRuntimeSource, PluginSDK, type PluginSummary, type PluginSurfaceMeta, type RegisteredPlugin, type RegisteredServicePlugin, type RegisteredToolsPlugin, type RegisteredUIPlugin, type RemotePlugin, type RenderMode, type ScannerAPI, SearchBar, type SessionSnapshot, type SharingAPI, type SpatialEvent, type SpatialIntent, type SpatialPoint, type SpatialSnapshot, type SpatialStateEvent, type StateEvent, type StateListener, type StateQuality, type StorageAPI, type Subscription, SurfaceCard, type SurfaceCardProps, SurfaceDispatcher, SurfaceDrawer, type SurfaceDrawerProps, SurfaceFullscreen, type SurfaceFullscreenProps, type SurfaceManifest, type SurfacePlacement, type SurfaceType, type SurfaceVisibility, TabBar, type ThemeTokens, type TrustedToolDeclaration, type VisionAPI, type VisionFrameQuality, type VisionLabel, type VisionObject, type VisionText, type VisualSnapshot, type VisualStartOptions, type VoiceAPI, type VoiceCaller, type VoiceDeviceToolCall, type VoiceDeviceToolReason, type VoiceEvent, type VoicePersona, type VoiceSessionProvider, type VoiceSnapshot, type VoiceStartOptions, type VoiceState, type VoiceSubscription, type VoiceTranscriptEntry, type WebViewCookie, __resetSessionSyncForTest, aiLiveIndicator, bundleManager, callDeviceTool, clearDeviceTools, createNativeBridge, deviceStateHolders, deviceState as deviceStateV2, deviceToolsActionForSession, ensureDeviceToolsLoaded, explainCapability, gateAiLive, gateDeviceState, gatePluginNative, gateVoice, getCameraPreviewProvider, getHaptics, getNativeBundleHash, getNativeBundleUrl, getNativeComponentId, getPlugin, getPluginNative, handleServerDeviceToolCall, hasDeviceTool, implementedStateDomains, implementedVoice, isActionable, isDeviceToolCommand, isInlineSurface, isMountSurface, listDeviceTools, listPlugins, mayPicturesLeavePhone, nativePluginRegistry, neverObserved, pluginBundleUrl, register, registerBundleModules, registerDeviceTools, registerNativePlugin, registerPlatformDeviceTools, registerService, registerTools, releaseAiLive, releaseDeviceStateHolds, rendersNative, resetDeviceToolLoader, resolvePluginGrants, resolvePluginRuntimeProps, resolvePluginUrl, setAiLiveProvider, setBundleStorage, setCameraPreviewProvider, setDeviceStateProvider, setDeviceToolContextProvider, setDeviceToolResultPoster, setHapticsProvider, setPicturesToAiConsentProvider, setPluginNativeProvider, setPluginThemeColors, setVoiceProvider, supportsNative, supportsWebView, syncDeviceToolsForSession, unregisterDeviceTools, unregisterNativePlugin, useApi, useNativeBridge, usePlugin, usePluginRuntimeProps, usePluginTheme, useRemotePlugins, useSurfaceStore };

package/dist/index.js

@@ -75,6 +75,22 @@ var init_voice = __esm({
   }
 });
 
+// src/sdk/aiLive.ts
+var AI_LIVE_IDLE;
+var init_aiLive = __esm({
+  "src/sdk/aiLive.ts"() {
+    "use strict";
+    AI_LIVE_IDLE = Object.freeze({
+      state: "idle",
+      camera: null,
+      sent: 0,
+      skipped: 0,
+      lastEpisode: null,
+      lastError: null
+    });
+  }
+});
+
 // src/sdk/device-state.ts
 var device_state_exports = {};
 __export(device_state_exports, {
@@ -477,6 +493,18 @@ function setCameraPreviewProvider(Component) {
 function getCameraPreviewProvider() {
   return _cameraPreview;
 }
+function setPicturesToAiConsentProvider(provider) {
+  _picturesToAiConsent = provider;
+}
+function mayPicturesLeavePhone(declared) {
+  if (declared?.picturesToAi !== true) return false;
+  if (!_picturesToAiConsent) return false;
+  try {
+    return _picturesToAiConsent() === true;
+  } catch {
+    return false;
+  }
+}
 function registerNativePlugin(id, plugin) {
   _nativeRegistry.set(id, plugin);
 }
@@ -498,7 +526,7 @@ function registerBundleModules(modules) {
 function resolveBundleModule(name) {
   return _bundleModules.get(name);
 }
-var _haptics, noopNative, _pluginNative, _cameraPreview, _nativeRegistry, nativePluginRegistry, _bundleModules;
+var _haptics, noopNative, _pluginNative, _cameraPreview, _picturesToAiConsent, _nativeRegistry, nativePluginRegistry, _bundleModules;
 var init_host = __esm({
   "src/host.ts"() {
     "use strict";
@@ -528,6 +556,7 @@ var init_host = __esm({
     };
     _pluginNative = noopNative;
     _cameraPreview = null;
+    _picturesToAiConsent = null;
     _nativeRegistry = /* @__PURE__ */ new Map();
     nativePluginRegistry = {
       has(id) {
@@ -705,6 +734,7 @@ var init_useApi = __esm({
 // src/sdk/index.ts
 var sdk_exports = {};
 __export(sdk_exports, {
+  AI_LIVE_IDLE: () => AI_LIVE_IDLE,
   ActionButton: () => ActionButton,
   ActivityIndicator: () => import_react_native.ActivityIndicator,
   Animated: () => import_react_native.Animated,
@@ -749,6 +779,7 @@ var init_sdk = __esm({
     init_registration();
     init_registration();
     init_voice();
+    init_aiLive();
     init_device_state();
     init_primitives();
     init_components();
@@ -1064,6 +1095,93 @@ var init_voiceHost = __esm({
   }
 });
 
+// src/aiLiveHost.ts
+function setAiLiveProvider(provider) {
+  _provider3 = provider || null;
+}
+function denial2(why) {
+  return Object.assign(new Error(`aiLive: ${why}`), { code: "RESTRICTED" });
+}
+function deniedAiLive(why) {
+  const fail = () => {
+    throw denial2(why);
+  };
+  return {
+    async start() {
+      return fail();
+    },
+    async stop() {
+    },
+    getSnapshot() {
+      return { state: "error", camera: null, sent: 0, skipped: 0, lastEpisode: null, lastError: why };
+    },
+    onEpisode() {
+      return fail();
+    },
+    subscribe() {
+      return fail();
+    }
+  };
+}
+function gateAiLive(capabilities, pluginId, holder, camera) {
+  if (capabilities?.picturesToAi !== true) {
+    return deniedAiLive("not declared in this plugin's capabilities. Add `capabilities: { picturesToAi: true }` to send camera pictures to the workspace's chosen AI.");
+  }
+  if (!pluginId || !holder) return deniedAiLive("no plugin identity \u2014 it fails closed without a screen to bind the session to.");
+  if (!_provider3) return deniedAiLive("no live AI in this build \u2014 the app has not called setAiLiveProvider.");
+  const provider = _provider3;
+  return {
+    async start(opts) {
+      if (!mayPicturesLeavePhone(capabilities)) {
+        throw denial2(`the person on this phone has not allowed camera pictures to go to AI. They can turn on "Pictures to AI" in the app's settings (What Ada can access).`);
+      }
+      try {
+        return await provider.start(opts || {}, holder, { visual: camera() });
+      } catch (e) {
+        await releaseDeviceStateHolds(aiLiveCameraHolder(holder));
+        throw e;
+      }
+    },
+    async stop() {
+      try {
+        await provider.stop(holder);
+      } finally {
+        await releaseDeviceStateHolds(aiLiveCameraHolder(holder));
+      }
+    },
+    getSnapshot: () => provider.getSnapshot(holder),
+    onEpisode: (cb) => provider.onEpisode(cb, holder),
+    subscribe: (cb) => provider.subscribe(cb, holder)
+  };
+}
+async function releaseAiLive(holder) {
+  try {
+    await _provider3?.stop(holder);
+  } catch {
+  }
+  try {
+    _provider3?.forget(holder);
+  } catch {
+  }
+  await releaseDeviceStateHolds(aiLiveCameraHolder(holder));
+}
+var _provider3, aiLiveCameraHolder, NOTHING, aiLiveIndicator;
+var init_aiLiveHost = __esm({
+  "src/aiLiveHost.ts"() {
+    "use strict";
+    init_host();
+    init_deviceStateHost();
+    _provider3 = null;
+    aiLiveCameraHolder = (holder) => `${holder}#aiLive`;
+    NOTHING = { unsubscribe() {
+    } };
+    aiLiveIndicator = {
+      isSending: () => !!_provider3?.isSending(),
+      subscribe: (cb) => _provider3?.subscribeSending(cb) ?? NOTHING
+    };
+  }
+});
+
 // src/pluginRuntime.ts
 function mergeCaps(decl) {
   if (!decl) return void 0;
@@ -1079,7 +1197,8 @@ function resolvePluginRuntimeProps(src, holder = src.pluginId) {
   const native = gatePluginNative(getPluginNative(), capabilities, src.pluginId);
   const deviceState = gateDeviceState(stateDomains, holder);
   const voice = gateVoice(!!capabilities?.voice, src.pluginId);
-  return { device: native, native, deviceState, voice };
+  const aiLive = gateAiLive(capabilities, src.pluginId, holder, () => gateDeviceState(["visual"], aiLiveCameraHolder(holder)).visual);
+  return { device: native, native, deviceState, voice, aiLive };
 }
 var init_pluginRuntime = __esm({
   "src/pluginRuntime.ts"() {
@@ -1087,6 +1206,7 @@ var init_pluginRuntime = __esm({
     init_host();
     init_deviceStateHost();
     init_voiceHost();
+    init_aiLiveHost();
   }
 });
 
@@ -3326,6 +3446,7 @@ var init_loader = __esm({
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  AI_LIVE_IDLE: () => AI_LIVE_IDLE,
   ActionButton: () => ActionButton,
   ActivityIndicator: () => import_react_native.ActivityIndicator,
   AdaOrb: () => AdaOrb,
@@ -3361,6 +3482,7 @@ __export(index_exports, {
   TouchableOpacity: () => import_react_native.TouchableOpacity,
   View: () => import_react_native.View,
   __resetSessionSyncForTest: () => __resetSessionSyncForTest,
+  aiLiveIndicator: () => aiLiveIndicator,
   bundleManager: () => bundleManager,
   callDeviceTool: () => callDeviceTool,
   clearDeviceTools: () => clearDeviceTools,
@@ -3370,6 +3492,7 @@ __export(index_exports, {
   deviceToolsActionForSession: () => deviceToolsActionForSession,
   ensureDeviceToolsLoaded: () => ensureDeviceToolsLoaded,
   explainCapability: () => explainCapability,
+  gateAiLive: () => gateAiLive,
   gateDeviceState: () => gateDeviceState,
   gatePluginNative: () => gatePluginNative,
   gateVoice: () => gateVoice,
@@ -3390,6 +3513,7 @@ __export(index_exports, {
   isMountSurface: () => isMountSurface,
   listDeviceTools: () => listDeviceTools,
   listPlugins: () => listPlugins,
+  mayPicturesLeavePhone: () => mayPicturesLeavePhone,
   nativePluginRegistry: () => nativePluginRegistry,
   neverObserved: () => neverObserved,
   pluginBundleUrl: () => pluginBundleUrl,
@@ -3400,18 +3524,21 @@ __export(index_exports, {
   registerPlatformDeviceTools: () => registerPlatformDeviceTools,
   registerService: () => registerService,
   registerTools: () => registerTools,
+  releaseAiLive: () => releaseAiLive,
   releaseDeviceStateHolds: () => releaseDeviceStateHolds,
   rendersNative: () => rendersNative,
   resetDeviceToolLoader: () => resetDeviceToolLoader,
   resolvePluginGrants: () => resolvePluginGrants,
   resolvePluginRuntimeProps: () => resolvePluginRuntimeProps,
   resolvePluginUrl: () => resolvePluginUrl,
+  setAiLiveProvider: () => setAiLiveProvider,
   setBundleStorage: () => setBundleStorage,
   setCameraPreviewProvider: () => setCameraPreviewProvider,
   setDeviceStateProvider: () => setDeviceStateProvider,
   setDeviceToolContextProvider: () => setDeviceToolContextProvider,
   setDeviceToolResultPoster: () => setDeviceToolResultPoster,
   setHapticsProvider: () => setHapticsProvider,
+  setPicturesToAiConsentProvider: () => setPicturesToAiConsentProvider,
   setPluginNativeProvider: () => setPluginNativeProvider,
   setPluginThemeColors: () => setPluginThemeColors,
   setVoiceProvider: () => setVoiceProvider,
@@ -3434,6 +3561,7 @@ init_device_state();
 init_device_state();
 init_deviceStateHost();
 init_voiceHost();
+init_aiLiveHost();
 init_host();
 init_pluginRuntime();
 init_registry();
@@ -3661,6 +3789,7 @@ function usePlugin(pluginId, manifest) {
 var import_react12 = require("react");
 init_pluginRuntime();
 init_deviceStateHost();
+init_aiLiveHost();
 var mounts = 0;
 function usePluginRuntimeProps(src, deps) {
   const holderRef = (0, import_react12.useRef)(null);
@@ -3673,6 +3802,9 @@ function usePluginRuntimeProps(src, deps) {
   (0, import_react12.useEffect)(() => () => {
     void releaseDeviceStateHolds(holder);
   }, [holder]);
+  (0, import_react12.useEffect)(() => () => {
+    void releaseAiLive(holder);
+  }, [holder]);
   const grantedKey = granted.join(",");
   (0, import_react12.useEffect)(() => {
     void releaseDeviceStateHolds(holder, granted);
@@ -3761,6 +3893,7 @@ init_store();
 init_SurfaceDispatcher();
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  AI_LIVE_IDLE,
   ActionButton,
   ActivityIndicator,
   AdaOrb,
@@ -3796,6 +3929,7 @@ init_SurfaceDispatcher();
   TouchableOpacity,
   View,
   __resetSessionSyncForTest,
+  aiLiveIndicator,
   bundleManager,
   callDeviceTool,
   clearDeviceTools,
@@ -3805,6 +3939,7 @@ init_SurfaceDispatcher();
   deviceToolsActionForSession,
   ensureDeviceToolsLoaded,
   explainCapability,
+  gateAiLive,
   gateDeviceState,
   gatePluginNative,
   gateVoice,
@@ -3825,6 +3960,7 @@ init_SurfaceDispatcher();
   isMountSurface,
   listDeviceTools,
   listPlugins,
+  mayPicturesLeavePhone,
   nativePluginRegistry,
   neverObserved,
   pluginBundleUrl,
@@ -3835,18 +3971,21 @@ init_SurfaceDispatcher();
   registerPlatformDeviceTools,
   registerService,
   registerTools,
+  releaseAiLive,
   releaseDeviceStateHolds,
   rendersNative,
   resetDeviceToolLoader,
   resolvePluginGrants,
   resolvePluginRuntimeProps,
   resolvePluginUrl,
+  setAiLiveProvider,
   setBundleStorage,
   setCameraPreviewProvider,
   setDeviceStateProvider,
   setDeviceToolContextProvider,
   setDeviceToolResultPoster,
   setHapticsProvider,
+  setPicturesToAiConsentProvider,
   setPluginNativeProvider,
   setPluginThemeColors,
   setVoiceProvider,