@ateam-ai-mobile/plugin-runtime 0.1.20 → 0.1.22

package/src/pluginRuntime.ts

@@ -16,9 +16,11 @@
 import { getPluginNative, gatePluginNative } from './host';
 import { gateDeviceState } from './deviceStateHost';
 import { gateVoice } from './voiceHost';
+import { gateAiLive, aiLiveCameraHolder } from './aiLiveHost';
 import type { PluginNative, NativeCapabilities } from './sdk/types';
 import type { DeviceState, DeviceBridgeDomain } from './sdk/device-state';
 import type { VoiceAPI } from './sdk/voice';
+import type { AiLiveAPI } from './sdk/aiLive';
 
 /** The gated surface a plugin Component receives. `device` and `native` are the
  *  SAME object (native is the deprecated alias). */
@@ -27,6 +29,8 @@ export interface PluginRuntimeProps {
   native: PluginNative;
   deviceState: DeviceState;
   voice: VoiceAPI;
+  /** The workspace's chosen AI watching the phone camera, live (`picturesToAi`). */
+  aiLive: AiLiveAPI;
 }
 
 /** One declaration source — either the server manifest or the registry entry. */
@@ -105,6 +109,11 @@ export function resolvePluginRuntimeProps(src: PluginRuntimeSource, holder: stri
   // session is isolated per solution: declaring voice runs YOUR conversation, it
   // does not attach to another plugin's live one.
   const voice = gateVoice(!!(capabilities as any)?.voice, src.pluginId);
+  // Pictures to the chosen AI: gated by the TRUSTED `picturesToAi` declaration,
+  // bound to this surface's holder so closing the screen stops it. The camera its
+  // sender reads is the SDK's own hold (not a domain the plugin declared, and never
+  // handed to the plugin), built here because this is where device gates are called.
+  const aiLive = gateAiLive(capabilities, src.pluginId, holder, () => gateDeviceState(['visual'], aiLiveCameraHolder(holder)).visual);
   // device and native are intentionally the SAME gated object (R4 rename).
-  return { device: native, native, deviceState, voice };
+  return { device: native, native, deviceState, voice, aiLive };
 }

package/src/sdk/aiLive.ts

@@ -0,0 +1,111 @@
+/**
+ * props.aiLive — the workspace's CHOSEN AI watching this phone's camera, live.
+ *
+ * The workspace admin picks and tests a vision setup in AI Models ("Drinking", a
+ * hosted model, the classes it must notice). A screen that wants that AI to watch
+ * the phone camera calls start(); the SDK then sends pictures to the workspace's
+ * live session at the rate and size the setup was TESTED at, and hands the screen
+ * what the AI saw through onEpisode ("drinking" started / ended). The screen never
+ * touches a picture: it gets events, not pixels.
+ *
+ * Two people must say yes before a picture leaves the phone (owner decision
+ * 2026-10-25): the solution declares `native: { picturesToAi: true }`, AND the
+ * person on the phone allowed "Camera pictures to AI" in the app's settings. Both
+ * are asked at start() AND for as long as it sends: taking the yes back (the
+ * switch, the camera's own switch, the OS permission) stops it, and no picture is
+ * sent after that (GPT review 2026-10-30). Owner decision 2026-10-29: only while
+ * the screen that asked is open, and with "Sending to AI" on screen — the
+ * indicator is the PLATFORM's (AiLiveIndicator in @ateam-ai-mobile/ui), a plugin
+ * cannot hide it, and closing the screen stops it.
+ *
+ * Each phone session is the phone's OWN live session, capped at US$0.50 by Core,
+ * and it is one connection: start() opening it starts the session, and its end
+ * (stop(), the screen closing, the app gone, the network lost) ends it. Live
+ * sessions are per person (workspace + actor): other people's never get in the
+ * way, and when the person already has one running (started on the AI Models
+ * page or on another device), start() refuses rather than join it.
+ */
+import type { Subscription } from './device-state';
+
+export type AiLiveCamera = 'front' | 'back';
+
+/** What the AI saw over time, as the live session's episode events. */
+export interface AiLiveEpisode {
+  /** episode.started / .interrupted / .recovered / .ongoing / .ended */
+  type: string;
+  /** The task class id from the chosen setup, e.g. "drinking". */
+  class: string;
+  episodeId: string;
+  /** Whose camera saw it: the signed-in person whose live session this is (live sessions are per
+   *  person, workspace + actor). Always there: an event without it never reaches the screen. */
+  actorId: string;
+  /** When (ms since the epoch, the session's clock). */
+  at: number;
+  [key: string]: unknown;
+}
+
+export type AiLiveState =
+  /** Nothing running for this screen. */
+  | 'idle'
+  /** Asking for the live session, opening the camera and the connection. */
+  | 'starting'
+  /** Pictures are going to the AI ("Sending to AI" is on screen). */
+  | 'sending'
+  /** Stopped by stop(), by the screen closing, or by the person taking the yes back (lastError says so). */
+  | 'stopped'
+  /** It could not start or it broke; `lastError` says why in plain words. */
+  | 'error';
+
+export interface AiLiveSnapshot {
+  state: AiLiveState;
+  camera: AiLiveCamera | null;
+  /** Pictures sent since start. */
+  sent: number;
+  /** Picture slots that sent nothing (no fresh camera frame, or the connection was busy). */
+  skipped: number;
+  lastEpisode: AiLiveEpisode | null;
+  lastError: string | null;
+}
+
+export interface AiLiveStartOptions {
+  /** Which camera: 'back' (default) or 'front', the selfie camera. */
+  camera?: AiLiveCamera;
+}
+
+export interface AiLiveAPI {
+  /** Start sending this phone's camera to the chosen AI, in a live session of the
+   *  phone's own (capped at US$0.50). Refuses by name when either side has not said
+   *  yes, when another screen is sending already, or when the person already has a
+   *  live session running elsewhere. */
+  start(opts?: AiLiveStartOptions): Promise<AiLiveSnapshot>;
+  /** Stop sending. Closing the screen does the same. */
+  stop(): Promise<void>;
+  getSnapshot(): AiLiveSnapshot;
+  /** Called with every episode event the AI makes while this screen sends. */
+  onEpisode(cb: (ev: AiLiveEpisode) => void): Subscription;
+  /** Called whenever the snapshot changes. */
+  subscribe(cb: (snap: AiLiveSnapshot) => void): Subscription;
+}
+
+/** What the host supplies (setAiLiveProvider). `owner` is the screen's holder id,
+ *  captured by the gate — a plugin never supplies its own. */
+export interface AiLiveProvider {
+  /** `ctx.allowed` is the two-sided consent, asked again right before every picture:
+   *  once it answers no, the provider sends nothing more and stops. */
+  start(opts: AiLiveStartOptions, owner: string, ctx: { visual: import('./device-state').DeviceState['visual']; allowed: () => boolean }): Promise<AiLiveSnapshot>;
+  /** `why` (optional) is what the screen then reads as lastError, e.g. the yes taken back. */
+  stop(owner: string, why?: string): Promise<void>;
+  getSnapshot(owner: string): AiLiveSnapshot;
+  onEpisode(cb: (ev: AiLiveEpisode) => void, owner: string): Subscription;
+  subscribe(cb: (snap: AiLiveSnapshot) => void, owner: string): Subscription;
+  /** The screen is gone: drop its callbacks and its last snapshot (after stop). */
+  forget(owner: string): void;
+  /** Whether ANY screen is sending right now — what the platform's indicator shows. */
+  isSending(): boolean;
+  /** Any change of isSending(). */
+  subscribeSending(cb: () => void): Subscription;
+}
+
+export const AI_LIVE_IDLE: AiLiveSnapshot = Object.freeze({
+  state: 'idle', camera: null, sent: 0, skipped: 0, lastEpisode: null, lastError: null,
+}) as AiLiveSnapshot;

package/src/sdk/index.ts

@@ -76,6 +76,18 @@ export type {
 // provider and its own surfaces call it as this caller. See VoiceSessionProvider.
 export { PLATFORM_VOICE_CALLER } from './voice';
 
+// AI live — the workspace's chosen AI watching the phone camera (props.aiLive).
+export type {
+  AiLiveAPI,
+  AiLiveCamera,
+  AiLiveEpisode,
+  AiLiveProvider,
+  AiLiveSnapshot,
+  AiLiveStartOptions,
+  AiLiveState,
+} from './aiLive';
+export { AI_LIVE_IDLE } from './aiLive';
+
 // Device state — the SEMANTIC layer: the types of what `props.deviceState`
 // returns, and the two pure helpers for reading a snapshot. They are here, in
 // the one module a bundle can require, because a bundle has no other SDK name

package/src/sdk/types.ts

@@ -6,6 +6,7 @@
  */
 import type { DeviceState, DeviceBridgeDomain } from './device-state';
 import type { VoiceAPI } from './voice';
+import type { AiLiveAPI } from './aiLive';
 
 // ── Plugin Context ──────────────────────────────────────────
 
@@ -258,11 +259,16 @@ export interface BluetoothDevice {
 //   deviceState.visual.getSnapshot()  → "is there a current frame, how old"
 //   native.vision.readText(frame)     → "what does the sign say"
 //
-// FRAMES NEVER LEAVE THE PHONE. That is Arie's decision (2026-08-15) and it is
-// the whole reason this is a platform capability rather than an API call from
-// a plugin: analysis runs on-device, so there is no per-frame cost and no image
-// on anyone's server. An implementation that POSTs the frame somewhere would
-// satisfy the types and violate the design.
+// FRAMES STAY ON THE PHONE UNLESS TWO PEOPLE SAY YES. Arie's decision
+// (2026-08-15: frames never leave the phone) is the default and the reason this
+// is a platform capability rather than an API call from a plugin: analysis runs
+// on-device, so there is no per-frame cost and no image on anyone's server.
+// Amended 2026-10-25: a picture MAY leave only when the solution's developer
+// declares `picturesToAi` AND the person on the phone allowed it in the host's
+// settings. `mayPicturesLeavePhone()` (host.ts) is the one check; any path that
+// sends a picture off the phone must call it and refuse when it says no. No
+// such path exists yet. An implementation that POSTs a frame without that check
+// would satisfy the types and violate the design.
 //
 // IT TAKES A HANDLE, NOT PIXELS. `MediaHandle.id` from a VisualFrameRef — the
 // core rule that media crosses by reference. A plugin never holds bytes.
@@ -595,6 +601,11 @@ export interface NativeCapabilities {
   nfc?: boolean;
   /** Real biometric auth (Face ID / Touch ID / fingerprint). */
   biometricsV2?: boolean;
+  /** The developer's half of "camera pictures may go to an AI service off the
+   *  phone". Declaring it grants nothing by itself: the person on the phone must
+   *  also allow it in the host's settings. Both are checked together by
+   *  `mayPicturesLeavePhone()`. Not declared = pictures never leave. */
+  picturesToAi?: boolean;
 }
 
 // ── Lifecycle hooks ─────────────────────────────────────────
@@ -677,6 +688,14 @@ export interface PluginProps {
    * plugin gets a refusal (reads say so, actions throw) — never a fake `idle`.
    */
   voice: VoiceAPI;
+  /**
+   * The workspace's CHOSEN AI watching the phone camera, live — gated to plugins
+   * that declared `capabilities: { picturesToAi: true }` AND used only when the
+   * person on the phone allowed "Pictures to AI". The screen gets episodes
+   * ("drinking" started), never pictures; "Sending to AI" is platform-owned.
+   * REQUIRED like `voice`: an undeclared plugin gets a refusal, never a fake idle.
+   */
+  aiLive: AiLiveAPI;
   theme: ThemeTokens;
   context: PluginContext;
   /** Engine-supplied props from `sys.focusUiPlugin(pluginId, props)`.