@ateam-ai-mobile/plugin-runtime 0.1.5 → 0.1.7

package/src/sdk/index.ts

@@ -5,9 +5,9 @@
  */
 
 // Registration API
-export { register, registerService } from './registration';
-import { register, registerService } from './registration';
-export const PluginSDK = { register, registerService } as const;
+export { register, registerService, registerTools } from './registration';
+import { register, registerService, registerTools } from './registration';
+export const PluginSDK = { register, registerService, registerTools } as const;
 
 // Types
 export type {
@@ -24,9 +24,17 @@ export type {
   ConnectivityAPI,
   FileSystemAPI,
   CookiesAPI,
+  WebViewCookie,
   SharingAPI,
   ScannerAPI,
   NfcAPI,
+  BluetoothAPI,
+  BluetoothDevice,
+  VisionAPI,
+  VisionText,
+  VisionLabel,
+  VisionObject,
+  VisionFrameQuality,
   BiometricsAPI,
   BiometricsAPIv2,
   NativeCapabilities,
@@ -36,8 +44,28 @@ export type {
   RegisteredPlugin,
   RegisteredUIPlugin,
   RegisteredServicePlugin,
+  RegisteredToolsPlugin,
+  DeviceToolContext,
+  DeviceToolDefinition,
+  DeviceToolFn,
 } from './types';
 
+// Voice — the live conversational SESSION surface (props.voice).
+export type {
+  VoiceAPI,
+  VoiceState,
+  VoiceSnapshot,
+  VoiceTranscriptEntry,
+  VoiceStartOptions,
+  VoiceEvent,
+  VoiceSubscription,
+  VoiceCaller,
+  VoiceSessionProvider,
+} from './voice';
+// The privileged platform caller (a value) — the host injects the owner-aware
+// provider and its own surfaces call it as this caller. See VoiceSessionProvider.
+export { PLATFORM_VOICE_CALLER } from './voice';
+
 // RN primitives (plugins import from the SDK, not from react-native)
 export {
   View,
@@ -72,5 +100,9 @@ export {
 } from './components';
 
 // Theme + API hooks
+// Showing the camera is a COMPONENT, not a native module handed to bundles —
+// one door, one lock. See CameraPreview.tsx for why.
+export { CameraPreview } from './CameraPreview';
+export type { CameraPreviewProps } from './CameraPreview';
 export { usePluginTheme, setPluginThemeColors } from './usePluginTheme';
 export { useApi } from './useApi';

package/src/sdk/registration.ts

@@ -3,29 +3,68 @@
  * Solution developers use these to register their plugins.
  */
 import type {
+  DeviceToolDefinition,
   NativeCapabilities,
   PluginHooks,
   PluginProps,
   RegisteredUIPlugin,
   RegisteredServicePlugin,
+  RegisteredToolsPlugin,
 } from './types';
+import type { DeviceBridgeDomain } from './device-state';
 
 /**
  * Register a UI or hybrid plugin (has a visual component).
+ *
+ * `tools` is optional: a plugin may ship UI and local tools together. Tools are
+ * picked up when the bundle loads, independently of whether the UI is mounted.
  */
 export function register(
   id: string,
   config: {
     type?: 'ui' | 'hybrid';
     capabilities?: NativeCapabilities;
+    /** L3 DeviceState domains this plugin's UI reads via props.deviceState (e.g.
+     *  ['spatial','visual']). Carried through so a HOST-registered plugin (no
+     *  server manifest) can declare its domains — resolvePluginRuntimeProps reads
+     *  this as the trusted source when there is no manifest. Dropping it here left
+     *  every registered UI plugin with no L3, silently. Mirrors registerTools. */
+    stateDomains?: DeviceBridgeDomain[];
     Component: React.FC<PluginProps>;
+    tools?: DeviceToolDefinition[];
   },
 ): RegisteredUIPlugin {
   return {
     id,
     type: config.type || 'ui',
     capabilities: config.capabilities,
+    stateDomains: config.stateDomains,
     Component: config.Component,
+    tools: config.tools,
+  };
+}
+
+/**
+ * Register a tools-only plugin — a `tools/` section with no UI at all.
+ *
+ * Nothing here mounts. The tools become callable as soon as the bundle is
+ * evaluated, and stay callable with nothing of the plugin on screen.
+ */
+export function registerTools(
+  id: string,
+  config: {
+    capabilities?: NativeCapabilities;
+    /** L3 DeviceState domains the tools may read via ctx.deviceState (e.g. ['visual']). */
+    stateDomains?: DeviceBridgeDomain[];
+    tools: DeviceToolDefinition[];
+  },
+): RegisteredToolsPlugin {
+  return {
+    id,
+    type: 'tools',
+    capabilities: config.capabilities,
+    stateDomains: config.stateDomains,
+    tools: config.tools,
   };
 }
 

package/src/sdk/types.ts

@@ -4,6 +4,8 @@
  * This is the developer contract for native mobile plugins.
  * Both UI and service plugins use these interfaces.
  */
+import type { DeviceState, DeviceBridgeDomain } from './device-state';
+import type { VoiceAPI } from './voice';
 
 // ── Plugin Context ──────────────────────────────────────────
 
@@ -55,8 +57,27 @@ export interface PluginBridge {
 
 export interface CameraAPI {
   takePhoto(opts?: { quality?: number; facing?: 'front' | 'back' }): Promise<{ uri: string; width: number; height: number }>;
+  /**
+   * @deprecated Use `native.scanner.scanOnce()`. Scanning needs a MOUNTED
+   * camera, which a bare function call cannot provide; this refuses by name
+   * and points there.
+   */
   scanBarcode(opts?: { formats?: string[] }): Promise<{ data: string; format: string }>;
   pickImage(opts?: { multiple?: boolean }): Promise<{ uri: string }[]>;
+  /**
+   * Record a video clip the user deliberately makes — a damage claim, a
+   * how-it-sounds clip, something to attach.
+   *
+   * NOT the Visual domain. That is an ephemeral frame ring a reasoning loop
+   * reads and nobody keeps; this produces a FILE. Different jobs, and the
+   * grammar says so: this is a verb that throws, Visual is a state read that
+   * says `unknown`.
+   *
+   * Rides the same mounted camera surface as `scanner.scanOnce`, and throws
+   * the same coded failures. `withAudio` needs microphone permission and
+   * REFUSES without it rather than quietly recording silence.
+   */
+  recordVideo(opts?: { maxDurationMs?: number; withAudio?: boolean }): Promise<{ uri: string; durationMs: number | null }>;
 }
 
 export interface LocationAPI {
@@ -88,6 +109,16 @@ export interface HapticsAPI {
   impact(style: 'light' | 'medium' | 'heavy'): void;
   notification(type: 'success' | 'warning' | 'error'): void;
   selection(): void;
+  /**
+   * Shorthand for notification('error').
+   *
+   * Documented in the platform guide and already CALLED by shipped plugins
+   * (connections-panel), which only compiled because they reached it through
+   * optional chaining — `native?.haptics?.error?.()` silently does nothing when
+   * the method is absent, so a failure the user was meant to FEEL produced no
+   * buzz and no error. Declared here so the call site is checked.
+   */
+  error(): void;
 }
 
 export interface ConnectivityAPI {
@@ -139,11 +170,22 @@ export interface ScannerAPI {
   /** Prompt the user for camera permission. */
   requestPermissions(): Promise<{ granted: boolean; status: string }>;
   /**
-   * Scanner is camera-based; the plugin mounts the scanner component
-   * directly from device-bridge. This API exposes only the permission
-   * wrapper at the plugin-sdk level — the actual scan is a React
-   * component lifecycle.
+   * Show the scanner and resolve with the FIRST code read.
+   *
+   * The plugin no longer mounts anything: the host renders one
+   * `CameraTaskHost`, this raises it, and the promise settles when a code is
+   * found. That is why the capture lifecycle is owned in exactly one place
+   * instead of by every plugin that wants to scan.
+   *
+   * THROWS rather than resolving empty, with a code that says which:
+   *   TIMEOUT           nothing was found in time
+   *   CANCELLED         the user dismissed it — a choice, not a fault
+   *   PERMISSION_DENIED camera permission refused
+   *   BUSY              another camera task is running
+   *   UNAVAILABLE       no camera surface is mounted in this app
+   * An empty string would read as a code that scanned to nothing.
    */
+  scanOnce(opts?: { formats?: string[]; timeoutMs?: number }): Promise<{ data: string; format: string }>;
 }
 
 // ── NFC API (tag read) ────────────────────────────────────
@@ -171,10 +213,171 @@ export interface BiometricsAPIv2 {
 }
 
 
+// ── Bluetooth / BLE ───────────────────────────────────────
+//
+// SHAPE ONLY — every method throws CapabilityUnavailableError in this build.
+// It is declared anyway, and that is the point: `bluetooth` was already a
+// manifest flag with nothing behind it, so a plugin could declare it, pass the
+// gate, and reach `undefined`. Calling `native.bluetooth.scan()` then failed
+// with "cannot read properties of undefined" — a crash that tells the author
+// nothing and a model less than nothing.
+//
+// A declared-but-unbuilt capability must FAIL BY NAME. That is the whole rule:
+// absence is an answer, but only when it says what is absent.
+//
+// WHY IT IS KEPT RATHER THAN DROPPED: BLE beacons are one of the few indoor
+// position signals that work where GPS does not, which is exactly the case Ada
+// Guide runs into the moment the user walks into a building. Also pairing,
+// loyalty tags, and proximity triggers. The shape is here so a solution can be
+// designed against it before the native side exists.
+//
+// IMPLEMENTING IT is a real project, not a wiring job: react-native-ble-plx,
+// iOS background modes, NSBluetoothAlwaysUsageDescription, and a permission
+// flow of its own.
+export interface BluetoothDevice {
+  id: string;
+  name: string | null;
+  /** Signal strength in dBm. Null when not reported — never 0, which is a
+   *  legitimate (and very strong) reading. */
+  rssi: number | null;
+  /** Manufacturer/service payload, if the advertisement carried one. */
+  serviceUUIDs?: string[];
+}
+
+// ── Vision — on-device image understanding ────────────────
+//
+// SHAPE ONLY. Every method refuses by name until a native module lands.
+//
+// WHY IT IS HERE AND NOT IN THE VISUAL DOMAIN. The freeze is explicit
+// (device-state.ts §3): "Visual is NOT the camera API… Everything interpretive
+// — what is in the scene, what the sign says, which corridor to take — sits
+// ABOVE Device Bridge and is not modelled here." Visual owns the frame
+// TIMELINE; deciding what a frame MEANS is this layer's job. Two different
+// questions, deliberately kept apart:
+//
+//   deviceState.visual.getSnapshot()  → "is there a current frame, how old"
+//   native.vision.readText(frame)     → "what does the sign say"
+//
+// FRAMES NEVER LEAVE THE PHONE. That is Arie's decision (2026-08-15) and it is
+// the whole reason this is a platform capability rather than an API call from
+// a plugin: analysis runs on-device, so there is no per-frame cost and no image
+// on anyone's server. An implementation that POSTs the frame somewhere would
+// satisfy the types and violate the design.
+//
+// IT TAKES A HANDLE, NOT PIXELS. `MediaHandle.id` from a VisualFrameRef — the
+// core rule that media crosses by reference. A plugin never holds bytes.
+//
+// NATIVE SIDE (for whoever implements it): iOS Vision —
+// VNRecognizeTextRequest for `readText`, VNClassifyImageRequest for `describe`;
+// Android MLKit text recognition + image labeling. Both are on-device and free
+// per call.
+export interface VisionText {
+  /** The recognised string. */
+  text: string;
+  /** 0..1 recogniser confidence. Null when the platform gives no estimate —
+   *  never 1, which would read as certainty nothing measured. */
+  confidence: number | null;
+  /** Normalised 0..1 box in the frame, origin top-left. Lets a guide say
+   *  "the sign on your right" rather than just quoting words. */
+  box?: { x: number; y: number; width: number; height: number };
+}
+
+export interface VisionLabel {
+  /** A label from the platform's classifier vocabulary — NOT free text. */
+  label: string;
+  confidence: number | null;
+}
+
+export interface VisionObject {
+  /** A detector-vocabulary class name (e.g. "Car", "Traffic light", "Person") —
+   *  NOT free text. */
+  label: string;
+  /** 0..1 detector confidence. Null when the platform gives no estimate — never
+   *  1, which would read as certainty nothing measured. */
+  confidence: number | null;
+  /** Normalised 0..1 box in the frame, origin top-left. Where the thing is,
+   *  so a guide can say "on your right" rather than just naming it. */
+  box?: { x: number; y: number; width: number; height: number };
+}
+
+/**
+ * How usable a frame is, as measured rather than guessed.
+ *
+ * This one is the exception to the split above: it is a MEASUREMENT of the
+ * image, not an interpretation of the scene, so its results belong on
+ * `VisualFrameRef.quality` — which currently reports `unknown` for all three
+ * because nothing looks at the pixels. Wiring this is what lets a guide say
+ * "it's too dark for me to tell" instead of describing a black frame.
+ */
+export interface VisionFrameQuality {
+  blur: 'sharp' | 'slight' | 'severe' | 'unknown';
+  exposure: 'good' | 'dark' | 'bright' | 'unknown';
+  /** True when the lens is covered. Null = not assessed. */
+  obstructed: boolean | null;
+}
+
+export interface VisionAPI {
+  /** Is on-device vision available at all? False is a real answer. */
+  isSupported(): Promise<boolean>;
+  /**
+   * Read text in a frame (OCR).
+   *
+   * Returns an EMPTY ARRAY when it looked and found no text — that is a real
+   * finding, distinct from throwing, which means it could not look.
+   */
+  readText(frameId: string, opts?: { languages?: string[] }): Promise<VisionText[]>;
+  /**
+   * Label what is in a frame.
+   *
+   * Deliberately labels, not prose. A sentence is the SKILL's job — this layer
+   * must not put words in the model's mouth, and a classifier that returns
+   * "a bathroom sign ahead on the left" is asserting a spatial relationship it
+   * did not measure.
+   */
+  describe(frameId: string, opts?: { maxLabels?: number }): Promise<VisionLabel[]>;
+  /**
+   * Detect the objects in a frame, each with a box.
+   *
+   * Distinct from `describe`: describe LABELS the whole scene ("kitchen"),
+   * detectObjects LOCATES the things in it ("Person" on the left, "Car" ahead) —
+   * which is what a guidance loop reads to say where something is. Returns an
+   * EMPTY ARRAY when it looked and found nothing, distinct from throwing (could
+   * not look). On-device (bundled detector); frames never leave the phone.
+   */
+  detectObjects(
+    frameId: string,
+    opts?: { maxItems?: number; minConfidence?: number },
+  ): Promise<VisionObject[]>;
+  /** Measure whether a frame is usable. Feeds VisualFrameRef.quality. */
+  assessQuality(frameId: string): Promise<VisionFrameQuality>;
+}
+
+export interface BluetoothAPI {
+  /** Is BLE hardware present and permitted? */
+  isSupported(): Promise<boolean>;
+  /** Is the radio switched on? (Separate from permission — both can block.) */
+  isEnabled(): Promise<boolean>;
+  /**
+   * Scan for advertising devices for `timeoutMs`, then stop.
+   *
+   * Bounded rather than open-ended on purpose: a scan left running is a
+   * battery drain the caller cannot see, and the mistake is invisible until a
+   * user notices their phone is flat.
+   */
+  scan(opts?: { timeoutMs?: number; serviceUUIDs?: string[] }): Promise<BluetoothDevice[]>;
+}
+
 export interface PluginNative {
   camera: CameraAPI;
   location: LocationAPI;
-  storage: StorageAPI;
+  /**
+   * @deprecated The gate never hands this out — it uses the per-plugin
+   * `storageFor(pluginId)` and FAILS CLOSED (blocks) when it can't scope. A
+   * single shared storage across plugins was a cross-plugin read/write hole;
+   * there is no shared bucket any more. Present only so older providers still
+   * satisfy the type; new providers implement `storageFor`.
+   */
+  storage?: StorageAPI;
   /** @deprecated use `biometricsV2` for the native-backed API. This stub
    *  remains for backwards compatibility. */
   biometrics: BiometricsAPI;
@@ -182,11 +385,41 @@ export interface PluginNative {
   notifications: NotificationsAPI;
   haptics: HapticsAPI;
   connectivity: ConnectivityAPI;
-  fileSystem: FileSystemAPI;
+  /**
+   * @deprecated The gate never hands this out — it uses the per-plugin
+   * `fileSystemFor(pluginId)` and FAILS CLOSED (blocks) when it can't scope. A
+   * single shared filesystem across plugins was a cross-plugin read/write hole;
+   * there is no shared bucket any more. Present only so older providers still
+   * satisfy the type; new providers implement `fileSystemFor`.
+   */
+  fileSystem?: FileSystemAPI;
+  /**
+   * Per-plugin filesystem: returns a FileSystemAPI whose sandbox root is scoped
+   * to THIS plugin (`plugin-files/<pluginId>/`), so one plugin cannot read,
+   * write or reach another plugin's files even via traversal. `gatePluginNative`
+   * uses it (passing the plugin's id) to isolate each plugin. If it is absent or
+   * no id is known, the gate FAILS CLOSED (blocks) rather than fall back to a
+   * shared filesystem — there is no shared bucket.
+   */
+  fileSystemFor?(pluginId: string): FileSystemAPI;
+  /**
+   * Per-plugin storage: returns a StorageAPI whose KV namespace AND SQLite
+   * database are scoped to THIS plugin, so one plugin cannot read, overwrite,
+   * or query another plugin's key-value entries or tables. `gatePluginNative`
+   * uses it (passing the plugin's id) to isolate each plugin. Identity is
+   * threaded through the TRUSTED provider here — never a caller-supplied
+   * key/table prefix. If it is absent or no id is known, the gate FAILS CLOSED
+   * (blocks) rather than fall back to a shared store — there is no shared bucket.
+   */
+  storageFor?(pluginId: string): StorageAPI;
   cookies: CookiesAPI;
   sharing: SharingAPI;
   scanner: ScannerAPI;
   nfc: NfcAPI;
+  /** SHAPE ONLY in this build — see BluetoothAPI. Present so it fails by name. */
+  bluetooth: BluetoothAPI;
+  /** SHAPE ONLY in this build — see VisionAPI. On-device image understanding. */
+  vision: VisionAPI;
 }
 
 // ── Native capability flags (manifest declaration) ──────────
@@ -199,7 +432,17 @@ export interface NativeCapabilities {
   backgroundTasks?: boolean;
   fileSystem?: boolean;
   notifications?: boolean;
+  /** BLE. Declaring it passes the gate but every call refuses by name until
+   *  the native side exists — see BluetoothAPI. */
   bluetooth?: boolean;
+  /** On-device image understanding (OCR, labels, frame quality). Refuses by
+   *  name until the native module lands — see VisionAPI. */
+  vision?: boolean;
+  /** The live voice SESSION (props.voice). A privacy surface — a declaring
+   *  plugin can read the conversation transcript — so it is gated like any
+   *  other capability, and coupled to the platform's unsuppressable mic
+   *  indicator. See VoiceAPI. */
+  voice?: boolean;
   haptics?: boolean;
   /** Read the platform cookie jar after a WebView login flow. */
   cookies?: boolean;
@@ -258,7 +501,43 @@ export interface ThemeTokens {
 
 export interface PluginProps {
   bridge: PluginBridge;
-  native: PluginNative;
+  /**
+   * L2 Device Primitives — the camera/location/storage/haptics/… surface, gated
+   * to the capabilities this plugin declared. This is the canonical name (R4);
+   * the underlying L1 native modules keep the `native` name, but what a plugin
+   * receives is `props.device`.
+   */
+  device: PluginNative;
+  /**
+   * @deprecated Renamed to `device` in R4. Still passed (the SAME gated object)
+   * so existing/remote plugin bundles keep working — migrate to `props.device`.
+   */
+  native?: PluginNative;
+  /**
+   * L3 DeviceState — the SEMANTIC layer (`props.deviceState.spatial/visual/…`),
+   * gated to the domains this plugin declared in `stateDomains`. Reads answer
+   * `unknown` rather than throwing; a domain the build has no engine for, or the
+   * plugin did not declare, returns an explicit refusal.
+   *
+   * REQUIRED, deliberately: an optional field is exactly the compile-time escape
+   * hatch that let a renderer ship without it (props.deviceState === undefined,
+   * which crashed on drawer/card). Every renderer builds this through
+   * resolvePluginRuntimeProps, so the constructor side always has it; making it
+   * required means a NEW renderer that forgets to pass it fails to compile.
+   */
+  deviceState: DeviceState;
+  /**
+   * The live conversational SESSION, gated to plugins that declared
+   * `capabilities: { voice: true }`. A SESSION, not a device sensor: start/stop
+   * the loop, read + subscribe to the transcript, and speak first via
+   * `prompt()`. The mic-active indicator is platform-owned and cannot be
+   * suppressed from here.
+   *
+   * REQUIRED, like `deviceState`: every renderer builds it through
+   * resolvePluginRuntimeProps, so the object is always present. An undeclared
+   * plugin gets a refusal (reads say so, actions throw) — never a fake `idle`.
+   */
+  voice: VoiceAPI;
   theme: ThemeTokens;
   context: PluginContext;
   /** Engine-supplied props from `sys.focusUiPlugin(pluginId, props)`.
@@ -290,6 +569,10 @@ export interface PluginManifest {
   hooks?: PluginHooks;
   channels?: string[] | { events: string[]; actions: string[] };
   capabilities?: Record<string, unknown>;
+  /** L3 DeviceState domains this plugin reads (props.deviceState), as Core passes
+   *  them through from the bundle's declaration. Absent → fall back to the loaded
+   *  bundle's own `stateDomains`. Read via resolvePluginRuntimeProps. */
+  stateDomains?: DeviceBridgeDomain[];
   _source?: string;
   _connectorId?: string;
   _mcpEndpoint?: string;
@@ -301,7 +584,13 @@ export interface RegisteredUIPlugin {
   id: string;
   type: 'ui' | 'hybrid';
   capabilities?: NativeCapabilities;
+  /** L3 DeviceState domains this plugin's UI reads (props.deviceState). The
+   *  trusted source for a HOST-registered plugin (no server manifest), read by
+   *  resolvePluginRuntimeProps. Carried by register(), same as registerTools. */
+  stateDomains?: DeviceBridgeDomain[];
   Component: React.FC<PluginProps>;
+  /** Optional `tools/` section — a plugin may ship both UI and local tools. */
+  tools?: DeviceToolDefinition[];
 }
 
 export interface RegisteredServicePlugin {
@@ -313,4 +602,52 @@ export interface RegisteredServicePlugin {
   onStop?: () => Promise<void> | void;
 }
 
-export type RegisteredPlugin = RegisteredUIPlugin | RegisteredServicePlugin;
+// ── Device tools (a plugin's `tools/` section) ──────────────
+
+/** What a device tool implementation is handed at call time. */
+export interface DeviceToolContext {
+  bridge: PluginBridge;
+  /** Device Bridge — camera, location, sensors, storage. */
+  native: PluginNative;
+  /**
+   * L3 semantic DeviceState (visual/spatial/activity/…), gated to the domains
+   * the plugin declared in `stateDomains`. Every domain is always present; an
+   * undeclared one answers with an explicit "not declared" rather than data —
+   * the same gating a UI plugin's `props.deviceState` gets. This is what lets an
+   * EXTERNAL tool (e.g. Ada Guide's `guide.observe`) read `ctx.deviceState.visual`
+   * over the public surface, with no SDK-internal access.
+   */
+  deviceState: DeviceState;
+  /** Cooperative cancellation: caller cancelled, or the session went away. */
+  signal?: AbortSignal;
+}
+
+export type DeviceToolFn = (args: any, ctx: DeviceToolContext) => Promise<any> | any;
+
+export interface DeviceToolDefinition {
+  /** Domain name, as the catalog and the caller see it. e.g. "guide.observe" */
+  name: string;
+  description?: string;
+  /** JSON Schema for arguments — advertised to Core, not enforced here. */
+  input_schema?: Record<string, unknown>;
+  run: DeviceToolFn;
+}
+
+/** A plugin that ships tools and no UI at all. */
+export interface RegisteredToolsPlugin {
+  id: string;
+  type: 'tools';
+  capabilities?: NativeCapabilities;
+  /**
+   * L3 DeviceState domains this plugin's tools may read via `ctx.deviceState`
+   * (e.g. `['visual']`). Undeclared domains answer "not declared". Same
+   * mechanism as a UI plugin's manifest `stateDomains`.
+   */
+  stateDomains?: DeviceBridgeDomain[];
+  tools: DeviceToolDefinition[];
+}
+
+export type RegisteredPlugin =
+  | RegisteredUIPlugin
+  | RegisteredServicePlugin
+  | RegisteredToolsPlugin;