@ateam-ai-mobile/expo 0.1.20 → 0.1.22

package/dist/bootstrap.js

@@ -175,8 +175,8 @@ __export(bootstrap_exports, {
   bootstrapExpoAdas: () => bootstrapExpoAdas
 });
 module.exports = __toCommonJS(bootstrap_exports);
-var import_core2 = require("@ateam-ai-mobile/core");
-var import_plugin_runtime3 = require("@ateam-ai-mobile/plugin-runtime");
+var import_core3 = require("@ateam-ai-mobile/core");
+var import_plugin_runtime4 = require("@ateam-ai-mobile/plugin-runtime");
 var import_ui2 = require("@ateam-ai-mobile/ui");
 
 // src/audio/index.ts
@@ -1901,6 +1901,12 @@ async function describe(id, maxLabels = 5) {
 async function detectObjects(id, maxItems = 10, minConfidence = 0.25) {
   return verb("detectObjects")(id, maxItems, minConfidence);
 }
+async function jpegById(id, maxSide = 768, quality = 0.7) {
+  if (native2 && typeof native2.jpegById !== "function") {
+    throw new Error("sending pictures to AI needs a newer app build \u2014 this one cannot make a picture from a camera frame");
+  }
+  return verb("jpegById")(id, maxSide, quality);
+}
 
 // src/maps/index.ts
 var import_react_native3 = require("react-native");
@@ -2917,6 +2923,420 @@ function createDeviceProvider(config2) {
   };
 }
 
+// src/aiLive/provider.ts
+var import_core2 = require("@ateam-ai-mobile/core");
+var import_plugin_runtime3 = require("@ateam-ai-mobile/plugin-runtime");
+
+// src/aiLive/streamProtocol.ts
+var STREAM_PATH = "/ws/stream";
+var SUBPROTOCOL = "adas.v1";
+var HEADER_BYTES = 14;
+var ENVELOPE_VERSION = 1;
+var CLOSE = Object.freeze({
+  AUTH_MISSING: 4001,
+  AUTH_FAILED: 4002,
+  NO_SUCH_WORKSPACE: 4003,
+  NOT_ENTITLED: 4004,
+  BAD_REQUEST: 4400,
+  UNKNOWN_KIND: 4404,
+  TIMEOUT: 4408,
+  OVER_LIMIT: 4429,
+  TOO_BIG: 1009,
+  CONSUMER_FAILING: 1011
+});
+function encodeFrame({ seq, tCapture, payload }) {
+  const out = new Uint8Array(HEADER_BYTES + payload.length);
+  const v = new DataView(out.buffer);
+  v.setUint8(0, ENVELOPE_VERSION);
+  v.setUint8(1, 0);
+  v.setUint32(2, seq >>> 0, false);
+  v.setFloat64(6, tCapture, false);
+  out.set(payload, HEADER_BYTES);
+  return out;
+}
+function openMessage(kind, params) {
+  return JSON.stringify(params ? { type: "open", kind, params } : { type: "open", kind });
+}
+var CLOSE_MESSAGE = JSON.stringify({ type: "close" });
+function parseServerEvent(text2) {
+  if (typeof text2 !== "string") return null;
+  try {
+    const m = JSON.parse(text2);
+    return m && typeof m.event === "string" ? { event: m.event, data: m.data } : null;
+  } catch {
+    return null;
+  }
+}
+var WHY = {
+  [CLOSE.AUTH_MISSING]: "The login or the workspace was missing. Sign in again and try once more.",
+  [CLOSE.AUTH_FAILED]: "The login was refused. Sign in again and try once more.",
+  [CLOSE.NO_SUCH_WORKSPACE]: "This workspace was not found.",
+  [CLOSE.NOT_ENTITLED]: "This workspace may not stream right now.",
+  [CLOSE.TIMEOUT]: "The connection to the AI timed out.",
+  [CLOSE.OVER_LIMIT]: "Too many cameras are sending to this workspace. Stop another one first.",
+  [CLOSE.TOO_BIG]: "A picture was too big to send.",
+  [CLOSE.CONSUMER_FAILING]: "The AI stopped taking pictures. Check the live session in AI Models."
+};
+function describeClose(code, reason) {
+  if (code === CLOSE.BAD_REQUEST && reason) return reason;
+  return WHY[code] || (reason ? reason : `The connection to the AI closed (${code}).`);
+}
+var B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+var B64_INDEX = (() => {
+  const t = new Int16Array(256).fill(-1);
+  for (let i = 0; i < 64; i++) t[B64.charCodeAt(i)] = i;
+  return t;
+})();
+function base64ToBytes(b64) {
+  const clean = b64.replace(/[^A-Za-z0-9+/]/g, "");
+  const out = new Uint8Array(Math.floor(clean.length * 3 / 4));
+  let o = 0, acc = 0, bits = 0;
+  for (let i = 0; i < clean.length; i++) {
+    acc = acc << 6 | B64_INDEX[clean.charCodeAt(i)];
+    bits += 6;
+    if (bits >= 8) {
+      bits -= 8;
+      out[o++] = acc >> bits & 255;
+    }
+  }
+  return o === out.length ? out : out.subarray(0, o);
+}
+
+// src/aiLive/provider.ts
+var LIVE_KIND = "ai-models.live";
+var FRESH_MS2 = 3e3;
+var MAX_BUFFERED = 512 * 1024;
+var DEFAULT_CAMERA = { fps: 1, maxPx: 768 };
+var OPEN_TIMEOUT_MS = 2e4;
+var CLOSE_WAIT_MS = 3e3;
+var isEpisode = (d) => !!d && typeof d === "object" && typeof d.type === "string" && typeof d.class === "string" && typeof d.episodeId === "string" && typeof d.actorId === "string" && d.actorId.length > 0 && Number.isFinite(d.at);
+var defaultAiLiveDeps = () => ({
+  WebSocketImpl: globalThis.WebSocket,
+  wsBase: import_core2.getCoreWsUrl,
+  token: import_core2.getToken,
+  tenant: import_core2.getTenant,
+  jpegById,
+  now: Date.now,
+  setInterval: (fn, ms) => setInterval(fn, ms),
+  clearInterval: (h) => clearInterval(h),
+  setTimeout: (fn, ms) => setTimeout(fn, ms),
+  clearTimeout: (h) => clearTimeout(h)
+});
+var idle = () => ({ state: "idle", camera: null, sent: 0, skipped: 0, lastEpisode: null, lastError: null });
+function createAiLiveProvider(overrides = {}) {
+  const deps = { ...defaultAiLiveDeps(), ...overrides };
+  let owner = null;
+  let snap = idle();
+  let ws = null;
+  let loop = null;
+  let visual = null;
+  let allowed = () => false;
+  let stopReason = null;
+  let tearingDown = null;
+  const waitingToStart = /* @__PURE__ */ new Map();
+  const closedOf = /* @__PURE__ */ new WeakMap();
+  let closingOnPurpose = false;
+  let seq = 0;
+  let busy = false;
+  let gen = 0;
+  let lastFrameId = null;
+  let camera = DEFAULT_CAMERA;
+  let lastOwner = null;
+  const episodeCbs = /* @__PURE__ */ new Map();
+  const snapCbs = /* @__PURE__ */ new Map();
+  const sendingCbs = /* @__PURE__ */ new Set();
+  let wasSending = false;
+  const add = (m, who, cb) => {
+    let set3 = m.get(who);
+    if (!set3) {
+      set3 = /* @__PURE__ */ new Set();
+      m.set(who, set3);
+    }
+    set3.add(cb);
+    return { unsubscribe: () => {
+      set3.delete(cb);
+    } };
+  };
+  const set2 = (patch) => {
+    snap = { ...snap, ...patch };
+    for (const cb of lastOwner && snapCbs.get(lastOwner) || []) {
+      try {
+        cb(snap);
+      } catch {
+      }
+    }
+    const sending = snap.state === "starting" || snap.state === "sending";
+    if (sending !== wasSending) {
+      wasSending = sending;
+      for (const cb of sendingCbs) {
+        try {
+          cb();
+        } catch {
+        }
+      }
+    }
+  };
+  function stopTimers() {
+    if (loop != null) {
+      deps.clearInterval(loop);
+      loop = null;
+    }
+  }
+  async function slot() {
+    const mySeq = seq++;
+    const g = gen;
+    const sock = ws;
+    const current = () => g === gen && ws === sock && sock?.readyState === 1;
+    const consentGone = () => {
+      if (allowed()) return false;
+      if (g === gen) void teardown("stopped", import_plugin_runtime3.CONSENT_TAKEN_BACK);
+      return true;
+    };
+    if (consentGone()) return;
+    if (busy || !current()) {
+      set2({ skipped: snap.skipped + 1 });
+      return;
+    }
+    if ((sock.bufferedAmount || 0) > MAX_BUFFERED) {
+      set2({ skipped: snap.skipped + 1 });
+      return;
+    }
+    busy = true;
+    try {
+      const v = await visual.getSnapshot();
+      if (!current()) return;
+      const f = v?.latest;
+      const id = f?.handle?.id;
+      const t = Number(f?.timestamp);
+      if (!id || id === lastFrameId || !(deps.now() - t < FRESH_MS2)) {
+        set2({ skipped: snap.skipped + 1 });
+        return;
+      }
+      const b64 = await deps.jpegById(id, camera.maxPx, 0.7);
+      if (!current() || consentGone()) return;
+      const frame = encodeFrame({ seq: mySeq, tCapture: t, payload: base64ToBytes(b64) });
+      sock.send(frame.buffer);
+      lastFrameId = id;
+      set2({ sent: snap.sent + 1 });
+    } catch (e) {
+      if (g === gen) set2({ skipped: snap.skipped + 1, lastError: String(e?.message || e) });
+    } finally {
+      if (g === gen) busy = false;
+    }
+  }
+  function openSocket() {
+    return new Promise((resolve, reject) => {
+      const token = deps.token();
+      const tenant = deps.tenant();
+      if (!token || !tenant) {
+        reject(new Error("not signed in to a workspace"));
+        return;
+      }
+      const url = `${deps.wsBase()}${STREAM_PATH}?tenant=${encodeURIComponent(tenant)}`;
+      let opened = false;
+      const sock = new deps.WebSocketImpl(url, [SUBPROTOCOL, `adas.bearer.${token}`]);
+      ws = sock;
+      sock.binaryType = "arraybuffer";
+      let markClosed = () => {
+      };
+      closedOf.set(sock, new Promise((r) => {
+        markClosed = r;
+      }));
+      const gaveUp = deps.setTimeout(() => {
+        if (opened) return;
+        reject(new Error("Core did not answer in time. Try again."));
+        try {
+          sock.close();
+        } catch {
+        }
+      }, OPEN_TIMEOUT_MS);
+      sock.onopen = () => {
+        try {
+          sock.send(openMessage(LIVE_KIND, { session: "own", source: "phone" }));
+        } catch (e) {
+          reject(e);
+        }
+      };
+      sock.onmessage = (m) => {
+        if (ws !== sock) return;
+        const ev = parseServerEvent(m?.data);
+        if (!ev) return;
+        if (ev.event === "session" && !opened) {
+          const c = ev.data?.camera;
+          camera = c && Number(c.fps) > 0 && Number(c.maxPx) > 0 ? { fps: Number(c.fps), maxPx: Number(c.maxPx) } : DEFAULT_CAMERA;
+          return;
+        }
+        if (ev.event === "opened" && !opened) {
+          opened = true;
+          deps.clearTimeout(gaveUp);
+          resolve();
+          return;
+        }
+        if (ev.event === "episode" && isEpisode(ev.data)) {
+          const episode = ev.data;
+          set2({ lastEpisode: episode });
+          for (const cb of owner && episodeCbs.get(owner) || []) {
+            try {
+              cb(episode);
+            } catch {
+            }
+          }
+        }
+      };
+      sock.onerror = () => {
+      };
+      sock.onclose = (c) => {
+        markClosed();
+        const why = describeClose(Number(c?.code), c?.reason);
+        if (!opened) {
+          deps.clearTimeout(gaveUp);
+          reject(new Error(why));
+          return;
+        }
+        if (closingOnPurpose || ws !== sock) return;
+        void teardown("error", why);
+      };
+    });
+  }
+  function teardown(state, lastError) {
+    if (tearingDown) return tearingDown;
+    const p = teardownNow(state, lastError);
+    tearingDown = p;
+    void p.then(() => {
+    }, () => {
+    }).then(() => {
+      if (tearingDown === p) tearingDown = null;
+    });
+    return p;
+  }
+  async function teardownNow(state, lastError) {
+    stopReason = lastError;
+    allowed = () => false;
+    gen++;
+    stopTimers();
+    const sock = ws;
+    ws = null;
+    closingOnPurpose = true;
+    try {
+      if (sock && sock.readyState === 1) sock.send(CLOSE_MESSAGE);
+    } catch {
+    }
+    try {
+      sock?.close?.();
+    } catch {
+    }
+    closingOnPurpose = false;
+    const v = visual;
+    visual = null;
+    let gaveUp = null;
+    const closed = sock && closedOf.get(sock);
+    await Promise.all([
+      (async () => {
+        try {
+          await v?.stop?.();
+        } catch {
+        }
+      })(),
+      closed ? Promise.race([closed, new Promise((r) => {
+        gaveUp = deps.setTimeout(r, CLOSE_WAIT_MS);
+      })]) : Promise.resolve()
+    ]);
+    if (gaveUp != null) deps.clearTimeout(gaveUp);
+    owner = null;
+    set2({ state, lastError });
+  }
+  return {
+    async start(opts, who, ctx) {
+      if (tearingDown) {
+        const wait = { cancelled: false };
+        waitingToStart.set(who, wait);
+        try {
+          while (tearingDown) {
+            try {
+              await tearingDown;
+            } catch {
+            }
+          }
+        } finally {
+          if (waitingToStart.get(who) === wait) waitingToStart.delete(who);
+        }
+        if (wait.cancelled) throw Object.assign(new Error("aiLive: stopped before it started."), { code: "STOPPED" });
+      }
+      if (owner && owner !== who) throw Object.assign(new Error("aiLive: another screen is sending to AI right now. Close it first."), { code: "CONFLICT" });
+      if (owner === who) return snap;
+      const cam = opts?.camera === "front" ? "front" : "back";
+      if (opts?.camera && opts.camera !== "front" && opts.camera !== "back") throw new Error(`aiLive: unknown camera "${String(opts.camera)}" \u2014 use "front" or "back"`);
+      owner = who;
+      lastOwner = who;
+      visual = ctx.visual;
+      allowed = typeof ctx.allowed === "function" ? ctx.allowed : () => false;
+      gen++;
+      seq = 0;
+      lastFrameId = null;
+      busy = false;
+      const g = gen;
+      const cameraOfScreen = ctx.visual;
+      snap = { ...idle(), camera: cam };
+      set2({ state: "starting" });
+      const stillStarting = async () => {
+        if (g === gen && !allowed()) await teardown("stopped", import_plugin_runtime3.CONSENT_TAKEN_BACK);
+        if (g !== gen) throw Object.assign(new Error(stopReason || "aiLive: stopped before it started sending."), { code: "STOPPED" });
+      };
+      try {
+        await stillStarting();
+        camera = DEFAULT_CAMERA;
+        if (g !== gen) throw Object.assign(new Error(stopReason || "aiLive: stopped before it started sending."), { code: "STOPPED" });
+        await openSocket();
+        await stillStarting();
+        await cameraOfScreen.start({ intent: "stream", facing: cam });
+        if (g !== gen) {
+          try {
+            await cameraOfScreen.stop?.();
+          } catch {
+          }
+        }
+        await stillStarting();
+        if (g !== gen) throw Object.assign(new Error(stopReason || "aiLive: stopped before it started sending."), { code: "STOPPED" });
+        loop = deps.setInterval(() => {
+          void slot();
+        }, Math.max(200, Math.round(1e3 / camera.fps)));
+        set2({ state: "sending" });
+        return snap;
+      } catch (e) {
+        if (g !== gen) {
+          throw e?.code === "STOPPED" ? e : Object.assign(new Error(stopReason || "aiLive: stopped before it started sending."), { code: "STOPPED" });
+        }
+        await teardown("error", String(e?.message || e));
+        throw e;
+      }
+    },
+    async stop(who, why) {
+      const waiting = waitingToStart.get(who);
+      if (waiting) waiting.cancelled = true;
+      if (owner !== who) return;
+      await teardown("stopped", why ?? null);
+    },
+    getSnapshot: (who) => lastOwner === who ? snap : idle(),
+    onEpisode: (cb, who) => add(episodeCbs, who, cb),
+    subscribe: (cb, who) => add(snapCbs, who, cb),
+    forget(who) {
+      episodeCbs.delete(who);
+      snapCbs.delete(who);
+      if (lastOwner === who && owner === null) {
+        lastOwner = null;
+        snap = idle();
+      }
+    },
+    isSending: () => snap.state === "starting" || snap.state === "sending",
+    subscribeSending(cb) {
+      sendingCbs.add(cb);
+      return { unsubscribe: () => {
+        sendingCbs.delete(cb);
+      } };
+    }
+  };
+}
+
 // src/oauth/expoOAuth.tsx
 var import_react_native7 = require("react-native");
 var import_google_signin = require("@react-native-google-signin/google-signin");
@@ -2996,7 +3416,7 @@ function bootstrapExpoAdas(opts) {
     deviceProvider: createDeviceProvider({ reachabilityUrl: opts.reachabilityUrl })
   });
   for (const plugin of opts.nativePlugins ?? []) {
-    for (const id of plugin.ids) (0, import_plugin_runtime3.registerNativePlugin)(id, plugin.resolver);
+    for (const id of plugin.ids) (0, import_plugin_runtime4.registerNativePlugin)(id, plugin.resolver);
   }
   const relay = opts.urls.relay;
   (0, import_ui2.setPluginWebViewExtraHosts)(() => relay ? [relay()] : []);
@@ -3008,14 +3428,15 @@ function bootstrapExpoAdas(opts) {
   (0, import_ui2.setCallAudioProvider)(createCallAudioProvider());
   if (opts.onReachabilityChange) {
     const onReach = opts.onReachabilityChange;
-    (0, import_core2.setAcsReachabilityReporter)({ up: () => onReach(true), down: () => onReach(false) });
+    (0, import_core3.setAcsReachabilityReporter)({ up: () => onReach(true), down: () => onReach(false) });
   }
-  (0, import_plugin_runtime3.setDeviceStateProvider)(
+  (0, import_plugin_runtime4.setDeviceStateProvider)(
     createDeviceState({ connectivity: connectivity_exports, location: location_exports })
   );
-  (0, import_plugin_runtime3.registerPlatformDeviceTools)();
+  (0, import_plugin_runtime4.setAiLiveProvider)(createAiLiveProvider());
+  (0, import_plugin_runtime4.registerPlatformDeviceTools)();
   (0, import_ui2.setUIActionNavigator)(opts.navigateToPlugin);
-  (0, import_core2.initAdas)({
+  (0, import_core3.initAdas)({
     coreUrl: opts.urls.core,
     ...opts.urls.voice ? { voiceUrl: opts.urls.voice } : {},
     ...opts.urls.consumerAuth ? { consumerAuthUrl: opts.urls.consumerAuth } : {},