@opengeni/react-native 1.4.5-canary.37615713321001

package/src/environment.tsx

@@ -0,0 +1,214 @@
+import {
+  createContext,
+  type ReactNode,
+  useContext,
+  useEffect,
+  useLayoutEffect,
+  useMemo,
+  useState,
+} from "react";
+import type { OpenGeniReactNativeAdapters } from "./adapters";
+import { sha256HexToArrayBuffer } from "./adapters";
+
+interface EventTargetLike {
+  addEventListener(type: string, listener: () => void): void;
+  removeEventListener(type: string, listener: () => void): void;
+  dispatch(type: string): void;
+}
+
+function createEventTarget(): EventTargetLike {
+  const listeners = new Map<string, Set<() => void>>();
+  return {
+    addEventListener: (type, listener) => {
+      const existing = listeners.get(type) ?? new Set<() => void>();
+      existing.add(listener);
+      listeners.set(type, existing);
+    },
+    removeEventListener: (type, listener) => {
+      listeners.get(type)?.delete(listener);
+    },
+    dispatch: (type) => {
+      for (const listener of listeners.get(type) ?? []) listener();
+    },
+  };
+}
+
+interface InstalledEnvironment {
+  cleanup(): void;
+}
+
+/** Install only the browser-compatible primitives currently read by the official session hooks. */
+export function installOpenGeniReactNativeEnvironment(
+  adapters: OpenGeniReactNativeAdapters,
+): InstalledEnvironment {
+  const globals = globalThis;
+  const windowTarget = createEventTarget();
+  const documentTarget = createEventTarget();
+  const existingWindow = Reflect.get(globals, "window");
+  const existingSessionStorage = Reflect.get(globals, "sessionStorage");
+  const existingDocument = Reflect.get(globals, "document");
+  const existingCrypto = Reflect.get(globals, "crypto");
+  const nativeWindow = createDelegatingObject(isObject(existingWindow) ? existingWindow : globals);
+  let lifecycleState = adapters.lifecycle.currentState();
+
+  const sessionStorage = {
+    getItem: (key: string) => adapters.persistence.getItem(key),
+    setItem: (key: string, value: string) => adapters.persistence.setItem(key, value),
+    removeItem: (key: string) => adapters.persistence.removeItem(key),
+    clear: () => adapters.persistence.clear(),
+    key: () => null,
+    get length() {
+      return 0;
+    },
+  };
+
+  const nativeCrypto = createDelegatingObject(isObject(existingCrypto) ? existingCrypto : null);
+  const subtleValue = isObject(existingCrypto) ? Reflect.get(existingCrypto, "subtle") : undefined;
+  const nativeSubtle = createDelegatingObject(isObject(subtleValue) ? subtleValue : null);
+  Reflect.set(nativeCrypto, "randomUUID", () => adapters.crypto.randomUUID());
+  Reflect.set(
+    nativeSubtle,
+    "digest",
+    async (algorithm: AlgorithmIdentifier, data: BufferSource) => {
+      const name = typeof algorithm === "string" ? algorithm : algorithm.name;
+      if (name.toUpperCase() !== "SHA-256") {
+        throw new TypeError(`Unsupported native digest algorithm: ${name}`);
+      }
+      const bytes =
+        data instanceof ArrayBuffer
+          ? new Uint8Array(data)
+          : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+      return sha256HexToArrayBuffer(await adapters.crypto.sha256Hex(bytes));
+    },
+  );
+  Reflect.set(nativeCrypto, "subtle", nativeSubtle);
+  Reflect.set(globals, "crypto", nativeCrypto);
+
+  Reflect.set(nativeWindow, "sessionStorage", sessionStorage);
+  if (typeof Reflect.get(nativeWindow, "setTimeout") !== "function") {
+    Reflect.set(nativeWindow, "setTimeout", globalThis.setTimeout.bind(globalThis));
+  }
+  if (typeof Reflect.get(nativeWindow, "clearTimeout") !== "function") {
+    Reflect.set(nativeWindow, "clearTimeout", globalThis.clearTimeout.bind(globalThis));
+  }
+  if (typeof Reflect.get(nativeWindow, "addEventListener") !== "function") {
+    Reflect.set(nativeWindow, "addEventListener", (type: string, listener: () => void) =>
+      windowTarget.addEventListener(type, listener),
+    );
+  }
+  if (typeof Reflect.get(nativeWindow, "removeEventListener") !== "function") {
+    Reflect.set(nativeWindow, "removeEventListener", (type: string, listener: () => void) =>
+      windowTarget.removeEventListener(type, listener),
+    );
+  }
+  Reflect.set(globals, "window", nativeWindow);
+  Reflect.set(globals, "sessionStorage", sessionStorage);
+
+  const nativeDocument = {
+    get visibilityState() {
+      return lifecycleState === "active" ? "visible" : "hidden";
+    },
+    addEventListener: (type: string, listener: () => void) =>
+      documentTarget.addEventListener(type, listener),
+    removeEventListener: (type: string, listener: () => void) =>
+      documentTarget.removeEventListener(type, listener),
+  };
+  Reflect.set(globals, "document", nativeDocument);
+
+  const unsubscribe = adapters.lifecycle.subscribe((next) => {
+    const wasActive = lifecycleState === "active";
+    lifecycleState = next;
+    documentTarget.dispatch("visibilitychange");
+    if (!wasActive && next === "active") windowTarget.dispatch("pageshow");
+  });
+
+  return {
+    cleanup: () => {
+      unsubscribe();
+      restoreGlobal(globals, "window", nativeWindow, existingWindow);
+      restoreGlobal(globals, "sessionStorage", sessionStorage, existingSessionStorage);
+      restoreGlobal(globals, "document", nativeDocument, existingDocument);
+      restoreGlobal(globals, "crypto", nativeCrypto, existingCrypto);
+    },
+  };
+}
+
+function restoreGlobal(
+  target: typeof globalThis,
+  key: string,
+  installedValue: unknown,
+  previousValue: unknown,
+): void {
+  if (Reflect.get(target, key) !== installedValue) return;
+  if (previousValue === undefined) Reflect.deleteProperty(target, key);
+  else Reflect.set(target, key, previousValue);
+}
+
+function isObject(value: unknown): value is object {
+  return (typeof value === "object" && value !== null) || typeof value === "function";
+}
+
+function createDelegatingObject(prototype: object | null): object {
+  const value = {};
+  Object.setPrototypeOf(value, prototype);
+  return value;
+}
+
+export interface NativeEnvironmentContextValue {
+  adapters: OpenGeniReactNativeAdapters;
+  active: boolean;
+}
+
+const NativeEnvironmentContext = createContext<NativeEnvironmentContextValue | null>(null);
+
+export interface OpenGeniReactNativeProviderProps {
+  adapters: OpenGeniReactNativeAdapters;
+  children: ReactNode;
+  loadingFallback?: ReactNode;
+}
+
+export function OpenGeniReactNativeProvider({
+  adapters,
+  children,
+  loadingFallback = null,
+}: OpenGeniReactNativeProviderProps) {
+  const [hydrated, setHydrated] = useState(false);
+  const [active, setActive] = useState(adapters.lifecycle.currentState() === "active");
+
+  useEffect(() => {
+    let mounted = true;
+    setHydrated(false);
+    void adapters.persistence.hydrate().then(() => {
+      if (!mounted) return;
+      setHydrated(true);
+    });
+    const unsubscribe = adapters.lifecycle.subscribe((state) => setActive(state === "active"));
+    return () => {
+      mounted = false;
+      unsubscribe();
+    };
+  }, [adapters]);
+
+  useLayoutEffect(() => {
+    const installed = installOpenGeniReactNativeEnvironment(adapters);
+    return () => {
+      // React removes parent layout effects before passive effects in deleted children.
+      // Keep the browser primitives alive until those official hook cleanups have run.
+      globalThis.setTimeout(() => installed.cleanup(), 0);
+    };
+  }, [adapters]);
+
+  const value = useMemo(() => ({ adapters, active }), [active, adapters]);
+  if (!hydrated) return loadingFallback;
+  return (
+    <NativeEnvironmentContext.Provider value={value}>{children}</NativeEnvironmentContext.Provider>
+  );
+}
+
+export function useOpenGeniReactNativeEnvironment(): NativeEnvironmentContextValue {
+  const value = useContext(NativeEnvironmentContext);
+  if (!value) {
+    throw new Error("OpenGeni native hooks require OpenGeniReactNativeProvider");
+  }
+  return value;
+}

package/src/expo.ts

@@ -0,0 +1,295 @@
+// @opengeni/react-native/expo: ready-made adapters for Expo hosts.
+//
+// Hosts own persistence (where drafts and pending operations live, and how they are
+// namespaced per signed-in principal); everything else here is the standard Expo
+// implementation of the adapter contract.
+import { AppState } from "react-native";
+import { useMemo } from "react";
+import * as Crypto from "expo-crypto";
+import * as DocumentPicker from "expo-document-picker";
+import { fetch as expoFetch } from "expo/fetch";
+import {
+  RecordingPresets,
+  requestRecordingPermissionsAsync,
+  setAudioModeAsync,
+  useAudioRecorder,
+} from "expo-audio";
+import { Directory, File, Paths } from "expo-file-system";
+import * as ImagePicker from "expo-image-picker";
+import type { FetchLike, OpenGeniClient } from "@opengeni/sdk";
+import type {
+  NativeLifecycleState,
+  NativePersistenceAdapter,
+  NativePickedFile,
+  OpenGeniReactNativeAdapters,
+} from "./adapters";
+import {
+  NativeTranscriptionError,
+  type NativeVoiceRecorder,
+  type NativeVoiceRecording,
+} from "./voice-input";
+
+export { createExpoCallAdapter, parseNativeCallEvent } from "./realtime/expo-call";
+
+function lifecycleState(value: string | null | undefined): NativeLifecycleState {
+  if (value === "active") return "active";
+  if (value === "background") return "background";
+  return "inactive";
+}
+
+function bytesToHex(bytes: Uint8Array): string {
+  return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+function base64ToBytes(base64: string): Uint8Array {
+  const binary = globalThis.atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
+  return bytes;
+}
+
+function fallbackFilename(uri: string, contentType: string): string {
+  const raw = uri.split("/").at(-1)?.split("?")[0];
+  if (raw) {
+    try {
+      return decodeURIComponent(raw);
+    } catch {
+      return raw;
+    }
+  }
+  return contentType.startsWith("image/") ? "image" : "document";
+}
+
+function isRuntimeFetchResponse(value: unknown): value is Response {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "status" in value &&
+    typeof value.status === "number" &&
+    "headers" in value &&
+    typeof value.headers === "object" &&
+    value.headers !== null &&
+    "json" in value &&
+    typeof value.json === "function" &&
+    "text" in value &&
+    typeof value.text === "function" &&
+    "arrayBuffer" in value &&
+    typeof value.arrayBuffer === "function" &&
+    "clone" in value &&
+    typeof value.clone === "function"
+  );
+}
+
+/** Streaming fetch backed by Expo's native implementation (required for live event streams). */
+export const expoStreamingFetch: FetchLike = async (input, init) => {
+  const response = await expoFetch(input as string, init as never);
+  // Preserve Expo's native response object: re-wrapping its body corrupts JSON reads on iOS.
+  if (!isRuntimeFetchResponse(response)) {
+    throw new TypeError("Expo fetch returned an incompatible response");
+  }
+  return response;
+};
+
+export const expoLifecycleAdapter: OpenGeniReactNativeAdapters["lifecycle"] = {
+  currentState: () => lifecycleState(AppState.currentState),
+  subscribe: (listener) => {
+    const subscription = AppState.addEventListener("change", (state) =>
+      listener(lifecycleState(state)),
+    );
+    return () => subscription.remove();
+  },
+};
+
+export const expoCryptoAdapter: OpenGeniReactNativeAdapters["crypto"] = {
+  randomUUID: Crypto.randomUUID,
+  sha256Hex: async (bytes) => {
+    const digestInput: Uint8Array<ArrayBuffer> =
+      bytes.buffer instanceof ArrayBuffer
+        ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+        : Uint8Array.from(bytes);
+    const digest = await Crypto.digest(Crypto.CryptoDigestAlgorithm.SHA256, digestInput);
+    return bytesToHex(new Uint8Array(digest));
+  },
+};
+
+export const expoFileAdapter: OpenGeniReactNativeAdapters["files"] = {
+  pickDocuments: async () => {
+    const result = await DocumentPicker.getDocumentAsync({
+      copyToCacheDirectory: true,
+      multiple: true,
+    });
+    if (result.canceled) return [];
+    return result.assets.map(
+      (asset): NativePickedFile => ({
+        id: Crypto.randomUUID(),
+        uri: asset.uri,
+        name: asset.name,
+        contentType: asset.mimeType ?? "application/octet-stream",
+        sizeBytes: asset.size ?? null,
+        kind: "document",
+      }),
+    );
+  },
+  pickImages: async () => {
+    const result = await ImagePicker.launchImageLibraryAsync({
+      allowsMultipleSelection: true,
+      mediaTypes: ["images"],
+      quality: 1,
+    });
+    if (result.canceled) return [];
+    return result.assets.map((asset): NativePickedFile => {
+      const contentType = asset.mimeType ?? "image/jpeg";
+      return {
+        id: Crypto.randomUUID(),
+        uri: asset.uri,
+        name: asset.fileName ?? fallbackFilename(asset.uri, contentType),
+        contentType,
+        sizeBytes: asset.fileSize ?? null,
+        kind: "image",
+        previewUri: asset.uri,
+      };
+    });
+  },
+  // Expo FileSystem's iOS `bytes()` requests write authority that read-only photo-picker
+  // exports do not grant; `base64()` uses read authority, so decode once here.
+  readBytes: async (source) => base64ToBytes(await new File(source.uri).base64()),
+};
+
+/** Standard Expo adapters; the host supplies persistence. */
+export function createExpoOpenGeniAdapters(input: {
+  persistence: NativePersistenceAdapter;
+}): OpenGeniReactNativeAdapters {
+  return {
+    lifecycle: expoLifecycleAdapter,
+    crypto: expoCryptoAdapter,
+    files: expoFileAdapter,
+    fetch: expoStreamingFetch,
+    persistence: input.persistence,
+  };
+}
+
+// Mono AAC in an MPEG-4 container: small uploads, accepted by every transcription provider.
+const DICTATION_RECORDING = {
+  ...RecordingPresets.HIGH_QUALITY,
+  numberOfChannels: 1,
+  bitRate: 64_000,
+  isMeteringEnabled: true,
+};
+
+function discardRecordingFile(uri: string | null) {
+  if (!uri) return;
+  try {
+    const file = new File(uri);
+    if (file.exists) file.delete();
+  } catch {
+    // A leftover cache file is harmless; the system clears the cache directory.
+  }
+}
+
+/** Move a finished recording out of the recorder's scratch path into its own file. */
+async function keepRecordingFile(uri: string): Promise<string> {
+  try {
+    const directory = new Directory(Paths.document, "opengeni-dictation");
+    if (!directory.exists) directory.create({ intermediates: true, idempotent: true });
+    const kept = new File(directory, `${Crypto.randomUUID()}.m4a`);
+    await new File(uri).copy(kept);
+    discardRecordingFile(uri);
+    return kept.uri;
+  } catch {
+    // Keeping the original path is still better than losing the recording.
+    return uri;
+  }
+}
+
+/** Dictation recorder backed by expo-audio. */
+export function useExpoVoiceRecorder(): NativeVoiceRecorder {
+  const recorder = useAudioRecorder(DICTATION_RECORDING);
+  return useMemo<NativeVoiceRecorder>(
+    () => ({
+      async requestPermission() {
+        const permission = await requestRecordingPermissionsAsync();
+        return permission.granted;
+      },
+      async start() {
+        await setAudioModeAsync({ allowsRecording: true, playsInSilentMode: true });
+        await recorder.prepareToRecordAsync();
+        recorder.record();
+      },
+      async stop() {
+        const durationSeconds = recorder.getStatus().durationMillis / 1000;
+        await recorder.stop();
+        await setAudioModeAsync({ allowsRecording: false }).catch(() => undefined);
+        const uri = recorder.uri;
+        if (!uri) return null;
+        // Kept until its text is in the draft: a following segment or a retry
+        // must never find the recorder has reused or cleared this file.
+        return { uri: await keepRecordingFile(uri), mimeType: "audio/mp4", durationSeconds };
+      },
+      async cancel() {
+        if (recorder.isRecording) await recorder.stop();
+        await setAudioModeAsync({ allowsRecording: false }).catch(() => undefined);
+        discardRecordingFile(recorder.uri);
+      },
+      discard(recording) {
+        discardRecordingFile(recording.uri);
+      },
+      durationSeconds() {
+        return recorder.getStatus().durationMillis / 1000;
+      },
+      level() {
+        // Metering is in decibels (silence is about -160); map the useful -55…0 range.
+        const decibels = recorder.getStatus().metering;
+        if (typeof decibels !== "number") return null;
+        return Math.max(0, Math.min(1, (decibels + 55) / 55));
+      },
+    }),
+    [recorder],
+  );
+}
+
+/**
+ * Send a recording to the deployment's transcription API. The device file is
+ * streamed as multipart form data and kept: dictation deletes it only once its
+ * text is in the draft. Failures carry the HTTP status (null when offline).
+ */
+export async function transcribeExpoRecording(input: {
+  client: Pick<OpenGeniClient, "fetchApi">;
+  workspaceId: string;
+  recording: NativeVoiceRecording;
+}): Promise<string> {
+  const form = new FormData();
+  // expo-file-system's File is a Blob, which Expo's fetch uploads from disk.
+  form.append("audio", new File(input.recording.uri) as unknown as Blob, "dictation.m4a");
+  form.append("mimeType", input.recording.mimeType);
+  form.append("durationSeconds", input.recording.durationSeconds.toFixed(3));
+  try {
+    const response = await input.client.fetchApi(
+      `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/transcriptions`,
+      { method: "POST", body: form },
+    );
+    const text = await response.text();
+    let body: { text?: unknown; error?: { message?: unknown }; message?: unknown } = {};
+    try {
+      body = JSON.parse(text) as typeof body;
+    } catch {
+      // A non-JSON body is reported by status below.
+    }
+    if (!response.ok) {
+      const message =
+        typeof body.error?.message === "string"
+          ? body.error.message
+          : typeof body.message === "string"
+            ? body.message
+            : `Transcription failed (${response.status}).`;
+      throw new NativeTranscriptionError(message, response.status);
+    }
+    return typeof body.text === "string" ? body.text : "";
+  } catch (cause) {
+    if (cause instanceof NativeTranscriptionError) throw cause;
+    // No response at all: offline, timed out or dropped. The recording is kept.
+    throw new NativeTranscriptionError(
+      cause instanceof Error ? cause.message : "Network request failed",
+      null,
+    );
+  }
+}

package/src/index.ts

@@ -0,0 +1,98 @@
+export type {
+  AsyncKeyValueStorage,
+  NativeCryptoAdapter,
+  NativeFileAdapter,
+  NativeLifecycleAdapter,
+  NativeLifecycleState,
+  NativePersistenceAdapter,
+  NativePickedFile,
+  OpenGeniReactNativeAdapters,
+} from "./adapters";
+export { createHydratedPersistenceAdapter, sha256HexToArrayBuffer } from "./adapters";
+export {
+  installOpenGeniReactNativeEnvironment,
+  OpenGeniReactNativeProvider,
+  useOpenGeniReactNativeEnvironment,
+  type NativeEnvironmentContextValue,
+  type OpenGeniReactNativeProviderProps,
+} from "./environment";
+export {
+  NATIVE_ATTACHMENT_PREPARATION_TIMEOUT_MS,
+  prepareNativeAttachmentUpload,
+  useNativeFileAttachments,
+  type NativeAttachment,
+  type NativeAttachmentStatus,
+  type NativeFileAttachmentsResult,
+} from "./attachments";
+export {
+  useOpenGeniNativeSession,
+  type OpenGeniNativeSessionClient,
+  type OpenGeniNativeSessionController,
+} from "./use-native-session";
+export {
+  appendDictation,
+  isRetryableTranscriptionError,
+  NativeTranscriptionError,
+  useNativeVoiceInput,
+  type NativeVoiceInput,
+  type NativeVoiceInputStatus,
+  type NativeVoiceRecorder,
+  type NativeVoiceRecording,
+} from "./voice-input";
+export {
+  boundedJson,
+  formatNativeRelativeTime,
+  nativeHumanInputRequestPreview,
+  nativeSessionStatusTone,
+  timelineAccessibilityLabel,
+  validateHumanInputAnswers,
+  type NativeSessionStatusTone,
+} from "./view-model";
+export {
+  DEFAULT_OPENGENI_NATIVE_LABELS,
+  DEFAULT_OPENGENI_NATIVE_THEME,
+  type OpenGeniNativeLabels,
+  type OpenGeniNativeTheme,
+} from "./presentation";
+export {
+  OpenGeniNativeAttachmentStrip,
+  OpenGeniNativeSessionView,
+  type NativeComposerActionContext,
+  type NativeComposerActionRenderer,
+  type NativeComposerRenderContext,
+  type NativeComposerRenderer,
+  type NativeComposerSurfaceContext,
+  type NativeComposerSurfaceRenderer,
+  type NativeMessageRenderer,
+  type NativeToolRenderer,
+  type NativeTurnSummaryContext,
+  type NativeTurnSummaryRenderer,
+  type NativeTurnSummaryResult,
+  type OpenGeniNativeSessionViewProps,
+} from "./session-view";
+export {
+  createMemoryRealtimeOwnerStorage,
+  createNativeRealtimeControllerFactory,
+  createNativeRemoteAudio,
+  type NativeRealtimeControllerFactoryOptions,
+  type NativeWebRtcAdapter,
+} from "./realtime/native-realtime";
+export {
+  useNativeSessionRealtime,
+  type UseNativeSessionRealtimeOptions,
+} from "./realtime/use-native-session-realtime";
+export {
+  nativeRealtimeModelSupported,
+  useNativeRealtimeCall,
+  type NativeCallAdapter,
+  type NativeCallAudioRoute,
+  type NativeCallEvent,
+  type NativeCallPhase,
+  type NativeCallRealtime,
+  type NativeRealtimeCall,
+} from "./realtime/native-call";
+export {
+  DEFAULT_OPENGENI_NATIVE_CALL_LABELS,
+  OpenGeniNativeCallView,
+  type OpenGeniNativeCallLabels,
+} from "./realtime/call-view";