@mentra/engine 0.1.0-dev.0

package/src/services/PhoneVideoCoordinator.ts

@@ -0,0 +1,189 @@
+/**
+ * PhoneVideoCoordinator — owns local-miniapp startVideoRecording()/
+ * stopVideoRecording() end-to-end.
+ *
+ * Architecture:
+ *   miniapp → SDK → LocalMiniappRuntime → videoRecording runtime hook
+ *          → coordinator.startRecording(packageName, opts)
+ *            ├── (precheck) glasses connected
+ *            └── BluetoothSdk.startVideoRecording(requestId, save, sound, settings)
+ *
+ * Unlike photo, this has no cloud upload/long-poll; the promise resolves once
+ * the glasses report a correlated video_recording_status for the request. The
+ * active recordings map enforces single-recording (rejecting overlapping starts
+ * so we never hand back a phantom id), validates stop ownership, and lets a
+ * closed/crashed miniapp be cleaned up on unregister.
+ */
+
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+import {useGlassesStore} from "../stores/glasses"
+import {isGlassesConnected} from "./GlassesReadiness"
+
+export interface VideoRecordingOpts {
+  width?: number
+  height?: number
+  fps?: number
+  sound?: boolean
+  save?: boolean
+}
+
+export interface VideoRecordingStarted {
+  recordingId: string
+}
+
+export class VideoError extends Error {
+  constructor(
+    public readonly code: string,
+    message: string,
+  ) {
+    super(message)
+    this.name = "VideoError"
+  }
+}
+
+interface ActiveRecording {
+  packageName: string
+  recordingId: string
+}
+
+export class PhoneVideoCoordinator {
+  // recordingId → active recording. Used to enforce single-recording, validate
+  // stop requests, and clean up on miniapp unregister.
+  private readonly activeRecordings = new Map<string, ActiveRecording>()
+
+  // Set while a start is dispatching, before it lands in activeRecordings, so a
+  // second start that races the first is still rejected.
+  private starting = false
+
+  async startRecording(packageName: string, opts: VideoRecordingOpts): Promise<VideoRecordingStarted> {
+    // Fail fast if glasses aren't connected — the BLE command would otherwise
+    // be sent into the void.
+    if (!isGlassesConnected(useGlassesStore.getState().connection)) {
+      throw new VideoError("GLASSES_NOT_CONNECTED", "Glasses are not connected")
+    }
+
+    // The glasses can only record one video at a time. A second start while one
+    // is already active (or in-flight) is ignored device-side, but we'd still
+    // mint and return a *new* recordingId — and stopping that id would not stop
+    // the real recording, leaving it to run until the max-recording timeout.
+    // Reject up front so the caller gets an honest error instead of a phantom id.
+    if (this.starting || this.activeRecordings.size > 0) {
+      throw new VideoError("ALREADY_RECORDING", "A video recording is already in progress")
+    }
+
+    this.starting = true
+    try {
+      const recordingId = `miniapp_video_${Date.now()}_${Math.round(Math.random() * 1e6)}`
+
+      // Only forward settings when at least one field is set; otherwise let the
+      // glasses use their saved button-video defaults. The native serializers
+      // send each provided field independently, so an fps-only override still
+      // applies (the glasses merge the missing fields onto their saved defaults).
+      const settings =
+        opts.width != null || opts.height != null || opts.fps != null
+          ? {width: opts.width, height: opts.height, fps: opts.fps}
+          : undefined
+
+      try {
+        const status = await BluetoothSdk.startVideoRecording(
+          recordingId,
+          opts.save ?? false,
+          opts.sound ?? true,
+          settings,
+        )
+        if (status.status !== "recording_started") {
+          throw new VideoError(
+            status.status || "VIDEO_RECORDING_FAILED",
+            status.details || "Glasses did not start recording",
+          )
+        }
+      } catch (err) {
+        throw this.toVideoError(err, "BLE_SEND_FAILED")
+      }
+
+      this.activeRecordings.set(recordingId, {packageName, recordingId})
+      return {recordingId}
+    } finally {
+      this.starting = false
+    }
+  }
+
+  async stopRecording(
+    packageName: string,
+    recordingId?: string,
+    opts?: {uploadUrl?: string; uploadAuthToken?: string},
+  ): Promise<void> {
+    if (!recordingId) {
+      throw new VideoError("INVALID_RECORDING_ID", "recordingId is required to stop a recording")
+    }
+
+    // Validate ownership before touching the glasses: a miniapp may only stop a
+    // recording it started, and we shouldn't forward an unknown id.
+    const active = this.activeRecordings.get(recordingId)
+    if (!active || active.packageName !== packageName) {
+      throw new VideoError("RECORDING_NOT_OWNED", "recordingId is not an active recording owned by this app")
+    }
+
+    try {
+      // When uploadUrl is set the glasses upload the finished clip there (multipart
+      // POST); otherwise the recording stays on the glasses (local save).
+      const status = await BluetoothSdk.stopVideoRecording(recordingId, opts?.uploadUrl, opts?.uploadAuthToken)
+      if (status.status !== "recording_stopped") {
+        throw new VideoError(status.status || "VIDEO_STOP_FAILED", status.details || "Glasses did not stop recording")
+      }
+    } catch (err) {
+      const videoError = this.toVideoError(err, "BLE_SEND_FAILED")
+      if (this.isAlreadyStopped(videoError)) {
+        this.activeRecordings.delete(recordingId)
+        return
+      }
+      // Leave the recording tracked so a retry (or unregister cleanup) can stop
+      // it — only drop it once the glasses report a successful stop.
+      throw videoError
+    }
+    this.activeRecordings.delete(recordingId)
+  }
+
+  /**
+   * Stop every recording owned by a package. Called when a miniapp unregisters
+   * (close/crash): the miniapp loses its recordingId, so without this the
+   * glasses keep recording until the max-recording timeout or thermal shutdown.
+   * Mirrors the streaming coordinator's per-app cleanup on disconnect.
+   */
+  async stopForApp(packageName: string): Promise<void> {
+    const owned = [...this.activeRecordings.values()].filter((r) => r.packageName === packageName)
+    for (const rec of owned) {
+      try {
+        const status = await BluetoothSdk.stopVideoRecording(rec.recordingId)
+        // Only drop tracking once the glasses report a successful stop (mirrors
+        // stopRecording). If we deleted on failure the glasses could still be
+        // recording while the single-recording guard reads "idle", letting the
+        // next start hand back a phantom id that can't stop the real recording.
+        if (status.status === "recording_stopped") {
+          this.activeRecordings.delete(rec.recordingId)
+        }
+      } catch (err) {
+        const videoError = this.toVideoError(err, "BLE_SEND_FAILED")
+        if (this.isAlreadyStopped(videoError)) {
+          this.activeRecordings.delete(rec.recordingId)
+          continue
+        }
+        console.warn(`[PhoneVideoCoordinator] failed to stop ${rec.recordingId} for ${packageName} on cleanup`, videoError)
+      }
+    }
+  }
+
+  private toVideoError(err: unknown, fallbackCode: string): VideoError {
+    if (err instanceof VideoError) return err
+    const code = (err as {code?: string})?.code
+    const message = err instanceof Error ? err.message : String(err)
+    return new VideoError(code || fallbackCode, message)
+  }
+
+  private isAlreadyStopped(error: VideoError): boolean {
+    return error.code === "not_recording" || error.code === "recording_stopped"
+  }
+}
+
+// Singleton — coordinator state is process-wide (mirrors phonePhotoCoordinator).
+export const phoneVideoCoordinator = new PhoneVideoCoordinator()

package/src/services/STTModelManager.ts

@@ -0,0 +1,360 @@
+import BluetoothSdk from "@mentra/bluetooth-sdk"
+import {Platform} from "react-native"
+import * as RNFS from "@dr.pogodin/react-native-fs"
+
+export interface LanguageInfo {
+  code: string
+  displayName: string
+  size: number
+  language: string
+  downloaded: boolean
+  path?: string
+  type: "transducer" | "ctc"
+}
+
+export interface DownloadProgress {
+  jobId: number
+  bytesWritten: number
+  contentLength: number
+  percentage: number
+}
+
+export interface ExtractionProgress {
+  percentage: number
+  currentFile?: string
+}
+
+export interface LanguageConfig {
+  code: string
+  displayName: string
+  fileName: string
+  downloadUrl?: string
+  size: number
+  type: "transducer" | "ctc"
+  requiredFiles: string[]
+  languageCode: string
+}
+
+const DEFAULT_LANGUAGE = "en"
+
+class STTModelManager {
+  private static instance: STTModelManager
+  private downloadJobId?: number
+  private currentLanguage = DEFAULT_LANGUAGE
+  private modelBaseUrl = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/"
+
+  private languages: Record<string, LanguageConfig> = {
+    en: {
+      code: "en",
+      displayName: "English",
+      fileName: "sherpa-onnx-streaming-zipformer-en-2023-06-21-mobile",
+      size: 349 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "en-US",
+    },
+    fr: {
+      code: "fr",
+      displayName: "Français",
+      fileName: "sherpa-onnx-streaming-zipformer-fr-kroko-2025-08-06",
+      size: 57 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "fr-FR",
+    },
+    de: {
+      code: "de",
+      displayName: "Deutsch",
+      fileName: "sherpa-onnx-streaming-zipformer-de-kroko-2025-08-06",
+      size: 58 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "de-DE",
+    },
+    es: {
+      code: "es",
+      displayName: "Español",
+      fileName: "sherpa-onnx-streaming-zipformer-es-kroko-2025-08-06",
+      size: 124 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "es-ES",
+    },
+    zh: {
+      code: "zh",
+      displayName: "中文",
+      fileName: "sherpa-onnx-streaming-zipformer-zh-2025-06-30",
+      size: 150 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "zh-CN",
+    },
+    ko: {
+      code: "ko",
+      displayName: "한국어",
+      fileName: "sherpa-onnx-streaming-zipformer-korean-2024-06-16",
+      size: 200 * 1024 * 1024,
+      type: "transducer",
+      requiredFiles: ["encoder.onnx", "decoder.onnx", "joiner.onnx", "tokens.txt"],
+      languageCode: "ko-KR",
+    },
+  }
+
+  private constructor() {}
+
+  static getInstance(): STTModelManager {
+    if (!STTModelManager.instance) {
+      STTModelManager.instance = new STTModelManager()
+    }
+    return STTModelManager.instance
+  }
+
+  async getCurrentLanguageFromPreferences(): Promise<string> {
+    try {
+      const path = await BluetoothSdk.getSttModelPath()
+      const code = path && path.length > 0 ? this.getLanguageFromPath(path) : ""
+      if (code && this.languages[code]) {
+        this.currentLanguage = code
+        return code
+      }
+      return ""
+    } catch (error) {
+      console.error("Error getting current STT language from preferences:", error)
+      return ""
+    }
+  }
+
+  getCurrentLanguage(): string {
+    return this.currentLanguage
+  }
+
+  setCurrentLanguage(code: string): void {
+    if (this.languages[code]) {
+      this.currentLanguage = code
+    }
+  }
+
+  getBcp47ForLanguage(code?: string): string {
+    const id = code || this.currentLanguage
+    return this.languages[id]?.languageCode ?? "en-US"
+  }
+
+  getAvailableLanguages(): LanguageConfig[] {
+    return Object.values(this.languages)
+  }
+
+  getModelDirectory(): string {
+    const baseDir = Platform.OS === "ios" ? RNFS.DocumentDirectoryPath : RNFS.DocumentDirectoryPath
+    return `${baseDir}/stt_models`
+  }
+
+  getModelPath(code?: string): string {
+    const id = code || this.currentLanguage
+    return `${this.getModelDirectory()}/${id}`
+  }
+
+  /** Last path segment is the language code (the on-disk folder name). */
+  getLanguageFromPath(path: string): string {
+    return path.split("/").pop() || ""
+  }
+
+  async isModelAvailable(code?: string): Promise<boolean> {
+    try {
+      const id = code || this.currentLanguage
+      const language = this.languages[id]
+      if (!language) return false
+
+      const modelPath = this.getModelPath(id)
+      for (const file of language.requiredFiles) {
+        const filePath = `${modelPath}/${file}`
+        const exists = await RNFS.exists(filePath)
+        if (!exists) {
+          console.log(`Missing required STT file: ${file} at ${filePath}`)
+          return false
+        }
+      }
+
+      return await BluetoothSdk.validateSttModel(modelPath)
+    } catch (error) {
+      console.error("Error checking STT model availability:", error)
+      return false
+    }
+  }
+
+  async getLanguageInfo(code?: string): Promise<LanguageInfo> {
+    const id = code || this.currentLanguage
+    const language = this.languages[id]
+    if (!language) {
+      throw new Error(`Language ${id} not found`)
+    }
+
+    const downloaded = await this.isModelAvailable(id)
+    const path = downloaded ? this.getModelPath(id) : undefined
+
+    return {
+      code: id,
+      displayName: language.displayName,
+      size: language.size,
+      language: language.languageCode,
+      downloaded,
+      path,
+      type: language.type,
+    }
+  }
+
+  async getAllLanguageInfo(): Promise<LanguageInfo[]> {
+    const infos: LanguageInfo[] = []
+    for (const code of Object.keys(this.languages)) {
+      infos.push(await this.getLanguageInfo(code))
+    }
+    return infos
+  }
+
+  async downloadModel(
+    code?: string,
+    onProgress?: (progress: DownloadProgress) => void,
+    onExtractionProgress?: (progress: ExtractionProgress) => void,
+  ): Promise<void> {
+    const id = code || this.currentLanguage
+    const language = this.languages[id]
+    if (!language) {
+      throw new Error(`Language ${id} not found`)
+    }
+
+    const modelUrl = language.downloadUrl ?? `${this.modelBaseUrl}${language.fileName}.tar.bz2`
+    const tempPath = `${RNFS.TemporaryDirectoryPath}/${language.fileName}.tar.bz2`
+    const modelDir = this.getModelDirectory()
+    const finalPath = this.getModelPath(id)
+
+    try {
+      await RNFS.mkdir(modelDir, {NSURLIsExcludedFromBackupKey: true})
+
+      const downloadOptions = {
+        fromUrl: modelUrl,
+        toFile: tempPath,
+        progress: (res: RNFS.DownloadProgressCallbackResultT) => {
+          const percentage = Math.round((res.bytesWritten / res.contentLength) * 100)
+          onProgress?.({
+            jobId: res.jobId,
+            bytesWritten: res.bytesWritten,
+            contentLength: res.contentLength,
+            percentage,
+          })
+        },
+        progressDivider: 10,
+        begin: (res: RNFS.DownloadBeginCallbackResultT) => {
+          console.log("Download started:", res)
+        },
+        connectionTimeout: 30000,
+        readTimeout: 30000,
+      }
+
+      const result = await RNFS.downloadFile(downloadOptions)
+      this.downloadJobId = result.jobId
+
+      const downloadResult = await result.promise
+
+      if (downloadResult.statusCode !== 200) {
+        throw new Error(`Download failed with status code: ${downloadResult.statusCode}`)
+      }
+
+      onExtractionProgress?.({percentage: 0})
+
+      const subscription = BluetoothSdk.addListener("extraction_progress", (event) => {
+        onExtractionProgress?.({percentage: event.percentage})
+      })
+
+      try {
+        const extractionResult = await BluetoothSdk.extractTarBz2(tempPath, finalPath)
+        if (!extractionResult) {
+          throw new Error("Native extraction returned failure status")
+        }
+      } catch (extractError) {
+        console.error("Native extraction failed:", extractError)
+        throw extractError
+      } finally {
+        subscription.remove()
+      }
+
+      onExtractionProgress?.({percentage: 100})
+
+      await RNFS.unlink(tempPath)
+
+      if (id === this.currentLanguage) {
+        await this.setNativeModelPath(finalPath, language.languageCode)
+      }
+    } catch (error) {
+      // Best-effort cleanup; never let it mask the original error.
+      try {
+        if (await RNFS.exists(tempPath)) await RNFS.unlink(tempPath)
+      } catch (cleanupError) {
+        console.warn("STTModelManager: temp cleanup failed:", cleanupError)
+      }
+      try {
+        if (await RNFS.exists(finalPath)) await RNFS.unlink(finalPath)
+      } catch (cleanupError) {
+        console.warn("STTModelManager: final cleanup failed:", cleanupError)
+      }
+      throw error
+    }
+  }
+
+  async cancelDownload(): Promise<void> {
+    if (this.downloadJobId !== undefined) {
+      await RNFS.stopDownload(this.downloadJobId)
+      this.downloadJobId = undefined
+    }
+  }
+
+  async deleteModel(code?: string): Promise<void> {
+    const id = code || this.currentLanguage
+    const modelPath = this.getModelPath(id)
+    if (await RNFS.exists(modelPath)) {
+      await RNFS.unlink(modelPath)
+    }
+  }
+
+  async activateLanguage(code: string): Promise<void> {
+    const language = this.languages[code]
+    if (!language) {
+      throw new Error(`Language ${code} not found`)
+    }
+
+    const isAvailable = await this.isModelAvailable(code)
+    if (!isAvailable) {
+      throw new Error(`Language ${code} model is not downloaded`)
+    }
+
+    this.currentLanguage = code
+    const modelPath = this.getModelPath(code)
+    await this.setNativeModelPath(modelPath, language.languageCode)
+  }
+
+  private async setNativeModelPath(path: string, languageCode: string): Promise<void> {
+    BluetoothSdk.setSttModelDetails(path, languageCode)
+  }
+
+  async getStorageInfo(): Promise<{free: number; total: number}> {
+    const fsInfo = await RNFS.getFSInfo()
+    return {
+      free: fsInfo.freeSpace,
+      total: fsInfo.totalSpace,
+    }
+  }
+
+  formatBytes(bytes: number): string {
+    return STTModelManager.formatBytes(bytes)
+  }
+
+  static formatBytes(bytes: number): string {
+    if (bytes === 0) return "0 Bytes"
+    const k = 1024
+    const sizes = ["Bytes", "KB", "MB", "GB"]
+    const i = Math.floor(Math.log(bytes) / Math.log(k))
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i]
+  }
+}
+
+const instance = STTModelManager.getInstance()
+export {STTModelManager}
+export default instance