@glassly/engine 0.1.0-dev.0

package/src/services/SceneRenderer.ts

@@ -0,0 +1,320 @@
+/**
+ * SceneRenderer — the device-aware boundary of the scene display pipeline.
+ *
+ * LocalDisplayManager owns WHO may draw (arbitration/boot/expiry); this service
+ * owns WHAT gets sent for a scene: capabilities lookup, legacy-layout→scene
+ * conversion (sugar and old bundles ride the same pipeline), process → diff →
+ * SceneFrame emission, retained-scene replay, and the degrade branch for
+ * devices that can't position elements.
+ *
+ * Everything device-specific here is DATA (capability fields, display profile);
+ * the code paths are generic. Spec: notes/miniapp-display-render-implementation-spec.md
+ */
+
+import displayProcessor from "./DisplayProcessor"
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import {useDisplayStore} from "../stores/display"
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+import {useGlassesStore} from "../stores/glasses"
+import {DeviceTypes} from "../types"
+import {getEffectiveCapabilities} from "./effectiveCapabilities"
+import {
+  degradeScene,
+  diffScene,
+  processScene,
+  resolveAnchors,
+  SceneStore,
+  type FrameElement,
+  type SceneBreakMode,
+  type SceneDisplayCapabilities,
+  type SceneElementInput,
+  type SceneFrame,
+} from "../utils/display/scene"
+
+const LOG_TAG = "SCENE_RENDERER"
+
+/** Legacy layout shape (matches DisplayPayload.layout). */
+export type LegacyLayout = {layoutType: string; [key: string]: unknown}
+
+export type SceneEmitResult =
+  /** A SceneFrame went to native. */
+  | {kind: "scene"; degraded: boolean; dropped: string[]}
+  /** Degrade path: caller routes this legacy layout through the existing pipeline (null layout ⇒ clear). */
+  | {kind: "legacy"; layout: LegacyLayout | null; degraded: boolean; dropped: string[]}
+  /** No display capabilities on the connected device. */
+  | {kind: "no-display"}
+
+class SceneRenderer {
+  private static instance: SceneRenderer | null = null
+  private store = new SceneStore()
+
+  public static getInstance(): SceneRenderer {
+    if (!SceneRenderer.instance) {
+      SceneRenderer.instance = new SceneRenderer()
+    }
+    return SceneRenderer.instance
+  }
+
+  // ===========================================================================
+  // Capabilities
+  // ===========================================================================
+
+  /**
+   * Scene capabilities of the currently-selected device, or null when it has no
+   * display block. Read on demand — same model source the CONNECT_ACK uses.
+   */
+  public currentCapabilities(): SceneDisplayCapabilities | null {
+    // dev read these via the (since-removed) settings/glassesStatus runtime
+    // hooks; engine reads its own stores directly.
+    const model =
+      (useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) as string | undefined) ??
+      (useGlassesStore.getState().deviceModel as string | undefined)
+    const display = getEffectiveCapabilities((model ?? DeviceTypes.NONE) as DeviceTypes)?.display
+    if (!display) return null
+    return {
+      width: display.width ?? display.resolution?.width ?? displayProcessor.getProfile().displayWidthPx,
+      height: display.height ?? display.resolution?.height ?? displayProcessor.getProfile().displayHeightPx ?? 288,
+      canPosition: display.canPosition === true,
+      maxTextElements: display.maxTextElements ?? 6,
+      maxImageElements: display.maxImageElements ?? 0,
+      maxImagePx: display.maxImagePx,
+      shapes: display.shapes ?? [],
+      intensityLevels: display.intensityLevels ?? 2,
+      partialUpdate: display.partialUpdate ?? false,
+    }
+  }
+
+  // ===========================================================================
+  // Legacy layout → scene conversion (sugar + old bundles on the scene path)
+  // ===========================================================================
+
+  /**
+   * Convert a legacy layout into scene elements for a positioning device.
+   * Returns null for layouts that must stay on the legacy path (dashboard_card,
+   * clear_view, unknown types). Sugar semantics: full-canvas / split boxes.
+   */
+  public convertLegacyLayout(layout: LegacyLayout, caps: SceneDisplayCapabilities): SceneElementInput[] | null {
+    const W = caps.width
+    const H = caps.height
+    const breakMode = layout.breakMode as SceneBreakMode | undefined
+
+    switch (layout.layoutType) {
+      case "text_wall":
+        return [
+          {
+            type: "text",
+            id: "sugar:wall",
+            box: {x: 0, y: 0, w: W, h: H},
+            text: String(layout.text ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      case "double_text_wall": {
+        const half = Math.floor(W / 2)
+        return [
+          {
+            type: "text",
+            id: "sugar:left",
+            box: {x: 0, y: 0, w: half, h: H},
+            text: String(layout.topText ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+          {
+            type: "text",
+            id: "sugar:right",
+            box: {x: half, y: 0, w: W - half, h: H},
+            text: String(layout.bottomText ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      }
+      case "reference_card": {
+        const titleH = Math.max(24, Math.floor(H / 5))
+        return [
+          {type: "text", id: "sugar:title", box: {x: 0, y: 0, w: W, h: titleH}, text: String(layout.title ?? "")},
+          {
+            type: "text",
+            id: "sugar:body",
+            box: {x: 0, y: titleH, w: W, h: H - titleH},
+            text: String(layout.text ?? ""),
+            ...(breakMode ? {style: {breakMode}} : {}),
+          },
+        ]
+      }
+      case "bitmap_view": {
+        // Dimensionless legacy bitmaps historically landed in the device's
+        // default ~200×100 container (G2 seeded 200×100 at 188,44) — NOT the
+        // full canvas, which would tile a small image across the whole frame.
+        const w = (layout.width as number | undefined) ?? Math.min(200, W)
+        const h = (layout.height as number | undefined) ?? Math.min(100, H)
+        return [
+          {
+            type: "image",
+            id: "sugar:bmp",
+            box: {
+              x: (layout.x as number | undefined) ?? Math.max(0, Math.round((W - w) / 2)),
+              y: (layout.y as number | undefined) ?? Math.max(0, Math.round((H - h) / 4)),
+              w,
+              h,
+            },
+            data: String(layout.data ?? ""),
+          },
+        ]
+      }
+      case "positioned_text": {
+        const border = (layout.borderWidth as number | undefined) ?? 0
+        const radius = (layout.borderRadius as number | undefined) ?? 0
+        return [
+          {
+            type: "text",
+            id: "sugar:pos",
+            box: {
+              x: (layout.x as number | undefined) ?? 0,
+              y: (layout.y as number | undefined) ?? 0,
+              w: (layout.width as number | undefined) ?? W,
+              h: (layout.height as number | undefined) ?? H,
+            },
+            text: String(layout.text ?? ""),
+            ...(border || radius ? {style: {border, radius}} : {}),
+          },
+        ]
+      }
+      default:
+        return null
+    }
+  }
+
+  // ===========================================================================
+  // Emit
+  // ===========================================================================
+
+  /**
+   * Process + diff + send a scene for (app, view). On non-positioning devices
+   * this returns the degraded legacy layout instead — the caller routes it
+   * through the existing DisplayProcessor path (which wraps it, keeping G1/Z100
+   * output byte-identical to today).
+   */
+  public emitScene(appId: string, view: "main" | "dashboard", elements: SceneElementInput[]): SceneEmitResult {
+    const caps = this.currentCapabilities()
+    if (!caps) return {kind: "no-display"}
+
+    // Corner-anchored boxes resolve against the REAL canvas before either
+    // branch: processScene clamps concrete coords, and degradeScene's reading
+    // order sorts by resolved y (a bottom-anchored y:0 must not sort first).
+    elements = resolveAnchors(elements, caps.width, caps.height)
+
+    if (!caps.canPosition) {
+      const {layout, degraded, dropped} = degradeScene(elements)
+      return {kind: "legacy", layout, degraded, dropped}
+    }
+
+    const processed = processScene(elements, caps, displayProcessor.getProfile())
+    // A stale baseline means the glasses were cleared since the last commit —
+    // diff from empty so everything repaints (the retained frame stays valid
+    // as the replay source only).
+    const prevFrame = this.store.isBaselineStale(appId, view) ? [] : this.store.lastFrame(appId, view)
+    const {elements: diffed, removed} = diffScene(
+      prevFrame,
+      processed.elements,
+      this.store.nextSyntheticId(appId, view),
+    )
+    const canvas = {width: caps.width, height: caps.height}
+    this.store.commit(appId, view, diffed, canvas)
+
+    const frame: SceneFrame = {
+      appId,
+      view,
+      sceneEpoch: this.store.currentEpoch(appId, view),
+      elements: diffed,
+      removed,
+      canvas,
+    }
+    this.sendFrame(frame)
+    return {kind: "scene", degraded: processed.degraded, dropped: processed.dropped}
+  }
+
+  /**
+   * Re-emit an app's retained scene (reconnect / arbitration restore). Replay
+   * is always create-based — the device rebuilds from scratch (spec §4).
+   * Returns false when nothing is retained.
+   */
+  public replayApp(appId: string, view: "main" | "dashboard"): boolean {
+    const frame = this.store.buildReplayFrame(appId, view)
+    if (!frame) return false
+    this.sendFrame(frame)
+    return true
+  }
+
+  /** Forget an app's retained scenes (app stop). */
+  public clearApp(appId: string): void {
+    this.store.clear(appId)
+  }
+
+  /**
+   * Reset the diff baseline for (app, view) without touching the device.
+   * Used when a legacy layout bypasses the scene path (clear_view /
+   * dashboard_card / unknown layoutTypes go to native raw): after that, the
+   * retained frame no longer matches the glasses, so the next render() must
+   * diff from empty — otherwise "unchanged" elements would be skipped and
+   * never repainted.
+   */
+  public resetBaseline(appId: string, view: "main" | "dashboard"): void {
+    this.store.commit(appId, view, [])
+  }
+
+  /**
+   * A system-level clear wiped the view (durationMs expiry, unmount fallback,
+   * boot-window expiry): mark every app's baseline for that view stale. Unlike
+   * resetBaseline this KEEPS the retained frames — they remain the
+   * restore-as-replay source; only incremental diffing restarts from empty.
+   */
+  public markViewStale(view: "main" | "dashboard"): void {
+    this.store.markViewStale(view)
+  }
+
+  private sendFrame(frame: SceneFrame): void {
+    try {
+      // Frames carry the canvas they were laid out against; only a frame from
+      // a path that predates the tag falls back to the current capabilities.
+      const caps = this.currentCapabilities()
+      if (!frame.canvas && caps) frame = {...frame, canvas: {width: caps.width, height: caps.height}}
+      // sendDisplayEvent hook -> direct BluetoothSdk.displayEvent (the exact
+      // wiring the deleted MantleManager hook provided).
+      void Promise.resolve(BluetoothSdk.displayEvent({view: frame.view, scene: frame})).catch((err) => {
+        console.error(`${LOG_TAG}: native scene send failed:`, err)
+      })
+
+      // Keep the phone-side glasses mirror on the same frame as native. Legacy
+      // layouts update this store in LocalDisplayManager.sendToNative; scene
+      // frames bypass that method, so without an explicit write every
+      // positioning device (including G2) leaves the mirror blank/stale.
+      useDisplayStore.getState().setDisplayEvent({
+        view: frame.view,
+        layout: {
+          layoutType: "scene",
+          width: frame.canvas?.width,
+          height: frame.canvas?.height,
+          elements: frame.elements,
+        },
+      })
+    } catch (err) {
+      console.error(`${LOG_TAG}: native scene send failed:`, err)
+    }
+  }
+
+  // ===========================================================================
+  // Test hooks
+  // ===========================================================================
+
+  public _resetForTest(): void {
+    this.store.clearAll()
+  }
+
+  public _lastFrameForTest(appId: string, view: string): FrameElement[] {
+    return this.store.lastFrame(appId, view)
+  }
+}
+
+const sceneRenderer = SceneRenderer.getInstance()
+export default sceneRenderer
+export {SceneRenderer}

package/src/services/SentenceTtsPipeline.ts

@@ -0,0 +1,106 @@
+export interface SentenceAudio {
+  audioUrl: string
+  cleanup?: () => Promise<void> | void
+}
+
+export interface SentencePlaybackResult {
+  success: boolean
+  completed: boolean
+  error: string | null
+  duration: number | null
+}
+
+export interface SentencePipelineRun {
+  cancelled: boolean
+}
+
+interface SentenceTtsPipelineOptions {
+  sentences: string[]
+  firstAudio: SentenceAudio
+  run: SentencePipelineRun
+  synthesize: (sentence: string) => Promise<SentenceAudio>
+  play: (audio: SentenceAudio, index: number) => Promise<SentencePlaybackResult>
+  onCleanupError?: (error: unknown) => void
+}
+
+async function cleanupAudio(audio: SentenceAudio, onError?: (error: unknown) => void): Promise<void> {
+  try {
+    await Promise.resolve(audio.cleanup?.())
+  } catch (error) {
+    onError?.(error)
+  }
+}
+
+/**
+ * Play pre-segmented TTS with one-sentence lookahead.
+ *
+ * The next sentence begins synthesis before playback of the current sentence,
+ * minimizing the transition delay without requiring incremental waveform
+ * support from the TTS model.
+ */
+export async function runSentenceTtsPipeline({
+  sentences,
+  firstAudio,
+  run,
+  synthesize,
+  play,
+  onCleanupError,
+}: SentenceTtsPipelineOptions): Promise<SentencePlaybackResult> {
+  let current = firstAudio
+  let totalDuration = 0
+
+  for (let index = 0; index < sentences.length; index++) {
+    if (run.cancelled) {
+      await cleanupAudio(current, onCleanupError)
+      return {success: true, completed: false, error: null, duration: totalDuration}
+    }
+
+    // Attach both handlers immediately so a fast native rejection cannot
+    // become an unhandled promise while the current sentence is playing.
+    const nextOutcome =
+      index + 1 < sentences.length
+        ? synthesize(sentences[index + 1]).then(
+            (audio) => ({audio, error: null}),
+            (error: unknown) => ({audio: null, error}),
+          )
+        : null
+
+    const playback = await play(current, index)
+    totalDuration += playback.duration ?? 0
+    await cleanupAudio(current, onCleanupError)
+
+    if (run.cancelled) {
+      if (nextOutcome) {
+        void nextOutcome.then(({audio}) => {
+          if (audio) void cleanupAudio(audio, onCleanupError)
+        })
+      }
+      return {success: true, completed: false, error: null, duration: totalDuration}
+    }
+    if (!playback.success || !playback.completed) {
+      if (nextOutcome) {
+        void nextOutcome.then(({audio}) => {
+          if (audio) void cleanupAudio(audio, onCleanupError)
+        })
+      }
+      return playback
+    }
+    if (!nextOutcome) {
+      return {success: true, completed: true, error: null, duration: totalDuration}
+    }
+
+    const next = await nextOutcome
+    if (!next.audio) {
+      const message = next.error instanceof Error ? next.error.message : String(next.error)
+      return {
+        success: false,
+        completed: false,
+        error: `Offline TTS failed for sentence ${index + 2}: ${message}`,
+        duration: totalDuration,
+      }
+    }
+    current = next.audio
+  }
+
+  return {success: true, completed: true, error: null, duration: totalDuration}
+}

package/src/services/SpeakerIdModelManager.ts

@@ -0,0 +1,151 @@
+/**
+ * SpeakerIdModelManager — runtime download of the on-device speaker-embedding
+ * model (single .onnx, no extraction; ~28 MB).
+ *
+ * The model lives under a different GitHub release tag than the ASR models
+ * STTModelManager pulls from — note the upstream typo in the tag name: it
+ * really is `speaker-recongition-models`. The download is sha256-pinned; a
+ * hash mismatch deletes the file and fails.
+ */
+
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import * as RNFS from "@dr.pogodin/react-native-fs"
+
+import {SPEAKER_ID_MODEL_ID} from "./SpeakerProfileStore"
+
+/** Upstream's typo — the release tag really is spelled this way. */
+const RELEASE_TAG = "speaker-recongition-models"
+const MODEL_SHA256 = "aa3cfc16963a10586a9393f5035d6d6b57e98d358b347f80c2a30bf4f00ceba2"
+const MODEL_SIZE_BYTES = 28.3 * 1024 * 1024
+
+export interface SpeakerIdModelProgress {
+  bytesWritten: number
+  contentLength: number
+  percentage: number
+}
+
+class SpeakerIdModelManager {
+  private static instance: SpeakerIdModelManager
+  private downloadJobId?: number
+  private downloadPromise: Promise<void> | null = null
+
+  private constructor() {}
+
+  static getInstance(): SpeakerIdModelManager {
+    if (!SpeakerIdModelManager.instance) {
+      SpeakerIdModelManager.instance = new SpeakerIdModelManager()
+    }
+    return SpeakerIdModelManager.instance
+  }
+
+  get modelId(): string {
+    return SPEAKER_ID_MODEL_ID
+  }
+
+  get sizeBytes(): number {
+    return MODEL_SIZE_BYTES
+  }
+
+  getModelPath(): string {
+    return `${RNFS.DocumentDirectoryPath}/speaker_id_models/${SPEAKER_ID_MODEL_ID}.onnx`
+  }
+
+  private getDownloadUrl(): string {
+    return `https://github.com/k2-fsa/sherpa-onnx/releases/download/${RELEASE_TAG}/${SPEAKER_ID_MODEL_ID}.onnx`
+  }
+
+  async isModelAvailable(): Promise<boolean> {
+    try {
+      const path = this.getModelPath()
+      if (!(await RNFS.exists(path))) return false
+      const stat = await RNFS.stat(path)
+      return Number(stat.size) > 0
+    } catch {
+      return false
+    }
+  }
+
+  /** Point the native engine at the model if it's already on disk. */
+  async activateIfDownloaded(): Promise<boolean> {
+    if (!(await this.isModelAvailable())) return false
+    await BluetoothSdk.setSpeakerIdModelPath(this.getModelPath())
+    return true
+  }
+
+  async downloadModel(onProgress?: (progress: SpeakerIdModelProgress) => void): Promise<void> {
+    if (this.downloadPromise) return this.downloadPromise
+    this.downloadPromise = this.downloadInner(onProgress).finally(() => {
+      this.downloadPromise = null
+    })
+    return this.downloadPromise
+  }
+
+  private async downloadInner(onProgress?: (progress: SpeakerIdModelProgress) => void): Promise<void> {
+    const finalPath = this.getModelPath()
+    const dir = finalPath.slice(0, finalPath.lastIndexOf("/"))
+    const tempPath = `${RNFS.TemporaryDirectoryPath}/${SPEAKER_ID_MODEL_ID}.onnx.partial`
+
+    console.log(`SpeakerID: model download starting → ${this.getDownloadUrl()}`)
+    try {
+      await RNFS.mkdir(dir, {NSURLIsExcludedFromBackupKey: true})
+
+      const result = await RNFS.downloadFile({
+        fromUrl: this.getDownloadUrl(),
+        toFile: tempPath,
+        progress: (res: RNFS.DownloadProgressCallbackResultT) => {
+          onProgress?.({
+            bytesWritten: res.bytesWritten,
+            contentLength: res.contentLength,
+            percentage: res.contentLength > 0 ? Math.round((res.bytesWritten / res.contentLength) * 100) : 0,
+          })
+        },
+        progressDivider: 5,
+        connectionTimeout: 30000,
+        readTimeout: 30000,
+      })
+      this.downloadJobId = result.jobId
+      const downloadResult = await result.promise
+      this.downloadJobId = undefined
+
+      if (downloadResult.statusCode !== 200) {
+        throw new Error(`Speaker ID model download failed with status ${downloadResult.statusCode}`)
+      }
+
+      const hash = await RNFS.hash(tempPath, "sha256")
+      if (hash.toLowerCase() !== MODEL_SHA256) {
+        throw new Error("Speaker ID model checksum mismatch — download corrupted")
+      }
+
+      if (await RNFS.exists(finalPath)) await RNFS.unlink(finalPath)
+      await RNFS.moveFile(tempPath, finalPath)
+      await BluetoothSdk.setSpeakerIdModelPath(finalPath)
+      console.log(`SpeakerID: model downloaded, checksum ok, activated at ${finalPath}`)
+    } catch (error) {
+      console.warn(`SpeakerID: model download failed: ${error instanceof Error ? error.message : String(error)}`)
+      try {
+        if (await RNFS.exists(tempPath)) await RNFS.unlink(tempPath)
+      } catch (cleanupError) {
+        console.warn("SpeakerIdModelManager: temp cleanup failed:", cleanupError)
+      }
+      throw error
+    }
+  }
+
+  async cancelDownload(): Promise<void> {
+    if (this.downloadJobId !== undefined) {
+      await RNFS.stopDownload(this.downloadJobId)
+      this.downloadJobId = undefined
+    }
+  }
+
+  async deleteModel(): Promise<void> {
+    const path = this.getModelPath()
+    if (await RNFS.exists(path)) {
+      await RNFS.unlink(path)
+    }
+  }
+}
+
+const instance = SpeakerIdModelManager.getInstance()
+export {SpeakerIdModelManager}
+export default instance

package/src/services/SpeakerIdSync.ts

@@ -0,0 +1,79 @@
+/**
+ * SpeakerIdSync — keeps the native SpeakerIdEngine in step with the engine
+ * layer: the "Identify enrolled speakers" setting drives the engine's enabled
+ * flag, profile centroids are pushed into native memory whenever the profile
+ * store changes (MMKV is the source of truth; native holds them in memory
+ * only — deleting a profile drops it from native on the same push), and the
+ * model path is activated when the model is already on disk. Also starts the
+ * SpeakerLabelResolver that annotates cloud transcripts.
+ *
+ * Started by `engine.start()`. Idempotent.
+ */
+
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+import speakerIdModelManager from "./SpeakerIdModelManager"
+import speakerLabelResolver from "./SpeakerLabelResolver"
+import {speakerProfileStore} from "./speakerProfileStoreInstance"
+
+let subs: Array<() => void> = []
+
+async function pushEnabled(): Promise<void> {
+  const enabled = useSettingsStore.getState().getSetting(SETTINGS.speaker_id_enabled.key) === true
+  console.log(`SpeakerID: pushing identify enabled=${enabled} to native`)
+  await BluetoothSdk.setSpeakerIdEnabled(enabled)
+}
+
+async function pushProfiles(): Promise<void> {
+  const profiles = await speakerProfileStore.list()
+  console.log(`SpeakerID: pushing ${profiles.length} profile(s) to native: ${profiles.map((p) => p.name).join(", ")}`)
+  await BluetoothSdk.setSpeakerProfiles(profiles.map((p) => ({id: p.id, name: p.name, embedding: p.centroid})))
+}
+
+export function startSpeakerIdSync(): void {
+  if (subs.length) return
+
+  speakerLabelResolver.start()
+  subs.push(() => speakerLabelResolver.stop())
+
+  subs.push(
+    useSettingsStore.subscribe(
+      (s) => s.getSetting(SETTINGS.speaker_id_enabled.key),
+      () => {
+        void pushEnabled().catch((err) => console.warn("SpeakerIdSync: enable push failed:", err))
+      },
+    ),
+  )
+
+  subs.push(
+    speakerProfileStore.onChanged(() => {
+      void pushProfiles().catch((err) => console.warn("SpeakerIdSync: profile push failed:", err))
+    }),
+  )
+
+  // Initial seed. Profile push needs the user identity, which may not resolve
+  // yet (e.g. first launch before sign-in) — that's fine, the next store
+  // change re-pushes; native simply has no profiles until then.
+  void speakerIdModelManager
+    .activateIfDownloaded()
+    .then((activated) => {
+      console.log(
+        `SpeakerID: model ${activated ? `activated (${speakerIdModelManager.getModelPath()})` : "not downloaded yet"}`,
+      )
+      return pushEnabled()
+    })
+    .then(() => pushProfiles())
+    .catch((err) => console.warn("SpeakerIdSync: initial sync failed:", err instanceof Error ? err.message : err))
+}
+
+export function stopSpeakerIdSync(): void {
+  for (const unsub of subs) {
+    try {
+      unsub()
+    } catch {
+      // ignore
+    }
+  }
+  subs = []
+}