@glassly/engine 0.1.0-dev.0

package/src/services/LocalMiniappStorage.ts

@@ -0,0 +1,82 @@
+/**
+ * Phone-local SimpleStorage for bundled miniapps.
+ *
+ * Keys are scoped to the stable Glassly user id and package name. Access tokens
+ * must never be part of the namespace: Core access tokens rotate and are held
+ * in memory only, so using one makes every app restart look like a new user.
+ */
+
+export interface LocalMiniappStorageBackend {
+  get(key: string): unknown | null
+  has(key: string): boolean
+  set(key: string, value: unknown): void
+  remove(key: string): void
+  keys(): string[]
+}
+
+export interface LocalMiniappStorageHooks {
+  getUserId: () => Promise<string>
+  backend: LocalMiniappStorageBackend
+}
+
+const KEY_ROOT = "localstorage_"
+
+export class LocalMiniappStorage {
+  constructor(private readonly hooks: LocalMiniappStorageHooks) {}
+
+  async get(packageName: string, key: string): Promise<unknown | null> {
+    return this.hooks.backend.get((await this.prefix(packageName)) + key)
+  }
+
+  async set(packageName: string, key: string, value: unknown): Promise<void> {
+    this.hooks.backend.set((await this.prefix(packageName)) + key, value)
+  }
+
+  async delete(packageName: string, key: string): Promise<void> {
+    this.hooks.backend.remove((await this.prefix(packageName)) + key)
+  }
+
+  async keys(packageName: string): Promise<string[]> {
+    const prefix = await this.prefix(packageName)
+    return this.hooks.backend
+      .keys()
+      .filter((key) => key.startsWith(prefix))
+      .map((key) => key.slice(prefix.length))
+  }
+
+  async clear(packageName: string): Promise<void> {
+    const prefix = await this.prefix(packageName)
+    for (const key of this.hooks.backend.keys()) {
+      if (key.startsWith(prefix)) this.hooks.backend.remove(key)
+    }
+  }
+
+  async has(packageName: string, key: string): Promise<boolean> {
+    return this.hooks.backend.has((await this.prefix(packageName)) + key)
+  }
+
+  async getAll(packageName: string): Promise<Record<string, string>> {
+    const prefix = await this.prefix(packageName)
+    const values: Record<string, string> = {}
+    for (const key of this.hooks.backend.keys()) {
+      if (!key.startsWith(prefix)) continue
+      if (!this.hooks.backend.has(key)) continue
+      const value = this.hooks.backend.get(key)
+      values[key.slice(prefix.length)] = typeof value === "string" ? value : String(value ?? "")
+    }
+    return values
+  }
+
+  async setMultiple(packageName: string, values: Record<string, unknown>): Promise<void> {
+    const prefix = await this.prefix(packageName)
+    for (const [key, value] of Object.entries(values)) {
+      this.hooks.backend.set(prefix + key, value ?? null)
+    }
+  }
+
+  private async prefix(packageName: string): Promise<string> {
+    const userId = (await this.hooks.getUserId()).trim()
+    if (!userId) throw new Error("Glassly user identity is unavailable")
+    return `${KEY_ROOT}${userId}_${packageName}_`
+  }
+}

package/src/services/LocalMiniappUserIdentity.ts

@@ -0,0 +1,79 @@
+export interface LocalMiniappUserIdentityBackend {
+  get(): string | null
+  set(userId: string): void
+  remove(): void
+}
+
+/**
+ * Prefix marking an id that was minted on-device rather than issued by Core.
+ * Core ids are opaque, so the prefix is what lets a later sign-in recognise a
+ * local namespace and decide whether to migrate or abandon it.
+ */
+export const LOCAL_USER_ID_PREFIX = "local_"
+
+/** True for ids minted by {@link LocalMiniappUserIdentity.resolveLocal}. */
+export function isLocalUserId(userId: string): boolean {
+  return userId.startsWith(LOCAL_USER_ID_PREFIX)
+}
+
+/**
+ * Keeps the stable Glassly user id available to local miniapps across process
+ * restarts. The id is non-secret and is deliberately stored separately from
+ * the short-lived Core access token.
+ */
+export class LocalMiniappUserIdentity {
+  private current: string | null = null
+
+  constructor(private readonly backend: LocalMiniappUserIdentityBackend) {}
+
+  async resolve(resolveOnline: () => Promise<string>): Promise<string> {
+    const available = this.get()
+    if (available) return available
+
+    const resolved = (await resolveOnline()).trim()
+    if (!resolved) throw new Error("Glassly user identity is unavailable")
+    this.remember(resolved)
+    return resolved
+  }
+
+  /**
+   * Namespace for when Core can never answer — the user has turned Glassly
+   * cloud off in Privacy settings, so there is no client to exchange a token
+   * with and {@link resolve} would throw forever.
+   *
+   * Storage still needs *a* stable namespace: local miniapps (maps, captions)
+   * are offline-capable by design and only break because `prefix()` has no id
+   * to scope keys with. The minted id is device-local, non-secret, and
+   * persisted through the same backend, so it survives restarts exactly like a
+   * Core id. A cached Core id always wins, so flipping cloud off does not
+   * strand data written while signed in.
+   */
+  resolveLocal(): string {
+    const available = this.get()
+    if (available) return available
+
+    const minted = `${LOCAL_USER_ID_PREFIX}${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`
+    this.remember(minted)
+    return minted
+  }
+
+  get(): string | null {
+    if (this.current) return this.current
+    const cached = this.backend.get()?.trim() ?? ""
+    if (!cached) return null
+    this.current = cached
+    return cached
+  }
+
+  remember(userId: string): void {
+    const resolved = userId.trim()
+    if (!resolved) throw new Error("Glassly user identity is unavailable")
+    this.backend.set(resolved)
+    this.current = resolved
+  }
+
+  forget(): void {
+    this.current = null
+    this.backend.remove()
+  }
+}

package/src/services/LocalSttFallbackCoordinator.ts

@@ -0,0 +1,247 @@
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import {ISLAND_SETTINGS_KEYS} from "../runtime/config"
+import {useSettingsStore} from "../stores/settings"
+import {cloudClientService} from "./CloudClientService"
+import sttModelManager from "./STTModelManager"
+
+/**
+ * Decides whether on-device Sherpa STT should produce transcripts for local
+ * miniapps, instead of (or in addition to) cloud transcription. Local STT is
+ * active when the cloud is unavailable (disconnected or refusing STT for plan
+ * quota) or any subscription requires `forceLocal`.
+ *
+ * Activation flips `localSttFallbackActive` in the settings store, which
+ * the bluetooth-sdk native side reads to decide whether to feed PCM into
+ * the SherpaOnnxTranscriber. Local transcripts then flow back through the
+ * `local_transcription` Bridge event; the host wires that event to
+ * `localMiniappRuntime.forwardEvent`, with this coordinator providing the
+ * subscription-active gate so we never publish transcripts to miniapps that
+ * aren't asking for them. LocalMiniappRuntime filters mixed local/cloud
+ * delivery per miniapp.
+ */
+/**
+ * Why local STT is currently active, mirrored into the
+ * `local_stt_fallback_reason` setting ("" while inactive) so UI can show it.
+ * When several causes hold at once, the primary-path outage wins over
+ * `force_local` (a forced subscription would keep local STT on either way).
+ */
+export type LocalSttFallbackReason =
+  | "force_local"
+  | "cloud_disconnected"
+  | "cloud_disabled"
+  | "stt_quota"
+  | "provider_unreachable"
+
+class LocalSttFallbackCoordinator {
+  private static instance: LocalSttFallbackCoordinator
+
+  private hasTranscriptionSubscription = false
+  private hasForceLocalSubscription = false
+  private activeLanguage: string | null = null
+  /**
+   * Default to "cloud is up" so we never accidentally activate local STT before
+   * the cloud client has had a chance to start. We attach lazily on the first
+   * subscription/reconcile pass; once attached, this field reflects the real
+   * cloud-client runtime status.
+   */
+  private cloudConnected = true
+  private localActive = false
+  private cloudAdapterAttached = false
+  /**
+   * Soniox-direct (user API key) state, pushed by SonioxDirectService. While
+   * enabled, IT is the primary transcription path — cloud connectivity is
+   * irrelevant and sherpa engages on Soniox health instead. Defaults healthy
+   * for the same reason cloudConnected defaults true: never activate local STT
+   * before the primary has had a chance to come up.
+   */
+  private sonioxDirectEnabled = false
+  private sonioxDirectHealthy = true
+  /**
+   * Cloud STT refused for plan quota (`stream.quota` blocked=true). The cloud
+   * connection stays up, so this is tracked separately from `cloudConnected`:
+   * for transcription purposes the cloud path is down, and sherpa takes over
+   * if its model is on-device. Soniox-direct is unmetered and unaffected.
+   */
+  private sttQuotaBlocked = false
+
+  private constructor() {
+    // Reset the persisted mirror flag on boot — the in-memory state in this
+    // coordinator is the source of truth, and a stale "true" left from the
+    // previous session would cause native to feed Sherpa before any miniapp
+    // registered a subscription.
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, false)
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackReason, "")
+  }
+
+  /**
+   * Lazy-attach the engine cloud connection listener. The coordinator is
+   * constructed at module load, before engine.start() brings up the cloud
+   * client, so we defer until the first reconcile.
+   */
+  private attachCloudAdapterIfReady(): void {
+    if (this.cloudAdapterAttached) return
+    this.cloudConnected = cloudClientService.isConnected()
+    cloudClientService.onStatusChanged((status) => {
+      const connected = status.status === "connected"
+      if (this.cloudConnected === connected) return
+      this.log(`cloud connection -> ${connected ? "up" : "down"}`)
+      this.cloudConnected = connected
+      void this.reconcile()
+    })
+    this.cloudAdapterAttached = true
+  }
+
+  static getInstance(): LocalSttFallbackCoordinator {
+    if (!LocalSttFallbackCoordinator.instance) {
+      LocalSttFallbackCoordinator.instance = new LocalSttFallbackCoordinator()
+    }
+    return LocalSttFallbackCoordinator.instance
+  }
+
+  isActive(): boolean {
+    return this.localActive
+  }
+
+  getActiveLanguage(): string | null {
+    return this.activeLanguage
+  }
+
+  /** The cause of the current activation, or null while inactive. */
+  getReason(): LocalSttFallbackReason | null {
+    return this.localActive ? this.currentReason() : null
+  }
+
+  onSubscriptionChange(hasTranscription: boolean, language: string | null, hasForceLocal = false): void {
+    this.log(`onSubscriptionChange(hasTx=${hasTranscription}, lang=${language}, forceLocal=${hasForceLocal})`)
+    this.hasTranscriptionSubscription = hasTranscription
+    this.hasForceLocalSubscription = hasTranscription && hasForceLocal
+    this.activeLanguage = hasTranscription ? language : null
+    void this.reconcile()
+  }
+
+  /**
+   * Informational hook, retained for API stability. No longer called: the
+   * authoritative switching signal is the cloud-client adapter (local miniapps
+   * are powered only by V2). v1 cloud transcripts deliberately do NOT drive this
+   * coordinator anymore. Kept for future hysteresis if connection-state alone
+   * proves too coarse.
+   */
+  onCloudTranscript(): void {}
+
+  /**
+   * Pushed when the cloud announces/clears an STT quota block. Awaitable so
+   * the caller can inspect `isActive()` afterwards to phrase its user notice
+   * ("continuing on-device" vs "transcription paused").
+   */
+  async onSttQuotaBlockedChange(blocked: boolean): Promise<void> {
+    if (this.sttQuotaBlocked === blocked) return
+    this.log(`stt quota -> ${blocked ? "blocked" : "restored"}`)
+    this.sttQuotaBlocked = blocked
+    await this.reconcile()
+  }
+
+  isSttQuotaBlocked(): boolean {
+    return this.sttQuotaBlocked
+  }
+
+  /** Pushed by SonioxDirectService on every enable/health transition. */
+  onSonioxDirectStateChange(enabled: boolean, healthy: boolean): void {
+    if (this.sonioxDirectEnabled === enabled && this.sonioxDirectHealthy === healthy) return
+    this.log(`soniox-direct -> enabled=${enabled} healthy=${healthy}`)
+    this.sonioxDirectEnabled = enabled
+    this.sonioxDirectHealthy = healthy
+    void this.reconcile()
+  }
+
+  /** Whether the primary transcription path (Soniox-direct when a user key is
+   *  set, the cloud otherwise) is up. */
+  private primaryUp(): boolean {
+    // Cloud turned off in Privacy settings: the cloud path is not merely down,
+    // it is never coming back, so on-device STT is the primary. Checked ahead
+    // of `cloudConnected` because that field defaults to true (so local STT
+    // doesn't pre-empt cloud at boot) and would otherwise report a path that
+    // will never connect as up. Soniox-direct is a user-supplied third-party
+    // key, not Glassly cloud, so it stays available.
+    if (!this.sonioxDirectEnabled && cloudClientService.isCloudDisabled()) return false
+    return this.sonioxDirectEnabled ? this.sonioxDirectHealthy : this.cloudConnected && !this.sttQuotaBlocked
+  }
+
+  private async reconcile(): Promise<void> {
+    this.attachCloudAdapterIfReady()
+    const shouldBeActive = this.shouldUseLocalStt()
+    if (shouldBeActive && !this.localActive) {
+      await this.startLocalStt()
+    } else if (!shouldBeActive && this.localActive) {
+      this.stopLocalStt(this.hasTranscriptionSubscription ? "primary transcription recovered" : "subscription gone")
+    } else if (shouldBeActive && this.localActive) {
+      // Still active, but the cause may have shifted (e.g. quota block landing
+      // while already offline for a disconnect) — keep the mirror current.
+      this.publishReason()
+    }
+  }
+
+  /**
+   * Ranks the concurrent causes: a dead-forever cloud path first, then the
+   * primary provider's own failure, then transient outages, with `force_local`
+   * only when the primary is actually up. Mirrors `primaryUp()`.
+   */
+  private currentReason(): LocalSttFallbackReason {
+    if (!this.sonioxDirectEnabled && cloudClientService.isCloudDisabled()) return "cloud_disabled"
+    if (this.sonioxDirectEnabled && !this.sonioxDirectHealthy) return "provider_unreachable"
+    if (!this.sonioxDirectEnabled && !this.cloudConnected) return "cloud_disconnected"
+    if (!this.sonioxDirectEnabled && this.sttQuotaBlocked) return "stt_quota"
+    return "force_local"
+  }
+
+  private publishReason(): void {
+    const reason = this.localActive ? this.currentReason() : ""
+    const store = useSettingsStore.getState()
+    if (store.getSetting(ISLAND_SETTINGS_KEYS.localSttFallbackReason) !== reason) {
+      store.setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackReason, reason)
+    }
+  }
+
+  private async startLocalStt(): Promise<void> {
+    this.log("starting local stt")
+    const modelAvailable = await sttModelManager.isModelAvailable()
+    if (!modelAvailable) {
+      this.log("local stt model is not available yet — skipping activation")
+      return
+    }
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed before start completed")
+      return
+    }
+    try {
+      // Direct btsdk call now (was a host restartTranscriber hook).
+      await BluetoothSdk.restartTranscriber()
+    } catch (err) {
+      this.log(`restartTranscriber failed: ${err}`)
+    }
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed during transcriber restart")
+      return
+    }
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, true)
+    this.localActive = true
+    this.publishReason()
+  }
+
+  private stopLocalStt(reason: string): void {
+    this.log(`stopping local stt: ${reason}`)
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, false)
+    this.localActive = false
+    this.publishReason()
+  }
+
+  private shouldUseLocalStt(): boolean {
+    return this.hasTranscriptionSubscription && (this.hasForceLocalSubscription || !this.primaryUp())
+  }
+
+  private log(msg: string): void {
+    console.log(`[LocalSttFallback] ${msg}`)
+  }
+}
+
+const localSttFallbackCoordinator = LocalSttFallbackCoordinator.getInstance()
+export default localSttFallbackCoordinator

package/src/services/MapboxTokenService.ts

@@ -0,0 +1,118 @@
+/**
+ * MapboxTokenService
+ *
+ * Single source of truth for which Mapbox access token the app uses. The app
+ * binary deliberately ships NO baked Mapbox key: either the user supplies
+ * their own key (Settings > Keys, `mapbox_api_key`, "use my account"), or the
+ * token comes from Glassly Cloud at runtime.
+ *
+ * Two resolution levels:
+ *
+ * - `resolveMapboxAccessToken()` (sync): the USER key only. Gates phone-side
+ *   direct REST calls (Directions/Geocoding/Search) — without a user key those
+ *   operations go through the cloud maps proxy instead, so the org token is
+ *   only ever billed via the cloud's own auth/cache/rate-limit point.
+ *
+ * - `resolveMapboxRenderToken()` (async): the token for consumers that CANNOT
+ *   be proxied — map tile rendering (miniapp WebView via session.keys.get)
+ *   and the native Mapbox Navigation SDK. User key first; else the public
+ *   render token fetched once from the cloud runtime (/api/maps/render-token)
+ *   and cached. Empty when neither exists (e.g. Glassly Cloud disabled and no
+ *   user key) — callers must fail with a clear error, never fall back to a
+ *   baked credential.
+ */
+import {SETTINGS, useSettingsStore} from "../stores/settings"
+import {cloudClientService} from "./CloudClientService"
+
+const MAPBOX_TOKENS_URL = "https://api.mapbox.com/tokens/v2"
+
+/** How long a cloud-fetched render token is trusted before re-fetching. */
+const CLOUD_TOKEN_TTL_MS = 60 * 60 * 1000
+
+let cloudRenderToken: {token: string; fetchedAt: number} | null = null
+let cloudRenderTokenInflight: Promise<string> | null = null
+
+/** The user-supplied Mapbox key from Settings > Keys ("" = not set). */
+export function userMapboxApiKey(): string {
+  const raw = useSettingsStore.getState().getSetting(SETTINGS.mapbox_api_key.key)
+  return typeof raw === "string" ? raw.trim() : ""
+}
+
+/**
+ * The token for phone-side DIRECT Mapbox REST calls: the user's key or
+ * nothing. Empty means "route the operation through the cloud maps proxy
+ * instead" (see NavigationHandlers) — the cloud-served org token is
+ * deliberately not used for direct REST from the phone.
+ */
+export function resolveMapboxAccessToken(): string {
+  return userMapboxApiKey()
+}
+
+/**
+ * The token for the consumers that cannot be proxied (map tiles, native nav
+ * SDK): user key first, else the cloud-served public render token (cached
+ * with a TTL, in-flight-deduped). "" when neither source yields one — the
+ * caller must surface that as a failure. Never fetches while Glassly Cloud is
+ * disabled: cloud-off + no user key is a deliberate hard stop.
+ */
+export async function resolveMapboxRenderToken(): Promise<string> {
+  const userKey = userMapboxApiKey()
+  if (userKey) return userKey
+  if (cloudClientService.isCloudDisabled()) return ""
+
+  const now = Date.now()
+  if (cloudRenderToken && now - cloudRenderToken.fetchedAt < CLOUD_TOKEN_TTL_MS) {
+    return cloudRenderToken.token
+  }
+  if (cloudRenderTokenInflight) return cloudRenderTokenInflight
+
+  cloudRenderTokenInflight = (async () => {
+    try {
+      const {token} = await cloudClientService.maps.renderToken()
+      const trimmed = typeof token === "string" ? token.trim() : ""
+      if (trimmed) {
+        cloudRenderToken = {token: trimmed, fetchedAt: Date.now()}
+        return trimmed
+      }
+      // A definite "this deployment serves none" — remember briefly via the
+      // same TTL slot so we don't hammer the endpoint, but as an empty token.
+      cloudRenderToken = {token: "", fetchedAt: Date.now()}
+      return ""
+    } catch (err) {
+      // Transient failure (offline, not connected yet): keep any stale cached
+      // token rather than going dark, and retry on the next call.
+      console.warn("[MapboxTokenService] cloud render-token fetch failed:", err)
+      return cloudRenderToken?.token ?? ""
+    } finally {
+      cloudRenderTokenInflight = null
+    }
+  })()
+  return cloudRenderTokenInflight
+}
+
+/** Test hook: drop the cached cloud render token. */
+export function resetCloudRenderTokenCache(): void {
+  cloudRenderToken = null
+  cloudRenderTokenInflight = null
+}
+
+/**
+ * Cheap key validation for the Keys screen: GET /tokens/v2 with the candidate
+ * token. Mapbox answers 200 `{code: "TokenValid"}` for a live token and
+ * 401 with TokenMalformed/TokenInvalid/TokenExpired otherwise — no metered
+ * API usage. Network failure → indeterminate (the caller may offer
+ * "save anyway").
+ */
+export async function validateMapboxApiKey(
+  apiKey: string,
+): Promise<{ok: true} | {ok: false; reason: "invalid_key" | "network"}> {
+  try {
+    const res = await fetch(`${MAPBOX_TOKENS_URL}?access_token=${encodeURIComponent(apiKey.trim())}`, {method: "GET"})
+    if (res.status === 401 || res.status === 403) return {ok: false, reason: "invalid_key"}
+    if (!res.ok) return {ok: false, reason: "network"}
+    const json = (await res.json()) as {code?: string}
+    return json.code === "TokenValid" ? {ok: true} : {ok: false, reason: "invalid_key"}
+  } catch {
+    return {ok: false, reason: "network"}
+  }
+}

package/src/services/MemoryCapture.ts

@@ -0,0 +1,61 @@
+/**
+ * MemoryCapture — decide what the runtime writes into the shared memory
+ * timeline as speech flows past.
+ *
+ * Kept as pure logic (no store, no settings, no singletons) so the rules can be
+ * unit-tested directly; LocalMiniappRuntime owns the side effects. Mirrors the
+ * split already used for TranscriptionRouting.
+ */
+
+/** The subset of a forwarded transcription/translation event we care about. */
+export interface CaptureCandidate {
+  text?: string
+  /** Translation events only: the words actually spoken, pre-translation. */
+  originalText?: string
+  isFinal?: boolean
+  speakerId?: string
+  /** Resolved enrolled-speaker name (on-device speaker ID), when present. */
+  resolvedSpeakerName?: string
+  transcribeLanguage?: string
+}
+
+/** What to append, once a candidate is accepted. */
+export interface CaptureDecision {
+  text: string
+  speakerId: string | null
+  resolvedSpeakerName: string | null
+  language: string | null
+}
+
+/**
+ * Extract the line to store, or null when the event isn't capturable.
+ *
+ * For translation we take `originalText` — what was SPOKEN — not the
+ * translated `text`. A MemoryEntry holds a single text + language, so the
+ * translated variant has nowhere to live; recording the source keeps
+ * translation- and transcription-driven sessions consistent, and means a
+ * translation-only conversation still lands in the timeline.
+ *
+ * Only finals are capturable: interims are superseded within the same
+ * utterance and would bloat the timeline with partial text.
+ */
+export function captureDecisionFor(stream: string, event: CaptureCandidate | null): CaptureDecision | null {
+  const isTranslation = stream.startsWith("translation:")
+  if (!isTranslation && !stream.startsWith("transcription:")) return null
+  if (!event?.isFinal) return null
+
+  // When a translation event omits originalText there is no source text to
+  // record — storing the translation under the source language would be wrong.
+  const spoken = isTranslation ? event.originalText : event.text
+  if (typeof spoken !== "string" || !spoken.trim()) return null
+
+  const language = event.transcribeLanguage
+  return {
+    text: spoken.trim(),
+    speakerId: typeof event.speakerId === "string" ? event.speakerId : null,
+    resolvedSpeakerName: typeof event.resolvedSpeakerName === "string" ? event.resolvedSpeakerName : null,
+    // "auto" is a subscription mode, not a language — an undetected source
+    // language is genuinely unknown, so store null rather than the sentinel.
+    language: typeof language === "string" && language !== "auto" ? language : null,
+  }
+}