@mentra/engine 0.1.0-beta.0

package/src/services/LocalSttFallbackCoordinator.ts

@@ -0,0 +1,147 @@
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+import {ISLAND_SETTINGS_KEYS} from "../runtime/config"
+import {useSettingsStore} from "../stores/settings"
+import {cloudClientService} from "./CloudClientService"
+import sttModelManager from "./STTModelManager"
+
+/**
+ * Decides whether on-device Sherpa STT should produce transcripts for local
+ * miniapps, instead of (or in addition to) cloud transcription. Local STT is
+ * active when the cloud is unavailable or any subscription requires
+ * `forceLocal`.
+ *
+ * Activation flips `localSttFallbackActive` in the settings store, which
+ * the bluetooth-sdk native side reads to decide whether to feed PCM into
+ * the SherpaOnnxTranscriber. Local transcripts then flow back through the
+ * `local_transcription` Bridge event; the host wires that event to
+ * `localMiniappRuntime.forwardEvent`, with this coordinator providing the
+ * subscription-active gate so we never publish transcripts to miniapps that
+ * aren't asking for them. LocalMiniappRuntime filters mixed local/cloud
+ * delivery per miniapp.
+ */
+class LocalSttFallbackCoordinator {
+  private static instance: LocalSttFallbackCoordinator
+
+  private hasTranscriptionSubscription = false
+  private hasForceLocalSubscription = false
+  private activeLanguage: string | null = null
+  /**
+   * Default to "cloud is up" so we never accidentally activate local STT before
+   * the cloud client has had a chance to start. We attach lazily on the first
+   * subscription/reconcile pass; once attached, this field reflects the real
+   * cloud-client runtime status.
+   */
+  private cloudConnected = true
+  private localActive = false
+  private cloudAdapterAttached = false
+
+  private constructor() {
+    // Reset the persisted mirror flag on boot — the in-memory state in this
+    // coordinator is the source of truth, and a stale "true" left from the
+    // previous session would cause native to feed Sherpa before any miniapp
+    // registered a subscription.
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, false)
+  }
+
+  /**
+   * Lazy-attach the engine cloud connection listener. The coordinator is
+   * constructed at module load, before engine.start() brings up the cloud
+   * client, so we defer until the first reconcile.
+   */
+  private attachCloudAdapterIfReady(): void {
+    if (this.cloudAdapterAttached) return
+    this.cloudConnected = cloudClientService.isConnected()
+    cloudClientService.onStatusChanged((status) => {
+      const connected = status.status === "connected"
+      if (this.cloudConnected === connected) return
+      this.log(`cloud connection -> ${connected ? "up" : "down"}`)
+      this.cloudConnected = connected
+      void this.reconcile()
+    })
+    this.cloudAdapterAttached = true
+  }
+
+  static getInstance(): LocalSttFallbackCoordinator {
+    if (!LocalSttFallbackCoordinator.instance) {
+      LocalSttFallbackCoordinator.instance = new LocalSttFallbackCoordinator()
+    }
+    return LocalSttFallbackCoordinator.instance
+  }
+
+  isActive(): boolean {
+    return this.localActive
+  }
+
+  getActiveLanguage(): string | null {
+    return this.activeLanguage
+  }
+
+  onSubscriptionChange(hasTranscription: boolean, language: string | null, hasForceLocal = false): void {
+    this.log(`onSubscriptionChange(hasTx=${hasTranscription}, lang=${language}, forceLocal=${hasForceLocal})`)
+    this.hasTranscriptionSubscription = hasTranscription
+    this.hasForceLocalSubscription = hasTranscription && hasForceLocal
+    this.activeLanguage = hasTranscription ? language : null
+    void this.reconcile()
+  }
+
+  /**
+   * Informational hook, retained for API stability. No longer called: the
+   * authoritative switching signal is the cloud-client adapter (local miniapps
+   * are powered only by V2). v1 cloud transcripts deliberately do NOT drive this
+   * coordinator anymore. Kept for future hysteresis if connection-state alone
+   * proves too coarse.
+   */
+  onCloudTranscript(): void {}
+
+  private async reconcile(): Promise<void> {
+    this.attachCloudAdapterIfReady()
+    const shouldBeActive = this.shouldUseLocalStt()
+    if (shouldBeActive && !this.localActive) {
+      await this.startLocalStt()
+    } else if (!shouldBeActive && this.localActive) {
+      this.stopLocalStt(this.hasTranscriptionSubscription ? "cloud reconnected" : "subscription gone")
+    }
+  }
+
+  private async startLocalStt(): Promise<void> {
+    this.log("starting local stt")
+    const modelAvailable = await sttModelManager.isModelAvailable()
+    if (!modelAvailable) {
+      this.log("local stt model is not available yet — skipping activation")
+      return
+    }
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed before start completed")
+      return
+    }
+    try {
+      // Direct btsdk call now (was a host restartTranscriber hook).
+      await BluetoothSdk.restartTranscriber()
+    } catch (err) {
+      this.log(`restartTranscriber failed: ${err}`)
+    }
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed during transcriber restart")
+      return
+    }
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, true)
+    this.localActive = true
+  }
+
+  private stopLocalStt(reason: string): void {
+    this.log(`stopping local stt: ${reason}`)
+    useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, false)
+    this.localActive = false
+  }
+
+  private shouldUseLocalStt(): boolean {
+    return this.hasTranscriptionSubscription && (this.hasForceLocalSubscription || !this.cloudConnected)
+  }
+
+  private log(msg: string): void {
+    console.log(`[LocalSttFallback] ${msg}`)
+  }
+}
+
+const localSttFallbackCoordinator = LocalSttFallbackCoordinator.getInstance()
+export default localSttFallbackCoordinator

package/src/services/MentraJSCrashController.ts

@@ -0,0 +1,189 @@
+/**
+ * MentraJSCrashController — crash recovery state machine for the
+ * per-miniapp JSContexts owned by MentraJSRouter.
+ *
+ *   RUNNING
+ *     │  crash detected (uncaught exception, native bridge throw,
+ *     │   ping miss, soft-watchdog kill, etc.)
+ *     ▼
+ *   CRASHED ──── respawn allowed? ──── no ──► CRASHLOOP_DISABLED
+ *     │                                          (persistent banner)
+ *     ▼ yes
+ *   BACKOFF(2s → 8s → 30s)
+ *     │  timer fires
+ *     ▼
+ *   RUNNING (and stays here on a clean 60s window;
+ *            otherwise crash count increments)
+ *
+ * Retry policy: zero retries by default — the first crash escalates
+ * straight to CRASHLOOP_DISABLED, matching how Android / iOS treat crashed
+ * apps. Callers that prefer auto-respawn opt in by passing
+ * `{maxRetries: 3, backoffMs: [2_000, 8_000, 30_000]}`; the BACKOFF state
+ * machine and exponential delays are implemented and unit-tested for that
+ * case.
+ *
+ * The controller is purely a policy engine — it doesn't actually call
+ * mentraJsSpawn / mentraJsKill. Callers (MentraJSRouter) read the
+ * Outcome returned from onCrash() and drive the host accordingly. This
+ * makes the controller unit-testable with synthetic timer.
+ */
+
+export type CrashState =
+  | {kind: "RUNNING"; sinceMs: number}
+  | {kind: "CRASHED"; reason: string; crashedAtMs: number}
+  | {kind: "BACKOFF"; until: number; attempt: number; reason: string}
+  | {kind: "CRASHLOOP_DISABLED"; sinceMs: number; reason: string}
+
+export interface CrashOutcome {
+  /** Caller must call `respawn(packageName)` after this delay (ms from now). */
+  scheduleRespawnAfterMs?: number
+  /** Caller should surface "X restarted" toast (2nd-within-5m crash). */
+  showRestartToast?: boolean
+  /** Caller should surface persistent "crashloop disabled" banner. */
+  surfaceCrashloopBanner?: boolean
+  /** Caller may push the crash event to dev portal / Sentry as warn-level. */
+  sentryLevel?: "info" | "warning" | "error"
+}
+
+export interface CrashControllerOptions {
+  /** Override the wall clock for tests. Defaults to Date.now. */
+  now?: () => number
+  /** Backoff sequence in ms. Defaults to [2_000, 8_000, 30_000]. */
+  backoffMs?: number[]
+  /** Window in ms — N crashes inside this window escalates. Defaults to 5min. */
+  crashWindowMs?: number
+  /** Max retries before CRASHLOOP_DISABLED. Defaults to 0 — first crash
+   *  escalates immediately, matching Android / iOS app conventions. */
+  maxRetries?: number
+  /** Clean-uptime threshold that resets the retry counter. Defaults to 60s. */
+  cleanUptimeResetMs?: number
+}
+
+interface PackageState {
+  state: CrashState
+  /** Crash timestamps inside the current window (sorted ascending). */
+  crashesInWindow: number[]
+  /** Total respawns since the package was first registered (for telemetry). */
+  totalRespawns: number
+}
+
+const DEFAULT_BACKOFFS = [2_000, 8_000, 30_000]
+const DEFAULT_WINDOW_MS = 5 * 60_000
+const DEFAULT_MAX_RETRIES = 0
+const DEFAULT_CLEAN_UPTIME_RESET_MS = 60_000
+
+export class MentraJSCrashController {
+  private readonly now: () => number
+  private readonly backoffMs: number[]
+  private readonly crashWindowMs: number
+  private readonly maxRetries: number
+  private readonly cleanUptimeResetMs: number
+  private readonly packages: Map<string, PackageState> = new Map()
+
+  constructor(opts: CrashControllerOptions = {}) {
+    this.now = opts.now ?? (() => Date.now())
+    this.backoffMs = opts.backoffMs ?? DEFAULT_BACKOFFS
+    this.crashWindowMs = opts.crashWindowMs ?? DEFAULT_WINDOW_MS
+    this.maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES
+    this.cleanUptimeResetMs = opts.cleanUptimeResetMs ?? DEFAULT_CLEAN_UPTIME_RESET_MS
+  }
+
+  /** Tracking starts when MentraJSRouter spawns a fresh JSContext. */
+  onSpawn(packageName: string): void {
+    const at = this.now()
+    const prev = this.packages.get(packageName)
+    if (prev) {
+      // Carry forward the crash history; a respawn doesn't reset retry
+      // budget unless the previous RUNNING state survived >= cleanUptimeResetMs.
+      prev.state = {kind: "RUNNING", sinceMs: at}
+      prev.totalRespawns += 1
+    } else {
+      this.packages.set(packageName, {
+        state: {kind: "RUNNING", sinceMs: at},
+        crashesInWindow: [],
+        totalRespawns: 0,
+      })
+    }
+  }
+
+  /** Called when MentraJSRouter explicitly tears down (user-initiated). */
+  onKill(packageName: string): void {
+    this.packages.delete(packageName)
+  }
+
+  /**
+   * Record a crash and return the policy outcome. Caller is responsible for
+   * scheduling the respawn (timer ownership belongs at the orchestration
+   * layer that holds the host context, not here).
+   */
+  onCrash(packageName: string, reason: string): CrashOutcome {
+    const at = this.now()
+    const entry = this.packages.get(packageName) ?? {
+      state: {kind: "RUNNING", sinceMs: at},
+      crashesInWindow: [],
+      totalRespawns: 0,
+    }
+
+    // If the previous RUNNING window was long enough, reset the retry
+    // counter — a crash after a stable session shouldn't burn budget.
+    if (entry.state.kind === "RUNNING") {
+      const uptime = at - entry.state.sinceMs
+      if (uptime >= this.cleanUptimeResetMs) {
+        entry.crashesInWindow = []
+      }
+    }
+
+    // Drop crashes that fell out of the window before counting the new one.
+    const cutoff = at - this.crashWindowMs
+    entry.crashesInWindow = entry.crashesInWindow.filter((t) => t >= cutoff)
+    entry.crashesInWindow.push(at)
+    this.packages.set(packageName, entry)
+
+    const crashCount = entry.crashesInWindow.length
+
+    if (crashCount > this.maxRetries) {
+      entry.state = {kind: "CRASHLOOP_DISABLED", sinceMs: at, reason}
+      return {
+        surfaceCrashloopBanner: true,
+        sentryLevel: "error",
+      }
+    }
+
+    const attempt = crashCount - 1
+    const delayMs = this.backoffMs[Math.min(attempt, this.backoffMs.length - 1)]!
+    entry.state = {kind: "BACKOFF", until: at + delayMs, attempt: crashCount, reason}
+
+    return {
+      scheduleRespawnAfterMs: delayMs,
+      showRestartToast: crashCount >= 2,
+      sentryLevel: crashCount === 1 ? "info" : "warning",
+    }
+  }
+
+  /** Inspect current state — useful for tests and diagnostics. */
+  stateFor(packageName: string): CrashState | null {
+    return this.packages.get(packageName)?.state ?? null
+  }
+
+  /** Total respawns since registration — surfaced to Sentry as a counter. */
+  respawnCount(packageName: string): number {
+    return this.packages.get(packageName)?.totalRespawns ?? 0
+  }
+
+  /** Currently-disabled packages. Host UI uses this to show banners. */
+  disabledPackages(): string[] {
+    const out: string[] = []
+    for (const [name, entry] of this.packages) {
+      if (entry.state.kind === "CRASHLOOP_DISABLED") out.push(name)
+    }
+    return out
+  }
+
+  /** Manually re-enable a disabled package (user "Try again" tap). */
+  reset(packageName: string): void {
+    const entry = this.packages.get(packageName)
+    if (!entry) return
+    entry.crashesInWindow = []
+    entry.state = {kind: "RUNNING", sinceMs: this.now()}
+  }
+}

package/src/services/MentraJSCrashloopReportService.ts

@@ -0,0 +1,66 @@
+import {submitAutomaticReport} from "../facades/reports"
+import {useAppStatusStore} from "../stores/apps"
+import {
+  logAutomaticReportSubmissionStatus,
+  toAutomaticReportSubmissionStatus,
+  type AutomaticReportSubmissionStatus,
+} from "./AutomaticReportResult"
+import {getMiniappEngine} from "./MiniappEngine"
+import {islandNotifications, type IslandNotification} from "./NotificationsEmitter"
+
+const LOG_TAG = "MentraJSCrashloopReport"
+
+let unsubscribe: (() => void) | null = null
+
+export async function submitMentraJSCrashloopReport(params: {
+  packageName: string
+  reason: string
+  lastLogLines?: string[]
+}): Promise<AutomaticReportSubmissionStatus> {
+  const {packageName, reason, lastLogLines = []} = params
+  const app = useAppStatusStore.getState().apps.find((a) => a.packageName === packageName)
+  const appName = app?.name ?? packageName
+  const result = await submitAutomaticReport({
+    kind: "automatic",
+    trigger: {
+      type: "automatic",
+      source: "miniapp_crashloop",
+      reason: "mentrajs_crashloop_disabled",
+      sourceAppletPackageName: packageName,
+      sourceAppletName: appName,
+    },
+    report: {
+      expectedBehavior: `${appName} should run without crashing.`,
+      actualBehavior: JSON.stringify({reason, lastLogLines}, null, 2),
+      systemPriority: "critical",
+    },
+    throttleKey: `mentrajs_crashloop:${packageName}`,
+  })
+
+  const status = toAutomaticReportSubmissionStatus(result)
+  logAutomaticReportSubmissionStatus(LOG_TAG, status, packageName)
+  return status
+}
+
+function handleNotification(notification: IslandNotification): void {
+  if (notification.kind !== "miniapp_crashloop" || !notification.packageName) return
+  const packageName = notification.packageName
+  const lastLogLines = getMiniappEngine()?.router.logRing.snapshot(packageName) ?? []
+  void submitMentraJSCrashloopReport({
+    packageName,
+    reason: notification.reason,
+    lastLogLines,
+  }).catch((error) => {
+    console.error(`[${LOG_TAG}] Unexpected error:`, error)
+  })
+}
+
+export function startMentraJSCrashloopReportService(): void {
+  if (unsubscribe) return
+  unsubscribe = islandNotifications.subscribe(handleNotification)
+}
+
+export function stopMentraJSCrashloopReportService(): void {
+  unsubscribe?.()
+  unsubscribe = null
+}

package/src/services/MentraJSLogPipeline.ts

@@ -0,0 +1,166 @@
+/**
+ * MentraJSLogPipeline — the three log-side utilities that
+ * MentraJSRouter's `__log` handler runs every miniapp log line through:
+ *
+ *   1. redactSecrets — walks a value (string / array / object) and
+ *      replaces anything matching the secret-key regex
+ *      (token|password|secret|auth|bearer|key|api[_-]?key) with
+ *      "[REDACTED]". Conservative: over-redaction is preferable to
+ *      leaking a real secret into Sentry breadcrumbs.
+ *
+ *   2. MentraJSLogThrottle — token-bucket rate limiter, 100 lines/min
+ *      sustained with a 500-line burst, per miniapp. Excess is dropped
+ *      and surfaced once as a `[throttled N]` summary. Stops a
+ *      misbehaving miniapp from drowning the host's log stream.
+ *
+ *   3. MentraJSLogRingBuffer — fixed-size circular buffer (default 200)
+ *      holding the most recent log lines per miniapp. Dumped into the
+ *      crash report so post-mortem tooling sees the miniapp's last words.
+ *
+ * All three are pure (no React Native imports) and unit-tested in
+ * MentraJSLogPipeline.test.ts.
+ */
+
+const SECRET_KEY_PATTERN = /\b(token|password|secret|auth|bearer|key|api[_-]?key)\b/i
+
+/**
+ * Redact obvious secret-like strings from a log line. Walks the JSON
+ * argument list, replacing any value that looks like a secret with
+ * `[REDACTED]`. Conservative — false negatives are fine, false positives
+ * are bad (devs lose visibility into their own data).
+ *
+ * Heuristics:
+ *   1. Top-level strings matching SECRET_KEY_PATTERN get the entire value
+ *      redacted (e.g. `console.log("token=abc123")` → `[REDACTED]`).
+ *   2. Object keys matching the pattern get their values replaced.
+ *   3. Arrays are walked element-by-element.
+ *
+ * Returns a new structure — never mutates the input.
+ */
+export function redactSecrets(value: unknown): unknown {
+  if (typeof value === "string") {
+    return SECRET_KEY_PATTERN.test(value) ? "[REDACTED]" : value
+  }
+  if (Array.isArray(value)) {
+    return value.map((v) => redactSecrets(v))
+  }
+  if (value && typeof value === "object") {
+    const out: Record<string, unknown> = {}
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      if (SECRET_KEY_PATTERN.test(k)) {
+        out[k] = "[REDACTED]"
+      } else {
+        out[k] = redactSecrets(v)
+      }
+    }
+    return out
+  }
+  return value
+}
+
+/**
+ * Token bucket throttler. Per-package: 100 logs/min sustained with a 500
+ * burst. Excess lines are dropped; the first drop in each refill window
+ * emits a `[throttled N]` synthetic line so the dev knows messages
+ * were lost.
+ */
+export interface ThrottleOptions {
+  /** Tokens added per second. Default 100/60 = ~1.67. */
+  tokensPerSecond?: number
+  /** Bucket capacity. Default 500. */
+  bucketCapacity?: number
+  /** Clock override for tests. */
+  now?: () => number
+}
+
+interface PackageBucket {
+  tokens: number
+  lastRefillAtMs: number
+  /** Drops accumulated since the last `[throttled N]` synthetic line. */
+  pendingDrops: number
+}
+
+export class MentraJSLogThrottle {
+  private readonly tokensPerSecond: number
+  private readonly bucketCapacity: number
+  private readonly now: () => number
+  private readonly buckets: Map<string, PackageBucket> = new Map()
+
+  constructor(opts: ThrottleOptions = {}) {
+    this.tokensPerSecond = opts.tokensPerSecond ?? 100 / 60
+    this.bucketCapacity = opts.bucketCapacity ?? 500
+    this.now = opts.now ?? (() => Date.now())
+  }
+
+  /**
+   * Try to consume one token for the named package. Returns either:
+   *   - `{allowed: true}` — caller emits the original log line.
+   *   - `{allowed: false, throttledLine}` — caller may emit the
+   *     synthetic throttled line (the first drop in each window).
+   *     If `throttledLine` is undefined the caller drops silently
+   *     (subsequent drops in the same window).
+   */
+  consume(packageName: string): {allowed: true} | {allowed: false; throttledLine?: string} {
+    const at = this.now()
+    let bucket = this.buckets.get(packageName)
+    if (!bucket) {
+      bucket = {
+        tokens: this.bucketCapacity,
+        lastRefillAtMs: at,
+        pendingDrops: 0,
+      }
+      this.buckets.set(packageName, bucket)
+    } else {
+      const elapsedSec = (at - bucket.lastRefillAtMs) / 1000
+      bucket.tokens = Math.min(this.bucketCapacity, bucket.tokens + elapsedSec * this.tokensPerSecond)
+      bucket.lastRefillAtMs = at
+    }
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1
+      if (bucket.pendingDrops > 0) {
+        const dropped = bucket.pendingDrops
+        bucket.pendingDrops = 0
+        return {allowed: false, throttledLine: `[throttled ${dropped}]`}
+      }
+      return {allowed: true}
+    }
+    bucket.pendingDrops += 1
+    return {allowed: false}
+  }
+
+  /** Test/dev — reset all buckets. */
+  resetForTests(): void {
+    this.buckets.clear()
+  }
+}
+
+/**
+ * In-memory ring buffer holding the last N lines per package. Useful as a
+ * "last words" context attached to crash reports.
+ */
+export class MentraJSLogRingBuffer {
+  private readonly capacity: number
+  private readonly buffers: Map<string, string[]> = new Map()
+
+  constructor(capacityPerPackage = 200) {
+    this.capacity = capacityPerPackage
+  }
+
+  push(packageName: string, line: string): void {
+    let buf = this.buffers.get(packageName)
+    if (!buf) {
+      buf = []
+      this.buffers.set(packageName, buf)
+    }
+    buf.push(line)
+    if (buf.length > this.capacity) buf.shift()
+  }
+
+  snapshot(packageName: string): string[] {
+    return [...(this.buffers.get(packageName) ?? [])]
+  }
+
+  clear(packageName: string): void {
+    this.buffers.delete(packageName)
+  }
+}