@mentra/miniapp 3.3.0 → 3.4.0-beta.1237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +5 -5
  2. package/dist/background/index.d.ts +5 -2
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js.map +1 -1
  5. package/dist/index.d.ts +5 -2
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/modules/blob.d.ts +2 -1
  9. package/dist/modules/blob.d.ts.map +1 -1
  10. package/dist/modules/blob.js +2 -1
  11. package/dist/modules/blob.js.map +1 -1
  12. package/dist/modules/events.d.ts +16 -6
  13. package/dist/modules/events.d.ts.map +1 -1
  14. package/dist/modules/events.js +53 -47
  15. package/dist/modules/events.js.map +1 -1
  16. package/dist/modules/mic.d.ts +26 -4
  17. package/dist/modules/mic.d.ts.map +1 -1
  18. package/dist/modules/mic.js +13 -4
  19. package/dist/modules/mic.js.map +1 -1
  20. package/dist/modules/miniapps.d.ts +44 -57
  21. package/dist/modules/miniapps.d.ts.map +1 -1
  22. package/dist/modules/miniapps.js +21 -24
  23. package/dist/modules/miniapps.js.map +1 -1
  24. package/dist/modules/speaker.d.ts +41 -10
  25. package/dist/modules/speaker.d.ts.map +1 -1
  26. package/dist/modules/speaker.js +17 -6
  27. package/dist/modules/speaker.js.map +1 -1
  28. package/dist/modules/speechProcessing.d.ts +29 -0
  29. package/dist/modules/speechProcessing.d.ts.map +1 -0
  30. package/dist/modules/speechProcessing.js +22 -0
  31. package/dist/modules/speechProcessing.js.map +1 -0
  32. package/dist/modules/system.d.ts +8 -3
  33. package/dist/modules/system.d.ts.map +1 -1
  34. package/dist/modules/system.js +1 -1
  35. package/dist/modules/system.js.map +1 -1
  36. package/dist/modules/telemetry.d.ts +25 -0
  37. package/dist/modules/telemetry.d.ts.map +1 -0
  38. package/dist/modules/telemetry.js +61 -0
  39. package/dist/modules/telemetry.js.map +1 -0
  40. package/dist/modules/transcription.d.ts +70 -3
  41. package/dist/modules/transcription.d.ts.map +1 -1
  42. package/dist/modules/transcription.js +68 -2
  43. package/dist/modules/transcription.js.map +1 -1
  44. package/dist/protocol.d.ts +29 -15
  45. package/dist/protocol.d.ts.map +1 -1
  46. package/dist/protocol.js +30 -15
  47. package/dist/protocol.js.map +1 -1
  48. package/dist/react/MentraProvider.d.ts +13 -1
  49. package/dist/react/MentraProvider.d.ts.map +1 -1
  50. package/dist/react/MentraProvider.js +31 -1
  51. package/dist/react/MentraProvider.js.map +1 -1
  52. package/dist/session.d.ts +16 -3
  53. package/dist/session.d.ts.map +1 -1
  54. package/dist/session.js +6 -13
  55. package/dist/session.js.map +1 -1
  56. package/dist/ui/index.d.ts +7 -4
  57. package/dist/ui/index.d.ts.map +1 -1
  58. package/dist/ui/index.js.map +1 -1
  59. package/package.json +2 -2
  60. package/src/background/index.ts +6 -3
  61. package/src/index.ts +11 -3
  62. package/src/modules/blob.ts +2 -1
  63. package/src/modules/events.ts +65 -51
  64. package/src/modules/mic.ts +30 -4
  65. package/src/modules/miniapps.ts +59 -71
  66. package/src/modules/speaker.ts +56 -12
  67. package/src/modules/speechProcessing.ts +42 -0
  68. package/src/modules/system.ts +8 -3
  69. package/src/modules/telemetry.ts +62 -0
  70. package/src/modules/transcription.ts +106 -9
  71. package/src/protocol.ts +32 -16
  72. package/src/react/MentraProvider.tsx +23 -2
  73. package/src/session.ts +20 -17
  74. package/src/ui/index.ts +7 -4
@@ -9,13 +9,17 @@ import { MiniappSession } from "../session"
9
9
 
10
10
  export interface ShareOptions {
11
11
  text?: string
12
+ /**
13
+ * An http(s) link, or the `file://` uri of one of this miniapp's blobs
14
+ * (`session.blob`). The host refuses any other file or scheme.
15
+ */
12
16
  url?: string
13
17
  title?: string
14
18
  /** Base64-encoded file data for file sharing. */
15
19
  base64?: string
16
20
  /** MIME type when sharing base64 data. */
17
21
  mimeType?: string
18
- /** Filename when sharing base64 data. */
22
+ /** File name when sharing base64 data: one name; path separators become `_`. */
19
23
  filename?: string
20
24
  }
21
25
 
@@ -25,9 +29,10 @@ export interface ShareResult {
25
29
  }
26
30
 
27
31
  export interface DownloadOptions {
28
- /** URL to download from, OR base64 data. */
32
+ /** An http: or https: URL to download from, OR base64 data. */
29
33
  url?: string
30
34
  base64?: string
35
+ /** One file name; path separators become `_`. */
31
36
  filename?: string
32
37
  mimeType?: string
33
38
  }
@@ -58,7 +63,7 @@ export class SystemModule {
58
63
  return result ?? { success: false }
59
64
  }
60
65
 
61
- /** Open a URL in the system browser. Blocks dangerous schemes (javascript:, file:). */
66
+ /** Open a URL in the system browser. Never opens local files or scripts (file:, content:, javascript:, data:). */
62
67
  openUrl(url: string): void {
63
68
  this.session.sendOneShot({
64
69
  type: MiniappRequestType.OPEN_URL,
@@ -0,0 +1,62 @@
1
+ /**
2
+ * @fileoverview TelemetryModule (`session.telemetry`): a miniapp's own analytics, shown in its organization's Fleet
3
+ * Management.
4
+ *
5
+ * `capture(event, properties?)` sends one fire-and-forget message to the Mentra App and returns nothing. It never throws
6
+ * and never waits. The SDK checks nothing: the Mentra App attributes the event to this miniapp and applies the limits
7
+ * (names, values, 25 properties, the rate), and on a Mentra App without the `telemetry` host feature it does nothing.
8
+ * Before the handshake says which, it holds the latest 100 captures and sends them once it knows.
9
+ */
10
+
11
+ import {MiniappRequestType} from "../protocol"
12
+ import type {MiniappSession} from "../session"
13
+
14
+ export type TelemetryValue = string | number | boolean
15
+ export type TelemetryProperties = Record<string, TelemetryValue>
16
+
17
+ /** Captures held until the handshake says whether the Mentra App takes them. */
18
+ const MAX_HELD = 100
19
+
20
+ export class TelemetryModule {
21
+ private held: object[] = []
22
+ private waiting = false
23
+
24
+ constructor(private readonly session: MiniappSession) {}
25
+
26
+ /**
27
+ * Captures one event, in the style of PostHog: `session.telemetry.capture("call_ended", {platform: "teams",
28
+ * duration_s: 312})`. Names are lowercase letters, digits and underscores; values are flat. Never personal data: no
29
+ * names, emails, phone numbers, typed text or content.
30
+ */
31
+ capture(event: string, properties?: TelemetryProperties): void {
32
+ try {
33
+ const message = {type: MiniappRequestType.TELEMETRY_CAPTURE, event, properties: properties ?? {}}
34
+ if (this.session.handshakeDone) {
35
+ if (this.session.hostFeatures?.telemetry === true) this.session.sendOneShot(message)
36
+ return
37
+ }
38
+ if (this.held.length >= MAX_HELD) this.held.shift()
39
+ this.held.push(message)
40
+ if (!this.waiting) {
41
+ this.waiting = true
42
+ const off = this.session.on("ready", () => {
43
+ off()
44
+ this.waiting = false
45
+ const held = this.held.splice(0)
46
+ if (this.session.hostFeatures?.telemetry !== true) return
47
+ for (const one of held) {
48
+ // Each on its own: one capture that cannot be sent never takes the others, or the session's other
49
+ // listeners, down with it.
50
+ try {
51
+ this.session.sendOneShot(one)
52
+ } catch {
53
+ // Lost.
54
+ }
55
+ }
56
+ })
57
+ }
58
+ } catch {
59
+ // Analytics never break the miniapp: a value that cannot be sent is lost.
60
+ }
61
+ }
62
+ }
@@ -6,7 +6,10 @@
6
6
  * session.transcription.on(handler) // auto-detect
7
7
  * session.transcription.forLanguage("en-US", handler) // single language
8
8
  * session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
9
+ * session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
9
10
  * session.transcription.configure({languageHints, vocabulary, diarization})
11
+ * session.transcription.onModelStatus(handler) // on-device model state
12
+ * session.transcription.finishUtterance() // end the on-device line now
10
13
  * session.transcription.stop() // tear down all
11
14
  *
12
15
  * Hoisted to top-level (vs. nested under `session.mic`) because transcription
@@ -23,6 +26,7 @@ import {MiniappSession} from "../session"
23
26
  import type {TranscriptionData, UnsubscribeFn} from "./events"
24
27
  import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
25
28
  import type {LanguageHint, TranscriptionLanguage} from "./languages"
29
+ import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
26
30
 
27
31
  /** Configuration for cloud-side transcription behavior. */
28
32
  export interface TranscriptionConfig {
@@ -41,16 +45,56 @@ export interface TranscriptionConfig {
41
45
 
42
46
  export interface TranscriptionOptions {
43
47
  /**
44
- * Use only on-device STT for this listener; never deliver cloud transcripts.
45
- * Other listeners on the same stream keep their own routing choice. Requires
46
- * a downloaded local STT model.
48
+ * Where this listener's speech is transcribed. Default `"preferCloud"`.
49
+ *
50
+ * - `"preferCloud"`: the cloud; the phone's on-device model only while the
51
+ * cloud is unreachable.
52
+ * - `"preferLocal"`: the on-device model while it runs in this listener's
53
+ * language, otherwise the cloud. `on()` follows the model's language,
54
+ * unless `configure({languageHints})` names one the model can't
55
+ * transcribe; then it stays on the cloud.
56
+ * - `"localOnly"`: on-device only; never the cloud.
57
+ *
58
+ * The phone runs one on-device model, the one picked on its Speech page, and
59
+ * downloads it when a `preferLocal` or `localOnly` listener needs it; follow
60
+ * it with `onModelStatus`. A listener moves between the cloud and on-device
61
+ * during a pause in speech, once both have finished with what was said
62
+ * before it, so it never misses or repeats speech; `TranscriptionData.source`
63
+ * says where each result came from. Each listener routes on its own: other
64
+ * listeners on the same stream keep their choice.
47
65
  */
66
+ processing?: SpeechProcessing
67
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
48
68
  forceLocal?: boolean
49
69
  }
50
70
 
71
+ /**
72
+ * State of the on-device speech-to-text model behind this miniapp's on-device
73
+ * transcription. The phone runs the one model picked on its Speech page for
74
+ * every miniapp.
75
+ */
76
+ export interface TranscriptionModelStatus {
77
+ /** BCP-47 language of the picked model, e.g. "en-US". */
78
+ language: string
79
+ /**
80
+ * - `downloading`: the model is being downloaded (`progress` is 0–100).
81
+ * - `loading`: the model is loading into memory.
82
+ * - `ready`: on-device transcripts are flowing.
83
+ * - `unavailable`: no on-device transcription right now; `reason` says why
84
+ * (for example no `preferLocal` or `localOnly` listener, a failed
85
+ * download, or a listener in a language other than the picked model's).
86
+ */
87
+ state: "downloading" | "loading" | "ready" | "unavailable"
88
+ progress?: number
89
+ reason?: string
90
+ }
91
+
51
92
  export class TranscriptionModule {
52
93
  private readonly unsubs = new Set<UnsubscribeFn>()
53
94
  private currentConfig: TranscriptionConfig | null = null
95
+ private readonly modelStatusHandlers = new Set<(status: TranscriptionModelStatus) => void>()
96
+ private modelStatusUnsub: UnsubscribeFn | null = null
97
+ private modelStatus: TranscriptionModelStatus | null = null
54
98
 
55
99
  constructor(private readonly session: MiniappSession) {}
56
100
 
@@ -63,8 +107,11 @@ export class TranscriptionModule {
63
107
  * event) preserves existing semantics.
64
108
  */
65
109
  on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
110
+ const processing = resolveSpeechProcessing(options, "transcription.on(options)")
66
111
  return this.track(
67
- this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, options),
112
+ this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, {
113
+ processing,
114
+ }),
68
115
  )
69
116
  }
70
117
 
@@ -90,16 +137,15 @@ export class TranscriptionModule {
90
137
  const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
91
138
  requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
92
139
  )
140
+ const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)")
93
141
  if (langs.length === 0) return () => {}
94
142
 
95
143
  const unsubs: UnsubscribeFn[] = []
96
144
  for (const lang of langs) {
97
145
  unsubs.push(
98
- this.session._subscribe(
99
- `${MiniappStreamType.TRANSCRIPTION}:${lang}`,
100
- handler as (data: unknown) => void,
101
- options,
102
- ),
146
+ this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void, {
147
+ processing,
148
+ }),
103
149
  )
104
150
  }
105
151
  const combined: UnsubscribeFn = () => {
@@ -141,6 +187,57 @@ export class TranscriptionModule {
141
187
  .then(() => undefined)
142
188
  }
143
189
 
190
+ /**
191
+ * Follow the on-device speech-to-text model's state. You get the current
192
+ * state right away, then every change. Use it to show "downloading 40%" or
193
+ * to wait for `ready` before relying on on-device transcripts.
194
+ */
195
+ onModelStatus(handler: (status: TranscriptionModelStatus) => void): UnsubscribeFn {
196
+ this.modelStatusHandlers.add(handler)
197
+ if (!this.modelStatusUnsub) {
198
+ // One stream subscription per module; the host sends its current state on subscribe.
199
+ this.modelStatusUnsub = this.session._subscribe(MiniappStreamType.TRANSCRIPTION_MODEL_STATUS, (data) => {
200
+ const status = data as TranscriptionModelStatus
201
+ this.modelStatus = status
202
+ for (const listener of [...this.modelStatusHandlers]) listener(status)
203
+ })
204
+ } else if (this.modelStatus) {
205
+ // Later handlers get the last state locally, since the host only replays on subscribe.
206
+ const current = this.modelStatus
207
+ queueMicrotask(() => {
208
+ if (this.modelStatusHandlers.has(handler) && this.modelStatus === current) handler(current)
209
+ })
210
+ }
211
+ return this.track(() => {
212
+ this.modelStatusHandlers.delete(handler)
213
+ if (this.modelStatusHandlers.size > 0) return
214
+ this.modelStatusUnsub?.()
215
+ this.modelStatusUnsub = null
216
+ this.modelStatus = null
217
+ })
218
+ }
219
+
220
+ /**
221
+ * End the current on-device utterance now, instead of waiting for the
222
+ * recognizer's own end-of-speech pause. Call it when your own end-of-turn
223
+ * rule fires (for example after `mic.onVoiceActivity` reports enough
224
+ * silence).
225
+ *
226
+ * Resolves with the final text once that final transcript has been
227
+ * delivered to your on-device handlers, or `""` when nothing was pending.
228
+ * Resolves `null` when no on-device recognizer is running or the picked
229
+ * model can't finish early (models that transcribe in fixed chunks); their
230
+ * final text still arrives on its own.
231
+ *
232
+ * Applies only to on-device transcription and needs a `localOnly` listener
233
+ * or one that is getting on-device text right now. The phone runs one
234
+ * recognizer for every miniapp, so this ends the current line for every
235
+ * on-device subscriber, Captions included.
236
+ */
237
+ finishUtterance(): Promise<string | null> {
238
+ return this.session.sendRequest<string | null>({type: MiniappRequestType.TRANSCRIPTION_FINISH})
239
+ }
240
+
144
241
  /** Tear down every transcription subscription this module owns. */
145
242
  stop(): void {
146
243
  for (const u of this.unsubs) {
package/src/protocol.ts CHANGED
@@ -26,7 +26,16 @@ export enum MiniappRequestType {
26
26
  /** Request a fresh miniapp-scoped backend auth token. */
27
27
  AUTH_REFRESH = "miniapp_auth_refresh",
28
28
 
29
- /** Update the set of streams the miniapp is subscribed to. */
29
+ /**
30
+ * Update the set of streams the miniapp is subscribed to. `subscriptions`
31
+ * lists stream names; a transcription stream is `{stream, processing}`, one
32
+ * entry per processing its listeners ask for, and `transcriptionListeners`
33
+ * lists every transcription listener as `{id, stream, processing}`. Both
34
+ * also carry `forceLocal: true` for `localOnly`, the form hosts before
35
+ * `processing` read. Hosts accept that older form,
36
+ * `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
37
+ * `processing`.
38
+ */
30
39
  SUBSCRIBE = "miniapp_subscribe",
31
40
 
32
41
  /**
@@ -45,7 +54,7 @@ export enum MiniappRequestType {
45
54
  /** Stop any audio playback this miniapp initiated. */
46
55
  STOP_AUDIO = "miniapp_stop_audio",
47
56
 
48
- /** Speak text via cloud TTS — phone constructs the URL. */
57
+ /** Speak text via cloud or on-device TTS, as its `processing` says. */
49
58
  SPEAK = "miniapp_speak",
50
59
 
51
60
  /**
@@ -189,6 +198,17 @@ export enum MiniappRequestType {
189
198
  * so future reconnect logic can re-send.
190
199
  */
191
200
  TRANSCRIPTION_CONFIG = "miniapp_transcription_config",
201
+ /**
202
+ * End the current on-device utterance now (transcription.finishUtterance).
203
+ * Result: the final text, "" when nothing was pending, or null when the
204
+ * model can't finish early.
205
+ */
206
+ TRANSCRIPTION_FINISH = "miniapp_transcription_finish",
207
+
208
+ // ----- Telemetry -----
209
+
210
+ /** A miniapp's own analytics event (`session.telemetry.capture`). One-shot; hosts with `hostFeatures.telemetry`. */
211
+ TELEMETRY_CAPTURE = "miniapp_telemetry_capture",
192
212
 
193
213
  // ----- Deferred in v1 -----
194
214
 
@@ -225,11 +245,11 @@ export enum MiniappRequestType {
225
245
  MINIAPPS_START = "miniapp_apps_start",
226
246
  /** Stop another miniapp. */
227
247
  MINIAPPS_STOP = "miniapp_apps_stop",
228
- /** Install/update an approved bundle. Bundled host-trusted Stores only. */
248
+ /** Install the miniapp at a miniapp URL. System-list miniapps only. */
229
249
  MINIAPPS_INSTALL = "miniapp_apps_install",
230
250
  /** Preflight a release's host/SDK/hardware requirements without downloading it. */
231
251
  MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
232
- /** Uninstall a package owned by the calling Store. Bundled host-trusted Stores only. */
252
+ /** Uninstall a miniapp installed from a miniapp URL. System-list miniapps only. */
233
253
  MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
234
254
  /** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
235
255
  ACTION_INVOKE = "miniapp_action_invoke",
@@ -294,9 +314,6 @@ export enum MiniappResponseType {
294
314
  /** Push: host color scheme (light/dark) changed. */
295
315
  COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
296
316
 
297
- /** Push: a Store installation moved to another host-side phase. */
298
- MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
299
-
300
317
  /**
301
318
  * Push: speaker playback state transitioned. Per-miniapp; the runtime sends
302
319
  * to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
@@ -391,6 +408,10 @@ export enum MiniappStreamType {
391
408
  // Speech / audio (cloud or local)
392
409
  TRANSCRIPTION = "transcription", // language variant: "transcription:en-US"
393
410
  TRANSLATION = "translation", // language variant: "translation:en-US"
411
+ /** State of the on-device speech-to-text model (session.transcription.onModelStatus). */
412
+ TRANSCRIPTION_MODEL_STATUS = "transcription_model_status",
413
+ /** On-device "Hey Mentra" keyword spotter (session.mic.onWakeWord). */
414
+ WAKE_WORD = "wake_word",
394
415
  AUDIO_CHUNK = "audio_chunk",
395
416
  VAD = "vad",
396
417
 
@@ -444,7 +465,7 @@ export enum MiniappErrorCode {
444
465
  TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
445
466
  TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
446
467
 
447
- /** `speak({forceLocal: true})` but the on-device offline TTS model isn't downloaded/ready. */
468
+ /** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
448
469
  TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
449
470
 
450
471
  /** Not connected / pre-ACK and transport closed. */
@@ -459,21 +480,16 @@ export enum MiniappErrorCode {
459
480
  // ----- Inter-miniapp interop -----
460
481
  /**
461
482
  * Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
462
- * setWifiAdbState) reject with this code.
483
+ * setWifiAdbState) reject with this code, as installs do from any miniapp off
484
+ * the system list. Also: an uninstall of a miniapp that isn't a URL install.
463
485
  */
464
486
  NOT_PERMITTED = "NOT_PERMITTED",
465
487
  /** Target miniapp is not installed. */
466
488
  APP_NOT_FOUND = "APP_NOT_FOUND",
467
489
  /** Target miniapp is incompatible with the connected glasses. */
468
490
  APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
469
- /** Bundle download, validation, or activation failed. */
491
+ /** The miniapp didn't install; the error's `reason` says why (`InstallMiniappFailureReason`). */
470
492
  INSTALL_FAILED = "INSTALL_FAILED",
471
- /** Another Store installation is already in progress for the same package. */
472
- INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
473
- /** An automatic update was deferred because the target is running or starting. */
474
- APP_RUNNING = "APP_RUNNING",
475
- /** Another Store owns the installed package's update provenance. */
476
- STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
477
493
  /** Target miniapp does not declare the requested action. */
478
494
  ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
479
495
  /** Target connected but never registered a handler for the action (within the buffer window). */
@@ -5,6 +5,8 @@
5
5
  * miniapp. Currently:
6
6
  * - Keeps `<html class="dark">` in sync with the host's color scheme so the
7
7
  * shared theme CSS switches automatically.
8
+ * - Calls `mentra.ready()` once the app has mounted, which lifts the host's
9
+ * splash and fires the background's `session.ui.onOpen` handlers.
8
10
  *
9
11
  * Usage:
10
12
  * createRoot(root).render(
@@ -17,7 +19,7 @@
17
19
  * things themselves can opt out without forking the provider.
18
20
  */
19
21
 
20
- import {type ReactNode} from "react"
22
+ import {type ReactNode, useEffect} from "react"
21
23
 
22
24
  import {useColorScheme} from "./useColorScheme"
23
25
 
@@ -32,11 +34,30 @@ export interface MentraProviderProps {
32
34
  * (e.g. you integrate a theming library like next-themes).
33
35
  */
34
36
  syncColorScheme?: boolean
37
+ /**
38
+ * Call `mentra.ready()` after the first mount. A parent's effects run after
39
+ * its children's, so every `mentra.on` subscription the app makes on mount
40
+ * is attached before the background is told the UI is open. Without
41
+ * `ready()` the host keeps its splash up and reports the miniapp as failed.
42
+ *
43
+ * Default: true. Set to false to call `mentra.ready()` yourself, e.g. to keep
44
+ * the splash up until the first screen has its data.
45
+ */
46
+ autoReady?: boolean
35
47
  }
36
48
 
37
- export function MentraProvider({children, syncColorScheme = true}: MentraProviderProps) {
49
+ function getBridge(): {ready?: () => void} | null {
50
+ if (typeof window === "undefined") return null
51
+ return (window as unknown as {mentra?: {ready?: () => void}}).mentra ?? null
52
+ }
53
+
54
+ export function MentraProvider({children, syncColorScheme = true, autoReady = true}: MentraProviderProps) {
38
55
  const scheme = useColorScheme()
39
56
 
57
+ useEffect(() => {
58
+ if (autoReady) getBridge()?.ready?.()
59
+ }, [autoReady])
60
+
40
61
  // Runs during render — before children paint, before effects fire.
41
62
  if (syncColorScheme && typeof document !== "undefined") {
42
63
  const el = document.documentElement
package/src/session.ts CHANGED
@@ -26,6 +26,7 @@ import {ConfigurationModule} from "./modules/configuration"
26
26
  import {DashboardAPI} from "./modules/dashboard"
27
27
  import {DisplayManager} from "./modules/display"
28
28
  import {EventManager, type TranscriptionEventRoute, type UnsubscribeFn} from "./modules/events"
29
+ import type {SpeechProcessingOptions} from "./modules/speechProcessing"
29
30
  import {GlassesModule} from "./modules/glasses"
30
31
  import {HeadingModule} from "./modules/heading"
31
32
  import {ImuModule} from "./modules/imu"
@@ -45,6 +46,7 @@ import {StreamModule} from "./modules/stream"
45
46
  import {SystemModule} from "./modules/system"
46
47
  import {MiniappsModule} from "./modules/miniapps"
47
48
  import {ActionsModule} from "./modules/actions"
49
+ import {TelemetryModule} from "./modules/telemetry"
48
50
  import {BlobModule} from "./modules/blob"
49
51
  import {
50
52
  MeetingModule,
@@ -130,6 +132,13 @@ export interface HostFeatures {
130
132
  captureAudio?: boolean
131
133
  /** Host keeps the UI closed until a background session that announced it sends READY. */
132
134
  initReady?: boolean
135
+ /**
136
+ * Host installs miniapps from a miniapp URL through `session.miniapps.install({url})`. Only a miniapp on the
137
+ * Mentra App's system list may call it; older Mentra Apps answer NOT_PERMITTED.
138
+ */
139
+ miniappInstall?: boolean
140
+ /** Host takes the miniapp's own events from `session.telemetry.capture` (Fleet Management's miniapp telemetry). */
141
+ telemetry?: boolean
133
142
  }
134
143
 
135
144
  export interface MiniappAuthState {
@@ -185,6 +194,8 @@ export class ConnectAckTimeoutError extends Error {
185
194
  export interface MiniappRequestError {
186
195
  code: string
187
196
  message: string
197
+ /** A finer cause, where a request documents one (`INSTALL_FAILED` from `session.miniapps.install`). */
198
+ reason?: string
188
199
  }
189
200
 
190
201
  // ---------------------------------------------------------------------------
@@ -296,6 +307,8 @@ export class MiniappSession<TChannels extends object = any> {
296
307
  * declared action; `handle` (open to all) exposes one of your own.
297
308
  */
298
309
  public readonly actions: ActionsModule
310
+ /** The miniapp's own analytics, shown in its organization's Fleet Management (see {@link TelemetryModule}). */
311
+ public readonly telemetry: TelemetryModule
299
312
 
300
313
  /** Phone-declared glasses capabilities. Null until CONNECT_ACK arrives. */
301
314
  public capabilities: GlassesCapabilities | null = null
@@ -403,6 +416,12 @@ export class MiniappSession<TChannels extends object = any> {
403
416
  this.ui = new UIModuleImpl<TChannels>(this)
404
417
  this.miniapps = new MiniappsModule(this)
405
418
  this.actions = new ActionsModule(this)
419
+ this.telemetry = new TelemetryModule(this)
420
+ }
421
+
422
+ /** @internal Whether CONNECT_ACK arrived, so `hostFeatures` is the host's answer and not just unknown. */
423
+ get handshakeDone(): boolean {
424
+ return this.handshakeComplete
406
425
  }
407
426
 
408
427
  /**
@@ -476,7 +495,7 @@ export class MiniappSession<TChannels extends object = any> {
476
495
  _subscribe(
477
496
  streamType: string,
478
497
  handler: (data: unknown) => void,
479
- options: {forceLocal?: boolean} = {},
498
+ options: SpeechProcessingOptions = {},
480
499
  ): UnsubscribeFn {
481
500
  return this.events.subscribe(streamType, handler, options)
482
501
  }
@@ -999,22 +1018,6 @@ export class MiniappSession<TChannels extends object = any> {
999
1018
  return
1000
1019
  }
1001
1020
 
1002
- case MiniappResponseType.MINIAPPS_INSTALL_PROGRESS: {
1003
- const {packageName, version, phase} = payload as Record<string, unknown>
1004
- if (
1005
- typeof packageName === "string" &&
1006
- typeof version === "string" &&
1007
- (phase === "downloading" ||
1008
- phase === "verifying" ||
1009
- phase === "extracting" ||
1010
- phase === "activating" ||
1011
- phase === "complete")
1012
- ) {
1013
- this.miniapps._deliverInstallProgress({packageName, version, phase})
1014
- }
1015
- return
1016
- }
1017
-
1018
1021
  case MiniappResponseType.REQUEST_RESULT: {
1019
1022
  const requestId = envelope.requestId
1020
1023
  if (!requestId) return
package/src/ui/index.ts CHANGED
@@ -71,10 +71,13 @@ export interface MentraUiGlobal<TChannels extends object = Record<string, unknow
71
71
  onClose(cb: () => void): () => void
72
72
 
73
73
  /**
74
- * MUST be called once on UI bootstrap. Tells the host the WebView is
75
- * mounted and ready to receive messages from background. Idempotent
76
- * — subsequent calls no-op. Background's `session.ui.onOpen` handlers
77
- * fire after this lands.
74
+ * Tells the host the WebView is mounted and ready to receive messages
75
+ * from background. `<MentraProvider>` calls it once the app has mounted;
76
+ * call it yourself only without the provider (or with `autoReady={false}`).
77
+ * The host's splash stays up until it lands, and a page that loads
78
+ * without calling it is reported as failed. Idempotent — subsequent
79
+ * calls no-op. Background's `session.ui.onOpen` handlers fire after this
80
+ * lands.
78
81
  */
79
82
  ready(): void
80
83
  }