@mentra/miniapp 3.3.0 → 3.4.0-beta.1237

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +5 -5
  2. package/dist/background/index.d.ts +5 -2
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js.map +1 -1
  5. package/dist/index.d.ts +5 -2
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/modules/blob.d.ts +2 -1
  9. package/dist/modules/blob.d.ts.map +1 -1
  10. package/dist/modules/blob.js +2 -1
  11. package/dist/modules/blob.js.map +1 -1
  12. package/dist/modules/events.d.ts +16 -6
  13. package/dist/modules/events.d.ts.map +1 -1
  14. package/dist/modules/events.js +53 -47
  15. package/dist/modules/events.js.map +1 -1
  16. package/dist/modules/mic.d.ts +26 -4
  17. package/dist/modules/mic.d.ts.map +1 -1
  18. package/dist/modules/mic.js +13 -4
  19. package/dist/modules/mic.js.map +1 -1
  20. package/dist/modules/miniapps.d.ts +44 -57
  21. package/dist/modules/miniapps.d.ts.map +1 -1
  22. package/dist/modules/miniapps.js +21 -24
  23. package/dist/modules/miniapps.js.map +1 -1
  24. package/dist/modules/speaker.d.ts +41 -10
  25. package/dist/modules/speaker.d.ts.map +1 -1
  26. package/dist/modules/speaker.js +17 -6
  27. package/dist/modules/speaker.js.map +1 -1
  28. package/dist/modules/speechProcessing.d.ts +29 -0
  29. package/dist/modules/speechProcessing.d.ts.map +1 -0
  30. package/dist/modules/speechProcessing.js +22 -0
  31. package/dist/modules/speechProcessing.js.map +1 -0
  32. package/dist/modules/system.d.ts +8 -3
  33. package/dist/modules/system.d.ts.map +1 -1
  34. package/dist/modules/system.js +1 -1
  35. package/dist/modules/system.js.map +1 -1
  36. package/dist/modules/telemetry.d.ts +25 -0
  37. package/dist/modules/telemetry.d.ts.map +1 -0
  38. package/dist/modules/telemetry.js +61 -0
  39. package/dist/modules/telemetry.js.map +1 -0
  40. package/dist/modules/transcription.d.ts +70 -3
  41. package/dist/modules/transcription.d.ts.map +1 -1
  42. package/dist/modules/transcription.js +68 -2
  43. package/dist/modules/transcription.js.map +1 -1
  44. package/dist/protocol.d.ts +29 -15
  45. package/dist/protocol.d.ts.map +1 -1
  46. package/dist/protocol.js +30 -15
  47. package/dist/protocol.js.map +1 -1
  48. package/dist/react/MentraProvider.d.ts +13 -1
  49. package/dist/react/MentraProvider.d.ts.map +1 -1
  50. package/dist/react/MentraProvider.js +31 -1
  51. package/dist/react/MentraProvider.js.map +1 -1
  52. package/dist/session.d.ts +16 -3
  53. package/dist/session.d.ts.map +1 -1
  54. package/dist/session.js +6 -13
  55. package/dist/session.js.map +1 -1
  56. package/dist/ui/index.d.ts +7 -4
  57. package/dist/ui/index.d.ts.map +1 -1
  58. package/dist/ui/index.js.map +1 -1
  59. package/package.json +2 -2
  60. package/src/background/index.ts +6 -3
  61. package/src/index.ts +11 -3
  62. package/src/modules/blob.ts +2 -1
  63. package/src/modules/events.ts +65 -51
  64. package/src/modules/mic.ts +30 -4
  65. package/src/modules/miniapps.ts +59 -71
  66. package/src/modules/speaker.ts +56 -12
  67. package/src/modules/speechProcessing.ts +42 -0
  68. package/src/modules/system.ts +8 -3
  69. package/src/modules/telemetry.ts +62 -0
  70. package/src/modules/transcription.ts +106 -9
  71. package/src/protocol.ts +32 -16
  72. package/src/react/MentraProvider.tsx +23 -2
  73. package/src/session.ts +20 -17
  74. package/src/ui/index.ts +7 -4
@@ -33,10 +33,16 @@ import {EventEmitter} from "eventemitter3"
33
33
  import {makeRequestId} from "../envelope"
34
34
  import {MiniappRequestType, MiniappStreamType} from "../protocol"
35
35
  import {MiniappSession} from "../session"
36
+ import {resolveSpeechProcessing, type SpeechProcessing, type SpeechProcessingOptions} from "./speechProcessing"
36
37
 
37
38
  export type UnsubscribeFn = () => void
38
39
 
39
- /** @internal Host-selected delivery target for a transcription event. */
40
+ /**
41
+ * @internal Delivery class the host tags a transcription event with. Hosts
42
+ * address each event to the listeners that should get it; SDKs from before
43
+ * `processing` filter on this tag instead: `"forceLocal"` reaches `localOnly`
44
+ * listeners, `"default"` the others, `"all"` everyone.
45
+ */
40
46
  export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
41
47
 
42
48
  // ---------------------------------------------------------------------------
@@ -48,6 +54,8 @@ export interface TranscriptionData {
48
54
  text: string
49
55
  isFinal: boolean
50
56
  language?: string
57
+ /** Where the text was transcribed: `"cloud"`, or `"local"` for the phone's on-device model. */
58
+ source: "cloud" | "local"
51
59
  }
52
60
 
53
61
  export interface TranslationData {
@@ -150,6 +158,10 @@ export interface NotificationDismissedData {
150
158
  export interface VadData {
151
159
  /** True while the user is speaking (voice detected), false when silent. */
152
160
  status: boolean
161
+ /** 0 while speaking; on a silence event, how long the audio has already been silent. */
162
+ silentForMs?: number
163
+ /** Unix epoch milliseconds of the transition, when the host reports it. */
164
+ timestamp?: number
153
165
  }
154
166
 
155
167
  export interface TouchData {
@@ -173,14 +185,12 @@ export class EventManager {
173
185
  private listenerSeq = 0
174
186
  private transcriptionListeners = new Map<
175
187
  string,
176
- {stream: string; forceLocal: boolean; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
188
+ {stream: string; processing: SpeechProcessing; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
177
189
  >()
178
190
  private retireListeners = new Set<() => void>()
179
191
  private readonly emitter = new EventEmitter()
180
192
  /** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
181
193
  private readonly refCounts = new Map<string, number>()
182
- /** Transcription stream -> refs that require on-device-only processing. */
183
- private readonly forceLocalRefCounts = new Map<string, number>()
184
194
 
185
195
  constructor(private readonly session: MiniappSession) {}
186
196
 
@@ -195,20 +205,24 @@ export class EventManager {
195
205
  * the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
196
206
  * via a separate envelope `type`; the EventManager only routes them
197
207
  * locally.
208
+ *
209
+ * `processing` applies to transcription streams only.
198
210
  */
199
- subscribe(stream: string, handler: (data: unknown) => void, options: {forceLocal?: boolean} = {}): UnsubscribeFn {
211
+ subscribe(stream: string, handler: (data: unknown) => void, options: SpeechProcessingOptions = {}): UnsubscribeFn {
200
212
  const isInternal = stream.startsWith("_")
201
- const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
202
213
  const isTranscription = stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
214
+ const processing = isTranscription ? resolveSpeechProcessing(options, "events.subscribe(options)") : "preferCloud"
215
+ const localOnly = processing === "localOnly"
203
216
  const listenerId = isTranscription ? `${this.listenerScope}:${++this.listenerSeq}` : undefined
204
217
  let active = true
205
218
  const listener = (data: unknown, route?: TranscriptionEventRoute) => {
206
219
  if (!active) return
207
- if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) {
220
+ if (isTranscription) {
221
+ // The host tags every transcription event with the class it is for.
208
222
  // Hosts that predate routed transcription events omit `route`. Treat
209
- // those events as the default/cloud path so a forceLocal listener can
210
- // never accidentally receive cloud text.
211
- if (forceLocal ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
223
+ // those events as the cloud path so a localOnly listener can never
224
+ // accidentally receive cloud text.
225
+ if (localOnly ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
212
226
  }
213
227
  handler(data)
214
228
  }
@@ -217,21 +231,12 @@ export class EventManager {
217
231
  active = false
218
232
  }
219
233
  this.retireListeners.add(retire)
220
- if (listenerId) this.transcriptionListeners.set(listenerId, {stream, forceLocal, invoke: listener})
234
+ if (listenerId) this.transcriptionListeners.set(listenerId, {stream, processing, invoke: listener})
221
235
  if (!isInternal) {
222
236
  const before = this.refCounts.get(stream) ?? 0
223
237
  this.refCounts.set(stream, before + 1)
224
- const forceLocalBefore = this.forceLocalRefCounts.get(stream) ?? 0
225
- const defaultBefore = before - forceLocalBefore
226
- if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
227
- if (
228
- isTranscription ||
229
- before === 0 ||
230
- (forceLocal && forceLocalBefore === 0) ||
231
- (!forceLocal && defaultBefore === 0)
232
- ) {
233
- this.sendSubscriptionUpdate()
234
- }
238
+ // Every transcription listener is registered with the host by id.
239
+ if (isTranscription || before === 0) this.sendSubscriptionUpdate()
235
240
  }
236
241
  return () => {
237
242
  if (!active) return
@@ -241,24 +246,12 @@ export class EventManager {
241
246
  this.emitter.off(stream, listener)
242
247
  if (isInternal) return
243
248
  const current = this.refCounts.get(stream) ?? 0
244
- const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
245
- let subscriptionChanged = current <= 1
246
249
  if (current <= 1) {
247
250
  this.refCounts.delete(stream)
248
251
  } else {
249
252
  this.refCounts.set(stream, current - 1)
250
253
  }
251
- if (forceLocal) {
252
- if (forceLocalCurrent <= 1) {
253
- this.forceLocalRefCounts.delete(stream)
254
- subscriptionChanged = true
255
- } else {
256
- this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
257
- }
258
- } else if (current - forceLocalCurrent <= 1) {
259
- subscriptionChanged = true
260
- }
261
- if (isTranscription || subscriptionChanged) this.sendSubscriptionUpdate()
254
+ if (isTranscription || current <= 1) this.sendSubscriptionUpdate()
262
255
  }
263
256
  }
264
257
 
@@ -269,7 +262,6 @@ export class EventManager {
269
262
  this.transcriptionListeners.clear()
270
263
  this.emitter.removeAllListeners()
271
264
  this.refCounts.clear()
272
- this.forceLocalRefCounts.clear()
273
265
  this.sendSubscriptionUpdate()
274
266
  }
275
267
 
@@ -284,6 +276,7 @@ export class EventManager {
284
276
  transcriptionRoute?: TranscriptionEventRoute,
285
277
  listenerId?: string,
286
278
  ): void {
279
+ if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) data = withTranscriptionSource(data)
287
280
  if (listenerId !== undefined) {
288
281
  const listener = this.transcriptionListeners.get(listenerId)
289
282
  if (listener && (listener.stream === stream || listener.stream === "transcription:auto"))
@@ -318,27 +311,48 @@ export class EventManager {
318
311
  }
319
312
 
320
313
  private sendSubscriptionUpdate(): void {
321
- const subscriptions = Array.from(this.refCounts.keys()).map((stream) => {
322
- if (stream === MiniappStreamType.LOCATION_UPDATE) return {stream: "location_stream", rate: "realtime"}
323
-
324
- const forceLocalRefs = this.forceLocalRefCounts.get(stream) ?? 0
325
- if (forceLocalRefs === 0) return stream
326
-
327
- const totalRefs = this.refCounts.get(stream) ?? 0
328
- return {
329
- stream,
330
- forceLocal: true,
331
- ...(totalRefs > forceLocalRefs ? {includeCloud: true} : {}),
332
- }
333
- })
314
+ const subscriptions: Array<string | Record<string, unknown>> = []
315
+ for (const stream of this.refCounts.keys()) {
316
+ if (stream === MiniappStreamType.LOCATION_UPDATE)
317
+ subscriptions.push({stream: "location_stream", rate: "realtime"})
318
+ else if (!stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) subscriptions.push(stream)
319
+ }
320
+ // One entry per transcription stream and processing its listeners ask for.
321
+ const transcription = new Set<string>()
322
+ for (const {stream, processing} of this.transcriptionListeners.values()) {
323
+ const key = `${processing} ${stream}`
324
+ if (transcription.has(key)) continue
325
+ transcription.add(key)
326
+ subscriptions.push({stream, ...processingOnWire(processing)})
327
+ }
334
328
  this.session.sendOneShot({
335
329
  type: MiniappRequestType.SUBSCRIBE,
336
330
  subscriptions,
337
- transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, forceLocal}]) => ({
331
+ transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, processing}]) => ({
338
332
  id,
339
333
  stream,
340
- forceLocal,
334
+ ...processingOnWire(processing),
341
335
  })),
342
336
  })
343
337
  }
344
338
  }
339
+
340
+ /**
341
+ * The wire form of a transcription listener's processing. Hosts before
342
+ * `processing` (Mentra App 3.4.0) read only `forceLocal`, so `localOnly` keeps
343
+ * sending it: there a localOnly listener still never gets cloud text.
344
+ */
345
+ function processingOnWire(processing: SpeechProcessing): {processing: SpeechProcessing; forceLocal?: true} {
346
+ return processing === "localOnly" ? {processing, forceLocal: true} : {processing}
347
+ }
348
+
349
+ /**
350
+ * Hosts before `source` (Mentra App 3.4.0) leave it out. Their on-device
351
+ * transcripts carry `type: "local_transcription"`, their cloud ones do not.
352
+ */
353
+ function withTranscriptionSource(data: unknown): unknown {
354
+ if (typeof data !== "object" || data === null) return data
355
+ const event = data as {source?: unknown; type?: unknown}
356
+ if (event.source === "cloud" || event.source === "local") return data
357
+ return {...event, source: event.type === "local_transcription" ? "local" : "cloud"}
358
+ }
@@ -31,6 +31,22 @@ export interface MicLease {
31
31
  release(): Promise<void>
32
32
  }
33
33
 
34
+ /**
35
+ * On-device wake-word ("Hey Mentra") event. It describes only the wake-word
36
+ * model: `loading` while the phone loads it, `ready` once it is listening,
37
+ * `detected` for each detection, and `error` when it cannot run (`message`
38
+ * says why). The state of the on-device speech-to-text model is reported
39
+ * separately by `session.transcription.onModelStatus`.
40
+ */
41
+ export interface WakeWordEvent {
42
+ kind: "loading" | "ready" | "detected" | "error"
43
+ /** The phrase this event is about. Currently always "Hey Mentra". */
44
+ phrase: string
45
+ /** Unix epoch milliseconds. */
46
+ timestamp: number
47
+ message?: string
48
+ }
49
+
34
50
  export class MicModule {
35
51
  /** All active unsubscribe functions for stop() to tear down at once. */
36
52
  private readonly unsubs = new Set<UnsubscribeFn>()
@@ -38,17 +54,27 @@ export class MicModule {
38
54
  constructor(private readonly session: MiniappSession) {}
39
55
 
40
56
  /**
41
- * Subscribe to voice activity detection (VAD) events. `data.status` is
42
- * `true` while the user is speaking, `false` when silent.
57
+ * Subscribe to on-device voice activity detection. One event when the user
58
+ * starts speaking (`status: true`) and one when they stop (`status: false`,
59
+ * with `silentForMs` saying how long the audio has already been quiet). A
60
+ * gap in microphone audio counts as silence. Requires MICROPHONE permission.
43
61
  */
44
62
  onVoiceActivity(handler: (data: VadData) => void): UnsubscribeFn {
45
63
  return this.track(this.session._subscribe(MiniappStreamType.VAD, handler as (data: unknown) => void))
46
64
  }
47
65
 
48
66
  /**
49
- * Subscribe to raw audio chunks. Format depends on the phone's mic mode
50
- * (PCM or LC3, base64-encoded).
67
+ * Subscribe to the on-device "Hey Mentra" wake word. Every subscribed
68
+ * miniapp receives every detection. You get the detector's current state
69
+ * right away, then every change. The phone loads the wake-word model while
70
+ * any miniapp subscribes and downloads it on first use. Requires MICROPHONE
71
+ * permission.
51
72
  */
73
+ onWakeWord(handler: (event: WakeWordEvent) => void): UnsubscribeFn {
74
+ return this.track(this.session._subscribe(MiniappStreamType.WAKE_WORD, handler as (data: unknown) => void))
75
+ }
76
+
77
+ /** Subscribe to raw audio chunks in the phone's PCM/LC3 microphone format. */
52
78
  onAudioChunk(handler: (data: AudioChunkData) => void): UnsubscribeFn {
53
79
  return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
54
80
  }
@@ -7,10 +7,12 @@
7
7
  * layer (invoke / handle) lives on `session.actions`, not here — these are
8
8
  * operations on the app as a whole, not on a declared action.
9
9
  *
10
- * Installation (`install`, `checkInstallCompatibility`, `uninstall`) is
11
- * narrower still: the host accepts it only from a Store miniapp bundled with
12
- * the Mentra App build. The host downloads, verifies, and activates the
13
- * release through its own installer; the Store only selects the release.
10
+ * Installing (`install`, `checkInstallCompatibility`, `uninstall`) is
11
+ * narrower still: only a miniapp on the Mentra App's system list, running the
12
+ * release the Mentra App installed for it, may install miniapps. Development
13
+ * builds, which may use the calls above, may not. The host installs from a
14
+ * miniapp URL through the same installer as every other way in, with no install
15
+ * sheet: the caller asks the person first.
14
16
  */
15
17
 
16
18
  import {MiniappRequestType} from "../protocol"
@@ -55,10 +57,6 @@ export interface MiniappInfo {
55
57
  compatibility: MiniappCompatibility
56
58
  /** True when this package identity is backed by a ZIP in the Mentra App build. */
57
59
  system: boolean
58
- /** Store package that owns updates of this active release, when it is Store-managed. */
59
- storeOwnerPackageName?: string
60
- /** Store the host build assigns to update this bundled package's active release, if any. */
61
- systemStoreOwnerPackageName?: string
62
60
  /** Declared actions (empty if the miniapp declares none). */
63
61
  actions: MiniappActionInfo[]
64
62
  }
@@ -74,49 +72,54 @@ export interface MiniappHardwareRequirement {
74
72
  description?: string
75
73
  }
76
74
 
77
- /** Backend-neutral release request, accepted only from a bundled, host-trusted Store. */
75
+ /** A miniapp to install. */
78
76
  export interface InstallMiniappRequest {
79
- packageName: string
80
- version: string
81
- bundleUrl: string
82
- /** SHA-256 of the release ZIP; the host refuses any other bytes. */
83
- bundleSha256: string
84
- /**
85
- * Bearer for `bundleUrl`, when the Store's backend requires one. The host
86
- * forwards it untouched and keeps no Store address of its own: entitlement is
87
- * settled between this miniapp and its backend, integrity by the SHA-256 and
88
- * the bundle's publisher signature.
89
- */
90
- bundleAuthorization?: string
91
- /**
92
- * Automatic update: never stop a running miniapp. A running or starting
93
- * target defers the request with `APP_RUNNING`.
94
- */
95
- onlyIfStopped?: boolean
77
+ /** The miniapp's URL: the HTTPS address that serves its `miniapp.json`. */
78
+ url: string
96
79
  /**
97
- * With `onlyIfStopped`: when the target is running, the host stages the
98
- * release and asks the user whether to update now. Declining, or a prompt the
99
- * host does not show (the Mentra App is in the background, another prompt is
100
- * open, or this miniapp was asked within the last day), still answers
101
- * `APP_RUNNING`.
80
+ * The package the person agreed to install. When given, a URL that serves another package installs nothing and
81
+ * fails with `not_a_miniapp`.
102
82
  */
103
- askIfRunning?: boolean
104
- /** Minimum Mentra App version declared by the release manifest. */
105
- minHostVersion?: string
106
- /** Mentra Miniapp SDK version declared by the release manifest. */
107
- sdkVersion?: string
108
- /** Glasses hardware requirements declared by the release manifest. */
109
- hardwareRequirements?: MiniappHardwareRequirement[]
110
- releaseId?: string
111
- channel?: string
83
+ packageName?: string
112
84
  }
113
85
 
114
86
  export interface InstallMiniappResult {
87
+ /**
88
+ * `installed`: this release installed. `opened`: nothing installed, because the miniapp is installed from this URL
89
+ * already, or a newer release of it is; {@link MiniappsModule.open} it to show it. `deferred`: the miniapp is
90
+ * running, so its update waits until it stops, or until the person answers the Mentra App's update prompt.
91
+ */
92
+ status: "installed" | "opened" | "deferred"
115
93
  packageName: string
94
+ /** The installed version: the new release, or the one that stays. */
116
95
  version: string
117
- installedByStore: string
118
96
  }
119
97
 
98
+ /**
99
+ * Why an install failed: the `reason` of an `INSTALL_FAILED` rejection from {@link MiniappsModule.install}.
100
+ * - `not_a_miniapp`: the URL serves no valid `miniapp.json` or bundle.
101
+ * - `domain_mismatch`: the package name doesn't belong to the URL's domain.
102
+ * - `archived`: Mentra archived this URL; it doesn't install.
103
+ * - `needs_newer_app`: the miniapp needs a newer Mentra App, or hardware the glasses lack.
104
+ * - `access_denied`: its server refused this account, or this deployment doesn't allow installing from a URL.
105
+ * - `age_restricted`: the person's declared age is below the miniapp's age rating.
106
+ * - `network`: the URL couldn't be reached.
107
+ * - `cancelled`: the Mentra App signed out or switched deployments meanwhile.
108
+ * - `integrity`: the bundle doesn't match what `miniapp.json` names.
109
+ * - `signature_invalid`: the bundle's signature doesn't match the installed miniapp's publisher.
110
+ */
111
+ export type InstallMiniappFailureReason =
112
+ | "not_a_miniapp"
113
+ | "domain_mismatch"
114
+ | "archived"
115
+ | "needs_newer_app"
116
+ | "access_denied"
117
+ | "age_restricted"
118
+ | "network"
119
+ | "cancelled"
120
+ | "integrity"
121
+ | "signature_invalid"
122
+
120
123
  export interface InstallMiniappCompatibilityRequest {
121
124
  minHostVersion?: string | null
122
125
  sdkVersion?: string | null
@@ -127,17 +130,7 @@ export type InstallMiniappCompatibility =
127
130
  | {compatible: true}
128
131
  | {compatible: false; blocker: "host" | "sdk" | "hardware"; reason: string}
129
132
 
130
- export type InstallMiniappPhase = "downloading" | "verifying" | "extracting" | "activating" | "complete"
131
-
132
- export interface InstallMiniappProgress {
133
- packageName: string
134
- version: string
135
- phase: InstallMiniappPhase
136
- }
137
-
138
133
  export class MiniappsModule {
139
- private readonly progressHandlers = new Set<(progress: InstallMiniappProgress) => void>()
140
-
141
134
  constructor(private readonly session: MiniappSession) {}
142
135
 
143
136
  /**
@@ -185,24 +178,27 @@ export class MiniappsModule {
185
178
  }
186
179
 
187
180
  /**
188
- * Download, verify, and install a release through the phone host. Accepted
189
- * only from a Store bundled with the Mentra App build; other callers receive
190
- * NOT_PERMITTED.
181
+ * Install the miniapp at a miniapp URL, with no install sheet: ask the person first. The host fetches its
182
+ * `miniapp.json`, checks the package name against the URL's domain, registers the URL with Mentra, checks the
183
+ * phone can run it, and downloads, verifies and installs its bundle.
184
+ *
185
+ * Only a miniapp on the Mentra App's system list may install; any other caller gets NOT_PERMITTED. A failed
186
+ * install rejects with INSTALL_FAILED, whose `reason` is an {@link InstallMiniappFailureReason}.
191
187
  */
192
188
  async install(request: InstallMiniappRequest): Promise<InstallMiniappResult> {
193
189
  return this.session.sendRequest<InstallMiniappResult>(
194
190
  {
195
- ...request,
196
191
  type: MiniappRequestType.MINIAPPS_INSTALL,
192
+ url: request.url,
193
+ ...(request.packageName !== undefined ? {packageName: request.packageName} : {}),
197
194
  },
198
- // Downloading, validating, extracting, asking the user and relaunching
199
- // can legitimately outlast the SDK's default request timeout. The request
200
- // still settles on the host response or session disconnect.
195
+ // Fetching, registering, downloading and installing can legitimately outlast the SDK's default request
196
+ // timeout. The request still settles on the host response or session disconnect.
201
197
  {timeoutMs: 0},
202
198
  )
203
199
  }
204
200
 
205
- /** Check host, Mentra Miniapp SDK, and glasses compatibility before offering or applying an update. */
201
+ /** Check a release's Mentra App, SDK and glasses requirements against this phone. System-list miniapps only. */
206
202
  async checkInstallCompatibility(request: InstallMiniappCompatibilityRequest): Promise<InstallMiniappCompatibility> {
207
203
  return this.session.sendRequest<InstallMiniappCompatibility>({
208
204
  type: MiniappRequestType.MINIAPPS_INSTALL_CHECK,
@@ -212,18 +208,10 @@ export class MiniappsModule {
212
208
  })
213
209
  }
214
210
 
215
- /** Subscribe to host-owned install progress. Returns an unsubscribe function. */
216
- onInstallProgress(handler: (progress: InstallMiniappProgress) => void): () => void {
217
- this.progressHandlers.add(handler)
218
- return () => this.progressHandlers.delete(handler)
219
- }
220
-
221
- /** @internal */
222
- _deliverInstallProgress(progress: InstallMiniappProgress): void {
223
- for (const handler of this.progressHandlers) handler(progress)
224
- }
225
-
226
- /** Uninstall a package this Store installed. Bundled Stores only. */
211
+ /**
212
+ * Uninstall a miniapp installed from a miniapp URL. A miniapp the Mentra App bundles, one a workspace manages and a
213
+ * development build stay, with NOT_PERMITTED. System-list miniapps only.
214
+ */
227
215
  async uninstall(packageName: string): Promise<void> {
228
216
  await this.session.sendRequest<void>({
229
217
  type: MiniappRequestType.MINIAPPS_UNINSTALL,
@@ -5,13 +5,15 @@
5
5
  * audio chunks, VAD) lives on session.mic — the split is by I/O direction.
6
6
  *
7
7
  * Imperative surface:
8
- * speaker.play({audioUrl}) — play an arbitrary URL via the phone's
8
+ * speaker.play({audioUrl}) — play an http(s) or data: URL, or one of this
9
+ * miniapp's blobs, via the phone's
9
10
  * AudioPlaybackService.
10
- * speaker.speak(text) — send a SPEAK request. Phone streams cloud
11
- * TTS when connected and falls back to local
12
- * offline TTS when cloud is unavailable.
11
+ * speaker.speak(text) — send a SPEAK request. By default the phone
12
+ * streams cloud TTS when connected and falls
13
+ * back to its on-device voice; the
14
+ * `processing` option picks the order.
13
15
  * Resolves when playback completes; rejects
14
- * with a TTS_* error code on cloud failure.
16
+ * with a TTS_* error code on failure.
15
17
  * speaker.createStream(opts) — open a live PCM output stream (16-bit LE).
16
18
  * Returns a SpeakerStreamWriter: write() PCM
17
19
  * chunks as they arrive (e.g. from a WebSocket
@@ -38,8 +40,13 @@ import {MiniappErrorCode, MiniappRequestType} from "../protocol"
38
40
  import {MiniappSession} from "../session"
39
41
  import {bytesToBase64, toUint8Array} from "./base64"
40
42
  import type {UnsubscribeFn} from "./events"
43
+ import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
41
44
 
42
45
  export interface PlayAudioOptions {
46
+ /**
47
+ * An http(s) or `data:` URL, or the `file://` uri of one of this miniapp's
48
+ * blobs (`session.blob`). The host refuses any other file or scheme.
49
+ */
43
50
  audioUrl: string
44
51
  /** Start playback at this offset in milliseconds. Defaults to zero. */
45
52
  startPositionMs?: number
@@ -58,10 +65,17 @@ export interface SpeakOptions {
58
65
  */
59
66
  enableSanitization?: boolean
60
67
  /**
61
- * Force this call to use on-device offline TTS, skipping cloud TTS
62
- * entirely — even when cloud is connected. Rejects with `TTS_LOCAL_UNAVAILABLE`
63
- * if the offline model isn't ready instead of falling back to cloud.
68
+ * Where the speech is synthesized. Default `"preferCloud"`.
69
+ *
70
+ * - `"preferCloud"`: cloud TTS while the cloud is connected, otherwise the
71
+ * phone's on-device voice.
72
+ * - `"preferLocal"`: the on-device voice when its model is ready and can say
73
+ * it (an explicit cloud `voice_id` cannot), otherwise cloud TTS.
74
+ * - `"localOnly"`: the on-device voice only; never the cloud. Rejects with
75
+ * `TTS_LOCAL_UNAVAILABLE` when the on-device model isn't ready.
64
76
  */
77
+ processing?: SpeechProcessing
78
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
65
79
  forceLocal?: boolean
66
80
  }
67
81
 
@@ -80,6 +94,22 @@ export type SpeakerStreamSampleRate = 16000 | 24000 | 48000
80
94
  export interface SpeakerStreamOptions {
81
95
  /** PCM sample rate in Hz. Default 16000. */
82
96
  sampleRate?: SpeakerStreamSampleRate
97
+ /**
98
+ * Open the native stream now and keep it idle until the first write, so a
99
+ * later sound only has to write samples. The speaker reports `idle` and other
100
+ * audio is left alone until then. If the audio route changes before the
101
+ * first write (Bluetooth or glasses connect/disconnect, a call, an audio
102
+ * interruption), the phone closes the stream and the next write fails with
103
+ * "not open"; open a fresh stream then.
104
+ */
105
+ prewarm?: boolean
106
+ /**
107
+ * How much audio (80–500 ms) the phone queues before playback starts, and
108
+ * the cushion against gaps after that. Default 500. A sound shorter than
109
+ * this still plays once you close the stream. Use a small value such as 80
110
+ * for short cues that should start immediately.
111
+ */
112
+ jitterMs?: number
83
113
  /** Channel count. v1 is mono only. Default 1. */
84
114
  channels?: 1
85
115
  /** Playback volume 0..1. Default 1. */
@@ -118,7 +148,10 @@ export class SpeakerStreamWriter {
118
148
  private settled = false
119
149
  private writeChain: Promise<void> = Promise.resolve()
120
150
 
121
- constructor(private readonly session: MiniappSession, readonly streamId: string) {}
151
+ constructor(
152
+ private readonly session: MiniappSession,
153
+ readonly streamId: string,
154
+ ) {}
122
155
 
123
156
  /**
124
157
  * Append raw PCM bytes (16-bit LE). Auto-chunked. Resolves with the host's
@@ -255,8 +288,9 @@ export class SpeakerModule {
255
288
  }
256
289
 
257
290
  /**
258
- * Speak text through the phone. The host streams cloud TTS when connected,
259
- * then falls back to local offline TTS.
291
+ * Speak text through the phone, in the cloud or on-device as
292
+ * `options.processing` says (default: cloud TTS when connected, then the
293
+ * on-device voice).
260
294
  *
261
295
  * Rejects with a MiniappRequestError containing a `code` field on cloud-side
262
296
  * TTS failures: `TTS_TEXT_TOO_LONG`, `TTS_INVALID_VOICE`, `TTS_UPSTREAM_ERROR`.
@@ -268,6 +302,7 @@ export class SpeakerModule {
268
302
  if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
269
303
  throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
270
304
  }
305
+ const processing = resolveSpeechProcessing(options, "speaker.speak(options)")
271
306
  try {
272
307
  // Like play(): resolves only when TTS playback completes, so opt out of the
273
308
  // default request timeout (long text can outlast it). Settled by the host
@@ -281,7 +316,9 @@ export class SpeakerModule {
281
316
  volume: options.volume,
282
317
  stopOtherAudio: options.stopOtherAudio ?? false,
283
318
  enableSanitization: options.enableSanitization ?? true,
284
- forceLocal: options.forceLocal ?? false,
319
+ processing,
320
+ // Hosts before `processing` (Mentra App 3.4.0) read only this flag.
321
+ forceLocal: processing === "localOnly",
285
322
  },
286
323
  {timeoutMs: 0},
287
324
  )
@@ -312,9 +349,16 @@ export class SpeakerModule {
312
349
  ) {
313
350
  throw new RangeError("speaker stream volume must be between 0 and 1")
314
351
  }
352
+ if (
353
+ options.jitterMs !== undefined &&
354
+ (!Number.isFinite(options.jitterMs) || options.jitterMs < 80 || options.jitterMs > 500)
355
+ )
356
+ throw new RangeError("speaker stream jitterMs must be between 80 and 500")
315
357
  const res = await this.session.sendRequest<{streamId: string}>({
316
358
  type: MiniappRequestType.SPEAKER_STREAM_OPEN,
317
359
  sampleRate: options.sampleRate ?? 16000,
360
+ ...(options.jitterMs === undefined ? {} : {jitterMs: options.jitterMs}),
361
+ ...(options.prewarm === undefined ? {} : {prewarm: options.prewarm}),
318
362
  channels: options.channels ?? 1,
319
363
  volume: options.volume,
320
364
  stopOtherAudio: options.stopOtherAudio ?? true,
@@ -0,0 +1,42 @@
1
+ /**
2
+ * @fileoverview Where speech is processed: on the phone or in the cloud.
3
+ *
4
+ * Transcription listeners (`session.transcription.on` / `forLanguage`) and
5
+ * speech requests (`session.speaker.speak`) take the same `processing` option.
6
+ */
7
+
8
+ import {MiniappValidationError} from "./languages"
9
+
10
+ /**
11
+ * Where speech is processed.
12
+ *
13
+ * - `"preferCloud"` (default): the cloud; the phone's on-device models only
14
+ * while the cloud is unreachable.
15
+ * - `"preferLocal"`: on-device when a model for the language is ready,
16
+ * otherwise the cloud.
17
+ * - `"localOnly"`: on-device only; never the cloud.
18
+ */
19
+ export type SpeechProcessing = "preferCloud" | "preferLocal" | "localOnly"
20
+
21
+ const SPEECH_PROCESSING: readonly SpeechProcessing[] = ["preferCloud", "preferLocal", "localOnly"]
22
+
23
+ /** The `processing` option, with the deprecated `forceLocal` it replaces. */
24
+ export interface SpeechProcessingOptions {
25
+ processing?: SpeechProcessing
26
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0. */
27
+ forceLocal?: boolean
28
+ }
29
+
30
+ /**
31
+ * @internal The processing an options object asks for. `processing` wins;
32
+ * without it, the deprecated `forceLocal: true` means `"localOnly"`.
33
+ * Throws `MiniappValidationError` for an unknown `processing` value.
34
+ */
35
+ export function resolveSpeechProcessing(options: SpeechProcessingOptions, where: string): SpeechProcessing {
36
+ const {processing} = options
37
+ if (processing === undefined) return options.forceLocal === true ? "localOnly" : "preferCloud"
38
+ if (SPEECH_PROCESSING.includes(processing)) return processing
39
+ throw new MiniappValidationError(
40
+ `${where}: unknown processing ${JSON.stringify(processing)}. Use ${SPEECH_PROCESSING.map((value) => `"${value}"`).join(", ")}.`,
41
+ )
42
+ }