@mentra/miniapp 3.4.0-dev.1181 → 3.4.0-dev.1183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +2 -2
  2. package/dist/background/index.d.ts +1 -0
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js.map +1 -1
  5. package/dist/index.d.ts +1 -0
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/modules/events.d.ts +12 -6
  9. package/dist/modules/events.d.ts.map +1 -1
  10. package/dist/modules/events.js +53 -47
  11. package/dist/modules/events.js.map +1 -1
  12. package/dist/modules/speaker.d.ts +19 -9
  13. package/dist/modules/speaker.d.ts.map +1 -1
  14. package/dist/modules/speaker.js +10 -5
  15. package/dist/modules/speaker.js.map +1 -1
  16. package/dist/modules/speechProcessing.d.ts +29 -0
  17. package/dist/modules/speechProcessing.d.ts.map +1 -0
  18. package/dist/modules/speechProcessing.js +22 -0
  19. package/dist/modules/speechProcessing.js.map +1 -0
  20. package/dist/modules/transcription.d.ts +27 -10
  21. package/dist/modules/transcription.d.ts.map +1 -1
  22. package/dist/modules/transcription.js +14 -6
  23. package/dist/modules/transcription.js.map +1 -1
  24. package/dist/protocol.d.ts +12 -3
  25. package/dist/protocol.d.ts.map +1 -1
  26. package/dist/protocol.js +12 -3
  27. package/dist/protocol.js.map +1 -1
  28. package/dist/session.d.ts +2 -3
  29. package/dist/session.d.ts.map +1 -1
  30. package/dist/session.js.map +1 -1
  31. package/package.json +2 -2
  32. package/src/background/index.ts +1 -0
  33. package/src/index.ts +1 -0
  34. package/src/modules/events.ts +61 -51
  35. package/src/modules/speaker.ts +23 -10
  36. package/src/modules/speechProcessing.ts +42 -0
  37. package/src/modules/transcription.ts +35 -16
  38. package/src/protocol.ts +12 -3
  39. package/src/session.ts +2 -1
@@ -33,10 +33,16 @@ import {EventEmitter} from "eventemitter3"
33
33
  import {makeRequestId} from "../envelope"
34
34
  import {MiniappRequestType, MiniappStreamType} from "../protocol"
35
35
  import {MiniappSession} from "../session"
36
+ import {resolveSpeechProcessing, type SpeechProcessing, type SpeechProcessingOptions} from "./speechProcessing"
36
37
 
37
38
  export type UnsubscribeFn = () => void
38
39
 
39
- /** @internal Host-selected delivery target for a transcription event. */
40
+ /**
41
+ * @internal Delivery class the host tags a transcription event with. Hosts
42
+ * address each event to the listeners that should get it; SDKs from before
43
+ * `processing` filter on this tag instead: `"forceLocal"` reaches `localOnly`
44
+ * listeners, `"default"` the others, `"all"` everyone.
45
+ */
40
46
  export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
41
47
 
42
48
  // ---------------------------------------------------------------------------
@@ -48,6 +54,8 @@ export interface TranscriptionData {
48
54
  text: string
49
55
  isFinal: boolean
50
56
  language?: string
57
+ /** Where the text was transcribed: `"cloud"`, or `"local"` for the phone's on-device model. */
58
+ source: "cloud" | "local"
51
59
  }
52
60
 
53
61
  export interface TranslationData {
@@ -177,14 +185,12 @@ export class EventManager {
177
185
  private listenerSeq = 0
178
186
  private transcriptionListeners = new Map<
179
187
  string,
180
- {stream: string; forceLocal: boolean; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
188
+ {stream: string; processing: SpeechProcessing; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
181
189
  >()
182
190
  private retireListeners = new Set<() => void>()
183
191
  private readonly emitter = new EventEmitter()
184
192
  /** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
185
193
  private readonly refCounts = new Map<string, number>()
186
- /** Transcription stream -> refs that require on-device-only processing. */
187
- private readonly forceLocalRefCounts = new Map<string, number>()
188
194
 
189
195
  constructor(private readonly session: MiniappSession) {}
190
196
 
@@ -199,20 +205,24 @@ export class EventManager {
199
205
  * the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
200
206
  * via a separate envelope `type`; the EventManager only routes them
201
207
  * locally.
208
+ *
209
+ * `processing` applies to transcription streams only.
202
210
  */
203
- subscribe(stream: string, handler: (data: unknown) => void, options: {forceLocal?: boolean} = {}): UnsubscribeFn {
211
+ subscribe(stream: string, handler: (data: unknown) => void, options: SpeechProcessingOptions = {}): UnsubscribeFn {
204
212
  const isInternal = stream.startsWith("_")
205
- const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
206
213
  const isTranscription = stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
214
+ const processing = isTranscription ? resolveSpeechProcessing(options, "events.subscribe(options)") : "preferCloud"
215
+ const localOnly = processing === "localOnly"
207
216
  const listenerId = isTranscription ? `${this.listenerScope}:${++this.listenerSeq}` : undefined
208
217
  let active = true
209
218
  const listener = (data: unknown, route?: TranscriptionEventRoute) => {
210
219
  if (!active) return
211
- if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) {
220
+ if (isTranscription) {
221
+ // The host tags every transcription event with the class it is for.
212
222
  // Hosts that predate routed transcription events omit `route`. Treat
213
- // those events as the default/cloud path so a forceLocal listener can
214
- // never accidentally receive cloud text.
215
- if (forceLocal ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
223
+ // those events as the cloud path so a localOnly listener can never
224
+ // accidentally receive cloud text.
225
+ if (localOnly ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
216
226
  }
217
227
  handler(data)
218
228
  }
@@ -221,21 +231,12 @@ export class EventManager {
221
231
  active = false
222
232
  }
223
233
  this.retireListeners.add(retire)
224
- if (listenerId) this.transcriptionListeners.set(listenerId, {stream, forceLocal, invoke: listener})
234
+ if (listenerId) this.transcriptionListeners.set(listenerId, {stream, processing, invoke: listener})
225
235
  if (!isInternal) {
226
236
  const before = this.refCounts.get(stream) ?? 0
227
237
  this.refCounts.set(stream, before + 1)
228
- const forceLocalBefore = this.forceLocalRefCounts.get(stream) ?? 0
229
- const defaultBefore = before - forceLocalBefore
230
- if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
231
- if (
232
- isTranscription ||
233
- before === 0 ||
234
- (forceLocal && forceLocalBefore === 0) ||
235
- (!forceLocal && defaultBefore === 0)
236
- ) {
237
- this.sendSubscriptionUpdate()
238
- }
238
+ // Every transcription listener is registered with the host by id.
239
+ if (isTranscription || before === 0) this.sendSubscriptionUpdate()
239
240
  }
240
241
  return () => {
241
242
  if (!active) return
@@ -245,24 +246,12 @@ export class EventManager {
245
246
  this.emitter.off(stream, listener)
246
247
  if (isInternal) return
247
248
  const current = this.refCounts.get(stream) ?? 0
248
- const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
249
- let subscriptionChanged = current <= 1
250
249
  if (current <= 1) {
251
250
  this.refCounts.delete(stream)
252
251
  } else {
253
252
  this.refCounts.set(stream, current - 1)
254
253
  }
255
- if (forceLocal) {
256
- if (forceLocalCurrent <= 1) {
257
- this.forceLocalRefCounts.delete(stream)
258
- subscriptionChanged = true
259
- } else {
260
- this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
261
- }
262
- } else if (current - forceLocalCurrent <= 1) {
263
- subscriptionChanged = true
264
- }
265
- if (isTranscription || subscriptionChanged) this.sendSubscriptionUpdate()
254
+ if (isTranscription || current <= 1) this.sendSubscriptionUpdate()
266
255
  }
267
256
  }
268
257
 
@@ -273,7 +262,6 @@ export class EventManager {
273
262
  this.transcriptionListeners.clear()
274
263
  this.emitter.removeAllListeners()
275
264
  this.refCounts.clear()
276
- this.forceLocalRefCounts.clear()
277
265
  this.sendSubscriptionUpdate()
278
266
  }
279
267
 
@@ -288,6 +276,7 @@ export class EventManager {
288
276
  transcriptionRoute?: TranscriptionEventRoute,
289
277
  listenerId?: string,
290
278
  ): void {
279
+ if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) data = withTranscriptionSource(data)
291
280
  if (listenerId !== undefined) {
292
281
  const listener = this.transcriptionListeners.get(listenerId)
293
282
  if (listener && (listener.stream === stream || listener.stream === "transcription:auto"))
@@ -322,27 +311,48 @@ export class EventManager {
322
311
  }
323
312
 
324
313
  private sendSubscriptionUpdate(): void {
325
- const subscriptions = Array.from(this.refCounts.keys()).map((stream) => {
326
- if (stream === MiniappStreamType.LOCATION_UPDATE) return {stream: "location_stream", rate: "realtime"}
327
-
328
- const forceLocalRefs = this.forceLocalRefCounts.get(stream) ?? 0
329
- if (forceLocalRefs === 0) return stream
330
-
331
- const totalRefs = this.refCounts.get(stream) ?? 0
332
- return {
333
- stream,
334
- forceLocal: true,
335
- ...(totalRefs > forceLocalRefs ? {includeCloud: true} : {}),
336
- }
337
- })
314
+ const subscriptions: Array<string | Record<string, unknown>> = []
315
+ for (const stream of this.refCounts.keys()) {
316
+ if (stream === MiniappStreamType.LOCATION_UPDATE)
317
+ subscriptions.push({stream: "location_stream", rate: "realtime"})
318
+ else if (!stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) subscriptions.push(stream)
319
+ }
320
+ // One entry per transcription stream and processing its listeners ask for.
321
+ const transcription = new Set<string>()
322
+ for (const {stream, processing} of this.transcriptionListeners.values()) {
323
+ const key = `${processing} ${stream}`
324
+ if (transcription.has(key)) continue
325
+ transcription.add(key)
326
+ subscriptions.push({stream, ...processingOnWire(processing)})
327
+ }
338
328
  this.session.sendOneShot({
339
329
  type: MiniappRequestType.SUBSCRIBE,
340
330
  subscriptions,
341
- transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, forceLocal}]) => ({
331
+ transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, processing}]) => ({
342
332
  id,
343
333
  stream,
344
- forceLocal,
334
+ ...processingOnWire(processing),
345
335
  })),
346
336
  })
347
337
  }
348
338
  }
339
+
340
+ /**
341
+ * The wire form of a transcription listener's processing. Hosts before
342
+ * `processing` (Mentra App 3.4.0) read only `forceLocal`, so `localOnly` keeps
343
+ * sending it: there a localOnly listener still never gets cloud text.
344
+ */
345
+ function processingOnWire(processing: SpeechProcessing): {processing: SpeechProcessing; forceLocal?: true} {
346
+ return processing === "localOnly" ? {processing, forceLocal: true} : {processing}
347
+ }
348
+
349
+ /**
350
+ * Hosts before `source` (Mentra App 3.4.0) leave it out. Their on-device
351
+ * transcripts carry `type: "local_transcription"`, their cloud ones do not.
352
+ */
353
+ function withTranscriptionSource(data: unknown): unknown {
354
+ if (typeof data !== "object" || data === null) return data
355
+ const event = data as {source?: unknown; type?: unknown}
356
+ if (event.source === "cloud" || event.source === "local") return data
357
+ return {...event, source: event.type === "local_transcription" ? "local" : "cloud"}
358
+ }
@@ -8,11 +8,12 @@
8
8
  * speaker.play({audioUrl}) — play an http(s) or data: URL, or one of this
9
9
  * miniapp's blobs, via the phone's
10
10
  * AudioPlaybackService.
11
- * speaker.speak(text) — send a SPEAK request. Phone streams cloud
12
- * TTS when connected and falls back to local
13
- * offline TTS when cloud is unavailable.
11
+ * speaker.speak(text) — send a SPEAK request. By default the phone
12
+ * streams cloud TTS when connected and falls
13
+ * back to its on-device voice; the
14
+ * `processing` option picks the order.
14
15
  * Resolves when playback completes; rejects
15
- * with a TTS_* error code on cloud failure.
16
+ * with a TTS_* error code on failure.
16
17
  * speaker.createStream(opts) — open a live PCM output stream (16-bit LE).
17
18
  * Returns a SpeakerStreamWriter: write() PCM
18
19
  * chunks as they arrive (e.g. from a WebSocket
@@ -39,6 +40,7 @@ import {MiniappErrorCode, MiniappRequestType} from "../protocol"
39
40
  import {MiniappSession} from "../session"
40
41
  import {bytesToBase64, toUint8Array} from "./base64"
41
42
  import type {UnsubscribeFn} from "./events"
43
+ import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
42
44
 
43
45
  export interface PlayAudioOptions {
44
46
  /**
@@ -63,10 +65,17 @@ export interface SpeakOptions {
63
65
  */
64
66
  enableSanitization?: boolean
65
67
  /**
66
- * Force this call to use on-device offline TTS, skipping cloud TTS
67
- * entirely — even when cloud is connected. Rejects with `TTS_LOCAL_UNAVAILABLE`
68
- * if the offline model isn't ready instead of falling back to cloud.
68
+ * Where the speech is synthesized. Default `"preferCloud"`.
69
+ *
70
+ * - `"preferCloud"`: cloud TTS while the cloud is connected, otherwise the
71
+ * phone's on-device voice.
72
+ * - `"preferLocal"`: the on-device voice when its model is ready and can say
73
+ * it (an explicit cloud `voice_id` cannot), otherwise cloud TTS.
74
+ * - `"localOnly"`: the on-device voice only; never the cloud. Rejects with
75
+ * `TTS_LOCAL_UNAVAILABLE` when the on-device model isn't ready.
69
76
  */
77
+ processing?: SpeechProcessing
78
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
70
79
  forceLocal?: boolean
71
80
  }
72
81
 
@@ -279,8 +288,9 @@ export class SpeakerModule {
279
288
  }
280
289
 
281
290
  /**
282
- * Speak text through the phone. The host streams cloud TTS when connected,
283
- * then falls back to local offline TTS.
291
+ * Speak text through the phone, in the cloud or on-device as
292
+ * `options.processing` says (default: cloud TTS when connected, then the
293
+ * on-device voice).
284
294
  *
285
295
  * Rejects with a MiniappRequestError containing a `code` field on cloud-side
286
296
  * TTS failures: `TTS_TEXT_TOO_LONG`, `TTS_INVALID_VOICE`, `TTS_UPSTREAM_ERROR`.
@@ -292,6 +302,7 @@ export class SpeakerModule {
292
302
  if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
293
303
  throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
294
304
  }
305
+ const processing = resolveSpeechProcessing(options, "speaker.speak(options)")
295
306
  try {
296
307
  // Like play(): resolves only when TTS playback completes, so opt out of the
297
308
  // default request timeout (long text can outlast it). Settled by the host
@@ -305,7 +316,9 @@ export class SpeakerModule {
305
316
  volume: options.volume,
306
317
  stopOtherAudio: options.stopOtherAudio ?? false,
307
318
  enableSanitization: options.enableSanitization ?? true,
308
- forceLocal: options.forceLocal ?? false,
319
+ processing,
320
+ // Hosts before `processing` (Mentra App 3.4.0) read only this flag.
321
+ forceLocal: processing === "localOnly",
309
322
  },
310
323
  {timeoutMs: 0},
311
324
  )
@@ -0,0 +1,42 @@
1
+ /**
2
+ * @fileoverview Where speech is processed: on the phone or in the cloud.
3
+ *
4
+ * Transcription listeners (`session.transcription.on` / `forLanguage`) and
5
+ * speech requests (`session.speaker.speak`) take the same `processing` option.
6
+ */
7
+
8
+ import {MiniappValidationError} from "./languages"
9
+
10
+ /**
11
+ * Where speech is processed.
12
+ *
13
+ * - `"preferCloud"` (default): the cloud; the phone's on-device models only
14
+ * while the cloud is unreachable.
15
+ * - `"preferLocal"`: on-device when a model for the language is ready,
16
+ * otherwise the cloud.
17
+ * - `"localOnly"`: on-device only; never the cloud.
18
+ */
19
+ export type SpeechProcessing = "preferCloud" | "preferLocal" | "localOnly"
20
+
21
+ const SPEECH_PROCESSING: readonly SpeechProcessing[] = ["preferCloud", "preferLocal", "localOnly"]
22
+
23
+ /** The `processing` option, with the deprecated `forceLocal` it replaces. */
24
+ export interface SpeechProcessingOptions {
25
+ processing?: SpeechProcessing
26
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0. */
27
+ forceLocal?: boolean
28
+ }
29
+
30
+ /**
31
+ * @internal The processing an options object asks for. `processing` wins;
32
+ * without it, the deprecated `forceLocal: true` means `"localOnly"`.
33
+ * Throws `MiniappValidationError` for an unknown `processing` value.
34
+ */
35
+ export function resolveSpeechProcessing(options: SpeechProcessingOptions, where: string): SpeechProcessing {
36
+ const {processing} = options
37
+ if (processing === undefined) return options.forceLocal === true ? "localOnly" : "preferCloud"
38
+ if (SPEECH_PROCESSING.includes(processing)) return processing
39
+ throw new MiniappValidationError(
40
+ `${where}: unknown processing ${JSON.stringify(processing)}. Use ${SPEECH_PROCESSING.map((value) => `"${value}"`).join(", ")}.`,
41
+ )
42
+ }
@@ -6,6 +6,7 @@
6
6
  * session.transcription.on(handler) // auto-detect
7
7
  * session.transcription.forLanguage("en-US", handler) // single language
8
8
  * session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
9
+ * session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
9
10
  * session.transcription.configure({languageHints, vocabulary, diarization})
10
11
  * session.transcription.onModelStatus(handler) // on-device model state
11
12
  * session.transcription.finishUtterance() // end the on-device line now
@@ -25,6 +26,7 @@ import {MiniappSession} from "../session"
25
26
  import type {TranscriptionData, UnsubscribeFn} from "./events"
26
27
  import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
27
28
  import type {LanguageHint, TranscriptionLanguage} from "./languages"
29
+ import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
28
30
 
29
31
  /** Configuration for cloud-side transcription behavior. */
30
32
  export interface TranscriptionConfig {
@@ -43,11 +45,26 @@ export interface TranscriptionConfig {
43
45
 
44
46
  export interface TranscriptionOptions {
45
47
  /**
46
- * Use only on-device STT for this listener; never deliver cloud transcripts.
47
- * Other listeners on the same stream keep their own routing choice. The
48
- * phone uses the speech model picked on its Speech page and downloads it if
49
- * needed; follow that with `onModelStatus`.
48
+ * Where this listener's speech is transcribed. Default `"preferCloud"`.
49
+ *
50
+ * - `"preferCloud"`: the cloud; the phone's on-device model only while the
51
+ * cloud is unreachable.
52
+ * - `"preferLocal"`: the on-device model while it runs in this listener's
53
+ * language, otherwise the cloud. `on()` follows the model's language,
54
+ * unless `configure({languageHints})` names one the model can't
55
+ * transcribe; then it stays on the cloud.
56
+ * - `"localOnly"`: on-device only; never the cloud.
57
+ *
58
+ * The phone runs one on-device model, the one picked on its Speech page, and
59
+ * downloads it when a `preferLocal` or `localOnly` listener needs it; follow
60
+ * it with `onModelStatus`. A listener moves between the cloud and on-device
61
+ * during a pause in speech, once both have finished with what was said
62
+ * before it, so it never misses or repeats speech; `TranscriptionData.source`
63
+ * says where each result came from. Each listener routes on its own: other
64
+ * listeners on the same stream keep their choice.
50
65
  */
66
+ processing?: SpeechProcessing
67
+ /** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
51
68
  forceLocal?: boolean
52
69
  }
53
70
 
@@ -64,8 +81,8 @@ export interface TranscriptionModelStatus {
64
81
  * - `loading`: the model is loading into memory.
65
82
  * - `ready`: on-device transcripts are flowing.
66
83
  * - `unavailable`: no on-device transcription right now; `reason` says why
67
- * (for example no on-device subscription, a failed download, or a
68
- * subscription in a language other than the picked model's).
84
+ * (for example no `preferLocal` or `localOnly` listener, a failed
85
+ * download, or a listener in a language other than the picked model's).
69
86
  */
70
87
  state: "downloading" | "loading" | "ready" | "unavailable"
71
88
  progress?: number
@@ -90,8 +107,11 @@ export class TranscriptionModule {
90
107
  * event) preserves existing semantics.
91
108
  */
92
109
  on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
110
+ const processing = resolveSpeechProcessing(options, "transcription.on(options)")
93
111
  return this.track(
94
- this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, options),
112
+ this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, {
113
+ processing,
114
+ }),
95
115
  )
96
116
  }
97
117
 
@@ -117,16 +137,15 @@ export class TranscriptionModule {
117
137
  const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
118
138
  requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
119
139
  )
140
+ const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)")
120
141
  if (langs.length === 0) return () => {}
121
142
 
122
143
  const unsubs: UnsubscribeFn[] = []
123
144
  for (const lang of langs) {
124
145
  unsubs.push(
125
- this.session._subscribe(
126
- `${MiniappStreamType.TRANSCRIPTION}:${lang}`,
127
- handler as (data: unknown) => void,
128
- options,
129
- ),
146
+ this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void, {
147
+ processing,
148
+ }),
130
149
  )
131
150
  }
132
151
  const combined: UnsubscribeFn = () => {
@@ -210,10 +229,10 @@ export class TranscriptionModule {
210
229
  * model can't finish early (models that transcribe in fixed chunks); their
211
230
  * final text still arrives on its own.
212
231
  *
213
- * Applies only to on-device transcription and needs an active
214
- * `{forceLocal: true}` subscription. The phone runs one recognizer for every
215
- * miniapp, so this ends the current line for every on-device subscriber,
216
- * Captions included.
232
+ * Applies only to on-device transcription and needs a `localOnly` listener
233
+ * or one that is getting on-device text right now. The phone runs one
234
+ * recognizer for every miniapp, so this ends the current line for every
235
+ * on-device subscriber, Captions included.
217
236
  */
218
237
  finishUtterance(): Promise<string | null> {
219
238
  return this.session.sendRequest<string | null>({type: MiniappRequestType.TRANSCRIPTION_FINISH})
package/src/protocol.ts CHANGED
@@ -26,7 +26,16 @@ export enum MiniappRequestType {
26
26
  /** Request a fresh miniapp-scoped backend auth token. */
27
27
  AUTH_REFRESH = "miniapp_auth_refresh",
28
28
 
29
- /** Update the set of streams the miniapp is subscribed to. */
29
+ /**
30
+ * Update the set of streams the miniapp is subscribed to. `subscriptions`
31
+ * lists stream names; a transcription stream is `{stream, processing}`, one
32
+ * entry per processing its listeners ask for, and `transcriptionListeners`
33
+ * lists every transcription listener as `{id, stream, processing}`. Both
34
+ * also carry `forceLocal: true` for `localOnly`, the form hosts before
35
+ * `processing` read. Hosts accept that older form,
36
+ * `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
37
+ * `processing`.
38
+ */
30
39
  SUBSCRIBE = "miniapp_subscribe",
31
40
 
32
41
  /**
@@ -45,7 +54,7 @@ export enum MiniappRequestType {
45
54
  /** Stop any audio playback this miniapp initiated. */
46
55
  STOP_AUDIO = "miniapp_stop_audio",
47
56
 
48
- /** Speak text via cloud TTS — phone constructs the URL. */
57
+ /** Speak text via cloud or on-device TTS, as its `processing` says. */
49
58
  SPEAK = "miniapp_speak",
50
59
 
51
60
  /**
@@ -451,7 +460,7 @@ export enum MiniappErrorCode {
451
460
  TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
452
461
  TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
453
462
 
454
- /** `speak({forceLocal: true})` but the on-device offline TTS model isn't downloaded/ready. */
463
+ /** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
455
464
  TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
456
465
 
457
466
  /** Not connected / pre-ACK and transport closed. */
package/src/session.ts CHANGED
@@ -26,6 +26,7 @@ import {ConfigurationModule} from "./modules/configuration"
26
26
  import {DashboardAPI} from "./modules/dashboard"
27
27
  import {DisplayManager} from "./modules/display"
28
28
  import {EventManager, type TranscriptionEventRoute, type UnsubscribeFn} from "./modules/events"
29
+ import type {SpeechProcessingOptions} from "./modules/speechProcessing"
29
30
  import {GlassesModule} from "./modules/glasses"
30
31
  import {HeadingModule} from "./modules/heading"
31
32
  import {ImuModule} from "./modules/imu"
@@ -483,7 +484,7 @@ export class MiniappSession<TChannels extends object = any> {
483
484
  _subscribe(
484
485
  streamType: string,
485
486
  handler: (data: unknown) => void,
486
- options: {forceLocal?: boolean} = {},
487
+ options: SpeechProcessingOptions = {},
487
488
  ): UnsubscribeFn {
488
489
  return this.events.subscribe(streamType, handler, options)
489
490
  }