@mentra/miniapp 3.4.0-dev.1182 → 3.4.0-dev.1184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/background/index.d.ts +1 -0
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/events.d.ts +12 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/speaker.d.ts +19 -9
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +10 -5
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/transcription.d.ts +27 -10
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +14 -6
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +12 -3
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +12 -3
- package/dist/protocol.js.map +1 -1
- package/dist/session.d.ts +2 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +1 -0
- package/src/index.ts +1 -0
- package/src/modules/events.ts +61 -51
- package/src/modules/speaker.ts +23 -10
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/transcription.ts +35 -16
- package/src/protocol.ts +12 -3
- package/src/session.ts +2 -1
package/src/modules/events.ts
CHANGED
|
@@ -33,10 +33,16 @@ import {EventEmitter} from "eventemitter3"
|
|
|
33
33
|
import {makeRequestId} from "../envelope"
|
|
34
34
|
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
35
35
|
import {MiniappSession} from "../session"
|
|
36
|
+
import {resolveSpeechProcessing, type SpeechProcessing, type SpeechProcessingOptions} from "./speechProcessing"
|
|
36
37
|
|
|
37
38
|
export type UnsubscribeFn = () => void
|
|
38
39
|
|
|
39
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* @internal Delivery class the host tags a transcription event with. Hosts
|
|
42
|
+
* address each event to the listeners that should get it; SDKs from before
|
|
43
|
+
* `processing` filter on this tag instead: `"forceLocal"` reaches `localOnly`
|
|
44
|
+
* listeners, `"default"` the others, `"all"` everyone.
|
|
45
|
+
*/
|
|
40
46
|
export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
|
|
41
47
|
|
|
42
48
|
// ---------------------------------------------------------------------------
|
|
@@ -48,6 +54,8 @@ export interface TranscriptionData {
|
|
|
48
54
|
text: string
|
|
49
55
|
isFinal: boolean
|
|
50
56
|
language?: string
|
|
57
|
+
/** Where the text was transcribed: `"cloud"`, or `"local"` for the phone's on-device model. */
|
|
58
|
+
source: "cloud" | "local"
|
|
51
59
|
}
|
|
52
60
|
|
|
53
61
|
export interface TranslationData {
|
|
@@ -177,14 +185,12 @@ export class EventManager {
|
|
|
177
185
|
private listenerSeq = 0
|
|
178
186
|
private transcriptionListeners = new Map<
|
|
179
187
|
string,
|
|
180
|
-
{stream: string;
|
|
188
|
+
{stream: string; processing: SpeechProcessing; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
|
|
181
189
|
>()
|
|
182
190
|
private retireListeners = new Set<() => void>()
|
|
183
191
|
private readonly emitter = new EventEmitter()
|
|
184
192
|
/** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
|
|
185
193
|
private readonly refCounts = new Map<string, number>()
|
|
186
|
-
/** Transcription stream -> refs that require on-device-only processing. */
|
|
187
|
-
private readonly forceLocalRefCounts = new Map<string, number>()
|
|
188
194
|
|
|
189
195
|
constructor(private readonly session: MiniappSession) {}
|
|
190
196
|
|
|
@@ -199,20 +205,24 @@ export class EventManager {
|
|
|
199
205
|
* the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
|
|
200
206
|
* via a separate envelope `type`; the EventManager only routes them
|
|
201
207
|
* locally.
|
|
208
|
+
*
|
|
209
|
+
* `processing` applies to transcription streams only.
|
|
202
210
|
*/
|
|
203
|
-
subscribe(stream: string, handler: (data: unknown) => void, options:
|
|
211
|
+
subscribe(stream: string, handler: (data: unknown) => void, options: SpeechProcessingOptions = {}): UnsubscribeFn {
|
|
204
212
|
const isInternal = stream.startsWith("_")
|
|
205
|
-
const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
206
213
|
const isTranscription = stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
214
|
+
const processing = isTranscription ? resolveSpeechProcessing(options, "events.subscribe(options)") : "preferCloud"
|
|
215
|
+
const localOnly = processing === "localOnly"
|
|
207
216
|
const listenerId = isTranscription ? `${this.listenerScope}:${++this.listenerSeq}` : undefined
|
|
208
217
|
let active = true
|
|
209
218
|
const listener = (data: unknown, route?: TranscriptionEventRoute) => {
|
|
210
219
|
if (!active) return
|
|
211
|
-
if (
|
|
220
|
+
if (isTranscription) {
|
|
221
|
+
// The host tags every transcription event with the class it is for.
|
|
212
222
|
// Hosts that predate routed transcription events omit `route`. Treat
|
|
213
|
-
// those events as the
|
|
214
|
-
//
|
|
215
|
-
if (
|
|
223
|
+
// those events as the cloud path so a localOnly listener can never
|
|
224
|
+
// accidentally receive cloud text.
|
|
225
|
+
if (localOnly ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
|
|
216
226
|
}
|
|
217
227
|
handler(data)
|
|
218
228
|
}
|
|
@@ -221,21 +231,12 @@ export class EventManager {
|
|
|
221
231
|
active = false
|
|
222
232
|
}
|
|
223
233
|
this.retireListeners.add(retire)
|
|
224
|
-
if (listenerId) this.transcriptionListeners.set(listenerId, {stream,
|
|
234
|
+
if (listenerId) this.transcriptionListeners.set(listenerId, {stream, processing, invoke: listener})
|
|
225
235
|
if (!isInternal) {
|
|
226
236
|
const before = this.refCounts.get(stream) ?? 0
|
|
227
237
|
this.refCounts.set(stream, before + 1)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
|
|
231
|
-
if (
|
|
232
|
-
isTranscription ||
|
|
233
|
-
before === 0 ||
|
|
234
|
-
(forceLocal && forceLocalBefore === 0) ||
|
|
235
|
-
(!forceLocal && defaultBefore === 0)
|
|
236
|
-
) {
|
|
237
|
-
this.sendSubscriptionUpdate()
|
|
238
|
-
}
|
|
238
|
+
// Every transcription listener is registered with the host by id.
|
|
239
|
+
if (isTranscription || before === 0) this.sendSubscriptionUpdate()
|
|
239
240
|
}
|
|
240
241
|
return () => {
|
|
241
242
|
if (!active) return
|
|
@@ -245,24 +246,12 @@ export class EventManager {
|
|
|
245
246
|
this.emitter.off(stream, listener)
|
|
246
247
|
if (isInternal) return
|
|
247
248
|
const current = this.refCounts.get(stream) ?? 0
|
|
248
|
-
const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
|
|
249
|
-
let subscriptionChanged = current <= 1
|
|
250
249
|
if (current <= 1) {
|
|
251
250
|
this.refCounts.delete(stream)
|
|
252
251
|
} else {
|
|
253
252
|
this.refCounts.set(stream, current - 1)
|
|
254
253
|
}
|
|
255
|
-
if (
|
|
256
|
-
if (forceLocalCurrent <= 1) {
|
|
257
|
-
this.forceLocalRefCounts.delete(stream)
|
|
258
|
-
subscriptionChanged = true
|
|
259
|
-
} else {
|
|
260
|
-
this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
|
|
261
|
-
}
|
|
262
|
-
} else if (current - forceLocalCurrent <= 1) {
|
|
263
|
-
subscriptionChanged = true
|
|
264
|
-
}
|
|
265
|
-
if (isTranscription || subscriptionChanged) this.sendSubscriptionUpdate()
|
|
254
|
+
if (isTranscription || current <= 1) this.sendSubscriptionUpdate()
|
|
266
255
|
}
|
|
267
256
|
}
|
|
268
257
|
|
|
@@ -273,7 +262,6 @@ export class EventManager {
|
|
|
273
262
|
this.transcriptionListeners.clear()
|
|
274
263
|
this.emitter.removeAllListeners()
|
|
275
264
|
this.refCounts.clear()
|
|
276
|
-
this.forceLocalRefCounts.clear()
|
|
277
265
|
this.sendSubscriptionUpdate()
|
|
278
266
|
}
|
|
279
267
|
|
|
@@ -288,6 +276,7 @@ export class EventManager {
|
|
|
288
276
|
transcriptionRoute?: TranscriptionEventRoute,
|
|
289
277
|
listenerId?: string,
|
|
290
278
|
): void {
|
|
279
|
+
if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) data = withTranscriptionSource(data)
|
|
291
280
|
if (listenerId !== undefined) {
|
|
292
281
|
const listener = this.transcriptionListeners.get(listenerId)
|
|
293
282
|
if (listener && (listener.stream === stream || listener.stream === "transcription:auto"))
|
|
@@ -322,27 +311,48 @@ export class EventManager {
|
|
|
322
311
|
}
|
|
323
312
|
|
|
324
313
|
private sendSubscriptionUpdate(): void {
|
|
325
|
-
const subscriptions
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
if (
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
314
|
+
const subscriptions: Array<string | Record<string, unknown>> = []
|
|
315
|
+
for (const stream of this.refCounts.keys()) {
|
|
316
|
+
if (stream === MiniappStreamType.LOCATION_UPDATE)
|
|
317
|
+
subscriptions.push({stream: "location_stream", rate: "realtime"})
|
|
318
|
+
else if (!stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) subscriptions.push(stream)
|
|
319
|
+
}
|
|
320
|
+
// One entry per transcription stream and processing its listeners ask for.
|
|
321
|
+
const transcription = new Set<string>()
|
|
322
|
+
for (const {stream, processing} of this.transcriptionListeners.values()) {
|
|
323
|
+
const key = `${processing} ${stream}`
|
|
324
|
+
if (transcription.has(key)) continue
|
|
325
|
+
transcription.add(key)
|
|
326
|
+
subscriptions.push({stream, ...processingOnWire(processing)})
|
|
327
|
+
}
|
|
338
328
|
this.session.sendOneShot({
|
|
339
329
|
type: MiniappRequestType.SUBSCRIBE,
|
|
340
330
|
subscriptions,
|
|
341
|
-
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream,
|
|
331
|
+
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, processing}]) => ({
|
|
342
332
|
id,
|
|
343
333
|
stream,
|
|
344
|
-
|
|
334
|
+
...processingOnWire(processing),
|
|
345
335
|
})),
|
|
346
336
|
})
|
|
347
337
|
}
|
|
348
338
|
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* The wire form of a transcription listener's processing. Hosts before
|
|
342
|
+
* `processing` (Mentra App 3.4.0) read only `forceLocal`, so `localOnly` keeps
|
|
343
|
+
* sending it: there a localOnly listener still never gets cloud text.
|
|
344
|
+
*/
|
|
345
|
+
function processingOnWire(processing: SpeechProcessing): {processing: SpeechProcessing; forceLocal?: true} {
|
|
346
|
+
return processing === "localOnly" ? {processing, forceLocal: true} : {processing}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Hosts before `source` (Mentra App 3.4.0) leave it out. Their on-device
|
|
351
|
+
* transcripts carry `type: "local_transcription"`, their cloud ones do not.
|
|
352
|
+
*/
|
|
353
|
+
function withTranscriptionSource(data: unknown): unknown {
|
|
354
|
+
if (typeof data !== "object" || data === null) return data
|
|
355
|
+
const event = data as {source?: unknown; type?: unknown}
|
|
356
|
+
if (event.source === "cloud" || event.source === "local") return data
|
|
357
|
+
return {...event, source: event.type === "local_transcription" ? "local" : "cloud"}
|
|
358
|
+
}
|
package/src/modules/speaker.ts
CHANGED
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* speaker.play({audioUrl}) — play an http(s) or data: URL, or one of this
|
|
9
9
|
* miniapp's blobs, via the phone's
|
|
10
10
|
* AudioPlaybackService.
|
|
11
|
-
* speaker.speak(text) — send a SPEAK request.
|
|
12
|
-
* TTS when connected and falls
|
|
13
|
-
*
|
|
11
|
+
* speaker.speak(text) — send a SPEAK request. By default the phone
|
|
12
|
+
* streams cloud TTS when connected and falls
|
|
13
|
+
* back to its on-device voice; the
|
|
14
|
+
* `processing` option picks the order.
|
|
14
15
|
* Resolves when playback completes; rejects
|
|
15
|
-
* with a TTS_* error code on
|
|
16
|
+
* with a TTS_* error code on failure.
|
|
16
17
|
* speaker.createStream(opts) — open a live PCM output stream (16-bit LE).
|
|
17
18
|
* Returns a SpeakerStreamWriter: write() PCM
|
|
18
19
|
* chunks as they arrive (e.g. from a WebSocket
|
|
@@ -39,6 +40,7 @@ import {MiniappErrorCode, MiniappRequestType} from "../protocol"
|
|
|
39
40
|
import {MiniappSession} from "../session"
|
|
40
41
|
import {bytesToBase64, toUint8Array} from "./base64"
|
|
41
42
|
import type {UnsubscribeFn} from "./events"
|
|
43
|
+
import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
|
|
42
44
|
|
|
43
45
|
export interface PlayAudioOptions {
|
|
44
46
|
/**
|
|
@@ -63,10 +65,17 @@ export interface SpeakOptions {
|
|
|
63
65
|
*/
|
|
64
66
|
enableSanitization?: boolean
|
|
65
67
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
68
|
+
* Where the speech is synthesized. Default `"preferCloud"`.
|
|
69
|
+
*
|
|
70
|
+
* - `"preferCloud"`: cloud TTS while the cloud is connected, otherwise the
|
|
71
|
+
* phone's on-device voice.
|
|
72
|
+
* - `"preferLocal"`: the on-device voice when its model is ready and can say
|
|
73
|
+
* it (an explicit cloud `voice_id` cannot), otherwise cloud TTS.
|
|
74
|
+
* - `"localOnly"`: the on-device voice only; never the cloud. Rejects with
|
|
75
|
+
* `TTS_LOCAL_UNAVAILABLE` when the on-device model isn't ready.
|
|
69
76
|
*/
|
|
77
|
+
processing?: SpeechProcessing
|
|
78
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
70
79
|
forceLocal?: boolean
|
|
71
80
|
}
|
|
72
81
|
|
|
@@ -279,8 +288,9 @@ export class SpeakerModule {
|
|
|
279
288
|
}
|
|
280
289
|
|
|
281
290
|
/**
|
|
282
|
-
* Speak text through the phone
|
|
283
|
-
*
|
|
291
|
+
* Speak text through the phone, in the cloud or on-device as
|
|
292
|
+
* `options.processing` says (default: cloud TTS when connected, then the
|
|
293
|
+
* on-device voice).
|
|
284
294
|
*
|
|
285
295
|
* Rejects with a MiniappRequestError containing a `code` field on cloud-side
|
|
286
296
|
* TTS failures: `TTS_TEXT_TOO_LONG`, `TTS_INVALID_VOICE`, `TTS_UPSTREAM_ERROR`.
|
|
@@ -292,6 +302,7 @@ export class SpeakerModule {
|
|
|
292
302
|
if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
|
|
293
303
|
throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
|
|
294
304
|
}
|
|
305
|
+
const processing = resolveSpeechProcessing(options, "speaker.speak(options)")
|
|
295
306
|
try {
|
|
296
307
|
// Like play(): resolves only when TTS playback completes, so opt out of the
|
|
297
308
|
// default request timeout (long text can outlast it). Settled by the host
|
|
@@ -305,7 +316,9 @@ export class SpeakerModule {
|
|
|
305
316
|
volume: options.volume,
|
|
306
317
|
stopOtherAudio: options.stopOtherAudio ?? false,
|
|
307
318
|
enableSanitization: options.enableSanitization ?? true,
|
|
308
|
-
|
|
319
|
+
processing,
|
|
320
|
+
// Hosts before `processing` (Mentra App 3.4.0) read only this flag.
|
|
321
|
+
forceLocal: processing === "localOnly",
|
|
309
322
|
},
|
|
310
323
|
{timeoutMs: 0},
|
|
311
324
|
)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Where speech is processed: on the phone or in the cloud.
|
|
3
|
+
*
|
|
4
|
+
* Transcription listeners (`session.transcription.on` / `forLanguage`) and
|
|
5
|
+
* speech requests (`session.speaker.speak`) take the same `processing` option.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import {MiniappValidationError} from "./languages"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Where speech is processed.
|
|
12
|
+
*
|
|
13
|
+
* - `"preferCloud"` (default): the cloud; the phone's on-device models only
|
|
14
|
+
* while the cloud is unreachable.
|
|
15
|
+
* - `"preferLocal"`: on-device when a model for the language is ready,
|
|
16
|
+
* otherwise the cloud.
|
|
17
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
18
|
+
*/
|
|
19
|
+
export type SpeechProcessing = "preferCloud" | "preferLocal" | "localOnly"
|
|
20
|
+
|
|
21
|
+
const SPEECH_PROCESSING: readonly SpeechProcessing[] = ["preferCloud", "preferLocal", "localOnly"]
|
|
22
|
+
|
|
23
|
+
/** The `processing` option, with the deprecated `forceLocal` it replaces. */
|
|
24
|
+
export interface SpeechProcessingOptions {
|
|
25
|
+
processing?: SpeechProcessing
|
|
26
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0. */
|
|
27
|
+
forceLocal?: boolean
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @internal The processing an options object asks for. `processing` wins;
|
|
32
|
+
* without it, the deprecated `forceLocal: true` means `"localOnly"`.
|
|
33
|
+
* Throws `MiniappValidationError` for an unknown `processing` value.
|
|
34
|
+
*/
|
|
35
|
+
export function resolveSpeechProcessing(options: SpeechProcessingOptions, where: string): SpeechProcessing {
|
|
36
|
+
const {processing} = options
|
|
37
|
+
if (processing === undefined) return options.forceLocal === true ? "localOnly" : "preferCloud"
|
|
38
|
+
if (SPEECH_PROCESSING.includes(processing)) return processing
|
|
39
|
+
throw new MiniappValidationError(
|
|
40
|
+
`${where}: unknown processing ${JSON.stringify(processing)}. Use ${SPEECH_PROCESSING.map((value) => `"${value}"`).join(", ")}.`,
|
|
41
|
+
)
|
|
42
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* session.transcription.on(handler) // auto-detect
|
|
7
7
|
* session.transcription.forLanguage("en-US", handler) // single language
|
|
8
8
|
* session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
|
|
9
|
+
* session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
|
|
9
10
|
* session.transcription.configure({languageHints, vocabulary, diarization})
|
|
10
11
|
* session.transcription.onModelStatus(handler) // on-device model state
|
|
11
12
|
* session.transcription.finishUtterance() // end the on-device line now
|
|
@@ -25,6 +26,7 @@ import {MiniappSession} from "../session"
|
|
|
25
26
|
import type {TranscriptionData, UnsubscribeFn} from "./events"
|
|
26
27
|
import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
|
|
27
28
|
import type {LanguageHint, TranscriptionLanguage} from "./languages"
|
|
29
|
+
import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
|
|
28
30
|
|
|
29
31
|
/** Configuration for cloud-side transcription behavior. */
|
|
30
32
|
export interface TranscriptionConfig {
|
|
@@ -43,11 +45,26 @@ export interface TranscriptionConfig {
|
|
|
43
45
|
|
|
44
46
|
export interface TranscriptionOptions {
|
|
45
47
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
48
|
+
* Where this listener's speech is transcribed. Default `"preferCloud"`.
|
|
49
|
+
*
|
|
50
|
+
* - `"preferCloud"`: the cloud; the phone's on-device model only while the
|
|
51
|
+
* cloud is unreachable.
|
|
52
|
+
* - `"preferLocal"`: the on-device model while it runs in this listener's
|
|
53
|
+
* language, otherwise the cloud. `on()` follows the model's language,
|
|
54
|
+
* unless `configure({languageHints})` names one the model can't
|
|
55
|
+
* transcribe; then it stays on the cloud.
|
|
56
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
57
|
+
*
|
|
58
|
+
* The phone runs one on-device model, the one picked on its Speech page, and
|
|
59
|
+
* downloads it when a `preferLocal` or `localOnly` listener needs it; follow
|
|
60
|
+
* it with `onModelStatus`. A listener moves between the cloud and on-device
|
|
61
|
+
* during a pause in speech, once both have finished with what was said
|
|
62
|
+
* before it, so it never misses or repeats speech; `TranscriptionData.source`
|
|
63
|
+
* says where each result came from. Each listener routes on its own: other
|
|
64
|
+
* listeners on the same stream keep their choice.
|
|
50
65
|
*/
|
|
66
|
+
processing?: SpeechProcessing
|
|
67
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
51
68
|
forceLocal?: boolean
|
|
52
69
|
}
|
|
53
70
|
|
|
@@ -64,8 +81,8 @@ export interface TranscriptionModelStatus {
|
|
|
64
81
|
* - `loading`: the model is loading into memory.
|
|
65
82
|
* - `ready`: on-device transcripts are flowing.
|
|
66
83
|
* - `unavailable`: no on-device transcription right now; `reason` says why
|
|
67
|
-
* (for example no
|
|
68
|
-
*
|
|
84
|
+
* (for example no `preferLocal` or `localOnly` listener, a failed
|
|
85
|
+
* download, or a listener in a language other than the picked model's).
|
|
69
86
|
*/
|
|
70
87
|
state: "downloading" | "loading" | "ready" | "unavailable"
|
|
71
88
|
progress?: number
|
|
@@ -90,8 +107,11 @@ export class TranscriptionModule {
|
|
|
90
107
|
* event) preserves existing semantics.
|
|
91
108
|
*/
|
|
92
109
|
on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
|
|
110
|
+
const processing = resolveSpeechProcessing(options, "transcription.on(options)")
|
|
93
111
|
return this.track(
|
|
94
|
-
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void,
|
|
112
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, {
|
|
113
|
+
processing,
|
|
114
|
+
}),
|
|
95
115
|
)
|
|
96
116
|
}
|
|
97
117
|
|
|
@@ -117,16 +137,15 @@ export class TranscriptionModule {
|
|
|
117
137
|
const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
|
|
118
138
|
requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
|
|
119
139
|
)
|
|
140
|
+
const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)")
|
|
120
141
|
if (langs.length === 0) return () => {}
|
|
121
142
|
|
|
122
143
|
const unsubs: UnsubscribeFn[] = []
|
|
123
144
|
for (const lang of langs) {
|
|
124
145
|
unsubs.push(
|
|
125
|
-
this.session._subscribe(
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
options,
|
|
129
|
-
),
|
|
146
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void, {
|
|
147
|
+
processing,
|
|
148
|
+
}),
|
|
130
149
|
)
|
|
131
150
|
}
|
|
132
151
|
const combined: UnsubscribeFn = () => {
|
|
@@ -210,10 +229,10 @@ export class TranscriptionModule {
|
|
|
210
229
|
* model can't finish early (models that transcribe in fixed chunks); their
|
|
211
230
|
* final text still arrives on its own.
|
|
212
231
|
*
|
|
213
|
-
* Applies only to on-device transcription and needs
|
|
214
|
-
*
|
|
215
|
-
* miniapp, so this ends the current line for every
|
|
216
|
-
* Captions included.
|
|
232
|
+
* Applies only to on-device transcription and needs a `localOnly` listener
|
|
233
|
+
* or one that is getting on-device text right now. The phone runs one
|
|
234
|
+
* recognizer for every miniapp, so this ends the current line for every
|
|
235
|
+
* on-device subscriber, Captions included.
|
|
217
236
|
*/
|
|
218
237
|
finishUtterance(): Promise<string | null> {
|
|
219
238
|
return this.session.sendRequest<string | null>({type: MiniappRequestType.TRANSCRIPTION_FINISH})
|
package/src/protocol.ts
CHANGED
|
@@ -26,7 +26,16 @@ export enum MiniappRequestType {
|
|
|
26
26
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
27
27
|
AUTH_REFRESH = "miniapp_auth_refresh",
|
|
28
28
|
|
|
29
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Update the set of streams the miniapp is subscribed to. `subscriptions`
|
|
31
|
+
* lists stream names; a transcription stream is `{stream, processing}`, one
|
|
32
|
+
* entry per processing its listeners ask for, and `transcriptionListeners`
|
|
33
|
+
* lists every transcription listener as `{id, stream, processing}`. Both
|
|
34
|
+
* also carry `forceLocal: true` for `localOnly`, the form hosts before
|
|
35
|
+
* `processing` read. Hosts accept that older form,
|
|
36
|
+
* `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
|
|
37
|
+
* `processing`.
|
|
38
|
+
*/
|
|
30
39
|
SUBSCRIBE = "miniapp_subscribe",
|
|
31
40
|
|
|
32
41
|
/**
|
|
@@ -45,7 +54,7 @@ export enum MiniappRequestType {
|
|
|
45
54
|
/** Stop any audio playback this miniapp initiated. */
|
|
46
55
|
STOP_AUDIO = "miniapp_stop_audio",
|
|
47
56
|
|
|
48
|
-
/** Speak text via cloud TTS
|
|
57
|
+
/** Speak text via cloud or on-device TTS, as its `processing` says. */
|
|
49
58
|
SPEAK = "miniapp_speak",
|
|
50
59
|
|
|
51
60
|
/**
|
|
@@ -451,7 +460,7 @@ export enum MiniappErrorCode {
|
|
|
451
460
|
TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
|
|
452
461
|
TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
|
|
453
462
|
|
|
454
|
-
/** `speak({
|
|
463
|
+
/** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
|
|
455
464
|
TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
|
|
456
465
|
|
|
457
466
|
/** Not connected / pre-ACK and transport closed. */
|
package/src/session.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {ConfigurationModule} from "./modules/configuration"
|
|
|
26
26
|
import {DashboardAPI} from "./modules/dashboard"
|
|
27
27
|
import {DisplayManager} from "./modules/display"
|
|
28
28
|
import {EventManager, type TranscriptionEventRoute, type UnsubscribeFn} from "./modules/events"
|
|
29
|
+
import type {SpeechProcessingOptions} from "./modules/speechProcessing"
|
|
29
30
|
import {GlassesModule} from "./modules/glasses"
|
|
30
31
|
import {HeadingModule} from "./modules/heading"
|
|
31
32
|
import {ImuModule} from "./modules/imu"
|
|
@@ -483,7 +484,7 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
483
484
|
_subscribe(
|
|
484
485
|
streamType: string,
|
|
485
486
|
handler: (data: unknown) => void,
|
|
486
|
-
options:
|
|
487
|
+
options: SpeechProcessingOptions = {},
|
|
487
488
|
): UnsubscribeFn {
|
|
488
489
|
return this.events.subscribe(streamType, handler, options)
|
|
489
490
|
}
|