@mentra/miniapp 3.3.0 → 3.4.0-beta.1237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/background/index.d.ts +5 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/blob.d.ts +2 -1
- package/dist/modules/blob.d.ts.map +1 -1
- package/dist/modules/blob.js +2 -1
- package/dist/modules/blob.js.map +1 -1
- package/dist/modules/events.d.ts +16 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/mic.d.ts +26 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +13 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +44 -57
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +21 -24
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +41 -10
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +17 -6
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/system.d.ts +8 -3
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +1 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/telemetry.d.ts +25 -0
- package/dist/modules/telemetry.d.ts.map +1 -0
- package/dist/modules/telemetry.js +61 -0
- package/dist/modules/telemetry.js.map +1 -0
- package/dist/modules/transcription.d.ts +70 -3
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +68 -2
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +29 -15
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +30 -15
- package/dist/protocol.js.map +1 -1
- package/dist/react/MentraProvider.d.ts +13 -1
- package/dist/react/MentraProvider.d.ts.map +1 -1
- package/dist/react/MentraProvider.js +31 -1
- package/dist/react/MentraProvider.js.map +1 -1
- package/dist/session.d.ts +16 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +6 -13
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +7 -4
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +6 -3
- package/src/index.ts +11 -3
- package/src/modules/blob.ts +2 -1
- package/src/modules/events.ts +65 -51
- package/src/modules/mic.ts +30 -4
- package/src/modules/miniapps.ts +59 -71
- package/src/modules/speaker.ts +56 -12
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/system.ts +8 -3
- package/src/modules/telemetry.ts +62 -0
- package/src/modules/transcription.ts +106 -9
- package/src/protocol.ts +32 -16
- package/src/react/MentraProvider.tsx +23 -2
- package/src/session.ts +20 -17
- package/src/ui/index.ts +7 -4
package/src/modules/events.ts
CHANGED
|
@@ -33,10 +33,16 @@ import {EventEmitter} from "eventemitter3"
|
|
|
33
33
|
import {makeRequestId} from "../envelope"
|
|
34
34
|
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
35
35
|
import {MiniappSession} from "../session"
|
|
36
|
+
import {resolveSpeechProcessing, type SpeechProcessing, type SpeechProcessingOptions} from "./speechProcessing"
|
|
36
37
|
|
|
37
38
|
export type UnsubscribeFn = () => void
|
|
38
39
|
|
|
39
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* @internal Delivery class the host tags a transcription event with. Hosts
|
|
42
|
+
* address each event to the listeners that should get it; SDKs from before
|
|
43
|
+
* `processing` filter on this tag instead: `"forceLocal"` reaches `localOnly`
|
|
44
|
+
* listeners, `"default"` the others, `"all"` everyone.
|
|
45
|
+
*/
|
|
40
46
|
export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
|
|
41
47
|
|
|
42
48
|
// ---------------------------------------------------------------------------
|
|
@@ -48,6 +54,8 @@ export interface TranscriptionData {
|
|
|
48
54
|
text: string
|
|
49
55
|
isFinal: boolean
|
|
50
56
|
language?: string
|
|
57
|
+
/** Where the text was transcribed: `"cloud"`, or `"local"` for the phone's on-device model. */
|
|
58
|
+
source: "cloud" | "local"
|
|
51
59
|
}
|
|
52
60
|
|
|
53
61
|
export interface TranslationData {
|
|
@@ -150,6 +158,10 @@ export interface NotificationDismissedData {
|
|
|
150
158
|
export interface VadData {
|
|
151
159
|
/** True while the user is speaking (voice detected), false when silent. */
|
|
152
160
|
status: boolean
|
|
161
|
+
/** 0 while speaking; on a silence event, how long the audio has already been silent. */
|
|
162
|
+
silentForMs?: number
|
|
163
|
+
/** Unix epoch milliseconds of the transition, when the host reports it. */
|
|
164
|
+
timestamp?: number
|
|
153
165
|
}
|
|
154
166
|
|
|
155
167
|
export interface TouchData {
|
|
@@ -173,14 +185,12 @@ export class EventManager {
|
|
|
173
185
|
private listenerSeq = 0
|
|
174
186
|
private transcriptionListeners = new Map<
|
|
175
187
|
string,
|
|
176
|
-
{stream: string;
|
|
188
|
+
{stream: string; processing: SpeechProcessing; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
|
|
177
189
|
>()
|
|
178
190
|
private retireListeners = new Set<() => void>()
|
|
179
191
|
private readonly emitter = new EventEmitter()
|
|
180
192
|
/** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
|
|
181
193
|
private readonly refCounts = new Map<string, number>()
|
|
182
|
-
/** Transcription stream -> refs that require on-device-only processing. */
|
|
183
|
-
private readonly forceLocalRefCounts = new Map<string, number>()
|
|
184
194
|
|
|
185
195
|
constructor(private readonly session: MiniappSession) {}
|
|
186
196
|
|
|
@@ -195,20 +205,24 @@ export class EventManager {
|
|
|
195
205
|
* the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
|
|
196
206
|
* via a separate envelope `type`; the EventManager only routes them
|
|
197
207
|
* locally.
|
|
208
|
+
*
|
|
209
|
+
* `processing` applies to transcription streams only.
|
|
198
210
|
*/
|
|
199
|
-
subscribe(stream: string, handler: (data: unknown) => void, options:
|
|
211
|
+
subscribe(stream: string, handler: (data: unknown) => void, options: SpeechProcessingOptions = {}): UnsubscribeFn {
|
|
200
212
|
const isInternal = stream.startsWith("_")
|
|
201
|
-
const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
202
213
|
const isTranscription = stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
214
|
+
const processing = isTranscription ? resolveSpeechProcessing(options, "events.subscribe(options)") : "preferCloud"
|
|
215
|
+
const localOnly = processing === "localOnly"
|
|
203
216
|
const listenerId = isTranscription ? `${this.listenerScope}:${++this.listenerSeq}` : undefined
|
|
204
217
|
let active = true
|
|
205
218
|
const listener = (data: unknown, route?: TranscriptionEventRoute) => {
|
|
206
219
|
if (!active) return
|
|
207
|
-
if (
|
|
220
|
+
if (isTranscription) {
|
|
221
|
+
// The host tags every transcription event with the class it is for.
|
|
208
222
|
// Hosts that predate routed transcription events omit `route`. Treat
|
|
209
|
-
// those events as the
|
|
210
|
-
//
|
|
211
|
-
if (
|
|
223
|
+
// those events as the cloud path so a localOnly listener can never
|
|
224
|
+
// accidentally receive cloud text.
|
|
225
|
+
if (localOnly ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
|
|
212
226
|
}
|
|
213
227
|
handler(data)
|
|
214
228
|
}
|
|
@@ -217,21 +231,12 @@ export class EventManager {
|
|
|
217
231
|
active = false
|
|
218
232
|
}
|
|
219
233
|
this.retireListeners.add(retire)
|
|
220
|
-
if (listenerId) this.transcriptionListeners.set(listenerId, {stream,
|
|
234
|
+
if (listenerId) this.transcriptionListeners.set(listenerId, {stream, processing, invoke: listener})
|
|
221
235
|
if (!isInternal) {
|
|
222
236
|
const before = this.refCounts.get(stream) ?? 0
|
|
223
237
|
this.refCounts.set(stream, before + 1)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
|
|
227
|
-
if (
|
|
228
|
-
isTranscription ||
|
|
229
|
-
before === 0 ||
|
|
230
|
-
(forceLocal && forceLocalBefore === 0) ||
|
|
231
|
-
(!forceLocal && defaultBefore === 0)
|
|
232
|
-
) {
|
|
233
|
-
this.sendSubscriptionUpdate()
|
|
234
|
-
}
|
|
238
|
+
// Every transcription listener is registered with the host by id.
|
|
239
|
+
if (isTranscription || before === 0) this.sendSubscriptionUpdate()
|
|
235
240
|
}
|
|
236
241
|
return () => {
|
|
237
242
|
if (!active) return
|
|
@@ -241,24 +246,12 @@ export class EventManager {
|
|
|
241
246
|
this.emitter.off(stream, listener)
|
|
242
247
|
if (isInternal) return
|
|
243
248
|
const current = this.refCounts.get(stream) ?? 0
|
|
244
|
-
const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
|
|
245
|
-
let subscriptionChanged = current <= 1
|
|
246
249
|
if (current <= 1) {
|
|
247
250
|
this.refCounts.delete(stream)
|
|
248
251
|
} else {
|
|
249
252
|
this.refCounts.set(stream, current - 1)
|
|
250
253
|
}
|
|
251
|
-
if (
|
|
252
|
-
if (forceLocalCurrent <= 1) {
|
|
253
|
-
this.forceLocalRefCounts.delete(stream)
|
|
254
|
-
subscriptionChanged = true
|
|
255
|
-
} else {
|
|
256
|
-
this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
|
|
257
|
-
}
|
|
258
|
-
} else if (current - forceLocalCurrent <= 1) {
|
|
259
|
-
subscriptionChanged = true
|
|
260
|
-
}
|
|
261
|
-
if (isTranscription || subscriptionChanged) this.sendSubscriptionUpdate()
|
|
254
|
+
if (isTranscription || current <= 1) this.sendSubscriptionUpdate()
|
|
262
255
|
}
|
|
263
256
|
}
|
|
264
257
|
|
|
@@ -269,7 +262,6 @@ export class EventManager {
|
|
|
269
262
|
this.transcriptionListeners.clear()
|
|
270
263
|
this.emitter.removeAllListeners()
|
|
271
264
|
this.refCounts.clear()
|
|
272
|
-
this.forceLocalRefCounts.clear()
|
|
273
265
|
this.sendSubscriptionUpdate()
|
|
274
266
|
}
|
|
275
267
|
|
|
@@ -284,6 +276,7 @@ export class EventManager {
|
|
|
284
276
|
transcriptionRoute?: TranscriptionEventRoute,
|
|
285
277
|
listenerId?: string,
|
|
286
278
|
): void {
|
|
279
|
+
if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) data = withTranscriptionSource(data)
|
|
287
280
|
if (listenerId !== undefined) {
|
|
288
281
|
const listener = this.transcriptionListeners.get(listenerId)
|
|
289
282
|
if (listener && (listener.stream === stream || listener.stream === "transcription:auto"))
|
|
@@ -318,27 +311,48 @@ export class EventManager {
|
|
|
318
311
|
}
|
|
319
312
|
|
|
320
313
|
private sendSubscriptionUpdate(): void {
|
|
321
|
-
const subscriptions
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
314
|
+
const subscriptions: Array<string | Record<string, unknown>> = []
|
|
315
|
+
for (const stream of this.refCounts.keys()) {
|
|
316
|
+
if (stream === MiniappStreamType.LOCATION_UPDATE)
|
|
317
|
+
subscriptions.push({stream: "location_stream", rate: "realtime"})
|
|
318
|
+
else if (!stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) subscriptions.push(stream)
|
|
319
|
+
}
|
|
320
|
+
// One entry per transcription stream and processing its listeners ask for.
|
|
321
|
+
const transcription = new Set<string>()
|
|
322
|
+
for (const {stream, processing} of this.transcriptionListeners.values()) {
|
|
323
|
+
const key = `${processing} ${stream}`
|
|
324
|
+
if (transcription.has(key)) continue
|
|
325
|
+
transcription.add(key)
|
|
326
|
+
subscriptions.push({stream, ...processingOnWire(processing)})
|
|
327
|
+
}
|
|
334
328
|
this.session.sendOneShot({
|
|
335
329
|
type: MiniappRequestType.SUBSCRIBE,
|
|
336
330
|
subscriptions,
|
|
337
|
-
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream,
|
|
331
|
+
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, processing}]) => ({
|
|
338
332
|
id,
|
|
339
333
|
stream,
|
|
340
|
-
|
|
334
|
+
...processingOnWire(processing),
|
|
341
335
|
})),
|
|
342
336
|
})
|
|
343
337
|
}
|
|
344
338
|
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* The wire form of a transcription listener's processing. Hosts before
|
|
342
|
+
* `processing` (Mentra App 3.4.0) read only `forceLocal`, so `localOnly` keeps
|
|
343
|
+
* sending it: there a localOnly listener still never gets cloud text.
|
|
344
|
+
*/
|
|
345
|
+
function processingOnWire(processing: SpeechProcessing): {processing: SpeechProcessing; forceLocal?: true} {
|
|
346
|
+
return processing === "localOnly" ? {processing, forceLocal: true} : {processing}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Hosts before `source` (Mentra App 3.4.0) leave it out. Their on-device
|
|
351
|
+
* transcripts carry `type: "local_transcription"`, their cloud ones do not.
|
|
352
|
+
*/
|
|
353
|
+
function withTranscriptionSource(data: unknown): unknown {
|
|
354
|
+
if (typeof data !== "object" || data === null) return data
|
|
355
|
+
const event = data as {source?: unknown; type?: unknown}
|
|
356
|
+
if (event.source === "cloud" || event.source === "local") return data
|
|
357
|
+
return {...event, source: event.type === "local_transcription" ? "local" : "cloud"}
|
|
358
|
+
}
|
package/src/modules/mic.ts
CHANGED
|
@@ -31,6 +31,22 @@ export interface MicLease {
|
|
|
31
31
|
release(): Promise<void>
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* On-device wake-word ("Hey Mentra") event. It describes only the wake-word
|
|
36
|
+
* model: `loading` while the phone loads it, `ready` once it is listening,
|
|
37
|
+
* `detected` for each detection, and `error` when it cannot run (`message`
|
|
38
|
+
* says why). The state of the on-device speech-to-text model is reported
|
|
39
|
+
* separately by `session.transcription.onModelStatus`.
|
|
40
|
+
*/
|
|
41
|
+
export interface WakeWordEvent {
|
|
42
|
+
kind: "loading" | "ready" | "detected" | "error"
|
|
43
|
+
/** The phrase this event is about. Currently always "Hey Mentra". */
|
|
44
|
+
phrase: string
|
|
45
|
+
/** Unix epoch milliseconds. */
|
|
46
|
+
timestamp: number
|
|
47
|
+
message?: string
|
|
48
|
+
}
|
|
49
|
+
|
|
34
50
|
export class MicModule {
|
|
35
51
|
/** All active unsubscribe functions for stop() to tear down at once. */
|
|
36
52
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
@@ -38,17 +54,27 @@ export class MicModule {
|
|
|
38
54
|
constructor(private readonly session: MiniappSession) {}
|
|
39
55
|
|
|
40
56
|
/**
|
|
41
|
-
* Subscribe to voice activity detection
|
|
42
|
-
* `true`
|
|
57
|
+
* Subscribe to on-device voice activity detection. One event when the user
|
|
58
|
+
* starts speaking (`status: true`) and one when they stop (`status: false`,
|
|
59
|
+
* with `silentForMs` saying how long the audio has already been quiet). A
|
|
60
|
+
* gap in microphone audio counts as silence. Requires MICROPHONE permission.
|
|
43
61
|
*/
|
|
44
62
|
onVoiceActivity(handler: (data: VadData) => void): UnsubscribeFn {
|
|
45
63
|
return this.track(this.session._subscribe(MiniappStreamType.VAD, handler as (data: unknown) => void))
|
|
46
64
|
}
|
|
47
65
|
|
|
48
66
|
/**
|
|
49
|
-
* Subscribe to
|
|
50
|
-
*
|
|
67
|
+
* Subscribe to the on-device "Hey Mentra" wake word. Every subscribed
|
|
68
|
+
* miniapp receives every detection. You get the detector's current state
|
|
69
|
+
* right away, then every change. The phone loads the wake-word model while
|
|
70
|
+
* any miniapp subscribes and downloads it on first use. Requires MICROPHONE
|
|
71
|
+
* permission.
|
|
51
72
|
*/
|
|
73
|
+
onWakeWord(handler: (event: WakeWordEvent) => void): UnsubscribeFn {
|
|
74
|
+
return this.track(this.session._subscribe(MiniappStreamType.WAKE_WORD, handler as (data: unknown) => void))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Subscribe to raw audio chunks in the phone's PCM/LC3 microphone format. */
|
|
52
78
|
onAudioChunk(handler: (data: AudioChunkData) => void): UnsubscribeFn {
|
|
53
79
|
return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
|
|
54
80
|
}
|
package/src/modules/miniapps.ts
CHANGED
|
@@ -7,10 +7,12 @@
|
|
|
7
7
|
* layer (invoke / handle) lives on `session.actions`, not here — these are
|
|
8
8
|
* operations on the app as a whole, not on a declared action.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
* narrower still:
|
|
12
|
-
* the Mentra App
|
|
13
|
-
*
|
|
10
|
+
* Installing (`install`, `checkInstallCompatibility`, `uninstall`) is
|
|
11
|
+
* narrower still: only a miniapp on the Mentra App's system list, running the
|
|
12
|
+
* release the Mentra App installed for it, may install miniapps. Development
|
|
13
|
+
* builds, which may use the calls above, may not. The host installs from a
|
|
14
|
+
* miniapp URL through the same installer as every other way in, with no install
|
|
15
|
+
* sheet: the caller asks the person first.
|
|
14
16
|
*/
|
|
15
17
|
|
|
16
18
|
import {MiniappRequestType} from "../protocol"
|
|
@@ -55,10 +57,6 @@ export interface MiniappInfo {
|
|
|
55
57
|
compatibility: MiniappCompatibility
|
|
56
58
|
/** True when this package identity is backed by a ZIP in the Mentra App build. */
|
|
57
59
|
system: boolean
|
|
58
|
-
/** Store package that owns updates of this active release, when it is Store-managed. */
|
|
59
|
-
storeOwnerPackageName?: string
|
|
60
|
-
/** Store the host build assigns to update this bundled package's active release, if any. */
|
|
61
|
-
systemStoreOwnerPackageName?: string
|
|
62
60
|
/** Declared actions (empty if the miniapp declares none). */
|
|
63
61
|
actions: MiniappActionInfo[]
|
|
64
62
|
}
|
|
@@ -74,49 +72,54 @@ export interface MiniappHardwareRequirement {
|
|
|
74
72
|
description?: string
|
|
75
73
|
}
|
|
76
74
|
|
|
77
|
-
/**
|
|
75
|
+
/** A miniapp to install. */
|
|
78
76
|
export interface InstallMiniappRequest {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
bundleUrl: string
|
|
82
|
-
/** SHA-256 of the release ZIP; the host refuses any other bytes. */
|
|
83
|
-
bundleSha256: string
|
|
84
|
-
/**
|
|
85
|
-
* Bearer for `bundleUrl`, when the Store's backend requires one. The host
|
|
86
|
-
* forwards it untouched and keeps no Store address of its own: entitlement is
|
|
87
|
-
* settled between this miniapp and its backend, integrity by the SHA-256 and
|
|
88
|
-
* the bundle's publisher signature.
|
|
89
|
-
*/
|
|
90
|
-
bundleAuthorization?: string
|
|
91
|
-
/**
|
|
92
|
-
* Automatic update: never stop a running miniapp. A running or starting
|
|
93
|
-
* target defers the request with `APP_RUNNING`.
|
|
94
|
-
*/
|
|
95
|
-
onlyIfStopped?: boolean
|
|
77
|
+
/** The miniapp's URL: the HTTPS address that serves its `miniapp.json`. */
|
|
78
|
+
url: string
|
|
96
79
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
* host does not show (the Mentra App is in the background, another prompt is
|
|
100
|
-
* open, or this miniapp was asked within the last day), still answers
|
|
101
|
-
* `APP_RUNNING`.
|
|
80
|
+
* The package the person agreed to install. When given, a URL that serves another package installs nothing and
|
|
81
|
+
* fails with `not_a_miniapp`.
|
|
102
82
|
*/
|
|
103
|
-
|
|
104
|
-
/** Minimum Mentra App version declared by the release manifest. */
|
|
105
|
-
minHostVersion?: string
|
|
106
|
-
/** Mentra Miniapp SDK version declared by the release manifest. */
|
|
107
|
-
sdkVersion?: string
|
|
108
|
-
/** Glasses hardware requirements declared by the release manifest. */
|
|
109
|
-
hardwareRequirements?: MiniappHardwareRequirement[]
|
|
110
|
-
releaseId?: string
|
|
111
|
-
channel?: string
|
|
83
|
+
packageName?: string
|
|
112
84
|
}
|
|
113
85
|
|
|
114
86
|
export interface InstallMiniappResult {
|
|
87
|
+
/**
|
|
88
|
+
* `installed`: this release installed. `opened`: nothing installed, because the miniapp is installed from this URL
|
|
89
|
+
* already, or a newer release of it is; {@link MiniappsModule.open} it to show it. `deferred`: the miniapp is
|
|
90
|
+
* running, so its update waits until it stops, or until the person answers the Mentra App's update prompt.
|
|
91
|
+
*/
|
|
92
|
+
status: "installed" | "opened" | "deferred"
|
|
115
93
|
packageName: string
|
|
94
|
+
/** The installed version: the new release, or the one that stays. */
|
|
116
95
|
version: string
|
|
117
|
-
installedByStore: string
|
|
118
96
|
}
|
|
119
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Why an install failed: the `reason` of an `INSTALL_FAILED` rejection from {@link MiniappsModule.install}.
|
|
100
|
+
* - `not_a_miniapp`: the URL serves no valid `miniapp.json` or bundle.
|
|
101
|
+
* - `domain_mismatch`: the package name doesn't belong to the URL's domain.
|
|
102
|
+
* - `archived`: Mentra archived this URL; it doesn't install.
|
|
103
|
+
* - `needs_newer_app`: the miniapp needs a newer Mentra App, or hardware the glasses lack.
|
|
104
|
+
* - `access_denied`: its server refused this account, or this deployment doesn't allow installing from a URL.
|
|
105
|
+
* - `age_restricted`: the person's declared age is below the miniapp's age rating.
|
|
106
|
+
* - `network`: the URL couldn't be reached.
|
|
107
|
+
* - `cancelled`: the Mentra App signed out or switched deployments meanwhile.
|
|
108
|
+
* - `integrity`: the bundle doesn't match what `miniapp.json` names.
|
|
109
|
+
* - `signature_invalid`: the bundle's signature doesn't match the installed miniapp's publisher.
|
|
110
|
+
*/
|
|
111
|
+
export type InstallMiniappFailureReason =
|
|
112
|
+
| "not_a_miniapp"
|
|
113
|
+
| "domain_mismatch"
|
|
114
|
+
| "archived"
|
|
115
|
+
| "needs_newer_app"
|
|
116
|
+
| "access_denied"
|
|
117
|
+
| "age_restricted"
|
|
118
|
+
| "network"
|
|
119
|
+
| "cancelled"
|
|
120
|
+
| "integrity"
|
|
121
|
+
| "signature_invalid"
|
|
122
|
+
|
|
120
123
|
export interface InstallMiniappCompatibilityRequest {
|
|
121
124
|
minHostVersion?: string | null
|
|
122
125
|
sdkVersion?: string | null
|
|
@@ -127,17 +130,7 @@ export type InstallMiniappCompatibility =
|
|
|
127
130
|
| {compatible: true}
|
|
128
131
|
| {compatible: false; blocker: "host" | "sdk" | "hardware"; reason: string}
|
|
129
132
|
|
|
130
|
-
export type InstallMiniappPhase = "downloading" | "verifying" | "extracting" | "activating" | "complete"
|
|
131
|
-
|
|
132
|
-
export interface InstallMiniappProgress {
|
|
133
|
-
packageName: string
|
|
134
|
-
version: string
|
|
135
|
-
phase: InstallMiniappPhase
|
|
136
|
-
}
|
|
137
|
-
|
|
138
133
|
export class MiniappsModule {
|
|
139
|
-
private readonly progressHandlers = new Set<(progress: InstallMiniappProgress) => void>()
|
|
140
|
-
|
|
141
134
|
constructor(private readonly session: MiniappSession) {}
|
|
142
135
|
|
|
143
136
|
/**
|
|
@@ -185,24 +178,27 @@ export class MiniappsModule {
|
|
|
185
178
|
}
|
|
186
179
|
|
|
187
180
|
/**
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
*
|
|
181
|
+
* Install the miniapp at a miniapp URL, with no install sheet: ask the person first. The host fetches its
|
|
182
|
+
* `miniapp.json`, checks the package name against the URL's domain, registers the URL with Mentra, checks the
|
|
183
|
+
* phone can run it, and downloads, verifies and installs its bundle.
|
|
184
|
+
*
|
|
185
|
+
* Only a miniapp on the Mentra App's system list may install; any other caller gets NOT_PERMITTED. A failed
|
|
186
|
+
* install rejects with INSTALL_FAILED, whose `reason` is an {@link InstallMiniappFailureReason}.
|
|
191
187
|
*/
|
|
192
188
|
async install(request: InstallMiniappRequest): Promise<InstallMiniappResult> {
|
|
193
189
|
return this.session.sendRequest<InstallMiniappResult>(
|
|
194
190
|
{
|
|
195
|
-
...request,
|
|
196
191
|
type: MiniappRequestType.MINIAPPS_INSTALL,
|
|
192
|
+
url: request.url,
|
|
193
|
+
...(request.packageName !== undefined ? {packageName: request.packageName} : {}),
|
|
197
194
|
},
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
// still settles on the host response or session disconnect.
|
|
195
|
+
// Fetching, registering, downloading and installing can legitimately outlast the SDK's default request
|
|
196
|
+
// timeout. The request still settles on the host response or session disconnect.
|
|
201
197
|
{timeoutMs: 0},
|
|
202
198
|
)
|
|
203
199
|
}
|
|
204
200
|
|
|
205
|
-
/** Check
|
|
201
|
+
/** Check a release's Mentra App, SDK and glasses requirements against this phone. System-list miniapps only. */
|
|
206
202
|
async checkInstallCompatibility(request: InstallMiniappCompatibilityRequest): Promise<InstallMiniappCompatibility> {
|
|
207
203
|
return this.session.sendRequest<InstallMiniappCompatibility>({
|
|
208
204
|
type: MiniappRequestType.MINIAPPS_INSTALL_CHECK,
|
|
@@ -212,18 +208,10 @@ export class MiniappsModule {
|
|
|
212
208
|
})
|
|
213
209
|
}
|
|
214
210
|
|
|
215
|
-
/**
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/** @internal */
|
|
222
|
-
_deliverInstallProgress(progress: InstallMiniappProgress): void {
|
|
223
|
-
for (const handler of this.progressHandlers) handler(progress)
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/** Uninstall a package this Store installed. Bundled Stores only. */
|
|
211
|
+
/**
|
|
212
|
+
* Uninstall a miniapp installed from a miniapp URL. A miniapp the Mentra App bundles, one a workspace manages and a
|
|
213
|
+
* development build stay, with NOT_PERMITTED. System-list miniapps only.
|
|
214
|
+
*/
|
|
227
215
|
async uninstall(packageName: string): Promise<void> {
|
|
228
216
|
await this.session.sendRequest<void>({
|
|
229
217
|
type: MiniappRequestType.MINIAPPS_UNINSTALL,
|
package/src/modules/speaker.ts
CHANGED
|
@@ -5,13 +5,15 @@
|
|
|
5
5
|
* audio chunks, VAD) lives on session.mic — the split is by I/O direction.
|
|
6
6
|
*
|
|
7
7
|
* Imperative surface:
|
|
8
|
-
* speaker.play({audioUrl}) — play an
|
|
8
|
+
* speaker.play({audioUrl}) — play an http(s) or data: URL, or one of this
|
|
9
|
+
* miniapp's blobs, via the phone's
|
|
9
10
|
* AudioPlaybackService.
|
|
10
|
-
* speaker.speak(text) — send a SPEAK request.
|
|
11
|
-
* TTS when connected and falls
|
|
12
|
-
*
|
|
11
|
+
* speaker.speak(text) — send a SPEAK request. By default the phone
|
|
12
|
+
* streams cloud TTS when connected and falls
|
|
13
|
+
* back to its on-device voice; the
|
|
14
|
+
* `processing` option picks the order.
|
|
13
15
|
* Resolves when playback completes; rejects
|
|
14
|
-
* with a TTS_* error code on
|
|
16
|
+
* with a TTS_* error code on failure.
|
|
15
17
|
* speaker.createStream(opts) — open a live PCM output stream (16-bit LE).
|
|
16
18
|
* Returns a SpeakerStreamWriter: write() PCM
|
|
17
19
|
* chunks as they arrive (e.g. from a WebSocket
|
|
@@ -38,8 +40,13 @@ import {MiniappErrorCode, MiniappRequestType} from "../protocol"
|
|
|
38
40
|
import {MiniappSession} from "../session"
|
|
39
41
|
import {bytesToBase64, toUint8Array} from "./base64"
|
|
40
42
|
import type {UnsubscribeFn} from "./events"
|
|
43
|
+
import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
|
|
41
44
|
|
|
42
45
|
export interface PlayAudioOptions {
|
|
46
|
+
/**
|
|
47
|
+
* An http(s) or `data:` URL, or the `file://` uri of one of this miniapp's
|
|
48
|
+
* blobs (`session.blob`). The host refuses any other file or scheme.
|
|
49
|
+
*/
|
|
43
50
|
audioUrl: string
|
|
44
51
|
/** Start playback at this offset in milliseconds. Defaults to zero. */
|
|
45
52
|
startPositionMs?: number
|
|
@@ -58,10 +65,17 @@ export interface SpeakOptions {
|
|
|
58
65
|
*/
|
|
59
66
|
enableSanitization?: boolean
|
|
60
67
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
68
|
+
* Where the speech is synthesized. Default `"preferCloud"`.
|
|
69
|
+
*
|
|
70
|
+
* - `"preferCloud"`: cloud TTS while the cloud is connected, otherwise the
|
|
71
|
+
* phone's on-device voice.
|
|
72
|
+
* - `"preferLocal"`: the on-device voice when its model is ready and can say
|
|
73
|
+
* it (an explicit cloud `voice_id` cannot), otherwise cloud TTS.
|
|
74
|
+
* - `"localOnly"`: the on-device voice only; never the cloud. Rejects with
|
|
75
|
+
* `TTS_LOCAL_UNAVAILABLE` when the on-device model isn't ready.
|
|
64
76
|
*/
|
|
77
|
+
processing?: SpeechProcessing
|
|
78
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
65
79
|
forceLocal?: boolean
|
|
66
80
|
}
|
|
67
81
|
|
|
@@ -80,6 +94,22 @@ export type SpeakerStreamSampleRate = 16000 | 24000 | 48000
|
|
|
80
94
|
export interface SpeakerStreamOptions {
|
|
81
95
|
/** PCM sample rate in Hz. Default 16000. */
|
|
82
96
|
sampleRate?: SpeakerStreamSampleRate
|
|
97
|
+
/**
|
|
98
|
+
* Open the native stream now and keep it idle until the first write, so a
|
|
99
|
+
* later sound only has to write samples. The speaker reports `idle` and other
|
|
100
|
+
* audio is left alone until then. If the audio route changes before the
|
|
101
|
+
* first write (Bluetooth or glasses connect/disconnect, a call, an audio
|
|
102
|
+
* interruption), the phone closes the stream and the next write fails with
|
|
103
|
+
* "not open"; open a fresh stream then.
|
|
104
|
+
*/
|
|
105
|
+
prewarm?: boolean
|
|
106
|
+
/**
|
|
107
|
+
* How much audio (80–500 ms) the phone queues before playback starts, and
|
|
108
|
+
* the cushion against gaps after that. Default 500. A sound shorter than
|
|
109
|
+
* this still plays once you close the stream. Use a small value such as 80
|
|
110
|
+
* for short cues that should start immediately.
|
|
111
|
+
*/
|
|
112
|
+
jitterMs?: number
|
|
83
113
|
/** Channel count. v1 is mono only. Default 1. */
|
|
84
114
|
channels?: 1
|
|
85
115
|
/** Playback volume 0..1. Default 1. */
|
|
@@ -118,7 +148,10 @@ export class SpeakerStreamWriter {
|
|
|
118
148
|
private settled = false
|
|
119
149
|
private writeChain: Promise<void> = Promise.resolve()
|
|
120
150
|
|
|
121
|
-
constructor(
|
|
151
|
+
constructor(
|
|
152
|
+
private readonly session: MiniappSession,
|
|
153
|
+
readonly streamId: string,
|
|
154
|
+
) {}
|
|
122
155
|
|
|
123
156
|
/**
|
|
124
157
|
* Append raw PCM bytes (16-bit LE). Auto-chunked. Resolves with the host's
|
|
@@ -255,8 +288,9 @@ export class SpeakerModule {
|
|
|
255
288
|
}
|
|
256
289
|
|
|
257
290
|
/**
|
|
258
|
-
* Speak text through the phone
|
|
259
|
-
*
|
|
291
|
+
* Speak text through the phone, in the cloud or on-device as
|
|
292
|
+
* `options.processing` says (default: cloud TTS when connected, then the
|
|
293
|
+
* on-device voice).
|
|
260
294
|
*
|
|
261
295
|
* Rejects with a MiniappRequestError containing a `code` field on cloud-side
|
|
262
296
|
* TTS failures: `TTS_TEXT_TOO_LONG`, `TTS_INVALID_VOICE`, `TTS_UPSTREAM_ERROR`.
|
|
@@ -268,6 +302,7 @@ export class SpeakerModule {
|
|
|
268
302
|
if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
|
|
269
303
|
throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
|
|
270
304
|
}
|
|
305
|
+
const processing = resolveSpeechProcessing(options, "speaker.speak(options)")
|
|
271
306
|
try {
|
|
272
307
|
// Like play(): resolves only when TTS playback completes, so opt out of the
|
|
273
308
|
// default request timeout (long text can outlast it). Settled by the host
|
|
@@ -281,7 +316,9 @@ export class SpeakerModule {
|
|
|
281
316
|
volume: options.volume,
|
|
282
317
|
stopOtherAudio: options.stopOtherAudio ?? false,
|
|
283
318
|
enableSanitization: options.enableSanitization ?? true,
|
|
284
|
-
|
|
319
|
+
processing,
|
|
320
|
+
// Hosts before `processing` (Mentra App 3.4.0) read only this flag.
|
|
321
|
+
forceLocal: processing === "localOnly",
|
|
285
322
|
},
|
|
286
323
|
{timeoutMs: 0},
|
|
287
324
|
)
|
|
@@ -312,9 +349,16 @@ export class SpeakerModule {
|
|
|
312
349
|
) {
|
|
313
350
|
throw new RangeError("speaker stream volume must be between 0 and 1")
|
|
314
351
|
}
|
|
352
|
+
if (
|
|
353
|
+
options.jitterMs !== undefined &&
|
|
354
|
+
(!Number.isFinite(options.jitterMs) || options.jitterMs < 80 || options.jitterMs > 500)
|
|
355
|
+
)
|
|
356
|
+
throw new RangeError("speaker stream jitterMs must be between 80 and 500")
|
|
315
357
|
const res = await this.session.sendRequest<{streamId: string}>({
|
|
316
358
|
type: MiniappRequestType.SPEAKER_STREAM_OPEN,
|
|
317
359
|
sampleRate: options.sampleRate ?? 16000,
|
|
360
|
+
...(options.jitterMs === undefined ? {} : {jitterMs: options.jitterMs}),
|
|
361
|
+
...(options.prewarm === undefined ? {} : {prewarm: options.prewarm}),
|
|
318
362
|
channels: options.channels ?? 1,
|
|
319
363
|
volume: options.volume,
|
|
320
364
|
stopOtherAudio: options.stopOtherAudio ?? true,
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Where speech is processed: on the phone or in the cloud.
|
|
3
|
+
*
|
|
4
|
+
* Transcription listeners (`session.transcription.on` / `forLanguage`) and
|
|
5
|
+
* speech requests (`session.speaker.speak`) take the same `processing` option.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import {MiniappValidationError} from "./languages"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Where speech is processed.
|
|
12
|
+
*
|
|
13
|
+
* - `"preferCloud"` (default): the cloud; the phone's on-device models only
|
|
14
|
+
* while the cloud is unreachable.
|
|
15
|
+
* - `"preferLocal"`: on-device when a model for the language is ready,
|
|
16
|
+
* otherwise the cloud.
|
|
17
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
18
|
+
*/
|
|
19
|
+
export type SpeechProcessing = "preferCloud" | "preferLocal" | "localOnly"
|
|
20
|
+
|
|
21
|
+
const SPEECH_PROCESSING: readonly SpeechProcessing[] = ["preferCloud", "preferLocal", "localOnly"]
|
|
22
|
+
|
|
23
|
+
/** The `processing` option, with the deprecated `forceLocal` it replaces. */
|
|
24
|
+
export interface SpeechProcessingOptions {
|
|
25
|
+
processing?: SpeechProcessing
|
|
26
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0. */
|
|
27
|
+
forceLocal?: boolean
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* @internal The processing an options object asks for. `processing` wins;
|
|
32
|
+
* without it, the deprecated `forceLocal: true` means `"localOnly"`.
|
|
33
|
+
* Throws `MiniappValidationError` for an unknown `processing` value.
|
|
34
|
+
*/
|
|
35
|
+
export function resolveSpeechProcessing(options: SpeechProcessingOptions, where: string): SpeechProcessing {
|
|
36
|
+
const {processing} = options
|
|
37
|
+
if (processing === undefined) return options.forceLocal === true ? "localOnly" : "preferCloud"
|
|
38
|
+
if (SPEECH_PROCESSING.includes(processing)) return processing
|
|
39
|
+
throw new MiniappValidationError(
|
|
40
|
+
`${where}: unknown processing ${JSON.stringify(processing)}. Use ${SPEECH_PROCESSING.map((value) => `"${value}"`).join(", ")}.`,
|
|
41
|
+
)
|
|
42
|
+
}
|