@mentra/miniapp 3.3.0 → 3.4.0-beta.1237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/background/index.d.ts +5 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/blob.d.ts +2 -1
- package/dist/modules/blob.d.ts.map +1 -1
- package/dist/modules/blob.js +2 -1
- package/dist/modules/blob.js.map +1 -1
- package/dist/modules/events.d.ts +16 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/mic.d.ts +26 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +13 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +44 -57
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +21 -24
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +41 -10
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +17 -6
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/system.d.ts +8 -3
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +1 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/telemetry.d.ts +25 -0
- package/dist/modules/telemetry.d.ts.map +1 -0
- package/dist/modules/telemetry.js +61 -0
- package/dist/modules/telemetry.js.map +1 -0
- package/dist/modules/transcription.d.ts +70 -3
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +68 -2
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +29 -15
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +30 -15
- package/dist/protocol.js.map +1 -1
- package/dist/react/MentraProvider.d.ts +13 -1
- package/dist/react/MentraProvider.d.ts.map +1 -1
- package/dist/react/MentraProvider.js +31 -1
- package/dist/react/MentraProvider.js.map +1 -1
- package/dist/session.d.ts +16 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +6 -13
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +7 -4
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +6 -3
- package/src/index.ts +11 -3
- package/src/modules/blob.ts +2 -1
- package/src/modules/events.ts +65 -51
- package/src/modules/mic.ts +30 -4
- package/src/modules/miniapps.ts +59 -71
- package/src/modules/speaker.ts +56 -12
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/system.ts +8 -3
- package/src/modules/telemetry.ts +62 -0
- package/src/modules/transcription.ts +106 -9
- package/src/protocol.ts +32 -16
- package/src/react/MentraProvider.tsx +23 -2
- package/src/session.ts +20 -17
- package/src/ui/index.ts +7 -4
package/src/modules/system.ts
CHANGED
|
@@ -9,13 +9,17 @@ import { MiniappSession } from "../session"
|
|
|
9
9
|
|
|
10
10
|
export interface ShareOptions {
|
|
11
11
|
text?: string
|
|
12
|
+
/**
|
|
13
|
+
* An http(s) link, or the `file://` uri of one of this miniapp's blobs
|
|
14
|
+
* (`session.blob`). The host refuses any other file or scheme.
|
|
15
|
+
*/
|
|
12
16
|
url?: string
|
|
13
17
|
title?: string
|
|
14
18
|
/** Base64-encoded file data for file sharing. */
|
|
15
19
|
base64?: string
|
|
16
20
|
/** MIME type when sharing base64 data. */
|
|
17
21
|
mimeType?: string
|
|
18
|
-
/**
|
|
22
|
+
/** File name when sharing base64 data: one name; path separators become `_`. */
|
|
19
23
|
filename?: string
|
|
20
24
|
}
|
|
21
25
|
|
|
@@ -25,9 +29,10 @@ export interface ShareResult {
|
|
|
25
29
|
}
|
|
26
30
|
|
|
27
31
|
export interface DownloadOptions {
|
|
28
|
-
/** URL to download from, OR base64 data. */
|
|
32
|
+
/** An http: or https: URL to download from, OR base64 data. */
|
|
29
33
|
url?: string
|
|
30
34
|
base64?: string
|
|
35
|
+
/** One file name; path separators become `_`. */
|
|
31
36
|
filename?: string
|
|
32
37
|
mimeType?: string
|
|
33
38
|
}
|
|
@@ -58,7 +63,7 @@ export class SystemModule {
|
|
|
58
63
|
return result ?? { success: false }
|
|
59
64
|
}
|
|
60
65
|
|
|
61
|
-
/** Open a URL in the system browser.
|
|
66
|
+
/** Open a URL in the system browser. Never opens local files or scripts (file:, content:, javascript:, data:). */
|
|
62
67
|
openUrl(url: string): void {
|
|
63
68
|
this.session.sendOneShot({
|
|
64
69
|
type: MiniappRequestType.OPEN_URL,
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview TelemetryModule (`session.telemetry`): a miniapp's own analytics, shown in its organization's Fleet
|
|
3
|
+
* Management.
|
|
4
|
+
*
|
|
5
|
+
* `capture(event, properties?)` sends one fire-and-forget message to the Mentra App and returns nothing. It never throws
|
|
6
|
+
* and never waits. The SDK checks nothing: the Mentra App attributes the event to this miniapp and applies the limits
|
|
7
|
+
* (names, values, 25 properties, the rate), and on a Mentra App without the `telemetry` host feature it does nothing.
|
|
8
|
+
* Before the handshake says which, it holds the latest 100 captures and sends them once it knows.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import {MiniappRequestType} from "../protocol"
|
|
12
|
+
import type {MiniappSession} from "../session"
|
|
13
|
+
|
|
14
|
+
export type TelemetryValue = string | number | boolean
|
|
15
|
+
export type TelemetryProperties = Record<string, TelemetryValue>
|
|
16
|
+
|
|
17
|
+
/** Captures held until the handshake says whether the Mentra App takes them. */
|
|
18
|
+
const MAX_HELD = 100
|
|
19
|
+
|
|
20
|
+
export class TelemetryModule {
|
|
21
|
+
private held: object[] = []
|
|
22
|
+
private waiting = false
|
|
23
|
+
|
|
24
|
+
constructor(private readonly session: MiniappSession) {}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Captures one event, in the style of PostHog: `session.telemetry.capture("call_ended", {platform: "teams",
|
|
28
|
+
* duration_s: 312})`. Names are lowercase letters, digits and underscores; values are flat. Never personal data: no
|
|
29
|
+
* names, emails, phone numbers, typed text or content.
|
|
30
|
+
*/
|
|
31
|
+
capture(event: string, properties?: TelemetryProperties): void {
|
|
32
|
+
try {
|
|
33
|
+
const message = {type: MiniappRequestType.TELEMETRY_CAPTURE, event, properties: properties ?? {}}
|
|
34
|
+
if (this.session.handshakeDone) {
|
|
35
|
+
if (this.session.hostFeatures?.telemetry === true) this.session.sendOneShot(message)
|
|
36
|
+
return
|
|
37
|
+
}
|
|
38
|
+
if (this.held.length >= MAX_HELD) this.held.shift()
|
|
39
|
+
this.held.push(message)
|
|
40
|
+
if (!this.waiting) {
|
|
41
|
+
this.waiting = true
|
|
42
|
+
const off = this.session.on("ready", () => {
|
|
43
|
+
off()
|
|
44
|
+
this.waiting = false
|
|
45
|
+
const held = this.held.splice(0)
|
|
46
|
+
if (this.session.hostFeatures?.telemetry !== true) return
|
|
47
|
+
for (const one of held) {
|
|
48
|
+
// Each on its own: one capture that cannot be sent never takes the others, or the session's other
|
|
49
|
+
// listeners, down with it.
|
|
50
|
+
try {
|
|
51
|
+
this.session.sendOneShot(one)
|
|
52
|
+
} catch {
|
|
53
|
+
// Lost.
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
})
|
|
57
|
+
}
|
|
58
|
+
} catch {
|
|
59
|
+
// Analytics never break the miniapp: a value that cannot be sent is lost.
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* session.transcription.on(handler) // auto-detect
|
|
7
7
|
* session.transcription.forLanguage("en-US", handler) // single language
|
|
8
8
|
* session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
|
|
9
|
+
* session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
|
|
9
10
|
* session.transcription.configure({languageHints, vocabulary, diarization})
|
|
11
|
+
* session.transcription.onModelStatus(handler) // on-device model state
|
|
12
|
+
* session.transcription.finishUtterance() // end the on-device line now
|
|
10
13
|
* session.transcription.stop() // tear down all
|
|
11
14
|
*
|
|
12
15
|
* Hoisted to top-level (vs. nested under `session.mic`) because transcription
|
|
@@ -23,6 +26,7 @@ import {MiniappSession} from "../session"
|
|
|
23
26
|
import type {TranscriptionData, UnsubscribeFn} from "./events"
|
|
24
27
|
import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
|
|
25
28
|
import type {LanguageHint, TranscriptionLanguage} from "./languages"
|
|
29
|
+
import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
|
|
26
30
|
|
|
27
31
|
/** Configuration for cloud-side transcription behavior. */
|
|
28
32
|
export interface TranscriptionConfig {
|
|
@@ -41,16 +45,56 @@ export interface TranscriptionConfig {
|
|
|
41
45
|
|
|
42
46
|
export interface TranscriptionOptions {
|
|
43
47
|
/**
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
48
|
+
* Where this listener's speech is transcribed. Default `"preferCloud"`.
|
|
49
|
+
*
|
|
50
|
+
* - `"preferCloud"`: the cloud; the phone's on-device model only while the
|
|
51
|
+
* cloud is unreachable.
|
|
52
|
+
* - `"preferLocal"`: the on-device model while it runs in this listener's
|
|
53
|
+
* language, otherwise the cloud. `on()` follows the model's language,
|
|
54
|
+
* unless `configure({languageHints})` names one the model can't
|
|
55
|
+
* transcribe; then it stays on the cloud.
|
|
56
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
57
|
+
*
|
|
58
|
+
* The phone runs one on-device model, the one picked on its Speech page, and
|
|
59
|
+
* downloads it when a `preferLocal` or `localOnly` listener needs it; follow
|
|
60
|
+
* it with `onModelStatus`. A listener moves between the cloud and on-device
|
|
61
|
+
* during a pause in speech, once both have finished with what was said
|
|
62
|
+
* before it, so it never misses or repeats speech; `TranscriptionData.source`
|
|
63
|
+
* says where each result came from. Each listener routes on its own: other
|
|
64
|
+
* listeners on the same stream keep their choice.
|
|
47
65
|
*/
|
|
66
|
+
processing?: SpeechProcessing
|
|
67
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
48
68
|
forceLocal?: boolean
|
|
49
69
|
}
|
|
50
70
|
|
|
71
|
+
/**
|
|
72
|
+
* State of the on-device speech-to-text model behind this miniapp's on-device
|
|
73
|
+
* transcription. The phone runs the one model picked on its Speech page for
|
|
74
|
+
* every miniapp.
|
|
75
|
+
*/
|
|
76
|
+
export interface TranscriptionModelStatus {
|
|
77
|
+
/** BCP-47 language of the picked model, e.g. "en-US". */
|
|
78
|
+
language: string
|
|
79
|
+
/**
|
|
80
|
+
* - `downloading`: the model is being downloaded (`progress` is 0–100).
|
|
81
|
+
* - `loading`: the model is loading into memory.
|
|
82
|
+
* - `ready`: on-device transcripts are flowing.
|
|
83
|
+
* - `unavailable`: no on-device transcription right now; `reason` says why
|
|
84
|
+
* (for example no `preferLocal` or `localOnly` listener, a failed
|
|
85
|
+
* download, or a listener in a language other than the picked model's).
|
|
86
|
+
*/
|
|
87
|
+
state: "downloading" | "loading" | "ready" | "unavailable"
|
|
88
|
+
progress?: number
|
|
89
|
+
reason?: string
|
|
90
|
+
}
|
|
91
|
+
|
|
51
92
|
export class TranscriptionModule {
|
|
52
93
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
53
94
|
private currentConfig: TranscriptionConfig | null = null
|
|
95
|
+
private readonly modelStatusHandlers = new Set<(status: TranscriptionModelStatus) => void>()
|
|
96
|
+
private modelStatusUnsub: UnsubscribeFn | null = null
|
|
97
|
+
private modelStatus: TranscriptionModelStatus | null = null
|
|
54
98
|
|
|
55
99
|
constructor(private readonly session: MiniappSession) {}
|
|
56
100
|
|
|
@@ -63,8 +107,11 @@ export class TranscriptionModule {
|
|
|
63
107
|
* event) preserves existing semantics.
|
|
64
108
|
*/
|
|
65
109
|
on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
|
|
110
|
+
const processing = resolveSpeechProcessing(options, "transcription.on(options)")
|
|
66
111
|
return this.track(
|
|
67
|
-
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void,
|
|
112
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, {
|
|
113
|
+
processing,
|
|
114
|
+
}),
|
|
68
115
|
)
|
|
69
116
|
}
|
|
70
117
|
|
|
@@ -90,16 +137,15 @@ export class TranscriptionModule {
|
|
|
90
137
|
const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
|
|
91
138
|
requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
|
|
92
139
|
)
|
|
140
|
+
const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)")
|
|
93
141
|
if (langs.length === 0) return () => {}
|
|
94
142
|
|
|
95
143
|
const unsubs: UnsubscribeFn[] = []
|
|
96
144
|
for (const lang of langs) {
|
|
97
145
|
unsubs.push(
|
|
98
|
-
this.session._subscribe(
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
options,
|
|
102
|
-
),
|
|
146
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void, {
|
|
147
|
+
processing,
|
|
148
|
+
}),
|
|
103
149
|
)
|
|
104
150
|
}
|
|
105
151
|
const combined: UnsubscribeFn = () => {
|
|
@@ -141,6 +187,57 @@ export class TranscriptionModule {
|
|
|
141
187
|
.then(() => undefined)
|
|
142
188
|
}
|
|
143
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Follow the on-device speech-to-text model's state. You get the current
|
|
192
|
+
* state right away, then every change. Use it to show "downloading 40%" or
|
|
193
|
+
* to wait for `ready` before relying on on-device transcripts.
|
|
194
|
+
*/
|
|
195
|
+
onModelStatus(handler: (status: TranscriptionModelStatus) => void): UnsubscribeFn {
|
|
196
|
+
this.modelStatusHandlers.add(handler)
|
|
197
|
+
if (!this.modelStatusUnsub) {
|
|
198
|
+
// One stream subscription per module; the host sends its current state on subscribe.
|
|
199
|
+
this.modelStatusUnsub = this.session._subscribe(MiniappStreamType.TRANSCRIPTION_MODEL_STATUS, (data) => {
|
|
200
|
+
const status = data as TranscriptionModelStatus
|
|
201
|
+
this.modelStatus = status
|
|
202
|
+
for (const listener of [...this.modelStatusHandlers]) listener(status)
|
|
203
|
+
})
|
|
204
|
+
} else if (this.modelStatus) {
|
|
205
|
+
// Later handlers get the last state locally, since the host only replays on subscribe.
|
|
206
|
+
const current = this.modelStatus
|
|
207
|
+
queueMicrotask(() => {
|
|
208
|
+
if (this.modelStatusHandlers.has(handler) && this.modelStatus === current) handler(current)
|
|
209
|
+
})
|
|
210
|
+
}
|
|
211
|
+
return this.track(() => {
|
|
212
|
+
this.modelStatusHandlers.delete(handler)
|
|
213
|
+
if (this.modelStatusHandlers.size > 0) return
|
|
214
|
+
this.modelStatusUnsub?.()
|
|
215
|
+
this.modelStatusUnsub = null
|
|
216
|
+
this.modelStatus = null
|
|
217
|
+
})
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* End the current on-device utterance now, instead of waiting for the
|
|
222
|
+
* recognizer's own end-of-speech pause. Call it when your own end-of-turn
|
|
223
|
+
* rule fires (for example after `mic.onVoiceActivity` reports enough
|
|
224
|
+
* silence).
|
|
225
|
+
*
|
|
226
|
+
* Resolves with the final text once that final transcript has been
|
|
227
|
+
* delivered to your on-device handlers, or `""` when nothing was pending.
|
|
228
|
+
* Resolves `null` when no on-device recognizer is running or the picked
|
|
229
|
+
* model can't finish early (models that transcribe in fixed chunks); their
|
|
230
|
+
* final text still arrives on its own.
|
|
231
|
+
*
|
|
232
|
+
* Applies only to on-device transcription and needs a `localOnly` listener
|
|
233
|
+
* or one that is getting on-device text right now. The phone runs one
|
|
234
|
+
* recognizer for every miniapp, so this ends the current line for every
|
|
235
|
+
* on-device subscriber, Captions included.
|
|
236
|
+
*/
|
|
237
|
+
finishUtterance(): Promise<string | null> {
|
|
238
|
+
return this.session.sendRequest<string | null>({type: MiniappRequestType.TRANSCRIPTION_FINISH})
|
|
239
|
+
}
|
|
240
|
+
|
|
144
241
|
/** Tear down every transcription subscription this module owns. */
|
|
145
242
|
stop(): void {
|
|
146
243
|
for (const u of this.unsubs) {
|
package/src/protocol.ts
CHANGED
|
@@ -26,7 +26,16 @@ export enum MiniappRequestType {
|
|
|
26
26
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
27
27
|
AUTH_REFRESH = "miniapp_auth_refresh",
|
|
28
28
|
|
|
29
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Update the set of streams the miniapp is subscribed to. `subscriptions`
|
|
31
|
+
* lists stream names; a transcription stream is `{stream, processing}`, one
|
|
32
|
+
* entry per processing its listeners ask for, and `transcriptionListeners`
|
|
33
|
+
* lists every transcription listener as `{id, stream, processing}`. Both
|
|
34
|
+
* also carry `forceLocal: true` for `localOnly`, the form hosts before
|
|
35
|
+
* `processing` read. Hosts accept that older form,
|
|
36
|
+
* `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
|
|
37
|
+
* `processing`.
|
|
38
|
+
*/
|
|
30
39
|
SUBSCRIBE = "miniapp_subscribe",
|
|
31
40
|
|
|
32
41
|
/**
|
|
@@ -45,7 +54,7 @@ export enum MiniappRequestType {
|
|
|
45
54
|
/** Stop any audio playback this miniapp initiated. */
|
|
46
55
|
STOP_AUDIO = "miniapp_stop_audio",
|
|
47
56
|
|
|
48
|
-
/** Speak text via cloud TTS
|
|
57
|
+
/** Speak text via cloud or on-device TTS, as its `processing` says. */
|
|
49
58
|
SPEAK = "miniapp_speak",
|
|
50
59
|
|
|
51
60
|
/**
|
|
@@ -189,6 +198,17 @@ export enum MiniappRequestType {
|
|
|
189
198
|
* so future reconnect logic can re-send.
|
|
190
199
|
*/
|
|
191
200
|
TRANSCRIPTION_CONFIG = "miniapp_transcription_config",
|
|
201
|
+
/**
|
|
202
|
+
* End the current on-device utterance now (transcription.finishUtterance).
|
|
203
|
+
* Result: the final text, "" when nothing was pending, or null when the
|
|
204
|
+
* model can't finish early.
|
|
205
|
+
*/
|
|
206
|
+
TRANSCRIPTION_FINISH = "miniapp_transcription_finish",
|
|
207
|
+
|
|
208
|
+
// ----- Telemetry -----
|
|
209
|
+
|
|
210
|
+
/** A miniapp's own analytics event (`session.telemetry.capture`). One-shot; hosts with `hostFeatures.telemetry`. */
|
|
211
|
+
TELEMETRY_CAPTURE = "miniapp_telemetry_capture",
|
|
192
212
|
|
|
193
213
|
// ----- Deferred in v1 -----
|
|
194
214
|
|
|
@@ -225,11 +245,11 @@ export enum MiniappRequestType {
|
|
|
225
245
|
MINIAPPS_START = "miniapp_apps_start",
|
|
226
246
|
/** Stop another miniapp. */
|
|
227
247
|
MINIAPPS_STOP = "miniapp_apps_stop",
|
|
228
|
-
/** Install
|
|
248
|
+
/** Install the miniapp at a miniapp URL. System-list miniapps only. */
|
|
229
249
|
MINIAPPS_INSTALL = "miniapp_apps_install",
|
|
230
250
|
/** Preflight a release's host/SDK/hardware requirements without downloading it. */
|
|
231
251
|
MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
|
|
232
|
-
/** Uninstall a
|
|
252
|
+
/** Uninstall a miniapp installed from a miniapp URL. System-list miniapps only. */
|
|
233
253
|
MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
|
|
234
254
|
/** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
|
|
235
255
|
ACTION_INVOKE = "miniapp_action_invoke",
|
|
@@ -294,9 +314,6 @@ export enum MiniappResponseType {
|
|
|
294
314
|
/** Push: host color scheme (light/dark) changed. */
|
|
295
315
|
COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
|
|
296
316
|
|
|
297
|
-
/** Push: a Store installation moved to another host-side phase. */
|
|
298
|
-
MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
|
|
299
|
-
|
|
300
317
|
/**
|
|
301
318
|
* Push: speaker playback state transitioned. Per-miniapp; the runtime sends
|
|
302
319
|
* to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
|
|
@@ -391,6 +408,10 @@ export enum MiniappStreamType {
|
|
|
391
408
|
// Speech / audio (cloud or local)
|
|
392
409
|
TRANSCRIPTION = "transcription", // language variant: "transcription:en-US"
|
|
393
410
|
TRANSLATION = "translation", // language variant: "translation:en-US"
|
|
411
|
+
/** State of the on-device speech-to-text model (session.transcription.onModelStatus). */
|
|
412
|
+
TRANSCRIPTION_MODEL_STATUS = "transcription_model_status",
|
|
413
|
+
/** On-device "Hey Mentra" keyword spotter (session.mic.onWakeWord). */
|
|
414
|
+
WAKE_WORD = "wake_word",
|
|
394
415
|
AUDIO_CHUNK = "audio_chunk",
|
|
395
416
|
VAD = "vad",
|
|
396
417
|
|
|
@@ -444,7 +465,7 @@ export enum MiniappErrorCode {
|
|
|
444
465
|
TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
|
|
445
466
|
TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
|
|
446
467
|
|
|
447
|
-
/** `speak({
|
|
468
|
+
/** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
|
|
448
469
|
TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
|
|
449
470
|
|
|
450
471
|
/** Not connected / pre-ACK and transport closed. */
|
|
@@ -459,21 +480,16 @@ export enum MiniappErrorCode {
|
|
|
459
480
|
// ----- Inter-miniapp interop -----
|
|
460
481
|
/**
|
|
461
482
|
* Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
|
|
462
|
-
* setWifiAdbState) reject with this code
|
|
483
|
+
* setWifiAdbState) reject with this code, as installs do from any miniapp off
|
|
484
|
+
* the system list. Also: an uninstall of a miniapp that isn't a URL install.
|
|
463
485
|
*/
|
|
464
486
|
NOT_PERMITTED = "NOT_PERMITTED",
|
|
465
487
|
/** Target miniapp is not installed. */
|
|
466
488
|
APP_NOT_FOUND = "APP_NOT_FOUND",
|
|
467
489
|
/** Target miniapp is incompatible with the connected glasses. */
|
|
468
490
|
APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
|
|
469
|
-
/**
|
|
491
|
+
/** The miniapp didn't install; the error's `reason` says why (`InstallMiniappFailureReason`). */
|
|
470
492
|
INSTALL_FAILED = "INSTALL_FAILED",
|
|
471
|
-
/** Another Store installation is already in progress for the same package. */
|
|
472
|
-
INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
|
|
473
|
-
/** An automatic update was deferred because the target is running or starting. */
|
|
474
|
-
APP_RUNNING = "APP_RUNNING",
|
|
475
|
-
/** Another Store owns the installed package's update provenance. */
|
|
476
|
-
STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
|
|
477
493
|
/** Target miniapp does not declare the requested action. */
|
|
478
494
|
ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
|
|
479
495
|
/** Target connected but never registered a handler for the action (within the buffer window). */
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* miniapp. Currently:
|
|
6
6
|
* - Keeps `<html class="dark">` in sync with the host's color scheme so the
|
|
7
7
|
* shared theme CSS switches automatically.
|
|
8
|
+
* - Calls `mentra.ready()` once the app has mounted, which lifts the host's
|
|
9
|
+
* splash and fires the background's `session.ui.onOpen` handlers.
|
|
8
10
|
*
|
|
9
11
|
* Usage:
|
|
10
12
|
* createRoot(root).render(
|
|
@@ -17,7 +19,7 @@
|
|
|
17
19
|
* things themselves can opt out without forking the provider.
|
|
18
20
|
*/
|
|
19
21
|
|
|
20
|
-
import {type ReactNode} from "react"
|
|
22
|
+
import {type ReactNode, useEffect} from "react"
|
|
21
23
|
|
|
22
24
|
import {useColorScheme} from "./useColorScheme"
|
|
23
25
|
|
|
@@ -32,11 +34,30 @@ export interface MentraProviderProps {
|
|
|
32
34
|
* (e.g. you integrate a theming library like next-themes).
|
|
33
35
|
*/
|
|
34
36
|
syncColorScheme?: boolean
|
|
37
|
+
/**
|
|
38
|
+
* Call `mentra.ready()` after the first mount. A parent's effects run after
|
|
39
|
+
* its children's, so every `mentra.on` subscription the app makes on mount
|
|
40
|
+
* is attached before the background is told the UI is open. Without
|
|
41
|
+
* `ready()` the host keeps its splash up and reports the miniapp as failed.
|
|
42
|
+
*
|
|
43
|
+
* Default: true. Set to false to call `mentra.ready()` yourself, e.g. to keep
|
|
44
|
+
* the splash up until the first screen has its data.
|
|
45
|
+
*/
|
|
46
|
+
autoReady?: boolean
|
|
35
47
|
}
|
|
36
48
|
|
|
37
|
-
|
|
49
|
+
function getBridge(): {ready?: () => void} | null {
|
|
50
|
+
if (typeof window === "undefined") return null
|
|
51
|
+
return (window as unknown as {mentra?: {ready?: () => void}}).mentra ?? null
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function MentraProvider({children, syncColorScheme = true, autoReady = true}: MentraProviderProps) {
|
|
38
55
|
const scheme = useColorScheme()
|
|
39
56
|
|
|
57
|
+
useEffect(() => {
|
|
58
|
+
if (autoReady) getBridge()?.ready?.()
|
|
59
|
+
}, [autoReady])
|
|
60
|
+
|
|
40
61
|
// Runs during render — before children paint, before effects fire.
|
|
41
62
|
if (syncColorScheme && typeof document !== "undefined") {
|
|
42
63
|
const el = document.documentElement
|
package/src/session.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {ConfigurationModule} from "./modules/configuration"
|
|
|
26
26
|
import {DashboardAPI} from "./modules/dashboard"
|
|
27
27
|
import {DisplayManager} from "./modules/display"
|
|
28
28
|
import {EventManager, type TranscriptionEventRoute, type UnsubscribeFn} from "./modules/events"
|
|
29
|
+
import type {SpeechProcessingOptions} from "./modules/speechProcessing"
|
|
29
30
|
import {GlassesModule} from "./modules/glasses"
|
|
30
31
|
import {HeadingModule} from "./modules/heading"
|
|
31
32
|
import {ImuModule} from "./modules/imu"
|
|
@@ -45,6 +46,7 @@ import {StreamModule} from "./modules/stream"
|
|
|
45
46
|
import {SystemModule} from "./modules/system"
|
|
46
47
|
import {MiniappsModule} from "./modules/miniapps"
|
|
47
48
|
import {ActionsModule} from "./modules/actions"
|
|
49
|
+
import {TelemetryModule} from "./modules/telemetry"
|
|
48
50
|
import {BlobModule} from "./modules/blob"
|
|
49
51
|
import {
|
|
50
52
|
MeetingModule,
|
|
@@ -130,6 +132,13 @@ export interface HostFeatures {
|
|
|
130
132
|
captureAudio?: boolean
|
|
131
133
|
/** Host keeps the UI closed until a background session that announced it sends READY. */
|
|
132
134
|
initReady?: boolean
|
|
135
|
+
/**
|
|
136
|
+
* Host installs miniapps from a miniapp URL through `session.miniapps.install({url})`. Only a miniapp on the
|
|
137
|
+
* Mentra App's system list may call it; older Mentra Apps answer NOT_PERMITTED.
|
|
138
|
+
*/
|
|
139
|
+
miniappInstall?: boolean
|
|
140
|
+
/** Host takes the miniapp's own events from `session.telemetry.capture` (Fleet Management's miniapp telemetry). */
|
|
141
|
+
telemetry?: boolean
|
|
133
142
|
}
|
|
134
143
|
|
|
135
144
|
export interface MiniappAuthState {
|
|
@@ -185,6 +194,8 @@ export class ConnectAckTimeoutError extends Error {
|
|
|
185
194
|
export interface MiniappRequestError {
|
|
186
195
|
code: string
|
|
187
196
|
message: string
|
|
197
|
+
/** A finer cause, where a request documents one (`INSTALL_FAILED` from `session.miniapps.install`). */
|
|
198
|
+
reason?: string
|
|
188
199
|
}
|
|
189
200
|
|
|
190
201
|
// ---------------------------------------------------------------------------
|
|
@@ -296,6 +307,8 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
296
307
|
* declared action; `handle` (open to all) exposes one of your own.
|
|
297
308
|
*/
|
|
298
309
|
public readonly actions: ActionsModule
|
|
310
|
+
/** The miniapp's own analytics, shown in its organization's Fleet Management (see {@link TelemetryModule}). */
|
|
311
|
+
public readonly telemetry: TelemetryModule
|
|
299
312
|
|
|
300
313
|
/** Phone-declared glasses capabilities. Null until CONNECT_ACK arrives. */
|
|
301
314
|
public capabilities: GlassesCapabilities | null = null
|
|
@@ -403,6 +416,12 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
403
416
|
this.ui = new UIModuleImpl<TChannels>(this)
|
|
404
417
|
this.miniapps = new MiniappsModule(this)
|
|
405
418
|
this.actions = new ActionsModule(this)
|
|
419
|
+
this.telemetry = new TelemetryModule(this)
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** @internal Whether CONNECT_ACK arrived, so `hostFeatures` is the host's answer and not just unknown. */
|
|
423
|
+
get handshakeDone(): boolean {
|
|
424
|
+
return this.handshakeComplete
|
|
406
425
|
}
|
|
407
426
|
|
|
408
427
|
/**
|
|
@@ -476,7 +495,7 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
476
495
|
_subscribe(
|
|
477
496
|
streamType: string,
|
|
478
497
|
handler: (data: unknown) => void,
|
|
479
|
-
options:
|
|
498
|
+
options: SpeechProcessingOptions = {},
|
|
480
499
|
): UnsubscribeFn {
|
|
481
500
|
return this.events.subscribe(streamType, handler, options)
|
|
482
501
|
}
|
|
@@ -999,22 +1018,6 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
999
1018
|
return
|
|
1000
1019
|
}
|
|
1001
1020
|
|
|
1002
|
-
case MiniappResponseType.MINIAPPS_INSTALL_PROGRESS: {
|
|
1003
|
-
const {packageName, version, phase} = payload as Record<string, unknown>
|
|
1004
|
-
if (
|
|
1005
|
-
typeof packageName === "string" &&
|
|
1006
|
-
typeof version === "string" &&
|
|
1007
|
-
(phase === "downloading" ||
|
|
1008
|
-
phase === "verifying" ||
|
|
1009
|
-
phase === "extracting" ||
|
|
1010
|
-
phase === "activating" ||
|
|
1011
|
-
phase === "complete")
|
|
1012
|
-
) {
|
|
1013
|
-
this.miniapps._deliverInstallProgress({packageName, version, phase})
|
|
1014
|
-
}
|
|
1015
|
-
return
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
1021
|
case MiniappResponseType.REQUEST_RESULT: {
|
|
1019
1022
|
const requestId = envelope.requestId
|
|
1020
1023
|
if (!requestId) return
|
package/src/ui/index.ts
CHANGED
|
@@ -71,10 +71,13 @@ export interface MentraUiGlobal<TChannels extends object = Record<string, unknow
|
|
|
71
71
|
onClose(cb: () => void): () => void
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
74
|
+
* Tells the host the WebView is mounted and ready to receive messages
|
|
75
|
+
* from background. `<MentraProvider>` calls it once the app has mounted;
|
|
76
|
+
* call it yourself only without the provider (or with `autoReady={false}`).
|
|
77
|
+
* The host's splash stays up until it lands, and a page that loads
|
|
78
|
+
* without calling it is reported as failed. Idempotent — subsequent
|
|
79
|
+
* calls no-op. Background's `session.ui.onOpen` handlers fire after this
|
|
80
|
+
* lands.
|
|
78
81
|
*/
|
|
79
82
|
ready(): void
|
|
80
83
|
}
|