@mentra/miniapp 3.4.0-dev.999 → 3.4.1-beta.1409
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/dist/background/index.d.ts +5 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/background/register.d.ts +3 -1
- package/dist/background/register.d.ts.map +1 -1
- package/dist/background/register.js +19 -2
- package/dist/background/register.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/blob.d.ts +2 -1
- package/dist/modules/blob.d.ts.map +1 -1
- package/dist/modules/blob.js +2 -1
- package/dist/modules/blob.js.map +1 -1
- package/dist/modules/events.d.ts +16 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/mic.d.ts +26 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +13 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +44 -57
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +21 -24
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +41 -10
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +17 -6
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/stream.d.ts +39 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +13 -1
- package/dist/modules/stream.js.map +1 -1
- package/dist/modules/system.d.ts +8 -3
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +1 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/telemetry.d.ts +25 -0
- package/dist/modules/telemetry.d.ts.map +1 -0
- package/dist/modules/telemetry.js +61 -0
- package/dist/modules/telemetry.js.map +1 -0
- package/dist/modules/transcription.d.ts +70 -3
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +68 -2
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +29 -15
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +30 -15
- package/dist/protocol.js.map +1 -1
- package/dist/react/MentraProvider.d.ts +13 -1
- package/dist/react/MentraProvider.d.ts.map +1 -1
- package/dist/react/MentraProvider.js +31 -1
- package/dist/react/MentraProvider.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +13 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.js +40 -1
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -1
- package/dist/react/stream-preview/transport.d.ts.map +1 -1
- package/dist/react/stream-preview/transport.js +2 -0
- package/dist/react/stream-preview/transport.js.map +1 -1
- package/dist/session.d.ts +59 -6
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +203 -43
- package/dist/session.js.map +1 -1
- package/dist/transport/auto.js +6 -0
- package/dist/transport/auto.js.map +1 -1
- package/dist/transport/dispatch.d.ts +2 -0
- package/dist/transport/dispatch.d.ts.map +1 -1
- package/dist/transport/dispatch.js +7 -2
- package/dist/transport/dispatch.js.map +1 -1
- package/dist/transport/local-socket.d.ts +6 -0
- package/dist/transport/local-socket.d.ts.map +1 -1
- package/dist/transport/local-socket.js +34 -2
- package/dist/transport/local-socket.js.map +1 -1
- package/dist/transport/types.d.ts +4 -1
- package/dist/transport/types.d.ts.map +1 -1
- package/dist/ui/index.d.ts +7 -4
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +6 -3
- package/src/background/register.ts +24 -2
- package/src/index.ts +13 -3
- package/src/modules/blob.ts +2 -1
- package/src/modules/events.ts +65 -51
- package/src/modules/mic.ts +30 -4
- package/src/modules/miniapps.ts +59 -71
- package/src/modules/speaker.ts +56 -12
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/stream.ts +44 -1
- package/src/modules/system.ts +8 -3
- package/src/modules/telemetry.ts +62 -0
- package/src/modules/transcription.ts +106 -9
- package/src/protocol.ts +32 -16
- package/src/react/MentraProvider.tsx +23 -2
- package/src/react/stream-preview/PreviewConnection.ts +54 -3
- package/src/react/stream-preview/transport.ts +4 -8
- package/src/session.ts +206 -52
- package/src/transport/auto.ts +5 -0
- package/src/transport/dispatch.ts +7 -2
- package/src/transport/local-socket.ts +30 -1
- package/src/transport/types.ts +4 -1
- package/src/ui/index.ts +7 -4
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* session.transcription.on(handler) // auto-detect
|
|
7
7
|
* session.transcription.forLanguage("en-US", handler) // single language
|
|
8
8
|
* session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
|
|
9
|
+
* session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
|
|
9
10
|
* session.transcription.configure({languageHints, vocabulary, diarization})
|
|
11
|
+
* session.transcription.onModelStatus(handler) // on-device model state
|
|
12
|
+
* session.transcription.finishUtterance() // end the on-device line now
|
|
10
13
|
* session.transcription.stop() // tear down all
|
|
11
14
|
*
|
|
12
15
|
* Hoisted to top-level (vs. nested under `session.mic`) because transcription
|
|
@@ -23,6 +26,7 @@ import {MiniappSession} from "../session"
|
|
|
23
26
|
import type {TranscriptionData, UnsubscribeFn} from "./events"
|
|
24
27
|
import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
|
|
25
28
|
import type {LanguageHint, TranscriptionLanguage} from "./languages"
|
|
29
|
+
import {resolveSpeechProcessing, type SpeechProcessing} from "./speechProcessing"
|
|
26
30
|
|
|
27
31
|
/** Configuration for cloud-side transcription behavior. */
|
|
28
32
|
export interface TranscriptionConfig {
|
|
@@ -41,16 +45,56 @@ export interface TranscriptionConfig {
|
|
|
41
45
|
|
|
42
46
|
export interface TranscriptionOptions {
|
|
43
47
|
/**
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
48
|
+
* Where this listener's speech is transcribed. Default `"preferCloud"`.
|
|
49
|
+
*
|
|
50
|
+
* - `"preferCloud"`: the cloud; the phone's on-device model only while the
|
|
51
|
+
* cloud is unreachable.
|
|
52
|
+
* - `"preferLocal"`: the on-device model while it runs in this listener's
|
|
53
|
+
* language, otherwise the cloud. `on()` follows the model's language,
|
|
54
|
+
* unless `configure({languageHints})` names one the model can't
|
|
55
|
+
* transcribe; then it stays on the cloud.
|
|
56
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
57
|
+
*
|
|
58
|
+
* The phone runs one on-device model, the one picked on its Speech page, and
|
|
59
|
+
* downloads it when a `preferLocal` or `localOnly` listener needs it; follow
|
|
60
|
+
* it with `onModelStatus`. A listener moves between the cloud and on-device
|
|
61
|
+
* during a pause in speech, once both have finished with what was said
|
|
62
|
+
* before it, so it never misses or repeats speech; `TranscriptionData.source`
|
|
63
|
+
* says where each result came from. Each listener routes on its own: other
|
|
64
|
+
* listeners on the same stream keep their choice.
|
|
47
65
|
*/
|
|
66
|
+
processing?: SpeechProcessing
|
|
67
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
48
68
|
forceLocal?: boolean
|
|
49
69
|
}
|
|
50
70
|
|
|
71
|
+
/**
|
|
72
|
+
* State of the on-device speech-to-text model behind this miniapp's on-device
|
|
73
|
+
* transcription. The phone runs the one model picked on its Speech page for
|
|
74
|
+
* every miniapp.
|
|
75
|
+
*/
|
|
76
|
+
export interface TranscriptionModelStatus {
|
|
77
|
+
/** BCP-47 language of the picked model, e.g. "en-US". */
|
|
78
|
+
language: string
|
|
79
|
+
/**
|
|
80
|
+
* - `downloading`: the model is being downloaded (`progress` is 0–100).
|
|
81
|
+
* - `loading`: the model is loading into memory.
|
|
82
|
+
* - `ready`: on-device transcripts are flowing.
|
|
83
|
+
* - `unavailable`: no on-device transcription right now; `reason` says why
|
|
84
|
+
* (for example no `preferLocal` or `localOnly` listener, a failed
|
|
85
|
+
* download, or a listener in a language other than the picked model's).
|
|
86
|
+
*/
|
|
87
|
+
state: "downloading" | "loading" | "ready" | "unavailable"
|
|
88
|
+
progress?: number
|
|
89
|
+
reason?: string
|
|
90
|
+
}
|
|
91
|
+
|
|
51
92
|
export class TranscriptionModule {
|
|
52
93
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
53
94
|
private currentConfig: TranscriptionConfig | null = null
|
|
95
|
+
private readonly modelStatusHandlers = new Set<(status: TranscriptionModelStatus) => void>()
|
|
96
|
+
private modelStatusUnsub: UnsubscribeFn | null = null
|
|
97
|
+
private modelStatus: TranscriptionModelStatus | null = null
|
|
54
98
|
|
|
55
99
|
constructor(private readonly session: MiniappSession) {}
|
|
56
100
|
|
|
@@ -63,8 +107,11 @@ export class TranscriptionModule {
|
|
|
63
107
|
* event) preserves existing semantics.
|
|
64
108
|
*/
|
|
65
109
|
on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
|
|
110
|
+
const processing = resolveSpeechProcessing(options, "transcription.on(options)")
|
|
66
111
|
return this.track(
|
|
67
|
-
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void,
|
|
112
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, {
|
|
113
|
+
processing,
|
|
114
|
+
}),
|
|
68
115
|
)
|
|
69
116
|
}
|
|
70
117
|
|
|
@@ -90,16 +137,15 @@ export class TranscriptionModule {
|
|
|
90
137
|
const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
|
|
91
138
|
requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
|
|
92
139
|
)
|
|
140
|
+
const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)")
|
|
93
141
|
if (langs.length === 0) return () => {}
|
|
94
142
|
|
|
95
143
|
const unsubs: UnsubscribeFn[] = []
|
|
96
144
|
for (const lang of langs) {
|
|
97
145
|
unsubs.push(
|
|
98
|
-
this.session._subscribe(
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
options,
|
|
102
|
-
),
|
|
146
|
+
this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void, {
|
|
147
|
+
processing,
|
|
148
|
+
}),
|
|
103
149
|
)
|
|
104
150
|
}
|
|
105
151
|
const combined: UnsubscribeFn = () => {
|
|
@@ -141,6 +187,57 @@ export class TranscriptionModule {
|
|
|
141
187
|
.then(() => undefined)
|
|
142
188
|
}
|
|
143
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Follow the on-device speech-to-text model's state. You get the current
|
|
192
|
+
* state right away, then every change. Use it to show "downloading 40%" or
|
|
193
|
+
* to wait for `ready` before relying on on-device transcripts.
|
|
194
|
+
*/
|
|
195
|
+
onModelStatus(handler: (status: TranscriptionModelStatus) => void): UnsubscribeFn {
|
|
196
|
+
this.modelStatusHandlers.add(handler)
|
|
197
|
+
if (!this.modelStatusUnsub) {
|
|
198
|
+
// One stream subscription per module; the host sends its current state on subscribe.
|
|
199
|
+
this.modelStatusUnsub = this.session._subscribe(MiniappStreamType.TRANSCRIPTION_MODEL_STATUS, (data) => {
|
|
200
|
+
const status = data as TranscriptionModelStatus
|
|
201
|
+
this.modelStatus = status
|
|
202
|
+
for (const listener of [...this.modelStatusHandlers]) listener(status)
|
|
203
|
+
})
|
|
204
|
+
} else if (this.modelStatus) {
|
|
205
|
+
// Later handlers get the last state locally, since the host only replays on subscribe.
|
|
206
|
+
const current = this.modelStatus
|
|
207
|
+
queueMicrotask(() => {
|
|
208
|
+
if (this.modelStatusHandlers.has(handler) && this.modelStatus === current) handler(current)
|
|
209
|
+
})
|
|
210
|
+
}
|
|
211
|
+
return this.track(() => {
|
|
212
|
+
this.modelStatusHandlers.delete(handler)
|
|
213
|
+
if (this.modelStatusHandlers.size > 0) return
|
|
214
|
+
this.modelStatusUnsub?.()
|
|
215
|
+
this.modelStatusUnsub = null
|
|
216
|
+
this.modelStatus = null
|
|
217
|
+
})
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* End the current on-device utterance now, instead of waiting for the
|
|
222
|
+
* recognizer's own end-of-speech pause. Call it when your own end-of-turn
|
|
223
|
+
* rule fires (for example after `mic.onVoiceActivity` reports enough
|
|
224
|
+
* silence).
|
|
225
|
+
*
|
|
226
|
+
* Resolves with the final text once that final transcript has been
|
|
227
|
+
* delivered to your on-device handlers, or `""` when nothing was pending.
|
|
228
|
+
* Resolves `null` when no on-device recognizer is running or the picked
|
|
229
|
+
* model can't finish early (models that transcribe in fixed chunks); their
|
|
230
|
+
* final text still arrives on its own.
|
|
231
|
+
*
|
|
232
|
+
* Applies only to on-device transcription and needs a `localOnly` listener
|
|
233
|
+
* or one that is getting on-device text right now. The phone runs one
|
|
234
|
+
* recognizer for every miniapp, so this ends the current line for every
|
|
235
|
+
* on-device subscriber, Captions included.
|
|
236
|
+
*/
|
|
237
|
+
finishUtterance(): Promise<string | null> {
|
|
238
|
+
return this.session.sendRequest<string | null>({type: MiniappRequestType.TRANSCRIPTION_FINISH})
|
|
239
|
+
}
|
|
240
|
+
|
|
144
241
|
/** Tear down every transcription subscription this module owns. */
|
|
145
242
|
stop(): void {
|
|
146
243
|
for (const u of this.unsubs) {
|
package/src/protocol.ts
CHANGED
|
@@ -26,7 +26,16 @@ export enum MiniappRequestType {
|
|
|
26
26
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
27
27
|
AUTH_REFRESH = "miniapp_auth_refresh",
|
|
28
28
|
|
|
29
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Update the set of streams the miniapp is subscribed to. `subscriptions`
|
|
31
|
+
* lists stream names; a transcription stream is `{stream, processing}`, one
|
|
32
|
+
* entry per processing its listeners ask for, and `transcriptionListeners`
|
|
33
|
+
* lists every transcription listener as `{id, stream, processing}`. Both
|
|
34
|
+
* also carry `forceLocal: true` for `localOnly`, the form hosts before
|
|
35
|
+
* `processing` read. Hosts accept that older form,
|
|
36
|
+
* `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
|
|
37
|
+
* `processing`.
|
|
38
|
+
*/
|
|
30
39
|
SUBSCRIBE = "miniapp_subscribe",
|
|
31
40
|
|
|
32
41
|
/**
|
|
@@ -45,7 +54,7 @@ export enum MiniappRequestType {
|
|
|
45
54
|
/** Stop any audio playback this miniapp initiated. */
|
|
46
55
|
STOP_AUDIO = "miniapp_stop_audio",
|
|
47
56
|
|
|
48
|
-
/** Speak text via cloud TTS
|
|
57
|
+
/** Speak text via cloud or on-device TTS, as its `processing` says. */
|
|
49
58
|
SPEAK = "miniapp_speak",
|
|
50
59
|
|
|
51
60
|
/**
|
|
@@ -189,6 +198,17 @@ export enum MiniappRequestType {
|
|
|
189
198
|
* so future reconnect logic can re-send.
|
|
190
199
|
*/
|
|
191
200
|
TRANSCRIPTION_CONFIG = "miniapp_transcription_config",
|
|
201
|
+
/**
|
|
202
|
+
* End the current on-device utterance now (transcription.finishUtterance).
|
|
203
|
+
* Result: the final text, "" when nothing was pending, or null when the
|
|
204
|
+
* model can't finish early.
|
|
205
|
+
*/
|
|
206
|
+
TRANSCRIPTION_FINISH = "miniapp_transcription_finish",
|
|
207
|
+
|
|
208
|
+
// ----- Telemetry -----
|
|
209
|
+
|
|
210
|
+
/** A miniapp's own analytics event (`session.telemetry.capture`). One-shot; hosts with `hostFeatures.telemetry`. */
|
|
211
|
+
TELEMETRY_CAPTURE = "miniapp_telemetry_capture",
|
|
192
212
|
|
|
193
213
|
// ----- Deferred in v1 -----
|
|
194
214
|
|
|
@@ -225,11 +245,11 @@ export enum MiniappRequestType {
|
|
|
225
245
|
MINIAPPS_START = "miniapp_apps_start",
|
|
226
246
|
/** Stop another miniapp. */
|
|
227
247
|
MINIAPPS_STOP = "miniapp_apps_stop",
|
|
228
|
-
/** Install
|
|
248
|
+
/** Install the miniapp at a miniapp URL. System-list miniapps only. */
|
|
229
249
|
MINIAPPS_INSTALL = "miniapp_apps_install",
|
|
230
250
|
/** Preflight a release's host/SDK/hardware requirements without downloading it. */
|
|
231
251
|
MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
|
|
232
|
-
/** Uninstall a
|
|
252
|
+
/** Uninstall a miniapp installed from a miniapp URL. System-list miniapps only. */
|
|
233
253
|
MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
|
|
234
254
|
/** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
|
|
235
255
|
ACTION_INVOKE = "miniapp_action_invoke",
|
|
@@ -294,9 +314,6 @@ export enum MiniappResponseType {
|
|
|
294
314
|
/** Push: host color scheme (light/dark) changed. */
|
|
295
315
|
COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
|
|
296
316
|
|
|
297
|
-
/** Push: a Store installation moved to another host-side phase. */
|
|
298
|
-
MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
|
|
299
|
-
|
|
300
317
|
/**
|
|
301
318
|
* Push: speaker playback state transitioned. Per-miniapp; the runtime sends
|
|
302
319
|
* to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
|
|
@@ -391,6 +408,10 @@ export enum MiniappStreamType {
|
|
|
391
408
|
// Speech / audio (cloud or local)
|
|
392
409
|
TRANSCRIPTION = "transcription", // language variant: "transcription:en-US"
|
|
393
410
|
TRANSLATION = "translation", // language variant: "translation:en-US"
|
|
411
|
+
/** State of the on-device speech-to-text model (session.transcription.onModelStatus). */
|
|
412
|
+
TRANSCRIPTION_MODEL_STATUS = "transcription_model_status",
|
|
413
|
+
/** On-device "Hey Mentra" keyword spotter (session.mic.onWakeWord). */
|
|
414
|
+
WAKE_WORD = "wake_word",
|
|
394
415
|
AUDIO_CHUNK = "audio_chunk",
|
|
395
416
|
VAD = "vad",
|
|
396
417
|
|
|
@@ -444,7 +465,7 @@ export enum MiniappErrorCode {
|
|
|
444
465
|
TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
|
|
445
466
|
TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
|
|
446
467
|
|
|
447
|
-
/** `speak({
|
|
468
|
+
/** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
|
|
448
469
|
TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
|
|
449
470
|
|
|
450
471
|
/** Not connected / pre-ACK and transport closed. */
|
|
@@ -459,21 +480,16 @@ export enum MiniappErrorCode {
|
|
|
459
480
|
// ----- Inter-miniapp interop -----
|
|
460
481
|
/**
|
|
461
482
|
* Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
|
|
462
|
-
* setWifiAdbState) reject with this code
|
|
483
|
+
* setWifiAdbState) reject with this code, as installs do from any miniapp off
|
|
484
|
+
* the system list. Also: an uninstall of a miniapp that isn't a URL install.
|
|
463
485
|
*/
|
|
464
486
|
NOT_PERMITTED = "NOT_PERMITTED",
|
|
465
487
|
/** Target miniapp is not installed. */
|
|
466
488
|
APP_NOT_FOUND = "APP_NOT_FOUND",
|
|
467
489
|
/** Target miniapp is incompatible with the connected glasses. */
|
|
468
490
|
APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
|
|
469
|
-
/**
|
|
491
|
+
/** The miniapp didn't install; the error's `reason` says why (`InstallMiniappFailureReason`). */
|
|
470
492
|
INSTALL_FAILED = "INSTALL_FAILED",
|
|
471
|
-
/** Another Store installation is already in progress for the same package. */
|
|
472
|
-
INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
|
|
473
|
-
/** An automatic update was deferred because the target is running or starting. */
|
|
474
|
-
APP_RUNNING = "APP_RUNNING",
|
|
475
|
-
/** Another Store owns the installed package's update provenance. */
|
|
476
|
-
STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
|
|
477
493
|
/** Target miniapp does not declare the requested action. */
|
|
478
494
|
ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
|
|
479
495
|
/** Target connected but never registered a handler for the action (within the buffer window). */
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* miniapp. Currently:
|
|
6
6
|
* - Keeps `<html class="dark">` in sync with the host's color scheme so the
|
|
7
7
|
* shared theme CSS switches automatically.
|
|
8
|
+
* - Calls `mentra.ready()` once the app has mounted, which lifts the host's
|
|
9
|
+
* splash and fires the background's `session.ui.onOpen` handlers.
|
|
8
10
|
*
|
|
9
11
|
* Usage:
|
|
10
12
|
* createRoot(root).render(
|
|
@@ -17,7 +19,7 @@
|
|
|
17
19
|
* things themselves can opt out without forking the provider.
|
|
18
20
|
*/
|
|
19
21
|
|
|
20
|
-
import {type ReactNode} from "react"
|
|
22
|
+
import {type ReactNode, useEffect} from "react"
|
|
21
23
|
|
|
22
24
|
import {useColorScheme} from "./useColorScheme"
|
|
23
25
|
|
|
@@ -32,11 +34,30 @@ export interface MentraProviderProps {
|
|
|
32
34
|
* (e.g. you integrate a theming library like next-themes).
|
|
33
35
|
*/
|
|
34
36
|
syncColorScheme?: boolean
|
|
37
|
+
/**
|
|
38
|
+
* Call `mentra.ready()` after the first mount. A parent's effects run after
|
|
39
|
+
* its children's, so every `mentra.on` subscription the app makes on mount
|
|
40
|
+
* is attached before the background is told the UI is open. Without
|
|
41
|
+
* `ready()` the host keeps its splash up and reports the miniapp as failed.
|
|
42
|
+
*
|
|
43
|
+
* Default: true. Set to false to call `mentra.ready()` yourself, e.g. to keep
|
|
44
|
+
* the splash up until the first screen has its data.
|
|
45
|
+
*/
|
|
46
|
+
autoReady?: boolean
|
|
35
47
|
}
|
|
36
48
|
|
|
37
|
-
|
|
49
|
+
function getBridge(): {ready?: () => void} | null {
|
|
50
|
+
if (typeof window === "undefined") return null
|
|
51
|
+
return (window as unknown as {mentra?: {ready?: () => void}}).mentra ?? null
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function MentraProvider({children, syncColorScheme = true, autoReady = true}: MentraProviderProps) {
|
|
38
55
|
const scheme = useColorScheme()
|
|
39
56
|
|
|
57
|
+
useEffect(() => {
|
|
58
|
+
if (autoReady) getBridge()?.ready?.()
|
|
59
|
+
}, [autoReady])
|
|
60
|
+
|
|
40
61
|
// Runs during render — before children paint, before effects fire.
|
|
41
62
|
if (syncColorScheme && typeof document !== "undefined") {
|
|
42
63
|
const el = document.documentElement
|
|
@@ -104,6 +104,41 @@ export interface PreviewConnectionDeps {
|
|
|
104
104
|
isDocumentVisible?: () => boolean
|
|
105
105
|
/** Delay helper. Returns a cancel function. Defaults to `setTimeout`. */
|
|
106
106
|
after?: (ms: number, fn: () => void) => () => void
|
|
107
|
+
/** Reloads this document once, for a port the host added after it loaded. Defaults to the page's own. */
|
|
108
|
+
documentReload?: PreviewDocumentReload
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Android's WebMessage port exists only in documents created after the host added it. A document
|
|
113
|
+
* that loaded before has none, and only a new document gets one.
|
|
114
|
+
*/
|
|
115
|
+
export interface PreviewDocumentReload {
|
|
116
|
+
/** Reload now, unless this page already reloaded for the port without opening since. False when it did not. */
|
|
117
|
+
request(): boolean
|
|
118
|
+
/** The transport opened: a later document missing its port may reload again. */
|
|
119
|
+
clear(): void
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const PORT_RELOAD_KEY = "mentra.preview.portReload"
|
|
123
|
+
|
|
124
|
+
const defaultDocumentReload: PreviewDocumentReload = {
|
|
125
|
+
request() {
|
|
126
|
+
try {
|
|
127
|
+
if (typeof window === "undefined" || window.sessionStorage.getItem(PORT_RELOAD_KEY)) return false
|
|
128
|
+
window.sessionStorage.setItem(PORT_RELOAD_KEY, "1")
|
|
129
|
+
window.location.reload()
|
|
130
|
+
return true
|
|
131
|
+
} catch {
|
|
132
|
+
return false
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
clear() {
|
|
136
|
+
try {
|
|
137
|
+
if (typeof window !== "undefined") window.sessionStorage.removeItem(PORT_RELOAD_KEY)
|
|
138
|
+
} catch {
|
|
139
|
+
// Storage unavailable: nothing was marked either.
|
|
140
|
+
}
|
|
141
|
+
},
|
|
107
142
|
}
|
|
108
143
|
|
|
109
144
|
function defaultListen(target: "document" | "window", type: string, handler: () => void): () => void {
|
|
@@ -113,8 +148,8 @@ function defaultListen(target: "document" | "window", type: string, handler: ()
|
|
|
113
148
|
? null
|
|
114
149
|
: document
|
|
115
150
|
: typeof window === "undefined"
|
|
116
|
-
|
|
117
|
-
|
|
151
|
+
? null
|
|
152
|
+
: window
|
|
118
153
|
if (!node) return () => {}
|
|
119
154
|
node.addEventListener(type, handler)
|
|
120
155
|
return () => node.removeEventListener(type, handler)
|
|
@@ -162,6 +197,7 @@ export class PreviewConnection {
|
|
|
162
197
|
private readonly now: () => number
|
|
163
198
|
private readonly isVisible: () => boolean
|
|
164
199
|
private readonly after: (ms: number, fn: () => void) => () => void
|
|
200
|
+
private readonly documentReload: PreviewDocumentReload
|
|
165
201
|
private readonly unsubscribers: Array<() => void> = []
|
|
166
202
|
private counters = {handshakes: 0, reconnects: 0, staleReplies: 0, staleLocalOps: 0, tierChanges: 0}
|
|
167
203
|
|
|
@@ -170,6 +206,7 @@ export class PreviewConnection {
|
|
|
170
206
|
this.now = deps.now ?? (() => (typeof performance !== "undefined" ? performance.now() : Date.now()))
|
|
171
207
|
this.isVisible = deps.isDocumentVisible ?? defaultIsVisible
|
|
172
208
|
this.after = deps.after ?? defaultAfter
|
|
209
|
+
this.documentReload = deps.documentReload ?? defaultDocumentReload
|
|
173
210
|
this.state = deps.channel ? "idle" : "unavailable"
|
|
174
211
|
this.creditLoop = new CreditLoop({
|
|
175
212
|
sendAck: (gen, seq) => this.transport?.sendText(ackMessage(gen, seq)),
|
|
@@ -412,7 +449,7 @@ export class PreviewConnection {
|
|
|
412
449
|
this.counters.handshakes += 1
|
|
413
450
|
this.setState("connecting")
|
|
414
451
|
const gen = ++this.connectionGen
|
|
415
|
-
let transport: PreviewTransport
|
|
452
|
+
let transport: PreviewTransport | null = null
|
|
416
453
|
try {
|
|
417
454
|
transport = this.createTransport(config.transport, {
|
|
418
455
|
url: config.url,
|
|
@@ -425,10 +462,23 @@ export class PreviewConnection {
|
|
|
425
462
|
} catch (error) {
|
|
426
463
|
const reason = (error as {reason?: string}).reason ?? "connect_failed"
|
|
427
464
|
previewTraceWarn("transport_failed", {reason, docGen: this.docGen, transport: config.transport})
|
|
465
|
+
// A connect can fail after the transport attached to the port: a retry must not leave it listening.
|
|
466
|
+
transport?.close()
|
|
428
467
|
if (gen === this.connectionGen) {
|
|
429
468
|
this.config = null
|
|
430
469
|
this.setState("error", {code: "transport_failed", reason})
|
|
431
470
|
this.sink?.onError("transport_failed")
|
|
471
|
+
if (reason === "port_missing" && this.documentReload.request()) {
|
|
472
|
+
previewTraceWarn("port_reload", {docGen: this.docGen})
|
|
473
|
+
return false
|
|
474
|
+
}
|
|
475
|
+
// Retried like a transport lost later, within the same budget and cooldown. After this
|
|
476
|
+
// handshake has settled: a sync from inside it would only rejoin it.
|
|
477
|
+
this.after(0, () => {
|
|
478
|
+
if (gen === this.connectionGen && this.state === "error" && this.isVisible() && this.wantsFrames()) {
|
|
479
|
+
this.scheduleReconnect(reason)
|
|
480
|
+
}
|
|
481
|
+
})
|
|
432
482
|
}
|
|
433
483
|
return false
|
|
434
484
|
}
|
|
@@ -438,6 +488,7 @@ export class PreviewConnection {
|
|
|
438
488
|
}
|
|
439
489
|
this.transport = transport
|
|
440
490
|
this.applied = NOTHING_APPLIED
|
|
491
|
+
this.documentReload.clear()
|
|
441
492
|
previewTrace("handshake_ok", {
|
|
442
493
|
docGen: this.docGen,
|
|
443
494
|
transport: config.transport,
|
|
@@ -58,10 +58,7 @@ export class WebSocketTransport implements PreviewTransport {
|
|
|
58
58
|
private opened = false
|
|
59
59
|
private closed = false
|
|
60
60
|
|
|
61
|
-
constructor(
|
|
62
|
-
private readonly url: string,
|
|
63
|
-
private readonly token: string,
|
|
64
|
-
) {}
|
|
61
|
+
constructor(private readonly url: string, private readonly token: string) {}
|
|
65
62
|
|
|
66
63
|
connect(): Promise<void> {
|
|
67
64
|
return new Promise((resolve, reject) => {
|
|
@@ -168,10 +165,7 @@ export class WebMessagePortTransport implements PreviewTransport {
|
|
|
168
165
|
if (typeof event.data !== "string") this.report("port_unexpected_payload")
|
|
169
166
|
}
|
|
170
167
|
|
|
171
|
-
constructor(
|
|
172
|
-
private readonly token: string,
|
|
173
|
-
private readonly portName: string = DEFAULT_PREVIEW_PORT_NAME,
|
|
174
|
-
) {}
|
|
168
|
+
constructor(private readonly token: string, private readonly portName: string = DEFAULT_PREVIEW_PORT_NAME) {}
|
|
175
169
|
|
|
176
170
|
connect(): Promise<void> {
|
|
177
171
|
const port =
|
|
@@ -184,6 +178,8 @@ export class WebMessagePortTransport implements PreviewTransport {
|
|
|
184
178
|
try {
|
|
185
179
|
port.postMessage(helloMessage(this.token))
|
|
186
180
|
} catch {
|
|
181
|
+
port.removeEventListener?.("message", this.listener)
|
|
182
|
+
this.port = null
|
|
187
183
|
return Promise.reject(new PreviewTransportError("hello_send_failed"))
|
|
188
184
|
}
|
|
189
185
|
previewTrace("transport_open", {transport: "webmessage"})
|