@mentra/miniapp 3.3.0-dev.944 → 3.4.0-beta.1237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/dist/background/index.d.ts +5 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/background/register.d.ts +3 -1
- package/dist/background/register.d.ts.map +1 -1
- package/dist/background/register.js +19 -2
- package/dist/background/register.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/blob.d.ts +2 -1
- package/dist/modules/blob.d.ts.map +1 -1
- package/dist/modules/blob.js +2 -1
- package/dist/modules/blob.js.map +1 -1
- package/dist/modules/events.d.ts +16 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/mic.d.ts +26 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +13 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +44 -57
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +21 -24
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +41 -10
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +17 -6
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/system.d.ts +8 -3
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +1 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/telemetry.d.ts +25 -0
- package/dist/modules/telemetry.d.ts.map +1 -0
- package/dist/modules/telemetry.js +61 -0
- package/dist/modules/telemetry.js.map +1 -0
- package/dist/modules/transcription.d.ts +70 -3
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +68 -2
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +29 -15
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +30 -15
- package/dist/protocol.js.map +1 -1
- package/dist/react/MentraProvider.d.ts +13 -1
- package/dist/react/MentraProvider.d.ts.map +1 -1
- package/dist/react/MentraProvider.js +31 -1
- package/dist/react/MentraProvider.js.map +1 -1
- package/dist/session.d.ts +59 -6
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +203 -43
- package/dist/session.js.map +1 -1
- package/dist/transport/auto.js +6 -0
- package/dist/transport/auto.js.map +1 -1
- package/dist/transport/dispatch.d.ts +2 -0
- package/dist/transport/dispatch.d.ts.map +1 -1
- package/dist/transport/dispatch.js +7 -2
- package/dist/transport/dispatch.js.map +1 -1
- package/dist/transport/local-socket.d.ts +6 -0
- package/dist/transport/local-socket.d.ts.map +1 -1
- package/dist/transport/local-socket.js +34 -2
- package/dist/transport/local-socket.js.map +1 -1
- package/dist/transport/types.d.ts +4 -1
- package/dist/transport/types.d.ts.map +1 -1
- package/dist/ui/index.d.ts +7 -4
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +6 -3
- package/src/background/register.ts +24 -2
- package/src/index.ts +11 -3
- package/src/modules/blob.ts +2 -1
- package/src/modules/events.ts +65 -51
- package/src/modules/mic.ts +30 -4
- package/src/modules/miniapps.ts +59 -71
- package/src/modules/speaker.ts +56 -12
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/system.ts +8 -3
- package/src/modules/telemetry.ts +62 -0
- package/src/modules/transcription.ts +106 -9
- package/src/protocol.ts +32 -16
- package/src/react/MentraProvider.tsx +23 -2
- package/src/session.ts +206 -52
- package/src/transport/auto.ts +5 -0
- package/src/transport/dispatch.ts +7 -2
- package/src/transport/local-socket.ts +30 -1
- package/src/transport/types.ts +4 -1
- package/src/ui/index.ts +7 -4
|
@@ -12,6 +12,11 @@ const DEFAULT_URL = "ws://127.0.0.1:8765";
|
|
|
12
12
|
export class LocalSocketTransport {
|
|
13
13
|
constructor(options = {}) {
|
|
14
14
|
this.ws = null;
|
|
15
|
+
/**
|
|
16
|
+
* The socket an open() in progress is connecting. A later open() or close()
|
|
17
|
+
* abandons it, so a socket that connects late cannot replace the active one.
|
|
18
|
+
*/
|
|
19
|
+
this.connecting = null;
|
|
15
20
|
this.messageHandler = null;
|
|
16
21
|
this.disconnectHandler = null;
|
|
17
22
|
this.url = options.url ?? DEFAULT_URL;
|
|
@@ -22,8 +27,10 @@ export class LocalSocketTransport {
|
|
|
22
27
|
if (typeof WebSocket === "undefined") {
|
|
23
28
|
throw new Error("LocalSocketTransport: browser WebSocket global not available");
|
|
24
29
|
}
|
|
30
|
+
this.abandonConnecting();
|
|
25
31
|
await new Promise((resolve, reject) => {
|
|
26
32
|
const ws = new WebSocket(this.url);
|
|
33
|
+
this.connecting = ws;
|
|
27
34
|
let settled = false;
|
|
28
35
|
const settle = (fn) => {
|
|
29
36
|
if (settled)
|
|
@@ -32,6 +39,11 @@ export class LocalSocketTransport {
|
|
|
32
39
|
fn();
|
|
33
40
|
};
|
|
34
41
|
ws.onopen = () => {
|
|
42
|
+
if (this.connecting !== ws) {
|
|
43
|
+
settle(() => reject(new Error("LocalSocketTransport: open was superseded")));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
this.connecting = null;
|
|
35
47
|
this.ws = ws;
|
|
36
48
|
settle(() => resolve());
|
|
37
49
|
};
|
|
@@ -39,13 +51,20 @@ export class LocalSocketTransport {
|
|
|
39
51
|
settle(() => reject(new Error(`LocalSocketTransport: failed to connect to ${this.url}: ${String(ev)}`)));
|
|
40
52
|
};
|
|
41
53
|
ws.onmessage = (ev) => {
|
|
54
|
+
if (this.ws !== ws)
|
|
55
|
+
return;
|
|
42
56
|
const data = ev.data;
|
|
43
57
|
if (typeof data === "string")
|
|
44
58
|
this.messageHandler?.(data);
|
|
45
59
|
};
|
|
46
60
|
ws.onclose = (ev) => {
|
|
47
|
-
if (this.
|
|
48
|
-
this.
|
|
61
|
+
if (this.connecting === ws)
|
|
62
|
+
this.connecting = null;
|
|
63
|
+
settle(() => reject(new Error(`LocalSocketTransport: closed before opening: ${ev.code} ${ev.reason}`)));
|
|
64
|
+
// Only the active socket's close is a disconnect; an abandoned one was never in use.
|
|
65
|
+
if (this.ws !== ws)
|
|
66
|
+
return;
|
|
67
|
+
this.ws = null;
|
|
49
68
|
this.disconnectHandler?.(`closed: ${ev.code} ${ev.reason}`);
|
|
50
69
|
};
|
|
51
70
|
});
|
|
@@ -63,6 +82,7 @@ export class LocalSocketTransport {
|
|
|
63
82
|
this.disconnectHandler = handler;
|
|
64
83
|
}
|
|
65
84
|
close() {
|
|
85
|
+
this.abandonConnecting();
|
|
66
86
|
if (!this.ws)
|
|
67
87
|
return;
|
|
68
88
|
try {
|
|
@@ -76,5 +96,17 @@ export class LocalSocketTransport {
|
|
|
76
96
|
isOpen() {
|
|
77
97
|
return !!this.ws && this.ws.readyState === WebSocket.OPEN;
|
|
78
98
|
}
|
|
99
|
+
abandonConnecting() {
|
|
100
|
+
const ws = this.connecting;
|
|
101
|
+
this.connecting = null;
|
|
102
|
+
if (!ws)
|
|
103
|
+
return;
|
|
104
|
+
try {
|
|
105
|
+
ws.close();
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
// ignore
|
|
109
|
+
}
|
|
110
|
+
}
|
|
79
111
|
}
|
|
80
112
|
//# sourceMappingURL=local-socket.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-socket.js","sourceRoot":"","sources":["../../src/transport/local-socket.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,MAAM,WAAW,GAAG,qBAAqB,CAAA;AAEzC,MAAM,OAAO,oBAAoB;
|
|
1
|
+
{"version":3,"file":"local-socket.js","sourceRoot":"","sources":["../../src/transport/local-socket.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,MAAM,WAAW,GAAG,qBAAqB,CAAA;AAEzC,MAAM,OAAO,oBAAoB;IAW/B,YAAY,UAAuC,EAAE;QAT7C,OAAE,GAAqB,IAAI,CAAA;QACnC;;;WAGG;QACK,eAAU,GAAqB,IAAI,CAAA;QACnC,mBAAc,GAAmC,IAAI,CAAA;QACrD,sBAAiB,GAAsC,IAAI,CAAA;QAGjE,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,WAAW,CAAA;IACvC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;YAAE,OAAM;QAC5D,IAAI,OAAO,SAAS,KAAK,WAAW,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;QACjF,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAClC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAA;YACpB,IAAI,OAAO,GAAG,KAAK,CAAA;YACnB,MAAM,MAAM,GAAG,CAAC,EAAc,EAAE,EAAE;gBAChC,IAAI,OAAO;oBAAE,OAAM;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,EAAE,EAAE,CAAA;YACN,CAAC,CAAA;YAED,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE;gBACf,IAAI,IAAI,CAAC,UAAU,KAAK,EAAE,EAAE,CAAC;oBAC3B,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,CAAC,CAAA;oBAC5E,OAAM;gBACR,CAAC;gBACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;gBACtB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAA;gBACZ,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACzB,CAAC,CAAA;YACD,EAAE,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAClB,MAAM,CAAC,GAAG,EAAE,CACV,MAAM,CAAC,IAAI,KAAK,CAAC,8CAA8C,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAC3F,CAAA;YACH,CAAC,CAAA;YACD,EAAE,CAAC,SAAS,GAAG,CAAC,EAAgB,EAAE,EAAE;gBAClC,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,OAAM;gBAC1B,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAA;gBACpB,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,IAAI,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,CAAA;YAC3D,CAAC,CAAA;YACD,EAAE,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAClB,IAAI,IAAI,CAAC,UAAU,KAAK,EAAE;oBAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;gBAClD,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gDAAgD,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;gBACvG,qFAAqF;gBACrF,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,OAAM;gBAC1B,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;gBACd,IAAI,CAAC,iBAAiB,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7D,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,GAAW;QACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC;IAED,SAAS,CAAC,OAAgC;QACxC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAA;IAC/B,CAAC;IAED,YAAY,CAAC,OAAmC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAA;IAClC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,iBAAiB,EAAE,CAAA;QACxB,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAM;QACpB,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IAChB,CAAC;IAED,MAAM;QACJ,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,CAAA;IAC3D,CAAC;IAEO,iBAAiB;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAA;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QACtB,IAAI,CAAC,EAAE;YAAE,OAAM;QACf,IAAI,CAAC;YACH,EAAE,CAAC,KAAK,EAAE,CAAA;QACZ,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;CACF"}
|
|
@@ -10,7 +10,10 @@
|
|
|
10
10
|
export type TransportMessageHandler = (raw: string) => void;
|
|
11
11
|
export type TransportDisconnectHandler = (reason: string) => void;
|
|
12
12
|
export interface Transport {
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Establish whatever connection the transport needs. May be a no-op. Resolves
|
|
15
|
+
* at once when already open: a retried handshake calls it again.
|
|
16
|
+
*/
|
|
14
17
|
open(): Promise<void>;
|
|
15
18
|
/** Send a serialized envelope. */
|
|
16
19
|
send(raw: string): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transport/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,uBAAuB,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAC3D,MAAM,MAAM,0BAA0B,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAA;AAEjE,MAAM,WAAW,SAAS;IACxB
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transport/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,uBAAuB,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAC3D,MAAM,MAAM,0BAA0B,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAA;AAEjE,MAAM,WAAW,SAAS;IACxB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACrB,kCAAkC;IAClC,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,6DAA6D;IAC7D,SAAS,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI,CAAA;IACjD,oEAAoE;IACpE,YAAY,CAAC,OAAO,EAAE,0BAA0B,GAAG,IAAI,CAAA;IACvD,+CAA+C;IAC/C,KAAK,IAAI,IAAI,CAAA;IACb,MAAM,IAAI,OAAO,CAAA;CAClB"}
|
package/dist/ui/index.d.ts
CHANGED
|
@@ -54,10 +54,13 @@ export interface MentraUiGlobal<TChannels extends object = Record<string, unknow
|
|
|
54
54
|
*/
|
|
55
55
|
onClose(cb: () => void): () => void;
|
|
56
56
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
57
|
+
* Tells the host the WebView is mounted and ready to receive messages
|
|
58
|
+
* from background. `<MentraProvider>` calls it once the app has mounted;
|
|
59
|
+
* call it yourself only without the provider (or with `autoReady={false}`).
|
|
60
|
+
* The host's splash stays up until it lands, and a page that loads
|
|
61
|
+
* without calling it is reported as failed. Idempotent — subsequent
|
|
62
|
+
* calls no-op. Background's `session.ui.onOpen` handlers fire after this
|
|
63
|
+
* lands.
|
|
61
64
|
*/
|
|
62
65
|
ready(): void;
|
|
63
66
|
}
|
package/dist/ui/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ui/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAC,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,eAAe,CAAA;AACnG,OAAO,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAA;AAEnE,MAAM,WAAW,cAAc,CAAC,SAAS,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAChF;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACrC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,CAAC,EACrD,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,GACpB,IAAI,CAAA;IAEP;;;OAGG;IACH,EAAE,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACnC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,CAAC,EACrD,EAAE,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,GAClC,MAAM,IAAI,CAAA;IAEb;;;;;;;;OAQG;IACH,OAAO,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACxC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,KAAK,EACrD,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC7B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEhC;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAA;IAElC;;;;OAIG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAA;IAEnC
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ui/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAC,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,eAAe,CAAA;AACnG,OAAO,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAA;AAEnE,MAAM,WAAW,cAAc,CAAC,SAAS,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAChF;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACrC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,CAAC,EACrD,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,GACpB,IAAI,CAAA;IAEP;;;OAGG;IACH,EAAE,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACnC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,CAAC,EACrD,EAAE,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,GAClC,MAAM,IAAI,CAAA;IAEb;;;;;;;;OAQG;IACH,OAAO,CAAC,CAAC,SAAS,MAAM,SAAS,GAAG,MAAM,EACxC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,KAAK,EACrD,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC7B,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEhC;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAA;IAElC;;;;OAIG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAA;IAEnC;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI,CAAA;CACd;AAKD,OAAO,EACL,cAAc,EACd,KAAK,mBAAmB,GACzB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAA;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,0BAA0B,CAAA;AACxD,OAAO,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAC,WAAW,EAAC,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAC,qBAAqB,EAAC,MAAM,gCAAgC,CAAA;AACpE,OAAO,EAAC,MAAM,EAAE,KAAK,WAAW,EAAC,MAAM,iBAAiB,CAAA;AACxD,OAAO,EAAC,eAAe,EAAE,KAAK,eAAe,EAAC,MAAM,0BAA0B,CAAA;AAC9E,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAA;AACpD,YAAY,EAAC,kBAAkB,EAAE,mBAAmB,EAAE,UAAU,EAAE,sBAAsB,EAAC,MAAM,wBAAwB,CAAA;AAEvH;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,MAAM,WAAW,CAAC,SAAS,SAAS,MAAM,IAAI,cAAc,CAAC,SAAS,CAAC,CAAA"}
|
package/dist/ui/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ui/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ui/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAC,cAAc,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAA;AA8DnE,uEAAuE;AACvE,sEAAsE;AACtE,+DAA+D;AAC/D,OAAO,EACL,cAAc,GAEf,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAA;AACtD,OAAO,EAAC,eAAe,EAAC,MAAM,0BAA0B,CAAA;AACxD,OAAO,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAC,WAAW,EAAC,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAC,qBAAqB,EAAC,MAAM,gCAAgC,CAAA;AACpE,OAAO,EAAC,MAAM,EAAmB,MAAM,iBAAiB,CAAA;AACxD,OAAO,EAAC,eAAe,EAAuB,MAAM,0BAA0B,CAAA;AAC9E,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mentra/miniapp",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.4.0-beta.1237",
|
|
4
4
|
"description": "SDK for building MentraOS local miniapps (browser/WebView)",
|
|
5
5
|
"author": "Mentra Labs",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
}
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
|
-
"@mentra/cloud-protocol": "3.
|
|
60
|
+
"@mentra/cloud-protocol": "3.4.0-beta.1237",
|
|
61
61
|
"eventemitter3": "^5.0.1"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
package/src/background/index.ts
CHANGED
|
@@ -55,13 +55,13 @@ export type {LanguageHint, TranscriptionLanguage} from "../modules/languages"
|
|
|
55
55
|
// (the registering side lives in the background JSContext). `InvokeOptions` is
|
|
56
56
|
// for system miniapps that call other apps' actions via `session.actions.invoke`.
|
|
57
57
|
export type {ActionContext, ActionHandler, InvokeOptions} from "../modules/actions"
|
|
58
|
+
export type {TelemetryModule, TelemetryProperties, TelemetryValue} from "../modules/telemetry"
|
|
58
59
|
export type {
|
|
59
60
|
InstallMiniappCompatibility,
|
|
60
61
|
InstallMiniappCompatibilityRequest,
|
|
61
62
|
InstallMiniappRequest,
|
|
62
63
|
InstallMiniappResult,
|
|
63
|
-
|
|
64
|
-
InstallMiniappProgress,
|
|
64
|
+
InstallMiniappFailureReason,
|
|
65
65
|
ListMiniappsOptions,
|
|
66
66
|
MiniappActionInfo,
|
|
67
67
|
MiniappCompatibility,
|
|
@@ -173,7 +173,8 @@ export type {
|
|
|
173
173
|
ScanQrOptions,
|
|
174
174
|
ScanQrResult,
|
|
175
175
|
} from "../modules/system"
|
|
176
|
-
export type {TranscriptionConfig, TranscriptionOptions} from "../modules/transcription"
|
|
176
|
+
export type {TranscriptionConfig, TranscriptionModelStatus, TranscriptionOptions} from "../modules/transcription"
|
|
177
|
+
export type {SpeechProcessing} from "../modules/speechProcessing"
|
|
177
178
|
export type {PermissionErrorEvent} from "../modules/permissions"
|
|
178
179
|
export type {
|
|
179
180
|
AuthUpdatePayload,
|
|
@@ -216,3 +217,5 @@ export type {
|
|
|
216
217
|
ComputedRoute,
|
|
217
218
|
ComputeRouteResult,
|
|
218
219
|
} from "../modules/navigation"
|
|
220
|
+
|
|
221
|
+
export type {WakeWordEvent} from "../modules/mic"
|
|
@@ -13,7 +13,9 @@
|
|
|
13
13
|
* 2. Call the user-supplied handler synchronously so subscriptions
|
|
14
14
|
* are registered before any host events fan out.
|
|
15
15
|
* 3. Call `session.connect()` so the host receives CONNECT and the
|
|
16
|
-
* first CONNECT_ACK populates `userId`, `capabilities`, etc.
|
|
16
|
+
* first CONNECT_ACK populates `userId`, `capabilities`, etc. A host that
|
|
17
|
+
* has not answered within `connectTimeoutMs` gets CONNECT again; that is
|
|
18
|
+
* a stalled host, not a miniapp crash.
|
|
17
19
|
* 4. Send READY once the handler's returned promise settles. Hosts that
|
|
18
20
|
* advertise `hostFeatures.initReady` keep the UI closed until then.
|
|
19
21
|
*
|
|
@@ -22,7 +24,7 @@
|
|
|
22
24
|
* re-evaluated and the handler fires again with a fresh session.
|
|
23
25
|
*/
|
|
24
26
|
|
|
25
|
-
import {MiniappSession, type MiniappSessionOptions} from "../session"
|
|
27
|
+
import {MiniappSession, NotConnectedError, type MiniappSessionOptions} from "../session"
|
|
26
28
|
|
|
27
29
|
export type TypedMiniappSession<TChannels extends object> = MiniappSession<TChannels>
|
|
28
30
|
|
|
@@ -38,6 +40,18 @@ interface InitGlobals {
|
|
|
38
40
|
* Optional because tests / pure-Node environments don't have it.
|
|
39
41
|
*/
|
|
40
42
|
__hostError?: (payloadJson: string) => void
|
|
43
|
+
/**
|
|
44
|
+
* The session from the latest init. Global rather than module state so a
|
|
45
|
+
* bundle evaluated again in the same context also finds it.
|
|
46
|
+
*/
|
|
47
|
+
__mentraBackgroundSession?: MiniappSession
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A later init replaces the session: disconnecting it ends its handshake, so it neither retries nor reports. */
|
|
51
|
+
function releaseBackgroundSession(g: InitGlobals): void {
|
|
52
|
+
const previous = g.__mentraBackgroundSession
|
|
53
|
+
g.__mentraBackgroundSession = undefined
|
|
54
|
+
previous?.disconnect()
|
|
41
55
|
}
|
|
42
56
|
|
|
43
57
|
/**
|
|
@@ -62,8 +76,13 @@ export function registerMiniapp<TChannels extends object = Record<string, unknow
|
|
|
62
76
|
options: MiniappSessionOptions = {},
|
|
63
77
|
): void {
|
|
64
78
|
const g = globalThis as unknown as InitGlobals
|
|
79
|
+
releaseBackgroundSession(g)
|
|
65
80
|
g.__mentraInitCallback = (sessionId: string) => {
|
|
81
|
+
releaseBackgroundSession(g)
|
|
66
82
|
const session = new MiniappSession<TChannels>(options)
|
|
83
|
+
g.__mentraBackgroundSession = session
|
|
84
|
+
// Before the handler runs, so its own connect() or waitForReady() shares the retried handshake.
|
|
85
|
+
session.retryHandshakeOnTimeout()
|
|
67
86
|
// The handler's returned promise defines readiness: the host keeps the UI
|
|
68
87
|
// closed until it settles, so handlers registered after an await still
|
|
69
88
|
// exist before the first UI request arrives.
|
|
@@ -95,6 +114,9 @@ export function registerMiniapp<TChannels extends object = Record<string, unknow
|
|
|
95
114
|
},
|
|
96
115
|
)
|
|
97
116
|
connected.catch((err: unknown) => {
|
|
117
|
+
// Replaced by a later init, or disconnected during the handshake: a
|
|
118
|
+
// deliberate teardown, not a crash.
|
|
119
|
+
if (g.__mentraBackgroundSession !== session || err instanceof NotConnectedError) return
|
|
98
120
|
// eslint-disable-next-line no-console
|
|
99
121
|
console.error("[mentra-miniapp] session.connect() rejected:", err)
|
|
100
122
|
// Surface to the host's crash controller as a structured uncaught
|
package/src/index.ts
CHANGED
|
@@ -108,6 +108,7 @@ export type {
|
|
|
108
108
|
SpeakerStreamWriteResult,
|
|
109
109
|
} from "./modules/speaker"
|
|
110
110
|
export {SpeakerStreamWriter, SPEAKER_STREAM_MAX_BUFFERED_MS, SPEAKER_WRITE_CHUNK_BYTES} from "./modules/speaker"
|
|
111
|
+
export type {SpeechProcessing} from "./modules/speechProcessing"
|
|
111
112
|
export type {
|
|
112
113
|
CameraFovPreset,
|
|
113
114
|
CameraFovRequest,
|
|
@@ -156,8 +157,7 @@ export type {
|
|
|
156
157
|
InstallMiniappCompatibilityRequest,
|
|
157
158
|
InstallMiniappRequest,
|
|
158
159
|
InstallMiniappResult,
|
|
159
|
-
|
|
160
|
-
InstallMiniappProgress,
|
|
160
|
+
InstallMiniappFailureReason,
|
|
161
161
|
MiniappHardwareRequirement,
|
|
162
162
|
} from "./modules/miniapps"
|
|
163
163
|
export type {ActionContext, ActionHandler, InvokeOptions} from "./modules/actions"
|
|
@@ -167,6 +167,7 @@ export type {ActionContext, ActionHandler, InvokeOptions} from "./modules/action
|
|
|
167
167
|
export type {DisplayManager} from "./modules/display"
|
|
168
168
|
export type {MiniappsModule} from "./modules/miniapps"
|
|
169
169
|
export type {ActionsModule} from "./modules/actions"
|
|
170
|
+
export type {TelemetryModule, TelemetryProperties, TelemetryValue} from "./modules/telemetry"
|
|
170
171
|
export type {GlassesModule} from "./modules/glasses"
|
|
171
172
|
export type {HeadingModule} from "./modules/heading"
|
|
172
173
|
export type {ImuModule} from "./modules/imu"
|
|
@@ -201,7 +202,12 @@ export type {
|
|
|
201
202
|
} from "./modules/navigation"
|
|
202
203
|
export type {PermissionsModule, PermissionErrorEvent} from "./modules/permissions"
|
|
203
204
|
export type {PhoneModule, PhoneNotificationsModule, PhoneCalendarModule} from "./modules/phone"
|
|
204
|
-
export type {
|
|
205
|
+
export type {
|
|
206
|
+
TranscriptionModule,
|
|
207
|
+
TranscriptionConfig,
|
|
208
|
+
TranscriptionModelStatus,
|
|
209
|
+
TranscriptionOptions,
|
|
210
|
+
} from "./modules/transcription"
|
|
205
211
|
export type {TranslationModule} from "./modules/translation"
|
|
206
212
|
export type {SpeakerModule} from "./modules/speaker"
|
|
207
213
|
export type {
|
|
@@ -248,3 +254,5 @@ export {
|
|
|
248
254
|
|
|
249
255
|
// Permission types
|
|
250
256
|
export type {PermissionType, PermissionRecord} from "./session"
|
|
257
|
+
|
|
258
|
+
export type {WakeWordEvent} from "./modules/mic"
|
package/src/modules/blob.ts
CHANGED
|
@@ -201,7 +201,8 @@ export class BlobModule {
|
|
|
201
201
|
/**
|
|
202
202
|
* Download `url` straight into a blob under `key`, host-side. The bytes never
|
|
203
203
|
* cross the bridge (like the web Cache API's `cache.add`). `opts.headers` lets
|
|
204
|
-
* you pass auth. Resolves to the stored blob's metadata.
|
|
204
|
+
* you pass auth. Resolves to the stored blob's metadata. Only http: and https:
|
|
205
|
+
* URLs are downloaded; any other scheme is refused.
|
|
205
206
|
*/
|
|
206
207
|
setFromUrl(key: string, url: string, opts: BlobSetFromUrlOptions = {}): Promise<BlobMeta> {
|
|
207
208
|
return this.session.sendRequest<BlobMeta>({
|
package/src/modules/events.ts
CHANGED
|
@@ -33,10 +33,16 @@ import {EventEmitter} from "eventemitter3"
|
|
|
33
33
|
import {makeRequestId} from "../envelope"
|
|
34
34
|
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
35
35
|
import {MiniappSession} from "../session"
|
|
36
|
+
import {resolveSpeechProcessing, type SpeechProcessing, type SpeechProcessingOptions} from "./speechProcessing"
|
|
36
37
|
|
|
37
38
|
export type UnsubscribeFn = () => void
|
|
38
39
|
|
|
39
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* @internal Delivery class the host tags a transcription event with. Hosts
|
|
42
|
+
* address each event to the listeners that should get it; SDKs from before
|
|
43
|
+
* `processing` filter on this tag instead: `"forceLocal"` reaches `localOnly`
|
|
44
|
+
* listeners, `"default"` the others, `"all"` everyone.
|
|
45
|
+
*/
|
|
40
46
|
export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
|
|
41
47
|
|
|
42
48
|
// ---------------------------------------------------------------------------
|
|
@@ -48,6 +54,8 @@ export interface TranscriptionData {
|
|
|
48
54
|
text: string
|
|
49
55
|
isFinal: boolean
|
|
50
56
|
language?: string
|
|
57
|
+
/** Where the text was transcribed: `"cloud"`, or `"local"` for the phone's on-device model. */
|
|
58
|
+
source: "cloud" | "local"
|
|
51
59
|
}
|
|
52
60
|
|
|
53
61
|
export interface TranslationData {
|
|
@@ -150,6 +158,10 @@ export interface NotificationDismissedData {
|
|
|
150
158
|
export interface VadData {
|
|
151
159
|
/** True while the user is speaking (voice detected), false when silent. */
|
|
152
160
|
status: boolean
|
|
161
|
+
/** 0 while speaking; on a silence event, how long the audio has already been silent. */
|
|
162
|
+
silentForMs?: number
|
|
163
|
+
/** Unix epoch milliseconds of the transition, when the host reports it. */
|
|
164
|
+
timestamp?: number
|
|
153
165
|
}
|
|
154
166
|
|
|
155
167
|
export interface TouchData {
|
|
@@ -173,14 +185,12 @@ export class EventManager {
|
|
|
173
185
|
private listenerSeq = 0
|
|
174
186
|
private transcriptionListeners = new Map<
|
|
175
187
|
string,
|
|
176
|
-
{stream: string;
|
|
188
|
+
{stream: string; processing: SpeechProcessing; invoke: (data: unknown, route?: TranscriptionEventRoute) => void}
|
|
177
189
|
>()
|
|
178
190
|
private retireListeners = new Set<() => void>()
|
|
179
191
|
private readonly emitter = new EventEmitter()
|
|
180
192
|
/** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
|
|
181
193
|
private readonly refCounts = new Map<string, number>()
|
|
182
|
-
/** Transcription stream -> refs that require on-device-only processing. */
|
|
183
|
-
private readonly forceLocalRefCounts = new Map<string, number>()
|
|
184
194
|
|
|
185
195
|
constructor(private readonly session: MiniappSession) {}
|
|
186
196
|
|
|
@@ -195,20 +205,24 @@ export class EventManager {
|
|
|
195
205
|
* the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
|
|
196
206
|
* via a separate envelope `type`; the EventManager only routes them
|
|
197
207
|
* locally.
|
|
208
|
+
*
|
|
209
|
+
* `processing` applies to transcription streams only.
|
|
198
210
|
*/
|
|
199
|
-
subscribe(stream: string, handler: (data: unknown) => void, options:
|
|
211
|
+
subscribe(stream: string, handler: (data: unknown) => void, options: SpeechProcessingOptions = {}): UnsubscribeFn {
|
|
200
212
|
const isInternal = stream.startsWith("_")
|
|
201
|
-
const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
202
213
|
const isTranscription = stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
214
|
+
const processing = isTranscription ? resolveSpeechProcessing(options, "events.subscribe(options)") : "preferCloud"
|
|
215
|
+
const localOnly = processing === "localOnly"
|
|
203
216
|
const listenerId = isTranscription ? `${this.listenerScope}:${++this.listenerSeq}` : undefined
|
|
204
217
|
let active = true
|
|
205
218
|
const listener = (data: unknown, route?: TranscriptionEventRoute) => {
|
|
206
219
|
if (!active) return
|
|
207
|
-
if (
|
|
220
|
+
if (isTranscription) {
|
|
221
|
+
// The host tags every transcription event with the class it is for.
|
|
208
222
|
// Hosts that predate routed transcription events omit `route`. Treat
|
|
209
|
-
// those events as the
|
|
210
|
-
//
|
|
211
|
-
if (
|
|
223
|
+
// those events as the cloud path so a localOnly listener can never
|
|
224
|
+
// accidentally receive cloud text.
|
|
225
|
+
if (localOnly ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
|
|
212
226
|
}
|
|
213
227
|
handler(data)
|
|
214
228
|
}
|
|
@@ -217,21 +231,12 @@ export class EventManager {
|
|
|
217
231
|
active = false
|
|
218
232
|
}
|
|
219
233
|
this.retireListeners.add(retire)
|
|
220
|
-
if (listenerId) this.transcriptionListeners.set(listenerId, {stream,
|
|
234
|
+
if (listenerId) this.transcriptionListeners.set(listenerId, {stream, processing, invoke: listener})
|
|
221
235
|
if (!isInternal) {
|
|
222
236
|
const before = this.refCounts.get(stream) ?? 0
|
|
223
237
|
this.refCounts.set(stream, before + 1)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
|
|
227
|
-
if (
|
|
228
|
-
isTranscription ||
|
|
229
|
-
before === 0 ||
|
|
230
|
-
(forceLocal && forceLocalBefore === 0) ||
|
|
231
|
-
(!forceLocal && defaultBefore === 0)
|
|
232
|
-
) {
|
|
233
|
-
this.sendSubscriptionUpdate()
|
|
234
|
-
}
|
|
238
|
+
// Every transcription listener is registered with the host by id.
|
|
239
|
+
if (isTranscription || before === 0) this.sendSubscriptionUpdate()
|
|
235
240
|
}
|
|
236
241
|
return () => {
|
|
237
242
|
if (!active) return
|
|
@@ -241,24 +246,12 @@ export class EventManager {
|
|
|
241
246
|
this.emitter.off(stream, listener)
|
|
242
247
|
if (isInternal) return
|
|
243
248
|
const current = this.refCounts.get(stream) ?? 0
|
|
244
|
-
const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
|
|
245
|
-
let subscriptionChanged = current <= 1
|
|
246
249
|
if (current <= 1) {
|
|
247
250
|
this.refCounts.delete(stream)
|
|
248
251
|
} else {
|
|
249
252
|
this.refCounts.set(stream, current - 1)
|
|
250
253
|
}
|
|
251
|
-
if (
|
|
252
|
-
if (forceLocalCurrent <= 1) {
|
|
253
|
-
this.forceLocalRefCounts.delete(stream)
|
|
254
|
-
subscriptionChanged = true
|
|
255
|
-
} else {
|
|
256
|
-
this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
|
|
257
|
-
}
|
|
258
|
-
} else if (current - forceLocalCurrent <= 1) {
|
|
259
|
-
subscriptionChanged = true
|
|
260
|
-
}
|
|
261
|
-
if (isTranscription || subscriptionChanged) this.sendSubscriptionUpdate()
|
|
254
|
+
if (isTranscription || current <= 1) this.sendSubscriptionUpdate()
|
|
262
255
|
}
|
|
263
256
|
}
|
|
264
257
|
|
|
@@ -269,7 +262,6 @@ export class EventManager {
|
|
|
269
262
|
this.transcriptionListeners.clear()
|
|
270
263
|
this.emitter.removeAllListeners()
|
|
271
264
|
this.refCounts.clear()
|
|
272
|
-
this.forceLocalRefCounts.clear()
|
|
273
265
|
this.sendSubscriptionUpdate()
|
|
274
266
|
}
|
|
275
267
|
|
|
@@ -284,6 +276,7 @@ export class EventManager {
|
|
|
284
276
|
transcriptionRoute?: TranscriptionEventRoute,
|
|
285
277
|
listenerId?: string,
|
|
286
278
|
): void {
|
|
279
|
+
if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) data = withTranscriptionSource(data)
|
|
287
280
|
if (listenerId !== undefined) {
|
|
288
281
|
const listener = this.transcriptionListeners.get(listenerId)
|
|
289
282
|
if (listener && (listener.stream === stream || listener.stream === "transcription:auto"))
|
|
@@ -318,27 +311,48 @@ export class EventManager {
|
|
|
318
311
|
}
|
|
319
312
|
|
|
320
313
|
private sendSubscriptionUpdate(): void {
|
|
321
|
-
const subscriptions
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
314
|
+
const subscriptions: Array<string | Record<string, unknown>> = []
|
|
315
|
+
for (const stream of this.refCounts.keys()) {
|
|
316
|
+
if (stream === MiniappStreamType.LOCATION_UPDATE)
|
|
317
|
+
subscriptions.push({stream: "location_stream", rate: "realtime"})
|
|
318
|
+
else if (!stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) subscriptions.push(stream)
|
|
319
|
+
}
|
|
320
|
+
// One entry per transcription stream and processing its listeners ask for.
|
|
321
|
+
const transcription = new Set<string>()
|
|
322
|
+
for (const {stream, processing} of this.transcriptionListeners.values()) {
|
|
323
|
+
const key = `${processing} ${stream}`
|
|
324
|
+
if (transcription.has(key)) continue
|
|
325
|
+
transcription.add(key)
|
|
326
|
+
subscriptions.push({stream, ...processingOnWire(processing)})
|
|
327
|
+
}
|
|
334
328
|
this.session.sendOneShot({
|
|
335
329
|
type: MiniappRequestType.SUBSCRIBE,
|
|
336
330
|
subscriptions,
|
|
337
|
-
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream,
|
|
331
|
+
transcriptionListeners: [...this.transcriptionListeners].map(([id, {stream, processing}]) => ({
|
|
338
332
|
id,
|
|
339
333
|
stream,
|
|
340
|
-
|
|
334
|
+
...processingOnWire(processing),
|
|
341
335
|
})),
|
|
342
336
|
})
|
|
343
337
|
}
|
|
344
338
|
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* The wire form of a transcription listener's processing. Hosts before
|
|
342
|
+
* `processing` (Mentra App 3.4.0) read only `forceLocal`, so `localOnly` keeps
|
|
343
|
+
* sending it: there a localOnly listener still never gets cloud text.
|
|
344
|
+
*/
|
|
345
|
+
function processingOnWire(processing: SpeechProcessing): {processing: SpeechProcessing; forceLocal?: true} {
|
|
346
|
+
return processing === "localOnly" ? {processing, forceLocal: true} : {processing}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Hosts before `source` (Mentra App 3.4.0) leave it out. Their on-device
|
|
351
|
+
* transcripts carry `type: "local_transcription"`, their cloud ones do not.
|
|
352
|
+
*/
|
|
353
|
+
function withTranscriptionSource(data: unknown): unknown {
|
|
354
|
+
if (typeof data !== "object" || data === null) return data
|
|
355
|
+
const event = data as {source?: unknown; type?: unknown}
|
|
356
|
+
if (event.source === "cloud" || event.source === "local") return data
|
|
357
|
+
return {...event, source: event.type === "local_transcription" ? "local" : "cloud"}
|
|
358
|
+
}
|
package/src/modules/mic.ts
CHANGED
|
@@ -31,6 +31,22 @@ export interface MicLease {
|
|
|
31
31
|
release(): Promise<void>
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* On-device wake-word ("Hey Mentra") event. It describes only the wake-word
|
|
36
|
+
* model: `loading` while the phone loads it, `ready` once it is listening,
|
|
37
|
+
* `detected` for each detection, and `error` when it cannot run (`message`
|
|
38
|
+
* says why). The state of the on-device speech-to-text model is reported
|
|
39
|
+
* separately by `session.transcription.onModelStatus`.
|
|
40
|
+
*/
|
|
41
|
+
export interface WakeWordEvent {
|
|
42
|
+
kind: "loading" | "ready" | "detected" | "error"
|
|
43
|
+
/** The phrase this event is about. Currently always "Hey Mentra". */
|
|
44
|
+
phrase: string
|
|
45
|
+
/** Unix epoch milliseconds. */
|
|
46
|
+
timestamp: number
|
|
47
|
+
message?: string
|
|
48
|
+
}
|
|
49
|
+
|
|
34
50
|
export class MicModule {
|
|
35
51
|
/** All active unsubscribe functions for stop() to tear down at once. */
|
|
36
52
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
@@ -38,17 +54,27 @@ export class MicModule {
|
|
|
38
54
|
constructor(private readonly session: MiniappSession) {}
|
|
39
55
|
|
|
40
56
|
/**
|
|
41
|
-
* Subscribe to voice activity detection
|
|
42
|
-
* `true`
|
|
57
|
+
* Subscribe to on-device voice activity detection. One event when the user
|
|
58
|
+
* starts speaking (`status: true`) and one when they stop (`status: false`,
|
|
59
|
+
* with `silentForMs` saying how long the audio has already been quiet). A
|
|
60
|
+
* gap in microphone audio counts as silence. Requires MICROPHONE permission.
|
|
43
61
|
*/
|
|
44
62
|
onVoiceActivity(handler: (data: VadData) => void): UnsubscribeFn {
|
|
45
63
|
return this.track(this.session._subscribe(MiniappStreamType.VAD, handler as (data: unknown) => void))
|
|
46
64
|
}
|
|
47
65
|
|
|
48
66
|
/**
|
|
49
|
-
* Subscribe to
|
|
50
|
-
*
|
|
67
|
+
* Subscribe to the on-device "Hey Mentra" wake word. Every subscribed
|
|
68
|
+
* miniapp receives every detection. You get the detector's current state
|
|
69
|
+
* right away, then every change. The phone loads the wake-word model while
|
|
70
|
+
* any miniapp subscribes and downloads it on first use. Requires MICROPHONE
|
|
71
|
+
* permission.
|
|
51
72
|
*/
|
|
73
|
+
onWakeWord(handler: (event: WakeWordEvent) => void): UnsubscribeFn {
|
|
74
|
+
return this.track(this.session._subscribe(MiniappStreamType.WAKE_WORD, handler as (data: unknown) => void))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Subscribe to raw audio chunks in the phone's PCM/LC3 microphone format. */
|
|
52
78
|
onAudioChunk(handler: (data: AudioChunkData) => void): UnsubscribeFn {
|
|
53
79
|
return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
|
|
54
80
|
}
|