@mentra/miniapp 3.3.0 → 3.4.0-beta.1241
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/background/index.d.ts +5 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/blob.d.ts +2 -1
- package/dist/modules/blob.d.ts.map +1 -1
- package/dist/modules/blob.js +2 -1
- package/dist/modules/blob.js.map +1 -1
- package/dist/modules/events.d.ts +16 -6
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +53 -47
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/mic.d.ts +26 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +13 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +44 -57
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +21 -24
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +41 -10
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +17 -6
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/speechProcessing.d.ts +29 -0
- package/dist/modules/speechProcessing.d.ts.map +1 -0
- package/dist/modules/speechProcessing.js +22 -0
- package/dist/modules/speechProcessing.js.map +1 -0
- package/dist/modules/system.d.ts +8 -3
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +1 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/telemetry.d.ts +25 -0
- package/dist/modules/telemetry.d.ts.map +1 -0
- package/dist/modules/telemetry.js +61 -0
- package/dist/modules/telemetry.js.map +1 -0
- package/dist/modules/transcription.d.ts +70 -3
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +68 -2
- package/dist/modules/transcription.js.map +1 -1
- package/dist/protocol.d.ts +29 -15
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +30 -15
- package/dist/protocol.js.map +1 -1
- package/dist/react/MentraProvider.d.ts +13 -1
- package/dist/react/MentraProvider.d.ts.map +1 -1
- package/dist/react/MentraProvider.js +31 -1
- package/dist/react/MentraProvider.js.map +1 -1
- package/dist/session.d.ts +16 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +6 -13
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +7 -4
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js.map +1 -1
- package/package.json +2 -2
- package/src/background/index.ts +6 -3
- package/src/index.ts +11 -3
- package/src/modules/blob.ts +2 -1
- package/src/modules/events.ts +65 -51
- package/src/modules/mic.ts +30 -4
- package/src/modules/miniapps.ts +59 -71
- package/src/modules/speaker.ts +56 -12
- package/src/modules/speechProcessing.ts +42 -0
- package/src/modules/system.ts +8 -3
- package/src/modules/telemetry.ts +62 -0
- package/src/modules/transcription.ts +106 -9
- package/src/protocol.ts +32 -16
- package/src/react/MentraProvider.tsx +23 -2
- package/src/session.ts +20 -17
- package/src/ui/index.ts +7 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system.d.ts","sourceRoot":"","sources":["../../src/modules/system.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,
|
|
1
|
+
{"version":3,"file":"system.d.ts","sourceRoot":"","sources":["../../src/modules/system.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,MAAM,YAAY,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAA;CAAC,GAAG;IAAC,SAAS,EAAE,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,SAAS,CAAA;CAAC,CAAA;AAElG,qBAAa,YAAY;IACX,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,cAAc;IAEpD,sDAAsD;IAChD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAQxD,kHAAkH;IAClH,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAO1B,yCAAyC;IACnC,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlD,kFAAkF;IAC5E,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAQjE;;;;OAIG;IACG,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC;CAUjE"}
|
package/dist/modules/system.js
CHANGED
|
@@ -16,7 +16,7 @@ export class SystemModule {
|
|
|
16
16
|
});
|
|
17
17
|
return result ?? { success: false };
|
|
18
18
|
}
|
|
19
|
-
/** Open a URL in the system browser.
|
|
19
|
+
/** Open a URL in the system browser. Never opens local files or scripts (file:, content:, javascript:, data:). */
|
|
20
20
|
openUrl(url) {
|
|
21
21
|
this.session.sendOneShot({
|
|
22
22
|
type: MiniappRequestType.OPEN_URL,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system.js","sourceRoot":"","sources":["../../src/modules/system.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"system.js","sourceRoot":"","sources":["../../src/modules/system.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AA+ChD,MAAM,OAAO,YAAY;IACvB,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;IAAG,CAAC;IAExD,sDAAsD;IACtD,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAc;YACzD,IAAI,EAAE,kBAAkB,CAAC,KAAK;YAC9B,GAAG,OAAO;SACX,CAAC,CAAA;QACF,OAAO,MAAM,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,kHAAkH;IAClH,OAAO,CAAC,GAAW;QACjB,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACvB,IAAI,EAAE,kBAAkB,CAAC,QAAQ;YACjC,GAAG;SACJ,CAAC,CAAA;IACJ,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAO;YACnC,IAAI,EAAE,kBAAkB,CAAC,cAAc;YACvC,IAAI;SACL,CAAC,CAAA;IACJ,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAiB;YAC5D,IAAI,EAAE,kBAAkB,CAAC,QAAQ;YACjC,GAAG,OAAO;SACX,CAAC,CAAA;QACF,OAAO,MAAM,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACrC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,UAAyB,EAAE;QACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAC3C;YACE,IAAI,EAAE,kBAAkB,CAAC,OAAO;YAChC,GAAG,OAAO;SACX,EACD,EAAC,SAAS,EAAE,CAAC,EAAC,CACf,CAAA;QACD,OAAO,MAAM,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;IACtC,CAAC;CACF"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview TelemetryModule (`session.telemetry`): a miniapp's own analytics, shown in its organization's Fleet
|
|
3
|
+
* Management.
|
|
4
|
+
*
|
|
5
|
+
* `capture(event, properties?)` sends one fire-and-forget message to the Mentra App and returns nothing. It never throws
|
|
6
|
+
* and never waits. The SDK checks nothing: the Mentra App attributes the event to this miniapp and applies the limits
|
|
7
|
+
* (names, values, 25 properties, the rate), and on a Mentra App without the `telemetry` host feature it does nothing.
|
|
8
|
+
* Before the handshake says which, it holds the latest 100 captures and sends them once it knows.
|
|
9
|
+
*/
|
|
10
|
+
import type { MiniappSession } from "../session";
|
|
11
|
+
export type TelemetryValue = string | number | boolean;
|
|
12
|
+
export type TelemetryProperties = Record<string, TelemetryValue>;
|
|
13
|
+
export declare class TelemetryModule {
|
|
14
|
+
private readonly session;
|
|
15
|
+
private held;
|
|
16
|
+
private waiting;
|
|
17
|
+
constructor(session: MiniappSession);
|
|
18
|
+
/**
|
|
19
|
+
* Captures one event, in the style of PostHog: `session.telemetry.capture("call_ended", {platform: "teams",
|
|
20
|
+
* duration_s: 312})`. Names are lowercase letters, digits and underscores; values are flat. Never personal data: no
|
|
21
|
+
* names, emails, phone numbers, typed text or content.
|
|
22
|
+
*/
|
|
23
|
+
capture(event: string, properties?: TelemetryProperties): void;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/modules/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAA;AACtD,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;AAKhE,qBAAa,eAAe;IAId,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,OAAO,CAAQ;gBAEM,OAAO,EAAE,cAAc;IAEpD;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,mBAAmB,GAAG,IAAI;CA+B/D"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview TelemetryModule (`session.telemetry`): a miniapp's own analytics, shown in its organization's Fleet
|
|
3
|
+
* Management.
|
|
4
|
+
*
|
|
5
|
+
* `capture(event, properties?)` sends one fire-and-forget message to the Mentra App and returns nothing. It never throws
|
|
6
|
+
* and never waits. The SDK checks nothing: the Mentra App attributes the event to this miniapp and applies the limits
|
|
7
|
+
* (names, values, 25 properties, the rate), and on a Mentra App without the `telemetry` host feature it does nothing.
|
|
8
|
+
* Before the handshake says which, it holds the latest 100 captures and sends them once it knows.
|
|
9
|
+
*/
|
|
10
|
+
import { MiniappRequestType } from "../protocol";
|
|
11
|
+
/** Captures held until the handshake says whether the Mentra App takes them. */
|
|
12
|
+
const MAX_HELD = 100;
|
|
13
|
+
export class TelemetryModule {
|
|
14
|
+
constructor(session) {
|
|
15
|
+
this.session = session;
|
|
16
|
+
this.held = [];
|
|
17
|
+
this.waiting = false;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Captures one event, in the style of PostHog: `session.telemetry.capture("call_ended", {platform: "teams",
|
|
21
|
+
* duration_s: 312})`. Names are lowercase letters, digits and underscores; values are flat. Never personal data: no
|
|
22
|
+
* names, emails, phone numbers, typed text or content.
|
|
23
|
+
*/
|
|
24
|
+
capture(event, properties) {
|
|
25
|
+
try {
|
|
26
|
+
const message = { type: MiniappRequestType.TELEMETRY_CAPTURE, event, properties: properties ?? {} };
|
|
27
|
+
if (this.session.handshakeDone) {
|
|
28
|
+
if (this.session.hostFeatures?.telemetry === true)
|
|
29
|
+
this.session.sendOneShot(message);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (this.held.length >= MAX_HELD)
|
|
33
|
+
this.held.shift();
|
|
34
|
+
this.held.push(message);
|
|
35
|
+
if (!this.waiting) {
|
|
36
|
+
this.waiting = true;
|
|
37
|
+
const off = this.session.on("ready", () => {
|
|
38
|
+
off();
|
|
39
|
+
this.waiting = false;
|
|
40
|
+
const held = this.held.splice(0);
|
|
41
|
+
if (this.session.hostFeatures?.telemetry !== true)
|
|
42
|
+
return;
|
|
43
|
+
for (const one of held) {
|
|
44
|
+
// Each on its own: one capture that cannot be sent never takes the others, or the session's other
|
|
45
|
+
// listeners, down with it.
|
|
46
|
+
try {
|
|
47
|
+
this.session.sendOneShot(one);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Lost.
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
// Analytics never break the miniapp: a value that cannot be sent is lost.
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=telemetry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../../src/modules/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAC,kBAAkB,EAAC,MAAM,aAAa,CAAA;AAM9C,gFAAgF;AAChF,MAAM,QAAQ,GAAG,GAAG,CAAA;AAEpB,MAAM,OAAO,eAAe;IAI1B,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;QAH5C,SAAI,GAAa,EAAE,CAAA;QACnB,YAAO,GAAG,KAAK,CAAA;IAEgC,CAAC;IAExD;;;;OAIG;IACH,OAAO,CAAC,KAAa,EAAE,UAAgC;QACrD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAC,IAAI,EAAE,kBAAkB,CAAC,iBAAiB,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,IAAI,EAAE,EAAC,CAAA;YACjG,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,SAAS,KAAK,IAAI;oBAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;gBACpF,OAAM;YACR,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,QAAQ;gBAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAA;YACnD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;gBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;oBACxC,GAAG,EAAE,CAAA;oBACL,IAAI,CAAC,OAAO,GAAG,KAAK,CAAA;oBACpB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;oBAChC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,SAAS,KAAK,IAAI;wBAAE,OAAM;oBACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,kGAAkG;wBAClG,2BAA2B;wBAC3B,IAAI,CAAC;4BACH,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;wBAC/B,CAAC;wBAAC,MAAM,CAAC;4BACP,QAAQ;wBACV,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0EAA0E;QAC5E,CAAC;IACH,CAAC;CACF"}
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* session.transcription.on(handler) // auto-detect
|
|
7
7
|
* session.transcription.forLanguage("en-US", handler) // single language
|
|
8
8
|
* session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
|
|
9
|
+
* session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
|
|
9
10
|
* session.transcription.configure({languageHints, vocabulary, diarization})
|
|
11
|
+
* session.transcription.onModelStatus(handler) // on-device model state
|
|
12
|
+
* session.transcription.finishUtterance() // end the on-device line now
|
|
10
13
|
* session.transcription.stop() // tear down all
|
|
11
14
|
*
|
|
12
15
|
* Hoisted to top-level (vs. nested under `session.mic`) because transcription
|
|
@@ -20,6 +23,7 @@
|
|
|
20
23
|
import { MiniappSession } from "../session";
|
|
21
24
|
import type { TranscriptionData, UnsubscribeFn } from "./events";
|
|
22
25
|
import type { LanguageHint, TranscriptionLanguage } from "./languages";
|
|
26
|
+
import { type SpeechProcessing } from "./speechProcessing";
|
|
23
27
|
/** Configuration for cloud-side transcription behavior. */
|
|
24
28
|
export interface TranscriptionConfig {
|
|
25
29
|
/**
|
|
@@ -36,16 +40,55 @@ export interface TranscriptionConfig {
|
|
|
36
40
|
}
|
|
37
41
|
export interface TranscriptionOptions {
|
|
38
42
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
43
|
+
* Where this listener's speech is transcribed. Default `"preferCloud"`.
|
|
44
|
+
*
|
|
45
|
+
* - `"preferCloud"`: the cloud; the phone's on-device model only while the
|
|
46
|
+
* cloud is unreachable.
|
|
47
|
+
* - `"preferLocal"`: the on-device model while it runs in this listener's
|
|
48
|
+
* language, otherwise the cloud. `on()` follows the model's language,
|
|
49
|
+
* unless `configure({languageHints})` names one the model can't
|
|
50
|
+
* transcribe; then it stays on the cloud.
|
|
51
|
+
* - `"localOnly"`: on-device only; never the cloud.
|
|
52
|
+
*
|
|
53
|
+
* The phone runs one on-device model, the one picked on its Speech page, and
|
|
54
|
+
* downloads it when a `preferLocal` or `localOnly` listener needs it; follow
|
|
55
|
+
* it with `onModelStatus`. A listener moves between the cloud and on-device
|
|
56
|
+
* during a pause in speech, once both have finished with what was said
|
|
57
|
+
* before it, so it never misses or repeats speech; `TranscriptionData.source`
|
|
58
|
+
* says where each result came from. Each listener routes on its own: other
|
|
59
|
+
* listeners on the same stream keep their choice.
|
|
42
60
|
*/
|
|
61
|
+
processing?: SpeechProcessing;
|
|
62
|
+
/** @deprecated Use `processing: "localOnly"`. Removed in 4.0; ignored when `processing` is set. */
|
|
43
63
|
forceLocal?: boolean;
|
|
44
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* State of the on-device speech-to-text model behind this miniapp's on-device
|
|
67
|
+
* transcription. The phone runs the one model picked on its Speech page for
|
|
68
|
+
* every miniapp.
|
|
69
|
+
*/
|
|
70
|
+
export interface TranscriptionModelStatus {
|
|
71
|
+
/** BCP-47 language of the picked model, e.g. "en-US". */
|
|
72
|
+
language: string;
|
|
73
|
+
/**
|
|
74
|
+
* - `downloading`: the model is being downloaded (`progress` is 0–100).
|
|
75
|
+
* - `loading`: the model is loading into memory.
|
|
76
|
+
* - `ready`: on-device transcripts are flowing.
|
|
77
|
+
* - `unavailable`: no on-device transcription right now; `reason` says why
|
|
78
|
+
* (for example no `preferLocal` or `localOnly` listener, a failed
|
|
79
|
+
* download, or a listener in a language other than the picked model's).
|
|
80
|
+
*/
|
|
81
|
+
state: "downloading" | "loading" | "ready" | "unavailable";
|
|
82
|
+
progress?: number;
|
|
83
|
+
reason?: string;
|
|
84
|
+
}
|
|
45
85
|
export declare class TranscriptionModule {
|
|
46
86
|
private readonly session;
|
|
47
87
|
private readonly unsubs;
|
|
48
88
|
private currentConfig;
|
|
89
|
+
private readonly modelStatusHandlers;
|
|
90
|
+
private modelStatusUnsub;
|
|
91
|
+
private modelStatus;
|
|
49
92
|
constructor(session: MiniappSession);
|
|
50
93
|
/**
|
|
51
94
|
* Subscribe to all transcription events (auto-detect language). The
|
|
@@ -83,6 +126,30 @@ export declare class TranscriptionModule {
|
|
|
83
126
|
* so the runtime can re-apply on reconnect.
|
|
84
127
|
*/
|
|
85
128
|
configure(config: TranscriptionConfig): Promise<void>;
|
|
129
|
+
/**
|
|
130
|
+
* Follow the on-device speech-to-text model's state. You get the current
|
|
131
|
+
* state right away, then every change. Use it to show "downloading 40%" or
|
|
132
|
+
* to wait for `ready` before relying on on-device transcripts.
|
|
133
|
+
*/
|
|
134
|
+
onModelStatus(handler: (status: TranscriptionModelStatus) => void): UnsubscribeFn;
|
|
135
|
+
/**
|
|
136
|
+
* End the current on-device utterance now, instead of waiting for the
|
|
137
|
+
* recognizer's own end-of-speech pause. Call it when your own end-of-turn
|
|
138
|
+
* rule fires (for example after `mic.onVoiceActivity` reports enough
|
|
139
|
+
* silence).
|
|
140
|
+
*
|
|
141
|
+
* Resolves with the final text once that final transcript has been
|
|
142
|
+
* delivered to your on-device handlers, or `""` when nothing was pending.
|
|
143
|
+
* Resolves `null` when no on-device recognizer is running or the picked
|
|
144
|
+
* model can't finish early (models that transcribe in fixed chunks); their
|
|
145
|
+
* final text still arrives on its own.
|
|
146
|
+
*
|
|
147
|
+
* Applies only to on-device transcription and needs a `localOnly` listener
|
|
148
|
+
* or one that is getting on-device text right now. The phone runs one
|
|
149
|
+
* recognizer for every miniapp, so this ends the current line for every
|
|
150
|
+
* on-device subscriber, Captions included.
|
|
151
|
+
*/
|
|
152
|
+
finishUtterance(): Promise<string | null>;
|
|
86
153
|
/** Tear down every transcription subscription this module owns. */
|
|
87
154
|
stop(): void;
|
|
88
155
|
/** True iff `MICROPHONE` is declared in the miniapp's manifest. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transcription.d.ts","sourceRoot":"","sources":["../../src/modules/transcription.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"transcription.d.ts","sourceRoot":"","sources":["../../src/modules/transcription.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EAAC,cAAc,EAAC,MAAM,YAAY,CAAA;AACzC,OAAO,KAAK,EAAC,iBAAiB,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAE9D,OAAO,KAAK,EAAC,YAAY,EAAE,qBAAqB,EAAC,MAAM,aAAa,CAAA;AACpE,OAAO,EAA0B,KAAK,gBAAgB,EAAC,MAAM,oBAAoB,CAAA;AAEjF,2DAA2D;AAC3D,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,YAAY,EAAE,CAAA;IAC9B,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,6DAA6D;IAC7D,WAAW,CAAC,EAAE,OAAO,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;GAIG;AACH,MAAM,WAAW,wBAAwB;IACvC,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;;;OAOG;IACH,KAAK,EAAE,aAAa,GAAG,SAAS,GAAG,OAAO,GAAG,aAAa,CAAA;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,OAAO;IANpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,aAAa,CAAmC;IACxD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwD;IAC5F,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,WAAW,CAAwC;gBAE9B,OAAO,EAAE,cAAc;IAEpD;;;;;;;OAOG;IACH,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,aAAa;IASjG;;;;;;;;;;;;;OAaG;IACH,WAAW,CACT,QAAQ,EAAE,qBAAqB,GAAG,qBAAqB,EAAE,EACzD,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,EAC1C,OAAO,GAAE,oBAAyB,GACjC,aAAa;IA2BhB;;;;;;;;;;OAUG;IACH,SAAS,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBrD;;;;OAIG;IACH,aAAa,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,wBAAwB,KAAK,IAAI,GAAG,aAAa;IAyBjF;;;;;;;;;;;;;;;;OAgBG;IACH,eAAe,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAIzC,mEAAmE;IACnE,IAAI,IAAI,IAAI;IAWZ,mEAAmE;IACnE,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,0EAA0E;IAC1E,UAAU,IAAI,mBAAmB,GAAG,IAAI;IAMxC,OAAO,CAAC,KAAK;CAOd"}
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* session.transcription.on(handler) // auto-detect
|
|
7
7
|
* session.transcription.forLanguage("en-US", handler) // single language
|
|
8
8
|
* session.transcription.forLanguage(["en-US", "es-ES"], handler) // multi-language
|
|
9
|
+
* session.transcription.on(handler, {processing: "preferLocal"}) // on-device when it can
|
|
9
10
|
* session.transcription.configure({languageHints, vocabulary, diarization})
|
|
11
|
+
* session.transcription.onModelStatus(handler) // on-device model state
|
|
12
|
+
* session.transcription.finishUtterance() // end the on-device line now
|
|
10
13
|
* session.transcription.stop() // tear down all
|
|
11
14
|
*
|
|
12
15
|
* Hoisted to top-level (vs. nested under `session.mic`) because transcription
|
|
@@ -19,11 +22,15 @@
|
|
|
19
22
|
*/
|
|
20
23
|
import { MiniappRequestType, MiniappStreamType } from "../protocol";
|
|
21
24
|
import { requireLanguageHint, requireTranscriptionLanguage } from "./languages";
|
|
25
|
+
import { resolveSpeechProcessing } from "./speechProcessing";
|
|
22
26
|
export class TranscriptionModule {
|
|
23
27
|
constructor(session) {
|
|
24
28
|
this.session = session;
|
|
25
29
|
this.unsubs = new Set();
|
|
26
30
|
this.currentConfig = null;
|
|
31
|
+
this.modelStatusHandlers = new Set();
|
|
32
|
+
this.modelStatusUnsub = null;
|
|
33
|
+
this.modelStatus = null;
|
|
27
34
|
}
|
|
28
35
|
/**
|
|
29
36
|
* Subscribe to all transcription events (auto-detect language). The
|
|
@@ -34,7 +41,10 @@ export class TranscriptionModule {
|
|
|
34
41
|
* event) preserves existing semantics.
|
|
35
42
|
*/
|
|
36
43
|
on(handler, options = {}) {
|
|
37
|
-
|
|
44
|
+
const processing = resolveSpeechProcessing(options, "transcription.on(options)");
|
|
45
|
+
return this.track(this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler, {
|
|
46
|
+
processing,
|
|
47
|
+
}));
|
|
38
48
|
}
|
|
39
49
|
/**
|
|
40
50
|
* Subscribe to transcription for one or more specific languages. Each call
|
|
@@ -52,11 +62,14 @@ export class TranscriptionModule {
|
|
|
52
62
|
*/
|
|
53
63
|
forLanguage(language, handler, options = {}) {
|
|
54
64
|
const langs = (Array.isArray(language) ? language : [language]).map((lang) => requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"));
|
|
65
|
+
const processing = resolveSpeechProcessing(options, "transcription.forLanguage(options)");
|
|
55
66
|
if (langs.length === 0)
|
|
56
67
|
return () => { };
|
|
57
68
|
const unsubs = [];
|
|
58
69
|
for (const lang of langs) {
|
|
59
|
-
unsubs.push(this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler,
|
|
70
|
+
unsubs.push(this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler, {
|
|
71
|
+
processing,
|
|
72
|
+
}));
|
|
60
73
|
}
|
|
61
74
|
const combined = () => {
|
|
62
75
|
for (const u of unsubs) {
|
|
@@ -94,6 +107,59 @@ export class TranscriptionModule {
|
|
|
94
107
|
})
|
|
95
108
|
.then(() => undefined);
|
|
96
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Follow the on-device speech-to-text model's state. You get the current
|
|
112
|
+
* state right away, then every change. Use it to show "downloading 40%" or
|
|
113
|
+
* to wait for `ready` before relying on on-device transcripts.
|
|
114
|
+
*/
|
|
115
|
+
onModelStatus(handler) {
|
|
116
|
+
this.modelStatusHandlers.add(handler);
|
|
117
|
+
if (!this.modelStatusUnsub) {
|
|
118
|
+
// One stream subscription per module; the host sends its current state on subscribe.
|
|
119
|
+
this.modelStatusUnsub = this.session._subscribe(MiniappStreamType.TRANSCRIPTION_MODEL_STATUS, (data) => {
|
|
120
|
+
const status = data;
|
|
121
|
+
this.modelStatus = status;
|
|
122
|
+
for (const listener of [...this.modelStatusHandlers])
|
|
123
|
+
listener(status);
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
else if (this.modelStatus) {
|
|
127
|
+
// Later handlers get the last state locally, since the host only replays on subscribe.
|
|
128
|
+
const current = this.modelStatus;
|
|
129
|
+
queueMicrotask(() => {
|
|
130
|
+
if (this.modelStatusHandlers.has(handler) && this.modelStatus === current)
|
|
131
|
+
handler(current);
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return this.track(() => {
|
|
135
|
+
this.modelStatusHandlers.delete(handler);
|
|
136
|
+
if (this.modelStatusHandlers.size > 0)
|
|
137
|
+
return;
|
|
138
|
+
this.modelStatusUnsub?.();
|
|
139
|
+
this.modelStatusUnsub = null;
|
|
140
|
+
this.modelStatus = null;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* End the current on-device utterance now, instead of waiting for the
|
|
145
|
+
* recognizer's own end-of-speech pause. Call it when your own end-of-turn
|
|
146
|
+
* rule fires (for example after `mic.onVoiceActivity` reports enough
|
|
147
|
+
* silence).
|
|
148
|
+
*
|
|
149
|
+
* Resolves with the final text once that final transcript has been
|
|
150
|
+
* delivered to your on-device handlers, or `""` when nothing was pending.
|
|
151
|
+
* Resolves `null` when no on-device recognizer is running or the picked
|
|
152
|
+
* model can't finish early (models that transcribe in fixed chunks); their
|
|
153
|
+
* final text still arrives on its own.
|
|
154
|
+
*
|
|
155
|
+
* Applies only to on-device transcription and needs a `localOnly` listener
|
|
156
|
+
* or one that is getting on-device text right now. The phone runs one
|
|
157
|
+
* recognizer for every miniapp, so this ends the current line for every
|
|
158
|
+
* on-device subscriber, Captions included.
|
|
159
|
+
*/
|
|
160
|
+
finishUtterance() {
|
|
161
|
+
return this.session.sendRequest({ type: MiniappRequestType.TRANSCRIPTION_FINISH });
|
|
162
|
+
}
|
|
97
163
|
/** Tear down every transcription subscription this module owns. */
|
|
98
164
|
stop() {
|
|
99
165
|
for (const u of this.unsubs) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transcription.js","sourceRoot":"","sources":["../../src/modules/transcription.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"transcription.js","sourceRoot":"","sources":["../../src/modules/transcription.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAC,kBAAkB,EAAE,iBAAiB,EAAC,MAAM,aAAa,CAAA;AAGjE,OAAO,EAAC,mBAAmB,EAAE,4BAA4B,EAAC,MAAM,aAAa,CAAA;AAE7E,OAAO,EAAC,uBAAuB,EAAwB,MAAM,oBAAoB,CAAA;AA+DjF,MAAM,OAAO,mBAAmB;IAO9B,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;QANnC,WAAM,GAAG,IAAI,GAAG,EAAiB,CAAA;QAC1C,kBAAa,GAA+B,IAAI,CAAA;QACvC,wBAAmB,GAAG,IAAI,GAAG,EAA8C,CAAA;QACpF,qBAAgB,GAAyB,IAAI,CAAA;QAC7C,gBAAW,GAAoC,IAAI,CAAA;IAEJ,CAAC;IAExD;;;;;;;OAOG;IACH,EAAE,CAAC,OAA0C,EAAE,UAAgC,EAAE;QAC/E,MAAM,UAAU,GAAG,uBAAuB,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAA;QAChF,OAAO,IAAI,CAAC,KAAK,CACf,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,iBAAiB,CAAC,aAAa,OAAO,EAAE,OAAkC,EAAE;YACrG,UAAU;SACX,CAAC,CACH,CAAA;IACH,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,WAAW,CACT,QAAyD,EACzD,OAA0C,EAC1C,UAAgC,EAAE;QAElC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC3E,4BAA4B,CAAC,IAAI,EAAE,qCAAqC,CAAC,CAC1E,CAAA;QACD,MAAM,UAAU,GAAG,uBAAuB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAA;QACzF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;QAEvC,MAAM,MAAM,GAAoB,EAAE,CAAA;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,iBAAiB,CAAC,aAAa,IAAI,IAAI,EAAE,EAAE,OAAkC,EAAE;gBACxG,UAAU;aACX,CAAC,CACH,CAAA;QACH,CAAC;QACD,MAAM,QAAQ,GAAkB,GAAG,EAAE;YACnC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,CAAC,EAAE,CAAA;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;YACH,CAAC;QACH,CAAC,CAAA;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC7B,CAAC;IAED;;;;;;;;;;OAUG;IACH,SAAS,CAAC,MAA2B;QACnC,MAAM,UAAU,GAAwB;YACtC,GAAG,MAAM;YACT,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChD,mBAAmB,CAAC,IAAI,EAAE,wCAAwC,CAAC,CACpE;SACF,CAAA;QACD,IAAI,CAAC,aAAa,GAAG,EAAC,GAAG,UAAU,EAAC,CAAA;QACpC,OAAO,IAAI,CAAC,OAAO;aAChB,WAAW,CAAO;YACjB,IAAI,EAAE,kBAAkB,CAAC,oBAAoB;YAC7C,MAAM,EAAE,EAAC,GAAG,UAAU,EAAC;SACxB,CAAC;aACD,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;IAC1B,CAAC;IAED;;;;OAIG;IACH,aAAa,CAAC,OAAmD;QAC/D,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,qFAAqF;YACrF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,0BAA0B,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrG,MAAM,MAAM,GAAG,IAAgC,CAAA;gBAC/C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAA;gBACzB,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC;oBAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;YACxE,CAAC,CAAC,CAAA;QACJ,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,uFAAuF;YACvF,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAA;YAChC,cAAc,CAAC,GAAG,EAAE;gBAClB,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO;oBAAE,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7F,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE;YACrB,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxC,IAAI,IAAI,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC;gBAAE,OAAM;YAC7C,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAA;YACzB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAA;YAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACzB,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAgB,EAAC,IAAI,EAAE,kBAAkB,CAAC,oBAAoB,EAAC,CAAC,CAAA;IACjG,CAAC;IAED,mEAAmE;IACnE,IAAI;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,CAAC,EAAE,CAAA;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;IAED,mEAAmE;IACnE,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAA;IAC1D,CAAC;IAED,0EAA0E;IAC1E,UAAU;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAC,GAAG,IAAI,CAAC,aAAa,EAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC5D,CAAC;IAED,2EAA2E;IAEnE,KAAK,CAAC,KAAoB;QAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtB,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzB,KAAK,EAAE,CAAA;QACT,CAAC,CAAA;IACH,CAAC;CACF"}
|
package/dist/protocol.d.ts
CHANGED
|
@@ -18,7 +18,16 @@ export declare enum MiniappRequestType {
|
|
|
18
18
|
READY = "miniapp_ready",
|
|
19
19
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
20
20
|
AUTH_REFRESH = "miniapp_auth_refresh",
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Update the set of streams the miniapp is subscribed to. `subscriptions`
|
|
23
|
+
* lists stream names; a transcription stream is `{stream, processing}`, one
|
|
24
|
+
* entry per processing its listeners ask for, and `transcriptionListeners`
|
|
25
|
+
* lists every transcription listener as `{id, stream, processing}`. Both
|
|
26
|
+
* also carry `forceLocal: true` for `localOnly`, the form hosts before
|
|
27
|
+
* `processing` read. Hosts accept that older form,
|
|
28
|
+
* `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
|
|
29
|
+
* `processing`.
|
|
30
|
+
*/
|
|
22
31
|
SUBSCRIBE = "miniapp_subscribe",
|
|
23
32
|
/**
|
|
24
33
|
* Push a legacy display layout to the glasses. The SDK no longer emits this
|
|
@@ -32,7 +41,7 @@ export declare enum MiniappRequestType {
|
|
|
32
41
|
PLAY_AUDIO = "miniapp_play_audio",
|
|
33
42
|
/** Stop any audio playback this miniapp initiated. */
|
|
34
43
|
STOP_AUDIO = "miniapp_stop_audio",
|
|
35
|
-
/** Speak text via cloud TTS
|
|
44
|
+
/** Speak text via cloud or on-device TTS, as its `processing` says. */
|
|
36
45
|
SPEAK = "miniapp_speak",
|
|
37
46
|
/**
|
|
38
47
|
* Live PCM output stream to the phone's audio playback service
|
|
@@ -161,6 +170,14 @@ export declare enum MiniappRequestType {
|
|
|
161
170
|
* so future reconnect logic can re-send.
|
|
162
171
|
*/
|
|
163
172
|
TRANSCRIPTION_CONFIG = "miniapp_transcription_config",
|
|
173
|
+
/**
|
|
174
|
+
* End the current on-device utterance now (transcription.finishUtterance).
|
|
175
|
+
* Result: the final text, "" when nothing was pending, or null when the
|
|
176
|
+
* model can't finish early.
|
|
177
|
+
*/
|
|
178
|
+
TRANSCRIPTION_FINISH = "miniapp_transcription_finish",
|
|
179
|
+
/** A miniapp's own analytics event (`session.telemetry.capture`). One-shot; hosts with `hostFeatures.telemetry`. */
|
|
180
|
+
TELEMETRY_CAPTURE = "miniapp_telemetry_capture",
|
|
164
181
|
/** Dashboard content update. Noop in v1. */
|
|
165
182
|
DASHBOARD_CONTENT_UPDATE = "miniapp_dashboard_content_update",
|
|
166
183
|
PHOTO = "miniapp_photo",
|
|
@@ -189,11 +206,11 @@ export declare enum MiniappRequestType {
|
|
|
189
206
|
MINIAPPS_START = "miniapp_apps_start",
|
|
190
207
|
/** Stop another miniapp. */
|
|
191
208
|
MINIAPPS_STOP = "miniapp_apps_stop",
|
|
192
|
-
/** Install
|
|
209
|
+
/** Install the miniapp at a miniapp URL. System-list miniapps only. */
|
|
193
210
|
MINIAPPS_INSTALL = "miniapp_apps_install",
|
|
194
211
|
/** Preflight a release's host/SDK/hardware requirements without downloading it. */
|
|
195
212
|
MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
|
|
196
|
-
/** Uninstall a
|
|
213
|
+
/** Uninstall a miniapp installed from a miniapp URL. System-list miniapps only. */
|
|
197
214
|
MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
|
|
198
215
|
/** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
|
|
199
216
|
ACTION_INVOKE = "miniapp_action_invoke",
|
|
@@ -246,8 +263,6 @@ export declare enum MiniappResponseType {
|
|
|
246
263
|
VISIBILITY_CHANGE = "miniapp_visibility_change",
|
|
247
264
|
/** Push: host color scheme (light/dark) changed. */
|
|
248
265
|
COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
|
|
249
|
-
/** Push: a Store installation moved to another host-side phase. */
|
|
250
|
-
MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
|
|
251
266
|
/**
|
|
252
267
|
* Push: speaker playback state transitioned. Per-miniapp; the runtime sends
|
|
253
268
|
* to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
|
|
@@ -322,6 +337,10 @@ export declare enum MiniappStreamType {
|
|
|
322
337
|
GLASSES_WIFI = "glasses_wifi",
|
|
323
338
|
TRANSCRIPTION = "transcription",// language variant: "transcription:en-US"
|
|
324
339
|
TRANSLATION = "translation",// language variant: "translation:en-US"
|
|
340
|
+
/** State of the on-device speech-to-text model (session.transcription.onModelStatus). */
|
|
341
|
+
TRANSCRIPTION_MODEL_STATUS = "transcription_model_status",
|
|
342
|
+
/** On-device "Hey Mentra" keyword spotter (session.mic.onWakeWord). */
|
|
343
|
+
WAKE_WORD = "wake_word",
|
|
325
344
|
AUDIO_CHUNK = "audio_chunk",
|
|
326
345
|
VAD = "vad",
|
|
327
346
|
LOCATION_UPDATE = "location_update",
|
|
@@ -360,7 +379,7 @@ export declare enum MiniappErrorCode {
|
|
|
360
379
|
TTS_TEXT_TOO_LONG = "TTS_TEXT_TOO_LONG",
|
|
361
380
|
TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
|
|
362
381
|
TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
|
|
363
|
-
/** `speak({
|
|
382
|
+
/** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
|
|
364
383
|
TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
|
|
365
384
|
/** Not connected / pre-ACK and transport closed. */
|
|
366
385
|
NOT_CONNECTED = "NOT_CONNECTED",
|
|
@@ -370,21 +389,16 @@ export declare enum MiniappErrorCode {
|
|
|
370
389
|
MIC_SOURCE_CONFLICT = "MIC_SOURCE_CONFLICT",
|
|
371
390
|
/**
|
|
372
391
|
* Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
|
|
373
|
-
* setWifiAdbState) reject with this code
|
|
392
|
+
* setWifiAdbState) reject with this code, as installs do from any miniapp off
|
|
393
|
+
* the system list. Also: an uninstall of a miniapp that isn't a URL install.
|
|
374
394
|
*/
|
|
375
395
|
NOT_PERMITTED = "NOT_PERMITTED",
|
|
376
396
|
/** Target miniapp is not installed. */
|
|
377
397
|
APP_NOT_FOUND = "APP_NOT_FOUND",
|
|
378
398
|
/** Target miniapp is incompatible with the connected glasses. */
|
|
379
399
|
APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
|
|
380
|
-
/**
|
|
400
|
+
/** The miniapp didn't install; the error's `reason` says why (`InstallMiniappFailureReason`). */
|
|
381
401
|
INSTALL_FAILED = "INSTALL_FAILED",
|
|
382
|
-
/** Another Store installation is already in progress for the same package. */
|
|
383
|
-
INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
|
|
384
|
-
/** An automatic update was deferred because the target is running or starting. */
|
|
385
|
-
APP_RUNNING = "APP_RUNNING",
|
|
386
|
-
/** Another Store owns the installed package's update provenance. */
|
|
387
|
-
STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
|
|
388
402
|
/** Target miniapp does not declare the requested action. */
|
|
389
403
|
ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
|
|
390
404
|
/** Target connected but never registered a handler for the action (within the buffer window). */
|
package/dist/protocol.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,oBAAY,kBAAkB;IAC5B,8EAA8E;IAC9E,OAAO,oBAAoB;IAE3B;;;;OAIG;IACH,KAAK,kBAAkB;IAEvB,yDAAyD;IACzD,YAAY,yBAAyB;IAErC
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,oBAAY,kBAAkB;IAC5B,8EAA8E;IAC9E,OAAO,oBAAoB;IAE3B;;;;OAIG;IACH,KAAK,kBAAkB;IAEvB,yDAAyD;IACzD,YAAY,yBAAyB;IAErC;;;;;;;;;OASG;IACH,SAAS,sBAAsB;IAE/B;;;;OAIG;IACH,OAAO,oBAAoB;IAE3B,uEAAuE;IACvE,MAAM,mBAAmB;IAEzB,8EAA8E;IAC9E,UAAU,uBAAuB;IAEjC,sDAAsD;IACtD,UAAU,uBAAuB;IAEjC,uEAAuE;IACvE,KAAK,kBAAkB;IAEvB;;;;;OAKG;IACH,mBAAmB,gCAAgC;IACnD,oBAAoB,iCAAiC;IACrD,oBAAoB,iCAAiC;IACrD,oBAAoB,iCAAiC;IAErD,mCAAmC;IACnC,OAAO,oBAAoB;IAE3B,8BAA8B;IAC9B,aAAa,0BAA0B;IAEvC,wDAAwD;IACxD,oBAAoB,iCAAiC;IAErD,0DAA0D;IAC1D,gBAAgB,6BAA6B;IAC7C,gDAAgD;IAChD,eAAe,4BAA4B;IAC3C,mFAAmF;IACnF,kBAAkB,+BAA+B;IACjD,0FAA0F;IAC1F,6BAA6B,0CAA0C;IACvE,qFAAqF;IACrF,6BAA6B,0CAA0C;IACvE,sFAAsF;IACtF,oBAAoB,iCAAiC;IACrD,sFAAsF;IACtF,wBAAwB,qCAAqC;IAC7D;;;;6EAIyE;IACzE,0BAA0B,uCAAuC;IACjE;;sFAEkF;IAClF,6BAA6B,0CAA0C;IACvE;4DACwD;IACxD,wBAAwB,qCAAqC;IAC7D,iFAAiF;IACjF,6BAA6B,0CAA0C;IAEvE,kCAAkC;IAClC,WAAW,wBAAwB;IACnC,WAAW,wBAAwB;IACnC,cAAc,2BAA2B;IACzC,YAAY,yBAAyB;IACrC,aAAa,0BAA0B;IACvC,WAAW,wBAAwB;IACnC,eAAe,4BAA4B;IAC3C,oBAAoB,iCAAiC;IACrD,aAAa,0BAA0B;IAEvC,iCAAiC;IACjC,UAAU,uBAAuB;IAEjC,2EAA2E;IAC3E,cAAc,2BAA2B;IAEzC;;;;OAIG;IACH,eAAe,4BAA4B;IAE3C,kGAAkG;IAClG,mBAAmB,gCAAgC;IACnD,0EAA0E;IAC1E,6BAA6B,0CAA0C;IACvE;;;;OAIG;IACH,WAAW,wBAAwB;IACnC,gDAAgD;IAChD,WAAW,wBAAwB;IAEnC;;;OAGG;IACH,kBAAkB,+BAA+B;IAEjD,4CAA4C;IAC5C,KAAK,kBAAkB;IACvB,wCAAwC;IACxC,QAAQ,qBAAqB;IAC7B,yCAAyC;IACzC,cAAc,2BAA2B;IACzC,mEAAmE;IACnE,QAAQ,qBAAqB;IAC7B;;;OAGG;IACH,OAAO,oBAAoB;IAE3B;;;;;OAKG;IACH,WAAW,wBAAwB;IACnC,UAAU,uBAAuB;IACjC,WAAW,wBAAwB;IACnC,UAAU,uBAAuB;IACjC,QAAQ,qBAAqB;IAC7B,SAAS,sBAAsB;IAC/B,WAAW,wBAAwB;IACnC,UAAU,uBAAuB;IACjC,UAAU,uBAAuB;IACjC,cAAc,2BAA2B;IACzC,SAAS,sBAAsB;IAC/B,eAAe,4BAA4B;IAC3C,kFAAkF;IAClF,iBAAiB,8BAA8B;IAC/C,iFAAiF;IACjF,WAAW,wBAAwB;IACnC,wFAAwF;IACxF,UAAU,uBAAuB;IAEjC,0EAA0E;IAC1E,IAAI,iBAAiB;IAErB;;;;OAIG;IACH,oBAAoB,iCAAiC;IACrD;;;;OAIG;IACH,oBAAoB,iCAAiC;IAIrD,oHAAoH;IACpH,iBAAiB,8BAA8B;IAI/C,4CAA4C;IAC5C,wBAAwB,qCAAqC;IAG7D,KAAK,kBAAkB;IACvB,qBAAqB,kCAAkC;IACvD,oBAAoB,iCAAiC;IACrD,YAAY,yBAAyB;IACrC,WAAW,wBAAwB;IACnC,oBAAoB,iCAAiC;IACrD,mBAAmB,gCAAgC;IACnD;;;OAGG;IACH,oBAAoB,iCAAiC;IACrD,6CAA6C;IAC7C,mBAAmB,gCAAgC;IAEnD,oGAAoG;IACpG,kBAAkB,+BAA+B;IACjD,uFAAuF;IACvF,qBAAqB,kCAAkC;IACvD,gFAAgF;IAChF,yBAAyB,sCAAsC;IAG/D,+EAA+E;IAC/E,aAAa,sBAAsB;IACnC,kDAAkD;IAClD,cAAc,uBAAuB;IACrC,4BAA4B;IAC5B,aAAa,sBAAsB;IACnC,uEAAuE;IACvE,gBAAgB,yBAAyB;IACzC,mFAAmF;IACnF,sBAAsB,+BAA+B;IACrD,mFAAmF;IACnF,kBAAkB,2BAA2B;IAC7C,iFAAiF;IACjF,aAAa,0BAA0B;IACvC,kFAAkF;IAClF,aAAa,0BAA0B;IAEvC;;;OAGG;IACH,oBAAoB,iCAAiC;IACrD,yBAAyB,sCAAsC;IAC/D,cAAc,2BAA2B;IACzC,cAAc,2BAA2B;IACzC,YAAY,yBAAyB;IACrC,aAAa,0BAA0B;IACvC,qFAAqF;IACrF,WAAW,wBAAwB;IACnC,+EAA+E;IAC/E,aAAa,0BAA0B;IACvC,iBAAiB,8BAA8B;IAC/C,+EAA+E;IAC/E,yBAAyB,sCAAsC;IAC/D,2BAA2B,wCAAwC;IACnE,iBAAiB,8BAA8B;IAC/C,uFAAuF;IACvF,6BAA6B,0CAA0C;IACvE,8BAA8B,2CAA2C;IACzE,sGAAsG;IACtG,iBAAiB,8BAA8B;IAC/C,2FAA2F;IAC3F,kBAAkB,+BAA+B;IACjD,wEAAwE;IACxE,iBAAiB,8BAA8B;IAC/C;;;OAGG;IACH,qBAAqB,kCAAkC;CACxD;AAMD,oBAAY,mBAAmB;IAC7B,sEAAsE;IACtE,WAAW,wBAAwB;IAEnC,sDAAsD;IACtD,KAAK,kBAAkB;IAEvB,0EAA0E;IAC1E,cAAc,2BAA2B;IAEzC,0CAA0C;IAC1C,mBAAmB,gCAAgC;IAEnD,2DAA2D;IAC3D,iBAAiB,8BAA8B;IAE/C,oDAAoD;IACpD,mBAAmB,gCAAgC;IAEnD;;;;OAIG;IACH,aAAa,0BAA0B;IAEvC;;;;OAIG;IACH,kBAAkB,+BAA+B;IAEjD;;;OAGG;IACH,WAAW,wBAAwB;IAEnC,qEAAqE;IACrE,IAAI,iBAAiB;IAErB;;;;;OAKG;IACH,WAAW,wBAAwB;IAEnC;;;OAGG;IACH,aAAa,0BAA0B;IAEvC;;;OAGG;IACH,uBAAuB,oCAAoC;IAE3D;;;;OAIG;IACH,aAAa,0BAA0B;IAEvC;;;OAGG;IACH,qBAAqB,kCAAkC;IAEvD;;;;;OAKG;IACH,eAAe,4BAA4B;IAE3C,kDAAkD;IAClD,KAAK,kBAAkB;CACxB;AAMD,oBAAY,iBAAiB;IAE3B,YAAY,iBAAiB;IAC7B,WAAW,gBAAgB;IAC3B,aAAa,kBAAkB;IAC/B;;;;OAIG;IACH,UAAU,eAAe;IAGzB,eAAe,oBAAoB;IACnC,aAAa,kBAAkB;IAC/B,kBAAkB,uBAAuB;IACzC,2FAA2F;IAC3F,YAAY,iBAAiB;IAG7B,aAAa,kBAAkB,CAAE,0CAA0C;IAC3E,WAAW,gBAAgB,CAAE,wCAAwC;IACrE,yFAAyF;IACzF,0BAA0B,+BAA+B;IACzD,uEAAuE;IACvE,SAAS,cAAc;IACvB,WAAW,gBAAgB;IAC3B,GAAG,QAAQ;IAGX,eAAe,oBAAoB;IACnC,4DAA4D;IAC5D,cAAc,mBAAmB;IACjC,8EAA8E;IAC9E,iBAAiB,sBAAsB;IACvC,gFAAgF;IAChF,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;IACzC;;;;;;OAMG;IACH,4BAA4B,iCAAiC;IAE7D,WAAW,gBAAgB;IAC3B,aAAa,kBAAkB;CAChC;AAMD,oBAAY,gBAAgB;IAC1B,qDAAqD;IACrD,gBAAgB,qBAAqB;IAErC,2FAA2F;IAC3F,uBAAuB,4BAA4B;IAEnD,iEAAiE;IACjE,iBAAiB,sBAAsB;IAEvC,2DAA2D;IAC3D,eAAe,oBAAoB;IAEnC,+EAA+E;IAC/E,eAAe,oBAAoB;IAEnC,mDAAmD;IACnD,QAAQ,aAAa;IAErB,wDAAwD;IACxD,iBAAiB,sBAAsB;IACvC,iBAAiB,sBAAsB;IACvC,kBAAkB,uBAAuB;IAEzC,6FAA6F;IAC7F,qBAAqB,0BAA0B;IAE/C,oDAAoD;IACpD,aAAa,kBAAkB;IAE/B,0EAA0E;IAC1E,oBAAoB,yBAAyB;IAE7C,6EAA6E;IAC7E,mBAAmB,wBAAwB;IAG3C;;;;OAIG;IACH,aAAa,kBAAkB;IAC/B,uCAAuC;IACvC,aAAa,kBAAkB;IAC/B,iEAAiE;IACjE,kBAAkB,uBAAuB;IACzC,iGAAiG;IACjG,cAAc,mBAAmB;IACjC,4DAA4D;IAC5D,gBAAgB,qBAAqB;IACrC,iGAAiG;IACjG,iBAAiB,sBAAsB;IACvC,4EAA4E;IAC5E,WAAW,gBAAgB;IAC3B,iEAAiE;IACjE,cAAc,mBAAmB;IACjC,4DAA4D;IAC5D,iBAAiB,sBAAsB;IAGvC,6DAA6D;IAC7D,cAAc,mBAAmB;IACjC,6DAA6D;IAC7D,mBAAmB,wBAAwB;IAC3C,iEAAiE;IACjE,cAAc,mBAAmB;IACjC,gFAAgF;IAChF,mBAAmB,wBAAwB;IAC3C,gEAAgE;IAChE,iBAAiB,sBAAsB;IACvC,wEAAwE;IACxE,oBAAoB,yBAAyB;IAC7C,kFAAkF;IAClF,kBAAkB,uBAAuB;CAC1C;AAMD;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,SAAS,CAAA;AAE9C;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GACxB,mBAAmB,GACnB,mBAAmB,GACnB,cAAc,GACd,mBAAmB,GACnB,cAAc,GACd,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,aAAa,GACb,mBAAmB,GACnB,sBAAsB,CAAA;AAE1B,eAAO,MAAM,mBAAmB,EAAE,SAAS,gBAAgB,EAY1D,CAAA;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,gBAAgB,CAE5E;AAED,yEAAyE;AACzE,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,kFAAkF;AAClF,eAAO,MAAM,kBAAkB,aAAa,CAAA"}
|
package/dist/protocol.js
CHANGED
|
@@ -22,7 +22,16 @@ export var MiniappRequestType;
|
|
|
22
22
|
MiniappRequestType["READY"] = "miniapp_ready";
|
|
23
23
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
24
24
|
MiniappRequestType["AUTH_REFRESH"] = "miniapp_auth_refresh";
|
|
25
|
-
/**
|
|
25
|
+
/**
|
|
26
|
+
* Update the set of streams the miniapp is subscribed to. `subscriptions`
|
|
27
|
+
* lists stream names; a transcription stream is `{stream, processing}`, one
|
|
28
|
+
* entry per processing its listeners ask for, and `transcriptionListeners`
|
|
29
|
+
* lists every transcription listener as `{id, stream, processing}`. Both
|
|
30
|
+
* also carry `forceLocal: true` for `localOnly`, the form hosts before
|
|
31
|
+
* `processing` read. Hosts accept that older form,
|
|
32
|
+
* `{stream, forceLocal?, includeCloud?}`, from SDKs that predate
|
|
33
|
+
* `processing`.
|
|
34
|
+
*/
|
|
26
35
|
MiniappRequestType["SUBSCRIBE"] = "miniapp_subscribe";
|
|
27
36
|
/**
|
|
28
37
|
* Push a legacy display layout to the glasses. The SDK no longer emits this
|
|
@@ -36,7 +45,7 @@ export var MiniappRequestType;
|
|
|
36
45
|
MiniappRequestType["PLAY_AUDIO"] = "miniapp_play_audio";
|
|
37
46
|
/** Stop any audio playback this miniapp initiated. */
|
|
38
47
|
MiniappRequestType["STOP_AUDIO"] = "miniapp_stop_audio";
|
|
39
|
-
/** Speak text via cloud TTS
|
|
48
|
+
/** Speak text via cloud or on-device TTS, as its `processing` says. */
|
|
40
49
|
MiniappRequestType["SPEAK"] = "miniapp_speak";
|
|
41
50
|
/**
|
|
42
51
|
* Live PCM output stream to the phone's audio playback service
|
|
@@ -165,6 +174,15 @@ export var MiniappRequestType;
|
|
|
165
174
|
* so future reconnect logic can re-send.
|
|
166
175
|
*/
|
|
167
176
|
MiniappRequestType["TRANSCRIPTION_CONFIG"] = "miniapp_transcription_config";
|
|
177
|
+
/**
|
|
178
|
+
* End the current on-device utterance now (transcription.finishUtterance).
|
|
179
|
+
* Result: the final text, "" when nothing was pending, or null when the
|
|
180
|
+
* model can't finish early.
|
|
181
|
+
*/
|
|
182
|
+
MiniappRequestType["TRANSCRIPTION_FINISH"] = "miniapp_transcription_finish";
|
|
183
|
+
// ----- Telemetry -----
|
|
184
|
+
/** A miniapp's own analytics event (`session.telemetry.capture`). One-shot; hosts with `hostFeatures.telemetry`. */
|
|
185
|
+
MiniappRequestType["TELEMETRY_CAPTURE"] = "miniapp_telemetry_capture";
|
|
168
186
|
// ----- Deferred in v1 -----
|
|
169
187
|
/** Dashboard content update. Noop in v1. */
|
|
170
188
|
MiniappRequestType["DASHBOARD_CONTENT_UPDATE"] = "miniapp_dashboard_content_update";
|
|
@@ -196,11 +214,11 @@ export var MiniappRequestType;
|
|
|
196
214
|
MiniappRequestType["MINIAPPS_START"] = "miniapp_apps_start";
|
|
197
215
|
/** Stop another miniapp. */
|
|
198
216
|
MiniappRequestType["MINIAPPS_STOP"] = "miniapp_apps_stop";
|
|
199
|
-
/** Install
|
|
217
|
+
/** Install the miniapp at a miniapp URL. System-list miniapps only. */
|
|
200
218
|
MiniappRequestType["MINIAPPS_INSTALL"] = "miniapp_apps_install";
|
|
201
219
|
/** Preflight a release's host/SDK/hardware requirements without downloading it. */
|
|
202
220
|
MiniappRequestType["MINIAPPS_INSTALL_CHECK"] = "miniapp_apps_install_check";
|
|
203
|
-
/** Uninstall a
|
|
221
|
+
/** Uninstall a miniapp installed from a miniapp URL. System-list miniapps only. */
|
|
204
222
|
MiniappRequestType["MINIAPPS_UNINSTALL"] = "miniapp_apps_uninstall";
|
|
205
223
|
/** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
|
|
206
224
|
MiniappRequestType["ACTION_INVOKE"] = "miniapp_action_invoke";
|
|
@@ -257,8 +275,6 @@ export var MiniappResponseType;
|
|
|
257
275
|
MiniappResponseType["VISIBILITY_CHANGE"] = "miniapp_visibility_change";
|
|
258
276
|
/** Push: host color scheme (light/dark) changed. */
|
|
259
277
|
MiniappResponseType["COLOR_SCHEME_CHANGE"] = "miniapp_color_scheme_change";
|
|
260
|
-
/** Push: a Store installation moved to another host-side phase. */
|
|
261
|
-
MiniappResponseType["MINIAPPS_INSTALL_PROGRESS"] = "miniapp_apps_install_progress";
|
|
262
278
|
/**
|
|
263
279
|
* Push: speaker playback state transitioned. Per-miniapp; the runtime sends
|
|
264
280
|
* to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
|
|
@@ -340,6 +356,10 @@ export var MiniappStreamType;
|
|
|
340
356
|
// Speech / audio (cloud or local)
|
|
341
357
|
MiniappStreamType["TRANSCRIPTION"] = "transcription";
|
|
342
358
|
MiniappStreamType["TRANSLATION"] = "translation";
|
|
359
|
+
/** State of the on-device speech-to-text model (session.transcription.onModelStatus). */
|
|
360
|
+
MiniappStreamType["TRANSCRIPTION_MODEL_STATUS"] = "transcription_model_status";
|
|
361
|
+
/** On-device "Hey Mentra" keyword spotter (session.mic.onWakeWord). */
|
|
362
|
+
MiniappStreamType["WAKE_WORD"] = "wake_word";
|
|
343
363
|
MiniappStreamType["AUDIO_CHUNK"] = "audio_chunk";
|
|
344
364
|
MiniappStreamType["VAD"] = "vad";
|
|
345
365
|
// Phone sensors
|
|
@@ -384,7 +404,7 @@ export var MiniappErrorCode;
|
|
|
384
404
|
MiniappErrorCode["TTS_TEXT_TOO_LONG"] = "TTS_TEXT_TOO_LONG";
|
|
385
405
|
MiniappErrorCode["TTS_INVALID_VOICE"] = "TTS_INVALID_VOICE";
|
|
386
406
|
MiniappErrorCode["TTS_UPSTREAM_ERROR"] = "TTS_UPSTREAM_ERROR";
|
|
387
|
-
/** `speak({
|
|
407
|
+
/** `speak({processing: "localOnly"})` but the on-device TTS model isn't downloaded/ready. */
|
|
388
408
|
MiniappErrorCode["TTS_LOCAL_UNAVAILABLE"] = "TTS_LOCAL_UNAVAILABLE";
|
|
389
409
|
/** Not connected / pre-ACK and transport closed. */
|
|
390
410
|
MiniappErrorCode["NOT_CONNECTED"] = "NOT_CONNECTED";
|
|
@@ -395,21 +415,16 @@ export var MiniappErrorCode;
|
|
|
395
415
|
// ----- Inter-miniapp interop -----
|
|
396
416
|
/**
|
|
397
417
|
* Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
|
|
398
|
-
* setWifiAdbState) reject with this code
|
|
418
|
+
* setWifiAdbState) reject with this code, as installs do from any miniapp off
|
|
419
|
+
* the system list. Also: an uninstall of a miniapp that isn't a URL install.
|
|
399
420
|
*/
|
|
400
421
|
MiniappErrorCode["NOT_PERMITTED"] = "NOT_PERMITTED";
|
|
401
422
|
/** Target miniapp is not installed. */
|
|
402
423
|
MiniappErrorCode["APP_NOT_FOUND"] = "APP_NOT_FOUND";
|
|
403
424
|
/** Target miniapp is incompatible with the connected glasses. */
|
|
404
425
|
MiniappErrorCode["APP_NOT_COMPATIBLE"] = "APP_NOT_COMPATIBLE";
|
|
405
|
-
/**
|
|
426
|
+
/** The miniapp didn't install; the error's `reason` says why (`InstallMiniappFailureReason`). */
|
|
406
427
|
MiniappErrorCode["INSTALL_FAILED"] = "INSTALL_FAILED";
|
|
407
|
-
/** Another Store installation is already in progress for the same package. */
|
|
408
|
-
MiniappErrorCode["INSTALL_IN_PROGRESS"] = "INSTALL_IN_PROGRESS";
|
|
409
|
-
/** An automatic update was deferred because the target is running or starting. */
|
|
410
|
-
MiniappErrorCode["APP_RUNNING"] = "APP_RUNNING";
|
|
411
|
-
/** Another Store owns the installed package's update provenance. */
|
|
412
|
-
MiniappErrorCode["STORE_OWNERSHIP_CONFLICT"] = "STORE_OWNERSHIP_CONFLICT";
|
|
413
428
|
/** Target miniapp does not declare the requested action. */
|
|
414
429
|
MiniappErrorCode["ACTION_NOT_FOUND"] = "ACTION_NOT_FOUND";
|
|
415
430
|
/** Target connected but never registered a handler for the action (within the buffer window). */
|