@mentra/miniapp 0.3.0-dev.1 → 3.1.0-dev.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/background/index.d.ts +3 -1
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/hardware.d.ts +24 -0
- package/dist/hardware.d.ts.map +1 -0
- package/dist/hardware.js +20 -0
- package/dist/hardware.js.map +1 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/modules/glasses.d.ts +8 -0
- package/dist/modules/glasses.d.ts.map +1 -1
- package/dist/modules/glasses.js +13 -0
- package/dist/modules/glasses.js.map +1 -1
- package/dist/modules/languages.d.ts +42 -0
- package/dist/modules/languages.d.ts.map +1 -0
- package/dist/modules/languages.js +57 -0
- package/dist/modules/languages.js.map +1 -0
- package/dist/modules/mic.d.ts +17 -0
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +28 -1
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +2 -0
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +5 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +2 -3
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +6 -2
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +1 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/modules/system.d.ts +20 -1
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +13 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/transcription.d.ts +26 -7
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +28 -8
- package/dist/modules/transcription.js.map +1 -1
- package/dist/modules/translation.d.ts +9 -3
- package/dist/modules/translation.d.ts.map +1 -1
- package/dist/modules/translation.js +5 -3
- package/dist/modules/translation.js.map +1 -1
- package/dist/protocol.d.ts +18 -1
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +18 -1
- package/dist/protocol.js.map +1 -1
- package/dist/react/capsuleHeaderLayout.d.ts +3 -0
- package/dist/react/capsuleHeaderLayout.d.ts.map +1 -0
- package/dist/react/capsuleHeaderLayout.js +5 -0
- package/dist/react/capsuleHeaderLayout.js.map +1 -0
- package/dist/react/useCapsuleHeaderStyle.d.ts.map +1 -1
- package/dist/react/useCapsuleHeaderStyle.js +6 -1
- package/dist/react/useCapsuleHeaderStyle.js.map +1 -1
- package/dist/transport/mock.js +3 -0
- package/dist/transport/mock.js.map +1 -1
- package/package.json +6 -2
- package/src/background/index.ts +15 -1
- package/src/hardware.ts +25 -0
- package/src/index.ts +21 -7
- package/src/modules/glasses.ts +14 -0
- package/src/modules/languages.ts +72 -0
- package/src/modules/mic.ts +30 -1
- package/src/modules/miniapps.ts +2 -0
- package/src/modules/speaker.ts +7 -3
- package/src/modules/stream.ts +7 -2
- package/src/modules/system.ts +26 -1
- package/src/modules/transcription.ts +43 -13
- package/src/modules/translation.ts +22 -8
- package/src/protocol.ts +20 -1
- package/src/react/capsuleHeaderLayout.ts +11 -0
- package/src/react/useCapsuleHeaderStyle.ts +6 -1
- package/src/transport/mock.ts +4 -0
package/src/hardware.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Hardware components that a miniapp can declare in its manifest. */
|
|
2
|
+
export enum HardwareType {
|
|
3
|
+
CAMERA = "CAMERA",
|
|
4
|
+
DISPLAY = "DISPLAY",
|
|
5
|
+
MICROPHONE = "MICROPHONE",
|
|
6
|
+
SPEAKER = "SPEAKER",
|
|
7
|
+
IMU = "IMU",
|
|
8
|
+
BUTTON = "BUTTON",
|
|
9
|
+
LIGHT = "LIGHT",
|
|
10
|
+
WIFI = "WIFI",
|
|
11
|
+
EXIST = "EXIST",
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Whether a missing component prevents the miniapp from running. */
|
|
15
|
+
export enum HardwareRequirementLevel {
|
|
16
|
+
REQUIRED = "REQUIRED",
|
|
17
|
+
OPTIONAL = "OPTIONAL",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** One hardware requirement from a miniapp manifest. */
|
|
21
|
+
export interface HardwareRequirement {
|
|
22
|
+
type: HardwareType
|
|
23
|
+
level: HardwareRequirementLevel
|
|
24
|
+
description?: string
|
|
25
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -36,14 +36,21 @@ export type {MentraOSGlobals, MiniappCapsuleMenuRect, MiniappColorScheme, Miniap
|
|
|
36
36
|
|
|
37
37
|
export {MiniappErrorCode, MiniappRequestType, MiniappResponseType, MiniappStreamType} from "./protocol"
|
|
38
38
|
export {CLOUD_STATUS_STREAM} from "./modules/cloud"
|
|
39
|
+
|
|
40
|
+
// Language registry (issue 021): canonical language types + the validation
|
|
41
|
+
// error the typed transcription/translation surfaces throw.
|
|
42
|
+
export {
|
|
43
|
+
MiniappValidationError,
|
|
44
|
+
SUPPORTED_LANGUAGE_HINTS,
|
|
45
|
+
SUPPORTED_TRANSCRIPTION_LANGUAGES,
|
|
46
|
+
isTranscriptionLanguage,
|
|
47
|
+
} from "./modules/languages"
|
|
48
|
+
export type {LanguageHint, TranscriptionLanguage} from "./modules/languages"
|
|
39
49
|
export type {CloudClientAudioTransport, CloudClientConnectionStatus, CloudClientStatus} from "./modules/cloud"
|
|
40
50
|
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
// type) per @mentra/types' Bun-compat convention.
|
|
45
|
-
export {HardwareType, HardwareRequirementLevel} from "@mentra/types"
|
|
46
|
-
export type {HardwareRequirement} from "@mentra/types"
|
|
51
|
+
// Canonical miniapp manifest hardware contract.
|
|
52
|
+
export {HardwareType, HardwareRequirementLevel} from "./hardware"
|
|
53
|
+
export type {HardwareRequirement} from "./hardware"
|
|
47
54
|
|
|
48
55
|
// Transports — exported for advanced uses (forced transport injection, tests)
|
|
49
56
|
export {createTransport} from "./transport/auto"
|
|
@@ -120,7 +127,14 @@ export type {
|
|
|
120
127
|
StreamStatus,
|
|
121
128
|
StreamVideoConfig,
|
|
122
129
|
} from "./modules/stream"
|
|
123
|
-
export type {
|
|
130
|
+
export type {
|
|
131
|
+
ShareOptions,
|
|
132
|
+
ShareResult,
|
|
133
|
+
DownloadOptions,
|
|
134
|
+
DownloadResult,
|
|
135
|
+
ScanQrOptions,
|
|
136
|
+
ScanQrResult,
|
|
137
|
+
} from "./modules/system"
|
|
124
138
|
export type {MiniappInfo, MiniappActionInfo, MiniappCompatibility, ListMiniappsOptions} from "./modules/miniapps"
|
|
125
139
|
export type {ActionContext, ActionHandler, InvokeOptions} from "./modules/actions"
|
|
126
140
|
|
package/src/modules/glasses.ts
CHANGED
|
@@ -41,4 +41,18 @@ export class GlassesModule {
|
|
|
41
41
|
reason,
|
|
42
42
|
})
|
|
43
43
|
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Enable or disable Wi-Fi ADB (wireless debugging) on Mentra Live.
|
|
47
|
+
* Persisted on the glasses; boot applies the saved preference (default off).
|
|
48
|
+
*
|
|
49
|
+
* SYSTEM-only — rejects with `NOT_PERMITTED` unless this miniapp is a system
|
|
50
|
+
* app. Third-party miniapps must not expose wireless debugging controls.
|
|
51
|
+
*/
|
|
52
|
+
async setWifiAdbState(enabled: boolean): Promise<void> {
|
|
53
|
+
await this.session.sendRequest<void>({
|
|
54
|
+
type: MiniappRequestType.SET_WIFI_ADB_STATE,
|
|
55
|
+
enabled,
|
|
56
|
+
})
|
|
57
|
+
}
|
|
44
58
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Language validation at the SDK boundary (issue 021, WP2).
|
|
3
|
+
*
|
|
4
|
+
* The registry itself lives in `@mentra/cloud-protocol/languages` (single
|
|
5
|
+
* source of truth shared with the cloud). This module re-exports the types so
|
|
6
|
+
* miniapps import everything from `@mentra/miniapp`, and adds the throwing
|
|
7
|
+
* validators the typed module surfaces (`transcription.*`, `translation.*`)
|
|
8
|
+
* call on every language parameter.
|
|
9
|
+
*
|
|
10
|
+
* Contract (design doc docs/issues/021-typed-language-subscriptions):
|
|
11
|
+
* - Valid by construction: params are typed as literal unions, so TS users
|
|
12
|
+
* get autocomplete and compile errors.
|
|
13
|
+
* - Loud at runtime: JS users (or `any` casts) get a thrown
|
|
14
|
+
* `MiniappValidationError` with a suggestion, never silent fallback.
|
|
15
|
+
* - Canonical on the wire: bare codes ("fr") canonicalize to their default
|
|
16
|
+
* tag ("fr-FR") before becoming stream keys, so the phone/cloud only ever
|
|
17
|
+
* see registry tags.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import {
|
|
21
|
+
isTranscriptionLanguage,
|
|
22
|
+
suggestTranscriptionLanguage,
|
|
23
|
+
toLanguageHint,
|
|
24
|
+
toTranscriptionLanguage,
|
|
25
|
+
SUPPORTED_LANGUAGE_HINTS,
|
|
26
|
+
SUPPORTED_TRANSCRIPTION_LANGUAGES,
|
|
27
|
+
} from "@mentra/cloud-protocol/languages"
|
|
28
|
+
import type {LanguageHint, TranscriptionLanguage} from "@mentra/cloud-protocol/languages"
|
|
29
|
+
|
|
30
|
+
export {SUPPORTED_LANGUAGE_HINTS, SUPPORTED_TRANSCRIPTION_LANGUAGES, isTranscriptionLanguage}
|
|
31
|
+
export type {LanguageHint, TranscriptionLanguage}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Thrown synchronously by SDK methods when a parameter fails validation.
|
|
35
|
+
* Deliberately a hard throw (not a warning, not a silent default): an invalid
|
|
36
|
+
* language previously produced a subscription that silently transcribed the
|
|
37
|
+
* wrong language or silently never spoke (OS-1746).
|
|
38
|
+
*/
|
|
39
|
+
export class MiniappValidationError extends Error {
|
|
40
|
+
constructor(message: string) {
|
|
41
|
+
super(message)
|
|
42
|
+
this.name = "MiniappValidationError"
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Canonicalize a subscription language or throw. Accepts registry tags
|
|
48
|
+
* ("fr-FR") and bare registry codes ("fr" -> "fr-FR").
|
|
49
|
+
*/
|
|
50
|
+
export function requireTranscriptionLanguage(value: string, param: string): TranscriptionLanguage {
|
|
51
|
+
const canonical = toTranscriptionLanguage(value)
|
|
52
|
+
if (canonical) return canonical
|
|
53
|
+
const suggestion = suggestTranscriptionLanguage(value)
|
|
54
|
+
throw new MiniappValidationError(
|
|
55
|
+
`${param}: unknown language ${JSON.stringify(value)}` +
|
|
56
|
+
(suggestion ? ` — did you mean ${JSON.stringify(suggestion)}?` : "") +
|
|
57
|
+
` Supported values are BCP-47 tags from SUPPORTED_TRANSCRIPTION_LANGUAGES (e.g. "en-US", "fr-FR").`,
|
|
58
|
+
)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Normalize a language hint to the bare code the STT provider accepts, or
|
|
63
|
+
* throw. Accepts bare codes ("fr") and registry tags ("fr-FR" -> "fr").
|
|
64
|
+
*/
|
|
65
|
+
export function requireLanguageHint(value: string, param: string): LanguageHint {
|
|
66
|
+
const hint = toLanguageHint(value)
|
|
67
|
+
if (hint) return hint
|
|
68
|
+
throw new MiniappValidationError(
|
|
69
|
+
`${param}: unknown language hint ${JSON.stringify(value)}. ` +
|
|
70
|
+
`Hints are bare ISO 639-1 codes from SUPPORTED_LANGUAGE_HINTS (e.g. "en", "fr").`,
|
|
71
|
+
)
|
|
72
|
+
}
|
package/src/modules/mic.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* PERMISSION_NOT_DECLARED otherwise.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {MiniappStreamType} from "../protocol"
|
|
18
|
+
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
19
19
|
import {MiniappSession} from "../session"
|
|
20
20
|
import type {AudioChunkData, UnsubscribeFn, VadData} from "./events"
|
|
21
21
|
|
|
@@ -41,6 +41,35 @@ export class MicModule {
|
|
|
41
41
|
return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Temporarily override glasses-side voice activity detection (GX8002) for
|
|
46
|
+
* this miniapp's lifetime. When disabled, mic gating falls back to the
|
|
47
|
+
* loudness gate only (if it is enabled). The Mentra App's configured value
|
|
48
|
+
* is restored when this miniapp disconnects.
|
|
49
|
+
*
|
|
50
|
+
* Requires `MICROPHONE` in the miniapp manifest.
|
|
51
|
+
*/
|
|
52
|
+
setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void> {
|
|
53
|
+
return this.session.sendRequest<void>({
|
|
54
|
+
type: MiniappRequestType.MIC_SET_VAD_ENABLED,
|
|
55
|
+
enabled,
|
|
56
|
+
})
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Temporarily override the center-mic loudness gate ("Barrier") for this
|
|
61
|
+
* miniapp's lifetime. It blocks quiet/self-talk audio independent of VAD.
|
|
62
|
+
* The Mentra App's configured value is restored when this miniapp disconnects.
|
|
63
|
+
*
|
|
64
|
+
* Requires `MICROPHONE` in the miniapp manifest.
|
|
65
|
+
*/
|
|
66
|
+
setLoudnessGateEnabled(enabled: boolean): Promise<void> {
|
|
67
|
+
return this.session.sendRequest<void>({
|
|
68
|
+
type: MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED,
|
|
69
|
+
enabled,
|
|
70
|
+
})
|
|
71
|
+
}
|
|
72
|
+
|
|
44
73
|
/**
|
|
45
74
|
* Tear down every subscription this module owns. Useful when a component
|
|
46
75
|
* is unmounting and wants to free everything at once without tracking
|
package/src/modules/miniapps.ts
CHANGED
|
@@ -20,6 +20,8 @@ export interface MiniappActionInfo {
|
|
|
20
20
|
* to an MCP tool `inputSchema`. Undefined for actions that take no params.
|
|
21
21
|
*/
|
|
22
22
|
parameters?: Record<string, unknown>
|
|
23
|
+
/** JSON-Schema descriptor for the structured value returned by the action. */
|
|
24
|
+
outputSchema?: Record<string, unknown>
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
/**
|
package/src/modules/speaker.ts
CHANGED
|
@@ -50,6 +50,11 @@ export interface SpeakOptions {
|
|
|
50
50
|
voice_settings?: Record<string, unknown>
|
|
51
51
|
volume?: number
|
|
52
52
|
stopOtherAudio?: boolean
|
|
53
|
+
/**
|
|
54
|
+
* Normalize display-oriented markup and symbols into natural spoken words.
|
|
55
|
+
* Defaults to true. Set false when exact characters must reach the TTS engine.
|
|
56
|
+
*/
|
|
57
|
+
enableSanitization?: boolean
|
|
53
58
|
/**
|
|
54
59
|
* Force this call to use on-device offline TTS, skipping cloud TTS
|
|
55
60
|
* entirely — even when cloud is connected. Rejects with `TTS_LOCAL_UNAVAILABLE`
|
|
@@ -253,9 +258,7 @@ export class SpeakerModule {
|
|
|
253
258
|
async speak(text: string, options?: SpeakOptions): Promise<SpeakResult>
|
|
254
259
|
async speak(sentences: string[], options?: SpeakOptions): Promise<SpeakResult>
|
|
255
260
|
async speak(text: string | string[], options: SpeakOptions = {}): Promise<SpeakResult> {
|
|
256
|
-
const normalized = Array.isArray(text)
|
|
257
|
-
? text.map((sentence) => sentence.trim()).filter(Boolean)
|
|
258
|
-
: text.trim()
|
|
261
|
+
const normalized = Array.isArray(text) ? text.map((sentence) => sentence.trim()).filter(Boolean) : text.trim()
|
|
259
262
|
if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
|
|
260
263
|
throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
|
|
261
264
|
}
|
|
@@ -271,6 +274,7 @@ export class SpeakerModule {
|
|
|
271
274
|
voice_settings: options.voice_settings,
|
|
272
275
|
volume: options.volume,
|
|
273
276
|
stopOtherAudio: options.stopOtherAudio ?? false,
|
|
277
|
+
enableSanitization: options.enableSanitization ?? true,
|
|
274
278
|
forceLocal: options.forceLocal ?? false,
|
|
275
279
|
},
|
|
276
280
|
{timeoutMs: 0},
|
package/src/modules/stream.ts
CHANGED
|
@@ -77,8 +77,12 @@ export interface StartStreamOptions {
|
|
|
77
77
|
* and automatic recording.
|
|
78
78
|
* - "whip": sub-second WebRTC playback (use `webrtcUrl`/WHEP), but no
|
|
79
79
|
* HLS/DASH and no recording.
|
|
80
|
+
* - "rtmp": RTMPS ingest over TCP 443 → HLS playback. Use when WHIP/SRT
|
|
81
|
+
* UDP is blocked (then play `hlsUrl`, not WHEP).
|
|
80
82
|
*/
|
|
81
|
-
ingest?: "srt" | "whip"
|
|
83
|
+
ingest?: "srt" | "whip" | "rtmp"
|
|
84
|
+
/** Optional Bearer token for direct WHIP Authorization (custom authenticated endpoints). */
|
|
85
|
+
authToken?: string
|
|
82
86
|
}
|
|
83
87
|
|
|
84
88
|
export interface StreamResult {
|
|
@@ -98,7 +102,7 @@ export interface StreamResult {
|
|
|
98
102
|
webrtcUrl?: string
|
|
99
103
|
}
|
|
100
104
|
|
|
101
|
-
/** Live encoder telemetry
|
|
105
|
+
/** Live encoder and device telemetry emitted periodically by supported glasses firmware. */
|
|
102
106
|
export interface StreamLiveStats {
|
|
103
107
|
/** Current encoded video bitrate in bits per second. */
|
|
104
108
|
bitrate?: number
|
|
@@ -141,6 +145,7 @@ export class StreamModule {
|
|
|
141
145
|
video: options.video,
|
|
142
146
|
audio: options.audio,
|
|
143
147
|
sound: options.sound ?? true,
|
|
148
|
+
...(options.authToken ? {authToken: options.authToken} : {}),
|
|
144
149
|
})
|
|
145
150
|
}
|
|
146
151
|
return this.session.sendRequest<StreamResult>({
|
package/src/modules/system.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview SystemModule — OS-level utilities (share, open URL, clipboard, download).
|
|
2
|
+
* @fileoverview SystemModule — OS-level utilities (share, open URL, clipboard, download, QR scan).
|
|
3
3
|
*
|
|
4
4
|
* These bridge to native phone capabilities via LocalMiniappRuntime.
|
|
5
5
|
*/
|
|
@@ -37,6 +37,15 @@ export interface DownloadResult {
|
|
|
37
37
|
cancelled?: boolean
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
export interface ScanQrOptions {
|
|
41
|
+
/** Header title on the phone scanner overlay. */
|
|
42
|
+
title?: string
|
|
43
|
+
/** Hint shown over the camera viewfinder. */
|
|
44
|
+
hint?: string
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type ScanQrResult = {data: string; cancelled?: false} | {cancelled: true; data?: undefined}
|
|
48
|
+
|
|
40
49
|
export class SystemModule {
|
|
41
50
|
constructor(private readonly session: MiniappSession) {}
|
|
42
51
|
|
|
@@ -73,4 +82,20 @@ export class SystemModule {
|
|
|
73
82
|
})
|
|
74
83
|
return result ?? { success: false }
|
|
75
84
|
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Open the phone camera QR scanner as a host overlay. Does not unmount the
|
|
88
|
+
* miniapp (so live sessions survive the scan). Resolves with the raw QR
|
|
89
|
+
* payload, or `{cancelled: true}` if the user dismisses the scanner.
|
|
90
|
+
*/
|
|
91
|
+
async scanQr(options: ScanQrOptions = {}): Promise<ScanQrResult> {
|
|
92
|
+
const result = await this.session.sendRequest<ScanQrResult>(
|
|
93
|
+
{
|
|
94
|
+
type: MiniappRequestType.SCAN_QR,
|
|
95
|
+
...options,
|
|
96
|
+
},
|
|
97
|
+
{timeoutMs: 0},
|
|
98
|
+
)
|
|
99
|
+
return result ?? { cancelled: true }
|
|
100
|
+
}
|
|
76
101
|
}
|
|
@@ -21,11 +21,18 @@
|
|
|
21
21
|
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
22
22
|
import {MiniappSession} from "../session"
|
|
23
23
|
import type {TranscriptionData, UnsubscribeFn} from "./events"
|
|
24
|
+
import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
|
|
25
|
+
import type {LanguageHint, TranscriptionLanguage} from "./languages"
|
|
24
26
|
|
|
25
27
|
/** Configuration for cloud-side transcription behavior. */
|
|
26
28
|
export interface TranscriptionConfig {
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Language hints to improve auto-detection accuracy. Bare ISO 639-1 codes
|
|
31
|
+
* from the registry (e.g. ["en", "ja"]); registry BCP-47 tags are accepted
|
|
32
|
+
* and normalized ("en-US" -> "en"). Anything else throws
|
|
33
|
+
* `MiniappValidationError`.
|
|
34
|
+
*/
|
|
35
|
+
languageHints?: LanguageHint[]
|
|
29
36
|
/** Custom vocabulary / boosted terms (e.g. ["MentraOS", "HIPAA"]). */
|
|
30
37
|
vocabulary?: string[]
|
|
31
38
|
/** Enable speaker diarisation. Defaults vary by provider. */
|
|
@@ -65,15 +72,24 @@ export class TranscriptionModule {
|
|
|
65
72
|
* Subscribe to transcription for one or more specific languages. Each call
|
|
66
73
|
* is independent; multiple can be active simultaneously.
|
|
67
74
|
*
|
|
68
|
-
*
|
|
75
|
+
* Languages are validated against the platform registry
|
|
76
|
+
* (`SUPPORTED_TRANSCRIPTION_LANGUAGES`): registry BCP-47 tags pass through,
|
|
77
|
+
* bare registry codes canonicalize ("fr" -> "fr-FR"), anything else throws
|
|
78
|
+
* `MiniappValidationError` immediately — an invalid language must never
|
|
79
|
+
* become a subscription that silently transcribes the wrong thing
|
|
80
|
+
* (OS-1746 / issue 021).
|
|
81
|
+
*
|
|
82
|
+
* @param language - Registry tag(s), e.g. `"en-US"` or `["en-US", "es-ES"]`.
|
|
69
83
|
* @param handler - Called for every event in any of the listed languages.
|
|
70
84
|
*/
|
|
71
85
|
forLanguage(
|
|
72
|
-
language:
|
|
86
|
+
language: TranscriptionLanguage | TranscriptionLanguage[],
|
|
73
87
|
handler: (data: TranscriptionData) => void,
|
|
74
88
|
options: TranscriptionOptions = {},
|
|
75
89
|
): UnsubscribeFn {
|
|
76
|
-
const langs = Array.isArray(language) ? language : [language]
|
|
90
|
+
const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
|
|
91
|
+
requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
|
|
92
|
+
)
|
|
77
93
|
if (langs.length === 0) return () => {}
|
|
78
94
|
|
|
79
95
|
const unsubs: UnsubscribeFn[] = []
|
|
@@ -100,15 +116,29 @@ export class TranscriptionModule {
|
|
|
100
116
|
|
|
101
117
|
/**
|
|
102
118
|
* Apply transcription configuration (language hints, custom vocabulary,
|
|
103
|
-
* diarisation toggle).
|
|
104
|
-
*
|
|
119
|
+
* diarisation toggle). Hints are validated against the registry and
|
|
120
|
+
* normalized to bare codes ("en-US" -> "en"); unknown values throw
|
|
121
|
+
* `MiniappValidationError` synchronously.
|
|
122
|
+
*
|
|
123
|
+
* Sent to the phone runtime as a REQUEST (not a one-shot) so the returned
|
|
124
|
+
* promise rejects if the runtime cannot apply it — a config that the
|
|
125
|
+
* platform ignores must be loud, not silent (issue 021 S3). Cached locally
|
|
126
|
+
* so the runtime can re-apply on reconnect.
|
|
105
127
|
*/
|
|
106
|
-
configure(config: TranscriptionConfig): void {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
128
|
+
configure(config: TranscriptionConfig): Promise<void> {
|
|
129
|
+
const normalized: TranscriptionConfig = {
|
|
130
|
+
...config,
|
|
131
|
+
languageHints: config.languageHints?.map((hint) =>
|
|
132
|
+
requireLanguageHint(hint, "transcription.configure(languageHints)"),
|
|
133
|
+
),
|
|
134
|
+
}
|
|
135
|
+
this.currentConfig = {...normalized}
|
|
136
|
+
return this.session
|
|
137
|
+
.sendRequest<void>({
|
|
138
|
+
type: MiniappRequestType.TRANSCRIPTION_CONFIG,
|
|
139
|
+
config: {...normalized},
|
|
140
|
+
})
|
|
141
|
+
.then(() => undefined)
|
|
112
142
|
}
|
|
113
143
|
|
|
114
144
|
/** Tear down every transcription subscription this module owns. */
|
|
@@ -20,9 +20,17 @@
|
|
|
20
20
|
import {MiniappStreamType} from "../protocol"
|
|
21
21
|
import {MiniappSession} from "../session"
|
|
22
22
|
import type {TranslationData, UnsubscribeFn} from "./events"
|
|
23
|
+
import {requireTranscriptionLanguage} from "./languages"
|
|
24
|
+
import type {TranscriptionLanguage} from "./languages"
|
|
23
25
|
|
|
24
26
|
export type TranslationHandler = (data: TranslationData) => void
|
|
25
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Translation source: a registry language, or "auto" to let the cloud detect
|
|
30
|
+
* the spoken language.
|
|
31
|
+
*/
|
|
32
|
+
export type TranslationSource = TranscriptionLanguage | "auto"
|
|
33
|
+
|
|
26
34
|
export class TranslationModule {
|
|
27
35
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
28
36
|
|
|
@@ -51,8 +59,10 @@ export class TranslationModule {
|
|
|
51
59
|
* regardless of source. Useful for "I'm a Spanish speaker, translate
|
|
52
60
|
* whatever I hear to Spanish."
|
|
53
61
|
*/
|
|
54
|
-
to(target:
|
|
55
|
-
const targets = Array.isArray(target) ? target : [target]
|
|
62
|
+
to(target: TranscriptionLanguage | TranscriptionLanguage[], handler: TranslationHandler): UnsubscribeFn {
|
|
63
|
+
const targets = (Array.isArray(target) ? target : [target]).map((t) =>
|
|
64
|
+
requireTranscriptionLanguage(t, "translation.to(target)"),
|
|
65
|
+
)
|
|
56
66
|
if (targets.length === 0) return () => {}
|
|
57
67
|
const unsubs: UnsubscribeFn[] = []
|
|
58
68
|
for (const t of targets) {
|
|
@@ -81,17 +91,21 @@ export class TranslationModule {
|
|
|
81
91
|
* from the same source.
|
|
82
92
|
*/
|
|
83
93
|
fromTo(
|
|
84
|
-
source:
|
|
85
|
-
target:
|
|
94
|
+
source: TranslationSource,
|
|
95
|
+
target: TranscriptionLanguage | TranscriptionLanguage[],
|
|
86
96
|
handler: TranslationHandler,
|
|
87
97
|
): UnsubscribeFn {
|
|
88
|
-
const
|
|
98
|
+
const validSource =
|
|
99
|
+
source === "auto" ? "auto" : requireTranscriptionLanguage(source, "translation.fromTo(source)")
|
|
100
|
+
const targets = (Array.isArray(target) ? target : [target]).map((t) =>
|
|
101
|
+
requireTranscriptionLanguage(t, "translation.fromTo(target)"),
|
|
102
|
+
)
|
|
89
103
|
if (targets.length === 0) return () => {}
|
|
90
104
|
const unsubs: UnsubscribeFn[] = []
|
|
91
105
|
for (const t of targets) {
|
|
92
106
|
unsubs.push(
|
|
93
107
|
this.session._subscribe(
|
|
94
|
-
`${MiniappStreamType.TRANSLATION}:${
|
|
108
|
+
`${MiniappStreamType.TRANSLATION}:${validSource}:${t}`,
|
|
95
109
|
handler as (data: unknown) => void,
|
|
96
110
|
),
|
|
97
111
|
)
|
|
@@ -113,8 +127,8 @@ export class TranslationModule {
|
|
|
113
127
|
* SDK v3 parity. This alias will be removed in a future release.
|
|
114
128
|
*/
|
|
115
129
|
forLanguagePair(
|
|
116
|
-
fromLang:
|
|
117
|
-
toLang:
|
|
130
|
+
fromLang: TranslationSource,
|
|
131
|
+
toLang: TranscriptionLanguage,
|
|
118
132
|
handler: TranslationHandler,
|
|
119
133
|
): UnsubscribeFn {
|
|
120
134
|
return this.fromTo(fromLang, toLang, handler)
|
package/src/protocol.ts
CHANGED
|
@@ -117,6 +117,17 @@ export enum MiniappRequestType {
|
|
|
117
117
|
*/
|
|
118
118
|
IMU_SET_ENABLED = "miniapp_imu_set_enabled",
|
|
119
119
|
|
|
120
|
+
/** Explicitly enable/disable glasses-side VAD (GX8002). Mirrors the app's mic settings toggle. */
|
|
121
|
+
MIC_SET_VAD_ENABLED = "miniapp_mic_set_vad_enabled",
|
|
122
|
+
/** Explicitly enable/disable the center-mic loudness gate ("Barrier"). */
|
|
123
|
+
MIC_SET_LOUDNESS_GATE_ENABLED = "miniapp_mic_set_loudness_gate_enabled",
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Enable or disable Wi-Fi ADB (wireless debugging) on Mentra Live.
|
|
127
|
+
* SYSTEM-only — host rejects with NOT_PERMITTED for non-system callers.
|
|
128
|
+
*/
|
|
129
|
+
SET_WIFI_ADB_STATE = "miniapp_set_wifi_adb_state",
|
|
130
|
+
|
|
120
131
|
/** Share content via the OS share sheet. */
|
|
121
132
|
SHARE = "miniapp_share",
|
|
122
133
|
/** Open a URL in the system browser. */
|
|
@@ -125,6 +136,11 @@ export enum MiniappRequestType {
|
|
|
125
136
|
COPY_CLIPBOARD = "miniapp_copy_clipboard",
|
|
126
137
|
/** Download a file (triggers OS share sheet for save location). */
|
|
127
138
|
DOWNLOAD = "miniapp_download",
|
|
139
|
+
/**
|
|
140
|
+
* Open a phone-camera QR scanner overlay. Host must not unmount the miniapp
|
|
141
|
+
* (clearing foreground fires UI_CLOSE and can hang up live sessions).
|
|
142
|
+
*/
|
|
143
|
+
SCAN_QR = "miniapp_scan_qr",
|
|
128
144
|
|
|
129
145
|
/**
|
|
130
146
|
* Phone-local persistent binary blob storage, scoped to (userId, packageName).
|
|
@@ -343,7 +359,10 @@ export enum MiniappErrorCode {
|
|
|
343
359
|
NOT_CONNECTED = "NOT_CONNECTED",
|
|
344
360
|
|
|
345
361
|
// ----- Inter-miniapp interop -----
|
|
346
|
-
/**
|
|
362
|
+
/**
|
|
363
|
+
* Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
|
|
364
|
+
* setWifiAdbState) reject with this code.
|
|
365
|
+
*/
|
|
347
366
|
NOT_PERMITTED = "NOT_PERMITTED",
|
|
348
367
|
/** Target miniapp is not installed. */
|
|
349
368
|
APP_NOT_FOUND = "APP_NOT_FOUND",
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type {MiniappCapsuleMenuRect} from "../globals"
|
|
2
|
+
|
|
3
|
+
export function getCapsuleHeaderPaddingRight(
|
|
4
|
+
capsuleMenu: MiniappCapsuleMenuRect,
|
|
5
|
+
viewportWidth: number,
|
|
6
|
+
safeAreaRight: number,
|
|
7
|
+
rightGap: number,
|
|
8
|
+
): number {
|
|
9
|
+
const capsuleOffsetFromContentRight = viewportWidth - capsuleMenu.left - safeAreaRight
|
|
10
|
+
return Math.max(rightGap, capsuleOffsetFromContentRight + rightGap)
|
|
11
|
+
}
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
import type {CSSProperties} from "react"
|
|
17
17
|
|
|
18
|
+
import {getCapsuleHeaderPaddingRight} from "./capsuleHeaderLayout"
|
|
18
19
|
import {useSafeArea} from "./useSafeArea"
|
|
19
20
|
|
|
20
21
|
export interface UseCapsuleHeaderStyleOptions {
|
|
@@ -39,7 +40,11 @@ export function useCapsuleHeaderStyle(
|
|
|
39
40
|
? capsuleMenu.top + capsuleMenu.height / 2 - insets.top
|
|
40
41
|
: fallbackMarginTop + height / 2
|
|
41
42
|
const marginTop = Math.max(0, center - height / 2)
|
|
42
|
-
const paddingRight = capsuleMenu
|
|
43
|
+
const paddingRight = capsuleMenu
|
|
44
|
+
? typeof window === "undefined"
|
|
45
|
+
? capsuleMenu.width + rightGap
|
|
46
|
+
: getCapsuleHeaderPaddingRight(capsuleMenu, window.innerWidth, insets.right, rightGap)
|
|
47
|
+
: leftPadding
|
|
43
48
|
|
|
44
49
|
return {
|
|
45
50
|
display: "flex",
|
package/src/transport/mock.ts
CHANGED
|
@@ -243,8 +243,12 @@ function syntheticDataFor(requestId: string, requestType: string, requestPayload
|
|
|
243
243
|
case MiniappRequestType.COPY_CLIPBOARD:
|
|
244
244
|
case MiniappRequestType.DOWNLOAD:
|
|
245
245
|
case MiniappRequestType.REQUEST_WIFI_SETUP:
|
|
246
|
+
case MiniappRequestType.SET_WIFI_ADB_STATE:
|
|
246
247
|
return {ok: true}
|
|
247
248
|
|
|
249
|
+
case MiniappRequestType.SCAN_QR:
|
|
250
|
+
return {cancelled: true}
|
|
251
|
+
|
|
248
252
|
default:
|
|
249
253
|
return null
|
|
250
254
|
}
|