@mentra/miniapp 0.3.0-dev.0 → 3.1.0-dev.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/background/index.d.ts +7 -4
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/background/register.d.ts +3 -2
- package/dist/background/register.d.ts.map +1 -1
- package/dist/background/register.js.map +1 -1
- package/dist/hardware.d.ts +24 -0
- package/dist/hardware.d.ts.map +1 -0
- package/dist/hardware.js +20 -0
- package/dist/hardware.js.map +1 -0
- package/dist/index.d.ts +8 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/modules/camera.d.ts +50 -4
- package/dist/modules/camera.d.ts.map +1 -1
- package/dist/modules/camera.js +20 -5
- package/dist/modules/camera.js.map +1 -1
- package/dist/modules/events.d.ts +10 -17
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +56 -11
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/glasses.d.ts +8 -0
- package/dist/modules/glasses.d.ts.map +1 -1
- package/dist/modules/glasses.js +13 -0
- package/dist/modules/glasses.js.map +1 -1
- package/dist/modules/languages.d.ts +42 -0
- package/dist/modules/languages.d.ts.map +1 -0
- package/dist/modules/languages.js +57 -0
- package/dist/modules/languages.js.map +1 -0
- package/dist/modules/mic.d.ts +17 -0
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +28 -1
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +2 -0
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/phone.d.ts +38 -4
- package/dist/modules/phone.d.ts.map +1 -1
- package/dist/modules/phone.js +22 -6
- package/dist/modules/phone.js.map +1 -1
- package/dist/modules/speaker.d.ts +12 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +7 -8
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +6 -2
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +1 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/modules/system.d.ts +20 -1
- package/dist/modules/system.d.ts.map +1 -1
- package/dist/modules/system.js +13 -1
- package/dist/modules/system.js.map +1 -1
- package/dist/modules/transcription.d.ts +35 -8
- package/dist/modules/transcription.d.ts.map +1 -1
- package/dist/modules/transcription.js +32 -12
- package/dist/modules/transcription.js.map +1 -1
- package/dist/modules/translation.d.ts +9 -3
- package/dist/modules/translation.d.ts.map +1 -1
- package/dist/modules/translation.js +5 -3
- package/dist/modules/translation.js.map +1 -1
- package/dist/protocol.d.ts +24 -2
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +24 -2
- package/dist/protocol.js.map +1 -1
- package/dist/react/capsuleHeaderLayout.d.ts +3 -0
- package/dist/react/capsuleHeaderLayout.d.ts.map +1 -0
- package/dist/react/capsuleHeaderLayout.js +5 -0
- package/dist/react/capsuleHeaderLayout.js.map +1 -0
- package/dist/react/useCapsuleHeaderStyle.d.ts.map +1 -1
- package/dist/react/useCapsuleHeaderStyle.js +6 -1
- package/dist/react/useCapsuleHeaderStyle.js.map +1 -1
- package/dist/session.d.ts +5 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +7 -3
- package/dist/session.js.map +1 -1
- package/dist/sub-path-types.test-d.d.ts +12 -0
- package/dist/sub-path-types.test-d.d.ts.map +1 -1
- package/dist/sub-path-types.test-d.js.map +1 -1
- package/dist/transport/mock.js +5 -0
- package/dist/transport/mock.js.map +1 -1
- package/package.json +16 -3
- package/src/background/index.ts +18 -4
- package/src/background/register.ts +10 -3
- package/src/hardware.ts +25 -0
- package/src/index.ts +23 -9
- package/src/modules/camera.ts +75 -13
- package/src/modules/events.ts +54 -27
- package/src/modules/glasses.ts +14 -0
- package/src/modules/languages.ts +72 -0
- package/src/modules/mic.ts +30 -1
- package/src/modules/miniapps.ts +2 -0
- package/src/modules/phone.ts +59 -24
- package/src/modules/speaker.ts +21 -2
- package/src/modules/stream.ts +7 -2
- package/src/modules/system.ts +26 -1
- package/src/modules/transcription.ts +63 -16
- package/src/modules/translation.ts +22 -8
- package/src/protocol.ts +29 -3
- package/src/react/capsuleHeaderLayout.ts +11 -0
- package/src/react/useCapsuleHeaderStyle.ts +6 -1
- package/src/session.ts +19 -7
- package/src/sub-path-types.test-d.ts +12 -1
- package/src/transport/mock.ts +7 -0
package/src/index.ts
CHANGED
|
@@ -36,14 +36,21 @@ export type {MentraOSGlobals, MiniappCapsuleMenuRect, MiniappColorScheme, Miniap
|
|
|
36
36
|
|
|
37
37
|
export {MiniappErrorCode, MiniappRequestType, MiniappResponseType, MiniappStreamType} from "./protocol"
|
|
38
38
|
export {CLOUD_STATUS_STREAM} from "./modules/cloud"
|
|
39
|
+
|
|
40
|
+
// Language registry (issue 021): canonical language types + the validation
|
|
41
|
+
// error the typed transcription/translation surfaces throw.
|
|
42
|
+
export {
|
|
43
|
+
MiniappValidationError,
|
|
44
|
+
SUPPORTED_LANGUAGE_HINTS,
|
|
45
|
+
SUPPORTED_TRANSCRIPTION_LANGUAGES,
|
|
46
|
+
isTranscriptionLanguage,
|
|
47
|
+
} from "./modules/languages"
|
|
48
|
+
export type {LanguageHint, TranscriptionLanguage} from "./modules/languages"
|
|
39
49
|
export type {CloudClientAudioTransport, CloudClientConnectionStatus, CloudClientStatus} from "./modules/cloud"
|
|
40
50
|
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
// type) per @mentra/types' Bun-compat convention.
|
|
45
|
-
export {HardwareType, HardwareRequirementLevel} from "@mentra/types"
|
|
46
|
-
export type {HardwareRequirement} from "@mentra/types"
|
|
51
|
+
// Canonical miniapp manifest hardware contract.
|
|
52
|
+
export {HardwareType, HardwareRequirementLevel} from "./hardware"
|
|
53
|
+
export type {HardwareRequirement} from "./hardware"
|
|
47
54
|
|
|
48
55
|
// Transports — exported for advanced uses (forced transport injection, tests)
|
|
49
56
|
export {createTransport} from "./transport/auto"
|
|
@@ -71,7 +78,6 @@ export type {
|
|
|
71
78
|
AudioChunkData,
|
|
72
79
|
BatteryData,
|
|
73
80
|
ButtonPressData,
|
|
74
|
-
CalendarEventData,
|
|
75
81
|
ConnectionData,
|
|
76
82
|
HeadingData,
|
|
77
83
|
HeadPositionData,
|
|
@@ -85,6 +91,7 @@ export type {
|
|
|
85
91
|
VadData,
|
|
86
92
|
WifiData,
|
|
87
93
|
} from "./modules/events"
|
|
94
|
+
export type {CalendarEvent, CalendarListOptions, CalendarListResult} from "./modules/phone"
|
|
88
95
|
export type {
|
|
89
96
|
PlayAudioOptions,
|
|
90
97
|
SpeakOptions,
|
|
@@ -120,7 +127,14 @@ export type {
|
|
|
120
127
|
StreamStatus,
|
|
121
128
|
StreamVideoConfig,
|
|
122
129
|
} from "./modules/stream"
|
|
123
|
-
export type {
|
|
130
|
+
export type {
|
|
131
|
+
ShareOptions,
|
|
132
|
+
ShareResult,
|
|
133
|
+
DownloadOptions,
|
|
134
|
+
DownloadResult,
|
|
135
|
+
ScanQrOptions,
|
|
136
|
+
ScanQrResult,
|
|
137
|
+
} from "./modules/system"
|
|
124
138
|
export type {MiniappInfo, MiniappActionInfo, MiniappCompatibility, ListMiniappsOptions} from "./modules/miniapps"
|
|
125
139
|
export type {ActionContext, ActionHandler, InvokeOptions} from "./modules/actions"
|
|
126
140
|
|
|
@@ -163,7 +177,7 @@ export type {
|
|
|
163
177
|
} from "./modules/navigation"
|
|
164
178
|
export type {PermissionsModule, PermissionErrorEvent} from "./modules/permissions"
|
|
165
179
|
export type {PhoneModule, PhoneNotificationsModule, PhoneCalendarModule} from "./modules/phone"
|
|
166
|
-
export type {TranscriptionModule, TranscriptionConfig} from "./modules/transcription"
|
|
180
|
+
export type {TranscriptionModule, TranscriptionConfig, TranscriptionOptions} from "./modules/transcription"
|
|
167
181
|
export type {TranslationModule} from "./modules/translation"
|
|
168
182
|
export type {SpeakerModule} from "./modules/speaker"
|
|
169
183
|
|
package/src/modules/camera.ts
CHANGED
|
@@ -33,6 +33,11 @@ export interface TakePhotoOptions {
|
|
|
33
33
|
size?: "low" | "medium" | "high" | "max"
|
|
34
34
|
/** Capture at maximum source quality and optimize BLE delivery for readable text. */
|
|
35
35
|
mode?: "photo" | "text"
|
|
36
|
+
/**
|
|
37
|
+
* Image delivery path. `auto` tries direct Wi-Fi upload and falls back to
|
|
38
|
+
* BLE; `direct` disables BLE fallback; `ble` always relays through the phone.
|
|
39
|
+
*/
|
|
40
|
+
transferMethod?: "auto" | "direct" | "ble"
|
|
36
41
|
compress?: "none" | "low" | "medium" | "high"
|
|
37
42
|
sound?: boolean
|
|
38
43
|
saveToGallery?: boolean
|
|
@@ -42,6 +47,38 @@ export interface TakePhotoOptions {
|
|
|
42
47
|
* manual exposure; ignored otherwise.
|
|
43
48
|
*/
|
|
44
49
|
exposureTimeNs?: number
|
|
50
|
+
/** Sensor ISO for this capture only. Only used when `exposureTimeNs` enables manual exposure. */
|
|
51
|
+
iso?: number | null
|
|
52
|
+
/** After AE convergence, divide the metered exposure by this factor (scan mode). */
|
|
53
|
+
aeExposureDivisor?: number
|
|
54
|
+
/** Cap ISO after AE metering (scan mode). */
|
|
55
|
+
isoCap?: number
|
|
56
|
+
/** Request per-capture noise reduction. Sent on the wire; glasses may log `not_implemented`. */
|
|
57
|
+
noiseReduction?: boolean
|
|
58
|
+
/** Request per-capture edge enhancement. Sent on the wire; glasses may log `not_implemented`. */
|
|
59
|
+
edgeEnhancement?: boolean
|
|
60
|
+
/**
|
|
61
|
+
* ZSL buffering. Manual and scan stills force it off because fixed sensor controls take priority.
|
|
62
|
+
*/
|
|
63
|
+
zsl?: boolean
|
|
64
|
+
/**
|
|
65
|
+
* MFNR still capture. Manual and scan stills force it off because fixed sensor controls take priority.
|
|
66
|
+
*/
|
|
67
|
+
mfnr?: boolean
|
|
68
|
+
/** ISP digital gain hint. */
|
|
69
|
+
ispDigitalGain?: number
|
|
70
|
+
/** ISP analog gain hint. */
|
|
71
|
+
ispAnalogGain?: string
|
|
72
|
+
/**
|
|
73
|
+
* Override the SDK's default 60s request timeout for this capture only.
|
|
74
|
+
* Leave unset for a normal user-triggered capture (the full 60s ceiling is
|
|
75
|
+
* appropriate there). Callers that speculatively fire a capture and may
|
|
76
|
+
* abandon it if the result isn't needed (e.g. a short-lived non-visual
|
|
77
|
+
* assistant turn racing a capture against a ~1s latency cap) should pass a
|
|
78
|
+
* short value here so the abandoned `takePhoto()` promise settles quickly
|
|
79
|
+
* instead of sitting on the default 60s ceiling.
|
|
80
|
+
*/
|
|
81
|
+
timeoutMs?: number
|
|
45
82
|
}
|
|
46
83
|
|
|
47
84
|
export interface PhotoTaken {
|
|
@@ -54,8 +91,15 @@ export interface PhotoTaken {
|
|
|
54
91
|
|
|
55
92
|
export interface WarmUpCameraOptions {
|
|
56
93
|
size?: "low" | "medium" | "high" | "max"
|
|
94
|
+
/** Match the upcoming capture mode so warm-up and capture share one ASG camera session. */
|
|
95
|
+
mode?: "photo" | "text"
|
|
57
96
|
exposureTimeNs?: number
|
|
97
|
+
/** Ready-state hold in milliseconds. Defaults to 15 seconds and is capped at 60 seconds. */
|
|
58
98
|
durationMs?: number
|
|
99
|
+
/** ZSL preview buffering for the warm-up session. */
|
|
100
|
+
zsl?: boolean
|
|
101
|
+
/** MFNR still capture for the warm-up session. */
|
|
102
|
+
mfnr?: boolean
|
|
59
103
|
}
|
|
60
104
|
|
|
61
105
|
export interface StartVideoRecordingOptions {
|
|
@@ -100,10 +144,11 @@ export class CameraModule {
|
|
|
100
144
|
}
|
|
101
145
|
|
|
102
146
|
/**
|
|
103
|
-
* Apply camera FOV/ROI
|
|
147
|
+
* Apply a session-owned camera FOV/ROI override on the glasses.
|
|
104
148
|
*
|
|
105
149
|
* Resolves after the ASG client reports that the setting was applied to camera
|
|
106
|
-
* hardware after the restart cooldown.
|
|
150
|
+
* hardware after the restart cooldown. The host restores the prior override or persistent base
|
|
151
|
+
* when this miniapp closes. Requires CAMERA permission declared in miniapp.json.
|
|
107
152
|
*/
|
|
108
153
|
async setFov(request: CameraFovRequest): Promise<CameraFovResult> {
|
|
109
154
|
return this.session.sendRequest<CameraFovResult>({
|
|
@@ -122,23 +167,37 @@ export class CameraModule {
|
|
|
122
167
|
* `session.capabilities.hasCamera` before calling.
|
|
123
168
|
*/
|
|
124
169
|
async takePhoto(options: TakePhotoOptions = {}): Promise<PhotoTaken> {
|
|
125
|
-
return this.session.sendRequest<PhotoTaken>(
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
170
|
+
return this.session.sendRequest<PhotoTaken>(
|
|
171
|
+
{
|
|
172
|
+
type: MiniappRequestType.PHOTO,
|
|
173
|
+
size: options.size ?? "medium",
|
|
174
|
+
mode: options.mode ?? "photo",
|
|
175
|
+
...(options.transferMethod !== undefined ? {transferMethod: options.transferMethod} : {}),
|
|
176
|
+
compress: options.compress ?? "none",
|
|
177
|
+
sound: options.sound ?? true,
|
|
178
|
+
saveToGallery: options.saveToGallery ?? false,
|
|
179
|
+
exposureTimeNs: options.exposureTimeNs,
|
|
180
|
+
iso: options.iso,
|
|
181
|
+
aeExposureDivisor: options.aeExposureDivisor,
|
|
182
|
+
isoCap: options.isoCap,
|
|
183
|
+
noiseReduction: options.noiseReduction,
|
|
184
|
+
edgeEnhancement: options.edgeEnhancement,
|
|
185
|
+
zsl: options.zsl,
|
|
186
|
+
mfnr: options.mfnr,
|
|
187
|
+
ispDigitalGain: options.ispDigitalGain,
|
|
188
|
+
ispAnalogGain: options.ispAnalogGain,
|
|
189
|
+
},
|
|
190
|
+
options.timeoutMs != null ? {timeoutMs: options.timeoutMs} : undefined,
|
|
191
|
+
)
|
|
134
192
|
}
|
|
135
193
|
|
|
136
194
|
/**
|
|
137
195
|
* Pre-warm the glasses camera so the next takePhoto() is near-instant.
|
|
138
|
-
* The camera stays warm for ~durationMs (default 15s); call warmUp() again to
|
|
196
|
+
* The camera stays warm for ~durationMs (default 15s, maximum 60s); call warmUp() again to
|
|
139
197
|
* extend it. Warm with the same `size` you'll capture with — a mismatched size
|
|
140
198
|
* forces the camera to reconfigure and loses the speedup. Requires CAMERA
|
|
141
|
-
* permission in miniapp.json. Resolves once the camera reports ready.
|
|
199
|
+
* permission in miniapp.json. Resolves once the camera reports ready. The host automatically
|
|
200
|
+
* releases this miniapp's request-owned lease when the miniapp closes.
|
|
142
201
|
*
|
|
143
202
|
* Warm-ups are serialized: only one runs at a time and none may start while a
|
|
144
203
|
* photo is being captured. If the camera is busy (a capture is in flight, or
|
|
@@ -150,8 +209,11 @@ export class CameraModule {
|
|
|
150
209
|
await this.session.sendRequest<void>({
|
|
151
210
|
type: MiniappRequestType.CAMERA_WARM_UP,
|
|
152
211
|
size: options.size ?? "medium",
|
|
212
|
+
mode: options.mode ?? "photo",
|
|
153
213
|
exposureTimeNs: options.exposureTimeNs,
|
|
154
214
|
durationMs: options.durationMs ?? 15000,
|
|
215
|
+
zsl: options.zsl,
|
|
216
|
+
mfnr: options.mfnr,
|
|
155
217
|
})
|
|
156
218
|
}
|
|
157
219
|
|
package/src/modules/events.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* - session.imu.onHeadPosition(...)
|
|
10
10
|
* - session.location.onUpdate(...)
|
|
11
11
|
* - session.glasses.onBattery(...) / onConnection(...)
|
|
12
|
-
* - session.phone.
|
|
12
|
+
* - session.phone.notifications.on(...) / session.phone.onBattery(...)
|
|
13
13
|
*
|
|
14
14
|
* This module's only public method is `subscribe(rawStreamType, handler)` —
|
|
15
15
|
* a forward-compat escape hatch for new event types not yet wrapped on a
|
|
@@ -35,6 +35,9 @@ import {MiniappSession} from "../session"
|
|
|
35
35
|
|
|
36
36
|
export type UnsubscribeFn = () => void
|
|
37
37
|
|
|
38
|
+
/** @internal Host-selected delivery target for a transcription event. */
|
|
39
|
+
export type TranscriptionEventRoute = "default" | "forceLocal" | "all"
|
|
40
|
+
|
|
38
41
|
// ---------------------------------------------------------------------------
|
|
39
42
|
// Shared event data shapes — re-exported by index.ts so consumers can type
|
|
40
43
|
// their handlers without importing this internal module.
|
|
@@ -135,20 +138,6 @@ export interface NotificationDismissedData {
|
|
|
135
138
|
timestamp: number
|
|
136
139
|
}
|
|
137
140
|
|
|
138
|
-
export interface CalendarEventData {
|
|
139
|
-
eventId: string
|
|
140
|
-
title: string
|
|
141
|
-
/** ISO 8601 start time. */
|
|
142
|
-
dtStart: string
|
|
143
|
-
/** ISO 8601 end time. */
|
|
144
|
-
dtEnd: string
|
|
145
|
-
timezone: string
|
|
146
|
-
allDay: boolean
|
|
147
|
-
location: string
|
|
148
|
-
notes: string
|
|
149
|
-
calendarId: string
|
|
150
|
-
}
|
|
151
|
-
|
|
152
141
|
export interface VadData {
|
|
153
142
|
/** True while the user is speaking (voice detected), false when silent. */
|
|
154
143
|
status: boolean
|
|
@@ -172,8 +161,10 @@ export interface AudioChunkData {
|
|
|
172
161
|
|
|
173
162
|
export class EventManager {
|
|
174
163
|
private readonly emitter = new EventEmitter()
|
|
175
|
-
/** Stream -> ref count. Outbound SUBSCRIBE
|
|
164
|
+
/** Stream -> ref count. Outbound SUBSCRIBE tracks active streams and routing changes. */
|
|
176
165
|
private readonly refCounts = new Map<string, number>()
|
|
166
|
+
/** Transcription stream -> refs that require on-device-only processing. */
|
|
167
|
+
private readonly forceLocalRefCounts = new Map<string, number>()
|
|
177
168
|
|
|
178
169
|
constructor(private readonly session: MiniappSession) {}
|
|
179
170
|
|
|
@@ -189,26 +180,51 @@ export class EventManager {
|
|
|
189
180
|
* via a separate envelope `type`; the EventManager only routes them
|
|
190
181
|
* locally.
|
|
191
182
|
*/
|
|
192
|
-
subscribe(stream: string, handler: (data: unknown) => void): UnsubscribeFn {
|
|
193
|
-
this.emitter.on(stream, handler)
|
|
183
|
+
subscribe(stream: string, handler: (data: unknown) => void, options: {forceLocal?: boolean} = {}): UnsubscribeFn {
|
|
194
184
|
const isInternal = stream.startsWith("_")
|
|
185
|
+
const forceLocal = options.forceLocal === true && stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)
|
|
186
|
+
const listener = (data: unknown, route?: TranscriptionEventRoute) => {
|
|
187
|
+
if (stream.startsWith(`${MiniappStreamType.TRANSCRIPTION}:`)) {
|
|
188
|
+
// Hosts that predate routed transcription events omit `route`. Treat
|
|
189
|
+
// those events as the default/cloud path so a forceLocal listener can
|
|
190
|
+
// never accidentally receive cloud text.
|
|
191
|
+
if (forceLocal ? route !== "forceLocal" && route !== "all" : route === "forceLocal") return
|
|
192
|
+
}
|
|
193
|
+
handler(data)
|
|
194
|
+
}
|
|
195
|
+
this.emitter.on(stream, listener)
|
|
195
196
|
if (!isInternal) {
|
|
196
197
|
const before = this.refCounts.get(stream) ?? 0
|
|
197
198
|
this.refCounts.set(stream, before + 1)
|
|
198
|
-
|
|
199
|
+
const forceLocalBefore = this.forceLocalRefCounts.get(stream) ?? 0
|
|
200
|
+
const defaultBefore = before - forceLocalBefore
|
|
201
|
+
if (forceLocal) this.forceLocalRefCounts.set(stream, forceLocalBefore + 1)
|
|
202
|
+
if (before === 0 || (forceLocal && forceLocalBefore === 0) || (!forceLocal && defaultBefore === 0)) {
|
|
199
203
|
this.sendSubscriptionUpdate()
|
|
200
204
|
}
|
|
201
205
|
}
|
|
202
206
|
return () => {
|
|
203
|
-
this.emitter.off(stream,
|
|
207
|
+
this.emitter.off(stream, listener)
|
|
204
208
|
if (isInternal) return
|
|
205
209
|
const current = this.refCounts.get(stream) ?? 0
|
|
210
|
+
const forceLocalCurrent = this.forceLocalRefCounts.get(stream) ?? 0
|
|
211
|
+
let subscriptionChanged = current <= 1
|
|
206
212
|
if (current <= 1) {
|
|
207
213
|
this.refCounts.delete(stream)
|
|
208
|
-
this.sendSubscriptionUpdate()
|
|
209
214
|
} else {
|
|
210
215
|
this.refCounts.set(stream, current - 1)
|
|
211
216
|
}
|
|
217
|
+
if (forceLocal) {
|
|
218
|
+
if (forceLocalCurrent <= 1) {
|
|
219
|
+
this.forceLocalRefCounts.delete(stream)
|
|
220
|
+
subscriptionChanged = true
|
|
221
|
+
} else {
|
|
222
|
+
this.forceLocalRefCounts.set(stream, forceLocalCurrent - 1)
|
|
223
|
+
}
|
|
224
|
+
} else if (current - forceLocalCurrent <= 1) {
|
|
225
|
+
subscriptionChanged = true
|
|
226
|
+
}
|
|
227
|
+
if (subscriptionChanged) this.sendSubscriptionUpdate()
|
|
212
228
|
}
|
|
213
229
|
}
|
|
214
230
|
|
|
@@ -216,6 +232,7 @@ export class EventManager {
|
|
|
216
232
|
unsubscribeAll(): void {
|
|
217
233
|
this.emitter.removeAllListeners()
|
|
218
234
|
this.refCounts.clear()
|
|
235
|
+
this.forceLocalRefCounts.clear()
|
|
219
236
|
this.sendSubscriptionUpdate()
|
|
220
237
|
}
|
|
221
238
|
|
|
@@ -224,8 +241,8 @@ export class EventManager {
|
|
|
224
241
|
// -------------------------------------------------------------------------
|
|
225
242
|
|
|
226
243
|
/** @internal */
|
|
227
|
-
_forwardEvent(stream: string, data: unknown): void {
|
|
228
|
-
this.emitter.emit(stream, data)
|
|
244
|
+
_forwardEvent(stream: string, data: unknown, transcriptionRoute?: TranscriptionEventRoute): void {
|
|
245
|
+
this.emitter.emit(stream, data, transcriptionRoute)
|
|
229
246
|
|
|
230
247
|
// Wildcard fan-out: handlers register under wildcard patterns
|
|
231
248
|
// ("transcription:auto", "translation:*:<target>", …) but the host
|
|
@@ -235,7 +252,7 @@ export class EventManager {
|
|
|
235
252
|
// LocalMiniappRuntime.forwardEvent matches on the host side —
|
|
236
253
|
// otherwise a wildcard subscriber never fires.
|
|
237
254
|
if (stream.startsWith("transcription:") && stream !== "transcription:auto") {
|
|
238
|
-
this.emitter.emit("transcription:auto", data)
|
|
255
|
+
this.emitter.emit("transcription:auto", data, transcriptionRoute)
|
|
239
256
|
} else if (stream.startsWith("translation:")) {
|
|
240
257
|
const parts = stream.split(":")
|
|
241
258
|
const patterns = new Set<string>(["translation:auto"])
|
|
@@ -253,9 +270,19 @@ export class EventManager {
|
|
|
253
270
|
}
|
|
254
271
|
|
|
255
272
|
private sendSubscriptionUpdate(): void {
|
|
256
|
-
const subscriptions = Array.from(this.refCounts.keys()).map((stream) =>
|
|
257
|
-
stream === MiniappStreamType.LOCATION_UPDATE
|
|
258
|
-
|
|
273
|
+
const subscriptions = Array.from(this.refCounts.keys()).map((stream) => {
|
|
274
|
+
if (stream === MiniappStreamType.LOCATION_UPDATE) return {stream: "location_stream", rate: "realtime"}
|
|
275
|
+
|
|
276
|
+
const forceLocalRefs = this.forceLocalRefCounts.get(stream) ?? 0
|
|
277
|
+
if (forceLocalRefs === 0) return stream
|
|
278
|
+
|
|
279
|
+
const totalRefs = this.refCounts.get(stream) ?? 0
|
|
280
|
+
return {
|
|
281
|
+
stream,
|
|
282
|
+
forceLocal: true,
|
|
283
|
+
...(totalRefs > forceLocalRefs ? {includeCloud: true} : {}),
|
|
284
|
+
}
|
|
285
|
+
})
|
|
259
286
|
this.session.sendOneShot({
|
|
260
287
|
type: MiniappRequestType.SUBSCRIBE,
|
|
261
288
|
subscriptions,
|
package/src/modules/glasses.ts
CHANGED
|
@@ -41,4 +41,18 @@ export class GlassesModule {
|
|
|
41
41
|
reason,
|
|
42
42
|
})
|
|
43
43
|
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Enable or disable Wi-Fi ADB (wireless debugging) on Mentra Live.
|
|
47
|
+
* Persisted on the glasses; boot applies the saved preference (default off).
|
|
48
|
+
*
|
|
49
|
+
* SYSTEM-only — rejects with `NOT_PERMITTED` unless this miniapp is a system
|
|
50
|
+
* app. Third-party miniapps must not expose wireless debugging controls.
|
|
51
|
+
*/
|
|
52
|
+
async setWifiAdbState(enabled: boolean): Promise<void> {
|
|
53
|
+
await this.session.sendRequest<void>({
|
|
54
|
+
type: MiniappRequestType.SET_WIFI_ADB_STATE,
|
|
55
|
+
enabled,
|
|
56
|
+
})
|
|
57
|
+
}
|
|
44
58
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Language validation at the SDK boundary (issue 021, WP2).
|
|
3
|
+
*
|
|
4
|
+
* The registry itself lives in `@mentra/cloud-protocol/languages` (single
|
|
5
|
+
* source of truth shared with the cloud). This module re-exports the types so
|
|
6
|
+
* miniapps import everything from `@mentra/miniapp`, and adds the throwing
|
|
7
|
+
* validators the typed module surfaces (`transcription.*`, `translation.*`)
|
|
8
|
+
* call on every language parameter.
|
|
9
|
+
*
|
|
10
|
+
* Contract (design doc docs/issues/021-typed-language-subscriptions):
|
|
11
|
+
* - Valid by construction: params are typed as literal unions, so TS users
|
|
12
|
+
* get autocomplete and compile errors.
|
|
13
|
+
* - Loud at runtime: JS users (or `any` casts) get a thrown
|
|
14
|
+
* `MiniappValidationError` with a suggestion, never silent fallback.
|
|
15
|
+
* - Canonical on the wire: bare codes ("fr") canonicalize to their default
|
|
16
|
+
* tag ("fr-FR") before becoming stream keys, so the phone/cloud only ever
|
|
17
|
+
* see registry tags.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import {
|
|
21
|
+
isTranscriptionLanguage,
|
|
22
|
+
suggestTranscriptionLanguage,
|
|
23
|
+
toLanguageHint,
|
|
24
|
+
toTranscriptionLanguage,
|
|
25
|
+
SUPPORTED_LANGUAGE_HINTS,
|
|
26
|
+
SUPPORTED_TRANSCRIPTION_LANGUAGES,
|
|
27
|
+
} from "@mentra/cloud-protocol/languages"
|
|
28
|
+
import type {LanguageHint, TranscriptionLanguage} from "@mentra/cloud-protocol/languages"
|
|
29
|
+
|
|
30
|
+
export {SUPPORTED_LANGUAGE_HINTS, SUPPORTED_TRANSCRIPTION_LANGUAGES, isTranscriptionLanguage}
|
|
31
|
+
export type {LanguageHint, TranscriptionLanguage}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Thrown synchronously by SDK methods when a parameter fails validation.
|
|
35
|
+
* Deliberately a hard throw (not a warning, not a silent default): an invalid
|
|
36
|
+
* language previously produced a subscription that silently transcribed the
|
|
37
|
+
* wrong language or silently never spoke (OS-1746).
|
|
38
|
+
*/
|
|
39
|
+
export class MiniappValidationError extends Error {
|
|
40
|
+
constructor(message: string) {
|
|
41
|
+
super(message)
|
|
42
|
+
this.name = "MiniappValidationError"
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Canonicalize a subscription language or throw. Accepts registry tags
|
|
48
|
+
* ("fr-FR") and bare registry codes ("fr" -> "fr-FR").
|
|
49
|
+
*/
|
|
50
|
+
export function requireTranscriptionLanguage(value: string, param: string): TranscriptionLanguage {
|
|
51
|
+
const canonical = toTranscriptionLanguage(value)
|
|
52
|
+
if (canonical) return canonical
|
|
53
|
+
const suggestion = suggestTranscriptionLanguage(value)
|
|
54
|
+
throw new MiniappValidationError(
|
|
55
|
+
`${param}: unknown language ${JSON.stringify(value)}` +
|
|
56
|
+
(suggestion ? ` — did you mean ${JSON.stringify(suggestion)}?` : "") +
|
|
57
|
+
` Supported values are BCP-47 tags from SUPPORTED_TRANSCRIPTION_LANGUAGES (e.g. "en-US", "fr-FR").`,
|
|
58
|
+
)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Normalize a language hint to the bare code the STT provider accepts, or
|
|
63
|
+
* throw. Accepts bare codes ("fr") and registry tags ("fr-FR" -> "fr").
|
|
64
|
+
*/
|
|
65
|
+
export function requireLanguageHint(value: string, param: string): LanguageHint {
|
|
66
|
+
const hint = toLanguageHint(value)
|
|
67
|
+
if (hint) return hint
|
|
68
|
+
throw new MiniappValidationError(
|
|
69
|
+
`${param}: unknown language hint ${JSON.stringify(value)}. ` +
|
|
70
|
+
`Hints are bare ISO 639-1 codes from SUPPORTED_LANGUAGE_HINTS (e.g. "en", "fr").`,
|
|
71
|
+
)
|
|
72
|
+
}
|
package/src/modules/mic.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* PERMISSION_NOT_DECLARED otherwise.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {MiniappStreamType} from "../protocol"
|
|
18
|
+
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
19
19
|
import {MiniappSession} from "../session"
|
|
20
20
|
import type {AudioChunkData, UnsubscribeFn, VadData} from "./events"
|
|
21
21
|
|
|
@@ -41,6 +41,35 @@ export class MicModule {
|
|
|
41
41
|
return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Temporarily override glasses-side voice activity detection (GX8002) for
|
|
46
|
+
* this miniapp's lifetime. When disabled, mic gating falls back to the
|
|
47
|
+
* loudness gate only (if it is enabled). The Mentra App's configured value
|
|
48
|
+
* is restored when this miniapp disconnects.
|
|
49
|
+
*
|
|
50
|
+
* Requires `MICROPHONE` in the miniapp manifest.
|
|
51
|
+
*/
|
|
52
|
+
setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void> {
|
|
53
|
+
return this.session.sendRequest<void>({
|
|
54
|
+
type: MiniappRequestType.MIC_SET_VAD_ENABLED,
|
|
55
|
+
enabled,
|
|
56
|
+
})
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Temporarily override the center-mic loudness gate ("Barrier") for this
|
|
61
|
+
* miniapp's lifetime. It blocks quiet/self-talk audio independent of VAD.
|
|
62
|
+
* The Mentra App's configured value is restored when this miniapp disconnects.
|
|
63
|
+
*
|
|
64
|
+
* Requires `MICROPHONE` in the miniapp manifest.
|
|
65
|
+
*/
|
|
66
|
+
setLoudnessGateEnabled(enabled: boolean): Promise<void> {
|
|
67
|
+
return this.session.sendRequest<void>({
|
|
68
|
+
type: MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED,
|
|
69
|
+
enabled,
|
|
70
|
+
})
|
|
71
|
+
}
|
|
72
|
+
|
|
44
73
|
/**
|
|
45
74
|
* Tear down every subscription this module owns. Useful when a component
|
|
46
75
|
* is unmounting and wants to free everything at once without tracking
|
package/src/modules/miniapps.ts
CHANGED
|
@@ -20,6 +20,8 @@ export interface MiniappActionInfo {
|
|
|
20
20
|
* to an MCP tool `inputSchema`. Undefined for actions that take no params.
|
|
21
21
|
*/
|
|
22
22
|
parameters?: Record<string, unknown>
|
|
23
|
+
/** JSON-Schema descriptor for the structured value returned by the action. */
|
|
24
|
+
outputSchema?: Record<string, unknown>
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
/**
|
package/src/modules/phone.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Mirrors cloud SDK v3's PhoneManager structure. Sub-namespaced by concern:
|
|
5
5
|
*
|
|
6
6
|
* session.phone.notifications.{on, hasPermission, stop}
|
|
7
|
-
* session.phone.calendar.{
|
|
7
|
+
* session.phone.calendar.{listEvents, hasPermission}
|
|
8
8
|
* session.phone.onBattery(...) // stays flat
|
|
9
9
|
*
|
|
10
10
|
* Imperative phone-OS calls (share, openUrl, copyToClipboard, download) live
|
|
@@ -16,15 +16,9 @@
|
|
|
16
16
|
* subscribing on iOS is a no-op even though the API is present.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import {MiniappStreamType} from "../protocol"
|
|
19
|
+
import {MiniappRequestType, MiniappStreamType} from "../protocol"
|
|
20
20
|
import {MiniappSession} from "../session"
|
|
21
|
-
import type {
|
|
22
|
-
BatteryData,
|
|
23
|
-
CalendarEventData,
|
|
24
|
-
NotificationDismissedData,
|
|
25
|
-
PhoneNotificationData,
|
|
26
|
-
UnsubscribeFn,
|
|
27
|
-
} from "./events"
|
|
21
|
+
import type {BatteryData, NotificationDismissedData, PhoneNotificationData, UnsubscribeFn} from "./events"
|
|
28
22
|
|
|
29
23
|
class TrackedSubs {
|
|
30
24
|
private readonly unsubs = new Set<UnsubscribeFn>()
|
|
@@ -55,9 +49,7 @@ export class PhoneNotificationsModule extends TrackedSubs {
|
|
|
55
49
|
}
|
|
56
50
|
|
|
57
51
|
on(handler: (data: PhoneNotificationData) => void): UnsubscribeFn {
|
|
58
|
-
return this.track(
|
|
59
|
-
this.session._subscribe(MiniappStreamType.PHONE_NOTIFICATION, handler as (data: unknown) => void),
|
|
60
|
-
)
|
|
52
|
+
return this.track(this.session._subscribe(MiniappStreamType.PHONE_NOTIFICATION, handler as (data: unknown) => void))
|
|
61
53
|
}
|
|
62
54
|
|
|
63
55
|
/**
|
|
@@ -71,10 +63,7 @@ export class PhoneNotificationsModule extends TrackedSubs {
|
|
|
71
63
|
*/
|
|
72
64
|
onDismissed(handler: (data: NotificationDismissedData) => void): UnsubscribeFn {
|
|
73
65
|
return this.track(
|
|
74
|
-
this.session._subscribe(
|
|
75
|
-
MiniappStreamType.PHONE_NOTIFICATION_DISMISSED,
|
|
76
|
-
handler as (data: unknown) => void,
|
|
77
|
-
),
|
|
66
|
+
this.session._subscribe(MiniappStreamType.PHONE_NOTIFICATION_DISMISSED, handler as (data: unknown) => void),
|
|
78
67
|
)
|
|
79
68
|
}
|
|
80
69
|
|
|
@@ -84,15 +73,55 @@ export class PhoneNotificationsModule extends TrackedSubs {
|
|
|
84
73
|
}
|
|
85
74
|
}
|
|
86
75
|
|
|
87
|
-
export
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
76
|
+
export interface CalendarListOptions {
|
|
77
|
+
/** Inclusive start of the query window. */
|
|
78
|
+
startsAt: string | Date
|
|
79
|
+
/** Exclusive end of the query window. Must be after startsAt. */
|
|
80
|
+
endsAt: string | Date
|
|
81
|
+
/** Maximum events to return. Defaults to 50 and may not exceed 100. */
|
|
82
|
+
limit?: number
|
|
83
|
+
}
|
|
91
84
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
85
|
+
export interface CalendarEvent {
|
|
86
|
+
/** Stable occurrence id, unique across calendars and recurring instances. */
|
|
87
|
+
id: string
|
|
88
|
+
calendarId: string
|
|
89
|
+
title: string
|
|
90
|
+
/** ISO 8601 start time. All-day events preserve their calendar timezone offset. */
|
|
91
|
+
startsAt: string
|
|
92
|
+
/** ISO 8601 end time. All-day events preserve their calendar timezone offset. */
|
|
93
|
+
endsAt: string
|
|
94
|
+
timezone?: string
|
|
95
|
+
allDay: boolean
|
|
96
|
+
location?: string
|
|
97
|
+
notes?: string
|
|
98
|
+
url?: string
|
|
99
|
+
/** Deduplicated HTTPS links found in the event URL, location, and notes. */
|
|
100
|
+
links: string[]
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface CalendarListResult {
|
|
104
|
+
events: CalendarEvent[]
|
|
105
|
+
/** True when more matching events existed than the requested limit. */
|
|
106
|
+
truncated: boolean
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export class PhoneCalendarModule {
|
|
110
|
+
constructor(private readonly session: MiniappSession) {}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Read calendar events in a bounded window. CALENDAR must be declared in
|
|
114
|
+
* miniapp.json and granted before the miniapp opens.
|
|
115
|
+
*/
|
|
116
|
+
listEvents(options: CalendarListOptions): Promise<CalendarListResult> {
|
|
117
|
+
const startsAt = normalizeCalendarDate(options?.startsAt, "startsAt")
|
|
118
|
+
const endsAt = normalizeCalendarDate(options?.endsAt, "endsAt")
|
|
119
|
+
return this.session.sendRequest<CalendarListResult>({
|
|
120
|
+
type: MiniappRequestType.CALENDAR_LIST_EVENTS,
|
|
121
|
+
startsAt,
|
|
122
|
+
endsAt,
|
|
123
|
+
...(options.limit === undefined ? {} : {limit: options.limit}),
|
|
124
|
+
})
|
|
96
125
|
}
|
|
97
126
|
|
|
98
127
|
/** True iff `CALENDAR` is declared in the miniapp's manifest. */
|
|
@@ -101,6 +130,12 @@ export class PhoneCalendarModule extends TrackedSubs {
|
|
|
101
130
|
}
|
|
102
131
|
}
|
|
103
132
|
|
|
133
|
+
function normalizeCalendarDate(value: string | Date | undefined, field: string): string {
|
|
134
|
+
const date = value instanceof Date ? value : new Date(value ?? "")
|
|
135
|
+
if (Number.isNaN(date.getTime())) throw new TypeError(`${field} must be a valid Date or ISO 8601 string`)
|
|
136
|
+
return date.toISOString()
|
|
137
|
+
}
|
|
138
|
+
|
|
104
139
|
export class PhoneModule {
|
|
105
140
|
public readonly notifications: PhoneNotificationsModule
|
|
106
141
|
public readonly calendar: PhoneCalendarModule
|