@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.942
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/background/index.d.ts +6 -3
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/background/register.d.ts +6 -0
- package/dist/background/register.d.ts.map +1 -1
- package/dist/background/register.js +19 -4
- package/dist/background/register.js.map +1 -1
- package/dist/index.d.ts +7 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/display.d.ts +26 -0
- package/dist/modules/display.d.ts.map +1 -1
- package/dist/modules/display.js +1 -0
- package/dist/modules/display.js.map +1 -1
- package/dist/modules/events.d.ts +8 -4
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +46 -6
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/meeting.d.ts +129 -0
- package/dist/modules/meeting.d.ts.map +1 -1
- package/dist/modules/meeting.js +200 -14
- package/dist/modules/meeting.js.map +1 -1
- package/dist/modules/mic.d.ts +5 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +5 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +101 -4
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +59 -4
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +2 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +5 -0
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +46 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +107 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/protocol.d.ts +86 -1
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +82 -0
- package/dist/protocol.js.map +1 -1
- package/dist/react/StreamPreview.d.ts +41 -0
- package/dist/react/StreamPreview.d.ts.map +1 -0
- package/dist/react/StreamPreview.js +144 -0
- package/dist/react/StreamPreview.js.map +1 -0
- package/dist/react/index.d.ts +3 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
- package/dist/react/stream-preview/PreviewConnection.js +493 -0
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
- package/dist/react/stream-preview/controlChannel.d.ts +75 -0
- package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
- package/dist/react/stream-preview/controlChannel.js +55 -0
- package/dist/react/stream-preview/controlChannel.js.map +1 -0
- package/dist/react/stream-preview/creditLoop.d.ts +81 -0
- package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
- package/dist/react/stream-preview/creditLoop.js +146 -0
- package/dist/react/stream-preview/creditLoop.js.map +1 -0
- package/dist/react/stream-preview/protocol.d.ts +113 -0
- package/dist/react/stream-preview/protocol.d.ts.map +1 -0
- package/dist/react/stream-preview/protocol.js +148 -0
- package/dist/react/stream-preview/protocol.js.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.js +71 -0
- package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
- package/dist/react/stream-preview/testFrames.d.ts +38 -0
- package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
- package/dist/react/stream-preview/testFrames.js +50 -0
- package/dist/react/stream-preview/testFrames.js.map +1 -0
- package/dist/react/stream-preview/tiers.d.ts +21 -0
- package/dist/react/stream-preview/tiers.d.ts.map +1 -0
- package/dist/react/stream-preview/tiers.js +33 -0
- package/dist/react/stream-preview/tiers.js.map +1 -0
- package/dist/react/stream-preview/trace.d.ts +23 -0
- package/dist/react/stream-preview/trace.d.ts.map +1 -0
- package/dist/react/stream-preview/trace.js +71 -0
- package/dist/react/stream-preview/trace.js.map +1 -0
- package/dist/react/stream-preview/transport.d.ts +79 -0
- package/dist/react/stream-preview/transport.d.ts.map +1 -0
- package/dist/react/stream-preview/transport.js +209 -0
- package/dist/react/stream-preview/transport.js.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.js +398 -0
- package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
- package/dist/react/useHistoryState.d.ts +35 -0
- package/dist/react/useHistoryState.d.ts.map +1 -0
- package/dist/react/useHistoryState.js +117 -0
- package/dist/react/useHistoryState.js.map +1 -0
- package/dist/session.d.ts +25 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +61 -1
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +3 -0
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js +2 -0
- package/dist/ui/index.js.map +1 -1
- package/package.json +8 -3
- package/src/background/index.ts +28 -1
- package/src/background/register.ts +28 -6
- package/src/index.ts +29 -1
- package/src/modules/display.ts +29 -0
- package/src/modules/events.ts +51 -6
- package/src/modules/meeting.ts +292 -15
- package/src/modules/mic.ts +5 -4
- package/src/modules/miniapps.ts +145 -4
- package/src/modules/speaker.ts +10 -4
- package/src/modules/stream.ts +153 -1
- package/src/protocol.ts +125 -0
- package/src/react/StreamPreview.tsx +187 -0
- package/src/react/index.ts +3 -0
- package/src/react/stream-preview/PreviewConnection.ts +587 -0
- package/src/react/stream-preview/controlChannel.ts +119 -0
- package/src/react/stream-preview/creditLoop.ts +189 -0
- package/src/react/stream-preview/protocol.ts +227 -0
- package/src/react/stream-preview/resizeCoalescer.ts +88 -0
- package/src/react/stream-preview/testFrames.ts +86 -0
- package/src/react/stream-preview/tiers.ts +37 -0
- package/src/react/stream-preview/trace.ts +81 -0
- package/src/react/stream-preview/transport.ts +242 -0
- package/src/react/stream-preview/yuvRenderer.ts +453 -0
- package/src/react/useHistoryState.ts +135 -0
- package/src/session.ts +76 -0
- package/src/ui/index.ts +3 -0
package/src/modules/meeting.ts
CHANGED
|
@@ -108,6 +108,21 @@ export interface MeetingJoinOptions {
|
|
|
108
108
|
origin?: MeetingOrigin
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
export interface MeetingCreateOptions {
|
|
112
|
+
provider: MeetingProvider
|
|
113
|
+
subject?: string
|
|
114
|
+
durationMinutes?: number
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export interface CreatedMeeting {
|
|
118
|
+
provider: MeetingProvider
|
|
119
|
+
joinUrl: string
|
|
120
|
+
/** Opaque ownership receipt. Keep it to retire this meeting later. */
|
|
121
|
+
meetingRef: string
|
|
122
|
+
identityMode: MeetingIdentityMode
|
|
123
|
+
guestReason?: "no-entra-identity" | "teams-license-unavailable"
|
|
124
|
+
}
|
|
125
|
+
|
|
111
126
|
export type MeetingParticipantState = "idle" | "connecting" | "connected" | "lobby" | "hold" | "disconnected"
|
|
112
127
|
|
|
113
128
|
/** A remote participant as reported by the phone-native meeting client. */
|
|
@@ -123,12 +138,27 @@ export interface MeetingParticipant {
|
|
|
123
138
|
export type MeetingIdentityMode = "guest" | "teams-user"
|
|
124
139
|
export type MeetingGuestReason = "no-entra-identity" | "teams-license-unavailable" | "legacy-credential"
|
|
125
140
|
|
|
141
|
+
/**
|
|
142
|
+
* Identity resolved before joining. `teams-user` means Microsoft accepted the
|
|
143
|
+
* Teams calling license, not that the account owns a particular M365 SKU or
|
|
144
|
+
* has permission to organize every meeting. Credentials never cross this API.
|
|
145
|
+
* Acquisition/consent/provider errors reject instead of reporting guest mode.
|
|
146
|
+
*/
|
|
147
|
+
export interface MeetingIdentity {
|
|
148
|
+
identityMode: MeetingIdentityMode
|
|
149
|
+
guestReason?: MeetingGuestReason
|
|
150
|
+
/** Present when the host has a signed-in Entra account, including unlicensed accounts. */
|
|
151
|
+
account?: {displayName?: string; email?: string}
|
|
152
|
+
}
|
|
153
|
+
|
|
126
154
|
/** Host policy; this contains no credentials or deployment endpoints. */
|
|
127
155
|
export interface MeetingConfiguration {
|
|
128
156
|
enabled: boolean
|
|
129
157
|
credentialSource: "runtime" | "miniapp"
|
|
130
158
|
externalBackendAllowed: boolean
|
|
131
159
|
managedStreams: boolean
|
|
160
|
+
/** Omitted by older hosts. Runtime creation requires Graph organizer configuration. */
|
|
161
|
+
creationSource?: "runtime" | "miniapp"
|
|
132
162
|
}
|
|
133
163
|
|
|
134
164
|
export interface MeetingState {
|
|
@@ -137,6 +167,11 @@ export interface MeetingState {
|
|
|
137
167
|
guestReason?: MeetingGuestReason
|
|
138
168
|
state: MeetingPhase
|
|
139
169
|
muted: boolean
|
|
170
|
+
/**
|
|
171
|
+
* Whether remote participants receive the glasses camera. False after `setVideoEnabled(false)`;
|
|
172
|
+
* every new join starts true. Omitted by hosts that cannot toggle outgoing video.
|
|
173
|
+
*/
|
|
174
|
+
videoEnabled?: boolean
|
|
140
175
|
error?: string
|
|
141
176
|
/** Provider termination details, including Teams' invalid meeting-link codes. */
|
|
142
177
|
endReason?: MeetingEndReason
|
|
@@ -225,12 +260,16 @@ export function parseMeetingCapabilities(raw: unknown): MeetingCapabilities | un
|
|
|
225
260
|
if (!value || typeof value !== "object") return undefined
|
|
226
261
|
const capability = value as Record<string, unknown>
|
|
227
262
|
const rawLobby = (raw as Record<string, unknown>).manageLobby
|
|
228
|
-
const lobby = rawLobby && typeof rawLobby === "object" ? rawLobby as Record<string, unknown> : undefined
|
|
263
|
+
const lobby = rawLobby && typeof rawLobby === "object" ? (rawLobby as Record<string, unknown>) : undefined
|
|
229
264
|
return {
|
|
230
|
-
...(lobby
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
265
|
+
...(lobby
|
|
266
|
+
? {
|
|
267
|
+
manageLobby: {
|
|
268
|
+
allowed: typeof lobby.allowed === "boolean" ? lobby.allowed : null,
|
|
269
|
+
reason: typeof lobby.reason === "string" && lobby.reason ? lobby.reason : null,
|
|
270
|
+
},
|
|
271
|
+
}
|
|
272
|
+
: {}),
|
|
234
273
|
hangUpForEveryone: {
|
|
235
274
|
allowed: typeof capability.allowed === "boolean" ? capability.allowed : null,
|
|
236
275
|
reason: typeof capability.reason === "string" && capability.reason ? capability.reason : null,
|
|
@@ -277,14 +316,7 @@ export interface MeetingSoftApRecovery {
|
|
|
277
316
|
|
|
278
317
|
const SOFTAP_STEPS: ReadonlySet<string> = new Set(["hotspot", "scopedJoin", "acsJoin", "publish", "live"])
|
|
279
318
|
const SOFTAP_STEP_STATUSES: ReadonlySet<string> = new Set(["pending", "running", "done", "failed"])
|
|
280
|
-
const SOFTAP_PHASES: ReadonlySet<string> = new Set([
|
|
281
|
-
"idle",
|
|
282
|
-
"starting",
|
|
283
|
-
"recovering",
|
|
284
|
-
"live",
|
|
285
|
-
"stopping",
|
|
286
|
-
"failed",
|
|
287
|
-
])
|
|
319
|
+
const SOFTAP_PHASES: ReadonlySet<string> = new Set(["idle", "starting", "recovering", "live", "stopping", "failed"])
|
|
288
320
|
|
|
289
321
|
/** Tolerant parse of a host `softap` payload. Unknown steps are dropped; a malformed payload reads as absent. */
|
|
290
322
|
export function parseMeetingSoftApProgress(raw: unknown): MeetingSoftApProgress | undefined {
|
|
@@ -367,6 +399,81 @@ export function parseMeetingParticipants(raw: unknown): MeetingParticipant[] | u
|
|
|
367
399
|
|
|
368
400
|
export type MeetingStateHandler = (state: MeetingState) => void
|
|
369
401
|
|
|
402
|
+
/** The host resumed a photo pause because it hit the ceiling. */
|
|
403
|
+
export interface MeetingVideoPublisherEvent {
|
|
404
|
+
pauseId: string
|
|
405
|
+
status: "expired"
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Why [MeetingModule.captureStill] failed. `unsupported` means this host, build, or transport
|
|
410
|
+
* cannot do it at all (older Mentra App, iOS today, a non-Direct-link call): use the
|
|
411
|
+
* pause-based path instead. Everything else is a failed attempt of a supported one.
|
|
412
|
+
*/
|
|
413
|
+
export type MeetingStillFailure =
|
|
414
|
+
| "unsupported"
|
|
415
|
+
| "not_live"
|
|
416
|
+
| "busy"
|
|
417
|
+
| "glasses_rejected"
|
|
418
|
+
| "upload_timeout"
|
|
419
|
+
| "hold_failed"
|
|
420
|
+
| "cancelled"
|
|
421
|
+
|
|
422
|
+
/** Rejection of [MeetingModule.captureStill]. */
|
|
423
|
+
export interface MeetingStillError {
|
|
424
|
+
code: string
|
|
425
|
+
message: string
|
|
426
|
+
reason: MeetingStillFailure
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** `card` on the tile, glasses `uploading` the capture, still `shown` on the tile. */
|
|
430
|
+
export type MeetingStillPhase = "card" | "uploading" | "shown"
|
|
431
|
+
|
|
432
|
+
/** @internal Push correlated to one [MeetingModule.captureStill] call by `stillId`. */
|
|
433
|
+
export interface MeetingStillProgressEvent {
|
|
434
|
+
stillId: string
|
|
435
|
+
phase: MeetingStillPhase
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
let stillSeq = 0
|
|
439
|
+
|
|
440
|
+
function mintStillId(): string {
|
|
441
|
+
stillSeq = (stillSeq + 1) % 0xffff
|
|
442
|
+
return `still-${Date.now().toString(36)}-${stillSeq.toString(36)}-${Math.random().toString(36).slice(2, 8)}`
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
export interface MeetingStillResult {
|
|
446
|
+
ok: true
|
|
447
|
+
requestId: string
|
|
448
|
+
/** Size of the JPEG the glasses delivered. */
|
|
449
|
+
bytes: number
|
|
450
|
+
/** Host clock when the first still frame reached the Teams sender. */
|
|
451
|
+
shownAt: number
|
|
452
|
+
timings: {cardMs: number; stillMs: number; heldMs: number; totalMs: number}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const STILL_FAILURES: ReadonlySet<string> = new Set<MeetingStillFailure>([
|
|
456
|
+
"unsupported",
|
|
457
|
+
"not_live",
|
|
458
|
+
"busy",
|
|
459
|
+
"glasses_rejected",
|
|
460
|
+
"upload_timeout",
|
|
461
|
+
"hold_failed",
|
|
462
|
+
"cancelled",
|
|
463
|
+
])
|
|
464
|
+
|
|
465
|
+
function toStillError(error: unknown): MeetingStillError {
|
|
466
|
+
const raw = (error && typeof error === "object" ? error : {}) as {code?: unknown; message?: unknown; reason?: unknown}
|
|
467
|
+
const code = typeof raw.code === "string" ? raw.code : MiniappErrorCode.INTERNAL
|
|
468
|
+
const message = typeof raw.message === "string" ? raw.message : String(error)
|
|
469
|
+
let reason: MeetingStillFailure
|
|
470
|
+
if (typeof raw.reason === "string" && STILL_FAILURES.has(raw.reason)) reason = raw.reason as MeetingStillFailure
|
|
471
|
+
else if (code === MiniappErrorCode.NOT_IMPLEMENTED) reason = "unsupported"
|
|
472
|
+
else if (code === MiniappErrorCode.REQUEST_ABORTED) reason = "cancelled"
|
|
473
|
+
else reason = "glasses_rejected"
|
|
474
|
+
return {code, message, reason}
|
|
475
|
+
}
|
|
476
|
+
|
|
370
477
|
function isMiniappRequestError(error: unknown): error is MiniappRequestError {
|
|
371
478
|
return Boolean(error && typeof error === "object" && "code" in error)
|
|
372
479
|
}
|
|
@@ -426,6 +533,47 @@ export class MeetingModule {
|
|
|
426
533
|
return this.session.sendRequest<MeetingConfiguration>({type: MiniappRequestType.MEETING_GET_CONFIGURATION})
|
|
427
534
|
}
|
|
428
535
|
|
|
536
|
+
/** Resolve the current calling identity without creating or joining a meeting. */
|
|
537
|
+
async getIdentity(): Promise<MeetingIdentity> {
|
|
538
|
+
try {
|
|
539
|
+
return await this.session.sendRequest<MeetingIdentity>(
|
|
540
|
+
{type: MiniappRequestType.MEETING_GET_IDENTITY},
|
|
541
|
+
{timeoutMs: 30_000},
|
|
542
|
+
)
|
|
543
|
+
} catch (error) {
|
|
544
|
+
mapHostError(error)
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** Create a Teams meeting in the selected Runtime without exposing provider credentials. */
|
|
549
|
+
async create(options: MeetingCreateOptions): Promise<CreatedMeeting> {
|
|
550
|
+
if (options.provider !== "acs-teams") {
|
|
551
|
+
throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: "Unsupported meeting provider"}
|
|
552
|
+
}
|
|
553
|
+
try {
|
|
554
|
+
return await this.session.sendRequest<CreatedMeeting>(
|
|
555
|
+
{
|
|
556
|
+
type: MiniappRequestType.MEETING_CREATE,
|
|
557
|
+
provider: options.provider,
|
|
558
|
+
subject: options.subject,
|
|
559
|
+
durationMinutes: options.durationMinutes,
|
|
560
|
+
},
|
|
561
|
+
{timeoutMs: 0},
|
|
562
|
+
)
|
|
563
|
+
} catch (error) {
|
|
564
|
+
mapHostError(error)
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** Delete a meeting created by this caller. Does not hang up an active call. */
|
|
569
|
+
async retire(meetingRef: string): Promise<void> {
|
|
570
|
+
try {
|
|
571
|
+
await this.session.sendRequest<void>({type: MiniappRequestType.MEETING_RETIRE, meetingRef}, {timeoutMs: 0})
|
|
572
|
+
} catch (error) {
|
|
573
|
+
mapHostError(error)
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
429
577
|
async leave(): Promise<void> {
|
|
430
578
|
try {
|
|
431
579
|
await this.session.sendRequest<void>({type: MiniappRequestType.MEETING_LEAVE})
|
|
@@ -434,6 +582,105 @@ export class MeetingModule {
|
|
|
434
582
|
}
|
|
435
583
|
}
|
|
436
584
|
|
|
585
|
+
/**
|
|
586
|
+
* Stop the glasses camera publisher without leaving the meeting.
|
|
587
|
+
* On Direct link this stops only `start_stream`. Cloudflare callers pause their own stream.
|
|
588
|
+
*/
|
|
589
|
+
async pauseVideoPublisher(options: {reason: "photo"}): Promise<{pauseId: string}> {
|
|
590
|
+
try {
|
|
591
|
+
return await this.session.sendRequest<{pauseId: string}>({
|
|
592
|
+
type: MiniappRequestType.MEETING_PAUSE_VIDEO_PUBLISHER,
|
|
593
|
+
reason: options.reason,
|
|
594
|
+
})
|
|
595
|
+
} catch (error) {
|
|
596
|
+
mapHostError(error)
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/** A stale `pauseId` is a no-op. Leave, End, and a new join cancel the pause themselves. */
|
|
601
|
+
async resumeVideoPublisher(pauseId: string): Promise<void> {
|
|
602
|
+
try {
|
|
603
|
+
await this.session.sendRequest<void>({
|
|
604
|
+
type: MiniappRequestType.MEETING_RESUME_VIDEO_PUBLISHER,
|
|
605
|
+
pauseId,
|
|
606
|
+
})
|
|
607
|
+
} catch (error) {
|
|
608
|
+
mapHostError(error)
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/** Resolves when the first card frame has been handed to the Teams sender. */
|
|
613
|
+
async showCard(options: {kind: "taking-photo"}): Promise<void> {
|
|
614
|
+
try {
|
|
615
|
+
await this.session.sendRequest<void>({
|
|
616
|
+
type: MiniappRequestType.MEETING_SHOW_CARD,
|
|
617
|
+
kind: options.kind,
|
|
618
|
+
})
|
|
619
|
+
} catch (error) {
|
|
620
|
+
mapHostError(error)
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Replace the card with a still. Resolves when the first still frame is handed to the sender.
|
|
626
|
+
* `durationMs` is the caller's clock, starting at `shownAt`.
|
|
627
|
+
*/
|
|
628
|
+
async showImage(options: {imageUrl: string; durationMs: number}): Promise<{shownAt: number}> {
|
|
629
|
+
try {
|
|
630
|
+
return await this.session.sendRequest<{shownAt: number}>({
|
|
631
|
+
type: MiniappRequestType.MEETING_SHOW_IMAGE,
|
|
632
|
+
imageUrl: options.imageUrl,
|
|
633
|
+
durationMs: options.durationMs,
|
|
634
|
+
})
|
|
635
|
+
} catch (error) {
|
|
636
|
+
mapHostError(error)
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** The caller's `durationMs` has elapsed. Glasses frames may reach the tile again. */
|
|
641
|
+
async showLive(): Promise<void> {
|
|
642
|
+
try {
|
|
643
|
+
await this.session.sendRequest<void>({type: MiniappRequestType.MEETING_SHOW_LIVE})
|
|
644
|
+
} catch (error) {
|
|
645
|
+
mapHostError(error)
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Share one full-size glasses photo on the outgoing video of a Direct link call, without
|
|
651
|
+
* stopping the glasses camera stream. The host shows the "Taking a photo..." card, has the glasses
|
|
652
|
+
* shoot and upload straight to the phone, holds the still for `durationMs`, then returns to live
|
|
653
|
+
* video. Do not combine with [pauseVideoPublisher] or [showCard]/[showImage]; this replaces them.
|
|
654
|
+
*
|
|
655
|
+
* `onProgress` reports each phase as the host reaches it, so a caller can say "taking",
|
|
656
|
+
* "sharing" and "on the call" at the right moments; the promise itself settles after the hold.
|
|
657
|
+
*
|
|
658
|
+
* Rejects with a [MeetingStillError]. `reason: "unsupported"` means use the pause-based path.
|
|
659
|
+
*/
|
|
660
|
+
async captureStill(options: {
|
|
661
|
+
durationMs: number
|
|
662
|
+
onProgress?: (phase: MeetingStillPhase) => void
|
|
663
|
+
}): Promise<MeetingStillResult> {
|
|
664
|
+
const stillId = mintStillId()
|
|
665
|
+
const onProgress = options.onProgress
|
|
666
|
+
const unsubscribe = onProgress
|
|
667
|
+
? this.session.on("meetingStill", (event) => {
|
|
668
|
+
if (event.stillId === stillId) onProgress(event.phase)
|
|
669
|
+
})
|
|
670
|
+
: undefined
|
|
671
|
+
try {
|
|
672
|
+
return await this.session.sendRequest<MeetingStillResult>(
|
|
673
|
+
{type: MiniappRequestType.MEETING_CAPTURE_STILL, durationMs: options.durationMs, stillId},
|
|
674
|
+
// Capture, upload, and the hold itself; the host bounds each step.
|
|
675
|
+
{timeoutMs: options.durationMs + 75_000},
|
|
676
|
+
)
|
|
677
|
+
} catch (error) {
|
|
678
|
+
throw toStillError(error)
|
|
679
|
+
} finally {
|
|
680
|
+
unsubscribe?.()
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
437
684
|
/**
|
|
438
685
|
* End the meeting for everyone, then leave.
|
|
439
686
|
*
|
|
@@ -475,6 +722,23 @@ export class MeetingModule {
|
|
|
475
722
|
}
|
|
476
723
|
}
|
|
477
724
|
|
|
725
|
+
/**
|
|
726
|
+
* Stop or resume the glasses camera the meeting receives. The call and the glasses stream stay
|
|
727
|
+
* up, so a local `<StreamPreview>` keeps drawing while remote participants see the camera off.
|
|
728
|
+
* Check `state.videoEnabled !== undefined` before offering it; older hosts reject.
|
|
729
|
+
*/
|
|
730
|
+
async setVideoEnabled(enabled: boolean): Promise<void> {
|
|
731
|
+
try {
|
|
732
|
+
const result = await this.session.sendRequest<MeetingState | null>({
|
|
733
|
+
type: MiniappRequestType.MEETING_SET_VIDEO_ENABLED,
|
|
734
|
+
enabled,
|
|
735
|
+
})
|
|
736
|
+
if (result) this._applyState(result)
|
|
737
|
+
} catch (error) {
|
|
738
|
+
mapHostError(error)
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
478
742
|
/**
|
|
479
743
|
* Repoint the host at a new WHEP URL mid-call, the recovery path for a re-published stream.
|
|
480
744
|
*
|
|
@@ -515,14 +779,27 @@ export class MeetingModule {
|
|
|
515
779
|
return this.session.on("meetingState", handler)
|
|
516
780
|
}
|
|
517
781
|
|
|
782
|
+
/** Fired when the host's pause ceiling resumes the publisher. A resume this caller sent is silent. */
|
|
783
|
+
onVideoPublisher(handler: (event: MeetingVideoPublisherEvent) => void): UnsubscribeFn {
|
|
784
|
+
return this.session.on("meetingVideoPublisher", handler)
|
|
785
|
+
}
|
|
786
|
+
|
|
518
787
|
/** @internal — applied by MiniappSession on inbound MEETING_STATE. */
|
|
519
788
|
_applyState(event: MeetingState): void {
|
|
520
789
|
this._state = {
|
|
521
790
|
identityMode:
|
|
522
|
-
event.identityMode === "guest" || event.identityMode === "teams-user"
|
|
523
|
-
|
|
791
|
+
event.identityMode === "guest" || event.identityMode === "teams-user"
|
|
792
|
+
? event.identityMode
|
|
793
|
+
: this._state.identityMode,
|
|
794
|
+
guestReason:
|
|
795
|
+
event.identityMode === "guest"
|
|
796
|
+
? event.guestReason
|
|
797
|
+
: event.identityMode === "teams-user"
|
|
798
|
+
? undefined
|
|
799
|
+
: this._state.guestReason,
|
|
524
800
|
state: event.state,
|
|
525
801
|
muted: Boolean(event.muted),
|
|
802
|
+
videoEnabled: typeof event.videoEnabled === "boolean" ? event.videoEnabled : undefined,
|
|
526
803
|
error: event.error,
|
|
527
804
|
endReason: parseMeetingEndReason(event.endReason),
|
|
528
805
|
meetingUrl: event.meetingUrl,
|
package/src/modules/mic.ts
CHANGED
|
@@ -54,10 +54,11 @@ export class MicModule {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
57
|
+
* Request glasses-side voice activity detection (GX8002) for this miniapp's
|
|
58
|
+
* lifetime. Enabling is subject to the user's "Allow voice activity detection"
|
|
59
|
+
* setting and continuous-audio requirements from other consumers. When disabled,
|
|
60
|
+
* audio frames continue; the independent loudness gate can still silence quiet
|
|
61
|
+
* input. The request is released when this miniapp disconnects.
|
|
61
62
|
*
|
|
62
63
|
* Requires `MICROPHONE` in the miniapp manifest.
|
|
63
64
|
*/
|
package/src/modules/miniapps.ts
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview MiniappsModule — `session.miniapps`.
|
|
3
3
|
*
|
|
4
|
-
* Discover and control the lifecycle of OTHER miniapps: list, start,
|
|
5
|
-
* SYSTEM-only — every call rejects with NOT_PERMITTED unless the caller
|
|
6
|
-
* system app (Mentra AI is the first consumer). The action *capability*
|
|
7
|
-
* (invoke / handle) lives on `session.actions`, not here — these
|
|
4
|
+
* Discover and control the lifecycle of OTHER miniapps: list, start, open,
|
|
5
|
+
* stop. SYSTEM-only — every call rejects with NOT_PERMITTED unless the caller
|
|
6
|
+
* is a system app (Mentra AI is the first consumer). The action *capability*
|
|
7
|
+
* layer (invoke / handle) lives on `session.actions`, not here — these are
|
|
8
8
|
* operations on the app as a whole, not on a declared action.
|
|
9
|
+
*
|
|
10
|
+
* Installation (`install`, `checkInstallCompatibility`, `uninstall`) is
|
|
11
|
+
* narrower still: the host accepts it only from a Store miniapp bundled with
|
|
12
|
+
* the Mentra App build. The host downloads, verifies, and activates the
|
|
13
|
+
* release through its own installer; the Store only selects the release.
|
|
9
14
|
*/
|
|
10
15
|
|
|
11
16
|
import {MiniappRequestType} from "../protocol"
|
|
@@ -22,6 +27,11 @@ export interface MiniappActionInfo {
|
|
|
22
27
|
parameters?: Record<string, unknown>
|
|
23
28
|
/** JSON-Schema descriptor for the structured value returned by the action. */
|
|
24
29
|
outputSchema?: Record<string, unknown>
|
|
30
|
+
/**
|
|
31
|
+
* `persistent` (default) starts the target as normal user-visible activity.
|
|
32
|
+
* `transient` wakes it invisibly for the duration of the invocation only.
|
|
33
|
+
*/
|
|
34
|
+
lifecycle: "persistent" | "transient"
|
|
25
35
|
}
|
|
26
36
|
|
|
27
37
|
/**
|
|
@@ -43,6 +53,12 @@ export interface MiniappInfo {
|
|
|
43
53
|
version: string
|
|
44
54
|
running: boolean
|
|
45
55
|
compatibility: MiniappCompatibility
|
|
56
|
+
/** True when this package identity is backed by a ZIP in the Mentra App build. */
|
|
57
|
+
system: boolean
|
|
58
|
+
/** Store package that owns updates of this active release, when it is Store-managed. */
|
|
59
|
+
storeOwnerPackageName?: string
|
|
60
|
+
/** Store the host build assigns to update this bundled package's active release, if any. */
|
|
61
|
+
systemStoreOwnerPackageName?: string
|
|
46
62
|
/** Declared actions (empty if the miniapp declares none). */
|
|
47
63
|
actions: MiniappActionInfo[]
|
|
48
64
|
}
|
|
@@ -52,7 +68,76 @@ export interface ListMiniappsOptions {
|
|
|
52
68
|
includeIncompatible?: boolean
|
|
53
69
|
}
|
|
54
70
|
|
|
71
|
+
export interface MiniappHardwareRequirement {
|
|
72
|
+
type: string
|
|
73
|
+
level: string
|
|
74
|
+
description?: string
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Backend-neutral release request, accepted only from a bundled, host-trusted Store. */
|
|
78
|
+
export interface InstallMiniappRequest {
|
|
79
|
+
packageName: string
|
|
80
|
+
version: string
|
|
81
|
+
bundleUrl: string
|
|
82
|
+
/** SHA-256 of the release ZIP; the host refuses any other bytes. */
|
|
83
|
+
bundleSha256: string
|
|
84
|
+
/**
|
|
85
|
+
* Bearer for `bundleUrl`, when the Store's backend requires one. The host
|
|
86
|
+
* forwards it untouched and keeps no Store address of its own: entitlement is
|
|
87
|
+
* settled between this miniapp and its backend, integrity by the SHA-256 and
|
|
88
|
+
* the bundle's publisher signature.
|
|
89
|
+
*/
|
|
90
|
+
bundleAuthorization?: string
|
|
91
|
+
/**
|
|
92
|
+
* Automatic update: never stop a running miniapp. A running or starting
|
|
93
|
+
* target defers the request with `APP_RUNNING`.
|
|
94
|
+
*/
|
|
95
|
+
onlyIfStopped?: boolean
|
|
96
|
+
/**
|
|
97
|
+
* With `onlyIfStopped`: when the target is running, the host stages the
|
|
98
|
+
* release and asks the user whether to update now. Declining, or a prompt the
|
|
99
|
+
* host does not show (the Mentra App is in the background, another prompt is
|
|
100
|
+
* open, or this miniapp was asked within the last day), still answers
|
|
101
|
+
* `APP_RUNNING`.
|
|
102
|
+
*/
|
|
103
|
+
askIfRunning?: boolean
|
|
104
|
+
/** Minimum Mentra App version declared by the release manifest. */
|
|
105
|
+
minHostVersion?: string
|
|
106
|
+
/** Mentra Miniapp SDK version declared by the release manifest. */
|
|
107
|
+
sdkVersion?: string
|
|
108
|
+
/** Glasses hardware requirements declared by the release manifest. */
|
|
109
|
+
hardwareRequirements?: MiniappHardwareRequirement[]
|
|
110
|
+
releaseId?: string
|
|
111
|
+
channel?: string
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface InstallMiniappResult {
|
|
115
|
+
packageName: string
|
|
116
|
+
version: string
|
|
117
|
+
installedByStore: string
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface InstallMiniappCompatibilityRequest {
|
|
121
|
+
minHostVersion?: string | null
|
|
122
|
+
sdkVersion?: string | null
|
|
123
|
+
hardwareRequirements?: MiniappHardwareRequirement[]
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export type InstallMiniappCompatibility =
|
|
127
|
+
| {compatible: true}
|
|
128
|
+
| {compatible: false; blocker: "host" | "sdk" | "hardware"; reason: string}
|
|
129
|
+
|
|
130
|
+
export type InstallMiniappPhase = "downloading" | "verifying" | "extracting" | "activating" | "complete"
|
|
131
|
+
|
|
132
|
+
export interface InstallMiniappProgress {
|
|
133
|
+
packageName: string
|
|
134
|
+
version: string
|
|
135
|
+
phase: InstallMiniappPhase
|
|
136
|
+
}
|
|
137
|
+
|
|
55
138
|
export class MiniappsModule {
|
|
139
|
+
private readonly progressHandlers = new Set<(progress: InstallMiniappProgress) => void>()
|
|
140
|
+
|
|
56
141
|
constructor(private readonly session: MiniappSession) {}
|
|
57
142
|
|
|
58
143
|
/**
|
|
@@ -82,6 +167,15 @@ export class MiniappsModule {
|
|
|
82
167
|
})
|
|
83
168
|
}
|
|
84
169
|
|
|
170
|
+
/** Start another miniapp and foreground its phone UI. SYSTEM-only. */
|
|
171
|
+
async open(packageName: string): Promise<void> {
|
|
172
|
+
await this.session.sendRequest<void>({
|
|
173
|
+
type: MiniappRequestType.MINIAPPS_START,
|
|
174
|
+
packageName,
|
|
175
|
+
foreground: true,
|
|
176
|
+
})
|
|
177
|
+
}
|
|
178
|
+
|
|
85
179
|
/** Stop another miniapp. SYSTEM-only. */
|
|
86
180
|
async stop(packageName: string): Promise<void> {
|
|
87
181
|
await this.session.sendRequest<void>({
|
|
@@ -89,4 +183,51 @@ export class MiniappsModule {
|
|
|
89
183
|
packageName,
|
|
90
184
|
})
|
|
91
185
|
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Download, verify, and install a release through the phone host. Accepted
|
|
189
|
+
* only from a Store bundled with the Mentra App build; other callers receive
|
|
190
|
+
* NOT_PERMITTED.
|
|
191
|
+
*/
|
|
192
|
+
async install(request: InstallMiniappRequest): Promise<InstallMiniappResult> {
|
|
193
|
+
return this.session.sendRequest<InstallMiniappResult>(
|
|
194
|
+
{
|
|
195
|
+
...request,
|
|
196
|
+
type: MiniappRequestType.MINIAPPS_INSTALL,
|
|
197
|
+
},
|
|
198
|
+
// Downloading, validating, extracting, asking the user and relaunching
|
|
199
|
+
// can legitimately outlast the SDK's default request timeout. The request
|
|
200
|
+
// still settles on the host response or session disconnect.
|
|
201
|
+
{timeoutMs: 0},
|
|
202
|
+
)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Check host, Mentra Miniapp SDK, and glasses compatibility before offering or applying an update. */
|
|
206
|
+
async checkInstallCompatibility(request: InstallMiniappCompatibilityRequest): Promise<InstallMiniappCompatibility> {
|
|
207
|
+
return this.session.sendRequest<InstallMiniappCompatibility>({
|
|
208
|
+
type: MiniappRequestType.MINIAPPS_INSTALL_CHECK,
|
|
209
|
+
minHostVersion: request.minHostVersion ?? undefined,
|
|
210
|
+
sdkVersion: request.sdkVersion ?? undefined,
|
|
211
|
+
hardwareRequirements: request.hardwareRequirements ?? [],
|
|
212
|
+
})
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Subscribe to host-owned install progress. Returns an unsubscribe function. */
|
|
216
|
+
onInstallProgress(handler: (progress: InstallMiniappProgress) => void): () => void {
|
|
217
|
+
this.progressHandlers.add(handler)
|
|
218
|
+
return () => this.progressHandlers.delete(handler)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** @internal */
|
|
222
|
+
_deliverInstallProgress(progress: InstallMiniappProgress): void {
|
|
223
|
+
for (const handler of this.progressHandlers) handler(progress)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Uninstall a package this Store installed. Bundled Stores only. */
|
|
227
|
+
async uninstall(packageName: string): Promise<void> {
|
|
228
|
+
await this.session.sendRequest<void>({
|
|
229
|
+
type: MiniappRequestType.MINIAPPS_UNINSTALL,
|
|
230
|
+
packageName,
|
|
231
|
+
})
|
|
232
|
+
}
|
|
92
233
|
}
|
package/src/modules/speaker.ts
CHANGED
|
@@ -41,6 +41,8 @@ import type {UnsubscribeFn} from "./events"
|
|
|
41
41
|
|
|
42
42
|
export interface PlayAudioOptions {
|
|
43
43
|
audioUrl: string
|
|
44
|
+
/** Start playback at this offset in milliseconds. Defaults to zero. */
|
|
45
|
+
startPositionMs?: number
|
|
44
46
|
volume?: number
|
|
45
47
|
stopOtherAudio?: boolean
|
|
46
48
|
}
|
|
@@ -116,10 +118,7 @@ export class SpeakerStreamWriter {
|
|
|
116
118
|
private settled = false
|
|
117
119
|
private writeChain: Promise<void> = Promise.resolve()
|
|
118
120
|
|
|
119
|
-
constructor(
|
|
120
|
-
private readonly session: MiniappSession,
|
|
121
|
-
readonly streamId: string,
|
|
122
|
-
) {}
|
|
121
|
+
constructor(private readonly session: MiniappSession, readonly streamId: string) {}
|
|
123
122
|
|
|
124
123
|
/**
|
|
125
124
|
* Append raw PCM bytes (16-bit LE). Auto-chunked. Resolves with the host's
|
|
@@ -233,6 +232,12 @@ export class SpeakerModule {
|
|
|
233
232
|
|
|
234
233
|
/** Play a URL. Resolves when playback completes on the phone. */
|
|
235
234
|
async play(options: PlayAudioOptions): Promise<void> {
|
|
235
|
+
if (
|
|
236
|
+
options.startPositionMs !== undefined &&
|
|
237
|
+
(!Number.isFinite(options.startPositionMs) || options.startPositionMs < 0)
|
|
238
|
+
) {
|
|
239
|
+
throw new RangeError("startPositionMs must be a finite, non-negative number")
|
|
240
|
+
}
|
|
236
241
|
// Playback length is unbounded (a clip can run for minutes), and the host
|
|
237
242
|
// only sends its REQUEST_RESULT when playback finishes or is interrupted, so
|
|
238
243
|
// opt out of the default request timeout — otherwise a long clip would reject
|
|
@@ -241,6 +246,7 @@ export class SpeakerModule {
|
|
|
241
246
|
{
|
|
242
247
|
type: MiniappRequestType.PLAY_AUDIO,
|
|
243
248
|
audioUrl: options.audioUrl,
|
|
249
|
+
...(options.startPositionMs !== undefined ? {startPositionMs: options.startPositionMs} : {}),
|
|
244
250
|
volume: options.volume,
|
|
245
251
|
stopOtherAudio: options.stopOtherAudio ?? false,
|
|
246
252
|
},
|