@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.1000
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/background/index.d.ts +6 -3
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/background/register.d.ts +6 -0
- package/dist/background/register.d.ts.map +1 -1
- package/dist/background/register.js +19 -4
- package/dist/background/register.js.map +1 -1
- package/dist/index.d.ts +7 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/display.d.ts +26 -0
- package/dist/modules/display.d.ts.map +1 -1
- package/dist/modules/display.js +1 -0
- package/dist/modules/display.js.map +1 -1
- package/dist/modules/events.d.ts +8 -4
- package/dist/modules/events.d.ts.map +1 -1
- package/dist/modules/events.js +46 -6
- package/dist/modules/events.js.map +1 -1
- package/dist/modules/meeting.d.ts +129 -0
- package/dist/modules/meeting.d.ts.map +1 -1
- package/dist/modules/meeting.js +200 -14
- package/dist/modules/meeting.js.map +1 -1
- package/dist/modules/mic.d.ts +5 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +5 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/miniapps.d.ts +101 -4
- package/dist/modules/miniapps.d.ts.map +1 -1
- package/dist/modules/miniapps.js +59 -4
- package/dist/modules/miniapps.js.map +1 -1
- package/dist/modules/speaker.d.ts +2 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +5 -0
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +46 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +107 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/protocol.d.ts +86 -1
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +82 -0
- package/dist/protocol.js.map +1 -1
- package/dist/react/StreamPreview.d.ts +41 -0
- package/dist/react/StreamPreview.d.ts.map +1 -0
- package/dist/react/StreamPreview.js +144 -0
- package/dist/react/StreamPreview.js.map +1 -0
- package/dist/react/index.d.ts +3 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
- package/dist/react/stream-preview/PreviewConnection.js +493 -0
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
- package/dist/react/stream-preview/controlChannel.d.ts +75 -0
- package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
- package/dist/react/stream-preview/controlChannel.js +55 -0
- package/dist/react/stream-preview/controlChannel.js.map +1 -0
- package/dist/react/stream-preview/creditLoop.d.ts +81 -0
- package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
- package/dist/react/stream-preview/creditLoop.js +146 -0
- package/dist/react/stream-preview/creditLoop.js.map +1 -0
- package/dist/react/stream-preview/protocol.d.ts +113 -0
- package/dist/react/stream-preview/protocol.d.ts.map +1 -0
- package/dist/react/stream-preview/protocol.js +148 -0
- package/dist/react/stream-preview/protocol.js.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.js +71 -0
- package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
- package/dist/react/stream-preview/testFrames.d.ts +38 -0
- package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
- package/dist/react/stream-preview/testFrames.js +50 -0
- package/dist/react/stream-preview/testFrames.js.map +1 -0
- package/dist/react/stream-preview/tiers.d.ts +21 -0
- package/dist/react/stream-preview/tiers.d.ts.map +1 -0
- package/dist/react/stream-preview/tiers.js +33 -0
- package/dist/react/stream-preview/tiers.js.map +1 -0
- package/dist/react/stream-preview/trace.d.ts +23 -0
- package/dist/react/stream-preview/trace.d.ts.map +1 -0
- package/dist/react/stream-preview/trace.js +71 -0
- package/dist/react/stream-preview/trace.js.map +1 -0
- package/dist/react/stream-preview/transport.d.ts +79 -0
- package/dist/react/stream-preview/transport.d.ts.map +1 -0
- package/dist/react/stream-preview/transport.js +209 -0
- package/dist/react/stream-preview/transport.js.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.js +398 -0
- package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
- package/dist/react/useHistoryState.d.ts +35 -0
- package/dist/react/useHistoryState.d.ts.map +1 -0
- package/dist/react/useHistoryState.js +117 -0
- package/dist/react/useHistoryState.js.map +1 -0
- package/dist/session.d.ts +25 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +61 -1
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +3 -0
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js +2 -0
- package/dist/ui/index.js.map +1 -1
- package/package.json +8 -3
- package/src/background/index.ts +28 -1
- package/src/background/register.ts +28 -6
- package/src/index.ts +29 -1
- package/src/modules/display.ts +29 -0
- package/src/modules/events.ts +51 -6
- package/src/modules/meeting.ts +292 -15
- package/src/modules/mic.ts +5 -4
- package/src/modules/miniapps.ts +145 -4
- package/src/modules/speaker.ts +10 -4
- package/src/modules/stream.ts +153 -1
- package/src/protocol.ts +125 -0
- package/src/react/StreamPreview.tsx +187 -0
- package/src/react/index.ts +3 -0
- package/src/react/stream-preview/PreviewConnection.ts +587 -0
- package/src/react/stream-preview/controlChannel.ts +119 -0
- package/src/react/stream-preview/creditLoop.ts +189 -0
- package/src/react/stream-preview/protocol.ts +227 -0
- package/src/react/stream-preview/resizeCoalescer.ts +88 -0
- package/src/react/stream-preview/testFrames.ts +86 -0
- package/src/react/stream-preview/tiers.ts +37 -0
- package/src/react/stream-preview/trace.ts +81 -0
- package/src/react/stream-preview/transport.ts +242 -0
- package/src/react/stream-preview/yuvRenderer.ts +453 -0
- package/src/react/useHistoryState.ts +135 -0
- package/src/session.ts +76 -0
- package/src/ui/index.ts +3 -0
package/src/modules/stream.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* to your own ingest URL with no relay.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {MiniappRequestType} from "../protocol"
|
|
11
|
+
import {MiniappRequestType, type PreviewErrorCode, type PreviewSource} from "../protocol"
|
|
12
12
|
import {MiniappSession} from "../session"
|
|
13
13
|
|
|
14
14
|
export interface StreamVideoConfig {
|
|
@@ -133,9 +133,161 @@ export interface StreamStatus {
|
|
|
133
133
|
stats?: StreamLiveStats
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
export interface StreamPreviewOptions {
|
|
137
|
+
/** Defaults to `"call"`. `"glasses"` is reserved and rejects with `unsupported`. */
|
|
138
|
+
source?: PreviewSource
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export type PreviewHandleState = "held" | "ended"
|
|
142
|
+
|
|
143
|
+
export interface PreviewHandleStatus {
|
|
144
|
+
state: PreviewHandleState
|
|
145
|
+
/** Set when `state` is `"ended"`: `stopped`, `source_ended`, `runtime_stopped`, ... */
|
|
146
|
+
reason?: string
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** A preview request or lease failed. `code` is one of {@link PreviewErrorCode} when the host sent one. */
|
|
150
|
+
export class PreviewError extends Error {
|
|
151
|
+
readonly code: PreviewErrorCode | string
|
|
152
|
+
|
|
153
|
+
constructor(code: PreviewErrorCode | string, message?: string) {
|
|
154
|
+
super(message ?? code)
|
|
155
|
+
this.name = "PreviewError"
|
|
156
|
+
this.code = code
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The background-owned lease on a preview source.
|
|
162
|
+
*
|
|
163
|
+
* The lease is what lets a `<StreamPreview>` in the UI show frames. It survives the UI closing and
|
|
164
|
+
* the WebView being destroyed, and ends on `stop()`, when the source ends (the meeting is over), or
|
|
165
|
+
* when the background runtime stops. Hiding the component does not release it.
|
|
166
|
+
*/
|
|
167
|
+
export interface PreviewHandle {
|
|
168
|
+
readonly source: PreviewSource
|
|
169
|
+
readonly handleId: string
|
|
170
|
+
/** Correlation id stamped on every `PREVIEW_TRACE` log line for this lease. */
|
|
171
|
+
readonly previewTraceId: string
|
|
172
|
+
readonly state: PreviewHandleState
|
|
173
|
+
/** Release the lease. Idempotent. */
|
|
174
|
+
stop(): Promise<void>
|
|
175
|
+
/** Lease transitions. Fires `{state: "ended", reason}` exactly once. */
|
|
176
|
+
onStatus(cb: (status: PreviewHandleStatus) => void): () => void
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
interface PreviewStartResult {
|
|
180
|
+
handleId: string
|
|
181
|
+
previewTraceId: string
|
|
182
|
+
source: PreviewSource
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function toPreviewError(error: unknown): PreviewError {
|
|
186
|
+
if (error instanceof PreviewError) return error
|
|
187
|
+
const value = (error ?? {}) as {code?: unknown; message?: unknown}
|
|
188
|
+
const code = typeof value.code === "string" ? value.code : "unsupported"
|
|
189
|
+
const message = typeof value.message === "string" ? value.message : undefined
|
|
190
|
+
return new PreviewError(code, message)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
class PreviewHandleImpl implements PreviewHandle {
|
|
194
|
+
state: PreviewHandleState = "held"
|
|
195
|
+
private readonly listeners = new Set<(status: PreviewHandleStatus) => void>()
|
|
196
|
+
|
|
197
|
+
constructor(
|
|
198
|
+
readonly source: PreviewSource,
|
|
199
|
+
readonly handleId: string,
|
|
200
|
+
readonly previewTraceId: string,
|
|
201
|
+
private readonly release: (handle: PreviewHandleImpl) => Promise<void>,
|
|
202
|
+
) {}
|
|
203
|
+
|
|
204
|
+
async stop(): Promise<void> {
|
|
205
|
+
if (this.state === "ended") return
|
|
206
|
+
this.end("stopped")
|
|
207
|
+
await this.release(this)
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
onStatus(cb: (status: PreviewHandleStatus) => void): () => void {
|
|
211
|
+
this.listeners.add(cb)
|
|
212
|
+
return () => this.listeners.delete(cb)
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** @internal */
|
|
216
|
+
end(reason: string): void {
|
|
217
|
+
if (this.state === "ended") return
|
|
218
|
+
this.state = "ended"
|
|
219
|
+
this.emit({state: "ended", reason})
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private emit(status: PreviewHandleStatus): void {
|
|
223
|
+
for (const listener of [...this.listeners]) {
|
|
224
|
+
try {
|
|
225
|
+
listener(status)
|
|
226
|
+
} catch {
|
|
227
|
+
// A throwing listener must not stop the others from hearing that the lease ended.
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
136
233
|
export class StreamModule {
|
|
234
|
+
private readonly previewHandles = new Map<string, PreviewHandleImpl>()
|
|
235
|
+
|
|
137
236
|
constructor(private readonly session: MiniappSession) {}
|
|
138
237
|
|
|
238
|
+
/**
|
|
239
|
+
* Take the preview lease on a source so a `<StreamPreview>` in the UI can show its frames.
|
|
240
|
+
*
|
|
241
|
+
* For `"call"` the caller must declare `CAMERA` and own the active meeting (`session.meeting`).
|
|
242
|
+
* Only one lease exists at a time; a second request fails with `preview_busy`. Rejects with a
|
|
243
|
+
* {@link PreviewError}. Never await this on a call's connect path: preview is additive.
|
|
244
|
+
*/
|
|
245
|
+
async preview(options: StreamPreviewOptions = {}): Promise<PreviewHandle> {
|
|
246
|
+
const source = options.source ?? "call"
|
|
247
|
+
if (source !== "call") {
|
|
248
|
+
throw new PreviewError("unsupported", `Preview source "${String(source)}" is not supported yet`)
|
|
249
|
+
}
|
|
250
|
+
let result: PreviewStartResult
|
|
251
|
+
try {
|
|
252
|
+
result = await this.session.sendRequest<PreviewStartResult>({
|
|
253
|
+
type: MiniappRequestType.STREAM_PREVIEW_START,
|
|
254
|
+
source,
|
|
255
|
+
})
|
|
256
|
+
} catch (error) {
|
|
257
|
+
throw toPreviewError(error)
|
|
258
|
+
}
|
|
259
|
+
const handle = new PreviewHandleImpl(result.source ?? source, result.handleId, result.previewTraceId, (h) =>
|
|
260
|
+
this.releasePreview(h),
|
|
261
|
+
)
|
|
262
|
+
this.previewHandles.set(handle.handleId, handle)
|
|
263
|
+
handle.onStatus((status) => {
|
|
264
|
+
if (status.state === "ended") this.previewHandles.delete(handle.handleId)
|
|
265
|
+
})
|
|
266
|
+
return handle
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
private async releasePreview(handle: PreviewHandleImpl): Promise<void> {
|
|
270
|
+
this.previewHandles.delete(handle.handleId)
|
|
271
|
+
try {
|
|
272
|
+
await this.session.sendRequest<void>({
|
|
273
|
+
type: MiniappRequestType.STREAM_PREVIEW_STOP,
|
|
274
|
+
handleId: handle.handleId,
|
|
275
|
+
})
|
|
276
|
+
} catch (error) {
|
|
277
|
+
throw toPreviewError(error)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** @internal — applied by MiniappSession on inbound STREAM_PREVIEW_STATUS. */
|
|
282
|
+
_applyPreviewStatus(payload: Record<string, unknown>): void {
|
|
283
|
+
if (typeof payload.handleId !== "string") return
|
|
284
|
+
const handle = this.previewHandles.get(payload.handleId)
|
|
285
|
+
if (!handle) return
|
|
286
|
+
if (payload.state === "ended") {
|
|
287
|
+
handle.end(typeof payload.reason === "string" ? payload.reason : "ended")
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
139
291
|
/**
|
|
140
292
|
* Start a live video stream from the glasses camera.
|
|
141
293
|
*
|
package/src/protocol.ts
CHANGED
|
@@ -16,6 +16,13 @@ export enum MiniappRequestType {
|
|
|
16
16
|
/** Handshake: miniapp announces itself and asks phone to bind the session. */
|
|
17
17
|
CONNECT = "miniapp_connect",
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* One-shot: the background's `registerMiniapp` handler has settled, so its
|
|
21
|
+
* `session.ui.handle` handlers exist. Sent only to hosts advertising
|
|
22
|
+
* `hostFeatures.initReady`; the host keeps the UI closed until it arrives.
|
|
23
|
+
*/
|
|
24
|
+
READY = "miniapp_ready",
|
|
25
|
+
|
|
19
26
|
/** Request a fresh miniapp-scoped backend auth token. */
|
|
20
27
|
AUTH_REFRESH = "miniapp_auth_refresh",
|
|
21
28
|
|
|
@@ -196,6 +203,13 @@ export enum MiniappRequestType {
|
|
|
196
203
|
STREAM_STOP = "miniapp_stream_stop",
|
|
197
204
|
MANAGED_STREAM_START = "miniapp_managed_stream_start",
|
|
198
205
|
MANAGED_STREAM_STOP = "miniapp_managed_stream_stop",
|
|
206
|
+
/**
|
|
207
|
+
* Take the single raw decoded-frame preview lease on a source. See
|
|
208
|
+
* `session.stream.preview()`. Replies `{handleId, previewTraceId, source}`.
|
|
209
|
+
*/
|
|
210
|
+
STREAM_PREVIEW_START = "miniapp_stream_preview_start",
|
|
211
|
+
/** Release a preview lease by `handleId`. */
|
|
212
|
+
STREAM_PREVIEW_STOP = "miniapp_stream_preview_stop",
|
|
199
213
|
|
|
200
214
|
/** Ask the host to open the glasses Wi-Fi setup flow (mirrors the cloud SDK's requestWifiSetup). */
|
|
201
215
|
REQUEST_WIFI_SETUP = "miniapp_request_wifi_setup",
|
|
@@ -211,6 +225,12 @@ export enum MiniappRequestType {
|
|
|
211
225
|
MINIAPPS_START = "miniapp_apps_start",
|
|
212
226
|
/** Stop another miniapp. */
|
|
213
227
|
MINIAPPS_STOP = "miniapp_apps_stop",
|
|
228
|
+
/** Install/update an approved bundle. Bundled host-trusted Stores only. */
|
|
229
|
+
MINIAPPS_INSTALL = "miniapp_apps_install",
|
|
230
|
+
/** Preflight a release's host/SDK/hardware requirements without downloading it. */
|
|
231
|
+
MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
|
|
232
|
+
/** Uninstall a package owned by the calling Store. Bundled host-trusted Stores only. */
|
|
233
|
+
MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
|
|
214
234
|
/** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
|
|
215
235
|
ACTION_INVOKE = "miniapp_action_invoke",
|
|
216
236
|
/** Target → host: the result of a delivered ACTION_CALL, correlated by callId. */
|
|
@@ -220,7 +240,10 @@ export enum MiniappRequestType {
|
|
|
220
240
|
* Phone-native meeting (ACS Teams). Join/leave/mute live in the MentraOS
|
|
221
241
|
* host so the miniapp never holds the ACS Calling SDK.
|
|
222
242
|
*/
|
|
243
|
+
MEETING_GET_IDENTITY = "miniapp_meeting_get_identity",
|
|
223
244
|
MEETING_GET_CONFIGURATION = "miniapp_meeting_get_configuration",
|
|
245
|
+
MEETING_CREATE = "miniapp_meeting_create",
|
|
246
|
+
MEETING_RETIRE = "miniapp_meeting_retire",
|
|
224
247
|
MEETING_JOIN = "miniapp_meeting_join",
|
|
225
248
|
MEETING_LEAVE = "miniapp_meeting_leave",
|
|
226
249
|
/** Terminate the meeting for everyone, not just this device. See `meeting.end()`. */
|
|
@@ -228,8 +251,24 @@ export enum MiniappRequestType {
|
|
|
228
251
|
/** Admit one waiting participant, when the host has Teams lobby permission. */
|
|
229
252
|
MEETING_ADMIT = "miniapp_meeting_admit",
|
|
230
253
|
MEETING_SET_MUTED = "miniapp_meeting_set_muted",
|
|
254
|
+
/** Stop or resume the glasses camera the meeting receives, without leaving. */
|
|
255
|
+
MEETING_SET_VIDEO_ENABLED = "miniapp_meeting_set_video_enabled",
|
|
231
256
|
MEETING_UPDATE_VIDEO_SOURCE = "miniapp_meeting_update_video_source",
|
|
232
257
|
MEETING_GET_STATE = "miniapp_meeting_get_state",
|
|
258
|
+
/** Stop the glasses publisher for a photo. The meeting, hotspot, and audio stay up. */
|
|
259
|
+
MEETING_PAUSE_VIDEO_PUBLISHER = "miniapp_meeting_pause_video_publisher",
|
|
260
|
+
MEETING_RESUME_VIDEO_PUBLISHER = "miniapp_meeting_resume_video_publisher",
|
|
261
|
+
/** Full-frame "Taking a photo..." card on the outgoing video. Resolves after the first card frame. */
|
|
262
|
+
MEETING_SHOW_CARD = "miniapp_meeting_show_card",
|
|
263
|
+
/** Replace the card with a still. `durationMs` is counted by the caller from `shownAt`. */
|
|
264
|
+
MEETING_SHOW_IMAGE = "miniapp_meeting_show_image",
|
|
265
|
+
/** Stop holding a card or still so glasses frames reach Teams again. */
|
|
266
|
+
MEETING_SHOW_LIVE = "miniapp_meeting_show_live",
|
|
267
|
+
/**
|
|
268
|
+
* Direct link only: card, full-size glasses photo, still, and live again, all run by the host
|
|
269
|
+
* without stopping the glasses publisher. Resolves after the still has been held `durationMs`.
|
|
270
|
+
*/
|
|
271
|
+
MEETING_CAPTURE_STILL = "miniapp_meeting_capture_still",
|
|
233
272
|
}
|
|
234
273
|
|
|
235
274
|
// ============================================================================
|
|
@@ -255,6 +294,9 @@ export enum MiniappResponseType {
|
|
|
255
294
|
/** Push: host color scheme (light/dark) changed. */
|
|
256
295
|
COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
|
|
257
296
|
|
|
297
|
+
/** Push: a Store installation moved to another host-side phase. */
|
|
298
|
+
MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
|
|
299
|
+
|
|
258
300
|
/**
|
|
259
301
|
* Push: speaker playback state transitioned. Per-miniapp; the runtime sends
|
|
260
302
|
* to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
|
|
@@ -292,6 +334,25 @@ export enum MiniappResponseType {
|
|
|
292
334
|
*/
|
|
293
335
|
MEETING_STATE = "miniapp_meeting_state",
|
|
294
336
|
|
|
337
|
+
/**
|
|
338
|
+
* Push: a photo pause hit its ceiling and the publisher was started again.
|
|
339
|
+
* Carries {pauseId, status: "expired"}. A resume the miniapp sends itself is not this event.
|
|
340
|
+
*/
|
|
341
|
+
MEETING_VIDEO_PUBLISHER = "miniapp_meeting_video_publisher",
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Push: progress of one `meeting.captureStill`. Carries {stillId, phase} with phase `card`
|
|
345
|
+
* (card on the tile), `uploading` (the glasses captured and are uploading), or `shown` (the
|
|
346
|
+
* still is on the tile). The request result still settles the call.
|
|
347
|
+
*/
|
|
348
|
+
MEETING_STILL = "miniapp_meeting_still",
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Push: a preview lease changed state. Carries {handleId, state: "held" | "ended", reason?}.
|
|
352
|
+
* See PreviewHandle.onStatus().
|
|
353
|
+
*/
|
|
354
|
+
STREAM_PREVIEW_STATUS = "miniapp_stream_preview_status",
|
|
355
|
+
|
|
295
356
|
/**
|
|
296
357
|
* Push: phone is about to tear down the miniapp's session. Gives the SDK
|
|
297
358
|
* a brief window (~50ms grace on the phone side) to fire one last
|
|
@@ -405,6 +466,14 @@ export enum MiniappErrorCode {
|
|
|
405
466
|
APP_NOT_FOUND = "APP_NOT_FOUND",
|
|
406
467
|
/** Target miniapp is incompatible with the connected glasses. */
|
|
407
468
|
APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
|
|
469
|
+
/** Bundle download, validation, or activation failed. */
|
|
470
|
+
INSTALL_FAILED = "INSTALL_FAILED",
|
|
471
|
+
/** Another Store installation is already in progress for the same package. */
|
|
472
|
+
INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
|
|
473
|
+
/** An automatic update was deferred because the target is running or starting. */
|
|
474
|
+
APP_RUNNING = "APP_RUNNING",
|
|
475
|
+
/** Another Store owns the installed package's update provenance. */
|
|
476
|
+
STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
|
|
408
477
|
/** Target miniapp does not declare the requested action. */
|
|
409
478
|
ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
|
|
410
479
|
/** Target connected but never registered a handler for the action (within the buffer window). */
|
|
@@ -432,3 +501,59 @@ export enum MiniappErrorCode {
|
|
|
432
501
|
/** `blob.importFile` failed (not the user cancelling — that resolves to null). */
|
|
433
502
|
BLOB_IMPORT_FAILED = "BLOB_IMPORT_FAILED",
|
|
434
503
|
}
|
|
504
|
+
|
|
505
|
+
// ============================================================================
|
|
506
|
+
// Stream preview
|
|
507
|
+
// ============================================================================
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Where a raw preview comes from. `"call"` is the decoded video of the phone-native meeting the
|
|
511
|
+
* caller owns. `"glasses"` is reserved and currently rejects with `unsupported`.
|
|
512
|
+
*/
|
|
513
|
+
export type PreviewSource = "call" | "glasses"
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Typed preview failures and non-terminal statuses, shared by the background handle, the UI
|
|
517
|
+
* component and the host. `waiting_for_lease` and `paused_background` are statuses, not errors.
|
|
518
|
+
*/
|
|
519
|
+
export type PreviewErrorCode =
|
|
520
|
+
| "permission_denied"
|
|
521
|
+
| "not_meeting_owner"
|
|
522
|
+
| "preview_busy"
|
|
523
|
+
| "waiting_for_lease"
|
|
524
|
+
| "source_ended"
|
|
525
|
+
| "unsupported"
|
|
526
|
+
| "ack_timeout"
|
|
527
|
+
| "transport_failed"
|
|
528
|
+
| "pack_failed"
|
|
529
|
+
| "paused_background"
|
|
530
|
+
| "diagnostics_disabled"
|
|
531
|
+
|
|
532
|
+
export const PREVIEW_ERROR_CODES: readonly PreviewErrorCode[] = [
|
|
533
|
+
"permission_denied",
|
|
534
|
+
"not_meeting_owner",
|
|
535
|
+
"preview_busy",
|
|
536
|
+
"waiting_for_lease",
|
|
537
|
+
"source_ended",
|
|
538
|
+
"unsupported",
|
|
539
|
+
"ack_timeout",
|
|
540
|
+
"transport_failed",
|
|
541
|
+
"pack_failed",
|
|
542
|
+
"paused_background",
|
|
543
|
+
"diagnostics_disabled",
|
|
544
|
+
]
|
|
545
|
+
|
|
546
|
+
export function isPreviewErrorCode(value: unknown): value is PreviewErrorCode {
|
|
547
|
+
return typeof value === "string" && (PREVIEW_ERROR_CODES as readonly string[]).includes(value)
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/** Lease state pushed to the background with `STREAM_PREVIEW_STATUS`. */
|
|
551
|
+
export interface PreviewStatus {
|
|
552
|
+
handleId: string
|
|
553
|
+
state: "held" | "ended"
|
|
554
|
+
/** Why the lease ended: `stopped`, `source_ended`, `runtime_stopped`, ... */
|
|
555
|
+
reason?: string
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** The single reserved UI channel the host answers itself for preview control. */
|
|
559
|
+
export const PREVIEW_UI_CHANNEL = "_preview"
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `<StreamPreview>` — draws the raw decoded frames of the source the miniapp's background has
|
|
3
|
+
* leased with `session.stream.preview()`.
|
|
4
|
+
*
|
|
5
|
+
* The component owns only the renderer (canvas, WebGL context, textures). Mounting it shows the
|
|
6
|
+
* preview; unmounting it hides the preview and stops production, but keeps the document's
|
|
7
|
+
* connection open so the next mount starts without a handshake. Releasing the lease is the
|
|
8
|
+
* background's job (`handle.stop()`), not this component's.
|
|
9
|
+
*
|
|
10
|
+
* Frame bytes never enter React state: frames go from the transport to the renderer directly.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {useEffect, useRef, type CSSProperties} from "react"
|
|
14
|
+
|
|
15
|
+
import type {PreviewConnectionState, PreviewSink} from "./stream-preview/PreviewConnection"
|
|
16
|
+
import {getPreviewConnection} from "./stream-preview/PreviewConnection"
|
|
17
|
+
import {ResizeCoalescer, type PreviewBox} from "./stream-preview/resizeCoalescer"
|
|
18
|
+
import {quantizePreviewTier, sameTier, type PreviewTier} from "./stream-preview/tiers"
|
|
19
|
+
import {previewTrace, previewTraceWarn} from "./stream-preview/trace"
|
|
20
|
+
import {
|
|
21
|
+
createYuvRenderer,
|
|
22
|
+
type PreviewFit,
|
|
23
|
+
type YuvRenderer,
|
|
24
|
+
type YuvRendererOptions,
|
|
25
|
+
} from "./stream-preview/yuvRenderer"
|
|
26
|
+
|
|
27
|
+
export type {PreviewFit} from "./stream-preview/yuvRenderer"
|
|
28
|
+
export type {PreviewConnectionState} from "./stream-preview/PreviewConnection"
|
|
29
|
+
|
|
30
|
+
export interface StreamPreviewStatus {
|
|
31
|
+
state: PreviewConnectionState
|
|
32
|
+
/** The Mentra App is in the background; production is paused and resumes on return. */
|
|
33
|
+
paused: boolean
|
|
34
|
+
/** Why the state changed, when the host said (`source_ended`, `stopped`, ...). */
|
|
35
|
+
reason?: string
|
|
36
|
+
/** Error code for `error` / `unsupported`. */
|
|
37
|
+
code?: string
|
|
38
|
+
/** Latest once-a-second host counters plus page-side counters. */
|
|
39
|
+
stats?: Record<string, unknown>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface StreamPreviewProps {
|
|
43
|
+
/** `contain` letterboxes (default); `cover` fills and crops. The picture is never stretched. */
|
|
44
|
+
fit?: PreviewFit
|
|
45
|
+
onStatus?: (status: StreamPreviewStatus) => void
|
|
46
|
+
/** Typed preview error codes: `source_ended`, `unsupported`, `transport_failed`, ... */
|
|
47
|
+
onError?: (code: string) => void
|
|
48
|
+
className?: string
|
|
49
|
+
style?: CSSProperties
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
type RendererFactory = (canvas: HTMLCanvasElement, options: YuvRendererOptions) => YuvRenderer | null
|
|
53
|
+
|
|
54
|
+
let rendererFactory: RendererFactory = createYuvRenderer
|
|
55
|
+
|
|
56
|
+
/** Test seam: WebGL is not available in headless DOMs. */
|
|
57
|
+
export function setStreamPreviewRendererFactoryForTests(factory: RendererFactory | null): void {
|
|
58
|
+
rendererFactory = factory ?? createYuvRenderer
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const DEFAULT_STYLE: CSSProperties = {display: "block", width: "100%", height: "100%"}
|
|
62
|
+
|
|
63
|
+
function measure(canvas: HTMLCanvasElement): PreviewBox {
|
|
64
|
+
const rect = canvas.getBoundingClientRect()
|
|
65
|
+
const ratio = typeof window === "undefined" ? 1 : window.devicePixelRatio || 1
|
|
66
|
+
return {width: Math.round(rect.width * ratio), height: Math.round(rect.height * ratio)}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isDocumentVisible(): boolean {
|
|
70
|
+
return typeof document === "undefined" || document.visibilityState !== "hidden"
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function StreamPreview({fit = "contain", onStatus, onError, className, style}: StreamPreviewProps) {
|
|
74
|
+
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
|
75
|
+
const rendererRef = useRef<YuvRenderer | null>(null)
|
|
76
|
+
const fitRef = useRef(fit)
|
|
77
|
+
const onStatusRef = useRef(onStatus)
|
|
78
|
+
const onErrorRef = useRef(onError)
|
|
79
|
+
fitRef.current = fit
|
|
80
|
+
onStatusRef.current = onStatus
|
|
81
|
+
onErrorRef.current = onError
|
|
82
|
+
|
|
83
|
+
useEffect(() => {
|
|
84
|
+
rendererRef.current?.setFit(fit)
|
|
85
|
+
}, [fit])
|
|
86
|
+
|
|
87
|
+
useEffect(() => {
|
|
88
|
+
const canvas = canvasRef.current
|
|
89
|
+
if (!canvas) return
|
|
90
|
+
const connection = getPreviewConnection()
|
|
91
|
+
const mountEpoch = connection.nextMountEpoch()
|
|
92
|
+
let contextLosses = 0
|
|
93
|
+
let status: StreamPreviewStatus = {state: connection.currentState, paused: false}
|
|
94
|
+
let box: PreviewBox = {width: 0, height: 0}
|
|
95
|
+
let tier: PreviewTier | null = null
|
|
96
|
+
let running = isDocumentVisible()
|
|
97
|
+
let unmounted = false
|
|
98
|
+
|
|
99
|
+
const emit = (next: Partial<StreamPreviewStatus>) => {
|
|
100
|
+
status = {...status, ...next}
|
|
101
|
+
onStatusRef.current?.(status)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const buildRenderer = () => {
|
|
105
|
+
rendererRef.current?.dispose()
|
|
106
|
+
rendererRef.current = rendererFactory(canvas, {
|
|
107
|
+
fit: fitRef.current,
|
|
108
|
+
onError: (reason) => {
|
|
109
|
+
previewTraceWarn("renderer_error", {reason, mountEpoch, docGen: connection.documentGeneration})
|
|
110
|
+
connection.reportRendererError(mountEpoch, reason)
|
|
111
|
+
if (reason === "webgl_context_lost") contextLosses += 1
|
|
112
|
+
},
|
|
113
|
+
})
|
|
114
|
+
if (!rendererRef.current) connection.reportRendererError(mountEpoch, "webgl_unavailable")
|
|
115
|
+
}
|
|
116
|
+
buildRenderer()
|
|
117
|
+
|
|
118
|
+
// Context loss rebuilds only the renderer. The connection and the lease are not involved,
|
|
119
|
+
// and frames that arrive meanwhile are still acked so the sender does not stall.
|
|
120
|
+
const onContextRestored = () => {
|
|
121
|
+
if (unmounted) return
|
|
122
|
+
previewTrace("renderer_rebuilt", {mountEpoch, contextLosses})
|
|
123
|
+
buildRenderer()
|
|
124
|
+
}
|
|
125
|
+
canvas.addEventListener("webglcontextrestored", onContextRestored)
|
|
126
|
+
|
|
127
|
+
const sink: PreviewSink = {
|
|
128
|
+
mountEpoch,
|
|
129
|
+
draw: (frame) => rendererRef.current?.drawFrame(frame) ?? null,
|
|
130
|
+
onState: (state, detail) =>
|
|
131
|
+
emit({state, paused: detail.paused ?? false, reason: detail.reason, code: detail.code}),
|
|
132
|
+
onStatus: (stats) => emit({stats: {...stats, contextLosses}}),
|
|
133
|
+
onError: (code) => onErrorRef.current?.(code),
|
|
134
|
+
}
|
|
135
|
+
const detach = connection.attach(sink)
|
|
136
|
+
previewTrace("mount", {mountEpoch, docGen: connection.documentGeneration})
|
|
137
|
+
|
|
138
|
+
const push = () => connection.update(mountEpoch, {boxWidth: box.width, boxHeight: box.height, tier, running})
|
|
139
|
+
|
|
140
|
+
const coalescer = new ResizeCoalescer({
|
|
141
|
+
apply: (next) => {
|
|
142
|
+
box = next
|
|
143
|
+
const nextTier = quantizePreviewTier(next.width, next.height)
|
|
144
|
+
if (sameTier(nextTier, tier) && tier !== null) return
|
|
145
|
+
tier = nextTier
|
|
146
|
+
if (!tier) previewTrace("hidden_zero_size", {mountEpoch})
|
|
147
|
+
push()
|
|
148
|
+
},
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
let resizeObserver: ResizeObserver | null = null
|
|
152
|
+
const onWindowResize = () => coalescer.push(measure(canvas))
|
|
153
|
+
if (typeof ResizeObserver === "function") {
|
|
154
|
+
resizeObserver = new ResizeObserver(() => coalescer.push(measure(canvas)))
|
|
155
|
+
resizeObserver.observe(canvas)
|
|
156
|
+
} else if (typeof window !== "undefined") {
|
|
157
|
+
window.addEventListener("resize", onWindowResize)
|
|
158
|
+
}
|
|
159
|
+
coalescer.push(measure(canvas))
|
|
160
|
+
|
|
161
|
+
const onVisibility = () => {
|
|
162
|
+
const visible = isDocumentVisible()
|
|
163
|
+
if (visible === running) return
|
|
164
|
+
running = visible
|
|
165
|
+
previewTrace("visibility_change", {mountEpoch, visible})
|
|
166
|
+
push()
|
|
167
|
+
}
|
|
168
|
+
document.addEventListener("visibilitychange", onVisibility)
|
|
169
|
+
|
|
170
|
+
return () => {
|
|
171
|
+
unmounted = true
|
|
172
|
+
previewTrace("unmount", {mountEpoch, docGen: connection.documentGeneration})
|
|
173
|
+
document.removeEventListener("visibilitychange", onVisibility)
|
|
174
|
+
resizeObserver?.disconnect()
|
|
175
|
+
if (typeof window !== "undefined") window.removeEventListener("resize", onWindowResize)
|
|
176
|
+
coalescer.cancel()
|
|
177
|
+
canvas.removeEventListener("webglcontextrestored", onContextRestored)
|
|
178
|
+
running = false
|
|
179
|
+
push()
|
|
180
|
+
detach()
|
|
181
|
+
rendererRef.current?.dispose()
|
|
182
|
+
rendererRef.current = null
|
|
183
|
+
}
|
|
184
|
+
}, [])
|
|
185
|
+
|
|
186
|
+
return <canvas ref={canvasRef} className={className} style={{...DEFAULT_STYLE, ...style}} />
|
|
187
|
+
}
|
package/src/react/index.ts
CHANGED
|
@@ -10,3 +10,6 @@ export type {MiniappHeaderProps} from "./MiniappHeader"
|
|
|
10
10
|
export {MentraProvider} from "./MentraProvider"
|
|
11
11
|
export type {MentraProviderProps} from "./MentraProvider"
|
|
12
12
|
export {useRpc, type RpcCallable} from "./useRpc"
|
|
13
|
+
export {useHistoryState, type SetHistoryState} from "./useHistoryState"
|
|
14
|
+
export {StreamPreview} from "./StreamPreview"
|
|
15
|
+
export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "./StreamPreview"
|