@glassly/miniapp-cli 0.1.15-dev.0 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/api.ts +2 -1
- package/src/sim/client/app.css +5 -0
- package/src/sim/client/main.ts +7 -2
- package/src/sim/device.test.ts +57 -28
- package/src/sim/device.ts +172 -82
- package/src/sim/host.test.ts +369 -10
- package/src/sim/host.ts +664 -88
- package/src/sim/models.json +599 -0
- package/src/sim/regenerate-models.ts +34 -0
- package/src/sim/registry.ts +394 -0
- package/src/sim/scenarios.test.ts +74 -0
- package/src/sim/script.test.ts +31 -0
- package/src/sim/script.ts +48 -0
- package/src/sim/server.ts +49 -1
- package/src/sim/shots.test.ts +59 -0
- package/src/sim/shots.ts +161 -0
- package/src/sim/sim.ts +114 -12
- package/src/sim/storage.ts +8 -7
package/src/sim/host.ts
CHANGED
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import {
|
|
2
|
+
degradeScene,
|
|
2
3
|
diffScene,
|
|
3
4
|
processScene,
|
|
4
5
|
resolveAnchors,
|
|
5
6
|
SceneStore,
|
|
6
7
|
TextMeasurer,
|
|
8
|
+
TextWrapper,
|
|
9
|
+
type DiffableElement,
|
|
7
10
|
type FrameElement,
|
|
11
|
+
type SceneDisplayCapabilities,
|
|
8
12
|
type SceneElementInput,
|
|
9
13
|
} from "@glassly/miniapp/display"
|
|
14
|
+
// A namespace import, so an older @glassly/miniapp without the host rules still loads.
|
|
15
|
+
import {existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync} from "fs"
|
|
16
|
+
import {join} from "path"
|
|
17
|
+
import * as hostRules from "@glassly/miniapp/host"
|
|
10
18
|
import type {MirrorElement, MirrorSnapshot} from "@glassly/miniapp/mirror"
|
|
11
19
|
import {
|
|
12
20
|
MiniappErrorCode,
|
|
@@ -17,7 +25,9 @@ import {
|
|
|
17
25
|
|
|
18
26
|
import type {SimDevice} from "./device.js"
|
|
19
27
|
import type {PhoneKind} from "./phone.js"
|
|
20
|
-
import type {
|
|
28
|
+
import type {ScriptLine} from "./script.js"
|
|
29
|
+
import {SimRegistry} from "./registry.js"
|
|
30
|
+
import {createSimStorage, type SimStorage} from "./storage.js"
|
|
21
31
|
|
|
22
32
|
export type SimLogSource = "background" | "ui" | "host" | "bridge"
|
|
23
33
|
|
|
@@ -49,12 +59,21 @@ export interface SimManifest {
|
|
|
49
59
|
version?: string
|
|
50
60
|
type?: string
|
|
51
61
|
permissions?: {type?: string}[]
|
|
62
|
+
actions?: unknown
|
|
63
|
+
widgets?: unknown
|
|
52
64
|
}
|
|
53
65
|
|
|
54
66
|
export interface SimHostOptions {
|
|
55
67
|
manifest: SimManifest
|
|
56
68
|
device: SimDevice
|
|
57
69
|
storage: SimStorage
|
|
70
|
+
/** Settings and the memory timeline; in memory when absent. */
|
|
71
|
+
settings?: SimStorage
|
|
72
|
+
memory?: SimStorage
|
|
73
|
+
/** Where memory exports are written and imports read; none disables them. */
|
|
74
|
+
stateDir?: string
|
|
75
|
+
/** The simulator's installed miniapps; a registry of this one alone when absent. */
|
|
76
|
+
registry?: SimRegistry
|
|
58
77
|
toBackground: (raw: string) => void
|
|
59
78
|
emit: (event: SimHostEvent) => void
|
|
60
79
|
frameLatencyMs?: number
|
|
@@ -71,6 +90,21 @@ export interface SimLocation {
|
|
|
71
90
|
type View = "main" | "dashboard"
|
|
72
91
|
|
|
73
92
|
const LOG_LIMIT = 2000
|
|
93
|
+
const NOTIFICATION_TITLE_MAX = 200
|
|
94
|
+
const NOTIFICATION_CONTENT_MAX = 2_000
|
|
95
|
+
const NOTIFICATION_ID_MAX = 128
|
|
96
|
+
const CALL_TIMEOUT_MS = 10_000
|
|
97
|
+
const DEFAULT_WORDS_PER_MINUTE = 160
|
|
98
|
+
const ENDPOINT_DEBOUNCE_MS = 500
|
|
99
|
+
|
|
100
|
+
// Stands in for the phone's installed apps on the iOS catalog, which has no icons.
|
|
101
|
+
const SAMPLE_NOTIFICATION_APPS = [
|
|
102
|
+
{packageName: "com.apple.MobileSMS", appName: "Messages", icon: null},
|
|
103
|
+
{packageName: "com.apple.mobilemail", appName: "Mail", icon: null},
|
|
104
|
+
{packageName: "com.apple.mobilecal", appName: "Calendar", icon: null},
|
|
105
|
+
{packageName: "net.whatsapp.WhatsApp", appName: "WhatsApp", icon: null},
|
|
106
|
+
{packageName: "com.tinyspeck.chatanywhere", appName: "Slack", icon: null},
|
|
107
|
+
]
|
|
74
108
|
const EFFECT_LIMIT = 200
|
|
75
109
|
const MAX_SETTLE_MS = 255 * 33
|
|
76
110
|
|
|
@@ -102,6 +136,26 @@ function permissionForStream(stream: string): string | null {
|
|
|
102
136
|
return null
|
|
103
137
|
}
|
|
104
138
|
|
|
139
|
+
/** What stock RayNeo iO firmware shows: the frame's text in paint order on its one subtitle page. */
|
|
140
|
+
function joinText(elements: DiffableElement[], caps: SceneDisplayCapabilities): DiffableElement[] {
|
|
141
|
+
const text = elements
|
|
142
|
+
.filter((el) => el.type === "text" && el.text)
|
|
143
|
+
.map((el) => (el as {text: string}).text)
|
|
144
|
+
.join("\n")
|
|
145
|
+
if (!text) return []
|
|
146
|
+
return [{id: "subtitle", type: "text", text, box: {x: 0, y: 0, w: caps.width, h: caps.height}} as DiffableElement]
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function baseLanguage(tag: string): string {
|
|
150
|
+
return tag.split("-")[0].toLowerCase()
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function localDate(ms: number): string {
|
|
154
|
+
const d = new Date(ms)
|
|
155
|
+
const pad = (n: number) => String(n).padStart(2, "0")
|
|
156
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`
|
|
157
|
+
}
|
|
158
|
+
|
|
105
159
|
function settleTimeOf(elements: FrameElement[]): number {
|
|
106
160
|
let ms = 0
|
|
107
161
|
for (const el of elements) {
|
|
@@ -140,7 +194,8 @@ interface StreamRoute {
|
|
|
140
194
|
* requests it covers. No I/O of its own; the server and the tests drive it.
|
|
141
195
|
*/
|
|
142
196
|
export class SimHost {
|
|
143
|
-
|
|
197
|
+
readonly manifest: SimManifest
|
|
198
|
+
private readonly registry: SimRegistry
|
|
144
199
|
private readonly device: SimDevice
|
|
145
200
|
private readonly storage: SimStorage
|
|
146
201
|
private readonly toBackground: (raw: string) => void
|
|
@@ -148,13 +203,19 @@ export class SimHost {
|
|
|
148
203
|
private readonly frameLatencyMs: number
|
|
149
204
|
private readonly speechMsPerChar: number
|
|
150
205
|
private readonly declared: Set<string>
|
|
206
|
+
private readonly settings: SimStorage
|
|
207
|
+
private readonly memory: InstanceType<typeof hostRules.MemoryStore> | null
|
|
208
|
+
private readonly stateDir: string | undefined
|
|
151
209
|
private readonly measurer: TextMeasurer
|
|
210
|
+
private readonly wrapper: TextWrapper
|
|
152
211
|
|
|
153
212
|
private scenes = new SceneStore()
|
|
154
213
|
private handshook = false
|
|
155
214
|
private background: "stopped" | "running" | "crashed" = "stopped"
|
|
156
215
|
private subscriptions = new Map<string, StreamRoute>()
|
|
157
216
|
private claimedGestures: string[] = []
|
|
217
|
+
private derivedSettings: Record<string, unknown> = {}
|
|
218
|
+
private gestureClaims = hostRules.emptyGestureClaims?.()
|
|
158
219
|
private overlay = false
|
|
159
220
|
private revision = 0
|
|
160
221
|
private frameSeq = 0
|
|
@@ -172,6 +233,11 @@ export class SimHost {
|
|
|
172
233
|
private readonly unsimulated = new Set<string>()
|
|
173
234
|
|
|
174
235
|
private speakerState = "idle"
|
|
236
|
+
private utteranceSeq = 0
|
|
237
|
+
private openUtterance: string | null = null
|
|
238
|
+
private openTranslations: Record<string, string> = {}
|
|
239
|
+
private speechEpoch = Date.now()
|
|
240
|
+
private playback: {timers: ReturnType<typeof setTimeout>[]; done: () => void} | null = null
|
|
175
241
|
private speech: {timer: ReturnType<typeof setTimeout>; finish: (completed: boolean) => void} | null = null
|
|
176
242
|
|
|
177
243
|
private glassesConnected = true
|
|
@@ -187,12 +253,25 @@ export class SimHost {
|
|
|
187
253
|
this.manifest = options.manifest
|
|
188
254
|
this.device = options.device
|
|
189
255
|
this.storage = options.storage
|
|
256
|
+
this.settings = options.settings ?? createSimStorage()
|
|
257
|
+
this.stateDir = options.stateDir
|
|
258
|
+
const memory = options.memory ?? createSimStorage()
|
|
259
|
+
this.memory = hostRules.MemoryStore
|
|
260
|
+
? new hostRules.MemoryStore({
|
|
261
|
+
getUserId: async () => "sim-user",
|
|
262
|
+
backend: {get: (key) => memory.get(key), set: (key, value) => memory.set(key, value), remove: (key) => memory.delete(key)},
|
|
263
|
+
writeDebounceMs: 0,
|
|
264
|
+
})
|
|
265
|
+
: null
|
|
266
|
+
this.registry = options.registry ?? new SimRegistry()
|
|
267
|
+
this.registry.add(this)
|
|
190
268
|
this.toBackground = options.toBackground
|
|
191
269
|
this.emit = options.emit
|
|
192
270
|
this.frameLatencyMs = options.frameLatencyMs ?? 30
|
|
193
271
|
this.speechMsPerChar = options.speechMsPerChar ?? 55
|
|
194
272
|
this.declared = new Set((options.manifest.permissions ?? []).map((p) => String(p.type ?? "").toUpperCase()))
|
|
195
273
|
this.measurer = new TextMeasurer(options.device.profile)
|
|
274
|
+
this.wrapper = new TextWrapper(this.measurer, {breakMode: "character-no-hyphen"})
|
|
196
275
|
}
|
|
197
276
|
|
|
198
277
|
backgroundStarted(): void {
|
|
@@ -211,10 +290,14 @@ export class SimHost {
|
|
|
211
290
|
this.handshook = false
|
|
212
291
|
this.subscriptions.clear()
|
|
213
292
|
this.claimedGestures = []
|
|
293
|
+
this.gestureClaims = hostRules.emptyGestureClaims?.()
|
|
214
294
|
this.overlay = false
|
|
215
295
|
this.scenes = new SceneStore()
|
|
216
296
|
for (const view of ["main", "dashboard"] as View[]) this.clearExpiry(view)
|
|
217
297
|
this.finishSpeech(false)
|
|
298
|
+
this.stopPlaying()
|
|
299
|
+
this.openUtterance = null
|
|
300
|
+
this.openTranslations = {}
|
|
218
301
|
this.speakerState = "idle"
|
|
219
302
|
this.lastDropped = ""
|
|
220
303
|
for (const settle of [...this.uiRequests.values()]) settle({ok: false, error: {message: "background restarted"}})
|
|
@@ -244,31 +327,41 @@ export class SimHost {
|
|
|
244
327
|
}
|
|
245
328
|
|
|
246
329
|
private dispatch(type: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
330
|
+
if (this.registry.handle(this, type, payload, requestId)) return
|
|
247
331
|
switch (type) {
|
|
248
332
|
case MiniappRequestType.CONNECT:
|
|
249
333
|
return this.handleConnect(requestId)
|
|
250
334
|
case MiniappRequestType.SUBSCRIBE:
|
|
251
335
|
return this.handleSubscribe(payload, requestId)
|
|
252
336
|
case MiniappRequestType.CLAIM_GESTURES:
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
this.
|
|
257
|
-
return this.result(requestId, true)
|
|
337
|
+
if (payload.scope !== undefined && (typeof payload.scope !== "string" || payload.scope.length === 0)) {
|
|
338
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "Gesture claim scope must be a nonempty string")
|
|
339
|
+
}
|
|
340
|
+
return this.claimGestures(payload, requestId)
|
|
258
341
|
case MiniappRequestType.TEXT_FIELD:
|
|
342
|
+
if (typeof payload.open !== "boolean") {
|
|
343
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "Text field open must be a boolean")
|
|
344
|
+
}
|
|
259
345
|
return this.result(requestId, true)
|
|
260
346
|
case MiniappRequestType.RENDER:
|
|
261
347
|
return this.handleRender(payload, requestId)
|
|
262
348
|
case MiniappRequestType.SHOW_VIEW:
|
|
263
349
|
return this.handleShowView(payload, requestId)
|
|
264
350
|
case MiniappRequestType.MIRROR_GET:
|
|
265
|
-
if (!this.
|
|
351
|
+
if (!this.declared.has("DISPLAY_MIRROR")) {
|
|
352
|
+
return this.result(requestId, false, undefined, {
|
|
353
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
354
|
+
message: "DISPLAY_MIRROR permission not declared in miniapp.json",
|
|
355
|
+
permission: "DISPLAY_MIRROR",
|
|
356
|
+
})
|
|
357
|
+
}
|
|
266
358
|
return this.result(requestId, true, this.snapshot())
|
|
267
359
|
case MiniappRequestType.PING:
|
|
268
360
|
return this.send({type: MiniappResponseType.PONG}, requestId)
|
|
269
361
|
case MiniappResponseType.PONG:
|
|
270
|
-
case MiniappRequestType.TRANSCRIPTION_CONFIG:
|
|
271
362
|
return
|
|
363
|
+
case MiniappRequestType.TRANSCRIPTION_CONFIG:
|
|
364
|
+
return this.result(requestId, true, {applied: true})
|
|
272
365
|
case MiniappRequestType.AUTH_REFRESH:
|
|
273
366
|
return this.result(requestId, false, undefined, {
|
|
274
367
|
code: MiniappErrorCode.NOT_CONNECTED,
|
|
@@ -276,12 +369,12 @@ export class SimHost {
|
|
|
276
369
|
})
|
|
277
370
|
|
|
278
371
|
case MiniappRequestType.STORAGE_GET:
|
|
279
|
-
return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(payload))})
|
|
372
|
+
return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(type, payload))})
|
|
280
373
|
case MiniappRequestType.STORAGE_SET:
|
|
281
|
-
this.storage.set(this.key(payload), payload.value
|
|
374
|
+
this.storage.set(this.key(type, payload), payload.value ?? null)
|
|
282
375
|
return this.result(requestId, true)
|
|
283
376
|
case MiniappRequestType.STORAGE_DELETE:
|
|
284
|
-
this.storage.delete(this.key(payload))
|
|
377
|
+
this.storage.delete(this.key(type, payload))
|
|
285
378
|
return this.result(requestId, true)
|
|
286
379
|
case MiniappRequestType.STORAGE_LIST:
|
|
287
380
|
return this.result(requestId, true, {keys: this.storage.keys()})
|
|
@@ -289,25 +382,32 @@ export class SimHost {
|
|
|
289
382
|
this.storage.clear()
|
|
290
383
|
return this.result(requestId, true)
|
|
291
384
|
case MiniappRequestType.STORAGE_HAS:
|
|
292
|
-
return this.result(requestId, true, {has: this.storage.
|
|
293
|
-
case MiniappRequestType.STORAGE_GET_ALL:
|
|
294
|
-
|
|
385
|
+
return this.result(requestId, true, {has: this.storage.has(this.key(type, payload))})
|
|
386
|
+
case MiniappRequestType.STORAGE_GET_ALL: {
|
|
387
|
+
const values: Record<string, string> = {}
|
|
388
|
+
for (const [key, value] of Object.entries(this.storage.getAll())) {
|
|
389
|
+
values[key] = typeof value === "string" ? value : String(value ?? "")
|
|
390
|
+
}
|
|
391
|
+
return this.result(requestId, true, {values})
|
|
392
|
+
}
|
|
295
393
|
case MiniappRequestType.STORAGE_SET_MULTIPLE: {
|
|
296
394
|
const values = payload.values
|
|
297
395
|
if (!values || typeof values !== "object") throw new Error("storage_set_multiple requires a values object")
|
|
298
|
-
for (const [key, value] of Object.entries(values)) this.storage.set(key, value
|
|
396
|
+
for (const [key, value] of Object.entries(values)) this.storage.set(key, value ?? null)
|
|
299
397
|
return this.result(requestId, true)
|
|
300
398
|
}
|
|
301
399
|
case MiniappRequestType.STORAGE_FLUSH:
|
|
302
400
|
return this.result(requestId, true)
|
|
303
401
|
|
|
304
402
|
case MiniappRequestType.LOCATION_POLL:
|
|
305
|
-
if (!this.requirePermission(requestId, "LOCATION", "to read the location")) return
|
|
306
403
|
return this.result(requestId, true, this.location)
|
|
307
404
|
|
|
308
405
|
case MiniappRequestType.SPEAK:
|
|
309
406
|
return this.handleSpeak(String(payload.text ?? ""), requestId)
|
|
310
407
|
case MiniappRequestType.PLAY_AUDIO:
|
|
408
|
+
if (typeof payload.audioUrl !== "string" || !payload.audioUrl) {
|
|
409
|
+
return this.fail(requestId, MiniappErrorCode.INTERNAL, "play_audio requires an audioUrl")
|
|
410
|
+
}
|
|
311
411
|
this.effect("audio", {url: payload.audioUrl})
|
|
312
412
|
return this.startSpeech(1500, requestId, undefined)
|
|
313
413
|
case MiniappRequestType.STOP_AUDIO:
|
|
@@ -330,23 +430,69 @@ export class SimHost {
|
|
|
330
430
|
this.effect("download", {...payload, type: undefined})
|
|
331
431
|
return this.result(requestId, true, {success: true})
|
|
332
432
|
|
|
333
|
-
case MiniappRequestType.NOTIFICATION_POST:
|
|
334
|
-
|
|
335
|
-
const id = typeof payload.id === "string" && payload.id ? payload.id : `sim-${++this.notificationSeq}`
|
|
336
|
-
this.effect("notification", {id, title: payload.title, content: payload.content})
|
|
337
|
-
return this.result(requestId, true, {id})
|
|
338
|
-
}
|
|
433
|
+
case MiniappRequestType.NOTIFICATION_POST:
|
|
434
|
+
return this.handleNotificationPost(payload, requestId)
|
|
339
435
|
case MiniappRequestType.NOTIFICATION_DISMISS:
|
|
340
|
-
if (!this.requirePermission(requestId, "POST_NOTIFICATIONS"
|
|
341
|
-
|
|
436
|
+
if (!this.requirePermission(requestId, "POST_NOTIFICATIONS")) return
|
|
437
|
+
if (typeof payload.id !== "string" || !payload.id.trim()) {
|
|
438
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "dismiss needs the id post() returned")
|
|
439
|
+
}
|
|
440
|
+
this.effect("notification_dismiss", {id: payload.id.trim()})
|
|
342
441
|
return this.result(requestId, true)
|
|
343
442
|
|
|
443
|
+
case MiniappRequestType.SETTINGS_GET: {
|
|
444
|
+
const {selectSettingsBridgeKeys} = hostRules
|
|
445
|
+
if (!selectSettingsBridgeKeys) break
|
|
446
|
+
if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
|
|
447
|
+
const values: Record<string, unknown> = {}
|
|
448
|
+
for (const key of selectSettingsBridgeKeys(payload.keys)) values[key] = this.settingValue(key)
|
|
449
|
+
return this.result(requestId, true, {values})
|
|
450
|
+
}
|
|
451
|
+
case MiniappRequestType.SETTINGS_SET: {
|
|
452
|
+
const {validateSettingsSet} = hostRules
|
|
453
|
+
if (!validateSettingsSet) break
|
|
454
|
+
if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
|
|
455
|
+
const validated = validateSettingsSet(payload)
|
|
456
|
+
if (!validated.ok) return this.fail(requestId, validated.code, validated.message)
|
|
457
|
+
const changed = JSON.stringify(this.settingValue(validated.key)) !== JSON.stringify(validated.value ?? null)
|
|
458
|
+
this.settings.set(validated.key, validated.value)
|
|
459
|
+
this.result(requestId, true)
|
|
460
|
+
if (changed) this.push(MiniappStreamType.SETTINGS_CHANGED, {key: validated.key, value: this.settingValue(validated.key)})
|
|
461
|
+
return
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
case MiniappRequestType.NOTIFICATION_APPS_LIST: {
|
|
465
|
+
if (!this.requirePermission(requestId, "READ_NOTIFICATIONS")) return
|
|
466
|
+
const posts = this.declared.has("POST_NOTIFICATIONS")
|
|
467
|
+
const self = {packageName: this.manifest.packageName, appName: this.manifest.name || this.manifest.packageName, icon: null}
|
|
468
|
+
return this.result(requestId, true, [...SAMPLE_NOTIFICATION_APPS, ...(posts ? [self] : [])])
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
|
|
344
473
|
case MiniappRequestType.SYSTEM_EXIT:
|
|
345
474
|
case MiniappRequestType.MINIAPPS_STOP_SELF:
|
|
346
475
|
this.result(requestId, true)
|
|
347
476
|
this.log("host", "info", "The miniapp asked to stop (session.system.exit)")
|
|
348
477
|
return this.emit({type: "exit", reason: "the miniapp called exit"})
|
|
349
478
|
|
|
479
|
+
case MiniappRequestType.MEMORY_APPEND_LINE:
|
|
480
|
+
case MiniappRequestType.MEMORY_ADD_NOTE:
|
|
481
|
+
case MiniappRequestType.MEMORY_REMOVE_ENTRY:
|
|
482
|
+
case MiniappRequestType.MEMORY_REMOVE_SCOPE:
|
|
483
|
+
case MiniappRequestType.MEMORY_SESSION_LIST:
|
|
484
|
+
case MiniappRequestType.MEMORY_QUERY:
|
|
485
|
+
case MiniappRequestType.MEMORY_WINDOW:
|
|
486
|
+
case MiniappRequestType.MEMORY_EXPORT_TEXT:
|
|
487
|
+
case MiniappRequestType.MEMORY_EXPORT_FILE:
|
|
488
|
+
case MiniappRequestType.MEMORY_IMPORT_FILE:
|
|
489
|
+
case MiniappRequestType.MEMORY_SUMMARY_ADD:
|
|
490
|
+
case MiniappRequestType.MEMORY_SUMMARY_LIST:
|
|
491
|
+
case MiniappRequestType.MEMORY_PENDING_SUMMARY_HOUR:
|
|
492
|
+
case MiniappRequestType.MEMORY_CLEAR:
|
|
493
|
+
if (!this.memory) break
|
|
494
|
+
return void this.handleMemory(this.memory, type, payload, requestId)
|
|
495
|
+
|
|
350
496
|
case "UI_SEND":
|
|
351
497
|
return this.handleUiSend(payload)
|
|
352
498
|
case "UI_CANCEL":
|
|
@@ -364,12 +510,237 @@ export class SimHost {
|
|
|
364
510
|
})
|
|
365
511
|
}
|
|
366
512
|
|
|
367
|
-
private
|
|
368
|
-
|
|
513
|
+
private claimGestures(payload: Record<string, unknown>, requestId?: string): void {
|
|
514
|
+
const gestures = Array.isArray(payload.gestures) ? payload.gestures.filter((g): g is string => typeof g === "string") : []
|
|
515
|
+
const {setGestureClaims, gestureClaimant} = hostRules
|
|
516
|
+
if (!this.gestureClaims || !setGestureClaims || !gestureClaimant) {
|
|
517
|
+
this.claimedGestures = gestures
|
|
518
|
+
} else {
|
|
519
|
+
const {state, changes} = setGestureClaims(this.gestureClaims, this.manifest.packageName, gestures, payload.scope as string | undefined)
|
|
520
|
+
this.gestureClaims = state
|
|
521
|
+
const held = new Set(state.registrations.flatMap((entry) => [...entry.gestures]))
|
|
522
|
+
this.claimedGestures = [...held].filter((gesture) => gestureClaimant(state, gesture) === this.manifest.packageName)
|
|
523
|
+
for (const change of changes) this.push(MiniappStreamType.GESTURE_CLAIM, {gesture: change.gesture, held: change.held})
|
|
524
|
+
}
|
|
525
|
+
this.emit({type: "status"})
|
|
526
|
+
this.result(requestId, true)
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
private key(type: string, payload: Record<string, unknown>): string {
|
|
530
|
+
if (typeof payload.key !== "string" || !payload.key) throw new Error(`${type.replace(/^miniapp_/, "")} requires a key`)
|
|
369
531
|
return payload.key
|
|
370
532
|
}
|
|
371
533
|
|
|
534
|
+
/** A bridged key's value: stored, else the Glassly App's default; derived keys follow simulator state. */
|
|
535
|
+
// The phone pushes settings_changed when a derived key's value moves; these follow simulator state.
|
|
536
|
+
private publishDerivedSettings(): void {
|
|
537
|
+
for (const key of ["phone_active", "foreground_app_on_screen"]) {
|
|
538
|
+
const value = this.settingValue(key)
|
|
539
|
+
if (this.derivedSettings[key] === value) continue
|
|
540
|
+
const known = key in this.derivedSettings
|
|
541
|
+
this.derivedSettings[key] = value
|
|
542
|
+
if (known) this.push(MiniappStreamType.SETTINGS_CHANGED, {key, value})
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
private settingValue(key: string): unknown {
|
|
547
|
+
if (key === "phone_active") return this.visibility === "foreground"
|
|
548
|
+
if (key === "foreground_app_on_screen") return this.visibleView() === "main" && this.scenes.lastFrame(this.manifest.packageName, "main").length > 0
|
|
549
|
+
if (key === "controllers") return []
|
|
550
|
+
return this.settings.has(key) ? this.settings.get(key) : (hostRules.settingsBridgeDefault?.(key) ?? null)
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// Mirrors the MEMORY_* handlers in the Glassly App's LocalMiniappRuntime.
|
|
554
|
+
private async handleMemory(
|
|
555
|
+
memory: InstanceType<typeof hostRules.MemoryStore>,
|
|
556
|
+
type: string,
|
|
557
|
+
payload: Record<string, unknown>,
|
|
558
|
+
requestId?: string,
|
|
559
|
+
): Promise<void> {
|
|
560
|
+
if (!this.declared.has("MEMORY")) {
|
|
561
|
+
return this.result(requestId, false, undefined, {
|
|
562
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
563
|
+
message: `MEMORY permission not declared in miniapp.json. Add {"type": "MEMORY"} to the "permissions" array.`,
|
|
564
|
+
permission: "MEMORY",
|
|
565
|
+
operation: type,
|
|
566
|
+
})
|
|
567
|
+
}
|
|
568
|
+
const str = (v: unknown): string => (typeof v === "string" ? v : "")
|
|
569
|
+
const num = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) ? v : undefined)
|
|
570
|
+
try {
|
|
571
|
+
switch (type) {
|
|
572
|
+
case MiniappRequestType.MEMORY_APPEND_LINE: {
|
|
573
|
+
const text = str(payload.text)
|
|
574
|
+
const at = num(payload.at)
|
|
575
|
+
if (!text || at == null) {
|
|
576
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_append_line requires text and at")
|
|
577
|
+
}
|
|
578
|
+
await memory.appendLine({
|
|
579
|
+
text,
|
|
580
|
+
speakerId: typeof payload.speakerId === "string" ? payload.speakerId : null,
|
|
581
|
+
language: typeof payload.language === "string" ? payload.language : null,
|
|
582
|
+
at,
|
|
583
|
+
})
|
|
584
|
+
return this.result(requestId, true)
|
|
585
|
+
}
|
|
586
|
+
case MiniappRequestType.MEMORY_ADD_NOTE: {
|
|
587
|
+
const text = str(payload.text)
|
|
588
|
+
if (!text.trim()) return this.result(requestId, true, {entry: null})
|
|
589
|
+
const source = payload.source === "voice" || payload.source === "auto" ? payload.source : "ui"
|
|
590
|
+
return this.result(requestId, true, {entry: await memory.addNote(text, source, num(payload.at))})
|
|
591
|
+
}
|
|
592
|
+
case MiniappRequestType.MEMORY_REMOVE_ENTRY:
|
|
593
|
+
return this.result(requestId, true, {removed: await memory.removeEntry(str(payload.id))})
|
|
594
|
+
case MiniappRequestType.MEMORY_REMOVE_SCOPE: {
|
|
595
|
+
const at = num(payload.at)
|
|
596
|
+
if (at == null) return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_remove_scope requires at")
|
|
597
|
+
await memory.removeScope(payload.scope === "day" ? "day" : "session", at)
|
|
598
|
+
return this.result(requestId, true)
|
|
599
|
+
}
|
|
600
|
+
case MiniappRequestType.MEMORY_SESSION_LIST:
|
|
601
|
+
return this.result(requestId, true, {sessions: await memory.sessionList()})
|
|
602
|
+
case MiniappRequestType.MEMORY_QUERY: {
|
|
603
|
+
const terms = Array.isArray(payload.terms)
|
|
604
|
+
? payload.terms
|
|
605
|
+
.filter((t): t is string => typeof t === "string" && t.trim().length > 0)
|
|
606
|
+
.slice(0, 8)
|
|
607
|
+
.map((t) => t.slice(0, 64))
|
|
608
|
+
: undefined
|
|
609
|
+
const found = await memory.search({
|
|
610
|
+
sinceMs: num(payload.sinceMs),
|
|
611
|
+
untilMs: num(payload.untilMs),
|
|
612
|
+
contains: typeof payload.contains === "string" ? payload.contains : undefined,
|
|
613
|
+
terms,
|
|
614
|
+
limit: num(payload.limit),
|
|
615
|
+
})
|
|
616
|
+
return this.result(requestId, true, {entries: found.entries, windowCount: found.windowCount})
|
|
617
|
+
}
|
|
618
|
+
case MiniappRequestType.MEMORY_WINDOW: {
|
|
619
|
+
const sinceMs = num(payload.sinceMs)
|
|
620
|
+
if (sinceMs == null) return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_window requires sinceMs")
|
|
621
|
+
return this.result(requestId, true, {entries: await memory.window(sinceMs, num(payload.untilMs))})
|
|
622
|
+
}
|
|
623
|
+
case MiniappRequestType.MEMORY_EXPORT_TEXT: {
|
|
624
|
+
const scope = payload.scope === "session" || payload.scope === "day" ? payload.scope : "all"
|
|
625
|
+
return this.result(requestId, true, {text: await memory.exportText(scope, num(payload.at))})
|
|
626
|
+
}
|
|
627
|
+
case MiniappRequestType.MEMORY_EXPORT_FILE: {
|
|
628
|
+
if (!this.stateDir) return this.result(requestId, true, {success: false, cancelled: true})
|
|
629
|
+
const backup = await memory.exportBackup()
|
|
630
|
+
const file = join(this.stateDir, `glassly-memory-${localDate(backup.exportedAt)}.json`)
|
|
631
|
+
mkdirSync(this.stateDir, {recursive: true})
|
|
632
|
+
writeFileSync(file, JSON.stringify(backup))
|
|
633
|
+
this.effect("memory_export", {file})
|
|
634
|
+
return this.result(requestId, true, {success: true})
|
|
635
|
+
}
|
|
636
|
+
case MiniappRequestType.MEMORY_IMPORT_FILE: {
|
|
637
|
+
const file = this.latestMemoryBackup()
|
|
638
|
+
if (!file) return this.result(requestId, true, {imported: null})
|
|
639
|
+
let data: unknown
|
|
640
|
+
try {
|
|
641
|
+
data = JSON.parse(readFileSync(file, "utf-8"))
|
|
642
|
+
} catch {
|
|
643
|
+
throw new hostRules.MemoryBackupError("Not a Glassly memory backup")
|
|
644
|
+
}
|
|
645
|
+
this.effect("memory_import", {file})
|
|
646
|
+
return this.result(requestId, true, {imported: await memory.importBackup(data)})
|
|
647
|
+
}
|
|
648
|
+
case MiniappRequestType.MEMORY_SUMMARY_ADD: {
|
|
649
|
+
const hourStart = num(payload.hourStart)
|
|
650
|
+
if (hourStart == null || typeof payload.text !== "string") {
|
|
651
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_summary_add requires hourStart and text")
|
|
652
|
+
}
|
|
653
|
+
await memory.addSummary({hourStart, text: payload.text, createdAt: num(payload.createdAt) ?? Date.now()})
|
|
654
|
+
return this.result(requestId, true)
|
|
655
|
+
}
|
|
656
|
+
case MiniappRequestType.MEMORY_SUMMARY_LIST:
|
|
657
|
+
return this.result(requestId, true, {summaries: await memory.summaryList()})
|
|
658
|
+
case MiniappRequestType.MEMORY_PENDING_SUMMARY_HOUR:
|
|
659
|
+
return this.result(requestId, true, {hourStart: await memory.pendingSummaryHour(num(payload.nowMs))})
|
|
660
|
+
case MiniappRequestType.MEMORY_CLEAR:
|
|
661
|
+
await memory.clear()
|
|
662
|
+
return this.result(requestId, true)
|
|
663
|
+
}
|
|
664
|
+
} catch (err) {
|
|
665
|
+
if (err instanceof hostRules.MemoryBackupError) {
|
|
666
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, err.message)
|
|
667
|
+
}
|
|
668
|
+
this.fail(requestId, MiniappErrorCode.INTERNAL, err instanceof Error ? err.message : "Memory error")
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/** The newest memory backup in the simulator folder: what the phone's file picker would hand over. */
|
|
673
|
+
private latestMemoryBackup(): string | null {
|
|
674
|
+
if (!this.stateDir || !existsSync(this.stateDir)) return null
|
|
675
|
+
const files = readdirSync(this.stateDir).filter((name) => /^glassly-memory-.*\.json$/.test(name)).sort()
|
|
676
|
+
return files.length ? join(this.stateDir, files[files.length - 1]) : null
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** The phone calling one of this miniapp's declared actions, as a system caller would. */
|
|
680
|
+
callAction(actionId: string, params: Record<string, unknown> = {}, timeoutMs = CALL_TIMEOUT_MS, app = this.packageName): Promise<unknown> {
|
|
681
|
+
return this.registry.callAction(app, actionId, params, timeoutMs)
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* The phone placing one of this miniapp's widgets in a box (answers the card
|
|
686
|
+
* frame the provider draws) or sending it an event (answers its result).
|
|
687
|
+
*/
|
|
688
|
+
callWidget(
|
|
689
|
+
widgetId: string,
|
|
690
|
+
call: {box: {size: "XS" | "XSM" | "SM" | "MD" | "LG"; width: number; height: number}} | {event: Record<string, unknown>},
|
|
691
|
+
timeoutMs = CALL_TIMEOUT_MS,
|
|
692
|
+
app = this.packageName,
|
|
693
|
+
): Promise<unknown> {
|
|
694
|
+
return "box" in call
|
|
695
|
+
? this.registry.placeForCli(app, widgetId, call.box, timeoutMs)
|
|
696
|
+
: this.registry.sendForCli(app, widgetId, call.event, timeoutMs)
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
get packageName(): string {
|
|
700
|
+
return this.manifest.packageName
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/** Connected and running: requests to it are answered. */
|
|
704
|
+
get connected(): boolean {
|
|
705
|
+
return this.handshook && this.background === "running"
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/** Sends the miniapp a message from the host. */
|
|
709
|
+
deliver(payload: Record<string, unknown>): void {
|
|
710
|
+
this.send(payload)
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
reply(requestId: string | undefined, ok: boolean, data?: unknown, error?: Record<string, unknown>): void {
|
|
714
|
+
this.result(requestId, ok, data, error)
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** Empties storage, settings and the memory timeline, as a fresh install would. */
|
|
718
|
+
async reset(): Promise<void> {
|
|
719
|
+
this.storage.clear()
|
|
720
|
+
this.settings.clear()
|
|
721
|
+
await this.memory?.clear()
|
|
722
|
+
this.log("host", "info", "Storage, settings and memory cleared")
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
private handleNotificationPost(payload: Record<string, unknown>, requestId?: string): void {
|
|
726
|
+
if (!this.requirePermission(requestId, "POST_NOTIFICATIONS")) return
|
|
727
|
+
const title = typeof payload.title === "string" ? payload.title.slice(0, NOTIFICATION_TITLE_MAX) : ""
|
|
728
|
+
const content = typeof payload.content === "string" ? payload.content.slice(0, NOTIFICATION_CONTENT_MAX) : ""
|
|
729
|
+
if (!title.trim() && !content.trim()) {
|
|
730
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "A notification needs a title or content")
|
|
731
|
+
}
|
|
732
|
+
if (payload.id !== undefined && (typeof payload.id !== "string" || !payload.id.trim())) {
|
|
733
|
+
return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "A notification id must be a non-empty string")
|
|
734
|
+
}
|
|
735
|
+
const id =
|
|
736
|
+
typeof payload.id === "string" ? payload.id.trim().slice(0, NOTIFICATION_ID_MAX) : `sim-${++this.notificationSeq}`
|
|
737
|
+
this.effect("notification", {id, title, content})
|
|
738
|
+
this.result(requestId, true, {id})
|
|
739
|
+
}
|
|
740
|
+
|
|
372
741
|
private handleConnect(requestId?: string): void {
|
|
742
|
+
this.derivedSettings = {}
|
|
743
|
+
this.publishDerivedSettings()
|
|
373
744
|
const permissions: Record<string, boolean> = {}
|
|
374
745
|
for (const canonical of new Set(Object.values(PERMISSION_TYPE_TO_CANONICAL))) permissions[canonical] = false
|
|
375
746
|
for (const type of this.declared) {
|
|
@@ -399,8 +770,9 @@ export class SimHost {
|
|
|
399
770
|
}
|
|
400
771
|
|
|
401
772
|
private capabilities(): Record<string, unknown> {
|
|
402
|
-
const display = this.device.capabilities.display as Record<string, unknown>
|
|
403
|
-
|
|
773
|
+
const display = this.device.capabilities.display as Record<string, unknown> | null
|
|
774
|
+
if (!display) return this.device.capabilities
|
|
775
|
+
const views = this.holdsDashboard() && this.device.dashboardView ? ["main", "dashboard"] : ["main"]
|
|
404
776
|
return {...this.device.capabilities, display: {...display, views}}
|
|
405
777
|
}
|
|
406
778
|
|
|
@@ -425,9 +797,11 @@ export class SimHost {
|
|
|
425
797
|
for (const stream of next.keys()) {
|
|
426
798
|
const required = permissionForStream(stream)
|
|
427
799
|
if (!required || this.declared.has(required)) continue
|
|
800
|
+
// The Glassly App gates the settings stream with its generic permission check.
|
|
801
|
+
const purpose = stream === MiniappStreamType.SETTINGS_CHANGED ? "" : ` (required for "${stream}")`
|
|
428
802
|
const error = {
|
|
429
803
|
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
430
|
-
message: `${required} permission not declared in miniapp.json
|
|
804
|
+
message: `${required} permission not declared in miniapp.json${purpose}. Add {"type": "${required}"} to the "permissions" array.`,
|
|
431
805
|
permission: required,
|
|
432
806
|
subscription: stream,
|
|
433
807
|
}
|
|
@@ -448,6 +822,9 @@ export class SimHost {
|
|
|
448
822
|
else if (stream === "glasses_connection") this.push(stream, this.connectionData())
|
|
449
823
|
else if (stream === "glasses_wifi") this.push(stream, {connected: false})
|
|
450
824
|
else if (stream === MiniappStreamType.MIRROR_UPDATE) this.push(stream, this.snapshot())
|
|
825
|
+
else if (stream === MiniappStreamType.SETTINGS_CHANGED) {
|
|
826
|
+
for (const key of hostRules.selectSettingsBridgeKeys?.(undefined) ?? []) this.push(stream, {key, value: this.settingValue(key)})
|
|
827
|
+
}
|
|
451
828
|
}
|
|
452
829
|
|
|
453
830
|
private connectionData(): Record<string, unknown> {
|
|
@@ -460,9 +837,9 @@ export class SimHost {
|
|
|
460
837
|
}
|
|
461
838
|
}
|
|
462
839
|
|
|
463
|
-
|
|
840
|
+
requirePermission(requestId: string | undefined, permission: string): boolean {
|
|
464
841
|
if (this.declared.has(permission)) return true
|
|
465
|
-
const message = `${permission} permission not declared in miniapp.json
|
|
842
|
+
const message = `${permission} permission not declared in miniapp.json. Add {"type": "${permission}"} to the "permissions" array.`
|
|
466
843
|
this.log("host", "error", message)
|
|
467
844
|
this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission})
|
|
468
845
|
return false
|
|
@@ -476,7 +853,7 @@ export class SimHost {
|
|
|
476
853
|
})
|
|
477
854
|
}
|
|
478
855
|
const view: View = payload.view === "dashboard" ? "dashboard" : "main"
|
|
479
|
-
if (view === "dashboard" && this.
|
|
856
|
+
if (view === "dashboard" && !this.holdsDashboard()) {
|
|
480
857
|
return this.result(requestId, true, {
|
|
481
858
|
status: "displayed",
|
|
482
859
|
presented: false,
|
|
@@ -484,31 +861,21 @@ export class SimHost {
|
|
|
484
861
|
})
|
|
485
862
|
}
|
|
486
863
|
const caps = this.device.scene
|
|
864
|
+
if (!caps) return this.result(requestId, true, {status: "blocked", reason: "no display connected"})
|
|
487
865
|
const appId = this.manifest.packageName
|
|
488
866
|
const input = resolveAnchors(payload.elements as SceneElementInput[], caps.width, caps.height)
|
|
867
|
+
if (!caps.canPosition) return this.renderTextWall(input, view, caps, Number(payload.durationMs), requestId)
|
|
489
868
|
const processed = processScene(input, caps, this.device.profile)
|
|
490
869
|
const {elements} = diffScene(
|
|
491
870
|
this.scenes.lastFrame(appId, view),
|
|
492
|
-
processed.elements,
|
|
871
|
+
this.device.joinsText ? joinText(processed.elements, caps) : processed.elements,
|
|
493
872
|
this.scenes.nextSyntheticId(appId, view),
|
|
494
873
|
)
|
|
495
874
|
this.scenes.commit(appId, view, elements, {width: caps.width, height: caps.height})
|
|
496
875
|
this.frameSeq++
|
|
497
876
|
|
|
498
|
-
this.
|
|
499
|
-
|
|
500
|
-
if (durationMs > 0 && elements.length > 0) {
|
|
501
|
-
this.expiry[view] = setTimeout(() => {
|
|
502
|
-
this.scenes.commit(appId, view, [])
|
|
503
|
-
if (this.visibleView() === view) this.publishDisplay()
|
|
504
|
-
}, durationMs)
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
const dropped = processed.dropped.join(", ")
|
|
508
|
-
if (dropped !== this.lastDropped) {
|
|
509
|
-
this.lastDropped = dropped
|
|
510
|
-
if (dropped) this.log("host", "warn", `render() dropped elements: ${dropped}`)
|
|
511
|
-
}
|
|
877
|
+
this.scheduleExpiry(view, Number(payload.durationMs), elements.length > 0)
|
|
878
|
+
this.reportDropped(processed.dropped)
|
|
512
879
|
|
|
513
880
|
const base = {status: "displayed", degraded: processed.degraded, dropped: processed.dropped}
|
|
514
881
|
if (this.visibleView() !== view) {
|
|
@@ -525,6 +892,56 @@ export class SimHost {
|
|
|
525
892
|
setTimeout(() => this.result(requestId, true, {...base, presented: true}), this.frameLatencyMs + settleMs)
|
|
526
893
|
}
|
|
527
894
|
|
|
895
|
+
/** Glasses that cannot position elements get one wrapped block of text, as the Glassly App sends them. */
|
|
896
|
+
private renderTextWall(
|
|
897
|
+
input: SceneElementInput[],
|
|
898
|
+
view: View,
|
|
899
|
+
caps: SceneDisplayCapabilities,
|
|
900
|
+
durationMs: number,
|
|
901
|
+
requestId?: string,
|
|
902
|
+
): void {
|
|
903
|
+
const appId = this.manifest.packageName
|
|
904
|
+
const {layout, degraded, dropped} = degradeScene(input)
|
|
905
|
+
const wrap = (text: unknown, maxWidthPx?: number) =>
|
|
906
|
+
this.wrapper.wrap(String(text ?? ""), maxWidthPx ? {maxWidthPx} : undefined).lines.join("\n")
|
|
907
|
+
const {width: w, height: h} = caps
|
|
908
|
+
const text = (id: string, value: string, x: number, width: number): FrameElement =>
|
|
909
|
+
({id, type: "text", text: value, box: {x, y: 0, w: width, h}, change: "created"}) as FrameElement
|
|
910
|
+
// The Glassly App composes two columns into one text wall; side by side halves approximate it.
|
|
911
|
+
const elements: FrameElement[] = !layout
|
|
912
|
+
? []
|
|
913
|
+
: layout.layoutType === "double_text_wall"
|
|
914
|
+
? [
|
|
915
|
+
text("left", wrap(layout.topText, this.device.profile.displayWidthPx / 2), 0, w / 2),
|
|
916
|
+
text("right", wrap(layout.bottomText, this.device.profile.displayWidthPx / 2), w / 2, w / 2),
|
|
917
|
+
]
|
|
918
|
+
: [text("text", wrap(layout.text), 0, w)]
|
|
919
|
+
this.scenes.commit(appId, view, elements, {width: w, height: h})
|
|
920
|
+
this.frameSeq++
|
|
921
|
+
this.scheduleExpiry(view, durationMs, elements.length > 0)
|
|
922
|
+
this.reportDropped(dropped)
|
|
923
|
+
if (this.visibleView() === view) this.publishDisplay()
|
|
924
|
+
this.settlesAt = Date.now()
|
|
925
|
+
this.result(requestId, true, {status: "displayed", degraded, dropped})
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
private scheduleExpiry(view: View, durationMs: number, hasContent: boolean): void {
|
|
929
|
+
this.clearExpiry(view)
|
|
930
|
+
if (!(durationMs > 0) || !hasContent) return
|
|
931
|
+
const appId = this.manifest.packageName
|
|
932
|
+
this.expiry[view] = setTimeout(() => {
|
|
933
|
+
this.scenes.commit(appId, view, [])
|
|
934
|
+
if (this.visibleView() === view) this.publishDisplay()
|
|
935
|
+
}, durationMs)
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
private reportDropped(list: string[]): void {
|
|
939
|
+
const dropped = list.join(", ")
|
|
940
|
+
if (dropped === this.lastDropped) return
|
|
941
|
+
this.lastDropped = dropped
|
|
942
|
+
if (dropped) this.log("host", "warn", `render() dropped elements: ${dropped}`)
|
|
943
|
+
}
|
|
944
|
+
|
|
528
945
|
private handleShowView(payload: Record<string, unknown>, requestId?: string): void {
|
|
529
946
|
const view = payload.view
|
|
530
947
|
if (view !== "main" && view !== "dashboard") {
|
|
@@ -533,12 +950,15 @@ export class SimHost {
|
|
|
533
950
|
message: 'showView takes "main" or "dashboard"',
|
|
534
951
|
})
|
|
535
952
|
}
|
|
536
|
-
if (this.
|
|
953
|
+
if (!this.holdsDashboard()) {
|
|
537
954
|
return this.result(requestId, false, undefined, {
|
|
538
955
|
code: MiniappErrorCode.NOT_PERMITTED,
|
|
539
956
|
message: 'The dashboard view belongs to miniapps of type "dashboard"',
|
|
540
957
|
})
|
|
541
958
|
}
|
|
959
|
+
if (view === "dashboard" && !this.device.dashboardView) {
|
|
960
|
+
return this.result(requestId, true, {status: "unsupported", reason: "glasses cannot show the dashboard"})
|
|
961
|
+
}
|
|
542
962
|
const overlay = view === "dashboard"
|
|
543
963
|
if (overlay !== this.overlay) {
|
|
544
964
|
this.overlay = overlay
|
|
@@ -549,6 +969,11 @@ export class SimHost {
|
|
|
549
969
|
this.result(requestId, true, {status: "shown"})
|
|
550
970
|
}
|
|
551
971
|
|
|
972
|
+
/** The app under test holds the dashboard role when it is a dashboard miniapp; the wearer picks one on the phone. */
|
|
973
|
+
private holdsDashboard(): boolean {
|
|
974
|
+
return this.manifest.type === "dashboard"
|
|
975
|
+
}
|
|
976
|
+
|
|
552
977
|
private clearExpiry(view: View): void {
|
|
553
978
|
const timer = this.expiry[view]
|
|
554
979
|
if (timer) clearTimeout(timer)
|
|
@@ -561,6 +986,19 @@ export class SimHost {
|
|
|
561
986
|
|
|
562
987
|
snapshot(): MirrorSnapshot {
|
|
563
988
|
const caps = this.device.scene
|
|
989
|
+
if (!caps) {
|
|
990
|
+
return {
|
|
991
|
+
revision: this.revision,
|
|
992
|
+
updatedAt: Date.now(),
|
|
993
|
+
connected: this.glassesConnected,
|
|
994
|
+
model: this.device.capabilities.modelName,
|
|
995
|
+
canvas: null,
|
|
996
|
+
view: null,
|
|
997
|
+
appId: null,
|
|
998
|
+
content: {kind: "unavailable", reason: "no-display"},
|
|
999
|
+
glyphWidths: {},
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
564
1002
|
const view = this.visibleView()
|
|
565
1003
|
const appId = this.manifest.packageName
|
|
566
1004
|
const glyphWidths: Record<string, number> = {}
|
|
@@ -593,6 +1031,7 @@ export class SimHost {
|
|
|
593
1031
|
const snapshot = this.snapshot()
|
|
594
1032
|
this.emit({type: "display", snapshot})
|
|
595
1033
|
this.push(MiniappStreamType.MIRROR_UPDATE, snapshot)
|
|
1034
|
+
this.publishDerivedSettings()
|
|
596
1035
|
}
|
|
597
1036
|
|
|
598
1037
|
private handleSpeak(text: string, requestId?: string): void {
|
|
@@ -719,61 +1158,185 @@ export class SimHost {
|
|
|
719
1158
|
}
|
|
720
1159
|
|
|
721
1160
|
touch(gesture: string, extra: Record<string, unknown> = {}): void {
|
|
1161
|
+
if (!this.device.gestures.includes(gesture)) {
|
|
1162
|
+
throw new Error(
|
|
1163
|
+
`${this.device.label} does not send "${gesture}". It sends: ${this.device.gestures.join(", ") || "no gestures"}`,
|
|
1164
|
+
)
|
|
1165
|
+
}
|
|
722
1166
|
const data = {...extra, kind: gesture, gestureName: gesture, timestamp: Date.now()}
|
|
723
1167
|
this.inject([MiniappStreamType.TOUCH_EVENT, `${MiniappStreamType.TOUCH_EVENT}:${gesture}`], data)
|
|
724
1168
|
}
|
|
725
1169
|
|
|
726
1170
|
button(pressType: "short" | "long" = "short", buttonId = "main"): void {
|
|
1171
|
+
if (!this.device.sendsButtonPress) throw new Error(`${this.device.label} has no button that sends presses`)
|
|
727
1172
|
this.inject([MiniappStreamType.BUTTON_PRESS], {buttonId, pressType, timestamp: Date.now()})
|
|
728
1173
|
}
|
|
729
1174
|
|
|
1175
|
+
/**
|
|
1176
|
+
* One transcription result, as the phone sends it: interims keep the open
|
|
1177
|
+
* utterance's id and a final closes it.
|
|
1178
|
+
*/
|
|
730
1179
|
transcription(text: string, options: {isFinal?: boolean; language?: string; speakerId?: string} = {}): void {
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
const
|
|
1180
|
+
const isFinal = options.isFinal ?? true
|
|
1181
|
+
this.openUtterance ??= `utt_sim_${++this.utteranceSeq}`
|
|
1182
|
+
const now = Date.now() - this.speechEpoch
|
|
1183
|
+
this.deliverTranscription({
|
|
734
1184
|
text,
|
|
735
|
-
isFinal
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
}
|
|
742
|
-
|
|
743
|
-
const explicit = streams.find(
|
|
744
|
-
(stream) => stream !== "transcription:auto" && stream.slice(14).split("-")[0].toLowerCase() === base,
|
|
745
|
-
)
|
|
746
|
-
const target = explicit ?? (streams.includes("transcription:auto") ? "transcription:auto" : null)
|
|
747
|
-
if (!target) return this.undelivered(`transcription:${language}`)
|
|
748
|
-
this.pushRouted(explicit ?? `transcription:${language}`, data, this.subscriptions.get(target))
|
|
1185
|
+
isFinal,
|
|
1186
|
+
utteranceId: this.openUtterance,
|
|
1187
|
+
language: options.language ?? "en-US",
|
|
1188
|
+
speakerId: options.speakerId ?? "1",
|
|
1189
|
+
startTime: now,
|
|
1190
|
+
endTime: now,
|
|
1191
|
+
})
|
|
1192
|
+
if (isFinal) this.openUtterance = null
|
|
749
1193
|
}
|
|
750
1194
|
|
|
1195
|
+
/**
|
|
1196
|
+
* One translation result, as the phone sends it: to `target`, or to every
|
|
1197
|
+
* translation target a miniapp subscribed to when absent.
|
|
1198
|
+
*/
|
|
751
1199
|
translation(
|
|
752
1200
|
text: string,
|
|
753
|
-
options: {source?: string; target?: string; isFinal?: boolean; originalText?: string} = {},
|
|
1201
|
+
options: {source?: string; target?: string; isFinal?: boolean; originalText?: string; speakerId?: string} = {},
|
|
754
1202
|
): void {
|
|
755
|
-
const
|
|
756
|
-
const
|
|
757
|
-
const
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
`
|
|
762
|
-
|
|
763
|
-
]
|
|
764
|
-
const match = patterns.find((pattern) => this.subscriptions.has(pattern))
|
|
765
|
-
if (!match) return this.undelivered(patterns[0])
|
|
766
|
-
this.pushRouted(
|
|
767
|
-
patterns[0],
|
|
768
|
-
{
|
|
1203
|
+
const isFinal = options.isFinal ?? true
|
|
1204
|
+
const source = options.source ?? "es"
|
|
1205
|
+
const targets = options.target ? [options.target] : this.translationTargets(source)
|
|
1206
|
+
if (targets.length === 0) return this.undelivered(`translation:${source}:*`)
|
|
1207
|
+
const now = Date.now() - this.speechEpoch
|
|
1208
|
+
for (const target of targets) {
|
|
1209
|
+
const id = (this.openTranslations[target] ??= `utt_sim_tr_${++this.utteranceSeq}`)
|
|
1210
|
+
this.deliverTranslation(source, target, {
|
|
769
1211
|
text,
|
|
770
|
-
isFinal: options.isFinal ?? true,
|
|
771
|
-
sourceLanguage: source,
|
|
772
|
-
targetLanguage: target,
|
|
773
1212
|
...(options.originalText ? {originalText: options.originalText} : {}),
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
1213
|
+
isFinal,
|
|
1214
|
+
utteranceId: id,
|
|
1215
|
+
speakerId: options.speakerId ?? "1",
|
|
1216
|
+
startTime: now,
|
|
1217
|
+
endTime: now,
|
|
1218
|
+
})
|
|
1219
|
+
if (isFinal) delete this.openTranslations[target]
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
/**
|
|
1224
|
+
* Plays a transcript script as speech: each line's words arrive as growing
|
|
1225
|
+
* interims at the speaking rate, then its final half a second later, on
|
|
1226
|
+
* every transcription and translation stream a miniapp listens to.
|
|
1227
|
+
*/
|
|
1228
|
+
play(lines: readonly ScriptLine[], options: {wordsPerMinute?: number; gapMs?: number} = {}): Promise<{utterances: number; durationMs: number}> {
|
|
1229
|
+
this.stopPlaying()
|
|
1230
|
+
const msPerWord = 60_000 / Math.max(30, options.wordsPerMinute ?? DEFAULT_WORDS_PER_MINUTE)
|
|
1231
|
+
const gapMs = options.gapMs ?? 300
|
|
1232
|
+
const speakers = new Map<string, string>()
|
|
1233
|
+
let cursor = 0
|
|
1234
|
+
let language = "en"
|
|
1235
|
+
const steps: {at: number; run: () => void}[] = []
|
|
1236
|
+
for (const line of lines) {
|
|
1237
|
+
language = line.language ?? language
|
|
1238
|
+
const spoken = language
|
|
1239
|
+
const start = Math.max(line.atMs ?? cursor, cursor)
|
|
1240
|
+
const words = line.text.split(/\s+/).filter(Boolean)
|
|
1241
|
+
const speakerId = speakers.get(line.speaker ?? "") ?? String(speakers.size + 1)
|
|
1242
|
+
speakers.set(line.speaker ?? "", speakerId)
|
|
1243
|
+
const id = `utt_sim_${++this.utteranceSeq}`
|
|
1244
|
+
const translationIds: Record<string, string> = {}
|
|
1245
|
+
const finalAt = start + words.length * msPerWord + ENDPOINT_DEBOUNCE_MS
|
|
1246
|
+
const emit = (count: number, isFinal: boolean, at: number) => {
|
|
1247
|
+
const result = {isFinal, speakerId, startTime: Math.round(start), endTime: Math.round(at)}
|
|
1248
|
+
this.deliverTranscription({...result, text: words.slice(0, count).join(" "), utteranceId: id, language: spoken})
|
|
1249
|
+
for (const target of this.translationTargets(spoken)) {
|
|
1250
|
+
translationIds[target] ??= `utt_sim_tr_${++this.utteranceSeq}`
|
|
1251
|
+
const passthrough = baseLanguage(target) === baseLanguage(spoken)
|
|
1252
|
+
const original = words.slice(0, count).join(" ")
|
|
1253
|
+
const full =
|
|
1254
|
+
line.translations[target] ??
|
|
1255
|
+
line.translations[baseLanguage(target)] ??
|
|
1256
|
+
`[${baseLanguage(spoken)}→${baseLanguage(target)}] ${line.text}`
|
|
1257
|
+
const translated = full.split(/\s+/)
|
|
1258
|
+
// Translations trail the source text by a word, as the real service does.
|
|
1259
|
+
const shown = isFinal ? translated.length : Math.max(0, Math.round(((count - 1) / words.length) * translated.length))
|
|
1260
|
+
this.deliverTranslation(spoken, target, {
|
|
1261
|
+
...result,
|
|
1262
|
+
text: passthrough ? original : translated.slice(0, shown).join(" "),
|
|
1263
|
+
...(passthrough ? {} : {originalText: original}),
|
|
1264
|
+
utteranceId: translationIds[target],
|
|
1265
|
+
})
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
words.forEach((_, i) => {
|
|
1269
|
+
const at = start + (i + 1) * msPerWord
|
|
1270
|
+
steps.push({at, run: () => emit(i + 1, false, at)})
|
|
1271
|
+
})
|
|
1272
|
+
steps.push({at: finalAt, run: () => emit(words.length, true, finalAt)})
|
|
1273
|
+
cursor = finalAt + gapMs
|
|
1274
|
+
}
|
|
1275
|
+
this.speechEpoch = Date.now()
|
|
1276
|
+
if (![...this.subscriptions.keys()].some((stream) => /^(transcription|translation):/.test(stream))) {
|
|
1277
|
+
this.log("host", "warn", "Nothing is subscribed to transcriptions or translations; the script plays to no one")
|
|
1278
|
+
}
|
|
1279
|
+
return new Promise((resolve) => {
|
|
1280
|
+
const done = () => {
|
|
1281
|
+
this.playback = null
|
|
1282
|
+
resolve({utterances: lines.length, durationMs: Math.round(cursor)})
|
|
1283
|
+
}
|
|
1284
|
+
const timers = steps.map((step) => setTimeout(step.run, step.at))
|
|
1285
|
+
timers.push(setTimeout(done, cursor))
|
|
1286
|
+
this.playback = {timers, done}
|
|
1287
|
+
})
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
stopPlaying(): void {
|
|
1291
|
+
const playback = this.playback
|
|
1292
|
+
if (!playback) return
|
|
1293
|
+
for (const timer of playback.timers) clearTimeout(timer)
|
|
1294
|
+
playback.done()
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
/** Concrete translation targets subscribed to; the phone runs no translation for a wildcard target. */
|
|
1298
|
+
private translationTargets(spoken: string): string[] {
|
|
1299
|
+
const targets = new Set<string>()
|
|
1300
|
+
for (const stream of this.subscriptions.keys()) {
|
|
1301
|
+
const [kind, source, target] = stream.split(":")
|
|
1302
|
+
if (kind !== "translation" || !target || target === "*") continue
|
|
1303
|
+
if (source !== "*" && source !== "auto" && baseLanguage(source) !== baseLanguage(spoken)) continue
|
|
1304
|
+
targets.add(target)
|
|
1305
|
+
}
|
|
1306
|
+
return [...targets]
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// An auto subscriber gets the detected bare code; with none, a specific
|
|
1310
|
+
// subscription in the spoken language gets its own tag (LocalMiniappRuntime).
|
|
1311
|
+
private deliverTranscription(result: {
|
|
1312
|
+
text: string
|
|
1313
|
+
isFinal: boolean
|
|
1314
|
+
utteranceId: string
|
|
1315
|
+
language: string
|
|
1316
|
+
speakerId: string
|
|
1317
|
+
startTime: number
|
|
1318
|
+
endTime: number
|
|
1319
|
+
}): void {
|
|
1320
|
+
const {language, ...rest} = result
|
|
1321
|
+
const base = baseLanguage(language)
|
|
1322
|
+
const data = {type: "transcription", ...rest, provider: "soniox"}
|
|
1323
|
+
const streams = [...this.subscriptions.keys()].filter((stream) => stream.startsWith("transcription:"))
|
|
1324
|
+
if (streams.includes("transcription:auto")) {
|
|
1325
|
+
const specific = streams.find((stream) => stream !== "transcription:auto" && baseLanguage(stream.slice(14)) === base)
|
|
1326
|
+
return this.pushRouted(specific ?? `transcription:${base}`, {...data, transcribeLanguage: base, detectedLanguage: base}, this.subscriptions.get("transcription:auto"))
|
|
1327
|
+
}
|
|
1328
|
+
const matching = streams.filter((stream) => baseLanguage(stream.slice(14)) === base)
|
|
1329
|
+
if (matching.length === 0) return this.undelivered(`transcription:${language}`)
|
|
1330
|
+
for (const stream of matching) this.pushRouted(stream, {...data, transcribeLanguage: stream.slice(14)}, this.subscriptions.get(stream))
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
private deliverTranslation(spoken: string, target: string, result: Record<string, unknown>): void {
|
|
1334
|
+
const source = [...this.subscriptions.keys()].some((stream) => stream.startsWith(`translation:${spoken}:`)) ? spoken : baseLanguage(spoken)
|
|
1335
|
+
const route = [...this.subscriptions.entries()].find(([stream]) => stream.startsWith("translation:") && stream.endsWith(`:${target}`))
|
|
1336
|
+
this.pushRouted(
|
|
1337
|
+
`translation:${source}:${target}`,
|
|
1338
|
+
{type: "translation", ...result, transcribeLanguage: source, translateLanguage: target, didTranslate: true, provider: "soniox"},
|
|
1339
|
+
route?.[1],
|
|
777
1340
|
)
|
|
778
1341
|
}
|
|
779
1342
|
|
|
@@ -828,6 +1391,7 @@ export class SimHost {
|
|
|
828
1391
|
setVisibility(visibility: "foreground" | "background"): void {
|
|
829
1392
|
this.visibility = visibility
|
|
830
1393
|
this.send({type: MiniappResponseType.VISIBILITY_CHANGE, visibility})
|
|
1394
|
+
this.publishDerivedSettings()
|
|
831
1395
|
this.emit({type: "status"})
|
|
832
1396
|
}
|
|
833
1397
|
|
|
@@ -855,6 +1419,7 @@ export class SimHost {
|
|
|
855
1419
|
}
|
|
856
1420
|
|
|
857
1421
|
private undelivered(stream: string): void {
|
|
1422
|
+
if (this.playback) return
|
|
858
1423
|
this.log("host", "warn", `Nothing is subscribed to "${stream}"; the event was not delivered`)
|
|
859
1424
|
}
|
|
860
1425
|
|
|
@@ -884,6 +1449,10 @@ export class SimHost {
|
|
|
884
1449
|
this.toBackground(JSON.stringify(requestId ? {payload, requestId} : {payload}))
|
|
885
1450
|
}
|
|
886
1451
|
|
|
1452
|
+
private fail(requestId: string | undefined, code: string, message: string): void {
|
|
1453
|
+
this.result(requestId, false, undefined, {code, message})
|
|
1454
|
+
}
|
|
1455
|
+
|
|
887
1456
|
private result(requestId: string | undefined, ok: boolean, data?: unknown, error?: Record<string, unknown>): void {
|
|
888
1457
|
if (!requestId) return
|
|
889
1458
|
this.send(
|
|
@@ -929,7 +1498,14 @@ export class SimHost {
|
|
|
929
1498
|
packageName: this.manifest.packageName,
|
|
930
1499
|
name: this.manifest.name,
|
|
931
1500
|
version: this.manifest.version,
|
|
932
|
-
device: {
|
|
1501
|
+
device: {
|
|
1502
|
+
id: this.device.id,
|
|
1503
|
+
label: this.device.label,
|
|
1504
|
+
model: this.device.capabilities.modelName,
|
|
1505
|
+
canvas: snapshot.canvas,
|
|
1506
|
+
gestures: this.device.gestures,
|
|
1507
|
+
buttonPress: this.device.sendsButtonPress,
|
|
1508
|
+
},
|
|
933
1509
|
background: this.background,
|
|
934
1510
|
connected: this.handshook,
|
|
935
1511
|
paused: !this.glassesConnected,
|