@glassly/miniapp-cli 0.1.14-dev.0 → 0.1.15-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/sim/host.ts CHANGED
@@ -1,12 +1,20 @@
1
1
  import {
2
+ degradeScene,
2
3
  diffScene,
3
4
  processScene,
4
5
  resolveAnchors,
5
6
  SceneStore,
6
7
  TextMeasurer,
8
+ TextWrapper,
9
+ type DiffableElement,
7
10
  type FrameElement,
11
+ type SceneDisplayCapabilities,
8
12
  type SceneElementInput,
9
13
  } from "@glassly/miniapp/display"
14
+ // A namespace import, so an older @glassly/miniapp without the host rules still loads.
15
+ import {existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync} from "fs"
16
+ import {join} from "path"
17
+ import * as hostRules from "@glassly/miniapp/host"
10
18
  import type {MirrorElement, MirrorSnapshot} from "@glassly/miniapp/mirror"
11
19
  import {
12
20
  MiniappErrorCode,
@@ -17,7 +25,9 @@ import {
17
25
 
18
26
  import type {SimDevice} from "./device.js"
19
27
  import type {PhoneKind} from "./phone.js"
20
- import type {SimStorage} from "./storage.js"
28
+ import type {ScriptLine} from "./script.js"
29
+ import {SimRegistry} from "./registry.js"
30
+ import {createSimStorage, type SimStorage} from "./storage.js"
21
31
 
22
32
  export type SimLogSource = "background" | "ui" | "host" | "bridge"
23
33
 
@@ -49,12 +59,21 @@ export interface SimManifest {
49
59
  version?: string
50
60
  type?: string
51
61
  permissions?: {type?: string}[]
62
+ actions?: unknown
63
+ widgets?: unknown
52
64
  }
53
65
 
54
66
  export interface SimHostOptions {
55
67
  manifest: SimManifest
56
68
  device: SimDevice
57
69
  storage: SimStorage
70
+ /** Settings and the memory timeline; in memory when absent. */
71
+ settings?: SimStorage
72
+ memory?: SimStorage
73
+ /** Where memory exports are written and imports read; none disables them. */
74
+ stateDir?: string
75
+ /** The simulator's installed miniapps; a registry of this one alone when absent. */
76
+ registry?: SimRegistry
58
77
  toBackground: (raw: string) => void
59
78
  emit: (event: SimHostEvent) => void
60
79
  frameLatencyMs?: number
@@ -71,6 +90,21 @@ export interface SimLocation {
71
90
  type View = "main" | "dashboard"
72
91
 
73
92
  const LOG_LIMIT = 2000
93
+ const NOTIFICATION_TITLE_MAX = 200
94
+ const NOTIFICATION_CONTENT_MAX = 2_000
95
+ const NOTIFICATION_ID_MAX = 128
96
+ const CALL_TIMEOUT_MS = 10_000
97
+ const DEFAULT_WORDS_PER_MINUTE = 160
98
+ const ENDPOINT_DEBOUNCE_MS = 500
99
+
100
+ // Stands in for the phone's installed apps on the iOS catalog, which has no icons.
101
+ const SAMPLE_NOTIFICATION_APPS = [
102
+ {packageName: "com.apple.MobileSMS", appName: "Messages", icon: null},
103
+ {packageName: "com.apple.mobilemail", appName: "Mail", icon: null},
104
+ {packageName: "com.apple.mobilecal", appName: "Calendar", icon: null},
105
+ {packageName: "net.whatsapp.WhatsApp", appName: "WhatsApp", icon: null},
106
+ {packageName: "com.tinyspeck.chatanywhere", appName: "Slack", icon: null},
107
+ ]
74
108
  const EFFECT_LIMIT = 200
75
109
  const MAX_SETTLE_MS = 255 * 33
76
110
 
@@ -102,6 +136,26 @@ function permissionForStream(stream: string): string | null {
102
136
  return null
103
137
  }
104
138
 
139
+ /** What stock RayNeo iO firmware shows: the frame's text in paint order on its one subtitle page. */
140
+ function joinText(elements: DiffableElement[], caps: SceneDisplayCapabilities): DiffableElement[] {
141
+ const text = elements
142
+ .filter((el) => el.type === "text" && el.text)
143
+ .map((el) => (el as {text: string}).text)
144
+ .join("\n")
145
+ if (!text) return []
146
+ return [{id: "subtitle", type: "text", text, box: {x: 0, y: 0, w: caps.width, h: caps.height}} as DiffableElement]
147
+ }
148
+
149
+ function baseLanguage(tag: string): string {
150
+ return tag.split("-")[0].toLowerCase()
151
+ }
152
+
153
+ function localDate(ms: number): string {
154
+ const d = new Date(ms)
155
+ const pad = (n: number) => String(n).padStart(2, "0")
156
+ return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`
157
+ }
158
+
105
159
  function settleTimeOf(elements: FrameElement[]): number {
106
160
  let ms = 0
107
161
  for (const el of elements) {
@@ -140,7 +194,8 @@ interface StreamRoute {
140
194
  * requests it covers. No I/O of its own; the server and the tests drive it.
141
195
  */
142
196
  export class SimHost {
143
- private readonly manifest: SimManifest
197
+ readonly manifest: SimManifest
198
+ private readonly registry: SimRegistry
144
199
  private readonly device: SimDevice
145
200
  private readonly storage: SimStorage
146
201
  private readonly toBackground: (raw: string) => void
@@ -148,13 +203,19 @@ export class SimHost {
148
203
  private readonly frameLatencyMs: number
149
204
  private readonly speechMsPerChar: number
150
205
  private readonly declared: Set<string>
206
+ private readonly settings: SimStorage
207
+ private readonly memory: InstanceType<typeof hostRules.MemoryStore> | null
208
+ private readonly stateDir: string | undefined
151
209
  private readonly measurer: TextMeasurer
210
+ private readonly wrapper: TextWrapper
152
211
 
153
212
  private scenes = new SceneStore()
154
213
  private handshook = false
155
214
  private background: "stopped" | "running" | "crashed" = "stopped"
156
215
  private subscriptions = new Map<string, StreamRoute>()
157
216
  private claimedGestures: string[] = []
217
+ private derivedSettings: Record<string, unknown> = {}
218
+ private gestureClaims = hostRules.emptyGestureClaims?.()
158
219
  private overlay = false
159
220
  private revision = 0
160
221
  private frameSeq = 0
@@ -172,6 +233,11 @@ export class SimHost {
172
233
  private readonly unsimulated = new Set<string>()
173
234
 
174
235
  private speakerState = "idle"
236
+ private utteranceSeq = 0
237
+ private openUtterance: string | null = null
238
+ private openTranslations: Record<string, string> = {}
239
+ private speechEpoch = Date.now()
240
+ private playback: {timers: ReturnType<typeof setTimeout>[]; done: () => void} | null = null
175
241
  private speech: {timer: ReturnType<typeof setTimeout>; finish: (completed: boolean) => void} | null = null
176
242
 
177
243
  private glassesConnected = true
@@ -187,12 +253,25 @@ export class SimHost {
187
253
  this.manifest = options.manifest
188
254
  this.device = options.device
189
255
  this.storage = options.storage
256
+ this.settings = options.settings ?? createSimStorage()
257
+ this.stateDir = options.stateDir
258
+ const memory = options.memory ?? createSimStorage()
259
+ this.memory = hostRules.MemoryStore
260
+ ? new hostRules.MemoryStore({
261
+ getUserId: async () => "sim-user",
262
+ backend: {get: (key) => memory.get(key), set: (key, value) => memory.set(key, value), remove: (key) => memory.delete(key)},
263
+ writeDebounceMs: 0,
264
+ })
265
+ : null
266
+ this.registry = options.registry ?? new SimRegistry()
267
+ this.registry.add(this)
190
268
  this.toBackground = options.toBackground
191
269
  this.emit = options.emit
192
270
  this.frameLatencyMs = options.frameLatencyMs ?? 30
193
271
  this.speechMsPerChar = options.speechMsPerChar ?? 55
194
272
  this.declared = new Set((options.manifest.permissions ?? []).map((p) => String(p.type ?? "").toUpperCase()))
195
273
  this.measurer = new TextMeasurer(options.device.profile)
274
+ this.wrapper = new TextWrapper(this.measurer, {breakMode: "character-no-hyphen"})
196
275
  }
197
276
 
198
277
  backgroundStarted(): void {
@@ -211,10 +290,14 @@ export class SimHost {
211
290
  this.handshook = false
212
291
  this.subscriptions.clear()
213
292
  this.claimedGestures = []
293
+ this.gestureClaims = hostRules.emptyGestureClaims?.()
214
294
  this.overlay = false
215
295
  this.scenes = new SceneStore()
216
296
  for (const view of ["main", "dashboard"] as View[]) this.clearExpiry(view)
217
297
  this.finishSpeech(false)
298
+ this.stopPlaying()
299
+ this.openUtterance = null
300
+ this.openTranslations = {}
218
301
  this.speakerState = "idle"
219
302
  this.lastDropped = ""
220
303
  for (const settle of [...this.uiRequests.values()]) settle({ok: false, error: {message: "background restarted"}})
@@ -244,31 +327,41 @@ export class SimHost {
244
327
  }
245
328
 
246
329
  private dispatch(type: string, payload: Record<string, unknown>, requestId?: string): void {
330
+ if (this.registry.handle(this, type, payload, requestId)) return
247
331
  switch (type) {
248
332
  case MiniappRequestType.CONNECT:
249
333
  return this.handleConnect(requestId)
250
334
  case MiniappRequestType.SUBSCRIBE:
251
335
  return this.handleSubscribe(payload, requestId)
252
336
  case MiniappRequestType.CLAIM_GESTURES:
253
- this.claimedGestures = Array.isArray(payload.gestures)
254
- ? payload.gestures.filter((g): g is string => typeof g === "string")
255
- : []
256
- this.emit({type: "status"})
257
- return this.result(requestId, true)
337
+ if (payload.scope !== undefined && (typeof payload.scope !== "string" || payload.scope.length === 0)) {
338
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "Gesture claim scope must be a nonempty string")
339
+ }
340
+ return this.claimGestures(payload, requestId)
258
341
  case MiniappRequestType.TEXT_FIELD:
342
+ if (typeof payload.open !== "boolean") {
343
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "Text field open must be a boolean")
344
+ }
259
345
  return this.result(requestId, true)
260
346
  case MiniappRequestType.RENDER:
261
347
  return this.handleRender(payload, requestId)
262
348
  case MiniappRequestType.SHOW_VIEW:
263
349
  return this.handleShowView(payload, requestId)
264
350
  case MiniappRequestType.MIRROR_GET:
265
- if (!this.requirePermission(requestId, "DISPLAY_MIRROR", "to read the display mirror")) return
351
+ if (!this.declared.has("DISPLAY_MIRROR")) {
352
+ return this.result(requestId, false, undefined, {
353
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
354
+ message: "DISPLAY_MIRROR permission not declared in miniapp.json",
355
+ permission: "DISPLAY_MIRROR",
356
+ })
357
+ }
266
358
  return this.result(requestId, true, this.snapshot())
267
359
  case MiniappRequestType.PING:
268
360
  return this.send({type: MiniappResponseType.PONG}, requestId)
269
361
  case MiniappResponseType.PONG:
270
- case MiniappRequestType.TRANSCRIPTION_CONFIG:
271
362
  return
363
+ case MiniappRequestType.TRANSCRIPTION_CONFIG:
364
+ return this.result(requestId, true, {applied: true})
272
365
  case MiniappRequestType.AUTH_REFRESH:
273
366
  return this.result(requestId, false, undefined, {
274
367
  code: MiniappErrorCode.NOT_CONNECTED,
@@ -276,12 +369,12 @@ export class SimHost {
276
369
  })
277
370
 
278
371
  case MiniappRequestType.STORAGE_GET:
279
- return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(payload))})
372
+ return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(type, payload))})
280
373
  case MiniappRequestType.STORAGE_SET:
281
- this.storage.set(this.key(payload), payload.value == null ? null : String(payload.value))
374
+ this.storage.set(this.key(type, payload), payload.value ?? null)
282
375
  return this.result(requestId, true)
283
376
  case MiniappRequestType.STORAGE_DELETE:
284
- this.storage.delete(this.key(payload))
377
+ this.storage.delete(this.key(type, payload))
285
378
  return this.result(requestId, true)
286
379
  case MiniappRequestType.STORAGE_LIST:
287
380
  return this.result(requestId, true, {keys: this.storage.keys()})
@@ -289,25 +382,32 @@ export class SimHost {
289
382
  this.storage.clear()
290
383
  return this.result(requestId, true)
291
384
  case MiniappRequestType.STORAGE_HAS:
292
- return this.result(requestId, true, {has: this.storage.get(this.key(payload)) !== null})
293
- case MiniappRequestType.STORAGE_GET_ALL:
294
- return this.result(requestId, true, {values: this.storage.getAll()})
385
+ return this.result(requestId, true, {has: this.storage.has(this.key(type, payload))})
386
+ case MiniappRequestType.STORAGE_GET_ALL: {
387
+ const values: Record<string, string> = {}
388
+ for (const [key, value] of Object.entries(this.storage.getAll())) {
389
+ values[key] = typeof value === "string" ? value : String(value ?? "")
390
+ }
391
+ return this.result(requestId, true, {values})
392
+ }
295
393
  case MiniappRequestType.STORAGE_SET_MULTIPLE: {
296
394
  const values = payload.values
297
395
  if (!values || typeof values !== "object") throw new Error("storage_set_multiple requires a values object")
298
- for (const [key, value] of Object.entries(values)) this.storage.set(key, value == null ? null : String(value))
396
+ for (const [key, value] of Object.entries(values)) this.storage.set(key, value ?? null)
299
397
  return this.result(requestId, true)
300
398
  }
301
399
  case MiniappRequestType.STORAGE_FLUSH:
302
400
  return this.result(requestId, true)
303
401
 
304
402
  case MiniappRequestType.LOCATION_POLL:
305
- if (!this.requirePermission(requestId, "LOCATION", "to read the location")) return
306
403
  return this.result(requestId, true, this.location)
307
404
 
308
405
  case MiniappRequestType.SPEAK:
309
406
  return this.handleSpeak(String(payload.text ?? ""), requestId)
310
407
  case MiniappRequestType.PLAY_AUDIO:
408
+ if (typeof payload.audioUrl !== "string" || !payload.audioUrl) {
409
+ return this.fail(requestId, MiniappErrorCode.INTERNAL, "play_audio requires an audioUrl")
410
+ }
311
411
  this.effect("audio", {url: payload.audioUrl})
312
412
  return this.startSpeech(1500, requestId, undefined)
313
413
  case MiniappRequestType.STOP_AUDIO:
@@ -330,23 +430,69 @@ export class SimHost {
330
430
  this.effect("download", {...payload, type: undefined})
331
431
  return this.result(requestId, true, {success: true})
332
432
 
333
- case MiniappRequestType.NOTIFICATION_POST: {
334
- if (!this.requirePermission(requestId, "POST_NOTIFICATIONS", "to post a notification")) return
335
- const id = typeof payload.id === "string" && payload.id ? payload.id : `sim-${++this.notificationSeq}`
336
- this.effect("notification", {id, title: payload.title, content: payload.content})
337
- return this.result(requestId, true, {id})
338
- }
433
+ case MiniappRequestType.NOTIFICATION_POST:
434
+ return this.handleNotificationPost(payload, requestId)
339
435
  case MiniappRequestType.NOTIFICATION_DISMISS:
340
- if (!this.requirePermission(requestId, "POST_NOTIFICATIONS", "to dismiss a notification")) return
341
- this.effect("notification_dismiss", {id: payload.id})
436
+ if (!this.requirePermission(requestId, "POST_NOTIFICATIONS")) return
437
+ if (typeof payload.id !== "string" || !payload.id.trim()) {
438
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "dismiss needs the id post() returned")
439
+ }
440
+ this.effect("notification_dismiss", {id: payload.id.trim()})
342
441
  return this.result(requestId, true)
343
442
 
443
+ case MiniappRequestType.SETTINGS_GET: {
444
+ const {selectSettingsBridgeKeys} = hostRules
445
+ if (!selectSettingsBridgeKeys) break
446
+ if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
447
+ const values: Record<string, unknown> = {}
448
+ for (const key of selectSettingsBridgeKeys(payload.keys)) values[key] = this.settingValue(key)
449
+ return this.result(requestId, true, {values})
450
+ }
451
+ case MiniappRequestType.SETTINGS_SET: {
452
+ const {validateSettingsSet} = hostRules
453
+ if (!validateSettingsSet) break
454
+ if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
455
+ const validated = validateSettingsSet(payload)
456
+ if (!validated.ok) return this.fail(requestId, validated.code, validated.message)
457
+ const changed = JSON.stringify(this.settingValue(validated.key)) !== JSON.stringify(validated.value ?? null)
458
+ this.settings.set(validated.key, validated.value)
459
+ this.result(requestId, true)
460
+ if (changed) this.push(MiniappStreamType.SETTINGS_CHANGED, {key: validated.key, value: this.settingValue(validated.key)})
461
+ return
462
+ }
463
+
464
+ case MiniappRequestType.NOTIFICATION_APPS_LIST: {
465
+ if (!this.requirePermission(requestId, "READ_NOTIFICATIONS")) return
466
+ const posts = this.declared.has("POST_NOTIFICATIONS")
467
+ const self = {packageName: this.manifest.packageName, appName: this.manifest.name || this.manifest.packageName, icon: null}
468
+ return this.result(requestId, true, [...SAMPLE_NOTIFICATION_APPS, ...(posts ? [self] : [])])
469
+ }
470
+
471
+
472
+
344
473
  case MiniappRequestType.SYSTEM_EXIT:
345
474
  case MiniappRequestType.MINIAPPS_STOP_SELF:
346
475
  this.result(requestId, true)
347
476
  this.log("host", "info", "The miniapp asked to stop (session.system.exit)")
348
477
  return this.emit({type: "exit", reason: "the miniapp called exit"})
349
478
 
479
+ case MiniappRequestType.MEMORY_APPEND_LINE:
480
+ case MiniappRequestType.MEMORY_ADD_NOTE:
481
+ case MiniappRequestType.MEMORY_REMOVE_ENTRY:
482
+ case MiniappRequestType.MEMORY_REMOVE_SCOPE:
483
+ case MiniappRequestType.MEMORY_SESSION_LIST:
484
+ case MiniappRequestType.MEMORY_QUERY:
485
+ case MiniappRequestType.MEMORY_WINDOW:
486
+ case MiniappRequestType.MEMORY_EXPORT_TEXT:
487
+ case MiniappRequestType.MEMORY_EXPORT_FILE:
488
+ case MiniappRequestType.MEMORY_IMPORT_FILE:
489
+ case MiniappRequestType.MEMORY_SUMMARY_ADD:
490
+ case MiniappRequestType.MEMORY_SUMMARY_LIST:
491
+ case MiniappRequestType.MEMORY_PENDING_SUMMARY_HOUR:
492
+ case MiniappRequestType.MEMORY_CLEAR:
493
+ if (!this.memory) break
494
+ return void this.handleMemory(this.memory, type, payload, requestId)
495
+
350
496
  case "UI_SEND":
351
497
  return this.handleUiSend(payload)
352
498
  case "UI_CANCEL":
@@ -364,12 +510,237 @@ export class SimHost {
364
510
  })
365
511
  }
366
512
 
367
- private key(payload: Record<string, unknown>): string {
368
- if (typeof payload.key !== "string" || !payload.key) throw new Error("storage request requires a key")
513
+ private claimGestures(payload: Record<string, unknown>, requestId?: string): void {
514
+ const gestures = Array.isArray(payload.gestures) ? payload.gestures.filter((g): g is string => typeof g === "string") : []
515
+ const {setGestureClaims, gestureClaimant} = hostRules
516
+ if (!this.gestureClaims || !setGestureClaims || !gestureClaimant) {
517
+ this.claimedGestures = gestures
518
+ } else {
519
+ const {state, changes} = setGestureClaims(this.gestureClaims, this.manifest.packageName, gestures, payload.scope as string | undefined)
520
+ this.gestureClaims = state
521
+ const held = new Set(state.registrations.flatMap((entry) => [...entry.gestures]))
522
+ this.claimedGestures = [...held].filter((gesture) => gestureClaimant(state, gesture) === this.manifest.packageName)
523
+ for (const change of changes) this.push(MiniappStreamType.GESTURE_CLAIM, {gesture: change.gesture, held: change.held})
524
+ }
525
+ this.emit({type: "status"})
526
+ this.result(requestId, true)
527
+ }
528
+
529
+ private key(type: string, payload: Record<string, unknown>): string {
530
+ if (typeof payload.key !== "string" || !payload.key) throw new Error(`${type.replace(/^miniapp_/, "")} requires a key`)
369
531
  return payload.key
370
532
  }
371
533
 
534
+ /** A bridged key's value: stored, else the Glassly App's default; derived keys follow simulator state. */
535
+ // The phone pushes settings_changed when a derived key's value moves; these follow simulator state.
536
+ private publishDerivedSettings(): void {
537
+ for (const key of ["phone_active", "foreground_app_on_screen"]) {
538
+ const value = this.settingValue(key)
539
+ if (this.derivedSettings[key] === value) continue
540
+ const known = key in this.derivedSettings
541
+ this.derivedSettings[key] = value
542
+ if (known) this.push(MiniappStreamType.SETTINGS_CHANGED, {key, value})
543
+ }
544
+ }
545
+
546
+ private settingValue(key: string): unknown {
547
+ if (key === "phone_active") return this.visibility === "foreground"
548
+ if (key === "foreground_app_on_screen") return this.visibleView() === "main" && this.scenes.lastFrame(this.manifest.packageName, "main").length > 0
549
+ if (key === "controllers") return []
550
+ return this.settings.has(key) ? this.settings.get(key) : (hostRules.settingsBridgeDefault?.(key) ?? null)
551
+ }
552
+
553
+ // Mirrors the MEMORY_* handlers in the Glassly App's LocalMiniappRuntime.
554
+ private async handleMemory(
555
+ memory: InstanceType<typeof hostRules.MemoryStore>,
556
+ type: string,
557
+ payload: Record<string, unknown>,
558
+ requestId?: string,
559
+ ): Promise<void> {
560
+ if (!this.declared.has("MEMORY")) {
561
+ return this.result(requestId, false, undefined, {
562
+ code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
563
+ message: `MEMORY permission not declared in miniapp.json. Add {"type": "MEMORY"} to the "permissions" array.`,
564
+ permission: "MEMORY",
565
+ operation: type,
566
+ })
567
+ }
568
+ const str = (v: unknown): string => (typeof v === "string" ? v : "")
569
+ const num = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) ? v : undefined)
570
+ try {
571
+ switch (type) {
572
+ case MiniappRequestType.MEMORY_APPEND_LINE: {
573
+ const text = str(payload.text)
574
+ const at = num(payload.at)
575
+ if (!text || at == null) {
576
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_append_line requires text and at")
577
+ }
578
+ await memory.appendLine({
579
+ text,
580
+ speakerId: typeof payload.speakerId === "string" ? payload.speakerId : null,
581
+ language: typeof payload.language === "string" ? payload.language : null,
582
+ at,
583
+ })
584
+ return this.result(requestId, true)
585
+ }
586
+ case MiniappRequestType.MEMORY_ADD_NOTE: {
587
+ const text = str(payload.text)
588
+ if (!text.trim()) return this.result(requestId, true, {entry: null})
589
+ const source = payload.source === "voice" || payload.source === "auto" ? payload.source : "ui"
590
+ return this.result(requestId, true, {entry: await memory.addNote(text, source, num(payload.at))})
591
+ }
592
+ case MiniappRequestType.MEMORY_REMOVE_ENTRY:
593
+ return this.result(requestId, true, {removed: await memory.removeEntry(str(payload.id))})
594
+ case MiniappRequestType.MEMORY_REMOVE_SCOPE: {
595
+ const at = num(payload.at)
596
+ if (at == null) return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_remove_scope requires at")
597
+ await memory.removeScope(payload.scope === "day" ? "day" : "session", at)
598
+ return this.result(requestId, true)
599
+ }
600
+ case MiniappRequestType.MEMORY_SESSION_LIST:
601
+ return this.result(requestId, true, {sessions: await memory.sessionList()})
602
+ case MiniappRequestType.MEMORY_QUERY: {
603
+ const terms = Array.isArray(payload.terms)
604
+ ? payload.terms
605
+ .filter((t): t is string => typeof t === "string" && t.trim().length > 0)
606
+ .slice(0, 8)
607
+ .map((t) => t.slice(0, 64))
608
+ : undefined
609
+ const found = await memory.search({
610
+ sinceMs: num(payload.sinceMs),
611
+ untilMs: num(payload.untilMs),
612
+ contains: typeof payload.contains === "string" ? payload.contains : undefined,
613
+ terms,
614
+ limit: num(payload.limit),
615
+ })
616
+ return this.result(requestId, true, {entries: found.entries, windowCount: found.windowCount})
617
+ }
618
+ case MiniappRequestType.MEMORY_WINDOW: {
619
+ const sinceMs = num(payload.sinceMs)
620
+ if (sinceMs == null) return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_window requires sinceMs")
621
+ return this.result(requestId, true, {entries: await memory.window(sinceMs, num(payload.untilMs))})
622
+ }
623
+ case MiniappRequestType.MEMORY_EXPORT_TEXT: {
624
+ const scope = payload.scope === "session" || payload.scope === "day" ? payload.scope : "all"
625
+ return this.result(requestId, true, {text: await memory.exportText(scope, num(payload.at))})
626
+ }
627
+ case MiniappRequestType.MEMORY_EXPORT_FILE: {
628
+ if (!this.stateDir) return this.result(requestId, true, {success: false, cancelled: true})
629
+ const backup = await memory.exportBackup()
630
+ const file = join(this.stateDir, `glassly-memory-${localDate(backup.exportedAt)}.json`)
631
+ mkdirSync(this.stateDir, {recursive: true})
632
+ writeFileSync(file, JSON.stringify(backup))
633
+ this.effect("memory_export", {file})
634
+ return this.result(requestId, true, {success: true})
635
+ }
636
+ case MiniappRequestType.MEMORY_IMPORT_FILE: {
637
+ const file = this.latestMemoryBackup()
638
+ if (!file) return this.result(requestId, true, {imported: null})
639
+ let data: unknown
640
+ try {
641
+ data = JSON.parse(readFileSync(file, "utf-8"))
642
+ } catch {
643
+ throw new hostRules.MemoryBackupError("Not a Glassly memory backup")
644
+ }
645
+ this.effect("memory_import", {file})
646
+ return this.result(requestId, true, {imported: await memory.importBackup(data)})
647
+ }
648
+ case MiniappRequestType.MEMORY_SUMMARY_ADD: {
649
+ const hourStart = num(payload.hourStart)
650
+ if (hourStart == null || typeof payload.text !== "string") {
651
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "memory_summary_add requires hourStart and text")
652
+ }
653
+ await memory.addSummary({hourStart, text: payload.text, createdAt: num(payload.createdAt) ?? Date.now()})
654
+ return this.result(requestId, true)
655
+ }
656
+ case MiniappRequestType.MEMORY_SUMMARY_LIST:
657
+ return this.result(requestId, true, {summaries: await memory.summaryList()})
658
+ case MiniappRequestType.MEMORY_PENDING_SUMMARY_HOUR:
659
+ return this.result(requestId, true, {hourStart: await memory.pendingSummaryHour(num(payload.nowMs))})
660
+ case MiniappRequestType.MEMORY_CLEAR:
661
+ await memory.clear()
662
+ return this.result(requestId, true)
663
+ }
664
+ } catch (err) {
665
+ if (err instanceof hostRules.MemoryBackupError) {
666
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, err.message)
667
+ }
668
+ this.fail(requestId, MiniappErrorCode.INTERNAL, err instanceof Error ? err.message : "Memory error")
669
+ }
670
+ }
671
+
672
+ /** The newest memory backup in the simulator folder: what the phone's file picker would hand over. */
673
+ private latestMemoryBackup(): string | null {
674
+ if (!this.stateDir || !existsSync(this.stateDir)) return null
675
+ const files = readdirSync(this.stateDir).filter((name) => /^glassly-memory-.*\.json$/.test(name)).sort()
676
+ return files.length ? join(this.stateDir, files[files.length - 1]) : null
677
+ }
678
+
679
+ /** The phone calling one of this miniapp's declared actions, as a system caller would. */
680
+ callAction(actionId: string, params: Record<string, unknown> = {}, timeoutMs = CALL_TIMEOUT_MS, app = this.packageName): Promise<unknown> {
681
+ return this.registry.callAction(app, actionId, params, timeoutMs)
682
+ }
683
+
684
+ /**
685
+ * The phone placing one of this miniapp's widgets in a box (answers the card
686
+ * frame the provider draws) or sending it an event (answers its result).
687
+ */
688
+ callWidget(
689
+ widgetId: string,
690
+ call: {box: {size: "XS" | "XSM" | "SM" | "MD" | "LG"; width: number; height: number}} | {event: Record<string, unknown>},
691
+ timeoutMs = CALL_TIMEOUT_MS,
692
+ app = this.packageName,
693
+ ): Promise<unknown> {
694
+ return "box" in call
695
+ ? this.registry.placeForCli(app, widgetId, call.box, timeoutMs)
696
+ : this.registry.sendForCli(app, widgetId, call.event, timeoutMs)
697
+ }
698
+
699
+ get packageName(): string {
700
+ return this.manifest.packageName
701
+ }
702
+
703
+ /** Connected and running: requests to it are answered. */
704
+ get connected(): boolean {
705
+ return this.handshook && this.background === "running"
706
+ }
707
+
708
+ /** Sends the miniapp a message from the host. */
709
+ deliver(payload: Record<string, unknown>): void {
710
+ this.send(payload)
711
+ }
712
+
713
+ reply(requestId: string | undefined, ok: boolean, data?: unknown, error?: Record<string, unknown>): void {
714
+ this.result(requestId, ok, data, error)
715
+ }
716
+
717
+ /** Empties storage, settings and the memory timeline, as a fresh install would. */
718
+ async reset(): Promise<void> {
719
+ this.storage.clear()
720
+ this.settings.clear()
721
+ await this.memory?.clear()
722
+ this.log("host", "info", "Storage, settings and memory cleared")
723
+ }
724
+
725
+ private handleNotificationPost(payload: Record<string, unknown>, requestId?: string): void {
726
+ if (!this.requirePermission(requestId, "POST_NOTIFICATIONS")) return
727
+ const title = typeof payload.title === "string" ? payload.title.slice(0, NOTIFICATION_TITLE_MAX) : ""
728
+ const content = typeof payload.content === "string" ? payload.content.slice(0, NOTIFICATION_CONTENT_MAX) : ""
729
+ if (!title.trim() && !content.trim()) {
730
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "A notification needs a title or content")
731
+ }
732
+ if (payload.id !== undefined && (typeof payload.id !== "string" || !payload.id.trim())) {
733
+ return this.fail(requestId, MiniappErrorCode.INVALID_ARGUMENT, "A notification id must be a non-empty string")
734
+ }
735
+ const id =
736
+ typeof payload.id === "string" ? payload.id.trim().slice(0, NOTIFICATION_ID_MAX) : `sim-${++this.notificationSeq}`
737
+ this.effect("notification", {id, title, content})
738
+ this.result(requestId, true, {id})
739
+ }
740
+
372
741
  private handleConnect(requestId?: string): void {
742
+ this.derivedSettings = {}
743
+ this.publishDerivedSettings()
373
744
  const permissions: Record<string, boolean> = {}
374
745
  for (const canonical of new Set(Object.values(PERMISSION_TYPE_TO_CANONICAL))) permissions[canonical] = false
375
746
  for (const type of this.declared) {
@@ -399,8 +770,9 @@ export class SimHost {
399
770
  }
400
771
 
401
772
  private capabilities(): Record<string, unknown> {
402
- const display = this.device.capabilities.display as Record<string, unknown>
403
- const views = this.manifest.type === "dashboard" ? ["main", "dashboard"] : ["main"]
773
+ const display = this.device.capabilities.display as Record<string, unknown> | null
774
+ if (!display) return this.device.capabilities
775
+ const views = this.holdsDashboard() && this.device.dashboardView ? ["main", "dashboard"] : ["main"]
404
776
  return {...this.device.capabilities, display: {...display, views}}
405
777
  }
406
778
 
@@ -425,9 +797,11 @@ export class SimHost {
425
797
  for (const stream of next.keys()) {
426
798
  const required = permissionForStream(stream)
427
799
  if (!required || this.declared.has(required)) continue
800
+ // The Glassly App gates the settings stream with its generic permission check.
801
+ const purpose = stream === MiniappStreamType.SETTINGS_CHANGED ? "" : ` (required for "${stream}")`
428
802
  const error = {
429
803
  code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
430
- message: `${required} permission not declared in miniapp.json (required for "${stream}"). Add {"type": "${required}"} to the "permissions" array.`,
804
+ message: `${required} permission not declared in miniapp.json${purpose}. Add {"type": "${required}"} to the "permissions" array.`,
431
805
  permission: required,
432
806
  subscription: stream,
433
807
  }
@@ -448,6 +822,9 @@ export class SimHost {
448
822
  else if (stream === "glasses_connection") this.push(stream, this.connectionData())
449
823
  else if (stream === "glasses_wifi") this.push(stream, {connected: false})
450
824
  else if (stream === MiniappStreamType.MIRROR_UPDATE) this.push(stream, this.snapshot())
825
+ else if (stream === MiniappStreamType.SETTINGS_CHANGED) {
826
+ for (const key of hostRules.selectSettingsBridgeKeys?.(undefined) ?? []) this.push(stream, {key, value: this.settingValue(key)})
827
+ }
451
828
  }
452
829
 
453
830
  private connectionData(): Record<string, unknown> {
@@ -460,9 +837,9 @@ export class SimHost {
460
837
  }
461
838
  }
462
839
 
463
- private requirePermission(requestId: string | undefined, permission: string, purpose: string): boolean {
840
+ requirePermission(requestId: string | undefined, permission: string): boolean {
464
841
  if (this.declared.has(permission)) return true
465
- const message = `${permission} permission not declared in miniapp.json (required ${purpose}). Add {"type": "${permission}"} to the "permissions" array.`
842
+ const message = `${permission} permission not declared in miniapp.json. Add {"type": "${permission}"} to the "permissions" array.`
466
843
  this.log("host", "error", message)
467
844
  this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission})
468
845
  return false
@@ -476,7 +853,7 @@ export class SimHost {
476
853
  })
477
854
  }
478
855
  const view: View = payload.view === "dashboard" ? "dashboard" : "main"
479
- if (view === "dashboard" && this.manifest.type !== "dashboard") {
856
+ if (view === "dashboard" && !this.holdsDashboard()) {
480
857
  return this.result(requestId, true, {
481
858
  status: "displayed",
482
859
  presented: false,
@@ -484,31 +861,21 @@ export class SimHost {
484
861
  })
485
862
  }
486
863
  const caps = this.device.scene
864
+ if (!caps) return this.result(requestId, true, {status: "blocked", reason: "no display connected"})
487
865
  const appId = this.manifest.packageName
488
866
  const input = resolveAnchors(payload.elements as SceneElementInput[], caps.width, caps.height)
867
+ if (!caps.canPosition) return this.renderTextWall(input, view, caps, Number(payload.durationMs), requestId)
489
868
  const processed = processScene(input, caps, this.device.profile)
490
869
  const {elements} = diffScene(
491
870
  this.scenes.lastFrame(appId, view),
492
- processed.elements,
871
+ this.device.joinsText ? joinText(processed.elements, caps) : processed.elements,
493
872
  this.scenes.nextSyntheticId(appId, view),
494
873
  )
495
874
  this.scenes.commit(appId, view, elements, {width: caps.width, height: caps.height})
496
875
  this.frameSeq++
497
876
 
498
- this.clearExpiry(view)
499
- const durationMs = Number(payload.durationMs)
500
- if (durationMs > 0 && elements.length > 0) {
501
- this.expiry[view] = setTimeout(() => {
502
- this.scenes.commit(appId, view, [])
503
- if (this.visibleView() === view) this.publishDisplay()
504
- }, durationMs)
505
- }
506
-
507
- const dropped = processed.dropped.join(", ")
508
- if (dropped !== this.lastDropped) {
509
- this.lastDropped = dropped
510
- if (dropped) this.log("host", "warn", `render() dropped elements: ${dropped}`)
511
- }
877
+ this.scheduleExpiry(view, Number(payload.durationMs), elements.length > 0)
878
+ this.reportDropped(processed.dropped)
512
879
 
513
880
  const base = {status: "displayed", degraded: processed.degraded, dropped: processed.dropped}
514
881
  if (this.visibleView() !== view) {
@@ -525,6 +892,56 @@ export class SimHost {
525
892
  setTimeout(() => this.result(requestId, true, {...base, presented: true}), this.frameLatencyMs + settleMs)
526
893
  }
527
894
 
895
+ /** Glasses that cannot position elements get one wrapped block of text, as the Glassly App sends them. */
896
+ private renderTextWall(
897
+ input: SceneElementInput[],
898
+ view: View,
899
+ caps: SceneDisplayCapabilities,
900
+ durationMs: number,
901
+ requestId?: string,
902
+ ): void {
903
+ const appId = this.manifest.packageName
904
+ const {layout, degraded, dropped} = degradeScene(input)
905
+ const wrap = (text: unknown, maxWidthPx?: number) =>
906
+ this.wrapper.wrap(String(text ?? ""), maxWidthPx ? {maxWidthPx} : undefined).lines.join("\n")
907
+ const {width: w, height: h} = caps
908
+ const text = (id: string, value: string, x: number, width: number): FrameElement =>
909
+ ({id, type: "text", text: value, box: {x, y: 0, w: width, h}, change: "created"}) as FrameElement
910
+ // The Glassly App composes two columns into one text wall; side by side halves approximate it.
911
+ const elements: FrameElement[] = !layout
912
+ ? []
913
+ : layout.layoutType === "double_text_wall"
914
+ ? [
915
+ text("left", wrap(layout.topText, this.device.profile.displayWidthPx / 2), 0, w / 2),
916
+ text("right", wrap(layout.bottomText, this.device.profile.displayWidthPx / 2), w / 2, w / 2),
917
+ ]
918
+ : [text("text", wrap(layout.text), 0, w)]
919
+ this.scenes.commit(appId, view, elements, {width: w, height: h})
920
+ this.frameSeq++
921
+ this.scheduleExpiry(view, durationMs, elements.length > 0)
922
+ this.reportDropped(dropped)
923
+ if (this.visibleView() === view) this.publishDisplay()
924
+ this.settlesAt = Date.now()
925
+ this.result(requestId, true, {status: "displayed", degraded, dropped})
926
+ }
927
+
928
+ private scheduleExpiry(view: View, durationMs: number, hasContent: boolean): void {
929
+ this.clearExpiry(view)
930
+ if (!(durationMs > 0) || !hasContent) return
931
+ const appId = this.manifest.packageName
932
+ this.expiry[view] = setTimeout(() => {
933
+ this.scenes.commit(appId, view, [])
934
+ if (this.visibleView() === view) this.publishDisplay()
935
+ }, durationMs)
936
+ }
937
+
938
+ private reportDropped(list: string[]): void {
939
+ const dropped = list.join(", ")
940
+ if (dropped === this.lastDropped) return
941
+ this.lastDropped = dropped
942
+ if (dropped) this.log("host", "warn", `render() dropped elements: ${dropped}`)
943
+ }
944
+
528
945
  private handleShowView(payload: Record<string, unknown>, requestId?: string): void {
529
946
  const view = payload.view
530
947
  if (view !== "main" && view !== "dashboard") {
@@ -533,12 +950,15 @@ export class SimHost {
533
950
  message: 'showView takes "main" or "dashboard"',
534
951
  })
535
952
  }
536
- if (this.manifest.type !== "dashboard") {
953
+ if (!this.holdsDashboard()) {
537
954
  return this.result(requestId, false, undefined, {
538
955
  code: MiniappErrorCode.NOT_PERMITTED,
539
956
  message: 'The dashboard view belongs to miniapps of type "dashboard"',
540
957
  })
541
958
  }
959
+ if (view === "dashboard" && !this.device.dashboardView) {
960
+ return this.result(requestId, true, {status: "unsupported", reason: "glasses cannot show the dashboard"})
961
+ }
542
962
  const overlay = view === "dashboard"
543
963
  if (overlay !== this.overlay) {
544
964
  this.overlay = overlay
@@ -549,6 +969,11 @@ export class SimHost {
549
969
  this.result(requestId, true, {status: "shown"})
550
970
  }
551
971
 
972
+ /** The app under test holds the dashboard role when it is a dashboard miniapp; the wearer picks one on the phone. */
973
+ private holdsDashboard(): boolean {
974
+ return this.manifest.type === "dashboard"
975
+ }
976
+
552
977
  private clearExpiry(view: View): void {
553
978
  const timer = this.expiry[view]
554
979
  if (timer) clearTimeout(timer)
@@ -561,6 +986,19 @@ export class SimHost {
561
986
 
562
987
  snapshot(): MirrorSnapshot {
563
988
  const caps = this.device.scene
989
+ if (!caps) {
990
+ return {
991
+ revision: this.revision,
992
+ updatedAt: Date.now(),
993
+ connected: this.glassesConnected,
994
+ model: this.device.capabilities.modelName,
995
+ canvas: null,
996
+ view: null,
997
+ appId: null,
998
+ content: {kind: "unavailable", reason: "no-display"},
999
+ glyphWidths: {},
1000
+ }
1001
+ }
564
1002
  const view = this.visibleView()
565
1003
  const appId = this.manifest.packageName
566
1004
  const glyphWidths: Record<string, number> = {}
@@ -593,6 +1031,7 @@ export class SimHost {
593
1031
  const snapshot = this.snapshot()
594
1032
  this.emit({type: "display", snapshot})
595
1033
  this.push(MiniappStreamType.MIRROR_UPDATE, snapshot)
1034
+ this.publishDerivedSettings()
596
1035
  }
597
1036
 
598
1037
  private handleSpeak(text: string, requestId?: string): void {
@@ -719,61 +1158,185 @@ export class SimHost {
719
1158
  }
720
1159
 
721
1160
  touch(gesture: string, extra: Record<string, unknown> = {}): void {
1161
+ if (!this.device.gestures.includes(gesture)) {
1162
+ throw new Error(
1163
+ `${this.device.label} does not send "${gesture}". It sends: ${this.device.gestures.join(", ") || "no gestures"}`,
1164
+ )
1165
+ }
722
1166
  const data = {...extra, kind: gesture, gestureName: gesture, timestamp: Date.now()}
723
1167
  this.inject([MiniappStreamType.TOUCH_EVENT, `${MiniappStreamType.TOUCH_EVENT}:${gesture}`], data)
724
1168
  }
725
1169
 
726
1170
  button(pressType: "short" | "long" = "short", buttonId = "main"): void {
1171
+ if (!this.device.sendsButtonPress) throw new Error(`${this.device.label} has no button that sends presses`)
727
1172
  this.inject([MiniappStreamType.BUTTON_PRESS], {buttonId, pressType, timestamp: Date.now()})
728
1173
  }
729
1174
 
1175
+ /**
1176
+ * One transcription result, as the phone sends it: interims keep the open
1177
+ * utterance's id and a final closes it.
1178
+ */
730
1179
  transcription(text: string, options: {isFinal?: boolean; language?: string; speakerId?: string} = {}): void {
731
- const language = options.language ?? "en-US"
732
- const base = language.split("-")[0].toLowerCase()
733
- const data = {
1180
+ const isFinal = options.isFinal ?? true
1181
+ this.openUtterance ??= `utt_sim_${++this.utteranceSeq}`
1182
+ const now = Date.now() - this.speechEpoch
1183
+ this.deliverTranscription({
734
1184
  text,
735
- isFinal: options.isFinal ?? true,
736
- language,
737
- detectedLanguage: language,
738
- transcribeLanguage: language,
739
- ...(options.speakerId ? {speakerId: options.speakerId} : {}),
740
- timestamp: Date.now(),
741
- }
742
- const streams = [...this.subscriptions.keys()].filter((stream) => stream.startsWith("transcription:"))
743
- const explicit = streams.find(
744
- (stream) => stream !== "transcription:auto" && stream.slice(14).split("-")[0].toLowerCase() === base,
745
- )
746
- const target = explicit ?? (streams.includes("transcription:auto") ? "transcription:auto" : null)
747
- if (!target) return this.undelivered(`transcription:${language}`)
748
- this.pushRouted(explicit ?? `transcription:${language}`, data, this.subscriptions.get(target))
1185
+ isFinal,
1186
+ utteranceId: this.openUtterance,
1187
+ language: options.language ?? "en-US",
1188
+ speakerId: options.speakerId ?? "1",
1189
+ startTime: now,
1190
+ endTime: now,
1191
+ })
1192
+ if (isFinal) this.openUtterance = null
749
1193
  }
750
1194
 
1195
+ /**
1196
+ * One translation result, as the phone sends it: to `target`, or to every
1197
+ * translation target a miniapp subscribed to when absent.
1198
+ */
751
1199
  translation(
752
1200
  text: string,
753
- options: {source?: string; target?: string; isFinal?: boolean; originalText?: string} = {},
1201
+ options: {source?: string; target?: string; isFinal?: boolean; originalText?: string; speakerId?: string} = {},
754
1202
  ): void {
755
- const source = options.source ?? "en-US"
756
- const target = options.target ?? "es-ES"
757
- const patterns = [
758
- `translation:${source}:${target}`,
759
- "translation:auto",
760
- "translation:*:*",
761
- `translation:*:${target}`,
762
- `translation:${source}:*`,
763
- ]
764
- const match = patterns.find((pattern) => this.subscriptions.has(pattern))
765
- if (!match) return this.undelivered(patterns[0])
766
- this.pushRouted(
767
- patterns[0],
768
- {
1203
+ const isFinal = options.isFinal ?? true
1204
+ const source = options.source ?? "es"
1205
+ const targets = options.target ? [options.target] : this.translationTargets(source)
1206
+ if (targets.length === 0) return this.undelivered(`translation:${source}:*`)
1207
+ const now = Date.now() - this.speechEpoch
1208
+ for (const target of targets) {
1209
+ const id = (this.openTranslations[target] ??= `utt_sim_tr_${++this.utteranceSeq}`)
1210
+ this.deliverTranslation(source, target, {
769
1211
  text,
770
- isFinal: options.isFinal ?? true,
771
- sourceLanguage: source,
772
- targetLanguage: target,
773
1212
  ...(options.originalText ? {originalText: options.originalText} : {}),
774
- timestamp: Date.now(),
775
- },
776
- this.subscriptions.get(match),
1213
+ isFinal,
1214
+ utteranceId: id,
1215
+ speakerId: options.speakerId ?? "1",
1216
+ startTime: now,
1217
+ endTime: now,
1218
+ })
1219
+ if (isFinal) delete this.openTranslations[target]
1220
+ }
1221
+ }
1222
+
1223
+ /**
1224
+ * Plays a transcript script as speech: each line's words arrive as growing
1225
+ * interims at the speaking rate, then its final half a second later, on
1226
+ * every transcription and translation stream a miniapp listens to.
1227
+ */
1228
+ play(lines: readonly ScriptLine[], options: {wordsPerMinute?: number; gapMs?: number} = {}): Promise<{utterances: number; durationMs: number}> {
1229
+ this.stopPlaying()
1230
+ const msPerWord = 60_000 / Math.max(30, options.wordsPerMinute ?? DEFAULT_WORDS_PER_MINUTE)
1231
+ const gapMs = options.gapMs ?? 300
1232
+ const speakers = new Map<string, string>()
1233
+ let cursor = 0
1234
+ let language = "en"
1235
+ const steps: {at: number; run: () => void}[] = []
1236
+ for (const line of lines) {
1237
+ language = line.language ?? language
1238
+ const spoken = language
1239
+ const start = Math.max(line.atMs ?? cursor, cursor)
1240
+ const words = line.text.split(/\s+/).filter(Boolean)
1241
+ const speakerId = speakers.get(line.speaker ?? "") ?? String(speakers.size + 1)
1242
+ speakers.set(line.speaker ?? "", speakerId)
1243
+ const id = `utt_sim_${++this.utteranceSeq}`
1244
+ const translationIds: Record<string, string> = {}
1245
+ const finalAt = start + words.length * msPerWord + ENDPOINT_DEBOUNCE_MS
1246
+ const emit = (count: number, isFinal: boolean, at: number) => {
1247
+ const result = {isFinal, speakerId, startTime: Math.round(start), endTime: Math.round(at)}
1248
+ this.deliverTranscription({...result, text: words.slice(0, count).join(" "), utteranceId: id, language: spoken})
1249
+ for (const target of this.translationTargets(spoken)) {
1250
+ translationIds[target] ??= `utt_sim_tr_${++this.utteranceSeq}`
1251
+ const passthrough = baseLanguage(target) === baseLanguage(spoken)
1252
+ const original = words.slice(0, count).join(" ")
1253
+ const full =
1254
+ line.translations[target] ??
1255
+ line.translations[baseLanguage(target)] ??
1256
+ `[${baseLanguage(spoken)}→${baseLanguage(target)}] ${line.text}`
1257
+ const translated = full.split(/\s+/)
1258
+ // Translations trail the source text by a word, as the real service does.
1259
+ const shown = isFinal ? translated.length : Math.max(0, Math.round(((count - 1) / words.length) * translated.length))
1260
+ this.deliverTranslation(spoken, target, {
1261
+ ...result,
1262
+ text: passthrough ? original : translated.slice(0, shown).join(" "),
1263
+ ...(passthrough ? {} : {originalText: original}),
1264
+ utteranceId: translationIds[target],
1265
+ })
1266
+ }
1267
+ }
1268
+ words.forEach((_, i) => {
1269
+ const at = start + (i + 1) * msPerWord
1270
+ steps.push({at, run: () => emit(i + 1, false, at)})
1271
+ })
1272
+ steps.push({at: finalAt, run: () => emit(words.length, true, finalAt)})
1273
+ cursor = finalAt + gapMs
1274
+ }
1275
+ this.speechEpoch = Date.now()
1276
+ if (![...this.subscriptions.keys()].some((stream) => /^(transcription|translation):/.test(stream))) {
1277
+ this.log("host", "warn", "Nothing is subscribed to transcriptions or translations; the script plays to no one")
1278
+ }
1279
+ return new Promise((resolve) => {
1280
+ const done = () => {
1281
+ this.playback = null
1282
+ resolve({utterances: lines.length, durationMs: Math.round(cursor)})
1283
+ }
1284
+ const timers = steps.map((step) => setTimeout(step.run, step.at))
1285
+ timers.push(setTimeout(done, cursor))
1286
+ this.playback = {timers, done}
1287
+ })
1288
+ }
1289
+
1290
+ stopPlaying(): void {
1291
+ const playback = this.playback
1292
+ if (!playback) return
1293
+ for (const timer of playback.timers) clearTimeout(timer)
1294
+ playback.done()
1295
+ }
1296
+
1297
+ /** Concrete translation targets subscribed to; the phone runs no translation for a wildcard target. */
1298
+ private translationTargets(spoken: string): string[] {
1299
+ const targets = new Set<string>()
1300
+ for (const stream of this.subscriptions.keys()) {
1301
+ const [kind, source, target] = stream.split(":")
1302
+ if (kind !== "translation" || !target || target === "*") continue
1303
+ if (source !== "*" && source !== "auto" && baseLanguage(source) !== baseLanguage(spoken)) continue
1304
+ targets.add(target)
1305
+ }
1306
+ return [...targets]
1307
+ }
1308
+
1309
+ // An auto subscriber gets the detected bare code; with none, a specific
1310
+ // subscription in the spoken language gets its own tag (LocalMiniappRuntime).
1311
+ private deliverTranscription(result: {
1312
+ text: string
1313
+ isFinal: boolean
1314
+ utteranceId: string
1315
+ language: string
1316
+ speakerId: string
1317
+ startTime: number
1318
+ endTime: number
1319
+ }): void {
1320
+ const {language, ...rest} = result
1321
+ const base = baseLanguage(language)
1322
+ const data = {type: "transcription", ...rest, provider: "soniox"}
1323
+ const streams = [...this.subscriptions.keys()].filter((stream) => stream.startsWith("transcription:"))
1324
+ if (streams.includes("transcription:auto")) {
1325
+ const specific = streams.find((stream) => stream !== "transcription:auto" && baseLanguage(stream.slice(14)) === base)
1326
+ return this.pushRouted(specific ?? `transcription:${base}`, {...data, transcribeLanguage: base, detectedLanguage: base}, this.subscriptions.get("transcription:auto"))
1327
+ }
1328
+ const matching = streams.filter((stream) => baseLanguage(stream.slice(14)) === base)
1329
+ if (matching.length === 0) return this.undelivered(`transcription:${language}`)
1330
+ for (const stream of matching) this.pushRouted(stream, {...data, transcribeLanguage: stream.slice(14)}, this.subscriptions.get(stream))
1331
+ }
1332
+
1333
+ private deliverTranslation(spoken: string, target: string, result: Record<string, unknown>): void {
1334
+ const source = [...this.subscriptions.keys()].some((stream) => stream.startsWith(`translation:${spoken}:`)) ? spoken : baseLanguage(spoken)
1335
+ const route = [...this.subscriptions.entries()].find(([stream]) => stream.startsWith("translation:") && stream.endsWith(`:${target}`))
1336
+ this.pushRouted(
1337
+ `translation:${source}:${target}`,
1338
+ {type: "translation", ...result, transcribeLanguage: source, translateLanguage: target, didTranslate: true, provider: "soniox"},
1339
+ route?.[1],
777
1340
  )
778
1341
  }
779
1342
 
@@ -828,6 +1391,7 @@ export class SimHost {
828
1391
  setVisibility(visibility: "foreground" | "background"): void {
829
1392
  this.visibility = visibility
830
1393
  this.send({type: MiniappResponseType.VISIBILITY_CHANGE, visibility})
1394
+ this.publishDerivedSettings()
831
1395
  this.emit({type: "status"})
832
1396
  }
833
1397
 
@@ -855,6 +1419,7 @@ export class SimHost {
855
1419
  }
856
1420
 
857
1421
  private undelivered(stream: string): void {
1422
+ if (this.playback) return
858
1423
  this.log("host", "warn", `Nothing is subscribed to "${stream}"; the event was not delivered`)
859
1424
  }
860
1425
 
@@ -884,6 +1449,10 @@ export class SimHost {
884
1449
  this.toBackground(JSON.stringify(requestId ? {payload, requestId} : {payload}))
885
1450
  }
886
1451
 
1452
+ private fail(requestId: string | undefined, code: string, message: string): void {
1453
+ this.result(requestId, false, undefined, {code, message})
1454
+ }
1455
+
887
1456
  private result(requestId: string | undefined, ok: boolean, data?: unknown, error?: Record<string, unknown>): void {
888
1457
  if (!requestId) return
889
1458
  this.send(
@@ -929,7 +1498,14 @@ export class SimHost {
929
1498
  packageName: this.manifest.packageName,
930
1499
  name: this.manifest.name,
931
1500
  version: this.manifest.version,
932
- device: {id: this.device.id, model: this.device.capabilities.modelName, canvas: snapshot.canvas},
1501
+ device: {
1502
+ id: this.device.id,
1503
+ label: this.device.label,
1504
+ model: this.device.capabilities.modelName,
1505
+ canvas: snapshot.canvas,
1506
+ gestures: this.device.gestures,
1507
+ buttonPress: this.device.sendsButtonPress,
1508
+ },
933
1509
  background: this.background,
934
1510
  connected: this.handshook,
935
1511
  paused: !this.glassesConnected,