@glassly/miniapp-cli 0.1.17-dev.0 → 0.1.17-dev.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/sim/host.ts CHANGED
@@ -12,7 +12,7 @@ import {
12
12
  type SceneElementInput,
13
13
  } from "@glassly/miniapp/display"
14
14
  // A namespace import, so an older @glassly/miniapp without the host rules still loads.
15
- import {existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync} from "fs"
15
+ import {existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync} from "fs"
16
16
  import {join} from "path"
17
17
  import * as hostRules from "@glassly/miniapp/host"
18
18
  import type {MirrorElement, MirrorSnapshot} from "@glassly/miniapp/mirror"
@@ -28,6 +28,8 @@ import type {PhoneKind} from "./phone.js"
28
28
  import type {ScriptLine} from "./script.js"
29
29
  import {SimRegistry} from "./registry.js"
30
30
  import {createSimStorage, type SimStorage} from "./storage.js"
31
+ import {FORCED_MESSAGES, TYPE_HANDLERS, type SimTypeContext} from "./types/index.js"
32
+ import {MIC_SAMPLE_RATE} from "./wav.js"
31
33
 
32
34
  export type SimLogSource = "background" | "ui" | "host" | "bridge"
33
35
 
@@ -61,6 +63,7 @@ export interface SimManifest {
61
63
  permissions?: {type?: string}[]
62
64
  actions?: unknown
63
65
  widgets?: unknown
66
+ hardwareRequirements?: unknown
64
67
  }
65
68
 
66
69
  export interface SimHostOptions {
@@ -132,7 +135,7 @@ function permissionForStream(stream: string): string | null {
132
135
  if (stream.startsWith("transcription") || stream.startsWith("translation")) return "MICROPHONE"
133
136
  if (stream === "location_update" || stream === "heading_update") return "LOCATION"
134
137
  if (stream === "phone_notification" || stream === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
135
- if (stream === MiniappStreamType.SETTINGS_CHANGED) return "PLATFORM_SETTINGS"
138
+ if (stream === MiniappStreamType.SETTINGS_CHANGED) return "isOpenSettingsKey" in hostRules ? null : "PLATFORM_SETTINGS"
136
139
  return null
137
140
  }
138
141
 
@@ -213,9 +216,7 @@ export class SimHost {
213
216
  private handshook = false
214
217
  private background: "stopped" | "running" | "crashed" = "stopped"
215
218
  private subscriptions = new Map<string, StreamRoute>()
216
- private claimedGestures: string[] = []
217
219
  private derivedSettings: Record<string, unknown> = {}
218
- private gestureClaims = hostRules.emptyGestureClaims?.()
219
220
  private overlay = false
220
221
  private revision = 0
221
222
  private frameSeq = 0
@@ -231,6 +232,11 @@ export class SimHost {
231
232
  private readonly logEntries: SimLogEntry[] = []
232
233
  private readonly effects: SimEffect[] = []
233
234
  private readonly unsimulated = new Set<string>()
235
+ private readonly faked = new Map<string, string>()
236
+ private readonly forced = new Map<string, {code: string; message: string}>()
237
+ private endTypeContext: () => void = () => {}
238
+ private typeContext: SimTypeContext = this.createTypeContext()
239
+ private audioTimers: ReturnType<typeof setTimeout>[] = []
234
240
 
235
241
  private speakerState = "idle"
236
242
  private utteranceSeq = 0
@@ -281,16 +287,19 @@ export class SimHost {
281
287
  }
282
288
 
283
289
  backgroundStopped(crashed: boolean): void {
290
+ this.registry.releaseWidgetHost(this.manifest.packageName)
284
291
  this.resetSession()
285
292
  this.background = crashed ? "crashed" : "stopped"
286
293
  this.emit({type: "status"})
287
294
  }
288
295
 
289
296
  private resetSession(): void {
297
+ this.endTypeContext()
298
+ this.typeContext = this.createTypeContext()
299
+ for (const timer of this.audioTimers.splice(0)) clearTimeout(timer)
290
300
  this.handshook = false
291
301
  this.subscriptions.clear()
292
- this.claimedGestures = []
293
- this.gestureClaims = hostRules.emptyGestureClaims?.()
302
+ this.registry.dropGestureClaims(this.manifest.packageName)
294
303
  this.overlay = false
295
304
  this.scenes = new SceneStore()
296
305
  for (const view of ["main", "dashboard"] as View[]) this.clearExpiry(view)
@@ -327,6 +336,13 @@ export class SimHost {
327
336
  }
328
337
 
329
338
  private dispatch(type: string, payload: Record<string, unknown>, requestId?: string): void {
339
+ const forced = requestId ? this.forced.get(type) : undefined
340
+ if (forced) {
341
+ this.forced.delete(type)
342
+ this.log("host", "info", `Failing ${type} with ${forced.code}, as \`glassly sim fail\` asked`)
343
+ this.emit({type: "status"})
344
+ return this.fail(requestId, forced.code, forced.message)
345
+ }
330
346
  if (this.registry.handle(this, type, payload, requestId)) return
331
347
  switch (type) {
332
348
  case MiniappRequestType.CONNECT:
@@ -362,11 +378,12 @@ export class SimHost {
362
378
  return
363
379
  case MiniappRequestType.TRANSCRIPTION_CONFIG:
364
380
  return this.result(requestId, true, {applied: true})
365
- case MiniappRequestType.AUTH_REFRESH:
366
- return this.result(requestId, false, undefined, {
367
- code: MiniappErrorCode.NOT_CONNECTED,
368
- message: "Miniapp auth is not configured in the simulator",
369
- })
381
+ case MiniappRequestType.AUTH_REFRESH: {
382
+ const auth = {userId: "sim-user", token: `glassly-sim.${this.manifest.packageName}.${Date.now()}`, expiresAt: Date.now() + 3_600_000}
383
+ this.typeContext.fake(type, "a placeholder token for user sim-user that no server accepts")
384
+ this.send({type: MiniappResponseType.AUTH_UPDATE, auth})
385
+ return this.result(requestId, true, {auth})
386
+ }
370
387
 
371
388
  case MiniappRequestType.STORAGE_GET:
372
389
  return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(type, payload))})
@@ -402,8 +419,17 @@ export class SimHost {
402
419
  case MiniappRequestType.LOCATION_POLL:
403
420
  return this.result(requestId, true, this.location)
404
421
 
405
- case MiniappRequestType.SPEAK:
406
- return this.handleSpeak(String(payload.text ?? ""), requestId)
422
+ case MiniappRequestType.SPEAK: {
423
+ const raw = payload.text
424
+ const trimmed = (typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw.filter((s): s is string => typeof s === "string") : [])
425
+ .map((sentence) => sentence.trim())
426
+ .filter(Boolean)
427
+ const sentences = hostRules.prepareTtsSentences?.(trimmed, payload.enableSanitization !== false) ?? trimmed
428
+ if (sentences.length === 0) {
429
+ return this.fail(requestId, MiniappErrorCode.INTERNAL, "speak requires text or a non-empty sentence list with speakable content")
430
+ }
431
+ return this.handleSpeak(sentences.join(" "), requestId)
432
+ }
407
433
  case MiniappRequestType.PLAY_AUDIO:
408
434
  if (typeof payload.audioUrl !== "string" || !payload.audioUrl) {
409
435
  return this.fail(requestId, MiniappErrorCode.INTERNAL, "play_audio requires an audioUrl")
@@ -412,21 +438,25 @@ export class SimHost {
412
438
  return this.startSpeech(1500, requestId, undefined)
413
439
  case MiniappRequestType.STOP_AUDIO:
414
440
  this.effect("stop_audio", {})
415
- return this.finishSpeech(false)
441
+ this.finishSpeech(false)
442
+ return this.result(requestId, true)
416
443
 
417
444
  case MiniappRequestType.RGB_LED:
418
445
  this.effect("led", {...payload, type: undefined})
419
446
  return this.result(requestId, true)
420
447
  case MiniappRequestType.OPEN_URL:
421
- this.effect("open_url", {url: payload.url})
422
- return this.result(requestId, true)
448
+ return this.effect("open_url", {url: payload.url})
423
449
  case MiniappRequestType.COPY_CLIPBOARD:
450
+ if (typeof payload.text !== "string") return this.log("host", "warn", "copy_clipboard without text; the phone ignores it and never answers")
424
451
  this.effect("clipboard", {text: payload.text})
425
452
  return this.result(requestId, true)
426
453
  case MiniappRequestType.SHARE:
427
454
  this.effect("share", {...payload, type: undefined})
428
455
  return this.result(requestId, true, {success: true})
429
456
  case MiniappRequestType.DOWNLOAD:
457
+ if (typeof payload.base64 !== "string" && typeof payload.url !== "string") {
458
+ return this.log("host", "warn", "download without base64 or url; the phone ignores it and never answers")
459
+ }
430
460
  this.effect("download", {...payload, type: undefined})
431
461
  return this.result(requestId, true, {success: true})
432
462
 
@@ -443,21 +473,21 @@ export class SimHost {
443
473
  case MiniappRequestType.SETTINGS_GET: {
444
474
  const {selectSettingsBridgeKeys} = hostRules
445
475
  if (!selectSettingsBridgeKeys) break
446
- if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
476
+ if (!this.mayUseSettings("get", payload, requestId)) return
447
477
  const values: Record<string, unknown> = {}
448
- for (const key of selectSettingsBridgeKeys(payload.keys)) values[key] = this.settingValue(key)
478
+ for (const key of selectSettingsBridgeKeys(payload.keys, this.platformSettings)) values[key] = this.settingValue(key)
449
479
  return this.result(requestId, true, {values})
450
480
  }
451
481
  case MiniappRequestType.SETTINGS_SET: {
452
482
  const {validateSettingsSet} = hostRules
453
483
  if (!validateSettingsSet) break
454
- if (!this.requirePermission(requestId, "PLATFORM_SETTINGS")) return
484
+ if (!this.mayUseSettings("set", payload, requestId)) return
455
485
  const validated = validateSettingsSet(payload)
456
486
  if (!validated.ok) return this.fail(requestId, validated.code, validated.message)
457
487
  const changed = JSON.stringify(this.settingValue(validated.key)) !== JSON.stringify(validated.value ?? null)
458
488
  this.settings.set(validated.key, validated.value)
459
489
  this.result(requestId, true)
460
- if (changed) this.push(MiniappStreamType.SETTINGS_CHANGED, {key: validated.key, value: this.settingValue(validated.key)})
490
+ if (changed) for (const app of this.registry.apps()) app.settingChanged(validated.key)
461
491
  return
462
492
  }
463
493
 
@@ -472,7 +502,7 @@ export class SimHost {
472
502
 
473
503
  case MiniappRequestType.SYSTEM_EXIT:
474
504
  case MiniappRequestType.MINIAPPS_STOP_SELF:
475
- this.result(requestId, true)
505
+ if (type === MiniappRequestType.SYSTEM_EXIT) this.result(requestId, true)
476
506
  this.log("host", "info", "The miniapp asked to stop (session.system.exit)")
477
507
  return this.emit({type: "exit", reason: "the miniapp called exit"})
478
508
 
@@ -499,6 +529,8 @@ export class SimHost {
499
529
  return this.emit({type: "ui", frame: {type: "cancel", requestId: payload.requestId}})
500
530
  }
501
531
 
532
+ for (const handle of TYPE_HANDLERS) if (handle(this.typeContext, type, payload, requestId)) return
533
+
502
534
  if (!this.unsimulated.has(type)) {
503
535
  this.unsimulated.add(type)
504
536
  this.log("host", "warn", `"${type}" is not simulated; the request was answered with NOT_IMPLEMENTED`)
@@ -512,20 +544,58 @@ export class SimHost {
512
544
 
513
545
  private claimGestures(payload: Record<string, unknown>, requestId?: string): void {
514
546
  const gestures = Array.isArray(payload.gestures) ? payload.gestures.filter((g): g is string => typeof g === "string") : []
515
- const {setGestureClaims, gestureClaimant} = hostRules
516
- if (!this.gestureClaims || !setGestureClaims || !gestureClaimant) {
517
- this.claimedGestures = gestures
518
- } else {
519
- const {state, changes} = setGestureClaims(this.gestureClaims, this.manifest.packageName, gestures, payload.scope as string | undefined)
520
- this.gestureClaims = state
521
- const held = new Set(state.registrations.flatMap((entry) => [...entry.gestures]))
522
- this.claimedGestures = [...held].filter((gesture) => gestureClaimant(state, gesture) === this.manifest.packageName)
523
- for (const change of changes) this.push(MiniappStreamType.GESTURE_CLAIM, {gesture: change.gesture, held: change.held})
524
- }
547
+ this.registry.claimGestures(this, gestures, payload.scope as string | undefined)
525
548
  this.emit({type: "status"})
526
549
  this.result(requestId, true)
527
550
  }
528
551
 
552
+ failNext(type: string, code: string, message?: string): string {
553
+ const wire = type.startsWith("miniapp_") ? type : `miniapp_${type.toLowerCase()}`
554
+ if (!(Object.values(MiniappRequestType) as string[]).includes(wire)) throw new Error(`Unknown request type "${type}"`)
555
+ if (!code) throw new Error("glassly sim fail needs an error code")
556
+ this.forced.set(wire, {code, message: message ?? FORCED_MESSAGES[code] ?? `${code} (forced by glassly sim fail)`})
557
+ this.emit({type: "status"})
558
+ return wire
559
+ }
560
+
561
+ // Each background session gets its own context; the old one goes quiet, so a
562
+ // module's leftover timers cannot reach the next session.
563
+ private createTypeContext(): SimTypeContext {
564
+ let live = true
565
+ const onEnd: (() => void)[] = []
566
+ this.endTypeContext = () => {
567
+ live = false
568
+ for (const callback of onEnd.splice(0)) callback()
569
+ }
570
+ const context = {
571
+ declares: (permission: string) => this.declared.has(permission),
572
+ requirePermission: (requestId, permission, operation) => this.requirePermission(requestId, permission, operation),
573
+ reply: (requestId, ok, data, error) => live && this.result(requestId, ok, data, error),
574
+ fail: (requestId, code, message) => live && this.fail(requestId, code, message),
575
+ deliver: (payload) => live && this.send(payload),
576
+ push: (stream, data) => live && this.push(stream, data),
577
+ fake: (type, note) => {
578
+ if (this.faked.get(type) === note) return
579
+ this.faked.set(type, note)
580
+ this.emit({type: "status"})
581
+ },
582
+ effect: (kind, detail) => this.effect(kind, detail),
583
+ log: (level, message) => this.log("host", level, message),
584
+ onEnd: (callback) => onEnd.push(callback),
585
+ } as Omit<SimTypeContext, "packageName" | "device" | "stateDir" | "glassesConnected">
586
+ return Object.defineProperties(context, {
587
+ packageName: {get: () => this.manifest.packageName},
588
+ device: {get: () => this.device},
589
+ stateDir: {get: () => this.stateDir},
590
+ glassesConnected: {get: () => this.glassesConnected},
591
+ }) as SimTypeContext
592
+ }
593
+
594
+ gestureClaimChanged(gesture: string, held: boolean): void {
595
+ this.push(MiniappStreamType.GESTURE_CLAIM, {gesture, held})
596
+ this.emit({type: "status"})
597
+ }
598
+
529
599
  private key(type: string, payload: Record<string, unknown>): string {
530
600
  if (typeof payload.key !== "string" || !payload.key) throw new Error(`${type.replace(/^miniapp_/, "")} requires a key`)
531
601
  return payload.key
@@ -539,14 +609,35 @@ export class SimHost {
539
609
  if (this.derivedSettings[key] === value) continue
540
610
  const known = key in this.derivedSettings
541
611
  this.derivedSettings[key] = value
542
- if (known) this.push(MiniappStreamType.SETTINGS_CHANGED, {key, value})
612
+ if (known) this.pushSetting(key)
543
613
  }
544
614
  }
545
615
 
616
+ settingChanged(key: string): void {
617
+ this.pushSetting(key)
618
+ }
619
+
620
+ private get platformSettings(): boolean {
621
+ return this.declared.has("PLATFORM_SETTINGS")
622
+ }
623
+
624
+ private mayUseSettings(kind: "get" | "set", payload: Record<string, unknown>, requestId?: string): boolean {
625
+ const open = hostRules.settingsRequestNeedsPermission ? !hostRules.settingsRequestNeedsPermission(kind, payload) : false
626
+ return open || this.requirePermission(requestId, "PLATFORM_SETTINGS")
627
+ }
628
+
629
+ private pushSetting(key: string): void {
630
+ if (hostRules.settingsBridgeAccess?.(key, this.platformSettings) === null) return
631
+ this.push(MiniappStreamType.SETTINGS_CHANGED, {key, value: this.settingValue(key)})
632
+ }
633
+
546
634
  private settingValue(key: string): unknown {
547
635
  if (key === "phone_active") return this.visibility === "foreground"
548
636
  if (key === "foreground_app_on_screen") return this.visibleView() === "main" && this.scenes.lastFrame(this.manifest.packageName, "main").length > 0
549
637
  if (key === "controllers") return []
638
+ if (key === hostRules.READING_APP_SPEED && hostRules.readingAppSpeedFor) {
639
+ return hostRules.readingAppSpeedFor(this.settings.get(hostRules.READING_APP_SPEEDS), this.manifest.packageName)
640
+ }
550
641
  return this.settings.has(key) ? this.settings.get(key) : (hostRules.settingsBridgeDefault?.(key) ?? null)
551
642
  }
552
643
 
@@ -700,6 +791,14 @@ export class SimHost {
700
791
  return this.manifest.packageName
701
792
  }
702
793
 
794
+ get running(): boolean {
795
+ return this.background === "running"
796
+ }
797
+
798
+ get deviceCapabilities(): Record<string, unknown> {
799
+ return this.device.capabilities
800
+ }
801
+
703
802
  /** Connected and running: requests to it are answered. */
704
803
  get connected(): boolean {
705
804
  return this.handshook && this.background === "running"
@@ -719,7 +818,10 @@ export class SimHost {
719
818
  this.storage.clear()
720
819
  this.settings.clear()
721
820
  await this.memory?.clear()
722
- this.log("host", "info", "Storage, settings and memory cleared")
821
+ this.endTypeContext()
822
+ this.typeContext = this.createTypeContext()
823
+ if (this.stateDir) rmSync(join(this.stateDir, "blobs"), {recursive: true, force: true})
824
+ this.log("host", "info", "Storage, settings, memory and blobs cleared")
723
825
  }
724
826
 
725
827
  private handleNotificationPost(payload: Record<string, unknown>, requestId?: string): void {
@@ -823,7 +925,9 @@ export class SimHost {
823
925
  else if (stream === "glasses_wifi") this.push(stream, {connected: false})
824
926
  else if (stream === MiniappStreamType.MIRROR_UPDATE) this.push(stream, this.snapshot())
825
927
  else if (stream === MiniappStreamType.SETTINGS_CHANGED) {
826
- for (const key of hostRules.selectSettingsBridgeKeys?.(undefined) ?? []) this.push(stream, {key, value: this.settingValue(key)})
928
+ for (const key of hostRules.selectSettingsBridgeKeys?.(undefined, this.platformSettings) ?? []) {
929
+ this.push(stream, {key, value: this.settingValue(key)})
930
+ }
827
931
  }
828
932
  }
829
933
 
@@ -837,11 +941,11 @@ export class SimHost {
837
941
  }
838
942
  }
839
943
 
840
- requirePermission(requestId: string | undefined, permission: string): boolean {
944
+ requirePermission(requestId: string | undefined, permission: string, operation?: string): boolean {
841
945
  if (this.declared.has(permission)) return true
842
946
  const message = `${permission} permission not declared in miniapp.json. Add {"type": "${permission}"} to the "permissions" array.`
843
947
  this.log("host", "error", message)
844
- this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission})
948
+ this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission, ...(operation ? {operation} : {})})
845
949
  return false
846
950
  }
847
951
 
@@ -885,9 +989,9 @@ export class SimHost {
885
989
  reason: view === "main" ? "dashboard active" : "dashboard view not showing",
886
990
  })
887
991
  }
888
- this.publishDisplay()
889
992
  const settleMs = caps.animation ? settleTimeOf(elements) : 0
890
993
  this.settlesAt = Date.now() + settleMs
994
+ this.publishDisplay()
891
995
  if (!requestId) return
892
996
  setTimeout(() => this.result(requestId, true, {...base, presented: true}), this.frameLatencyMs + settleMs)
893
997
  }
@@ -920,8 +1024,8 @@ export class SimHost {
920
1024
  this.frameSeq++
921
1025
  this.scheduleExpiry(view, durationMs, elements.length > 0)
922
1026
  this.reportDropped(dropped)
923
- if (this.visibleView() === view) this.publishDisplay()
924
1027
  this.settlesAt = Date.now()
1028
+ if (this.visibleView() === view) this.publishDisplay()
925
1029
  this.result(requestId, true, {status: "displayed", degraded, dropped})
926
1030
  }
927
1031
 
@@ -1164,7 +1268,14 @@ export class SimHost {
1164
1268
  )
1165
1269
  }
1166
1270
  const data = {...extra, kind: gesture, gestureName: gesture, timestamp: Date.now()}
1167
- this.inject([MiniappStreamType.TOUCH_EVENT, `${MiniappStreamType.TOUCH_EVENT}:${gesture}`], data)
1271
+ const streams = [MiniappStreamType.TOUCH_EVENT, `${MiniappStreamType.TOUCH_EVENT}:${gesture}`]
1272
+ const receivers = this.registry.touchReceivers(gesture).filter((host) => host.listensTo(streams))
1273
+ if (receivers.length === 0) return this.undelivered(streams[0])
1274
+ for (const host of receivers) host.inject(streams, data)
1275
+ }
1276
+
1277
+ private listensTo(streams: string[]): boolean {
1278
+ return streams.some((stream) => this.subscriptions.has(stream))
1168
1279
  }
1169
1280
 
1170
1281
  button(pressType: "short" | "long" = "short", buttonId = "main"): void {
@@ -1202,12 +1313,17 @@ export class SimHost {
1202
1313
  ): void {
1203
1314
  const isFinal = options.isFinal ?? true
1204
1315
  const source = options.source ?? "es"
1205
- const targets = options.target ? [options.target] : this.translationTargets(source)
1206
- if (targets.length === 0) return this.undelivered(`translation:${source}:*`)
1316
+ let lanes = this.translationLanes(source)
1317
+ if (options.target) {
1318
+ lanes = lanes.filter((lane) => lane.target === options.target)
1319
+ if (lanes.length === 0) lanes = [{source: baseLanguage(source), target: options.target, stream: ""}]
1320
+ }
1321
+ if (lanes.length === 0) return this.undelivered(`translation:${source}:*`)
1207
1322
  const now = Date.now() - this.speechEpoch
1208
- for (const target of targets) {
1209
- const id = (this.openTranslations[target] ??= `utt_sim_tr_${++this.utteranceSeq}`)
1210
- this.deliverTranslation(source, target, {
1323
+ for (const lane of lanes) {
1324
+ const key = `${lane.source}:${lane.target}`
1325
+ const id = (this.openTranslations[key] ??= `utt_sim_tr_${++this.utteranceSeq}`)
1326
+ this.deliverTranslation(lane, {
1211
1327
  text,
1212
1328
  ...(options.originalText ? {originalText: options.originalText} : {}),
1213
1329
  isFinal,
@@ -1216,7 +1332,7 @@ export class SimHost {
1216
1332
  startTime: now,
1217
1333
  endTime: now,
1218
1334
  })
1219
- if (isFinal) delete this.openTranslations[target]
1335
+ if (isFinal) delete this.openTranslations[key]
1220
1336
  }
1221
1337
  }
1222
1338
 
@@ -1246,8 +1362,10 @@ export class SimHost {
1246
1362
  const emit = (count: number, isFinal: boolean, at: number) => {
1247
1363
  const result = {isFinal, speakerId, startTime: Math.round(start), endTime: Math.round(at)}
1248
1364
  this.deliverTranscription({...result, text: words.slice(0, count).join(" "), utteranceId: id, language: spoken})
1249
- for (const target of this.translationTargets(spoken)) {
1250
- translationIds[target] ??= `utt_sim_tr_${++this.utteranceSeq}`
1365
+ for (const lane of this.translationLanes(spoken)) {
1366
+ const {target} = lane
1367
+ const key = `${lane.source}:${target}`
1368
+ translationIds[key] ??= `utt_sim_tr_${++this.utteranceSeq}`
1251
1369
  const passthrough = baseLanguage(target) === baseLanguage(spoken)
1252
1370
  const original = words.slice(0, count).join(" ")
1253
1371
  const full =
@@ -1257,11 +1375,11 @@ export class SimHost {
1257
1375
  const translated = full.split(/\s+/)
1258
1376
  // Translations trail the source text by a word, as the real service does.
1259
1377
  const shown = isFinal ? translated.length : Math.max(0, Math.round(((count - 1) / words.length) * translated.length))
1260
- this.deliverTranslation(spoken, target, {
1378
+ this.deliverTranslation(lane, {
1261
1379
  ...result,
1262
1380
  text: passthrough ? original : translated.slice(0, shown).join(" "),
1263
1381
  ...(passthrough ? {} : {originalText: original}),
1264
- utteranceId: translationIds[target],
1382
+ utteranceId: translationIds[key],
1265
1383
  })
1266
1384
  }
1267
1385
  }
@@ -1287,6 +1405,26 @@ export class SimHost {
1287
1405
  })
1288
1406
  }
1289
1407
 
1408
+ feedAudio(pcm: Uint8Array, options: {realtime?: boolean} = {}): Promise<{chunks: number; durationMs: number}> {
1409
+ if (!this.subscriptions.has(MiniappStreamType.AUDIO_CHUNK)) {
1410
+ this.log("host", "warn", "The miniapp is not subscribed to audio_chunk (session.mic.onAudioChunk), so the audio goes nowhere")
1411
+ }
1412
+ const chunkBytes = (MIC_SAMPLE_RATE * 2) / 100
1413
+ const chunks: Uint8Array[] = []
1414
+ for (let at = 0; at < pcm.byteLength; at += chunkBytes) chunks.push(pcm.subarray(at, at + chunkBytes))
1415
+ const send = (chunk: Uint8Array) =>
1416
+ this.push(MiniappStreamType.AUDIO_CHUNK, {data: Buffer.from(chunk).toString("base64"), sampleRate: MIC_SAMPLE_RATE, format: "pcm_s16le"})
1417
+ const durationMs = Math.round((pcm.byteLength / 2 / MIC_SAMPLE_RATE) * 1000)
1418
+ if (options.realtime === false) {
1419
+ chunks.forEach(send)
1420
+ return Promise.resolve({chunks: chunks.length, durationMs})
1421
+ }
1422
+ return new Promise((resolve) => {
1423
+ chunks.forEach((chunk, i) => this.audioTimers.push(setTimeout(() => send(chunk), i * 10)))
1424
+ setTimeout(() => resolve({chunks: chunks.length, durationMs}), chunks.length * 10)
1425
+ })
1426
+ }
1427
+
1290
1428
  stopPlaying(): void {
1291
1429
  const playback = this.playback
1292
1430
  if (!playback) return
@@ -1294,16 +1432,20 @@ export class SimHost {
1294
1432
  playback.done()
1295
1433
  }
1296
1434
 
1297
- /** Concrete translation targets subscribed to; the phone runs no translation for a wildcard target. */
1298
- private translationTargets(spoken: string): string[] {
1299
- const targets = new Set<string>()
1435
+ // One lane per translation job the phone runs: a specific source keeps its
1436
+ // subscribed tag, a wildcard source gets the detected bare code. The phone
1437
+ // runs no translation for a wildcard target.
1438
+ private translationLanes(spoken: string): {source: string; target: string; stream: string}[] {
1439
+ const lanes = new Map<string, {source: string; target: string; stream: string}>()
1300
1440
  for (const stream of this.subscriptions.keys()) {
1301
- const [kind, source, target] = stream.split(":")
1441
+ const [kind, subscribed, target] = stream.split(":")
1302
1442
  if (kind !== "translation" || !target || target === "*") continue
1303
- if (source !== "*" && source !== "auto" && baseLanguage(source) !== baseLanguage(spoken)) continue
1304
- targets.add(target)
1443
+ const wildcard = subscribed === "*" || subscribed === "auto"
1444
+ if (!wildcard && baseLanguage(subscribed) !== baseLanguage(spoken)) continue
1445
+ const source = wildcard ? baseLanguage(spoken) : subscribed
1446
+ if (!lanes.has(`${source}:${target}`)) lanes.set(`${source}:${target}`, {source, target, stream})
1305
1447
  }
1306
- return [...targets]
1448
+ return [...lanes.values()]
1307
1449
  }
1308
1450
 
1309
1451
  // An auto subscriber gets the detected bare code; with none, a specific
@@ -1330,13 +1472,12 @@ export class SimHost {
1330
1472
  for (const stream of matching) this.pushRouted(stream, {...data, transcribeLanguage: stream.slice(14)}, this.subscriptions.get(stream))
1331
1473
  }
1332
1474
 
1333
- private deliverTranslation(spoken: string, target: string, result: Record<string, unknown>): void {
1334
- const source = [...this.subscriptions.keys()].some((stream) => stream.startsWith(`translation:${spoken}:`)) ? spoken : baseLanguage(spoken)
1335
- const route = [...this.subscriptions.entries()].find(([stream]) => stream.startsWith("translation:") && stream.endsWith(`:${target}`))
1475
+ private deliverTranslation(lane: {source: string; target: string; stream?: string}, result: Record<string, unknown>): void {
1476
+ const {source, target} = lane
1336
1477
  this.pushRouted(
1337
1478
  `translation:${source}:${target}`,
1338
1479
  {type: "translation", ...result, transcribeLanguage: source, translateLanguage: target, didTranslate: true, provider: "soniox"},
1339
- route?.[1],
1480
+ lane.stream ? this.subscriptions.get(lane.stream) : undefined,
1340
1481
  )
1341
1482
  }
1342
1483
 
@@ -1510,7 +1651,7 @@ export class SimHost {
1510
1651
  connected: this.handshook,
1511
1652
  paused: !this.glassesConnected,
1512
1653
  subscriptions: [...this.subscriptions.keys()],
1513
- claimedGestures: this.claimedGestures,
1654
+ claimedGestures: this.registry.heldGestures(this.manifest.packageName),
1514
1655
  permissions: [...this.declared],
1515
1656
  ui: {open: this.uiClients.size > 0, clients: this.uiClients.size},
1516
1657
  display: {
@@ -1530,6 +1671,8 @@ export class SimHost {
1530
1671
  colorScheme: this.colorScheme,
1531
1672
  phone: this.phone,
1532
1673
  unsimulated: [...this.unsimulated],
1674
+ faked: Object.fromEntries(this.faked),
1675
+ failNext: Object.fromEntries([...this.forced].map(([type, {code}]) => [type, code])),
1533
1676
  logSeq: this.logSeq,
1534
1677
  }
1535
1678
  }