@glassly/miniapp-cli 0.1.17-dev.0 → 0.1.17-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/sim/host.test.ts +447 -8
- package/src/sim/host.ts +204 -61
- package/src/sim/registry.ts +112 -22
- package/src/sim/script.test.ts +4 -0
- package/src/sim/script.ts +3 -3
- package/src/sim/server.ts +60 -14
- package/src/sim/sim.ts +4 -4
- package/src/sim/types/audio.test.ts +122 -0
- package/src/sim/types/audio.ts +269 -0
- package/src/sim/types/blobs-paths.ts +88 -0
- package/src/sim/types/blobs-store.ts +406 -0
- package/src/sim/types/blobs.test.ts +191 -0
- package/src/sim/types/blobs.ts +112 -0
- package/src/sim/types/context.ts +19 -0
- package/src/sim/types/hardware.test.ts +102 -0
- package/src/sim/types/hardware.ts +320 -0
- package/src/sim/types/index.ts +17 -0
- package/src/sim/types/navigation.test.ts +98 -0
- package/src/sim/types/navigation.ts +320 -0
- package/src/sim/types/services.test.ts +140 -0
- package/src/sim/types/services.ts +237 -0
- package/src/sim/wav.test.ts +36 -0
- package/src/sim/wav.ts +34 -0
package/src/sim/host.ts
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
type SceneElementInput,
|
|
13
13
|
} from "@glassly/miniapp/display"
|
|
14
14
|
// A namespace import, so an older @glassly/miniapp without the host rules still loads.
|
|
15
|
-
import {existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync} from "fs"
|
|
15
|
+
import {existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync} from "fs"
|
|
16
16
|
import {join} from "path"
|
|
17
17
|
import * as hostRules from "@glassly/miniapp/host"
|
|
18
18
|
import type {MirrorElement, MirrorSnapshot} from "@glassly/miniapp/mirror"
|
|
@@ -28,6 +28,8 @@ import type {PhoneKind} from "./phone.js"
|
|
|
28
28
|
import type {ScriptLine} from "./script.js"
|
|
29
29
|
import {SimRegistry} from "./registry.js"
|
|
30
30
|
import {createSimStorage, type SimStorage} from "./storage.js"
|
|
31
|
+
import {FORCED_MESSAGES, TYPE_HANDLERS, type SimTypeContext} from "./types/index.js"
|
|
32
|
+
import {MIC_SAMPLE_RATE} from "./wav.js"
|
|
31
33
|
|
|
32
34
|
export type SimLogSource = "background" | "ui" | "host" | "bridge"
|
|
33
35
|
|
|
@@ -61,6 +63,7 @@ export interface SimManifest {
|
|
|
61
63
|
permissions?: {type?: string}[]
|
|
62
64
|
actions?: unknown
|
|
63
65
|
widgets?: unknown
|
|
66
|
+
hardwareRequirements?: unknown
|
|
64
67
|
}
|
|
65
68
|
|
|
66
69
|
export interface SimHostOptions {
|
|
@@ -132,7 +135,7 @@ function permissionForStream(stream: string): string | null {
|
|
|
132
135
|
if (stream.startsWith("transcription") || stream.startsWith("translation")) return "MICROPHONE"
|
|
133
136
|
if (stream === "location_update" || stream === "heading_update") return "LOCATION"
|
|
134
137
|
if (stream === "phone_notification" || stream === "phone_notification_dismissed") return "READ_NOTIFICATIONS"
|
|
135
|
-
if (stream === MiniappStreamType.SETTINGS_CHANGED) return "PLATFORM_SETTINGS"
|
|
138
|
+
if (stream === MiniappStreamType.SETTINGS_CHANGED) return "isOpenSettingsKey" in hostRules ? null : "PLATFORM_SETTINGS"
|
|
136
139
|
return null
|
|
137
140
|
}
|
|
138
141
|
|
|
@@ -213,9 +216,7 @@ export class SimHost {
|
|
|
213
216
|
private handshook = false
|
|
214
217
|
private background: "stopped" | "running" | "crashed" = "stopped"
|
|
215
218
|
private subscriptions = new Map<string, StreamRoute>()
|
|
216
|
-
private claimedGestures: string[] = []
|
|
217
219
|
private derivedSettings: Record<string, unknown> = {}
|
|
218
|
-
private gestureClaims = hostRules.emptyGestureClaims?.()
|
|
219
220
|
private overlay = false
|
|
220
221
|
private revision = 0
|
|
221
222
|
private frameSeq = 0
|
|
@@ -231,6 +232,11 @@ export class SimHost {
|
|
|
231
232
|
private readonly logEntries: SimLogEntry[] = []
|
|
232
233
|
private readonly effects: SimEffect[] = []
|
|
233
234
|
private readonly unsimulated = new Set<string>()
|
|
235
|
+
private readonly faked = new Map<string, string>()
|
|
236
|
+
private readonly forced = new Map<string, {code: string; message: string}>()
|
|
237
|
+
private endTypeContext: () => void = () => {}
|
|
238
|
+
private typeContext: SimTypeContext = this.createTypeContext()
|
|
239
|
+
private audioTimers: ReturnType<typeof setTimeout>[] = []
|
|
234
240
|
|
|
235
241
|
private speakerState = "idle"
|
|
236
242
|
private utteranceSeq = 0
|
|
@@ -281,16 +287,19 @@ export class SimHost {
|
|
|
281
287
|
}
|
|
282
288
|
|
|
283
289
|
backgroundStopped(crashed: boolean): void {
|
|
290
|
+
this.registry.releaseWidgetHost(this.manifest.packageName)
|
|
284
291
|
this.resetSession()
|
|
285
292
|
this.background = crashed ? "crashed" : "stopped"
|
|
286
293
|
this.emit({type: "status"})
|
|
287
294
|
}
|
|
288
295
|
|
|
289
296
|
private resetSession(): void {
|
|
297
|
+
this.endTypeContext()
|
|
298
|
+
this.typeContext = this.createTypeContext()
|
|
299
|
+
for (const timer of this.audioTimers.splice(0)) clearTimeout(timer)
|
|
290
300
|
this.handshook = false
|
|
291
301
|
this.subscriptions.clear()
|
|
292
|
-
this.
|
|
293
|
-
this.gestureClaims = hostRules.emptyGestureClaims?.()
|
|
302
|
+
this.registry.dropGestureClaims(this.manifest.packageName)
|
|
294
303
|
this.overlay = false
|
|
295
304
|
this.scenes = new SceneStore()
|
|
296
305
|
for (const view of ["main", "dashboard"] as View[]) this.clearExpiry(view)
|
|
@@ -327,6 +336,13 @@ export class SimHost {
|
|
|
327
336
|
}
|
|
328
337
|
|
|
329
338
|
private dispatch(type: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
339
|
+
const forced = requestId ? this.forced.get(type) : undefined
|
|
340
|
+
if (forced) {
|
|
341
|
+
this.forced.delete(type)
|
|
342
|
+
this.log("host", "info", `Failing ${type} with ${forced.code}, as \`glassly sim fail\` asked`)
|
|
343
|
+
this.emit({type: "status"})
|
|
344
|
+
return this.fail(requestId, forced.code, forced.message)
|
|
345
|
+
}
|
|
330
346
|
if (this.registry.handle(this, type, payload, requestId)) return
|
|
331
347
|
switch (type) {
|
|
332
348
|
case MiniappRequestType.CONNECT:
|
|
@@ -362,11 +378,12 @@ export class SimHost {
|
|
|
362
378
|
return
|
|
363
379
|
case MiniappRequestType.TRANSCRIPTION_CONFIG:
|
|
364
380
|
return this.result(requestId, true, {applied: true})
|
|
365
|
-
case MiniappRequestType.AUTH_REFRESH:
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
})
|
|
381
|
+
case MiniappRequestType.AUTH_REFRESH: {
|
|
382
|
+
const auth = {userId: "sim-user", token: `glassly-sim.${this.manifest.packageName}.${Date.now()}`, expiresAt: Date.now() + 3_600_000}
|
|
383
|
+
this.typeContext.fake(type, "a placeholder token for user sim-user that no server accepts")
|
|
384
|
+
this.send({type: MiniappResponseType.AUTH_UPDATE, auth})
|
|
385
|
+
return this.result(requestId, true, {auth})
|
|
386
|
+
}
|
|
370
387
|
|
|
371
388
|
case MiniappRequestType.STORAGE_GET:
|
|
372
389
|
return this.result(requestId, true, {key: payload.key, value: this.storage.get(this.key(type, payload))})
|
|
@@ -402,8 +419,17 @@ export class SimHost {
|
|
|
402
419
|
case MiniappRequestType.LOCATION_POLL:
|
|
403
420
|
return this.result(requestId, true, this.location)
|
|
404
421
|
|
|
405
|
-
case MiniappRequestType.SPEAK:
|
|
406
|
-
|
|
422
|
+
case MiniappRequestType.SPEAK: {
|
|
423
|
+
const raw = payload.text
|
|
424
|
+
const trimmed = (typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw.filter((s): s is string => typeof s === "string") : [])
|
|
425
|
+
.map((sentence) => sentence.trim())
|
|
426
|
+
.filter(Boolean)
|
|
427
|
+
const sentences = hostRules.prepareTtsSentences?.(trimmed, payload.enableSanitization !== false) ?? trimmed
|
|
428
|
+
if (sentences.length === 0) {
|
|
429
|
+
return this.fail(requestId, MiniappErrorCode.INTERNAL, "speak requires text or a non-empty sentence list with speakable content")
|
|
430
|
+
}
|
|
431
|
+
return this.handleSpeak(sentences.join(" "), requestId)
|
|
432
|
+
}
|
|
407
433
|
case MiniappRequestType.PLAY_AUDIO:
|
|
408
434
|
if (typeof payload.audioUrl !== "string" || !payload.audioUrl) {
|
|
409
435
|
return this.fail(requestId, MiniappErrorCode.INTERNAL, "play_audio requires an audioUrl")
|
|
@@ -412,21 +438,25 @@ export class SimHost {
|
|
|
412
438
|
return this.startSpeech(1500, requestId, undefined)
|
|
413
439
|
case MiniappRequestType.STOP_AUDIO:
|
|
414
440
|
this.effect("stop_audio", {})
|
|
415
|
-
|
|
441
|
+
this.finishSpeech(false)
|
|
442
|
+
return this.result(requestId, true)
|
|
416
443
|
|
|
417
444
|
case MiniappRequestType.RGB_LED:
|
|
418
445
|
this.effect("led", {...payload, type: undefined})
|
|
419
446
|
return this.result(requestId, true)
|
|
420
447
|
case MiniappRequestType.OPEN_URL:
|
|
421
|
-
this.effect("open_url", {url: payload.url})
|
|
422
|
-
return this.result(requestId, true)
|
|
448
|
+
return this.effect("open_url", {url: payload.url})
|
|
423
449
|
case MiniappRequestType.COPY_CLIPBOARD:
|
|
450
|
+
if (typeof payload.text !== "string") return this.log("host", "warn", "copy_clipboard without text; the phone ignores it and never answers")
|
|
424
451
|
this.effect("clipboard", {text: payload.text})
|
|
425
452
|
return this.result(requestId, true)
|
|
426
453
|
case MiniappRequestType.SHARE:
|
|
427
454
|
this.effect("share", {...payload, type: undefined})
|
|
428
455
|
return this.result(requestId, true, {success: true})
|
|
429
456
|
case MiniappRequestType.DOWNLOAD:
|
|
457
|
+
if (typeof payload.base64 !== "string" && typeof payload.url !== "string") {
|
|
458
|
+
return this.log("host", "warn", "download without base64 or url; the phone ignores it and never answers")
|
|
459
|
+
}
|
|
430
460
|
this.effect("download", {...payload, type: undefined})
|
|
431
461
|
return this.result(requestId, true, {success: true})
|
|
432
462
|
|
|
@@ -443,21 +473,21 @@ export class SimHost {
|
|
|
443
473
|
case MiniappRequestType.SETTINGS_GET: {
|
|
444
474
|
const {selectSettingsBridgeKeys} = hostRules
|
|
445
475
|
if (!selectSettingsBridgeKeys) break
|
|
446
|
-
if (!this.
|
|
476
|
+
if (!this.mayUseSettings("get", payload, requestId)) return
|
|
447
477
|
const values: Record<string, unknown> = {}
|
|
448
|
-
for (const key of selectSettingsBridgeKeys(payload.keys)) values[key] = this.settingValue(key)
|
|
478
|
+
for (const key of selectSettingsBridgeKeys(payload.keys, this.platformSettings)) values[key] = this.settingValue(key)
|
|
449
479
|
return this.result(requestId, true, {values})
|
|
450
480
|
}
|
|
451
481
|
case MiniappRequestType.SETTINGS_SET: {
|
|
452
482
|
const {validateSettingsSet} = hostRules
|
|
453
483
|
if (!validateSettingsSet) break
|
|
454
|
-
if (!this.
|
|
484
|
+
if (!this.mayUseSettings("set", payload, requestId)) return
|
|
455
485
|
const validated = validateSettingsSet(payload)
|
|
456
486
|
if (!validated.ok) return this.fail(requestId, validated.code, validated.message)
|
|
457
487
|
const changed = JSON.stringify(this.settingValue(validated.key)) !== JSON.stringify(validated.value ?? null)
|
|
458
488
|
this.settings.set(validated.key, validated.value)
|
|
459
489
|
this.result(requestId, true)
|
|
460
|
-
if (changed)
|
|
490
|
+
if (changed) for (const app of this.registry.apps()) app.settingChanged(validated.key)
|
|
461
491
|
return
|
|
462
492
|
}
|
|
463
493
|
|
|
@@ -472,7 +502,7 @@ export class SimHost {
|
|
|
472
502
|
|
|
473
503
|
case MiniappRequestType.SYSTEM_EXIT:
|
|
474
504
|
case MiniappRequestType.MINIAPPS_STOP_SELF:
|
|
475
|
-
this.result(requestId, true)
|
|
505
|
+
if (type === MiniappRequestType.SYSTEM_EXIT) this.result(requestId, true)
|
|
476
506
|
this.log("host", "info", "The miniapp asked to stop (session.system.exit)")
|
|
477
507
|
return this.emit({type: "exit", reason: "the miniapp called exit"})
|
|
478
508
|
|
|
@@ -499,6 +529,8 @@ export class SimHost {
|
|
|
499
529
|
return this.emit({type: "ui", frame: {type: "cancel", requestId: payload.requestId}})
|
|
500
530
|
}
|
|
501
531
|
|
|
532
|
+
for (const handle of TYPE_HANDLERS) if (handle(this.typeContext, type, payload, requestId)) return
|
|
533
|
+
|
|
502
534
|
if (!this.unsimulated.has(type)) {
|
|
503
535
|
this.unsimulated.add(type)
|
|
504
536
|
this.log("host", "warn", `"${type}" is not simulated; the request was answered with NOT_IMPLEMENTED`)
|
|
@@ -512,20 +544,58 @@ export class SimHost {
|
|
|
512
544
|
|
|
513
545
|
private claimGestures(payload: Record<string, unknown>, requestId?: string): void {
|
|
514
546
|
const gestures = Array.isArray(payload.gestures) ? payload.gestures.filter((g): g is string => typeof g === "string") : []
|
|
515
|
-
|
|
516
|
-
if (!this.gestureClaims || !setGestureClaims || !gestureClaimant) {
|
|
517
|
-
this.claimedGestures = gestures
|
|
518
|
-
} else {
|
|
519
|
-
const {state, changes} = setGestureClaims(this.gestureClaims, this.manifest.packageName, gestures, payload.scope as string | undefined)
|
|
520
|
-
this.gestureClaims = state
|
|
521
|
-
const held = new Set(state.registrations.flatMap((entry) => [...entry.gestures]))
|
|
522
|
-
this.claimedGestures = [...held].filter((gesture) => gestureClaimant(state, gesture) === this.manifest.packageName)
|
|
523
|
-
for (const change of changes) this.push(MiniappStreamType.GESTURE_CLAIM, {gesture: change.gesture, held: change.held})
|
|
524
|
-
}
|
|
547
|
+
this.registry.claimGestures(this, gestures, payload.scope as string | undefined)
|
|
525
548
|
this.emit({type: "status"})
|
|
526
549
|
this.result(requestId, true)
|
|
527
550
|
}
|
|
528
551
|
|
|
552
|
+
failNext(type: string, code: string, message?: string): string {
|
|
553
|
+
const wire = type.startsWith("miniapp_") ? type : `miniapp_${type.toLowerCase()}`
|
|
554
|
+
if (!(Object.values(MiniappRequestType) as string[]).includes(wire)) throw new Error(`Unknown request type "${type}"`)
|
|
555
|
+
if (!code) throw new Error("glassly sim fail needs an error code")
|
|
556
|
+
this.forced.set(wire, {code, message: message ?? FORCED_MESSAGES[code] ?? `${code} (forced by glassly sim fail)`})
|
|
557
|
+
this.emit({type: "status"})
|
|
558
|
+
return wire
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Each background session gets its own context; the old one goes quiet, so a
|
|
562
|
+
// module's leftover timers cannot reach the next session.
|
|
563
|
+
private createTypeContext(): SimTypeContext {
|
|
564
|
+
let live = true
|
|
565
|
+
const onEnd: (() => void)[] = []
|
|
566
|
+
this.endTypeContext = () => {
|
|
567
|
+
live = false
|
|
568
|
+
for (const callback of onEnd.splice(0)) callback()
|
|
569
|
+
}
|
|
570
|
+
const context = {
|
|
571
|
+
declares: (permission: string) => this.declared.has(permission),
|
|
572
|
+
requirePermission: (requestId, permission, operation) => this.requirePermission(requestId, permission, operation),
|
|
573
|
+
reply: (requestId, ok, data, error) => live && this.result(requestId, ok, data, error),
|
|
574
|
+
fail: (requestId, code, message) => live && this.fail(requestId, code, message),
|
|
575
|
+
deliver: (payload) => live && this.send(payload),
|
|
576
|
+
push: (stream, data) => live && this.push(stream, data),
|
|
577
|
+
fake: (type, note) => {
|
|
578
|
+
if (this.faked.get(type) === note) return
|
|
579
|
+
this.faked.set(type, note)
|
|
580
|
+
this.emit({type: "status"})
|
|
581
|
+
},
|
|
582
|
+
effect: (kind, detail) => this.effect(kind, detail),
|
|
583
|
+
log: (level, message) => this.log("host", level, message),
|
|
584
|
+
onEnd: (callback) => onEnd.push(callback),
|
|
585
|
+
} as Omit<SimTypeContext, "packageName" | "device" | "stateDir" | "glassesConnected">
|
|
586
|
+
return Object.defineProperties(context, {
|
|
587
|
+
packageName: {get: () => this.manifest.packageName},
|
|
588
|
+
device: {get: () => this.device},
|
|
589
|
+
stateDir: {get: () => this.stateDir},
|
|
590
|
+
glassesConnected: {get: () => this.glassesConnected},
|
|
591
|
+
}) as SimTypeContext
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
gestureClaimChanged(gesture: string, held: boolean): void {
|
|
595
|
+
this.push(MiniappStreamType.GESTURE_CLAIM, {gesture, held})
|
|
596
|
+
this.emit({type: "status"})
|
|
597
|
+
}
|
|
598
|
+
|
|
529
599
|
private key(type: string, payload: Record<string, unknown>): string {
|
|
530
600
|
if (typeof payload.key !== "string" || !payload.key) throw new Error(`${type.replace(/^miniapp_/, "")} requires a key`)
|
|
531
601
|
return payload.key
|
|
@@ -539,14 +609,35 @@ export class SimHost {
|
|
|
539
609
|
if (this.derivedSettings[key] === value) continue
|
|
540
610
|
const known = key in this.derivedSettings
|
|
541
611
|
this.derivedSettings[key] = value
|
|
542
|
-
if (known) this.
|
|
612
|
+
if (known) this.pushSetting(key)
|
|
543
613
|
}
|
|
544
614
|
}
|
|
545
615
|
|
|
616
|
+
settingChanged(key: string): void {
|
|
617
|
+
this.pushSetting(key)
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
private get platformSettings(): boolean {
|
|
621
|
+
return this.declared.has("PLATFORM_SETTINGS")
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
private mayUseSettings(kind: "get" | "set", payload: Record<string, unknown>, requestId?: string): boolean {
|
|
625
|
+
const open = hostRules.settingsRequestNeedsPermission ? !hostRules.settingsRequestNeedsPermission(kind, payload) : false
|
|
626
|
+
return open || this.requirePermission(requestId, "PLATFORM_SETTINGS")
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private pushSetting(key: string): void {
|
|
630
|
+
if (hostRules.settingsBridgeAccess?.(key, this.platformSettings) === null) return
|
|
631
|
+
this.push(MiniappStreamType.SETTINGS_CHANGED, {key, value: this.settingValue(key)})
|
|
632
|
+
}
|
|
633
|
+
|
|
546
634
|
private settingValue(key: string): unknown {
|
|
547
635
|
if (key === "phone_active") return this.visibility === "foreground"
|
|
548
636
|
if (key === "foreground_app_on_screen") return this.visibleView() === "main" && this.scenes.lastFrame(this.manifest.packageName, "main").length > 0
|
|
549
637
|
if (key === "controllers") return []
|
|
638
|
+
if (key === hostRules.READING_APP_SPEED && hostRules.readingAppSpeedFor) {
|
|
639
|
+
return hostRules.readingAppSpeedFor(this.settings.get(hostRules.READING_APP_SPEEDS), this.manifest.packageName)
|
|
640
|
+
}
|
|
550
641
|
return this.settings.has(key) ? this.settings.get(key) : (hostRules.settingsBridgeDefault?.(key) ?? null)
|
|
551
642
|
}
|
|
552
643
|
|
|
@@ -700,6 +791,14 @@ export class SimHost {
|
|
|
700
791
|
return this.manifest.packageName
|
|
701
792
|
}
|
|
702
793
|
|
|
794
|
+
get running(): boolean {
|
|
795
|
+
return this.background === "running"
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
get deviceCapabilities(): Record<string, unknown> {
|
|
799
|
+
return this.device.capabilities
|
|
800
|
+
}
|
|
801
|
+
|
|
703
802
|
/** Connected and running: requests to it are answered. */
|
|
704
803
|
get connected(): boolean {
|
|
705
804
|
return this.handshook && this.background === "running"
|
|
@@ -719,7 +818,10 @@ export class SimHost {
|
|
|
719
818
|
this.storage.clear()
|
|
720
819
|
this.settings.clear()
|
|
721
820
|
await this.memory?.clear()
|
|
722
|
-
this.
|
|
821
|
+
this.endTypeContext()
|
|
822
|
+
this.typeContext = this.createTypeContext()
|
|
823
|
+
if (this.stateDir) rmSync(join(this.stateDir, "blobs"), {recursive: true, force: true})
|
|
824
|
+
this.log("host", "info", "Storage, settings, memory and blobs cleared")
|
|
723
825
|
}
|
|
724
826
|
|
|
725
827
|
private handleNotificationPost(payload: Record<string, unknown>, requestId?: string): void {
|
|
@@ -823,7 +925,9 @@ export class SimHost {
|
|
|
823
925
|
else if (stream === "glasses_wifi") this.push(stream, {connected: false})
|
|
824
926
|
else if (stream === MiniappStreamType.MIRROR_UPDATE) this.push(stream, this.snapshot())
|
|
825
927
|
else if (stream === MiniappStreamType.SETTINGS_CHANGED) {
|
|
826
|
-
for (const key of hostRules.selectSettingsBridgeKeys?.(undefined) ?? [])
|
|
928
|
+
for (const key of hostRules.selectSettingsBridgeKeys?.(undefined, this.platformSettings) ?? []) {
|
|
929
|
+
this.push(stream, {key, value: this.settingValue(key)})
|
|
930
|
+
}
|
|
827
931
|
}
|
|
828
932
|
}
|
|
829
933
|
|
|
@@ -837,11 +941,11 @@ export class SimHost {
|
|
|
837
941
|
}
|
|
838
942
|
}
|
|
839
943
|
|
|
840
|
-
requirePermission(requestId: string | undefined, permission: string): boolean {
|
|
944
|
+
requirePermission(requestId: string | undefined, permission: string, operation?: string): boolean {
|
|
841
945
|
if (this.declared.has(permission)) return true
|
|
842
946
|
const message = `${permission} permission not declared in miniapp.json. Add {"type": "${permission}"} to the "permissions" array.`
|
|
843
947
|
this.log("host", "error", message)
|
|
844
|
-
this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission})
|
|
948
|
+
this.result(requestId, false, undefined, {code: MiniappErrorCode.PERMISSION_NOT_DECLARED, message, permission, ...(operation ? {operation} : {})})
|
|
845
949
|
return false
|
|
846
950
|
}
|
|
847
951
|
|
|
@@ -885,9 +989,9 @@ export class SimHost {
|
|
|
885
989
|
reason: view === "main" ? "dashboard active" : "dashboard view not showing",
|
|
886
990
|
})
|
|
887
991
|
}
|
|
888
|
-
this.publishDisplay()
|
|
889
992
|
const settleMs = caps.animation ? settleTimeOf(elements) : 0
|
|
890
993
|
this.settlesAt = Date.now() + settleMs
|
|
994
|
+
this.publishDisplay()
|
|
891
995
|
if (!requestId) return
|
|
892
996
|
setTimeout(() => this.result(requestId, true, {...base, presented: true}), this.frameLatencyMs + settleMs)
|
|
893
997
|
}
|
|
@@ -920,8 +1024,8 @@ export class SimHost {
|
|
|
920
1024
|
this.frameSeq++
|
|
921
1025
|
this.scheduleExpiry(view, durationMs, elements.length > 0)
|
|
922
1026
|
this.reportDropped(dropped)
|
|
923
|
-
if (this.visibleView() === view) this.publishDisplay()
|
|
924
1027
|
this.settlesAt = Date.now()
|
|
1028
|
+
if (this.visibleView() === view) this.publishDisplay()
|
|
925
1029
|
this.result(requestId, true, {status: "displayed", degraded, dropped})
|
|
926
1030
|
}
|
|
927
1031
|
|
|
@@ -1164,7 +1268,14 @@ export class SimHost {
|
|
|
1164
1268
|
)
|
|
1165
1269
|
}
|
|
1166
1270
|
const data = {...extra, kind: gesture, gestureName: gesture, timestamp: Date.now()}
|
|
1167
|
-
|
|
1271
|
+
const streams = [MiniappStreamType.TOUCH_EVENT, `${MiniappStreamType.TOUCH_EVENT}:${gesture}`]
|
|
1272
|
+
const receivers = this.registry.touchReceivers(gesture).filter((host) => host.listensTo(streams))
|
|
1273
|
+
if (receivers.length === 0) return this.undelivered(streams[0])
|
|
1274
|
+
for (const host of receivers) host.inject(streams, data)
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
private listensTo(streams: string[]): boolean {
|
|
1278
|
+
return streams.some((stream) => this.subscriptions.has(stream))
|
|
1168
1279
|
}
|
|
1169
1280
|
|
|
1170
1281
|
button(pressType: "short" | "long" = "short", buttonId = "main"): void {
|
|
@@ -1202,12 +1313,17 @@ export class SimHost {
|
|
|
1202
1313
|
): void {
|
|
1203
1314
|
const isFinal = options.isFinal ?? true
|
|
1204
1315
|
const source = options.source ?? "es"
|
|
1205
|
-
|
|
1206
|
-
if (
|
|
1316
|
+
let lanes = this.translationLanes(source)
|
|
1317
|
+
if (options.target) {
|
|
1318
|
+
lanes = lanes.filter((lane) => lane.target === options.target)
|
|
1319
|
+
if (lanes.length === 0) lanes = [{source: baseLanguage(source), target: options.target, stream: ""}]
|
|
1320
|
+
}
|
|
1321
|
+
if (lanes.length === 0) return this.undelivered(`translation:${source}:*`)
|
|
1207
1322
|
const now = Date.now() - this.speechEpoch
|
|
1208
|
-
for (const
|
|
1209
|
-
const
|
|
1210
|
-
this.
|
|
1323
|
+
for (const lane of lanes) {
|
|
1324
|
+
const key = `${lane.source}:${lane.target}`
|
|
1325
|
+
const id = (this.openTranslations[key] ??= `utt_sim_tr_${++this.utteranceSeq}`)
|
|
1326
|
+
this.deliverTranslation(lane, {
|
|
1211
1327
|
text,
|
|
1212
1328
|
...(options.originalText ? {originalText: options.originalText} : {}),
|
|
1213
1329
|
isFinal,
|
|
@@ -1216,7 +1332,7 @@ export class SimHost {
|
|
|
1216
1332
|
startTime: now,
|
|
1217
1333
|
endTime: now,
|
|
1218
1334
|
})
|
|
1219
|
-
if (isFinal) delete this.openTranslations[
|
|
1335
|
+
if (isFinal) delete this.openTranslations[key]
|
|
1220
1336
|
}
|
|
1221
1337
|
}
|
|
1222
1338
|
|
|
@@ -1246,8 +1362,10 @@ export class SimHost {
|
|
|
1246
1362
|
const emit = (count: number, isFinal: boolean, at: number) => {
|
|
1247
1363
|
const result = {isFinal, speakerId, startTime: Math.round(start), endTime: Math.round(at)}
|
|
1248
1364
|
this.deliverTranscription({...result, text: words.slice(0, count).join(" "), utteranceId: id, language: spoken})
|
|
1249
|
-
for (const
|
|
1250
|
-
|
|
1365
|
+
for (const lane of this.translationLanes(spoken)) {
|
|
1366
|
+
const {target} = lane
|
|
1367
|
+
const key = `${lane.source}:${target}`
|
|
1368
|
+
translationIds[key] ??= `utt_sim_tr_${++this.utteranceSeq}`
|
|
1251
1369
|
const passthrough = baseLanguage(target) === baseLanguage(spoken)
|
|
1252
1370
|
const original = words.slice(0, count).join(" ")
|
|
1253
1371
|
const full =
|
|
@@ -1257,11 +1375,11 @@ export class SimHost {
|
|
|
1257
1375
|
const translated = full.split(/\s+/)
|
|
1258
1376
|
// Translations trail the source text by a word, as the real service does.
|
|
1259
1377
|
const shown = isFinal ? translated.length : Math.max(0, Math.round(((count - 1) / words.length) * translated.length))
|
|
1260
|
-
this.deliverTranslation(
|
|
1378
|
+
this.deliverTranslation(lane, {
|
|
1261
1379
|
...result,
|
|
1262
1380
|
text: passthrough ? original : translated.slice(0, shown).join(" "),
|
|
1263
1381
|
...(passthrough ? {} : {originalText: original}),
|
|
1264
|
-
utteranceId: translationIds[
|
|
1382
|
+
utteranceId: translationIds[key],
|
|
1265
1383
|
})
|
|
1266
1384
|
}
|
|
1267
1385
|
}
|
|
@@ -1287,6 +1405,26 @@ export class SimHost {
|
|
|
1287
1405
|
})
|
|
1288
1406
|
}
|
|
1289
1407
|
|
|
1408
|
+
feedAudio(pcm: Uint8Array, options: {realtime?: boolean} = {}): Promise<{chunks: number; durationMs: number}> {
|
|
1409
|
+
if (!this.subscriptions.has(MiniappStreamType.AUDIO_CHUNK)) {
|
|
1410
|
+
this.log("host", "warn", "The miniapp is not subscribed to audio_chunk (session.mic.onAudioChunk), so the audio goes nowhere")
|
|
1411
|
+
}
|
|
1412
|
+
const chunkBytes = (MIC_SAMPLE_RATE * 2) / 100
|
|
1413
|
+
const chunks: Uint8Array[] = []
|
|
1414
|
+
for (let at = 0; at < pcm.byteLength; at += chunkBytes) chunks.push(pcm.subarray(at, at + chunkBytes))
|
|
1415
|
+
const send = (chunk: Uint8Array) =>
|
|
1416
|
+
this.push(MiniappStreamType.AUDIO_CHUNK, {data: Buffer.from(chunk).toString("base64"), sampleRate: MIC_SAMPLE_RATE, format: "pcm_s16le"})
|
|
1417
|
+
const durationMs = Math.round((pcm.byteLength / 2 / MIC_SAMPLE_RATE) * 1000)
|
|
1418
|
+
if (options.realtime === false) {
|
|
1419
|
+
chunks.forEach(send)
|
|
1420
|
+
return Promise.resolve({chunks: chunks.length, durationMs})
|
|
1421
|
+
}
|
|
1422
|
+
return new Promise((resolve) => {
|
|
1423
|
+
chunks.forEach((chunk, i) => this.audioTimers.push(setTimeout(() => send(chunk), i * 10)))
|
|
1424
|
+
setTimeout(() => resolve({chunks: chunks.length, durationMs}), chunks.length * 10)
|
|
1425
|
+
})
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1290
1428
|
stopPlaying(): void {
|
|
1291
1429
|
const playback = this.playback
|
|
1292
1430
|
if (!playback) return
|
|
@@ -1294,16 +1432,20 @@ export class SimHost {
|
|
|
1294
1432
|
playback.done()
|
|
1295
1433
|
}
|
|
1296
1434
|
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1435
|
+
// One lane per translation job the phone runs: a specific source keeps its
|
|
1436
|
+
// subscribed tag, a wildcard source gets the detected bare code. The phone
|
|
1437
|
+
// runs no translation for a wildcard target.
|
|
1438
|
+
private translationLanes(spoken: string): {source: string; target: string; stream: string}[] {
|
|
1439
|
+
const lanes = new Map<string, {source: string; target: string; stream: string}>()
|
|
1300
1440
|
for (const stream of this.subscriptions.keys()) {
|
|
1301
|
-
const [kind,
|
|
1441
|
+
const [kind, subscribed, target] = stream.split(":")
|
|
1302
1442
|
if (kind !== "translation" || !target || target === "*") continue
|
|
1303
|
-
|
|
1304
|
-
|
|
1443
|
+
const wildcard = subscribed === "*" || subscribed === "auto"
|
|
1444
|
+
if (!wildcard && baseLanguage(subscribed) !== baseLanguage(spoken)) continue
|
|
1445
|
+
const source = wildcard ? baseLanguage(spoken) : subscribed
|
|
1446
|
+
if (!lanes.has(`${source}:${target}`)) lanes.set(`${source}:${target}`, {source, target, stream})
|
|
1305
1447
|
}
|
|
1306
|
-
return [...
|
|
1448
|
+
return [...lanes.values()]
|
|
1307
1449
|
}
|
|
1308
1450
|
|
|
1309
1451
|
// An auto subscriber gets the detected bare code; with none, a specific
|
|
@@ -1330,13 +1472,12 @@ export class SimHost {
|
|
|
1330
1472
|
for (const stream of matching) this.pushRouted(stream, {...data, transcribeLanguage: stream.slice(14)}, this.subscriptions.get(stream))
|
|
1331
1473
|
}
|
|
1332
1474
|
|
|
1333
|
-
private deliverTranslation(
|
|
1334
|
-
const source
|
|
1335
|
-
const route = [...this.subscriptions.entries()].find(([stream]) => stream.startsWith("translation:") && stream.endsWith(`:${target}`))
|
|
1475
|
+
private deliverTranslation(lane: {source: string; target: string; stream?: string}, result: Record<string, unknown>): void {
|
|
1476
|
+
const {source, target} = lane
|
|
1336
1477
|
this.pushRouted(
|
|
1337
1478
|
`translation:${source}:${target}`,
|
|
1338
1479
|
{type: "translation", ...result, transcribeLanguage: source, translateLanguage: target, didTranslate: true, provider: "soniox"},
|
|
1339
|
-
|
|
1480
|
+
lane.stream ? this.subscriptions.get(lane.stream) : undefined,
|
|
1340
1481
|
)
|
|
1341
1482
|
}
|
|
1342
1483
|
|
|
@@ -1510,7 +1651,7 @@ export class SimHost {
|
|
|
1510
1651
|
connected: this.handshook,
|
|
1511
1652
|
paused: !this.glassesConnected,
|
|
1512
1653
|
subscriptions: [...this.subscriptions.keys()],
|
|
1513
|
-
claimedGestures: this.
|
|
1654
|
+
claimedGestures: this.registry.heldGestures(this.manifest.packageName),
|
|
1514
1655
|
permissions: [...this.declared],
|
|
1515
1656
|
ui: {open: this.uiClients.size > 0, clients: this.uiClients.size},
|
|
1516
1657
|
display: {
|
|
@@ -1530,6 +1671,8 @@ export class SimHost {
|
|
|
1530
1671
|
colorScheme: this.colorScheme,
|
|
1531
1672
|
phone: this.phone,
|
|
1532
1673
|
unsimulated: [...this.unsimulated],
|
|
1674
|
+
faked: Object.fromEntries(this.faked),
|
|
1675
|
+
failNext: Object.fromEntries([...this.forced].map(([type, {code}]) => [type, code])),
|
|
1533
1676
|
logSeq: this.logSeq,
|
|
1534
1677
|
}
|
|
1535
1678
|
}
|