@glassly/engine 0.3.3-dev.670 → 0.3.4-beta.689

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/build/engine.d.ts +38 -14
  2. package/build/engine.d.ts.map +1 -1
  3. package/build/engine.js +3 -0
  4. package/build/engine.js.map +1 -1
  5. package/build/facades/ai.d.ts +32 -0
  6. package/build/facades/ai.d.ts.map +1 -0
  7. package/build/facades/ai.js +38 -0
  8. package/build/facades/ai.js.map +1 -0
  9. package/build/facades/glasses.d.ts +30 -24
  10. package/build/facades/glasses.d.ts.map +1 -1
  11. package/build/facades/glasses.js +49 -30
  12. package/build/facades/glasses.js.map +1 -1
  13. package/build/facades/glassesSettings.d.ts.map +1 -1
  14. package/build/facades/glassesSettings.js +1 -0
  15. package/build/facades/glassesSettings.js.map +1 -1
  16. package/build/facades/pairing.d.ts +6 -5
  17. package/build/facades/pairing.d.ts.map +1 -1
  18. package/build/facades/pairing.js +14 -14
  19. package/build/facades/pairing.js.map +1 -1
  20. package/build/generated/releaseMetadata.js +4 -4
  21. package/build/generated/releaseMetadata.js.map +1 -1
  22. package/build/index.d.ts +5 -2
  23. package/build/index.d.ts.map +1 -1
  24. package/build/index.js +2 -0
  25. package/build/index.js.map +1 -1
  26. package/build/services/DeviceEventRouter.d.ts.map +1 -1
  27. package/build/services/DeviceEventRouter.js +9 -1
  28. package/build/services/DeviceEventRouter.js.map +1 -1
  29. package/build/services/GestureDotsService.d.ts +14 -3
  30. package/build/services/GestureDotsService.d.ts.map +1 -1
  31. package/build/services/GestureDotsService.js +48 -13
  32. package/build/services/GestureDotsService.js.map +1 -1
  33. package/build/services/GlassesTouchInput.d.ts.map +1 -1
  34. package/build/services/GlassesTouchInput.js +11 -9
  35. package/build/services/GlassesTouchInput.js.map +1 -1
  36. package/build/services/LlmRouting.d.ts +35 -0
  37. package/build/services/LlmRouting.d.ts.map +1 -0
  38. package/build/services/LlmRouting.js +25 -0
  39. package/build/services/LlmRouting.js.map +1 -0
  40. package/build/services/LocalLlmModelManager.d.ts +79 -0
  41. package/build/services/LocalLlmModelManager.d.ts.map +1 -0
  42. package/build/services/LocalLlmModelManager.js +278 -0
  43. package/build/services/LocalLlmModelManager.js.map +1 -0
  44. package/build/services/LocalLlmPrompt.d.ts +90 -0
  45. package/build/services/LocalLlmPrompt.d.ts.map +1 -0
  46. package/build/services/LocalLlmPrompt.js +189 -0
  47. package/build/services/LocalLlmPrompt.js.map +1 -0
  48. package/build/services/LocalLlmProvider.d.ts +35 -0
  49. package/build/services/LocalLlmProvider.d.ts.map +1 -0
  50. package/build/services/LocalLlmProvider.js +218 -0
  51. package/build/services/LocalLlmProvider.js.map +1 -0
  52. package/build/services/LocalMiniappRuntime.d.ts +9 -1
  53. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  54. package/build/services/LocalMiniappRuntime.js +91 -17
  55. package/build/services/LocalMiniappRuntime.js.map +1 -1
  56. package/build/services/STTModelManager.d.ts +25 -2
  57. package/build/services/STTModelManager.d.ts.map +1 -1
  58. package/build/services/STTModelManager.js +137 -6
  59. package/build/services/STTModelManager.js.map +1 -1
  60. package/build/services/controllers.d.ts +21 -0
  61. package/build/services/controllers.d.ts.map +1 -0
  62. package/build/services/controllers.js +62 -0
  63. package/build/services/controllers.js.map +1 -0
  64. package/build/services/effectiveCapabilities.d.ts.map +1 -1
  65. package/build/services/effectiveCapabilities.js +18 -24
  66. package/build/services/effectiveCapabilities.js.map +1 -1
  67. package/build/services/neuralBand/NeuralBandInput.d.ts +17 -6
  68. package/build/services/neuralBand/NeuralBandInput.d.ts.map +1 -1
  69. package/build/services/neuralBand/NeuralBandInput.js +40 -23
  70. package/build/services/neuralBand/NeuralBandInput.js.map +1 -1
  71. package/build/services/neuralBand/NeuralBandSettingsSync.d.ts.map +1 -1
  72. package/build/services/neuralBand/NeuralBandSettingsSync.js +23 -26
  73. package/build/services/neuralBand/NeuralBandSettingsSync.js.map +1 -1
  74. package/build/services/settingsBridge.d.ts +26 -18
  75. package/build/services/settingsBridge.d.ts.map +1 -1
  76. package/build/services/settingsBridge.js +36 -15
  77. package/build/services/settingsBridge.js.map +1 -1
  78. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
  79. package/build/stores/bluetoothSettingKeys.js +3 -1
  80. package/build/stores/bluetoothSettingKeys.js.map +1 -1
  81. package/build/stores/core.d.ts.map +1 -1
  82. package/build/stores/core.js +0 -1
  83. package/build/stores/core.js.map +1 -1
  84. package/build/stores/glasses.d.ts.map +1 -1
  85. package/build/stores/glasses.js +2 -6
  86. package/build/stores/glasses.js.map +1 -1
  87. package/build/stores/settings.d.ts.map +1 -1
  88. package/build/stores/settings.js +29 -3
  89. package/build/stores/settings.js.map +1 -1
  90. package/package.json +12 -8
  91. package/src/engine.ts +3 -0
  92. package/src/facades/ai.ts +43 -0
  93. package/src/facades/glasses.ts +60 -31
  94. package/src/facades/glassesSettings.ts +1 -0
  95. package/src/facades/pairing.ts +18 -14
  96. package/src/generated/releaseMetadata.ts +4 -4
  97. package/src/index.ts +16 -2
  98. package/src/services/DeviceEventRouter.ts +8 -1
  99. package/src/services/GestureDotsService.ts +52 -15
  100. package/src/services/GlassesTouchInput.ts +11 -9
  101. package/src/services/LlmRouting.ts +49 -0
  102. package/src/services/LocalLlmModelManager.ts +327 -0
  103. package/src/services/LocalLlmPrompt.ts +246 -0
  104. package/src/services/LocalLlmProvider.ts +290 -0
  105. package/src/services/LocalMiniappRuntime.ts +137 -21
  106. package/src/services/STTModelManager.ts +140 -8
  107. package/src/services/controllers.ts +77 -0
  108. package/src/services/effectiveCapabilities.ts +16 -32
  109. package/src/services/neuralBand/NeuralBandInput.ts +50 -26
  110. package/src/services/neuralBand/NeuralBandSettingsSync.ts +28 -26
  111. package/src/services/settingsBridge.ts +60 -22
  112. package/src/stores/bluetoothSettingKeys.ts +3 -1
  113. package/src/stores/core.ts +0 -1
  114. package/src/stores/glasses.ts +2 -6
  115. package/src/stores/settings.ts +29 -3
@@ -2,6 +2,8 @@ import BluetoothSdk from "@glassly/bluetooth-sdk"
2
2
  import {Platform} from "react-native"
3
3
  import * as RNFS from "@dr.pogodin/react-native-fs"
4
4
 
5
+ export type SttModelType = "transducer" | "ctc" | "nemotron" | "apple"
6
+
5
7
  export interface LanguageInfo {
6
8
  code: string
7
9
  displayName: string
@@ -9,7 +11,7 @@ export interface LanguageInfo {
9
11
  language: string
10
12
  downloaded: boolean
11
13
  path?: string
12
- type: "transducer" | "ctc" | "nemotron"
14
+ type: SttModelType
13
15
  }
14
16
 
15
17
  export interface DownloadProgress {
@@ -30,19 +32,39 @@ export interface LanguageConfig {
30
32
  fileName: string
31
33
  downloadUrl?: string
32
34
  size: number
33
- type: "transducer" | "ctc" | "nemotron"
35
+ type: SttModelType
34
36
  requiredFiles: string[]
35
37
  languageCode: string
36
38
  }
37
39
 
38
40
  const DEFAULT_LANGUAGE = "en"
39
41
 
42
+ /**
43
+ * Apple SpeechTranscriber entries are keyed `apple:<bcp47>`; the same string is
44
+ * what native stores as the model path, since the OS owns the assets and there
45
+ * is no directory to point at.
46
+ */
47
+ export const APPLE_STT_PREFIX = "apple:"
48
+
49
+ export function isAppleSttCode(code: string): boolean {
50
+ return code.startsWith(APPLE_STT_PREFIX)
51
+ }
52
+
53
+ /** SpeechTranscriber ships with iOS 26. */
54
+ export function appleSttSupported(): boolean {
55
+ return Platform.OS === "ios" && parseInt(String(Platform.Version), 10) >= 26
56
+ }
57
+
40
58
  class STTModelManager {
41
59
  private static instance: STTModelManager
42
60
  private downloadJobId?: number
43
61
  private currentLanguage = DEFAULT_LANGUAGE
44
62
  private modelReadyListeners = new Set<() => void>()
45
63
  private modelBaseUrl = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/"
64
+ /** Apple locales reported by the OS, refreshed on each list load. */
65
+ private appleLanguages: Record<string, LanguageConfig> = {}
66
+ private appleInstalled = new Set<string>()
67
+ private appleRefresh: Promise<void> | null = null
46
68
 
47
69
  private languages: Record<string, LanguageConfig> = {
48
70
  "en": {
@@ -188,7 +210,10 @@ class STTModelManager {
188
210
  try {
189
211
  const path = await BluetoothSdk.getSttModelPath()
190
212
  const code = path && path.length > 0 ? this.getLanguageFromPath(path) : ""
191
- if (code && this.languages[code]) {
213
+ if (code && isAppleSttCode(code)) {
214
+ await this.refreshAppleLanguages()
215
+ }
216
+ if (code && this.config(code)) {
192
217
  this.currentLanguage = code
193
218
  return code
194
219
  }
@@ -204,18 +229,59 @@ class STTModelManager {
204
229
  }
205
230
 
206
231
  setCurrentLanguage(code: string): void {
207
- if (this.languages[code]) {
232
+ if (this.config(code)) {
208
233
  this.currentLanguage = code
209
234
  }
210
235
  }
211
236
 
212
237
  getBcp47ForLanguage(code?: string): string {
213
238
  const id = code || this.currentLanguage
214
- return this.languages[id]?.languageCode ?? "en-US"
239
+ return this.config(id)?.languageCode ?? "en-US"
215
240
  }
216
241
 
242
+ /** Sherpa models first, then whatever Apple locales the last refresh reported. */
217
243
  getAvailableLanguages(): LanguageConfig[] {
218
- return Object.values(this.languages)
244
+ return [...Object.values(this.languages), ...Object.values(this.appleLanguages)]
245
+ }
246
+
247
+ private config(code: string): LanguageConfig | undefined {
248
+ return this.languages[code] ?? this.appleLanguages[code]
249
+ }
250
+
251
+ /**
252
+ * Re-read the Apple locale list and install states from the OS. A no-op below
253
+ * iOS 26 and on Android, where the Apple section simply never appears.
254
+ */
255
+ async refreshAppleLanguages(): Promise<void> {
256
+ if (!appleSttSupported()) return
257
+ if (this.appleRefresh) return this.appleRefresh
258
+ this.appleRefresh = (async () => {
259
+ try {
260
+ const rows = await BluetoothSdk.getAppleSpeechLocales()
261
+ const next: Record<string, LanguageConfig> = {}
262
+ const installed = new Set<string>()
263
+ for (const row of rows) {
264
+ const code = APPLE_STT_PREFIX + row.code
265
+ next[code] = {
266
+ code,
267
+ displayName: row.name,
268
+ fileName: "",
269
+ size: 0,
270
+ type: "apple",
271
+ requiredFiles: [],
272
+ languageCode: row.code,
273
+ }
274
+ if (row.status === "installed") installed.add(code)
275
+ }
276
+ this.appleLanguages = next
277
+ this.appleInstalled = installed
278
+ } catch (error) {
279
+ console.warn("STTModelManager: Apple locale refresh failed:", error)
280
+ } finally {
281
+ this.appleRefresh = null
282
+ }
283
+ })()
284
+ return this.appleRefresh
219
285
  }
220
286
 
221
287
  getModelDirectory(): string {
@@ -225,6 +291,7 @@ class STTModelManager {
225
291
 
226
292
  getModelPath(code?: string): string {
227
293
  const id = code || this.currentLanguage
294
+ if (isAppleSttCode(id)) return id
228
295
  return `${this.getModelDirectory()}/${id}`
229
296
  }
230
297
 
@@ -236,6 +303,14 @@ class STTModelManager {
236
303
  async isModelAvailable(code?: string): Promise<boolean> {
237
304
  try {
238
305
  const id = code || this.currentLanguage
306
+ if (isAppleSttCode(id)) {
307
+ if (!appleSttSupported()) return false
308
+ const status = await BluetoothSdk.getAppleSpeechLocaleStatus(id.slice(APPLE_STT_PREFIX.length))
309
+ const installed = status === "installed"
310
+ if (installed) this.appleInstalled.add(id)
311
+ else this.appleInstalled.delete(id)
312
+ return installed
313
+ }
239
314
  const language = this.languages[id]
240
315
  if (!language) return false
241
316
 
@@ -258,7 +333,7 @@ class STTModelManager {
258
333
 
259
334
  async getLanguageInfo(code?: string): Promise<LanguageInfo> {
260
335
  const id = code || this.currentLanguage
261
- const language = this.languages[id]
336
+ const language = this.config(id)
262
337
  if (!language) {
263
338
  throw new Error(`Language ${id} not found`)
264
339
  }
@@ -282,15 +357,61 @@ class STTModelManager {
282
357
  for (const code of Object.keys(this.languages)) {
283
358
  infos.push(await this.getLanguageInfo(code))
284
359
  }
360
+ // One OS round-trip covers every Apple row's install state.
361
+ await this.refreshAppleLanguages()
362
+ for (const language of Object.values(this.appleLanguages)) {
363
+ const downloaded = this.appleInstalled.has(language.code)
364
+ infos.push({
365
+ code: language.code,
366
+ displayName: language.displayName,
367
+ size: 0,
368
+ language: language.languageCode,
369
+ downloaded,
370
+ path: downloaded ? language.code : undefined,
371
+ type: "apple",
372
+ })
373
+ }
285
374
  return infos
286
375
  }
287
376
 
377
+ /** Downloads the Apple locale's assets through the OS, surfacing its progress as a download. */
378
+ private async installAppleModel(
379
+ code: string,
380
+ onProgress?: (progress: DownloadProgress) => void,
381
+ onExtractionProgress?: (progress: ExtractionProgress) => void,
382
+ ): Promise<void> {
383
+ if (!appleSttSupported()) {
384
+ throw new Error("Apple on-device speech requires iOS 26")
385
+ }
386
+ const locale = code.slice(APPLE_STT_PREFIX.length)
387
+ const subscription = BluetoothSdk.addListener("apple_speech_install_progress", (event) => {
388
+ if (event.locale !== locale) return
389
+ onProgress?.({jobId: 0, bytesWritten: 0, contentLength: 0, percentage: event.percentage})
390
+ })
391
+ try {
392
+ const status = await BluetoothSdk.installAppleSpeechLocale(locale)
393
+ if (status !== "installed") {
394
+ throw new Error(`Apple speech assets for ${locale} are ${status}`)
395
+ }
396
+ } finally {
397
+ subscription.remove()
398
+ }
399
+ onExtractionProgress?.({percentage: 100})
400
+ this.appleInstalled.add(code)
401
+ if (code === this.currentLanguage) {
402
+ await this.setNativeModelPath(code, locale)
403
+ }
404
+ }
405
+
288
406
  async downloadModel(
289
407
  code?: string,
290
408
  onProgress?: (progress: DownloadProgress) => void,
291
409
  onExtractionProgress?: (progress: ExtractionProgress) => void,
292
410
  ): Promise<void> {
293
411
  const id = code || this.currentLanguage
412
+ if (isAppleSttCode(id)) {
413
+ return this.installAppleModel(id, onProgress, onExtractionProgress)
414
+ }
294
415
  const language = this.languages[id]
295
416
  if (!language) {
296
417
  throw new Error(`Language ${id} not found`)
@@ -383,6 +504,14 @@ class STTModelManager {
383
504
 
384
505
  async deleteModel(code?: string): Promise<void> {
385
506
  const id = code || this.currentLanguage
507
+ if (isAppleSttCode(id)) {
508
+ // The OS owns the assets; only the app's reservation can be dropped.
509
+ this.appleInstalled.delete(id)
510
+ if (appleSttSupported()) {
511
+ await BluetoothSdk.releaseAppleSpeechLocale(id.slice(APPLE_STT_PREFIX.length))
512
+ }
513
+ return
514
+ }
386
515
  const modelPath = this.getModelPath(id)
387
516
  if (await RNFS.exists(modelPath)) {
388
517
  await RNFS.unlink(modelPath)
@@ -390,7 +519,10 @@ class STTModelManager {
390
519
  }
391
520
 
392
521
  async activateLanguage(code: string): Promise<void> {
393
- const language = this.languages[code]
522
+ if (isAppleSttCode(code) && !this.appleLanguages[code]) {
523
+ await this.refreshAppleLanguages()
524
+ }
525
+ const language = this.config(code)
394
526
  if (!language) {
395
527
  throw new Error(`Language ${code} not found`)
396
528
  }
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Controller registry read-model: the paired records (the native-authoritative
3
+ * `controllers` setting) and the live driver statuses (glasses status
4
+ * `controllers`), plus what each model adds to the input surfaces. Pure.
5
+ */
6
+ import type {ControllerStatus, PairedController} from "@glassly/bluetooth-sdk"
7
+
8
+ import type {ButtonCapabilities} from "../types"
9
+ import {ControllerTypes} from "../types/enums"
10
+
11
+ export type InputSurface = NonNullable<ButtonCapabilities["buttons"]>[number]
12
+
13
+ /** The Neural Band's thumb swipes, pinches and dial. */
14
+ const NEURAL_BAND_SURFACE: InputSurface = {
15
+ type: "swipe2d",
16
+ events: [
17
+ "TAP",
18
+ "DOUBLE_TAP",
19
+ "PRESS_HOLD",
20
+ "SWIPE_UP",
21
+ "SWIPE_DOWN",
22
+ "SWIPE_LEFT",
23
+ "SWIPE_RIGHT",
24
+ "DIAL_UP",
25
+ "DIAL_DOWN",
26
+ ],
27
+ }
28
+
29
+ // The R1 swipes up/down like the temples it stands in for, so it adds nothing.
30
+ const INPUT_SURFACES: Readonly<Record<string, readonly InputSurface[]>> = {
31
+ [ControllerTypes.NEURAL_BAND]: [NEURAL_BAND_SURFACE],
32
+ [ControllerTypes.R1]: [],
33
+ }
34
+
35
+ /** The input surfaces a connected controller of `model` adds to the glasses'. */
36
+ export function controllerInputSurfaces(model: string): InputSurface[] {
37
+ return (INPUT_SURFACES[model] ?? []).map((surface) => ({...surface, events: [...surface.events]}))
38
+ }
39
+
40
+ /** The model swipes in two dimensions. */
41
+ export function hasInput2d(model: string): boolean {
42
+ return (INPUT_SURFACES[model] ?? []).some((surface) => surface.type === "swipe2d")
43
+ }
44
+
45
+ export function isControllerModel(model: string | undefined): boolean {
46
+ return model !== undefined && (Object.values(ControllerTypes) as string[]).includes(model)
47
+ }
48
+
49
+ /** The `controllers` setting as records; malformed entries are dropped. */
50
+ export function pairedControllersFrom(value: unknown): PairedController[] {
51
+ if (!Array.isArray(value)) return []
52
+ const records: PairedController[] = []
53
+ for (const entry of value) {
54
+ if (!entry || typeof entry !== "object") continue
55
+ const {id, model, name} = entry as Record<string, unknown>
56
+ if (typeof id !== "string" || !id || typeof model !== "string") continue
57
+ records.push({id, model, name: typeof name === "string" ? name : id})
58
+ }
59
+ return records
60
+ }
61
+
62
+ /** Delivering input: link up and the route or claim done. */
63
+ export function isControllerReady(status: ControllerStatus | undefined): boolean {
64
+ return status?.connected === true && status.fullyBooted === true
65
+ }
66
+
67
+ export function readyControllers(statuses: Record<string, ControllerStatus> | undefined): ControllerStatus[] {
68
+ return Object.values(statuses ?? {}).filter(isControllerReady)
69
+ }
70
+
71
+ /** Change key over the controllers delivering input. */
72
+ export function readyControllersKey(statuses: Record<string, ControllerStatus> | undefined): string {
73
+ return readyControllers(statuses)
74
+ .map((status) => `${status.id}=${status.model}`)
75
+ .sort()
76
+ .join(",")
77
+ }
@@ -12,21 +12,15 @@
12
12
  * that draws vector shapes keeps 128 scene slots, so shape-heavy frames
13
13
  * (a game board, a chart) get a larger shared text/shape budget.
14
14
  *
15
- * A connected controller that swipes in two dimensions (the Meta Neural Band)
16
- * adds its surface to `button.buttons` as a `swipe2d` entry, so a miniapp can
15
+ * Every connected controller adds its model's input surfaces to
16
+ * `button.buttons` (the Meta Neural Band a `swipe2d` entry), so a miniapp can
17
17
  * tell four-way swipes from the temples' and the R1 ring's up/down.
18
18
  */
19
19
  import {useGlassesStore} from "../stores/glasses"
20
20
  import {SETTINGS, useSettingsStore} from "../stores/settings"
21
- import {
22
- ALL_DISPLAY_SHAPES,
23
- DeviceTypes,
24
- getModelCapabilities,
25
- type ButtonCapabilities,
26
- type Capabilities,
27
- } from "../types"
21
+ import {ALL_DISPLAY_SHAPES, DeviceTypes, getModelCapabilities, type Capabilities} from "../types"
22
+ import {controllerInputSurfaces, readyControllers, type InputSurface} from "./controllers"
28
23
  import {resolveNativeNavigation} from "./nativeNavigationGate"
29
- import {input2dFrom} from "./settingsBridge"
30
24
 
31
25
  const CFW_MAX_TEXT_ELEMENTS = 24
32
26
  const CFW_MAX_IMAGE_ELEMENTS = 16
@@ -90,35 +84,25 @@ function glassesCapabilities(model: DeviceTypes): Capabilities {
90
84
  }
91
85
  }
92
86
 
93
- type InputSurface = NonNullable<ButtonCapabilities["buttons"]>[number]
94
-
95
- /** The Neural Band's thumb swipes, pinches and dial. */
96
- const NEURAL_BAND_SURFACE: InputSurface = {
97
- type: "swipe2d",
98
- events: [
99
- "TAP",
100
- "DOUBLE_TAP",
101
- "PRESS_HOLD",
102
- "SWIPE_UP",
103
- "SWIPE_DOWN",
104
- "SWIPE_LEFT",
105
- "SWIPE_RIGHT",
106
- "DIAL_UP",
107
- "DIAL_DOWN",
108
- ],
109
- }
110
-
111
87
  function withControllerInput(caps: Capabilities): Capabilities {
112
- const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key)
113
- if (!input2dFrom(useGlassesStore.getState(), controller)) return caps
114
88
  const buttons = caps.button?.buttons ?? []
89
+ const types = new Set(buttons.map((surface) => surface.type))
90
+ const added: InputSurface[] = []
91
+ for (const status of readyControllers(useGlassesStore.getState().controllers)) {
92
+ for (const surface of controllerInputSurfaces(status.model)) {
93
+ if (types.has(surface.type)) continue
94
+ types.add(surface.type)
95
+ added.push(surface)
96
+ }
97
+ }
98
+ if (added.length === 0) return caps
115
99
  return {
116
100
  ...caps,
117
101
  hasButton: true,
118
102
  button: {
119
103
  ...caps.button,
120
- count: (caps.button?.count ?? buttons.length) + 1,
121
- buttons: [...buttons, NEURAL_BAND_SURFACE],
104
+ count: (caps.button?.count ?? buttons.length) + added.length,
105
+ buttons: [...buttons, ...added],
122
106
  },
123
107
  }
124
108
  }
@@ -2,7 +2,9 @@
2
2
  * The Meta Neural Band's own event streams, past the gestures that ride `touch_event`:
3
3
  * claim progress (for the pairing screens), the handwriting model (typed into the
4
4
  * miniapp keyboard stream as synthesized key presses) and the air mouse (forwarded
5
- * to miniapps on the pointer stream).
5
+ * to miniapps on the pointer stream). State is per band (`controllerId`); the
6
+ * latest claim is also kept on its own, since the pairing screen runs before
7
+ * the band's id reaches JS.
6
8
  */
7
9
  import type {NeuralBandClaimEvent, NeuralBandHandwritingEvent, NeuralBandPointerEvent} from "@glassly/bluetooth-sdk"
8
10
  import {MiniappStreamType} from "@glassly/miniapp"
@@ -15,6 +17,8 @@ export interface NeuralBandClaimStatus {
15
17
  message?: string
16
18
  sessionExpired: boolean
17
19
  bandName?: string
20
+ /** The band being claimed; absent until a claim event arrives. */
21
+ controllerId?: string
18
22
  }
19
23
 
20
24
  export interface NeuralBandHandwritingStatus {
@@ -25,6 +29,11 @@ export interface NeuralBandHandwritingStatus {
25
29
  text: string
26
30
  }
27
31
 
32
+ export interface NeuralBandHandwritingUpdate {
33
+ id: string
34
+ status: NeuralBandHandwritingStatus
35
+ }
36
+
28
37
  export interface NeuralBandInputOptions {
29
38
  forward?: (stream: string, payload: unknown) => void
30
39
  now?: () => number
@@ -56,11 +65,15 @@ function codeFor(char: string): string {
56
65
  return "Unidentified"
57
66
  }
58
67
 
68
+ const IDLE_CLAIM: NeuralBandClaimStatus = {stage: "idle", sessionExpired: false}
69
+ const IDLE_HANDWRITING: NeuralBandHandwritingStatus = {phase: "idle", message: "", text: ""}
70
+
59
71
  export class NeuralBandInput {
60
- private claim: NeuralBandClaimStatus = {stage: "idle", sessionExpired: false}
61
- private handwriting: NeuralBandHandwritingStatus = {phase: "idle", message: "", text: ""}
72
+ private latestClaim: NeuralBandClaimStatus = IDLE_CLAIM
73
+ private claims = new Map<string, NeuralBandClaimStatus>()
74
+ private handwriting = new Map<string, NeuralBandHandwritingStatus>()
62
75
  private claimListeners = new Set<(status: NeuralBandClaimStatus) => void>()
63
- private handwritingListeners = new Set<(status: NeuralBandHandwritingStatus) => void>()
76
+ private handwritingListeners = new Set<(update: NeuralBandHandwritingUpdate) => void>()
64
77
  private readonly forward: (stream: string, payload: unknown) => void
65
78
  private readonly now: () => number
66
79
 
@@ -69,8 +82,10 @@ export class NeuralBandInput {
69
82
  this.now = options.now ?? (() => Date.now())
70
83
  }
71
84
 
72
- claimStatus(): NeuralBandClaimStatus {
73
- return this.claim
85
+ /** One band's claim, or the latest claim of any band without an id. */
86
+ claimStatus(controllerId?: string): NeuralBandClaimStatus {
87
+ if (controllerId === undefined) return this.latestClaim
88
+ return this.claims.get(controllerId) ?? IDLE_CLAIM
74
89
  }
75
90
 
76
91
  onClaimStatus(cb: (status: NeuralBandClaimStatus) => void): () => void {
@@ -78,42 +93,54 @@ export class NeuralBandInput {
78
93
  return () => this.claimListeners.delete(cb)
79
94
  }
80
95
 
81
- handwritingStatus(): NeuralBandHandwritingStatus {
82
- return this.handwriting
96
+ handwritingStatus(controllerId: string): NeuralBandHandwritingStatus {
97
+ return this.handwriting.get(controllerId) ?? IDLE_HANDWRITING
83
98
  }
84
99
 
85
- onHandwritingStatus(cb: (status: NeuralBandHandwritingStatus) => void): () => void {
100
+ onHandwritingStatus(cb: (update: NeuralBandHandwritingUpdate) => void): () => void {
86
101
  this.handwritingListeners.add(cb)
87
102
  return () => this.handwritingListeners.delete(cb)
88
103
  }
89
104
 
90
105
  /** A new pairing attempt starts from a clean slate. */
91
106
  resetClaim(): void {
92
- this.setClaim({stage: "idle", sessionExpired: false})
107
+ this.latestClaim = IDLE_CLAIM
108
+ for (const cb of this.claimListeners) cb(IDLE_CLAIM)
109
+ }
110
+
111
+ /** Drop an unpaired band's state. */
112
+ forget(controllerId: string): void {
113
+ this.claims.delete(controllerId)
114
+ this.handwriting.delete(controllerId)
93
115
  }
94
116
 
95
117
  handleClaim(event: NeuralBandClaimEvent): void {
96
- this.setClaim({
118
+ const status: NeuralBandClaimStatus = {
97
119
  stage: event.stage,
98
120
  message: event.message,
99
121
  sessionExpired: event.sessionExpired === true,
100
122
  bandName: event.bandName,
101
- })
123
+ controllerId: event.controllerId,
124
+ }
125
+ if (event.controllerId) this.claims.set(event.controllerId, status)
126
+ this.latestClaim = status
127
+ for (const cb of this.claimListeners) cb(status)
102
128
  }
103
129
 
104
130
  handleHandwriting(event: NeuralBandHandwritingEvent): void {
131
+ const id = event.controllerId
132
+ const current = this.handwritingStatus(id)
105
133
  if (event.kind === "state") {
106
- const text = event.phase === "preparing" ? "" : this.handwriting.text
107
- this.setHandwriting({phase: event.phase, message: event.message, problem: event.problem, text})
134
+ const text = event.phase === "preparing" ? "" : current.text
135
+ this.setHandwriting(id, {phase: event.phase, message: event.message, problem: event.problem, text})
108
136
  return
109
137
  }
110
- const previous = this.handwriting.text
111
- if (event.text !== previous) {
112
- for (const key of keyEventsForTextChange(previous, event.text, this.now())) {
138
+ if (event.text !== current.text) {
139
+ for (const key of keyEventsForTextChange(current.text, event.text, this.now())) {
113
140
  this.forward(MiniappStreamType.KEYBOARD_EVENT, key)
114
141
  }
115
142
  }
116
- this.setHandwriting({...this.handwriting, text: event.text})
143
+ this.setHandwriting(id, {...current, text: event.text})
117
144
  }
118
145
 
119
146
  handlePointer(event: NeuralBandPointerEvent): void {
@@ -123,24 +150,21 @@ export class NeuralBandInput {
123
150
  dx: event.dx,
124
151
  dy: event.dy,
125
152
  timestamp: event.timestamp,
153
+ controllerId: event.controllerId,
126
154
  })
127
155
  } else {
128
156
  this.forward(MiniappStreamType.POINTER_EVENT, {
129
157
  kind: "buttons",
130
158
  buttons: event.buttons,
131
159
  timestamp: event.timestamp,
160
+ controllerId: event.controllerId,
132
161
  })
133
162
  }
134
163
  }
135
164
 
136
- private setClaim(status: NeuralBandClaimStatus): void {
137
- this.claim = status
138
- for (const cb of this.claimListeners) cb(status)
139
- }
140
-
141
- private setHandwriting(status: NeuralBandHandwritingStatus): void {
142
- this.handwriting = status
143
- for (const cb of this.handwritingListeners) cb(status)
165
+ private setHandwriting(id: string, status: NeuralBandHandwritingStatus): void {
166
+ this.handwriting.set(id, status)
167
+ for (const cb of this.handwritingListeners) cb({id, status})
144
168
  }
145
169
  }
146
170
 
@@ -1,59 +1,61 @@
1
1
  /**
2
- * Pushes the Neural Band's user settings (air mouse, wearing hand) to the native
3
- * controller: when the band connects, and whenever either setting changes while
4
- * it is connected. The native side keeps nothing across connections.
2
+ * Pushes the Neural Band user settings (air mouse, wearing hand) to every
3
+ * connected band: to each band as it connects, and to all of them whenever
4
+ * either setting changes. The native side keeps nothing across connections.
5
5
  */
6
6
  import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
7
7
 
8
8
  import {useGlassesStore} from "../../stores/glasses"
9
9
  import {SETTINGS, useSettingsStore} from "../../stores/settings"
10
10
  import {ControllerTypes} from "../../types/enums"
11
+ import {readyControllers} from "../controllers"
11
12
 
12
13
  let unsubscribers: (() => void)[] = []
13
14
 
14
- function bandConnected(): boolean {
15
- const glasses = useGlassesStore.getState()
16
- const settings = useSettingsStore.getState()
17
- return (
18
- glasses.controllerConnected === true &&
19
- glasses.controllerFullyBooted === true &&
20
- settings.getSetting(SETTINGS.default_controller.key) === ControllerTypes.NEURAL_BAND
21
- )
15
+ function connectedBandIds(): string[] {
16
+ return readyControllers(useGlassesStore.getState().controllers)
17
+ .filter((status) => status.model === ControllerTypes.NEURAL_BAND)
18
+ .map((status) => status.id)
22
19
  }
23
20
 
24
- function pushAirMouse(): void {
25
- if (!bandConnected()) return
21
+ function pushAirMouse(id: string): void {
26
22
  const enabled = useSettingsStore.getState().getSetting(SETTINGS.neural_band_air_mouse.key) === true
27
- void BluetoothSdk.neuralBandSetPointer(enabled).catch((err) =>
23
+ void BluetoothSdk.neuralBandSetPointer(id, enabled).catch((err) =>
28
24
  console.warn("NeuralBandSettingsSync: air mouse push failed:", err),
29
25
  )
30
26
  }
31
27
 
32
- function pushHand(): void {
33
- if (!bandConnected()) return
28
+ function pushHand(id: string): void {
34
29
  const hand = useSettingsStore.getState().getSetting(SETTINGS.neural_band_hand.key) === "left" ? "left" : "right"
35
- void BluetoothSdk.neuralBandSetHand(hand).catch((err) =>
30
+ void BluetoothSdk.neuralBandSetHand(id, hand).catch((err) =>
36
31
  console.warn("NeuralBandSettingsSync: hand push failed:", err),
37
32
  )
38
33
  }
39
34
 
40
35
  export function startNeuralBandSettingsSync(): void {
41
36
  if (unsubscribers.length > 0) return
42
- let wasConnected = bandConnected()
37
+ let connected = new Set(connectedBandIds())
43
38
  unsubscribers = [
44
39
  useGlassesStore.subscribe(
45
- (s) => s.controllerConnected === true && s.controllerFullyBooted === true,
40
+ () => connectedBandIds().sort().join(","),
46
41
  () => {
47
- const connected = bandConnected()
48
- if (connected && !wasConnected) {
49
- pushAirMouse()
50
- pushHand()
42
+ const next = new Set(connectedBandIds())
43
+ for (const id of next) {
44
+ if (connected.has(id)) continue
45
+ pushAirMouse(id)
46
+ pushHand(id)
51
47
  }
52
- wasConnected = connected
48
+ connected = next
53
49
  },
54
50
  ),
55
- useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_air_mouse.key), pushAirMouse),
56
- useSettingsStore.subscribe((s) => s.getSetting(SETTINGS.neural_band_hand.key), pushHand),
51
+ useSettingsStore.subscribe(
52
+ (s) => s.getSetting(SETTINGS.neural_band_air_mouse.key),
53
+ () => connectedBandIds().forEach(pushAirMouse),
54
+ ),
55
+ useSettingsStore.subscribe(
56
+ (s) => s.getSetting(SETTINGS.neural_band_hand.key),
57
+ () => connectedBandIds().forEach(pushHand),
58
+ ),
57
59
  ]
58
60
  }
59
61