@glassly/engine 0.3.3-dev.670 → 0.3.4-dev.675
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/engine.d.ts +38 -14
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +3 -0
- package/build/engine.js.map +1 -1
- package/build/facades/ai.d.ts +32 -0
- package/build/facades/ai.d.ts.map +1 -0
- package/build/facades/ai.js +38 -0
- package/build/facades/ai.js.map +1 -0
- package/build/facades/glasses.d.ts +30 -24
- package/build/facades/glasses.d.ts.map +1 -1
- package/build/facades/glasses.js +49 -30
- package/build/facades/glasses.js.map +1 -1
- package/build/facades/glassesSettings.d.ts.map +1 -1
- package/build/facades/glassesSettings.js +1 -0
- package/build/facades/glassesSettings.js.map +1 -1
- package/build/facades/pairing.d.ts +6 -5
- package/build/facades/pairing.d.ts.map +1 -1
- package/build/facades/pairing.js +14 -14
- package/build/facades/pairing.js.map +1 -1
- package/build/generated/releaseMetadata.js +4 -4
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/index.d.ts +5 -2
- package/build/index.d.ts.map +1 -1
- package/build/index.js +2 -0
- package/build/index.js.map +1 -1
- package/build/services/DeviceEventRouter.d.ts.map +1 -1
- package/build/services/DeviceEventRouter.js +9 -1
- package/build/services/DeviceEventRouter.js.map +1 -1
- package/build/services/GestureDotsService.d.ts +4 -3
- package/build/services/GestureDotsService.d.ts.map +1 -1
- package/build/services/GestureDotsService.js +17 -12
- package/build/services/GestureDotsService.js.map +1 -1
- package/build/services/GlassesTouchInput.d.ts.map +1 -1
- package/build/services/GlassesTouchInput.js +11 -9
- package/build/services/GlassesTouchInput.js.map +1 -1
- package/build/services/LlmRouting.d.ts +35 -0
- package/build/services/LlmRouting.d.ts.map +1 -0
- package/build/services/LlmRouting.js +25 -0
- package/build/services/LlmRouting.js.map +1 -0
- package/build/services/LocalLlmModelManager.d.ts +79 -0
- package/build/services/LocalLlmModelManager.d.ts.map +1 -0
- package/build/services/LocalLlmModelManager.js +278 -0
- package/build/services/LocalLlmModelManager.js.map +1 -0
- package/build/services/LocalLlmPrompt.d.ts +90 -0
- package/build/services/LocalLlmPrompt.d.ts.map +1 -0
- package/build/services/LocalLlmPrompt.js +189 -0
- package/build/services/LocalLlmPrompt.js.map +1 -0
- package/build/services/LocalLlmProvider.d.ts +35 -0
- package/build/services/LocalLlmProvider.d.ts.map +1 -0
- package/build/services/LocalLlmProvider.js +218 -0
- package/build/services/LocalLlmProvider.js.map +1 -0
- package/build/services/LocalMiniappRuntime.d.ts +9 -1
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +91 -17
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/STTModelManager.d.ts +25 -2
- package/build/services/STTModelManager.d.ts.map +1 -1
- package/build/services/STTModelManager.js +137 -6
- package/build/services/STTModelManager.js.map +1 -1
- package/build/services/controllers.d.ts +21 -0
- package/build/services/controllers.d.ts.map +1 -0
- package/build/services/controllers.js +62 -0
- package/build/services/controllers.js.map +1 -0
- package/build/services/effectiveCapabilities.d.ts.map +1 -1
- package/build/services/effectiveCapabilities.js +18 -24
- package/build/services/effectiveCapabilities.js.map +1 -1
- package/build/services/neuralBand/NeuralBandInput.d.ts +17 -6
- package/build/services/neuralBand/NeuralBandInput.d.ts.map +1 -1
- package/build/services/neuralBand/NeuralBandInput.js +40 -23
- package/build/services/neuralBand/NeuralBandInput.js.map +1 -1
- package/build/services/neuralBand/NeuralBandSettingsSync.d.ts.map +1 -1
- package/build/services/neuralBand/NeuralBandSettingsSync.js +23 -26
- package/build/services/neuralBand/NeuralBandSettingsSync.js.map +1 -1
- package/build/services/settingsBridge.d.ts +26 -18
- package/build/services/settingsBridge.d.ts.map +1 -1
- package/build/services/settingsBridge.js +36 -15
- package/build/services/settingsBridge.js.map +1 -1
- package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
- package/build/stores/bluetoothSettingKeys.js +3 -1
- package/build/stores/bluetoothSettingKeys.js.map +1 -1
- package/build/stores/core.d.ts.map +1 -1
- package/build/stores/core.js +0 -1
- package/build/stores/core.js.map +1 -1
- package/build/stores/glasses.d.ts.map +1 -1
- package/build/stores/glasses.js +2 -6
- package/build/stores/glasses.js.map +1 -1
- package/build/stores/settings.d.ts.map +1 -1
- package/build/stores/settings.js +28 -2
- package/build/stores/settings.js.map +1 -1
- package/package.json +12 -8
- package/src/engine.ts +3 -0
- package/src/facades/ai.ts +43 -0
- package/src/facades/glasses.ts +60 -31
- package/src/facades/glassesSettings.ts +1 -0
- package/src/facades/pairing.ts +18 -14
- package/src/generated/releaseMetadata.ts +4 -4
- package/src/index.ts +16 -2
- package/src/services/DeviceEventRouter.ts +8 -1
- package/src/services/GestureDotsService.ts +20 -14
- package/src/services/GlassesTouchInput.ts +11 -9
- package/src/services/LlmRouting.ts +49 -0
- package/src/services/LocalLlmModelManager.ts +327 -0
- package/src/services/LocalLlmPrompt.ts +246 -0
- package/src/services/LocalLlmProvider.ts +290 -0
- package/src/services/LocalMiniappRuntime.ts +137 -21
- package/src/services/STTModelManager.ts +140 -8
- package/src/services/controllers.ts +77 -0
- package/src/services/effectiveCapabilities.ts +16 -32
- package/src/services/neuralBand/NeuralBandInput.ts +50 -26
- package/src/services/neuralBand/NeuralBandSettingsSync.ts +28 -26
- package/src/services/settingsBridge.ts +60 -22
- package/src/stores/bluetoothSettingKeys.ts +3 -1
- package/src/stores/core.ts +0 -1
- package/src/stores/glasses.ts +2 -6
- package/src/stores/settings.ts +28 -2
|
@@ -2,6 +2,8 @@ import BluetoothSdk from "@glassly/bluetooth-sdk"
|
|
|
2
2
|
import {Platform} from "react-native"
|
|
3
3
|
import * as RNFS from "@dr.pogodin/react-native-fs"
|
|
4
4
|
|
|
5
|
+
export type SttModelType = "transducer" | "ctc" | "nemotron" | "apple"
|
|
6
|
+
|
|
5
7
|
export interface LanguageInfo {
|
|
6
8
|
code: string
|
|
7
9
|
displayName: string
|
|
@@ -9,7 +11,7 @@ export interface LanguageInfo {
|
|
|
9
11
|
language: string
|
|
10
12
|
downloaded: boolean
|
|
11
13
|
path?: string
|
|
12
|
-
type:
|
|
14
|
+
type: SttModelType
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
export interface DownloadProgress {
|
|
@@ -30,19 +32,39 @@ export interface LanguageConfig {
|
|
|
30
32
|
fileName: string
|
|
31
33
|
downloadUrl?: string
|
|
32
34
|
size: number
|
|
33
|
-
type:
|
|
35
|
+
type: SttModelType
|
|
34
36
|
requiredFiles: string[]
|
|
35
37
|
languageCode: string
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
const DEFAULT_LANGUAGE = "en"
|
|
39
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Apple SpeechTranscriber entries are keyed `apple:<bcp47>`; the same string is
|
|
44
|
+
* what native stores as the model path, since the OS owns the assets and there
|
|
45
|
+
* is no directory to point at.
|
|
46
|
+
*/
|
|
47
|
+
export const APPLE_STT_PREFIX = "apple:"
|
|
48
|
+
|
|
49
|
+
export function isAppleSttCode(code: string): boolean {
|
|
50
|
+
return code.startsWith(APPLE_STT_PREFIX)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** SpeechTranscriber ships with iOS 26. */
|
|
54
|
+
export function appleSttSupported(): boolean {
|
|
55
|
+
return Platform.OS === "ios" && parseInt(String(Platform.Version), 10) >= 26
|
|
56
|
+
}
|
|
57
|
+
|
|
40
58
|
class STTModelManager {
|
|
41
59
|
private static instance: STTModelManager
|
|
42
60
|
private downloadJobId?: number
|
|
43
61
|
private currentLanguage = DEFAULT_LANGUAGE
|
|
44
62
|
private modelReadyListeners = new Set<() => void>()
|
|
45
63
|
private modelBaseUrl = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/"
|
|
64
|
+
/** Apple locales reported by the OS, refreshed on each list load. */
|
|
65
|
+
private appleLanguages: Record<string, LanguageConfig> = {}
|
|
66
|
+
private appleInstalled = new Set<string>()
|
|
67
|
+
private appleRefresh: Promise<void> | null = null
|
|
46
68
|
|
|
47
69
|
private languages: Record<string, LanguageConfig> = {
|
|
48
70
|
"en": {
|
|
@@ -188,7 +210,10 @@ class STTModelManager {
|
|
|
188
210
|
try {
|
|
189
211
|
const path = await BluetoothSdk.getSttModelPath()
|
|
190
212
|
const code = path && path.length > 0 ? this.getLanguageFromPath(path) : ""
|
|
191
|
-
if (code &&
|
|
213
|
+
if (code && isAppleSttCode(code)) {
|
|
214
|
+
await this.refreshAppleLanguages()
|
|
215
|
+
}
|
|
216
|
+
if (code && this.config(code)) {
|
|
192
217
|
this.currentLanguage = code
|
|
193
218
|
return code
|
|
194
219
|
}
|
|
@@ -204,18 +229,59 @@ class STTModelManager {
|
|
|
204
229
|
}
|
|
205
230
|
|
|
206
231
|
setCurrentLanguage(code: string): void {
|
|
207
|
-
if (this.
|
|
232
|
+
if (this.config(code)) {
|
|
208
233
|
this.currentLanguage = code
|
|
209
234
|
}
|
|
210
235
|
}
|
|
211
236
|
|
|
212
237
|
getBcp47ForLanguage(code?: string): string {
|
|
213
238
|
const id = code || this.currentLanguage
|
|
214
|
-
return this.
|
|
239
|
+
return this.config(id)?.languageCode ?? "en-US"
|
|
215
240
|
}
|
|
216
241
|
|
|
242
|
+
/** Sherpa models first, then whatever Apple locales the last refresh reported. */
|
|
217
243
|
getAvailableLanguages(): LanguageConfig[] {
|
|
218
|
-
return Object.values(this.languages)
|
|
244
|
+
return [...Object.values(this.languages), ...Object.values(this.appleLanguages)]
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
private config(code: string): LanguageConfig | undefined {
|
|
248
|
+
return this.languages[code] ?? this.appleLanguages[code]
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Re-read the Apple locale list and install states from the OS. A no-op below
|
|
253
|
+
* iOS 26 and on Android, where the Apple section simply never appears.
|
|
254
|
+
*/
|
|
255
|
+
async refreshAppleLanguages(): Promise<void> {
|
|
256
|
+
if (!appleSttSupported()) return
|
|
257
|
+
if (this.appleRefresh) return this.appleRefresh
|
|
258
|
+
this.appleRefresh = (async () => {
|
|
259
|
+
try {
|
|
260
|
+
const rows = await BluetoothSdk.getAppleSpeechLocales()
|
|
261
|
+
const next: Record<string, LanguageConfig> = {}
|
|
262
|
+
const installed = new Set<string>()
|
|
263
|
+
for (const row of rows) {
|
|
264
|
+
const code = APPLE_STT_PREFIX + row.code
|
|
265
|
+
next[code] = {
|
|
266
|
+
code,
|
|
267
|
+
displayName: row.name,
|
|
268
|
+
fileName: "",
|
|
269
|
+
size: 0,
|
|
270
|
+
type: "apple",
|
|
271
|
+
requiredFiles: [],
|
|
272
|
+
languageCode: row.code,
|
|
273
|
+
}
|
|
274
|
+
if (row.status === "installed") installed.add(code)
|
|
275
|
+
}
|
|
276
|
+
this.appleLanguages = next
|
|
277
|
+
this.appleInstalled = installed
|
|
278
|
+
} catch (error) {
|
|
279
|
+
console.warn("STTModelManager: Apple locale refresh failed:", error)
|
|
280
|
+
} finally {
|
|
281
|
+
this.appleRefresh = null
|
|
282
|
+
}
|
|
283
|
+
})()
|
|
284
|
+
return this.appleRefresh
|
|
219
285
|
}
|
|
220
286
|
|
|
221
287
|
getModelDirectory(): string {
|
|
@@ -225,6 +291,7 @@ class STTModelManager {
|
|
|
225
291
|
|
|
226
292
|
getModelPath(code?: string): string {
|
|
227
293
|
const id = code || this.currentLanguage
|
|
294
|
+
if (isAppleSttCode(id)) return id
|
|
228
295
|
return `${this.getModelDirectory()}/${id}`
|
|
229
296
|
}
|
|
230
297
|
|
|
@@ -236,6 +303,14 @@ class STTModelManager {
|
|
|
236
303
|
async isModelAvailable(code?: string): Promise<boolean> {
|
|
237
304
|
try {
|
|
238
305
|
const id = code || this.currentLanguage
|
|
306
|
+
if (isAppleSttCode(id)) {
|
|
307
|
+
if (!appleSttSupported()) return false
|
|
308
|
+
const status = await BluetoothSdk.getAppleSpeechLocaleStatus(id.slice(APPLE_STT_PREFIX.length))
|
|
309
|
+
const installed = status === "installed"
|
|
310
|
+
if (installed) this.appleInstalled.add(id)
|
|
311
|
+
else this.appleInstalled.delete(id)
|
|
312
|
+
return installed
|
|
313
|
+
}
|
|
239
314
|
const language = this.languages[id]
|
|
240
315
|
if (!language) return false
|
|
241
316
|
|
|
@@ -258,7 +333,7 @@ class STTModelManager {
|
|
|
258
333
|
|
|
259
334
|
async getLanguageInfo(code?: string): Promise<LanguageInfo> {
|
|
260
335
|
const id = code || this.currentLanguage
|
|
261
|
-
const language = this.
|
|
336
|
+
const language = this.config(id)
|
|
262
337
|
if (!language) {
|
|
263
338
|
throw new Error(`Language ${id} not found`)
|
|
264
339
|
}
|
|
@@ -282,15 +357,61 @@ class STTModelManager {
|
|
|
282
357
|
for (const code of Object.keys(this.languages)) {
|
|
283
358
|
infos.push(await this.getLanguageInfo(code))
|
|
284
359
|
}
|
|
360
|
+
// One OS round-trip covers every Apple row's install state.
|
|
361
|
+
await this.refreshAppleLanguages()
|
|
362
|
+
for (const language of Object.values(this.appleLanguages)) {
|
|
363
|
+
const downloaded = this.appleInstalled.has(language.code)
|
|
364
|
+
infos.push({
|
|
365
|
+
code: language.code,
|
|
366
|
+
displayName: language.displayName,
|
|
367
|
+
size: 0,
|
|
368
|
+
language: language.languageCode,
|
|
369
|
+
downloaded,
|
|
370
|
+
path: downloaded ? language.code : undefined,
|
|
371
|
+
type: "apple",
|
|
372
|
+
})
|
|
373
|
+
}
|
|
285
374
|
return infos
|
|
286
375
|
}
|
|
287
376
|
|
|
377
|
+
/** Downloads the Apple locale's assets through the OS, surfacing its progress as a download. */
|
|
378
|
+
private async installAppleModel(
|
|
379
|
+
code: string,
|
|
380
|
+
onProgress?: (progress: DownloadProgress) => void,
|
|
381
|
+
onExtractionProgress?: (progress: ExtractionProgress) => void,
|
|
382
|
+
): Promise<void> {
|
|
383
|
+
if (!appleSttSupported()) {
|
|
384
|
+
throw new Error("Apple on-device speech requires iOS 26")
|
|
385
|
+
}
|
|
386
|
+
const locale = code.slice(APPLE_STT_PREFIX.length)
|
|
387
|
+
const subscription = BluetoothSdk.addListener("apple_speech_install_progress", (event) => {
|
|
388
|
+
if (event.locale !== locale) return
|
|
389
|
+
onProgress?.({jobId: 0, bytesWritten: 0, contentLength: 0, percentage: event.percentage})
|
|
390
|
+
})
|
|
391
|
+
try {
|
|
392
|
+
const status = await BluetoothSdk.installAppleSpeechLocale(locale)
|
|
393
|
+
if (status !== "installed") {
|
|
394
|
+
throw new Error(`Apple speech assets for ${locale} are ${status}`)
|
|
395
|
+
}
|
|
396
|
+
} finally {
|
|
397
|
+
subscription.remove()
|
|
398
|
+
}
|
|
399
|
+
onExtractionProgress?.({percentage: 100})
|
|
400
|
+
this.appleInstalled.add(code)
|
|
401
|
+
if (code === this.currentLanguage) {
|
|
402
|
+
await this.setNativeModelPath(code, locale)
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
288
406
|
async downloadModel(
|
|
289
407
|
code?: string,
|
|
290
408
|
onProgress?: (progress: DownloadProgress) => void,
|
|
291
409
|
onExtractionProgress?: (progress: ExtractionProgress) => void,
|
|
292
410
|
): Promise<void> {
|
|
293
411
|
const id = code || this.currentLanguage
|
|
412
|
+
if (isAppleSttCode(id)) {
|
|
413
|
+
return this.installAppleModel(id, onProgress, onExtractionProgress)
|
|
414
|
+
}
|
|
294
415
|
const language = this.languages[id]
|
|
295
416
|
if (!language) {
|
|
296
417
|
throw new Error(`Language ${id} not found`)
|
|
@@ -383,6 +504,14 @@ class STTModelManager {
|
|
|
383
504
|
|
|
384
505
|
async deleteModel(code?: string): Promise<void> {
|
|
385
506
|
const id = code || this.currentLanguage
|
|
507
|
+
if (isAppleSttCode(id)) {
|
|
508
|
+
// The OS owns the assets; only the app's reservation can be dropped.
|
|
509
|
+
this.appleInstalled.delete(id)
|
|
510
|
+
if (appleSttSupported()) {
|
|
511
|
+
await BluetoothSdk.releaseAppleSpeechLocale(id.slice(APPLE_STT_PREFIX.length))
|
|
512
|
+
}
|
|
513
|
+
return
|
|
514
|
+
}
|
|
386
515
|
const modelPath = this.getModelPath(id)
|
|
387
516
|
if (await RNFS.exists(modelPath)) {
|
|
388
517
|
await RNFS.unlink(modelPath)
|
|
@@ -390,7 +519,10 @@ class STTModelManager {
|
|
|
390
519
|
}
|
|
391
520
|
|
|
392
521
|
async activateLanguage(code: string): Promise<void> {
|
|
393
|
-
|
|
522
|
+
if (isAppleSttCode(code) && !this.appleLanguages[code]) {
|
|
523
|
+
await this.refreshAppleLanguages()
|
|
524
|
+
}
|
|
525
|
+
const language = this.config(code)
|
|
394
526
|
if (!language) {
|
|
395
527
|
throw new Error(`Language ${code} not found`)
|
|
396
528
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Controller registry read-model: the paired records (the native-authoritative
|
|
3
|
+
* `controllers` setting) and the live driver statuses (glasses status
|
|
4
|
+
* `controllers`), plus what each model adds to the input surfaces. Pure.
|
|
5
|
+
*/
|
|
6
|
+
import type {ControllerStatus, PairedController} from "@glassly/bluetooth-sdk"
|
|
7
|
+
|
|
8
|
+
import type {ButtonCapabilities} from "../types"
|
|
9
|
+
import {ControllerTypes} from "../types/enums"
|
|
10
|
+
|
|
11
|
+
export type InputSurface = NonNullable<ButtonCapabilities["buttons"]>[number]
|
|
12
|
+
|
|
13
|
+
/** The Neural Band's thumb swipes, pinches and dial. */
|
|
14
|
+
const NEURAL_BAND_SURFACE: InputSurface = {
|
|
15
|
+
type: "swipe2d",
|
|
16
|
+
events: [
|
|
17
|
+
"TAP",
|
|
18
|
+
"DOUBLE_TAP",
|
|
19
|
+
"PRESS_HOLD",
|
|
20
|
+
"SWIPE_UP",
|
|
21
|
+
"SWIPE_DOWN",
|
|
22
|
+
"SWIPE_LEFT",
|
|
23
|
+
"SWIPE_RIGHT",
|
|
24
|
+
"DIAL_UP",
|
|
25
|
+
"DIAL_DOWN",
|
|
26
|
+
],
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// The R1 swipes up/down like the temples it stands in for, so it adds nothing.
|
|
30
|
+
const INPUT_SURFACES: Readonly<Record<string, readonly InputSurface[]>> = {
|
|
31
|
+
[ControllerTypes.NEURAL_BAND]: [NEURAL_BAND_SURFACE],
|
|
32
|
+
[ControllerTypes.R1]: [],
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The input surfaces a connected controller of `model` adds to the glasses'. */
|
|
36
|
+
export function controllerInputSurfaces(model: string): InputSurface[] {
|
|
37
|
+
return (INPUT_SURFACES[model] ?? []).map((surface) => ({...surface, events: [...surface.events]}))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The model swipes in two dimensions. */
|
|
41
|
+
export function hasInput2d(model: string): boolean {
|
|
42
|
+
return (INPUT_SURFACES[model] ?? []).some((surface) => surface.type === "swipe2d")
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function isControllerModel(model: string | undefined): boolean {
|
|
46
|
+
return model !== undefined && (Object.values(ControllerTypes) as string[]).includes(model)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The `controllers` setting as records; malformed entries are dropped. */
|
|
50
|
+
export function pairedControllersFrom(value: unknown): PairedController[] {
|
|
51
|
+
if (!Array.isArray(value)) return []
|
|
52
|
+
const records: PairedController[] = []
|
|
53
|
+
for (const entry of value) {
|
|
54
|
+
if (!entry || typeof entry !== "object") continue
|
|
55
|
+
const {id, model, name} = entry as Record<string, unknown>
|
|
56
|
+
if (typeof id !== "string" || !id || typeof model !== "string") continue
|
|
57
|
+
records.push({id, model, name: typeof name === "string" ? name : id})
|
|
58
|
+
}
|
|
59
|
+
return records
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Delivering input: link up and the route or claim done. */
|
|
63
|
+
export function isControllerReady(status: ControllerStatus | undefined): boolean {
|
|
64
|
+
return status?.connected === true && status.fullyBooted === true
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function readyControllers(statuses: Record<string, ControllerStatus> | undefined): ControllerStatus[] {
|
|
68
|
+
return Object.values(statuses ?? {}).filter(isControllerReady)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Change key over the controllers delivering input. */
|
|
72
|
+
export function readyControllersKey(statuses: Record<string, ControllerStatus> | undefined): string {
|
|
73
|
+
return readyControllers(statuses)
|
|
74
|
+
.map((status) => `${status.id}=${status.model}`)
|
|
75
|
+
.sort()
|
|
76
|
+
.join(",")
|
|
77
|
+
}
|
|
@@ -12,21 +12,15 @@
|
|
|
12
12
|
* that draws vector shapes keeps 128 scene slots, so shape-heavy frames
|
|
13
13
|
* (a game board, a chart) get a larger shared text/shape budget.
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
15
|
+
* Every connected controller adds its model's input surfaces to
|
|
16
|
+
* `button.buttons` (the Meta Neural Band a `swipe2d` entry), so a miniapp can
|
|
17
17
|
* tell four-way swipes from the temples' and the R1 ring's up/down.
|
|
18
18
|
*/
|
|
19
19
|
import {useGlassesStore} from "../stores/glasses"
|
|
20
20
|
import {SETTINGS, useSettingsStore} from "../stores/settings"
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
DeviceTypes,
|
|
24
|
-
getModelCapabilities,
|
|
25
|
-
type ButtonCapabilities,
|
|
26
|
-
type Capabilities,
|
|
27
|
-
} from "../types"
|
|
21
|
+
import {ALL_DISPLAY_SHAPES, DeviceTypes, getModelCapabilities, type Capabilities} from "../types"
|
|
22
|
+
import {controllerInputSurfaces, readyControllers, type InputSurface} from "./controllers"
|
|
28
23
|
import {resolveNativeNavigation} from "./nativeNavigationGate"
|
|
29
|
-
import {input2dFrom} from "./settingsBridge"
|
|
30
24
|
|
|
31
25
|
const CFW_MAX_TEXT_ELEMENTS = 24
|
|
32
26
|
const CFW_MAX_IMAGE_ELEMENTS = 16
|
|
@@ -90,35 +84,25 @@ function glassesCapabilities(model: DeviceTypes): Capabilities {
|
|
|
90
84
|
}
|
|
91
85
|
}
|
|
92
86
|
|
|
93
|
-
type InputSurface = NonNullable<ButtonCapabilities["buttons"]>[number]
|
|
94
|
-
|
|
95
|
-
/** The Neural Band's thumb swipes, pinches and dial. */
|
|
96
|
-
const NEURAL_BAND_SURFACE: InputSurface = {
|
|
97
|
-
type: "swipe2d",
|
|
98
|
-
events: [
|
|
99
|
-
"TAP",
|
|
100
|
-
"DOUBLE_TAP",
|
|
101
|
-
"PRESS_HOLD",
|
|
102
|
-
"SWIPE_UP",
|
|
103
|
-
"SWIPE_DOWN",
|
|
104
|
-
"SWIPE_LEFT",
|
|
105
|
-
"SWIPE_RIGHT",
|
|
106
|
-
"DIAL_UP",
|
|
107
|
-
"DIAL_DOWN",
|
|
108
|
-
],
|
|
109
|
-
}
|
|
110
|
-
|
|
111
87
|
function withControllerInput(caps: Capabilities): Capabilities {
|
|
112
|
-
const controller = useSettingsStore.getState().getSetting(SETTINGS.default_controller.key)
|
|
113
|
-
if (!input2dFrom(useGlassesStore.getState(), controller)) return caps
|
|
114
88
|
const buttons = caps.button?.buttons ?? []
|
|
89
|
+
const types = new Set(buttons.map((surface) => surface.type))
|
|
90
|
+
const added: InputSurface[] = []
|
|
91
|
+
for (const status of readyControllers(useGlassesStore.getState().controllers)) {
|
|
92
|
+
for (const surface of controllerInputSurfaces(status.model)) {
|
|
93
|
+
if (types.has(surface.type)) continue
|
|
94
|
+
types.add(surface.type)
|
|
95
|
+
added.push(surface)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (added.length === 0) return caps
|
|
115
99
|
return {
|
|
116
100
|
...caps,
|
|
117
101
|
hasButton: true,
|
|
118
102
|
button: {
|
|
119
103
|
...caps.button,
|
|
120
|
-
count: (caps.button?.count ?? buttons.length) +
|
|
121
|
-
buttons: [...buttons,
|
|
104
|
+
count: (caps.button?.count ?? buttons.length) + added.length,
|
|
105
|
+
buttons: [...buttons, ...added],
|
|
122
106
|
},
|
|
123
107
|
}
|
|
124
108
|
}
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
* The Meta Neural Band's own event streams, past the gestures that ride `touch_event`:
|
|
3
3
|
* claim progress (for the pairing screens), the handwriting model (typed into the
|
|
4
4
|
* miniapp keyboard stream as synthesized key presses) and the air mouse (forwarded
|
|
5
|
-
* to miniapps on the pointer stream).
|
|
5
|
+
* to miniapps on the pointer stream). State is per band (`controllerId`); the
|
|
6
|
+
* latest claim is also kept on its own, since the pairing screen runs before
|
|
7
|
+
* the band's id reaches JS.
|
|
6
8
|
*/
|
|
7
9
|
import type {NeuralBandClaimEvent, NeuralBandHandwritingEvent, NeuralBandPointerEvent} from "@glassly/bluetooth-sdk"
|
|
8
10
|
import {MiniappStreamType} from "@glassly/miniapp"
|
|
@@ -15,6 +17,8 @@ export interface NeuralBandClaimStatus {
|
|
|
15
17
|
message?: string
|
|
16
18
|
sessionExpired: boolean
|
|
17
19
|
bandName?: string
|
|
20
|
+
/** The band being claimed; absent until a claim event arrives. */
|
|
21
|
+
controllerId?: string
|
|
18
22
|
}
|
|
19
23
|
|
|
20
24
|
export interface NeuralBandHandwritingStatus {
|
|
@@ -25,6 +29,11 @@ export interface NeuralBandHandwritingStatus {
|
|
|
25
29
|
text: string
|
|
26
30
|
}
|
|
27
31
|
|
|
32
|
+
export interface NeuralBandHandwritingUpdate {
|
|
33
|
+
id: string
|
|
34
|
+
status: NeuralBandHandwritingStatus
|
|
35
|
+
}
|
|
36
|
+
|
|
28
37
|
export interface NeuralBandInputOptions {
|
|
29
38
|
forward?: (stream: string, payload: unknown) => void
|
|
30
39
|
now?: () => number
|
|
@@ -56,11 +65,15 @@ function codeFor(char: string): string {
|
|
|
56
65
|
return "Unidentified"
|
|
57
66
|
}
|
|
58
67
|
|
|
68
|
+
const IDLE_CLAIM: NeuralBandClaimStatus = {stage: "idle", sessionExpired: false}
|
|
69
|
+
const IDLE_HANDWRITING: NeuralBandHandwritingStatus = {phase: "idle", message: "", text: ""}
|
|
70
|
+
|
|
59
71
|
export class NeuralBandInput {
|
|
60
|
-
private
|
|
61
|
-
private
|
|
72
|
+
private latestClaim: NeuralBandClaimStatus = IDLE_CLAIM
|
|
73
|
+
private claims = new Map<string, NeuralBandClaimStatus>()
|
|
74
|
+
private handwriting = new Map<string, NeuralBandHandwritingStatus>()
|
|
62
75
|
private claimListeners = new Set<(status: NeuralBandClaimStatus) => void>()
|
|
63
|
-
private handwritingListeners = new Set<(
|
|
76
|
+
private handwritingListeners = new Set<(update: NeuralBandHandwritingUpdate) => void>()
|
|
64
77
|
private readonly forward: (stream: string, payload: unknown) => void
|
|
65
78
|
private readonly now: () => number
|
|
66
79
|
|
|
@@ -69,8 +82,10 @@ export class NeuralBandInput {
|
|
|
69
82
|
this.now = options.now ?? (() => Date.now())
|
|
70
83
|
}
|
|
71
84
|
|
|
72
|
-
|
|
73
|
-
|
|
85
|
+
/** One band's claim, or the latest claim of any band without an id. */
|
|
86
|
+
claimStatus(controllerId?: string): NeuralBandClaimStatus {
|
|
87
|
+
if (controllerId === undefined) return this.latestClaim
|
|
88
|
+
return this.claims.get(controllerId) ?? IDLE_CLAIM
|
|
74
89
|
}
|
|
75
90
|
|
|
76
91
|
onClaimStatus(cb: (status: NeuralBandClaimStatus) => void): () => void {
|
|
@@ -78,42 +93,54 @@ export class NeuralBandInput {
|
|
|
78
93
|
return () => this.claimListeners.delete(cb)
|
|
79
94
|
}
|
|
80
95
|
|
|
81
|
-
handwritingStatus(): NeuralBandHandwritingStatus {
|
|
82
|
-
return this.handwriting
|
|
96
|
+
handwritingStatus(controllerId: string): NeuralBandHandwritingStatus {
|
|
97
|
+
return this.handwriting.get(controllerId) ?? IDLE_HANDWRITING
|
|
83
98
|
}
|
|
84
99
|
|
|
85
|
-
onHandwritingStatus(cb: (
|
|
100
|
+
onHandwritingStatus(cb: (update: NeuralBandHandwritingUpdate) => void): () => void {
|
|
86
101
|
this.handwritingListeners.add(cb)
|
|
87
102
|
return () => this.handwritingListeners.delete(cb)
|
|
88
103
|
}
|
|
89
104
|
|
|
90
105
|
/** A new pairing attempt starts from a clean slate. */
|
|
91
106
|
resetClaim(): void {
|
|
92
|
-
this.
|
|
107
|
+
this.latestClaim = IDLE_CLAIM
|
|
108
|
+
for (const cb of this.claimListeners) cb(IDLE_CLAIM)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Drop an unpaired band's state. */
|
|
112
|
+
forget(controllerId: string): void {
|
|
113
|
+
this.claims.delete(controllerId)
|
|
114
|
+
this.handwriting.delete(controllerId)
|
|
93
115
|
}
|
|
94
116
|
|
|
95
117
|
handleClaim(event: NeuralBandClaimEvent): void {
|
|
96
|
-
|
|
118
|
+
const status: NeuralBandClaimStatus = {
|
|
97
119
|
stage: event.stage,
|
|
98
120
|
message: event.message,
|
|
99
121
|
sessionExpired: event.sessionExpired === true,
|
|
100
122
|
bandName: event.bandName,
|
|
101
|
-
|
|
123
|
+
controllerId: event.controllerId,
|
|
124
|
+
}
|
|
125
|
+
if (event.controllerId) this.claims.set(event.controllerId, status)
|
|
126
|
+
this.latestClaim = status
|
|
127
|
+
for (const cb of this.claimListeners) cb(status)
|
|
102
128
|
}
|
|
103
129
|
|
|
104
130
|
handleHandwriting(event: NeuralBandHandwritingEvent): void {
|
|
131
|
+
const id = event.controllerId
|
|
132
|
+
const current = this.handwritingStatus(id)
|
|
105
133
|
if (event.kind === "state") {
|
|
106
|
-
const text = event.phase === "preparing" ? "" :
|
|
107
|
-
this.setHandwriting({phase: event.phase, message: event.message, problem: event.problem, text})
|
|
134
|
+
const text = event.phase === "preparing" ? "" : current.text
|
|
135
|
+
this.setHandwriting(id, {phase: event.phase, message: event.message, problem: event.problem, text})
|
|
108
136
|
return
|
|
109
137
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
for (const key of keyEventsForTextChange(previous, event.text, this.now())) {
|
|
138
|
+
if (event.text !== current.text) {
|
|
139
|
+
for (const key of keyEventsForTextChange(current.text, event.text, this.now())) {
|
|
113
140
|
this.forward(MiniappStreamType.KEYBOARD_EVENT, key)
|
|
114
141
|
}
|
|
115
142
|
}
|
|
116
|
-
this.setHandwriting({...
|
|
143
|
+
this.setHandwriting(id, {...current, text: event.text})
|
|
117
144
|
}
|
|
118
145
|
|
|
119
146
|
handlePointer(event: NeuralBandPointerEvent): void {
|
|
@@ -123,24 +150,21 @@ export class NeuralBandInput {
|
|
|
123
150
|
dx: event.dx,
|
|
124
151
|
dy: event.dy,
|
|
125
152
|
timestamp: event.timestamp,
|
|
153
|
+
controllerId: event.controllerId,
|
|
126
154
|
})
|
|
127
155
|
} else {
|
|
128
156
|
this.forward(MiniappStreamType.POINTER_EVENT, {
|
|
129
157
|
kind: "buttons",
|
|
130
158
|
buttons: event.buttons,
|
|
131
159
|
timestamp: event.timestamp,
|
|
160
|
+
controllerId: event.controllerId,
|
|
132
161
|
})
|
|
133
162
|
}
|
|
134
163
|
}
|
|
135
164
|
|
|
136
|
-
private
|
|
137
|
-
this.
|
|
138
|
-
for (const cb of this.
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
private setHandwriting(status: NeuralBandHandwritingStatus): void {
|
|
142
|
-
this.handwriting = status
|
|
143
|
-
for (const cb of this.handwritingListeners) cb(status)
|
|
165
|
+
private setHandwriting(id: string, status: NeuralBandHandwritingStatus): void {
|
|
166
|
+
this.handwriting.set(id, status)
|
|
167
|
+
for (const cb of this.handwritingListeners) cb({id, status})
|
|
144
168
|
}
|
|
145
169
|
}
|
|
146
170
|
|
|
@@ -1,59 +1,61 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pushes the Neural Band
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Pushes the Neural Band user settings (air mouse, wearing hand) to every
|
|
3
|
+
* connected band: to each band as it connects, and to all of them whenever
|
|
4
|
+
* either setting changes. The native side keeps nothing across connections.
|
|
5
5
|
*/
|
|
6
6
|
import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
|
|
7
7
|
|
|
8
8
|
import {useGlassesStore} from "../../stores/glasses"
|
|
9
9
|
import {SETTINGS, useSettingsStore} from "../../stores/settings"
|
|
10
10
|
import {ControllerTypes} from "../../types/enums"
|
|
11
|
+
import {readyControllers} from "../controllers"
|
|
11
12
|
|
|
12
13
|
let unsubscribers: (() => void)[] = []
|
|
13
14
|
|
|
14
|
-
function
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
glasses.controllerConnected === true &&
|
|
19
|
-
glasses.controllerFullyBooted === true &&
|
|
20
|
-
settings.getSetting(SETTINGS.default_controller.key) === ControllerTypes.NEURAL_BAND
|
|
21
|
-
)
|
|
15
|
+
function connectedBandIds(): string[] {
|
|
16
|
+
return readyControllers(useGlassesStore.getState().controllers)
|
|
17
|
+
.filter((status) => status.model === ControllerTypes.NEURAL_BAND)
|
|
18
|
+
.map((status) => status.id)
|
|
22
19
|
}
|
|
23
20
|
|
|
24
|
-
function pushAirMouse(): void {
|
|
25
|
-
if (!bandConnected()) return
|
|
21
|
+
function pushAirMouse(id: string): void {
|
|
26
22
|
const enabled = useSettingsStore.getState().getSetting(SETTINGS.neural_band_air_mouse.key) === true
|
|
27
|
-
void BluetoothSdk.neuralBandSetPointer(enabled).catch((err) =>
|
|
23
|
+
void BluetoothSdk.neuralBandSetPointer(id, enabled).catch((err) =>
|
|
28
24
|
console.warn("NeuralBandSettingsSync: air mouse push failed:", err),
|
|
29
25
|
)
|
|
30
26
|
}
|
|
31
27
|
|
|
32
|
-
function pushHand(): void {
|
|
33
|
-
if (!bandConnected()) return
|
|
28
|
+
function pushHand(id: string): void {
|
|
34
29
|
const hand = useSettingsStore.getState().getSetting(SETTINGS.neural_band_hand.key) === "left" ? "left" : "right"
|
|
35
|
-
void BluetoothSdk.neuralBandSetHand(hand).catch((err) =>
|
|
30
|
+
void BluetoothSdk.neuralBandSetHand(id, hand).catch((err) =>
|
|
36
31
|
console.warn("NeuralBandSettingsSync: hand push failed:", err),
|
|
37
32
|
)
|
|
38
33
|
}
|
|
39
34
|
|
|
40
35
|
export function startNeuralBandSettingsSync(): void {
|
|
41
36
|
if (unsubscribers.length > 0) return
|
|
42
|
-
let
|
|
37
|
+
let connected = new Set(connectedBandIds())
|
|
43
38
|
unsubscribers = [
|
|
44
39
|
useGlassesStore.subscribe(
|
|
45
|
-
(
|
|
40
|
+
() => connectedBandIds().sort().join(","),
|
|
46
41
|
() => {
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
42
|
+
const next = new Set(connectedBandIds())
|
|
43
|
+
for (const id of next) {
|
|
44
|
+
if (connected.has(id)) continue
|
|
45
|
+
pushAirMouse(id)
|
|
46
|
+
pushHand(id)
|
|
51
47
|
}
|
|
52
|
-
|
|
48
|
+
connected = next
|
|
53
49
|
},
|
|
54
50
|
),
|
|
55
|
-
useSettingsStore.subscribe(
|
|
56
|
-
|
|
51
|
+
useSettingsStore.subscribe(
|
|
52
|
+
(s) => s.getSetting(SETTINGS.neural_band_air_mouse.key),
|
|
53
|
+
() => connectedBandIds().forEach(pushAirMouse),
|
|
54
|
+
),
|
|
55
|
+
useSettingsStore.subscribe(
|
|
56
|
+
(s) => s.getSetting(SETTINGS.neural_band_hand.key),
|
|
57
|
+
() => connectedBandIds().forEach(pushHand),
|
|
58
|
+
),
|
|
57
59
|
]
|
|
58
60
|
}
|
|
59
61
|
|