@glassly/engine 0.3.3-dev.670 → 0.3.4-beta.689

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/build/engine.d.ts +38 -14
  2. package/build/engine.d.ts.map +1 -1
  3. package/build/engine.js +3 -0
  4. package/build/engine.js.map +1 -1
  5. package/build/facades/ai.d.ts +32 -0
  6. package/build/facades/ai.d.ts.map +1 -0
  7. package/build/facades/ai.js +38 -0
  8. package/build/facades/ai.js.map +1 -0
  9. package/build/facades/glasses.d.ts +30 -24
  10. package/build/facades/glasses.d.ts.map +1 -1
  11. package/build/facades/glasses.js +49 -30
  12. package/build/facades/glasses.js.map +1 -1
  13. package/build/facades/glassesSettings.d.ts.map +1 -1
  14. package/build/facades/glassesSettings.js +1 -0
  15. package/build/facades/glassesSettings.js.map +1 -1
  16. package/build/facades/pairing.d.ts +6 -5
  17. package/build/facades/pairing.d.ts.map +1 -1
  18. package/build/facades/pairing.js +14 -14
  19. package/build/facades/pairing.js.map +1 -1
  20. package/build/generated/releaseMetadata.js +4 -4
  21. package/build/generated/releaseMetadata.js.map +1 -1
  22. package/build/index.d.ts +5 -2
  23. package/build/index.d.ts.map +1 -1
  24. package/build/index.js +2 -0
  25. package/build/index.js.map +1 -1
  26. package/build/services/DeviceEventRouter.d.ts.map +1 -1
  27. package/build/services/DeviceEventRouter.js +9 -1
  28. package/build/services/DeviceEventRouter.js.map +1 -1
  29. package/build/services/GestureDotsService.d.ts +14 -3
  30. package/build/services/GestureDotsService.d.ts.map +1 -1
  31. package/build/services/GestureDotsService.js +48 -13
  32. package/build/services/GestureDotsService.js.map +1 -1
  33. package/build/services/GlassesTouchInput.d.ts.map +1 -1
  34. package/build/services/GlassesTouchInput.js +11 -9
  35. package/build/services/GlassesTouchInput.js.map +1 -1
  36. package/build/services/LlmRouting.d.ts +35 -0
  37. package/build/services/LlmRouting.d.ts.map +1 -0
  38. package/build/services/LlmRouting.js +25 -0
  39. package/build/services/LlmRouting.js.map +1 -0
  40. package/build/services/LocalLlmModelManager.d.ts +79 -0
  41. package/build/services/LocalLlmModelManager.d.ts.map +1 -0
  42. package/build/services/LocalLlmModelManager.js +278 -0
  43. package/build/services/LocalLlmModelManager.js.map +1 -0
  44. package/build/services/LocalLlmPrompt.d.ts +90 -0
  45. package/build/services/LocalLlmPrompt.d.ts.map +1 -0
  46. package/build/services/LocalLlmPrompt.js +189 -0
  47. package/build/services/LocalLlmPrompt.js.map +1 -0
  48. package/build/services/LocalLlmProvider.d.ts +35 -0
  49. package/build/services/LocalLlmProvider.d.ts.map +1 -0
  50. package/build/services/LocalLlmProvider.js +218 -0
  51. package/build/services/LocalLlmProvider.js.map +1 -0
  52. package/build/services/LocalMiniappRuntime.d.ts +9 -1
  53. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  54. package/build/services/LocalMiniappRuntime.js +91 -17
  55. package/build/services/LocalMiniappRuntime.js.map +1 -1
  56. package/build/services/STTModelManager.d.ts +25 -2
  57. package/build/services/STTModelManager.d.ts.map +1 -1
  58. package/build/services/STTModelManager.js +137 -6
  59. package/build/services/STTModelManager.js.map +1 -1
  60. package/build/services/controllers.d.ts +21 -0
  61. package/build/services/controllers.d.ts.map +1 -0
  62. package/build/services/controllers.js +62 -0
  63. package/build/services/controllers.js.map +1 -0
  64. package/build/services/effectiveCapabilities.d.ts.map +1 -1
  65. package/build/services/effectiveCapabilities.js +18 -24
  66. package/build/services/effectiveCapabilities.js.map +1 -1
  67. package/build/services/neuralBand/NeuralBandInput.d.ts +17 -6
  68. package/build/services/neuralBand/NeuralBandInput.d.ts.map +1 -1
  69. package/build/services/neuralBand/NeuralBandInput.js +40 -23
  70. package/build/services/neuralBand/NeuralBandInput.js.map +1 -1
  71. package/build/services/neuralBand/NeuralBandSettingsSync.d.ts.map +1 -1
  72. package/build/services/neuralBand/NeuralBandSettingsSync.js +23 -26
  73. package/build/services/neuralBand/NeuralBandSettingsSync.js.map +1 -1
  74. package/build/services/settingsBridge.d.ts +26 -18
  75. package/build/services/settingsBridge.d.ts.map +1 -1
  76. package/build/services/settingsBridge.js +36 -15
  77. package/build/services/settingsBridge.js.map +1 -1
  78. package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
  79. package/build/stores/bluetoothSettingKeys.js +3 -1
  80. package/build/stores/bluetoothSettingKeys.js.map +1 -1
  81. package/build/stores/core.d.ts.map +1 -1
  82. package/build/stores/core.js +0 -1
  83. package/build/stores/core.js.map +1 -1
  84. package/build/stores/glasses.d.ts.map +1 -1
  85. package/build/stores/glasses.js +2 -6
  86. package/build/stores/glasses.js.map +1 -1
  87. package/build/stores/settings.d.ts.map +1 -1
  88. package/build/stores/settings.js +29 -3
  89. package/build/stores/settings.js.map +1 -1
  90. package/package.json +12 -8
  91. package/src/engine.ts +3 -0
  92. package/src/facades/ai.ts +43 -0
  93. package/src/facades/glasses.ts +60 -31
  94. package/src/facades/glassesSettings.ts +1 -0
  95. package/src/facades/pairing.ts +18 -14
  96. package/src/generated/releaseMetadata.ts +4 -4
  97. package/src/index.ts +16 -2
  98. package/src/services/DeviceEventRouter.ts +8 -1
  99. package/src/services/GestureDotsService.ts +52 -15
  100. package/src/services/GlassesTouchInput.ts +11 -9
  101. package/src/services/LlmRouting.ts +49 -0
  102. package/src/services/LocalLlmModelManager.ts +327 -0
  103. package/src/services/LocalLlmPrompt.ts +246 -0
  104. package/src/services/LocalLlmProvider.ts +290 -0
  105. package/src/services/LocalMiniappRuntime.ts +137 -21
  106. package/src/services/STTModelManager.ts +140 -8
  107. package/src/services/controllers.ts +77 -0
  108. package/src/services/effectiveCapabilities.ts +16 -32
  109. package/src/services/neuralBand/NeuralBandInput.ts +50 -26
  110. package/src/services/neuralBand/NeuralBandSettingsSync.ts +28 -26
  111. package/src/services/settingsBridge.ts +60 -22
  112. package/src/stores/bluetoothSettingKeys.ts +3 -1
  113. package/src/stores/core.ts +0 -1
  114. package/src/stores/glasses.ts +2 -6
  115. package/src/stores/settings.ts +29 -3
@@ -12,7 +12,10 @@
12
12
  * that in its storage.
13
13
  */
14
14
 
15
+ import type {ControllerStatus, PairedController} from "@glassly/bluetooth-sdk"
16
+
15
17
  import {ControllerTypes} from "../types/enums"
18
+ import {hasInput2d, isControllerReady} from "./controllers"
16
19
 
17
20
  export type SettingsBridgeAccess = "read" | "write"
18
21
 
@@ -39,11 +42,11 @@ export const SETTINGS_BRIDGE_KEYS: Readonly<Record<string, SettingsBridgeAccess>
39
42
 
40
43
  /**
41
44
  * Read-only keys the engine computes: whether the wearer can see the phone,
42
- * whether the foreground miniapp is drawing on the glasses, the ring's
43
- * battery, and whether a 2D controller is connected. They change on their
44
- * own clocks; the runtime pushes them like stored keys.
45
+ * whether the foreground miniapp is drawing on the glasses, and the paired
46
+ * controllers with their battery and input. They change on their own clocks;
47
+ * the runtime pushes them like stored keys.
45
48
  */
46
- export const DERIVED_SETTINGS_KEYS = ["phone_active", "foreground_app_on_screen", "ring_battery", "input_2d"] as const
49
+ export const DERIVED_SETTINGS_KEYS = ["phone_active", "foreground_app_on_screen", "controllers"] as const
47
50
  export type DerivedSettingsKey = (typeof DERIVED_SETTINGS_KEYS)[number]
48
51
 
49
52
  export function isDerivedSettingsKey(key: string): key is DerivedSettingsKey {
@@ -91,31 +94,66 @@ export function validateSettingsSet(payload: Record<string, unknown>): SettingsS
91
94
  return {ok: true, key, value: payload.value}
92
95
  }
93
96
 
94
- /** The ring's battery as the dashboard shows it: the relayed ring first, else the controller. */
95
- export interface RingBattery {
97
+ /** One paired controller as the `controllers` derived key serves it. */
98
+ export interface ControllerSettingEntry {
99
+ id: string
100
+ model: string
101
+ name: string
102
+ /** Delivering input. */
96
103
  connected: boolean
97
- /** -1 when unknown. */
98
- level: number
104
+ /** level -1 when unknown. */
105
+ battery: {level: number; charging: boolean}
106
+ /** Swipes sideways as well as up and down (the Meta Neural Band). */
107
+ input2d: boolean
99
108
  }
100
109
 
101
- export function ringBatteryFrom(glasses: {
110
+ /** What the glasses report about the ring linked to them (G2 with the R1 routed through it). */
111
+ export interface RelayedRing {
102
112
  ringConnected: boolean
103
113
  ringBatteryLevel: number
104
- controllerConnected: boolean
105
- controllerBatteryLevel: number
106
- }): RingBattery {
107
- const relayed = glasses.ringConnected && glasses.ringBatteryLevel >= 0
108
- return {
109
- connected: relayed || glasses.controllerConnected,
110
- level: relayed ? glasses.ringBatteryLevel : glasses.controllerBatteryLevel,
111
- }
114
+ ringCharging: boolean
115
+ }
116
+
117
+ /**
118
+ * The R1 the glasses' ring report is about: a ring routed through the G2 keeps
119
+ * its phone link for telemetry, so prefer a linked ring, else the first one.
120
+ */
121
+ function relayedRingId(
122
+ records: PairedController[],
123
+ statuses: Record<string, ControllerStatus> | undefined,
124
+ ): string | undefined {
125
+ const rings = records.filter((record) => record.model === ControllerTypes.R1)
126
+ return (rings.find((ring) => statuses?.[ring.id]?.connected === true) ?? rings[0])?.id
112
127
  }
113
128
 
114
129
  /**
115
- * Whether the connected controller swipes in two dimensions (sideways as well
116
- * as up and down): the Meta Neural Band's thumb swipes. The R1 ring and the
117
- * glasses' temples are one-dimensional.
130
+ * One entry per paired record, registry order. While the glasses report a
131
+ * linked ring, that report is that ring's link and battery: a ring routed
132
+ * through the G2 reports there, not to the phone.
118
133
  */
119
- export function input2dFrom(glasses: {controllerConnected: boolean}, controllerType: unknown): boolean {
120
- return glasses.controllerConnected && controllerType === ControllerTypes.NEURAL_BAND
134
+ export function controllersSettingFrom(
135
+ records: PairedController[],
136
+ statuses: Record<string, ControllerStatus> | undefined,
137
+ relayed: RelayedRing,
138
+ ): ControllerSettingEntry[] {
139
+ const relayedId = relayed.ringConnected ? relayedRingId(records, statuses) : undefined
140
+ return records.map((record) => {
141
+ const status = statuses?.[record.id]
142
+ let connected = isControllerReady(status)
143
+ let battery = {level: status?.batteryLevel ?? -1, charging: status?.charging === true}
144
+ if (record.id === relayedId) {
145
+ connected = true
146
+ if (relayed.ringBatteryLevel >= 0) {
147
+ battery = {level: relayed.ringBatteryLevel, charging: relayed.ringCharging}
148
+ }
149
+ }
150
+ return {
151
+ id: record.id,
152
+ model: record.model,
153
+ name: record.name,
154
+ connected,
155
+ battery,
156
+ input2d: connected && hasInput2d(record.model),
157
+ }
158
+ })
121
159
  }
@@ -20,8 +20,10 @@ export const MENTRA_LIVE_SETTING_KEYS: string[] = [
20
20
  "pending_wearable",
21
21
  "device_name",
22
22
  "device_address",
23
- "default_controller",
23
+ "controllers",
24
24
  "pending_controller",
25
+ // Deprecated, kept so native can migrate and clear them:
26
+ "default_controller",
25
27
  "controller_device_name",
26
28
  "controller_address",
27
29
  ]
@@ -10,7 +10,6 @@ export interface CoreState extends PublicBluetoothStatus {
10
10
  const initialState: PublicBluetoothStatus = {
11
11
  // state:
12
12
  searching: false,
13
- searchingController: false,
14
13
  micRanking: ["glasses", "phone", "bluetooth", "bluetoothClassic"],
15
14
  systemMicUnavailable: false,
16
15
  currentMic: null,
@@ -168,12 +168,8 @@ const initialState: GlassesStore = {
168
168
  otaProgress: null,
169
169
  otaInProgress: false,
170
170
  mtkUpdatedThisSession: false,
171
- // ring:
172
- controllerConnected: false,
173
- controllerFullyBooted: false,
174
- controllerMacAddress: "",
175
- controllerBatteryLevel: -1,
176
- controllerSignalStrength: -1,
171
+ // controller drivers by registry id:
172
+ controllers: {},
177
173
  // iOS ANCS consent (G2 consumes notifications on-firmware):
178
174
  ancsAuthorized: false,
179
175
  // G2 Glassly CFW keyboard bridge (revision 39+):
@@ -386,6 +386,17 @@ export const SETTINGS: Record<string, Setting> = {
386
386
  saveOnServer: false,
387
387
  persist: true,
388
388
  },
389
+ // Which on-device LLM export serves session.ai calls routed to the phone
390
+ // (forceLocal / localFallback): a LocalLlmModelId ("qwen3-0.6b" |
391
+ // "qwen3-1.7b" | "qwen3-4b"). Device-local; the model files live on this
392
+ // phone. See LocalLlmModelManager.
393
+ local_llm_model: {
394
+ key: "local_llm_model",
395
+ defaultValue: () => "qwen3-1.7b",
396
+ writable: true,
397
+ saveOnServer: false,
398
+ persist: true,
399
+ },
389
400
  // Developer override for the ASG OTA manifest URL. null/empty = no override;
390
401
  // the normal selection applies (legacy-glasses gate, EXPO_PUBLIC_ASG_OTA_VERSION_URL,
391
402
  // glasses-reported URL, then production). See resolveOtaManifestUrl.
@@ -477,6 +488,17 @@ export const SETTINGS: Record<string, Setting> = {
477
488
  saveOnServer: false,
478
489
  persist: true,
479
490
  },
491
+ // Paired controllers (PairedController records) in pairing order. Native
492
+ // upserts on ready and removes on forget, echoing the array via save_setting.
493
+ controllers: {
494
+ key: "controllers",
495
+ defaultValue: () => [],
496
+ writable: true,
497
+ saveOnServer: false,
498
+ persist: true,
499
+ nativeAuthoritative: true,
500
+ },
501
+ // Deprecated single-slot identity: seeded to native for its one-shot migration into `controllers`; never read in JS.
480
502
  default_controller: {
481
503
  key: "default_controller",
482
504
  defaultValue: () => "",
@@ -487,7 +509,7 @@ export const SETTINGS: Record<string, Setting> = {
487
509
  },
488
510
  // Display name of the keyboard the glasses remember (G2 Glassly CFW). The
489
511
  // native bridge owns the address/bond; this only keeps the home card and
490
- // settings row populated before the glasses connect, like default_controller.
512
+ // settings row populated before the glasses connect.
491
513
  keyboard_name: {
492
514
  key: "keyboard_name",
493
515
  defaultValue: () => "",
@@ -503,6 +525,7 @@ export const SETTINGS: Record<string, Setting> = {
503
525
  persist: true,
504
526
  nativeAuthoritative: true,
505
527
  },
528
+ // Deprecated: see default_controller.
506
529
  controller_device_name: {
507
530
  key: "controller_device_name",
508
531
  defaultValue: () => "",
@@ -511,6 +534,7 @@ export const SETTINGS: Record<string, Setting> = {
511
534
  persist: true,
512
535
  nativeAuthoritative: true,
513
536
  },
537
+ // Deprecated: see default_controller.
514
538
  controller_address: {
515
539
  key: "controller_address",
516
540
  defaultValue: () => "",
@@ -731,7 +755,7 @@ export const SETTINGS: Record<string, Setting> = {
731
755
  // glasses settings:
732
756
  contextual_dashboard: {
733
757
  key: "contextual_dashboard",
734
- defaultValue: () => true,
758
+ defaultValue: () => false,
735
759
  writable: true,
736
760
  saveOnServer: true,
737
761
  persist: true,
@@ -1179,8 +1203,10 @@ export const BLUETOOTH_SETTING_KEYS: string[] = [
1179
1203
  SETTINGS.device_name.key,
1180
1204
  SETTINGS.device_address.key,
1181
1205
  SETTINGS.project_name.key,
1182
- SETTINGS.default_controller.key,
1206
+ SETTINGS.controllers.key,
1183
1207
  SETTINGS.pending_controller.key,
1208
+ // Deprecated, kept so native can migrate and clear them:
1209
+ SETTINGS.default_controller.key,
1184
1210
  SETTINGS.controller_device_name.key,
1185
1211
  SETTINGS.controller_address.key,
1186
1212
  // offline applets: