@glassly/engine 0.3.2-beta.572 → 0.3.2-beta.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/engine.d.ts +15 -1
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +7 -3
- package/build/engine.js.map +1 -1
- package/build/facades/dev.d.ts +2 -2
- package/build/facades/dev.d.ts.map +1 -1
- package/build/facades/dev.js +2 -2
- package/build/facades/dev.js.map +1 -1
- package/build/facades/glasses.d.ts +20 -0
- package/build/facades/glasses.d.ts.map +1 -1
- package/build/facades/glasses.js +21 -0
- package/build/facades/glasses.js.map +1 -1
- package/build/facades/pairing.js +1 -1
- package/build/facades/pairing.js.map +1 -1
- package/build/facades/reports.js +2 -2
- package/build/facades/reports.js.map +1 -1
- package/build/facades/session.d.ts +1 -1
- package/build/facades/session.js +2 -2
- package/build/facades/session.js.map +1 -1
- package/build/generated/releaseMetadata.js +3 -3
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/index.d.ts +3 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +3 -1
- package/build/index.js.map +1 -1
- package/build/runtime/bootstrap.d.ts +2 -2
- package/build/runtime/bootstrap.d.ts.map +1 -1
- package/build/runtime/bootstrap.js.map +1 -1
- package/build/services/CloudAudioSubscriptionSync.d.ts +2 -2
- package/build/services/CloudAudioSubscriptionSync.js +2 -2
- package/build/services/CloudAudioSubscriptionSync.js.map +1 -1
- package/build/services/CloudClientService.d.ts +2 -2
- package/build/services/CloudClientService.d.ts.map +1 -1
- package/build/services/CloudClientService.js +4 -4
- package/build/services/CloudClientService.js.map +1 -1
- package/build/services/CloudTranscriptE2EMetrics.d.ts +2 -2
- package/build/services/CloudTranscriptE2EMetrics.d.ts.map +1 -1
- package/build/services/CloudTranscriptE2EMetrics.js +3 -3
- package/build/services/CloudTranscriptE2EMetrics.js.map +1 -1
- package/build/services/DeviceEventRouter.d.ts.map +1 -1
- package/build/services/DeviceEventRouter.js +5 -0
- package/build/services/DeviceEventRouter.js.map +1 -1
- package/build/services/GestureDotsService.d.ts +9 -9
- package/build/services/GestureDotsService.d.ts.map +1 -1
- package/build/services/GestureDotsService.js +75 -50
- package/build/services/GestureDotsService.js.map +1 -1
- package/build/services/GlassesTouchInput.d.ts +1 -1
- package/build/services/GlassesTouchInput.d.ts.map +1 -1
- package/build/services/GlassesTouchInput.js +7 -4
- package/build/services/GlassesTouchInput.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +1 -1
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +26 -7
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/LocalSttFallbackCoordinator.d.ts +5 -2
- package/build/services/LocalSttFallbackCoordinator.d.ts.map +1 -1
- package/build/services/LocalSttFallbackCoordinator.js +7 -3
- package/build/services/LocalSttFallbackCoordinator.js.map +1 -1
- package/build/services/PhonePhotoCoordinator.d.ts +1 -1
- package/build/services/PhonePhotoCoordinator.js +4 -4
- package/build/services/PhonePhotoCoordinator.js.map +1 -1
- package/build/services/PhoneStreamCoordinator.d.ts +1 -1
- package/build/services/PhoneStreamCoordinator.js +1 -1
- package/build/services/PhoneStreamCoordinator.js.map +1 -1
- package/build/services/SupportProfileSync.d.ts +1 -1
- package/build/services/SupportProfileSync.js +2 -2
- package/build/services/SupportProfileSync.js.map +1 -1
- package/build/services/TranscriptionRouting.d.ts +6 -1
- package/build/services/TranscriptionRouting.d.ts.map +1 -1
- package/build/services/TranscriptionRouting.js +7 -3
- package/build/services/TranscriptionRouting.js.map +1 -1
- package/build/services/cloudStreamApi.d.ts +2 -2
- package/build/services/cloudStreamApi.js +2 -2
- package/build/services/cloudStreamApi.js.map +1 -1
- package/build/services/gestureDotGlyphs.d.ts +1 -1
- package/build/services/gestureDotGlyphs.d.ts.map +1 -1
- package/build/services/gestureDotGlyphs.js +5 -0
- package/build/services/gestureDotGlyphs.js.map +1 -1
- package/build/services/neuralBand/MetaAuth.d.ts +31 -0
- package/build/services/neuralBand/MetaAuth.d.ts.map +1 -0
- package/build/services/neuralBand/MetaAuth.js +161 -0
- package/build/services/neuralBand/MetaAuth.js.map +1 -0
- package/build/services/neuralBand/NeuralBandInput.d.ts +50 -0
- package/build/services/neuralBand/NeuralBandInput.d.ts.map +1 -0
- package/build/services/neuralBand/NeuralBandInput.js +114 -0
- package/build/services/neuralBand/NeuralBandInput.js.map +1 -0
- package/build/services/neuralBand/NeuralBandSettingsSync.d.ts +3 -0
- package/build/services/neuralBand/NeuralBandSettingsSync.d.ts.map +1 -0
- package/build/services/neuralBand/NeuralBandSettingsSync.js +52 -0
- package/build/services/neuralBand/NeuralBandSettingsSync.js.map +1 -0
- package/build/services/neuralBand/sha256.d.ts +7 -0
- package/build/services/neuralBand/sha256.d.ts.map +1 -0
- package/build/services/neuralBand/sha256.js +68 -0
- package/build/services/neuralBand/sha256.js.map +1 -0
- package/build/services/soniox/SonioxRealtimeClient.d.ts +2 -2
- package/build/services/soniox/SonioxRealtimeClient.js +2 -2
- package/build/services/soniox/SonioxRealtimeClient.js.map +1 -1
- package/build/services/soniox/SonioxTokenAssembler.d.ts +1 -1
- package/build/services/soniox/SonioxTokenAssembler.js +1 -1
- package/build/services/soniox/SonioxTokenAssembler.js.map +1 -1
- package/build/stores/cloudClientStatus.js +1 -1
- package/build/stores/cloudClientStatus.js.map +1 -1
- package/build/stores/settings.d.ts.map +1 -1
- package/build/stores/settings.js +25 -7
- package/build/stores/settings.js.map +1 -1
- package/build/types/enums.d.ts +2 -1
- package/build/types/enums.d.ts.map +1 -1
- package/build/types/enums.js +1 -0
- package/build/types/enums.js.map +1 -1
- package/build/utils/cloudClient/cloudSecureStore.js +2 -2
- package/build/utils/cloudClient/cloudSecureStore.js.map +1 -1
- package/package.json +7 -7
- package/src/engine.ts +7 -3
- package/src/facades/dev.ts +2 -2
- package/src/facades/glasses.ts +27 -0
- package/src/facades/pairing.ts +1 -1
- package/src/facades/reports.ts +2 -2
- package/src/facades/session.ts +2 -2
- package/src/generated/releaseMetadata.ts +3 -3
- package/src/index.ts +15 -1
- package/src/runtime/bootstrap.ts +2 -2
- package/src/services/CloudAudioSubscriptionSync.ts +2 -2
- package/src/services/CloudClientService.ts +4 -4
- package/src/services/CloudTranscriptE2EMetrics.ts +3 -3
- package/src/services/DeviceEventRouter.ts +5 -0
- package/src/services/GestureDotsService.ts +72 -49
- package/src/services/GlassesTouchInput.ts +13 -5
- package/src/services/LocalMiniappRuntime.ts +46 -24
- package/src/services/LocalSttFallbackCoordinator.ts +18 -4
- package/src/services/PhonePhotoCoordinator.ts +4 -4
- package/src/services/PhoneStreamCoordinator.ts +1 -1
- package/src/services/SupportProfileSync.ts +2 -2
- package/src/services/TranscriptionRouting.ts +12 -4
- package/src/services/cloudStreamApi.ts +3 -3
- package/src/services/gestureDotGlyphs.ts +6 -1
- package/src/services/neuralBand/MetaAuth.ts +192 -0
- package/src/services/neuralBand/NeuralBandInput.ts +148 -0
- package/src/services/neuralBand/NeuralBandSettingsSync.ts +63 -0
- package/src/services/neuralBand/sha256.ts +70 -0
- package/src/services/soniox/SonioxRealtimeClient.ts +2 -2
- package/src/services/soniox/SonioxTokenAssembler.ts +1 -1
- package/src/stores/cloudClientStatus.ts +1 -1
- package/src/stores/settings.ts +25 -7
- package/src/types/enums.ts +1 -0
- package/src/utils/cloudClient/cloudSecureStore.ts +2 -2
package/src/facades/dev.ts
CHANGED
|
@@ -94,12 +94,12 @@ export type DevRuntimeStatusSnapshot = ReturnType<typeof projectRuntimeStatus>
|
|
|
94
94
|
export const dev = {
|
|
95
95
|
/** Inject a glasses gesture through the normal dashboard/miniapp input policy. */
|
|
96
96
|
sendGesture,
|
|
97
|
-
/** The cloud
|
|
97
|
+
/** The cloud core/runtime URL overrides. */
|
|
98
98
|
cloudUrls: (): {core?: string; runtime?: string} => {
|
|
99
99
|
const s = useSettingsStore.getState()
|
|
100
100
|
return {core: s.getSetting(SETTINGS.cloud_core_url.key), runtime: s.getSetting(SETTINGS.cloud_runtime_url.key)}
|
|
101
101
|
},
|
|
102
|
-
/** Override the cloud
|
|
102
|
+
/** Override the cloud URLs and reconnect the live client onto them. */
|
|
103
103
|
setCloudUrls: (urls: {core?: string; runtime?: string}) => {
|
|
104
104
|
const next = {...currentCloudUrlOverrides(), ...urls}
|
|
105
105
|
const resolved = resolveCloudUrlReconnectTarget(next)
|
package/src/facades/glasses.ts
CHANGED
|
@@ -25,6 +25,10 @@ import {hasDefaultDevice} from "../services/DeviceStoreHydration"
|
|
|
25
25
|
import {projectPairingIdentity} from "../services/PairingIdentity"
|
|
26
26
|
import {isGlassesConnected, isGlassesReady} from "../services/GlassesReadiness"
|
|
27
27
|
import {pushAllBluetoothSettings} from "../services/GlassesSettingsSync"
|
|
28
|
+
import neuralBandInput, {
|
|
29
|
+
type NeuralBandClaimStatus,
|
|
30
|
+
type NeuralBandHandwritingStatus,
|
|
31
|
+
} from "../services/neuralBand/NeuralBandInput"
|
|
28
32
|
import {recordSupportProfileConnectionFailure} from "../services/SupportProfileSync"
|
|
29
33
|
import {getModelCapabilities, type DeviceTypes} from "../types"
|
|
30
34
|
import {glassesWifi} from "./glassesWifi"
|
|
@@ -267,6 +271,29 @@ export const glasses = {
|
|
|
267
271
|
},
|
|
268
272
|
},
|
|
269
273
|
|
|
274
|
+
// --- Meta Neural Band (a controller with a claim step and its own features) ---
|
|
275
|
+
neuralBand: {
|
|
276
|
+
/** Store the Meta account session the one-time band claim uses. */
|
|
277
|
+
setMetaSession: (accessToken: string, userId: string): Promise<void> =>
|
|
278
|
+
BluetoothSdk.neuralBandSetMetaSession(accessToken, userId),
|
|
279
|
+
clearMetaSession: (): Promise<void> => BluetoothSdk.neuralBandClearMetaSession(),
|
|
280
|
+
hasMetaSession: (): Promise<boolean> => BluetoothSdk.neuralBandHasMetaSession(),
|
|
281
|
+
/** A band identity is stored on this phone, so the remembered band connects without a claim. */
|
|
282
|
+
isClaimed: (): Promise<boolean> => BluetoothSdk.neuralBandIsClaimed(),
|
|
283
|
+
/** The latest claim progress, kept across screens. */
|
|
284
|
+
claimStatus: (): NeuralBandClaimStatus => neuralBandInput.claimStatus(),
|
|
285
|
+
resetClaim: (): void => neuralBandInput.resetClaim(),
|
|
286
|
+
onClaimStatus: (cb: (status: NeuralBandClaimStatus) => void): (() => void) => neuralBandInput.onClaimStatus(cb),
|
|
287
|
+
/** The band's handwriting model; writing arrives on the miniapp keyboard stream. */
|
|
288
|
+
setHandwriting: (enabled: boolean): Promise<void> => BluetoothSdk.neuralBandSetHandwriting(enabled),
|
|
289
|
+
handwritingStatus: (): NeuralBandHandwritingStatus => neuralBandInput.handwritingStatus(),
|
|
290
|
+
onHandwritingStatus: (cb: (status: NeuralBandHandwritingStatus) => void): (() => void) =>
|
|
291
|
+
neuralBandInput.onHandwritingStatus(cb),
|
|
292
|
+
/** The band's air mouse; movement reaches miniapps on the pointer stream. */
|
|
293
|
+
setPointer: (enabled: boolean): Promise<void> => BluetoothSdk.neuralBandSetPointer(enabled),
|
|
294
|
+
setHand: (hand: "left" | "right"): Promise<void> => BluetoothSdk.neuralBandSetHand(hand),
|
|
295
|
+
},
|
|
296
|
+
|
|
270
297
|
// --- audio (glasses media-volume + own-app playback hint) ---
|
|
271
298
|
audio: {
|
|
272
299
|
/** Read the glasses' current media volume. */
|
package/src/facades/pairing.ts
CHANGED
|
@@ -63,7 +63,7 @@ function projectTargetReady(options: Pick<PairingReadyWaitOptions, "deviceModel"
|
|
|
63
63
|
const {deviceModel, deviceName} = options
|
|
64
64
|
const glasses = useGlassesStore.getState()
|
|
65
65
|
|
|
66
|
-
if (deviceModel
|
|
66
|
+
if (deviceModel !== undefined && (Object.values(ControllerTypes) as string[]).includes(deviceModel)) {
|
|
67
67
|
if (!glasses.controllerConnected || !glasses.controllerFullyBooted) return false
|
|
68
68
|
const settings = useSettingsStore.getState()
|
|
69
69
|
const pairedModel = settings.getSetting(SETTINGS.default_controller.key)
|
package/src/facades/reports.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* reports facade — `engine.reports`: user/system report submission over the
|
|
3
|
-
* Cloud
|
|
3
|
+
* Cloud core client.
|
|
4
4
|
*
|
|
5
5
|
* Host/OEM code owns UI and wording. Engine owns Glassly mechanics: context
|
|
6
|
-
* collection, recent phone logs, Cloud
|
|
6
|
+
* collection, recent phone logs, Cloud calls, local automatic throttling,
|
|
7
7
|
* screenshots, and notifying connected glasses.
|
|
8
8
|
*/
|
|
9
9
|
import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
|
package/src/facades/session.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* session facade — `engine.session`: the cloud
|
|
2
|
+
* session facade — `engine.session`: the cloud live-session surface engine
|
|
3
3
|
* now owns (keystone #5). Exposes the connection status read-model (status +
|
|
4
4
|
* audio transport, projected from the engine-owned cloud-status store) and the
|
|
5
5
|
* liveness flag.
|
|
@@ -38,7 +38,7 @@ export const session = {
|
|
|
38
38
|
* Legacy Cloud-V1 websocket status. The socket's writer (the host
|
|
39
39
|
* WebSocketManager) was deleted in the Cloud V1 ripout, so this now always
|
|
40
40
|
* reports the store's initial DISCONNECTED; the host connection banners
|
|
41
|
-
* render `status()` (the cloud
|
|
41
|
+
* render `status()` (the cloud runtime) instead. Kept, with the backing
|
|
42
42
|
* connection store, for the engine-side legacy-surface removal (issue #3392).
|
|
43
43
|
*/
|
|
44
44
|
legacyWebsocketStatus: (): WebSocketStatus => useConnectionStore.getState().status,
|
|
@@ -12,9 +12,9 @@ export interface EngineReleaseMetadata {
|
|
|
12
12
|
export const ENGINE_RELEASE_METADATA: Readonly<EngineReleaseMetadata> = Object.freeze({
|
|
13
13
|
"schemaVersion": 1,
|
|
14
14
|
"familyBaseVersion": "0.3.2",
|
|
15
|
-
"releaseIdentity": "0.3.2-beta.
|
|
16
|
-
"releaseSetId": "glassly-0.3.2-beta.
|
|
17
|
-
"sourceCommit": "
|
|
15
|
+
"releaseIdentity": "0.3.2-beta.597",
|
|
16
|
+
"releaseSetId": "glassly-0.3.2-beta.597",
|
|
17
|
+
"sourceCommit": "4bbd4945a62883e5f384569b0f5c56a51e5c81cd",
|
|
18
18
|
"otaManifestUrl": null,
|
|
19
19
|
"otaManifestSha256": null
|
|
20
20
|
})
|
package/src/index.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* capability tables, timers). Judgment rule: read models, commands, pure
|
|
8
8
|
* functions and types belong here; anything that mutates runtime state or
|
|
9
9
|
* exposes a raw store/service remains private to the Glassly host.
|
|
10
|
-
* See cloud
|
|
10
|
+
* See cloud/docs/issues/020-glasses-status-boundary/integration-review.md §D.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
// The namespaced OEM-facing engine API (the "(A) host API"). See ./engine.
|
|
@@ -201,6 +201,20 @@ export {
|
|
|
201
201
|
} from "./utils/devMiniappLaunch"
|
|
202
202
|
export {HardwareCompatibility, type CompatibilityResult} from "./utils/hardware"
|
|
203
203
|
export {BgTimer, throttle, debounce} from "./utils/timers"
|
|
204
|
+
// Meta Neural Band: the account sign-in a claim needs (the WebView lives in the host).
|
|
205
|
+
export {
|
|
206
|
+
MetaAuthError,
|
|
207
|
+
authEntryUrl,
|
|
208
|
+
callbackMatches,
|
|
209
|
+
callbackQuery,
|
|
210
|
+
decryptBlob,
|
|
211
|
+
isLoginCallback,
|
|
212
|
+
login,
|
|
213
|
+
tokensQuery,
|
|
214
|
+
type MetaSession,
|
|
215
|
+
type MetaSsoTokens,
|
|
216
|
+
} from "./services/neuralBand/MetaAuth"
|
|
217
|
+
export type {NeuralBandClaimStatus, NeuralBandHandwritingStatus} from "./services/neuralBand/NeuralBandInput"
|
|
204
218
|
|
|
205
219
|
// Hardware types and capability profiles.
|
|
206
220
|
export {
|
package/src/runtime/bootstrap.ts
CHANGED
|
@@ -29,9 +29,9 @@ export interface IslandAuth {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
export interface IslandConfigValues {
|
|
32
|
-
/** cloud
|
|
32
|
+
/** cloud core service base URL (defaults resolved by the cloud client). */
|
|
33
33
|
coreUrl?: string
|
|
34
|
-
/** cloud
|
|
34
|
+
/** cloud runtime service base URL. */
|
|
35
35
|
runtimeUrl?: string
|
|
36
36
|
/** OEM identifier (Glassly is OEM #0); reserved for OEM auth/telemetry. */
|
|
37
37
|
oemId?: string
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Failure mode this protects:
|
|
7
7
|
* 1. Local captions subscribes to transcription.
|
|
8
|
-
* 2. The first Cloud
|
|
8
|
+
* 2. The first Cloud `PUT /api/audio/subscriptions` throws while the socket
|
|
9
9
|
* is reconnecting or Core is briefly unreachable.
|
|
10
10
|
* 3. A reconnect/resync recomputes the same desired subscription set.
|
|
11
11
|
*
|
|
12
12
|
* If we mark the set as "last applied" before the PUT succeeds, step 3 is
|
|
13
|
-
* incorrectly deduped and captions can wait forever even though Cloud
|
|
13
|
+
* incorrectly deduped and captions can wait forever even though Cloud later
|
|
14
14
|
* reconnects. So `begin()` only blocks exact duplicates while a write is in
|
|
15
15
|
* flight; only `succeeded()` promotes that key to the applied state.
|
|
16
16
|
*/
|
|
@@ -35,7 +35,7 @@ import {storage as mmkvStorage} from "../utils/storage"
|
|
|
35
35
|
import {useCloudClientStatusStore} from "../stores/cloudClientStatus"
|
|
36
36
|
import {islandNotifications} from "./NotificationsEmitter"
|
|
37
37
|
import {BgTimer} from "../utils/timers"
|
|
38
|
-
import {
|
|
38
|
+
import {logCloudTranscriptMetric} from "./CloudTranscriptE2EMetrics"
|
|
39
39
|
import {isLocalUserId, LocalMiniappUserIdentity} from "./LocalMiniappUserIdentity"
|
|
40
40
|
import {migrateLocalMiniappData, type LocalMiniappDataMigrationBackend} from "./LocalMiniappDataMigration"
|
|
41
41
|
import {nativeHttpResponseBody} from "./NativeHttpResponse"
|
|
@@ -580,7 +580,7 @@ function construct(): void {
|
|
|
580
580
|
})
|
|
581
581
|
transcriptUnsubscribe = c.runtime.onTranscript((data) => {
|
|
582
582
|
if (c !== client) return
|
|
583
|
-
|
|
583
|
+
logCloudTranscriptMetric(data)
|
|
584
584
|
emitTranscript(data)
|
|
585
585
|
})
|
|
586
586
|
translationUnsubscribe = c.runtime.onTranslation((data) => {
|
|
@@ -857,7 +857,7 @@ export const cloudClientService = {
|
|
|
857
857
|
stopDisableCloudWatcher()
|
|
858
858
|
},
|
|
859
859
|
|
|
860
|
-
/** Device-side managed photo
|
|
860
|
+
/** Device-side managed photo: presign now, deliver bytes, await ready. */
|
|
861
861
|
startManagedPhoto(opts: Record<string, unknown> = {}) {
|
|
862
862
|
if (!client) throw new Error("cloud client not connected")
|
|
863
863
|
return client.runtime.startManagedPhoto(opts)
|
|
@@ -867,7 +867,7 @@ export const cloudClientService = {
|
|
|
867
867
|
return client.runtime.awaitManagedPhotoReady(requestId)
|
|
868
868
|
},
|
|
869
869
|
|
|
870
|
-
/** Managed stream
|
|
870
|
+
/** Managed stream: provision ingest+playback on the runtime. */
|
|
871
871
|
startManagedStream(opts: Record<string, unknown> = {}) {
|
|
872
872
|
if (!client) throw new Error("cloud client not connected")
|
|
873
873
|
return client.runtime.startManagedStream(opts)
|
|
@@ -2,7 +2,7 @@ import type {TranscriptionData} from "@glassly/cloud-protocol"
|
|
|
2
2
|
|
|
3
3
|
import {buildE2EMetric, logE2EMetricPayload} from "../utils/e2eMetrics"
|
|
4
4
|
|
|
5
|
-
export function
|
|
5
|
+
export function buildCloudTranscriptMetric(data: TranscriptionData): Record<string, unknown> {
|
|
6
6
|
return buildE2EMetric("cloud_v2_transcript", {
|
|
7
7
|
text: data.text,
|
|
8
8
|
state: data.isFinal ? "final" : "interim",
|
|
@@ -22,6 +22,6 @@ export function buildCloudV2TranscriptMetric(data: TranscriptionData): Record<st
|
|
|
22
22
|
})
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
export function
|
|
26
|
-
logE2EMetricPayload(
|
|
25
|
+
export function logCloudTranscriptMetric(data: TranscriptionData): void {
|
|
26
|
+
logE2EMetricPayload(buildCloudTranscriptMetric(data), "cloud_v2_transcript")
|
|
27
27
|
}
|
|
@@ -35,6 +35,7 @@ import {retirePendingSelectionOnPromotion} from "./PairingIdentity"
|
|
|
35
35
|
import GlobalEventEmitter from "../utils/GlobalEventEmitter"
|
|
36
36
|
import {asgCameraApi} from "./asg/asgCameraApi"
|
|
37
37
|
import {dispatchTouchEvent, resetTouchInput} from "./GlassesTouchInput"
|
|
38
|
+
import neuralBandInput from "./neuralBand/NeuralBandInput"
|
|
38
39
|
import dashboardOpenTrace from "./debug/DashboardOpenTrace"
|
|
39
40
|
|
|
40
41
|
let subs: Array<{remove: () => void}> = []
|
|
@@ -233,6 +234,10 @@ export function startDeviceEventRouter(): void {
|
|
|
233
234
|
// Physical keyboard bridged through G2 Glassly CFW: raw HID transitions →
|
|
234
235
|
// layout-translated keyboard_event stream (with synthesized auto-repeat).
|
|
235
236
|
subs.push(BluetoothSdk.addListener("keyboard_key", (event) => glassesKeyboardInput.handle(event)))
|
|
237
|
+
// Meta Neural Band: its gestures ride touch_event above; these are its own streams.
|
|
238
|
+
subs.push(BluetoothSdk.addListener("neural_band_claim", (event) => neuralBandInput.handleClaim(event)))
|
|
239
|
+
subs.push(BluetoothSdk.addListener("neural_band_handwriting", (event) => neuralBandInput.handleHandwriting(event)))
|
|
240
|
+
subs.push(BluetoothSdk.addListener("neural_band_pointer", (event) => neuralBandInput.handlePointer(event)))
|
|
236
241
|
subs.push({
|
|
237
242
|
remove: useGlassesStore.subscribe(
|
|
238
243
|
(s) => s.keyboardConnected === true,
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Gesture dots — a debug indicator that flashes a small glyph top-right on the
|
|
3
3
|
* glasses when a touch gesture arrives, while nothing else is on the display:
|
|
4
|
-
* a filled circle for a single tap, an arrow for a swipe
|
|
5
|
-
* short pulse), a bullseye
|
|
4
|
+
* a filled circle for a single tap, two for a double tap, an arrow for a swipe
|
|
5
|
+
* (cleared after a short pulse), a bullseye for a recognized long press.
|
|
6
|
+
* The Neural Band's middle finger (index 1) draws the same shapes hollow. Extracted from
|
|
6
7
|
* the CFW dashboard service; it renders through
|
|
7
8
|
* the ordinary scene pipeline, so it works on any positioning device (stock
|
|
8
9
|
* G2 included), and is gated by the gesture_dots debug setting.
|
|
@@ -25,20 +26,27 @@ const DOTS_APP_ID = "system.gesture-dots"
|
|
|
25
26
|
const GESTURE_FLASH_MS = 700
|
|
26
27
|
const MARGIN_X = 16
|
|
27
28
|
const MARGIN_TOP = 10
|
|
28
|
-
|
|
29
|
-
const
|
|
29
|
+
/** Gestures that flash a glyph. */
|
|
30
|
+
const FLASH_GLYPHS: Record<string, GestureDotGlyph> = {
|
|
31
|
+
single_tap: "dot",
|
|
32
|
+
double_tap: "double",
|
|
33
|
+
long_press: "hold",
|
|
34
|
+
swipe_up: "up",
|
|
35
|
+
swipe_down: "down",
|
|
36
|
+
swipe_left: "left",
|
|
37
|
+
swipe_right: "right",
|
|
38
|
+
}
|
|
39
|
+
/** The band's middle finger draws the filled shapes hollow. */
|
|
40
|
+
const MIDDLE_FINGER_GLYPHS: Partial<Record<GestureDotGlyph, GestureDotGlyph>> = {dot: "ring", double: "doubleRing"}
|
|
30
41
|
|
|
31
42
|
class GestureDotsService {
|
|
32
43
|
private started = false
|
|
33
44
|
private flash: GestureDotGlyph | null = null
|
|
34
45
|
private flashTimer: number | null = null
|
|
35
|
-
/**
|
|
36
|
-
private
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
-
* event; the tap_end that follows on release is what clears it).
|
|
40
|
-
*/
|
|
41
|
-
private holdRecognized = false
|
|
46
|
+
/** Raw controller holds survive hold recognition/dial events until their own release. */
|
|
47
|
+
private presses = new Map<number, {raw: boolean; glyph: GestureDotGlyph}>()
|
|
48
|
+
/** Avoid sending identical images for successive phases of the same press. */
|
|
49
|
+
private renderedGlyph: GestureDotGlyph | null = null
|
|
42
50
|
/** Whether the indicator scene currently has anything on it (needs a clear when the flash ends). */
|
|
43
51
|
private visible = false
|
|
44
52
|
private unsubs: Array<() => void> = []
|
|
@@ -75,14 +83,14 @@ class GestureDotsService {
|
|
|
75
83
|
if (this.flashTimer !== null) BgTimer.clearTimeout(this.flashTimer)
|
|
76
84
|
this.flashTimer = null
|
|
77
85
|
this.flash = null
|
|
78
|
-
this.
|
|
79
|
-
this.holdRecognized = false
|
|
86
|
+
this.presses.clear()
|
|
80
87
|
}
|
|
81
88
|
|
|
82
89
|
/** Another surface owns the display; forget the indicator without painting over it. */
|
|
83
90
|
private relinquish(): void {
|
|
84
91
|
this.resetGesture()
|
|
85
92
|
this.visible = false
|
|
93
|
+
this.renderedGlyph = null
|
|
86
94
|
sceneRenderer.clearApp(DOTS_APP_ID)
|
|
87
95
|
this.trace("relinquish")
|
|
88
96
|
}
|
|
@@ -94,15 +102,16 @@ class GestureDotsService {
|
|
|
94
102
|
/**
|
|
95
103
|
* Flash a gesture on the indicator. Other gesture names are ignored.
|
|
96
104
|
* Claimed gestures cancel any earlier press/flash without starting a new
|
|
97
|
-
* one: the claimant may also consume the release edge.
|
|
105
|
+
* one: the claimant may also consume the release edge. `fingerIndex` is the
|
|
106
|
+
* Neural Band's finger on its pinches and press edges (1 = middle).
|
|
98
107
|
*/
|
|
99
|
-
noteGesture(gestureName: string, claimed = false): void {
|
|
108
|
+
noteGesture(gestureName: string, claimed = false, fingerIndex?: number): void {
|
|
100
109
|
if (!this.started || !this.enabled()) return
|
|
101
|
-
this.applyGesture(gestureName, claimed)
|
|
110
|
+
this.applyGesture(gestureName, claimed, fingerIndex)
|
|
102
111
|
this.trace("input", {gestureName, claimed})
|
|
103
112
|
}
|
|
104
113
|
|
|
105
|
-
private applyGesture(gestureName: string, claimed: boolean): void {
|
|
114
|
+
private applyGesture(gestureName: string, claimed: boolean, fingerIndex?: number): void {
|
|
106
115
|
if (localDisplayManager.hasActiveDisplay() || useDisplayStore.getState().overlayOwner !== null) {
|
|
107
116
|
this.relinquish()
|
|
108
117
|
return
|
|
@@ -112,49 +121,59 @@ class GestureDotsService {
|
|
|
112
121
|
this.render()
|
|
113
122
|
return
|
|
114
123
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
+
const finger = fingerIndex ?? 0
|
|
125
|
+
const press = this.presses.get(finger)
|
|
126
|
+
if (gestureName === "touch_down" || gestureName === "tap_start") {
|
|
127
|
+
if (press) {
|
|
128
|
+
// Both reports describe one press. Never restart its visual feedback.
|
|
129
|
+
if (gestureName === "touch_down") press.raw = true
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
this.clearFlash()
|
|
133
|
+
this.presses.set(finger, {raw: gestureName === "touch_down", glyph: finger === 1 ? "ring" : "dot"})
|
|
124
134
|
this.render()
|
|
125
135
|
return
|
|
126
136
|
}
|
|
127
|
-
if (gestureName === "
|
|
128
|
-
|
|
129
|
-
this.
|
|
137
|
+
if (gestureName === "tap_end") {
|
|
138
|
+
if (!press) return
|
|
139
|
+
this.presses.delete(finger)
|
|
140
|
+
// The band's resolved tap arrives after release (double-tap window).
|
|
141
|
+
// Keep the glyph lit across that gap so one pinch never flashes twice.
|
|
142
|
+
if (press.raw && this.presses.size === 0) this.startFlash(press.glyph)
|
|
130
143
|
this.render()
|
|
131
144
|
return
|
|
132
145
|
}
|
|
133
|
-
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
: gestureName === "swipe_up"
|
|
141
|
-
? "up"
|
|
142
|
-
: gestureName === "swipe_down"
|
|
143
|
-
? "down"
|
|
144
|
-
: null
|
|
145
|
-
if (!flash) {
|
|
146
|
-
if (released) this.render()
|
|
146
|
+
const shape = FLASH_GLYPHS[gestureName]
|
|
147
|
+
const glyph = shape && finger === 1 ? MIDDLE_FINGER_GLYPHS[shape] ?? shape : shape
|
|
148
|
+
if (press?.raw && (!glyph || gestureName === "long_press")) {
|
|
149
|
+
// Pinch-and-turn reports do not release a finger. A recognized hold
|
|
150
|
+
// stays visible until tap_end instead of expiring halfway through it.
|
|
151
|
+
if (glyph) press.glyph = glyph
|
|
152
|
+
this.render()
|
|
147
153
|
return
|
|
148
154
|
}
|
|
149
|
-
|
|
155
|
+
// Completed taps/swipes become one continuous pulse, even if the device
|
|
156
|
+
// omits tap_end. Older tap_start-only devices retain the same fallback.
|
|
157
|
+
this.presses.delete(finger)
|
|
158
|
+
if (glyph) this.startFlash(glyph)
|
|
159
|
+
this.render()
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private clearFlash(): void {
|
|
150
163
|
if (this.flashTimer !== null) BgTimer.clearTimeout(this.flashTimer)
|
|
164
|
+
this.flashTimer = null
|
|
165
|
+
this.flash = null
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
private startFlash(glyph: GestureDotGlyph): void {
|
|
169
|
+
this.clearFlash()
|
|
170
|
+
this.flash = glyph
|
|
151
171
|
this.flashTimer = BgTimer.setTimeout(() => {
|
|
152
172
|
this.flashTimer = null
|
|
153
173
|
this.flash = null
|
|
154
174
|
this.render()
|
|
155
175
|
this.trace("flash-ended")
|
|
156
176
|
}, GESTURE_FLASH_MS)
|
|
157
|
-
this.render()
|
|
158
177
|
}
|
|
159
178
|
|
|
160
179
|
/** Debug setting only; no payloads or retained history. Correlate seq with G2PERF. */
|
|
@@ -165,7 +184,7 @@ class GestureDotsService {
|
|
|
165
184
|
JSON.stringify({
|
|
166
185
|
t: Date.now(),
|
|
167
186
|
reason,
|
|
168
|
-
held: this.
|
|
187
|
+
held: this.presses.size > 0,
|
|
169
188
|
flash: this.flash,
|
|
170
189
|
overlay: useDisplayStore.getState().overlayOwner,
|
|
171
190
|
active: localDisplayManager.hasActiveDisplay(),
|
|
@@ -188,15 +207,18 @@ class GestureDotsService {
|
|
|
188
207
|
private render(): void {
|
|
189
208
|
if (!this.started) return
|
|
190
209
|
if (localDisplayManager.hasActiveDisplay() || useDisplayStore.getState().overlayOwner !== null) return
|
|
191
|
-
const shown =
|
|
210
|
+
const shown = Array.from(this.presses.values()).at(-1)?.glyph ?? this.flash
|
|
192
211
|
if (!this.enabled()) {
|
|
193
212
|
if (this.visible) {
|
|
194
213
|
this.visible = false
|
|
214
|
+
this.renderedGlyph = null
|
|
195
215
|
sceneRenderer.emitScene(DOTS_APP_ID, "main", [])
|
|
196
216
|
}
|
|
197
217
|
return
|
|
198
218
|
}
|
|
199
219
|
if (!shown && !this.visible) return
|
|
220
|
+
const glyph = shown ?? "blank"
|
|
221
|
+
if (glyph === this.renderedGlyph) return
|
|
200
222
|
const elements: SceneElementInput[] = [
|
|
201
223
|
{
|
|
202
224
|
type: "image",
|
|
@@ -208,13 +230,14 @@ class GestureDotsService {
|
|
|
208
230
|
h: GESTURE_DOT_GLYPH_SIZE,
|
|
209
231
|
anchor: "top-right",
|
|
210
232
|
},
|
|
211
|
-
data: GESTURE_DOT_GLYPHS[
|
|
233
|
+
data: GESTURE_DOT_GLYPHS[glyph],
|
|
212
234
|
},
|
|
213
235
|
]
|
|
214
236
|
this.visible = true
|
|
237
|
+
this.renderedGlyph = glyph
|
|
215
238
|
const result = sceneRenderer.emitScene(DOTS_APP_ID, "main", elements)
|
|
216
239
|
this.trace("render", {
|
|
217
|
-
glyph
|
|
240
|
+
glyph,
|
|
218
241
|
...(result?.kind === "scene" ? {seq: result.seq, sent: result.sent} : {}),
|
|
219
242
|
})
|
|
220
243
|
}
|
|
@@ -50,8 +50,11 @@ function shouldForwardScroll(event: TouchEvent): boolean {
|
|
|
50
50
|
export function dispatchTouchEvent(event: TouchEvent): void {
|
|
51
51
|
const gesture = event.gestureName
|
|
52
52
|
const claimant = localMiniappRuntime.gestureClaimant(gesture)
|
|
53
|
+
// The Neural Band's middle finger (index 1) shares the tap names but is left
|
|
54
|
+
// to miniapps: the engine's own double-tap behaviors are the index finger's.
|
|
55
|
+
const osDoubleTap = gesture === "double_tap" && event.index !== 1
|
|
53
56
|
|
|
54
|
-
if (
|
|
57
|
+
if (osDoubleTap) {
|
|
55
58
|
// A visible main miniapp opts out of OS exit by registering double tap (the
|
|
56
59
|
// terminal's back-to-menu). The latest claimant still receives it. Otherwise
|
|
57
60
|
// the tap means "leave the app". The controller shows the bounded OS
|
|
@@ -72,7 +75,7 @@ export function dispatchTouchEvent(event: TouchEvent): void {
|
|
|
72
75
|
// An overlay app claims the double tap on an empty screen to open the dashboard.
|
|
73
76
|
if (gesture === "double_tap" && isOverlayApp(claimant)) dashboardOpenTrace.trigger("double-tap", event.timestamp)
|
|
74
77
|
// The press may have started before this app claimed its release edge.
|
|
75
|
-
gestureDotsService.noteGesture(gesture, true)
|
|
78
|
+
gestureDotsService.noteGesture(gesture, true, event.index)
|
|
76
79
|
if (!shouldForwardScroll(event)) return
|
|
77
80
|
localMiniappRuntime.forwardEvent("touch_event", event, undefined, claimant)
|
|
78
81
|
return
|
|
@@ -81,7 +84,7 @@ export function dispatchTouchEvent(event: TouchEvent): void {
|
|
|
81
84
|
// Gesture-dots debug indicator (any device, gated by its setting). It
|
|
82
85
|
// only paints while the screen is otherwise empty. Process even suppressed
|
|
83
86
|
// scroll repeats here so their completion still clears a held dot.
|
|
84
|
-
gestureDotsService.noteGesture(gesture)
|
|
87
|
+
gestureDotsService.noteGesture(gesture, false, event.index)
|
|
85
88
|
|
|
86
89
|
if (!shouldForwardScroll(event)) return
|
|
87
90
|
localMiniappRuntime.forwardEvent("touch_event", event)
|
|
@@ -90,7 +93,7 @@ export function dispatchTouchEvent(event: TouchEvent): void {
|
|
|
90
93
|
// Glassly CFW glasses the dashboard miniapp claims it while it runs
|
|
91
94
|
// (ADR-0006); stock firmware keeps its native dashboard (ADR-0002). The
|
|
92
95
|
// modal nav UI owns the screen; showDashboard would fight it.
|
|
93
|
-
if (
|
|
96
|
+
if (osDoubleTap && !nativeNavigationService.isActive) {
|
|
94
97
|
useDisplayStore.getState().setDisplayEvent({layout: {layoutType: "unavailable", reason: "firmware-dashboard"}})
|
|
95
98
|
void Promise.resolve(BluetoothSdk.showDashboard()).catch((err) => {
|
|
96
99
|
console.error("DeviceEventRouter: showDashboard failed:", err)
|
|
@@ -113,7 +116,12 @@ export function sendGesture(
|
|
|
113
116
|
| "tap_then_hold"
|
|
114
117
|
| "touch_down"
|
|
115
118
|
| "swipe_up"
|
|
116
|
-
| "swipe_down"
|
|
119
|
+
| "swipe_down"
|
|
120
|
+
| "swipe_left"
|
|
121
|
+
| "swipe_right"
|
|
122
|
+
| "long_press"
|
|
123
|
+
| "dial_up"
|
|
124
|
+
| "dial_down",
|
|
117
125
|
): void {
|
|
118
126
|
const model = useGlassesStore.getState().deviceModel
|
|
119
127
|
dispatchTouchEvent({
|