@ateam-ai-mobile/expo 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/build.gradle +16 -0
- package/android/src/main/java/expo/modules/adavision/AdaVisionModule.kt +75 -0
- package/dist/attachmentCache-DhQTqi2E.d.mts +18 -0
- package/dist/attachmentCache-DhQTqi2E.d.ts +18 -0
- package/dist/bootstrap.d.mts +95 -0
- package/dist/bootstrap.d.ts +95 -0
- package/dist/bootstrap.js +2991 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/bootstrap.mjs +66 -0
- package/dist/bootstrap.mjs.map +1 -0
- package/dist/chunk-4BMWVN25.mjs +91 -0
- package/dist/chunk-4BMWVN25.mjs.map +1 -0
- package/dist/chunk-6LXF3JVX.mjs +96 -0
- package/dist/chunk-6LXF3JVX.mjs.map +1 -0
- package/dist/chunk-7FWYEHN5.mjs +203 -0
- package/dist/chunk-7FWYEHN5.mjs.map +1 -0
- package/dist/chunk-C7M5WAHA.mjs +1798 -0
- package/dist/chunk-C7M5WAHA.mjs.map +1 -0
- package/dist/chunk-FF6ZZZXH.mjs +71 -0
- package/dist/chunk-FF6ZZZXH.mjs.map +1 -0
- package/dist/chunk-GF2MPTAB.mjs +647 -0
- package/dist/chunk-GF2MPTAB.mjs.map +1 -0
- package/dist/chunk-J6XRNX7P.mjs +19 -0
- package/dist/chunk-J6XRNX7P.mjs.map +1 -0
- package/dist/chunk-KOXF2FZM.mjs +92 -0
- package/dist/chunk-KOXF2FZM.mjs.map +1 -0
- package/dist/chunk-L4V7OCJ3.mjs +71 -0
- package/dist/chunk-L4V7OCJ3.mjs.map +1 -0
- package/dist/chunk-PCAEUX2N.mjs +67 -0
- package/dist/chunk-PCAEUX2N.mjs.map +1 -0
- package/dist/chunk-WMJKH4XE.mjs +34 -0
- package/dist/chunk-WMJKH4XE.mjs.map +1 -0
- package/dist/chunk-XQSY6MHP.mjs +213 -0
- package/dist/chunk-XQSY6MHP.mjs.map +1 -0
- package/dist/chunk-Z5AI2SXX.mjs +1810 -0
- package/dist/chunk-Z5AI2SXX.mjs.map +1 -0
- package/dist/createBridge.js +41 -10
- package/dist/createBridge.js.map +1 -1
- package/dist/createBridge.mjs +7 -1
- package/dist/createBridge.mjs.map +1 -1
- package/dist/device/createDeviceProvider.d.mts +23 -0
- package/dist/device/createDeviceProvider.d.ts +23 -0
- package/dist/device/createDeviceProvider.js +1228 -0
- package/dist/device/createDeviceProvider.js.map +1 -0
- package/dist/device/createDeviceProvider.mjs +19 -0
- package/dist/device/createDeviceProvider.mjs.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.d.mts +10 -0
- package/dist/device-bridge/collectors/appEngagement.d.ts +10 -0
- package/dist/device-bridge/collectors/appEngagement.js +128 -0
- package/dist/device-bridge/collectors/appEngagement.js.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.mjs +12 -0
- package/dist/device-bridge/collectors/appEngagement.mjs.map +1 -0
- package/dist/device-bridge/collectors/health.d.mts +24 -0
- package/dist/device-bridge/collectors/health.d.ts +24 -0
- package/dist/device-bridge/collectors/health.js +238 -0
- package/dist/device-bridge/collectors/health.js.map +1 -0
- package/dist/device-bridge/collectors/health.mjs +18 -0
- package/dist/device-bridge/collectors/health.mjs.map +1 -0
- package/dist/device-bridge/cookies.d.mts +49 -0
- package/dist/device-bridge/cookies.d.ts +49 -0
- package/dist/device-bridge/cookies.js +90 -0
- package/dist/device-bridge/cookies.js.map +1 -0
- package/dist/device-bridge/cookies.mjs +14 -0
- package/dist/device-bridge/cookies.mjs.map +1 -0
- package/dist/device-bridge/index.d.mts +495 -0
- package/dist/device-bridge/index.d.ts +495 -0
- package/dist/device-bridge/index.js +2121 -0
- package/dist/device-bridge/index.js.map +1 -0
- package/dist/device-bridge/index.mjs +34 -0
- package/dist/device-bridge/index.mjs.map +1 -0
- package/dist/device-bridge/nfc.d.mts +30 -0
- package/dist/device-bridge/nfc.d.ts +30 -0
- package/dist/device-bridge/nfc.js +109 -0
- package/dist/device-bridge/nfc.js.map +1 -0
- package/dist/device-bridge/nfc.mjs +14 -0
- package/dist/device-bridge/nfc.mjs.map +1 -0
- package/dist/index.d.mts +509 -18
- package/dist/index.d.ts +509 -18
- package/dist/index.js +1706 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +24 -194
- package/dist/index.mjs.map +1 -1
- package/dist/maps/index.d.mts +56 -0
- package/dist/maps/index.d.ts +56 -0
- package/dist/maps/index.js +95 -0
- package/dist/maps/index.js.map +1 -0
- package/dist/maps/index.mjs +17 -0
- package/dist/maps/index.mjs.map +1 -0
- package/dist/oauth/expoOAuth.d.mts +18 -0
- package/dist/oauth/expoOAuth.d.ts +18 -0
- package/dist/oauth/expoOAuth.js +102 -0
- package/dist/oauth/expoOAuth.js.map +1 -0
- package/dist/oauth/expoOAuth.mjs +8 -0
- package/dist/oauth/expoOAuth.mjs.map +1 -0
- package/dist/types-BRanJYTn.d.mts +138 -0
- package/dist/types-BRanJYTn.d.ts +138 -0
- package/dist/vision/index.d.mts +83 -0
- package/dist/vision/index.d.ts +83 -0
- package/dist/vision/index.js +101 -0
- package/dist/vision/index.js.map +1 -0
- package/dist/vision/index.mjs +28 -0
- package/dist/vision/index.mjs.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AdaMaps/AdaMapsModule.swift +180 -0
- package/ios/AdaMaps.podspec +25 -0
- package/ios/AdaVision.podspec +32 -0
- package/ios/AdaVisionModule.swift +486 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/metadata.json +773 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/model.mil +720 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/weights/weight.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model1/coremldata.bin +0 -0
- package/package.json +113 -7
- package/src/args.ts +56 -0
- package/src/bootstrap.ts +159 -0
- package/src/camera/CameraTaskHost.tsx +175 -0
- package/src/camera/cameraTasks.ts +129 -0
- package/src/createBridge.ts +3 -1
- package/src/device/createDeviceProvider.ts +917 -0
- package/src/device-bridge/actions/calendar-actions.ts +217 -0
- package/src/device-bridge/actions/dnd.ts +74 -0
- package/src/device-bridge/actions/health-actions.ts +58 -0
- package/src/device-bridge/actions/index.ts +12 -0
- package/src/device-bridge/actions/messaging.ts +74 -0
- package/src/device-bridge/actions/navigation.ts +88 -0
- package/src/device-bridge/actions/queue-poller.ts +151 -0
- package/src/device-bridge/background.ts +137 -0
- package/src/device-bridge/biometrics.ts +64 -0
- package/src/device-bridge/capabilities.ts +218 -0
- package/src/device-bridge/collectors/appEngagement.ts +120 -0
- package/src/device-bridge/collectors/battery.ts +83 -0
- package/src/device-bridge/collectors/calendar.ts +162 -0
- package/src/device-bridge/collectors/camera.ts +27 -0
- package/src/device-bridge/collectors/connectivity.ts +60 -0
- package/src/device-bridge/collectors/contacts.ts +200 -0
- package/src/device-bridge/collectors/health.ts +299 -0
- package/src/device-bridge/collectors/index.ts +12 -0
- package/src/device-bridge/collectors/location.ts +236 -0
- package/src/device-bridge/collectors/mood.ts +141 -0
- package/src/device-bridge/collectors/notifications.ts +70 -0
- package/src/device-bridge/cookies.ts +89 -0
- package/src/device-bridge/dynamic-connector.ts +422 -0
- package/src/device-bridge/index.ts +467 -0
- package/src/device-bridge/nfc.ts +85 -0
- package/src/device-bridge/plugin.ts +71 -0
- package/src/device-bridge/relay-client.ts +237 -0
- package/src/device-bridge/scanner.ts +42 -0
- package/src/device-bridge/sharing.ts +35 -0
- package/src/device-bridge/types.ts +326 -0
- package/src/device-state/activityClassifier.ts +151 -0
- package/src/device-state/activityEngine.ts +252 -0
- package/src/device-state/collectors.ts +35 -0
- package/src/device-state/domains/activity.ts +93 -0
- package/src/device-state/domains/capability.ts +60 -0
- package/src/device-state/domains/connectivity.ts +60 -0
- package/src/device-state/domains/deviceContext.ts +57 -0
- package/src/device-state/domains/interaction.ts +60 -0
- package/src/device-state/domains/spatial.ts +367 -0
- package/src/device-state/domains/visual.ts +121 -0
- package/src/device-state/events.ts +49 -0
- package/src/device-state/headingSource.ts +119 -0
- package/src/device-state/index.ts +78 -0
- package/src/device-state/quality.ts +60 -0
- package/src/device-state/replay.ts +285 -0
- package/src/device-state/spatialEvents.ts +300 -0
- package/src/device-state/turnDetector.ts +121 -0
- package/src/device-state/visualEngine.ts +341 -0
- package/src/geo.ts +23 -0
- package/src/haptics.ts +4 -0
- package/src/index.ts +37 -0
- package/src/maps/index.ts +81 -0
- package/src/maps/normalize.ts +225 -0
- package/src/oauth/expoOAuth.tsx +113 -0
- package/src/vision/analysis.js +80 -0
- package/src/vision/index.ts +146 -0
- package/src/vision/ring.js +35 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Visual engine — a rolling frame timeline the Visual domain reads.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS IS SHAPED ODDLY. expo-camera can only produce frames through a
|
|
5
|
+
* MOUNTED `<CameraView>`; there is no headless capture API. So the engine is
|
|
6
|
+
* split in two:
|
|
7
|
+
*
|
|
8
|
+
* this module — the state: the ring, the session, the intent, the cadence.
|
|
9
|
+
* Pure TS, no React, so the domain and tests can use it.
|
|
10
|
+
* VisualEngineHost — a component the app mounts once. When a session is open
|
|
11
|
+
* it mounts the camera and feeds frames in; otherwise it
|
|
12
|
+
* renders nothing and the camera is off.
|
|
13
|
+
*
|
|
14
|
+
* The design's rule is that `getSnapshot()` must be a CACHED read that does not
|
|
15
|
+
* open the camera or take a picture — "avoid repeatedly starting the camera or
|
|
16
|
+
* capturing a new image for every plugin-tool call during an interactive
|
|
17
|
+
* session". The ring is what makes that possible: capture runs on the engine's
|
|
18
|
+
* own cadence, and reads serve whatever is already there.
|
|
19
|
+
*
|
|
20
|
+
* THE CADENCE IS THE ENGINE'S CHOICE, NOT THE PLUGIN'S. A plugin declares an
|
|
21
|
+
* INTENT — `navigation`, `document`, `ocr` — and this picks the interval. That
|
|
22
|
+
* is the whole point of the semantic layer: a caller asking for frames every
|
|
23
|
+
* 500ms is specifying an implementation, and cannot know the analysis latency,
|
|
24
|
+
* the battery state or the thermal headroom. Intent survives those changing;
|
|
25
|
+
* a number does not.
|
|
26
|
+
*
|
|
27
|
+
* EACH FRAME CARRIES ITS ORIENTATION — heading, read at capture time. Without
|
|
28
|
+
* it a frame cannot support a guidance instruction: "turn left at the corner"
|
|
29
|
+
* needs to know which way the lens faced. Reading it afterwards would pair a
|
|
30
|
+
* frame with a heading from after the user turned, which is how a guide
|
|
31
|
+
* describes the wrong junction with confidence.
|
|
32
|
+
*
|
|
33
|
+
* POSITION IS NOT ON THE FRAME, and that is the freeze's decision, not an
|
|
34
|
+
* omission. VisualFrameRef (API_FREEZE §4) carries `orientation` only; where
|
|
35
|
+
* the user IS belongs to Spatial. A caller that needs both correlates them by
|
|
36
|
+
* `timestamp` — both snapshots carry one. Duplicating position here would
|
|
37
|
+
* create a second copy that can disagree with Spatial's, which is precisely
|
|
38
|
+
* the two-descriptions-of-one-thing problem this whole layer exists to end.
|
|
39
|
+
*
|
|
40
|
+
* NOT IMPLEMENTED, and reported as `unknown`/null rather than guessed:
|
|
41
|
+
* per-frame blur, exposure and obstruction (need image analysis), and
|
|
42
|
+
* pitch/roll (need device-motion, not installed). A frame too dark to use must
|
|
43
|
+
* not be described as `exposure: 'good'` because nothing looked at it.
|
|
44
|
+
*/
|
|
45
|
+
import type { StateQuality } from '@ateam-ai-mobile/plugin-runtime';
|
|
46
|
+
|
|
47
|
+
/** A frame in the ring. `uri` is a local file the app can read; nothing is uploaded. */
|
|
48
|
+
export interface VisualFrame {
|
|
49
|
+
handle: { id: string; mimeType: string; expiresAt: number | null };
|
|
50
|
+
timestamp: number;
|
|
51
|
+
/**
|
|
52
|
+
* WHERE THE CAMERA WAS POINTING, stamped at capture time.
|
|
53
|
+
*
|
|
54
|
+
* A frame without this is nearly useless for guidance: "turn left at the
|
|
55
|
+
* corner" needs to know which way the lens faced and where the user stood.
|
|
56
|
+
* And it has to be captured WITH the frame, not read afterwards — by the
|
|
57
|
+
* time a snapshot is read the user has walked on, and pairing a frame with a
|
|
58
|
+
* later heading is how a guide describes the wrong junction confidently.
|
|
59
|
+
*
|
|
60
|
+
* Every field is nullable and null means NOT MEASURED. `headingDeg: 0` is due
|
|
61
|
+
* north, so a missing compass must not collapse to 0.
|
|
62
|
+
*/
|
|
63
|
+
orientation?: {
|
|
64
|
+
/** True heading if available, else magnetic. Null when the compass said nothing. */
|
|
65
|
+
headingDeg?: number | null;
|
|
66
|
+
/** Not measured — needs device-motion, which is not installed. */
|
|
67
|
+
pitchDeg?: number | null;
|
|
68
|
+
rollDeg?: number | null;
|
|
69
|
+
};
|
|
70
|
+
quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Supplies the spatial context stamped onto each frame.
|
|
75
|
+
*
|
|
76
|
+
* Injected rather than imported for the same reason the collectors are, and
|
|
77
|
+
* because it must be CHEAP: it is read once per capture, on the engine's
|
|
78
|
+
* cadence. Both getters return null rather than throwing — a frame with no
|
|
79
|
+
* heading is still a usable frame, and losing the picture because the compass
|
|
80
|
+
* was busy would be the wrong trade.
|
|
81
|
+
*/
|
|
82
|
+
export interface VisualContextProvider {
|
|
83
|
+
heading(): { headingDeg: number | null } | null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export type VisualIntentName =
|
|
87
|
+
| 'navigation' | 'document' | 'ocr' | 'shopping'
|
|
88
|
+
| 'inspection' | 'accessibility' | 'stream' | 'highQuality';
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Intent → capture cadence and window.
|
|
92
|
+
*
|
|
93
|
+
* These are the engine's judgement, not the freeze's — the docs say native
|
|
94
|
+
* owns sampling and deliberately do not fix numbers. Recorded here so the
|
|
95
|
+
* reasoning is visible and arguable rather than buried:
|
|
96
|
+
*
|
|
97
|
+
* - `navigation`: a walking user's scene changes continuously, so a steady
|
|
98
|
+
* cadence is right; ~1s is the ceiling at which a guidance instruction is
|
|
99
|
+
* still about the present. Faster mostly queues behind the analysis.
|
|
100
|
+
* - `document` / `ocr`: the subject is held still and the user is aiming. A
|
|
101
|
+
* slow cadence is enough, and a big window is pointless.
|
|
102
|
+
* - `stream`: the fastest we offer, for a caller that really is watching.
|
|
103
|
+
* - `highQuality`: rare, deliberate, single good frame.
|
|
104
|
+
*/
|
|
105
|
+
const PROFILES: Record<VisualIntentName, { intervalMs: number; windowMs: number }> = {
|
|
106
|
+
navigation: { intervalMs: 1000, windowMs: 5000 },
|
|
107
|
+
accessibility: { intervalMs: 1000, windowMs: 5000 },
|
|
108
|
+
stream: { intervalMs: 500, windowMs: 5000 },
|
|
109
|
+
inspection: { intervalMs: 2000, windowMs: 8000 },
|
|
110
|
+
shopping: { intervalMs: 2000, windowMs: 5000 },
|
|
111
|
+
document: { intervalMs: 3000, windowMs: 3000 },
|
|
112
|
+
ocr: { intervalMs: 3000, windowMs: 3000 },
|
|
113
|
+
highQuality: { intervalMs: 5000, windowMs: 5000 },
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export interface VisualSessionState {
|
|
117
|
+
id: string;
|
|
118
|
+
intent: VisualIntentName;
|
|
119
|
+
startedAt: number;
|
|
120
|
+
/** Set while the app is backgrounded. The session SURVIVES; the camera does not. */
|
|
121
|
+
suspended: boolean;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
type Listener = () => void;
|
|
125
|
+
|
|
126
|
+
class VisualEngine {
|
|
127
|
+
private frames: VisualFrame[] = [];
|
|
128
|
+
private session: VisualSessionState | null = null;
|
|
129
|
+
private listeners = new Set<Listener>();
|
|
130
|
+
/** Set by the host component while a camera is actually mounted and ready. */
|
|
131
|
+
private cameraReady = false;
|
|
132
|
+
/** Optional: supplies heading/position stamped onto each frame. */
|
|
133
|
+
private context: VisualContextProvider | null = null;
|
|
134
|
+
|
|
135
|
+
/** Host DI. Without it frames carry no orientation, and say so with nulls. */
|
|
136
|
+
setContextProvider(p: VisualContextProvider | null): void { this.context = p; }
|
|
137
|
+
|
|
138
|
+
// ── session ──
|
|
139
|
+
|
|
140
|
+
startSession(intent: VisualIntentName): VisualSessionState {
|
|
141
|
+
if (this.session) return this.session;
|
|
142
|
+
this.session = {
|
|
143
|
+
id: `visual_${Date.now()}`,
|
|
144
|
+
intent: PROFILES[intent] ? intent : 'navigation',
|
|
145
|
+
startedAt: Date.now(),
|
|
146
|
+
suspended: false,
|
|
147
|
+
};
|
|
148
|
+
this.notify();
|
|
149
|
+
return this.session;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
stopSession(): void {
|
|
153
|
+
this.session = null;
|
|
154
|
+
// The timeline belongs to the session. Keeping it would let a later read
|
|
155
|
+
// describe a room the user left.
|
|
156
|
+
this.frames = [];
|
|
157
|
+
this.cameraReady = false;
|
|
158
|
+
this.notify();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Background: suspend, do not end.
|
|
163
|
+
*
|
|
164
|
+
* iOS tears the camera down when the app leaves the foreground. The session
|
|
165
|
+
* is a STATE that survives that; what does not survive is the claim that the
|
|
166
|
+
* camera is live. While suspended the ring is kept but ages — a read gets
|
|
167
|
+
* old frames clearly marked old, rather than a fresh-looking lie.
|
|
168
|
+
*/
|
|
169
|
+
suspend(): void {
|
|
170
|
+
if (!this.session || this.session.suspended) return;
|
|
171
|
+
this.session.suspended = true;
|
|
172
|
+
this.cameraReady = false;
|
|
173
|
+
this.notify();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
resume(): void {
|
|
177
|
+
if (!this.session?.suspended) return;
|
|
178
|
+
this.session.suspended = false;
|
|
179
|
+
this.notify();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
getSession(): VisualSessionState | null { return this.session; }
|
|
183
|
+
|
|
184
|
+
/** Is a camera mounted and ready right now? Only the host knows. */
|
|
185
|
+
setCameraReady(ready: boolean): void {
|
|
186
|
+
this.cameraReady = ready;
|
|
187
|
+
this.notify();
|
|
188
|
+
}
|
|
189
|
+
isCameraReady(): boolean { return this.cameraReady && !this.session?.suspended; }
|
|
190
|
+
|
|
191
|
+
// ── the ring ──
|
|
192
|
+
|
|
193
|
+
profile() {
|
|
194
|
+
const intent = this.session?.intent || 'navigation';
|
|
195
|
+
return PROFILES[intent];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Feed from the native vision session (VisionSession, iOS) — the FAST frame
|
|
200
|
+
* source that replaces the takePictureAsync loop. The frame is IN-MEMORY: the
|
|
201
|
+
* native ring holds the pixels keyed by `id`, and native.vision resolves that
|
|
202
|
+
* id inside the trusted layer. No `uri` crosses here; the quality is a REAL
|
|
203
|
+
* measurement (not `unknown`), and `expiresAt` is the ring's real eviction time.
|
|
204
|
+
* The public VisualFrameRef strip to id-only is owned by the device-state session.
|
|
205
|
+
*/
|
|
206
|
+
pushFrameMeta(meta: {
|
|
207
|
+
id: string;
|
|
208
|
+
timestamp: number;
|
|
209
|
+
expiresAt?: number | null;
|
|
210
|
+
quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
|
|
211
|
+
}): void {
|
|
212
|
+
let heading: { headingDeg: number | null } | null = null;
|
|
213
|
+
try { heading = this.context?.heading() ?? null; } catch { /* a frame without a heading beats no frame */ }
|
|
214
|
+
|
|
215
|
+
this.frames.push({
|
|
216
|
+
// In-memory: no file. native.vision resolves the id inside the native ring.
|
|
217
|
+
handle: { id: meta.id, mimeType: 'image/none', expiresAt: meta.expiresAt ?? null },
|
|
218
|
+
timestamp: meta.timestamp,
|
|
219
|
+
orientation: { headingDeg: heading?.headingDeg ?? null, pitchDeg: null, rollDeg: null },
|
|
220
|
+
quality: meta.quality ?? { blur: 'unknown', exposure: 'unknown', obstructed: null },
|
|
221
|
+
});
|
|
222
|
+
this.evict();
|
|
223
|
+
this.notify();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// The native ring's actual retention (ms). The engine's effective window is
|
|
227
|
+
// min(intent window, retention) — set by whoever owns the clamp semantics; this
|
|
228
|
+
// just stores the input so that logic has one source of truth to read.
|
|
229
|
+
private _retentionMs: number | null = null;
|
|
230
|
+
setRetentionMs(ms: number | null): void { this._retentionMs = ms && ms > 0 ? ms : null; }
|
|
231
|
+
retentionMs(): number | null { return this._retentionMs; }
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* How far back the timeline ACTUALLY reaches: `min(intent window, native
|
|
235
|
+
* retention)`.
|
|
236
|
+
*
|
|
237
|
+
* The intent asks for a window; native decides what it can afford to keep.
|
|
238
|
+
* Full-res luma at ~24fps is tens of megabytes for a few seconds, so the ring
|
|
239
|
+
* is memory-bounded and shrinks under pressure — `retentionMs` is native
|
|
240
|
+
* publishing what it is really holding.
|
|
241
|
+
*
|
|
242
|
+
* EVERY FRAME IN THE TIMELINE CAN BE ANALYSED. That is the one sentence a
|
|
243
|
+
* plugin author has to hold, and this clamp is what makes it true. The
|
|
244
|
+
* alternative — listing frames past retention marked "not analysable" — was
|
|
245
|
+
* considered and rejected: a caller would have to check a flag before every
|
|
246
|
+
* use, and forgetting throws mid-guidance on a frame the model can see
|
|
247
|
+
* listed. Present-but-unusable is the same trap as an empty rowset reading
|
|
248
|
+
* as "no rows matched".
|
|
249
|
+
*
|
|
250
|
+
* If per-frame history beyond retention is ever needed, the shape is an
|
|
251
|
+
* AGGREGATE on the snapshot ("the view has been unusable for N ms"), never a
|
|
252
|
+
* per-entry flag.
|
|
253
|
+
*/
|
|
254
|
+
effectiveWindowMs(): number {
|
|
255
|
+
const intent = this.profile().windowMs;
|
|
256
|
+
const retention = this._retentionMs;
|
|
257
|
+
return retention === null ? intent : Math.min(intent, retention);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Time-based, not count-based: the design specifies a rolling WINDOW.
|
|
262
|
+
*
|
|
263
|
+
* Evicts on the EFFECTIVE window, so a frame whose pixels native has already
|
|
264
|
+
* dropped leaves the timeline at the same moment. Without this, `frameById`
|
|
265
|
+
* would start returning null for frames still being listed.
|
|
266
|
+
*/
|
|
267
|
+
private evict(): void {
|
|
268
|
+
const cutoff = Date.now() - this.effectiveWindowMs();
|
|
269
|
+
this.frames = this.frames.filter(f => f.timestamp >= cutoff);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
timeline(): { windowMs: number; frames: VisualFrame[] } {
|
|
273
|
+
this.evict();
|
|
274
|
+
// The EFFECTIVE window, never the intent's. Reporting the intent would have
|
|
275
|
+
// a guide reason about 8 seconds of history it does not have — precisely
|
|
276
|
+
// the confusion the clamp exists to prevent.
|
|
277
|
+
return { windowMs: this.effectiveWindowMs(), frames: [...this.frames] };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
latest(): VisualFrame | null {
|
|
281
|
+
this.evict();
|
|
282
|
+
return this.frames.length ? this.frames[this.frames.length - 1] : null;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Resolve a frame handle id (from a VisualFrameRef) back to the live frame so
|
|
287
|
+
* the mechanical layer — native.vision — can read its on-device file. The
|
|
288
|
+
* whole contract of that layer is that it takes an id, never pixels; this is
|
|
289
|
+
* where an id becomes a `handle.uri` it can open. Returns null when the id is
|
|
290
|
+
* unknown OR its frame has aged out of the rolling window (the cache file may
|
|
291
|
+
* also be gone by then) — a real "I can't look at that frame" answer, which
|
|
292
|
+
* the caller must treat as distinct from "I looked and found nothing".
|
|
293
|
+
*/
|
|
294
|
+
frameById(id: string): VisualFrame | null {
|
|
295
|
+
this.evict();
|
|
296
|
+
return this.frames.find(f => f.handle.id === id) ?? null;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Record a MEASURED quality back onto a frame (native.vision.assessQuality).
|
|
301
|
+
* Frames are minted with `quality: unknown` because nothing looks at the
|
|
302
|
+
* pixels at capture time; this is how a real measurement replaces that so a
|
|
303
|
+
* later read of VisualFrameRef.quality reflects it. No-op if the frame has
|
|
304
|
+
* aged out — we never resurrect an evicted frame just to label it.
|
|
305
|
+
*/
|
|
306
|
+
setFrameQuality(id: string, quality: NonNullable<VisualFrame['quality']>): void {
|
|
307
|
+
const f = this.frames.find(fr => fr.handle.id === id);
|
|
308
|
+
if (!f) return;
|
|
309
|
+
f.quality = quality;
|
|
310
|
+
this.notify();
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ── change notification ──
|
|
314
|
+
|
|
315
|
+
subscribe(fn: Listener): () => void {
|
|
316
|
+
this.listeners.add(fn);
|
|
317
|
+
return () => this.listeners.delete(fn);
|
|
318
|
+
}
|
|
319
|
+
private notify(): void {
|
|
320
|
+
for (const fn of this.listeners) { try { fn(); } catch { /* a bad listener is not the engine's problem */ } }
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** One engine per app. The camera is a singleton resource; so is this. */
|
|
325
|
+
export const visualEngine = new VisualEngine();
|
|
326
|
+
|
|
327
|
+
/** Freshness for a visual read — exported so the domain and host agree. */
|
|
328
|
+
export function frameQuality(frame: VisualFrame | null): StateQuality {
|
|
329
|
+
if (!frame) return { freshness: { state: 'unknown', observedAt: null, ageMs: null } };
|
|
330
|
+
const ageMs = Date.now() - frame.timestamp;
|
|
331
|
+
return {
|
|
332
|
+
freshness: {
|
|
333
|
+
// Tighter than the other domains on purpose: a two-second-old frame of a
|
|
334
|
+
// moving scene is not "fresh" in any useful sense.
|
|
335
|
+
state: ageMs < 2000 ? 'fresh' : ageMs < 10_000 ? 'aging' : 'stale',
|
|
336
|
+
observedAt: frame.timestamp,
|
|
337
|
+
ageMs,
|
|
338
|
+
},
|
|
339
|
+
confidence: { level: 'unknown' },
|
|
340
|
+
};
|
|
341
|
+
}
|
package/src/geo.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Great-circle distance — the SDK's one answer to "how far apart are these".
|
|
2
|
+
//
|
|
3
|
+
// Moved out of device-state/domains/spatial.ts when native.maps needed the same
|
|
4
|
+
// question answered (a search result's distance from `near`). Two haversines
|
|
5
|
+
// would be two answers that could drift; both callers import this one.
|
|
6
|
+
//
|
|
7
|
+
// No imports on purpose: it runs under plain `node --test` with the maps
|
|
8
|
+
// normaliser that uses it.
|
|
9
|
+
|
|
10
|
+
/** Metres along a path, great-circle per leg, rounded to the metre. */
|
|
11
|
+
export function distanceOf(path: ReadonlyArray<{ latitude: number; longitude: number }>): number {
|
|
12
|
+
const R = 6_371_000;
|
|
13
|
+
let total = 0;
|
|
14
|
+
for (let i = 1; i < path.length; i++) {
|
|
15
|
+
const a = path[i - 1], b = path[i];
|
|
16
|
+
const dLat = ((b.latitude - a.latitude) * Math.PI) / 180;
|
|
17
|
+
const dLon = ((b.longitude - a.longitude) * Math.PI) / 180;
|
|
18
|
+
const la1 = (a.latitude * Math.PI) / 180, la2 = (b.latitude * Math.PI) / 180;
|
|
19
|
+
const h = Math.sin(dLat / 2) ** 2 + Math.cos(la1) * Math.cos(la2) * Math.sin(dLon / 2) ** 2;
|
|
20
|
+
total += 2 * R * Math.asin(Math.sqrt(h));
|
|
21
|
+
}
|
|
22
|
+
return Math.round(total);
|
|
23
|
+
}
|
package/src/haptics.ts
CHANGED
|
@@ -24,4 +24,8 @@ export const expoHaptics: HapticsAPI = {
|
|
|
24
24
|
selection() {
|
|
25
25
|
Haptics.selectionAsync().catch(() => {});
|
|
26
26
|
},
|
|
27
|
+
// Shorthand for notification('error') — the form plugins actually call.
|
|
28
|
+
error() {
|
|
29
|
+
Haptics.notificationAsync(Haptics.NotificationFeedbackType.Error).catch(() => {});
|
|
30
|
+
},
|
|
27
31
|
};
|
package/src/index.ts
CHANGED
|
@@ -15,6 +15,8 @@ import {
|
|
|
15
15
|
setHapticsProvider,
|
|
16
16
|
setBundleStorage,
|
|
17
17
|
registerBundleModules,
|
|
18
|
+
setPluginNativeProvider,
|
|
19
|
+
type PluginNative,
|
|
18
20
|
} from '@ateam-ai-mobile/plugin-runtime';
|
|
19
21
|
import { setClipboardProvider } from '@ateam-ai-mobile/ui';
|
|
20
22
|
|
|
@@ -47,8 +49,18 @@ export function setupExpoProviders(opts?: {
|
|
|
47
49
|
* existing app so users keep their attachment previews / bundle cache. */
|
|
48
50
|
attachmentIndexKey?: string;
|
|
49
51
|
bundleIndexKey?: string;
|
|
52
|
+
/**
|
|
53
|
+
* The device-primitive implementation plugins receive as `props.device`.
|
|
54
|
+
* Build it with `createDeviceProvider(...)` from `@ateam-ai-mobile/expo/device`
|
|
55
|
+
* and pass it here so the SDK owns the implementation and this call registers
|
|
56
|
+
* it (setPluginNativeProvider). It is injected — not imported by this entry —
|
|
57
|
+
* because createDeviceProvider statically pulls optional expo modules that a
|
|
58
|
+
* host without them cannot build (same reason createBridge is a subpath).
|
|
59
|
+
*/
|
|
60
|
+
deviceProvider?: PluginNative;
|
|
50
61
|
}): { attachmentCache: ExpoAttachmentCache } {
|
|
51
62
|
setHapticsProvider(expoHaptics);
|
|
63
|
+
if (opts?.deviceProvider) setPluginNativeProvider(opts.deviceProvider);
|
|
52
64
|
setBundleStorage(new ExpoBundleStorage({ indexKey: opts?.bundleIndexKey }));
|
|
53
65
|
setClipboardProvider({
|
|
54
66
|
setString: (text: string) => ExpoClipboard.setStringAsync(text).then(() => {}),
|
|
@@ -65,3 +77,28 @@ export function setupExpoProviders(opts?: {
|
|
|
65
77
|
});
|
|
66
78
|
return { attachmentCache: new ExpoAttachmentCache({ indexKey: opts?.attachmentIndexKey }) };
|
|
67
79
|
}
|
|
80
|
+
|
|
81
|
+
// Device Bridge v2 — the semantic state engines this build provides.
|
|
82
|
+
// The host passes this to setDeviceStateProvider(); per-plugin gating by
|
|
83
|
+
// manifest.stateDomains happens in the SDK.
|
|
84
|
+
export { createDeviceState, IMPLEMENTED_DOMAINS } from './device-state';
|
|
85
|
+
export type { DeviceStateCollectors } from './device-state';
|
|
86
|
+
// The mounted half of the Visual engine. The app renders this ONCE; it draws
|
|
87
|
+
// nothing until a visual session is open. Required for deviceState.visual to
|
|
88
|
+
// produce frames at all — expo-camera has no headless capture.
|
|
89
|
+
// The rolling frame timeline. Exposed so the host's native.vision layer can
|
|
90
|
+
// resolve a frame id to its on-device file (frameById) and write a measured
|
|
91
|
+
// quality back onto the frame (setFrameQuality) — see visualEngine.ts.
|
|
92
|
+
export { visualEngine } from './device-state/visualEngine';
|
|
93
|
+
export type { VisualFrame } from './device-state/visualEngine';
|
|
94
|
+
// The mounted camera surface behind scanner.scanOnce() and
|
|
95
|
+
// camera.recordVideo(). Mount ONCE near the app root; it renders nothing
|
|
96
|
+
// until a plugin asks for a scan or a recording. Without it both APIs
|
|
97
|
+
// reject immediately with UNAVAILABLE — deliberately, since a plugin must
|
|
98
|
+
// never await a promise nothing can settle.
|
|
99
|
+
export { CameraTaskHost } from './camera/CameraTaskHost';
|
|
100
|
+
export { cameraTasks } from './camera/cameraTasks';
|
|
101
|
+
// Replay — run guidance logic against a RECORDED WALK instead of a phone.
|
|
102
|
+
// See docs/SPEC_device_walk_recorder.md for the fixture format.
|
|
103
|
+
export { createReplayDeviceState, parseWalkJsonl } from './device-state/replay';
|
|
104
|
+
export type { WalkFixture, WalkSample, WalkRefusal, ReplayClock, ReplaySession } from './device-state/replay';
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// ada-maps — place search + walking directions (Apple MapKit), our own native
|
|
2
|
+
// code, plus the SDK's one reverse geocode.
|
|
3
|
+
//
|
|
4
|
+
// NOT on-device — the opposite of ada-vision. Every call sends the query and the
|
|
5
|
+
// coordinates it is given to a map service over the network. None of it reads
|
|
6
|
+
// the device's location: the caller supplies every coordinate.
|
|
7
|
+
//
|
|
8
|
+
// search(...) — MKLocalSearch (AdaMaps module, iOS only)
|
|
9
|
+
// route(...) — MKDirections (AdaMaps module, iOS only)
|
|
10
|
+
// reverseGeocode(at) — expo-location's reverseGeocodeAsync, which IS CLGeocoder
|
|
11
|
+
// on iOS and the platform Geocoder on Android. No Swift for
|
|
12
|
+
// it, and the ONLY reverse geocode in the SDK: the location
|
|
13
|
+
// collector and the connector bridge call this too.
|
|
14
|
+
//
|
|
15
|
+
// requireOptionalNativeModule: on a binary WITHOUT the module the import is null,
|
|
16
|
+
// isSupported() is false, and search/route refuse cleanly (no crash). Android has
|
|
17
|
+
// no AdaMaps at all, so it refuses by name the same way.
|
|
18
|
+
|
|
19
|
+
import { Platform } from 'react-native';
|
|
20
|
+
import { requireOptionalNativeModule } from 'expo-modules-core';
|
|
21
|
+
import * as ExpoLocation from 'expo-location';
|
|
22
|
+
import type { MapsAddress, MapsCoordinate } from '@ateam-ai-mobile/plugin-runtime';
|
|
23
|
+
import { toMapsAddress } from './normalize';
|
|
24
|
+
import type { RawMapsPlace, RawMapsRoute } from './normalize';
|
|
25
|
+
|
|
26
|
+
interface AdaMapsNative {
|
|
27
|
+
isSupported(): boolean;
|
|
28
|
+
search(query: string, lat: number, lng: number, hasNear: boolean, radiusM: number, limit: number): Promise<RawMapsPlace[]>;
|
|
29
|
+
route(fromLat: number, fromLng: number, toLat: number, toLng: number): Promise<RawMapsRoute | null>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const native = requireOptionalNativeModule<AdaMapsNative>('AdaMaps');
|
|
33
|
+
|
|
34
|
+
/** What search/route are missing in THIS build, as the clause after "needs" —
|
|
35
|
+
* or null when they can run. A voice assistant reads it out, so it names the
|
|
36
|
+
* missing thing rather than giving a code. */
|
|
37
|
+
export function missingReason(): string | null {
|
|
38
|
+
if (Platform.OS !== 'ios') return `iOS (Apple MapKit) — ${Platform.OS} has no implementation yet`;
|
|
39
|
+
if (!native) return 'the app build with the AdaMaps module (this build predates it — a native rebuild is needed)';
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Is the native maps module present in this binary (and on a platform it runs on)? */
|
|
44
|
+
export function isSupported(): boolean {
|
|
45
|
+
try { return missingReason() === null && (native?.isSupported() ?? false); } catch { return false; }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// search/route hand their arguments to native AS GIVEN: a NaN or Infinity
|
|
49
|
+
// `limit` traps Swift's Int conversion and kills the app. createMapsAPI checks
|
|
50
|
+
// every argument first (searchArgs / coordinateArg in ./normalize); a host
|
|
51
|
+
// calling these directly must pass what those checks would accept.
|
|
52
|
+
|
|
53
|
+
/** MKLocalSearch around `near` (when given). [] = looked, found nothing; throws =
|
|
54
|
+
* couldn't look, with a string code OFFLINE / THROTTLED / UNAVAILABLE. */
|
|
55
|
+
export async function search(
|
|
56
|
+
query: string,
|
|
57
|
+
near: MapsCoordinate | null,
|
|
58
|
+
radiusM: number,
|
|
59
|
+
limit: number,
|
|
60
|
+
): Promise<RawMapsPlace[]> {
|
|
61
|
+
if (!native) throw new Error('ada-maps native module not present in this build');
|
|
62
|
+
// Without `near`, hasNear is false and Swift never reads lat/lng — the zeros
|
|
63
|
+
// are placeholders for a primitive argument, not a place.
|
|
64
|
+
return near
|
|
65
|
+
? native.search(query, near.latitude, near.longitude, true, radiusM, limit)
|
|
66
|
+
: native.search(query, 0, 0, false, radiusM, limit);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** MKDirections, walking, between two caller-supplied points. null = no route. */
|
|
70
|
+
export async function route(from: MapsCoordinate, to: MapsCoordinate): Promise<RawMapsRoute | null> {
|
|
71
|
+
if (!native) throw new Error('ada-maps native module not present in this build');
|
|
72
|
+
return native.route(from.latitude, from.longitude, to.latitude, to.longitude);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The address at a coordinate, or null when the geocoder has nothing there.
|
|
76
|
+
* Throws when it could not look (network, rate limit, Android without location
|
|
77
|
+
* permission) — callers decide whether that is fatal. */
|
|
78
|
+
export async function reverseGeocode(at: MapsCoordinate): Promise<MapsAddress | null> {
|
|
79
|
+
const [first] = await ExpoLocation.reverseGeocodeAsync({ latitude: at.latitude, longitude: at.longitude });
|
|
80
|
+
return first ? toMapsAddress(first) : null;
|
|
81
|
+
}
|