@ateam-ai-mobile/expo 0.1.4 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/android/build.gradle +16 -0
  2. package/android/src/main/java/expo/modules/adavision/AdaVisionModule.kt +75 -0
  3. package/dist/attachmentCache-DhQTqi2E.d.mts +18 -0
  4. package/dist/attachmentCache-DhQTqi2E.d.ts +18 -0
  5. package/dist/bootstrap.d.mts +95 -0
  6. package/dist/bootstrap.d.ts +95 -0
  7. package/dist/bootstrap.js +2473 -0
  8. package/dist/bootstrap.js.map +1 -0
  9. package/dist/bootstrap.mjs +64 -0
  10. package/dist/bootstrap.mjs.map +1 -0
  11. package/dist/chunk-4BMWVN25.mjs +91 -0
  12. package/dist/chunk-4BMWVN25.mjs.map +1 -0
  13. package/dist/chunk-4XYXJYN5.mjs +1513 -0
  14. package/dist/chunk-4XYXJYN5.mjs.map +1 -0
  15. package/dist/chunk-5HM6UA2K.mjs +571 -0
  16. package/dist/chunk-5HM6UA2K.mjs.map +1 -0
  17. package/dist/chunk-6LXF3JVX.mjs +96 -0
  18. package/dist/chunk-6LXF3JVX.mjs.map +1 -0
  19. package/dist/chunk-FF6ZZZXH.mjs +71 -0
  20. package/dist/chunk-FF6ZZZXH.mjs.map +1 -0
  21. package/dist/chunk-KOXF2FZM.mjs +92 -0
  22. package/dist/chunk-KOXF2FZM.mjs.map +1 -0
  23. package/dist/chunk-L4V7OCJ3.mjs +71 -0
  24. package/dist/chunk-L4V7OCJ3.mjs.map +1 -0
  25. package/dist/chunk-TZXWVYYR.mjs +67 -0
  26. package/dist/chunk-TZXWVYYR.mjs.map +1 -0
  27. package/dist/chunk-VQJPSLHP.mjs +1801 -0
  28. package/dist/chunk-VQJPSLHP.mjs.map +1 -0
  29. package/dist/chunk-WMJKH4XE.mjs +34 -0
  30. package/dist/chunk-WMJKH4XE.mjs.map +1 -0
  31. package/dist/chunk-XQSY6MHP.mjs +213 -0
  32. package/dist/chunk-XQSY6MHP.mjs.map +1 -0
  33. package/dist/createBridge.mjs +2 -0
  34. package/dist/createBridge.mjs.map +1 -1
  35. package/dist/device/createDeviceProvider.d.mts +23 -0
  36. package/dist/device/createDeviceProvider.d.ts +23 -0
  37. package/dist/device/createDeviceProvider.js +967 -0
  38. package/dist/device/createDeviceProvider.js.map +1 -0
  39. package/dist/device/createDeviceProvider.mjs +17 -0
  40. package/dist/device/createDeviceProvider.mjs.map +1 -0
  41. package/dist/device-bridge/collectors/appEngagement.d.mts +10 -0
  42. package/dist/device-bridge/collectors/appEngagement.d.ts +10 -0
  43. package/dist/device-bridge/collectors/appEngagement.js +128 -0
  44. package/dist/device-bridge/collectors/appEngagement.js.map +1 -0
  45. package/dist/device-bridge/collectors/appEngagement.mjs +12 -0
  46. package/dist/device-bridge/collectors/appEngagement.mjs.map +1 -0
  47. package/dist/device-bridge/collectors/health.d.mts +24 -0
  48. package/dist/device-bridge/collectors/health.d.ts +24 -0
  49. package/dist/device-bridge/collectors/health.js +238 -0
  50. package/dist/device-bridge/collectors/health.js.map +1 -0
  51. package/dist/device-bridge/collectors/health.mjs +18 -0
  52. package/dist/device-bridge/collectors/health.mjs.map +1 -0
  53. package/dist/device-bridge/cookies.d.mts +49 -0
  54. package/dist/device-bridge/cookies.d.ts +49 -0
  55. package/dist/device-bridge/cookies.js +90 -0
  56. package/dist/device-bridge/cookies.js.map +1 -0
  57. package/dist/device-bridge/cookies.mjs +14 -0
  58. package/dist/device-bridge/cookies.mjs.map +1 -0
  59. package/dist/device-bridge/index.d.mts +483 -0
  60. package/dist/device-bridge/index.d.ts +483 -0
  61. package/dist/device-bridge/index.js +2084 -0
  62. package/dist/device-bridge/index.js.map +1 -0
  63. package/dist/device-bridge/index.mjs +32 -0
  64. package/dist/device-bridge/index.mjs.map +1 -0
  65. package/dist/device-bridge/nfc.d.mts +30 -0
  66. package/dist/device-bridge/nfc.d.ts +30 -0
  67. package/dist/device-bridge/nfc.js +109 -0
  68. package/dist/device-bridge/nfc.js.map +1 -0
  69. package/dist/device-bridge/nfc.mjs +14 -0
  70. package/dist/device-bridge/nfc.mjs.map +1 -0
  71. package/dist/index.d.mts +493 -18
  72. package/dist/index.d.ts +493 -18
  73. package/dist/index.js +1409 -6
  74. package/dist/index.js.map +1 -1
  75. package/dist/index.mjs +23 -194
  76. package/dist/index.mjs.map +1 -1
  77. package/dist/oauth/expoOAuth.d.mts +18 -0
  78. package/dist/oauth/expoOAuth.d.ts +18 -0
  79. package/dist/oauth/expoOAuth.js +102 -0
  80. package/dist/oauth/expoOAuth.js.map +1 -0
  81. package/dist/oauth/expoOAuth.mjs +8 -0
  82. package/dist/oauth/expoOAuth.mjs.map +1 -0
  83. package/dist/types-BRanJYTn.d.mts +138 -0
  84. package/dist/types-BRanJYTn.d.ts +138 -0
  85. package/dist/vision/index.d.mts +81 -0
  86. package/dist/vision/index.d.ts +81 -0
  87. package/dist/vision/index.js +101 -0
  88. package/dist/vision/index.js.map +1 -0
  89. package/dist/vision/index.mjs +28 -0
  90. package/dist/vision/index.mjs.map +1 -0
  91. package/expo-module.config.json +9 -0
  92. package/ios/AdaVision.podspec +28 -0
  93. package/ios/AdaVisionModule.swift +432 -0
  94. package/ios/Resources/yolov8n-oiv7.mlmodelc/analytics/coremldata.bin +0 -0
  95. package/ios/Resources/yolov8n-oiv7.mlmodelc/coremldata.bin +0 -0
  96. package/ios/Resources/yolov8n-oiv7.mlmodelc/metadata.json +773 -0
  97. package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/analytics/coremldata.bin +0 -0
  98. package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/coremldata.bin +0 -0
  99. package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/model.mil +720 -0
  100. package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/weights/weight.bin +0 -0
  101. package/ios/Resources/yolov8n-oiv7.mlmodelc/model1/coremldata.bin +0 -0
  102. package/package.json +107 -7
  103. package/src/bootstrap.ts +159 -0
  104. package/src/camera/CameraTaskHost.tsx +175 -0
  105. package/src/camera/cameraTasks.ts +129 -0
  106. package/src/device/createDeviceProvider.ts +832 -0
  107. package/src/device-bridge/actions/calendar-actions.ts +217 -0
  108. package/src/device-bridge/actions/dnd.ts +74 -0
  109. package/src/device-bridge/actions/health-actions.ts +58 -0
  110. package/src/device-bridge/actions/index.ts +12 -0
  111. package/src/device-bridge/actions/messaging.ts +74 -0
  112. package/src/device-bridge/actions/navigation.ts +88 -0
  113. package/src/device-bridge/actions/queue-poller.ts +151 -0
  114. package/src/device-bridge/background.ts +137 -0
  115. package/src/device-bridge/biometrics.ts +64 -0
  116. package/src/device-bridge/capabilities.ts +218 -0
  117. package/src/device-bridge/collectors/appEngagement.ts +120 -0
  118. package/src/device-bridge/collectors/battery.ts +83 -0
  119. package/src/device-bridge/collectors/calendar.ts +162 -0
  120. package/src/device-bridge/collectors/camera.ts +27 -0
  121. package/src/device-bridge/collectors/connectivity.ts +60 -0
  122. package/src/device-bridge/collectors/contacts.ts +200 -0
  123. package/src/device-bridge/collectors/health.ts +299 -0
  124. package/src/device-bridge/collectors/index.ts +12 -0
  125. package/src/device-bridge/collectors/location.ts +224 -0
  126. package/src/device-bridge/collectors/mood.ts +141 -0
  127. package/src/device-bridge/collectors/notifications.ts +70 -0
  128. package/src/device-bridge/cookies.ts +89 -0
  129. package/src/device-bridge/dynamic-connector.ts +422 -0
  130. package/src/device-bridge/index.ts +467 -0
  131. package/src/device-bridge/nfc.ts +85 -0
  132. package/src/device-bridge/plugin.ts +71 -0
  133. package/src/device-bridge/relay-client.ts +237 -0
  134. package/src/device-bridge/scanner.ts +42 -0
  135. package/src/device-bridge/sharing.ts +35 -0
  136. package/src/device-bridge/types.ts +326 -0
  137. package/src/device-state/activityClassifier.ts +151 -0
  138. package/src/device-state/activityEngine.ts +252 -0
  139. package/src/device-state/collectors.ts +31 -0
  140. package/src/device-state/domains/activity.ts +93 -0
  141. package/src/device-state/domains/capability.ts +60 -0
  142. package/src/device-state/domains/connectivity.ts +60 -0
  143. package/src/device-state/domains/deviceContext.ts +55 -0
  144. package/src/device-state/domains/interaction.ts +59 -0
  145. package/src/device-state/domains/spatial.ts +283 -0
  146. package/src/device-state/domains/visual.ts +100 -0
  147. package/src/device-state/headingSource.ts +84 -0
  148. package/src/device-state/index.ts +77 -0
  149. package/src/device-state/quality.ts +60 -0
  150. package/src/device-state/replay.ts +220 -0
  151. package/src/device-state/turnDetector.ts +121 -0
  152. package/src/device-state/visualEngine.ts +341 -0
  153. package/src/haptics.ts +4 -0
  154. package/src/index.ts +37 -0
  155. package/src/oauth/expoOAuth.tsx +113 -0
  156. package/src/vision/analysis.js +80 -0
  157. package/src/vision/index.ts +144 -0
  158. package/src/vision/ring.js +35 -0
@@ -0,0 +1,341 @@
1
+ /**
2
+ * The Visual engine — a rolling frame timeline the Visual domain reads.
3
+ *
4
+ * WHY THIS IS SHAPED ODDLY. expo-camera can only produce frames through a
5
+ * MOUNTED `<CameraView>`; there is no headless capture API. So the engine is
6
+ * split in two:
7
+ *
8
+ * this module — the state: the ring, the session, the intent, the cadence.
9
+ * Pure TS, no React, so the domain and tests can use it.
10
+ * VisualEngineHost — a component the app mounts once. When a session is open
11
+ * it mounts the camera and feeds frames in; otherwise it
12
+ * renders nothing and the camera is off.
13
+ *
14
+ * The design's rule is that `getSnapshot()` must be a CACHED read that does not
15
+ * open the camera or take a picture — "avoid repeatedly starting the camera or
16
+ * capturing a new image for every plugin-tool call during an interactive
17
+ * session". The ring is what makes that possible: capture runs on the engine's
18
+ * own cadence, and reads serve whatever is already there.
19
+ *
20
+ * THE CADENCE IS THE ENGINE'S CHOICE, NOT THE PLUGIN'S. A plugin declares an
21
+ * INTENT — `navigation`, `document`, `ocr` — and this picks the interval. That
22
+ * is the whole point of the semantic layer: a caller asking for frames every
23
+ * 500ms is specifying an implementation, and cannot know the analysis latency,
24
+ * the battery state or the thermal headroom. Intent survives those changing;
25
+ * a number does not.
26
+ *
27
+ * EACH FRAME CARRIES ITS ORIENTATION — heading, read at capture time. Without
28
+ * it a frame cannot support a guidance instruction: "turn left at the corner"
29
+ * needs to know which way the lens faced. Reading it afterwards would pair a
30
+ * frame with a heading from after the user turned, which is how a guide
31
+ * describes the wrong junction with confidence.
32
+ *
33
+ * POSITION IS NOT ON THE FRAME, and that is the freeze's decision, not an
34
+ * omission. VisualFrameRef (API_FREEZE §4) carries `orientation` only; where
35
+ * the user IS belongs to Spatial. A caller that needs both correlates them by
36
+ * `timestamp` — both snapshots carry one. Duplicating position here would
37
+ * create a second copy that can disagree with Spatial's, which is precisely
38
+ * the two-descriptions-of-one-thing problem this whole layer exists to end.
39
+ *
40
+ * NOT IMPLEMENTED, and reported as `unknown`/null rather than guessed:
41
+ * per-frame blur, exposure and obstruction (need image analysis), and
42
+ * pitch/roll (need device-motion, not installed). A frame too dark to use must
43
+ * not be described as `exposure: 'good'` because nothing looked at it.
44
+ */
45
+ import type { StateQuality } from '@ateam-ai-mobile/plugin-runtime';
46
+
47
+ /** A frame in the ring. `uri` is a local file the app can read; nothing is uploaded. */
48
+ export interface VisualFrame {
49
+ handle: { id: string; mimeType: string; expiresAt: number | null };
50
+ timestamp: number;
51
+ /**
52
+ * WHERE THE CAMERA WAS POINTING, stamped at capture time.
53
+ *
54
+ * A frame without this is nearly useless for guidance: "turn left at the
55
+ * corner" needs to know which way the lens faced and where the user stood.
56
+ * And it has to be captured WITH the frame, not read afterwards — by the
57
+ * time a snapshot is read the user has walked on, and pairing a frame with a
58
+ * later heading is how a guide describes the wrong junction confidently.
59
+ *
60
+ * Every field is nullable and null means NOT MEASURED. `headingDeg: 0` is due
61
+ * north, so a missing compass must not collapse to 0.
62
+ */
63
+ orientation?: {
64
+ /** True heading if available, else magnetic. Null when the compass said nothing. */
65
+ headingDeg?: number | null;
66
+ /** Not measured — needs device-motion, which is not installed. */
67
+ pitchDeg?: number | null;
68
+ rollDeg?: number | null;
69
+ };
70
+ quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
71
+ }
72
+
73
+ /**
74
+ * Supplies the spatial context stamped onto each frame.
75
+ *
76
+ * Injected rather than imported for the same reason the collectors are, and
77
+ * because it must be CHEAP: it is read once per capture, on the engine's
78
+ * cadence. Both getters return null rather than throwing — a frame with no
79
+ * heading is still a usable frame, and losing the picture because the compass
80
+ * was busy would be the wrong trade.
81
+ */
82
+ export interface VisualContextProvider {
83
+ heading(): { headingDeg: number | null } | null;
84
+ }
85
+
86
+ export type VisualIntentName =
87
+ | 'navigation' | 'document' | 'ocr' | 'shopping'
88
+ | 'inspection' | 'accessibility' | 'stream' | 'highQuality';
89
+
90
+ /**
91
+ * Intent → capture cadence and window.
92
+ *
93
+ * These are the engine's judgement, not the freeze's — the docs say native
94
+ * owns sampling and deliberately do not fix numbers. Recorded here so the
95
+ * reasoning is visible and arguable rather than buried:
96
+ *
97
+ * - `navigation`: a walking user's scene changes continuously, so a steady
98
+ * cadence is right; ~1s is the ceiling at which a guidance instruction is
99
+ * still about the present. Faster mostly queues behind the analysis.
100
+ * - `document` / `ocr`: the subject is held still and the user is aiming. A
101
+ * slow cadence is enough, and a big window is pointless.
102
+ * - `stream`: the fastest we offer, for a caller that really is watching.
103
+ * - `highQuality`: rare, deliberate, single good frame.
104
+ */
105
+ const PROFILES: Record<VisualIntentName, { intervalMs: number; windowMs: number }> = {
106
+ navigation: { intervalMs: 1000, windowMs: 5000 },
107
+ accessibility: { intervalMs: 1000, windowMs: 5000 },
108
+ stream: { intervalMs: 500, windowMs: 5000 },
109
+ inspection: { intervalMs: 2000, windowMs: 8000 },
110
+ shopping: { intervalMs: 2000, windowMs: 5000 },
111
+ document: { intervalMs: 3000, windowMs: 3000 },
112
+ ocr: { intervalMs: 3000, windowMs: 3000 },
113
+ highQuality: { intervalMs: 5000, windowMs: 5000 },
114
+ };
115
+
116
+ export interface VisualSessionState {
117
+ id: string;
118
+ intent: VisualIntentName;
119
+ startedAt: number;
120
+ /** Set while the app is backgrounded. The session SURVIVES; the camera does not. */
121
+ suspended: boolean;
122
+ }
123
+
124
+ type Listener = () => void;
125
+
126
+ class VisualEngine {
127
+ private frames: VisualFrame[] = [];
128
+ private session: VisualSessionState | null = null;
129
+ private listeners = new Set<Listener>();
130
+ /** Set by the host component while a camera is actually mounted and ready. */
131
+ private cameraReady = false;
132
+ /** Optional: supplies heading/position stamped onto each frame. */
133
+ private context: VisualContextProvider | null = null;
134
+
135
+ /** Host DI. Without it frames carry no orientation, and say so with nulls. */
136
+ setContextProvider(p: VisualContextProvider | null): void { this.context = p; }
137
+
138
+ // ── session ──
139
+
140
+ startSession(intent: VisualIntentName): VisualSessionState {
141
+ if (this.session) return this.session;
142
+ this.session = {
143
+ id: `visual_${Date.now()}`,
144
+ intent: PROFILES[intent] ? intent : 'navigation',
145
+ startedAt: Date.now(),
146
+ suspended: false,
147
+ };
148
+ this.notify();
149
+ return this.session;
150
+ }
151
+
152
+ stopSession(): void {
153
+ this.session = null;
154
+ // The timeline belongs to the session. Keeping it would let a later read
155
+ // describe a room the user left.
156
+ this.frames = [];
157
+ this.cameraReady = false;
158
+ this.notify();
159
+ }
160
+
161
+ /**
162
+ * Background: suspend, do not end.
163
+ *
164
+ * iOS tears the camera down when the app leaves the foreground. The session
165
+ * is a STATE that survives that; what does not survive is the claim that the
166
+ * camera is live. While suspended the ring is kept but ages — a read gets
167
+ * old frames clearly marked old, rather than a fresh-looking lie.
168
+ */
169
+ suspend(): void {
170
+ if (!this.session || this.session.suspended) return;
171
+ this.session.suspended = true;
172
+ this.cameraReady = false;
173
+ this.notify();
174
+ }
175
+
176
+ resume(): void {
177
+ if (!this.session?.suspended) return;
178
+ this.session.suspended = false;
179
+ this.notify();
180
+ }
181
+
182
+ getSession(): VisualSessionState | null { return this.session; }
183
+
184
+ /** Is a camera mounted and ready right now? Only the host knows. */
185
+ setCameraReady(ready: boolean): void {
186
+ this.cameraReady = ready;
187
+ this.notify();
188
+ }
189
+ isCameraReady(): boolean { return this.cameraReady && !this.session?.suspended; }
190
+
191
+ // ── the ring ──
192
+
193
+ profile() {
194
+ const intent = this.session?.intent || 'navigation';
195
+ return PROFILES[intent];
196
+ }
197
+
198
+ /**
199
+ * Feed from the native vision session (VisionSession, iOS) — the FAST frame
200
+ * source that replaces the takePictureAsync loop. The frame is IN-MEMORY: the
201
+ * native ring holds the pixels keyed by `id`, and native.vision resolves that
202
+ * id inside the trusted layer. No `uri` crosses here; the quality is a REAL
203
+ * measurement (not `unknown`), and `expiresAt` is the ring's real eviction time.
204
+ * The public VisualFrameRef strip to id-only is owned by the device-state session.
205
+ */
206
+ pushFrameMeta(meta: {
207
+ id: string;
208
+ timestamp: number;
209
+ expiresAt?: number | null;
210
+ quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
211
+ }): void {
212
+ let heading: { headingDeg: number | null } | null = null;
213
+ try { heading = this.context?.heading() ?? null; } catch { /* a frame without a heading beats no frame */ }
214
+
215
+ this.frames.push({
216
+ // In-memory: no file. native.vision resolves the id inside the native ring.
217
+ handle: { id: meta.id, mimeType: 'image/none', expiresAt: meta.expiresAt ?? null },
218
+ timestamp: meta.timestamp,
219
+ orientation: { headingDeg: heading?.headingDeg ?? null, pitchDeg: null, rollDeg: null },
220
+ quality: meta.quality ?? { blur: 'unknown', exposure: 'unknown', obstructed: null },
221
+ });
222
+ this.evict();
223
+ this.notify();
224
+ }
225
+
226
+ // The native ring's actual retention (ms). The engine's effective window is
227
+ // min(intent window, retention) — set by whoever owns the clamp semantics; this
228
+ // just stores the input so that logic has one source of truth to read.
229
+ private _retentionMs: number | null = null;
230
+ setRetentionMs(ms: number | null): void { this._retentionMs = ms && ms > 0 ? ms : null; }
231
+ retentionMs(): number | null { return this._retentionMs; }
232
+
233
+ /**
234
+ * How far back the timeline ACTUALLY reaches: `min(intent window, native
235
+ * retention)`.
236
+ *
237
+ * The intent asks for a window; native decides what it can afford to keep.
238
+ * Full-res luma at ~24fps is tens of megabytes for a few seconds, so the ring
239
+ * is memory-bounded and shrinks under pressure — `retentionMs` is native
240
+ * publishing what it is really holding.
241
+ *
242
+ * EVERY FRAME IN THE TIMELINE CAN BE ANALYSED. That is the one sentence a
243
+ * plugin author has to hold, and this clamp is what makes it true. The
244
+ * alternative — listing frames past retention marked "not analysable" — was
245
+ * considered and rejected: a caller would have to check a flag before every
246
+ * use, and forgetting throws mid-guidance on a frame the model can see
247
+ * listed. Present-but-unusable is the same trap as an empty rowset reading
248
+ * as "no rows matched".
249
+ *
250
+ * If per-frame history beyond retention is ever needed, the shape is an
251
+ * AGGREGATE on the snapshot ("the view has been unusable for N ms"), never a
252
+ * per-entry flag.
253
+ */
254
+ effectiveWindowMs(): number {
255
+ const intent = this.profile().windowMs;
256
+ const retention = this._retentionMs;
257
+ return retention === null ? intent : Math.min(intent, retention);
258
+ }
259
+
260
+ /**
261
+ * Time-based, not count-based: the design specifies a rolling WINDOW.
262
+ *
263
+ * Evicts on the EFFECTIVE window, so a frame whose pixels native has already
264
+ * dropped leaves the timeline at the same moment. Without this, `frameById`
265
+ * would start returning null for frames still being listed.
266
+ */
267
+ private evict(): void {
268
+ const cutoff = Date.now() - this.effectiveWindowMs();
269
+ this.frames = this.frames.filter(f => f.timestamp >= cutoff);
270
+ }
271
+
272
+ timeline(): { windowMs: number; frames: VisualFrame[] } {
273
+ this.evict();
274
+ // The EFFECTIVE window, never the intent's. Reporting the intent would have
275
+ // a guide reason about 8 seconds of history it does not have — precisely
276
+ // the confusion the clamp exists to prevent.
277
+ return { windowMs: this.effectiveWindowMs(), frames: [...this.frames] };
278
+ }
279
+
280
+ latest(): VisualFrame | null {
281
+ this.evict();
282
+ return this.frames.length ? this.frames[this.frames.length - 1] : null;
283
+ }
284
+
285
+ /**
286
+ * Resolve a frame handle id (from a VisualFrameRef) back to the live frame so
287
+ * the mechanical layer — native.vision — can read its on-device file. The
288
+ * whole contract of that layer is that it takes an id, never pixels; this is
289
+ * where an id becomes a `handle.uri` it can open. Returns null when the id is
290
+ * unknown OR its frame has aged out of the rolling window (the cache file may
291
+ * also be gone by then) — a real "I can't look at that frame" answer, which
292
+ * the caller must treat as distinct from "I looked and found nothing".
293
+ */
294
+ frameById(id: string): VisualFrame | null {
295
+ this.evict();
296
+ return this.frames.find(f => f.handle.id === id) ?? null;
297
+ }
298
+
299
+ /**
300
+ * Record a MEASURED quality back onto a frame (native.vision.assessQuality).
301
+ * Frames are minted with `quality: unknown` because nothing looks at the
302
+ * pixels at capture time; this is how a real measurement replaces that so a
303
+ * later read of VisualFrameRef.quality reflects it. No-op if the frame has
304
+ * aged out — we never resurrect an evicted frame just to label it.
305
+ */
306
+ setFrameQuality(id: string, quality: NonNullable<VisualFrame['quality']>): void {
307
+ const f = this.frames.find(fr => fr.handle.id === id);
308
+ if (!f) return;
309
+ f.quality = quality;
310
+ this.notify();
311
+ }
312
+
313
+ // ── change notification ──
314
+
315
+ subscribe(fn: Listener): () => void {
316
+ this.listeners.add(fn);
317
+ return () => this.listeners.delete(fn);
318
+ }
319
+ private notify(): void {
320
+ for (const fn of this.listeners) { try { fn(); } catch { /* a bad listener is not the engine's problem */ } }
321
+ }
322
+ }
323
+
324
+ /** One engine per app. The camera is a singleton resource; so is this. */
325
+ export const visualEngine = new VisualEngine();
326
+
327
+ /** Freshness for a visual read — exported so the domain and host agree. */
328
+ export function frameQuality(frame: VisualFrame | null): StateQuality {
329
+ if (!frame) return { freshness: { state: 'unknown', observedAt: null, ageMs: null } };
330
+ const ageMs = Date.now() - frame.timestamp;
331
+ return {
332
+ freshness: {
333
+ // Tighter than the other domains on purpose: a two-second-old frame of a
334
+ // moving scene is not "fresh" in any useful sense.
335
+ state: ageMs < 2000 ? 'fresh' : ageMs < 10_000 ? 'aging' : 'stale',
336
+ observedAt: frame.timestamp,
337
+ ageMs,
338
+ },
339
+ confidence: { level: 'unknown' },
340
+ };
341
+ }
package/src/haptics.ts CHANGED
@@ -24,4 +24,8 @@ export const expoHaptics: HapticsAPI = {
24
24
  selection() {
25
25
  Haptics.selectionAsync().catch(() => {});
26
26
  },
27
+ // Shorthand for notification('error') — the form plugins actually call.
28
+ error() {
29
+ Haptics.notificationAsync(Haptics.NotificationFeedbackType.Error).catch(() => {});
30
+ },
27
31
  };
package/src/index.ts CHANGED
@@ -15,6 +15,8 @@ import {
15
15
  setHapticsProvider,
16
16
  setBundleStorage,
17
17
  registerBundleModules,
18
+ setPluginNativeProvider,
19
+ type PluginNative,
18
20
  } from '@ateam-ai-mobile/plugin-runtime';
19
21
  import { setClipboardProvider } from '@ateam-ai-mobile/ui';
20
22
 
@@ -47,8 +49,18 @@ export function setupExpoProviders(opts?: {
47
49
  * existing app so users keep their attachment previews / bundle cache. */
48
50
  attachmentIndexKey?: string;
49
51
  bundleIndexKey?: string;
52
+ /**
53
+ * The device-primitive implementation plugins receive as `props.device`.
54
+ * Build it with `createDeviceProvider(...)` from `@ateam-ai-mobile/expo/device`
55
+ * and pass it here so the SDK owns the implementation and this call registers
56
+ * it (setPluginNativeProvider). It is injected — not imported by this entry —
57
+ * because createDeviceProvider statically pulls optional expo modules that a
58
+ * host without them cannot build (same reason createBridge is a subpath).
59
+ */
60
+ deviceProvider?: PluginNative;
50
61
  }): { attachmentCache: ExpoAttachmentCache } {
51
62
  setHapticsProvider(expoHaptics);
63
+ if (opts?.deviceProvider) setPluginNativeProvider(opts.deviceProvider);
52
64
  setBundleStorage(new ExpoBundleStorage({ indexKey: opts?.bundleIndexKey }));
53
65
  setClipboardProvider({
54
66
  setString: (text: string) => ExpoClipboard.setStringAsync(text).then(() => {}),
@@ -65,3 +77,28 @@ export function setupExpoProviders(opts?: {
65
77
  });
66
78
  return { attachmentCache: new ExpoAttachmentCache({ indexKey: opts?.attachmentIndexKey }) };
67
79
  }
80
+
81
+ // Device Bridge v2 — the semantic state engines this build provides.
82
+ // The host passes this to setDeviceStateProvider(); per-plugin gating by
83
+ // manifest.stateDomains happens in the SDK.
84
+ export { createDeviceState, IMPLEMENTED_DOMAINS } from './device-state';
85
+ export type { DeviceStateCollectors } from './device-state';
86
+ // The mounted half of the Visual engine. The app renders this ONCE; it draws
87
+ // nothing until a visual session is open. Required for deviceState.visual to
88
+ // produce frames at all — expo-camera has no headless capture.
89
+ // The rolling frame timeline. Exposed so the host's native.vision layer can
90
+ // resolve a frame id to its on-device file (frameById) and write a measured
91
+ // quality back onto the frame (setFrameQuality) — see visualEngine.ts.
92
+ export { visualEngine } from './device-state/visualEngine';
93
+ export type { VisualFrame } from './device-state/visualEngine';
94
+ // The mounted camera surface behind scanner.scanOnce() and
95
+ // camera.recordVideo(). Mount ONCE near the app root; it renders nothing
96
+ // until a plugin asks for a scan or a recording. Without it both APIs
97
+ // reject immediately with UNAVAILABLE — deliberately, since a plugin must
98
+ // never await a promise nothing can settle.
99
+ export { CameraTaskHost } from './camera/CameraTaskHost';
100
+ export { cameraTasks } from './camera/cameraTasks';
101
+ // Replay — run guidance logic against a RECORDED WALK instead of a phone.
102
+ // See docs/SPEC_device_walk_recorder.md for the fixture format.
103
+ export { createReplayDeviceState, parseWalkJsonl } from './device-state/replay';
104
+ export type { WalkFixture, WalkSample, WalkRefusal, ReplayClock, ReplaySession } from './device-state/replay';
@@ -0,0 +1,113 @@
1
+ /**
2
+ * @ateam-ai-mobile/expo/oauth — the native OAuth provider for the relocated
3
+ * auth screens (LoginScreen / JoinScreen / OpenJoinScreen in @ateam-ai-mobile/ui).
4
+ *
5
+ * This is the ONLY place @react-native-google-signin + expo-apple-authentication
6
+ * are imported. The `ui` package stays native-free: its screens call
7
+ * getOAuthProvider() for the id_token acquisition + the real Apple HIG button.
8
+ *
9
+ * Deliberately a SUBPATH (`@ateam-ai-mobile/expo/oauth`), not part of the main
10
+ * entry — same rule as `/bridge` and `/device`: a host that doesn't install
11
+ * these native modules must still be able to build @ateam-ai-mobile/expo.
12
+ *
13
+ * Behaviour is ported verbatim from the app's old login.tsx so the iPhone
14
+ * parity gate sees no change:
15
+ * - Google: configure({ webClientId, iosClientId }) → hasPlayServices →
16
+ * signIn → normalize the v15-vs-older id_token shape. A missing iosClientId
17
+ * silently falls back to a web flow (unchanged); the caller gates on
18
+ * webClientId, so we never reach here without one for the login path.
19
+ * - a user-cancel (SIGN_IN_CANCELLED / IN_PROGRESS / ERR_REQUEST_CANCELED)
20
+ * resolves to null — the screen treats null as "no error", never a red flash.
21
+ */
22
+ import React from 'react';
23
+ import { Platform } from 'react-native';
24
+ import {
25
+ GoogleSignin,
26
+ statusCodes,
27
+ type ConfigureParams,
28
+ } from '@react-native-google-signin/google-signin';
29
+ import * as AppleAuthentication from 'expo-apple-authentication';
30
+ import type { OAuthProvider, OAuthGoogleConfig } from '@ateam-ai-mobile/ui';
31
+
32
+ const NOOP = () => {};
33
+
34
+ export interface ExpoOAuthConfig {
35
+ /**
36
+ * The host's bundled Google client ids, used by the join / open-join screens
37
+ * that carry no tenant config (they pass no cfg to signInWithGoogle). The
38
+ * login screen passes the tenant's configured ids and overrides these.
39
+ */
40
+ googleWebClientId?: string;
41
+ googleIosClientId?: string;
42
+ }
43
+
44
+ /**
45
+ * Build the native OAuth provider. Wire it once at startup:
46
+ * setOAuthProvider(createExpoOAuthProvider({ googleWebClientId, googleIosClientId }));
47
+ */
48
+ export function createExpoOAuthProvider(host: ExpoOAuthConfig = {}): OAuthProvider {
49
+ return {
50
+ async signInWithGoogle(cfg?: OAuthGoogleConfig): Promise<string | null> {
51
+ const webClientId = cfg?.webClientId || host.googleWebClientId || '';
52
+ const iosClientId = cfg?.iosClientId || host.googleIosClientId || '';
53
+ // configure is idempotent + cheap; do it at sign-in time so the tenant's
54
+ // ids are always current (replaces login.tsx's configure effect).
55
+ const params: ConfigureParams = { webClientId };
56
+ if (iosClientId) (params as any).iosClientId = iosClientId;
57
+ GoogleSignin.configure(params);
58
+ try {
59
+ // android-only; no-op on iOS. Called defensively, as before.
60
+ await GoogleSignin.hasPlayServices({ showPlayServicesUpdateDialog: true });
61
+ const result: any = await GoogleSignin.signIn();
62
+ // v15+ returns { data: { idToken } }; older returns { idToken }.
63
+ return result?.data?.idToken || result?.idToken || null;
64
+ } catch (err: any) {
65
+ const code = err?.code;
66
+ if (code === statusCodes.SIGN_IN_CANCELLED || code === statusCodes.IN_PROGRESS) {
67
+ return null;
68
+ }
69
+ throw err;
70
+ }
71
+ },
72
+
73
+ async signInWithApple(): Promise<string | null> {
74
+ try {
75
+ const credential = await AppleAuthentication.signInAsync({
76
+ requestedScopes: [
77
+ AppleAuthentication.AppleAuthenticationScope.FULL_NAME,
78
+ AppleAuthentication.AppleAuthenticationScope.EMAIL,
79
+ ],
80
+ });
81
+ return credential.identityToken || null;
82
+ } catch (err: any) {
83
+ // ERR_REQUEST_CANCELED = the user dismissed the sheet.
84
+ if (err?.code === 'ERR_REQUEST_CANCELED') return null;
85
+ throw err;
86
+ }
87
+ },
88
+
89
+ async isAppleAvailable(): Promise<boolean> {
90
+ if (Platform.OS !== 'ios') return false;
91
+ try {
92
+ return await AppleAuthentication.isAvailableAsync();
93
+ } catch {
94
+ return false;
95
+ }
96
+ },
97
+
98
+ renderAppleButton({ onPress, disabled, style }): React.ReactNode {
99
+ // Apple's HIG requires their own styled button when Apple Sign-In sits
100
+ // alongside other providers. Same chrome the app shipped: CONTINUE /
101
+ // BLACK / fully-rounded. Layout (width/height) comes from the screen.
102
+ return (
103
+ <AppleAuthentication.AppleAuthenticationButton
104
+ buttonType={AppleAuthentication.AppleAuthenticationButtonType.CONTINUE}
105
+ buttonStyle={AppleAuthentication.AppleAuthenticationButtonStyle.BLACK}
106
+ cornerRadius={999}
107
+ style={style as any}
108
+ onPress={disabled ? NOOP : onPress}
109
+ />
110
+ );
111
+ },
112
+ };
113
+ }
@@ -0,0 +1,80 @@
1
+ // Frame-quality analysis — the REFERENCE algorithm, in plain JS so it can be
2
+ // unit-tested exhaustively without a device. The native iOS (Objective-C++) and
3
+ // Android (Kotlin) implementations mirror this math and these thresholds exactly;
4
+ // this file is the spec they must match, and the thing the tests exercise.
5
+ //
6
+ // Input is a luma (grayscale) plane — one byte per pixel, 0..255. We subsample
7
+ // to a fixed grid so the work is bounded and thresholds are resolution-independent,
8
+ // then derive:
9
+ // exposure from mean luminance (good | dark | bright)
10
+ // obstructed from darkness + low variance (covered lens vs a real scene)
11
+ // blur from variance of the Laplacian (the classic focus metric)
12
+
13
+ export const THRESHOLDS = {
14
+ grid: 128,
15
+ exposureDark: 45,
16
+ exposureBright: 205,
17
+ obstructedMean: 22,
18
+ obstructedStd: 8,
19
+ blurSharp: 120,
20
+ blurSlight: 30,
21
+ };
22
+
23
+ /**
24
+ * @param {number} width
25
+ * @param {number} height
26
+ * @param {Uint8Array|number[]} luma row-major luma bytes
27
+ * @param {number} [rowStride] bytes per row (defaults to width)
28
+ * @returns {{blur:string, exposure:string, obstructed:boolean|null}}
29
+ */
30
+ export function assessLuma(width, height, luma, rowStride) {
31
+ const stride = rowStride || width;
32
+ const S = THRESHOLDS.grid;
33
+ if (!width || !height || !luma || luma.length === 0) {
34
+ return { blur: 'unknown', exposure: 'unknown', obstructed: null };
35
+ }
36
+
37
+ // Grid dimensions, capped at S. Sample coordinates span the FULL frame
38
+ // [0, width-1] / [0, height-1] — the first sample is the top-left pixel and
39
+ // the last is the bottom-right, so no edge (a bright window, an occlusion,
40
+ // strong detail) is ever omitted. A floor-step + capped-count scheme would
41
+ // systematically miss the right/bottom strip when the size isn't a multiple
42
+ // of S.
43
+ const cols = Math.min(S, width);
44
+ const rows = Math.min(S, height);
45
+ const sxAt = (gx) => (cols > 1 ? Math.round((gx * (width - 1)) / (cols - 1)) : 0);
46
+ const syAt = (gy) => (rows > 1 ? Math.round((gy * (height - 1)) / (rows - 1)) : 0);
47
+
48
+ const grid = new Uint8Array(cols * rows);
49
+ let n = 0, sum = 0, sumSq = 0;
50
+ for (let gy = 0; gy < rows; gy++) {
51
+ const rowBase = syAt(gy) * stride;
52
+ for (let gx = 0; gx < cols; gx++) {
53
+ const v = luma[rowBase + sxAt(gx)] & 0xff;
54
+ grid[gy * cols + gx] = v;
55
+ sum += v; sumSq += v * v; n++;
56
+ }
57
+ }
58
+ if (n === 0) return { blur: 'unknown', exposure: 'unknown', obstructed: null };
59
+
60
+ const mean = sum / n;
61
+ const variance = Math.max(0, sumSq / n - mean * mean);
62
+ const std = Math.sqrt(variance);
63
+
64
+ let lapSum = 0, lapSumSq = 0, lapN = 0;
65
+ for (let y = 1; y < rows - 1; y++) {
66
+ for (let x = 1; x < cols - 1; x++) {
67
+ const i = y * cols + x;
68
+ const lap = 4 * grid[i] - grid[i - 1] - grid[i + 1] - grid[i - cols] - grid[i + cols];
69
+ lapSum += lap; lapSumSq += lap * lap; lapN++;
70
+ }
71
+ }
72
+ const lapVar = lapN > 0 ? Math.max(0, lapSumSq / lapN - (lapSum / lapN) ** 2) : 0;
73
+
74
+ const t = THRESHOLDS;
75
+ const exposure = mean < t.exposureDark ? 'dark' : (mean > t.exposureBright ? 'bright' : 'good');
76
+ const obstructed = mean < t.obstructedMean && std < t.obstructedStd;
77
+ const blur = lapVar > t.blurSharp ? 'sharp' : (lapVar > t.blurSlight ? 'slight' : 'severe');
78
+
79
+ return { blur, exposure, obstructed };
80
+ }