@ateam-ai-mobile/expo 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/build.gradle +16 -0
- package/android/src/main/java/expo/modules/adavision/AdaVisionModule.kt +75 -0
- package/dist/attachmentCache-DhQTqi2E.d.mts +18 -0
- package/dist/attachmentCache-DhQTqi2E.d.ts +18 -0
- package/dist/bootstrap.d.mts +95 -0
- package/dist/bootstrap.d.ts +95 -0
- package/dist/bootstrap.js +2473 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/bootstrap.mjs +64 -0
- package/dist/bootstrap.mjs.map +1 -0
- package/dist/chunk-4BMWVN25.mjs +91 -0
- package/dist/chunk-4BMWVN25.mjs.map +1 -0
- package/dist/chunk-4XYXJYN5.mjs +1513 -0
- package/dist/chunk-4XYXJYN5.mjs.map +1 -0
- package/dist/chunk-5HM6UA2K.mjs +571 -0
- package/dist/chunk-5HM6UA2K.mjs.map +1 -0
- package/dist/chunk-6LXF3JVX.mjs +96 -0
- package/dist/chunk-6LXF3JVX.mjs.map +1 -0
- package/dist/chunk-FF6ZZZXH.mjs +71 -0
- package/dist/chunk-FF6ZZZXH.mjs.map +1 -0
- package/dist/chunk-KOXF2FZM.mjs +92 -0
- package/dist/chunk-KOXF2FZM.mjs.map +1 -0
- package/dist/chunk-L4V7OCJ3.mjs +71 -0
- package/dist/chunk-L4V7OCJ3.mjs.map +1 -0
- package/dist/chunk-TZXWVYYR.mjs +67 -0
- package/dist/chunk-TZXWVYYR.mjs.map +1 -0
- package/dist/chunk-VQJPSLHP.mjs +1801 -0
- package/dist/chunk-VQJPSLHP.mjs.map +1 -0
- package/dist/chunk-WMJKH4XE.mjs +34 -0
- package/dist/chunk-WMJKH4XE.mjs.map +1 -0
- package/dist/chunk-XQSY6MHP.mjs +213 -0
- package/dist/chunk-XQSY6MHP.mjs.map +1 -0
- package/dist/createBridge.mjs +2 -0
- package/dist/createBridge.mjs.map +1 -1
- package/dist/device/createDeviceProvider.d.mts +23 -0
- package/dist/device/createDeviceProvider.d.ts +23 -0
- package/dist/device/createDeviceProvider.js +967 -0
- package/dist/device/createDeviceProvider.js.map +1 -0
- package/dist/device/createDeviceProvider.mjs +17 -0
- package/dist/device/createDeviceProvider.mjs.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.d.mts +10 -0
- package/dist/device-bridge/collectors/appEngagement.d.ts +10 -0
- package/dist/device-bridge/collectors/appEngagement.js +128 -0
- package/dist/device-bridge/collectors/appEngagement.js.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.mjs +12 -0
- package/dist/device-bridge/collectors/appEngagement.mjs.map +1 -0
- package/dist/device-bridge/collectors/health.d.mts +24 -0
- package/dist/device-bridge/collectors/health.d.ts +24 -0
- package/dist/device-bridge/collectors/health.js +238 -0
- package/dist/device-bridge/collectors/health.js.map +1 -0
- package/dist/device-bridge/collectors/health.mjs +18 -0
- package/dist/device-bridge/collectors/health.mjs.map +1 -0
- package/dist/device-bridge/cookies.d.mts +49 -0
- package/dist/device-bridge/cookies.d.ts +49 -0
- package/dist/device-bridge/cookies.js +90 -0
- package/dist/device-bridge/cookies.js.map +1 -0
- package/dist/device-bridge/cookies.mjs +14 -0
- package/dist/device-bridge/cookies.mjs.map +1 -0
- package/dist/device-bridge/index.d.mts +483 -0
- package/dist/device-bridge/index.d.ts +483 -0
- package/dist/device-bridge/index.js +2084 -0
- package/dist/device-bridge/index.js.map +1 -0
- package/dist/device-bridge/index.mjs +32 -0
- package/dist/device-bridge/index.mjs.map +1 -0
- package/dist/device-bridge/nfc.d.mts +30 -0
- package/dist/device-bridge/nfc.d.ts +30 -0
- package/dist/device-bridge/nfc.js +109 -0
- package/dist/device-bridge/nfc.js.map +1 -0
- package/dist/device-bridge/nfc.mjs +14 -0
- package/dist/device-bridge/nfc.mjs.map +1 -0
- package/dist/index.d.mts +493 -18
- package/dist/index.d.ts +493 -18
- package/dist/index.js +1409 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +23 -194
- package/dist/index.mjs.map +1 -1
- package/dist/oauth/expoOAuth.d.mts +18 -0
- package/dist/oauth/expoOAuth.d.ts +18 -0
- package/dist/oauth/expoOAuth.js +102 -0
- package/dist/oauth/expoOAuth.js.map +1 -0
- package/dist/oauth/expoOAuth.mjs +8 -0
- package/dist/oauth/expoOAuth.mjs.map +1 -0
- package/dist/types-BRanJYTn.d.mts +138 -0
- package/dist/types-BRanJYTn.d.ts +138 -0
- package/dist/vision/index.d.mts +81 -0
- package/dist/vision/index.d.ts +81 -0
- package/dist/vision/index.js +101 -0
- package/dist/vision/index.js.map +1 -0
- package/dist/vision/index.mjs +28 -0
- package/dist/vision/index.mjs.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AdaVision.podspec +28 -0
- package/ios/AdaVisionModule.swift +432 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/metadata.json +773 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/model.mil +720 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/weights/weight.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model1/coremldata.bin +0 -0
- package/package.json +107 -7
- package/src/bootstrap.ts +159 -0
- package/src/camera/CameraTaskHost.tsx +175 -0
- package/src/camera/cameraTasks.ts +129 -0
- package/src/device/createDeviceProvider.ts +832 -0
- package/src/device-bridge/actions/calendar-actions.ts +217 -0
- package/src/device-bridge/actions/dnd.ts +74 -0
- package/src/device-bridge/actions/health-actions.ts +58 -0
- package/src/device-bridge/actions/index.ts +12 -0
- package/src/device-bridge/actions/messaging.ts +74 -0
- package/src/device-bridge/actions/navigation.ts +88 -0
- package/src/device-bridge/actions/queue-poller.ts +151 -0
- package/src/device-bridge/background.ts +137 -0
- package/src/device-bridge/biometrics.ts +64 -0
- package/src/device-bridge/capabilities.ts +218 -0
- package/src/device-bridge/collectors/appEngagement.ts +120 -0
- package/src/device-bridge/collectors/battery.ts +83 -0
- package/src/device-bridge/collectors/calendar.ts +162 -0
- package/src/device-bridge/collectors/camera.ts +27 -0
- package/src/device-bridge/collectors/connectivity.ts +60 -0
- package/src/device-bridge/collectors/contacts.ts +200 -0
- package/src/device-bridge/collectors/health.ts +299 -0
- package/src/device-bridge/collectors/index.ts +12 -0
- package/src/device-bridge/collectors/location.ts +224 -0
- package/src/device-bridge/collectors/mood.ts +141 -0
- package/src/device-bridge/collectors/notifications.ts +70 -0
- package/src/device-bridge/cookies.ts +89 -0
- package/src/device-bridge/dynamic-connector.ts +422 -0
- package/src/device-bridge/index.ts +467 -0
- package/src/device-bridge/nfc.ts +85 -0
- package/src/device-bridge/plugin.ts +71 -0
- package/src/device-bridge/relay-client.ts +237 -0
- package/src/device-bridge/scanner.ts +42 -0
- package/src/device-bridge/sharing.ts +35 -0
- package/src/device-bridge/types.ts +326 -0
- package/src/device-state/activityClassifier.ts +151 -0
- package/src/device-state/activityEngine.ts +252 -0
- package/src/device-state/collectors.ts +31 -0
- package/src/device-state/domains/activity.ts +93 -0
- package/src/device-state/domains/capability.ts +60 -0
- package/src/device-state/domains/connectivity.ts +60 -0
- package/src/device-state/domains/deviceContext.ts +55 -0
- package/src/device-state/domains/interaction.ts +59 -0
- package/src/device-state/domains/spatial.ts +283 -0
- package/src/device-state/domains/visual.ts +100 -0
- package/src/device-state/headingSource.ts +84 -0
- package/src/device-state/index.ts +77 -0
- package/src/device-state/quality.ts +60 -0
- package/src/device-state/replay.ts +220 -0
- package/src/device-state/turnDetector.ts +121 -0
- package/src/device-state/visualEngine.ts +341 -0
- package/src/haptics.ts +4 -0
- package/src/index.ts +37 -0
- package/src/oauth/expoOAuth.tsx +113 -0
- package/src/vision/analysis.js +80 -0
- package/src/vision/index.ts +144 -0
- package/src/vision/ring.js +35 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Visual engine — a rolling frame timeline the Visual domain reads.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS IS SHAPED ODDLY. expo-camera can only produce frames through a
|
|
5
|
+
* MOUNTED `<CameraView>`; there is no headless capture API. So the engine is
|
|
6
|
+
* split in two:
|
|
7
|
+
*
|
|
8
|
+
* this module — the state: the ring, the session, the intent, the cadence.
|
|
9
|
+
* Pure TS, no React, so the domain and tests can use it.
|
|
10
|
+
* VisualEngineHost — a component the app mounts once. When a session is open
|
|
11
|
+
* it mounts the camera and feeds frames in; otherwise it
|
|
12
|
+
* renders nothing and the camera is off.
|
|
13
|
+
*
|
|
14
|
+
* The design's rule is that `getSnapshot()` must be a CACHED read that does not
|
|
15
|
+
* open the camera or take a picture — "avoid repeatedly starting the camera or
|
|
16
|
+
* capturing a new image for every plugin-tool call during an interactive
|
|
17
|
+
* session". The ring is what makes that possible: capture runs on the engine's
|
|
18
|
+
* own cadence, and reads serve whatever is already there.
|
|
19
|
+
*
|
|
20
|
+
* THE CADENCE IS THE ENGINE'S CHOICE, NOT THE PLUGIN'S. A plugin declares an
|
|
21
|
+
* INTENT — `navigation`, `document`, `ocr` — and this picks the interval. That
|
|
22
|
+
* is the whole point of the semantic layer: a caller asking for frames every
|
|
23
|
+
* 500ms is specifying an implementation, and cannot know the analysis latency,
|
|
24
|
+
* the battery state or the thermal headroom. Intent survives those changing;
|
|
25
|
+
* a number does not.
|
|
26
|
+
*
|
|
27
|
+
* EACH FRAME CARRIES ITS ORIENTATION — heading, read at capture time. Without
|
|
28
|
+
* it a frame cannot support a guidance instruction: "turn left at the corner"
|
|
29
|
+
* needs to know which way the lens faced. Reading it afterwards would pair a
|
|
30
|
+
* frame with a heading from after the user turned, which is how a guide
|
|
31
|
+
* describes the wrong junction with confidence.
|
|
32
|
+
*
|
|
33
|
+
* POSITION IS NOT ON THE FRAME, and that is the freeze's decision, not an
|
|
34
|
+
* omission. VisualFrameRef (API_FREEZE §4) carries `orientation` only; where
|
|
35
|
+
* the user IS belongs to Spatial. A caller that needs both correlates them by
|
|
36
|
+
* `timestamp` — both snapshots carry one. Duplicating position here would
|
|
37
|
+
* create a second copy that can disagree with Spatial's, which is precisely
|
|
38
|
+
* the two-descriptions-of-one-thing problem this whole layer exists to end.
|
|
39
|
+
*
|
|
40
|
+
* NOT IMPLEMENTED, and reported as `unknown`/null rather than guessed:
|
|
41
|
+
* per-frame blur, exposure and obstruction (need image analysis), and
|
|
42
|
+
* pitch/roll (need device-motion, not installed). A frame too dark to use must
|
|
43
|
+
* not be described as `exposure: 'good'` because nothing looked at it.
|
|
44
|
+
*/
|
|
45
|
+
import type { StateQuality } from '@ateam-ai-mobile/plugin-runtime';
|
|
46
|
+
|
|
47
|
+
/** A frame in the ring. `uri` is a local file the app can read; nothing is uploaded. */
|
|
48
|
+
export interface VisualFrame {
|
|
49
|
+
handle: { id: string; mimeType: string; expiresAt: number | null };
|
|
50
|
+
timestamp: number;
|
|
51
|
+
/**
|
|
52
|
+
* WHERE THE CAMERA WAS POINTING, stamped at capture time.
|
|
53
|
+
*
|
|
54
|
+
* A frame without this is nearly useless for guidance: "turn left at the
|
|
55
|
+
* corner" needs to know which way the lens faced and where the user stood.
|
|
56
|
+
* And it has to be captured WITH the frame, not read afterwards — by the
|
|
57
|
+
* time a snapshot is read the user has walked on, and pairing a frame with a
|
|
58
|
+
* later heading is how a guide describes the wrong junction confidently.
|
|
59
|
+
*
|
|
60
|
+
* Every field is nullable and null means NOT MEASURED. `headingDeg: 0` is due
|
|
61
|
+
* north, so a missing compass must not collapse to 0.
|
|
62
|
+
*/
|
|
63
|
+
orientation?: {
|
|
64
|
+
/** True heading if available, else magnetic. Null when the compass said nothing. */
|
|
65
|
+
headingDeg?: number | null;
|
|
66
|
+
/** Not measured — needs device-motion, which is not installed. */
|
|
67
|
+
pitchDeg?: number | null;
|
|
68
|
+
rollDeg?: number | null;
|
|
69
|
+
};
|
|
70
|
+
quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Supplies the spatial context stamped onto each frame.
|
|
75
|
+
*
|
|
76
|
+
* Injected rather than imported for the same reason the collectors are, and
|
|
77
|
+
* because it must be CHEAP: it is read once per capture, on the engine's
|
|
78
|
+
* cadence. Both getters return null rather than throwing — a frame with no
|
|
79
|
+
* heading is still a usable frame, and losing the picture because the compass
|
|
80
|
+
* was busy would be the wrong trade.
|
|
81
|
+
*/
|
|
82
|
+
export interface VisualContextProvider {
|
|
83
|
+
heading(): { headingDeg: number | null } | null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export type VisualIntentName =
|
|
87
|
+
| 'navigation' | 'document' | 'ocr' | 'shopping'
|
|
88
|
+
| 'inspection' | 'accessibility' | 'stream' | 'highQuality';
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Intent → capture cadence and window.
|
|
92
|
+
*
|
|
93
|
+
* These are the engine's judgement, not the freeze's — the docs say native
|
|
94
|
+
* owns sampling and deliberately do not fix numbers. Recorded here so the
|
|
95
|
+
* reasoning is visible and arguable rather than buried:
|
|
96
|
+
*
|
|
97
|
+
* - `navigation`: a walking user's scene changes continuously, so a steady
|
|
98
|
+
* cadence is right; ~1s is the ceiling at which a guidance instruction is
|
|
99
|
+
* still about the present. Faster mostly queues behind the analysis.
|
|
100
|
+
* - `document` / `ocr`: the subject is held still and the user is aiming. A
|
|
101
|
+
* slow cadence is enough, and a big window is pointless.
|
|
102
|
+
* - `stream`: the fastest we offer, for a caller that really is watching.
|
|
103
|
+
* - `highQuality`: rare, deliberate, single good frame.
|
|
104
|
+
*/
|
|
105
|
+
const PROFILES: Record<VisualIntentName, { intervalMs: number; windowMs: number }> = {
|
|
106
|
+
navigation: { intervalMs: 1000, windowMs: 5000 },
|
|
107
|
+
accessibility: { intervalMs: 1000, windowMs: 5000 },
|
|
108
|
+
stream: { intervalMs: 500, windowMs: 5000 },
|
|
109
|
+
inspection: { intervalMs: 2000, windowMs: 8000 },
|
|
110
|
+
shopping: { intervalMs: 2000, windowMs: 5000 },
|
|
111
|
+
document: { intervalMs: 3000, windowMs: 3000 },
|
|
112
|
+
ocr: { intervalMs: 3000, windowMs: 3000 },
|
|
113
|
+
highQuality: { intervalMs: 5000, windowMs: 5000 },
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export interface VisualSessionState {
|
|
117
|
+
id: string;
|
|
118
|
+
intent: VisualIntentName;
|
|
119
|
+
startedAt: number;
|
|
120
|
+
/** Set while the app is backgrounded. The session SURVIVES; the camera does not. */
|
|
121
|
+
suspended: boolean;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
type Listener = () => void;
|
|
125
|
+
|
|
126
|
+
class VisualEngine {
|
|
127
|
+
private frames: VisualFrame[] = [];
|
|
128
|
+
private session: VisualSessionState | null = null;
|
|
129
|
+
private listeners = new Set<Listener>();
|
|
130
|
+
/** Set by the host component while a camera is actually mounted and ready. */
|
|
131
|
+
private cameraReady = false;
|
|
132
|
+
/** Optional: supplies heading/position stamped onto each frame. */
|
|
133
|
+
private context: VisualContextProvider | null = null;
|
|
134
|
+
|
|
135
|
+
/** Host DI. Without it frames carry no orientation, and say so with nulls. */
|
|
136
|
+
setContextProvider(p: VisualContextProvider | null): void { this.context = p; }
|
|
137
|
+
|
|
138
|
+
// ── session ──
|
|
139
|
+
|
|
140
|
+
startSession(intent: VisualIntentName): VisualSessionState {
|
|
141
|
+
if (this.session) return this.session;
|
|
142
|
+
this.session = {
|
|
143
|
+
id: `visual_${Date.now()}`,
|
|
144
|
+
intent: PROFILES[intent] ? intent : 'navigation',
|
|
145
|
+
startedAt: Date.now(),
|
|
146
|
+
suspended: false,
|
|
147
|
+
};
|
|
148
|
+
this.notify();
|
|
149
|
+
return this.session;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
stopSession(): void {
|
|
153
|
+
this.session = null;
|
|
154
|
+
// The timeline belongs to the session. Keeping it would let a later read
|
|
155
|
+
// describe a room the user left.
|
|
156
|
+
this.frames = [];
|
|
157
|
+
this.cameraReady = false;
|
|
158
|
+
this.notify();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Background: suspend, do not end.
|
|
163
|
+
*
|
|
164
|
+
* iOS tears the camera down when the app leaves the foreground. The session
|
|
165
|
+
* is a STATE that survives that; what does not survive is the claim that the
|
|
166
|
+
* camera is live. While suspended the ring is kept but ages — a read gets
|
|
167
|
+
* old frames clearly marked old, rather than a fresh-looking lie.
|
|
168
|
+
*/
|
|
169
|
+
suspend(): void {
|
|
170
|
+
if (!this.session || this.session.suspended) return;
|
|
171
|
+
this.session.suspended = true;
|
|
172
|
+
this.cameraReady = false;
|
|
173
|
+
this.notify();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
resume(): void {
|
|
177
|
+
if (!this.session?.suspended) return;
|
|
178
|
+
this.session.suspended = false;
|
|
179
|
+
this.notify();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
getSession(): VisualSessionState | null { return this.session; }
|
|
183
|
+
|
|
184
|
+
/** Is a camera mounted and ready right now? Only the host knows. */
|
|
185
|
+
setCameraReady(ready: boolean): void {
|
|
186
|
+
this.cameraReady = ready;
|
|
187
|
+
this.notify();
|
|
188
|
+
}
|
|
189
|
+
isCameraReady(): boolean { return this.cameraReady && !this.session?.suspended; }
|
|
190
|
+
|
|
191
|
+
// ── the ring ──
|
|
192
|
+
|
|
193
|
+
profile() {
|
|
194
|
+
const intent = this.session?.intent || 'navigation';
|
|
195
|
+
return PROFILES[intent];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Feed from the native vision session (VisionSession, iOS) — the FAST frame
|
|
200
|
+
* source that replaces the takePictureAsync loop. The frame is IN-MEMORY: the
|
|
201
|
+
* native ring holds the pixels keyed by `id`, and native.vision resolves that
|
|
202
|
+
* id inside the trusted layer. No `uri` crosses here; the quality is a REAL
|
|
203
|
+
* measurement (not `unknown`), and `expiresAt` is the ring's real eviction time.
|
|
204
|
+
* The public VisualFrameRef strip to id-only is owned by the device-state session.
|
|
205
|
+
*/
|
|
206
|
+
pushFrameMeta(meta: {
|
|
207
|
+
id: string;
|
|
208
|
+
timestamp: number;
|
|
209
|
+
expiresAt?: number | null;
|
|
210
|
+
quality?: { blur?: string; exposure?: string; obstructed?: boolean | null };
|
|
211
|
+
}): void {
|
|
212
|
+
let heading: { headingDeg: number | null } | null = null;
|
|
213
|
+
try { heading = this.context?.heading() ?? null; } catch { /* a frame without a heading beats no frame */ }
|
|
214
|
+
|
|
215
|
+
this.frames.push({
|
|
216
|
+
// In-memory: no file. native.vision resolves the id inside the native ring.
|
|
217
|
+
handle: { id: meta.id, mimeType: 'image/none', expiresAt: meta.expiresAt ?? null },
|
|
218
|
+
timestamp: meta.timestamp,
|
|
219
|
+
orientation: { headingDeg: heading?.headingDeg ?? null, pitchDeg: null, rollDeg: null },
|
|
220
|
+
quality: meta.quality ?? { blur: 'unknown', exposure: 'unknown', obstructed: null },
|
|
221
|
+
});
|
|
222
|
+
this.evict();
|
|
223
|
+
this.notify();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// The native ring's actual retention (ms). The engine's effective window is
|
|
227
|
+
// min(intent window, retention) — set by whoever owns the clamp semantics; this
|
|
228
|
+
// just stores the input so that logic has one source of truth to read.
|
|
229
|
+
private _retentionMs: number | null = null;
|
|
230
|
+
setRetentionMs(ms: number | null): void { this._retentionMs = ms && ms > 0 ? ms : null; }
|
|
231
|
+
retentionMs(): number | null { return this._retentionMs; }
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* How far back the timeline ACTUALLY reaches: `min(intent window, native
|
|
235
|
+
* retention)`.
|
|
236
|
+
*
|
|
237
|
+
* The intent asks for a window; native decides what it can afford to keep.
|
|
238
|
+
* Full-res luma at ~24fps is tens of megabytes for a few seconds, so the ring
|
|
239
|
+
* is memory-bounded and shrinks under pressure — `retentionMs` is native
|
|
240
|
+
* publishing what it is really holding.
|
|
241
|
+
*
|
|
242
|
+
* EVERY FRAME IN THE TIMELINE CAN BE ANALYSED. That is the one sentence a
|
|
243
|
+
* plugin author has to hold, and this clamp is what makes it true. The
|
|
244
|
+
* alternative — listing frames past retention marked "not analysable" — was
|
|
245
|
+
* considered and rejected: a caller would have to check a flag before every
|
|
246
|
+
* use, and forgetting throws mid-guidance on a frame the model can see
|
|
247
|
+
* listed. Present-but-unusable is the same trap as an empty rowset reading
|
|
248
|
+
* as "no rows matched".
|
|
249
|
+
*
|
|
250
|
+
* If per-frame history beyond retention is ever needed, the shape is an
|
|
251
|
+
* AGGREGATE on the snapshot ("the view has been unusable for N ms"), never a
|
|
252
|
+
* per-entry flag.
|
|
253
|
+
*/
|
|
254
|
+
effectiveWindowMs(): number {
|
|
255
|
+
const intent = this.profile().windowMs;
|
|
256
|
+
const retention = this._retentionMs;
|
|
257
|
+
return retention === null ? intent : Math.min(intent, retention);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Time-based, not count-based: the design specifies a rolling WINDOW.
|
|
262
|
+
*
|
|
263
|
+
* Evicts on the EFFECTIVE window, so a frame whose pixels native has already
|
|
264
|
+
* dropped leaves the timeline at the same moment. Without this, `frameById`
|
|
265
|
+
* would start returning null for frames still being listed.
|
|
266
|
+
*/
|
|
267
|
+
private evict(): void {
|
|
268
|
+
const cutoff = Date.now() - this.effectiveWindowMs();
|
|
269
|
+
this.frames = this.frames.filter(f => f.timestamp >= cutoff);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
timeline(): { windowMs: number; frames: VisualFrame[] } {
|
|
273
|
+
this.evict();
|
|
274
|
+
// The EFFECTIVE window, never the intent's. Reporting the intent would have
|
|
275
|
+
// a guide reason about 8 seconds of history it does not have — precisely
|
|
276
|
+
// the confusion the clamp exists to prevent.
|
|
277
|
+
return { windowMs: this.effectiveWindowMs(), frames: [...this.frames] };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
latest(): VisualFrame | null {
|
|
281
|
+
this.evict();
|
|
282
|
+
return this.frames.length ? this.frames[this.frames.length - 1] : null;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Resolve a frame handle id (from a VisualFrameRef) back to the live frame so
|
|
287
|
+
* the mechanical layer — native.vision — can read its on-device file. The
|
|
288
|
+
* whole contract of that layer is that it takes an id, never pixels; this is
|
|
289
|
+
* where an id becomes a `handle.uri` it can open. Returns null when the id is
|
|
290
|
+
* unknown OR its frame has aged out of the rolling window (the cache file may
|
|
291
|
+
* also be gone by then) — a real "I can't look at that frame" answer, which
|
|
292
|
+
* the caller must treat as distinct from "I looked and found nothing".
|
|
293
|
+
*/
|
|
294
|
+
frameById(id: string): VisualFrame | null {
|
|
295
|
+
this.evict();
|
|
296
|
+
return this.frames.find(f => f.handle.id === id) ?? null;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Record a MEASURED quality back onto a frame (native.vision.assessQuality).
|
|
301
|
+
* Frames are minted with `quality: unknown` because nothing looks at the
|
|
302
|
+
* pixels at capture time; this is how a real measurement replaces that so a
|
|
303
|
+
* later read of VisualFrameRef.quality reflects it. No-op if the frame has
|
|
304
|
+
* aged out — we never resurrect an evicted frame just to label it.
|
|
305
|
+
*/
|
|
306
|
+
setFrameQuality(id: string, quality: NonNullable<VisualFrame['quality']>): void {
|
|
307
|
+
const f = this.frames.find(fr => fr.handle.id === id);
|
|
308
|
+
if (!f) return;
|
|
309
|
+
f.quality = quality;
|
|
310
|
+
this.notify();
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// ── change notification ──
|
|
314
|
+
|
|
315
|
+
subscribe(fn: Listener): () => void {
|
|
316
|
+
this.listeners.add(fn);
|
|
317
|
+
return () => this.listeners.delete(fn);
|
|
318
|
+
}
|
|
319
|
+
private notify(): void {
|
|
320
|
+
for (const fn of this.listeners) { try { fn(); } catch { /* a bad listener is not the engine's problem */ } }
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** One engine per app. The camera is a singleton resource; so is this. */
|
|
325
|
+
export const visualEngine = new VisualEngine();
|
|
326
|
+
|
|
327
|
+
/** Freshness for a visual read — exported so the domain and host agree. */
|
|
328
|
+
export function frameQuality(frame: VisualFrame | null): StateQuality {
|
|
329
|
+
if (!frame) return { freshness: { state: 'unknown', observedAt: null, ageMs: null } };
|
|
330
|
+
const ageMs = Date.now() - frame.timestamp;
|
|
331
|
+
return {
|
|
332
|
+
freshness: {
|
|
333
|
+
// Tighter than the other domains on purpose: a two-second-old frame of a
|
|
334
|
+
// moving scene is not "fresh" in any useful sense.
|
|
335
|
+
state: ageMs < 2000 ? 'fresh' : ageMs < 10_000 ? 'aging' : 'stale',
|
|
336
|
+
observedAt: frame.timestamp,
|
|
337
|
+
ageMs,
|
|
338
|
+
},
|
|
339
|
+
confidence: { level: 'unknown' },
|
|
340
|
+
};
|
|
341
|
+
}
|
package/src/haptics.ts
CHANGED
|
@@ -24,4 +24,8 @@ export const expoHaptics: HapticsAPI = {
|
|
|
24
24
|
selection() {
|
|
25
25
|
Haptics.selectionAsync().catch(() => {});
|
|
26
26
|
},
|
|
27
|
+
// Shorthand for notification('error') — the form plugins actually call.
|
|
28
|
+
error() {
|
|
29
|
+
Haptics.notificationAsync(Haptics.NotificationFeedbackType.Error).catch(() => {});
|
|
30
|
+
},
|
|
27
31
|
};
|
package/src/index.ts
CHANGED
|
@@ -15,6 +15,8 @@ import {
|
|
|
15
15
|
setHapticsProvider,
|
|
16
16
|
setBundleStorage,
|
|
17
17
|
registerBundleModules,
|
|
18
|
+
setPluginNativeProvider,
|
|
19
|
+
type PluginNative,
|
|
18
20
|
} from '@ateam-ai-mobile/plugin-runtime';
|
|
19
21
|
import { setClipboardProvider } from '@ateam-ai-mobile/ui';
|
|
20
22
|
|
|
@@ -47,8 +49,18 @@ export function setupExpoProviders(opts?: {
|
|
|
47
49
|
* existing app so users keep their attachment previews / bundle cache. */
|
|
48
50
|
attachmentIndexKey?: string;
|
|
49
51
|
bundleIndexKey?: string;
|
|
52
|
+
/**
|
|
53
|
+
* The device-primitive implementation plugins receive as `props.device`.
|
|
54
|
+
* Build it with `createDeviceProvider(...)` from `@ateam-ai-mobile/expo/device`
|
|
55
|
+
* and pass it here so the SDK owns the implementation and this call registers
|
|
56
|
+
* it (setPluginNativeProvider). It is injected — not imported by this entry —
|
|
57
|
+
* because createDeviceProvider statically pulls optional expo modules that a
|
|
58
|
+
* host without them cannot build (same reason createBridge is a subpath).
|
|
59
|
+
*/
|
|
60
|
+
deviceProvider?: PluginNative;
|
|
50
61
|
}): { attachmentCache: ExpoAttachmentCache } {
|
|
51
62
|
setHapticsProvider(expoHaptics);
|
|
63
|
+
if (opts?.deviceProvider) setPluginNativeProvider(opts.deviceProvider);
|
|
52
64
|
setBundleStorage(new ExpoBundleStorage({ indexKey: opts?.bundleIndexKey }));
|
|
53
65
|
setClipboardProvider({
|
|
54
66
|
setString: (text: string) => ExpoClipboard.setStringAsync(text).then(() => {}),
|
|
@@ -65,3 +77,28 @@ export function setupExpoProviders(opts?: {
|
|
|
65
77
|
});
|
|
66
78
|
return { attachmentCache: new ExpoAttachmentCache({ indexKey: opts?.attachmentIndexKey }) };
|
|
67
79
|
}
|
|
80
|
+
|
|
81
|
+
// Device Bridge v2 — the semantic state engines this build provides.
|
|
82
|
+
// The host passes this to setDeviceStateProvider(); per-plugin gating by
|
|
83
|
+
// manifest.stateDomains happens in the SDK.
|
|
84
|
+
export { createDeviceState, IMPLEMENTED_DOMAINS } from './device-state';
|
|
85
|
+
export type { DeviceStateCollectors } from './device-state';
|
|
86
|
+
// The mounted half of the Visual engine. The app renders this ONCE; it draws
|
|
87
|
+
// nothing until a visual session is open. Required for deviceState.visual to
|
|
88
|
+
// produce frames at all — expo-camera has no headless capture.
|
|
89
|
+
// The rolling frame timeline. Exposed so the host's native.vision layer can
|
|
90
|
+
// resolve a frame id to its on-device file (frameById) and write a measured
|
|
91
|
+
// quality back onto the frame (setFrameQuality) — see visualEngine.ts.
|
|
92
|
+
export { visualEngine } from './device-state/visualEngine';
|
|
93
|
+
export type { VisualFrame } from './device-state/visualEngine';
|
|
94
|
+
// The mounted camera surface behind scanner.scanOnce() and
|
|
95
|
+
// camera.recordVideo(). Mount ONCE near the app root; it renders nothing
|
|
96
|
+
// until a plugin asks for a scan or a recording. Without it both APIs
|
|
97
|
+
// reject immediately with UNAVAILABLE — deliberately, since a plugin must
|
|
98
|
+
// never await a promise nothing can settle.
|
|
99
|
+
export { CameraTaskHost } from './camera/CameraTaskHost';
|
|
100
|
+
export { cameraTasks } from './camera/cameraTasks';
|
|
101
|
+
// Replay — run guidance logic against a RECORDED WALK instead of a phone.
|
|
102
|
+
// See docs/SPEC_device_walk_recorder.md for the fixture format.
|
|
103
|
+
export { createReplayDeviceState, parseWalkJsonl } from './device-state/replay';
|
|
104
|
+
export type { WalkFixture, WalkSample, WalkRefusal, ReplayClock, ReplaySession } from './device-state/replay';
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @ateam-ai-mobile/expo/oauth — the native OAuth provider for the relocated
|
|
3
|
+
* auth screens (LoginScreen / JoinScreen / OpenJoinScreen in @ateam-ai-mobile/ui).
|
|
4
|
+
*
|
|
5
|
+
* This is the ONLY place @react-native-google-signin + expo-apple-authentication
|
|
6
|
+
* are imported. The `ui` package stays native-free: its screens call
|
|
7
|
+
* getOAuthProvider() for the id_token acquisition + the real Apple HIG button.
|
|
8
|
+
*
|
|
9
|
+
* Deliberately a SUBPATH (`@ateam-ai-mobile/expo/oauth`), not part of the main
|
|
10
|
+
* entry — same rule as `/bridge` and `/device`: a host that doesn't install
|
|
11
|
+
* these native modules must still be able to build @ateam-ai-mobile/expo.
|
|
12
|
+
*
|
|
13
|
+
* Behaviour is ported verbatim from the app's old login.tsx so the iPhone
|
|
14
|
+
* parity gate sees no change:
|
|
15
|
+
* - Google: configure({ webClientId, iosClientId }) → hasPlayServices →
|
|
16
|
+
* signIn → normalize the v15-vs-older id_token shape. A missing iosClientId
|
|
17
|
+
* silently falls back to a web flow (unchanged); the caller gates on
|
|
18
|
+
* webClientId, so we never reach here without one for the login path.
|
|
19
|
+
* - a user-cancel (SIGN_IN_CANCELLED / IN_PROGRESS / ERR_REQUEST_CANCELED)
|
|
20
|
+
* resolves to null — the screen treats null as "no error", never a red flash.
|
|
21
|
+
*/
|
|
22
|
+
import React from 'react';
|
|
23
|
+
import { Platform } from 'react-native';
|
|
24
|
+
import {
|
|
25
|
+
GoogleSignin,
|
|
26
|
+
statusCodes,
|
|
27
|
+
type ConfigureParams,
|
|
28
|
+
} from '@react-native-google-signin/google-signin';
|
|
29
|
+
import * as AppleAuthentication from 'expo-apple-authentication';
|
|
30
|
+
import type { OAuthProvider, OAuthGoogleConfig } from '@ateam-ai-mobile/ui';
|
|
31
|
+
|
|
32
|
+
const NOOP = () => {};
|
|
33
|
+
|
|
34
|
+
export interface ExpoOAuthConfig {
|
|
35
|
+
/**
|
|
36
|
+
* The host's bundled Google client ids, used by the join / open-join screens
|
|
37
|
+
* that carry no tenant config (they pass no cfg to signInWithGoogle). The
|
|
38
|
+
* login screen passes the tenant's configured ids and overrides these.
|
|
39
|
+
*/
|
|
40
|
+
googleWebClientId?: string;
|
|
41
|
+
googleIosClientId?: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Build the native OAuth provider. Wire it once at startup:
|
|
46
|
+
* setOAuthProvider(createExpoOAuthProvider({ googleWebClientId, googleIosClientId }));
|
|
47
|
+
*/
|
|
48
|
+
export function createExpoOAuthProvider(host: ExpoOAuthConfig = {}): OAuthProvider {
|
|
49
|
+
return {
|
|
50
|
+
async signInWithGoogle(cfg?: OAuthGoogleConfig): Promise<string | null> {
|
|
51
|
+
const webClientId = cfg?.webClientId || host.googleWebClientId || '';
|
|
52
|
+
const iosClientId = cfg?.iosClientId || host.googleIosClientId || '';
|
|
53
|
+
// configure is idempotent + cheap; do it at sign-in time so the tenant's
|
|
54
|
+
// ids are always current (replaces login.tsx's configure effect).
|
|
55
|
+
const params: ConfigureParams = { webClientId };
|
|
56
|
+
if (iosClientId) (params as any).iosClientId = iosClientId;
|
|
57
|
+
GoogleSignin.configure(params);
|
|
58
|
+
try {
|
|
59
|
+
// android-only; no-op on iOS. Called defensively, as before.
|
|
60
|
+
await GoogleSignin.hasPlayServices({ showPlayServicesUpdateDialog: true });
|
|
61
|
+
const result: any = await GoogleSignin.signIn();
|
|
62
|
+
// v15+ returns { data: { idToken } }; older returns { idToken }.
|
|
63
|
+
return result?.data?.idToken || result?.idToken || null;
|
|
64
|
+
} catch (err: any) {
|
|
65
|
+
const code = err?.code;
|
|
66
|
+
if (code === statusCodes.SIGN_IN_CANCELLED || code === statusCodes.IN_PROGRESS) {
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
throw err;
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
|
|
73
|
+
async signInWithApple(): Promise<string | null> {
|
|
74
|
+
try {
|
|
75
|
+
const credential = await AppleAuthentication.signInAsync({
|
|
76
|
+
requestedScopes: [
|
|
77
|
+
AppleAuthentication.AppleAuthenticationScope.FULL_NAME,
|
|
78
|
+
AppleAuthentication.AppleAuthenticationScope.EMAIL,
|
|
79
|
+
],
|
|
80
|
+
});
|
|
81
|
+
return credential.identityToken || null;
|
|
82
|
+
} catch (err: any) {
|
|
83
|
+
// ERR_REQUEST_CANCELED = the user dismissed the sheet.
|
|
84
|
+
if (err?.code === 'ERR_REQUEST_CANCELED') return null;
|
|
85
|
+
throw err;
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
|
|
89
|
+
async isAppleAvailable(): Promise<boolean> {
|
|
90
|
+
if (Platform.OS !== 'ios') return false;
|
|
91
|
+
try {
|
|
92
|
+
return await AppleAuthentication.isAvailableAsync();
|
|
93
|
+
} catch {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
|
|
98
|
+
renderAppleButton({ onPress, disabled, style }): React.ReactNode {
|
|
99
|
+
// Apple's HIG requires their own styled button when Apple Sign-In sits
|
|
100
|
+
// alongside other providers. Same chrome the app shipped: CONTINUE /
|
|
101
|
+
// BLACK / fully-rounded. Layout (width/height) comes from the screen.
|
|
102
|
+
return (
|
|
103
|
+
<AppleAuthentication.AppleAuthenticationButton
|
|
104
|
+
buttonType={AppleAuthentication.AppleAuthenticationButtonType.CONTINUE}
|
|
105
|
+
buttonStyle={AppleAuthentication.AppleAuthenticationButtonStyle.BLACK}
|
|
106
|
+
cornerRadius={999}
|
|
107
|
+
style={style as any}
|
|
108
|
+
onPress={disabled ? NOOP : onPress}
|
|
109
|
+
/>
|
|
110
|
+
);
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Frame-quality analysis — the REFERENCE algorithm, in plain JS so it can be
|
|
2
|
+
// unit-tested exhaustively without a device. The native iOS (Objective-C++) and
|
|
3
|
+
// Android (Kotlin) implementations mirror this math and these thresholds exactly;
|
|
4
|
+
// this file is the spec they must match, and the thing the tests exercise.
|
|
5
|
+
//
|
|
6
|
+
// Input is a luma (grayscale) plane — one byte per pixel, 0..255. We subsample
|
|
7
|
+
// to a fixed grid so the work is bounded and thresholds are resolution-independent,
|
|
8
|
+
// then derive:
|
|
9
|
+
// exposure from mean luminance (good | dark | bright)
|
|
10
|
+
// obstructed from darkness + low variance (covered lens vs a real scene)
|
|
11
|
+
// blur from variance of the Laplacian (the classic focus metric)
|
|
12
|
+
|
|
13
|
+
export const THRESHOLDS = {
|
|
14
|
+
grid: 128,
|
|
15
|
+
exposureDark: 45,
|
|
16
|
+
exposureBright: 205,
|
|
17
|
+
obstructedMean: 22,
|
|
18
|
+
obstructedStd: 8,
|
|
19
|
+
blurSharp: 120,
|
|
20
|
+
blurSlight: 30,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @param {number} width
|
|
25
|
+
* @param {number} height
|
|
26
|
+
* @param {Uint8Array|number[]} luma row-major luma bytes
|
|
27
|
+
* @param {number} [rowStride] bytes per row (defaults to width)
|
|
28
|
+
* @returns {{blur:string, exposure:string, obstructed:boolean|null}}
|
|
29
|
+
*/
|
|
30
|
+
export function assessLuma(width, height, luma, rowStride) {
|
|
31
|
+
const stride = rowStride || width;
|
|
32
|
+
const S = THRESHOLDS.grid;
|
|
33
|
+
if (!width || !height || !luma || luma.length === 0) {
|
|
34
|
+
return { blur: 'unknown', exposure: 'unknown', obstructed: null };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Grid dimensions, capped at S. Sample coordinates span the FULL frame
|
|
38
|
+
// [0, width-1] / [0, height-1] — the first sample is the top-left pixel and
|
|
39
|
+
// the last is the bottom-right, so no edge (a bright window, an occlusion,
|
|
40
|
+
// strong detail) is ever omitted. A floor-step + capped-count scheme would
|
|
41
|
+
// systematically miss the right/bottom strip when the size isn't a multiple
|
|
42
|
+
// of S.
|
|
43
|
+
const cols = Math.min(S, width);
|
|
44
|
+
const rows = Math.min(S, height);
|
|
45
|
+
const sxAt = (gx) => (cols > 1 ? Math.round((gx * (width - 1)) / (cols - 1)) : 0);
|
|
46
|
+
const syAt = (gy) => (rows > 1 ? Math.round((gy * (height - 1)) / (rows - 1)) : 0);
|
|
47
|
+
|
|
48
|
+
const grid = new Uint8Array(cols * rows);
|
|
49
|
+
let n = 0, sum = 0, sumSq = 0;
|
|
50
|
+
for (let gy = 0; gy < rows; gy++) {
|
|
51
|
+
const rowBase = syAt(gy) * stride;
|
|
52
|
+
for (let gx = 0; gx < cols; gx++) {
|
|
53
|
+
const v = luma[rowBase + sxAt(gx)] & 0xff;
|
|
54
|
+
grid[gy * cols + gx] = v;
|
|
55
|
+
sum += v; sumSq += v * v; n++;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (n === 0) return { blur: 'unknown', exposure: 'unknown', obstructed: null };
|
|
59
|
+
|
|
60
|
+
const mean = sum / n;
|
|
61
|
+
const variance = Math.max(0, sumSq / n - mean * mean);
|
|
62
|
+
const std = Math.sqrt(variance);
|
|
63
|
+
|
|
64
|
+
let lapSum = 0, lapSumSq = 0, lapN = 0;
|
|
65
|
+
for (let y = 1; y < rows - 1; y++) {
|
|
66
|
+
for (let x = 1; x < cols - 1; x++) {
|
|
67
|
+
const i = y * cols + x;
|
|
68
|
+
const lap = 4 * grid[i] - grid[i - 1] - grid[i + 1] - grid[i - cols] - grid[i + cols];
|
|
69
|
+
lapSum += lap; lapSumSq += lap * lap; lapN++;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const lapVar = lapN > 0 ? Math.max(0, lapSumSq / lapN - (lapSum / lapN) ** 2) : 0;
|
|
73
|
+
|
|
74
|
+
const t = THRESHOLDS;
|
|
75
|
+
const exposure = mean < t.exposureDark ? 'dark' : (mean > t.exposureBright ? 'bright' : 'good');
|
|
76
|
+
const obstructed = mean < t.obstructedMean && std < t.obstructedStd;
|
|
77
|
+
const blur = lapVar > t.blurSharp ? 'sharp' : (lapVar > t.blurSlight ? 'slight' : 'severe');
|
|
78
|
+
|
|
79
|
+
return { blur, exposure, obstructed };
|
|
80
|
+
}
|