@ateam-ai-mobile/expo 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap.js +841 -323
- package/dist/bootstrap.js.map +1 -1
- package/dist/bootstrap.mjs +9 -7
- package/dist/bootstrap.mjs.map +1 -1
- package/dist/chunk-7FWYEHN5.mjs +203 -0
- package/dist/chunk-7FWYEHN5.mjs.map +1 -0
- package/dist/{chunk-4XYXJYN5.mjs → chunk-C7M5WAHA.mjs} +583 -298
- package/dist/chunk-C7M5WAHA.mjs.map +1 -0
- package/dist/{chunk-5HM6UA2K.mjs → chunk-GF2MPTAB.mjs} +83 -7
- package/dist/chunk-GF2MPTAB.mjs.map +1 -0
- package/dist/chunk-J6XRNX7P.mjs +19 -0
- package/dist/chunk-J6XRNX7P.mjs.map +1 -0
- package/dist/{chunk-TZXWVYYR.mjs → chunk-PCAEUX2N.mjs} +1 -1
- package/dist/chunk-PCAEUX2N.mjs.map +1 -0
- package/dist/{chunk-VQJPSLHP.mjs → chunk-Z5AI2SXX.mjs} +13 -4
- package/dist/chunk-Z5AI2SXX.mjs.map +1 -0
- package/dist/createBridge.js +41 -10
- package/dist/createBridge.js.map +1 -1
- package/dist/createBridge.mjs +5 -1
- package/dist/createBridge.mjs.map +1 -1
- package/dist/device/createDeviceProvider.js +278 -17
- package/dist/device/createDeviceProvider.js.map +1 -1
- package/dist/device/createDeviceProvider.mjs +5 -3
- package/dist/device-bridge/index.d.mts +13 -1
- package/dist/device-bridge/index.d.ts +13 -1
- package/dist/device-bridge/index.js +49 -12
- package/dist/device-bridge/index.js.map +1 -1
- package/dist/device-bridge/index.mjs +3 -1
- package/dist/index.d.mts +19 -3
- package/dist/index.d.ts +19 -3
- package/dist/index.js +594 -297
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -1
- package/dist/maps/index.d.mts +56 -0
- package/dist/maps/index.d.ts +56 -0
- package/dist/maps/index.js +95 -0
- package/dist/maps/index.js.map +1 -0
- package/dist/maps/index.mjs +17 -0
- package/dist/maps/index.mjs.map +1 -0
- package/dist/vision/index.d.mts +4 -2
- package/dist/vision/index.d.ts +4 -2
- package/dist/vision/index.js.map +1 -1
- package/dist/vision/index.mjs +1 -1
- package/expo-module.config.json +1 -1
- package/ios/AdaMaps/AdaMapsModule.swift +180 -0
- package/ios/AdaMaps.podspec +25 -0
- package/ios/AdaVision.podspec +4 -0
- package/ios/AdaVisionModule.swift +63 -9
- package/package.json +10 -4
- package/src/args.ts +56 -0
- package/src/createBridge.ts +3 -1
- package/src/device/createDeviceProvider.ts +88 -3
- package/src/device-bridge/collectors/location.ts +21 -9
- package/src/device-state/collectors.ts +5 -1
- package/src/device-state/domains/deviceContext.ts +5 -3
- package/src/device-state/domains/interaction.ts +2 -1
- package/src/device-state/domains/spatial.ts +200 -116
- package/src/device-state/domains/visual.ts +34 -13
- package/src/device-state/events.ts +49 -0
- package/src/device-state/headingSource.ts +43 -8
- package/src/device-state/index.ts +3 -2
- package/src/device-state/replay.ts +76 -11
- package/src/device-state/spatialEvents.ts +300 -0
- package/src/geo.ts +23 -0
- package/src/maps/index.ts +81 -0
- package/src/maps/normalize.ts +225 -0
- package/src/vision/index.ts +4 -2
- package/dist/chunk-4XYXJYN5.mjs.map +0 -1
- package/dist/chunk-5HM6UA2K.mjs.map +0 -1
- package/dist/chunk-TZXWVYYR.mjs.map +0 -1
- package/dist/chunk-VQJPSLHP.mjs.map +0 -1
|
@@ -154,6 +154,17 @@ enum LumaMath {
|
|
|
154
154
|
// frame/sec, not 24). Emits `visionFrame` {id, ts, quality, expiresAt} per
|
|
155
155
|
// published frame; `qualityById` / (later) readText resolve the id inside here.
|
|
156
156
|
// The buffer/pixels never cross to JS.
|
|
157
|
+
//
|
|
158
|
+
// ORIENTATION — frames are UPRIGHT PORTRAIT at the source. The sensor delivers
|
|
159
|
+
// landscape buffers; `orientUpright` rotates the data-output connection to
|
|
160
|
+
// portrait (the host app is portrait-locked) every time configureAndRun creates
|
|
161
|
+
// that connection, so AVFoundation hands over already-upright buffers. The luma
|
|
162
|
+
// quality read, the ring's CGImage and every Vision / Core ML call therefore see
|
|
163
|
+
// the same upright image the user is pointing at, and boxes come back in upright
|
|
164
|
+
// coordinates (x = 0 is the user's LEFT). This is the ONE rotation mechanism:
|
|
165
|
+
// VNImageRequestHandler is always given the default .up orientation — passing an
|
|
166
|
+
// orientation there as well would rotate twice. Back camera only (position:
|
|
167
|
+
// .back), so nothing is mirrored: image-left is scene-left.
|
|
157
168
|
final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
158
169
|
private let session = AVCaptureSession()
|
|
159
170
|
private let queue = DispatchQueue(label: "ai.ateam.adavision.session")
|
|
@@ -175,11 +186,41 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
175
186
|
private var ring: [Retained] = []
|
|
176
187
|
private static let ciContext = CIContext(options: nil)
|
|
177
188
|
|
|
189
|
+
// A straight copy: the buffer is already upright (see ORIENTATION above), so no
|
|
190
|
+
// rotation happens here.
|
|
178
191
|
private static func makeCGImage(_ pb: CVPixelBuffer) -> CGImage? {
|
|
179
192
|
let ci = CIImage(cvPixelBuffer: pb)
|
|
180
193
|
return ciContext.createCGImage(ci, from: ci.extent)
|
|
181
194
|
}
|
|
182
195
|
|
|
196
|
+
/// Rotate a video-data-output connection so its sample buffers arrive UPRIGHT
|
|
197
|
+
/// in portrait. AVFoundation physically rotates the buffers, so every consumer
|
|
198
|
+
/// downstream gets the upright image. 90° is portrait (videoOrientation
|
|
199
|
+
/// .portrait below iOS 17); 0° is the sensor's native landscape. Returns false
|
|
200
|
+
/// when the connection can't rotate — frames would then stay sensor-landscape.
|
|
201
|
+
private static func orientUpright(_ connection: AVCaptureConnection) -> Bool {
|
|
202
|
+
if #available(iOS 17.0, *) {
|
|
203
|
+
if connection.isVideoRotationAngleSupported(90) {
|
|
204
|
+
connection.videoRotationAngle = 90
|
|
205
|
+
return true
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
if connection.isVideoOrientationSupported {
|
|
209
|
+
connection.videoOrientation = .portrait
|
|
210
|
+
return true
|
|
211
|
+
}
|
|
212
|
+
return false
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/// Vision's normalized box (origin BOTTOM-left) → the API contract: normalized
|
|
216
|
+
/// 0..1, origin TOP-left, in the upright frame. A y-flip is the whole transform
|
|
217
|
+
/// because the image handed to Vision is already upright (.up) — no rotation
|
|
218
|
+
/// here, by design (see ORIENTATION above). One helper for readText AND
|
|
219
|
+
/// detectObjects, so the two can never disagree about where a thing is.
|
|
220
|
+
private static func topLeftBox(_ bb: CGRect) -> [String: CGFloat] {
|
|
221
|
+
return ["x": bb.origin.x, "y": 1.0 - bb.origin.y - bb.height, "width": bb.width, "height": bb.height]
|
|
222
|
+
}
|
|
223
|
+
|
|
183
224
|
func retentionMs() -> Int { return ringCap * intervalMs }
|
|
184
225
|
|
|
185
226
|
func qualityById(_ id: String) -> [String: Any]? {
|
|
@@ -210,6 +251,8 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
210
251
|
self.generation += 1 // new session → old ids can no longer resolve
|
|
211
252
|
self.session.beginConfiguration()
|
|
212
253
|
self.session.sessionPreset = .medium
|
|
254
|
+
// Back camera only. Offering the front camera would also need the connection
|
|
255
|
+
// mirrored (isVideoMirrored) so boxes match the mirrored view the user sees.
|
|
213
256
|
guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
|
|
214
257
|
let input = try? AVCaptureDeviceInput(device: device), self.session.canAddInput(input) else {
|
|
215
258
|
self.session.commitConfiguration()
|
|
@@ -225,6 +268,12 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
225
268
|
onReady(NSError(domain: "AdaVision", code: 5, userInfo: [NSLocalizedDescriptionKey: "Cannot attach video output"])); return
|
|
226
269
|
}
|
|
227
270
|
self.session.addOutput(out)
|
|
271
|
+
// addOutput just created this output's connection; stop() discards it and
|
|
272
|
+
// every start builds a fresh one, so rotating here covers every (re)creation.
|
|
273
|
+
let upright = out.connection(with: .video).map { VisionSession.orientUpright($0) } ?? false
|
|
274
|
+
if !upright {
|
|
275
|
+
NSLog("AdaVision: cannot rotate the camera data output to portrait — frames stay sensor-landscape, so box x/y will not match left/right")
|
|
276
|
+
}
|
|
228
277
|
self.session.commitConfiguration()
|
|
229
278
|
self.running = true
|
|
230
279
|
self.session.startRunning()
|
|
@@ -293,13 +342,10 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
293
342
|
if let err = err { completion(nil, err); return }
|
|
294
343
|
let out: [[String: Any]] = (req.results as? [VNRecognizedTextObservation])?.compactMap { obs in
|
|
295
344
|
guard let top = obs.topCandidates(1).first else { return nil }
|
|
296
|
-
let bb = obs.boundingBox // normalized, origin BOTTOM-left
|
|
297
345
|
return [
|
|
298
346
|
"text": top.string,
|
|
299
347
|
"confidence": Double(top.confidence),
|
|
300
|
-
|
|
301
|
-
// transform is a later step; this is the y-flip.
|
|
302
|
-
"box": ["x": bb.origin.x, "y": 1.0 - bb.origin.y - bb.height, "width": bb.width, "height": bb.height],
|
|
348
|
+
"box": VisionSession.topLeftBox(obs.boundingBox),
|
|
303
349
|
]
|
|
304
350
|
} ?? []
|
|
305
351
|
completion(out, nil)
|
|
@@ -356,8 +402,9 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
356
402
|
}()
|
|
357
403
|
|
|
358
404
|
// MARK: detectObjects — on-device object detection (bundled Core ML), by id.
|
|
359
|
-
// Same latest-wins discipline as readText/describe (serial `queue`). Boxes
|
|
360
|
-
// normalised 0..1,
|
|
405
|
+
// Same latest-wins discipline as readText/describe (serial `queue`). Boxes go
|
|
406
|
+
// through the same topLeftBox as readText: normalised 0..1, TOP-left origin, in
|
|
407
|
+
// the upright portrait frame.
|
|
361
408
|
func detectObjects(_ id: String, maxItems: Int, minConfidence: Double, completion: @escaping ([[String: Any]]?, Error?) -> Void) {
|
|
362
409
|
queue.async {
|
|
363
410
|
guard let cg = self.ring.first(where: { $0.id == id })?.image else {
|
|
@@ -376,11 +423,10 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
376
423
|
let out: [[String: Any]] = objs.compactMap { obs -> (Double, [String: Any])? in
|
|
377
424
|
// .labels is sorted by confidence; take the top class for this box.
|
|
378
425
|
guard let top = obs.labels.first, Double(top.confidence) >= minConfidence else { return nil }
|
|
379
|
-
let bb = obs.boundingBox // normalized, origin BOTTOM-left
|
|
380
426
|
return (Double(top.confidence), [
|
|
381
427
|
"label": top.identifier,
|
|
382
428
|
"confidence": Double(top.confidence),
|
|
383
|
-
"box":
|
|
429
|
+
"box": VisionSession.topLeftBox(obs.boundingBox),
|
|
384
430
|
])
|
|
385
431
|
}
|
|
386
432
|
.sorted { $0.0 > $1.0 }
|
|
@@ -388,7 +434,11 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
388
434
|
.map { $0.1 }
|
|
389
435
|
completion(Array(out), nil)
|
|
390
436
|
}
|
|
391
|
-
|
|
437
|
+
// Vision stretches the whole (portrait, 3:4) frame onto the model's square
|
|
438
|
+
// 640² input and maps the boxes back itself. The stretch is per-axis, so a
|
|
439
|
+
// normalised box means the same thing in the frame as in the model input —
|
|
440
|
+
// no letterbox math here, whatever the frame's aspect.
|
|
441
|
+
request.imageCropAndScaleOption = .scaleFill
|
|
392
442
|
do {
|
|
393
443
|
try VNImageRequestHandler(cgImage: cg, options: [:]).perform([request])
|
|
394
444
|
} catch {
|
|
@@ -403,6 +453,10 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
403
453
|
// A native view bound to the SAME AVCaptureSession the headless VisionSession
|
|
404
454
|
// runs — so it shows exactly what the reader sees, WITHOUT opening a second
|
|
405
455
|
// camera. The real Guide never mounts this; it exists only so a tester can aim.
|
|
456
|
+
// The preview layer has its OWN connection (portrait by default); orientUpright
|
|
457
|
+
// rotates only the data output's, so both show the same upright portrait. Note
|
|
458
|
+
// resizeAspectFill crops the frame's sides on a tall screen, so a thing at the
|
|
459
|
+
// frame's left/right edge can be detected yet sit just outside the preview.
|
|
406
460
|
final class AdaVisionSessionHub {
|
|
407
461
|
static let shared = AdaVisionSessionHub()
|
|
408
462
|
private(set) weak var session: AVCaptureSession?
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ateam-ai-mobile/expo",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "Expo-backed providers for the ADAS SDK — haptics, bundle disk cache, attachment cache, device bridge.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
"build": "tsup"
|
|
10
10
|
},
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@ateam-ai-mobile/core": "0.1.
|
|
13
|
-
"@ateam-ai-mobile/plugin-runtime": "0.1.
|
|
14
|
-
"@ateam-ai-mobile/ui": "0.1.
|
|
12
|
+
"@ateam-ai-mobile/core": "0.1.8",
|
|
13
|
+
"@ateam-ai-mobile/plugin-runtime": "0.1.8",
|
|
14
|
+
"@ateam-ai-mobile/ui": "0.1.8"
|
|
15
15
|
},
|
|
16
16
|
"peerDependencies": {
|
|
17
17
|
"@react-native-async-storage/async-storage": "*",
|
|
@@ -131,6 +131,12 @@
|
|
|
131
131
|
"import": "./dist/vision/index.mjs",
|
|
132
132
|
"require": "./dist/vision/index.js"
|
|
133
133
|
},
|
|
134
|
+
"./maps": {
|
|
135
|
+
"react-native": "./src/maps/index.ts",
|
|
136
|
+
"types": "./dist/maps/index.d.ts",
|
|
137
|
+
"import": "./dist/maps/index.mjs",
|
|
138
|
+
"require": "./dist/maps/index.js"
|
|
139
|
+
},
|
|
134
140
|
"./device": {
|
|
135
141
|
"react-native": "./src/device/createDeviceProvider.ts",
|
|
136
142
|
"types": "./dist/device/createDeviceProvider.d.ts",
|
package/src/args.ts
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// Argument checks for native verbs — one place, shared by every capability that
|
|
2
|
+
// hands caller numbers to Swift.
|
|
3
|
+
//
|
|
4
|
+
// Why this exists: expo-modules-core converts a JS number to a Swift Int with
|
|
5
|
+
// `Int(double.rounded())`, which TRAPS on NaN, ±Infinity or anything past
|
|
6
|
+
// Int.max. A plugin (or a model driving a device tool) passing `limit: Infinity`
|
|
7
|
+
// or `maxItems: NaN` would therefore kill the host app. Every count that crosses
|
|
8
|
+
// the bridge is checked here first and refused BY NAME instead.
|
|
9
|
+
//
|
|
10
|
+
// No imports on purpose: it runs under plain `node --test`.
|
|
11
|
+
|
|
12
|
+
/** A malformed call, refused before anything is sent. Retrying will not help;
|
|
13
|
+
* fixing the call will. */
|
|
14
|
+
export type ArgumentError = Error & { code: 'INVALID_ARGUMENT' };
|
|
15
|
+
|
|
16
|
+
/** What the caller passed, short enough to quote back in a refusal. NaN and
|
|
17
|
+
* Infinity are spelled out, unquoted (JSON would print them as null). */
|
|
18
|
+
export const shown = (v: unknown): string => {
|
|
19
|
+
let s: string | undefined;
|
|
20
|
+
try {
|
|
21
|
+
s = JSON.stringify(v, (_k, x) => (typeof x === 'number' && !Number.isFinite(x) ? `\u0001${x}\u0001` : x))
|
|
22
|
+
?.replace(/"\\u0001(NaN|-?Infinity)\\u0001"/g, '$1');
|
|
23
|
+
} catch { /* cyclic — fall back to String */ }
|
|
24
|
+
s = s ?? String(v);
|
|
25
|
+
return s.length > 80 ? `${s.slice(0, 77)}...` : s;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** `<capability>.<verb>: <problem>; <what was not done>.` with code INVALID_ARGUMENT. */
|
|
29
|
+
export function refuseArgument(capability: string, verb: string, problem: string, nothing: string): ArgumentError {
|
|
30
|
+
const e = new Error(`${capability}.${verb}: ${problem}; ${nothing}.`) as ArgumentError;
|
|
31
|
+
e.code = 'INVALID_ARGUMENT';
|
|
32
|
+
return e;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A count bound for a Swift Int: absent → `def`; otherwise a finite number ≥ 1,
|
|
36
|
+
* floored and capped at `max`. Anything else refuses by name. */
|
|
37
|
+
export function countArg(
|
|
38
|
+
capability: string, verb: string, name: string, value: unknown, def: number, max: number, nothing: string,
|
|
39
|
+
): number {
|
|
40
|
+
const v = value ?? def;
|
|
41
|
+
if (typeof v !== 'number' || !Number.isFinite(v) || v < 1) {
|
|
42
|
+
throw refuseArgument(capability, verb, `${name} must be a number from 1 (at most ${max} are used), not ${shown(value)}`, nothing);
|
|
43
|
+
}
|
|
44
|
+
return Math.min(max, Math.floor(v));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** A 0..1 fraction (a confidence): absent → `def`; NaN or out of range refuses. */
|
|
48
|
+
export function fractionArg(
|
|
49
|
+
capability: string, verb: string, name: string, value: unknown, def: number, nothing: string,
|
|
50
|
+
): number {
|
|
51
|
+
const v = value ?? def;
|
|
52
|
+
if (typeof v !== 'number' || !Number.isFinite(v) || v < 0 || v > 1) {
|
|
53
|
+
throw refuseArgument(capability, verb, `${name} must be a number from 0 to 1, not ${shown(value)}`, nothing);
|
|
54
|
+
}
|
|
55
|
+
return v;
|
|
56
|
+
}
|
package/src/createBridge.ts
CHANGED
|
@@ -16,6 +16,8 @@ import * as ExpoSMS from 'expo-sms';
|
|
|
16
16
|
import * as ExpoLinking from 'expo-linking';
|
|
17
17
|
import { Platform } from 'react-native';
|
|
18
18
|
import * as Device from 'expo-device';
|
|
19
|
+
// The SDK's one reverse geocode — the same call native.maps.reverseGeocode makes.
|
|
20
|
+
import { reverseGeocode } from './maps';
|
|
19
21
|
|
|
20
22
|
import type { Bridge } from '@ateam-ai-mobile/plugin-runtime';
|
|
21
23
|
|
|
@@ -361,7 +363,7 @@ export function createBridge(connectorId: string, deviceId: string): Bridge {
|
|
|
361
363
|
accuracy: ExpoLocation.Accuracy.Balanced,
|
|
362
364
|
});
|
|
363
365
|
|
|
364
|
-
const
|
|
366
|
+
const geo = await reverseGeocode({
|
|
365
367
|
latitude: location.coords.latitude,
|
|
366
368
|
longitude: location.coords.longitude,
|
|
367
369
|
});
|
|
@@ -26,6 +26,13 @@ import AsyncStorage from '@react-native-async-storage/async-storage';
|
|
|
26
26
|
// native.vision: the on-device analyzer (ada-vision module) — resolve a frame id
|
|
27
27
|
// to its local pixels, measure/read/classify on-device, never off the phone.
|
|
28
28
|
import * as adaVision from '../vision';
|
|
29
|
+
// native.maps: place search + walking routes (ada-maps module, MapKit) and the
|
|
30
|
+
// SDK's one reverse geocode. NOT on-device — queries go to Apple's servers.
|
|
31
|
+
import * as adaMaps from '../maps';
|
|
32
|
+
import { countArg, fractionArg, refuseArgument, shown } from '../args';
|
|
33
|
+
import {
|
|
34
|
+
coordinateArg, mapsFailure, searchArgs, toMapsPlaces, toMapsRoute, toReverseGeocodedPlace,
|
|
35
|
+
} from '../maps/normalize';
|
|
29
36
|
// Device-bridge owns every native module. plugin-sdk delegates so RN
|
|
30
37
|
// plugins never import native modules directly. When adding a new
|
|
31
38
|
// capability: add it to device-bridge, import here, declare in types.ts,
|
|
@@ -67,6 +74,7 @@ import type {
|
|
|
67
74
|
NfcAPI,
|
|
68
75
|
BluetoothAPI,
|
|
69
76
|
VisionAPI,
|
|
77
|
+
MapsAPI,
|
|
70
78
|
} from '@ateam-ai-mobile/plugin-runtime';
|
|
71
79
|
|
|
72
80
|
class CapabilityUnavailableError extends Error {
|
|
@@ -758,18 +766,28 @@ function createVisionAPI(): VisionAPI {
|
|
|
758
766
|
// pixels). [] = looked, found no text (a real finding); throws = could not
|
|
759
767
|
// look (unknown/aged-out id). describe (VNClassifyImage) is the next pass.
|
|
760
768
|
async readText(frameId: string, opts?: { languages?: string[] }) {
|
|
761
|
-
|
|
769
|
+
const langs = opts?.languages;
|
|
770
|
+
if (langs != null && (!Array.isArray(langs) || langs.some((l) => typeof l !== 'string'))) {
|
|
771
|
+
throw refuseArgument('vision', 'readText', `languages must be an array of language codes, not ${shown(langs)}`, 'nothing was read');
|
|
772
|
+
}
|
|
773
|
+
return adaVision.readText(frameId, langs);
|
|
762
774
|
},
|
|
763
775
|
// describe: on-device labels (VNClassifyImage), not prose — the sentence is
|
|
764
776
|
// the skill's job. [] = looked, recognised nothing.
|
|
765
777
|
async describe(frameId: string, opts?: { maxLabels?: number }) {
|
|
766
|
-
|
|
778
|
+
// Checked here: a NaN/Infinity count would trap Swift's Int conversion (src/args.ts).
|
|
779
|
+
return adaVision.describe(frameId, countArg('vision', 'describe', 'maxLabels', opts?.maxLabels, 5, 50, 'nothing was labelled'));
|
|
767
780
|
},
|
|
768
781
|
// detectObjects: on-device located objects (bundled detector), each with a
|
|
769
782
|
// box — what a guidance loop reads to say WHERE a thing is, not just that the
|
|
770
783
|
// scene contains it. [] = looked, found nothing; throws = could not look.
|
|
771
784
|
async detectObjects(frameId: string, opts?: { maxItems?: number; minConfidence?: number }) {
|
|
772
|
-
|
|
785
|
+
const nothing = 'nothing was detected';
|
|
786
|
+
return adaVision.detectObjects(
|
|
787
|
+
frameId,
|
|
788
|
+
countArg('vision', 'detectObjects', 'maxItems', opts?.maxItems, 10, 50, nothing),
|
|
789
|
+
fractionArg('vision', 'detectObjects', 'minConfidence', opts?.minConfidence, 0.25, nothing),
|
|
790
|
+
);
|
|
773
791
|
},
|
|
774
792
|
// assessQuality: resolve the frame id to its LOCAL file, measure on-device,
|
|
775
793
|
// and record the result back onto the frame so a later read of
|
|
@@ -792,6 +810,72 @@ function createVisionAPI(): VisionAPI {
|
|
|
792
810
|
};
|
|
793
811
|
}
|
|
794
812
|
|
|
813
|
+
// ── Maps — place search, reverse geocoding, walking routes ──
|
|
814
|
+
//
|
|
815
|
+
// NOT on-device, unlike vision: the query and the coordinates it is given go to
|
|
816
|
+
// Apple's map servers (the platform geocoder on Android). It never reads the
|
|
817
|
+
// user's location — the caller passes every coordinate — so it is not a second
|
|
818
|
+
// door to where the user is.
|
|
819
|
+
//
|
|
820
|
+
// search/route are the ada-maps module (MapKit, iOS only). reverseGeocode is the
|
|
821
|
+
// SDK's one reverse geocode (expo-location), the same call the location
|
|
822
|
+
// collector and the connector bridge make — so it works on Android and on a
|
|
823
|
+
// build without ada-maps.
|
|
824
|
+
function createMapsAPI(): MapsAPI {
|
|
825
|
+
// Refuse BY NAME before any native call: what this build or platform is
|
|
826
|
+
// missing, and that nothing left the phone.
|
|
827
|
+
const refuseIfMissing = (verb: string, nothing: string) => {
|
|
828
|
+
const missing = adaMaps.missingReason();
|
|
829
|
+
if (missing) {
|
|
830
|
+
throw new CapabilityUnavailableError(`maps.${verb}`, `maps.${verb} needs ${missing}; ${nothing}.`);
|
|
831
|
+
}
|
|
832
|
+
};
|
|
833
|
+
|
|
834
|
+
// Every argument is checked (searchArgs / coordinateArg) BEFORE the missing
|
|
835
|
+
// check and before anything crosses the bridge: a remote bundle is plain JS,
|
|
836
|
+
// and a malformed call refuses as INVALID_ARGUMENT instead of crashing Swift
|
|
837
|
+
// (NaN limit) or searching around 0,0 (`{ lat, lng }`).
|
|
838
|
+
return {
|
|
839
|
+
async isSupported() { return adaMaps.isSupported(); },
|
|
840
|
+
// [] = Apple looked and found nothing (a real finding); throws = could not
|
|
841
|
+
// look. The defaults (radiusM 3000, limit 8) live in searchArgs.
|
|
842
|
+
async search(query, opts) {
|
|
843
|
+
const args = searchArgs(query, opts);
|
|
844
|
+
refuseIfMissing('search', 'nothing was searched');
|
|
845
|
+
const raw = await adaMaps
|
|
846
|
+
.search(args.query, args.near, args.radiusM, args.limit)
|
|
847
|
+
.catch(err => { throw mapsFailure('search', err); });
|
|
848
|
+
return toMapsPlaces(raw, args.near);
|
|
849
|
+
},
|
|
850
|
+
// null = nothing at that point; throws = could not look.
|
|
851
|
+
async reverseGeocode(at) {
|
|
852
|
+
const point = coordinateArg('reverseGeocode', 'at', at, 'nothing was looked up');
|
|
853
|
+
const address = await adaMaps
|
|
854
|
+
.reverseGeocode(point)
|
|
855
|
+
.catch(err => { throw mapsFailure('reverseGeocode', err); });
|
|
856
|
+
return address ? toReverseGeocodedPlace(point, address) : null;
|
|
857
|
+
},
|
|
858
|
+
// null = Apple found no route. Never an empty 0 m route, which would read as
|
|
859
|
+
// "you have arrived". A remote bundle can pass any string as `mode`, so a
|
|
860
|
+
// mode this build cannot honour refuses rather than silently walking.
|
|
861
|
+
async route(from, to, opts) {
|
|
862
|
+
if (opts?.mode !== undefined && opts.mode !== 'walking') {
|
|
863
|
+
throw new CapabilityUnavailableError(
|
|
864
|
+
'maps.route',
|
|
865
|
+
`maps.route supports walking only, not "${String(opts.mode)}"; no route was requested.`,
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
const a = coordinateArg('route', 'from', from, 'no route was requested');
|
|
869
|
+
const b = coordinateArg('route', 'to', to, 'no route was requested');
|
|
870
|
+
refuseIfMissing('route', 'no route was requested');
|
|
871
|
+
const raw = await adaMaps
|
|
872
|
+
.route(a, b)
|
|
873
|
+
.catch(err => { throw mapsFailure('route', err); });
|
|
874
|
+
return raw ? toMapsRoute(raw) : null;
|
|
875
|
+
},
|
|
876
|
+
};
|
|
877
|
+
}
|
|
878
|
+
|
|
795
879
|
// ── Provider factory ───────────────────────────────────────────
|
|
796
880
|
|
|
797
881
|
export interface DeviceProviderConfig {
|
|
@@ -828,5 +912,6 @@ export function createDeviceProvider(config?: DeviceProviderConfig): PluginNativ
|
|
|
828
912
|
nfc: createNfcAPI(),
|
|
829
913
|
bluetooth: createBluetoothAPI(),
|
|
830
914
|
vision: createVisionAPI(),
|
|
915
|
+
maps: createMapsAPI(),
|
|
831
916
|
};
|
|
832
917
|
}
|
|
@@ -4,11 +4,9 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as ExpoLocation from 'expo-location';
|
|
7
|
-
|
|
8
|
-
//
|
|
9
|
-
|
|
10
|
-
// returns it, so the shape follows the installed expo-location.
|
|
11
|
-
type LocationGeocodedAddress = Awaited<ReturnType<typeof ExpoLocation.reverseGeocodeAsync>>[number];
|
|
7
|
+
import type { MapsAddress } from '@ateam-ai-mobile/plugin-runtime';
|
|
8
|
+
// The SDK's one reverse geocode — the same call native.maps.reverseGeocode makes.
|
|
9
|
+
import { reverseGeocode } from '../../maps';
|
|
12
10
|
import type { LocationData } from '../types';
|
|
13
11
|
|
|
14
12
|
/**
|
|
@@ -50,7 +48,7 @@ export async function collectCurrent(): Promise<LocationData> {
|
|
|
50
48
|
let locationType = 'unknown';
|
|
51
49
|
|
|
52
50
|
try {
|
|
53
|
-
const
|
|
51
|
+
const geocoded = await reverseGeocode({
|
|
54
52
|
latitude: location.coords.latitude,
|
|
55
53
|
longitude: location.coords.longitude,
|
|
56
54
|
});
|
|
@@ -177,11 +175,25 @@ function nonNegative(v: unknown): number | undefined {
|
|
|
177
175
|
return typeof v === 'number' && v >= 0 ? v : undefined;
|
|
178
176
|
}
|
|
179
177
|
|
|
178
|
+
const WATCH_TIER = {
|
|
179
|
+
navigation: ExpoLocation.Accuracy.BestForNavigation,
|
|
180
|
+
high: ExpoLocation.Accuracy.High,
|
|
181
|
+
balanced: ExpoLocation.Accuracy.Balanced,
|
|
182
|
+
low: ExpoLocation.Accuracy.Low,
|
|
183
|
+
} as const;
|
|
184
|
+
|
|
180
185
|
/**
|
|
181
186
|
* Stream position updates. Returns an unsubscribe function — call it, or the
|
|
182
187
|
* GPS keeps running and drains the battery.
|
|
188
|
+
*
|
|
189
|
+
* `accuracy` picks the GPS tier. The default stays Balanced (≈100 m on iOS),
|
|
190
|
+
* which is right for "which city block" and wrong for "turn at the next corner":
|
|
191
|
+
* a navigation session asks for BestForNavigation and pays the battery for it.
|
|
183
192
|
*/
|
|
184
|
-
export function watchPosition(
|
|
193
|
+
export function watchPosition(
|
|
194
|
+
cb: (pos: Position) => void,
|
|
195
|
+
opts?: { accuracy?: keyof typeof WATCH_TIER },
|
|
196
|
+
): () => void {
|
|
185
197
|
let subscription: { remove(): void } | null = null;
|
|
186
198
|
let cancelled = false;
|
|
187
199
|
|
|
@@ -190,7 +202,7 @@ export function watchPosition(cb: (pos: Position) => void): () => void {
|
|
|
190
202
|
await ensurePermission();
|
|
191
203
|
if (cancelled) return;
|
|
192
204
|
subscription = await ExpoLocation.watchPositionAsync(
|
|
193
|
-
{ accuracy:
|
|
205
|
+
{ accuracy: WATCH_TIER[opts?.accuracy ?? 'balanced'] },
|
|
194
206
|
fix => cb(toPosition(fix)),
|
|
195
207
|
);
|
|
196
208
|
// Unsubscribed while we were still asking for permission.
|
|
@@ -209,7 +221,7 @@ export function watchPosition(cb: (pos: Position) => void): () => void {
|
|
|
209
221
|
|
|
210
222
|
// ─── Helpers ───
|
|
211
223
|
|
|
212
|
-
function classifyLocation(geocoded:
|
|
224
|
+
function classifyLocation(geocoded: MapsAddress): string {
|
|
213
225
|
// Simple heuristic — can be enhanced with user-labeled places from memory
|
|
214
226
|
const name = (geocoded.name || '').toLowerCase();
|
|
215
227
|
const street = (geocoded.street || '').toLowerCase();
|
|
@@ -6,11 +6,15 @@
|
|
|
6
6
|
* in one of them. Injection also matches how the rest of this SDK takes host
|
|
7
7
|
* capabilities, and lets a test drive the engines without an Expo runtime.
|
|
8
8
|
*/
|
|
9
|
+
/** GPS tier for a position stream: turn-by-turn, precise, city-block, coarse. */
|
|
10
|
+
export type WatchAccuracy = 'navigation' | 'high' | 'balanced' | 'low';
|
|
11
|
+
|
|
9
12
|
export interface DeviceStateCollectors {
|
|
10
13
|
connectivity?: { collectConnectivity(): Promise<any> };
|
|
11
14
|
location?: {
|
|
12
15
|
getPosition(opts?: { accuracy?: 'low' | 'high' }): Promise<any>;
|
|
13
|
-
|
|
16
|
+
/** `accuracy` is the GPS tier the stream runs at (battery vs precision). */
|
|
17
|
+
watchPosition(cb: (pos: any) => void, opts?: { accuracy?: WatchAccuracy }): () => void;
|
|
14
18
|
hasPermission(): Promise<boolean>;
|
|
15
19
|
};
|
|
16
20
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import * as ExpoBattery from 'expo-battery';
|
|
4
4
|
import type { DeviceState, DeviceContextSnapshot } from '@ateam-ai-mobile/plugin-runtime';
|
|
5
5
|
import { qualityFrom, qualityUnknown } from '../quality';
|
|
6
|
+
import { stateEvent } from '../events';
|
|
6
7
|
|
|
7
8
|
export function createDeviceContextDomain(): DeviceState['deviceContext'] {
|
|
8
9
|
return {
|
|
@@ -43,11 +44,12 @@ export function createDeviceContextDomain(): DeviceState['deviceContext'] {
|
|
|
43
44
|
|
|
44
45
|
// Real events, not a placeholder: expo-battery exposes native listeners,
|
|
45
46
|
// so a plugin that subscribes genuinely hears about a change rather than
|
|
46
|
-
// sitting on a subscription that can never fire.
|
|
47
|
+
// sitting on a subscription that can never fire. Nudges without a payload:
|
|
48
|
+
// no Device Context payload schema is decided, so the subscriber re-reads.
|
|
47
49
|
subscribe(listener) {
|
|
48
50
|
const subs = [
|
|
49
|
-
ExpoBattery.addBatteryLevelListener(() => listener(
|
|
50
|
-
ExpoBattery.addBatteryStateListener(() => listener(
|
|
51
|
+
ExpoBattery.addBatteryLevelListener(() => listener(stateEvent('deviceContext', 'batteryLevelChanged'))),
|
|
52
|
+
ExpoBattery.addBatteryStateListener(() => listener(stateEvent('deviceContext', 'chargingChanged'))),
|
|
51
53
|
];
|
|
52
54
|
return { unsubscribe: () => subs.forEach(x => x?.remove?.()) };
|
|
53
55
|
},
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { AppState } from 'react-native';
|
|
4
4
|
import type { DeviceState, InteractionSnapshot } from '@ateam-ai-mobile/plugin-runtime';
|
|
5
5
|
import { qualityFrom } from '../quality';
|
|
6
|
+
import { stateEvent } from '../events';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Lifecycle only, and honest about the rest.
|
|
@@ -51,7 +52,7 @@ export function createInteractionDomain(): DeviceState['interaction'] {
|
|
|
51
52
|
// which is the one part of this domain that works.
|
|
52
53
|
subscribe(listener) {
|
|
53
54
|
const sub = AppState.addEventListener('change', next => {
|
|
54
|
-
listener(
|
|
55
|
+
listener(stateEvent('interaction', 'appStateChanged', { appState: mapState(next as string) }));
|
|
55
56
|
});
|
|
56
57
|
return { unsubscribe: () => sub.remove() };
|
|
57
58
|
},
|