@ateam-ai-mobile/expo 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/build.gradle +16 -0
- package/android/src/main/java/expo/modules/adavision/AdaVisionModule.kt +75 -0
- package/dist/attachmentCache-DhQTqi2E.d.mts +18 -0
- package/dist/attachmentCache-DhQTqi2E.d.ts +18 -0
- package/dist/bootstrap.d.mts +95 -0
- package/dist/bootstrap.d.ts +95 -0
- package/dist/bootstrap.js +2473 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/bootstrap.mjs +64 -0
- package/dist/bootstrap.mjs.map +1 -0
- package/dist/chunk-4BMWVN25.mjs +91 -0
- package/dist/chunk-4BMWVN25.mjs.map +1 -0
- package/dist/chunk-4XYXJYN5.mjs +1513 -0
- package/dist/chunk-4XYXJYN5.mjs.map +1 -0
- package/dist/chunk-5HM6UA2K.mjs +571 -0
- package/dist/chunk-5HM6UA2K.mjs.map +1 -0
- package/dist/chunk-6LXF3JVX.mjs +96 -0
- package/dist/chunk-6LXF3JVX.mjs.map +1 -0
- package/dist/chunk-FF6ZZZXH.mjs +71 -0
- package/dist/chunk-FF6ZZZXH.mjs.map +1 -0
- package/dist/chunk-KOXF2FZM.mjs +92 -0
- package/dist/chunk-KOXF2FZM.mjs.map +1 -0
- package/dist/chunk-L4V7OCJ3.mjs +71 -0
- package/dist/chunk-L4V7OCJ3.mjs.map +1 -0
- package/dist/chunk-TZXWVYYR.mjs +67 -0
- package/dist/chunk-TZXWVYYR.mjs.map +1 -0
- package/dist/chunk-VQJPSLHP.mjs +1801 -0
- package/dist/chunk-VQJPSLHP.mjs.map +1 -0
- package/dist/chunk-WMJKH4XE.mjs +34 -0
- package/dist/chunk-WMJKH4XE.mjs.map +1 -0
- package/dist/chunk-XQSY6MHP.mjs +213 -0
- package/dist/chunk-XQSY6MHP.mjs.map +1 -0
- package/dist/createBridge.mjs +2 -0
- package/dist/createBridge.mjs.map +1 -1
- package/dist/device/createDeviceProvider.d.mts +23 -0
- package/dist/device/createDeviceProvider.d.ts +23 -0
- package/dist/device/createDeviceProvider.js +967 -0
- package/dist/device/createDeviceProvider.js.map +1 -0
- package/dist/device/createDeviceProvider.mjs +17 -0
- package/dist/device/createDeviceProvider.mjs.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.d.mts +10 -0
- package/dist/device-bridge/collectors/appEngagement.d.ts +10 -0
- package/dist/device-bridge/collectors/appEngagement.js +128 -0
- package/dist/device-bridge/collectors/appEngagement.js.map +1 -0
- package/dist/device-bridge/collectors/appEngagement.mjs +12 -0
- package/dist/device-bridge/collectors/appEngagement.mjs.map +1 -0
- package/dist/device-bridge/collectors/health.d.mts +24 -0
- package/dist/device-bridge/collectors/health.d.ts +24 -0
- package/dist/device-bridge/collectors/health.js +238 -0
- package/dist/device-bridge/collectors/health.js.map +1 -0
- package/dist/device-bridge/collectors/health.mjs +18 -0
- package/dist/device-bridge/collectors/health.mjs.map +1 -0
- package/dist/device-bridge/cookies.d.mts +49 -0
- package/dist/device-bridge/cookies.d.ts +49 -0
- package/dist/device-bridge/cookies.js +90 -0
- package/dist/device-bridge/cookies.js.map +1 -0
- package/dist/device-bridge/cookies.mjs +14 -0
- package/dist/device-bridge/cookies.mjs.map +1 -0
- package/dist/device-bridge/index.d.mts +483 -0
- package/dist/device-bridge/index.d.ts +483 -0
- package/dist/device-bridge/index.js +2084 -0
- package/dist/device-bridge/index.js.map +1 -0
- package/dist/device-bridge/index.mjs +32 -0
- package/dist/device-bridge/index.mjs.map +1 -0
- package/dist/device-bridge/nfc.d.mts +30 -0
- package/dist/device-bridge/nfc.d.ts +30 -0
- package/dist/device-bridge/nfc.js +109 -0
- package/dist/device-bridge/nfc.js.map +1 -0
- package/dist/device-bridge/nfc.mjs +14 -0
- package/dist/device-bridge/nfc.mjs.map +1 -0
- package/dist/index.d.mts +493 -18
- package/dist/index.d.ts +493 -18
- package/dist/index.js +1409 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +23 -194
- package/dist/index.mjs.map +1 -1
- package/dist/oauth/expoOAuth.d.mts +18 -0
- package/dist/oauth/expoOAuth.d.ts +18 -0
- package/dist/oauth/expoOAuth.js +102 -0
- package/dist/oauth/expoOAuth.js.map +1 -0
- package/dist/oauth/expoOAuth.mjs +8 -0
- package/dist/oauth/expoOAuth.mjs.map +1 -0
- package/dist/types-BRanJYTn.d.mts +138 -0
- package/dist/types-BRanJYTn.d.ts +138 -0
- package/dist/vision/index.d.mts +81 -0
- package/dist/vision/index.d.ts +81 -0
- package/dist/vision/index.js +101 -0
- package/dist/vision/index.js.map +1 -0
- package/dist/vision/index.mjs +28 -0
- package/dist/vision/index.mjs.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AdaVision.podspec +28 -0
- package/ios/AdaVisionModule.swift +432 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/metadata.json +773 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/analytics/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/coremldata.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/model.mil +720 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model0/weights/weight.bin +0 -0
- package/ios/Resources/yolov8n-oiv7.mlmodelc/model1/coremldata.bin +0 -0
- package/package.json +107 -7
- package/src/bootstrap.ts +159 -0
- package/src/camera/CameraTaskHost.tsx +175 -0
- package/src/camera/cameraTasks.ts +129 -0
- package/src/device/createDeviceProvider.ts +832 -0
- package/src/device-bridge/actions/calendar-actions.ts +217 -0
- package/src/device-bridge/actions/dnd.ts +74 -0
- package/src/device-bridge/actions/health-actions.ts +58 -0
- package/src/device-bridge/actions/index.ts +12 -0
- package/src/device-bridge/actions/messaging.ts +74 -0
- package/src/device-bridge/actions/navigation.ts +88 -0
- package/src/device-bridge/actions/queue-poller.ts +151 -0
- package/src/device-bridge/background.ts +137 -0
- package/src/device-bridge/biometrics.ts +64 -0
- package/src/device-bridge/capabilities.ts +218 -0
- package/src/device-bridge/collectors/appEngagement.ts +120 -0
- package/src/device-bridge/collectors/battery.ts +83 -0
- package/src/device-bridge/collectors/calendar.ts +162 -0
- package/src/device-bridge/collectors/camera.ts +27 -0
- package/src/device-bridge/collectors/connectivity.ts +60 -0
- package/src/device-bridge/collectors/contacts.ts +200 -0
- package/src/device-bridge/collectors/health.ts +299 -0
- package/src/device-bridge/collectors/index.ts +12 -0
- package/src/device-bridge/collectors/location.ts +224 -0
- package/src/device-bridge/collectors/mood.ts +141 -0
- package/src/device-bridge/collectors/notifications.ts +70 -0
- package/src/device-bridge/cookies.ts +89 -0
- package/src/device-bridge/dynamic-connector.ts +422 -0
- package/src/device-bridge/index.ts +467 -0
- package/src/device-bridge/nfc.ts +85 -0
- package/src/device-bridge/plugin.ts +71 -0
- package/src/device-bridge/relay-client.ts +237 -0
- package/src/device-bridge/scanner.ts +42 -0
- package/src/device-bridge/sharing.ts +35 -0
- package/src/device-bridge/types.ts +326 -0
- package/src/device-state/activityClassifier.ts +151 -0
- package/src/device-state/activityEngine.ts +252 -0
- package/src/device-state/collectors.ts +31 -0
- package/src/device-state/domains/activity.ts +93 -0
- package/src/device-state/domains/capability.ts +60 -0
- package/src/device-state/domains/connectivity.ts +60 -0
- package/src/device-state/domains/deviceContext.ts +55 -0
- package/src/device-state/domains/interaction.ts +59 -0
- package/src/device-state/domains/spatial.ts +283 -0
- package/src/device-state/domains/visual.ts +100 -0
- package/src/device-state/headingSource.ts +84 -0
- package/src/device-state/index.ts +77 -0
- package/src/device-state/quality.ts +60 -0
- package/src/device-state/replay.ts +220 -0
- package/src/device-state/turnDetector.ts +121 -0
- package/src/device-state/visualEngine.ts +341 -0
- package/src/haptics.ts +4 -0
- package/src/index.ts +37 -0
- package/src/oauth/expoOAuth.tsx +113 -0
- package/src/vision/analysis.js +80 -0
- package/src/vision/index.ts +144 -0
- package/src/vision/ring.js +35 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/vision/index.ts
|
|
21
|
+
var vision_exports = {};
|
|
22
|
+
__export(vision_exports, {
|
|
23
|
+
CameraPreview: () => CameraPreview,
|
|
24
|
+
assessQuality: () => assessQuality,
|
|
25
|
+
describe: () => describe,
|
|
26
|
+
detectObjects: () => detectObjects,
|
|
27
|
+
isSupported: () => isSupported,
|
|
28
|
+
onVisionFrame: () => onVisionFrame,
|
|
29
|
+
qualityById: () => qualityById,
|
|
30
|
+
readText: () => readText,
|
|
31
|
+
startVision: () => startVision,
|
|
32
|
+
stopVision: () => stopVision,
|
|
33
|
+
visionRetentionMs: () => visionRetentionMs
|
|
34
|
+
});
|
|
35
|
+
module.exports = __toCommonJS(vision_exports);
|
|
36
|
+
var import_expo_modules_core = require("expo-modules-core");
|
|
37
|
+
var import_expo = require("expo");
|
|
38
|
+
var CameraPreview = (() => {
|
|
39
|
+
try {
|
|
40
|
+
return (0, import_expo.requireNativeView)("AdaVision");
|
|
41
|
+
} catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
})();
|
|
45
|
+
var native = (0, import_expo_modules_core.requireOptionalNativeModule)("AdaVision");
|
|
46
|
+
function isSupported() {
|
|
47
|
+
try {
|
|
48
|
+
return native?.isSupported() ?? false;
|
|
49
|
+
} catch {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
async function assessQuality(uri) {
|
|
54
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
55
|
+
return native.assessQuality(uri);
|
|
56
|
+
}
|
|
57
|
+
async function startVision(intervalMs = 1e3) {
|
|
58
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
59
|
+
return native.startVision(intervalMs);
|
|
60
|
+
}
|
|
61
|
+
function stopVision() {
|
|
62
|
+
native?.stopVision?.();
|
|
63
|
+
}
|
|
64
|
+
function onVisionFrame(cb) {
|
|
65
|
+
if (!native) return { remove() {
|
|
66
|
+
} };
|
|
67
|
+
return native.addListener("visionFrame", cb);
|
|
68
|
+
}
|
|
69
|
+
function visionRetentionMs() {
|
|
70
|
+
return native?.visionRetentionMs?.() ?? 0;
|
|
71
|
+
}
|
|
72
|
+
function qualityById(id) {
|
|
73
|
+
return native?.qualityById?.(id) ?? null;
|
|
74
|
+
}
|
|
75
|
+
async function readText(id, languages) {
|
|
76
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
77
|
+
return native.readText(id, languages ?? null);
|
|
78
|
+
}
|
|
79
|
+
async function describe(id, maxLabels = 5) {
|
|
80
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
81
|
+
return native.describe(id, maxLabels);
|
|
82
|
+
}
|
|
83
|
+
async function detectObjects(id, maxItems = 10, minConfidence = 0.25) {
|
|
84
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
85
|
+
return native.detectObjects(id, maxItems, minConfidence);
|
|
86
|
+
}
|
|
87
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
88
|
+
0 && (module.exports = {
|
|
89
|
+
CameraPreview,
|
|
90
|
+
assessQuality,
|
|
91
|
+
describe,
|
|
92
|
+
detectObjects,
|
|
93
|
+
isSupported,
|
|
94
|
+
onVisionFrame,
|
|
95
|
+
qualityById,
|
|
96
|
+
readText,
|
|
97
|
+
startVision,
|
|
98
|
+
stopVision,
|
|
99
|
+
visionRetentionMs
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/vision/index.ts"],"sourcesContent":["// ada-vision — on-device image understanding, our own native code (no camera lib).\n//\n// On-device, frames never leave the phone:\n// assessQuality(uri) — one-shot quality of a LOCAL frame file.\n// startVision()/onVisionFrame — the headless camera session that publishes a\n// timeline frame (id + quality, never pixels) at a\n// cadence; readText/describe/detectObjects resolve\n// a published id ON-DEVICE.\n//\n// The luma math mirrors ./analysis.js (which has the exhaustive unit tests).\n//\n// requireOptionalNativeModule: on a binary WITHOUT the native module, the import\n// is null, isSupported() is false, and the session calls refuse cleanly (no crash).\n\nimport type { ComponentType } from 'react';\nimport { requireOptionalNativeModule } from 'expo-modules-core';\nimport { requireNativeView } from 'expo';\n\n/** TEST-ONLY live preview bound to the same headless session (so a tester can\n * aim). The real Guide runs headless and never mounts this. Null on a binary\n * without the native view, so importing this module never crashes. */\nexport const CameraPreview: ComponentType<any> | null = (() => {\n try { return requireNativeView('AdaVision'); } catch { return null; }\n})();\n\nexport interface RawFrameQuality {\n blur: 'sharp' | 'slight' | 'severe' | 'unknown';\n exposure: 'good' | 'dark' | 'bright' | 'unknown';\n /** true = lens covered; false = clearly not; null = not assessed. */\n obstructed: boolean | null;\n}\n\ninterface Subscription { remove(): void }\n\n/** A published timeline frame from the vision session (metadata only — never\n * pixels; the buffer stays in the native ring, resolved by id). */\nexport interface VisionFrame {\n id: string;\n ts: number;\n expiresAt: number;\n blur: RawFrameQuality['blur'];\n exposure: RawFrameQuality['exposure'];\n obstructed: RawFrameQuality['obstructed'];\n}\n\n/** One OCR result on a frame — matches the SDK's VisionText. */\nexport interface VisionText {\n text: string;\n /** 0..1 recogniser confidence (never a 1.0 filler). */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\n/** One classifier label — a vocabulary term, NOT prose. Matches VisionLabel. */\nexport interface VisionLabel {\n label: string;\n confidence: number | null;\n}\n\n/** One detected object — a located item with a box, from the bundled detector. */\nexport interface VisionObject {\n /** Open-Images class name (e.g. \"Car\", \"Traffic light\", \"Building\", \"Person\"). */\n label: string;\n /** 0..1 detector confidence. */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\ninterface AdaVisionNative {\n isSupported(): boolean;\n assessQuality(uri: string): Promise<RawFrameQuality>;\n startVision(intervalMs: number): Promise<void>;\n stopVision(): void;\n visionRetentionMs(): number;\n qualityById(id: string): RawFrameQuality | null;\n readText(id: string, languages: string[] | null): Promise<VisionText[]>;\n describe(id: string, maxLabels: number): Promise<VisionLabel[]>;\n detectObjects(id: string, maxItems: number, minConfidence: number): Promise<VisionObject[]>;\n addListener(event: 'visionFrame', cb: (f: VisionFrame) => void): Subscription;\n}\n\nconst native = requireOptionalNativeModule<AdaVisionNative>('AdaVision');\n\n/** Is the native vision module present in this binary? */\nexport function isSupported(): boolean {\n try { return native?.isSupported() ?? false; } catch { return false; }\n}\n\n/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */\nexport async function assessQuality(uri: string): Promise<RawFrameQuality> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.assessQuality(uri);\n}\n\n// ── Vision session (feeds the Visual Engine) ──\n\n/** Start the vision session — publishes a timeline frame every `intervalMs`. */\nexport async function startVision(intervalMs = 1000): Promise<void> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.startVision(intervalMs);\n}\n\n/** Stop the vision session and release the camera. */\nexport function stopVision(): void {\n native?.stopVision?.();\n}\n\n/** Subscribe to published timeline frames (id + quality + expiry; no pixels). */\nexport function onVisionFrame(cb: (f: VisionFrame) => void): Subscription {\n if (!native) return { remove() {} };\n return native.addListener('visionFrame', cb);\n}\n\n/** How far back the native ring can still resolve a frame id (ms). */\nexport function visionRetentionMs(): number {\n return native?.visionRetentionMs?.() ?? 0;\n}\n\n/** Resolve a published frame id → its measured quality (null if evicted). */\nexport function qualityById(id: string): RawFrameQuality | null {\n return native?.qualityById?.(id) ?? null;\n}\n\n/** On-device OCR on a published frame. [] = looked, no text; throws = couldn't look. */\nexport async function readText(id: string, languages?: string[]): Promise<VisionText[]> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.readText(id, languages ?? null);\n}\n\n/** On-device image classification → labels (not prose). [] = nothing recognised. */\nexport async function describe(id: string, maxLabels = 5): Promise<VisionLabel[]> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.describe(id, maxLabels);\n}\n\n/** On-device object detection (bundled YOLOv8n, Open-Images vocabulary) → located\n * items with boxes. [] = looked, found nothing above `minConfidence`; throws =\n * couldn't look (id aged out, or the model isn't in this build). */\nexport async function detectObjects(id: string, maxItems = 10, minConfidence = 0.25): Promise<VisionObject[]> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.detectObjects(id, maxItems, minConfidence);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAeA,+BAA4C;AAC5C,kBAAkC;AAK3B,IAAM,iBAA4C,MAAM;AAC7D,MAAI;AAAE,eAAO,+BAAkB,WAAW;AAAA,EAAG,QAAQ;AAAE,WAAO;AAAA,EAAM;AACtE,GAAG;AA4DH,IAAM,aAAS,sDAA6C,WAAW;AAGhE,SAAS,cAAuB;AACrC,MAAI;AAAE,WAAO,QAAQ,YAAY,KAAK;AAAA,EAAO,QAAQ;AAAE,WAAO;AAAA,EAAO;AACvE;AAGA,eAAsB,cAAc,KAAuC;AACzE,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,cAAc,GAAG;AACjC;AAKA,eAAsB,YAAY,aAAa,KAAqB;AAClE,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,YAAY,UAAU;AACtC;AAGO,SAAS,aAAmB;AACjC,UAAQ,aAAa;AACvB;AAGO,SAAS,cAAc,IAA4C;AACxE,MAAI,CAAC,OAAQ,QAAO,EAAE,SAAS;AAAA,EAAC,EAAE;AAClC,SAAO,OAAO,YAAY,eAAe,EAAE;AAC7C;AAGO,SAAS,oBAA4B;AAC1C,SAAO,QAAQ,oBAAoB,KAAK;AAC1C;AAGO,SAAS,YAAY,IAAoC;AAC9D,SAAO,QAAQ,cAAc,EAAE,KAAK;AACtC;AAGA,eAAsB,SAAS,IAAY,WAA6C;AACtF,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,SAAS,IAAI,aAAa,IAAI;AAC9C;AAGA,eAAsB,SAAS,IAAY,YAAY,GAA2B;AAChF,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,SAAS,IAAI,SAAS;AACtC;AAKA,eAAsB,cAAc,IAAY,WAAW,IAAI,gBAAgB,MAA+B;AAC5G,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,cAAc,IAAI,UAAU,aAAa;AACzD;","names":[]}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CameraPreview,
|
|
3
|
+
assessQuality,
|
|
4
|
+
describe,
|
|
5
|
+
detectObjects,
|
|
6
|
+
isSupported,
|
|
7
|
+
onVisionFrame,
|
|
8
|
+
qualityById,
|
|
9
|
+
readText,
|
|
10
|
+
startVision,
|
|
11
|
+
stopVision,
|
|
12
|
+
visionRetentionMs
|
|
13
|
+
} from "../chunk-TZXWVYYR.mjs";
|
|
14
|
+
import "../chunk-WMJKH4XE.mjs";
|
|
15
|
+
export {
|
|
16
|
+
CameraPreview,
|
|
17
|
+
assessQuality,
|
|
18
|
+
describe,
|
|
19
|
+
detectObjects,
|
|
20
|
+
isSupported,
|
|
21
|
+
onVisionFrame,
|
|
22
|
+
qualityById,
|
|
23
|
+
readText,
|
|
24
|
+
startVision,
|
|
25
|
+
stopVision,
|
|
26
|
+
visionRetentionMs
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Pod::Spec.new do |s|
|
|
2
|
+
s.name = 'AdaVision'
|
|
3
|
+
s.version = '1.0.0'
|
|
4
|
+
s.summary = 'On-device frame quality (blur / exposure / obstructed).'
|
|
5
|
+
s.description = 'Local Expo module: on-device image-quality statistics. Frames never leave the phone.'
|
|
6
|
+
s.author = ''
|
|
7
|
+
s.homepage = 'https://ateam-ai.com'
|
|
8
|
+
s.platforms = { :ios => '16.0' }
|
|
9
|
+
s.source = { git: '' }
|
|
10
|
+
s.static_framework = true
|
|
11
|
+
|
|
12
|
+
s.dependency 'ExpoModulesCore'
|
|
13
|
+
# Our own headless camera stream reads the live luma buffer via AVFoundation —
|
|
14
|
+
# no third-party camera library. Vision + CoreML drive OCR / scene labels /
|
|
15
|
+
# the bundled object detector — all on-device.
|
|
16
|
+
s.frameworks = 'AVFoundation', 'CoreMedia', 'CoreVideo', 'Vision', 'CoreML'
|
|
17
|
+
|
|
18
|
+
s.pod_target_xcconfig = {
|
|
19
|
+
'DEFINES_MODULE' => 'YES',
|
|
20
|
+
'SWIFT_COMPILATION_MODE' => 'wholemodule'
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
s.source_files = "**/*.{h,m,mm,swift,hpp,cpp}"
|
|
24
|
+
# Precompiled on-device object-detection model (YOLOv8n, Open-Images vocabulary).
|
|
25
|
+
# Copied verbatim into the app bundle (already an .mlmodelc — no build-time
|
|
26
|
+
# compile step), loaded via Bundle.main at runtime. Frames never leave the phone.
|
|
27
|
+
s.resources = "Resources/*.mlmodelc"
|
|
28
|
+
end
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
import ExpoModulesCore
|
|
2
|
+
import AVFoundation
|
|
3
|
+
import QuartzCore
|
|
4
|
+
import Vision
|
|
5
|
+
import CoreML
|
|
6
|
+
import UIKit
|
|
7
|
+
|
|
8
|
+
// On-device frame quality + the headless vision session.
|
|
9
|
+
//
|
|
10
|
+
// assessQuality(uri) — one-shot quality of a LOCAL frame file.
|
|
11
|
+
// startVision()/visionFrame — the FAST path: a headless AVCaptureVideoDataOutput
|
|
12
|
+
// reads each live frame's luma plane, publishes a
|
|
13
|
+
// timeline frame (id + quality, never pixels) at a
|
|
14
|
+
// cadence, and resolves readText/describe/detectObjects
|
|
15
|
+
// by id INSIDE native. No photo, no JPEG, no file.
|
|
16
|
+
public class AdaVisionModule: Module {
|
|
17
|
+
private let vision = VisionSession()
|
|
18
|
+
|
|
19
|
+
public func definition() -> ModuleDefinition {
|
|
20
|
+
Name("AdaVision")
|
|
21
|
+
Events("visionFrame")
|
|
22
|
+
|
|
23
|
+
// Test-only live preview: a native view bound to the SAME headless session
|
|
24
|
+
// (not a second camera). The real Guide runs headless.
|
|
25
|
+
View(AdaVisionPreviewView.self) {}
|
|
26
|
+
|
|
27
|
+
Function("isSupported") { () -> Bool in true }
|
|
28
|
+
|
|
29
|
+
// ── the real vision session (feeds the Visual Engine) ──
|
|
30
|
+
// Runs the headless luma stream, PUBLISHES a timeline frame at `intervalMs`
|
|
31
|
+
// (retaining that frame in a bounded ring keyed by id, for resolve-by-id),
|
|
32
|
+
// and emits `visionFrame` {id, ts, quality} per published frame. Analysis
|
|
33
|
+
// still runs every camera frame; only the published frames are retained, so
|
|
34
|
+
// the ring is cheap (≈ one frame/sec, not 24/sec).
|
|
35
|
+
AsyncFunction("startVision") { (intervalMs: Int, promise: Promise) in
|
|
36
|
+
self.vision.start(intervalMs: max(200, intervalMs), emit: { [weak self] f in self?.sendEvent("visionFrame", f) }) { err in
|
|
37
|
+
if let err = err { promise.reject("AdaVision", err.localizedDescription) } else { promise.resolve(nil) }
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
Function("stopVision") { () in self.vision.stop() }
|
|
41
|
+
/** How far back the native ring can still resolve a frame id (ms). */
|
|
42
|
+
Function("visionRetentionMs") { () -> Int in self.vision.retentionMs() }
|
|
43
|
+
/** Resolve a published frame id → its measured quality (null if evicted). */
|
|
44
|
+
Function("qualityById") { (id: String) -> [String: Any]? in self.vision.qualityById(id) }
|
|
45
|
+
/** On-device OCR on a published frame (Apple Vision). Rejects if the id is
|
|
46
|
+
* unknown/aged out — "could not look", never an empty "found nothing". */
|
|
47
|
+
AsyncFunction("readText") { (id: String, languages: [String]?, promise: Promise) in
|
|
48
|
+
self.vision.readText(id, languages: languages) { results, err in
|
|
49
|
+
if let err = err { promise.reject("AdaVision", err.localizedDescription) }
|
|
50
|
+
else { promise.resolve(results ?? []) }
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** On-device image classification (Apple Vision) → LABELS, not prose. */
|
|
54
|
+
AsyncFunction("describe") { (id: String, maxLabels: Int, promise: Promise) in
|
|
55
|
+
self.vision.describe(id, maxLabels: maxLabels) { results, err in
|
|
56
|
+
if let err = err { promise.reject("AdaVision", err.localizedDescription) }
|
|
57
|
+
else { promise.resolve(results ?? []) }
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** On-device OBJECT DETECTION on a published frame (bundled YOLOv8n Core ML,
|
|
61
|
+
* Open-Images vocabulary — car, bus, traffic light, building, door, person…).
|
|
62
|
+
* Returns located items {label, confidence, box}. NMS is baked into the model,
|
|
63
|
+
* so Vision hands back decoded objects directly. Rejects if the id aged out or
|
|
64
|
+
* the detector model isn't in this build. */
|
|
65
|
+
AsyncFunction("detectObjects") { (id: String, maxItems: Int, minConfidence: Double, promise: Promise) in
|
|
66
|
+
self.vision.detectObjects(id, maxItems: maxItems, minConfidence: minConfidence) { results, err in
|
|
67
|
+
if let err = err { promise.reject("AdaVision", err.localizedDescription) }
|
|
68
|
+
else { promise.resolve(results ?? []) }
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
AsyncFunction("assessQuality") { (uri: String) throws -> [String: Any] in
|
|
73
|
+
guard let image = AdaVisionModule.loadImage(uri) else {
|
|
74
|
+
throw Exception(name: "AdaVisionUnreadable", description: "Could not read frame at \(uri)")
|
|
75
|
+
}
|
|
76
|
+
return AdaVisionModule.assess(image)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
OnDestroy { self.vision.stop() }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// MARK: file-backed one-shot (kept from v1)
|
|
83
|
+
static func loadImage(_ uri: String) -> UIImage? {
|
|
84
|
+
if let url = URL(string: uri), url.isFileURL, let data = try? Data(contentsOf: url) {
|
|
85
|
+
return UIImage(data: data)
|
|
86
|
+
}
|
|
87
|
+
let path = uri.hasPrefix("file://") ? String(uri.dropFirst("file://".count)) : uri
|
|
88
|
+
return UIImage(contentsOfFile: path)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
static func assess(_ image: UIImage) -> [String: Any] {
|
|
92
|
+
let side = 128
|
|
93
|
+
let unknown: [String: Any] = ["blur": "unknown", "exposure": "unknown", "obstructed": NSNull()]
|
|
94
|
+
guard let cg = image.cgImage else { return unknown }
|
|
95
|
+
var buf = [UInt8](repeating: 0, count: side * side)
|
|
96
|
+
guard let ctx = CGContext(data: &buf, width: side, height: side, bitsPerComponent: 8, bytesPerRow: side,
|
|
97
|
+
space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue) else { return unknown }
|
|
98
|
+
ctx.draw(cg, in: CGRect(x: 0, y: 0, width: side, height: side))
|
|
99
|
+
return LumaMath.assess(luma: buf, width: side, height: side, rowStride: side)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// MARK: - Shared luma math (mirrors modules/ada-vision/analysis.js exactly)
|
|
104
|
+
enum LumaMath {
|
|
105
|
+
static func assess(luma: UnsafePointer<UInt8>, width: Int, height: Int, rowStride: Int) -> [String: Any] {
|
|
106
|
+
let S = 128
|
|
107
|
+
if width == 0 || height == 0 { return ["blur": "unknown", "exposure": "unknown", "obstructed": NSNull()] }
|
|
108
|
+
// Sample coords span the FULL frame [0,width-1]/[0,height-1] (mirrors analysis.js).
|
|
109
|
+
let cols = min(S, width)
|
|
110
|
+
let rows = min(S, height)
|
|
111
|
+
func sxAt(_ gx: Int) -> Int { cols > 1 ? Int((Double(gx) * Double(width - 1) / Double(cols - 1)).rounded()) : 0 }
|
|
112
|
+
func syAt(_ gy: Int) -> Int { rows > 1 ? Int((Double(gy) * Double(height - 1) / Double(rows - 1)).rounded()) : 0 }
|
|
113
|
+
|
|
114
|
+
var grid = [UInt8](repeating: 0, count: cols * rows)
|
|
115
|
+
var n = 0; var sum = 0.0; var sumSq = 0.0
|
|
116
|
+
for gy in 0..<rows {
|
|
117
|
+
let rowBase = syAt(gy) * rowStride
|
|
118
|
+
for gx in 0..<cols {
|
|
119
|
+
let v = luma[rowBase + sxAt(gx)]
|
|
120
|
+
grid[gy * cols + gx] = v
|
|
121
|
+
let d = Double(v); sum += d; sumSq += d * d; n += 1
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if n == 0 { return ["blur": "unknown", "exposure": "unknown", "obstructed": NSNull()] }
|
|
125
|
+
|
|
126
|
+
let mean = sum / Double(n)
|
|
127
|
+
let std = (max(0, sumSq / Double(n) - mean * mean)).squareRoot()
|
|
128
|
+
|
|
129
|
+
var lapSum = 0.0; var lapSumSq = 0.0; var lapN = 0
|
|
130
|
+
for y in 1..<max(1, rows - 1) {
|
|
131
|
+
for x in 1..<max(1, cols - 1) {
|
|
132
|
+
let i = y * cols + x
|
|
133
|
+
let lap = 4.0 * Double(grid[i]) - Double(grid[i - 1]) - Double(grid[i + 1]) - Double(grid[i - cols]) - Double(grid[i + cols])
|
|
134
|
+
lapSum += lap; lapSumSq += lap * lap; lapN += 1
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
let lapVar = lapN > 0 ? max(0, lapSumSq / Double(lapN) - pow(lapSum / Double(lapN), 2)) : 0
|
|
138
|
+
|
|
139
|
+
let exposure = mean < 45 ? "dark" : (mean > 205 ? "bright" : "good")
|
|
140
|
+
let obstructed = (mean < 22 && std < 8)
|
|
141
|
+
let blur = lapVar > 120 ? "sharp" : (lapVar > 30 ? "slight" : "severe")
|
|
142
|
+
return ["blur": blur, "exposure": exposure, "obstructed": obstructed]
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
static func assess(luma: [UInt8], width: Int, height: Int, rowStride: Int) -> [String: Any] {
|
|
146
|
+
return luma.withUnsafeBufferPointer { assess(luma: $0.baseAddress!, width: width, height: height, rowStride: rowStride) }
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// MARK: - Vision session (feeds the Visual Engine)
|
|
151
|
+
//
|
|
152
|
+
// Headless luma stream that PUBLISHES a timeline frame at `intervalMs`, retaining
|
|
153
|
+
// only published frames in a bounded ring keyed by id (so the ring is cheap — one
|
|
154
|
+
// frame/sec, not 24). Emits `visionFrame` {id, ts, quality, expiresAt} per
|
|
155
|
+
// published frame; `qualityById` / (later) readText resolve the id inside here.
|
|
156
|
+
// The buffer/pixels never cross to JS.
|
|
157
|
+
final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
158
|
+
private let session = AVCaptureSession()
|
|
159
|
+
private let queue = DispatchQueue(label: "ai.ateam.adavision.session")
|
|
160
|
+
private var emit: (([String: Any]) -> Void)?
|
|
161
|
+
private var running = false
|
|
162
|
+
private var intervalMs = 1000
|
|
163
|
+
private var lastPublish: CFTimeInterval = 0
|
|
164
|
+
private let ringCap = 12
|
|
165
|
+
// Bumped every start(). Frame ids embed it (vf_<gen>_<ms>) and the ring is
|
|
166
|
+
// cleared on start, so an id minted by an OLD camera session can never resolve
|
|
167
|
+
// after a restart — qualityById simply won't find it. Two guarantees, not one:
|
|
168
|
+
// distinct ids across sessions AND a cleared ring.
|
|
169
|
+
private var generation = 0
|
|
170
|
+
|
|
171
|
+
// Each published frame keeps a CGImage COPY (independent of the capture pool) so
|
|
172
|
+
// native.vision.readText/describe can resolve the id → pixels INSIDE native. The
|
|
173
|
+
// pixels never cross to JS.
|
|
174
|
+
private struct Retained { let id: String; let ts: Double; let quality: [String: Any]; let image: CGImage? }
|
|
175
|
+
private var ring: [Retained] = []
|
|
176
|
+
private static let ciContext = CIContext(options: nil)
|
|
177
|
+
|
|
178
|
+
private static func makeCGImage(_ pb: CVPixelBuffer) -> CGImage? {
|
|
179
|
+
let ci = CIImage(cvPixelBuffer: pb)
|
|
180
|
+
return ciContext.createCGImage(ci, from: ci.extent)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
func retentionMs() -> Int { return ringCap * intervalMs }
|
|
184
|
+
|
|
185
|
+
func qualityById(_ id: String) -> [String: Any]? {
|
|
186
|
+
return queue.sync { ring.first(where: { $0.id == id })?.quality }
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
func start(intervalMs: Int, emit: @escaping ([String: Any]) -> Void, onReady: @escaping (Error?) -> Void) {
|
|
190
|
+
if running { onReady(nil); return }
|
|
191
|
+
self.intervalMs = intervalMs
|
|
192
|
+
let status = AVCaptureDevice.authorizationStatus(for: .video)
|
|
193
|
+
if status == .authorized {
|
|
194
|
+
configureAndRun(emit: emit, onReady: onReady)
|
|
195
|
+
} else if status == .notDetermined {
|
|
196
|
+
AVCaptureDevice.requestAccess(for: .video) { granted in
|
|
197
|
+
if granted { self.configureAndRun(emit: emit, onReady: onReady) }
|
|
198
|
+
else { onReady(NSError(domain: "AdaVision", code: 2, userInfo: [NSLocalizedDescriptionKey: "Camera permission denied"])) }
|
|
199
|
+
}
|
|
200
|
+
} else {
|
|
201
|
+
onReady(NSError(domain: "AdaVision", code: 3, userInfo: [NSLocalizedDescriptionKey: "Camera not permitted"]))
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private func configureAndRun(emit: @escaping ([String: Any]) -> Void, onReady: @escaping (Error?) -> Void) {
|
|
206
|
+
queue.async {
|
|
207
|
+
self.emit = emit
|
|
208
|
+
self.ring.removeAll()
|
|
209
|
+
self.lastPublish = 0
|
|
210
|
+
self.generation += 1 // new session → old ids can no longer resolve
|
|
211
|
+
self.session.beginConfiguration()
|
|
212
|
+
self.session.sessionPreset = .medium
|
|
213
|
+
guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
|
|
214
|
+
let input = try? AVCaptureDeviceInput(device: device), self.session.canAddInput(input) else {
|
|
215
|
+
self.session.commitConfiguration()
|
|
216
|
+
onReady(NSError(domain: "AdaVision", code: 4, userInfo: [NSLocalizedDescriptionKey: "No usable camera"])); return
|
|
217
|
+
}
|
|
218
|
+
self.session.addInput(input)
|
|
219
|
+
let out = AVCaptureVideoDataOutput()
|
|
220
|
+
out.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]
|
|
221
|
+
out.alwaysDiscardsLateVideoFrames = true
|
|
222
|
+
out.setSampleBufferDelegate(self, queue: self.queue)
|
|
223
|
+
guard self.session.canAddOutput(out) else {
|
|
224
|
+
self.session.removeInput(input); self.session.commitConfiguration()
|
|
225
|
+
onReady(NSError(domain: "AdaVision", code: 5, userInfo: [NSLocalizedDescriptionKey: "Cannot attach video output"])); return
|
|
226
|
+
}
|
|
227
|
+
self.session.addOutput(out)
|
|
228
|
+
self.session.commitConfiguration()
|
|
229
|
+
self.running = true
|
|
230
|
+
self.session.startRunning()
|
|
231
|
+
AdaVisionSessionHub.shared.set(self.session) // let a preview view attach to THIS session
|
|
232
|
+
onReady(nil)
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
func stop() {
|
|
237
|
+
queue.async {
|
|
238
|
+
if !self.running { return }
|
|
239
|
+
self.running = false
|
|
240
|
+
self.session.stopRunning()
|
|
241
|
+
for i in self.session.inputs { self.session.removeInput(i) }
|
|
242
|
+
for o in self.session.outputs { self.session.removeOutput(o) }
|
|
243
|
+
self.emit = nil
|
|
244
|
+
self.ring.removeAll()
|
|
245
|
+
AdaVisionSessionHub.shared.set(nil)
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
|
|
250
|
+
guard running, let pb = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
|
251
|
+
let now = CACurrentMediaTime()
|
|
252
|
+
if lastPublish != 0 && (now - lastPublish) * 1000.0 < Double(intervalMs) { return } // publish at cadence
|
|
253
|
+
lastPublish = now
|
|
254
|
+
|
|
255
|
+
// 1. Quality from the luma plane (manual lock for the raw read).
|
|
256
|
+
var q: [String: Any] = ["blur": "unknown", "exposure": "unknown", "obstructed": NSNull()]
|
|
257
|
+
CVPixelBufferLockBaseAddress(pb, .readOnly)
|
|
258
|
+
if let base = CVPixelBufferGetBaseAddressOfPlane(pb, 0) {
|
|
259
|
+
let w = CVPixelBufferGetWidthOfPlane(pb, 0)
|
|
260
|
+
let h = CVPixelBufferGetHeightOfPlane(pb, 0)
|
|
261
|
+
let s = CVPixelBufferGetBytesPerRowOfPlane(pb, 0)
|
|
262
|
+
q = LumaMath.assess(luma: base.assumingMemoryBound(to: UInt8.self), width: w, height: h, rowStride: s)
|
|
263
|
+
}
|
|
264
|
+
CVPixelBufferUnlockBaseAddress(pb, .readOnly)
|
|
265
|
+
|
|
266
|
+
// 2. Retain a CGImage COPY for readText/describe (CIImage manages its own
|
|
267
|
+
// access — done outside the manual lock above).
|
|
268
|
+
let img = VisionSession.makeCGImage(pb)
|
|
269
|
+
|
|
270
|
+
let id = "vf_\(generation)_\(Int(now * 1000))"
|
|
271
|
+
let ts = Date().timeIntervalSince1970 * 1000.0
|
|
272
|
+
ring.append(Retained(id: id, ts: ts, quality: q, image: img))
|
|
273
|
+
if ring.count > ringCap { ring.removeFirst(ring.count - ringCap) }
|
|
274
|
+
|
|
275
|
+
var payload = q
|
|
276
|
+
payload["id"] = id
|
|
277
|
+
payload["ts"] = ts
|
|
278
|
+
payload["expiresAt"] = ts + Double(retentionMs())
|
|
279
|
+
emit?(payload)
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// MARK: readText — on-device OCR (Apple Vision), resolved by id inside native.
|
|
283
|
+
// Latest-wins: reads run on the serial `queue`, so a burst never runs concurrently;
|
|
284
|
+
// the newest published frames are what the caller asks for by id.
|
|
285
|
+
func readText(_ id: String, languages: [String]?, completion: @escaping ([[String: Any]]?, Error?) -> Void) {
|
|
286
|
+
queue.async {
|
|
287
|
+
guard let cg = self.ring.first(where: { $0.id == id })?.image else {
|
|
288
|
+
completion(nil, NSError(domain: "AdaVision", code: 6,
|
|
289
|
+
userInfo: [NSLocalizedDescriptionKey: "No frame for id \(id) — unknown or aged out of the ring"]))
|
|
290
|
+
return
|
|
291
|
+
}
|
|
292
|
+
let request = VNRecognizeTextRequest { req, err in
|
|
293
|
+
if let err = err { completion(nil, err); return }
|
|
294
|
+
let out: [[String: Any]] = (req.results as? [VNRecognizedTextObservation])?.compactMap { obs in
|
|
295
|
+
guard let top = obs.topCandidates(1).first else { return nil }
|
|
296
|
+
let bb = obs.boundingBox // normalized, origin BOTTOM-left
|
|
297
|
+
return [
|
|
298
|
+
"text": top.string,
|
|
299
|
+
"confidence": Double(top.confidence),
|
|
300
|
+
// Normalize to TOP-left origin (the API contract). Full device-rotation
|
|
301
|
+
// transform is a later step; this is the y-flip.
|
|
302
|
+
"box": ["x": bb.origin.x, "y": 1.0 - bb.origin.y - bb.height, "width": bb.width, "height": bb.height],
|
|
303
|
+
]
|
|
304
|
+
} ?? []
|
|
305
|
+
completion(out, nil)
|
|
306
|
+
}
|
|
307
|
+
request.recognitionLevel = .accurate
|
|
308
|
+
request.usesLanguageCorrection = true
|
|
309
|
+
if let langs = languages, !langs.isEmpty { request.recognitionLanguages = langs }
|
|
310
|
+
do {
|
|
311
|
+
try VNImageRequestHandler(cgImage: cg, options: [:]).perform([request])
|
|
312
|
+
} catch {
|
|
313
|
+
completion(nil, error)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// MARK: describe — on-device image classification (Apple Vision VNClassifyImage).
|
|
319
|
+
// Returns LABELS from the platform vocabulary, not prose — the sentence is the
|
|
320
|
+
// skill's job. Filtered to meaningful confidence, top `maxLabels`.
|
|
321
|
+
func describe(_ id: String, maxLabels: Int, completion: @escaping ([[String: Any]]?, Error?) -> Void) {
|
|
322
|
+
queue.async {
|
|
323
|
+
guard let cg = self.ring.first(where: { $0.id == id })?.image else {
|
|
324
|
+
completion(nil, NSError(domain: "AdaVision", code: 6,
|
|
325
|
+
userInfo: [NSLocalizedDescriptionKey: "No frame for id \(id) — unknown or aged out of the ring"]))
|
|
326
|
+
return
|
|
327
|
+
}
|
|
328
|
+
let request = VNClassifyImageRequest { req, err in
|
|
329
|
+
if let err = err { completion(nil, err); return }
|
|
330
|
+
let obs = (req.results as? [VNClassificationObservation]) ?? []
|
|
331
|
+
let out: [[String: Any]] = obs
|
|
332
|
+
.filter { $0.hasMinimumRecall(0.0, forPrecision: 0.0) && $0.confidence >= 0.1 }
|
|
333
|
+
.prefix(max(1, maxLabels))
|
|
334
|
+
.map { ["label": $0.identifier, "confidence": Double($0.confidence)] }
|
|
335
|
+
completion(out, nil)
|
|
336
|
+
}
|
|
337
|
+
do {
|
|
338
|
+
try VNImageRequestHandler(cgImage: cg, options: [:]).perform([request])
|
|
339
|
+
} catch {
|
|
340
|
+
completion(nil, error)
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// The bundled object detector (YOLOv8n, Open-Images vocabulary), compiled into
|
|
346
|
+
// the app bundle as yolov8n-oiv7.mlmodelc. NMS is baked into the model pipeline,
|
|
347
|
+
// so the request yields VNRecognizedObjectObservation (label + box) directly —
|
|
348
|
+
// no hand-written decoder. Loaded once; nil if the model isn't in this build.
|
|
349
|
+
private static let detector: VNCoreMLModel? = {
|
|
350
|
+
guard let url = Bundle.main.url(forResource: "yolov8n-oiv7", withExtension: "mlmodelc") else { return nil }
|
|
351
|
+
let cfg = MLModelConfiguration()
|
|
352
|
+
cfg.computeUnits = .all // Neural Engine when available
|
|
353
|
+
guard let ml = try? MLModel(contentsOf: url, configuration: cfg),
|
|
354
|
+
let vn = try? VNCoreMLModel(for: ml) else { return nil }
|
|
355
|
+
return vn
|
|
356
|
+
}()
|
|
357
|
+
|
|
358
|
+
// MARK: detectObjects — on-device object detection (bundled Core ML), by id.
|
|
359
|
+
// Same latest-wins discipline as readText/describe (serial `queue`). Boxes are
|
|
360
|
+
// normalised 0..1, flipped to TOP-left origin to match the readText contract.
|
|
361
|
+
func detectObjects(_ id: String, maxItems: Int, minConfidence: Double, completion: @escaping ([[String: Any]]?, Error?) -> Void) {
|
|
362
|
+
queue.async {
|
|
363
|
+
guard let cg = self.ring.first(where: { $0.id == id })?.image else {
|
|
364
|
+
completion(nil, NSError(domain: "AdaVision", code: 6,
|
|
365
|
+
userInfo: [NSLocalizedDescriptionKey: "No frame for id \(id) — unknown or aged out of the ring"]))
|
|
366
|
+
return
|
|
367
|
+
}
|
|
368
|
+
guard let model = VisionSession.detector else {
|
|
369
|
+
completion(nil, NSError(domain: "AdaVision", code: 7,
|
|
370
|
+
userInfo: [NSLocalizedDescriptionKey: "Object-detection model not bundled in this build"]))
|
|
371
|
+
return
|
|
372
|
+
}
|
|
373
|
+
let request = VNCoreMLRequest(model: model) { req, err in
|
|
374
|
+
if let err = err { completion(nil, err); return }
|
|
375
|
+
let objs = (req.results as? [VNRecognizedObjectObservation]) ?? []
|
|
376
|
+
let out: [[String: Any]] = objs.compactMap { obs -> (Double, [String: Any])? in
|
|
377
|
+
// .labels is sorted by confidence; take the top class for this box.
|
|
378
|
+
guard let top = obs.labels.first, Double(top.confidence) >= minConfidence else { return nil }
|
|
379
|
+
let bb = obs.boundingBox // normalized, origin BOTTOM-left
|
|
380
|
+
return (Double(top.confidence), [
|
|
381
|
+
"label": top.identifier,
|
|
382
|
+
"confidence": Double(top.confidence),
|
|
383
|
+
"box": ["x": bb.origin.x, "y": 1.0 - bb.origin.y - bb.height, "width": bb.width, "height": bb.height],
|
|
384
|
+
])
|
|
385
|
+
}
|
|
386
|
+
.sorted { $0.0 > $1.0 }
|
|
387
|
+
.prefix(max(1, maxItems))
|
|
388
|
+
.map { $0.1 }
|
|
389
|
+
completion(Array(out), nil)
|
|
390
|
+
}
|
|
391
|
+
request.imageCropAndScaleOption = .scaleFill // match the model's 640² letterless input
|
|
392
|
+
do {
|
|
393
|
+
try VNImageRequestHandler(cgImage: cg, options: [:]).perform([request])
|
|
394
|
+
} catch {
|
|
395
|
+
completion(nil, error)
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// MARK: - Live preview (TEST ONLY)
|
|
402
|
+
//
|
|
403
|
+
// A native view bound to the SAME AVCaptureSession the headless VisionSession
|
|
404
|
+
// runs — so it shows exactly what the reader sees, WITHOUT opening a second
|
|
405
|
+
// camera. The real Guide never mounts this; it exists only so a tester can aim.
|
|
406
|
+
final class AdaVisionSessionHub {
|
|
407
|
+
static let shared = AdaVisionSessionHub()
|
|
408
|
+
private(set) weak var session: AVCaptureSession?
|
|
409
|
+
private var observers: [() -> Void] = []
|
|
410
|
+
func set(_ s: AVCaptureSession?) {
|
|
411
|
+
session = s
|
|
412
|
+
let obs = observers
|
|
413
|
+
DispatchQueue.main.async { obs.forEach { $0() } }
|
|
414
|
+
}
|
|
415
|
+
func observe(_ cb: @escaping () -> Void) { observers.append(cb) }
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
final class AdaVisionPreviewView: ExpoView {
|
|
419
|
+
private let previewLayer = AVCaptureVideoPreviewLayer()
|
|
420
|
+
required init(appContext: AppContext? = nil) {
|
|
421
|
+
super.init(appContext: appContext)
|
|
422
|
+
previewLayer.videoGravity = .resizeAspectFill
|
|
423
|
+
layer.addSublayer(previewLayer)
|
|
424
|
+
attach()
|
|
425
|
+
AdaVisionSessionHub.shared.observe { [weak self] in self?.attach() }
|
|
426
|
+
}
|
|
427
|
+
private func attach() { previewLayer.session = AdaVisionSessionHub.shared.session }
|
|
428
|
+
override func layoutSubviews() {
|
|
429
|
+
super.layoutSubviews()
|
|
430
|
+
previewLayer.frame = bounds
|
|
431
|
+
}
|
|
432
|
+
}
|
|
Binary file
|
|
Binary file
|