@ateam-ai-mobile/expo 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap.js +44 -27
- package/dist/bootstrap.js.map +1 -1
- package/dist/bootstrap.mjs +3 -3
- package/dist/{chunk-C7M5WAHA.mjs → chunk-42FP5DTO.mjs} +17 -5
- package/dist/chunk-42FP5DTO.mjs.map +1 -0
- package/dist/{chunk-PCAEUX2N.mjs → chunk-6JX222YX.mjs} +21 -10
- package/dist/chunk-6JX222YX.mjs.map +1 -0
- package/dist/{chunk-GF2MPTAB.mjs → chunk-DLDG77DX.mjs} +2 -2
- package/dist/device/createDeviceProvider.js +28 -23
- package/dist/device/createDeviceProvider.js.map +1 -1
- package/dist/device/createDeviceProvider.mjs +2 -2
- package/dist/index.d.mts +3 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.js +16 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/vision/index.d.mts +10 -3
- package/dist/vision/index.d.ts +10 -3
- package/dist/vision/index.js +20 -9
- package/dist/vision/index.js.map +1 -1
- package/dist/vision/index.mjs +1 -1
- package/ios/AdaVisionModule.swift +55 -13
- package/package.json +4 -4
- package/src/device-state/domains/visual.ts +7 -2
- package/src/device-state/visualEngine.ts +15 -2
- package/src/vision/index.ts +34 -10
- package/dist/chunk-C7M5WAHA.mjs.map +0 -1
- package/dist/chunk-PCAEUX2N.mjs.map +0 -1
- /package/dist/{chunk-GF2MPTAB.mjs.map → chunk-DLDG77DX.mjs.map} +0 -0
package/dist/index.mjs
CHANGED
package/dist/vision/index.d.mts
CHANGED
|
@@ -61,8 +61,15 @@ interface VisionObject {
|
|
|
61
61
|
declare function isSupported(): boolean;
|
|
62
62
|
/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */
|
|
63
63
|
declare function assessQuality(uri: string): Promise<RawFrameQuality>;
|
|
64
|
-
/**
|
|
65
|
-
|
|
64
|
+
/** Which camera the vision session reads. */
|
|
65
|
+
type VisionFacing = 'front' | 'back';
|
|
66
|
+
/**
|
|
67
|
+
* Start the vision session on `facing` — publishes a timeline frame every
|
|
68
|
+
* `intervalMs`. The back camera works on every binary; the front camera needs a
|
|
69
|
+
* binary built with it, and an older one REFUSES it rather than quietly reading
|
|
70
|
+
* the back camera instead.
|
|
71
|
+
*/
|
|
72
|
+
declare function startVision(intervalMs?: number, facing?: VisionFacing): Promise<void>;
|
|
66
73
|
/** Stop the vision session and release the camera. */
|
|
67
74
|
declare function stopVision(): void;
|
|
68
75
|
/** Subscribe to published timeline frames (id + quality + expiry; no pixels). */
|
|
@@ -80,4 +87,4 @@ declare function describe(id: string, maxLabels?: number): Promise<VisionLabel[]
|
|
|
80
87
|
* couldn't look (id aged out, or the model isn't in this build). */
|
|
81
88
|
declare function detectObjects(id: string, maxItems?: number, minConfidence?: number): Promise<VisionObject[]>;
|
|
82
89
|
|
|
83
|
-
export { CameraPreview, type RawFrameQuality, type VisionFrame, type VisionLabel, type VisionObject, type VisionText, assessQuality, describe, detectObjects, isSupported, onVisionFrame, qualityById, readText, startVision, stopVision, visionRetentionMs };
|
|
90
|
+
export { CameraPreview, type RawFrameQuality, type VisionFacing, type VisionFrame, type VisionLabel, type VisionObject, type VisionText, assessQuality, describe, detectObjects, isSupported, onVisionFrame, qualityById, readText, startVision, stopVision, visionRetentionMs };
|
package/dist/vision/index.d.ts
CHANGED
|
@@ -61,8 +61,15 @@ interface VisionObject {
|
|
|
61
61
|
declare function isSupported(): boolean;
|
|
62
62
|
/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */
|
|
63
63
|
declare function assessQuality(uri: string): Promise<RawFrameQuality>;
|
|
64
|
-
/**
|
|
65
|
-
|
|
64
|
+
/** Which camera the vision session reads. */
|
|
65
|
+
type VisionFacing = 'front' | 'back';
|
|
66
|
+
/**
|
|
67
|
+
* Start the vision session on `facing` — publishes a timeline frame every
|
|
68
|
+
* `intervalMs`. The back camera works on every binary; the front camera needs a
|
|
69
|
+
* binary built with it, and an older one REFUSES it rather than quietly reading
|
|
70
|
+
* the back camera instead.
|
|
71
|
+
*/
|
|
72
|
+
declare function startVision(intervalMs?: number, facing?: VisionFacing): Promise<void>;
|
|
66
73
|
/** Stop the vision session and release the camera. */
|
|
67
74
|
declare function stopVision(): void;
|
|
68
75
|
/** Subscribe to published timeline frames (id + quality + expiry; no pixels). */
|
|
@@ -80,4 +87,4 @@ declare function describe(id: string, maxLabels?: number): Promise<VisionLabel[]
|
|
|
80
87
|
* couldn't look (id aged out, or the model isn't in this build). */
|
|
81
88
|
declare function detectObjects(id: string, maxItems?: number, minConfidence?: number): Promise<VisionObject[]>;
|
|
82
89
|
|
|
83
|
-
export { CameraPreview, type RawFrameQuality, type VisionFrame, type VisionLabel, type VisionObject, type VisionText, assessQuality, describe, detectObjects, isSupported, onVisionFrame, qualityById, readText, startVision, stopVision, visionRetentionMs };
|
|
90
|
+
export { CameraPreview, type RawFrameQuality, type VisionFacing, type VisionFrame, type VisionLabel, type VisionObject, type VisionText, assessQuality, describe, detectObjects, isSupported, onVisionFrame, qualityById, readText, startVision, stopVision, visionRetentionMs };
|
package/dist/vision/index.js
CHANGED
|
@@ -43,6 +43,14 @@ var CameraPreview = (() => {
|
|
|
43
43
|
}
|
|
44
44
|
})();
|
|
45
45
|
var native = (0, import_expo_modules_core.requireOptionalNativeModule)("AdaVision");
|
|
46
|
+
function verb(name) {
|
|
47
|
+
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
48
|
+
const fn = native[name];
|
|
49
|
+
if (typeof fn !== "function") {
|
|
50
|
+
throw Object.assign(new Error(`UNAVAILABLE: ada-vision ${String(name)} is not implemented on this platform (iOS only)`), { code: "UNAVAILABLE" });
|
|
51
|
+
}
|
|
52
|
+
return fn.bind(native);
|
|
53
|
+
}
|
|
46
54
|
function isSupported() {
|
|
47
55
|
try {
|
|
48
56
|
return native?.isSupported() ?? false;
|
|
@@ -54,9 +62,15 @@ async function assessQuality(uri) {
|
|
|
54
62
|
if (!native) throw new Error("ada-vision native module not present in this build");
|
|
55
63
|
return native.assessQuality(uri);
|
|
56
64
|
}
|
|
57
|
-
async function startVision(intervalMs = 1e3) {
|
|
58
|
-
if (
|
|
59
|
-
|
|
65
|
+
async function startVision(intervalMs = 1e3, facing = "back") {
|
|
66
|
+
if (facing !== "front" && facing !== "back") {
|
|
67
|
+
throw new Error(`Unknown camera "${String(facing)}" \u2014 use "front" or "back"`);
|
|
68
|
+
}
|
|
69
|
+
if (facing === "back") return verb("startVision")(intervalMs);
|
|
70
|
+
if (native && typeof native.startVisionCamera !== "function") {
|
|
71
|
+
throw new Error("the front camera needs a newer app build \u2014 this one can read the back camera only");
|
|
72
|
+
}
|
|
73
|
+
return verb("startVisionCamera")(intervalMs, facing);
|
|
60
74
|
}
|
|
61
75
|
function stopVision() {
|
|
62
76
|
native?.stopVision?.();
|
|
@@ -73,16 +87,13 @@ function qualityById(id) {
|
|
|
73
87
|
return native?.qualityById?.(id) ?? null;
|
|
74
88
|
}
|
|
75
89
|
async function readText(id, languages) {
|
|
76
|
-
|
|
77
|
-
return native.readText(id, languages ?? null);
|
|
90
|
+
return verb("readText")(id, languages ?? null);
|
|
78
91
|
}
|
|
79
92
|
async function describe(id, maxLabels = 5) {
|
|
80
|
-
|
|
81
|
-
return native.describe(id, maxLabels);
|
|
93
|
+
return verb("describe")(id, maxLabels);
|
|
82
94
|
}
|
|
83
95
|
async function detectObjects(id, maxItems = 10, minConfidence = 0.25) {
|
|
84
|
-
|
|
85
|
-
return native.detectObjects(id, maxItems, minConfidence);
|
|
96
|
+
return verb("detectObjects")(id, maxItems, minConfidence);
|
|
86
97
|
}
|
|
87
98
|
// Annotate the CommonJS export names for ESM import in node:
|
|
88
99
|
0 && (module.exports = {
|
package/dist/vision/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/vision/index.ts"],"sourcesContent":["// ada-vision — on-device image understanding, our own native code (no camera lib).\n//\n// On-device, frames never leave the phone:\n// assessQuality(uri) — one-shot quality of a LOCAL frame file.\n// startVision()/onVisionFrame — the headless camera session that publishes a\n// timeline frame (id + quality, never pixels) at a\n// cadence; readText/describe/detectObjects resolve\n// a published id ON-DEVICE.\n//\n// The luma math mirrors ./analysis.js (which has the exhaustive unit tests).\n//\n// requireOptionalNativeModule: on a binary WITHOUT the native module, the import\n// is null, isSupported() is false, and the session calls refuse cleanly (no crash).\n\nimport type { ComponentType } from 'react';\nimport { requireOptionalNativeModule } from 'expo-modules-core';\nimport { requireNativeView } from 'expo';\n\n/** TEST-ONLY live preview bound to the same headless session (so a tester can\n * aim). The real Guide runs headless and never mounts this. Null on a binary\n * without the native view, so importing this module never crashes. */\nexport const CameraPreview: ComponentType<any> | null = (() => {\n try { return requireNativeView('AdaVision'); } catch { return null; }\n})();\n\nexport interface RawFrameQuality {\n blur: 'sharp' | 'slight' | 'severe' | 'unknown';\n exposure: 'good' | 'dark' | 'bright' | 'unknown';\n /** true = lens covered; false = clearly not; null = not assessed. */\n obstructed: boolean | null;\n}\n\ninterface Subscription { remove(): void }\n\n/** A published timeline frame from the vision session (metadata only — never\n * pixels; the buffer stays in the native ring, resolved by id). */\nexport interface VisionFrame {\n id: string;\n ts: number;\n expiresAt: number;\n blur: RawFrameQuality['blur'];\n exposure: RawFrameQuality['exposure'];\n obstructed: RawFrameQuality['obstructed'];\n}\n\n/** One OCR result on a frame — matches the SDK's VisionText. */\nexport interface VisionText {\n text: string;\n /** 0..1 recogniser confidence (never a 1.0 filler). */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left, in the UPRIGHT portrait frame (native\n * rotates frames at capture) — x runs from the user's left to their right. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\n/** One classifier label — a vocabulary term, NOT prose. Matches VisionLabel. */\nexport interface VisionLabel {\n label: string;\n confidence: number | null;\n}\n\n/** One detected object — a located item with a box, from the bundled detector. */\nexport interface VisionObject {\n /** Open-Images class name (e.g. \"Car\", \"Traffic light\", \"Building\", \"Person\"). */\n label: string;\n /** 0..1 detector confidence. */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left, in the UPRIGHT portrait frame (native\n * rotates frames at capture) — x runs from the user's left to their right. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\ninterface AdaVisionNative {\n isSupported(): boolean;\n assessQuality(uri: string): Promise<RawFrameQuality>;\n startVision(intervalMs: number): Promise<void>;\n stopVision(): void;\n visionRetentionMs(): number;\n qualityById(id: string): RawFrameQuality | null;\n readText(id: string, languages: string[] | null): Promise<VisionText[]>;\n describe(id: string, maxLabels: number): Promise<VisionLabel[]>;\n detectObjects(id: string, maxItems: number, minConfidence: number): Promise<VisionObject[]>;\n addListener(event: 'visionFrame', cb: (f: VisionFrame) => void): Subscription;\n}\n\nconst native = requireOptionalNativeModule<AdaVisionNative>('AdaVision');\n\n/** Is the native vision module present in this binary? */\nexport function isSupported(): boolean {\n try { return native?.isSupported() ?? false; } catch { return false; }\n}\n\n/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */\nexport async function assessQuality(uri: string): Promise<RawFrameQuality> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.assessQuality(uri);\n}\n\n// ── Vision session (feeds the Visual Engine) ──\n\n/** Start the vision session — publishes a timeline frame every `intervalMs`. */\nexport async function startVision(intervalMs = 1000): Promise<void> {\n if (
|
|
1
|
+
{"version":3,"sources":["../../src/vision/index.ts"],"sourcesContent":["// ada-vision — on-device image understanding, our own native code (no camera lib).\n//\n// On-device, frames never leave the phone:\n// assessQuality(uri) — one-shot quality of a LOCAL frame file.\n// startVision()/onVisionFrame — the headless camera session that publishes a\n// timeline frame (id + quality, never pixels) at a\n// cadence; readText/describe/detectObjects resolve\n// a published id ON-DEVICE.\n//\n// The luma math mirrors ./analysis.js (which has the exhaustive unit tests).\n//\n// requireOptionalNativeModule: on a binary WITHOUT the native module, the import\n// is null, isSupported() is false, and the session calls refuse cleanly (no crash).\n\nimport type { ComponentType } from 'react';\nimport { requireOptionalNativeModule } from 'expo-modules-core';\nimport { requireNativeView } from 'expo';\n\n/** TEST-ONLY live preview bound to the same headless session (so a tester can\n * aim). The real Guide runs headless and never mounts this. Null on a binary\n * without the native view, so importing this module never crashes. */\nexport const CameraPreview: ComponentType<any> | null = (() => {\n try { return requireNativeView('AdaVision'); } catch { return null; }\n})();\n\nexport interface RawFrameQuality {\n blur: 'sharp' | 'slight' | 'severe' | 'unknown';\n exposure: 'good' | 'dark' | 'bright' | 'unknown';\n /** true = lens covered; false = clearly not; null = not assessed. */\n obstructed: boolean | null;\n}\n\ninterface Subscription { remove(): void }\n\n/** A published timeline frame from the vision session (metadata only — never\n * pixels; the buffer stays in the native ring, resolved by id). */\nexport interface VisionFrame {\n id: string;\n ts: number;\n expiresAt: number;\n blur: RawFrameQuality['blur'];\n exposure: RawFrameQuality['exposure'];\n obstructed: RawFrameQuality['obstructed'];\n}\n\n/** One OCR result on a frame — matches the SDK's VisionText. */\nexport interface VisionText {\n text: string;\n /** 0..1 recogniser confidence (never a 1.0 filler). */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left, in the UPRIGHT portrait frame (native\n * rotates frames at capture) — x runs from the user's left to their right. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\n/** One classifier label — a vocabulary term, NOT prose. Matches VisionLabel. */\nexport interface VisionLabel {\n label: string;\n confidence: number | null;\n}\n\n/** One detected object — a located item with a box, from the bundled detector. */\nexport interface VisionObject {\n /** Open-Images class name (e.g. \"Car\", \"Traffic light\", \"Building\", \"Person\"). */\n label: string;\n /** 0..1 detector confidence. */\n confidence: number | null;\n /** Normalised 0..1 box, origin TOP-left, in the UPRIGHT portrait frame (native\n * rotates frames at capture) — x runs from the user's left to their right. */\n box?: { x: number; y: number; width: number; height: number };\n}\n\ninterface AdaVisionNative {\n isSupported(): boolean;\n assessQuality(uri: string): Promise<RawFrameQuality>;\n startVision(intervalMs: number): Promise<void>;\n /** Absent from binaries built before the front camera existed. */\n startVisionCamera?(intervalMs: number, facing: VisionFacing): Promise<void>;\n stopVision(): void;\n visionRetentionMs(): number;\n qualityById(id: string): RawFrameQuality | null;\n readText(id: string, languages: string[] | null): Promise<VisionText[]>;\n describe(id: string, maxLabels: number): Promise<VisionLabel[]>;\n detectObjects(id: string, maxItems: number, minConfidence: number): Promise<VisionObject[]>;\n addListener(event: 'visionFrame', cb: (f: VisionFrame) => void): Subscription;\n}\n\nconst native = requireOptionalNativeModule<AdaVisionNative>('AdaVision');\n\n/** The native function, or a refusal by name: Android's AdaVision has assessQuality only\n * (AI Models F4), so a verb it lacks says so instead of failing as \"not a function\". */\nfunction verb<K extends keyof AdaVisionNative>(name: K): AdaVisionNative[K] {\n if (!native) throw new Error('ada-vision native module not present in this build');\n const fn = native[name];\n if (typeof fn !== 'function') {\n throw Object.assign(new Error(`UNAVAILABLE: ada-vision ${String(name)} is not implemented on this platform (iOS only)`), { code: 'UNAVAILABLE' });\n }\n return (fn as Function).bind(native) as AdaVisionNative[K];\n}\n\n/** Is the native vision module present in this binary? */\nexport function isSupported(): boolean {\n try { return native?.isSupported() ?? false; } catch { return false; }\n}\n\n/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */\nexport async function assessQuality(uri: string): Promise<RawFrameQuality> {\n if (!native) throw new Error('ada-vision native module not present in this build');\n return native.assessQuality(uri);\n}\n\n// ── Vision session (feeds the Visual Engine) ──\n\n/** Which camera the vision session reads. */\nexport type VisionFacing = 'front' | 'back';\n\n/**\n * Start the vision session on `facing` — publishes a timeline frame every\n * `intervalMs`. The back camera works on every binary; the front camera needs a\n * binary built with it, and an older one REFUSES it rather than quietly reading\n * the back camera instead.\n */\nexport async function startVision(intervalMs = 1000, facing: VisionFacing = 'back'): Promise<void> {\n if (facing !== 'front' && facing !== 'back') {\n throw new Error(`Unknown camera \"${String(facing)}\" — use \"front\" or \"back\"`);\n }\n if (facing === 'back') return verb('startVision')(intervalMs);\n if (native && typeof native.startVisionCamera !== 'function') {\n throw new Error('the front camera needs a newer app build — this one can read the back camera only');\n }\n return verb('startVisionCamera')!(intervalMs, facing);\n}\n\n/** Stop the vision session and release the camera. */\nexport function stopVision(): void {\n native?.stopVision?.();\n}\n\n/** Subscribe to published timeline frames (id + quality + expiry; no pixels). */\nexport function onVisionFrame(cb: (f: VisionFrame) => void): Subscription {\n if (!native) return { remove() {} };\n return native.addListener('visionFrame', cb);\n}\n\n/** How far back the native ring can still resolve a frame id (ms). */\nexport function visionRetentionMs(): number {\n return native?.visionRetentionMs?.() ?? 0;\n}\n\n/** Resolve a published frame id → its measured quality (null if evicted). */\nexport function qualityById(id: string): RawFrameQuality | null {\n return native?.qualityById?.(id) ?? null;\n}\n\n/** On-device OCR on a published frame. [] = looked, no text; throws = couldn't look. */\nexport async function readText(id: string, languages?: string[]): Promise<VisionText[]> {\n return verb('readText')(id, languages ?? null);\n}\n\n/** On-device image classification → labels (not prose). [] = nothing recognised. */\nexport async function describe(id: string, maxLabels = 5): Promise<VisionLabel[]> {\n return verb('describe')(id, maxLabels);\n}\n\n/** On-device object detection (bundled YOLOv8n, Open-Images vocabulary) → located\n * items with boxes. [] = looked, found nothing above `minConfidence`; throws =\n * couldn't look (id aged out, or the model isn't in this build). */\nexport async function detectObjects(id: string, maxItems = 10, minConfidence = 0.25): Promise<VisionObject[]> {\n return verb('detectObjects')(id, maxItems, minConfidence);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAeA,+BAA4C;AAC5C,kBAAkC;AAK3B,IAAM,iBAA4C,MAAM;AAC7D,MAAI;AAAE,eAAO,+BAAkB,WAAW;AAAA,EAAG,QAAQ;AAAE,WAAO;AAAA,EAAM;AACtE,GAAG;AAgEH,IAAM,aAAS,sDAA6C,WAAW;AAIvE,SAAS,KAAsC,MAA6B;AAC1E,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,QAAM,KAAK,OAAO,IAAI;AACtB,MAAI,OAAO,OAAO,YAAY;AAC5B,UAAM,OAAO,OAAO,IAAI,MAAM,2BAA2B,OAAO,IAAI,CAAC,iDAAiD,GAAG,EAAE,MAAM,cAAc,CAAC;AAAA,EAClJ;AACA,SAAQ,GAAgB,KAAK,MAAM;AACrC;AAGO,SAAS,cAAuB;AACrC,MAAI;AAAE,WAAO,QAAQ,YAAY,KAAK;AAAA,EAAO,QAAQ;AAAE,WAAO;AAAA,EAAO;AACvE;AAGA,eAAsB,cAAc,KAAuC;AACzE,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,oDAAoD;AACjF,SAAO,OAAO,cAAc,GAAG;AACjC;AAaA,eAAsB,YAAY,aAAa,KAAM,SAAuB,QAAuB;AACjG,MAAI,WAAW,WAAW,WAAW,QAAQ;AAC3C,UAAM,IAAI,MAAM,mBAAmB,OAAO,MAAM,CAAC,gCAA2B;AAAA,EAC9E;AACA,MAAI,WAAW,OAAQ,QAAO,KAAK,aAAa,EAAE,UAAU;AAC5D,MAAI,UAAU,OAAO,OAAO,sBAAsB,YAAY;AAC5D,UAAM,IAAI,MAAM,wFAAmF;AAAA,EACrG;AACA,SAAO,KAAK,mBAAmB,EAAG,YAAY,MAAM;AACtD;AAGO,SAAS,aAAmB;AACjC,UAAQ,aAAa;AACvB;AAGO,SAAS,cAAc,IAA4C;AACxE,MAAI,CAAC,OAAQ,QAAO,EAAE,SAAS;AAAA,EAAC,EAAE;AAClC,SAAO,OAAO,YAAY,eAAe,EAAE;AAC7C;AAGO,SAAS,oBAA4B;AAC1C,SAAO,QAAQ,oBAAoB,KAAK;AAC1C;AAGO,SAAS,YAAY,IAAoC;AAC9D,SAAO,QAAQ,cAAc,EAAE,KAAK;AACtC;AAGA,eAAsB,SAAS,IAAY,WAA6C;AACtF,SAAO,KAAK,UAAU,EAAE,IAAI,aAAa,IAAI;AAC/C;AAGA,eAAsB,SAAS,IAAY,YAAY,GAA2B;AAChF,SAAO,KAAK,UAAU,EAAE,IAAI,SAAS;AACvC;AAKA,eAAsB,cAAc,IAAY,WAAW,IAAI,gBAAgB,MAA+B;AAC5G,SAAO,KAAK,eAAe,EAAE,IAAI,UAAU,aAAa;AAC1D;","names":[]}
|
package/dist/vision/index.mjs
CHANGED
|
@@ -33,9 +33,17 @@ public class AdaVisionModule: Module {
|
|
|
33
33
|
// still runs every camera frame; only the published frames are retained, so
|
|
34
34
|
// the ring is cheap (≈ one frame/sec, not 24/sec).
|
|
35
35
|
AsyncFunction("startVision") { (intervalMs: Int, promise: Promise) in
|
|
36
|
-
self.
|
|
37
|
-
|
|
36
|
+
self.startVision(intervalMs: intervalMs, facing: .back, promise: promise)
|
|
37
|
+
}
|
|
38
|
+
/** startVision with a chosen camera: "front" (the selfie camera, mirrored like
|
|
39
|
+
* the view the user sees) or "back". A separate function rather than a new
|
|
40
|
+
* startVision argument, so JS can tell a binary that has it from one that does
|
|
41
|
+
* not and refuse the front camera honestly on the older one. */
|
|
42
|
+
AsyncFunction("startVisionCamera") { (intervalMs: Int, facing: String, promise: Promise) in
|
|
43
|
+
guard let position = VisionSession.position(facing) else {
|
|
44
|
+
promise.reject("AdaVision", "Unknown camera \"\(facing)\" — use \"front\" or \"back\""); return
|
|
38
45
|
}
|
|
46
|
+
self.startVision(intervalMs: intervalMs, facing: position, promise: promise)
|
|
39
47
|
}
|
|
40
48
|
Function("stopVision") { () in self.vision.stop() }
|
|
41
49
|
/** How far back the native ring can still resolve a frame id (ms). */
|
|
@@ -79,6 +87,12 @@ public class AdaVisionModule: Module {
|
|
|
79
87
|
OnDestroy { self.vision.stop() }
|
|
80
88
|
}
|
|
81
89
|
|
|
90
|
+
private func startVision(intervalMs: Int, facing: AVCaptureDevice.Position, promise: Promise) {
|
|
91
|
+
vision.start(intervalMs: max(200, intervalMs), facing: facing, emit: { [weak self] f in self?.sendEvent("visionFrame", f) }) { err in
|
|
92
|
+
if let err = err { promise.reject("AdaVision", err.localizedDescription) } else { promise.resolve(nil) }
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
82
96
|
// MARK: file-backed one-shot (kept from v1)
|
|
83
97
|
static func loadImage(_ uri: String) -> UIImage? {
|
|
84
98
|
if let url = URL(string: uri), url.isFileURL, let data = try? Data(contentsOf: url) {
|
|
@@ -163,14 +177,16 @@ enum LumaMath {
|
|
|
163
177
|
// the same upright image the user is pointing at, and boxes come back in upright
|
|
164
178
|
// coordinates (x = 0 is the user's LEFT). This is the ONE rotation mechanism:
|
|
165
179
|
// VNImageRequestHandler is always given the default .up orientation — passing an
|
|
166
|
-
// orientation there as well would rotate twice.
|
|
167
|
-
// .
|
|
180
|
+
// orientation there as well would rotate twice. The BACK camera is not mirrored:
|
|
181
|
+
// image-left is scene-left. The FRONT camera is mirrored, like the selfie view the
|
|
182
|
+
// preview shows, so a box sits where the user sees the thing on screen.
|
|
168
183
|
final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
169
184
|
private let session = AVCaptureSession()
|
|
170
185
|
private let queue = DispatchQueue(label: "ai.ateam.adavision.session")
|
|
171
186
|
private var emit: (([String: Any]) -> Void)?
|
|
172
187
|
private var running = false
|
|
173
188
|
private var intervalMs = 1000
|
|
189
|
+
private var facing: AVCaptureDevice.Position = .back
|
|
174
190
|
private var lastPublish: CFTimeInterval = 0
|
|
175
191
|
private let ringCap = 12
|
|
176
192
|
// Bumped every start(). Frame ids embed it (vf_<gen>_<ms>) and the ring is
|
|
@@ -227,15 +243,25 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
227
243
|
return queue.sync { ring.first(where: { $0.id == id })?.quality }
|
|
228
244
|
}
|
|
229
245
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
246
|
+
/// "front" / "back" → the camera; nil for anything else (the caller refuses it).
|
|
247
|
+
static func position(_ facing: String) -> AVCaptureDevice.Position? {
|
|
248
|
+
switch facing {
|
|
249
|
+
case "front": return .front
|
|
250
|
+
case "back": return .back
|
|
251
|
+
default: return nil
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
func start(intervalMs: Int, facing: AVCaptureDevice.Position, emit: @escaping ([String: Any]) -> Void, onReady: @escaping (Error?) -> Void) {
|
|
256
|
+
// The camera asked for travels WITH this request into configureAndRun, which
|
|
257
|
+
// checks and changes the running state on `queue` only — two overlapping starts
|
|
258
|
+
// (one waiting on the permission prompt) can never open each other's camera.
|
|
233
259
|
let status = AVCaptureDevice.authorizationStatus(for: .video)
|
|
234
260
|
if status == .authorized {
|
|
235
|
-
configureAndRun(emit: emit, onReady: onReady)
|
|
261
|
+
configureAndRun(intervalMs: intervalMs, facing: facing, emit: emit, onReady: onReady)
|
|
236
262
|
} else if status == .notDetermined {
|
|
237
263
|
AVCaptureDevice.requestAccess(for: .video) { granted in
|
|
238
|
-
if granted { self.configureAndRun(emit: emit, onReady: onReady) }
|
|
264
|
+
if granted { self.configureAndRun(intervalMs: intervalMs, facing: facing, emit: emit, onReady: onReady) }
|
|
239
265
|
else { onReady(NSError(domain: "AdaVision", code: 2, userInfo: [NSLocalizedDescriptionKey: "Camera permission denied"])) }
|
|
240
266
|
}
|
|
241
267
|
} else {
|
|
@@ -243,8 +269,20 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
243
269
|
}
|
|
244
270
|
}
|
|
245
271
|
|
|
246
|
-
private func configureAndRun(emit: @escaping ([String: Any]) -> Void, onReady: @escaping (Error?) -> Void) {
|
|
272
|
+
private func configureAndRun(intervalMs: Int, facing: AVCaptureDevice.Position, emit: @escaping ([String: Any]) -> Void, onReady: @escaping (Error?) -> Void) {
|
|
247
273
|
queue.async {
|
|
274
|
+
// Already running on THIS camera: nothing to do. On the other camera, the
|
|
275
|
+
// running one is torn down first.
|
|
276
|
+
if self.running && self.facing == facing { onReady(nil); return }
|
|
277
|
+
if self.running {
|
|
278
|
+
// Switching cameras: drop the running one before configuring the other.
|
|
279
|
+
self.running = false
|
|
280
|
+
self.session.stopRunning()
|
|
281
|
+
for i in self.session.inputs { self.session.removeInput(i) }
|
|
282
|
+
for o in self.session.outputs { self.session.removeOutput(o) }
|
|
283
|
+
}
|
|
284
|
+
self.intervalMs = intervalMs
|
|
285
|
+
self.facing = facing
|
|
248
286
|
self.emit = emit
|
|
249
287
|
self.ring.removeAll()
|
|
250
288
|
self.lastPublish = 0
|
|
@@ -254,9 +292,7 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
254
292
|
// (a voice call can be running at the same time — see AdaAudioModule).
|
|
255
293
|
self.session.automaticallyConfiguresApplicationAudioSession = false
|
|
256
294
|
self.session.sessionPreset = .medium
|
|
257
|
-
|
|
258
|
-
// mirrored (isVideoMirrored) so boxes match the mirrored view the user sees.
|
|
259
|
-
guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
|
|
295
|
+
guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: self.facing),
|
|
260
296
|
let input = try? AVCaptureDeviceInput(device: device), self.session.canAddInput(input) else {
|
|
261
297
|
self.session.commitConfiguration()
|
|
262
298
|
onReady(NSError(domain: "AdaVision", code: 4, userInfo: [NSLocalizedDescriptionKey: "No usable camera"])); return
|
|
@@ -277,6 +313,12 @@ final class VisionSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
|
|
277
313
|
if !upright {
|
|
278
314
|
NSLog("AdaVision: cannot rotate the camera data output to portrait — frames stay sensor-landscape, so box x/y will not match left/right")
|
|
279
315
|
}
|
|
316
|
+
// The front camera's frames are mirrored to match its preview (which AVFoundation
|
|
317
|
+
// mirrors by itself), so what a box says is on the left is on the left of the screen.
|
|
318
|
+
if self.facing == .front, let c = out.connection(with: .video), c.isVideoMirroringSupported {
|
|
319
|
+
c.automaticallyAdjustsVideoMirroring = false
|
|
320
|
+
c.isVideoMirrored = true
|
|
321
|
+
}
|
|
280
322
|
self.session.commitConfiguration()
|
|
281
323
|
self.running = true
|
|
282
324
|
self.session.startRunning()
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ateam-ai-mobile/expo",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "Expo-backed providers for the ADAS SDK — haptics, bundle disk cache, attachment cache, device bridge.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
"build": "tsup"
|
|
10
10
|
},
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@ateam-ai-mobile/core": "0.1.
|
|
13
|
-
"@ateam-ai-mobile/plugin-runtime": "0.1.
|
|
14
|
-
"@ateam-ai-mobile/ui": "0.1.
|
|
12
|
+
"@ateam-ai-mobile/core": "0.1.13",
|
|
13
|
+
"@ateam-ai-mobile/plugin-runtime": "0.1.13",
|
|
14
|
+
"@ateam-ai-mobile/ui": "0.1.13"
|
|
15
15
|
},
|
|
16
16
|
"peerDependencies": {
|
|
17
17
|
"@react-native-async-storage/async-storage": "*",
|
|
@@ -23,7 +23,7 @@ function cameraState(): VisualSnapshot['camera'] {
|
|
|
23
23
|
// would let a guide speak about a scene nothing is watching.
|
|
24
24
|
available: !!session,
|
|
25
25
|
active: visualEngine.isCameraReady(),
|
|
26
|
-
facing: session ?
|
|
26
|
+
facing: session ? session.facing : 'unknown',
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
29
|
|
|
@@ -52,7 +52,12 @@ export function createVisualDomain(): DeviceState['visual'] {
|
|
|
52
52
|
},
|
|
53
53
|
|
|
54
54
|
async start(options) {
|
|
55
|
-
const
|
|
55
|
+
const facing = options?.facing ?? 'back';
|
|
56
|
+
if (facing !== 'front' && facing !== 'back') {
|
|
57
|
+
// Refused, never quietly swapped for the other camera.
|
|
58
|
+
throw unsupported(`visual.start facing "${String(facing)}" — use "front" or "back"`);
|
|
59
|
+
}
|
|
60
|
+
const s = visualEngine.startSession((options?.intent as any) || 'navigation', facing);
|
|
56
61
|
// Warm the compass alongside the camera. Started here rather than at app
|
|
57
62
|
// launch: a magnetometer subscription running all day for a feature
|
|
58
63
|
// nobody opened is exactly the kind of cost this layer is meant to avoid.
|
|
@@ -117,6 +117,8 @@ export interface VisualSessionState {
|
|
|
117
117
|
id: string;
|
|
118
118
|
intent: VisualIntentName;
|
|
119
119
|
startedAt: number;
|
|
120
|
+
/** The camera the session reads. The host opens THIS one. */
|
|
121
|
+
facing: 'front' | 'back';
|
|
120
122
|
/** Set while the app is backgrounded. The session SURVIVES; the camera does not. */
|
|
121
123
|
suspended: boolean;
|
|
122
124
|
}
|
|
@@ -137,12 +139,23 @@ class VisualEngine {
|
|
|
137
139
|
|
|
138
140
|
// ── session ──
|
|
139
141
|
|
|
140
|
-
startSession(intent: VisualIntentName): VisualSessionState {
|
|
141
|
-
if (this.session)
|
|
142
|
+
startSession(intent: VisualIntentName, facing: 'front' | 'back' = 'back'): VisualSessionState {
|
|
143
|
+
if (this.session) {
|
|
144
|
+
if (this.session.facing === facing) return this.session;
|
|
145
|
+
// Same session, other camera. A NEW object, so a host holding the old one
|
|
146
|
+
// sees the change; the frames are dropped because they came from the
|
|
147
|
+
// other camera.
|
|
148
|
+
this.session = { ...this.session, facing };
|
|
149
|
+
this.frames = [];
|
|
150
|
+
this.cameraReady = false;
|
|
151
|
+
this.notify();
|
|
152
|
+
return this.session;
|
|
153
|
+
}
|
|
142
154
|
this.session = {
|
|
143
155
|
id: `visual_${Date.now()}`,
|
|
144
156
|
intent: PROFILES[intent] ? intent : 'navigation',
|
|
145
157
|
startedAt: Date.now(),
|
|
158
|
+
facing,
|
|
146
159
|
suspended: false,
|
|
147
160
|
};
|
|
148
161
|
this.notify();
|
package/src/vision/index.ts
CHANGED
|
@@ -74,6 +74,8 @@ interface AdaVisionNative {
|
|
|
74
74
|
isSupported(): boolean;
|
|
75
75
|
assessQuality(uri: string): Promise<RawFrameQuality>;
|
|
76
76
|
startVision(intervalMs: number): Promise<void>;
|
|
77
|
+
/** Absent from binaries built before the front camera existed. */
|
|
78
|
+
startVisionCamera?(intervalMs: number, facing: VisionFacing): Promise<void>;
|
|
77
79
|
stopVision(): void;
|
|
78
80
|
visionRetentionMs(): number;
|
|
79
81
|
qualityById(id: string): RawFrameQuality | null;
|
|
@@ -85,6 +87,17 @@ interface AdaVisionNative {
|
|
|
85
87
|
|
|
86
88
|
const native = requireOptionalNativeModule<AdaVisionNative>('AdaVision');
|
|
87
89
|
|
|
90
|
+
/** The native function, or a refusal by name: Android's AdaVision has assessQuality only
|
|
91
|
+
* (AI Models F4), so a verb it lacks says so instead of failing as "not a function". */
|
|
92
|
+
function verb<K extends keyof AdaVisionNative>(name: K): AdaVisionNative[K] {
|
|
93
|
+
if (!native) throw new Error('ada-vision native module not present in this build');
|
|
94
|
+
const fn = native[name];
|
|
95
|
+
if (typeof fn !== 'function') {
|
|
96
|
+
throw Object.assign(new Error(`UNAVAILABLE: ada-vision ${String(name)} is not implemented on this platform (iOS only)`), { code: 'UNAVAILABLE' });
|
|
97
|
+
}
|
|
98
|
+
return (fn as Function).bind(native) as AdaVisionNative[K];
|
|
99
|
+
}
|
|
100
|
+
|
|
88
101
|
/** Is the native vision module present in this binary? */
|
|
89
102
|
export function isSupported(): boolean {
|
|
90
103
|
try { return native?.isSupported() ?? false; } catch { return false; }
|
|
@@ -98,10 +111,24 @@ export async function assessQuality(uri: string): Promise<RawFrameQuality> {
|
|
|
98
111
|
|
|
99
112
|
// ── Vision session (feeds the Visual Engine) ──
|
|
100
113
|
|
|
101
|
-
/**
|
|
102
|
-
export
|
|
103
|
-
|
|
104
|
-
|
|
114
|
+
/** Which camera the vision session reads. */
|
|
115
|
+
export type VisionFacing = 'front' | 'back';
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Start the vision session on `facing` — publishes a timeline frame every
|
|
119
|
+
* `intervalMs`. The back camera works on every binary; the front camera needs a
|
|
120
|
+
* binary built with it, and an older one REFUSES it rather than quietly reading
|
|
121
|
+
* the back camera instead.
|
|
122
|
+
*/
|
|
123
|
+
export async function startVision(intervalMs = 1000, facing: VisionFacing = 'back'): Promise<void> {
|
|
124
|
+
if (facing !== 'front' && facing !== 'back') {
|
|
125
|
+
throw new Error(`Unknown camera "${String(facing)}" — use "front" or "back"`);
|
|
126
|
+
}
|
|
127
|
+
if (facing === 'back') return verb('startVision')(intervalMs);
|
|
128
|
+
if (native && typeof native.startVisionCamera !== 'function') {
|
|
129
|
+
throw new Error('the front camera needs a newer app build — this one can read the back camera only');
|
|
130
|
+
}
|
|
131
|
+
return verb('startVisionCamera')!(intervalMs, facing);
|
|
105
132
|
}
|
|
106
133
|
|
|
107
134
|
/** Stop the vision session and release the camera. */
|
|
@@ -127,20 +154,17 @@ export function qualityById(id: string): RawFrameQuality | null {
|
|
|
127
154
|
|
|
128
155
|
/** On-device OCR on a published frame. [] = looked, no text; throws = couldn't look. */
|
|
129
156
|
export async function readText(id: string, languages?: string[]): Promise<VisionText[]> {
|
|
130
|
-
|
|
131
|
-
return native.readText(id, languages ?? null);
|
|
157
|
+
return verb('readText')(id, languages ?? null);
|
|
132
158
|
}
|
|
133
159
|
|
|
134
160
|
/** On-device image classification → labels (not prose). [] = nothing recognised. */
|
|
135
161
|
export async function describe(id: string, maxLabels = 5): Promise<VisionLabel[]> {
|
|
136
|
-
|
|
137
|
-
return native.describe(id, maxLabels);
|
|
162
|
+
return verb('describe')(id, maxLabels);
|
|
138
163
|
}
|
|
139
164
|
|
|
140
165
|
/** On-device object detection (bundled YOLOv8n, Open-Images vocabulary) → located
|
|
141
166
|
* items with boxes. [] = looked, found nothing above `minConfidence`; throws =
|
|
142
167
|
* couldn't look (id aged out, or the model isn't in this build). */
|
|
143
168
|
export async function detectObjects(id: string, maxItems = 10, minConfidence = 0.25): Promise<VisionObject[]> {
|
|
144
|
-
|
|
145
|
-
return native.detectObjects(id, maxItems, minConfidence);
|
|
169
|
+
return verb('detectObjects')(id, maxItems, minConfidence);
|
|
146
170
|
}
|