@trillboards/edge-sdk 0.2.194 → 0.2.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/pyproject.toml +1 -1
- package/deploy/detector/tests/test_loader.py +1 -1
- package/deploy/detector/tests/test_vehicle_end.py +215 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/loader.py +6 -0
- package/deploy/detector/trillboards_detector/registry.py +30 -1
- package/deploy/detector/trillboards_detector/server.py +39 -0
- package/deploy/detector/trillboards_detector/tasks.py +8 -1
- package/deploy/detector/trillboards_detector/vehicle_end.py +154 -0
- package/deploy/systemd/trillboards-detector.service +10 -3
- package/dist/DetectorFields.d.ts +121 -1
- package/dist/DetectorFields.d.ts.map +1 -1
- package/dist/DetectorFields.js +123 -3
- package/dist/DetectorFields.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/LocalDetectorClient.d.ts +21 -0
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +44 -0
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/PhoneHeldInHand.d.ts +38 -3
- package/dist/PhoneHeldInHand.d.ts.map +1 -1
- package/dist/PhoneHeldInHand.js +33 -6
- package/dist/PhoneHeldInHand.js.map +1 -1
- package/dist/ScopedVlmReview.d.ts +19 -4
- package/dist/ScopedVlmReview.d.ts.map +1 -1
- package/dist/ScopedVlmReview.js +88 -22
- package/dist/ScopedVlmReview.js.map +1 -1
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +2 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/localSensing.d.ts +4 -1
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +13 -9
- package/dist/localSensing.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { RTMPOSE_MODEL_ID, RTMPOSE_SHA256, type DetectionPoseRequest, type DetectionPoseResult } from './ActorPose';
|
|
2
|
+
import type { DetectionColorsRequest, DetectionColorsResult } from './DetectionColors';
|
|
2
3
|
import type { Detection } from './LocalDetectorClient';
|
|
3
4
|
import { type NormalizedPoint } from './zoneContainment';
|
|
4
5
|
/**
|
|
@@ -20,8 +21,21 @@ import { type NormalizedPoint } from './zoneContainment';
|
|
|
20
21
|
* Gemma 4 E2B as deployed 2/18 caught, 1 false 19.2 s per frame (Texas)
|
|
21
22
|
* The 2026-09-01 detector arm that lost to the VLM (DetectorFieldProducer
|
|
22
23
|
* `co_located` note) had no wrist ownership and no geometric polygon.
|
|
24
|
+
*
|
|
25
|
+
* `native_held_in_hand_staff_v1` (cameras whose staff wear the store's uniform,
|
|
26
|
+
* server-selected): the holder must also be staff, i.e. at least `staffFraction`
|
|
27
|
+
* of their person box is the uniform colour (the detector service's
|
|
28
|
+
* `detection_colors`, same Ace red as the unattended-customer rule). Both false
|
|
29
|
+
* alert episodes of the replayed 2026-10-30 Texas window were CUSTOMERS across the
|
|
30
|
+
* counter whose wrists sat by a phone or wallet (holder share 0.0001-0.0065), while
|
|
31
|
+
* staff holders measured 0.036 and up (219 phone frames) and 0.044 and up on the
|
|
32
|
+
* Gate 0 set (seen from above, the vest is mostly under hair and a plaid shirt, so
|
|
33
|
+
* 0.08 would drop 2 of its 18). At 0.02, through the detector service's code on all
|
|
34
|
+
* 1,956 labelled stills: phone frames caught 219/290 (unchanged), frames wrongly
|
|
35
|
+
* yes 32 -> 9; over the replay boxes' own look times, false alert episodes 1 and
|
|
36
|
+
* 2 -> 0 with long episodes caught 6/7 (unchanged); Gate 0 holds at 16/18, 0 false.
|
|
23
37
|
*/
|
|
24
|
-
export interface
|
|
38
|
+
export interface PhoneHeldInHandV1Policy {
|
|
25
39
|
version: 1;
|
|
26
40
|
method: 'native_held_in_hand_v1';
|
|
27
41
|
minPhoneConfidence: 0.4;
|
|
@@ -39,8 +53,21 @@ export interface PhoneHeldInHandPolicy {
|
|
|
39
53
|
sha256: typeof RTMPOSE_SHA256;
|
|
40
54
|
};
|
|
41
55
|
}
|
|
56
|
+
export interface PhoneHeldByStaffPolicy extends Omit<PhoneHeldInHandV1Policy, 'method'> {
|
|
57
|
+
method: 'native_held_in_hand_staff_v1';
|
|
58
|
+
/** The store's uniform on the holder's person box (detection_colors, Pillow 8-bit HSV). */
|
|
59
|
+
holder: {
|
|
60
|
+
hue: [[0, 10], [240, 255]];
|
|
61
|
+
minSaturation: 100;
|
|
62
|
+
minValue: 60;
|
|
63
|
+
staffFraction: 0.02;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export type PhoneHeldInHandPolicy = PhoneHeldInHandV1Policy | PhoneHeldByStaffPolicy;
|
|
67
|
+
/** Every held-in-hand method this build parses; the profile request names them (X-Detector-Field-Methods). */
|
|
68
|
+
export declare const PHONE_HELD_METHODS: readonly string[];
|
|
42
69
|
export interface PhoneHeldInHandTrace {
|
|
43
|
-
method: '
|
|
70
|
+
method: PhoneHeldInHandPolicy['method'];
|
|
44
71
|
imageSha256: string;
|
|
45
72
|
policySha256: string;
|
|
46
73
|
detectorSha256: string;
|
|
@@ -55,11 +82,16 @@ export interface PhoneHeldInHandTrace {
|
|
|
55
82
|
wrist: 'left_wrist' | 'right_wrist';
|
|
56
83
|
phoneConfidence: number;
|
|
57
84
|
reachRatio: number;
|
|
85
|
+
/** The holder's uniform share (staff method only). */
|
|
86
|
+
uniformFraction?: number;
|
|
58
87
|
};
|
|
59
88
|
}
|
|
60
89
|
declare const RF_SHA = "8e8197112112b79d7cc5d0a94ecef659c05317a658aadc7c7643374c539d7615";
|
|
61
90
|
export declare function parsePhoneHeldInHand(value: unknown): PhoneHeldInHandPolicy | undefined;
|
|
62
|
-
/**
|
|
91
|
+
/**
|
|
92
|
+
* What the detector call must ask for: every class the rule reads, pose on confident people, and (staff method)
|
|
93
|
+
* the uniform share of exactly those people: the same minimum score and count select the same boxes.
|
|
94
|
+
*/
|
|
63
95
|
export declare function heldInHandDetectorRequest(policy: PhoneHeldInHandPolicy): {
|
|
64
96
|
classes: string[];
|
|
65
97
|
minConfidence: number;
|
|
@@ -72,6 +104,7 @@ export declare function heldInHandDetectorRequest(policy: PhoneHeldInHandPolicy)
|
|
|
72
104
|
sha256: typeof RTMPOSE_SHA256;
|
|
73
105
|
};
|
|
74
106
|
detectionPose: DetectionPoseRequest;
|
|
107
|
+
detectionColors?: DetectionColorsRequest;
|
|
75
108
|
};
|
|
76
109
|
/** The route polygon, boundary-inclusive like every other zone rule (zoneContainment). */
|
|
77
110
|
export declare function insidePolygon(point: NormalizedPoint, polygon: ReadonlyArray<NormalizedPoint>): boolean;
|
|
@@ -89,6 +122,8 @@ export declare function decidePhoneHeldInHand(input: {
|
|
|
89
122
|
};
|
|
90
123
|
detections: ReadonlyArray<Detection>;
|
|
91
124
|
pose: DetectionPoseResult;
|
|
125
|
+
/** The staff method's verified detection_colors result for this frame (LocalDetectorClient checks its proof). */
|
|
126
|
+
colors?: DetectionColorsResult;
|
|
92
127
|
}): {
|
|
93
128
|
value: boolean;
|
|
94
129
|
trace: PhoneHeldInHandTrace;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PhoneHeldInHand.d.ts","sourceRoot":"","sources":["../src/PhoneHeldInHand.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"PhoneHeldInHand.d.ts","sourceRoot":"","sources":["../src/PhoneHeldInHand.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,gBAAgB,EAAC,cAAc,EAAC,KAAK,oBAAoB,EAAC,KAAK,mBAAmB,EAAC,MAAM,aAAa,CAAC;AAC/G,OAAO,KAAK,EAAC,sBAAsB,EAAC,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AACpF,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAiB,KAAK,eAAe,EAAC,MAAM,mBAAmB,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAC,CAAC,CAAC;IAAC,MAAM,EAAC,wBAAwB,CAAC;IAC3C,kBAAkB,EAAC,GAAG,CAAC;IAAC,mBAAmB,EAAC,GAAG,CAAC;IAAC,UAAU,EAAC,IAAI,CAAC;IAAC,SAAS,EAAC,CAAC,CAAC;IAC9E,QAAQ,EAAC;QAAC,OAAO,EAAC,eAAe,CAAC;QAAC,MAAM,EAAC,MAAM,CAAC;QAAC,MAAM,EAAC,OAAO,MAAM,CAAA;KAAC,CAAC;IACxE,IAAI,EAAC;QAAC,OAAO,EAAC,OAAO,gBAAgB,CAAC;QAAC,MAAM,EAAC,MAAM,CAAC;QAAC,MAAM,EAAC,OAAO,cAAc,CAAA;KAAC,CAAC;CACrF;AACD,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,uBAAuB,EAAC,QAAQ,CAAC;IACpF,MAAM,EAAC,8BAA8B,CAAC;IACtC,2FAA2F;IAC3F,MAAM,EAAC;QAAC,GAAG,EAAC,CAAC,CAAC,CAAC,EAAC,EAAE,CAAC,EAAC,CAAC,GAAG,EAAC,GAAG,CAAC,CAAC,CAAC;QAAA,aAAa,EAAC,GAAG,CAAC;QAAA,QAAQ,EAAC,EAAE,CAAC;QAAA,aAAa,EAAC,IAAI,CAAA;KAAC,CAAC;CAClF;AACD,MAAM,MAAM,qBAAqB,GAAC,uBAAuB,GAAC,sBAAsB,CAAC;AACjF,8GAA8G;AAC9G,eAAO,MAAM,kBAAkB,EAAC,SAAS,MAAM,EAA2E,CAAC;AAC3H,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAAC,WAAW,EAAC,MAAM,CAAC;IAAC,YAAY,EAAC,MAAM,CAAC;IAChF,cAAc,EAAC,MAAM,CAAC;IAAC,UAAU,EAAC,MAAM,CAAC;IACzC,uFAAuF;IACvF,MAAM,EAAC,MAAM,CAAC;IAAC,MAAM,EAAC,MAAM,CAAC;IAC7B,sFAAsF;IACtF,IAAI,EAAC,IAAI,GAAC;QAAC,UAAU,EAAC,MAAM,CAAC;QAAA,WAAW,EAAC,MAAM,CAAC;QAAA,KAAK,EAAC,YAAY,GAAC,aAAa,CAAC;QAC/E,eAAe,EAAC,MAAM,CAAC;QAAA,UAAU,EAAC,MAAM,CAAC;QACzC,sDAAsD;QACtD,eAAe,CAAC,EAAC,MAAM,CAAA;KAAC,CAAC;CAC5B;AACD,QAAA,MAAM,MAAM,qEAAmE,CAAC;AAKhF,wBAAgB,oBAAoB,CAAC,KAAK,EAAC,OAAO,GAAE,qBAAqB,GAAC,SAAS,CAclF;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAC,qBAAqB,GAAE;IACtE,OAAO,EAAC,MAAM,EAAE,CAAC;IAAA,aAAa,EAAC,MAAM,CAAC;IAAA,aAAa,EAAC;QAAC,OAAO,EAAC,MAAM,CAAC;QAAA,MAAM,EAAC,MAAM,CAAA;KAAC,CAAC;IACnF,IAAI,EAAC;QAAC,OAAO,EAAC,OAAO,gBAAgB,CAAC;QAAA,MAAM,EAAC,OAAO,cAAc,CAAA;KAAC,CAAC;IAAA,aAAa,EAAC,oBAAoB,CAAC;IACvG,eAAe,CAAC,EAAC,sBAAsB,CAAA;CAAC,CASzC;AAED,0FAA0F;AAC1F,wBAAgB,aAAa,CAAC,KAAK,EAAC,eAAe,EAAC,OAAO,EAAC,aAAa,CAAC,eAAe,CAAC,GAAE,OAAO,CAElG;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAC;IAC1C,IAAI,EAAC,MAAM,CAAC;IAAA,MAAM,EAAC,qBAAqB,CAAC;IAAA,MAAM,EAAC,aAAa,CAAC,eAAe,CAAC,CAAC;IAC/E,KAAK,EAAC;QAAC,KAAK,EAAC,MAAM,CAAC;QAAA,MAAM,EAAC,MAAM,CAAA;KAAC,CAAC;IAAA,UAAU,EAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IAAA,IAAI,EAAC,mBAAmB,CAAC;IAChG,iHAAiH;IACjH,MAAM,CAAC,EAAC,qBAAqB,CAAC;CAC/B,GAAE;IAAC,KAAK,EAAC,OAAO,CAAC;IAAA,KAAK,EAAC,oBAAoB,CAAA;CAAC,CAuC5C"}
|
package/dist/PhoneHeldInHand.js
CHANGED
|
@@ -1,21 +1,29 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PHONE_HELD_METHODS = void 0;
|
|
3
4
|
exports.parsePhoneHeldInHand = parsePhoneHeldInHand;
|
|
4
5
|
exports.heldInHandDetectorRequest = heldInHandDetectorRequest;
|
|
5
6
|
exports.insidePolygon = insidePolygon;
|
|
6
7
|
exports.decidePhoneHeldInHand = decidePhoneHeldInHand;
|
|
7
8
|
const crypto_1 = require("crypto");
|
|
9
|
+
const util_1 = require("util");
|
|
8
10
|
const ActorPose_1 = require("./ActorPose");
|
|
9
11
|
const zoneContainment_1 = require("./zoneContainment");
|
|
12
|
+
/** Every held-in-hand method this build parses; the profile request names them (X-Detector-Field-Methods). */
|
|
13
|
+
exports.PHONE_HELD_METHODS = Object.freeze(['native_held_in_hand_v1', 'native_held_in_hand_staff_v1']);
|
|
10
14
|
const RF_SHA = '8e8197112112b79d7cc5d0a94ecef659c05317a658aadc7c7643374c539d7615';
|
|
15
|
+
const STAFF_HOLDER = { hue: [[0, 10], [240, 255]], minSaturation: 100, minValue: 60, staffFraction: 0.02 };
|
|
11
16
|
const keys = (v, want) => !!v && typeof v === 'object' && !Array.isArray(v) && Object.keys(v).sort().join(',') === want;
|
|
12
17
|
const hash = (v) => (0, crypto_1.createHash)('sha256').update(v).digest('hex');
|
|
13
18
|
function parsePhoneHeldInHand(value) {
|
|
14
19
|
if (value === undefined)
|
|
15
20
|
return undefined;
|
|
16
21
|
const v = value;
|
|
17
|
-
|
|
18
|
-
|
|
22
|
+
const staff = v?.method === 'native_held_in_hand_staff_v1';
|
|
23
|
+
if (!keys(v, 'detector,' + (staff ? 'holder,' : '') + 'maxPeople,method,minPersonConfidence,minPhoneConfidence,pose,version,wristReach')
|
|
24
|
+
|| v.version !== 1 || v.method !== (staff ? 'native_held_in_hand_staff_v1' : 'native_held_in_hand_v1')
|
|
25
|
+
|| staff && !(0, util_1.isDeepStrictEqual)(v.holder, STAFF_HOLDER)
|
|
26
|
+
|| v.minPhoneConfidence !== .4 || v.minPersonConfidence !== .5
|
|
19
27
|
|| v.wristReach !== .25 || v.maxPeople !== 8
|
|
20
28
|
|| !keys(v.detector, 'format,modelId,sha256') || v.detector.modelId !== 'rfdetr_medium' || v.detector.format !== 'onnx'
|
|
21
29
|
|| v.detector.sha256 !== RF_SHA
|
|
@@ -24,13 +32,19 @@ function parsePhoneHeldInHand(value) {
|
|
|
24
32
|
throw Error('Invalid phone held-in-hand policy');
|
|
25
33
|
return JSON.parse(JSON.stringify(v));
|
|
26
34
|
}
|
|
27
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* What the detector call must ask for: every class the rule reads, pose on confident people, and (staff method)
|
|
37
|
+
* the uniform share of exactly those people: the same minimum score and count select the same boxes.
|
|
38
|
+
*/
|
|
28
39
|
function heldInHandDetectorRequest(policy) {
|
|
40
|
+
const holder = policy.method === 'native_held_in_hand_staff_v1' ? policy.holder : undefined;
|
|
29
41
|
return { classes: ['person', 'cell phone'],
|
|
30
42
|
minConfidence: Math.min(policy.minPhoneConfidence, policy.minPersonConfidence),
|
|
31
43
|
expectedModel: { modelId: policy.detector.modelId, sha256: policy.detector.sha256 },
|
|
32
44
|
pose: { modelId: policy.pose.modelId, sha256: policy.pose.sha256 },
|
|
33
|
-
detectionPose: { minScore: policy.minPersonConfidence, maxPeople: policy.maxPeople }
|
|
45
|
+
detectionPose: { minScore: policy.minPersonConfidence, maxPeople: policy.maxPeople },
|
|
46
|
+
...(holder ? { detectionColors: { version: 1, minScore: policy.minPersonConfidence, maxBoxes: policy.maxPeople,
|
|
47
|
+
hue: holder.hue.map(r => [r[0], r[1]]), minSaturation: holder.minSaturation, minValue: holder.minValue } } : {}) };
|
|
34
48
|
}
|
|
35
49
|
/** The route polygon, boundary-inclusive like every other zone rule (zoneContainment). */
|
|
36
50
|
function insidePolygon(point, polygon) {
|
|
@@ -45,6 +59,15 @@ function decidePhoneHeldInHand(input) {
|
|
|
45
59
|
const { width, height } = input.frame;
|
|
46
60
|
if (!(width > 1 && height > 1) || input.region.length < 3)
|
|
47
61
|
throw Error('Phone held-in-hand needs a frame and a polygon');
|
|
62
|
+
const holder = policy.method === 'native_held_in_hand_staff_v1' ? policy.holder : undefined;
|
|
63
|
+
const uniform = new Map();
|
|
64
|
+
if (holder) {
|
|
65
|
+
const asked = heldInHandDetectorRequest(policy).detectionColors;
|
|
66
|
+
if (!input.colors || !(0, util_1.isDeepStrictEqual)(input.colors.request, asked))
|
|
67
|
+
throw Error('Phone held-by-staff needs the uniform colour of each posed person');
|
|
68
|
+
for (const row of input.colors.people)
|
|
69
|
+
uniform.set(row.detectionIndex, row.fraction);
|
|
70
|
+
}
|
|
48
71
|
const centre = (d) => ({ x: (d.box.x + d.box.width / 2) / width, y: (d.box.y + d.box.height / 2) / height });
|
|
49
72
|
const phones = input.detections.map((d, index) => ({ d, index }))
|
|
50
73
|
.filter(({ d }) => d.label === 'cell phone' && d.confidence >= policy.minPhoneConfidence && insidePolygon(centre(d), input.region));
|
|
@@ -53,6 +76,9 @@ function decidePhoneHeldInHand(input) {
|
|
|
53
76
|
.filter(({ d }) => d && d.label === 'person' && d.confidence >= policy.minPersonConfidence && insidePolygon(centre(d), input.region));
|
|
54
77
|
let held = null;
|
|
55
78
|
for (const { row, d } of people) {
|
|
79
|
+
// Staff method: a holder without the uniform (a customer across the counter) cannot hold it.
|
|
80
|
+
if (holder && !((uniform.get(row.detectionIndex) ?? 0) >= holder.staffFraction))
|
|
81
|
+
continue;
|
|
56
82
|
const scale = Math.max(d.box.width, d.box.height);
|
|
57
83
|
for (const wrist of ['left_wrist', 'right_wrist']) {
|
|
58
84
|
const joint = row.joints.find(j => j.name === wrist);
|
|
@@ -63,12 +89,13 @@ function decidePhoneHeldInHand(input) {
|
|
|
63
89
|
const reachRatio = Math.hypot(joint.x - px, joint.y - py) / scale;
|
|
64
90
|
if (reachRatio <= policy.wristReach && (!held || reachRatio < held.reachRatio)) {
|
|
65
91
|
held = { phoneIndex: index, personIndex: row.detectionIndex, wrist,
|
|
66
|
-
phoneConfidence: phone.confidence, reachRatio: Math.round(reachRatio * 1000) / 1000
|
|
92
|
+
phoneConfidence: phone.confidence, reachRatio: Math.round(reachRatio * 1000) / 1000,
|
|
93
|
+
...(holder ? { uniformFraction: uniform.get(row.detectionIndex) } : {}) };
|
|
67
94
|
}
|
|
68
95
|
}
|
|
69
96
|
}
|
|
70
97
|
}
|
|
71
|
-
return { value: held !== null, trace: { method:
|
|
98
|
+
return { value: held !== null, trace: { method: policy.method, imageSha256: hash(input.jpeg),
|
|
72
99
|
policySha256: hash(JSON.stringify(policy)), detectorSha256: policy.detector.sha256, poseSha256: policy.pose.sha256,
|
|
73
100
|
phones: phones.length, people: people.length, held } };
|
|
74
101
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PhoneHeldInHand.js","sourceRoot":"","sources":["../src/PhoneHeldInHand.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"PhoneHeldInHand.js","sourceRoot":"","sources":["../src/PhoneHeldInHand.ts"],"names":[],"mappings":";;;AAsEA,oDAcC;AAMD,8DAYC;AAGD,sCAEC;AAMD,sDA4CC;AA7JD,mCAAkC;AAClC,+BAAuC;AACvC,2CAA+G;AAG/G,uDAAuE;AA+CvE,8GAA8G;AACjG,QAAA,kBAAkB,GAAmB,MAAM,CAAC,MAAM,CAAC,CAAC,wBAAwB,EAAC,8BAA8B,CAAC,CAAC,CAAC;AAY3H,MAAM,MAAM,GAAC,kEAAkE,CAAC;AAChF,MAAM,YAAY,GAAC,EAAC,GAAG,EAAC,CAAC,CAAC,CAAC,EAAC,EAAE,CAAC,EAAC,CAAC,GAAG,EAAC,GAAG,CAAC,CAAC,EAAC,aAAa,EAAC,GAAG,EAAC,QAAQ,EAAC,EAAE,EAAC,aAAa,EAAC,IAAI,EAAU,CAAC;AACtG,MAAM,IAAI,GAAC,CAAC,CAAS,EAAC,IAAW,EAAC,EAAE,CAAA,CAAC,CAAC,CAAC,IAAE,OAAO,CAAC,KAAG,QAAQ,IAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAG,IAAI,CAAC;AACxH,MAAM,IAAI,GAAC,CAAC,CAAe,EAAC,EAAE,CAAA,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE3E,SAAgB,oBAAoB,CAAC,KAAa;IAChD,IAAG,KAAK,KAAG,SAAS;QAAC,OAAO,SAAS,CAAC;IACtC,MAAM,CAAC,GAAC,KAA8B,CAAC;IACvC,MAAM,KAAK,GAAC,CAAC,EAAE,MAAM,KAAG,8BAA8B,CAAC;IACvD,IAAG,CAAC,IAAI,CAAC,CAAC,EAAC,WAAW,GAAC,CAAC,KAAK,CAAA,CAAC,CAAA,SAAS,CAAA,CAAC,CAAA,EAAE,CAAC,GAAC,iFAAiF,CAAC;WAC1H,CAAC,CAAC,OAAO,KAAG,CAAC,IAAE,CAAC,CAAC,MAAM,KAAG,CAAC,KAAK,CAAA,CAAC,CAAA,8BAA8B,CAAA,CAAC,CAAA,wBAAwB,CAAC;WACzF,KAAK,IAAE,CAAC,IAAA,wBAAiB,EAAE,CAA4B,CAAC,MAAM,EAAC,YAAY,CAAC;WAC5E,CAAC,CAAC,kBAAkB,KAAG,EAAE,IAAE,CAAC,CAAC,mBAAmB,KAAG,EAAE;WACrD,CAAC,CAAC,UAAU,KAAG,GAAG,IAAE,CAAC,CAAC,SAAS,KAAG,CAAC;WACnC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAC,uBAAuB,CAAC,IAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAG,eAAe,IAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAG,MAAM;WAC3G,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAG,MAAM;WAC1B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAC,uBAAuB,CAAC,IAAE,CAAC,CAAC,IAAI,CAAC,OAAO,KAAG,4BAAgB,IAAE,CAAC,CAAC,IAAI,CAAC,MAAM,KAAG,MAAM;WAChG,CAAC,CAAC,IAAI,CAAC,MAAM,KAAG,0BAAc;QAAC,MAAM,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACpF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,SAAgB,yBAAyB,CAAC,MAA4B;IAIpE,MAAM,MAAM,GAAC,MAAM,CAAC,MAAM,KAAG,8BAA8B,CAAA,CAAC,CAAA,MAAM,CAAC,MAAM,CAAA,CAAC,CAAA,SAAS,CAAC;IACpF,OAAO,EAAC,OAAO,EAAC,CAAC,QAAQ,EAAC,YAAY,CAAC;QACrC,aAAa,EAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAC,MAAM,CAAC,mBAAmB,CAAC;QAC5E,aAAa,EAAC,EAAC,OAAO,EAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAC,MAAM,EAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAC;QAC7E,IAAI,EAAC,EAAC,OAAO,EAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAC,MAAM,EAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAC;QAC5D,aAAa,EAAC,EAAC,QAAQ,EAAC,MAAM,CAAC,mBAAmB,EAAC,SAAS,EAAC,MAAM,CAAC,SAAS,EAAC;QAC9E,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,eAAe,EAAC,EAAC,OAAO,EAAC,CAAC,EAAC,QAAQ,EAAC,MAAM,CAAC,mBAAmB,EAAC,QAAQ,EAAC,MAAM,CAAC,SAAS;gBACpG,GAAG,EAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA,EAAE,CAAA,CAAC,CAAC,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC,CAAC,CAAoB,CAAC,EAAC,aAAa,EAAC,MAAM,CAAC,aAAa,EAAC,QAAQ,EAAC,MAAM,CAAC,QAAQ,EAAC,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAC,CAAC;AACjI,CAAC;AAED,0FAA0F;AAC1F,SAAgB,aAAa,CAAC,KAAqB,EAAC,OAAsC;IACxF,OAAO,IAAA,gCAAc,EAAC,KAAK,CAAC,CAAC,EAAC,KAAK,CAAC,CAAC,EAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,SAAgB,qBAAqB,CAAC,KAKrC;IACC,MAAM,MAAM,GAAC,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;IACjD,MAAM,EAAC,KAAK,EAAC,MAAM,EAAC,GAAC,KAAK,CAAC,KAAK,CAAC;IACjC,IAAG,CAAC,CAAC,KAAK,GAAC,CAAC,IAAE,MAAM,GAAC,CAAC,CAAC,IAAE,KAAK,CAAC,MAAM,CAAC,MAAM,GAAC,CAAC;QAAC,MAAM,KAAK,CAAC,gDAAgD,CAAC,CAAC;IAC7G,MAAM,MAAM,GAAC,MAAM,CAAC,MAAM,KAAG,8BAA8B,CAAA,CAAC,CAAA,MAAM,CAAC,MAAM,CAAA,CAAC,CAAA,SAAS,CAAC;IACpF,MAAM,OAAO,GAAC,IAAI,GAAG,EAAiB,CAAC;IACvC,IAAG,MAAM,EAAC,CAAC;QACT,MAAM,KAAK,GAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC;QAC9D,IAAG,CAAC,KAAK,CAAC,MAAM,IAAE,CAAC,IAAA,wBAAiB,EAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAC,KAAK,CAAC;YAAC,MAAM,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAClJ,KAAI,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM;YAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,EAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAC,CAAC,CAAW,EAAiB,EAAE,CAAA,CAAC,EAAC,CAAC,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAC,CAAC,CAAC,GAAC,KAAK,EAAC,CAAC,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAC,CAAC,CAAC,GAAC,MAAM,EAAC,CAAC,CAAC;IAClH,MAAM,MAAM,GAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,KAAK,EAAC,EAAE,CAAA,CAAC,EAAC,CAAC,EAAC,KAAK,EAAC,CAAC,CAAC;SACtD,MAAM,CAAC,CAAC,EAAC,CAAC,EAAC,EAAC,EAAE,CAAA,CAAC,CAAC,KAAK,KAAG,YAAY,IAAE,CAAC,CAAC,UAAU,IAAE,MAAM,CAAC,kBAAkB,IAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACzH,MAAM,MAAM,GAAC,KAAK,CAAC,IAAI,CAAC,MAAM;SAC3B,GAAG,CAAC,GAAG,CAAA,EAAE,CAAA,CAAC,EAAC,GAAG,EAAC,CAAC,EAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,CAAC,EAAC,CAAC,CAAC;SACxD,MAAM,CAAC,CAAC,EAAC,CAAC,EAAC,EAAC,EAAE,CAAA,CAAC,IAAE,CAAC,CAAC,KAAK,KAAG,QAAQ,IAAE,CAAC,CAAC,UAAU,IAAE,MAAM,CAAC,mBAAmB,IAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACzH,IAAI,IAAI,GAA8B,IAAI,CAAC;IAC3C,KAAI,MAAM,EAAC,GAAG,EAAC,CAAC,EAAC,IAAI,MAAM,EAAC,CAAC;QAC3B,6FAA6F;QAC7F,IAAG,MAAM,IAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAE,MAAM,CAAC,aAAa,CAAC;YAAC,SAAS;QACpF,MAAM,KAAK,GAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/C,KAAI,MAAM,KAAK,IAAI,CAAC,YAAY,EAAC,aAAa,CAAU,EAAC,CAAC;YACxD,MAAM,KAAK,GAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA,EAAE,CAAA,CAAC,CAAC,IAAI,KAAG,KAAK,CAAC,CAAC;YAC/C,IAAG,CAAC,KAAK,IAAE,KAAK,CAAC,CAAC,KAAG,IAAI,IAAE,KAAK,CAAC,CAAC,KAAG,IAAI;gBAAC,SAAS;YACnD,KAAI,MAAM,EAAC,CAAC,EAAC,KAAK,EAAC,KAAK,EAAC,IAAI,MAAM,EAAC,CAAC;gBACnC,MAAM,EAAE,GAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAC,CAAC,EAAC,EAAE,GAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAC,CAAC,CAAC;gBACzE,MAAM,UAAU,GAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAC,EAAE,EAAC,KAAK,CAAC,CAAC,GAAC,EAAE,CAAC,GAAC,KAAK,CAAC;gBACzD,IAAG,UAAU,IAAE,MAAM,CAAC,UAAU,IAAE,CAAC,CAAC,IAAI,IAAE,UAAU,GAAC,IAAI,CAAC,UAAU,CAAC,EAAC,CAAC;oBACrE,IAAI,GAAC,EAAC,UAAU,EAAC,KAAK,EAAC,WAAW,EAAC,GAAG,CAAC,cAAc,EAAC,KAAK;wBACzD,eAAe,EAAC,KAAK,CAAC,UAAU,EAAC,UAAU,EAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAC,IAAI,CAAC,GAAC,IAAI;wBAC5E,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,eAAe,EAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAE,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAC,CAAC;gBAC3E,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAC,KAAK,EAAC,IAAI,KAAG,IAAI,EAAC,KAAK,EAAC,EAAC,MAAM,EAAC,MAAM,CAAC,MAAM,EAAC,WAAW,EAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAChF,YAAY,EAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAC,cAAc,EAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAC,UAAU,EAAC,MAAM,CAAC,IAAI,CAAC,MAAM;YAC7G,MAAM,EAAC,MAAM,CAAC,MAAM,EAAC,MAAM,EAAC,MAAM,CAAC,MAAM,EAAC,IAAI,EAAC,EAAC,CAAC;AACrD,CAAC"}
|
|
@@ -6,7 +6,7 @@ import type { Detection } from './LocalDetectorClient';
|
|
|
6
6
|
import { type PhoneHeldInHandPolicy, type PhoneHeldInHandTrace } from './PhoneHeldInHand';
|
|
7
7
|
import type { DetectResult } from './LocalDetectorClient';
|
|
8
8
|
import { type PhonePersonViewsPolicy, type PhonePersonProgress, type PhonePersonTrace } from './PhonePersonViews';
|
|
9
|
-
import { type DetectedLabelsTrace, type DetectorFieldsPolicy, type UnattendedCustomerTrace, type VehicleGateMemory, type VehicleGateTrace } from './DetectorFields';
|
|
9
|
+
import { type DetectedLabelsTrace, type DetectorFieldsPolicy, type UnattendedCustomerTrace, type VehicleGateMemory, type VehicleGateTrace, type VehicleOrientationGateMethod, type VehicleOrientationMemory, type VehicleOrientationTrace } from './DetectorFields';
|
|
10
10
|
import type { LocalAnalyzeRequest } from './LiteRtVlmProvider';
|
|
11
11
|
/** Retained with the frame. Never read the current profile to review old evidence. */
|
|
12
12
|
export interface ScopedVlmContract {
|
|
@@ -42,7 +42,7 @@ export type DetectorFieldsScopeTrace = {
|
|
|
42
42
|
kind: 'gate';
|
|
43
43
|
field: string;
|
|
44
44
|
value: boolean;
|
|
45
|
-
gate: VehicleGateTrace;
|
|
45
|
+
gate: VehicleGateTrace | VehicleOrientationTrace;
|
|
46
46
|
} | {
|
|
47
47
|
kind: 'unavailable';
|
|
48
48
|
reason: string;
|
|
@@ -108,9 +108,16 @@ export declare function reviewScopedVlm(input: {
|
|
|
108
108
|
detectFields?: (jpeg: Buffer, policy: DetectorFieldsPolicy) => Promise<DetectResult>;
|
|
109
109
|
/** This camera's parked-vehicle memory, per gated field. In memory only. */
|
|
110
110
|
gateMemory?: {
|
|
111
|
-
get(field: string): VehicleGateMemory | undefined;
|
|
112
|
-
set(field: string, memory: VehicleGateMemory): void;
|
|
111
|
+
get(field: string): VehicleGateMemory | VehicleOrientationMemory | undefined;
|
|
112
|
+
set(field: string, memory: VehicleGateMemory | VehicleOrientationMemory): void;
|
|
113
113
|
};
|
|
114
|
+
/** The detector service's vehicle-end task: one back-vs-front score per box (vehicle_orientation_gate_v1). */
|
|
115
|
+
orientVehicles?: (jpeg: Buffer, method: VehicleOrientationGateMethod, boxes: Array<{
|
|
116
|
+
x: number;
|
|
117
|
+
y: number;
|
|
118
|
+
width: number;
|
|
119
|
+
height: number;
|
|
120
|
+
}>) => Promise<number[]>;
|
|
114
121
|
/** The frame's capture time; the vehicle gate's clock. */
|
|
115
122
|
capturedAtMs?: number;
|
|
116
123
|
progress?: ScopedVlmProgress;
|
|
@@ -130,5 +137,13 @@ export declare function unansweredDetectorFields(plans: ReadonlyArray<{
|
|
|
130
137
|
fields: string[];
|
|
131
138
|
detector?: 'native' | 'gate';
|
|
132
139
|
}>, traces: ReadonlyArray<ScopedVlmTrace>): Record<string, string>;
|
|
140
|
+
/**
|
|
141
|
+
* Who answered each natively decided question, by model id, for the upload's
|
|
142
|
+
* `detectorProduced`: ingest stamps those fields local_detector with the model.
|
|
143
|
+
* A detector-field decision names the policy's detector. A held-in-hand phone
|
|
144
|
+
* names its detector, whose phone and person boxes decide it (pose only places
|
|
145
|
+
* the wrists). A question Gemma answered, gated or not, is not listed.
|
|
146
|
+
*/
|
|
147
|
+
export declare function nativeAnswerProducers(contract: ScopedVlmContract, traces: ReadonlyArray<ScopedVlmTrace>): Record<string, string>;
|
|
133
148
|
export {};
|
|
134
149
|
//# sourceMappingURL=ScopedVlmReview.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ScopedVlmReview.d.ts","sourceRoot":"","sources":["../src/ScopedVlmReview.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwC,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC/F,OAAO,EAAwD,KAAK,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACjH,OAAO,EAAmD,KAAK,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAA8C,KAAK,qBAAqB,EAAE,KAAK,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AACrI,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAA8E,KAAK,sBAAsB,EAC9G,KAAK,mBAAmB,EAAE,KAAK,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EACL,KAAK,mBAAmB,EAAE,KAAK,oBAAoB,EAAE,KAAK,uBAAuB,EAAE,KAAK,iBAAiB,EACzG,KAAK,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"ScopedVlmReview.d.ts","sourceRoot":"","sources":["../src/ScopedVlmReview.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwC,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC/F,OAAO,EAAwD,KAAK,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACjH,OAAO,EAAmD,KAAK,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC/G,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAA8C,KAAK,qBAAqB,EAAE,KAAK,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AACrI,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAA8E,KAAK,sBAAsB,EAC9G,KAAK,mBAAmB,EAAE,KAAK,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EACL,KAAK,mBAAmB,EAAE,KAAK,oBAAoB,EAAE,KAAK,uBAAuB,EAAE,KAAK,iBAAiB,EACzG,KAAK,gBAAgB,EAAE,KAAK,4BAA4B,EAAE,KAAK,wBAAwB,EACvF,KAAK,uBAAuB,EAAC,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAE7D,sFAAsF;AACtF,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,CAAC,CAAC;IACX,QAAQ,EAAE,UAAU,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,oFAAoF;IACpF,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,gFAAgF;IAChF,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,IAAI,CAAC;IAC1B,oFAAoF;IACpF,qBAAqB,CAAC,EAAE,IAAI,CAAC;IAC7B,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;IAC1C,+EAA+E;IAC/E,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,iGAAiG;IACjG,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,sFAAsF;IACtF,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AACD,2EAA2E;AAC3E,MAAM,MAAM,wBAAwB,GAChC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,SAAS,EAAE,KAAK,CAAC,CAAC,uBAAuB,GAAG,mBAAmB,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;CAAE,GACzH;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,gBAAgB,GAAG,uBAAuB,CAAA;CAAE,GACjG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAC5C,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,aAAa,GAAG,eAAe,CAAC;IACvC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,eAAe,CAAC,EAAE,oBAAoB,CAAC;IACvC,cAAc,CAAC,EAAE,wBAAwB,CAAC;CAC3C;AACD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,CAAC,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,cAAc,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IACjF,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAC,mBAAmB,CAAC,CAAC;CAClD;AACD,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,iGAAiG;IACjG,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;CAC9B;AA0BD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY,CAS9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAC1F,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,cAAc,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,YAAY,EAAE,CA4IjG;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAsBxF;AAID,wBAAsB,eAAe,CAAC,KAAK,EAAE;IAC3C,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,iBAAiB,CAAC;IACvE,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAAC,cAAc,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;IAC9E,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAClD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,CAAC,OAAO,EAAE,mBAAmB,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5E,aAAa,CAAC,EAAE,CAAC,IAAI,EAAC,MAAM,EAAC,MAAM,EAAC,sBAAsB,KAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAClF,6EAA6E;IAC7E,eAAe,CAAC,EAAE,CAAC,IAAI,EAAC,MAAM,EAAC,MAAM,EAAC,qBAAqB,KAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACpF,oFAAoF;IACpF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAC,MAAM,EAAC,MAAM,EAAC,oBAAoB,KAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAChF,4EAA4E;IAC5E,UAAU,CAAC,EAAE;QAAE,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,GAAG,wBAAwB,GAAG,SAAS,CAAC;QAC1F,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,wBAAwB,GAAG,IAAI,CAAA;KAAE,CAAC;IACnF,8GAA8G;IAC9G,cAAc,CAAC,EAAE,CAAC,IAAI,EAAC,MAAM,EAAC,MAAM,EAAC,4BAA4B,EAAC,KAAK,EAAC,KAAK,CAAC;QAAC,CAAC,EAAC,MAAM,CAAC;QAAA,CAAC,EAAC,MAAM,CAAC;QAAA,KAAK,EAAC,MAAM,CAAC;QAAA,MAAM,EAAC,MAAM,CAAA;KAAC,CAAC,KACzH,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACtB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,CAAC;CACpD,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAAC,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAC7G,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC,CAqOvC;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,aAAa,CAAC;IAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAA;CAAE,CAAC,EAC/G,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAW/D;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,EAC/D,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAmB/D"}
|
package/dist/ScopedVlmReview.js
CHANGED
|
@@ -5,6 +5,7 @@ exports.planScopedVlmReview = planScopedVlmReview;
|
|
|
5
5
|
exports.cropLasso = cropLasso;
|
|
6
6
|
exports.reviewScopedVlm = reviewScopedVlm;
|
|
7
7
|
exports.unansweredDetectorFields = unansweredDetectorFields;
|
|
8
|
+
exports.nativeAnswerProducers = nativeAnswerProducers;
|
|
8
9
|
const crypto_1 = require("crypto");
|
|
9
10
|
const util_1 = require("util");
|
|
10
11
|
const jpegFrameSize_1 = require("./jpegFrameSize");
|
|
@@ -155,8 +156,10 @@ function planScopedVlmReview(schema, contract, regions, producerFields) {
|
|
|
155
156
|
// The detector's questions leave their scope: answered natively, or asked of
|
|
156
157
|
// the model only when the detector's gate opens. Same region, same prompt rule.
|
|
157
158
|
const detectorGroups = scopedGroups.flatMap(group => {
|
|
158
|
-
const
|
|
159
|
-
|
|
159
|
+
const gate = (f) => detectorMethod.get(f) === 'vehicle_arrival_gate_v1'
|
|
160
|
+
|| detectorMethod.get(f) === 'vehicle_orientation_gate_v1';
|
|
161
|
+
const native = group.fields.filter(f => detectorMethod.has(f) && !gate(f));
|
|
162
|
+
const gated = group.fields.filter(gate);
|
|
160
163
|
const rest = group.fields.filter(f => !detectorMethod.has(f));
|
|
161
164
|
if (gated.length > 1)
|
|
162
165
|
throw Error('One gated question per scope');
|
|
@@ -249,6 +252,10 @@ async function reviewScopedVlm(input) {
|
|
|
249
252
|
&& (!input.detectFields || !Number.isSafeInteger(input.capturedAtMs))) {
|
|
250
253
|
throw Error('Detector-answered questions require the local detector and the capture time; retain evidence');
|
|
251
254
|
}
|
|
255
|
+
if (fieldPolicy?.fields.some(f => f.method === 'vehicle_orientation_gate_v1')
|
|
256
|
+
&& input.progress?.completed?.length !== plans.length && !input.orientVehicles) {
|
|
257
|
+
throw Error('Vehicle orientation requires the local vehicle-end task; retain evidence');
|
|
258
|
+
}
|
|
252
259
|
let fieldDetection;
|
|
253
260
|
const detectForFields = () => (fieldDetection ?? (fieldDetection = input.detectFields(input.jpeg, fieldPolicy)));
|
|
254
261
|
if (phonePolicy && input.progress?.completed?.length !== plans.length
|
|
@@ -296,7 +303,7 @@ async function reviewScopedVlm(input) {
|
|
|
296
303
|
assertScopeAnswer(saved.structured, plan, saved.trace.detectorFields);
|
|
297
304
|
assertDetectorScope(saved, plan, imageSha256, fieldPolicy);
|
|
298
305
|
const held = saved.trace.phoneHeldInHand;
|
|
299
|
-
if (held !== undefined && (!heldPolicy || held.method !==
|
|
306
|
+
if (held !== undefined && (!heldPolicy || held.method !== heldPolicy.method
|
|
300
307
|
|| held.imageSha256 !== imageSha256 || saved.trace.imageSha256 !== imageSha256
|
|
301
308
|
|| plan.fields.some(field => saved.structured[field] !== (held.held !== null)))) {
|
|
302
309
|
throw Error('Invalid phone held-in-hand checkpoint result');
|
|
@@ -389,35 +396,65 @@ async function reviewScopedVlm(input) {
|
|
|
389
396
|
else {
|
|
390
397
|
const field = plan.fields[0];
|
|
391
398
|
const method = fieldPolicy.fields.find(f => f.field === field);
|
|
392
|
-
if (method.method
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
399
|
+
if (method.method === 'vehicle_orientation_gate_v1') {
|
|
400
|
+
// The box decides: the vehicle-end task looks at each newly parked vehicle twice.
|
|
401
|
+
const orient = (0, DetectorFields_1.planVehicleOrientation)({ jpeg: input.jpeg, policy: fieldPolicy, method, frame, detections: result.detections,
|
|
402
|
+
capturedAtMs: input.capturedAtMs, memory: input.gateMemory?.get(field) });
|
|
403
|
+
let scores = [];
|
|
404
|
+
if (orient.looks.length) {
|
|
405
|
+
try {
|
|
406
|
+
scores = await input.orientVehicles(input.jpeg, method, orient.looks);
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
if (!(error instanceof LocalDetectorClient_1.DetectorUnavailableError))
|
|
410
|
+
throw error;
|
|
411
|
+
scores = undefined;
|
|
412
|
+
detectorTrace = { kind: 'unavailable', reason: String(error.message).slice(0, 240) };
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (scores) {
|
|
416
|
+
const done = orient.commit(scores);
|
|
417
|
+
commitGate = () => input.gateMemory?.set(field, done.memory);
|
|
418
|
+
answer = { [plan.modelFields?.[field] ?? field]: done.value };
|
|
419
|
+
detectorTrace = { kind: 'gate', field, value: done.value, gate: done.trace };
|
|
420
|
+
}
|
|
421
|
+
else
|
|
422
|
+
answer = {};
|
|
405
423
|
}
|
|
406
424
|
else {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
425
|
+
if (method.method !== 'vehicle_arrival_gate_v1')
|
|
426
|
+
throw Error('Only the vehicle gate gates a question');
|
|
427
|
+
const gate = (0, DetectorFields_1.planVehicleGate)({ jpeg: input.jpeg, policy: fieldPolicy, method, frame, detections: result.detections,
|
|
428
|
+
capturedAtMs: input.capturedAtMs, memory: input.gateMemory?.get(field) });
|
|
429
|
+
let value;
|
|
430
|
+
if (gate.ask) {
|
|
431
|
+
const modelAnswer = await input.analyze({ jpeg, schema: modelSchema, prompt: plan.prompt });
|
|
432
|
+
const asked = modelAnswer[plan.modelFields?.[field] ?? field];
|
|
433
|
+
if (typeof asked !== 'boolean')
|
|
434
|
+
throw Error('Invalid gated model answer');
|
|
435
|
+
value = asked;
|
|
436
|
+
const next = gate.commit(value);
|
|
437
|
+
commitGate = () => input.gateMemory?.set(field, next);
|
|
438
|
+
}
|
|
439
|
+
else {
|
|
440
|
+
value = gate.value;
|
|
441
|
+
const next = gate.commit();
|
|
442
|
+
commitGate = () => input.gateMemory?.set(field, next);
|
|
443
|
+
}
|
|
444
|
+
answer = { [plan.modelFields?.[field] ?? field]: value };
|
|
445
|
+
detectorTrace = { kind: 'gate', field, value, gate: gate.trace };
|
|
410
446
|
}
|
|
411
|
-
answer = { [plan.modelFields?.[field] ?? field]: value };
|
|
412
|
-
detectorTrace = { kind: 'gate', field, value, gate: gate.trace };
|
|
413
447
|
}
|
|
414
448
|
}
|
|
415
449
|
else if (heldScope) {
|
|
416
450
|
const result = await detectHeld();
|
|
417
451
|
if (!result.detectionPose)
|
|
418
452
|
throw Error('Phone pose unavailable; retain evidence');
|
|
453
|
+
if (heldPolicy.method === 'native_held_in_hand_staff_v1' && !result.detectionColors)
|
|
454
|
+
throw Error('Phone holder uniform unavailable; retain evidence');
|
|
419
455
|
const decided = (0, PhoneHeldInHand_1.decidePhoneHeldInHand)({ jpeg: input.jpeg, policy: heldPolicy, region: plan.region ?? WHOLE_FRAME, frame: size,
|
|
420
|
-
detections: result.detections, pose: result.detectionPose
|
|
456
|
+
detections: result.detections, pose: result.detectionPose,
|
|
457
|
+
...(result.detectionColors ? { colors: result.detectionColors } : {}) });
|
|
421
458
|
answer = { phone_use: decided.value };
|
|
422
459
|
heldTrace = decided.trace;
|
|
423
460
|
}
|
|
@@ -499,6 +536,35 @@ function unansweredDetectorFields(plans, traces) {
|
|
|
499
536
|
});
|
|
500
537
|
return unavailable;
|
|
501
538
|
}
|
|
539
|
+
/**
|
|
540
|
+
* Who answered each natively decided question, by model id, for the upload's
|
|
541
|
+
* `detectorProduced`: ingest stamps those fields local_detector with the model.
|
|
542
|
+
* A detector-field decision names the policy's detector. A held-in-hand phone
|
|
543
|
+
* names its detector, whose phone and person boxes decide it (pose only places
|
|
544
|
+
* the wrists). A question Gemma answered, gated or not, is not listed.
|
|
545
|
+
*/
|
|
546
|
+
function nativeAnswerProducers(contract, traces) {
|
|
547
|
+
const out = {};
|
|
548
|
+
for (const trace of traces) {
|
|
549
|
+
const decided = trace.detectorFields;
|
|
550
|
+
if (decided?.kind === 'native') {
|
|
551
|
+
const detector = (0, DetectorFields_1.parseDetectorFields)(contract.detectorFields).detector.modelId;
|
|
552
|
+
for (const d of decided.decisions)
|
|
553
|
+
out[d.field] = detector;
|
|
554
|
+
}
|
|
555
|
+
if (decided?.kind === 'gate' && decided.gate.method === 'vehicle_orientation_gate_v1') {
|
|
556
|
+
const method = (0, DetectorFields_1.parseDetectorFields)(contract.detectorFields).fields
|
|
557
|
+
.find((f) => f.field === decided.field && f.method === 'vehicle_orientation_gate_v1');
|
|
558
|
+
out[decided.field] = method.orientation.model.modelId;
|
|
559
|
+
}
|
|
560
|
+
if (trace.phoneHeldInHand) {
|
|
561
|
+
const detector = (0, PhoneHeldInHand_1.parsePhoneHeldInHand)(contract.phoneHeldInHand).detector.modelId;
|
|
562
|
+
for (const field of trace.fields)
|
|
563
|
+
out[field] = detector;
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
return out;
|
|
567
|
+
}
|
|
502
568
|
/** A detector scope's saved answer must be the one its own trace recorded. */
|
|
503
569
|
function assertDetectorScope(saved, plan, imageSha256, policy) {
|
|
504
570
|
const t = saved.trace.detectorFields;
|