@camera.ui/server 2.2.8 → 2.3.0

package/dist/server/src/automations/pluginMethodRegistry.js

@@ -1,4 +1,5 @@
 import { PluginInterface } from '@camera.ui/sdk';
+import { segmentPicture } from './segment-picture.js';
 const IMAGE_METADATA = { fixed: { width: 0, height: 0 } };
 const AUDIO_METADATA = { fixed: { mimeType: 'audio/wav' } };
 const CONFIG_PARAM = { param: 'config' };
@@ -35,6 +36,34 @@ export const PLUGIN_METHOD_REGISTRY = {
             settingsMethod: 'faceDetectionSettings',
         },
     ],
+    [PluginInterface.FaceEmbedding]: [
+        {
+            id: 'embedFaceImages',
+            labelKey: 'components.automation_nodes.method_embed_faces',
+            params: [{ name: 'imageData', labelKey: 'components.automation_nodes.param_image', type: 'image', placeholder: '', binary: true }],
+            args: [{ images: 'imageData' }, CONFIG_PARAM],
+            settingsMethod: 'faceEmbeddingSettings',
+        },
+    ],
+    [PluginInterface.PersonEmbedding]: [
+        {
+            id: 'embedPersonImages',
+            labelKey: 'components.automation_nodes.method_embed_persons',
+            params: [{ name: 'imageData', labelKey: 'components.automation_nodes.param_image', type: 'image', placeholder: '', binary: true }],
+            args: [{ images: 'imageData' }, CONFIG_PARAM],
+            settingsMethod: 'personEmbeddingSettings',
+        },
+    ],
+    [PluginInterface.Segmentation]: [
+        {
+            id: 'segmentImages',
+            labelKey: 'components.automation_nodes.method_segment',
+            params: [{ name: 'imageData', labelKey: 'components.automation_nodes.param_image', type: 'image', placeholder: '', binary: true }],
+            args: [],
+            settingsMethod: 'segmentationSettings',
+            run: (plugin, contract, params) => segmentPicture(plugin, contract, params.imageData, params.config ?? {}),
+        },
+    ],
     [PluginInterface.LicensePlateDetection]: [
         {
             id: 'testLicensePlateDetection',
@@ -130,6 +159,14 @@ export function getMethodDef(methodId) {
     }
     return undefined;
 }
+export async function runRegistryMethod(def, plugin, contract, params) {
+    if (def.run)
+        return await def.run(plugin, contract, params);
+    const method = plugin[def.id];
+    if (!method)
+        throw new Error(`Method "${def.id}" is not implemented by the plugin`);
+    return await method(...buildArgsFromRegistry(def.id, params));
+}
 export function buildArgsFromRegistry(methodId, resolvedParams) {
     const def = getMethodDef(methodId);
     if (!def)
@@ -151,6 +188,8 @@ export function buildArgsFromRegistry(methodId, resolvedParams) {
             const format = (resolvedParams[`${arg.audioFrame}.format`] ?? resolvedParams['audio.format'] ?? 'pcm16');
             return { data, sampleRate, channels, format };
         }
+        if ('images' in arg)
+            return [resolvedParams[arg.images]];
         return resolvedParams[arg.param];
     });
 }

package/dist/server/src/automations/pluginMethodRegistry.js.map

@@ -1 +1 @@
-{"version":3,"file":"pluginMethodRegistry.js","sourceRoot":"","sources":["../../../../src/automations/pluginMethodRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAsBjD,MAAM,cAAc,GAAoB,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;AAC3E,MAAM,cAAc,GAAoB,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;AAC7E,MAAM,YAAY,GAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAE1D,MAAM,CAAC,MAAM,sBAAsB,GAAwD;IACzF,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE;QACjC;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,yBAAyB;SAC1C;QACD;YACE,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,mDAAmD;YAC7D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;KACF;IACD,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE;QAC/B;YACE,EAAE,EAAE,mBAAmB;YACvB,QAAQ,EAAE,8CAA8C;YACxD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,uBAAuB;SACxC;QACD;YACE,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,iDAAiD;YAC3D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,uBAAuB;SACxC;KACF;IACD,CAAC,eAAe,CAAC,qBAAqB,CAAC,EAAE;QACvC;YACE,EAAE,EAAE,2BAA2B;YAC/B,QAAQ,EAAE,+CAA+C;YACzD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,+BAA+B;SAChD;QACD;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,kDAAkD;YAC5D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,+BAA+B;SAChD;KACF;IACD,CAAC,eAAe,CAAC,mBAAmB,CAAC,EAAE;QACrC;YACE,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,oDAAoD;YAC9D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,6BAA6B;SAC9C;QACD;YACE,EAAE,EAAE,uBAAuB;YAC3B,QAAQ,EAAE,2DAA2D;YACrE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,6BAA6B;SAC9C;KACF;IACD,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE;QACjC;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;QACD;YACE,EAAE,EAAE,cAAc;YAClB,QAAQ,EAAE,kDAAkD;YAC5D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;KACF;IACD,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE;QAChC;YACE,EAAE,EAAE,oBAAoB;YACxB,QAAQ,EAAE,+CAA+C;YACzD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,wBAAwB;SACzC;QACD;YACE,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,iDAAiD;YAC3D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACnI,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,YAAY,CAAC;YAClD,cAAc,EAAE,wBAAwB;SACzC;KACF;IACD,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE;QAC/B;YACE,EAAE,EAAE,mBAAmB;YACvB,QAAQ,EAAE,8CAA8C;YACxD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,cAAc;SAC/B;QACD;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,cAAc;SAC/B;QACD;YACE,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,mDAAmD;YAC7D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,wCAAwC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;YAC3H,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;SAC1B;KACF;CACF,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QACtD,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,cAAuC;IAC7F,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAE/C,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,OAAO,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC;QACrC,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAW,CAAC;YACjD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;YACjG,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YACpG,MAAM,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,KAAK,CAAW,CAAC;YAC5G,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC3D,CAAC;QACD,IAAI,YAAY,IAAI,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAW,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,aAAa,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,CAAC;YAC1H,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,WAAW,CAAC,IAAI,cAAc,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/G,MAAM,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,OAAO,CAAW,CAAC;YACnH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAChD,CAAC;QACD,OAAO,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB;IAIvC,MAAM,MAAM,GAGR,EAAE,CAAC;IAEP,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC;QACtE,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;aAC3B,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"pluginMethodRegistry.js","sourceRoot":"","sources":["../../../../src/automations/pluginMethodRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA2BtD,MAAM,cAAc,GAAoB,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;AAC3E,MAAM,cAAc,GAAoB,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC;AAC7E,MAAM,YAAY,GAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAE1D,MAAM,CAAC,MAAM,sBAAsB,GAAwD;IACzF,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE;QACjC;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,yBAAyB;SAC1C;QACD;YACE,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,mDAAmD;YAC7D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;KACF;IACD,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE;QAC/B;YACE,EAAE,EAAE,mBAAmB;YACvB,QAAQ,EAAE,8CAA8C;YACxD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,uBAAuB;SACxC;QACD;YACE,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,iDAAiD;YAC3D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,uBAAuB;SACxC;KACF;IACD,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE;QAC/B;YACE,EAAE,EAAE,iBAAiB;YACrB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC7C,cAAc,EAAE,uBAAuB;SACxC;KACF;IACD,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE;QACjC;YACE,EAAE,EAAE,mBAAmB;YACvB,QAAQ,EAAE,kDAAkD;YAC5D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC7C,cAAc,EAAE,yBAAyB;SAC1C;KACF;IACD,CAAC,eAAe,CAAC,YAAY,CAAC,EAAE;QAC9B;YACE,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,4CAA4C;YACtD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,EAAE;YACR,cAAc,EAAE,sBAAsB;YACtC,GAAG,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,SAAmB,EAAG,MAAM,CAAC,MAA8C,IAAI,EAAE,CAAC;SAC9J;KACF;IACD,CAAC,eAAe,CAAC,qBAAqB,CAAC,EAAE;QACvC;YACE,EAAE,EAAE,2BAA2B;YAC/B,QAAQ,EAAE,+CAA+C;YACzD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,+BAA+B;SAChD;QACD;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,kDAAkD;YAC5D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,+BAA+B;SAChD;KACF;IACD,CAAC,eAAe,CAAC,mBAAmB,CAAC,EAAE;QACrC;YACE,EAAE,EAAE,yBAAyB;YAC7B,QAAQ,EAAE,oDAAoD;YAC9D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,6BAA6B;SAC9C;QACD;YACE,EAAE,EAAE,uBAAuB;YAC3B,QAAQ,EAAE,2DAA2D;YACrE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,6BAA6B;SAC9C;KACF;IACD,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE;QACjC;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;QACD;YACE,EAAE,EAAE,cAAc;YAClB,QAAQ,EAAE,kDAAkD;YAC5D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,yBAAyB;SAC1C;KACF;IACD,CAAC,eAAe,CAAC,cAAc,CAAC,EAAE;QAChC;YACE,EAAE,EAAE,oBAAoB;YACxB,QAAQ,EAAE,+CAA+C;YACzD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,wBAAwB;SACzC;QACD;YACE,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,iDAAiD;YAC3D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACnI,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,YAAY,CAAC;YAClD,cAAc,EAAE,wBAAwB;SACzC;KACF;IACD,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE;QAC/B;YACE,EAAE,EAAE,mBAAmB;YACvB,QAAQ,EAAE,8CAA8C;YACxD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,YAAY,CAAC;YAC5D,cAAc,EAAE,cAAc;SAC/B;QACD;YACE,EAAE,EAAE,qBAAqB;YACzB,QAAQ,EAAE,gDAAgD;YAC1D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,yCAAyC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAClI,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,YAAY,CAAC;YAC5C,cAAc,EAAE,cAAc;SAC/B;QACD;YACE,EAAE,EAAE,kBAAkB;YACtB,QAAQ,EAAE,mDAAmD;YAC7D,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,wCAAwC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;YAC3H,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;SAC1B;KACF;CACF,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QACtD,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAoB,EACpB,MAAqB,EACrB,QAAoC,EACpC,MAA+B;IAE/B,IAAI,GAAG,CAAC,GAAG;QAAE,OAAO,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACpF,OAAO,MAAM,MAAM,CAAC,GAAG,qBAAqB,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,cAAuC;IAC7F,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAE/C,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,OAAO,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC;QACrC,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAW,CAAC;YACjD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,QAAQ,CAAC,IAAI,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;YACjG,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YACpG,MAAM,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,KAAK,CAAW,CAAC;YAC5G,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC3D,CAAC;QACD,IAAI,YAAY,IAAI,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAW,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,aAAa,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,CAAC;YAC1H,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,WAAW,CAAC,IAAI,cAAc,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/G,MAAM,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,OAAO,CAAW,CAAC;YACnH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAChD,CAAC;QACD,IAAI,QAAQ,IAAI,GAAG;YAAE,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,OAAO,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB;IAIvC,MAAM,MAAM,GAGR,EAAE,CAAC;IAEP,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC;QACtE,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;aAC3B,CAAC,CAAC;SACJ,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/server/src/automations/segment-picture.d.ts

@@ -0,0 +1,16 @@
+import type { BoundingBox, ImageMetadata, ObjectDetectionPluginResponse, PluginContract, SegmentationImage, SegmentationPluginResponse } from '@camera.ui/sdk';
+export interface SegmentedObject {
+    label?: string;
+    confidence?: number;
+    box: BoundingBox;
+    outline: BoundingBox;
+}
+export interface SegmentedPicture {
+    detected: boolean;
+    detections: SegmentedObject[];
+}
+export interface SegmentingPlugin {
+    testObjectDetection?(imageData: Buffer, metadata: ImageMetadata, config: Record<string, unknown>): Promise<ObjectDetectionPluginResponse | undefined>;
+    segmentImages?(images: SegmentationImage[], config?: Record<string, unknown>): Promise<(SegmentationPluginResponse | undefined)[]>;
+}
+export declare function segmentPicture(plugin: SegmentingPlugin, contract: PluginContract | undefined, picture: Buffer, config: Record<string, unknown>): Promise<SegmentedPicture>;

package/dist/server/src/automations/segment-picture.js

@@ -0,0 +1,152 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { hasInterface, PluginInterface } from '@camera.ui/sdk';
+import { Scaler } from 'node-av/api';
+import { iou } from '../camera/decoder/detection-window.js';
+import { withPicture } from '../utils/image.js';
+const LABELS = new Set(['person', 'vehicle', 'animal']);
+const MIN_CONFIDENCE = 0.5;
+const MAX_OBJECTS = 8;
+const NMS_IOU = 0.45;
+const REGION_PADDING = 0.15;
+const REGION_MAX_EDGE = 640;
+const JPEG_QUALITY = 92;
+const WHOLE_PICTURE = { x: 0, y: 0, width: 1, height: 1 };
+export async function segmentPicture(plugin, contract, picture, config) {
+    if (!picture?.length)
+        throw new Error('A picture is needed');
+    const detections = await withPicture(picture, async (frame) => {
+        const objects = contract && hasInterface(contract, PluginInterface.ObjectDetection) ? await objectsIn(plugin, picture, frame) : [{ box: WHOLE_PICTURE }];
+        return await outline(plugin, frame, objects, config);
+    });
+    if (!detections)
+        throw new Error('The picture could not be read');
+    return { detected: detections.length > 0, detections };
+}
+async function objectsIn(plugin, picture, frame) {
+    const response = await plugin.testObjectDetection?.(picture, { width: frame.width, height: frame.height }, {});
+    const candidates = (response?.detections ?? [])
+        .filter((d) => d.box && LABELS.has(d.label.toLowerCase()) && d.confidence >= MIN_CONFIDENCE)
+        .map((d) => ({ label: d.label.toLowerCase(), confidence: d.confidence, box: d.box }))
+        .sort((a, b) => b.confidence - a.confidence);
+    const kept = [];
+    for (const candidate of candidates) {
+        if (kept.every((k) => k.label !== candidate.label || iou(k.box, candidate.box) <= NMS_IOU))
+            kept.push(candidate);
+    }
+    return kept.slice(0, MAX_OBJECTS);
+}
+async function outline(plugin, frame, objects, config) {
+    const env_1 = { stack: [], error: void 0, hasError: false };
+    try {
+        const scaler = __addDisposableResource(env_1, new Scaler(), false);
+        const sent = [];
+        const images = [];
+        for (const object of objects) {
+            const region = regionAround(object.box, frame.width, frame.height);
+            if (!region)
+                continue;
+            const scale = Math.min(1, REGION_MAX_EDGE / Math.max(region.width, region.height));
+            const resize = { width: Math.max(2, Math.round(region.width * scale) & ~1), height: Math.max(2, Math.round(region.height * scale) & ~1) };
+            const image = await scaler.toJpeg(frame, { crop: region, resize, quality: JPEG_QUALITY });
+            images.push({ image, box: boxInRegion(object.box, region, frame.width, frame.height) });
+            sent.push({ object, region });
+        }
+        if (!images.length)
+            return [];
+        const results = (await plugin.segmentImages?.(images, config)) ?? [];
+        const outlined = [];
+        sent.forEach(({ object, region }, i) => {
+            const mask = results[i]?.mask;
+            if (mask)
+                outlined.push({ ...object, outline: boxInPicture(mask.box, region, frame.width, frame.height) });
+        });
+        return outlined;
+    }
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
+    }
+    finally {
+        __disposeResources(env_1);
+    }
+}
+function regionAround(box, width, height) {
+    const padX = box.width * width * REGION_PADDING;
+    const padY = box.height * height * REGION_PADDING;
+    const side = Math.min(Math.max(box.width * width + 2 * padX, box.height * height + 2 * padY), width, height) & ~1;
+    if (side < 2)
+        return undefined;
+    const cx = (box.x + box.width / 2) * width;
+    const cy = (box.y + box.height / 2) * height;
+    const x = Math.min(Math.max(Math.round(cx - side / 2), 0), width - side) & ~1;
+    const y = Math.min(Math.max(Math.round(cy - side / 2), 0), height - side) & ~1;
+    return { x, y, width: side, height: side };
+}
+function boxInRegion(box, region, width, height) {
+    return {
+        x: (box.x * width - region.x) / region.width,
+        y: (box.y * height - region.y) / region.height,
+        width: (box.width * width) / region.width,
+        height: (box.height * height) / region.height,
+    };
+}
+function boxInPicture(box, region, width, height) {
+    return {
+        x: (region.x + box.x * region.width) / width,
+        y: (region.y + box.y * region.height) / height,
+        width: (box.width * region.width) / width,
+        height: (box.height * region.height) / height,
+    };
+}
+//# sourceMappingURL=segment-picture.js.map

package/dist/server/src/automations/segment-picture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"segment-picture.js","sourceRoot":"","sources":["../../../../src/automations/segment-picture.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,GAAG,EAAE,MAAM,uCAAuC,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAKhD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;AACxD,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,YAAY,GAAG,EAAE,CAAC;AACxB,MAAM,aAAa,GAAgB,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AA0BvE,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAwB,EACxB,QAAoC,EACpC,OAAe,EACf,MAA+B;IAE/B,IAAI,CAAC,OAAO,EAAE,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC5D,MAAM,OAAO,GAAG,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;QACzJ,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAClE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAwB,EAAE,OAAe,EAAE,KAAY;IAC9E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/G,MAAM,UAAU,GAAG,CAAC,QAAQ,EAAE,UAAU,IAAI,EAAE,CAAC;SAC5C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,cAAc,CAAC;SAC3F,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;SACpF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnH,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,MAAwB,EAAE,KAAY,EAAE,OAA2C,EAAE,MAA+B;;;QACzI,MAAM,MAAM,kCAAG,IAAI,MAAM,EAAE,QAAA,CAAC;QAC5B,MAAM,IAAI,GAAmE,EAAE,CAAC;QAChF,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACnE,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACnF,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1I,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;YAC1F,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxF,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,EAAE,CAAC;QAE9B,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,MAAM,QAAQ,GAAsB,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;YAC9B,IAAI,IAAI;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7G,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;;;;;;;;;CACjB;AAED,SAAS,YAAY,CAAC,GAAgB,EAAE,KAAa,EAAE,MAAc;IACnE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,cAAc,CAAC;IAChD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,MAAM,GAAG,cAAc,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAClH,IAAI,IAAI,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC3C,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/E,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,GAAgB,EAAE,MAAc,EAAE,KAAa,EAAE,MAAc;IAClF,OAAO;QACL,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK;QAC5C,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QAC9C,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK;QACzC,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAgB,EAAE,MAAc,EAAE,KAAa,EAAE,MAAc;IACnF,OAAO;QACL,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK;QAC5C,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM;QAC9C,KAAK,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK;QACzC,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM;KAC9C,CAAC;AACJ,CAAC"}

package/dist/server/src/camera/decoder/detection-coordinator.d.ts

@@ -3,7 +3,8 @@ import type { Logger } from '@camera.ui/common/logger';
 import type { RPCClient } from '@camera.ui/rpc';
 import type { CameraDetectionSettings, CameraFrameWorkerSettings, CameraUiSettings, CameraZones, PtzAutotrackSettings } from '@camera.ui/sdk';
 import type { CoordinatorSensorInfo } from '../../rpc/interfaces/detection.js';
-import type { TrackedClipEmbedding, TrackedFaceDetection } from './event-manager.js';
+import type { TrackedClipEmbedding, TrackedFaceDetection, TrackedPersonEmbedding } from './event-manager.js';
+import type { TraceFaceRead } from './event-trace.js';
 import type { CoordinatorSourceUrl, FrameWorkerPerfSnapshot, ObjectBenchmarkResult } from './types.js';
 export interface DetectionCoordinatorConfig {
     cameraId: string;
@@ -57,6 +58,7 @@ export declare class DetectionCoordinator {
     private trackAnchorBoxes;
     private farewellBoxes;
     private lastTrackedById;
+    private faceReads;
     private detectionWindow;
     private mainStreamActive;
     private idleSince;
@@ -90,7 +92,9 @@ export declare class DetectionCoordinator {
     vectorJobStarted(): void;
     vectorJobSettled(): void;
     acceptClipVectors(embeddings: TrackedClipEmbedding[], embeddingModel: string, capturedAt: number): void;
+    traceFaceReads(reads: TraceFaceRead[]): void;
     acceptFaceVectors(faces: TrackedFaceDetection[], embeddingModel: string, capturedAt: number): void;
+    acceptPersonVectors(persons: TrackedPersonEmbedding[], embeddingModel: string, capturedAt: number): void;
     dispose(): Promise<void>;
     onSensorAdded(sensor: CoordinatorSensorInfo): Promise<void>;
     onSensorRemoved(sensorId: string): Promise<void>;
@@ -105,7 +109,6 @@ export declare class DetectionCoordinator {
     private get cascadeEnabled();
     private get cascadeTimeoutSeconds();
     private get motionResolution();
-    private get targetFps();
     private get videoAspectRatio();
     private get mainStreamAvailable();
     private get mainStreamWanted();

package/dist/server/src/camera/decoder/detection-coordinator.js

@@ -35,14 +35,15 @@ import { PtzAutotracker } from './ptz/autotracker.js';
 import { SecondaryStage } from './secondary-stage.js';
 import { BufferedSource } from './sources/buffered-source.js';
 import { FrameSource } from './sources/frame-source.js';
-import { DETECT_TIMEOUT_MS, DETECTOR_METRIC_TYPES, ensureDetectionBoxes, isFullFrameBox, isTrainingSubject, MOTION_WIDTH_MAP, touchesFrameEdge } from './types.js';
+import { trainingValue } from './training-sink.js';
+import { DETECT_TIMEOUT_MS, DETECTOR_METRIC_TYPES, ensureDetectionBoxes, isFullFrameBox, isMovingTrack, isTrainingSubject, MOTION_WIDTH_MAP } from './types.js';
 const TRAINING_FRAME_MAX_WIDTH = 1280;
 const TRAINING_ATTRIBUTE_BONUS = 0.2;
+const TRAINING_SUGGESTION_WEIGHT = 0.5;
 const TRAINING_FRAME_QUALITY = 80;
 const MOMENT_EVENTS = new Set(['objectEntered', 'objectWoke', 'objectRecovered', 'bestShotUpdated']);
 const WITNESS_WINDOW_MS = 3000;
 const HELD_MOMENT_MS = 4000;
-const MOMENT_MOVING_SPEED = 0.05;
 const MOMENT_ATTRIBUTE_MIN_AREA = 600;
 const DEFAULT_CASCADE_TIMEOUT = 10;
 const OBJECT_DWELL_SECONDS = 2;
@@ -101,6 +102,7 @@ let DetectionCoordinator = class DetectionCoordinator {
     trackAnchorBoxes = [];
     farewellBoxes = [];
     lastTrackedById = new Map();
+    faceReads = [];
     detectionWindow = new DetectionWindow();
     mainStreamActive = false;
     idleSince = 0;
@@ -190,7 +192,6 @@ let DetectionCoordinator = class DetectionCoordinator {
             proxy: this.proxy,
             cameraId: this.config.cameraId,
             settings: this.config.ptzAutotrack,
-            getFps: () => this.targetFps,
             onSuppressionActivated: () => this.handleSuppressionActivated(),
         });
         this.cascadeUnsubscribe = this.cascade.onChange((event) => {
@@ -367,9 +368,15 @@ let DetectionCoordinator = class DetectionCoordinator {
     acceptClipVectors(embeddings, embeddingModel, capturedAt) {
         this.eventManager.acceptClipVectors(embeddings, embeddingModel, capturedAt);
     }
+    traceFaceReads(reads) {
+        this.faceReads.push(...reads);
+    }
     acceptFaceVectors(faces, embeddingModel, capturedAt) {
         this.eventManager.acceptFaceVectors(faces, embeddingModel, capturedAt);
     }
+    acceptPersonVectors(persons, embeddingModel, capturedAt) {
+        this.eventManager.acceptPersonVectors(persons, embeddingModel, capturedAt);
+    }
     async dispose() {
         this.heldMoments.clear();
         await this.stopVideoLoop();
@@ -509,6 +516,8 @@ let DetectionCoordinator = class DetectionCoordinator {
                             if (results.thumbnails && results.thumbnails.length > 0) {
                                 snapshot.thumbnails = results.thumbnails;
                             }
+                            if (results.trainingSuggestions)
+                                snapshot.trainingSuggestions = results.trainingSuggestions;
                             await this.attachTrainingFrame(snapshot, analysis);
                             this.eventManager.processResults(snapshot);
                         }
@@ -640,9 +649,6 @@ let DetectionCoordinator = class DetectionCoordinator {
     get motionResolution() {
         return this.config.detectionSettings.motion.resolution;
     }
-    get targetFps() {
-        return this.frameSource.fps;
-    }
     get videoAspectRatio() {
         const [w, h] = this.config.interfaceSettings.aspectRatio.split(':').map(Number);
         return w / h;
@@ -1299,6 +1305,10 @@ let DetectionCoordinator = class DetectionCoordinator {
         trace.rtp = analysis.rtp;
         trace.src = analysis.role?.replace('-resolution', '');
         trace.attrs = traceAttributes(results);
+        if (this.faceReads.length > 0) {
+            trace.reads = this.faceReads;
+            this.faceReads = [];
+        }
         const witness = [...this.witnessSeen].filter(([, at]) => Math.abs(trace.tMs - at) <= WITNESS_WINDOW_MS).map(([label]) => label);
         if (witness.length > 0)
             trace.witness = witness;
@@ -1322,6 +1332,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         let objectDetections = [];
         let staticDetections = [];
         let trainingExtras = [];
+        let trainingSuggestions = [];
         const results = { timestamp: t0 };
         let trace;
         // while the PTZ repositions: motion and object keep running so the
@@ -1454,10 +1465,11 @@ let DetectionCoordinator = class DetectionCoordinator {
                     results.object = { detected: objectDetections.length > 0, detections: visibleTracks };
                     staticDetections = pipelineResult.staticTracks;
                     trainingExtras = pipelineResult.trainingExtras;
+                    trainingSuggestions = pipelineResult.trainingSuggestions;
                     if (pipelineResult.crossings.length > 0)
                         results.lineCrossings = pipelineResult.crossings;
-                    // incl. extrapolated tracks, otherwise a single missed detector
-                    // frame flips the autotracker into LOST/REACQUIRE churn
+                    // incl. extrapolated tracks, they keep the target's id through a
+                    // missed detector frame
                     this.ptzAutotracker.handleObjectDetections(pipelineResult.tracked);
                     // live tracks anchor the next tick's window; a track that just died
                     // gets one farewell look at its last position
@@ -1508,7 +1520,8 @@ let DetectionCoordinator = class DetectionCoordinator {
             return;
         if (objectDetections.length > 0) {
             const images = this.hqUpgrade ?? analysis;
-            await this.runSecondariesAndThumbnails(images, objectDetections, results);
+            const stillPersons = staticDetections.filter((d) => d.label === 'person');
+            await this.runSecondariesAndThumbnails(images, objectDetections, results, stillPersons);
             if (!this.loopRunning)
                 return;
             await this.captureAttributeMoment(results, images, t0);
@@ -1526,6 +1539,9 @@ let DetectionCoordinator = class DetectionCoordinator {
             snapshot.staticObjects = staticDetections;
         if (trainingExtras.length > 0)
             snapshot.trainingExtras = trainingExtras;
+        const suggestions = [...trainingSuggestions, ...(results.trainingSuggestions ?? [])];
+        if (suggestions.length > 0)
+            snapshot.trainingSuggestions = suggestions;
         if (results.thumbnails && results.thumbnails.length > 0) {
             snapshot.thumbnails = results.thumbnails;
         }
@@ -1567,13 +1583,13 @@ let DetectionCoordinator = class DetectionCoordinator {
             this.thumbnailer.upgradeEventThumbnailAsync();
         }
     }
-    async runSecondariesAndThumbnails(analysis, objectDetections, results) {
+    async runSecondariesAndThumbnails(analysis, objectDetections, results, stillPersons = []) {
         if (objectDetections.length === 0)
             return;
         this.perf.objects += objectDetections.length;
         this.perf.framesWithObjects++;
         const secondaryStart = Date.now();
-        await this.secondaries.detect(analysis.frame, analysis.scaler, objectDetections, results);
+        await this.secondaries.detect(analysis.frame, analysis.scaler, objectDetections, results, stillPersons);
         this.perf.secondaryMs += Date.now() - secondaryStart;
         try {
             const jpegStart = Date.now();
@@ -1667,26 +1683,30 @@ let DetectionCoordinator = class DetectionCoordinator {
     async attachTrainingFrame(snapshot, analysis) {
         if (!this.eventManager.hasActiveEvent() && !this.snapshotWillStartEvent(snapshot))
             return;
-        // moment-style score per subject, an edge-clipped subject makes a poor
-        // sample; the sink sums only the boxes the scene memory does not know yet
+        // the sink sums only the boxes the scene memory does not know yet
+        const value = (label, confidence, box) => trainingValue(label, confidence, this.pipeline.threshold(label), box);
         const subjects = snapshot.objects
             .filter((d) => isTrainingSubject(d))
-            .map((d) => ({ label: d.label, box: d.box, score: d.confidence * Math.sqrt(d.box.width * d.box.height) * (touchesFrameEdge(d.box) ? 0.5 : 1) }));
-        // flat bonus: faces and plates are the scarce labels but their boxes are too
-        // small for the area term; fresh only, a buffered stale result describes
-        // an older frame and must not lift this one
+            .map((d) => ({ label: d.label, box: d.box, score: value(d.label, d.confidence, d.box) }));
+        // flat bonus: faces and plates are the scarce labels; fresh only, a
+        // buffered stale result describes an older frame and must not lift this one
         const now = Date.now();
         const facesFresh = snapshot.facesAt !== undefined && now - snapshot.facesAt <= SECONDARY_FRESH_MS;
         const platesFresh = snapshot.platesAt !== undefined && now - snapshot.platesAt <= SECONDARY_FRESH_MS;
         for (const face of facesFresh ? snapshot.faces : []) {
             if (face.box && !isFullFrameBox(face.box))
-                subjects.push({ label: 'face', box: face.box, score: TRAINING_ATTRIBUTE_BONUS * (touchesFrameEdge(face.box) ? 0.5 : 1) });
+                subjects.push({ label: 'face', box: face.box, score: TRAINING_ATTRIBUTE_BONUS });
         }
         for (const plate of platesFresh ? snapshot.plates : []) {
             if (plate.box && !isFullFrameBox(plate.box))
-                subjects.push({ label: 'license_plate', box: plate.box, score: TRAINING_ATTRIBUTE_BONUS * (touchesFrameEdge(plate.box) ? 0.5 : 1) });
+                subjects.push({ label: 'license_plate', box: plate.box, score: TRAINING_ATTRIBUTE_BONUS });
+        }
+        for (const hint of snapshot.trainingSuggestions ?? []) {
+            if (!isFullFrameBox(hint.box)) {
+                subjects.push({ label: hint.label, box: hint.box, score: TRAINING_SUGGESTION_WEIGHT * value(hint.label, hint.confidence, hint.box), hint: true });
+            }
         }
-        if (subjects.length === 0)
+        if (!subjects.some((s) => !s.hint))
             return;
         if (!this.eventManager.wantsTrainingFrame(subjects))
             return;
@@ -1697,6 +1717,7 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (jpeg) {
                 snapshot.trainingFrame = jpeg;
                 snapshot.trainingSubjects = subjects;
+                snapshot.trainingDetector = this.plugins.get(SensorType.Object)?.pluginId;
             }
         }
         catch (error) {
@@ -1827,7 +1848,7 @@ let DetectionCoordinator = class DetectionCoordinator {
     }
     async captureMoment(subject, score, trigger, tracked, analysis, at) {
         const box = { x: subject.x, y: subject.y, width: subject.width, height: subject.height };
-        const moving = tracked.filter((d) => (d.trackSpeed ?? 0) >= MOMENT_MOVING_SPEED).map((d) => d.box);
+        const moving = tracked.filter(isMovingTrack).map((d) => d.box);
         const target = { subject: box, base: unionBox([box, ...moving]), hint: directionOf(subject.velocityX, subject.velocityY) };
         const rendered = await this.renderMoment(target, analysis);
         if (!rendered)
@@ -2112,7 +2133,9 @@ let DetectionCoordinator = class DetectionCoordinator {
         const assist = this.plugins.get(SensorType.ObjectAssist);
         try {
             const inferStart = Date.now();
-            const boxed = await this.assistDetections(assist, frame, scaler, reported);
+            // the plugin delivers from its floor: only boxes above the user threshold
+            // may replace the camera's report
+            const boxed = this.pipeline.aboveThreshold(await this.assistDetections(assist, frame, scaler, reported));
             this.perf.assistMs += Date.now() - inferStart;
             this.perf.assistCount++;
             const reportedLabels = new Set(reported.map((d) => d.label.toLowerCase()));