framefields 2.2.0-beta.0 → 2.2.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/effects/index.d.mts +1 -1
- package/dist/effects/index.mjs +1 -1
- package/dist/index--1jnkoBj.d.mts.map +1 -1
- package/dist/index-DgJIBC_N.d.mts.map +1 -1
- package/dist/{index-Bg8HIYAO.d.mts → index-HyzpJrHe.d.mts} +49 -17
- package/dist/index-HyzpJrHe.d.mts.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +3 -3
- package/dist/preview/index.mjs +1 -1
- package/dist/{preview-RdaP0Rfv.mjs → preview-BXu7FyvN.mjs} +2 -2
- package/dist/{preview-RdaP0Rfv.mjs.map → preview-BXu7FyvN.mjs.map} +1 -1
- package/dist/preview-engine/{chunk-FFQ3K5FW.mjs → chunk-3NXJVSZY.mjs} +1 -1
- package/dist/preview-engine/{chunk-UKBUEM3G.mjs → chunk-6RSP766E.mjs} +1 -1
- package/dist/preview-engine/chunk-HCZCEOV5.mjs +174 -0
- package/dist/preview-engine/chunk-X6HUH6K4.mjs +1 -0
- package/dist/preview-engine/effects.mjs +1 -1
- package/dist/preview-engine/framefields.mjs +1 -1
- package/dist/preview-engine/player.mjs +1 -1
- package/dist/preview-engine/{renderers-57WDL6XN.mjs → renderers-NYK2NDZJ.mjs} +1 -1
- package/dist/renderer/index.mjs +1 -1
- package/dist/renderer-BkV_xpMb.mjs +3 -0
- package/dist/{renderer-mp1NN5zk.mjs → renderer-CqT2vxa4.mjs} +1263 -171
- package/dist/renderer-CqT2vxa4.mjs.map +1 -0
- package/dist/{renderer-CfR0GeQL.mjs → renderer-Dqdega0-.mjs} +6 -3
- package/dist/renderer-Dqdega0-.mjs.map +1 -0
- package/package.json +58 -55
- package/dist/index-Bg8HIYAO.d.mts.map +0 -1
- package/dist/preview-engine/chunk-3GNP7TJH.mjs +0 -98
- package/dist/preview-engine/chunk-VRGREB3V.mjs +0 -1
- package/dist/renderer-CfR0GeQL.mjs.map +0 -1
- package/dist/renderer-GSFP7v2z.mjs +0 -3
- package/dist/renderer-mp1NN5zk.mjs.map +0 -1
|
@@ -190,7 +190,7 @@ const COCO_CLASSES = [
|
|
|
190
190
|
];
|
|
191
191
|
|
|
192
192
|
//#endregion
|
|
193
|
-
//#region ../vision/dist/src-
|
|
193
|
+
//#region ../vision/dist/src-CQS94ONs.mjs
|
|
194
194
|
function computeIoU(boxA, boxB) {
|
|
195
195
|
const xA = Math.max(boxA.originX, boxB.originX);
|
|
196
196
|
const yA = Math.max(boxA.originY, boxB.originY);
|
|
@@ -263,11 +263,11 @@ var TemporalObjectTracker = class {
|
|
|
263
263
|
for (let d = 0; d < detections.length; d++) {
|
|
264
264
|
const det = detections[d];
|
|
265
265
|
if (trk.category !== det.category) continue;
|
|
266
|
-
const iou = computeIoU(trk.box, det.boundingBox);
|
|
267
|
-
if (iou >= this._iouThreshold) candidateMatches.push({
|
|
266
|
+
const iou$1 = computeIoU(trk.box, det.boundingBox);
|
|
267
|
+
if (iou$1 >= this._iouThreshold) candidateMatches.push({
|
|
268
268
|
trackIdx: t,
|
|
269
269
|
detIdx: d,
|
|
270
|
-
iou
|
|
270
|
+
iou: iou$1
|
|
271
271
|
});
|
|
272
272
|
}
|
|
273
273
|
}
|
|
@@ -858,11 +858,13 @@ function decodeRtmdetIns(out, opts) {
|
|
|
858
858
|
kept.sort((a, b) => b.detection.score - a.detection.score);
|
|
859
859
|
const detections = kept.map((k) => k.detection);
|
|
860
860
|
const masks = [];
|
|
861
|
-
if (!out.masks) return {
|
|
861
|
+
if (!out.masks && !out.maskPlane) return {
|
|
862
862
|
detections,
|
|
863
863
|
masks
|
|
864
864
|
};
|
|
865
|
-
const
|
|
865
|
+
const threshold = opts.maskThreshold ?? .5;
|
|
866
|
+
const feather = opts.featherRadius ?? .05;
|
|
867
|
+
const alphaOf = out.maskLogits ? logitToAlpha(threshold, feather) : probabilityToAlpha(threshold, feather);
|
|
866
868
|
const { maskWidth: mw, maskHeight: mh } = out;
|
|
867
869
|
const plane = mw * mh;
|
|
868
870
|
const gx = mw / transform.inputWidth;
|
|
@@ -870,7 +872,8 @@ function decodeRtmdetIns(out, opts) {
|
|
|
870
872
|
for (let d = 0; d < kept.length; d++) {
|
|
871
873
|
const { index, detection } = kept[d];
|
|
872
874
|
const box = detection.boundingBox;
|
|
873
|
-
const
|
|
875
|
+
const data = out.maskPlane ? out.maskPlane(index) : out.masks;
|
|
876
|
+
const base = out.maskPlane ? 0 : index * plane;
|
|
874
877
|
const bx0 = Math.max(0, Math.floor(box.originX) - MASK_BOX_PAD_PX);
|
|
875
878
|
const by0 = Math.max(0, Math.floor(box.originY) - MASK_BOX_PAD_PX);
|
|
876
879
|
const bx1 = Math.min(sw - 1, Math.ceil(box.originX + box.width) + MASK_BOX_PAD_PX);
|
|
@@ -888,7 +891,7 @@ function decodeRtmdetIns(out, opts) {
|
|
|
888
891
|
const mx0 = clamp$1(Math.floor(mx), 0, mw - 1);
|
|
889
892
|
const mx1 = Math.min(mw - 1, mx0 + 1);
|
|
890
893
|
const fx = clamp$1(mx - mx0, 0, 1);
|
|
891
|
-
const alpha = alphaOf(
|
|
894
|
+
const alpha = alphaOf(data[base + my0 * mw + mx0] * (1 - fx) * (1 - fy) + data[base + my0 * mw + mx1] * fx * (1 - fy) + data[base + my1 * mw + mx0] * (1 - fx) * fy + data[base + my1 * mw + mx1] * fx * fy);
|
|
892
895
|
if (alpha > 0) {
|
|
893
896
|
mask[row + sx] = alpha;
|
|
894
897
|
area += alpha / 255;
|
|
@@ -922,6 +925,21 @@ function probabilityToAlpha(threshold, feather) {
|
|
|
922
925
|
const inv = 255 / (2 * feather);
|
|
923
926
|
return (p) => p <= lo ? 0 : p >= threshold + feather ? 255 : (p - lo) * inv;
|
|
924
927
|
}
|
|
928
|
+
/**
|
|
929
|
+
* Logit → 0..255 alpha: `probabilityToAlpha` after a sigmoid, which is only
|
|
930
|
+
* evaluated inside the feather band.
|
|
931
|
+
*/
|
|
932
|
+
function logitToAlpha(threshold, feather) {
|
|
933
|
+
const toAlpha = probabilityToAlpha(threshold, feather);
|
|
934
|
+
const logit = (p) => p <= 0 ? -Infinity : p >= 1 ? Infinity : Math.log(p / (1 - p));
|
|
935
|
+
if (feather <= 0) {
|
|
936
|
+
const edge = logit(threshold);
|
|
937
|
+
return (l) => l > edge ? 255 : 0;
|
|
938
|
+
}
|
|
939
|
+
const lo = logit(threshold - feather);
|
|
940
|
+
const hi = logit(threshold + feather);
|
|
941
|
+
return (l) => l <= lo ? 0 : l >= hi ? 255 : toAlpha(1 / (1 + Math.exp(-l)));
|
|
942
|
+
}
|
|
925
943
|
function clamp$1(v, lo, hi) {
|
|
926
944
|
return v < lo ? lo : v > hi ? hi : v;
|
|
927
945
|
}
|
|
@@ -972,9 +990,9 @@ function decodeRtmo(out, opts) {
|
|
|
972
990
|
});
|
|
973
991
|
}
|
|
974
992
|
people.sort((a, b) => b.score - a.score);
|
|
975
|
-
const iou = opts.iouThreshold ?? .6;
|
|
993
|
+
const iou$1 = opts.iouThreshold ?? .6;
|
|
976
994
|
const kept = [];
|
|
977
|
-
for (const p of people) if (!kept.some((k) => isDuplicate(k, p, iou))) kept.push(p);
|
|
995
|
+
for (const p of people) if (!kept.some((k) => isDuplicate(k, p, iou$1))) kept.push(p);
|
|
978
996
|
return { people: kept };
|
|
979
997
|
}
|
|
980
998
|
function clamp(v, lo, hi) {
|
|
@@ -985,8 +1003,8 @@ function clamp(v, lo, hi) {
|
|
|
985
1003
|
* within 10% of the stronger detection's box size of each other (boxes may differ when one
|
|
986
1004
|
* of them also covers a flowing garment or a shadow).
|
|
987
1005
|
*/
|
|
988
|
-
function isDuplicate(a, b, iou) {
|
|
989
|
-
if (computeIoU(a.boundingBox, b.boundingBox) > iou) return true;
|
|
1006
|
+
function isDuplicate(a, b, iou$1) {
|
|
1007
|
+
if (computeIoU(a.boundingBox, b.boundingBox) > iou$1) return true;
|
|
990
1008
|
const scale = Math.sqrt(a.boundingBox.width * a.boundingBox.height);
|
|
991
1009
|
let sum = 0;
|
|
992
1010
|
let n = 0;
|
|
@@ -1028,6 +1046,286 @@ function decodeSelfie(alphas, matteWidth, matteHeight, opts) {
|
|
|
1028
1046
|
coverage: sw * sh > 0 ? sum / (255 * sw * sh) : 0
|
|
1029
1047
|
};
|
|
1030
1048
|
}
|
|
1049
|
+
/**
|
|
1050
|
+
* Frame preprocessing on the GPU: a texture becomes a model's NCHW float input
|
|
1051
|
+
* in a storage buffer, so a frame drawn by the renderer reaches the model
|
|
1052
|
+
* without a round trip through the CPU. The math is `imageToTensor`'s
|
|
1053
|
+
* (pixel-centre bilinear sampling, letterbox padding, channel order,
|
|
1054
|
+
* normalisation); see decode/preprocess.ts for the per-model contracts.
|
|
1055
|
+
*
|
|
1056
|
+
* Also: a content hash of a texture (two order-independent 32-bit sums, read
|
|
1057
|
+
* back as 8 bytes), so layers showing the same pixels can share a pass, and a
|
|
1058
|
+
* plain RGBA readback for paths that need pixels on the CPU.
|
|
1059
|
+
*/
|
|
1060
|
+
const TENSOR_WGSL = `
|
|
1061
|
+
struct Params {
|
|
1062
|
+
src: vec2u,
|
|
1063
|
+
dst: vec2u,
|
|
1064
|
+
scale: vec2f,
|
|
1065
|
+
offset: vec2f,
|
|
1066
|
+
lo: vec2i,
|
|
1067
|
+
hi: vec2i,
|
|
1068
|
+
mean: vec4f,
|
|
1069
|
+
invStd: vec4f,
|
|
1070
|
+
pad: vec4f,
|
|
1071
|
+
order: vec4u,
|
|
1072
|
+
};
|
|
1073
|
+
|
|
1074
|
+
@group(0) @binding(0) var<uniform> p: Params;
|
|
1075
|
+
@group(0) @binding(1) var src: texture_2d<f32>;
|
|
1076
|
+
@group(0) @binding(2) var<storage, read_write> dst: array<f32>;
|
|
1077
|
+
|
|
1078
|
+
@compute @workgroup_size(16, 16)
|
|
1079
|
+
fn main(@builtin(global_invocation_id) id: vec3u) {
|
|
1080
|
+
if (id.x >= p.dst.x || id.y >= p.dst.y) { return; }
|
|
1081
|
+
let x = i32(id.x);
|
|
1082
|
+
let y = i32(id.y);
|
|
1083
|
+
let plane = p.dst.x * p.dst.y;
|
|
1084
|
+
let o = id.y * p.dst.x + id.x;
|
|
1085
|
+
var v = p.pad.xyz;
|
|
1086
|
+
if (x >= p.lo.x && x <= p.hi.x && y >= p.lo.y && y <= p.hi.y) {
|
|
1087
|
+
let maxX = i32(p.src.x) - 1;
|
|
1088
|
+
let maxY = i32(p.src.y) - 1;
|
|
1089
|
+
let sy = max(0.0, (f32(y) + 0.5 - p.offset.y) / p.scale.y - 0.5);
|
|
1090
|
+
let iy0 = min(maxY, i32(floor(sy)));
|
|
1091
|
+
let iy1 = min(maxY, iy0 + 1);
|
|
1092
|
+
let fy = sy - f32(iy0);
|
|
1093
|
+
let sx = max(0.0, (f32(x) + 0.5 - p.offset.x) / p.scale.x - 0.5);
|
|
1094
|
+
let ix0 = min(maxX, i32(floor(sx)));
|
|
1095
|
+
let ix1 = min(maxX, ix0 + 1);
|
|
1096
|
+
let fx = sx - f32(ix0);
|
|
1097
|
+
var rgba =
|
|
1098
|
+
textureLoad(src, vec2i(ix0, iy0), 0) * ((1.0 - fx) * (1.0 - fy)) +
|
|
1099
|
+
textureLoad(src, vec2i(ix1, iy0), 0) * (fx * (1.0 - fy)) +
|
|
1100
|
+
textureLoad(src, vec2i(ix0, iy1), 0) * ((1.0 - fx) * fy) +
|
|
1101
|
+
textureLoad(src, vec2i(ix1, iy1), 0) * (fx * fy);
|
|
1102
|
+
rgba = rgba * 255.0;
|
|
1103
|
+
let s = vec3f(rgba[p.order.x], rgba[p.order.y], rgba[p.order.z]);
|
|
1104
|
+
v = (s - p.mean.xyz) * p.invStd.xyz;
|
|
1105
|
+
}
|
|
1106
|
+
dst[o] = v.x;
|
|
1107
|
+
dst[plane + o] = v.y;
|
|
1108
|
+
dst[2u * plane + o] = v.z;
|
|
1109
|
+
}
|
|
1110
|
+
`;
|
|
1111
|
+
const HASH_WGSL = `
|
|
1112
|
+
@group(0) @binding(0) var src: texture_2d<f32>;
|
|
1113
|
+
@group(0) @binding(1) var<storage, read_write> acc: array<atomic<u32>, 2>;
|
|
1114
|
+
|
|
1115
|
+
var<workgroup> local: array<atomic<u32>, 2>;
|
|
1116
|
+
|
|
1117
|
+
@compute @workgroup_size(16, 16)
|
|
1118
|
+
fn main(
|
|
1119
|
+
@builtin(global_invocation_id) id: vec3u,
|
|
1120
|
+
@builtin(local_invocation_index) li: u32,
|
|
1121
|
+
) {
|
|
1122
|
+
let dim = textureDimensions(src);
|
|
1123
|
+
if (id.x < dim.x && id.y < dim.y) {
|
|
1124
|
+
let c = pack4x8unorm(textureLoad(src, vec2i(id.xy), 0));
|
|
1125
|
+
let i = id.y * dim.x + id.x;
|
|
1126
|
+
let a = (c ^ (i * 0x9e3779b9u)) * 0x85ebca6bu;
|
|
1127
|
+
let b = (c + i * 0xc2b2ae35u) * 0x27d4eb2fu;
|
|
1128
|
+
atomicAdd(&local[0], a ^ (a >> 15u));
|
|
1129
|
+
atomicAdd(&local[1], b ^ (b >> 13u));
|
|
1130
|
+
}
|
|
1131
|
+
workgroupBarrier();
|
|
1132
|
+
if (li == 0u) {
|
|
1133
|
+
atomicAdd(&acc[0], atomicLoad(&local[0]));
|
|
1134
|
+
atomicAdd(&acc[1], atomicLoad(&local[1]));
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
`;
|
|
1138
|
+
const PARAMS_BYTES = 112;
|
|
1139
|
+
const pipelines = /* @__PURE__ */ new WeakMap();
|
|
1140
|
+
var FrameTensorPipeline = class FrameTensorPipeline$1 {
|
|
1141
|
+
tensorPipeline;
|
|
1142
|
+
hashPipeline;
|
|
1143
|
+
params;
|
|
1144
|
+
/** One per device. */
|
|
1145
|
+
static for(device) {
|
|
1146
|
+
let pipeline = pipelines.get(device);
|
|
1147
|
+
if (!pipeline) {
|
|
1148
|
+
pipeline = new FrameTensorPipeline$1(device);
|
|
1149
|
+
pipelines.set(device, pipeline);
|
|
1150
|
+
}
|
|
1151
|
+
return pipeline;
|
|
1152
|
+
}
|
|
1153
|
+
constructor(device) {
|
|
1154
|
+
this.device = device;
|
|
1155
|
+
this.tensorPipeline = device.createComputePipeline({
|
|
1156
|
+
label: "vision_frame_tensor",
|
|
1157
|
+
layout: "auto",
|
|
1158
|
+
compute: {
|
|
1159
|
+
module: device.createShaderModule({ code: TENSOR_WGSL }),
|
|
1160
|
+
entryPoint: "main"
|
|
1161
|
+
}
|
|
1162
|
+
});
|
|
1163
|
+
this.hashPipeline = device.createComputePipeline({
|
|
1164
|
+
label: "vision_frame_hash",
|
|
1165
|
+
layout: "auto",
|
|
1166
|
+
compute: {
|
|
1167
|
+
module: device.createShaderModule({ code: HASH_WGSL }),
|
|
1168
|
+
entryPoint: "main"
|
|
1169
|
+
}
|
|
1170
|
+
});
|
|
1171
|
+
this.params = device.createBuffer({
|
|
1172
|
+
label: "vision_frame_tensor_params",
|
|
1173
|
+
size: PARAMS_BYTES,
|
|
1174
|
+
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
/** Bytes of the NCHW float tensor for `spec`. */
|
|
1178
|
+
static tensorBytes(spec) {
|
|
1179
|
+
return 3 * spec.width * spec.height * 4;
|
|
1180
|
+
}
|
|
1181
|
+
/**
|
|
1182
|
+
* Writes `image` as the model input described by `spec` into `out` (at least
|
|
1183
|
+
* `tensorBytes(spec)`, STORAGE usage). Submits its own command buffer, so it
|
|
1184
|
+
* runs before anything submitted later (e.g. the inference reading `out`).
|
|
1185
|
+
*/
|
|
1186
|
+
toTensor(image, spec, out) {
|
|
1187
|
+
const transform = computeInputTransform(image.width, image.height, spec);
|
|
1188
|
+
const { scaleX, scaleY, offsetX, offsetY } = transform;
|
|
1189
|
+
const params = new ArrayBuffer(PARAMS_BYTES);
|
|
1190
|
+
const u32 = new Uint32Array(params);
|
|
1191
|
+
const f32 = new Float32Array(params);
|
|
1192
|
+
const i32 = new Int32Array(params);
|
|
1193
|
+
u32.set([
|
|
1194
|
+
image.width,
|
|
1195
|
+
image.height,
|
|
1196
|
+
spec.width,
|
|
1197
|
+
spec.height
|
|
1198
|
+
], 0);
|
|
1199
|
+
f32.set([
|
|
1200
|
+
scaleX,
|
|
1201
|
+
scaleY,
|
|
1202
|
+
offsetX,
|
|
1203
|
+
offsetY
|
|
1204
|
+
], 4);
|
|
1205
|
+
i32.set([
|
|
1206
|
+
Math.ceil(offsetX),
|
|
1207
|
+
Math.ceil(offsetY),
|
|
1208
|
+
Math.min(spec.width, Math.floor(offsetX + image.width * scaleX)) - 1,
|
|
1209
|
+
Math.min(spec.height, Math.floor(offsetY + image.height * scaleY)) - 1
|
|
1210
|
+
], 8);
|
|
1211
|
+
const inv = spec.std.map((s) => 1 / s);
|
|
1212
|
+
f32.set([...spec.mean, 0], 12);
|
|
1213
|
+
f32.set([...inv, 0], 16);
|
|
1214
|
+
f32.set([
|
|
1215
|
+
0,
|
|
1216
|
+
1,
|
|
1217
|
+
2
|
|
1218
|
+
].map((c) => (spec.padValue - spec.mean[c]) * inv[c]), 20);
|
|
1219
|
+
u32.set(spec.channels === "bgr" ? [
|
|
1220
|
+
2,
|
|
1221
|
+
1,
|
|
1222
|
+
0,
|
|
1223
|
+
3
|
|
1224
|
+
] : [
|
|
1225
|
+
0,
|
|
1226
|
+
1,
|
|
1227
|
+
2,
|
|
1228
|
+
3
|
|
1229
|
+
], 24);
|
|
1230
|
+
this.device.queue.writeBuffer(this.params, 0, params);
|
|
1231
|
+
const encoder$1 = this.device.createCommandEncoder({ label: "vision_frame_tensor" });
|
|
1232
|
+
const pass = encoder$1.beginComputePass();
|
|
1233
|
+
pass.setPipeline(this.tensorPipeline);
|
|
1234
|
+
pass.setBindGroup(0, this.device.createBindGroup({
|
|
1235
|
+
layout: this.tensorPipeline.getBindGroupLayout(0),
|
|
1236
|
+
entries: [
|
|
1237
|
+
{
|
|
1238
|
+
binding: 0,
|
|
1239
|
+
resource: { buffer: this.params }
|
|
1240
|
+
},
|
|
1241
|
+
{
|
|
1242
|
+
binding: 1,
|
|
1243
|
+
resource: image.texture.createView()
|
|
1244
|
+
},
|
|
1245
|
+
{
|
|
1246
|
+
binding: 2,
|
|
1247
|
+
resource: { buffer: out }
|
|
1248
|
+
}
|
|
1249
|
+
]
|
|
1250
|
+
}));
|
|
1251
|
+
pass.dispatchWorkgroups(Math.ceil(spec.width / 16), Math.ceil(spec.height / 16));
|
|
1252
|
+
pass.end();
|
|
1253
|
+
this.device.queue.submit([encoder$1.finish()]);
|
|
1254
|
+
return transform;
|
|
1255
|
+
}
|
|
1256
|
+
/** A key that is equal for textures with equal size and pixels. */
|
|
1257
|
+
async contentKey(image) {
|
|
1258
|
+
const acc = this.device.createBuffer({
|
|
1259
|
+
size: 8,
|
|
1260
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
|
|
1261
|
+
});
|
|
1262
|
+
const staging = this.device.createBuffer({
|
|
1263
|
+
size: 8,
|
|
1264
|
+
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST
|
|
1265
|
+
});
|
|
1266
|
+
const encoder$1 = this.device.createCommandEncoder({ label: "vision_frame_hash" });
|
|
1267
|
+
const pass = encoder$1.beginComputePass();
|
|
1268
|
+
pass.setPipeline(this.hashPipeline);
|
|
1269
|
+
pass.setBindGroup(0, this.device.createBindGroup({
|
|
1270
|
+
layout: this.hashPipeline.getBindGroupLayout(0),
|
|
1271
|
+
entries: [{
|
|
1272
|
+
binding: 0,
|
|
1273
|
+
resource: image.texture.createView()
|
|
1274
|
+
}, {
|
|
1275
|
+
binding: 1,
|
|
1276
|
+
resource: { buffer: acc }
|
|
1277
|
+
}]
|
|
1278
|
+
}));
|
|
1279
|
+
pass.dispatchWorkgroups(Math.ceil(image.width / 16), Math.ceil(image.height / 16));
|
|
1280
|
+
pass.end();
|
|
1281
|
+
encoder$1.copyBufferToBuffer(acc, 0, staging, 0, 8);
|
|
1282
|
+
this.device.queue.submit([encoder$1.finish()]);
|
|
1283
|
+
await staging.mapAsync(GPUMapMode.READ);
|
|
1284
|
+
const [a, b] = new Uint32Array(staging.getMappedRange().slice(0));
|
|
1285
|
+
staging.unmap();
|
|
1286
|
+
staging.destroy();
|
|
1287
|
+
acc.destroy();
|
|
1288
|
+
return `${image.width}x${image.height}:gpu:${a.toString(36)}:${b.toString(36)}`;
|
|
1289
|
+
}
|
|
1290
|
+
};
|
|
1291
|
+
/** Reads an 8-bit RGBA/BGRA texture back as RGBA pixels. */
|
|
1292
|
+
async function readTexturePixels(image) {
|
|
1293
|
+
const { device, texture, width, height } = image;
|
|
1294
|
+
const unaligned = width * 4;
|
|
1295
|
+
const bytesPerRow = Math.ceil(unaligned / 256) * 256;
|
|
1296
|
+
const staging = device.createBuffer({
|
|
1297
|
+
size: bytesPerRow * height,
|
|
1298
|
+
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
1299
|
+
label: "vision_frame_readback"
|
|
1300
|
+
});
|
|
1301
|
+
const encoder$1 = device.createCommandEncoder({ label: "vision_frame_readback" });
|
|
1302
|
+
encoder$1.copyTextureToBuffer({ texture }, {
|
|
1303
|
+
buffer: staging,
|
|
1304
|
+
bytesPerRow,
|
|
1305
|
+
rowsPerImage: height
|
|
1306
|
+
}, [
|
|
1307
|
+
width,
|
|
1308
|
+
height,
|
|
1309
|
+
1
|
|
1310
|
+
]);
|
|
1311
|
+
device.queue.submit([encoder$1.finish()]);
|
|
1312
|
+
await staging.mapAsync(GPUMapMode.READ);
|
|
1313
|
+
const mapped = new Uint8Array(staging.getMappedRange());
|
|
1314
|
+
const data = new Uint8ClampedArray(width * height * 4);
|
|
1315
|
+
for (let y = 0; y < height; y++) data.set(mapped.subarray(y * bytesPerRow, y * bytesPerRow + unaligned), y * unaligned);
|
|
1316
|
+
staging.unmap();
|
|
1317
|
+
staging.destroy();
|
|
1318
|
+
if (texture.format.startsWith("bgra")) for (let i = 0; i < data.length; i += 4) {
|
|
1319
|
+
const b = data[i];
|
|
1320
|
+
data[i] = data[i + 2];
|
|
1321
|
+
data[i + 2] = b;
|
|
1322
|
+
}
|
|
1323
|
+
return {
|
|
1324
|
+
data,
|
|
1325
|
+
width,
|
|
1326
|
+
height
|
|
1327
|
+
};
|
|
1328
|
+
}
|
|
1031
1329
|
/** COCO-17 keypoint indices (RTMO output order). */
|
|
1032
1330
|
const COCO17_KEYPOINTS = {
|
|
1033
1331
|
NOSE: 0,
|
|
@@ -1474,6 +1772,431 @@ function createNeutralLandmarks() {
|
|
|
1474
1772
|
});
|
|
1475
1773
|
return landmarks;
|
|
1476
1774
|
}
|
|
1775
|
+
/**
|
|
1776
|
+
* RTMDet-Ins post-processing, done here instead of inside the ONNX graph.
|
|
1777
|
+
*
|
|
1778
|
+
* The mmdeploy end2end export runs top-k, per-class NMS and mask assembly in
|
|
1779
|
+
* the graph and returns a full input-sized mask for each of up to 100
|
|
1780
|
+
* detections (40 on a typical frame: ~65 MB of floats). The runner instead
|
|
1781
|
+
* cuts the graph before its first data-dependent op (see `RTMDET_INS_HEAD`)
|
|
1782
|
+
* and reproduces the tail exactly:
|
|
1783
|
+
*
|
|
1784
|
+
* 1. top 5000 anchors by their best class score;
|
|
1785
|
+
* 2. ONNX NonMaxSuppression per class (IoU > 0.6 suppresses, scores > 0.05,
|
|
1786
|
+
* at most 200 per class);
|
|
1787
|
+
* 3. the 100 best (class, box) pairs;
|
|
1788
|
+
* 4. masks on demand: a 3-layer dynamic 1×1 conv over the 8 mask features
|
|
1789
|
+
* plus each pixel's offset from the anchor, at 1/8 of the input size.
|
|
1790
|
+
* The logits are upsampled (bilinear) and squashed by the decoder.
|
|
1791
|
+
*
|
|
1792
|
+
* Constants and layer layout match the export's graph; `models.e2e.test.ts`
|
|
1793
|
+
* checks the result against the end2end outputs.
|
|
1794
|
+
*/
|
|
1795
|
+
/** Head tensors of the mmdeploy RTMDet-Ins exports (all variants). */
|
|
1796
|
+
const RTMDET_INS_HEAD = {
|
|
1797
|
+
scores: "/Sigmoid_output_0",
|
|
1798
|
+
boxes: "/Concat_7_output_0",
|
|
1799
|
+
kernels: "/Concat_5_output_0",
|
|
1800
|
+
maskFeatures: "/bbox_head/mask_head/projection/Conv_output_0"
|
|
1801
|
+
};
|
|
1802
|
+
const PRE_NMS = 5e3;
|
|
1803
|
+
const IOU_THRESHOLD = Math.fround(.6);
|
|
1804
|
+
const SCORE_FLOOR = Math.fround(.05);
|
|
1805
|
+
const MAX_PER_CLASS = 200;
|
|
1806
|
+
const MAX_DETECTIONS = 100;
|
|
1807
|
+
const STRIDES = [
|
|
1808
|
+
8,
|
|
1809
|
+
16,
|
|
1810
|
+
32
|
|
1811
|
+
];
|
|
1812
|
+
/** Mask features are at stride 8; relative offsets are divided by stride × 8. */
|
|
1813
|
+
const MASK_STRIDE = 8;
|
|
1814
|
+
const MASK_CHANNELS = 8;
|
|
1815
|
+
const KERNEL_SIZE = 169;
|
|
1816
|
+
/** Steps 1–3: the detections the end2end graph would return, best first. */
|
|
1817
|
+
function selectInstances(head) {
|
|
1818
|
+
const { scores, boxes, anchors, classes } = head;
|
|
1819
|
+
const best = new Float32Array(anchors);
|
|
1820
|
+
for (let a = 0; a < anchors; a++) {
|
|
1821
|
+
let max = 0;
|
|
1822
|
+
const row = a * classes;
|
|
1823
|
+
for (let c = 0; c < classes; c++) {
|
|
1824
|
+
const s = scores[row + c];
|
|
1825
|
+
if (s > max) max = s;
|
|
1826
|
+
}
|
|
1827
|
+
best[a] = max;
|
|
1828
|
+
}
|
|
1829
|
+
const order = [];
|
|
1830
|
+
for (let a = 0; a < anchors; a++) if (best[a] > SCORE_FLOOR) order.push(a);
|
|
1831
|
+
order.sort((x, y) => best[y] - best[x] || x - y);
|
|
1832
|
+
if (order.length > PRE_NMS) order.length = PRE_NMS;
|
|
1833
|
+
const picked = [];
|
|
1834
|
+
const candidates = [];
|
|
1835
|
+
const kept = [];
|
|
1836
|
+
for (let c = 0; c < classes; c++) {
|
|
1837
|
+
candidates.length = 0;
|
|
1838
|
+
for (const a of order) if (scores[a * classes + c] > SCORE_FLOOR) candidates.push(a);
|
|
1839
|
+
if (candidates.length === 0) continue;
|
|
1840
|
+
candidates.sort((x, y) => scores[y * classes + c] - scores[x * classes + c] || x - y);
|
|
1841
|
+
kept.length = 0;
|
|
1842
|
+
for (const a of candidates) {
|
|
1843
|
+
if (kept.length >= MAX_PER_CLASS) break;
|
|
1844
|
+
let suppressed = false;
|
|
1845
|
+
for (const k of kept) if (iou(boxes, a, k) > IOU_THRESHOLD) {
|
|
1846
|
+
suppressed = true;
|
|
1847
|
+
break;
|
|
1848
|
+
}
|
|
1849
|
+
if (!suppressed) {
|
|
1850
|
+
kept.push(a);
|
|
1851
|
+
picked.push({
|
|
1852
|
+
anchor: a,
|
|
1853
|
+
classIndex: c,
|
|
1854
|
+
score: scores[a * classes + c]
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
picked.sort((x, y) => y.score - x.score);
|
|
1860
|
+
if (picked.length > MAX_DETECTIONS) picked.length = MAX_DETECTIONS;
|
|
1861
|
+
return picked;
|
|
1862
|
+
}
|
|
1863
|
+
function iou(boxes, a, b) {
|
|
1864
|
+
const a0 = boxes[a * 4];
|
|
1865
|
+
const a1 = boxes[a * 4 + 1];
|
|
1866
|
+
const a2 = boxes[a * 4 + 2];
|
|
1867
|
+
const a3 = boxes[a * 4 + 3];
|
|
1868
|
+
const b0 = boxes[b * 4];
|
|
1869
|
+
const b1 = boxes[b * 4 + 1];
|
|
1870
|
+
const b2 = boxes[b * 4 + 2];
|
|
1871
|
+
const b3 = boxes[b * 4 + 3];
|
|
1872
|
+
const w = Math.min(a2, b2) - Math.max(a0, b0);
|
|
1873
|
+
const h = Math.min(a3, b3) - Math.max(a1, b1);
|
|
1874
|
+
if (w <= 0 || h <= 0) return 0;
|
|
1875
|
+
const inter = w * h;
|
|
1876
|
+
const union = (a2 - a0) * (a3 - a1) + (b2 - b0) * (b3 - b1) - inter;
|
|
1877
|
+
return union > 0 ? inter / union : 0;
|
|
1878
|
+
}
|
|
1879
|
+
/** Anchor position and stride, in input pixels (mmdet point priors, offset 0). */
|
|
1880
|
+
function anchorPrior(anchor, inputWidth, inputHeight) {
|
|
1881
|
+
let base = 0;
|
|
1882
|
+
for (const stride of STRIDES) {
|
|
1883
|
+
const cols = Math.ceil(inputWidth / stride);
|
|
1884
|
+
const count = cols * Math.ceil(inputHeight / stride);
|
|
1885
|
+
if (anchor < base + count) {
|
|
1886
|
+
const i = anchor - base;
|
|
1887
|
+
return {
|
|
1888
|
+
x: i % cols * stride,
|
|
1889
|
+
y: Math.floor(i / cols) * stride,
|
|
1890
|
+
stride
|
|
1891
|
+
};
|
|
1892
|
+
}
|
|
1893
|
+
base += count;
|
|
1894
|
+
}
|
|
1895
|
+
throw new RangeError(`anchor ${anchor} is out of range`);
|
|
1896
|
+
}
|
|
1897
|
+
/**
|
|
1898
|
+
* Step 4: mask logits for one anchor, `maskHeight × maskWidth`, row-major.
|
|
1899
|
+
* Writes into `out` when it is large enough.
|
|
1900
|
+
*/
|
|
1901
|
+
function maskLogits(head, anchor, out) {
|
|
1902
|
+
const { kernels, maskFeatures, maskWidth: mw, maskHeight: mh } = head;
|
|
1903
|
+
const plane = mw * mh;
|
|
1904
|
+
const logits = out && out.length >= plane ? out : new Float32Array(plane);
|
|
1905
|
+
const prior = anchorPrior(anchor, head.inputWidth, head.inputHeight);
|
|
1906
|
+
const k = kernels.subarray(anchor * KERNEL_SIZE, (anchor + 1) * KERNEL_SIZE);
|
|
1907
|
+
const w1 = 0;
|
|
1908
|
+
const w2 = 80;
|
|
1909
|
+
const w3 = 144;
|
|
1910
|
+
const b1 = 152;
|
|
1911
|
+
const b2 = 160;
|
|
1912
|
+
const b3 = 168;
|
|
1913
|
+
const scale = 1 / (prior.stride * MASK_STRIDE);
|
|
1914
|
+
const input = new Float32Array(2 + MASK_CHANNELS);
|
|
1915
|
+
const hidden1 = new Float32Array(8);
|
|
1916
|
+
const hidden2 = new Float32Array(8);
|
|
1917
|
+
for (let y = 0; y < mh; y++) {
|
|
1918
|
+
const relY = (prior.y - y * MASK_STRIDE) * scale;
|
|
1919
|
+
for (let x = 0; x < mw; x++) {
|
|
1920
|
+
const p = y * mw + x;
|
|
1921
|
+
input[0] = (prior.x - x * MASK_STRIDE) * scale;
|
|
1922
|
+
input[1] = relY;
|
|
1923
|
+
for (let c = 0; c < MASK_CHANNELS; c++) input[2 + c] = maskFeatures[c * plane + p];
|
|
1924
|
+
for (let o = 0; o < 8; o++) {
|
|
1925
|
+
let sum$1 = k[b1 + o];
|
|
1926
|
+
const row = w1 + o * 10;
|
|
1927
|
+
for (let i = 0; i < 10; i++) sum$1 += k[row + i] * input[i];
|
|
1928
|
+
hidden1[o] = sum$1 > 0 ? sum$1 : 0;
|
|
1929
|
+
}
|
|
1930
|
+
for (let o = 0; o < 8; o++) {
|
|
1931
|
+
let sum$1 = k[b2 + o];
|
|
1932
|
+
const row = w2 + o * 8;
|
|
1933
|
+
for (let i = 0; i < 8; i++) sum$1 += k[row + i] * hidden1[i];
|
|
1934
|
+
hidden2[o] = sum$1 > 0 ? sum$1 : 0;
|
|
1935
|
+
}
|
|
1936
|
+
let sum = k[b3];
|
|
1937
|
+
for (let i = 0; i < 8; i++) sum += k[w3 + i] * hidden2[i];
|
|
1938
|
+
logits[p] = sum;
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
return logits;
|
|
1942
|
+
}
|
|
1943
|
+
/**
|
|
1944
|
+
* Minimal ONNX protobuf wire-format helpers: split a message into fields,
|
|
1945
|
+
* read strings, write length-delimited fields. Only what the graph editors
|
|
1946
|
+
* (onnx-subgraph.ts, onnx-float16.ts) need; unknown fields are copied as raw
|
|
1947
|
+
* bytes, so nothing else in a model is touched.
|
|
1948
|
+
*/
|
|
1949
|
+
const MODEL_GRAPH = 7;
|
|
1950
|
+
const GRAPH_NODE = 1;
|
|
1951
|
+
const GRAPH_INITIALIZER = 5;
|
|
1952
|
+
const GRAPH_INPUT = 11;
|
|
1953
|
+
const GRAPH_OUTPUT = 12;
|
|
1954
|
+
const GRAPH_VALUE_INFO = 13;
|
|
1955
|
+
const NODE_INPUT = 1;
|
|
1956
|
+
const NODE_OUTPUT$1 = 2;
|
|
1957
|
+
const VALUE_INFO_NAME = 1;
|
|
1958
|
+
const TENSOR_NAME = 8;
|
|
1959
|
+
const WIRE_VARINT = 0;
|
|
1960
|
+
const WIRE_I64 = 1;
|
|
1961
|
+
const WIRE_LEN = 2;
|
|
1962
|
+
const WIRE_I32 = 5;
|
|
1963
|
+
function readVarint(buf, pos) {
|
|
1964
|
+
let value = 0;
|
|
1965
|
+
let scale = 1;
|
|
1966
|
+
for (;;) {
|
|
1967
|
+
if (pos >= buf.length) throw new Error("ONNX model: truncated varint");
|
|
1968
|
+
const byte = buf[pos++];
|
|
1969
|
+
value += (byte & 127) * scale;
|
|
1970
|
+
if (byte < 128) return [value, pos];
|
|
1971
|
+
scale *= 128;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
function fields(buf) {
|
|
1975
|
+
const out = [];
|
|
1976
|
+
let pos = 0;
|
|
1977
|
+
while (pos < buf.length) {
|
|
1978
|
+
const start = pos;
|
|
1979
|
+
const [tag, afterTag] = readVarint(buf, pos);
|
|
1980
|
+
pos = afterTag;
|
|
1981
|
+
const number = Math.floor(tag / 8);
|
|
1982
|
+
const wire = tag & 7;
|
|
1983
|
+
let body = buf.subarray(pos, pos);
|
|
1984
|
+
if (wire === WIRE_VARINT) pos = readVarint(buf, pos)[1];
|
|
1985
|
+
else if (wire === WIRE_I64) pos += 8;
|
|
1986
|
+
else if (wire === WIRE_I32) pos += 4;
|
|
1987
|
+
else if (wire === WIRE_LEN) {
|
|
1988
|
+
const [length, afterLength] = readVarint(buf, pos);
|
|
1989
|
+
body = buf.subarray(afterLength, afterLength + length);
|
|
1990
|
+
pos = afterLength + length;
|
|
1991
|
+
} else throw new Error(`ONNX model: unsupported wire type ${wire}`);
|
|
1992
|
+
if (pos > buf.length) throw new Error("ONNX model: truncated field");
|
|
1993
|
+
out.push({
|
|
1994
|
+
number,
|
|
1995
|
+
wire,
|
|
1996
|
+
raw: buf.subarray(start, pos),
|
|
1997
|
+
body
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
return out;
|
|
2001
|
+
}
|
|
2002
|
+
const decoder = new TextDecoder();
|
|
2003
|
+
const encoder = new TextEncoder();
|
|
2004
|
+
function strings(buf, number) {
|
|
2005
|
+
return fields(buf).filter((f) => f.number === number && f.wire === WIRE_LEN).map((f) => decoder.decode(f.body));
|
|
2006
|
+
}
|
|
2007
|
+
function firstString(buf, number) {
|
|
2008
|
+
return strings(buf, number)[0];
|
|
2009
|
+
}
|
|
2010
|
+
function varint(value) {
|
|
2011
|
+
const bytes = [];
|
|
2012
|
+
while (value >= 128) {
|
|
2013
|
+
bytes.push(value % 128 | 128);
|
|
2014
|
+
value = Math.floor(value / 128);
|
|
2015
|
+
}
|
|
2016
|
+
bytes.push(value);
|
|
2017
|
+
return bytes;
|
|
2018
|
+
}
|
|
2019
|
+
function lengthDelimited(number, payload) {
|
|
2020
|
+
const head = [...varint(number * 8 + WIRE_LEN), ...varint(payload.length)];
|
|
2021
|
+
const out = new Uint8Array(head.length + payload.length);
|
|
2022
|
+
out.set(head, 0);
|
|
2023
|
+
out.set(payload, head.length);
|
|
2024
|
+
return out;
|
|
2025
|
+
}
|
|
2026
|
+
function concat(parts) {
|
|
2027
|
+
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
|
2028
|
+
let offset = 0;
|
|
2029
|
+
for (const part of parts) {
|
|
2030
|
+
out.set(part, offset);
|
|
2031
|
+
offset += part.length;
|
|
2032
|
+
}
|
|
2033
|
+
return out;
|
|
2034
|
+
}
|
|
2035
|
+
/**
|
|
2036
|
+
* Cuts an ONNX model down to the nodes that compute a chosen set of tensors,
|
|
2037
|
+
* which become the graph's outputs. Works on the protobuf wire format directly
|
|
2038
|
+
* (only the fields it needs are decoded; everything else is copied byte for
|
|
2039
|
+
* byte), so it needs no protobuf library and keeps the weights untouched.
|
|
2040
|
+
*
|
|
2041
|
+
* Used to run a detector's fixed-shape body on the GPU and do its
|
|
2042
|
+
* data-dependent tail (top-k, NMS, mask assembly) ourselves.
|
|
2043
|
+
*/
|
|
2044
|
+
/** ValueInfoProto for a float tensor of unknown shape. */
|
|
2045
|
+
function floatOutput(name) {
|
|
2046
|
+
const typeProto = lengthDelimited(1, new Uint8Array([8, 1]));
|
|
2047
|
+
return concat([lengthDelimited(VALUE_INFO_NAME, encoder.encode(name)), lengthDelimited(2, typeProto)]);
|
|
2048
|
+
}
|
|
2049
|
+
/**
|
|
2050
|
+
* Returns a model whose outputs are `outputs` (tensors of the original graph),
|
|
2051
|
+
* keeping only the nodes, initializers and inputs needed to compute them.
|
|
2052
|
+
* Output types come from the graph's value_info when present; otherwise they
|
|
2053
|
+
* are declared as float tensors.
|
|
2054
|
+
*/
|
|
2055
|
+
function extractSubgraph(model, outputs) {
|
|
2056
|
+
const modelFields = fields(model);
|
|
2057
|
+
const graphField = modelFields.find((f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN);
|
|
2058
|
+
if (!graphField) throw new Error("ONNX model has no graph");
|
|
2059
|
+
const graphFields = fields(graphField.body);
|
|
2060
|
+
const nodes = graphFields.filter((f) => f.number === GRAPH_NODE).map((f) => ({
|
|
2061
|
+
field: f,
|
|
2062
|
+
inputs: strings(f.body, NODE_INPUT),
|
|
2063
|
+
outputs: strings(f.body, NODE_OUTPUT$1)
|
|
2064
|
+
}));
|
|
2065
|
+
const producer = /* @__PURE__ */ new Map();
|
|
2066
|
+
nodes.forEach((node, index) => {
|
|
2067
|
+
for (const name of node.outputs) producer.set(name, index);
|
|
2068
|
+
});
|
|
2069
|
+
const keepNode = /* @__PURE__ */ new Set();
|
|
2070
|
+
const needed = /* @__PURE__ */ new Set();
|
|
2071
|
+
const stack = [...outputs];
|
|
2072
|
+
for (const name of outputs) if (!producer.has(name)) throw new Error(`ONNX model has no tensor '${name}'`);
|
|
2073
|
+
while (stack.length > 0) {
|
|
2074
|
+
const name = stack.pop();
|
|
2075
|
+
if (needed.has(name)) continue;
|
|
2076
|
+
needed.add(name);
|
|
2077
|
+
const index = producer.get(name);
|
|
2078
|
+
if (index === void 0 || keepNode.has(index)) continue;
|
|
2079
|
+
keepNode.add(index);
|
|
2080
|
+
for (const input of nodes[index].inputs) if (input) stack.push(input);
|
|
2081
|
+
}
|
|
2082
|
+
const valueInfo = /* @__PURE__ */ new Map();
|
|
2083
|
+
for (const f of graphFields) {
|
|
2084
|
+
if (f.number !== GRAPH_VALUE_INFO) continue;
|
|
2085
|
+
const name = firstString(f.body, VALUE_INFO_NAME);
|
|
2086
|
+
if (name !== void 0) valueInfo.set(name, f.body);
|
|
2087
|
+
}
|
|
2088
|
+
const parts = [];
|
|
2089
|
+
let nodeIndex = 0;
|
|
2090
|
+
for (const f of graphFields) switch (f.number) {
|
|
2091
|
+
case GRAPH_NODE:
|
|
2092
|
+
if (keepNode.has(nodeIndex)) parts.push(f.raw);
|
|
2093
|
+
nodeIndex++;
|
|
2094
|
+
break;
|
|
2095
|
+
case GRAPH_INITIALIZER: {
|
|
2096
|
+
const name = firstString(f.body, TENSOR_NAME);
|
|
2097
|
+
if (name !== void 0 && needed.has(name)) parts.push(f.raw);
|
|
2098
|
+
break;
|
|
2099
|
+
}
|
|
2100
|
+
case GRAPH_INPUT:
|
|
2101
|
+
case GRAPH_VALUE_INFO: {
|
|
2102
|
+
const name = firstString(f.body, VALUE_INFO_NAME);
|
|
2103
|
+
if (name !== void 0 && needed.has(name)) parts.push(f.raw);
|
|
2104
|
+
break;
|
|
2105
|
+
}
|
|
2106
|
+
case GRAPH_OUTPUT: break;
|
|
2107
|
+
default: parts.push(f.raw);
|
|
2108
|
+
}
|
|
2109
|
+
for (const name of outputs) parts.push(lengthDelimited(GRAPH_OUTPUT, valueInfo.get(name) ?? floatOutput(name)));
|
|
2110
|
+
const graph = concat(parts);
|
|
2111
|
+
return concat(modelFields.map((f) => f === graphField ? lengthDelimited(MODEL_GRAPH, graph) : f.raw));
|
|
2112
|
+
}
|
|
2113
|
+
/**
|
|
2114
|
+
* onnxruntime-web 1.30's WebGPU EP computes grouped convolutions wrong in its
|
|
2115
|
+
* default NHWC layout when a group has more than one channel and the count
|
|
2116
|
+
* isn't a multiple of 4 (RTMO's head: 18 per group → duplicate skeletons).
|
|
2117
|
+
* NCHW is correct but ~30% slower, so only models with such a convolution get
|
|
2118
|
+
* it. Depthwise convolutions (one channel per group: RTMDet-Ins, Selfie) are
|
|
2119
|
+
* fine in NHWC.
|
|
2120
|
+
*/
|
|
2121
|
+
const NODE_OP_TYPE = 4;
|
|
2122
|
+
const NODE_ATTRIBUTE = 5;
|
|
2123
|
+
const NODE_OUTPUT = 2;
|
|
2124
|
+
const ATTR_NAME = 1;
|
|
2125
|
+
const ATTR_I = 3;
|
|
2126
|
+
const ATTR_T = 5;
|
|
2127
|
+
const TENSOR_DIMS = 1;
|
|
2128
|
+
function varintValues(field) {
|
|
2129
|
+
const values = [];
|
|
2130
|
+
if (field.wire === WIRE_LEN) {
|
|
2131
|
+
let pos = 0;
|
|
2132
|
+
while (pos < field.body.length) {
|
|
2133
|
+
const [value, next] = readVarint(field.body, pos);
|
|
2134
|
+
values.push(value);
|
|
2135
|
+
pos = next;
|
|
2136
|
+
}
|
|
2137
|
+
} else if (field.wire === WIRE_VARINT) {
|
|
2138
|
+
let pos = 0;
|
|
2139
|
+
pos = readVarint(field.raw, pos)[1];
|
|
2140
|
+
values.push(readVarint(field.raw, pos)[0]);
|
|
2141
|
+
}
|
|
2142
|
+
return values;
|
|
2143
|
+
}
|
|
2144
|
+
function tensorDims(tensor) {
|
|
2145
|
+
return fields(tensor).filter((f) => f.number === TENSOR_DIMS).flatMap(varintValues);
|
|
2146
|
+
}
|
|
2147
|
+
/**
|
|
2148
|
+
* True when the model has a grouped convolution the NHWC kernels get wrong,
|
|
2149
|
+
* and when it can't be read (NCHW is always correct).
|
|
2150
|
+
*/
|
|
2151
|
+
function needsNchwLayout(model) {
|
|
2152
|
+
let graph;
|
|
2153
|
+
try {
|
|
2154
|
+
graph = fields(model).find((f) => f.number === MODEL_GRAPH && f.wire === WIRE_LEN);
|
|
2155
|
+
} catch {
|
|
2156
|
+
return true;
|
|
2157
|
+
}
|
|
2158
|
+
if (!graph) return true;
|
|
2159
|
+
const dims = /* @__PURE__ */ new Map();
|
|
2160
|
+
const convs = [];
|
|
2161
|
+
for (const f of fields(graph.body)) if (f.number === GRAPH_INITIALIZER) {
|
|
2162
|
+
const name = firstString(f.body, TENSOR_NAME);
|
|
2163
|
+
if (name !== void 0) dims.set(name, tensorDims(f.body));
|
|
2164
|
+
} else if (f.number === GRAPH_NODE) {
|
|
2165
|
+
const op = firstString(f.body, NODE_OP_TYPE);
|
|
2166
|
+
if (op !== "Conv" && op !== "Constant") continue;
|
|
2167
|
+
const attributes = fields(f.body).filter((a) => a.number === NODE_ATTRIBUTE).map((a) => a.body);
|
|
2168
|
+
if (op === "Constant") {
|
|
2169
|
+
const value = attributes.find((a) => firstString(a, ATTR_NAME) === "value");
|
|
2170
|
+
const tensor = value && fields(value).find((a) => a.number === ATTR_T);
|
|
2171
|
+
const output$1 = strings(f.body, NODE_OUTPUT)[0];
|
|
2172
|
+
if (tensor && output$1) dims.set(output$1, tensorDims(tensor.body));
|
|
2173
|
+
continue;
|
|
2174
|
+
}
|
|
2175
|
+
const groupAttr = attributes.find((a) => firstString(a, ATTR_NAME) === "group");
|
|
2176
|
+
const groupField = groupAttr && fields(groupAttr).find((a) => a.number === ATTR_I);
|
|
2177
|
+
const group = groupField ? varintValues(groupField)[0] ?? 1 : 1;
|
|
2178
|
+
const weight = strings(f.body, NODE_INPUT)[1];
|
|
2179
|
+
if (group > 1 && weight) convs.push({
|
|
2180
|
+
weight,
|
|
2181
|
+
group
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
return convs.some(({ weight }) => {
|
|
2185
|
+
const perGroup = dims.get(weight)?.[1];
|
|
2186
|
+
return perGroup !== void 0 && perGroup > 1 && perGroup % 4 !== 0;
|
|
2187
|
+
});
|
|
2188
|
+
}
|
|
2189
|
+
/**
|
|
2190
|
+
* Browser / WebGPU session provider .
|
|
2191
|
+
*
|
|
2192
|
+
* Mirrors `NodeSessionProvider` but uses `onnxruntime-web` with the WebGPU execution provider
|
|
2193
|
+
* and a WASM fallback. The ORT module is imported lazily behind an injectable loader, so:
|
|
2194
|
+
* - the Node core entry never pulls browser code, and
|
|
2195
|
+
* - unit tests can run fully offline with a fake module.
|
|
2196
|
+
*
|
|
2197
|
+
* `onnxruntime-web` is an OPTIONAL peer dependency — consumers of `@framefields/vision/web`
|
|
2198
|
+
* provide it; the main `@framefields/vision` entry never touches it.
|
|
2199
|
+
*/
|
|
1477
2200
|
const ORT_WEB_SPECIFIER = "onnxruntime-web";
|
|
1478
2201
|
const defaultLoader = () => import(
|
|
1479
2202
|
/* @vite-ignore */
|
|
@@ -1485,10 +2208,12 @@ var WebGPUProvider = class {
|
|
|
1485
2208
|
_providers;
|
|
1486
2209
|
_wasmFallback;
|
|
1487
2210
|
_ortPromise;
|
|
2211
|
+
device;
|
|
1488
2212
|
constructor(options = {}) {
|
|
1489
2213
|
this._loader = options.loader ?? defaultLoader;
|
|
1490
2214
|
this._providers = options.executionProviders ?? ["webgpu", "wasm"];
|
|
1491
2215
|
this._wasmFallback = options.wasmFallback !== false;
|
|
2216
|
+
this.device = options.device;
|
|
1492
2217
|
}
|
|
1493
2218
|
ort() {
|
|
1494
2219
|
if (!this._ortPromise) this._ortPromise = this._loader().catch((err) => {
|
|
@@ -1501,8 +2226,10 @@ var WebGPUProvider = class {
|
|
|
1501
2226
|
const ort = await this.ort();
|
|
1502
2227
|
let session;
|
|
1503
2228
|
try {
|
|
2229
|
+
const device = this.device;
|
|
2230
|
+
const layout = needsNchwLayout(modelBytes) ? "NCHW" : "NHWC";
|
|
1504
2231
|
session = await ort.InferenceSession.create(modelBytes, {
|
|
1505
|
-
executionProviders:
|
|
2232
|
+
executionProviders: this._providers.map((name) => withProviderOptions(name, device, layout)),
|
|
1506
2233
|
graphOptimizationLevel: "all",
|
|
1507
2234
|
logSeverityLevel: 3
|
|
1508
2235
|
});
|
|
@@ -1517,11 +2244,15 @@ var WebGPUProvider = class {
|
|
|
1517
2244
|
return {
|
|
1518
2245
|
run: async (input) => {
|
|
1519
2246
|
const inputName = session.inputNames.includes(input.name) ? input.name : session.inputNames[0] ?? input.name;
|
|
1520
|
-
const
|
|
2247
|
+
const tensor = input.gpuBuffer && ort.Tensor.fromGpuBuffer ? ort.Tensor.fromGpuBuffer(input.gpuBuffer, {
|
|
2248
|
+
dataType: "float32",
|
|
2249
|
+
dims: [...input.dims]
|
|
2250
|
+
}) : new ort.Tensor("float32", input.data, [...input.dims]);
|
|
2251
|
+
const results = await session.run({ [inputName]: tensor });
|
|
1521
2252
|
const outputs = {};
|
|
1522
|
-
for (const [name, tensor] of Object.entries(results)) outputs[name] = {
|
|
1523
|
-
data: tensor.data,
|
|
1524
|
-
dims: [...tensor.dims]
|
|
2253
|
+
for (const [name, tensor$1] of Object.entries(results)) outputs[name] = {
|
|
2254
|
+
data: tensor$1.data,
|
|
2255
|
+
dims: [...tensor$1.dims]
|
|
1525
2256
|
};
|
|
1526
2257
|
return outputs;
|
|
1527
2258
|
},
|
|
@@ -1531,30 +2262,62 @@ var WebGPUProvider = class {
|
|
|
1531
2262
|
};
|
|
1532
2263
|
}
|
|
1533
2264
|
};
|
|
2265
|
+
/**
|
|
2266
|
+
* The WebGPU EP runs in NCHW. Its default NHWC layout computes grouped convolutions wrong
|
|
2267
|
+
* when the channels per group aren't a multiple of 4 (onnxruntime-web 1.30) — RTMO's head
|
|
2268
|
+
* has group=8 convs over 104 channels (13 per group), which turned one dancer into several
|
|
2269
|
+
* skeletons. NCHW matches the CPU EP on every pinned model.
|
|
2270
|
+
*/
|
|
2271
|
+
function withProviderOptions(name, device, preferredLayout) {
|
|
2272
|
+
if (name !== "webgpu") return name;
|
|
2273
|
+
return device ? {
|
|
2274
|
+
name,
|
|
2275
|
+
preferredLayout,
|
|
2276
|
+
device
|
|
2277
|
+
} : {
|
|
2278
|
+
name,
|
|
2279
|
+
preferredLayout
|
|
2280
|
+
};
|
|
2281
|
+
}
|
|
1534
2282
|
const ORT_WEBGPU_SPECIFIER = "onnxruntime-web/webgpu";
|
|
1535
2283
|
const WEBGPU_SPECIFIER = "webgpu";
|
|
1536
2284
|
let nodeWebGPUReady;
|
|
2285
|
+
/** CPU rasterizers (lavapipe, llvmpipe, SwiftShader): slower than the CPU EP. */
|
|
2286
|
+
const SOFTWARE_ADAPTER = /llvmpipe|lavapipe|swiftshader|software/i;
|
|
1537
2287
|
/**
|
|
1538
|
-
* Ensure `globalThis.navigator.gpu` exists
|
|
1539
|
-
*
|
|
1540
|
-
*
|
|
2288
|
+
* Ensure `globalThis.navigator.gpu` exists and has a hardware adapter that
|
|
2289
|
+
* onnxruntime-web's WebGPU EP can use. In the renderer this is already the Dawn
|
|
2290
|
+
* device; otherwise imports the `webgpu` (Dawn) package and installs its
|
|
2291
|
+
* globals. Rejects when the only adapter is a software rasterizer.
|
|
1541
2292
|
*/
|
|
1542
2293
|
function ensureNodeWebGPU() {
|
|
1543
2294
|
if (!nodeWebGPUReady) {
|
|
1544
2295
|
nodeWebGPUReady = (async () => {
|
|
1545
2296
|
const g = globalThis;
|
|
1546
|
-
if (g.navigator?.gpu)
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
2297
|
+
if (!g.navigator?.gpu) {
|
|
2298
|
+
const { create, globals } = await import(
|
|
2299
|
+
/* @vite-ignore */
|
|
2300
|
+
WEBGPU_SPECIFIER
|
|
1550
2301
|
);
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
2302
|
+
Object.assign(globalThis, globals);
|
|
2303
|
+
g.navigator ??= {};
|
|
2304
|
+
Object.defineProperty(g.navigator, "gpu", {
|
|
2305
|
+
value: create([]),
|
|
2306
|
+
configurable: true,
|
|
2307
|
+
writable: true
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
const adapter = await (g.navigator?.gpu).requestAdapter({ powerPreference: "high-performance" });
|
|
2311
|
+
if (!adapter) throw new Error("No WebGPU adapter.");
|
|
2312
|
+
const info = adapter.info ?? {};
|
|
2313
|
+
const name = [
|
|
2314
|
+
info.vendor,
|
|
2315
|
+
info.architecture,
|
|
2316
|
+
info.device,
|
|
2317
|
+
info.description
|
|
2318
|
+
].filter(Boolean).join(" ");
|
|
2319
|
+
if (adapter.isFallbackAdapter === true || info.isFallbackAdapter === true || SOFTWARE_ADAPTER.test(name)) throw new Error(`WebGPU adapter is a software rasterizer (${name}).`);
|
|
2320
|
+
patchSupportedFeatures(adapter.features);
|
|
1558
2321
|
})();
|
|
1559
2322
|
nodeWebGPUReady.catch(() => {
|
|
1560
2323
|
nodeWebGPUReady = void 0;
|
|
@@ -1562,8 +2325,39 @@ function ensureNodeWebGPU() {
|
|
|
1562
2325
|
}
|
|
1563
2326
|
return nodeWebGPUReady;
|
|
1564
2327
|
}
|
|
2328
|
+
/**
|
|
2329
|
+
* Dawn's Node `GPUSupportedFeatures` has `has`/`keys`/`size` but no `forEach`,
|
|
2330
|
+
* which onnxruntime-web's WebGPU build calls on adapter and device features
|
|
2331
|
+
* (they share the class). Adds the missing set-like methods once.
|
|
2332
|
+
*/
|
|
2333
|
+
function patchSupportedFeatures(features) {
|
|
2334
|
+
const proto = Object.getPrototypeOf(features);
|
|
2335
|
+
if (!proto || typeof proto.keys !== "function") return;
|
|
2336
|
+
if (typeof proto.forEach !== "function") Object.defineProperty(proto, "forEach", {
|
|
2337
|
+
value(callback, thisArg) {
|
|
2338
|
+
for (const key of this.keys()) callback.call(thisArg, key, key, this);
|
|
2339
|
+
},
|
|
2340
|
+
configurable: true,
|
|
2341
|
+
writable: true
|
|
2342
|
+
});
|
|
2343
|
+
if (typeof proto.values !== "function") Object.defineProperty(proto, "values", {
|
|
2344
|
+
value() {
|
|
2345
|
+
return this.keys();
|
|
2346
|
+
},
|
|
2347
|
+
configurable: true,
|
|
2348
|
+
writable: true
|
|
2349
|
+
});
|
|
2350
|
+
if (typeof proto.entries !== "function") Object.defineProperty(proto, "entries", {
|
|
2351
|
+
*value() {
|
|
2352
|
+
for (const key of this.keys()) yield [key, key];
|
|
2353
|
+
},
|
|
2354
|
+
configurable: true,
|
|
2355
|
+
writable: true
|
|
2356
|
+
});
|
|
2357
|
+
}
|
|
1565
2358
|
var NodeWebGPUProvider = class {
|
|
1566
2359
|
kind = "webgpu";
|
|
2360
|
+
device;
|
|
1567
2361
|
_inner;
|
|
1568
2362
|
constructor(options = {}) {
|
|
1569
2363
|
const loader = options.loader ?? (async () => {
|
|
@@ -1573,8 +2367,10 @@ var NodeWebGPUProvider = class {
|
|
|
1573
2367
|
ORT_WEBGPU_SPECIFIER
|
|
1574
2368
|
);
|
|
1575
2369
|
if (mod.env?.wasm) mod.env.wasm.numThreads = 1;
|
|
2370
|
+
if (mod.env?.webgpu) mod.env.webgpu.powerPreference = "high-performance";
|
|
1576
2371
|
return mod;
|
|
1577
2372
|
});
|
|
2373
|
+
this.device = options.device;
|
|
1578
2374
|
this._inner = new WebGPUProvider({
|
|
1579
2375
|
wasmFallback: false,
|
|
1580
2376
|
...options,
|
|
@@ -1586,41 +2382,163 @@ var NodeWebGPUProvider = class {
|
|
|
1586
2382
|
}
|
|
1587
2383
|
};
|
|
1588
2384
|
let defaultProvider;
|
|
2385
|
+
let visionDevice;
|
|
2386
|
+
/**
|
|
2387
|
+
* The GPU device the default provider runs WebGPU sessions on — the renderer's,
|
|
2388
|
+
* so frames can go from its textures into the models without a CPU round trip.
|
|
2389
|
+
* Applies to sessions created afterwards; without it, onnxruntime-web creates
|
|
2390
|
+
* its own device.
|
|
2391
|
+
*/
|
|
2392
|
+
function setVisionDevice(device) {
|
|
2393
|
+
visionDevice = device;
|
|
2394
|
+
}
|
|
2395
|
+
/**
|
|
2396
|
+
* Hosts (the renderer) register a listener under this symbol to log the
|
|
2397
|
+
* backend; a global keeps it working however the packages are bundled.
|
|
2398
|
+
*/
|
|
2399
|
+
const VISION_BACKEND_LISTENER = Symbol.for("framefields.vision.backend");
|
|
2400
|
+
function reportBackend(info) {
|
|
2401
|
+
const listener = globalThis[VISION_BACKEND_LISTENER];
|
|
2402
|
+
if (typeof listener === "function") try {
|
|
2403
|
+
listener(info);
|
|
2404
|
+
} catch {}
|
|
2405
|
+
}
|
|
2406
|
+
function requestedVisionBackend() {
|
|
2407
|
+
const value = typeof process !== "undefined" ? process.env.FRAMEFIELDS_VISION_PROVIDER?.trim().toLowerCase() : void 0;
|
|
2408
|
+
return value === "webgpu" || value === "cuda" || value === "cpu" ? value : "auto";
|
|
2409
|
+
}
|
|
1589
2410
|
/**
|
|
1590
2411
|
* Prefers the GPU (onnxruntime-web's WebGPU EP on the process's WebGPU device —
|
|
1591
2412
|
* the Dawn device the renderer already creates, or one this ensures) and falls
|
|
1592
2413
|
* back to the CPU provider (onnxruntime-node) when a WebGPU session cannot be
|
|
1593
|
-
* created: no
|
|
1594
|
-
*
|
|
2414
|
+
* created: no hardware adapter, no Dawn, or onnxruntime-web not installed. The
|
|
2415
|
+
* choice is made on the first session and kept.
|
|
2416
|
+
*
|
|
2417
|
+
* `FRAMEFIELDS_VISION_PROVIDER` forces one backend instead: `webgpu`, `cpu`, or
|
|
2418
|
+
* `cuda` (onnxruntime-node's CUDA EP on Linux x64; needs the CUDA 12 and cuDNN 9
|
|
2419
|
+
* libraries). A forced backend that fails throws instead of falling back.
|
|
1595
2420
|
*/
|
|
1596
2421
|
var AutoSessionProvider = class {
|
|
1597
2422
|
_resolved;
|
|
2423
|
+
_resolving;
|
|
1598
2424
|
get kind() {
|
|
1599
2425
|
return this._resolved?.kind ?? "webgpu";
|
|
1600
2426
|
}
|
|
2427
|
+
get device() {
|
|
2428
|
+
return this._resolved?.device;
|
|
2429
|
+
}
|
|
1601
2430
|
async createSession(modelBytes) {
|
|
1602
|
-
if (
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
this.
|
|
1607
|
-
|
|
1608
|
-
}
|
|
1609
|
-
|
|
1610
|
-
|
|
2431
|
+
if (this._resolved) return this._resolved.createSession(modelBytes);
|
|
2432
|
+
if (!this._resolving) {
|
|
2433
|
+
this._resolving = this.resolve(modelBytes);
|
|
2434
|
+
this._resolving.catch(() => {
|
|
2435
|
+
this._resolving = void 0;
|
|
2436
|
+
});
|
|
2437
|
+
const { provider: provider$1, first } = await this._resolving;
|
|
2438
|
+
this._resolved = provider$1;
|
|
2439
|
+
if (first) return first;
|
|
2440
|
+
return provider$1.createSession(modelBytes);
|
|
2441
|
+
}
|
|
2442
|
+
const { provider } = await this._resolving;
|
|
2443
|
+
return provider.createSession(modelBytes);
|
|
2444
|
+
}
|
|
2445
|
+
async resolve(modelBytes) {
|
|
2446
|
+
const backend = requestedVisionBackend();
|
|
2447
|
+
if (backend === "cpu") {
|
|
2448
|
+
reportBackend({
|
|
2449
|
+
backend: "cpu",
|
|
2450
|
+
sharedDevice: false,
|
|
2451
|
+
reason: "FRAMEFIELDS_VISION_PROVIDER=cpu"
|
|
2452
|
+
});
|
|
2453
|
+
return { provider: new NodeSessionProvider() };
|
|
2454
|
+
}
|
|
2455
|
+
if (backend === "cuda") {
|
|
2456
|
+
reportBackend({
|
|
2457
|
+
backend: "cuda",
|
|
2458
|
+
sharedDevice: false
|
|
2459
|
+
});
|
|
2460
|
+
return { provider: new NodeSessionProvider({ executionProviders: ["cuda", "cpu"] }) };
|
|
2461
|
+
}
|
|
2462
|
+
const webgpu = new NodeWebGPUProvider({
|
|
2463
|
+
wasmFallback: false,
|
|
2464
|
+
device: visionDevice
|
|
2465
|
+
});
|
|
2466
|
+
try {
|
|
2467
|
+
const first = await webgpu.createSession(modelBytes);
|
|
2468
|
+
reportBackend({
|
|
2469
|
+
backend: "webgpu",
|
|
2470
|
+
sharedDevice: !!visionDevice
|
|
2471
|
+
});
|
|
2472
|
+
return {
|
|
2473
|
+
provider: webgpu,
|
|
2474
|
+
first
|
|
2475
|
+
};
|
|
2476
|
+
} catch (err) {
|
|
2477
|
+
if (backend === "webgpu") throw err;
|
|
2478
|
+
reportBackend({
|
|
2479
|
+
backend: "cpu",
|
|
2480
|
+
sharedDevice: false,
|
|
2481
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
2482
|
+
});
|
|
2483
|
+
return { provider: new NodeSessionProvider() };
|
|
1611
2484
|
}
|
|
1612
|
-
return this._resolved.createSession(modelBytes);
|
|
1613
2485
|
}
|
|
1614
2486
|
};
|
|
2487
|
+
let processProvider;
|
|
1615
2488
|
/**
|
|
1616
2489
|
* The provider for a runner created without one: GPU if possible, otherwise CPU.
|
|
2490
|
+
* One per process, so the GPU probe runs once however many runners there are.
|
|
1617
2491
|
*/
|
|
1618
2492
|
function createDefaultSessionProvider() {
|
|
1619
|
-
|
|
2493
|
+
if (defaultProvider) return defaultProvider();
|
|
2494
|
+
processProvider ??= new AutoSessionProvider();
|
|
2495
|
+
return processProvider;
|
|
2496
|
+
}
|
|
2497
|
+
/**
|
|
2498
|
+
* CPUs this process may use: the affinity mask, capped by a cgroup CPU quota
|
|
2499
|
+
* (containers). ONNX Runtime sizes its thread pool from the host's cores, which
|
|
2500
|
+
* oversubscribes a container limited to fewer CPUs than the host has.
|
|
2501
|
+
*/
|
|
2502
|
+
async function availableCpus() {
|
|
2503
|
+
const os$1 = await import("node:os");
|
|
2504
|
+
const cpus = typeof os$1.availableParallelism === "function" ? os$1.availableParallelism() : os$1.cpus().length;
|
|
2505
|
+
const quota = await cgroupCpuQuota();
|
|
2506
|
+
return Math.max(1, quota !== void 0 ? Math.min(cpus, quota) : cpus);
|
|
2507
|
+
}
|
|
2508
|
+
/** cgroup v2 `cpu.max` or v1 `cpu.cfs_quota_us`, in CPUs (rounded up); undefined when unlimited. */
|
|
2509
|
+
async function cgroupCpuQuota() {
|
|
2510
|
+
if (typeof process === "undefined" || process.platform !== "linux") return;
|
|
2511
|
+
const { readFile: readFile$1 } = await import("node:fs/promises");
|
|
2512
|
+
const read = (path$1) => readFile$1(path$1, "utf8").then((text) => text.trim(), () => void 0);
|
|
2513
|
+
const toCpus = (quota, period) => quota > 0 && period > 0 ? Math.ceil(quota / period) : void 0;
|
|
2514
|
+
const v2 = await read("/sys/fs/cgroup/cpu.max");
|
|
2515
|
+
if (v2 !== void 0) {
|
|
2516
|
+
const [quota, period] = v2.split(/\s+/);
|
|
2517
|
+
return quota === "max" ? void 0 : toCpus(Number(quota), Number(period));
|
|
2518
|
+
}
|
|
2519
|
+
const v1Quota = await read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us");
|
|
2520
|
+
const v1Period = await read("/sys/fs/cgroup/cpu/cpu.cfs_period_us");
|
|
2521
|
+
if (v1Quota === void 0 || v1Period === void 0) return void 0;
|
|
2522
|
+
return toCpus(Number(v1Quota), Number(v1Period));
|
|
2523
|
+
}
|
|
2524
|
+
/**
|
|
2525
|
+
* Intra-op threads for CPU sessions: `FRAMEFIELDS_VISION_THREADS` when set,
|
|
2526
|
+
* else the available CPUs on Linux (where containers cap them), else ONNX
|
|
2527
|
+
* Runtime's own default (undefined).
|
|
2528
|
+
*/
|
|
2529
|
+
async function cpuSessionThreads() {
|
|
2530
|
+
const env = typeof process !== "undefined" ? Number.parseInt(process.env.FRAMEFIELDS_VISION_THREADS ?? "", 10) : NaN;
|
|
2531
|
+
if (Number.isFinite(env) && env > 0) return env;
|
|
2532
|
+
if (typeof process === "undefined" || process.platform !== "linux") return;
|
|
2533
|
+
return availableCpus();
|
|
1620
2534
|
}
|
|
1621
2535
|
/** onnxruntime-node provider. The native module is imported lazily on first session. */
|
|
1622
2536
|
var NodeSessionProvider = class {
|
|
1623
2537
|
kind = "node";
|
|
2538
|
+
_executionProviders;
|
|
2539
|
+
constructor(options = {}) {
|
|
2540
|
+
this._executionProviders = options.executionProviders ?? ["cpu"];
|
|
2541
|
+
}
|
|
1624
2542
|
ortPromise;
|
|
1625
2543
|
async ort() {
|
|
1626
2544
|
if (!this.ortPromise) this.ortPromise = import("onnxruntime-node").catch((err) => {
|
|
@@ -1631,9 +2549,14 @@ var NodeSessionProvider = class {
|
|
|
1631
2549
|
}
|
|
1632
2550
|
async createSession(modelBytes) {
|
|
1633
2551
|
const ort = await this.ort();
|
|
2552
|
+
const threads = await cpuSessionThreads();
|
|
1634
2553
|
const session = await ort.InferenceSession.create(modelBytes, {
|
|
1635
|
-
executionProviders: [
|
|
1636
|
-
graphOptimizationLevel: "all"
|
|
2554
|
+
executionProviders: [...this._executionProviders],
|
|
2555
|
+
graphOptimizationLevel: "all",
|
|
2556
|
+
...threads !== void 0 ? {
|
|
2557
|
+
intraOpNumThreads: threads,
|
|
2558
|
+
interOpNumThreads: 1
|
|
2559
|
+
} : {}
|
|
1637
2560
|
});
|
|
1638
2561
|
return {
|
|
1639
2562
|
run: async (input) => {
|
|
@@ -1652,6 +2575,10 @@ var NodeSessionProvider = class {
|
|
|
1652
2575
|
};
|
|
1653
2576
|
}
|
|
1654
2577
|
};
|
|
2578
|
+
function isGpuImage(image) {
|
|
2579
|
+
return image.texture !== void 0;
|
|
2580
|
+
}
|
|
2581
|
+
const MAX_INSTANCE_PASSES = 4;
|
|
1655
2582
|
/**
|
|
1656
2583
|
* Lazy vision runner.
|
|
1657
2584
|
*
|
|
@@ -1666,38 +2593,58 @@ var VisionRunner = class VisionRunner$1 {
|
|
|
1666
2593
|
classes;
|
|
1667
2594
|
maskThreshold;
|
|
1668
2595
|
featherRadius;
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
/** One RTMDet-Ins pass per frame buffer, shared by detect() and segment(). */
|
|
1674
|
-
_instancePass;
|
|
1675
|
-
constructor(options) {
|
|
2596
|
+
_core;
|
|
2597
|
+
/** False for derived runners: their sessions belong to the runner they came from. */
|
|
2598
|
+
_ownsCore;
|
|
2599
|
+
constructor(options, core) {
|
|
1676
2600
|
this.variant = options.variant ?? "s";
|
|
1677
2601
|
this.confidence = options.confidence ?? .3;
|
|
1678
2602
|
this.classes = options.classes;
|
|
1679
2603
|
this.maskThreshold = options.maskThreshold ?? .5;
|
|
1680
2604
|
this.featherRadius = options.featherRadius;
|
|
1681
|
-
this.
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
2605
|
+
this._ownsCore = core === void 0;
|
|
2606
|
+
this._core = core ?? {
|
|
2607
|
+
store: options.store ?? new VisionModelStore({
|
|
2608
|
+
modelsDir: options.modelsDir,
|
|
2609
|
+
baseUrl: options.baseUrl,
|
|
2610
|
+
timeoutMs: options.timeoutMs,
|
|
2611
|
+
onProgress: options.onProgress
|
|
2612
|
+
}),
|
|
2613
|
+
provider: options.provider ?? createDefaultSessionProvider(),
|
|
2614
|
+
sessions: /* @__PURE__ */ new Map(),
|
|
2615
|
+
tensorScratch: /* @__PURE__ */ new Map(),
|
|
2616
|
+
runQueue: /* @__PURE__ */ new Map(),
|
|
2617
|
+
gpuInputs: /* @__PURE__ */ new Map(),
|
|
2618
|
+
instancePasses: []
|
|
2619
|
+
};
|
|
1688
2620
|
}
|
|
1689
2621
|
/** Cheap: no downloads, no sessions, no side effects. */
|
|
1690
2622
|
static create(options = {}) {
|
|
1691
2623
|
return new VisionRunner$1(options);
|
|
1692
2624
|
}
|
|
2625
|
+
/**
|
|
2626
|
+
* A runner with other decode options (confidence, classes, mask threshold,
|
|
2627
|
+
* feather) that shares this runner's models, sessions and per-frame RTMDet-Ins
|
|
2628
|
+
* pass — e.g. one per layer, without loading a model twice. Its `close()` is a
|
|
2629
|
+
* no-op; closing this runner releases the shared sessions.
|
|
2630
|
+
*/
|
|
2631
|
+
derive(options) {
|
|
2632
|
+
return new VisionRunner$1({
|
|
2633
|
+
variant: this.variant,
|
|
2634
|
+
confidence: options.confidence ?? this.confidence,
|
|
2635
|
+
classes: "classes" in options ? options.classes : this.classes,
|
|
2636
|
+
maskThreshold: options.maskThreshold ?? this.maskThreshold,
|
|
2637
|
+
featherRadius: "featherRadius" in options ? options.featherRadius : this.featherRadius
|
|
2638
|
+
}, this._core);
|
|
2639
|
+
}
|
|
1693
2640
|
get modelsDir() {
|
|
1694
|
-
return this.
|
|
2641
|
+
return this._core.store.modelsDir;
|
|
1695
2642
|
}
|
|
1696
2643
|
/** Every registry model keyed to its current download state (downloaded → "ready"). */
|
|
1697
2644
|
get downloadStatus() {
|
|
1698
2645
|
const statuses = /* @__PURE__ */ new Map();
|
|
1699
2646
|
for (const key of Object.keys(VISION_MODELS)) statuses.set(key, "pending");
|
|
1700
|
-
for (const [key, status] of this.
|
|
2647
|
+
for (const [key, status] of this._core.store.statuses()) statuses.set(key, status);
|
|
1701
2648
|
return statuses;
|
|
1702
2649
|
}
|
|
1703
2650
|
/** The model serving `task` (matte depends on frame aspect; defaults to square). */
|
|
@@ -1714,7 +2661,7 @@ var VisionRunner = class VisionRunner$1 {
|
|
|
1714
2661
|
keys.add("selfie-square");
|
|
1715
2662
|
keys.add("selfie-landscape");
|
|
1716
2663
|
} else keys.add(this.modelKeyFor(task));
|
|
1717
|
-
await this.
|
|
2664
|
+
await this._core.store.preload([...keys]);
|
|
1718
2665
|
await Promise.all([...keys].map((key) => this.session(key)));
|
|
1719
2666
|
}
|
|
1720
2667
|
/** COCO-80 boxes (sorted by score, source pixels). Masks are not decoded. */
|
|
@@ -1722,7 +2669,8 @@ var VisionRunner = class VisionRunner$1 {
|
|
|
1722
2669
|
const pass = await this.instancePass(image);
|
|
1723
2670
|
return decodeRtmdetIns({
|
|
1724
2671
|
...pass.outputs,
|
|
1725
|
-
masks: void 0
|
|
2672
|
+
masks: void 0,
|
|
2673
|
+
maskPlane: void 0
|
|
1726
2674
|
}, this.instanceDecodeOptions(image, pass.transform)).detections;
|
|
1727
2675
|
}
|
|
1728
2676
|
/** COCO-80 boxes plus a frame-aligned soft mask per instance. */
|
|
@@ -1761,35 +2709,35 @@ var VisionRunner = class VisionRunner$1 {
|
|
|
1761
2709
|
});
|
|
1762
2710
|
}
|
|
1763
2711
|
close() {
|
|
1764
|
-
|
|
1765
|
-
this.
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
2712
|
+
if (!this._ownsCore) return;
|
|
2713
|
+
const core = this._core;
|
|
2714
|
+
for (const promise of core.sessions.values()) promise.then((session) => session.release()).catch(() => void 0);
|
|
2715
|
+
core.sessions.clear();
|
|
2716
|
+
core.runQueue.clear();
|
|
2717
|
+
for (const { buffer } of core.gpuInputs.values()) buffer.destroy();
|
|
2718
|
+
core.gpuInputs.clear();
|
|
2719
|
+
core.instancePasses.length = 0;
|
|
2720
|
+
core.store.clearMemoryCache();
|
|
2721
|
+
}
|
|
2722
|
+
async instancePass(image) {
|
|
2723
|
+
const passes = this._core.instancePasses;
|
|
2724
|
+
const content = isGpuImage(image) ? await FrameTensorPipeline.for(image.device).contentKey(image) : contentKey(image);
|
|
2725
|
+
const byContent = passes.find((pass$1) => pass$1.content === content);
|
|
2726
|
+
if (byContent) return byContent.result;
|
|
1771
2727
|
const key = this.modelKeyFor("segment");
|
|
1772
|
-
const result = this.infer(key, image).then(({ outputs, transform }) => {
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
dets: dets.data,
|
|
1779
|
-
labels: output(outputs, "labels", key).data,
|
|
1780
|
-
masks: masks.data,
|
|
1781
|
-
count: dets.dims[1] ?? 0,
|
|
1782
|
-
maskHeight: masks.dims[masks.dims.length - 2] ?? 0,
|
|
1783
|
-
maskWidth: masks.dims[masks.dims.length - 1] ?? 0
|
|
1784
|
-
}
|
|
1785
|
-
};
|
|
1786
|
-
});
|
|
1787
|
-
this._instancePass = {
|
|
1788
|
-
frame: image.data,
|
|
2728
|
+
const result = this.infer(key, image).then(({ outputs, transform }) => ({
|
|
2729
|
+
transform,
|
|
2730
|
+
outputs: RTMDET_INS_HEAD.scores in outputs ? headPassOutputs(outputs, key, transform) : end2endPassOutputs(outputs, key)
|
|
2731
|
+
}));
|
|
2732
|
+
const pass = {
|
|
2733
|
+
content,
|
|
1789
2734
|
result
|
|
1790
2735
|
};
|
|
2736
|
+
passes.push(pass);
|
|
2737
|
+
if (passes.length > MAX_INSTANCE_PASSES) passes.shift();
|
|
1791
2738
|
result.catch(() => {
|
|
1792
|
-
|
|
2739
|
+
const index = passes.indexOf(pass);
|
|
2740
|
+
if (index >= 0) passes.splice(index, 1);
|
|
1793
2741
|
});
|
|
1794
2742
|
return result;
|
|
1795
2743
|
}
|
|
@@ -1805,44 +2753,178 @@ var VisionRunner = class VisionRunner$1 {
|
|
|
1805
2753
|
...this.featherRadius !== void 0 ? { featherRadius: this.featherRadius } : {}
|
|
1806
2754
|
};
|
|
1807
2755
|
}
|
|
1808
|
-
|
|
2756
|
+
/**
|
|
2757
|
+
* Runs one model on one image. Runs of the same model are queued: they share
|
|
2758
|
+
* the input scratch buffer, and one run already uses the session's threads.
|
|
2759
|
+
*/
|
|
2760
|
+
infer(key, image) {
|
|
2761
|
+
const queue = this._core.runQueue;
|
|
2762
|
+
const run = (queue.get(key) ?? Promise.resolve()).then(() => this.inferNow(key, image), () => this.inferNow(key, image));
|
|
2763
|
+
const tail = run.catch(() => void 0);
|
|
2764
|
+
queue.set(key, tail);
|
|
2765
|
+
tail.then(() => {
|
|
2766
|
+
if (queue.get(key) === tail) queue.delete(key);
|
|
2767
|
+
});
|
|
2768
|
+
return run;
|
|
2769
|
+
}
|
|
2770
|
+
async inferNow(key, frame) {
|
|
1809
2771
|
const session = await this.session(key);
|
|
1810
2772
|
const desc = VISION_MODELS[key];
|
|
1811
2773
|
const spec = preprocessSpec(desc.family, desc.input);
|
|
2774
|
+
const dims = [
|
|
2775
|
+
1,
|
|
2776
|
+
3,
|
|
2777
|
+
spec.height,
|
|
2778
|
+
spec.width
|
|
2779
|
+
];
|
|
2780
|
+
let image;
|
|
2781
|
+
if (isGpuImage(frame)) {
|
|
2782
|
+
if (this._core.provider.device === frame.device) {
|
|
2783
|
+
const buffer = this.gpuInput(key, frame.device, spec);
|
|
2784
|
+
const transform$1 = FrameTensorPipeline.for(frame.device).toTensor(frame, spec, buffer);
|
|
2785
|
+
return {
|
|
2786
|
+
outputs: await session.run({
|
|
2787
|
+
name: "input",
|
|
2788
|
+
data: EMPTY_TENSOR,
|
|
2789
|
+
gpuBuffer: buffer,
|
|
2790
|
+
dims
|
|
2791
|
+
}),
|
|
2792
|
+
transform: transform$1
|
|
2793
|
+
};
|
|
2794
|
+
}
|
|
2795
|
+
image = await readTexturePixels(frame);
|
|
2796
|
+
} else image = frame;
|
|
1812
2797
|
const size = 3 * spec.width * spec.height;
|
|
1813
|
-
|
|
2798
|
+
const scratchByKey = this._core.tensorScratch;
|
|
2799
|
+
let scratch = scratchByKey.get(key);
|
|
1814
2800
|
if (!scratch || scratch.length < size) {
|
|
1815
2801
|
scratch = new Float32Array(size);
|
|
1816
|
-
|
|
2802
|
+
scratchByKey.set(key, scratch);
|
|
1817
2803
|
}
|
|
1818
2804
|
const { tensor, transform } = imageToTensor(image, spec, scratch);
|
|
1819
2805
|
return {
|
|
1820
2806
|
outputs: await session.run({
|
|
1821
2807
|
name: "input",
|
|
1822
2808
|
data: tensor,
|
|
1823
|
-
dims
|
|
1824
|
-
1,
|
|
1825
|
-
3,
|
|
1826
|
-
spec.height,
|
|
1827
|
-
spec.width
|
|
1828
|
-
]
|
|
2809
|
+
dims
|
|
1829
2810
|
}),
|
|
1830
2811
|
transform
|
|
1831
2812
|
};
|
|
1832
2813
|
}
|
|
2814
|
+
gpuInput(key, device, spec) {
|
|
2815
|
+
const inputs = this._core.gpuInputs;
|
|
2816
|
+
const size = FrameTensorPipeline.tensorBytes(spec);
|
|
2817
|
+
const current = inputs.get(key);
|
|
2818
|
+
if (current?.device === device && current.buffer.size >= size) return current.buffer;
|
|
2819
|
+
current?.buffer.destroy();
|
|
2820
|
+
const buffer = device.createBuffer({
|
|
2821
|
+
label: `vision_input_${key}`,
|
|
2822
|
+
size,
|
|
2823
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
|
|
2824
|
+
});
|
|
2825
|
+
inputs.set(key, {
|
|
2826
|
+
device,
|
|
2827
|
+
buffer
|
|
2828
|
+
});
|
|
2829
|
+
return buffer;
|
|
2830
|
+
}
|
|
1833
2831
|
session(key) {
|
|
1834
|
-
|
|
2832
|
+
const { sessions, store, provider } = this._core;
|
|
2833
|
+
let promise = sessions.get(key);
|
|
1835
2834
|
if (!promise) {
|
|
1836
|
-
promise =
|
|
1837
|
-
|
|
1838
|
-
|
|
2835
|
+
promise = store.ensure(key).then((bytes) => provider.createSession(VISION_MODELS[key].family === "rtmdet-ins" ? rtmdetInsBody(bytes) : bytes)).catch((err) => {
|
|
2836
|
+
sessions.delete(key);
|
|
2837
|
+
store.evict(key);
|
|
1839
2838
|
throw err;
|
|
1840
2839
|
});
|
|
1841
|
-
|
|
2840
|
+
sessions.set(key, promise);
|
|
1842
2841
|
}
|
|
1843
2842
|
return promise;
|
|
1844
2843
|
}
|
|
1845
2844
|
};
|
|
2845
|
+
const EMPTY_TENSOR = new Float32Array(0);
|
|
2846
|
+
/**
|
|
2847
|
+
* The RTMDet-Ins graph up to its head (see `RTMDET_INS_HEAD`): no NMS or
|
|
2848
|
+
* full-size masks in the graph, so the GPU part has fixed shapes and only
|
|
2849
|
+
* ~9 MB comes back. Models without those tensors run end to end.
|
|
2850
|
+
*/
|
|
2851
|
+
function rtmdetInsBody(bytes) {
|
|
2852
|
+
try {
|
|
2853
|
+
return extractSubgraph(bytes, Object.values(RTMDET_INS_HEAD));
|
|
2854
|
+
} catch {
|
|
2855
|
+
return bytes;
|
|
2856
|
+
}
|
|
2857
|
+
}
|
|
2858
|
+
/** Outputs of the end2end graph: NMS done, one input-sized mask per detection. */
|
|
2859
|
+
function end2endPassOutputs(outputs, key) {
|
|
2860
|
+
const dets = output(outputs, "dets", key);
|
|
2861
|
+
const masks = output(outputs, "masks", key);
|
|
2862
|
+
return {
|
|
2863
|
+
dets: dets.data,
|
|
2864
|
+
labels: output(outputs, "labels", key).data,
|
|
2865
|
+
masks: masks.data,
|
|
2866
|
+
count: dets.dims[1] ?? 0,
|
|
2867
|
+
maskHeight: masks.dims[masks.dims.length - 2] ?? 0,
|
|
2868
|
+
maskWidth: masks.dims[masks.dims.length - 1] ?? 0
|
|
2869
|
+
};
|
|
2870
|
+
}
|
|
2871
|
+
/** Outputs of the head: selection here, mask logits only for detections asked for. */
|
|
2872
|
+
function headPassOutputs(outputs, key, transform) {
|
|
2873
|
+
const scores = output(outputs, RTMDET_INS_HEAD.scores, key);
|
|
2874
|
+
const features = output(outputs, RTMDET_INS_HEAD.maskFeatures, key);
|
|
2875
|
+
const head = {
|
|
2876
|
+
scores: scores.data,
|
|
2877
|
+
boxes: output(outputs, RTMDET_INS_HEAD.boxes, key).data,
|
|
2878
|
+
kernels: output(outputs, RTMDET_INS_HEAD.kernels, key).data,
|
|
2879
|
+
maskFeatures: features.data,
|
|
2880
|
+
anchors: scores.dims[1] ?? 0,
|
|
2881
|
+
classes: scores.dims[2] ?? 0,
|
|
2882
|
+
maskHeight: features.dims[2] ?? 0,
|
|
2883
|
+
maskWidth: features.dims[3] ?? 0,
|
|
2884
|
+
inputWidth: transform.inputWidth,
|
|
2885
|
+
inputHeight: transform.inputHeight
|
|
2886
|
+
};
|
|
2887
|
+
const picked = selectInstances(head);
|
|
2888
|
+
const dets = new Float32Array(picked.length * 5);
|
|
2889
|
+
const labels = new Int32Array(picked.length);
|
|
2890
|
+
picked.forEach(({ anchor, classIndex, score }, i) => {
|
|
2891
|
+
dets.set(head.boxes.subarray(anchor * 4, anchor * 4 + 4), i * 5);
|
|
2892
|
+
dets[i * 5 + 4] = score;
|
|
2893
|
+
labels[i] = classIndex;
|
|
2894
|
+
});
|
|
2895
|
+
const planes = /* @__PURE__ */ new Map();
|
|
2896
|
+
return {
|
|
2897
|
+
dets,
|
|
2898
|
+
labels,
|
|
2899
|
+
count: picked.length,
|
|
2900
|
+
maskWidth: head.maskWidth,
|
|
2901
|
+
maskHeight: head.maskHeight,
|
|
2902
|
+
maskLogits: true,
|
|
2903
|
+
maskPlane: (index) => {
|
|
2904
|
+
let plane = planes.get(index);
|
|
2905
|
+
if (!plane) {
|
|
2906
|
+
plane = maskLogits(head, picked[index].anchor);
|
|
2907
|
+
planes.set(index, plane);
|
|
2908
|
+
}
|
|
2909
|
+
return plane;
|
|
2910
|
+
}
|
|
2911
|
+
};
|
|
2912
|
+
}
|
|
2913
|
+
/** Size plus a 64-bit hash of the pixels: equal keys mean the same picture. */
|
|
2914
|
+
function contentKey(image) {
|
|
2915
|
+
const { data } = image;
|
|
2916
|
+
const words = data.byteOffset % 4 === 0 ? data.byteLength >>> 2 : 0;
|
|
2917
|
+
const view = new Uint32Array(data.buffer, data.byteOffset, words);
|
|
2918
|
+
let h1 = 2166136261;
|
|
2919
|
+
let h2 = 16777619 ^ data.byteLength;
|
|
2920
|
+
for (let i = 0; i < words; i++) {
|
|
2921
|
+
const v = view[i];
|
|
2922
|
+
h1 = Math.imul(h1 ^ v, 2654435761);
|
|
2923
|
+
h2 = Math.imul(h2 ^ v, 2246822519) ^ h1 >>> 15;
|
|
2924
|
+
}
|
|
2925
|
+
for (let i = words * 4; i < data.length; i++) h1 = Math.imul(h1 ^ data[i], 2654435761);
|
|
2926
|
+
return `${image.width}x${image.height}:${(h1 >>> 0).toString(36)}:${(h2 >>> 0).toString(36)}`;
|
|
2927
|
+
}
|
|
1846
2928
|
function output(outputs, name, key) {
|
|
1847
2929
|
const tensor = outputs[name];
|
|
1848
2930
|
if (!tensor) throw new Error(`Vision model '${key}' returned no '${name}' output (got: ${Object.keys(outputs).join(", ") || "none"}). Expected the ${VISION_MODELS[key].source}.`);
|
|
@@ -2034,16 +3116,16 @@ function matchPoseToTracks(people, trackedObjects, minIoU = .15) {
|
|
|
2034
3116
|
const pBox = getPersonBoundingBox(person);
|
|
2035
3117
|
const isNormalized = pBox.width <= 1.05 && pBox.height <= 1.05;
|
|
2036
3118
|
for (const trk of personTracks) {
|
|
2037
|
-
const iou = computeIoU(pBox, isNormalized && trk.boundingBox.normalizedWidth > 0 ? {
|
|
3119
|
+
const iou$1 = computeIoU(pBox, isNormalized && trk.boundingBox.normalizedWidth > 0 ? {
|
|
2038
3120
|
originX: trk.boundingBox.normalizedX,
|
|
2039
3121
|
originY: trk.boundingBox.normalizedY,
|
|
2040
3122
|
width: trk.boundingBox.normalizedWidth,
|
|
2041
3123
|
height: trk.boundingBox.normalizedHeight
|
|
2042
3124
|
} : trk.boundingBox);
|
|
2043
|
-
if (iou >= minIoU) candidateMatches.push({
|
|
3125
|
+
if (iou$1 >= minIoU) candidateMatches.push({
|
|
2044
3126
|
personIdx: p,
|
|
2045
3127
|
trackId: trk.trackId,
|
|
2046
|
-
iou
|
|
3128
|
+
iou: iou$1
|
|
2047
3129
|
});
|
|
2048
3130
|
}
|
|
2049
3131
|
}
|
|
@@ -2244,14 +3326,14 @@ var VisionBundle = class {
|
|
|
2244
3326
|
let bestIoU = .1;
|
|
2245
3327
|
for (const p of poseRes.people) {
|
|
2246
3328
|
const pBox = getPersonBoundingBox(p);
|
|
2247
|
-
const iou = computeIoU(pBox, pBox.width <= 1.05 && pBox.height <= 1.05 && trk.boundingBox.normalizedWidth > 0 ? {
|
|
3329
|
+
const iou$1 = computeIoU(pBox, pBox.width <= 1.05 && pBox.height <= 1.05 && trk.boundingBox.normalizedWidth > 0 ? {
|
|
2248
3330
|
originX: trk.boundingBox.normalizedX,
|
|
2249
3331
|
originY: trk.boundingBox.normalizedY,
|
|
2250
3332
|
width: trk.boundingBox.normalizedWidth,
|
|
2251
3333
|
height: trk.boundingBox.normalizedHeight
|
|
2252
3334
|
} : trk.boundingBox);
|
|
2253
|
-
if (iou > bestIoU) {
|
|
2254
|
-
bestIoU = iou;
|
|
3335
|
+
if (iou$1 > bestIoU) {
|
|
3336
|
+
bestIoU = iou$1;
|
|
2255
3337
|
bestPerson = p;
|
|
2256
3338
|
}
|
|
2257
3339
|
}
|
|
@@ -3014,7 +4096,7 @@ var VisionNode = class VisionNode$1 {
|
|
|
3014
4096
|
};
|
|
3015
4097
|
|
|
3016
4098
|
//#endregion
|
|
3017
|
-
//#region ../../nodes/node-vision/dist/renderers-
|
|
4099
|
+
//#region ../../nodes/node-vision/dist/renderers-CPLmY2dA.mjs
|
|
3018
4100
|
let scratchValues = new Uint8Array(0);
|
|
3019
4101
|
let scratchLengths = new Uint32Array(0);
|
|
3020
4102
|
function encodeMask(mask, width, height) {
|
|
@@ -3221,7 +4303,7 @@ var MatteCompositor = class {
|
|
|
3221
4303
|
return tex;
|
|
3222
4304
|
}
|
|
3223
4305
|
/** Clears `target` and draws `picture` through `mask` into it. */
|
|
3224
|
-
draw(encoder, target, picture, mask, mode, bounds) {
|
|
4306
|
+
draw(encoder$1, target, picture, mask, mode, bounds) {
|
|
3225
4307
|
const w = mask.width;
|
|
3226
4308
|
const h = mask.height;
|
|
3227
4309
|
const data = new Float32Array(8);
|
|
@@ -3267,7 +4349,7 @@ var MatteCompositor = class {
|
|
|
3267
4349
|
}
|
|
3268
4350
|
]
|
|
3269
4351
|
});
|
|
3270
|
-
const pass = encoder.beginRenderPass({
|
|
4352
|
+
const pass = encoder$1.beginRenderPass({
|
|
3271
4353
|
label: "vision_matte_composite",
|
|
3272
4354
|
colorAttachments: [{
|
|
3273
4355
|
view: target,
|
|
@@ -3287,7 +4369,13 @@ var MatteCompositor = class {
|
|
|
3287
4369
|
pass.end();
|
|
3288
4370
|
}
|
|
3289
4371
|
};
|
|
3290
|
-
|
|
4372
|
+
/**
|
|
4373
|
+
* Runners by model options (variant, models dir, mirror), each with the runners
|
|
4374
|
+
* derived from it by decode options — layers that differ only in classes or
|
|
4375
|
+
* confidence share sessions instead of reloading the models.
|
|
4376
|
+
*/
|
|
4377
|
+
const sharedRunners = /* @__PURE__ */ new Map();
|
|
4378
|
+
const MAX_SHARED_RUNNERS = 4;
|
|
3291
4379
|
let sharedSkeletonRenderer = null;
|
|
3292
4380
|
let sharedCompositor = null;
|
|
3293
4381
|
let sharedCompositorFormat = null;
|
|
@@ -3295,8 +4383,9 @@ let sharedCompositorFormat = null;
|
|
|
3295
4383
|
const frameCache = new VisionFrameCache();
|
|
3296
4384
|
/** The frame each node's child texture holds (drawn on its last call). */
|
|
3297
4385
|
const heldFrameByNode = /* @__PURE__ */ new Map();
|
|
3298
|
-
/** Per node: a copy of the held frame, and the composited matte. */
|
|
4386
|
+
/** Per node: a copy of the held frame, the frame being analysed, and the composited matte. */
|
|
3299
4387
|
const heldTextures = /* @__PURE__ */ new Map();
|
|
4388
|
+
const analysisTextures = /* @__PURE__ */ new Map();
|
|
3300
4389
|
const matteTextures = /* @__PURE__ */ new Map();
|
|
3301
4390
|
/** One tracker per vision node — track ids must not mix across nodes/sources. */
|
|
3302
4391
|
const objectTrackers = /* @__PURE__ */ new Map();
|
|
@@ -3335,6 +4424,7 @@ function nodeTexture(textures, device, key, like, usage, label) {
|
|
|
3335
4424
|
return tex;
|
|
3336
4425
|
}
|
|
3337
4426
|
const heldTexture = (device, key, like) => nodeTexture(heldTextures, device, key, like, GPUTextureUsage.COPY_DST | GPUTextureUsage.TEXTURE_BINDING, "vision_held_frame");
|
|
4427
|
+
const analysisTexture = (device, key, like) => nodeTexture(analysisTextures, device, key, like, GPUTextureUsage.COPY_DST | GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_SRC, "vision_analysis_frame");
|
|
3338
4428
|
const matteTexture = (device, key, like) => nodeTexture(matteTextures, device, key, like, GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_SRC, "vision_matte");
|
|
3339
4429
|
/** Everything that decides a frame's results besides the frame itself. */
|
|
3340
4430
|
function cacheSource(nodeKey, child, op, width, height) {
|
|
@@ -3384,27 +4474,42 @@ function selectSubject(selected, nodeKey, frame) {
|
|
|
3384
4474
|
/**
|
|
3385
4475
|
* Lazy shared runner — `create()` is a pure constructor (zero I/O); the first frame that
|
|
3386
4476
|
* requests a task triggers that task's model download at inference time (never at init).
|
|
3387
|
-
*
|
|
4477
|
+
* Nodes with the same model options share sessions; decode options (classes,
|
|
4478
|
+
* confidence, …) get a derived runner over the same sessions.
|
|
3388
4479
|
*/
|
|
3389
4480
|
function getSharedRunner(op) {
|
|
3390
|
-
const
|
|
4481
|
+
const modelOptions = {
|
|
3391
4482
|
variant: op.variant,
|
|
3392
|
-
confidence: op.confidence,
|
|
3393
|
-
classes: op.classes,
|
|
3394
|
-
maskThreshold: op.maskThreshold,
|
|
3395
|
-
featherRadius: op.featherRadius,
|
|
3396
4483
|
modelsDir: op.modelsDir,
|
|
3397
4484
|
baseUrl: op.baseUrl
|
|
3398
4485
|
};
|
|
3399
|
-
const
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
runner
|
|
4486
|
+
const modelKey = JSON.stringify(modelOptions);
|
|
4487
|
+
let entry = sharedRunners.get(modelKey);
|
|
4488
|
+
if (!entry) {
|
|
4489
|
+
if (sharedRunners.size >= MAX_SHARED_RUNNERS) {
|
|
4490
|
+
const [oldestKey, oldest] = sharedRunners.entries().next().value;
|
|
4491
|
+
oldest.runner.close();
|
|
4492
|
+
sharedRunners.delete(oldestKey);
|
|
4493
|
+
}
|
|
4494
|
+
entry = {
|
|
4495
|
+
runner: VisionRunner.create(modelOptions),
|
|
4496
|
+
derived: /* @__PURE__ */ new Map()
|
|
3405
4497
|
};
|
|
4498
|
+
sharedRunners.set(modelKey, entry);
|
|
4499
|
+
}
|
|
4500
|
+
const decodeOptions = {
|
|
4501
|
+
confidence: op.confidence,
|
|
4502
|
+
classes: op.classes,
|
|
4503
|
+
maskThreshold: op.maskThreshold,
|
|
4504
|
+
featherRadius: op.featherRadius
|
|
4505
|
+
};
|
|
4506
|
+
const decodeKey = JSON.stringify(decodeOptions);
|
|
4507
|
+
let runner = entry.derived.get(decodeKey);
|
|
4508
|
+
if (!runner) {
|
|
4509
|
+
runner = entry.runner.derive(decodeOptions);
|
|
4510
|
+
entry.derived.set(decodeKey, runner);
|
|
3406
4511
|
}
|
|
3407
|
-
return
|
|
4512
|
+
return runner;
|
|
3408
4513
|
}
|
|
3409
4514
|
function getObjectTracker(nodeKey, op) {
|
|
3410
4515
|
let tracker = objectTrackers.get(nodeKey);
|
|
@@ -3621,8 +4726,9 @@ function maskCentroid(mask) {
|
|
|
3621
4726
|
};
|
|
3622
4727
|
}
|
|
3623
4728
|
const VisionWebGPURenderer = async (args) => {
|
|
3624
|
-
const { ctx, encoder, pass, targetView, targetWidth, targetHeight, props, drawChild } = args;
|
|
4729
|
+
const { ctx, encoder: encoder$1, pass, targetView, targetWidth, targetHeight, props, drawChild } = args;
|
|
3625
4730
|
const { virtualMedia } = props;
|
|
4731
|
+
setVisionDevice(ctx.device);
|
|
3626
4732
|
const rawOp = virtualMedia?.operation;
|
|
3627
4733
|
if (rawOp?.op !== "Vision") return;
|
|
3628
4734
|
const op = rawOp;
|
|
@@ -3670,11 +4776,11 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3670
4776
|
const wantsPose = op.enablePose === true || mode === "skeleton";
|
|
3671
4777
|
const cacheable = !wantsSelfie && !wantsPose && (wantsDetection || wantsSegmentation);
|
|
3672
4778
|
const source = cacheable ? cacheSource(nodeKeyStr, childMedia, op, targetWidth, targetHeight) : "";
|
|
3673
|
-
const keyThresholdAt = (frame) => op.keyBackground === true ? backgroundKeyThreshold(op, compositionFrame - (frameIdx - frame), fps) : void 0;
|
|
3674
|
-
const lookup = (frame) => {
|
|
3675
|
-
if (!cacheable || frame === void 0) return void 0;
|
|
3676
|
-
const hit = frameCache.get(source, frame);
|
|
3677
|
-
if (!hit || hit.keyThreshold !== keyThresholdAt(frame)) return void 0;
|
|
4779
|
+
const keyThresholdAt = (frame$1) => op.keyBackground === true ? backgroundKeyThreshold(op, compositionFrame - (frameIdx - frame$1), fps) : void 0;
|
|
4780
|
+
const lookup = (frame$1) => {
|
|
4781
|
+
if (!cacheable || frame$1 === void 0) return void 0;
|
|
4782
|
+
const hit = frameCache.get(source, frame$1);
|
|
4783
|
+
if (!hit || hit.keyThreshold !== keyThresholdAt(frame$1)) return void 0;
|
|
3678
4784
|
if (wantsSegmentation && visionBundle && !hit.masks) return void 0;
|
|
3679
4785
|
return hit;
|
|
3680
4786
|
};
|
|
@@ -3685,43 +4791,33 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3685
4791
|
let heldTex;
|
|
3686
4792
|
if (isMatteMode && hasPreviousFrame && !exact) {
|
|
3687
4793
|
heldTex = heldTexture(ctx.device, nodeKeyStr, childTex);
|
|
3688
|
-
encoder.copyTextureToTexture({ texture: childTex }, { texture: heldTex }, [
|
|
4794
|
+
encoder$1.copyTextureToTexture({ texture: childTex }, { texture: heldTex }, [
|
|
3689
4795
|
targetWidth,
|
|
3690
4796
|
targetHeight,
|
|
3691
4797
|
1
|
|
3692
4798
|
]);
|
|
3693
4799
|
}
|
|
4800
|
+
let frame;
|
|
3694
4801
|
let framePixels = null;
|
|
3695
4802
|
if (hasPreviousFrame && !cached) {
|
|
3696
|
-
const
|
|
3697
|
-
const
|
|
3698
|
-
|
|
3699
|
-
const stagingBuffer = ctx.device.createBuffer({
|
|
3700
|
-
size: bufferSize,
|
|
3701
|
-
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3702
|
-
label: "vision_frame_staging"
|
|
3703
|
-
});
|
|
3704
|
-
const readbackEncoder = ctx.device.createCommandEncoder({ label: "vision_readback_encoder" });
|
|
3705
|
-
readbackEncoder.copyTextureToBuffer({ texture: childTex }, {
|
|
3706
|
-
buffer: stagingBuffer,
|
|
3707
|
-
bytesPerRow,
|
|
3708
|
-
rowsPerImage: targetHeight
|
|
3709
|
-
}, [
|
|
4803
|
+
const analysis = analysisTexture(ctx.device, nodeKeyStr, childTex);
|
|
4804
|
+
const copy = ctx.device.createCommandEncoder({ label: "vision_analysis_copy" });
|
|
4805
|
+
copy.copyTextureToTexture({ texture: childTex }, { texture: analysis }, [
|
|
3710
4806
|
targetWidth,
|
|
3711
4807
|
targetHeight,
|
|
3712
4808
|
1
|
|
3713
4809
|
]);
|
|
3714
|
-
ctx.device.queue.submit([
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
4810
|
+
ctx.device.queue.submit([copy.finish()]);
|
|
4811
|
+
frame = {
|
|
4812
|
+
texture: analysis,
|
|
4813
|
+
device: ctx.device,
|
|
4814
|
+
width: targetWidth,
|
|
4815
|
+
height: targetHeight
|
|
4816
|
+
};
|
|
4817
|
+
if (isMatteMode && op.keyBackground === true) framePixels = (await readTexturePixels(frame)).data;
|
|
3722
4818
|
}
|
|
3723
4819
|
const childView = childTex.createView();
|
|
3724
|
-
ctx.renderer.beginFrame(encoder, childView, {
|
|
4820
|
+
ctx.renderer.beginFrame(encoder$1, childView, {
|
|
3725
4821
|
r: 0,
|
|
3726
4822
|
g: 0,
|
|
3727
4823
|
b: 0,
|
|
@@ -3751,13 +4847,9 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3751
4847
|
frame: frameIdx
|
|
3752
4848
|
});
|
|
3753
4849
|
}
|
|
3754
|
-
} else if (
|
|
4850
|
+
} else if (frame) {
|
|
3755
4851
|
const runner = getSharedRunner(op);
|
|
3756
|
-
const image =
|
|
3757
|
-
data: framePixels,
|
|
3758
|
-
width: targetWidth,
|
|
3759
|
-
height: targetHeight
|
|
3760
|
-
};
|
|
4852
|
+
const image = frame;
|
|
3761
4853
|
let detections = [];
|
|
3762
4854
|
if (wantsDetection) {
|
|
3763
4855
|
detections = await runner.detect(image);
|
|
@@ -3784,7 +4876,7 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3784
4876
|
if (isMatteMode) {
|
|
3785
4877
|
subject = selectSubject(op.matteSource === "selfie" ? personMatteAsMask(personMatte) : mergeSubjectMask(frameMasks.length > 0 ? frameMasks : visionBundle?.getMaskResult(frameIdx) ?? []), nodeKeyStr, frameIdx);
|
|
3786
4878
|
const threshold = keyThresholdAt(heldFrame ?? frameIdx);
|
|
3787
|
-
if (subject && threshold !== void 0) subject = {
|
|
4879
|
+
if (subject && threshold !== void 0 && framePixels) subject = {
|
|
3788
4880
|
...subject,
|
|
3789
4881
|
mask: growMaskIntoForeground(subject.mask, subject.width, subject.height, framePixels, threshold, op.featherRadius)
|
|
3790
4882
|
};
|
|
@@ -3800,7 +4892,7 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3800
4892
|
if (isMatteMode) {
|
|
3801
4893
|
const picture = exact ? childTex : heldTex;
|
|
3802
4894
|
if (!subject || !picture) {
|
|
3803
|
-
ctx.renderer.beginFrame(encoder, targetView, {
|
|
4895
|
+
ctx.renderer.beginFrame(encoder$1, targetView, {
|
|
3804
4896
|
r: 0,
|
|
3805
4897
|
g: 0,
|
|
3806
4898
|
b: 0,
|
|
@@ -3814,9 +4906,9 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3814
4906
|
}
|
|
3815
4907
|
const maskTex = sharedCompositor.uploadMask(nodeKeyStr, subject.mask, subject.width, subject.height);
|
|
3816
4908
|
const matteTex = matteTexture(ctx.device, nodeKeyStr, childTex);
|
|
3817
|
-
sharedCompositor.draw(encoder, matteTex.createView(), picture, maskTex, mode, mode === "crop" ? maskBounds(subject) ?? void 0 : void 0);
|
|
4909
|
+
sharedCompositor.draw(encoder$1, matteTex.createView(), picture, maskTex, mode, mode === "crop" ? maskBounds(subject) ?? void 0 : void 0);
|
|
3818
4910
|
visionBundle?.setStencilTexture(matteTex);
|
|
3819
|
-
const outPass$1 = ctx.renderer.beginFrame(encoder, targetView, {
|
|
4911
|
+
const outPass$1 = ctx.renderer.beginFrame(encoder$1, targetView, {
|
|
3820
4912
|
r: 0,
|
|
3821
4913
|
g: 0,
|
|
3822
4914
|
b: 0,
|
|
@@ -3839,7 +4931,7 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3839
4931
|
width: targetWidth,
|
|
3840
4932
|
height: targetHeight
|
|
3841
4933
|
});
|
|
3842
|
-
const skelPass = ctx.renderer.beginFrame(encoder, targetView, {
|
|
4934
|
+
const skelPass = ctx.renderer.beginFrame(encoder$1, targetView, {
|
|
3843
4935
|
r: 0,
|
|
3844
4936
|
g: 0,
|
|
3845
4937
|
b: 0,
|
|
@@ -3856,7 +4948,7 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3856
4948
|
}
|
|
3857
4949
|
}
|
|
3858
4950
|
if (mode === "boxes" || mode === "tracking") {
|
|
3859
|
-
const outPass$1 = ctx.renderer.beginFrame(encoder, targetView, {
|
|
4951
|
+
const outPass$1 = ctx.renderer.beginFrame(encoder$1, targetView, {
|
|
3860
4952
|
r: 0,
|
|
3861
4953
|
g: 0,
|
|
3862
4954
|
b: 0,
|
|
@@ -3923,7 +5015,7 @@ const VisionWebGPURenderer = async (args) => {
|
|
|
3923
5015
|
outPass$1.end();
|
|
3924
5016
|
return;
|
|
3925
5017
|
}
|
|
3926
|
-
const outPass = ctx.renderer.beginFrame(encoder, targetView, {
|
|
5018
|
+
const outPass = ctx.renderer.beginFrame(encoder$1, targetView, {
|
|
3927
5019
|
r: 0,
|
|
3928
5020
|
g: 0,
|
|
3929
5021
|
b: 0,
|
|
@@ -3955,4 +5047,4 @@ var renderers_default = defineRenderer({ WebGPURenderer: VisionWebGPURenderer })
|
|
|
3955
5047
|
|
|
3956
5048
|
//#endregion
|
|
3957
5049
|
export { PoseSkeletonRenderer as a, TemporalObjectTracker as c, VisionRunner as d, analyzeSequence as f, COCO_CLASSES as h, COCO17_KEYPOINT_NAMES as i, VisionBundle as l, pinNodeToObject as m, COCO17_BONES as n, SegmentationTexturePool as o, pinNodeToLandmark as p, COCO17_KEYPOINTS as r, SpatialLandmarkTransformer as s, renderers_default as t, VisionNode as u };
|
|
3958
|
-
//# sourceMappingURL=renderer-
|
|
5050
|
+
//# sourceMappingURL=renderer-CqT2vxa4.mjs.map
|