@lumy-pack/scene-sieve 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -24
- package/dist/{errors.d.ts → cli/errors/classify-error.d.ts} +5 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/{utils → cli/options}/parse-options.d.ts +6 -1
- package/dist/cli.mjs +755 -319
- package/dist/constants/pipeline-defaults.d.ts +13 -0
- package/dist/core/{analyzer.d.ts → analyzer/analyzer.d.ts} +11 -6
- package/dist/core/{dbscan.d.ts → analyzer/clustering/dbscan.d.ts} +1 -1
- package/dist/core/analyzer/constants/vision-tuning.d.ts +12 -0
- package/dist/core/analyzer/features/feature-diff.d.ts +14 -0
- package/dist/core/analyzer/features/frame-features.d.ts +32 -0
- package/dist/core/analyzer/index.d.ts +3 -0
- package/dist/core/constants/workspace-layout.d.ts +9 -0
- package/dist/core/{extractor.d.ts → extractor/extractor.d.ts} +6 -4
- package/dist/core/extractor/index.d.ts +1 -0
- package/dist/core/index.d.ts +8 -9
- package/dist/core/input-resolver/index.d.ts +1 -0
- package/dist/core/{input-resolver.d.ts → input-resolver/input-resolver.d.ts} +11 -1
- package/dist/core/input-resolver/validation/validate-options.d.ts +8 -0
- package/dist/core/orchestrator/index.d.ts +3 -0
- package/dist/core/{orchestrator.d.ts → orchestrator/orchestrator.d.ts} +1 -1
- package/dist/core/{run-in-worker.d.ts → orchestrator/worker/run-in-worker.d.ts} +5 -1
- package/dist/core/pruner/index.d.ts +1 -0
- package/dist/core/{pruner.d.ts → pruner/pruner.d.ts} +2 -2
- package/dist/{utils/math.d.ts → core/pruner/scoring/normalize-scores.d.ts} +4 -0
- package/dist/core/segmenter/index.d.ts +1 -0
- package/dist/core/{segmenter.d.ts → segmenter/segmenter.d.ts} +11 -11
- package/dist/core/utils/metadata/build-video-metadata.d.ts +14 -0
- package/dist/core/workspace/index.d.ts +1 -0
- package/dist/core/{workspace.d.ts → workspace/workspace.d.ts} +1 -1
- package/dist/index.cjs +910 -434
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +913 -434
- package/dist/pipeline-worker.mjs +635 -278
- package/dist/types/index.d.ts +25 -0
- package/package.json +1 -1
- package/dist/constants.d.ts +0 -32
- /package/dist/{commands → cli/commands}/Sieve.d.ts +0 -0
- /package/dist/{utils → cli/commands}/command-registry.d.ts +0 -0
- /package/dist/{components → cli/components}/PhaseStep.d.ts +0 -0
- /package/dist/{components → cli/components}/ProgressBar.d.ts +0 -0
- /package/dist/core/{pipeline-worker.d.ts → orchestrator/worker/pipeline-worker.d.ts} +0 -0
- /package/dist/{utils → core/pruner/heap}/min-heap.d.ts +0 -0
- /package/dist/{utils → core/segmenter/scheduling}/concurrency.d.ts +0 -0
- /package/dist/{utils → core/utils/filesystem}/paths.d.ts +0 -0
- /package/dist/{utils → logging}/logger.d.ts +0 -0
package/dist/pipeline-worker.mjs
CHANGED
|
@@ -4,14 +4,14 @@ import { randomUUID } from "node:crypto";
|
|
|
4
4
|
import { filter, map } from "@winglet/common-utils";
|
|
5
5
|
import pc from "picocolors";
|
|
6
6
|
import sharp from "sharp";
|
|
7
|
-
import { homedir, tmpdir } from "node:os";
|
|
8
|
-
import { basename, extname, join, resolve } from "node:path";
|
|
9
7
|
import { mkdir, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
8
|
+
import { basename, extname, join, resolve } from "node:path";
|
|
10
9
|
import { path } from "@ffprobe-installer/ffprobe";
|
|
11
10
|
import { execa } from "execa";
|
|
12
11
|
import ffmpegPath from "ffmpeg-static";
|
|
12
|
+
import { homedir, tmpdir } from "node:os";
|
|
13
13
|
|
|
14
|
-
//#region src/
|
|
14
|
+
//#region src/logging/logger.ts
|
|
15
15
|
let debugMode = false;
|
|
16
16
|
let jsonMode = false;
|
|
17
17
|
function setDebugMode(enabled) {
|
|
@@ -42,25 +42,18 @@ const logger = {
|
|
|
42
42
|
};
|
|
43
43
|
|
|
44
44
|
//#endregion
|
|
45
|
-
//#region src/constants.ts
|
|
46
|
-
const APP_NAME = "scene-sieve";
|
|
45
|
+
//#region src/constants/pipeline-defaults.ts
|
|
47
46
|
const DEFAULT_THRESHOLD = .5;
|
|
48
|
-
const NORMALIZATION_ALPHA = .4;
|
|
49
|
-
const NORMALIZATION_MAD_COEFFICIENT = 1.4826;
|
|
50
|
-
const WORKSPACE_PREFIX = `${APP_NAME}-`;
|
|
51
|
-
const TEMP_BASE_DIR = tmpdir();
|
|
52
|
-
const FRAME_OUTPUT_EXTENSION = ".jpg";
|
|
53
|
-
const FRAME_FILENAME_PATTERN = "frame_%06d.jpg";
|
|
54
|
-
const DBSCAN_ALPHA = .03;
|
|
55
47
|
const IOU_THRESHOLD = .9;
|
|
48
|
+
|
|
49
|
+
//#endregion
|
|
50
|
+
//#region src/core/analyzer/constants/vision-tuning.ts
|
|
51
|
+
const DBSCAN_ALPHA = .03;
|
|
56
52
|
const DECAY_LAMBDA = .95;
|
|
57
53
|
const MATCH_DISTANCE_THRESHOLD = .25;
|
|
58
|
-
function getTempWorkspaceDir(sessionId) {
|
|
59
|
-
return join(TEMP_BASE_DIR, `${WORKSPACE_PREFIX}${sessionId}`);
|
|
60
|
-
}
|
|
61
54
|
|
|
62
55
|
//#endregion
|
|
63
|
-
//#region src/core/dbscan.ts
|
|
56
|
+
//#region src/core/analyzer/clustering/dbscan.ts
|
|
64
57
|
const UNVISITED = -2;
|
|
65
58
|
const NOISE = -1;
|
|
66
59
|
/**
|
|
@@ -140,7 +133,106 @@ function findNeighbors(points, idx, epsSquared) {
|
|
|
140
133
|
}
|
|
141
134
|
|
|
142
135
|
//#endregion
|
|
143
|
-
//#region src/core/analyzer.ts
|
|
136
|
+
//#region src/core/analyzer/features/feature-diff.ts
|
|
137
|
+
/**
|
|
138
|
+
* Match prev to next with Hamming k=2, crossCheck=false and strict ratio 0.25.
|
|
139
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
140
|
+
* @param prev - Previous frame's live features, owned by the caller.
|
|
141
|
+
* @param next - Next frame's live features, owned by the caller.
|
|
142
|
+
* @returns Unmatched next-frame coordinates without changing input ownership.
|
|
143
|
+
* @throws Propagates matching errors after releasing temporary native handles.
|
|
144
|
+
*/
|
|
145
|
+
function computeNewPoints(cvLib, prev, next) {
|
|
146
|
+
let matcher = null;
|
|
147
|
+
let matches = null;
|
|
148
|
+
try {
|
|
149
|
+
const matchedIndices = /* @__PURE__ */ new Set();
|
|
150
|
+
if (prev.descriptors.rows > 0 && next.descriptors.rows > 0) try {
|
|
151
|
+
matcher = new cvLib.BFMatcher(cvLib.NORM_HAMMING, false);
|
|
152
|
+
matches = new cvLib.DMatchVectorVector();
|
|
153
|
+
matcher.knnMatch(prev.descriptors, next.descriptors, matches, 2);
|
|
154
|
+
for (let i = 0; i < matches.size(); i++) {
|
|
155
|
+
const pair = matches.get(i);
|
|
156
|
+
try {
|
|
157
|
+
if (pair.size() < 2) continue;
|
|
158
|
+
const best = pair.get(0);
|
|
159
|
+
const second = pair.get(1);
|
|
160
|
+
if (best.distance < .25 * second.distance) matchedIndices.add(best.trainIdx);
|
|
161
|
+
} finally {
|
|
162
|
+
pair.delete();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} finally {
|
|
166
|
+
matcher?.delete();
|
|
167
|
+
}
|
|
168
|
+
const points = [];
|
|
169
|
+
for (let i = 0; i < next.keypoints.size(); i++) if (!matchedIndices.has(i)) {
|
|
170
|
+
const { x, y } = next.keypoints.get(i).pt;
|
|
171
|
+
points.push({
|
|
172
|
+
x,
|
|
173
|
+
y
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
return points;
|
|
177
|
+
} finally {
|
|
178
|
+
matches?.delete();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
//#endregion
|
|
183
|
+
//#region src/core/analyzer/features/frame-features.ts
|
|
184
|
+
/**
|
|
185
|
+
* Detect one frame's features without retaining its image or mask.
|
|
186
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
187
|
+
* @param akaze - Detector owned and released by the caller.
|
|
188
|
+
* @param frame - Grayscale bytes with matching width and height.
|
|
189
|
+
* @returns Feature handles that the caller must delete.
|
|
190
|
+
* @throws Propagates native errors after releasing partial allocations.
|
|
191
|
+
*/
|
|
192
|
+
function computeFrameFeatures(cvLib, akaze, frame) {
|
|
193
|
+
let image = null;
|
|
194
|
+
let mask = null;
|
|
195
|
+
let keypoints = null;
|
|
196
|
+
let descriptors = null;
|
|
197
|
+
try {
|
|
198
|
+
image = new cvLib.Mat(frame.height, frame.width, cvLib.CV_8UC1);
|
|
199
|
+
image.data.set(frame.data);
|
|
200
|
+
mask = new cvLib.Mat();
|
|
201
|
+
keypoints = new cvLib.KeyPointVector();
|
|
202
|
+
descriptors = new cvLib.Mat();
|
|
203
|
+
akaze.detectAndCompute(image, mask, keypoints, descriptors);
|
|
204
|
+
const ownedKeypoints = keypoints;
|
|
205
|
+
const ownedDescriptors = descriptors;
|
|
206
|
+
let deleted = false;
|
|
207
|
+
const features = {
|
|
208
|
+
width: frame.width,
|
|
209
|
+
height: frame.height,
|
|
210
|
+
keypoints: ownedKeypoints,
|
|
211
|
+
descriptors: ownedDescriptors,
|
|
212
|
+
/** Release the transferred handles exactly once. */
|
|
213
|
+
delete() {
|
|
214
|
+
if (deleted) return;
|
|
215
|
+
deleted = true;
|
|
216
|
+
try {
|
|
217
|
+
ownedKeypoints.delete();
|
|
218
|
+
} finally {
|
|
219
|
+
ownedDescriptors.delete();
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
keypoints = null;
|
|
224
|
+
descriptors = null;
|
|
225
|
+
return features;
|
|
226
|
+
} finally {
|
|
227
|
+
image?.delete();
|
|
228
|
+
mask?.delete();
|
|
229
|
+
keypoints?.delete();
|
|
230
|
+
descriptors?.delete();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
//#endregion
|
|
235
|
+
//#region src/core/analyzer/analyzer.ts
|
|
144
236
|
const OPENCV_INIT_TIMEOUT_MS = 3e4;
|
|
145
237
|
const require = createRequire(import.meta.url);
|
|
146
238
|
let cvReady = null;
|
|
@@ -263,77 +355,6 @@ var IoUTracker = class {
|
|
|
263
355
|
return maxWeight;
|
|
264
356
|
}
|
|
265
357
|
};
|
|
266
|
-
async function computeAKAZEDiff(cvLib, frame1, frame2) {
|
|
267
|
-
const cv = cvLib;
|
|
268
|
-
const mat1 = new cv.Mat(frame1.height, frame1.width, cv.CV_8UC1);
|
|
269
|
-
mat1.data.set(frame1.data);
|
|
270
|
-
const mat2 = new cv.Mat(frame2.height, frame2.width, cv.CV_8UC1);
|
|
271
|
-
mat2.data.set(frame2.data);
|
|
272
|
-
const kp1 = new cvLib.KeyPointVector();
|
|
273
|
-
const kp2 = new cvLib.KeyPointVector();
|
|
274
|
-
const desc1 = new cvLib.Mat();
|
|
275
|
-
const desc2 = new cvLib.Mat();
|
|
276
|
-
const mask1 = new cvLib.Mat();
|
|
277
|
-
const mask2 = new cvLib.Mat();
|
|
278
|
-
const akaze = new cvLib.AKAZE();
|
|
279
|
-
let matches = null;
|
|
280
|
-
try {
|
|
281
|
-
akaze.detectAndCompute(mat1, mask1, kp1, desc1);
|
|
282
|
-
akaze.detectAndCompute(mat2, mask2, kp2, desc2);
|
|
283
|
-
const matchedKp1Indices = /* @__PURE__ */ new Set();
|
|
284
|
-
const matchedKp2Indices = /* @__PURE__ */ new Set();
|
|
285
|
-
if (desc1.rows > 0 && desc2.rows > 0) {
|
|
286
|
-
const matcher = new cvLib.BFMatcher(cvLib.NORM_HAMMING, false);
|
|
287
|
-
try {
|
|
288
|
-
matches = new cvLib.DMatchVectorVector();
|
|
289
|
-
matcher.knnMatch(desc1, desc2, matches, 2);
|
|
290
|
-
for (let i = 0; i < matches.size(); i++) {
|
|
291
|
-
const pair = matches.get(i);
|
|
292
|
-
if (pair.size() < 2) continue;
|
|
293
|
-
const m0 = pair.get(0);
|
|
294
|
-
const m1 = pair.get(1);
|
|
295
|
-
if (m0.distance < .25 * m1.distance) {
|
|
296
|
-
matchedKp1Indices.add(m0.queryIdx);
|
|
297
|
-
matchedKp2Indices.add(m0.trainIdx);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
} finally {
|
|
301
|
-
matcher.delete();
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
const sNew = [];
|
|
305
|
-
for (let i = 0; i < kp2.size(); i++) if (!matchedKp2Indices.has(i)) {
|
|
306
|
-
const pt = kp2.get(i).pt;
|
|
307
|
-
sNew.push({
|
|
308
|
-
x: pt.x,
|
|
309
|
-
y: pt.y
|
|
310
|
-
});
|
|
311
|
-
}
|
|
312
|
-
const sLoss = [];
|
|
313
|
-
for (let i = 0; i < kp1.size(); i++) if (!matchedKp1Indices.has(i)) {
|
|
314
|
-
const pt = kp1.get(i).pt;
|
|
315
|
-
sLoss.push({
|
|
316
|
-
x: pt.x,
|
|
317
|
-
y: pt.y
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
return {
|
|
321
|
-
sNew,
|
|
322
|
-
sLoss
|
|
323
|
-
};
|
|
324
|
-
} finally {
|
|
325
|
-
mat1.delete();
|
|
326
|
-
mat2.delete();
|
|
327
|
-
kp1.delete();
|
|
328
|
-
kp2.delete();
|
|
329
|
-
desc1.delete();
|
|
330
|
-
desc2.delete();
|
|
331
|
-
mask1.delete();
|
|
332
|
-
mask2.delete();
|
|
333
|
-
akaze.delete();
|
|
334
|
-
if (matches) matches.delete();
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
358
|
/**
|
|
338
359
|
* Pixel-level difference fallback for AKAZE blind spots.
|
|
339
360
|
*
|
|
@@ -348,17 +369,30 @@ async function computeAKAZEDiff(cvLib, frame1, frame2) {
|
|
|
348
369
|
* 3. threshold → binary mask of significant changes
|
|
349
370
|
* 4. findContours → bounding rects of changed regions
|
|
350
371
|
* 5. Grid sampling within each bounding rect → Point2D[]
|
|
372
|
+
*
|
|
373
|
+
* @param cvLib - Initialized OpenCV runtime shared by the analyzer.
|
|
374
|
+
* @param frame1 - Previous grayscale frame, with the same dimensions as frame2.
|
|
375
|
+
* @param frame2 - Next grayscale frame, with the same dimensions as frame1.
|
|
376
|
+
* @returns Grid-sampled points from changed regions.
|
|
377
|
+
* @throws Propagates allocation or OpenCV errors after releasing acquired handles.
|
|
351
378
|
*/
|
|
352
379
|
function computePixelDiff(cvLib, frame1, frame2) {
|
|
353
380
|
const cv = cvLib;
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
381
|
+
let mat1 = null;
|
|
382
|
+
let mat2 = null;
|
|
383
|
+
let diff = null;
|
|
384
|
+
let blurred = null;
|
|
385
|
+
let binary = null;
|
|
386
|
+
let contours = null;
|
|
387
|
+
let hierarchy = null;
|
|
361
388
|
try {
|
|
389
|
+
mat1 = new cv.Mat(frame1.height, frame1.width, cv.CV_8UC1);
|
|
390
|
+
mat2 = new cv.Mat(frame2.height, frame2.width, cv.CV_8UC1);
|
|
391
|
+
diff = new cv.Mat();
|
|
392
|
+
blurred = new cv.Mat();
|
|
393
|
+
binary = new cv.Mat();
|
|
394
|
+
contours = new cv.MatVector();
|
|
395
|
+
hierarchy = new cv.Mat();
|
|
362
396
|
mat1.data.set(frame1.data);
|
|
363
397
|
mat2.data.set(frame2.data);
|
|
364
398
|
cv.absdiff(mat1, mat2, diff);
|
|
@@ -369,22 +403,26 @@ function computePixelDiff(cvLib, frame1, frame2) {
|
|
|
369
403
|
const points = [];
|
|
370
404
|
for (let c = 0; c < contours.size(); c++) {
|
|
371
405
|
const contour = contours.get(c);
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
x
|
|
376
|
-
|
|
377
|
-
|
|
406
|
+
try {
|
|
407
|
+
const rect = cv.boundingRect(contour);
|
|
408
|
+
if (rect.width * rect.height < 100) continue;
|
|
409
|
+
for (let y = rect.y; y < rect.y + rect.height; y += 8) for (let x = rect.x; x < rect.x + rect.width; x += 8) points.push({
|
|
410
|
+
x,
|
|
411
|
+
y
|
|
412
|
+
});
|
|
413
|
+
} finally {
|
|
414
|
+
contour.delete();
|
|
415
|
+
}
|
|
378
416
|
}
|
|
379
417
|
return points;
|
|
380
418
|
} finally {
|
|
381
|
-
mat1
|
|
382
|
-
mat2
|
|
383
|
-
diff
|
|
384
|
-
blurred
|
|
385
|
-
binary
|
|
386
|
-
contours
|
|
387
|
-
hierarchy
|
|
419
|
+
mat1?.delete();
|
|
420
|
+
mat2?.delete();
|
|
421
|
+
diff?.delete();
|
|
422
|
+
blurred?.delete();
|
|
423
|
+
binary?.delete();
|
|
424
|
+
contours?.delete();
|
|
425
|
+
hierarchy?.delete();
|
|
388
426
|
}
|
|
389
427
|
}
|
|
390
428
|
function computeInformationGain(clusters, clusterPoints, imageArea, animationIndices, animationWeights) {
|
|
@@ -403,47 +441,85 @@ function computeInformationGain(clusters, clusterPoints, imageArea, animationInd
|
|
|
403
441
|
}
|
|
404
442
|
return gain;
|
|
405
443
|
}
|
|
406
|
-
|
|
444
|
+
/**
|
|
445
|
+
* Analyze one batch, retaining its boundary frame for the following batch.
|
|
446
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
447
|
+
* @param akaze - Detector owned by analyzeFrames.
|
|
448
|
+
* @param frames - Boundary frame followed by new adjacent frames.
|
|
449
|
+
* @param carry - Boundary bytes and live features transferred from the previous batch.
|
|
450
|
+
* @param scale - Maximum preprocessing width.
|
|
451
|
+
* @param tracker - Stateful animation tracker shared across batches.
|
|
452
|
+
* @param pairOffset - Global position of this batch's first pair.
|
|
453
|
+
* @returns Scores, pair failure count, and ownership of the final frame's features.
|
|
454
|
+
*/
|
|
455
|
+
async function analyzeBatch(cvLib, akaze, frames, carry, scale, tracker, pairOffset) {
|
|
407
456
|
const edges = [];
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
const
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
457
|
+
let failures = 0;
|
|
458
|
+
let prev = carry?.features ?? null;
|
|
459
|
+
let next = null;
|
|
460
|
+
try {
|
|
461
|
+
const preprocessed = await Promise.all(map(frames, (f, index) => index === 0 && carry ? carry.preprocessed : preprocessFrame(f.extractPath, scale)));
|
|
462
|
+
const imageWidth = preprocessed[0]?.width ?? scale;
|
|
463
|
+
const imageHeight = preprocessed[0]?.height ?? Math.round(scale * 9 / 16);
|
|
464
|
+
const imageArea = imageWidth * imageHeight;
|
|
465
|
+
for (let i = 0; i < frames.length - 1; i++) {
|
|
466
|
+
const pairIndex = pairOffset + i;
|
|
467
|
+
try {
|
|
468
|
+
prev ??= computeFrameFeatures(cvLib, akaze, preprocessed[i]);
|
|
469
|
+
next = computeFrameFeatures(cvLib, akaze, preprocessed[i + 1]);
|
|
470
|
+
let dbscanResult = dbscan(computeNewPoints(cvLib, prev, next), imageWidth, imageHeight);
|
|
471
|
+
let clusters = dbscanResult.boundingBoxes;
|
|
472
|
+
if (clusters.length === 0) {
|
|
473
|
+
const pixelDiffPoints = computePixelDiff(cvLib, preprocessed[i], preprocessed[i + 1]);
|
|
474
|
+
if (pixelDiffPoints.length > 0) {
|
|
475
|
+
logger.debug(`Edge ${frames[i].id}->${frames[i + 1].id}: pixel-diff fallback (${pixelDiffPoints.length} points)`);
|
|
476
|
+
dbscanResult = dbscan(pixelDiffPoints, imageWidth, imageHeight, void 0, 2);
|
|
477
|
+
clusters = dbscanResult.boundingBoxes;
|
|
478
|
+
}
|
|
424
479
|
}
|
|
480
|
+
const clusterPointCounts = new Array(clusters.length).fill(0);
|
|
481
|
+
for (const label of dbscanResult.labels) if (label >= 0) clusterPointCounts[label]++;
|
|
482
|
+
const animationIndices = tracker.update(clusters, pairIndex);
|
|
483
|
+
const animationWeights = map(clusters, (_, ci) => animationIndices.has(ci) ? tracker.getAnimationWeight(ci, clusters) : 0);
|
|
484
|
+
const score = computeInformationGain(clusters, clusterPointCounts, imageArea, animationIndices, animationWeights);
|
|
485
|
+
logger.debug(`Edge ${frames[i].id}->${frames[i + 1].id} G(t)=${score.toFixed(6)}`);
|
|
486
|
+
edges.push({
|
|
487
|
+
sourceId: frames[i].id,
|
|
488
|
+
targetId: frames[i + 1].id,
|
|
489
|
+
score
|
|
490
|
+
});
|
|
491
|
+
} catch (err) {
|
|
492
|
+
logger.warn(`Frame pair analysis failed: ${String(err)}`);
|
|
493
|
+
failures++;
|
|
494
|
+
edges.push({
|
|
495
|
+
sourceId: frames[i].id,
|
|
496
|
+
targetId: frames[i + 1].id,
|
|
497
|
+
score: 0
|
|
498
|
+
});
|
|
499
|
+
} finally {
|
|
500
|
+
prev?.delete();
|
|
501
|
+
prev = next;
|
|
502
|
+
next = null;
|
|
425
503
|
}
|
|
426
|
-
const clusterPointCounts = new Array(clusters.length).fill(0);
|
|
427
|
-
for (const label of dbscanResult.labels) if (label >= 0) clusterPointCounts[label]++;
|
|
428
|
-
const animationIndices = tracker.update(clusters, pairIndex);
|
|
429
|
-
const animationWeights = map(clusters, (_, ci) => animationIndices.has(ci) ? tracker.getAnimationWeight(ci, clusters) : 0);
|
|
430
|
-
const score = computeInformationGain(clusters, clusterPointCounts, imageArea, animationIndices, animationWeights);
|
|
431
|
-
logger.debug(`Edge ${frames[i].id}->${frames[i + 1].id} G(t)=${score.toFixed(6)}`);
|
|
432
|
-
edges.push({
|
|
433
|
-
sourceId: frames[i].id,
|
|
434
|
-
targetId: frames[i + 1].id,
|
|
435
|
-
score
|
|
436
|
-
});
|
|
437
|
-
} catch (err) {
|
|
438
|
-
logger.debug(`Frame pair analysis failed: ${String(err)}`);
|
|
439
|
-
edges.push({
|
|
440
|
-
sourceId: frames[i].id,
|
|
441
|
-
targetId: frames[i + 1].id,
|
|
442
|
-
score: 0
|
|
443
|
-
});
|
|
444
504
|
}
|
|
505
|
+
const result = {
|
|
506
|
+
edges,
|
|
507
|
+
failures,
|
|
508
|
+
analysisResolution: {
|
|
509
|
+
width: imageWidth,
|
|
510
|
+
height: imageHeight
|
|
511
|
+
},
|
|
512
|
+
carry: prev ? {
|
|
513
|
+
preprocessed: preprocessed[preprocessed.length - 1],
|
|
514
|
+
features: prev
|
|
515
|
+
} : null
|
|
516
|
+
};
|
|
517
|
+
prev = null;
|
|
518
|
+
return result;
|
|
519
|
+
} finally {
|
|
520
|
+
prev?.delete();
|
|
521
|
+
next?.delete();
|
|
445
522
|
}
|
|
446
|
-
return edges;
|
|
447
523
|
}
|
|
448
524
|
/**
|
|
449
525
|
* Analyze adjacent frame pairs to compute information gain scores (G(t)).
|
|
@@ -454,35 +530,121 @@ async function analyzeBatch(cvLib, frames, scale, tracker, pairOffset) {
|
|
|
454
530
|
* 2. DBSCAN Spatial Clustering
|
|
455
531
|
* 3. Spatio-temporal IoU Tracking
|
|
456
532
|
* 4. G(t) Information Gain Scoring
|
|
533
|
+
* @param ctx - Frames, analysis options, and the progress callback for this run.
|
|
534
|
+
* @returns Adjacent scores and tracked animations in analysis coordinates.
|
|
535
|
+
* @throws Propagates runtime errors and rejects total failure of two or more pairs after cleanup.
|
|
457
536
|
*/
|
|
458
537
|
async function analyzeFrames(ctx) {
|
|
459
538
|
const { frames } = ctx;
|
|
460
539
|
if (frames.length < 2) return {
|
|
461
540
|
edges: [],
|
|
462
|
-
animations: []
|
|
541
|
+
animations: [],
|
|
542
|
+
analysisResolution: {
|
|
543
|
+
width: 0,
|
|
544
|
+
height: 0
|
|
545
|
+
}
|
|
463
546
|
};
|
|
464
547
|
logger.debug(`Analyzing ${frames.length} frames in batches of ${10}`);
|
|
465
548
|
const cvLib = await ensureOpenCV();
|
|
466
549
|
const edges = [];
|
|
467
|
-
const tracker = new IoUTracker(ctx.options.fps, ctx.options.iouThreshold, ctx.options.animationThreshold);
|
|
550
|
+
const tracker = new IoUTracker(ctx.effectiveFps ?? ctx.options.fps, ctx.options.iouThreshold, ctx.options.animationThreshold);
|
|
468
551
|
const scale = ctx.options.scale;
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
552
|
+
let akaze = null;
|
|
553
|
+
let carry = null;
|
|
554
|
+
let analysisResolution = {
|
|
555
|
+
width: 0,
|
|
556
|
+
height: 0
|
|
557
|
+
};
|
|
558
|
+
let failures = 0;
|
|
559
|
+
try {
|
|
560
|
+
akaze = new cvLib.AKAZE();
|
|
561
|
+
for (let i = 0; i < frames.length - 1; i += 10) {
|
|
562
|
+
const batch = [frames[i], ...frames.slice(i + 1, i + 1 + 10)];
|
|
563
|
+
const result = await analyzeBatch(cvLib, akaze, batch, carry, scale, tracker, i);
|
|
564
|
+
carry = result.carry;
|
|
565
|
+
failures += result.failures;
|
|
566
|
+
if (i === 0) analysisResolution = result.analysisResolution;
|
|
567
|
+
edges.push(...result.edges);
|
|
568
|
+
const progress = Math.min(100, (i + 10) / (frames.length - 1) * 100);
|
|
569
|
+
ctx.emitProgress(progress);
|
|
570
|
+
}
|
|
571
|
+
} finally {
|
|
572
|
+
carry?.features.delete();
|
|
573
|
+
akaze?.delete();
|
|
475
574
|
}
|
|
575
|
+
const pairs = frames.length - 1;
|
|
576
|
+
if (pairs >= 2 && failures === pairs) throw new Error(`All ${pairs} frame pairs failed analysis`);
|
|
476
577
|
const animations = tracker.flushAndGetAnimations();
|
|
477
578
|
logger.debug(`Computed ${edges.length} score edges and ${animations.length} animations`);
|
|
478
579
|
return {
|
|
479
580
|
edges,
|
|
480
|
-
animations
|
|
581
|
+
animations,
|
|
582
|
+
analysisResolution
|
|
583
|
+
};
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
//#endregion
|
|
587
|
+
//#region src/core/utils/metadata/build-video-metadata.ts
|
|
588
|
+
/**
|
|
589
|
+
* Read output dimensions and build consistent video and animation metadata.
|
|
590
|
+
* JPEG finalization does not resize, so the source dimensions match the output.
|
|
591
|
+
* @param ctx Pipeline state with source duration, effective FPS and analysis-space animations.
|
|
592
|
+
* @param selected Selected frames in output order; the first candidate is the fallback.
|
|
593
|
+
* @param analysisResolution Analysis dimensions; absent or zero dimensions imply no scaling.
|
|
594
|
+
* @returns Video metadata and new output-space animations, retaining zero-based frame IDs.
|
|
595
|
+
* @throws If sharp cannot read the selected or fallback image. Empty input performs no image I/O.
|
|
596
|
+
*/
|
|
597
|
+
async function buildVideoMetadata(ctx, selected, analysisResolution) {
|
|
598
|
+
const firstFrame = selected[0] ?? ctx.frames[0];
|
|
599
|
+
const dimensions = firstFrame ? await sharp(firstFrame.extractPath).metadata() : void 0;
|
|
600
|
+
const width = dimensions?.width ?? 0;
|
|
601
|
+
const height = dimensions?.height ?? 0;
|
|
602
|
+
const sx = analysisResolution?.width ? width / analysisResolution.width : 1;
|
|
603
|
+
const sy = analysisResolution?.height ? height / analysisResolution.height : 1;
|
|
604
|
+
const lastTimestamp = ctx.frames[ctx.frames.length - 1]?.timestamp ?? 0;
|
|
605
|
+
const duration = ctx.options.mode === "frames" ? lastTimestamp : ctx.sourceDurationSec ?? lastTimestamp;
|
|
606
|
+
return {
|
|
607
|
+
video: {
|
|
608
|
+
originalDurationMs: Math.round(duration * 1e3),
|
|
609
|
+
fps: ctx.options.mode === "frames" ? 1 : ctx.effectiveFps ?? ctx.options.fps,
|
|
610
|
+
resolution: {
|
|
611
|
+
width,
|
|
612
|
+
height
|
|
613
|
+
}
|
|
614
|
+
},
|
|
615
|
+
animations: (ctx.animations ?? []).map((animation) => {
|
|
616
|
+
const box = animation.boundingBox;
|
|
617
|
+
const x = Math.max(0, Math.min(width, Math.round(box.x * sx)));
|
|
618
|
+
const y = Math.max(0, Math.min(height, Math.round(box.y * sy)));
|
|
619
|
+
return {
|
|
620
|
+
...animation,
|
|
621
|
+
boundingBox: {
|
|
622
|
+
x,
|
|
623
|
+
y,
|
|
624
|
+
width: Math.max(0, Math.min(width - x, Math.round(box.width * sx))),
|
|
625
|
+
height: Math.max(0, Math.min(height - y, Math.round(box.height * sy)))
|
|
626
|
+
}
|
|
627
|
+
};
|
|
628
|
+
})
|
|
481
629
|
};
|
|
482
630
|
}
|
|
483
631
|
|
|
484
632
|
//#endregion
|
|
485
|
-
//#region src/
|
|
633
|
+
//#region src/core/constants/workspace-layout.ts
|
|
634
|
+
/**
|
|
635
|
+
* Temp workspace naming and frame file layout for pipeline runs.
|
|
636
|
+
*/
|
|
637
|
+
const APP_NAME = "scene-sieve";
|
|
638
|
+
const WORKSPACE_PREFIX = `${APP_NAME}-`;
|
|
639
|
+
const TEMP_BASE_DIR = tmpdir();
|
|
640
|
+
const FRAME_OUTPUT_EXTENSION = ".jpg";
|
|
641
|
+
const FRAME_FILENAME_PATTERN = "frame_%06d.jpg";
|
|
642
|
+
function getTempWorkspaceDir(sessionId) {
|
|
643
|
+
return join(TEMP_BASE_DIR, `${WORKSPACE_PREFIX}${sessionId}`);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
//#endregion
|
|
647
|
+
//#region src/core/utils/filesystem/paths.ts
|
|
486
648
|
async function ensureDir(dirPath) {
|
|
487
649
|
await mkdir(dirPath, { recursive: true });
|
|
488
650
|
}
|
|
@@ -520,13 +682,18 @@ function deriveOutputPath(inputPath) {
|
|
|
520
682
|
}
|
|
521
683
|
|
|
522
684
|
//#endregion
|
|
523
|
-
//#region src/core/extractor.ts
|
|
685
|
+
//#region src/core/extractor/extractor.ts
|
|
524
686
|
/**
|
|
525
687
|
* Extract frames from video/GIF using FFmpeg.
|
|
526
|
-
*
|
|
527
|
-
*
|
|
688
|
+
* @param ctx Pipeline context; records effectiveFps and sourceDurationSec for video input.
|
|
689
|
+
* @returns Extracted candidates, or the unchanged input array in frames mode.
|
|
690
|
+
* @throws When the input is missing, metadata has no video stream, or FFmpeg fails.
|
|
528
691
|
*/
|
|
529
692
|
async function extractFrames(ctx) {
|
|
693
|
+
if (ctx.options.mode === "frames") {
|
|
694
|
+
ctx.effectiveFps = 1;
|
|
695
|
+
return ctx.frames;
|
|
696
|
+
}
|
|
530
697
|
const framesDir = join(ctx.workspacePath, "frames");
|
|
531
698
|
const { inputPath, fps, maxFrames, scale } = ctx.options;
|
|
532
699
|
if (!inputPath) throw new Error("inputPath is required for frame extraction");
|
|
@@ -541,19 +708,30 @@ async function extractFrames(ctx) {
|
|
|
541
708
|
if (!(metadata.streams?.some((s) => s.codec_type === "video") ?? false)) throw new Error(`No video stream found in file: ${inputPath} (detected format: ${formatName})`);
|
|
542
709
|
logger.debug(`Detected format: ${formatName} (Duration: ${duration.toFixed(1)}s), path: ${inputPath}`);
|
|
543
710
|
await ensureDir(framesDir);
|
|
711
|
+
const frameLimit = Math.max(2, maxFrames);
|
|
544
712
|
let effectiveFps = fps;
|
|
545
713
|
if (duration > 0) {
|
|
546
|
-
const fpsCap =
|
|
714
|
+
const fpsCap = frameLimit / duration;
|
|
547
715
|
effectiveFps = Math.min(fps, fpsCap);
|
|
548
|
-
effectiveFps = Math.max(.5, effectiveFps);
|
|
549
716
|
logger.debug(`FPS: ${fps} → effective: ${effectiveFps.toFixed(2)} (maxFrames: ${maxFrames})`);
|
|
550
717
|
}
|
|
551
|
-
|
|
718
|
+
ctx.effectiveFps = effectiveFps;
|
|
719
|
+
ctx.sourceDurationSec = duration;
|
|
720
|
+
const frames = await extractByFps(inputPath, framesDir, effectiveFps, scale, frameLimit);
|
|
552
721
|
ctx.emitProgress(100);
|
|
553
722
|
logger.debug(`Extracted ${frames.length} frames`);
|
|
554
723
|
return frames;
|
|
555
724
|
}
|
|
556
|
-
|
|
725
|
+
/**
|
|
726
|
+
* Write scaled JPEG candidates through the bundled FFmpeg runtime.
|
|
727
|
+
* @param inputPath Readable video input.
|
|
728
|
+
* @param outputDir Existing frame directory.
|
|
729
|
+
* @param fps Positive effective sampling frequency.
|
|
730
|
+
* @param scale Output image height.
|
|
731
|
+
* @param frameLimit Maximum number of output frames.
|
|
732
|
+
* @returns Candidates with local output-grid timestamps; rejects on extraction failure.
|
|
733
|
+
*/
|
|
734
|
+
async function extractByFps(inputPath, outputDir, fps, scale, frameLimit) {
|
|
557
735
|
const outputPattern = join(outputDir, FRAME_FILENAME_PATTERN);
|
|
558
736
|
await execa(ffmpegPath, [
|
|
559
737
|
"-i",
|
|
@@ -562,9 +740,11 @@ async function extractByFps(inputPath, outputDir, fps, scale, duration) {
|
|
|
562
740
|
`fps=${fps},scale=-1:${scale}`,
|
|
563
741
|
"-q:v",
|
|
564
742
|
"2",
|
|
743
|
+
"-frames:v",
|
|
744
|
+
String(frameLimit),
|
|
565
745
|
outputPattern
|
|
566
746
|
]);
|
|
567
|
-
return buildFrameList(outputDir,
|
|
747
|
+
return buildFrameList(outputDir, fps);
|
|
568
748
|
}
|
|
569
749
|
async function getVideoMetadata(inputPath) {
|
|
570
750
|
const { stdout } = await execa(path, [
|
|
@@ -578,12 +758,18 @@ async function getVideoMetadata(inputPath) {
|
|
|
578
758
|
]);
|
|
579
759
|
return JSON.parse(stdout);
|
|
580
760
|
}
|
|
581
|
-
|
|
761
|
+
/**
|
|
762
|
+
* Read sorted JPEG paths and attach local output-grid times.
|
|
763
|
+
* @param framesDir Extracted frame directory; filesystem errors propagate.
|
|
764
|
+
* @param effectiveFps Positive frequency used by the fps filter.
|
|
765
|
+
* @returns Zero-based candidates without a segment seek offset.
|
|
766
|
+
*/
|
|
767
|
+
async function buildFrameList(framesDir, effectiveFps) {
|
|
582
768
|
const jpgFiles = filter(await readdir(framesDir), (f) => f.endsWith(".jpg")).sort();
|
|
583
769
|
if (jpgFiles.length === 0) return [];
|
|
584
770
|
return map(jpgFiles, (file, index) => ({
|
|
585
771
|
id: index,
|
|
586
|
-
timestamp:
|
|
772
|
+
timestamp: index / effectiveFps,
|
|
587
773
|
extractPath: join(framesDir, file)
|
|
588
774
|
}));
|
|
589
775
|
}
|
|
@@ -597,9 +783,10 @@ async function buildFrameList(framesDir, duration) {
|
|
|
597
783
|
* @param scale - Height scale for vision analysis
|
|
598
784
|
* @param startTime - Start time in seconds
|
|
599
785
|
* @param duration - Duration in seconds to extract
|
|
786
|
+
* @param frameLimit - Positive output limit; defaults to the range's grid capacity
|
|
600
787
|
* @returns Array of FrameNode with segment-local timestamps (starting from 0)
|
|
601
788
|
*/
|
|
602
|
-
async function extractFramesForRange(inputPath, outputDir, fps, scale, startTime, duration) {
|
|
789
|
+
async function extractFramesForRange(inputPath, outputDir, fps, scale, startTime, duration, frameLimit = Math.ceil(duration * fps)) {
|
|
603
790
|
const outputPattern = join(outputDir, FRAME_FILENAME_PATTERN);
|
|
604
791
|
await execa(ffmpegPath, [
|
|
605
792
|
"-ss",
|
|
@@ -612,13 +799,112 @@ async function extractFramesForRange(inputPath, outputDir, fps, scale, startTime
|
|
|
612
799
|
`fps=${fps},scale=-1:${scale}`,
|
|
613
800
|
"-q:v",
|
|
614
801
|
"2",
|
|
802
|
+
"-frames:v",
|
|
803
|
+
String(frameLimit),
|
|
615
804
|
outputPattern
|
|
616
805
|
]);
|
|
617
|
-
return buildFrameList(outputDir,
|
|
806
|
+
return buildFrameList(outputDir, fps);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
//#endregion
|
|
810
|
+
//#region src/core/input-resolver/validation/validate-options.ts
|
|
811
|
+
/**
|
|
812
|
+
* Reject invalid numeric options before defaults or pipeline effects are applied.
|
|
813
|
+
* @param options - Supplied options; omitted numeric fields use pipeline defaults.
|
|
814
|
+
* @returns Nothing when all supplied numeric fields satisfy their contracts.
|
|
815
|
+
* @throws An input error naming the invalid option and its received value.
|
|
816
|
+
*/
|
|
817
|
+
function validateOptions(options) {
|
|
818
|
+
for (const [name, min, max, integer, exclusiveMin, requirement] of [
|
|
819
|
+
[
|
|
820
|
+
"count",
|
|
821
|
+
1,
|
|
822
|
+
Infinity,
|
|
823
|
+
true,
|
|
824
|
+
false,
|
|
825
|
+
"an integer >= 1"
|
|
826
|
+
],
|
|
827
|
+
[
|
|
828
|
+
"threshold",
|
|
829
|
+
0,
|
|
830
|
+
1,
|
|
831
|
+
false,
|
|
832
|
+
true,
|
|
833
|
+
"in range (0, 1] and finite"
|
|
834
|
+
],
|
|
835
|
+
[
|
|
836
|
+
"fps",
|
|
837
|
+
0,
|
|
838
|
+
Infinity,
|
|
839
|
+
false,
|
|
840
|
+
true,
|
|
841
|
+
"finite and > 0"
|
|
842
|
+
],
|
|
843
|
+
[
|
|
844
|
+
"maxFrames",
|
|
845
|
+
2,
|
|
846
|
+
Infinity,
|
|
847
|
+
true,
|
|
848
|
+
false,
|
|
849
|
+
"an integer >= 2"
|
|
850
|
+
],
|
|
851
|
+
[
|
|
852
|
+
"scale",
|
|
853
|
+
16,
|
|
854
|
+
Infinity,
|
|
855
|
+
true,
|
|
856
|
+
false,
|
|
857
|
+
"an integer >= 16"
|
|
858
|
+
],
|
|
859
|
+
[
|
|
860
|
+
"quality",
|
|
861
|
+
1,
|
|
862
|
+
100,
|
|
863
|
+
true,
|
|
864
|
+
false,
|
|
865
|
+
"an integer in range [1, 100]"
|
|
866
|
+
],
|
|
867
|
+
[
|
|
868
|
+
"iouThreshold",
|
|
869
|
+
0,
|
|
870
|
+
1,
|
|
871
|
+
false,
|
|
872
|
+
false,
|
|
873
|
+
"finite and in range [0, 1]"
|
|
874
|
+
],
|
|
875
|
+
[
|
|
876
|
+
"animationThreshold",
|
|
877
|
+
1,
|
|
878
|
+
Infinity,
|
|
879
|
+
true,
|
|
880
|
+
false,
|
|
881
|
+
"an integer >= 1"
|
|
882
|
+
],
|
|
883
|
+
[
|
|
884
|
+
"maxSegmentDuration",
|
|
885
|
+
0,
|
|
886
|
+
Infinity,
|
|
887
|
+
false,
|
|
888
|
+
true,
|
|
889
|
+
"finite and > 0"
|
|
890
|
+
],
|
|
891
|
+
[
|
|
892
|
+
"concurrency",
|
|
893
|
+
1,
|
|
894
|
+
Infinity,
|
|
895
|
+
true,
|
|
896
|
+
false,
|
|
897
|
+
"an integer >= 1"
|
|
898
|
+
]
|
|
899
|
+
]) {
|
|
900
|
+
const value = options[name];
|
|
901
|
+
if (value === void 0) continue;
|
|
902
|
+
if (!Number.isFinite(value) || integer && !Number.isInteger(value) || (exclusiveMin ? value <= min : value < min) || value > max) throw new Error(`${name} must be ${requirement}, received: ${value}`);
|
|
903
|
+
}
|
|
618
904
|
}
|
|
619
905
|
|
|
620
906
|
//#endregion
|
|
621
|
-
//#region src/core/workspace.ts
|
|
907
|
+
//#region src/core/workspace/workspace.ts
|
|
622
908
|
async function createWorkspace(sessionId) {
|
|
623
909
|
const workspacePath = getTempWorkspaceDir(sessionId);
|
|
624
910
|
await ensureDir(join(workspacePath, "frames"));
|
|
@@ -649,17 +935,11 @@ async function finalizeOutput(ctx, selectedFrames) {
|
|
|
649
935
|
timestampMs: Math.round(frame.timestamp * 1e3)
|
|
650
936
|
});
|
|
651
937
|
}
|
|
938
|
+
const { video, animations } = await buildVideoMetadata(ctx, selectedFrames, ctx.analysisResolution);
|
|
652
939
|
const metadata = {
|
|
653
|
-
video
|
|
654
|
-
originalDurationMs: Math.round((ctx.frames.length > 0 ? ctx.frames[ctx.frames.length - 1].timestamp : 0) * 1e3),
|
|
655
|
-
fps: ctx.options.fps,
|
|
656
|
-
resolution: {
|
|
657
|
-
width: ctx.options.scale,
|
|
658
|
-
height: Math.round(ctx.options.scale * 9 / 16)
|
|
659
|
-
}
|
|
660
|
-
},
|
|
940
|
+
video,
|
|
661
941
|
frames: framesMetadata,
|
|
662
|
-
animations: map(
|
|
942
|
+
animations: map(animations, (anim) => ({
|
|
663
943
|
...anim,
|
|
664
944
|
startFrameId: anim.startFrameId + 1,
|
|
665
945
|
endFrameId: anim.endFrameId + 1,
|
|
@@ -732,13 +1012,19 @@ async function readFramesAsBuffers(frameNodes, quality) {
|
|
|
732
1012
|
}
|
|
733
1013
|
|
|
734
1014
|
//#endregion
|
|
735
|
-
//#region src/core/input-resolver.ts
|
|
1015
|
+
//#region src/core/input-resolver/input-resolver.ts
|
|
1016
|
+
/**
|
|
1017
|
+
* Validate supplied options and resolve defaults and paths for the pipeline.
|
|
1018
|
+
* @param options - Mode-specific input and optional numeric settings.
|
|
1019
|
+
* @returns Complete pipeline settings with absolute file input paths.
|
|
1020
|
+
* @throws An input error if a supplied numeric setting is invalid.
|
|
1021
|
+
*/
|
|
736
1022
|
function resolveOptions(options) {
|
|
1023
|
+
validateOptions(options);
|
|
737
1024
|
const mode = options.mode;
|
|
738
1025
|
const inputPath = mode === "file" ? resolveAbsolute(options.inputPath) : void 0;
|
|
739
1026
|
const outputPath = options.outputPath ?? (inputPath ? deriveOutputPath(inputPath) : join(process.cwd(), "scene-sieve-output"));
|
|
740
1027
|
const threshold = options.threshold ?? .5;
|
|
741
|
-
if (threshold <= 0 || threshold > 1) throw new Error(`threshold must be in range (0, 1], received: ${threshold}`);
|
|
742
1028
|
return {
|
|
743
1029
|
mode,
|
|
744
1030
|
inputPath,
|
|
@@ -763,6 +1049,10 @@ function resolveOptions(options) {
|
|
|
763
1049
|
* - 'file' mode: validate file exists and delegate to extractor (caller's responsibility)
|
|
764
1050
|
* - 'buffer' mode: write buffer as temp video file, return path via FrameNode trick (empty list)
|
|
765
1051
|
* - 'frames' mode: write frame buffers as JPGs, return FrameNode[]
|
|
1052
|
+
* @param options - Input source; encoded frames must have matching dimensions.
|
|
1053
|
+
* @param workspacePath - Workspace receiving temporary input files.
|
|
1054
|
+
* @returns Frame nodes or a resolved video path for extraction.
|
|
1055
|
+
* @throws Propagates metadata or write errors and rejects mismatched frame sizes.
|
|
766
1056
|
*/
|
|
767
1057
|
async function resolveInput(options, workspacePath) {
|
|
768
1058
|
if (options.mode === "file") return {
|
|
@@ -773,12 +1063,42 @@ async function resolveInput(options, workspacePath) {
|
|
|
773
1063
|
frames: [],
|
|
774
1064
|
resolvedInputPath: await writeInputBuffer(options.inputBuffer, workspacePath)
|
|
775
1065
|
};
|
|
776
|
-
if (options.mode === "frames")
|
|
1066
|
+
if (options.mode === "frames") {
|
|
1067
|
+
let dimensions;
|
|
1068
|
+
for (const buffer of options.inputFrames) {
|
|
1069
|
+
const { width, height } = await sharp(buffer).metadata();
|
|
1070
|
+
if (dimensions && (width !== dimensions.width || height !== dimensions.height)) throw new Error(`inputFrames must be the same size (${dimensions.width}x${dimensions.height}), received: ${width}x${height}`);
|
|
1071
|
+
dimensions = {
|
|
1072
|
+
width,
|
|
1073
|
+
height
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
return { frames: await writeInputFrames(options.inputFrames, workspacePath) };
|
|
1077
|
+
}
|
|
777
1078
|
throw new Error(`Unsupported input mode: ${options.mode}`);
|
|
778
1079
|
}
|
|
779
1080
|
|
|
780
1081
|
//#endregion
|
|
781
|
-
//#region src/
|
|
1082
|
+
//#region src/core/pruner/scoring/normalize-scores.ts
|
|
1083
|
+
const NORMALIZATION_ALPHA = .4;
|
|
1084
|
+
const NORMALIZATION_MAD_COEFFICIENT = 1.4826;
|
|
1085
|
+
const NORMALIZATION_MIN_SAMPLE_SIZE = 10;
|
|
1086
|
+
/**
|
|
1087
|
+
* Find the first position whose score is at least the requested value.
|
|
1088
|
+
* @param sorted - Finite positive scores sorted in ascending order.
|
|
1089
|
+
* @param value - A finite positive score present in sorted.
|
|
1090
|
+
* @returns The first matching rank, including the first position of any tie.
|
|
1091
|
+
*/
|
|
1092
|
+
function lowerBound(sorted, value) {
|
|
1093
|
+
let low = 0;
|
|
1094
|
+
let high = sorted.length;
|
|
1095
|
+
while (low < high) {
|
|
1096
|
+
const mid = Math.floor((low + high) / 2);
|
|
1097
|
+
if (sorted[mid] < value) low = mid + 1;
|
|
1098
|
+
else high = mid;
|
|
1099
|
+
}
|
|
1100
|
+
return low;
|
|
1101
|
+
}
|
|
782
1102
|
/**
|
|
783
1103
|
* Normalize raw scores to [0, 1] range via Robust Hybrid Normalization.
|
|
784
1104
|
*
|
|
@@ -823,13 +1143,13 @@ function normalizeScores(items) {
|
|
|
823
1143
|
});
|
|
824
1144
|
const cdf = map(safeScores, (s) => {
|
|
825
1145
|
if (s <= 0) return 0;
|
|
826
|
-
return sorted
|
|
1146
|
+
return lowerBound(sorted, s) / sorted.length;
|
|
827
1147
|
});
|
|
828
1148
|
return map(logisticZ, (z, i) => z * (1 - NORMALIZATION_ALPHA) + cdf[i] * NORMALIZATION_ALPHA);
|
|
829
1149
|
}
|
|
830
1150
|
|
|
831
1151
|
//#endregion
|
|
832
|
-
//#region src/
|
|
1152
|
+
//#region src/core/pruner/heap/min-heap.ts
|
|
833
1153
|
/**
|
|
834
1154
|
* Generic binary min-heap.
|
|
835
1155
|
*
|
|
@@ -880,7 +1200,7 @@ var MinHeap = class {
|
|
|
880
1200
|
};
|
|
881
1201
|
|
|
882
1202
|
//#endregion
|
|
883
|
-
//#region src/core/pruner.ts
|
|
1203
|
+
//#region src/core/pruner/pruner.ts
|
|
884
1204
|
/**
|
|
885
1205
|
* Edge-aware greedy merge with re-linking — O(N log N).
|
|
886
1206
|
*
|
|
@@ -994,7 +1314,7 @@ function suppressConsecutiveRuns(graph, passingIndices, normalizedScores) {
|
|
|
994
1314
|
return result;
|
|
995
1315
|
}
|
|
996
1316
|
/**
|
|
997
|
-
* Threshold-based pruning with NMS -- O(N).
|
|
1317
|
+
* Threshold-based pruning with NMS -- including normalization, O(N log N).
|
|
998
1318
|
*
|
|
999
1319
|
* 1. Scores are normalized to [0, 1] via percentile normalization.
|
|
1000
1320
|
* 2. Edges with normalized score >= threshold are collected.
|
|
@@ -1059,7 +1379,7 @@ function pruneByThresholdWithCap(graph, frames, threshold, maxCount) {
|
|
|
1059
1379
|
}
|
|
1060
1380
|
|
|
1061
1381
|
//#endregion
|
|
1062
|
-
//#region src/
|
|
1382
|
+
//#region src/core/segmenter/scheduling/concurrency.ts
|
|
1063
1383
|
/**
|
|
1064
1384
|
* Creates a concurrency limiter that runs at most `limit` tasks in parallel.
|
|
1065
1385
|
* Lightweight replacement for p-limit to avoid external dependency.
|
|
@@ -1081,7 +1401,7 @@ function concurrencyLimit(limit) {
|
|
|
1081
1401
|
}
|
|
1082
1402
|
|
|
1083
1403
|
//#endregion
|
|
1084
|
-
//#region src/core/segmenter.ts
|
|
1404
|
+
//#region src/core/segmenter/segmenter.ts
|
|
1085
1405
|
/**
|
|
1086
1406
|
* Determine whether segmentation should be used.
|
|
1087
1407
|
* Returns false for frames mode and GIF files.
|
|
@@ -1095,53 +1415,62 @@ function shouldSegment(resolvedOptions, originalOptions) {
|
|
|
1095
1415
|
return true;
|
|
1096
1416
|
}
|
|
1097
1417
|
/**
|
|
1098
|
-
*
|
|
1099
|
-
*
|
|
1100
|
-
*
|
|
1101
|
-
*
|
|
1102
|
-
*
|
|
1103
|
-
*
|
|
1418
|
+
* Partition the global extraction grid into nonempty logical segments.
|
|
1419
|
+
* @param totalDuration Positive source duration in seconds.
|
|
1420
|
+
* @param maxSegmentDuration Positive logical segment width in seconds.
|
|
1421
|
+
* @param maxFrames Candidate budget, defensively raised to at least two.
|
|
1422
|
+
* @param fps Positive requested sampling frequency.
|
|
1423
|
+
* @returns Contiguous plan indices with grid-aligned seeks and overlap-inclusive limits.
|
|
1104
1424
|
*/
|
|
1105
1425
|
function computeSegmentPlan(totalDuration, maxSegmentDuration, maxFrames, fps) {
|
|
1106
|
-
const
|
|
1426
|
+
const frameLimit = Math.max(2, maxFrames);
|
|
1427
|
+
const effectiveFps = Math.min(fps, frameLimit / totalDuration);
|
|
1107
1428
|
if (totalDuration <= maxSegmentDuration) return [{
|
|
1108
1429
|
index: 0,
|
|
1109
1430
|
startTime: 0,
|
|
1110
1431
|
endTime: totalDuration,
|
|
1111
1432
|
duration: totalDuration,
|
|
1112
|
-
allocatedFrames:
|
|
1433
|
+
allocatedFrames: frameLimit,
|
|
1113
1434
|
effectiveFps,
|
|
1114
1435
|
overlapBefore: 0,
|
|
1115
1436
|
overlapAfter: 0,
|
|
1116
1437
|
extractStartTime: 0,
|
|
1117
1438
|
extractDuration: totalDuration
|
|
1118
1439
|
}];
|
|
1119
|
-
const segmentCount = Math.ceil(totalDuration / maxSegmentDuration);
|
|
1120
|
-
const overlapTime = 1 / effectiveFps;
|
|
1121
1440
|
const segments = [];
|
|
1122
|
-
for (let
|
|
1123
|
-
const
|
|
1124
|
-
|
|
1125
|
-
const
|
|
1126
|
-
const
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1441
|
+
for (let slot = 0; slot < frameLimit; slot++) {
|
|
1442
|
+
const timestamp = slot / effectiveFps;
|
|
1443
|
+
if (timestamp >= totalDuration) break;
|
|
1444
|
+
const startTime = Math.floor(timestamp / maxSegmentDuration) * maxSegmentDuration;
|
|
1445
|
+
const previous = segments[segments.length - 1];
|
|
1446
|
+
if (previous?.startTime === startTime) {
|
|
1447
|
+
previous.allocatedFrames++;
|
|
1448
|
+
continue;
|
|
1449
|
+
}
|
|
1450
|
+
const endTime = Math.min(startTime + maxSegmentDuration, totalDuration);
|
|
1130
1451
|
segments.push({
|
|
1131
|
-
index:
|
|
1452
|
+
index: segments.length,
|
|
1132
1453
|
startTime,
|
|
1133
1454
|
endTime,
|
|
1134
|
-
duration,
|
|
1135
|
-
allocatedFrames:
|
|
1455
|
+
duration: endTime - startTime,
|
|
1456
|
+
allocatedFrames: 1,
|
|
1136
1457
|
effectiveFps,
|
|
1137
|
-
overlapBefore,
|
|
1138
|
-
overlapAfter,
|
|
1139
|
-
extractStartTime,
|
|
1140
|
-
extractDuration
|
|
1458
|
+
overlapBefore: 0,
|
|
1459
|
+
overlapAfter: 0,
|
|
1460
|
+
extractStartTime: timestamp,
|
|
1461
|
+
extractDuration: 0
|
|
1141
1462
|
});
|
|
1142
1463
|
}
|
|
1143
|
-
|
|
1144
|
-
|
|
1464
|
+
let firstSlot = 0;
|
|
1465
|
+
for (const segment of segments) {
|
|
1466
|
+
const nextSlot = firstSlot + segment.allocatedFrames;
|
|
1467
|
+
segment.overlapBefore = segment.index > 0 ? 1 : 0;
|
|
1468
|
+
segment.overlapAfter = segment.index < segments.length - 1 ? 1 : 0;
|
|
1469
|
+
segment.extractStartTime = (firstSlot - segment.overlapBefore) / effectiveFps;
|
|
1470
|
+
segment.extractDuration = (segment.overlapAfter ? Math.min(totalDuration, (nextSlot + 1) / effectiveFps) : totalDuration) - segment.extractStartTime;
|
|
1471
|
+
segment.allocatedFrames += segment.overlapBefore + segment.overlapAfter;
|
|
1472
|
+
firstSlot = nextSlot;
|
|
1473
|
+
}
|
|
1145
1474
|
return segments;
|
|
1146
1475
|
}
|
|
1147
1476
|
/**
|
|
@@ -1160,44 +1489,58 @@ function collectAllFrames(segmentResults) {
|
|
|
1160
1489
|
return allFrames;
|
|
1161
1490
|
}
|
|
1162
1491
|
/**
|
|
1163
|
-
* Sort frames
|
|
1164
|
-
*
|
|
1165
|
-
*
|
|
1492
|
+
* Sort frames in place and alias overlap duplicates to the first survivor.
|
|
1493
|
+
* @param frames Collected entries whose segment index and local ID identify a frame.
|
|
1494
|
+
* @param effectiveFps Positive sampling frequency; half a frame interval is the threshold.
|
|
1495
|
+
* @returns Timestamp-ordered survivors and duplicate keys pointing directly to survivor keys.
|
|
1166
1496
|
*/
|
|
1167
1497
|
function deduplicateFrames(frames, effectiveFps) {
|
|
1168
1498
|
frames.sort((a, b) => a.frame.timestamp - b.frame.timestamp);
|
|
1169
1499
|
const dupThreshold = 1 / (effectiveFps * 2);
|
|
1170
1500
|
const unique = [];
|
|
1501
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
1171
1502
|
for (const entry of frames) {
|
|
1172
1503
|
if (unique.length > 0) {
|
|
1173
1504
|
const last = unique[unique.length - 1];
|
|
1174
|
-
if (Math.abs(entry.frame.timestamp - last.frame.timestamp) < dupThreshold)
|
|
1505
|
+
if (Math.abs(entry.frame.timestamp - last.frame.timestamp) < dupThreshold) {
|
|
1506
|
+
aliases.set(`${entry.segmentIndex}:${entry.localId}`, `${last.segmentIndex}:${last.localId}`);
|
|
1507
|
+
continue;
|
|
1508
|
+
}
|
|
1175
1509
|
}
|
|
1176
1510
|
unique.push(entry);
|
|
1177
1511
|
}
|
|
1178
|
-
return
|
|
1512
|
+
return {
|
|
1513
|
+
unique,
|
|
1514
|
+
aliases
|
|
1515
|
+
};
|
|
1179
1516
|
}
|
|
1180
1517
|
/**
|
|
1181
|
-
* Assign sequential global IDs
|
|
1182
|
-
*
|
|
1518
|
+
* Assign sequential global IDs and retain duplicate local IDs as aliases.
|
|
1519
|
+
* @param uniqueFrames Timestamp-ordered survivors with distinct segment/local keys.
|
|
1520
|
+
* @param aliases Duplicate keys pointing directly to keys in uniqueFrames.
|
|
1521
|
+
* @returns Remapped frames and a global ID lookup covering survivors and duplicates.
|
|
1183
1522
|
*/
|
|
1184
|
-
function remapFrameIds(uniqueFrames) {
|
|
1523
|
+
function remapFrameIds(uniqueFrames, aliases) {
|
|
1185
1524
|
const globalIdMap = /* @__PURE__ */ new Map();
|
|
1525
|
+
const frames = uniqueFrames.map((entry, globalId) => {
|
|
1526
|
+
globalIdMap.set(`${entry.segmentIndex}:${entry.localId}`, globalId);
|
|
1527
|
+
return {
|
|
1528
|
+
id: globalId,
|
|
1529
|
+
timestamp: entry.frame.timestamp,
|
|
1530
|
+
extractPath: entry.frame.extractPath
|
|
1531
|
+
};
|
|
1532
|
+
});
|
|
1533
|
+
for (const [alias, survivor] of aliases) globalIdMap.set(alias, globalIdMap.get(survivor));
|
|
1186
1534
|
return {
|
|
1187
|
-
frames
|
|
1188
|
-
globalIdMap.set(`${entry.segmentIndex}:${entry.localId}`, globalId);
|
|
1189
|
-
return {
|
|
1190
|
-
id: globalId,
|
|
1191
|
-
timestamp: entry.frame.timestamp,
|
|
1192
|
-
extractPath: entry.frame.extractPath
|
|
1193
|
-
};
|
|
1194
|
-
}),
|
|
1535
|
+
frames,
|
|
1195
1536
|
globalIdMap
|
|
1196
1537
|
};
|
|
1197
1538
|
}
|
|
1198
1539
|
/**
|
|
1199
|
-
* Remap
|
|
1200
|
-
*
|
|
1540
|
+
* Remap edges, dropping missing endpoints and self loops while keeping the highest pair score.
|
|
1541
|
+
* @param segmentResults Segment-local edges in encounter order.
|
|
1542
|
+
* @param globalIdMap Survivor and duplicate local keys mapped to global IDs.
|
|
1543
|
+
* @returns One edge per surviving directed pair without changing its score.
|
|
1201
1544
|
*/
|
|
1202
1545
|
function remapEdges(segmentResults, globalIdMap) {
|
|
1203
1546
|
const edges = [];
|
|
@@ -1206,6 +1549,7 @@ function remapEdges(segmentResults, globalIdMap) {
|
|
|
1206
1549
|
const newSourceId = globalIdMap.get(`${result.segment.index}:${edge.sourceId}`);
|
|
1207
1550
|
const newTargetId = globalIdMap.get(`${result.segment.index}:${edge.targetId}`);
|
|
1208
1551
|
if (newSourceId === void 0 || newTargetId === void 0) continue;
|
|
1552
|
+
if (newSourceId === newTargetId) continue;
|
|
1209
1553
|
const edgeKey = `${newSourceId}-${newTargetId}`;
|
|
1210
1554
|
const existingIdx = edgeMap.get(edgeKey);
|
|
1211
1555
|
if (existingIdx !== void 0) {
|
|
@@ -1226,42 +1570,52 @@ function remapEdges(segmentResults, globalIdMap) {
|
|
|
1226
1570
|
return edges;
|
|
1227
1571
|
}
|
|
1228
1572
|
/**
|
|
1229
|
-
* Remap
|
|
1230
|
-
*
|
|
1573
|
+
* Remap animations, dropping missing or collapsed endpoints and retaining the first pair entry.
|
|
1574
|
+
* @param segmentResults Segment-local tracker entries in encounter order.
|
|
1575
|
+
* @param globalIdMap Survivor and duplicate local keys mapped to global IDs.
|
|
1576
|
+
* @returns One animation per directed pair with its original tracker duration and metadata.
|
|
1231
1577
|
*/
|
|
1232
1578
|
function remapAnimations(segmentResults, globalIdMap) {
|
|
1233
|
-
const animations =
|
|
1579
|
+
const animations = /* @__PURE__ */ new Map();
|
|
1234
1580
|
for (const result of segmentResults) for (const anim of result.animations) {
|
|
1235
1581
|
const newStartId = globalIdMap.get(`${result.segment.index}:${anim.startFrameId}`);
|
|
1236
1582
|
const newEndId = globalIdMap.get(`${result.segment.index}:${anim.endFrameId}`);
|
|
1237
1583
|
if (newStartId === void 0 || newEndId === void 0) continue;
|
|
1238
|
-
|
|
1584
|
+
if (newStartId === newEndId) continue;
|
|
1585
|
+
const animationKey = `${newStartId}-${newEndId}`;
|
|
1586
|
+
if (animations.has(animationKey)) continue;
|
|
1587
|
+
animations.set(animationKey, {
|
|
1239
1588
|
...anim,
|
|
1240
1589
|
startFrameId: newStartId,
|
|
1241
1590
|
endFrameId: newEndId
|
|
1242
1591
|
});
|
|
1243
1592
|
}
|
|
1244
|
-
return animations;
|
|
1593
|
+
return [...animations.values()];
|
|
1245
1594
|
}
|
|
1246
1595
|
/**
|
|
1247
1596
|
* Merge multiple segment results into a single unified frame/edge/animation set.
|
|
1248
|
-
*
|
|
1249
|
-
* -
|
|
1250
|
-
*
|
|
1251
|
-
* - Duplicate edges keep higher score
|
|
1597
|
+
* @param segmentResults Local frames, edges and tracker entries with distinct segment indices.
|
|
1598
|
+
* @returns Global timestamp-ordered frames, aliased edges and animations without self loops.
|
|
1599
|
+
* Duplicate edges keep the higher score; duplicate animations keep the first tracker entry.
|
|
1252
1600
|
*/
|
|
1253
1601
|
function mergeSegmentFrames(segmentResults) {
|
|
1254
1602
|
if (segmentResults.length === 0) return {
|
|
1255
1603
|
frames: [],
|
|
1256
1604
|
edges: [],
|
|
1257
|
-
animations: []
|
|
1605
|
+
animations: [],
|
|
1606
|
+
analysisResolution: {
|
|
1607
|
+
width: 0,
|
|
1608
|
+
height: 0
|
|
1609
|
+
}
|
|
1258
1610
|
};
|
|
1259
1611
|
const effectiveFps = segmentResults[0].segment.effectiveFps;
|
|
1260
|
-
const {
|
|
1612
|
+
const { unique, aliases } = deduplicateFrames(collectAllFrames(segmentResults), effectiveFps);
|
|
1613
|
+
const { frames, globalIdMap } = remapFrameIds(unique, aliases);
|
|
1261
1614
|
return {
|
|
1262
1615
|
frames,
|
|
1263
1616
|
edges: remapEdges(segmentResults, globalIdMap),
|
|
1264
|
-
animations: remapAnimations(segmentResults, globalIdMap)
|
|
1617
|
+
animations: remapAnimations(segmentResults, globalIdMap),
|
|
1618
|
+
analysisResolution: segmentResults[0].analysisResolution
|
|
1265
1619
|
};
|
|
1266
1620
|
}
|
|
1267
1621
|
function buildSegmentContext(segment, frames, segmentWorkspacePath, resolvedOptions, onProgress) {
|
|
@@ -1272,6 +1626,7 @@ function buildSegmentContext(segment, frames, segmentWorkspacePath, resolvedOpti
|
|
|
1272
1626
|
maxFrames: segment.allocatedFrames
|
|
1273
1627
|
},
|
|
1274
1628
|
workspacePath: segmentWorkspacePath,
|
|
1629
|
+
effectiveFps: segment.effectiveFps,
|
|
1275
1630
|
frames,
|
|
1276
1631
|
graph: [],
|
|
1277
1632
|
status: "ANALYZING",
|
|
@@ -1283,19 +1638,24 @@ function buildSegmentContext(segment, frames, segmentWorkspacePath, resolvedOpti
|
|
|
1283
1638
|
* Each segment uses an isolated workspace directory.
|
|
1284
1639
|
*/
|
|
1285
1640
|
async function processSegment(inputPath, segment, workspacePath, resolvedOptions, onProgress) {
|
|
1286
|
-
const frames = await extractFramesForRange(inputPath, join(workspacePath, "frames"), segment.effectiveFps, resolvedOptions.scale, segment.extractStartTime, segment.extractDuration);
|
|
1641
|
+
const frames = await extractFramesForRange(inputPath, join(workspacePath, "frames"), segment.effectiveFps, resolvedOptions.scale, segment.extractStartTime, segment.extractDuration, segment.allocatedFrames);
|
|
1287
1642
|
if (frames.length < 2) return {
|
|
1288
1643
|
segment,
|
|
1289
1644
|
frames,
|
|
1290
1645
|
edges: [],
|
|
1291
|
-
animations: []
|
|
1646
|
+
animations: [],
|
|
1647
|
+
analysisResolution: {
|
|
1648
|
+
width: 0,
|
|
1649
|
+
height: 0
|
|
1650
|
+
}
|
|
1292
1651
|
};
|
|
1293
|
-
const { edges, animations } = await analyzeFrames(buildSegmentContext(segment, frames, workspacePath, resolvedOptions, onProgress));
|
|
1652
|
+
const { edges, animations, analysisResolution } = await analyzeFrames(buildSegmentContext(segment, frames, workspacePath, resolvedOptions, onProgress));
|
|
1294
1653
|
return {
|
|
1295
1654
|
segment,
|
|
1296
1655
|
frames,
|
|
1297
1656
|
edges,
|
|
1298
|
-
animations
|
|
1657
|
+
animations,
|
|
1658
|
+
analysisResolution
|
|
1299
1659
|
};
|
|
1300
1660
|
}
|
|
1301
1661
|
/**
|
|
@@ -1335,7 +1695,7 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1335
1695
|
});
|
|
1336
1696
|
})));
|
|
1337
1697
|
options.onProgress?.("ANALYZING", 100);
|
|
1338
|
-
const { frames, edges, animations } = mergeSegmentFrames(results);
|
|
1698
|
+
const { frames, edges, animations, analysisResolution } = mergeSegmentFrames(results);
|
|
1339
1699
|
logger.debug(`Merged: ${frames.length} frames, ${edges.length} edges, ${animations.length} animations`);
|
|
1340
1700
|
options.onProgress?.("PRUNING", 0);
|
|
1341
1701
|
const survivingIds = pruneByThresholdWithCap(edges, frames, resolvedOptions.threshold, resolvedOptions.count);
|
|
@@ -1344,6 +1704,9 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1344
1704
|
options.onProgress?.("FINALIZING", 0);
|
|
1345
1705
|
const ctx = {
|
|
1346
1706
|
options: resolvedOptions,
|
|
1707
|
+
effectiveFps: segments[0]?.effectiveFps,
|
|
1708
|
+
sourceDurationSec: totalDuration,
|
|
1709
|
+
analysisResolution,
|
|
1347
1710
|
workspacePath: mainWorkspace,
|
|
1348
1711
|
frames,
|
|
1349
1712
|
graph: edges,
|
|
@@ -1357,21 +1720,14 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1357
1720
|
else outputFiles = await finalizeOutput(ctx, prunedFrames);
|
|
1358
1721
|
options.onProgress?.("FINALIZING", 100);
|
|
1359
1722
|
logger.success(`Segmented pipeline: ${prunedFrames.length} scenes from ${frames.length} frames (${segments.length} segments)`);
|
|
1723
|
+
const outputMetadata = await buildVideoMetadata(ctx, prunedFrames, analysisResolution);
|
|
1360
1724
|
return {
|
|
1361
1725
|
success: true,
|
|
1362
1726
|
originalFramesCount: frames.length,
|
|
1363
1727
|
prunedFramesCount: prunedFrames.length,
|
|
1364
1728
|
outputFiles,
|
|
1365
1729
|
outputBuffers,
|
|
1366
|
-
|
|
1367
|
-
video: {
|
|
1368
|
-
originalDurationMs: totalDuration * 1e3,
|
|
1369
|
-
fps: resolvedOptions.fps,
|
|
1370
|
-
resolution: {
|
|
1371
|
-
width: resolvedOptions.scale,
|
|
1372
|
-
height: Math.round(resolvedOptions.scale * 9 / 16)
|
|
1373
|
-
}
|
|
1374
|
-
},
|
|
1730
|
+
...outputMetadata,
|
|
1375
1731
|
executionTimeMs: Date.now() - pipelineStart
|
|
1376
1732
|
};
|
|
1377
1733
|
} catch (error) {
|
|
@@ -1385,9 +1741,9 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1385
1741
|
}
|
|
1386
1742
|
|
|
1387
1743
|
//#endregion
|
|
1388
|
-
//#region src/core/orchestrator.ts
|
|
1744
|
+
//#region src/core/orchestrator/orchestrator.ts
|
|
1389
1745
|
async function runPipeline(options) {
|
|
1390
|
-
|
|
1746
|
+
setDebugMode(options.debug ?? false);
|
|
1391
1747
|
const resolvedOptions = resolveOptions(options);
|
|
1392
1748
|
if (shouldSegment(resolvedOptions, options)) return runSegmentedPipeline(options, resolvedOptions);
|
|
1393
1749
|
const startTime = Date.now();
|
|
@@ -1407,19 +1763,27 @@ async function runPipeline(options) {
|
|
|
1407
1763
|
logger.debug(`Workspace created: ${ctx.workspacePath}`);
|
|
1408
1764
|
ctx.status = "EXTRACTING";
|
|
1409
1765
|
const { frames: resolvedFrames, resolvedInputPath } = await resolveInput(options, ctx.workspacePath);
|
|
1410
|
-
if (resolvedOptions.mode === "frames")
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1766
|
+
if (resolvedOptions.mode === "frames") {
|
|
1767
|
+
ctx.frames = resolvedFrames;
|
|
1768
|
+
ctx.effectiveFps = 1;
|
|
1769
|
+
} else {
|
|
1770
|
+
const extractCtx = {
|
|
1771
|
+
...ctx,
|
|
1772
|
+
options: {
|
|
1773
|
+
...resolvedOptions,
|
|
1774
|
+
inputPath: resolvedInputPath
|
|
1775
|
+
}
|
|
1776
|
+
};
|
|
1777
|
+
ctx.frames = await extractFrames(extractCtx);
|
|
1778
|
+
ctx.effectiveFps = extractCtx.effectiveFps;
|
|
1779
|
+
ctx.sourceDurationSec = extractCtx.sourceDurationSec;
|
|
1780
|
+
}
|
|
1418
1781
|
ctx.emitProgress(100);
|
|
1419
1782
|
ctx.status = "ANALYZING";
|
|
1420
|
-
const { edges, animations } = await analyzeFrames(ctx);
|
|
1783
|
+
const { edges, animations, analysisResolution } = await analyzeFrames(ctx);
|
|
1421
1784
|
ctx.graph = edges;
|
|
1422
1785
|
ctx.animations = animations;
|
|
1786
|
+
ctx.analysisResolution = analysisResolution;
|
|
1423
1787
|
ctx.status = "PRUNING";
|
|
1424
1788
|
const survivingIds = pruneByThresholdWithCap(ctx.graph, ctx.frames, resolvedOptions.threshold, resolvedOptions.count);
|
|
1425
1789
|
const prunedFrames = filter(ctx.frames, (f) => survivingIds.has(f.id));
|
|
@@ -1436,21 +1800,14 @@ async function runPipeline(options) {
|
|
|
1436
1800
|
}
|
|
1437
1801
|
ctx.status = "SUCCESS";
|
|
1438
1802
|
logger.success(`Extracted ${prunedFrames.length} scenes from ${ctx.frames.length} frames`);
|
|
1803
|
+
const metadata = await buildVideoMetadata(ctx, prunedFrames, analysisResolution);
|
|
1439
1804
|
return {
|
|
1440
1805
|
success: true,
|
|
1441
1806
|
originalFramesCount: ctx.frames.length,
|
|
1442
1807
|
prunedFramesCount: prunedFrames.length,
|
|
1443
1808
|
outputFiles,
|
|
1444
1809
|
outputBuffers,
|
|
1445
|
-
|
|
1446
|
-
video: {
|
|
1447
|
-
originalDurationMs: ctx.frames.length / ctx.options.fps * 1e3,
|
|
1448
|
-
fps: ctx.options.fps,
|
|
1449
|
-
resolution: {
|
|
1450
|
-
width: ctx.options.scale,
|
|
1451
|
-
height: Math.round(ctx.options.scale * 9 / 16)
|
|
1452
|
-
}
|
|
1453
|
-
},
|
|
1810
|
+
...metadata,
|
|
1454
1811
|
executionTimeMs: Date.now() - startTime
|
|
1455
1812
|
};
|
|
1456
1813
|
} catch (error) {
|
|
@@ -1465,7 +1822,7 @@ async function runPipeline(options) {
|
|
|
1465
1822
|
}
|
|
1466
1823
|
|
|
1467
1824
|
//#endregion
|
|
1468
|
-
//#region src/core/pipeline-worker.ts
|
|
1825
|
+
//#region src/core/orchestrator/worker/pipeline-worker.ts
|
|
1469
1826
|
runPipeline({
|
|
1470
1827
|
...workerData,
|
|
1471
1828
|
onProgress: (phase, percent) => {
|