@lumy-pack/scene-sieve 0.0.14 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -24
- package/dist/{errors.d.ts → cli/errors/classify-error.d.ts} +5 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/{utils → cli/options}/parse-options.d.ts +6 -1
- package/dist/cli.mjs +2437 -2126
- package/dist/constants/pipeline-defaults.d.ts +13 -0
- package/dist/core/{analyzer.d.ts → analyzer/analyzer.d.ts} +11 -6
- package/dist/core/{dbscan.d.ts → analyzer/clustering/dbscan.d.ts} +1 -1
- package/dist/core/analyzer/constants/vision-tuning.d.ts +12 -0
- package/dist/core/analyzer/features/feature-diff.d.ts +14 -0
- package/dist/core/analyzer/features/frame-features.d.ts +32 -0
- package/dist/core/analyzer/index.d.ts +3 -0
- package/dist/core/constants/workspace-layout.d.ts +9 -0
- package/dist/core/{extractor.d.ts → extractor/extractor.d.ts} +6 -4
- package/dist/core/extractor/index.d.ts +1 -0
- package/dist/core/index.d.ts +8 -9
- package/dist/core/input-resolver/index.d.ts +1 -0
- package/dist/core/{input-resolver.d.ts → input-resolver/input-resolver.d.ts} +11 -1
- package/dist/core/input-resolver/validation/validate-options.d.ts +8 -0
- package/dist/core/orchestrator/index.d.ts +3 -0
- package/dist/core/{orchestrator.d.ts → orchestrator/orchestrator.d.ts} +1 -1
- package/dist/core/{run-in-worker.d.ts → orchestrator/worker/run-in-worker.d.ts} +5 -1
- package/dist/core/pruner/index.d.ts +1 -0
- package/dist/core/{pruner.d.ts → pruner/pruner.d.ts} +2 -2
- package/dist/{utils/math.d.ts → core/pruner/scoring/normalize-scores.d.ts} +4 -0
- package/dist/core/segmenter/index.d.ts +1 -0
- package/dist/core/{segmenter.d.ts → segmenter/segmenter.d.ts} +11 -11
- package/dist/core/utils/metadata/build-video-metadata.d.ts +14 -0
- package/dist/core/workspace/index.d.ts +1 -0
- package/dist/core/{workspace.d.ts → workspace/workspace.d.ts} +1 -1
- package/dist/index.cjs +1854 -1486
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1836 -1455
- package/dist/pipeline-worker.mjs +1734 -1474
- package/dist/types/index.d.ts +25 -0
- package/package.json +5 -4
- package/dist/constants.d.ts +0 -32
- /package/dist/{commands → cli/commands}/Sieve.d.ts +0 -0
- /package/dist/{utils → cli/commands}/command-registry.d.ts +0 -0
- /package/dist/{components → cli/components}/PhaseStep.d.ts +0 -0
- /package/dist/{components → cli/components}/ProgressBar.d.ts +0 -0
- /package/dist/core/{pipeline-worker.d.ts → orchestrator/worker/pipeline-worker.d.ts} +0 -0
- /package/dist/{utils → core/pruner/heap}/min-heap.d.ts +0 -0
- /package/dist/{utils → core/segmenter/scheduling}/concurrency.d.ts +0 -0
- /package/dist/{utils → core/utils/filesystem}/paths.d.ts +0 -0
- /package/dist/{utils → logging}/logger.d.ts +0 -0
package/dist/pipeline-worker.mjs
CHANGED
|
@@ -1,1588 +1,1848 @@
|
|
|
1
|
-
|
|
2
|
-
import { parentPort, workerData } from "worker_threads";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
6
|
-
import { filter as filter6 } from "@winglet/common-utils";
|
|
7
|
-
|
|
8
|
-
// src/utils/logger.ts
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { filter, map } from "@winglet/common-utils";
|
|
9
5
|
import pc from "picocolors";
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
import sharp from "sharp";
|
|
7
|
+
import { mkdir, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
8
|
+
import { basename, extname, join, resolve } from "node:path";
|
|
9
|
+
import { path } from "@ffprobe-installer/ffprobe";
|
|
10
|
+
import { execa } from "execa";
|
|
11
|
+
import ffmpegPath from "ffmpeg-static";
|
|
12
|
+
import { homedir, tmpdir } from "node:os";
|
|
13
|
+
|
|
14
|
+
//#region src/logging/logger.ts
|
|
15
|
+
let debugMode = false;
|
|
16
|
+
let jsonMode = false;
|
|
12
17
|
function setDebugMode(enabled) {
|
|
13
|
-
|
|
18
|
+
debugMode = enabled;
|
|
14
19
|
}
|
|
15
20
|
function timestamp() {
|
|
16
|
-
|
|
21
|
+
return (/* @__PURE__ */ new Date()).toLocaleTimeString("en-US", { hour12: false });
|
|
17
22
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
console.warn(`${pc.yellow("warn")} ${message}`);
|
|
38
|
-
},
|
|
39
|
-
error(message) {
|
|
40
|
-
console.error(`${pc.red("error")} ${message}`);
|
|
41
|
-
},
|
|
42
|
-
debug(message) {
|
|
43
|
-
if (debugMode) {
|
|
44
|
-
if (jsonMode) {
|
|
45
|
-
process.stderr.write(`${pc.gray(`[${timestamp()}] debug`)} ${message}
|
|
46
|
-
`);
|
|
47
|
-
} else {
|
|
48
|
-
console.log(`${pc.gray(`[${timestamp()}] debug`)} ${message}`);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
23
|
+
const logger = {
|
|
24
|
+
info(message) {
|
|
25
|
+
if (jsonMode) process.stderr.write(`${pc.blue("info")} ${message}\n`);
|
|
26
|
+
else console.log(`${pc.blue("info")} ${message}`);
|
|
27
|
+
},
|
|
28
|
+
success(message) {
|
|
29
|
+
if (jsonMode) process.stderr.write(`${pc.green("done")} ${message}\n`);
|
|
30
|
+
else console.log(`\n${pc.green("done")} ${message}`);
|
|
31
|
+
},
|
|
32
|
+
warn(message) {
|
|
33
|
+
console.warn(`${pc.yellow("warn")} ${message}`);
|
|
34
|
+
},
|
|
35
|
+
error(message) {
|
|
36
|
+
console.error(`${pc.red("error")} ${message}`);
|
|
37
|
+
},
|
|
38
|
+
debug(message) {
|
|
39
|
+
if (debugMode) if (jsonMode) process.stderr.write(`${pc.gray(`[${timestamp()}] debug`)} ${message}\n`);
|
|
40
|
+
else console.log(`${pc.gray(`[${timestamp()}] debug`)} ${message}`);
|
|
41
|
+
}
|
|
52
42
|
};
|
|
53
43
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
44
|
+
//#endregion
|
|
45
|
+
//#region src/constants/pipeline-defaults.ts
|
|
46
|
+
const DEFAULT_THRESHOLD = .5;
|
|
47
|
+
const IOU_THRESHOLD = .9;
|
|
58
48
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
var DEFAULT_THRESHOLD = 0.5;
|
|
65
|
-
var DEFAULT_FPS = 5;
|
|
66
|
-
var DEFAULT_SCALE = 720;
|
|
67
|
-
var DEFAULT_QUALITY = 80;
|
|
68
|
-
var DEFAULT_MAX_FRAMES = 300;
|
|
69
|
-
var NORMALIZATION_LOGISTIC_K = 3;
|
|
70
|
-
var NORMALIZATION_ALPHA = 0.4;
|
|
71
|
-
var NORMALIZATION_MAD_COEFFICIENT = 1.4826;
|
|
72
|
-
var NORMALIZATION_MIN_SAMPLE_SIZE = 10;
|
|
73
|
-
var WORKSPACE_PREFIX = `${APP_NAME}-`;
|
|
74
|
-
var TEMP_BASE_DIR = tmpdir();
|
|
75
|
-
var FRAME_OUTPUT_EXTENSION = ".jpg";
|
|
76
|
-
var FRAME_FILENAME_PATTERN = "frame_%06d.jpg";
|
|
77
|
-
var OPENCV_BATCH_SIZE = 10;
|
|
78
|
-
var DBSCAN_ALPHA = 0.03;
|
|
79
|
-
var DBSCAN_MIN_PTS = 4;
|
|
80
|
-
var IOU_THRESHOLD = 0.9;
|
|
81
|
-
var DECAY_LAMBDA = 0.95;
|
|
82
|
-
var ANIMATION_FRAME_THRESHOLD = 5;
|
|
83
|
-
var MATCH_DISTANCE_THRESHOLD = 0.25;
|
|
84
|
-
var PIXELDIFF_GAUSSIAN_KERNEL = 3;
|
|
85
|
-
var PIXELDIFF_BINARY_THRESHOLD = 30;
|
|
86
|
-
var PIXELDIFF_CONTOUR_MIN_AREA = 100;
|
|
87
|
-
var PIXELDIFF_SAMPLE_SPACING = 8;
|
|
88
|
-
var DEFAULT_MAX_SEGMENT_DURATION = 300;
|
|
89
|
-
var DEFAULT_SEGMENT_CONCURRENCY = 2;
|
|
90
|
-
function getTempWorkspaceDir(sessionId) {
|
|
91
|
-
return join(TEMP_BASE_DIR, `${WORKSPACE_PREFIX}${sessionId}`);
|
|
92
|
-
}
|
|
49
|
+
//#endregion
|
|
50
|
+
//#region src/core/analyzer/constants/vision-tuning.ts
|
|
51
|
+
const DBSCAN_ALPHA = .03;
|
|
52
|
+
const DECAY_LAMBDA = .95;
|
|
53
|
+
const MATCH_DISTANCE_THRESHOLD = .25;
|
|
93
54
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
55
|
+
//#endregion
|
|
56
|
+
//#region src/core/analyzer/clustering/dbscan.ts
|
|
57
|
+
const UNVISITED = -2;
|
|
58
|
+
const NOISE = -1;
|
|
59
|
+
/**
|
|
60
|
+
* DBSCAN clustering with resolution-independent eps.
|
|
61
|
+
* eps = alpha * sqrt(width^2 + height^2)
|
|
62
|
+
*/
|
|
97
63
|
function dbscan(points, imageWidth, imageHeight, alpha, minPts) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
64
|
+
if (points.length === 0) return {
|
|
65
|
+
labels: [],
|
|
66
|
+
boundingBoxes: []
|
|
67
|
+
};
|
|
68
|
+
const eps = (alpha ?? .03) * Math.sqrt(imageWidth ** 2 + imageHeight ** 2);
|
|
69
|
+
const epsSquared = eps * eps;
|
|
70
|
+
const minPoints = minPts ?? 4;
|
|
71
|
+
const labels = new Array(points.length).fill(UNVISITED);
|
|
72
|
+
let clusterId = 0;
|
|
73
|
+
for (let i = 0; i < points.length; i++) {
|
|
74
|
+
if (labels[i] !== UNVISITED) continue;
|
|
75
|
+
const neighbors = findNeighbors(points, i, epsSquared);
|
|
76
|
+
if (neighbors.length < minPoints) {
|
|
77
|
+
labels[i] = NOISE;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
labels[i] = clusterId;
|
|
81
|
+
const seeds = [...neighbors];
|
|
82
|
+
const seedSet = new Set(seeds);
|
|
83
|
+
for (let si = 0; si < seeds.length; si++) {
|
|
84
|
+
const q = seeds[si];
|
|
85
|
+
if (labels[q] === NOISE) labels[q] = clusterId;
|
|
86
|
+
if (labels[q] !== UNVISITED) continue;
|
|
87
|
+
labels[q] = clusterId;
|
|
88
|
+
const qNeighbors = findNeighbors(points, q, epsSquared);
|
|
89
|
+
if (qNeighbors.length >= minPoints) {
|
|
90
|
+
for (const n of qNeighbors) if (!seedSet.has(n)) {
|
|
91
|
+
seedSet.add(n);
|
|
92
|
+
seeds.push(n);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
clusterId++;
|
|
97
|
+
}
|
|
98
|
+
const boundingBoxes = [];
|
|
99
|
+
for (let c = 0; c < clusterId; c++) {
|
|
100
|
+
let minX = Infinity;
|
|
101
|
+
let minY = Infinity;
|
|
102
|
+
let maxX = -Infinity;
|
|
103
|
+
let maxY = -Infinity;
|
|
104
|
+
for (let i = 0; i < points.length; i++) {
|
|
105
|
+
if (labels[i] !== c) continue;
|
|
106
|
+
const p = points[i];
|
|
107
|
+
if (p.x < minX) minX = p.x;
|
|
108
|
+
if (p.y < minY) minY = p.y;
|
|
109
|
+
if (p.x > maxX) maxX = p.x;
|
|
110
|
+
if (p.y > maxY) maxY = p.y;
|
|
111
|
+
}
|
|
112
|
+
boundingBoxes.push({
|
|
113
|
+
x: minX,
|
|
114
|
+
y: minY,
|
|
115
|
+
width: maxX - minX,
|
|
116
|
+
height: maxY - minY
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
labels,
|
|
121
|
+
boundingBoxes
|
|
122
|
+
};
|
|
157
123
|
}
|
|
158
124
|
function findNeighbors(points, idx, epsSquared) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return neighbors;
|
|
125
|
+
const p = points[idx];
|
|
126
|
+
const neighbors = [];
|
|
127
|
+
for (let i = 0; i < points.length; i++) {
|
|
128
|
+
if (i === idx) continue;
|
|
129
|
+
const q = points[i];
|
|
130
|
+
if ((p.x - q.x) ** 2 + (p.y - q.y) ** 2 <= epsSquared) neighbors.push(i);
|
|
131
|
+
}
|
|
132
|
+
return neighbors;
|
|
170
133
|
}
|
|
171
134
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
135
|
+
//#endregion
|
|
136
|
+
//#region src/core/analyzer/features/feature-diff.ts
|
|
137
|
+
/**
|
|
138
|
+
* Match prev to next with Hamming k=2, crossCheck=false and strict ratio 0.25.
|
|
139
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
140
|
+
* @param prev - Previous frame's live features, owned by the caller.
|
|
141
|
+
* @param next - Next frame's live features, owned by the caller.
|
|
142
|
+
* @returns Unmatched next-frame coordinates without changing input ownership.
|
|
143
|
+
* @throws Propagates matching errors after releasing temporary native handles.
|
|
144
|
+
*/
|
|
145
|
+
function computeNewPoints(cvLib, prev, next) {
|
|
146
|
+
let matcher = null;
|
|
147
|
+
let matches = null;
|
|
148
|
+
try {
|
|
149
|
+
const matchedIndices = /* @__PURE__ */ new Set();
|
|
150
|
+
if (prev.descriptors.rows > 0 && next.descriptors.rows > 0) try {
|
|
151
|
+
matcher = new cvLib.BFMatcher(cvLib.NORM_HAMMING, false);
|
|
152
|
+
matches = new cvLib.DMatchVectorVector();
|
|
153
|
+
matcher.knnMatch(prev.descriptors, next.descriptors, matches, 2);
|
|
154
|
+
for (let i = 0; i < matches.size(); i++) {
|
|
155
|
+
const pair = matches.get(i);
|
|
156
|
+
try {
|
|
157
|
+
if (pair.size() < 2) continue;
|
|
158
|
+
const best = pair.get(0);
|
|
159
|
+
const second = pair.get(1);
|
|
160
|
+
if (best.distance < .25 * second.distance) matchedIndices.add(best.trainIdx);
|
|
161
|
+
} finally {
|
|
162
|
+
pair.delete();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} finally {
|
|
166
|
+
matcher?.delete();
|
|
167
|
+
}
|
|
168
|
+
const points = [];
|
|
169
|
+
for (let i = 0; i < next.keypoints.size(); i++) if (!matchedIndices.has(i)) {
|
|
170
|
+
const { x, y } = next.keypoints.get(i).pt;
|
|
171
|
+
points.push({
|
|
172
|
+
x,
|
|
173
|
+
y
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
return points;
|
|
177
|
+
} finally {
|
|
178
|
+
matches?.delete();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
//#endregion
|
|
183
|
+
//#region src/core/analyzer/features/frame-features.ts
|
|
184
|
+
/**
|
|
185
|
+
* Detect one frame's features without retaining its image or mask.
|
|
186
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
187
|
+
* @param akaze - Detector owned and released by the caller.
|
|
188
|
+
* @param frame - Grayscale bytes with matching width and height.
|
|
189
|
+
* @returns Feature handles that the caller must delete.
|
|
190
|
+
* @throws Propagates native errors after releasing partial allocations.
|
|
191
|
+
*/
|
|
192
|
+
function computeFrameFeatures(cvLib, akaze, frame) {
|
|
193
|
+
let image = null;
|
|
194
|
+
let mask = null;
|
|
195
|
+
let keypoints = null;
|
|
196
|
+
let descriptors = null;
|
|
197
|
+
try {
|
|
198
|
+
image = new cvLib.Mat(frame.height, frame.width, cvLib.CV_8UC1);
|
|
199
|
+
image.data.set(frame.data);
|
|
200
|
+
mask = new cvLib.Mat();
|
|
201
|
+
keypoints = new cvLib.KeyPointVector();
|
|
202
|
+
descriptors = new cvLib.Mat();
|
|
203
|
+
akaze.detectAndCompute(image, mask, keypoints, descriptors);
|
|
204
|
+
const ownedKeypoints = keypoints;
|
|
205
|
+
const ownedDescriptors = descriptors;
|
|
206
|
+
let deleted = false;
|
|
207
|
+
const features = {
|
|
208
|
+
width: frame.width,
|
|
209
|
+
height: frame.height,
|
|
210
|
+
keypoints: ownedKeypoints,
|
|
211
|
+
descriptors: ownedDescriptors,
|
|
212
|
+
/** Release the transferred handles exactly once. */
|
|
213
|
+
delete() {
|
|
214
|
+
if (deleted) return;
|
|
215
|
+
deleted = true;
|
|
216
|
+
try {
|
|
217
|
+
ownedKeypoints.delete();
|
|
218
|
+
} finally {
|
|
219
|
+
ownedDescriptors.delete();
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
keypoints = null;
|
|
224
|
+
descriptors = null;
|
|
225
|
+
return features;
|
|
226
|
+
} finally {
|
|
227
|
+
image?.delete();
|
|
228
|
+
mask?.delete();
|
|
229
|
+
keypoints?.delete();
|
|
230
|
+
descriptors?.delete();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
//#endregion
|
|
235
|
+
//#region src/core/analyzer/analyzer.ts
|
|
236
|
+
const OPENCV_INIT_TIMEOUT_MS = 3e4;
|
|
237
|
+
const require = createRequire(import.meta.url);
|
|
238
|
+
let cvReady = null;
|
|
176
239
|
async function ensureOpenCV() {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
}
|
|
193
|
-
return cvReady;
|
|
240
|
+
if (!cvReady) cvReady = (async () => {
|
|
241
|
+
const cvObj = require("@techstark/opencv-js");
|
|
242
|
+
delete cvObj.then;
|
|
243
|
+
if (cvObj.Mat) return cvObj;
|
|
244
|
+
return new Promise((resolve, reject) => {
|
|
245
|
+
const timeout = setTimeout(() => {
|
|
246
|
+
reject(/* @__PURE__ */ new Error("OpenCV WASM initialization timed out after 30s"));
|
|
247
|
+
}, OPENCV_INIT_TIMEOUT_MS);
|
|
248
|
+
cvObj.onRuntimeInitialized = () => {
|
|
249
|
+
clearTimeout(timeout);
|
|
250
|
+
resolve(cvObj);
|
|
251
|
+
};
|
|
252
|
+
});
|
|
253
|
+
})();
|
|
254
|
+
return cvReady;
|
|
194
255
|
}
|
|
195
256
|
async function preprocessFrame(framePath, scale) {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
257
|
+
const { data, info } = await sharp(framePath).resize({
|
|
258
|
+
width: scale,
|
|
259
|
+
withoutEnlargement: true
|
|
260
|
+
}).grayscale().blur(1).raw().toBuffer({ resolveWithObject: true });
|
|
261
|
+
return {
|
|
262
|
+
data: new Uint8Array(data.buffer, data.byteOffset, data.byteLength),
|
|
263
|
+
width: info.width,
|
|
264
|
+
height: info.height
|
|
265
|
+
};
|
|
202
266
|
}
|
|
203
267
|
function computeIoU(a, b) {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
const aArea = a.width * a.height;
|
|
213
|
-
const bArea = b.width * b.height;
|
|
214
|
-
const union = aArea + bArea - intersection;
|
|
215
|
-
return union === 0 ? 0 : intersection / union;
|
|
268
|
+
const ix1 = Math.max(a.x, b.x);
|
|
269
|
+
const iy1 = Math.max(a.y, b.y);
|
|
270
|
+
const ix2 = Math.min(a.x + a.width, b.x + b.width);
|
|
271
|
+
const iy2 = Math.min(a.y + a.height, b.y + b.height);
|
|
272
|
+
const intersection = Math.max(0, ix2 - ix1) * Math.max(0, iy2 - iy1);
|
|
273
|
+
if (intersection === 0) return 0;
|
|
274
|
+
const union = a.width * a.height + b.width * b.height - intersection;
|
|
275
|
+
return union === 0 ? 0 : intersection / union;
|
|
216
276
|
}
|
|
217
277
|
var IoUTracker = class {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
return this.extractedAnimations;
|
|
298
|
-
}
|
|
299
|
-
getAnimationWeight(boxIndex, boxes) {
|
|
300
|
-
if (boxIndex >= boxes.length) return 0;
|
|
301
|
-
const box = boxes[boxIndex];
|
|
302
|
-
let maxWeight = 0;
|
|
303
|
-
for (const region of this.regions) {
|
|
304
|
-
if (region.consecutiveCount >= this.animationThreshold) {
|
|
305
|
-
const iou = computeIoU(box, region.box);
|
|
306
|
-
if (iou > this.iouThreshold) {
|
|
307
|
-
maxWeight = Math.max(maxWeight, region.weight);
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
return maxWeight;
|
|
312
|
-
}
|
|
278
|
+
fps;
|
|
279
|
+
iouThreshold;
|
|
280
|
+
animationThreshold;
|
|
281
|
+
regions = [];
|
|
282
|
+
extractedAnimations = [];
|
|
283
|
+
constructor(fps = 5, iouThreshold = IOU_THRESHOLD, animationThreshold = 5) {
|
|
284
|
+
this.fps = fps;
|
|
285
|
+
this.iouThreshold = iouThreshold;
|
|
286
|
+
this.animationThreshold = animationThreshold;
|
|
287
|
+
}
|
|
288
|
+
update(boxes, pairIndex) {
|
|
289
|
+
const animationIndices = /* @__PURE__ */ new Set();
|
|
290
|
+
const matched = /* @__PURE__ */ new Set();
|
|
291
|
+
for (let bi = 0; bi < boxes.length; bi++) {
|
|
292
|
+
const box = boxes[bi];
|
|
293
|
+
let bestIoU = 0;
|
|
294
|
+
let bestRegionIdx = -1;
|
|
295
|
+
for (let ri = 0; ri < this.regions.length; ri++) {
|
|
296
|
+
if (matched.has(ri)) continue;
|
|
297
|
+
const iou = computeIoU(box, this.regions[ri].box);
|
|
298
|
+
if (iou > bestIoU) {
|
|
299
|
+
bestIoU = iou;
|
|
300
|
+
bestRegionIdx = ri;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
if (bestIoU > this.iouThreshold && bestRegionIdx !== -1) {
|
|
304
|
+
const region = this.regions[bestRegionIdx];
|
|
305
|
+
const gap = pairIndex - region.lastSeen;
|
|
306
|
+
region.box = box;
|
|
307
|
+
region.consecutiveCount++;
|
|
308
|
+
region.lastSeen = pairIndex;
|
|
309
|
+
region.weight *= Math.pow(DECAY_LAMBDA, gap);
|
|
310
|
+
matched.add(bestRegionIdx);
|
|
311
|
+
if (region.consecutiveCount >= this.animationThreshold) animationIndices.add(bi);
|
|
312
|
+
} else this.regions.push({
|
|
313
|
+
box,
|
|
314
|
+
consecutiveCount: 1,
|
|
315
|
+
firstSeen: pairIndex,
|
|
316
|
+
lastSeen: pairIndex,
|
|
317
|
+
weight: 1
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
for (let ri = 0; ri < this.regions.length; ri++) if (!matched.has(ri)) {
|
|
321
|
+
const gap = pairIndex - this.regions[ri].lastSeen;
|
|
322
|
+
this.regions[ri].weight *= Math.pow(DECAY_LAMBDA, gap);
|
|
323
|
+
}
|
|
324
|
+
for (let i = 0; i < this.regions.length; i++) {
|
|
325
|
+
const region = this.regions[i];
|
|
326
|
+
if (region.weight <= .01 && !matched.has(i)) this.collectAnimation(region);
|
|
327
|
+
}
|
|
328
|
+
this.regions = filter(this.regions, (r, i) => r.weight > .01 || matched.has(i));
|
|
329
|
+
return animationIndices;
|
|
330
|
+
}
|
|
331
|
+
collectAnimation(region) {
|
|
332
|
+
if (region.consecutiveCount >= this.animationThreshold) {
|
|
333
|
+
const durationMs = region.consecutiveCount / this.fps * 1e3;
|
|
334
|
+
this.extractedAnimations.push({
|
|
335
|
+
type: "loading_spinner",
|
|
336
|
+
boundingBox: region.box,
|
|
337
|
+
startFrameId: region.firstSeen,
|
|
338
|
+
endFrameId: region.lastSeen,
|
|
339
|
+
durationMs
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
flushAndGetAnimations() {
|
|
344
|
+
for (const region of this.regions) this.collectAnimation(region);
|
|
345
|
+
this.regions = [];
|
|
346
|
+
return this.extractedAnimations;
|
|
347
|
+
}
|
|
348
|
+
getAnimationWeight(boxIndex, boxes) {
|
|
349
|
+
if (boxIndex >= boxes.length) return 0;
|
|
350
|
+
const box = boxes[boxIndex];
|
|
351
|
+
let maxWeight = 0;
|
|
352
|
+
for (const region of this.regions) if (region.consecutiveCount >= this.animationThreshold) {
|
|
353
|
+
if (computeIoU(box, region.box) > this.iouThreshold) maxWeight = Math.max(maxWeight, region.weight);
|
|
354
|
+
}
|
|
355
|
+
return maxWeight;
|
|
356
|
+
}
|
|
313
357
|
};
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
try {
|
|
336
|
-
matches = new cvLib.DMatchVectorVector();
|
|
337
|
-
matcher.knnMatch(desc1, desc2, matches, 2);
|
|
338
|
-
for (let i = 0; i < matches.size(); i++) {
|
|
339
|
-
const pair = matches.get(i);
|
|
340
|
-
if (pair.size() < 2) continue;
|
|
341
|
-
const m0 = pair.get(0);
|
|
342
|
-
const m1 = pair.get(1);
|
|
343
|
-
if (m0.distance < MATCH_DISTANCE_THRESHOLD * m1.distance) {
|
|
344
|
-
matchedKp1Indices.add(m0.queryIdx);
|
|
345
|
-
matchedKp2Indices.add(m0.trainIdx);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
} finally {
|
|
349
|
-
matcher.delete();
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
const sNew = [];
|
|
353
|
-
for (let i = 0; i < kp2.size(); i++) {
|
|
354
|
-
if (!matchedKp2Indices.has(i)) {
|
|
355
|
-
const pt = kp2.get(i).pt;
|
|
356
|
-
sNew.push({ x: pt.x, y: pt.y });
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
const sLoss = [];
|
|
360
|
-
for (let i = 0; i < kp1.size(); i++) {
|
|
361
|
-
if (!matchedKp1Indices.has(i)) {
|
|
362
|
-
const pt = kp1.get(i).pt;
|
|
363
|
-
sLoss.push({ x: pt.x, y: pt.y });
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
return { sNew, sLoss };
|
|
367
|
-
} finally {
|
|
368
|
-
mat1.delete();
|
|
369
|
-
mat2.delete();
|
|
370
|
-
kp1.delete();
|
|
371
|
-
kp2.delete();
|
|
372
|
-
desc1.delete();
|
|
373
|
-
desc2.delete();
|
|
374
|
-
mask1.delete();
|
|
375
|
-
mask2.delete();
|
|
376
|
-
akaze.delete();
|
|
377
|
-
if (matches) matches.delete();
|
|
378
|
-
}
|
|
379
|
-
}
|
|
358
|
+
/**
|
|
359
|
+
* Pixel-level difference fallback for AKAZE blind spots.
|
|
360
|
+
*
|
|
361
|
+
* When AKAZE produces sparse results (typical for UI screen recordings
|
|
362
|
+
* where form fields, dropdowns, or overlays change), this function
|
|
363
|
+
* detects changed regions via cv.absdiff and generates synthetic
|
|
364
|
+
* Point2D[] that feed into the existing DBSCAN → IoU → G(t) pipeline.
|
|
365
|
+
*
|
|
366
|
+
* Algorithm:
|
|
367
|
+
* 1. absdiff(frame1, frame2) → grayscale difference
|
|
368
|
+
* 2. GaussianBlur → reduce JPEG compression noise
|
|
369
|
+
* 3. threshold → binary mask of significant changes
|
|
370
|
+
* 4. findContours → bounding rects of changed regions
|
|
371
|
+
* 5. Grid sampling within each bounding rect → Point2D[]
|
|
372
|
+
*
|
|
373
|
+
* @param cvLib - Initialized OpenCV runtime shared by the analyzer.
|
|
374
|
+
* @param frame1 - Previous grayscale frame, with the same dimensions as frame2.
|
|
375
|
+
* @param frame2 - Next grayscale frame, with the same dimensions as frame1.
|
|
376
|
+
* @returns Grid-sampled points from changed regions.
|
|
377
|
+
* @throws Propagates allocation or OpenCV errors after releasing acquired handles.
|
|
378
|
+
*/
|
|
380
379
|
function computePixelDiff(cvLib, frame1, frame2) {
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
blurred.delete();
|
|
429
|
-
binary.delete();
|
|
430
|
-
contours.delete();
|
|
431
|
-
hierarchy.delete();
|
|
432
|
-
}
|
|
380
|
+
const cv = cvLib;
|
|
381
|
+
let mat1 = null;
|
|
382
|
+
let mat2 = null;
|
|
383
|
+
let diff = null;
|
|
384
|
+
let blurred = null;
|
|
385
|
+
let binary = null;
|
|
386
|
+
let contours = null;
|
|
387
|
+
let hierarchy = null;
|
|
388
|
+
try {
|
|
389
|
+
mat1 = new cv.Mat(frame1.height, frame1.width, cv.CV_8UC1);
|
|
390
|
+
mat2 = new cv.Mat(frame2.height, frame2.width, cv.CV_8UC1);
|
|
391
|
+
diff = new cv.Mat();
|
|
392
|
+
blurred = new cv.Mat();
|
|
393
|
+
binary = new cv.Mat();
|
|
394
|
+
contours = new cv.MatVector();
|
|
395
|
+
hierarchy = new cv.Mat();
|
|
396
|
+
mat1.data.set(frame1.data);
|
|
397
|
+
mat2.data.set(frame2.data);
|
|
398
|
+
cv.absdiff(mat1, mat2, diff);
|
|
399
|
+
const ksize = new cv.Size(3, 3);
|
|
400
|
+
cv.GaussianBlur(diff, blurred, ksize, 0);
|
|
401
|
+
cv.threshold(blurred, binary, 30, 255, cv.THRESH_BINARY);
|
|
402
|
+
cv.findContours(binary, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
|
|
403
|
+
const points = [];
|
|
404
|
+
for (let c = 0; c < contours.size(); c++) {
|
|
405
|
+
const contour = contours.get(c);
|
|
406
|
+
try {
|
|
407
|
+
const rect = cv.boundingRect(contour);
|
|
408
|
+
if (rect.width * rect.height < 100) continue;
|
|
409
|
+
for (let y = rect.y; y < rect.y + rect.height; y += 8) for (let x = rect.x; x < rect.x + rect.width; x += 8) points.push({
|
|
410
|
+
x,
|
|
411
|
+
y
|
|
412
|
+
});
|
|
413
|
+
} finally {
|
|
414
|
+
contour.delete();
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
return points;
|
|
418
|
+
} finally {
|
|
419
|
+
mat1?.delete();
|
|
420
|
+
mat2?.delete();
|
|
421
|
+
diff?.delete();
|
|
422
|
+
blurred?.delete();
|
|
423
|
+
binary?.delete();
|
|
424
|
+
contours?.delete();
|
|
425
|
+
hierarchy?.delete();
|
|
426
|
+
}
|
|
433
427
|
}
|
|
434
428
|
function computeInformationGain(clusters, clusterPoints, imageArea, animationIndices, animationWeights) {
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
}
|
|
450
|
-
return gain;
|
|
429
|
+
if (clusters.length === 0) return 0;
|
|
430
|
+
let gain = 0;
|
|
431
|
+
for (let i = 0; i < clusters.length; i++) {
|
|
432
|
+
const box = clusters[i];
|
|
433
|
+
const clusterArea = box.width * box.height;
|
|
434
|
+
if (clusterArea <= 0) continue;
|
|
435
|
+
let contribution = clusterArea / imageArea * (clusterPoints[i] / clusterArea);
|
|
436
|
+
if (animationIndices.has(i)) {
|
|
437
|
+
const animWeight = animationWeights[i] ?? 0;
|
|
438
|
+
contribution *= 1 - animWeight;
|
|
439
|
+
}
|
|
440
|
+
gain += contribution;
|
|
441
|
+
}
|
|
442
|
+
return gain;
|
|
451
443
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
444
|
+
/**
|
|
445
|
+
* Analyze one batch, retaining its boundary frame for the following batch.
|
|
446
|
+
* @param cvLib - Initialized OpenCV runtime.
|
|
447
|
+
* @param akaze - Detector owned by analyzeFrames.
|
|
448
|
+
* @param frames - Boundary frame followed by new adjacent frames.
|
|
449
|
+
* @param carry - Boundary bytes and live features transferred from the previous batch.
|
|
450
|
+
* @param scale - Maximum preprocessing width.
|
|
451
|
+
* @param tracker - Stateful animation tracker shared across batches.
|
|
452
|
+
* @param pairOffset - Global position of this batch's first pair.
|
|
453
|
+
* @returns Scores, pair failure count, and ownership of the final frame's features.
|
|
454
|
+
*/
|
|
455
|
+
async function analyzeBatch(cvLib, akaze, frames, carry, scale, tracker, pairOffset) {
|
|
456
|
+
const edges = [];
|
|
457
|
+
let failures = 0;
|
|
458
|
+
let prev = carry?.features ?? null;
|
|
459
|
+
let next = null;
|
|
460
|
+
try {
|
|
461
|
+
const preprocessed = await Promise.all(map(frames, (f, index) => index === 0 && carry ? carry.preprocessed : preprocessFrame(f.extractPath, scale)));
|
|
462
|
+
const imageWidth = preprocessed[0]?.width ?? scale;
|
|
463
|
+
const imageHeight = preprocessed[0]?.height ?? Math.round(scale * 9 / 16);
|
|
464
|
+
const imageArea = imageWidth * imageHeight;
|
|
465
|
+
for (let i = 0; i < frames.length - 1; i++) {
|
|
466
|
+
const pairIndex = pairOffset + i;
|
|
467
|
+
try {
|
|
468
|
+
prev ??= computeFrameFeatures(cvLib, akaze, preprocessed[i]);
|
|
469
|
+
next = computeFrameFeatures(cvLib, akaze, preprocessed[i + 1]);
|
|
470
|
+
let dbscanResult = dbscan(computeNewPoints(cvLib, prev, next), imageWidth, imageHeight);
|
|
471
|
+
let clusters = dbscanResult.boundingBoxes;
|
|
472
|
+
if (clusters.length === 0) {
|
|
473
|
+
const pixelDiffPoints = computePixelDiff(cvLib, preprocessed[i], preprocessed[i + 1]);
|
|
474
|
+
if (pixelDiffPoints.length > 0) {
|
|
475
|
+
logger.debug(`Edge ${frames[i].id}->${frames[i + 1].id}: pixel-diff fallback (${pixelDiffPoints.length} points)`);
|
|
476
|
+
dbscanResult = dbscan(pixelDiffPoints, imageWidth, imageHeight, void 0, 2);
|
|
477
|
+
clusters = dbscanResult.boundingBoxes;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
const clusterPointCounts = new Array(clusters.length).fill(0);
|
|
481
|
+
for (const label of dbscanResult.labels) if (label >= 0) clusterPointCounts[label]++;
|
|
482
|
+
const animationIndices = tracker.update(clusters, pairIndex);
|
|
483
|
+
const animationWeights = map(clusters, (_, ci) => animationIndices.has(ci) ? tracker.getAnimationWeight(ci, clusters) : 0);
|
|
484
|
+
const score = computeInformationGain(clusters, clusterPointCounts, imageArea, animationIndices, animationWeights);
|
|
485
|
+
logger.debug(`Edge ${frames[i].id}->${frames[i + 1].id} G(t)=${score.toFixed(6)}`);
|
|
486
|
+
edges.push({
|
|
487
|
+
sourceId: frames[i].id,
|
|
488
|
+
targetId: frames[i + 1].id,
|
|
489
|
+
score
|
|
490
|
+
});
|
|
491
|
+
} catch (err) {
|
|
492
|
+
logger.warn(`Frame pair analysis failed: ${String(err)}`);
|
|
493
|
+
failures++;
|
|
494
|
+
edges.push({
|
|
495
|
+
sourceId: frames[i].id,
|
|
496
|
+
targetId: frames[i + 1].id,
|
|
497
|
+
score: 0
|
|
498
|
+
});
|
|
499
|
+
} finally {
|
|
500
|
+
prev?.delete();
|
|
501
|
+
prev = next;
|
|
502
|
+
next = null;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
const result = {
|
|
506
|
+
edges,
|
|
507
|
+
failures,
|
|
508
|
+
analysisResolution: {
|
|
509
|
+
width: imageWidth,
|
|
510
|
+
height: imageHeight
|
|
511
|
+
},
|
|
512
|
+
carry: prev ? {
|
|
513
|
+
preprocessed: preprocessed[preprocessed.length - 1],
|
|
514
|
+
features: prev
|
|
515
|
+
} : null
|
|
516
|
+
};
|
|
517
|
+
prev = null;
|
|
518
|
+
return result;
|
|
519
|
+
} finally {
|
|
520
|
+
prev?.delete();
|
|
521
|
+
next?.delete();
|
|
522
|
+
}
|
|
526
523
|
}
|
|
524
|
+
/**
|
|
525
|
+
* Analyze adjacent frame pairs to compute information gain scores (G(t)).
|
|
526
|
+
* Processes frames in batches for memory efficiency.
|
|
527
|
+
*
|
|
528
|
+
* Pipeline:
|
|
529
|
+
* 1. AKAZE Feature Set Difference
|
|
530
|
+
* 2. DBSCAN Spatial Clustering
|
|
531
|
+
* 3. Spatio-temporal IoU Tracking
|
|
532
|
+
* 4. G(t) Information Gain Scoring
|
|
533
|
+
* @param ctx - Frames, analysis options, and the progress callback for this run.
|
|
534
|
+
* @returns Adjacent scores and tracked animations in analysis coordinates.
|
|
535
|
+
* @throws Propagates runtime errors and rejects total failure of two or more pairs after cleanup.
|
|
536
|
+
*/
|
|
527
537
|
async function analyzeFrames(ctx) {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
538
|
+
const { frames } = ctx;
|
|
539
|
+
if (frames.length < 2) return {
|
|
540
|
+
edges: [],
|
|
541
|
+
animations: [],
|
|
542
|
+
analysisResolution: {
|
|
543
|
+
width: 0,
|
|
544
|
+
height: 0
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
logger.debug(`Analyzing ${frames.length} frames in batches of ${10}`);
|
|
548
|
+
const cvLib = await ensureOpenCV();
|
|
549
|
+
const edges = [];
|
|
550
|
+
const tracker = new IoUTracker(ctx.effectiveFps ?? ctx.options.fps, ctx.options.iouThreshold, ctx.options.animationThreshold);
|
|
551
|
+
const scale = ctx.options.scale;
|
|
552
|
+
let akaze = null;
|
|
553
|
+
let carry = null;
|
|
554
|
+
let analysisResolution = {
|
|
555
|
+
width: 0,
|
|
556
|
+
height: 0
|
|
557
|
+
};
|
|
558
|
+
let failures = 0;
|
|
559
|
+
try {
|
|
560
|
+
akaze = new cvLib.AKAZE();
|
|
561
|
+
for (let i = 0; i < frames.length - 1; i += 10) {
|
|
562
|
+
const batch = [frames[i], ...frames.slice(i + 1, i + 1 + 10)];
|
|
563
|
+
const result = await analyzeBatch(cvLib, akaze, batch, carry, scale, tracker, i);
|
|
564
|
+
carry = result.carry;
|
|
565
|
+
failures += result.failures;
|
|
566
|
+
if (i === 0) analysisResolution = result.analysisResolution;
|
|
567
|
+
edges.push(...result.edges);
|
|
568
|
+
const progress = Math.min(100, (i + 10) / (frames.length - 1) * 100);
|
|
569
|
+
ctx.emitProgress(progress);
|
|
570
|
+
}
|
|
571
|
+
} finally {
|
|
572
|
+
carry?.features.delete();
|
|
573
|
+
akaze?.delete();
|
|
574
|
+
}
|
|
575
|
+
const pairs = frames.length - 1;
|
|
576
|
+
if (pairs >= 2 && failures === pairs) throw new Error(`All ${pairs} frame pairs failed analysis`);
|
|
577
|
+
const animations = tracker.flushAndGetAnimations();
|
|
578
|
+
logger.debug(`Computed ${edges.length} score edges and ${animations.length} animations`);
|
|
579
|
+
return {
|
|
580
|
+
edges,
|
|
581
|
+
animations,
|
|
582
|
+
analysisResolution
|
|
583
|
+
};
|
|
557
584
|
}
|
|
558
585
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
586
|
+
//#endregion
|
|
587
|
+
//#region src/core/utils/metadata/build-video-metadata.ts
|
|
588
|
+
/**
|
|
589
|
+
* Read output dimensions and build consistent video and animation metadata.
|
|
590
|
+
* JPEG finalization does not resize, so the source dimensions match the output.
|
|
591
|
+
* @param ctx Pipeline state with source duration, effective FPS and analysis-space animations.
|
|
592
|
+
* @param selected Selected frames in output order; the first candidate is the fallback.
|
|
593
|
+
* @param analysisResolution Analysis dimensions; absent or zero dimensions imply no scaling.
|
|
594
|
+
* @returns Video metadata and new output-space animations, retaining zero-based frame IDs.
|
|
595
|
+
* @throws If sharp cannot read the selected or fallback image. Empty input performs no image I/O.
|
|
596
|
+
*/
|
|
597
|
+
async function buildVideoMetadata(ctx, selected, analysisResolution) {
|
|
598
|
+
const firstFrame = selected[0] ?? ctx.frames[0];
|
|
599
|
+
const dimensions = firstFrame ? await sharp(firstFrame.extractPath).metadata() : void 0;
|
|
600
|
+
const width = dimensions?.width ?? 0;
|
|
601
|
+
const height = dimensions?.height ?? 0;
|
|
602
|
+
const sx = analysisResolution?.width ? width / analysisResolution.width : 1;
|
|
603
|
+
const sy = analysisResolution?.height ? height / analysisResolution.height : 1;
|
|
604
|
+
const lastTimestamp = ctx.frames[ctx.frames.length - 1]?.timestamp ?? 0;
|
|
605
|
+
const duration = ctx.options.mode === "frames" ? lastTimestamp : ctx.sourceDurationSec ?? lastTimestamp;
|
|
606
|
+
return {
|
|
607
|
+
video: {
|
|
608
|
+
originalDurationMs: Math.round(duration * 1e3),
|
|
609
|
+
fps: ctx.options.mode === "frames" ? 1 : ctx.effectiveFps ?? ctx.options.fps,
|
|
610
|
+
resolution: {
|
|
611
|
+
width,
|
|
612
|
+
height
|
|
613
|
+
}
|
|
614
|
+
},
|
|
615
|
+
animations: (ctx.animations ?? []).map((animation) => {
|
|
616
|
+
const box = animation.boundingBox;
|
|
617
|
+
const x = Math.max(0, Math.min(width, Math.round(box.x * sx)));
|
|
618
|
+
const y = Math.max(0, Math.min(height, Math.round(box.y * sy)));
|
|
619
|
+
return {
|
|
620
|
+
...animation,
|
|
621
|
+
boundingBox: {
|
|
622
|
+
x,
|
|
623
|
+
y,
|
|
624
|
+
width: Math.max(0, Math.min(width - x, Math.round(box.width * sx))),
|
|
625
|
+
height: Math.max(0, Math.min(height - y, Math.round(box.height * sy)))
|
|
626
|
+
}
|
|
627
|
+
};
|
|
628
|
+
})
|
|
629
|
+
};
|
|
630
|
+
}
|
|
566
631
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
632
|
+
//#endregion
|
|
633
|
+
//#region src/core/constants/workspace-layout.ts
|
|
634
|
+
/**
|
|
635
|
+
* Temp workspace naming and frame file layout for pipeline runs.
|
|
636
|
+
*/
|
|
637
|
+
const APP_NAME = "scene-sieve";
|
|
638
|
+
const WORKSPACE_PREFIX = `${APP_NAME}-`;
|
|
639
|
+
const TEMP_BASE_DIR = tmpdir();
|
|
640
|
+
const FRAME_OUTPUT_EXTENSION = ".jpg";
|
|
641
|
+
const FRAME_FILENAME_PATTERN = "frame_%06d.jpg";
|
|
642
|
+
function getTempWorkspaceDir(sessionId) {
|
|
643
|
+
return join(TEMP_BASE_DIR, `${WORKSPACE_PREFIX}${sessionId}`);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
//#endregion
|
|
647
|
+
//#region src/core/utils/filesystem/paths.ts
|
|
571
648
|
async function ensureDir(dirPath) {
|
|
572
|
-
|
|
649
|
+
await mkdir(dirPath, { recursive: true });
|
|
573
650
|
}
|
|
574
651
|
async function fileExists(filePath) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
652
|
+
try {
|
|
653
|
+
await stat(filePath);
|
|
654
|
+
return true;
|
|
655
|
+
} catch {
|
|
656
|
+
return false;
|
|
657
|
+
}
|
|
581
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Expand leading ~ to homedir. Node's path.resolve() does not expand ~,
|
|
661
|
+
* so paths like ~/Desktop/foo depend on process.cwd() and can produce
|
|
662
|
+
* different results when run from different directories.
|
|
663
|
+
*/
|
|
582
664
|
function expandTilde(p) {
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
}
|
|
587
|
-
return p;
|
|
665
|
+
if (p === "~") return homedir();
|
|
666
|
+
if (p.startsWith("~/") || p.startsWith("~\\")) return resolve(homedir(), p.slice(2));
|
|
667
|
+
return p;
|
|
588
668
|
}
|
|
669
|
+
/**
|
|
670
|
+
* Resolve path to absolute. Expands ~ to homedir first so that the result
|
|
671
|
+
* does not depend on process.cwd().
|
|
672
|
+
*/
|
|
589
673
|
function resolveAbsolute(p) {
|
|
590
|
-
|
|
674
|
+
return resolve(expandTilde(p));
|
|
591
675
|
}
|
|
676
|
+
/**
|
|
677
|
+
* Derive default output directory name from input file path.
|
|
678
|
+
* e.g., /path/to/video.mp4 -> /path/to/video_scenes
|
|
679
|
+
*/
|
|
592
680
|
function deriveOutputPath(inputPath) {
|
|
593
|
-
|
|
594
|
-
const name = basename(inputPath, extname(inputPath));
|
|
595
|
-
return resolve(dir, `${name}_scenes`);
|
|
681
|
+
return resolve(resolve(inputPath, ".."), `${basename(inputPath, extname(inputPath))}_scenes`);
|
|
596
682
|
}
|
|
597
683
|
|
|
598
|
-
|
|
684
|
+
//#endregion
|
|
685
|
+
//#region src/core/extractor/extractor.ts
|
|
686
|
+
/**
|
|
687
|
+
* Extract frames from video/GIF using FFmpeg.
|
|
688
|
+
* @param ctx Pipeline context; records effectiveFps and sourceDurationSec for video input.
|
|
689
|
+
* @returns Extracted candidates, or the unchanged input array in frames mode.
|
|
690
|
+
* @throws When the input is missing, metadata has no video stream, or FFmpeg fails.
|
|
691
|
+
*/
|
|
599
692
|
async function extractFrames(ctx) {
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
effectiveFps = Math.min(fps, fpsCap);
|
|
632
|
-
effectiveFps = Math.max(0.5, effectiveFps);
|
|
633
|
-
logger.debug(
|
|
634
|
-
`FPS: ${fps} \u2192 effective: ${effectiveFps.toFixed(2)} (maxFrames: ${maxFrames})`
|
|
635
|
-
);
|
|
636
|
-
}
|
|
637
|
-
const frames = await extractByFps(
|
|
638
|
-
inputPath,
|
|
639
|
-
framesDir,
|
|
640
|
-
effectiveFps,
|
|
641
|
-
scale,
|
|
642
|
-
duration
|
|
643
|
-
);
|
|
644
|
-
ctx.emitProgress(100);
|
|
645
|
-
logger.debug(`Extracted ${frames.length} frames`);
|
|
646
|
-
return frames;
|
|
693
|
+
if (ctx.options.mode === "frames") {
|
|
694
|
+
ctx.effectiveFps = 1;
|
|
695
|
+
return ctx.frames;
|
|
696
|
+
}
|
|
697
|
+
const framesDir = join(ctx.workspacePath, "frames");
|
|
698
|
+
const { inputPath, fps, maxFrames, scale } = ctx.options;
|
|
699
|
+
if (!inputPath) throw new Error("inputPath is required for frame extraction");
|
|
700
|
+
if (!await fileExists(inputPath)) throw new Error(`Input file not found: ${inputPath}`);
|
|
701
|
+
const metadata = await getVideoMetadata(inputPath).catch((err) => {
|
|
702
|
+
logger.debug(`ffprobe failed: ${err.message}`);
|
|
703
|
+
return null;
|
|
704
|
+
});
|
|
705
|
+
if (!metadata || !metadata.format) throw new Error(`Could not read file metadata: ${inputPath}`);
|
|
706
|
+
const formatName = metadata.format.format_name ?? "";
|
|
707
|
+
const duration = parseFloat(metadata.format.duration ?? "0");
|
|
708
|
+
if (!(metadata.streams?.some((s) => s.codec_type === "video") ?? false)) throw new Error(`No video stream found in file: ${inputPath} (detected format: ${formatName})`);
|
|
709
|
+
logger.debug(`Detected format: ${formatName} (Duration: ${duration.toFixed(1)}s), path: ${inputPath}`);
|
|
710
|
+
await ensureDir(framesDir);
|
|
711
|
+
const frameLimit = Math.max(2, maxFrames);
|
|
712
|
+
let effectiveFps = fps;
|
|
713
|
+
if (duration > 0) {
|
|
714
|
+
const fpsCap = frameLimit / duration;
|
|
715
|
+
effectiveFps = Math.min(fps, fpsCap);
|
|
716
|
+
logger.debug(`FPS: ${fps} → effective: ${effectiveFps.toFixed(2)} (maxFrames: ${maxFrames})`);
|
|
717
|
+
}
|
|
718
|
+
ctx.effectiveFps = effectiveFps;
|
|
719
|
+
ctx.sourceDurationSec = duration;
|
|
720
|
+
const frames = await extractByFps(inputPath, framesDir, effectiveFps, scale, frameLimit);
|
|
721
|
+
ctx.emitProgress(100);
|
|
722
|
+
logger.debug(`Extracted ${frames.length} frames`);
|
|
723
|
+
return frames;
|
|
647
724
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
725
|
+
/**
|
|
726
|
+
* Write scaled JPEG candidates through the bundled FFmpeg runtime.
|
|
727
|
+
* @param inputPath Readable video input.
|
|
728
|
+
* @param outputDir Existing frame directory.
|
|
729
|
+
* @param fps Positive effective sampling frequency.
|
|
730
|
+
* @param scale Output image height.
|
|
731
|
+
* @param frameLimit Maximum number of output frames.
|
|
732
|
+
* @returns Candidates with local output-grid timestamps; rejects on extraction failure.
|
|
733
|
+
*/
|
|
734
|
+
async function extractByFps(inputPath, outputDir, fps, scale, frameLimit) {
|
|
735
|
+
const outputPattern = join(outputDir, FRAME_FILENAME_PATTERN);
|
|
736
|
+
await execa(ffmpegPath, [
|
|
737
|
+
"-i",
|
|
738
|
+
inputPath,
|
|
739
|
+
"-vf",
|
|
740
|
+
`fps=${fps},scale=-1:${scale}`,
|
|
741
|
+
"-q:v",
|
|
742
|
+
"2",
|
|
743
|
+
"-frames:v",
|
|
744
|
+
String(frameLimit),
|
|
745
|
+
outputPattern
|
|
746
|
+
]);
|
|
747
|
+
return buildFrameList(outputDir, fps);
|
|
660
748
|
}
|
|
661
749
|
async function getVideoMetadata(inputPath) {
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
750
|
+
const { stdout } = await execa(path, [
|
|
751
|
+
"-v",
|
|
752
|
+
"quiet",
|
|
753
|
+
"-print_format",
|
|
754
|
+
"json",
|
|
755
|
+
"-show_format",
|
|
756
|
+
"-show_streams",
|
|
757
|
+
inputPath
|
|
758
|
+
]);
|
|
759
|
+
return JSON.parse(stdout);
|
|
672
760
|
}
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
761
|
+
/**
|
|
762
|
+
* Read sorted JPEG paths and attach local output-grid times.
|
|
763
|
+
* @param framesDir Extracted frame directory; filesystem errors propagate.
|
|
764
|
+
* @param effectiveFps Positive frequency used by the fps filter.
|
|
765
|
+
* @returns Zero-based candidates without a segment seek offset.
|
|
766
|
+
*/
|
|
767
|
+
async function buildFrameList(framesDir, effectiveFps) {
|
|
768
|
+
const jpgFiles = filter(await readdir(framesDir), (f) => f.endsWith(".jpg")).sort();
|
|
769
|
+
if (jpgFiles.length === 0) return [];
|
|
770
|
+
return map(jpgFiles, (file, index) => ({
|
|
771
|
+
id: index,
|
|
772
|
+
timestamp: index / effectiveFps,
|
|
773
|
+
extractPath: join(framesDir, file)
|
|
774
|
+
}));
|
|
684
775
|
}
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
776
|
+
/**
|
|
777
|
+
* Extract frames from a specific time range of a video using FFmpeg.
|
|
778
|
+
* Uses input seeking (-ss before -i) for fast seek + -t for duration.
|
|
779
|
+
*
|
|
780
|
+
* @param inputPath - Path to the video file
|
|
781
|
+
* @param outputDir - Directory to write extracted frames
|
|
782
|
+
* @param fps - Frames per second for extraction
|
|
783
|
+
* @param scale - Height scale for vision analysis
|
|
784
|
+
* @param startTime - Start time in seconds
|
|
785
|
+
* @param duration - Duration in seconds to extract
|
|
786
|
+
* @param frameLimit - Positive output limit; defaults to the range's grid capacity
|
|
787
|
+
* @returns Array of FrameNode with segment-local timestamps (starting from 0)
|
|
788
|
+
*/
|
|
789
|
+
async function extractFramesForRange(inputPath, outputDir, fps, scale, startTime, duration, frameLimit = Math.ceil(duration * fps)) {
|
|
790
|
+
const outputPattern = join(outputDir, FRAME_FILENAME_PATTERN);
|
|
791
|
+
await execa(ffmpegPath, [
|
|
792
|
+
"-ss",
|
|
793
|
+
String(startTime),
|
|
794
|
+
"-i",
|
|
795
|
+
inputPath,
|
|
796
|
+
"-t",
|
|
797
|
+
String(duration),
|
|
798
|
+
"-vf",
|
|
799
|
+
`fps=${fps},scale=-1:${scale}`,
|
|
800
|
+
"-q:v",
|
|
801
|
+
"2",
|
|
802
|
+
"-frames:v",
|
|
803
|
+
String(frameLimit),
|
|
804
|
+
outputPattern
|
|
805
|
+
]);
|
|
806
|
+
return buildFrameList(outputDir, fps);
|
|
701
807
|
}
|
|
702
808
|
|
|
703
|
-
|
|
704
|
-
|
|
809
|
+
//#endregion
|
|
810
|
+
//#region src/core/input-resolver/validation/validate-options.ts
|
|
811
|
+
/**
|
|
812
|
+
* Reject invalid numeric options before defaults or pipeline effects are applied.
|
|
813
|
+
* @param options - Supplied options; omitted numeric fields use pipeline defaults.
|
|
814
|
+
* @returns Nothing when all supplied numeric fields satisfy their contracts.
|
|
815
|
+
* @throws An input error naming the invalid option and its received value.
|
|
816
|
+
*/
|
|
817
|
+
function validateOptions(options) {
|
|
818
|
+
for (const [name, min, max, integer, exclusiveMin, requirement] of [
|
|
819
|
+
[
|
|
820
|
+
"count",
|
|
821
|
+
1,
|
|
822
|
+
Infinity,
|
|
823
|
+
true,
|
|
824
|
+
false,
|
|
825
|
+
"an integer >= 1"
|
|
826
|
+
],
|
|
827
|
+
[
|
|
828
|
+
"threshold",
|
|
829
|
+
0,
|
|
830
|
+
1,
|
|
831
|
+
false,
|
|
832
|
+
true,
|
|
833
|
+
"in range (0, 1] and finite"
|
|
834
|
+
],
|
|
835
|
+
[
|
|
836
|
+
"fps",
|
|
837
|
+
0,
|
|
838
|
+
Infinity,
|
|
839
|
+
false,
|
|
840
|
+
true,
|
|
841
|
+
"finite and > 0"
|
|
842
|
+
],
|
|
843
|
+
[
|
|
844
|
+
"maxFrames",
|
|
845
|
+
2,
|
|
846
|
+
Infinity,
|
|
847
|
+
true,
|
|
848
|
+
false,
|
|
849
|
+
"an integer >= 2"
|
|
850
|
+
],
|
|
851
|
+
[
|
|
852
|
+
"scale",
|
|
853
|
+
16,
|
|
854
|
+
Infinity,
|
|
855
|
+
true,
|
|
856
|
+
false,
|
|
857
|
+
"an integer >= 16"
|
|
858
|
+
],
|
|
859
|
+
[
|
|
860
|
+
"quality",
|
|
861
|
+
1,
|
|
862
|
+
100,
|
|
863
|
+
true,
|
|
864
|
+
false,
|
|
865
|
+
"an integer in range [1, 100]"
|
|
866
|
+
],
|
|
867
|
+
[
|
|
868
|
+
"iouThreshold",
|
|
869
|
+
0,
|
|
870
|
+
1,
|
|
871
|
+
false,
|
|
872
|
+
false,
|
|
873
|
+
"finite and in range [0, 1]"
|
|
874
|
+
],
|
|
875
|
+
[
|
|
876
|
+
"animationThreshold",
|
|
877
|
+
1,
|
|
878
|
+
Infinity,
|
|
879
|
+
true,
|
|
880
|
+
false,
|
|
881
|
+
"an integer >= 1"
|
|
882
|
+
],
|
|
883
|
+
[
|
|
884
|
+
"maxSegmentDuration",
|
|
885
|
+
0,
|
|
886
|
+
Infinity,
|
|
887
|
+
false,
|
|
888
|
+
true,
|
|
889
|
+
"finite and > 0"
|
|
890
|
+
],
|
|
891
|
+
[
|
|
892
|
+
"concurrency",
|
|
893
|
+
1,
|
|
894
|
+
Infinity,
|
|
895
|
+
true,
|
|
896
|
+
false,
|
|
897
|
+
"an integer >= 1"
|
|
898
|
+
]
|
|
899
|
+
]) {
|
|
900
|
+
const value = options[name];
|
|
901
|
+
if (value === void 0) continue;
|
|
902
|
+
if (!Number.isFinite(value) || integer && !Number.isInteger(value) || (exclusiveMin ? value <= min : value < min) || value > max) throw new Error(`${name} must be ${requirement}, received: ${value}`);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
705
905
|
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
import { join as join3 } from "path";
|
|
709
|
-
import { map as map3 } from "@winglet/common-utils";
|
|
710
|
-
import sharp2 from "sharp";
|
|
906
|
+
//#endregion
|
|
907
|
+
//#region src/core/workspace/workspace.ts
|
|
711
908
|
async function createWorkspace(sessionId) {
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
909
|
+
const workspacePath = getTempWorkspaceDir(sessionId);
|
|
910
|
+
await ensureDir(join(workspacePath, "frames"));
|
|
911
|
+
await ensureDir(join(workspacePath, "output"));
|
|
912
|
+
return workspacePath;
|
|
716
913
|
}
|
|
717
914
|
async function finalizeOutput(ctx, selectedFrames) {
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
await rm(outputPath, { recursive: true, force: true });
|
|
762
|
-
await rename(stagingDir, outputPath);
|
|
763
|
-
return outputFiles;
|
|
915
|
+
const stagingDir = join(ctx.workspacePath, "output");
|
|
916
|
+
const outputPath = ctx.options.outputPath;
|
|
917
|
+
const quality = ctx.options.quality;
|
|
918
|
+
const outputFiles = [];
|
|
919
|
+
const framesMetadata = [];
|
|
920
|
+
const totalFramesCount = ctx.frames.length;
|
|
921
|
+
const padding = Math.max(4, String(totalFramesCount).length);
|
|
922
|
+
for (let i = 0; i < selectedFrames.length; i++) {
|
|
923
|
+
const frame = selectedFrames[i];
|
|
924
|
+
const fileName = `frame_${String(frame.id + 1).padStart(padding, "0")}.jpg`;
|
|
925
|
+
const destPath = join(stagingDir, fileName);
|
|
926
|
+
await sharp(frame.extractPath).jpeg({
|
|
927
|
+
quality,
|
|
928
|
+
mozjpeg: true
|
|
929
|
+
}).toFile(destPath);
|
|
930
|
+
outputFiles.push(join(outputPath, fileName));
|
|
931
|
+
framesMetadata.push({
|
|
932
|
+
step: i + 1,
|
|
933
|
+
fileName,
|
|
934
|
+
frameId: frame.id + 1,
|
|
935
|
+
timestampMs: Math.round(frame.timestamp * 1e3)
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
const { video, animations } = await buildVideoMetadata(ctx, selectedFrames, ctx.analysisResolution);
|
|
939
|
+
const metadata = {
|
|
940
|
+
video,
|
|
941
|
+
frames: framesMetadata,
|
|
942
|
+
animations: map(animations, (anim) => ({
|
|
943
|
+
...anim,
|
|
944
|
+
startFrameId: anim.startFrameId + 1,
|
|
945
|
+
endFrameId: anim.endFrameId + 1,
|
|
946
|
+
durationMs: Math.round(anim.durationMs)
|
|
947
|
+
}))
|
|
948
|
+
};
|
|
949
|
+
await writeFile(join(stagingDir, ".metadata.json"), JSON.stringify(metadata, null, 2));
|
|
950
|
+
outputFiles.push(join(outputPath, ".metadata.json"));
|
|
951
|
+
await ensureDir(join(outputPath, ".."));
|
|
952
|
+
await rm(outputPath, {
|
|
953
|
+
recursive: true,
|
|
954
|
+
force: true
|
|
955
|
+
});
|
|
956
|
+
await rename(stagingDir, outputPath);
|
|
957
|
+
return outputFiles;
|
|
764
958
|
}
|
|
765
959
|
async function createSegmentWorkspace(parentWorkspacePath, segmentIndex) {
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
String(segmentIndex)
|
|
770
|
-
);
|
|
771
|
-
await ensureDir(join3(segmentPath, "frames"));
|
|
772
|
-
return segmentPath;
|
|
960
|
+
const segmentPath = join(parentWorkspacePath, "segments", String(segmentIndex));
|
|
961
|
+
await ensureDir(join(segmentPath, "frames"));
|
|
962
|
+
return segmentPath;
|
|
773
963
|
}
|
|
774
964
|
async function cleanupWorkspace(workspacePath) {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
965
|
+
if (!workspacePath) return;
|
|
966
|
+
try {
|
|
967
|
+
await rm(workspacePath, {
|
|
968
|
+
recursive: true,
|
|
969
|
+
force: true
|
|
970
|
+
});
|
|
971
|
+
} catch {}
|
|
780
972
|
}
|
|
781
|
-
|
|
973
|
+
/**
|
|
974
|
+
* Write a video buffer to a temp file in the workspace and return the path.
|
|
975
|
+
* Used by 'buffer' input mode.
|
|
976
|
+
*/
|
|
782
977
|
async function writeInputBuffer(buffer, workspacePath) {
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
978
|
+
const inputDir = join(workspacePath, "input");
|
|
979
|
+
await ensureDir(inputDir);
|
|
980
|
+
const tempPath = join(inputDir, "input.mp4");
|
|
981
|
+
await writeFile(tempPath, buffer);
|
|
982
|
+
return tempPath;
|
|
788
983
|
}
|
|
984
|
+
/**
|
|
985
|
+
* Write an array of frame Buffers as JPG files and return FrameNode[].
|
|
986
|
+
* Used by 'frames' input mode.
|
|
987
|
+
*/
|
|
789
988
|
async function writeInputFrames(frames, workspacePath) {
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
989
|
+
const framesDir = join(workspacePath, "frames");
|
|
990
|
+
await ensureDir(framesDir);
|
|
991
|
+
const frameNodes = [];
|
|
992
|
+
for (let i = 0; i < frames.length; i++) {
|
|
993
|
+
const extractPath = join(framesDir, `frame_${String(i).padStart(6, "0")}${FRAME_OUTPUT_EXTENSION}`);
|
|
994
|
+
await writeFile(extractPath, frames[i]);
|
|
995
|
+
frameNodes.push({
|
|
996
|
+
id: i,
|
|
997
|
+
timestamp: i,
|
|
998
|
+
extractPath
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
return frameNodes;
|
|
800
1002
|
}
|
|
1003
|
+
/**
|
|
1004
|
+
* Read selected FrameNode files as Buffers with JPEG compression.
|
|
1005
|
+
* Used to return output buffers in 'buffer' and 'frames' modes.
|
|
1006
|
+
*/
|
|
801
1007
|
async function readFramesAsBuffers(frameNodes, quality) {
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
)
|
|
807
|
-
);
|
|
1008
|
+
return Promise.all(map(frameNodes, (f) => sharp(f.extractPath).jpeg({
|
|
1009
|
+
quality,
|
|
1010
|
+
mozjpeg: true
|
|
1011
|
+
}).toBuffer()));
|
|
808
1012
|
}
|
|
809
1013
|
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
1014
|
+
//#endregion
|
|
1015
|
+
//#region src/core/input-resolver/input-resolver.ts
|
|
1016
|
+
/**
|
|
1017
|
+
* Validate supplied options and resolve defaults and paths for the pipeline.
|
|
1018
|
+
* @param options - Mode-specific input and optional numeric settings.
|
|
1019
|
+
* @returns Complete pipeline settings with absolute file input paths.
|
|
1020
|
+
* @throws An input error if a supplied numeric setting is invalid.
|
|
1021
|
+
*/
|
|
1022
|
+
function resolveOptions(options) {
|
|
1023
|
+
validateOptions(options);
|
|
1024
|
+
const mode = options.mode;
|
|
1025
|
+
const inputPath = mode === "file" ? resolveAbsolute(options.inputPath) : void 0;
|
|
1026
|
+
const outputPath = options.outputPath ?? (inputPath ? deriveOutputPath(inputPath) : join(process.cwd(), "scene-sieve-output"));
|
|
1027
|
+
const threshold = options.threshold ?? .5;
|
|
1028
|
+
return {
|
|
1029
|
+
mode,
|
|
1030
|
+
inputPath,
|
|
1031
|
+
count: options.count ?? 20,
|
|
1032
|
+
threshold,
|
|
1033
|
+
pruneMode: "threshold-with-cap",
|
|
1034
|
+
outputPath,
|
|
1035
|
+
fps: options.fps ?? 5,
|
|
1036
|
+
maxFrames: options.maxFrames ?? 300,
|
|
1037
|
+
scale: options.scale ?? 720,
|
|
1038
|
+
quality: options.quality ?? 80,
|
|
1039
|
+
iouThreshold: options.iouThreshold ?? .9,
|
|
1040
|
+
animationThreshold: options.animationThreshold ?? 5,
|
|
1041
|
+
debug: options.debug ?? false,
|
|
1042
|
+
maxSegmentDuration: options.maxSegmentDuration ?? 300,
|
|
1043
|
+
concurrency: options.concurrency ?? 2
|
|
1044
|
+
};
|
|
839
1045
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
1046
|
+
/**
|
|
1047
|
+
* Resolve the input source to a list of FrameNode[].
|
|
1048
|
+
*
|
|
1049
|
+
* - 'file' mode: validate file exists and delegate to extractor (caller's responsibility)
|
|
1050
|
+
* - 'buffer' mode: write buffer as temp video file, return path via FrameNode trick (empty list)
|
|
1051
|
+
* - 'frames' mode: write frame buffers as JPGs, return FrameNode[]
|
|
1052
|
+
* @param options - Input source; encoded frames must have matching dimensions.
|
|
1053
|
+
* @param workspacePath - Workspace receiving temporary input files.
|
|
1054
|
+
* @returns Frame nodes or a resolved video path for extraction.
|
|
1055
|
+
* @throws Propagates metadata or write errors and rejects mismatched frame sizes.
|
|
1056
|
+
*/
|
|
1057
|
+
async function resolveInput(options, workspacePath) {
|
|
1058
|
+
if (options.mode === "file") return {
|
|
1059
|
+
frames: [],
|
|
1060
|
+
resolvedInputPath: resolveAbsolute(options.inputPath)
|
|
1061
|
+
};
|
|
1062
|
+
if (options.mode === "buffer") return {
|
|
1063
|
+
frames: [],
|
|
1064
|
+
resolvedInputPath: await writeInputBuffer(options.inputBuffer, workspacePath)
|
|
1065
|
+
};
|
|
1066
|
+
if (options.mode === "frames") {
|
|
1067
|
+
let dimensions;
|
|
1068
|
+
for (const buffer of options.inputFrames) {
|
|
1069
|
+
const { width, height } = await sharp(buffer).metadata();
|
|
1070
|
+
if (dimensions && (width !== dimensions.width || height !== dimensions.height)) throw new Error(`inputFrames must be the same size (${dimensions.width}x${dimensions.height}), received: ${width}x${height}`);
|
|
1071
|
+
dimensions = {
|
|
1072
|
+
width,
|
|
1073
|
+
height
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
return { frames: await writeInputFrames(options.inputFrames, workspacePath) };
|
|
1077
|
+
}
|
|
1078
|
+
throw new Error(`Unsupported input mode: ${options.mode}`);
|
|
859
1079
|
}
|
|
860
1080
|
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1081
|
+
//#endregion
|
|
1082
|
+
//#region src/core/pruner/scoring/normalize-scores.ts
|
|
1083
|
+
const NORMALIZATION_ALPHA = .4;
|
|
1084
|
+
const NORMALIZATION_MAD_COEFFICIENT = 1.4826;
|
|
1085
|
+
const NORMALIZATION_MIN_SAMPLE_SIZE = 10;
|
|
1086
|
+
/**
|
|
1087
|
+
* Find the first position whose score is at least the requested value.
|
|
1088
|
+
* @param sorted - Finite positive scores sorted in ascending order.
|
|
1089
|
+
* @param value - A finite positive score present in sorted.
|
|
1090
|
+
* @returns The first matching rank, including the first position of any tie.
|
|
1091
|
+
*/
|
|
1092
|
+
function lowerBound(sorted, value) {
|
|
1093
|
+
let low = 0;
|
|
1094
|
+
let high = sorted.length;
|
|
1095
|
+
while (low < high) {
|
|
1096
|
+
const mid = Math.floor((low + high) / 2);
|
|
1097
|
+
if (sorted[mid] < value) low = mid + 1;
|
|
1098
|
+
else high = mid;
|
|
1099
|
+
}
|
|
1100
|
+
return low;
|
|
1101
|
+
}
|
|
1102
|
+
/**
|
|
1103
|
+
* Normalize raw scores to [0, 1] range via Robust Hybrid Normalization.
|
|
1104
|
+
*
|
|
1105
|
+
* This model combines two mathematical approaches to provide a stable "relative" threshold:
|
|
1106
|
+
*
|
|
1107
|
+
* 1. Logistic-Robust-Z (Intensity):
|
|
1108
|
+
* Calculates Z-scores using Median and Median Absolute Deviation (MAD).
|
|
1109
|
+
* Maps these to a sigmoid (logistic) curve. This suppresses noise (scores near median)
|
|
1110
|
+
* and highlights significant signals (outliers) without letting extreme outliers
|
|
1111
|
+
* crush other meaningful transitions.
|
|
1112
|
+
*
|
|
1113
|
+
* 2. CDF / Percentile Rank (Relative Position):
|
|
1114
|
+
* Maps each score to its percentile rank in the sequence. This ensures that 't'
|
|
1115
|
+
* always has a consistent meaning as a "relative rank" regardless of absolute values.
|
|
1116
|
+
*
|
|
1117
|
+
* The final score is a weighted sum (NORMALIZATION_ALPHA) of both.
|
|
1118
|
+
*
|
|
1119
|
+
* @param items - Array of items with scores to normalize
|
|
1120
|
+
* @returns normalized scores array (same length as input)
|
|
1121
|
+
*/
|
|
866
1122
|
function normalizeScores(items) {
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
});
|
|
894
|
-
const cdf = map4(safeScores, (s) => {
|
|
895
|
-
if (s <= 0) return 0;
|
|
896
|
-
const rank = sorted.findIndex((v) => v >= s);
|
|
897
|
-
return rank / sorted.length;
|
|
898
|
-
});
|
|
899
|
-
return map4(
|
|
900
|
-
logisticZ,
|
|
901
|
-
(z, i) => z * (1 - NORMALIZATION_ALPHA) + cdf[i] * NORMALIZATION_ALPHA
|
|
902
|
-
);
|
|
1123
|
+
if (items.length === 0) return [];
|
|
1124
|
+
const safeScores = map(items, (e) => Number.isFinite(e.score) && e.score > 0 ? e.score : 0);
|
|
1125
|
+
const positiveScores = filter(safeScores, (s) => s > 0);
|
|
1126
|
+
if (positiveScores.length === 0) return safeScores;
|
|
1127
|
+
const sorted = [...positiveScores].sort((a, b) => a - b);
|
|
1128
|
+
if (positiveScores.length <= 10) {
|
|
1129
|
+
const min = sorted[0];
|
|
1130
|
+
const max = sorted[sorted.length - 1];
|
|
1131
|
+
if (max === min) return map(safeScores, (s) => s > 0 ? 1 : 0);
|
|
1132
|
+
return map(safeScores, (s) => s <= 0 ? 0 : Math.max(0, Math.min((s - min) / (max - min), 1)));
|
|
1133
|
+
}
|
|
1134
|
+
const median = sorted[Math.floor(sorted.length / 2)];
|
|
1135
|
+
const absoluteDiffs = map(positiveScores, (v) => Math.abs(v - median));
|
|
1136
|
+
const mad = [...absoluteDiffs].sort((a, b) => a - b)[Math.floor(absoluteDiffs.length / 2)];
|
|
1137
|
+
const scale = mad === 0 ? median : mad * NORMALIZATION_MAD_COEFFICIENT;
|
|
1138
|
+
const logisticZ = map(safeScores, (s) => {
|
|
1139
|
+
if (s <= 0) return 0;
|
|
1140
|
+
if (scale === 0) return 1;
|
|
1141
|
+
const z = (s - median) / scale;
|
|
1142
|
+
return 1 / (1 + Math.exp(-3 * z));
|
|
1143
|
+
});
|
|
1144
|
+
const cdf = map(safeScores, (s) => {
|
|
1145
|
+
if (s <= 0) return 0;
|
|
1146
|
+
return lowerBound(sorted, s) / sorted.length;
|
|
1147
|
+
});
|
|
1148
|
+
return map(logisticZ, (z, i) => z * (1 - NORMALIZATION_ALPHA) + cdf[i] * NORMALIZATION_ALPHA);
|
|
903
1149
|
}
|
|
904
1150
|
|
|
905
|
-
|
|
1151
|
+
//#endregion
|
|
1152
|
+
//#region src/core/pruner/heap/min-heap.ts
|
|
1153
|
+
/**
|
|
1154
|
+
* Generic binary min-heap.
|
|
1155
|
+
*
|
|
1156
|
+
* Elements are ordered by a numeric `score` field.
|
|
1157
|
+
* push/pop: O(log N), size: O(1).
|
|
1158
|
+
*/
|
|
906
1159
|
var MinHeap = class {
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1160
|
+
h = [];
|
|
1161
|
+
get size() {
|
|
1162
|
+
return this.h.length;
|
|
1163
|
+
}
|
|
1164
|
+
push(entry) {
|
|
1165
|
+
this.h.push(entry);
|
|
1166
|
+
this.siftUp(this.h.length - 1);
|
|
1167
|
+
}
|
|
1168
|
+
pop() {
|
|
1169
|
+
const n = this.h.length;
|
|
1170
|
+
if (n === 0) return void 0;
|
|
1171
|
+
const top = this.h[0];
|
|
1172
|
+
const last = this.h.pop();
|
|
1173
|
+
if (n > 1) {
|
|
1174
|
+
this.h[0] = last;
|
|
1175
|
+
this.siftDown(0);
|
|
1176
|
+
}
|
|
1177
|
+
return top;
|
|
1178
|
+
}
|
|
1179
|
+
siftUp(i) {
|
|
1180
|
+
while (i > 0) {
|
|
1181
|
+
const p = i - 1 >> 1;
|
|
1182
|
+
if (this.h[p].score <= this.h[i].score) break;
|
|
1183
|
+
[this.h[p], this.h[i]] = [this.h[i], this.h[p]];
|
|
1184
|
+
i = p;
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
siftDown(i) {
|
|
1188
|
+
const n = this.h.length;
|
|
1189
|
+
for (;;) {
|
|
1190
|
+
let m = i;
|
|
1191
|
+
const l = 2 * i + 1;
|
|
1192
|
+
const r = 2 * i + 2;
|
|
1193
|
+
if (l < n && this.h[l].score < this.h[m].score) m = l;
|
|
1194
|
+
if (r < n && this.h[r].score < this.h[m].score) m = r;
|
|
1195
|
+
if (m === i) break;
|
|
1196
|
+
[this.h[m], this.h[i]] = [this.h[i], this.h[m]];
|
|
1197
|
+
i = m;
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
947
1200
|
};
|
|
948
1201
|
|
|
949
|
-
|
|
1202
|
+
//#endregion
|
|
1203
|
+
//#region src/core/pruner/pruner.ts
|
|
1204
|
+
/**
|
|
1205
|
+
* Edge-aware greedy merge with re-linking — O(N log N).
|
|
1206
|
+
*
|
|
1207
|
+
* 1. Build a doubly-linked list of frames
|
|
1208
|
+
* 2. Insert all edges into a min-heap
|
|
1209
|
+
* 3. Pop the lowest-score edge (most similar pair)
|
|
1210
|
+
* 4. Remove the later frame (tgtId), re-link neighbors
|
|
1211
|
+
* 5. Push synthetic edge with score = max(left, right)
|
|
1212
|
+
* 6. Repeat until surviving count === targetCount
|
|
1213
|
+
* 7. First and last frames are never removed (boundary preservation)
|
|
1214
|
+
*
|
|
1215
|
+
* Stale heap entries (involving removed frames) are lazily skipped on pop.
|
|
1216
|
+
*/
|
|
950
1217
|
function pruneTo(graph, frames, targetCount) {
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1218
|
+
if (frames.length <= targetCount) return new Set(map(frames, (f) => f.id));
|
|
1219
|
+
const prev = /* @__PURE__ */ new Map();
|
|
1220
|
+
const next = /* @__PURE__ */ new Map();
|
|
1221
|
+
for (let i = 0; i < frames.length; i++) {
|
|
1222
|
+
if (i > 0) prev.set(frames[i].id, frames[i - 1].id);
|
|
1223
|
+
if (i < frames.length - 1) next.set(frames[i].id, frames[i + 1].id);
|
|
1224
|
+
}
|
|
1225
|
+
const edgeScore = /* @__PURE__ */ new Map();
|
|
1226
|
+
const heap = new MinHeap();
|
|
1227
|
+
for (const edge of graph) {
|
|
1228
|
+
edgeScore.set(`${edge.sourceId}:${edge.targetId}`, edge.score);
|
|
1229
|
+
heap.push({
|
|
1230
|
+
score: edge.score,
|
|
1231
|
+
srcId: edge.sourceId,
|
|
1232
|
+
tgtId: edge.targetId
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
const surviving = new Set(map(frames, (f) => f.id));
|
|
1236
|
+
const firstId = frames[0].id;
|
|
1237
|
+
const lastId = frames[frames.length - 1].id;
|
|
1238
|
+
while (surviving.size > targetCount && heap.size > 0) {
|
|
1239
|
+
const entry = heap.pop();
|
|
1240
|
+
if (!surviving.has(entry.srcId) || !surviving.has(entry.tgtId)) continue;
|
|
1241
|
+
const key = `${entry.srcId}:${entry.tgtId}`;
|
|
1242
|
+
if (edgeScore.get(key) !== entry.score) continue;
|
|
1243
|
+
if (entry.tgtId === firstId || entry.tgtId === lastId) continue;
|
|
1244
|
+
surviving.delete(entry.tgtId);
|
|
1245
|
+
edgeScore.delete(key);
|
|
1246
|
+
const tgtNext = next.get(entry.tgtId);
|
|
1247
|
+
if (tgtNext !== void 0) {
|
|
1248
|
+
const rightKey = `${entry.tgtId}:${tgtNext}`;
|
|
1249
|
+
const rightScore = edgeScore.get(rightKey) ?? 0;
|
|
1250
|
+
edgeScore.delete(rightKey);
|
|
1251
|
+
const newScore = Math.max(entry.score, rightScore);
|
|
1252
|
+
edgeScore.set(`${entry.srcId}:${tgtNext}`, newScore);
|
|
1253
|
+
heap.push({
|
|
1254
|
+
score: newScore,
|
|
1255
|
+
srcId: entry.srcId,
|
|
1256
|
+
tgtId: tgtNext
|
|
1257
|
+
});
|
|
1258
|
+
next.set(entry.srcId, tgtNext);
|
|
1259
|
+
prev.set(tgtNext, entry.srcId);
|
|
1260
|
+
} else next.delete(entry.srcId);
|
|
1261
|
+
prev.delete(entry.tgtId);
|
|
1262
|
+
next.delete(entry.tgtId);
|
|
1263
|
+
}
|
|
1264
|
+
return surviving;
|
|
998
1265
|
}
|
|
1266
|
+
/**
|
|
1267
|
+
* Non-Maximum Suppression (NMS) for consecutive edge runs.
|
|
1268
|
+
*
|
|
1269
|
+
* Consecutive edges share overlapping frames (edge i: frame i->i+1,
|
|
1270
|
+
* edge i+1: frame i+1->i+2), so consecutive passing edges indicate
|
|
1271
|
+
* the same visual transition region. This function groups consecutive
|
|
1272
|
+
* passing edge indices into "runs" and keeps all distinct peaks per run.
|
|
1273
|
+
*
|
|
1274
|
+
* Multi-peak detection: within each run, strict local maxima (score higher
|
|
1275
|
+
* than both neighbors) are identified. Each local maximum represents a
|
|
1276
|
+
* distinct visual transition. If no strict local maxima exist (plateau or
|
|
1277
|
+
* monotonic sequence), the global peak of the run is selected as fallback.
|
|
1278
|
+
*
|
|
1279
|
+
* Single-element runs are unaffected (isolated transitions preserved).
|
|
1280
|
+
*
|
|
1281
|
+
* @param graph - full ScoreEdge array (for targetId lookup)
|
|
1282
|
+
* @param passingIndices - edge indices that passed threshold filtering (sorted ascending)
|
|
1283
|
+
* @param normalizedScores - normalized score array (same length as graph)
|
|
1284
|
+
* @returns Set of targetIds to add to surviving set (one or more per run)
|
|
1285
|
+
*/
|
|
999
1286
|
function suppressConsecutiveRuns(graph, passingIndices, normalizedScores) {
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
const idx = passingIndices[j];
|
|
1029
|
-
if (normalizedScores[idx] > normalizedScores[peakIdx]) {
|
|
1030
|
-
peakIdx = idx;
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
result.add(graph[peakIdx].targetId);
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
|
-
runStart = runEnd + 1;
|
|
1037
|
-
}
|
|
1038
|
-
return result;
|
|
1287
|
+
const result = /* @__PURE__ */ new Set();
|
|
1288
|
+
let runStart = 0;
|
|
1289
|
+
while (runStart < passingIndices.length) {
|
|
1290
|
+
let runEnd = runStart;
|
|
1291
|
+
while (runEnd + 1 < passingIndices.length && passingIndices[runEnd + 1] === passingIndices[runEnd] + 1) runEnd++;
|
|
1292
|
+
if (runEnd - runStart + 1 === 1) result.add(graph[passingIndices[runStart]].targetId);
|
|
1293
|
+
else {
|
|
1294
|
+
const peaks = [];
|
|
1295
|
+
for (let j = runStart; j <= runEnd; j++) {
|
|
1296
|
+
const idx = passingIndices[j];
|
|
1297
|
+
const score = normalizedScores[idx];
|
|
1298
|
+
const prevScore = j > runStart ? normalizedScores[passingIndices[j - 1]] : -Infinity;
|
|
1299
|
+
const nextScore = j < runEnd ? normalizedScores[passingIndices[j + 1]] : -Infinity;
|
|
1300
|
+
if (score > prevScore && score > nextScore) peaks.push(idx);
|
|
1301
|
+
}
|
|
1302
|
+
if (peaks.length > 0) for (const peakIdx of peaks) result.add(graph[peakIdx].targetId);
|
|
1303
|
+
else {
|
|
1304
|
+
let peakIdx = passingIndices[runStart];
|
|
1305
|
+
for (let j = runStart + 1; j <= runEnd; j++) {
|
|
1306
|
+
const idx = passingIndices[j];
|
|
1307
|
+
if (normalizedScores[idx] > normalizedScores[peakIdx]) peakIdx = idx;
|
|
1308
|
+
}
|
|
1309
|
+
result.add(graph[peakIdx].targetId);
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
runStart = runEnd + 1;
|
|
1313
|
+
}
|
|
1314
|
+
return result;
|
|
1039
1315
|
}
|
|
1316
|
+
/**
|
|
1317
|
+
* Threshold-based pruning with NMS -- including normalization, O(N log N).
|
|
1318
|
+
*
|
|
1319
|
+
* 1. Scores are normalized to [0, 1] via percentile normalization.
|
|
1320
|
+
* 2. Edges with normalized score >= threshold are collected.
|
|
1321
|
+
* 3. Non-Maximum Suppression groups consecutive passing edges and keeps
|
|
1322
|
+
* only the peak per run, preventing near-duplicate frame selection
|
|
1323
|
+
* from a single visual transition.
|
|
1324
|
+
*
|
|
1325
|
+
* First and last frames are always preserved (boundary protection).
|
|
1326
|
+
*/
|
|
1040
1327
|
function pruneByThreshold(graph, frames, threshold) {
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
}
|
|
1052
|
-
const nmsTargets = suppressConsecutiveRuns(graph, passingIndices, normalized);
|
|
1053
|
-
for (const id of nmsTargets) {
|
|
1054
|
-
surviving.add(id);
|
|
1055
|
-
}
|
|
1056
|
-
return surviving;
|
|
1328
|
+
if (frames.length === 0) return /* @__PURE__ */ new Set();
|
|
1329
|
+
const surviving = /* @__PURE__ */ new Set();
|
|
1330
|
+
surviving.add(frames[0].id);
|
|
1331
|
+
surviving.add(frames[frames.length - 1].id);
|
|
1332
|
+
const normalized = normalizeScores(graph);
|
|
1333
|
+
const passingIndices = [];
|
|
1334
|
+
for (let i = 0; i < graph.length; i++) if (normalized[i] >= threshold) passingIndices.push(i);
|
|
1335
|
+
const nmsTargets = suppressConsecutiveRuns(graph, passingIndices, normalized);
|
|
1336
|
+
for (const id of nmsTargets) surviving.add(id);
|
|
1337
|
+
return surviving;
|
|
1057
1338
|
}
|
|
1339
|
+
/**
|
|
1340
|
+
* Combined threshold + count pruning -- 2-stage pipeline.
|
|
1341
|
+
*
|
|
1342
|
+
* Stage 1: pruneByThreshold -- keep all frames with normalized score >= threshold
|
|
1343
|
+
* Stage 2: if result exceeds maxCount, rebuild subgraph with synthetic edges
|
|
1344
|
+
* (min-score over each gap) and apply pruneTo on the surviving subset
|
|
1345
|
+
*
|
|
1346
|
+
* Edge reconstruction: for consecutive survivors A, B with removed frames
|
|
1347
|
+
* [x1, x2, ...] between them, the synthetic edge score is:
|
|
1348
|
+
* min(score(A->x1), score(x1->x2), ..., score(xN->B))
|
|
1349
|
+
* This preserves the "weakest link" semantics.
|
|
1350
|
+
*/
|
|
1058
1351
|
function pruneByThresholdWithCap(graph, frames, threshold, maxCount) {
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
}
|
|
1087
|
-
syntheticEdges.push({
|
|
1088
|
-
sourceId: srcSurvivor.id,
|
|
1089
|
-
targetId: tgtSurvivor.id,
|
|
1090
|
-
score: minScore === Infinity ? 0 : minScore
|
|
1091
|
-
});
|
|
1092
|
-
}
|
|
1093
|
-
return pruneTo(syntheticEdges, survivingFrames, maxCount);
|
|
1352
|
+
const thresholdSurvivors = pruneByThreshold(graph, frames, threshold);
|
|
1353
|
+
if (thresholdSurvivors.size <= maxCount) return thresholdSurvivors;
|
|
1354
|
+
const survivingFrames = filter(frames, (f) => thresholdSurvivors.has(f.id));
|
|
1355
|
+
const idToOrigIdx = /* @__PURE__ */ new Map();
|
|
1356
|
+
for (let i = 0; i < frames.length; i++) idToOrigIdx.set(frames[i].id, i);
|
|
1357
|
+
const edgeLookup = /* @__PURE__ */ new Map();
|
|
1358
|
+
for (const e of graph) edgeLookup.set(`${e.sourceId}:${e.targetId}`, e.score);
|
|
1359
|
+
const syntheticEdges = [];
|
|
1360
|
+
for (let i = 0; i < survivingFrames.length - 1; i++) {
|
|
1361
|
+
const srcSurvivor = survivingFrames[i];
|
|
1362
|
+
const tgtSurvivor = survivingFrames[i + 1];
|
|
1363
|
+
const srcOrigIdx = idToOrigIdx.get(srcSurvivor.id);
|
|
1364
|
+
const tgtOrigIdx = idToOrigIdx.get(tgtSurvivor.id);
|
|
1365
|
+
let minScore = Infinity;
|
|
1366
|
+
for (let j = srcOrigIdx; j < tgtOrigIdx; j++) {
|
|
1367
|
+
const fromId = frames[j].id;
|
|
1368
|
+
const toId = frames[j + 1].id;
|
|
1369
|
+
const score = edgeLookup.get(`${fromId}:${toId}`) ?? 0;
|
|
1370
|
+
if (score < minScore) minScore = score;
|
|
1371
|
+
}
|
|
1372
|
+
syntheticEdges.push({
|
|
1373
|
+
sourceId: srcSurvivor.id,
|
|
1374
|
+
targetId: tgtSurvivor.id,
|
|
1375
|
+
score: minScore === Infinity ? 0 : minScore
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
return pruneTo(syntheticEdges, survivingFrames, maxCount);
|
|
1094
1379
|
}
|
|
1095
1380
|
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1381
|
+
//#endregion
|
|
1382
|
+
//#region src/core/segmenter/scheduling/concurrency.ts
|
|
1383
|
+
/**
|
|
1384
|
+
* Creates a concurrency limiter that runs at most `limit` tasks in parallel.
|
|
1385
|
+
* Lightweight replacement for p-limit to avoid external dependency.
|
|
1386
|
+
*/
|
|
1102
1387
|
function concurrencyLimit(limit) {
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
}
|
|
1117
|
-
};
|
|
1388
|
+
limit = Math.max(1, limit);
|
|
1389
|
+
let active = 0;
|
|
1390
|
+
const queue = [];
|
|
1391
|
+
return async (fn) => {
|
|
1392
|
+
while (active >= limit) await new Promise((resolve) => queue.push(resolve));
|
|
1393
|
+
active++;
|
|
1394
|
+
try {
|
|
1395
|
+
return await fn();
|
|
1396
|
+
} finally {
|
|
1397
|
+
active--;
|
|
1398
|
+
queue.shift()?.();
|
|
1399
|
+
}
|
|
1400
|
+
};
|
|
1118
1401
|
}
|
|
1119
1402
|
|
|
1120
|
-
|
|
1403
|
+
//#endregion
|
|
1404
|
+
//#region src/core/segmenter/segmenter.ts
|
|
1405
|
+
/**
|
|
1406
|
+
* Determine whether segmentation should be used.
|
|
1407
|
+
* Returns false for frames mode and GIF files.
|
|
1408
|
+
* Actual duration check happens inside runSegmentedPipeline after metadata fetch.
|
|
1409
|
+
*/
|
|
1121
1410
|
function shouldSegment(resolvedOptions, originalOptions) {
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1411
|
+
if (resolvedOptions.mode === "frames") return false;
|
|
1412
|
+
if (originalOptions.mode === "file") {
|
|
1413
|
+
if (originalOptions.inputPath.toLowerCase().endsWith(".gif")) return false;
|
|
1414
|
+
}
|
|
1415
|
+
return true;
|
|
1127
1416
|
}
|
|
1417
|
+
/**
|
|
1418
|
+
* Partition the global extraction grid into nonempty logical segments.
|
|
1419
|
+
* @param totalDuration Positive source duration in seconds.
|
|
1420
|
+
* @param maxSegmentDuration Positive logical segment width in seconds.
|
|
1421
|
+
* @param maxFrames Candidate budget, defensively raised to at least two.
|
|
1422
|
+
* @param fps Positive requested sampling frequency.
|
|
1423
|
+
* @returns Contiguous plan indices with grid-aligned seeks and overlap-inclusive limits.
|
|
1424
|
+
*/
|
|
1128
1425
|
function computeSegmentPlan(totalDuration, maxSegmentDuration, maxFrames, fps) {
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
});
|
|
1179
|
-
}
|
|
1180
|
-
const totalAllocated = segments.reduce(
|
|
1181
|
-
(sum, s) => sum + s.allocatedFrames,
|
|
1182
|
-
0
|
|
1183
|
-
);
|
|
1184
|
-
if (totalAllocated > maxFrames) {
|
|
1185
|
-
segments[segments.length - 1].allocatedFrames -= totalAllocated - maxFrames;
|
|
1186
|
-
}
|
|
1187
|
-
return segments;
|
|
1426
|
+
const frameLimit = Math.max(2, maxFrames);
|
|
1427
|
+
const effectiveFps = Math.min(fps, frameLimit / totalDuration);
|
|
1428
|
+
if (totalDuration <= maxSegmentDuration) return [{
|
|
1429
|
+
index: 0,
|
|
1430
|
+
startTime: 0,
|
|
1431
|
+
endTime: totalDuration,
|
|
1432
|
+
duration: totalDuration,
|
|
1433
|
+
allocatedFrames: frameLimit,
|
|
1434
|
+
effectiveFps,
|
|
1435
|
+
overlapBefore: 0,
|
|
1436
|
+
overlapAfter: 0,
|
|
1437
|
+
extractStartTime: 0,
|
|
1438
|
+
extractDuration: totalDuration
|
|
1439
|
+
}];
|
|
1440
|
+
const segments = [];
|
|
1441
|
+
for (let slot = 0; slot < frameLimit; slot++) {
|
|
1442
|
+
const timestamp = slot / effectiveFps;
|
|
1443
|
+
if (timestamp >= totalDuration) break;
|
|
1444
|
+
const startTime = Math.floor(timestamp / maxSegmentDuration) * maxSegmentDuration;
|
|
1445
|
+
const previous = segments[segments.length - 1];
|
|
1446
|
+
if (previous?.startTime === startTime) {
|
|
1447
|
+
previous.allocatedFrames++;
|
|
1448
|
+
continue;
|
|
1449
|
+
}
|
|
1450
|
+
const endTime = Math.min(startTime + maxSegmentDuration, totalDuration);
|
|
1451
|
+
segments.push({
|
|
1452
|
+
index: segments.length,
|
|
1453
|
+
startTime,
|
|
1454
|
+
endTime,
|
|
1455
|
+
duration: endTime - startTime,
|
|
1456
|
+
allocatedFrames: 1,
|
|
1457
|
+
effectiveFps,
|
|
1458
|
+
overlapBefore: 0,
|
|
1459
|
+
overlapAfter: 0,
|
|
1460
|
+
extractStartTime: timestamp,
|
|
1461
|
+
extractDuration: 0
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
let firstSlot = 0;
|
|
1465
|
+
for (const segment of segments) {
|
|
1466
|
+
const nextSlot = firstSlot + segment.allocatedFrames;
|
|
1467
|
+
segment.overlapBefore = segment.index > 0 ? 1 : 0;
|
|
1468
|
+
segment.overlapAfter = segment.index < segments.length - 1 ? 1 : 0;
|
|
1469
|
+
segment.extractStartTime = (firstSlot - segment.overlapBefore) / effectiveFps;
|
|
1470
|
+
segment.extractDuration = (segment.overlapAfter ? Math.min(totalDuration, (nextSlot + 1) / effectiveFps) : totalDuration) - segment.extractStartTime;
|
|
1471
|
+
segment.allocatedFrames += segment.overlapBefore + segment.overlapAfter;
|
|
1472
|
+
firstSlot = nextSlot;
|
|
1473
|
+
}
|
|
1474
|
+
return segments;
|
|
1188
1475
|
}
|
|
1476
|
+
/**
|
|
1477
|
+
* Collect all frames from every segment, adjusting timestamps by extractStartTime.
|
|
1478
|
+
*/
|
|
1189
1479
|
function collectAllFrames(segmentResults) {
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
localId: frame.id
|
|
1201
|
-
});
|
|
1202
|
-
}
|
|
1203
|
-
}
|
|
1204
|
-
return allFrames;
|
|
1480
|
+
const allFrames = [];
|
|
1481
|
+
for (const result of segmentResults) for (const frame of result.frames) allFrames.push({
|
|
1482
|
+
frame: {
|
|
1483
|
+
...frame,
|
|
1484
|
+
timestamp: frame.timestamp + result.segment.extractStartTime
|
|
1485
|
+
},
|
|
1486
|
+
segmentIndex: result.segment.index,
|
|
1487
|
+
localId: frame.id
|
|
1488
|
+
});
|
|
1489
|
+
return allFrames;
|
|
1205
1490
|
}
|
|
1491
|
+
/**
|
|
1492
|
+
* Sort frames in place and alias overlap duplicates to the first survivor.
|
|
1493
|
+
* @param frames Collected entries whose segment index and local ID identify a frame.
|
|
1494
|
+
* @param effectiveFps Positive sampling frequency; half a frame interval is the threshold.
|
|
1495
|
+
* @returns Timestamp-ordered survivors and duplicate keys pointing directly to survivor keys.
|
|
1496
|
+
*/
|
|
1206
1497
|
function deduplicateFrames(frames, effectiveFps) {
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1498
|
+
frames.sort((a, b) => a.frame.timestamp - b.frame.timestamp);
|
|
1499
|
+
const dupThreshold = 1 / (effectiveFps * 2);
|
|
1500
|
+
const unique = [];
|
|
1501
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
1502
|
+
for (const entry of frames) {
|
|
1503
|
+
if (unique.length > 0) {
|
|
1504
|
+
const last = unique[unique.length - 1];
|
|
1505
|
+
if (Math.abs(entry.frame.timestamp - last.frame.timestamp) < dupThreshold) {
|
|
1506
|
+
aliases.set(`${entry.segmentIndex}:${entry.localId}`, `${last.segmentIndex}:${last.localId}`);
|
|
1507
|
+
continue;
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
unique.push(entry);
|
|
1511
|
+
}
|
|
1512
|
+
return {
|
|
1513
|
+
unique,
|
|
1514
|
+
aliases
|
|
1515
|
+
};
|
|
1220
1516
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1517
|
+
/**
|
|
1518
|
+
* Assign sequential global IDs and retain duplicate local IDs as aliases.
|
|
1519
|
+
* @param uniqueFrames Timestamp-ordered survivors with distinct segment/local keys.
|
|
1520
|
+
* @param aliases Duplicate keys pointing directly to keys in uniqueFrames.
|
|
1521
|
+
* @returns Remapped frames and a global ID lookup covering survivors and duplicates.
|
|
1522
|
+
*/
|
|
1523
|
+
function remapFrameIds(uniqueFrames, aliases) {
|
|
1524
|
+
const globalIdMap = /* @__PURE__ */ new Map();
|
|
1525
|
+
const frames = uniqueFrames.map((entry, globalId) => {
|
|
1526
|
+
globalIdMap.set(`${entry.segmentIndex}:${entry.localId}`, globalId);
|
|
1527
|
+
return {
|
|
1528
|
+
id: globalId,
|
|
1529
|
+
timestamp: entry.frame.timestamp,
|
|
1530
|
+
extractPath: entry.frame.extractPath
|
|
1531
|
+
};
|
|
1532
|
+
});
|
|
1533
|
+
for (const [alias, survivor] of aliases) globalIdMap.set(alias, globalIdMap.get(survivor));
|
|
1534
|
+
return {
|
|
1535
|
+
frames,
|
|
1536
|
+
globalIdMap
|
|
1537
|
+
};
|
|
1232
1538
|
}
|
|
1539
|
+
/**
|
|
1540
|
+
* Remap edges, dropping missing endpoints and self loops while keeping the highest pair score.
|
|
1541
|
+
* @param segmentResults Segment-local edges in encounter order.
|
|
1542
|
+
* @param globalIdMap Survivor and duplicate local keys mapped to global IDs.
|
|
1543
|
+
* @returns One edge per surviving directed pair without changing its score.
|
|
1544
|
+
*/
|
|
1233
1545
|
function remapEdges(segmentResults, globalIdMap) {
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
targetId: newTargetId,
|
|
1260
|
-
score: edge.score
|
|
1261
|
-
});
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
return edges;
|
|
1546
|
+
const edges = [];
|
|
1547
|
+
const edgeMap = /* @__PURE__ */ new Map();
|
|
1548
|
+
for (const result of segmentResults) for (const edge of result.edges) {
|
|
1549
|
+
const newSourceId = globalIdMap.get(`${result.segment.index}:${edge.sourceId}`);
|
|
1550
|
+
const newTargetId = globalIdMap.get(`${result.segment.index}:${edge.targetId}`);
|
|
1551
|
+
if (newSourceId === void 0 || newTargetId === void 0) continue;
|
|
1552
|
+
if (newSourceId === newTargetId) continue;
|
|
1553
|
+
const edgeKey = `${newSourceId}-${newTargetId}`;
|
|
1554
|
+
const existingIdx = edgeMap.get(edgeKey);
|
|
1555
|
+
if (existingIdx !== void 0) {
|
|
1556
|
+
if (edges[existingIdx].score < edge.score) edges[existingIdx] = {
|
|
1557
|
+
sourceId: newSourceId,
|
|
1558
|
+
targetId: newTargetId,
|
|
1559
|
+
score: edge.score
|
|
1560
|
+
};
|
|
1561
|
+
} else {
|
|
1562
|
+
edgeMap.set(edgeKey, edges.length);
|
|
1563
|
+
edges.push({
|
|
1564
|
+
sourceId: newSourceId,
|
|
1565
|
+
targetId: newTargetId,
|
|
1566
|
+
score: edge.score
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
return edges;
|
|
1266
1571
|
}
|
|
1572
|
+
/**
|
|
1573
|
+
* Remap animations, dropping missing or collapsed endpoints and retaining the first pair entry.
|
|
1574
|
+
* @param segmentResults Segment-local tracker entries in encounter order.
|
|
1575
|
+
* @param globalIdMap Survivor and duplicate local keys mapped to global IDs.
|
|
1576
|
+
* @returns One animation per directed pair with its original tracker duration and metadata.
|
|
1577
|
+
*/
|
|
1267
1578
|
function remapAnimations(segmentResults, globalIdMap) {
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
}
|
|
1284
|
-
}
|
|
1285
|
-
return animations;
|
|
1579
|
+
const animations = /* @__PURE__ */ new Map();
|
|
1580
|
+
for (const result of segmentResults) for (const anim of result.animations) {
|
|
1581
|
+
const newStartId = globalIdMap.get(`${result.segment.index}:${anim.startFrameId}`);
|
|
1582
|
+
const newEndId = globalIdMap.get(`${result.segment.index}:${anim.endFrameId}`);
|
|
1583
|
+
if (newStartId === void 0 || newEndId === void 0) continue;
|
|
1584
|
+
if (newStartId === newEndId) continue;
|
|
1585
|
+
const animationKey = `${newStartId}-${newEndId}`;
|
|
1586
|
+
if (animations.has(animationKey)) continue;
|
|
1587
|
+
animations.set(animationKey, {
|
|
1588
|
+
...anim,
|
|
1589
|
+
startFrameId: newStartId,
|
|
1590
|
+
endFrameId: newEndId
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
return [...animations.values()];
|
|
1286
1594
|
}
|
|
1595
|
+
/**
|
|
1596
|
+
* Merge multiple segment results into a single unified frame/edge/animation set.
|
|
1597
|
+
* @param segmentResults Local frames, edges and tracker entries with distinct segment indices.
|
|
1598
|
+
* @returns Global timestamp-ordered frames, aliased edges and animations without self loops.
|
|
1599
|
+
* Duplicate edges keep the higher score; duplicate animations keep the first tracker entry.
|
|
1600
|
+
*/
|
|
1287
1601
|
function mergeSegmentFrames(segmentResults) {
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1602
|
+
if (segmentResults.length === 0) return {
|
|
1603
|
+
frames: [],
|
|
1604
|
+
edges: [],
|
|
1605
|
+
animations: [],
|
|
1606
|
+
analysisResolution: {
|
|
1607
|
+
width: 0,
|
|
1608
|
+
height: 0
|
|
1609
|
+
}
|
|
1610
|
+
};
|
|
1611
|
+
const effectiveFps = segmentResults[0].segment.effectiveFps;
|
|
1612
|
+
const { unique, aliases } = deduplicateFrames(collectAllFrames(segmentResults), effectiveFps);
|
|
1613
|
+
const { frames, globalIdMap } = remapFrameIds(unique, aliases);
|
|
1614
|
+
return {
|
|
1615
|
+
frames,
|
|
1616
|
+
edges: remapEdges(segmentResults, globalIdMap),
|
|
1617
|
+
animations: remapAnimations(segmentResults, globalIdMap),
|
|
1618
|
+
analysisResolution: segmentResults[0].analysisResolution
|
|
1619
|
+
};
|
|
1298
1620
|
}
|
|
1299
1621
|
function buildSegmentContext(segment, frames, segmentWorkspacePath, resolvedOptions, onProgress) {
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1622
|
+
return {
|
|
1623
|
+
options: {
|
|
1624
|
+
...resolvedOptions,
|
|
1625
|
+
fps: segment.effectiveFps,
|
|
1626
|
+
maxFrames: segment.allocatedFrames
|
|
1627
|
+
},
|
|
1628
|
+
workspacePath: segmentWorkspacePath,
|
|
1629
|
+
effectiveFps: segment.effectiveFps,
|
|
1630
|
+
frames,
|
|
1631
|
+
graph: [],
|
|
1632
|
+
status: "ANALYZING",
|
|
1633
|
+
emitProgress: onProgress
|
|
1634
|
+
};
|
|
1312
1635
|
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Extract frames for a single segment and analyze them.
|
|
1638
|
+
* Each segment uses an isolated workspace directory.
|
|
1639
|
+
*/
|
|
1313
1640
|
async function processSegment(inputPath, segment, workspacePath, resolvedOptions, onProgress) {
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
const { edges, animations } = await analyzeFrames(ctx);
|
|
1334
|
-
return { segment, frames, edges, animations };
|
|
1641
|
+
const frames = await extractFramesForRange(inputPath, join(workspacePath, "frames"), segment.effectiveFps, resolvedOptions.scale, segment.extractStartTime, segment.extractDuration, segment.allocatedFrames);
|
|
1642
|
+
if (frames.length < 2) return {
|
|
1643
|
+
segment,
|
|
1644
|
+
frames,
|
|
1645
|
+
edges: [],
|
|
1646
|
+
animations: [],
|
|
1647
|
+
analysisResolution: {
|
|
1648
|
+
width: 0,
|
|
1649
|
+
height: 0
|
|
1650
|
+
}
|
|
1651
|
+
};
|
|
1652
|
+
const { edges, animations, analysisResolution } = await analyzeFrames(buildSegmentContext(segment, frames, workspacePath, resolvedOptions, onProgress));
|
|
1653
|
+
return {
|
|
1654
|
+
segment,
|
|
1655
|
+
frames,
|
|
1656
|
+
edges,
|
|
1657
|
+
animations,
|
|
1658
|
+
analysisResolution
|
|
1659
|
+
};
|
|
1335
1660
|
}
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
status: "FINALIZING",
|
|
1417
|
-
emitProgress: (percent) => options2.onProgress?.("FINALIZING", percent)
|
|
1418
|
-
};
|
|
1419
|
-
let outputFiles = [];
|
|
1420
|
-
let outputBuffers;
|
|
1421
|
-
if (resolvedOptions.mode === "buffer" || resolvedOptions.mode === "frames") {
|
|
1422
|
-
outputBuffers = await readFramesAsBuffers(
|
|
1423
|
-
prunedFrames,
|
|
1424
|
-
resolvedOptions.quality
|
|
1425
|
-
);
|
|
1426
|
-
} else {
|
|
1427
|
-
outputFiles = await finalizeOutput(ctx, prunedFrames);
|
|
1428
|
-
}
|
|
1429
|
-
options2.onProgress?.("FINALIZING", 100);
|
|
1430
|
-
logger.success(
|
|
1431
|
-
`Segmented pipeline: ${prunedFrames.length} scenes from ${frames.length} frames (${segments.length} segments)`
|
|
1432
|
-
);
|
|
1433
|
-
return {
|
|
1434
|
-
success: true,
|
|
1435
|
-
originalFramesCount: frames.length,
|
|
1436
|
-
prunedFramesCount: prunedFrames.length,
|
|
1437
|
-
outputFiles,
|
|
1438
|
-
outputBuffers,
|
|
1439
|
-
animations,
|
|
1440
|
-
video: {
|
|
1441
|
-
originalDurationMs: totalDuration * 1e3,
|
|
1442
|
-
fps: resolvedOptions.fps,
|
|
1443
|
-
resolution: {
|
|
1444
|
-
width: resolvedOptions.scale,
|
|
1445
|
-
height: Math.round(resolvedOptions.scale * 9 / 16)
|
|
1446
|
-
}
|
|
1447
|
-
},
|
|
1448
|
-
executionTimeMs: Date.now() - pipelineStart
|
|
1449
|
-
};
|
|
1450
|
-
} catch (error) {
|
|
1451
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
1452
|
-
logger.error(`Segmented pipeline failed: ${err.message}`);
|
|
1453
|
-
throw err;
|
|
1454
|
-
} finally {
|
|
1455
|
-
if (!resolvedOptions.debug) {
|
|
1456
|
-
await cleanupWorkspace(mainWorkspace);
|
|
1457
|
-
} else {
|
|
1458
|
-
logger.debug(`Debug mode: workspace preserved at ${mainWorkspace}`);
|
|
1459
|
-
}
|
|
1460
|
-
}
|
|
1661
|
+
/**
|
|
1662
|
+
* Full segmented pipeline: metadata → plan → parallel extract+analyze → merge → prune → finalize.
|
|
1663
|
+
* Called from runPipeline when shouldSegment() returns true.
|
|
1664
|
+
*/
|
|
1665
|
+
async function runSegmentedPipeline(options, resolvedOptions) {
|
|
1666
|
+
const pipelineStart = Date.now();
|
|
1667
|
+
const sessionId = randomUUID();
|
|
1668
|
+
let mainWorkspace = "";
|
|
1669
|
+
try {
|
|
1670
|
+
mainWorkspace = await createWorkspace(sessionId);
|
|
1671
|
+
logger.debug(`Segmented pipeline: workspace at ${mainWorkspace}`);
|
|
1672
|
+
const { resolvedInputPath } = await resolveInput(options, mainWorkspace);
|
|
1673
|
+
const inputPath = resolvedInputPath ?? resolvedOptions.inputPath;
|
|
1674
|
+
if (!inputPath) throw new Error("No input path available for segmented pipeline");
|
|
1675
|
+
const metadata = await getVideoMetadata(inputPath);
|
|
1676
|
+
const totalDuration = parseFloat(metadata.format?.duration ?? "0");
|
|
1677
|
+
if (totalDuration <= 0) throw new Error(`Invalid video duration: ${totalDuration}`);
|
|
1678
|
+
logger.debug(`Video duration: ${totalDuration}s`);
|
|
1679
|
+
const segments = computeSegmentPlan(totalDuration, resolvedOptions.maxSegmentDuration, resolvedOptions.maxFrames, resolvedOptions.fps);
|
|
1680
|
+
logger.debug(`Segment plan: ${segments.length} segments`);
|
|
1681
|
+
const limit = concurrencyLimit(resolvedOptions.concurrency);
|
|
1682
|
+
const segmentProgresses = new Array(segments.length).fill(0);
|
|
1683
|
+
const weights = map(segments, (s) => s.duration / totalDuration);
|
|
1684
|
+
const emitOverallProgress = (phase) => {
|
|
1685
|
+
if (!options.onProgress) return;
|
|
1686
|
+
const overall = weights.reduce((sum, w, i) => sum + w * (segmentProgresses[i] ?? 0), 0);
|
|
1687
|
+
options.onProgress(phase, Math.min(100, overall));
|
|
1688
|
+
};
|
|
1689
|
+
options.onProgress?.("EXTRACTING", 0);
|
|
1690
|
+
const results = await Promise.all(map(segments, (segment) => limit(async () => {
|
|
1691
|
+
const segWorkspace = await createSegmentWorkspace(mainWorkspace, segment.index);
|
|
1692
|
+
return await processSegment(inputPath, segment, segWorkspace, resolvedOptions, (percent) => {
|
|
1693
|
+
segmentProgresses[segment.index] = percent;
|
|
1694
|
+
emitOverallProgress("ANALYZING");
|
|
1695
|
+
});
|
|
1696
|
+
})));
|
|
1697
|
+
options.onProgress?.("ANALYZING", 100);
|
|
1698
|
+
const { frames, edges, animations, analysisResolution } = mergeSegmentFrames(results);
|
|
1699
|
+
logger.debug(`Merged: ${frames.length} frames, ${edges.length} edges, ${animations.length} animations`);
|
|
1700
|
+
options.onProgress?.("PRUNING", 0);
|
|
1701
|
+
const survivingIds = pruneByThresholdWithCap(edges, frames, resolvedOptions.threshold, resolvedOptions.count);
|
|
1702
|
+
const prunedFrames = filter(frames, (f) => survivingIds.has(f.id));
|
|
1703
|
+
options.onProgress?.("PRUNING", 100);
|
|
1704
|
+
options.onProgress?.("FINALIZING", 0);
|
|
1705
|
+
const ctx = {
|
|
1706
|
+
options: resolvedOptions,
|
|
1707
|
+
effectiveFps: segments[0]?.effectiveFps,
|
|
1708
|
+
sourceDurationSec: totalDuration,
|
|
1709
|
+
analysisResolution,
|
|
1710
|
+
workspacePath: mainWorkspace,
|
|
1711
|
+
frames,
|
|
1712
|
+
graph: edges,
|
|
1713
|
+
animations,
|
|
1714
|
+
status: "FINALIZING",
|
|
1715
|
+
emitProgress: (percent) => options.onProgress?.("FINALIZING", percent)
|
|
1716
|
+
};
|
|
1717
|
+
let outputFiles = [];
|
|
1718
|
+
let outputBuffers;
|
|
1719
|
+
if (resolvedOptions.mode === "buffer" || resolvedOptions.mode === "frames") outputBuffers = await readFramesAsBuffers(prunedFrames, resolvedOptions.quality);
|
|
1720
|
+
else outputFiles = await finalizeOutput(ctx, prunedFrames);
|
|
1721
|
+
options.onProgress?.("FINALIZING", 100);
|
|
1722
|
+
logger.success(`Segmented pipeline: ${prunedFrames.length} scenes from ${frames.length} frames (${segments.length} segments)`);
|
|
1723
|
+
const outputMetadata = await buildVideoMetadata(ctx, prunedFrames, analysisResolution);
|
|
1724
|
+
return {
|
|
1725
|
+
success: true,
|
|
1726
|
+
originalFramesCount: frames.length,
|
|
1727
|
+
prunedFramesCount: prunedFrames.length,
|
|
1728
|
+
outputFiles,
|
|
1729
|
+
outputBuffers,
|
|
1730
|
+
...outputMetadata,
|
|
1731
|
+
executionTimeMs: Date.now() - pipelineStart
|
|
1732
|
+
};
|
|
1733
|
+
} catch (error) {
|
|
1734
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
1735
|
+
logger.error(`Segmented pipeline failed: ${err.message}`);
|
|
1736
|
+
throw err;
|
|
1737
|
+
} finally {
|
|
1738
|
+
if (!resolvedOptions.debug) await cleanupWorkspace(mainWorkspace);
|
|
1739
|
+
else logger.debug(`Debug mode: workspace preserved at ${mainWorkspace}`);
|
|
1740
|
+
}
|
|
1461
1741
|
}
|
|
1462
1742
|
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
animations: ctx.animations,
|
|
1543
|
-
video: {
|
|
1544
|
-
originalDurationMs: ctx.frames.length / ctx.options.fps * 1e3,
|
|
1545
|
-
fps: ctx.options.fps,
|
|
1546
|
-
resolution: {
|
|
1547
|
-
width: ctx.options.scale,
|
|
1548
|
-
height: Math.round(ctx.options.scale * 9 / 16)
|
|
1549
|
-
}
|
|
1550
|
-
},
|
|
1551
|
-
executionTimeMs: Date.now() - startTime
|
|
1552
|
-
};
|
|
1553
|
-
} catch (error) {
|
|
1554
|
-
ctx.status = "FAILED";
|
|
1555
|
-
ctx.error = error instanceof Error ? error : new Error(String(error));
|
|
1556
|
-
logger.error(`Pipeline failed: ${ctx.error.message}`);
|
|
1557
|
-
throw ctx.error;
|
|
1558
|
-
} finally {
|
|
1559
|
-
if (!resolvedOptions.debug) {
|
|
1560
|
-
await cleanupWorkspace(ctx.workspacePath);
|
|
1561
|
-
} else {
|
|
1562
|
-
logger.debug(`Debug mode: workspace preserved at ${ctx.workspacePath}`);
|
|
1563
|
-
}
|
|
1564
|
-
}
|
|
1743
|
+
//#endregion
|
|
1744
|
+
//#region src/core/orchestrator/orchestrator.ts
|
|
1745
|
+
async function runPipeline(options) {
|
|
1746
|
+
setDebugMode(options.debug ?? false);
|
|
1747
|
+
const resolvedOptions = resolveOptions(options);
|
|
1748
|
+
if (shouldSegment(resolvedOptions, options)) return runSegmentedPipeline(options, resolvedOptions);
|
|
1749
|
+
const startTime = Date.now();
|
|
1750
|
+
const sessionId = randomUUID();
|
|
1751
|
+
const ctx = {
|
|
1752
|
+
options: resolvedOptions,
|
|
1753
|
+
workspacePath: "",
|
|
1754
|
+
frames: [],
|
|
1755
|
+
graph: [],
|
|
1756
|
+
status: "INIT",
|
|
1757
|
+
emitProgress: (percent) => {
|
|
1758
|
+
if (options.onProgress && ctx.status !== "INIT" && ctx.status !== "SUCCESS" && ctx.status !== "FAILED") options.onProgress(ctx.status, percent);
|
|
1759
|
+
}
|
|
1760
|
+
};
|
|
1761
|
+
try {
|
|
1762
|
+
ctx.workspacePath = await createWorkspace(sessionId);
|
|
1763
|
+
logger.debug(`Workspace created: ${ctx.workspacePath}`);
|
|
1764
|
+
ctx.status = "EXTRACTING";
|
|
1765
|
+
const { frames: resolvedFrames, resolvedInputPath } = await resolveInput(options, ctx.workspacePath);
|
|
1766
|
+
if (resolvedOptions.mode === "frames") {
|
|
1767
|
+
ctx.frames = resolvedFrames;
|
|
1768
|
+
ctx.effectiveFps = 1;
|
|
1769
|
+
} else {
|
|
1770
|
+
const extractCtx = {
|
|
1771
|
+
...ctx,
|
|
1772
|
+
options: {
|
|
1773
|
+
...resolvedOptions,
|
|
1774
|
+
inputPath: resolvedInputPath
|
|
1775
|
+
}
|
|
1776
|
+
};
|
|
1777
|
+
ctx.frames = await extractFrames(extractCtx);
|
|
1778
|
+
ctx.effectiveFps = extractCtx.effectiveFps;
|
|
1779
|
+
ctx.sourceDurationSec = extractCtx.sourceDurationSec;
|
|
1780
|
+
}
|
|
1781
|
+
ctx.emitProgress(100);
|
|
1782
|
+
ctx.status = "ANALYZING";
|
|
1783
|
+
const { edges, animations, analysisResolution } = await analyzeFrames(ctx);
|
|
1784
|
+
ctx.graph = edges;
|
|
1785
|
+
ctx.animations = animations;
|
|
1786
|
+
ctx.analysisResolution = analysisResolution;
|
|
1787
|
+
ctx.status = "PRUNING";
|
|
1788
|
+
const survivingIds = pruneByThresholdWithCap(ctx.graph, ctx.frames, resolvedOptions.threshold, resolvedOptions.count);
|
|
1789
|
+
const prunedFrames = filter(ctx.frames, (f) => survivingIds.has(f.id));
|
|
1790
|
+
ctx.emitProgress(100);
|
|
1791
|
+
ctx.status = "FINALIZING";
|
|
1792
|
+
let outputFiles = [];
|
|
1793
|
+
let outputBuffers;
|
|
1794
|
+
if (resolvedOptions.mode === "buffer" || resolvedOptions.mode === "frames") {
|
|
1795
|
+
outputBuffers = await readFramesAsBuffers(prunedFrames, resolvedOptions.quality);
|
|
1796
|
+
ctx.emitProgress(100);
|
|
1797
|
+
} else {
|
|
1798
|
+
outputFiles = await finalizeOutput(ctx, prunedFrames);
|
|
1799
|
+
ctx.emitProgress(100);
|
|
1800
|
+
}
|
|
1801
|
+
ctx.status = "SUCCESS";
|
|
1802
|
+
logger.success(`Extracted ${prunedFrames.length} scenes from ${ctx.frames.length} frames`);
|
|
1803
|
+
const metadata = await buildVideoMetadata(ctx, prunedFrames, analysisResolution);
|
|
1804
|
+
return {
|
|
1805
|
+
success: true,
|
|
1806
|
+
originalFramesCount: ctx.frames.length,
|
|
1807
|
+
prunedFramesCount: prunedFrames.length,
|
|
1808
|
+
outputFiles,
|
|
1809
|
+
outputBuffers,
|
|
1810
|
+
...metadata,
|
|
1811
|
+
executionTimeMs: Date.now() - startTime
|
|
1812
|
+
};
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
ctx.status = "FAILED";
|
|
1815
|
+
ctx.error = error instanceof Error ? error : new Error(String(error));
|
|
1816
|
+
logger.error(`Pipeline failed: ${ctx.error.message}`);
|
|
1817
|
+
throw ctx.error;
|
|
1818
|
+
} finally {
|
|
1819
|
+
if (!resolvedOptions.debug) await cleanupWorkspace(ctx.workspacePath);
|
|
1820
|
+
else logger.debug(`Debug mode: workspace preserved at ${ctx.workspacePath}`);
|
|
1821
|
+
}
|
|
1565
1822
|
}
|
|
1566
1823
|
|
|
1567
|
-
|
|
1568
|
-
|
|
1824
|
+
//#endregion
|
|
1825
|
+
//#region src/core/orchestrator/worker/pipeline-worker.ts
|
|
1569
1826
|
runPipeline({
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1827
|
+
...workerData,
|
|
1828
|
+
onProgress: (phase, percent) => {
|
|
1829
|
+
parentPort?.postMessage({
|
|
1830
|
+
type: "progress",
|
|
1831
|
+
phase,
|
|
1832
|
+
percent
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1578
1835
|
}).then((result) => {
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1836
|
+
parentPort?.postMessage({
|
|
1837
|
+
type: "result",
|
|
1838
|
+
result
|
|
1839
|
+
});
|
|
1583
1840
|
}).catch((error) => {
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1841
|
+
parentPort?.postMessage({
|
|
1842
|
+
type: "error",
|
|
1843
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1844
|
+
});
|
|
1588
1845
|
});
|
|
1846
|
+
|
|
1847
|
+
//#endregion
|
|
1848
|
+
export { };
|