@lumy-pack/scene-sieve 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +1 -0
- package/dist/cli.mjs +73 -31
- package/dist/commands/Sieve.d.ts +19 -0
- package/dist/components/PhaseStep.d.ts +14 -0
- package/dist/components/ProgressBar.d.ts +7 -0
- package/dist/constants.d.ts +32 -0
- package/dist/core/analyzer.d.ts +62 -0
- package/dist/core/dbscan.d.ts +10 -0
- package/dist/core/extractor.d.ts +30 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/input-resolver.d.ts +13 -0
- package/dist/core/orchestrator.d.ts +2 -0
- package/dist/core/pipeline-worker.d.ts +1 -0
- package/dist/core/pruner.d.ts +61 -0
- package/dist/core/run-in-worker.d.ts +9 -0
- package/dist/core/segmenter.d.ts +38 -0
- package/dist/core/workspace.d.ts +25 -0
- package/dist/errors.d.ts +10 -0
- package/dist/index.cjs +65 -29
- package/dist/index.d.ts +2 -0
- package/dist/index.mjs +65 -29
- package/dist/pipeline-worker.mjs +65 -29
- package/dist/types/index.d.ts +129 -0
- package/dist/utils/command-registry.d.ts +24 -0
- package/dist/utils/concurrency.d.ts +5 -0
- package/dist/utils/logger.d.ts +8 -0
- package/dist/utils/math.d.ts +27 -0
- package/dist/utils/min-heap.d.ts +16 -0
- package/dist/utils/parse-options.d.ts +21 -0
- package/dist/utils/paths.d.ts +18 -0
- package/package.json +4 -2
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/cli.mjs
CHANGED
|
@@ -173,6 +173,7 @@ var init_dbscan = __esm({
|
|
|
173
173
|
|
|
174
174
|
// src/core/analyzer.ts
|
|
175
175
|
import { createRequire } from "module";
|
|
176
|
+
import { filter, map } from "@winglet/common-utils";
|
|
176
177
|
import sharp from "sharp";
|
|
177
178
|
async function ensureOpenCV() {
|
|
178
179
|
if (!cvReady) {
|
|
@@ -356,7 +357,7 @@ function computeInformationGain(clusters, clusterPoints, imageArea, animationInd
|
|
|
356
357
|
async function analyzeBatch(cvLib, frames, scale, tracker, pairOffset) {
|
|
357
358
|
const edges = [];
|
|
358
359
|
const preprocessed = await Promise.all(
|
|
359
|
-
|
|
360
|
+
map(frames, (f) => preprocessFrame(f.extractPath, scale))
|
|
360
361
|
);
|
|
361
362
|
const imageWidth = preprocessed[0]?.width ?? scale;
|
|
362
363
|
const imageHeight = preprocessed[0]?.height ?? Math.round(scale * 9 / 16);
|
|
@@ -398,7 +399,8 @@ async function analyzeBatch(cvLib, frames, scale, tracker, pairOffset) {
|
|
|
398
399
|
}
|
|
399
400
|
}
|
|
400
401
|
const animationIndices = tracker.update(clusters, pairIndex);
|
|
401
|
-
const animationWeights =
|
|
402
|
+
const animationWeights = map(
|
|
403
|
+
clusters,
|
|
402
404
|
(_, ci) => animationIndices.has(ci) ? tracker.getAnimationWeight(ci, clusters) : 0
|
|
403
405
|
);
|
|
404
406
|
const score = computeInformationGain(
|
|
@@ -524,7 +526,8 @@ var init_analyzer = __esm({
|
|
|
524
526
|
this.collectAnimation(region);
|
|
525
527
|
}
|
|
526
528
|
}
|
|
527
|
-
this.regions =
|
|
529
|
+
this.regions = filter(
|
|
530
|
+
this.regions,
|
|
528
531
|
(r, i) => r.weight > 0.01 || matched.has(i)
|
|
529
532
|
);
|
|
530
533
|
return animationIndices;
|
|
@@ -607,6 +610,7 @@ var init_paths = __esm({
|
|
|
607
610
|
import { readdir } from "fs/promises";
|
|
608
611
|
import { join as join2 } from "path";
|
|
609
612
|
import { path as ffprobePath } from "@ffprobe-installer/ffprobe";
|
|
613
|
+
import { filter as filter2, map as map2 } from "@winglet/common-utils";
|
|
610
614
|
import { execa } from "execa";
|
|
611
615
|
import ffmpegPath from "ffmpeg-static";
|
|
612
616
|
async function extractFrames(ctx) {
|
|
@@ -685,11 +689,11 @@ async function getVideoMetadata(inputPath) {
|
|
|
685
689
|
}
|
|
686
690
|
async function buildFrameList(framesDir, duration) {
|
|
687
691
|
const files = await readdir(framesDir);
|
|
688
|
-
const jpgFiles = files
|
|
692
|
+
const jpgFiles = filter2(files, (f) => f.endsWith(".jpg")).sort();
|
|
689
693
|
if (jpgFiles.length === 0) {
|
|
690
694
|
return [];
|
|
691
695
|
}
|
|
692
|
-
return jpgFiles
|
|
696
|
+
return map2(jpgFiles, (file, index) => ({
|
|
693
697
|
id: index,
|
|
694
698
|
timestamp: duration > 0 && jpgFiles.length > 1 ? duration * index / (jpgFiles.length - 1) : index,
|
|
695
699
|
extractPath: join2(framesDir, file)
|
|
@@ -724,6 +728,7 @@ var init_extractor = __esm({
|
|
|
724
728
|
// src/core/workspace.ts
|
|
725
729
|
import { readdir as readdir2, rename, rm, stat as stat2, writeFile } from "fs/promises";
|
|
726
730
|
import { join as join3 } from "path";
|
|
731
|
+
import { map as map3 } from "@winglet/common-utils";
|
|
727
732
|
import sharp2 from "sharp";
|
|
728
733
|
async function createWorkspace(sessionId) {
|
|
729
734
|
const workspacePath = getTempWorkspaceDir(sessionId);
|
|
@@ -764,7 +769,7 @@ async function finalizeOutput(ctx, selectedFrames) {
|
|
|
764
769
|
}
|
|
765
770
|
},
|
|
766
771
|
frames: framesMetadata,
|
|
767
|
-
animations: (ctx.animations || []
|
|
772
|
+
animations: map3(ctx.animations || [], (anim) => ({
|
|
768
773
|
...anim,
|
|
769
774
|
startFrameId: anim.startFrameId + 1,
|
|
770
775
|
endFrameId: anim.endFrameId + 1,
|
|
@@ -831,7 +836,8 @@ async function writeInputFrames(frames, workspacePath) {
|
|
|
831
836
|
}
|
|
832
837
|
async function readFramesAsBuffers(frameNodes, quality) {
|
|
833
838
|
return Promise.all(
|
|
834
|
-
|
|
839
|
+
map3(
|
|
840
|
+
frameNodes,
|
|
835
841
|
(f) => sharp2(f.extractPath).jpeg({ quality, mozjpeg: true }).toBuffer()
|
|
836
842
|
)
|
|
837
843
|
);
|
|
@@ -907,38 +913,42 @@ var init_input_resolver = __esm({
|
|
|
907
913
|
});
|
|
908
914
|
|
|
909
915
|
// src/utils/math.ts
|
|
916
|
+
import { filter as filter3, map as map4 } from "@winglet/common-utils";
|
|
910
917
|
function normalizeScores(items) {
|
|
911
918
|
if (items.length === 0) return [];
|
|
912
|
-
const safeScores =
|
|
919
|
+
const safeScores = map4(
|
|
920
|
+
items,
|
|
913
921
|
(e) => Number.isFinite(e.score) && e.score > 0 ? e.score : 0
|
|
914
922
|
);
|
|
915
|
-
const positiveScores = safeScores
|
|
923
|
+
const positiveScores = filter3(safeScores, (s) => s > 0);
|
|
916
924
|
if (positiveScores.length === 0) return safeScores;
|
|
917
925
|
const sorted = [...positiveScores].sort((a, b) => a - b);
|
|
918
926
|
if (positiveScores.length <= NORMALIZATION_MIN_SAMPLE_SIZE) {
|
|
919
927
|
const min = sorted[0];
|
|
920
928
|
const max = sorted[sorted.length - 1];
|
|
921
|
-
if (max === min) return safeScores
|
|
922
|
-
return
|
|
929
|
+
if (max === min) return map4(safeScores, (s) => s > 0 ? 1 : 0);
|
|
930
|
+
return map4(
|
|
931
|
+
safeScores,
|
|
923
932
|
(s) => s <= 0 ? 0 : Math.max(0, Math.min((s - min) / (max - min), 1))
|
|
924
933
|
);
|
|
925
934
|
}
|
|
926
935
|
const median = sorted[Math.floor(sorted.length / 2)];
|
|
927
|
-
const absoluteDiffs = positiveScores
|
|
936
|
+
const absoluteDiffs = map4(positiveScores, (v) => Math.abs(v - median));
|
|
928
937
|
const mad = [...absoluteDiffs].sort((a, b) => a - b)[Math.floor(absoluteDiffs.length / 2)];
|
|
929
938
|
const scale = mad === 0 ? median : mad * NORMALIZATION_MAD_COEFFICIENT;
|
|
930
|
-
const logisticZ = safeScores
|
|
939
|
+
const logisticZ = map4(safeScores, (s) => {
|
|
931
940
|
if (s <= 0) return 0;
|
|
932
941
|
if (scale === 0) return 1;
|
|
933
942
|
const z = (s - median) / scale;
|
|
934
943
|
return 1 / (1 + Math.exp(-NORMALIZATION_LOGISTIC_K * z));
|
|
935
944
|
});
|
|
936
|
-
const cdf = safeScores
|
|
945
|
+
const cdf = map4(safeScores, (s) => {
|
|
937
946
|
if (s <= 0) return 0;
|
|
938
947
|
const rank = sorted.findIndex((v) => v >= s);
|
|
939
948
|
return rank / sorted.length;
|
|
940
949
|
});
|
|
941
|
-
return
|
|
950
|
+
return map4(
|
|
951
|
+
logisticZ,
|
|
942
952
|
(z, i) => z * (1 - NORMALIZATION_ALPHA) + cdf[i] * NORMALIZATION_ALPHA
|
|
943
953
|
);
|
|
944
954
|
}
|
|
@@ -1000,9 +1010,10 @@ var init_min_heap = __esm({
|
|
|
1000
1010
|
});
|
|
1001
1011
|
|
|
1002
1012
|
// src/core/pruner.ts
|
|
1013
|
+
import { filter as filter4, map as map5 } from "@winglet/common-utils";
|
|
1003
1014
|
function pruneTo(graph, frames, targetCount) {
|
|
1004
1015
|
if (frames.length <= targetCount) {
|
|
1005
|
-
return new Set(frames
|
|
1016
|
+
return new Set(map5(frames, (f) => f.id));
|
|
1006
1017
|
}
|
|
1007
1018
|
const prev = /* @__PURE__ */ new Map();
|
|
1008
1019
|
const next = /* @__PURE__ */ new Map();
|
|
@@ -1020,7 +1031,7 @@ function pruneTo(graph, frames, targetCount) {
|
|
|
1020
1031
|
tgtId: edge.targetId
|
|
1021
1032
|
});
|
|
1022
1033
|
}
|
|
1023
|
-
const surviving = new Set(frames
|
|
1034
|
+
const surviving = new Set(map5(frames, (f) => f.id));
|
|
1024
1035
|
const firstId = frames[0].id;
|
|
1025
1036
|
const lastId = frames[frames.length - 1].id;
|
|
1026
1037
|
while (surviving.size > targetCount && heap.size > 0) {
|
|
@@ -1113,7 +1124,7 @@ function pruneByThresholdWithCap(graph, frames, threshold, maxCount) {
|
|
|
1113
1124
|
if (thresholdSurvivors.size <= maxCount) {
|
|
1114
1125
|
return thresholdSurvivors;
|
|
1115
1126
|
}
|
|
1116
|
-
const survivingFrames = frames
|
|
1127
|
+
const survivingFrames = filter4(frames, (f) => thresholdSurvivors.has(f.id));
|
|
1117
1128
|
const idToOrigIdx = /* @__PURE__ */ new Map();
|
|
1118
1129
|
for (let i = 0; i < frames.length; i++) {
|
|
1119
1130
|
idToOrigIdx.set(frames[i].id, i);
|
|
@@ -1180,6 +1191,7 @@ var init_concurrency = __esm({
|
|
|
1180
1191
|
// src/core/segmenter.ts
|
|
1181
1192
|
import { randomUUID } from "crypto";
|
|
1182
1193
|
import { join as join5 } from "path";
|
|
1194
|
+
import { filter as filter5, map as map6 } from "@winglet/common-utils";
|
|
1183
1195
|
function shouldSegment(resolvedOptions, originalOptions) {
|
|
1184
1196
|
if (resolvedOptions.mode === "frames") return false;
|
|
1185
1197
|
if (originalOptions.mode === "file") {
|
|
@@ -1297,18 +1309,30 @@ function remapEdges(segmentResults, globalIdMap) {
|
|
|
1297
1309
|
const edgeMap = /* @__PURE__ */ new Map();
|
|
1298
1310
|
for (const result of segmentResults) {
|
|
1299
1311
|
for (const edge of result.edges) {
|
|
1300
|
-
const newSourceId = globalIdMap.get(
|
|
1301
|
-
|
|
1312
|
+
const newSourceId = globalIdMap.get(
|
|
1313
|
+
`${result.segment.index}:${edge.sourceId}`
|
|
1314
|
+
);
|
|
1315
|
+
const newTargetId = globalIdMap.get(
|
|
1316
|
+
`${result.segment.index}:${edge.targetId}`
|
|
1317
|
+
);
|
|
1302
1318
|
if (newSourceId === void 0 || newTargetId === void 0) continue;
|
|
1303
1319
|
const edgeKey = `${newSourceId}-${newTargetId}`;
|
|
1304
1320
|
const existingIdx = edgeMap.get(edgeKey);
|
|
1305
1321
|
if (existingIdx !== void 0) {
|
|
1306
1322
|
if (edges[existingIdx].score < edge.score) {
|
|
1307
|
-
edges[existingIdx] = {
|
|
1323
|
+
edges[existingIdx] = {
|
|
1324
|
+
sourceId: newSourceId,
|
|
1325
|
+
targetId: newTargetId,
|
|
1326
|
+
score: edge.score
|
|
1327
|
+
};
|
|
1308
1328
|
}
|
|
1309
1329
|
} else {
|
|
1310
1330
|
edgeMap.set(edgeKey, edges.length);
|
|
1311
|
-
edges.push({
|
|
1331
|
+
edges.push({
|
|
1332
|
+
sourceId: newSourceId,
|
|
1333
|
+
targetId: newTargetId,
|
|
1334
|
+
score: edge.score
|
|
1335
|
+
});
|
|
1312
1336
|
}
|
|
1313
1337
|
}
|
|
1314
1338
|
}
|
|
@@ -1318,10 +1342,18 @@ function remapAnimations(segmentResults, globalIdMap) {
|
|
|
1318
1342
|
const animations = [];
|
|
1319
1343
|
for (const result of segmentResults) {
|
|
1320
1344
|
for (const anim of result.animations) {
|
|
1321
|
-
const newStartId = globalIdMap.get(
|
|
1322
|
-
|
|
1345
|
+
const newStartId = globalIdMap.get(
|
|
1346
|
+
`${result.segment.index}:${anim.startFrameId}`
|
|
1347
|
+
);
|
|
1348
|
+
const newEndId = globalIdMap.get(
|
|
1349
|
+
`${result.segment.index}:${anim.endFrameId}`
|
|
1350
|
+
);
|
|
1323
1351
|
if (newStartId === void 0 || newEndId === void 0) continue;
|
|
1324
|
-
animations.push({
|
|
1352
|
+
animations.push({
|
|
1353
|
+
...anim,
|
|
1354
|
+
startFrameId: newStartId,
|
|
1355
|
+
endFrameId: newEndId
|
|
1356
|
+
});
|
|
1325
1357
|
}
|
|
1326
1358
|
}
|
|
1327
1359
|
return animations;
|
|
@@ -1402,7 +1434,7 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1402
1434
|
logger.debug(`Segment plan: ${segments.length} segments`);
|
|
1403
1435
|
const limit = concurrencyLimit(resolvedOptions.concurrency);
|
|
1404
1436
|
const segmentProgresses = new Array(segments.length).fill(0);
|
|
1405
|
-
const weights = segments
|
|
1437
|
+
const weights = map6(segments, (s) => s.duration / totalDuration);
|
|
1406
1438
|
const emitOverallProgress = (phase) => {
|
|
1407
1439
|
if (!options.onProgress) return;
|
|
1408
1440
|
const overall = weights.reduce(
|
|
@@ -1413,7 +1445,8 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1413
1445
|
};
|
|
1414
1446
|
options.onProgress?.("EXTRACTING", 0);
|
|
1415
1447
|
const results = await Promise.all(
|
|
1416
|
-
|
|
1448
|
+
map6(
|
|
1449
|
+
segments,
|
|
1417
1450
|
(segment) => limit(async () => {
|
|
1418
1451
|
const segWorkspace = await createSegmentWorkspace(
|
|
1419
1452
|
mainWorkspace,
|
|
@@ -1445,7 +1478,7 @@ async function runSegmentedPipeline(options, resolvedOptions) {
|
|
|
1445
1478
|
resolvedOptions.threshold,
|
|
1446
1479
|
resolvedOptions.count
|
|
1447
1480
|
);
|
|
1448
|
-
const prunedFrames = frames
|
|
1481
|
+
const prunedFrames = filter5(frames, (f) => survivingIds.has(f.id));
|
|
1449
1482
|
options.onProgress?.("PRUNING", 100);
|
|
1450
1483
|
options.onProgress?.("FINALIZING", 0);
|
|
1451
1484
|
const ctx = {
|
|
@@ -1519,6 +1552,7 @@ __export(orchestrator_exports, {
|
|
|
1519
1552
|
runPipeline: () => runPipeline
|
|
1520
1553
|
});
|
|
1521
1554
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
1555
|
+
import { filter as filter6 } from "@winglet/common-utils";
|
|
1522
1556
|
async function runPipeline(options) {
|
|
1523
1557
|
const debug = options.debug ?? false;
|
|
1524
1558
|
if (debug) setDebugMode(true);
|
|
@@ -1572,7 +1606,7 @@ async function runPipeline(options) {
|
|
|
1572
1606
|
resolvedOptions.threshold,
|
|
1573
1607
|
resolvedOptions.count
|
|
1574
1608
|
);
|
|
1575
|
-
const prunedFrames = ctx.frames
|
|
1609
|
+
const prunedFrames = filter6(ctx.frames, (f) => survivingIds.has(f.id));
|
|
1576
1610
|
ctx.emitProgress(100);
|
|
1577
1611
|
ctx.status = "FINALIZING";
|
|
1578
1612
|
let outputFiles = [];
|
|
@@ -1636,7 +1670,6 @@ var init_orchestrator = __esm({
|
|
|
1636
1670
|
|
|
1637
1671
|
// src/cli.ts
|
|
1638
1672
|
import { createRequire as createRequire2 } from "module";
|
|
1639
|
-
import { Command } from "commander";
|
|
1640
1673
|
|
|
1641
1674
|
// ../shared/src/respond.ts
|
|
1642
1675
|
function respond(command, data, startTime, version2) {
|
|
@@ -1667,6 +1700,9 @@ function respondError(command, code, message, startTime, version2, details) {
|
|
|
1667
1700
|
process.exitCode = 1;
|
|
1668
1701
|
}
|
|
1669
1702
|
|
|
1703
|
+
// src/cli.ts
|
|
1704
|
+
import { Command } from "commander";
|
|
1705
|
+
|
|
1670
1706
|
// src/commands/Sieve.tsx
|
|
1671
1707
|
import { existsSync } from "fs";
|
|
1672
1708
|
import { Box as Box2, Text as Text3, useApp } from "ink";
|
|
@@ -2182,7 +2218,13 @@ if (process.argv.includes("--describe")) {
|
|
|
2182
2218
|
}
|
|
2183
2219
|
program.parseAsync(process.argv).catch((error) => {
|
|
2184
2220
|
if (process.argv.includes("--json")) {
|
|
2185
|
-
respondError(
|
|
2221
|
+
respondError(
|
|
2222
|
+
"extract",
|
|
2223
|
+
SieveErrorCode.UNKNOWN,
|
|
2224
|
+
error.message,
|
|
2225
|
+
Date.now(),
|
|
2226
|
+
version
|
|
2227
|
+
);
|
|
2186
2228
|
} else {
|
|
2187
2229
|
console.error("Fatal error:", error.message);
|
|
2188
2230
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Command } from 'commander';
|
|
2
|
+
import React from 'react';
|
|
3
|
+
export interface SieveViewProps {
|
|
4
|
+
input: string;
|
|
5
|
+
count?: number;
|
|
6
|
+
threshold?: number;
|
|
7
|
+
output?: string;
|
|
8
|
+
fps: number;
|
|
9
|
+
maxFrames: number;
|
|
10
|
+
scale: number;
|
|
11
|
+
quality: number;
|
|
12
|
+
iouThreshold?: number;
|
|
13
|
+
animationThreshold?: number;
|
|
14
|
+
maxSegmentDuration?: number;
|
|
15
|
+
concurrency?: number;
|
|
16
|
+
debug: boolean;
|
|
17
|
+
}
|
|
18
|
+
export declare function registerSieveCommand(program: Command, version: string): void;
|
|
19
|
+
export declare const SieveView: React.FC<SieveViewProps>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
export type PhaseStatus = 'pending' | 'running' | 'done' | 'failed';
|
|
3
|
+
export interface PhaseState {
|
|
4
|
+
label: string;
|
|
5
|
+
status: PhaseStatus;
|
|
6
|
+
hasProgress: boolean;
|
|
7
|
+
percent: number;
|
|
8
|
+
durationMs?: number;
|
|
9
|
+
}
|
|
10
|
+
interface PhaseStepProps {
|
|
11
|
+
phase: PhaseState;
|
|
12
|
+
}
|
|
13
|
+
export declare const PhaseStep: React.FC<PhaseStepProps>;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export declare const APP_NAME = "scene-sieve";
|
|
2
|
+
export declare const DEFAULT_COUNT = 20;
|
|
3
|
+
export declare const DEFAULT_THRESHOLD = 0.5;
|
|
4
|
+
export declare const DEFAULT_FPS = 5;
|
|
5
|
+
export declare const DEFAULT_SCALE = 720;
|
|
6
|
+
export declare const DEFAULT_QUALITY = 80;
|
|
7
|
+
export declare const DEFAULT_MAX_FRAMES = 300;
|
|
8
|
+
export declare const NORMALIZATION_MIN_PERCENTILE = 0.1;
|
|
9
|
+
export declare const NORMALIZATION_MAX_PERCENTILE = 0.9;
|
|
10
|
+
export declare const NORMALIZATION_LOGISTIC_K = 3;
|
|
11
|
+
export declare const NORMALIZATION_ALPHA = 0.4;
|
|
12
|
+
export declare const NORMALIZATION_MAD_COEFFICIENT = 1.4826;
|
|
13
|
+
export declare const NORMALIZATION_MIN_SAMPLE_SIZE = 10;
|
|
14
|
+
export declare const WORKSPACE_PREFIX = "scene-sieve-";
|
|
15
|
+
export declare const TEMP_BASE_DIR: string;
|
|
16
|
+
export declare const FRAME_OUTPUT_EXTENSION = ".jpg";
|
|
17
|
+
export declare const FRAME_FILENAME_PATTERN = "frame_%06d.jpg";
|
|
18
|
+
export declare const OPENCV_BATCH_SIZE = 10;
|
|
19
|
+
export declare const MIN_IFRAME_COUNT = 3;
|
|
20
|
+
export declare const DBSCAN_ALPHA = 0.03;
|
|
21
|
+
export declare const DBSCAN_MIN_PTS = 4;
|
|
22
|
+
export declare const IOU_THRESHOLD = 0.9;
|
|
23
|
+
export declare const DECAY_LAMBDA = 0.95;
|
|
24
|
+
export declare const ANIMATION_FRAME_THRESHOLD = 5;
|
|
25
|
+
export declare const MATCH_DISTANCE_THRESHOLD = 0.25;
|
|
26
|
+
export declare const PIXELDIFF_GAUSSIAN_KERNEL = 3;
|
|
27
|
+
export declare const PIXELDIFF_BINARY_THRESHOLD = 30;
|
|
28
|
+
export declare const PIXELDIFF_CONTOUR_MIN_AREA = 100;
|
|
29
|
+
export declare const PIXELDIFF_SAMPLE_SPACING = 8;
|
|
30
|
+
export declare const DEFAULT_MAX_SEGMENT_DURATION = 300;
|
|
31
|
+
export declare const DEFAULT_SEGMENT_CONCURRENCY = 2;
|
|
32
|
+
export declare function getTempWorkspaceDir(sessionId: string): string;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { AnalysisResult, AnimationMetadata, BoundingBox, ProcessContext } from '../types/index.js';
|
|
2
|
+
import type { Point2D } from './dbscan.js';
|
|
3
|
+
type CvLib = typeof import('@techstark/opencv-js');
|
|
4
|
+
export declare function preprocessFrame(framePath: string, scale: number): Promise<{
|
|
5
|
+
data: Uint8Array;
|
|
6
|
+
width: number;
|
|
7
|
+
height: number;
|
|
8
|
+
}>;
|
|
9
|
+
export declare function computeIoU(a: BoundingBox, b: BoundingBox): number;
|
|
10
|
+
export declare class IoUTracker {
|
|
11
|
+
private fps;
|
|
12
|
+
private iouThreshold;
|
|
13
|
+
private animationThreshold;
|
|
14
|
+
private regions;
|
|
15
|
+
private extractedAnimations;
|
|
16
|
+
constructor(fps?: number, iouThreshold?: number, animationThreshold?: number);
|
|
17
|
+
update(boxes: BoundingBox[], pairIndex: number): Set<number>;
|
|
18
|
+
private collectAnimation;
|
|
19
|
+
flushAndGetAnimations(): AnimationMetadata[];
|
|
20
|
+
getAnimationWeight(boxIndex: number, boxes: BoundingBox[]): number;
|
|
21
|
+
}
|
|
22
|
+
export interface AKAZEResult {
|
|
23
|
+
sNew: Point2D[];
|
|
24
|
+
sLoss: Point2D[];
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Pixel-level difference fallback for AKAZE blind spots.
|
|
28
|
+
*
|
|
29
|
+
* When AKAZE produces sparse results (typical for UI screen recordings
|
|
30
|
+
* where form fields, dropdowns, or overlays change), this function
|
|
31
|
+
* detects changed regions via cv.absdiff and generates synthetic
|
|
32
|
+
* Point2D[] that feed into the existing DBSCAN → IoU → G(t) pipeline.
|
|
33
|
+
*
|
|
34
|
+
* Algorithm:
|
|
35
|
+
* 1. absdiff(frame1, frame2) → grayscale difference
|
|
36
|
+
* 2. GaussianBlur → reduce JPEG compression noise
|
|
37
|
+
* 3. threshold → binary mask of significant changes
|
|
38
|
+
* 4. findContours → bounding rects of changed regions
|
|
39
|
+
* 5. Grid sampling within each bounding rect → Point2D[]
|
|
40
|
+
*/
|
|
41
|
+
export declare function computePixelDiff(cvLib: CvLib, frame1: {
|
|
42
|
+
data: Uint8Array;
|
|
43
|
+
width: number;
|
|
44
|
+
height: number;
|
|
45
|
+
}, frame2: {
|
|
46
|
+
data: Uint8Array;
|
|
47
|
+
width: number;
|
|
48
|
+
height: number;
|
|
49
|
+
}): Point2D[];
|
|
50
|
+
export declare function computeInformationGain(clusters: BoundingBox[], clusterPoints: number[], imageArea: number, animationIndices: Set<number>, animationWeights: number[]): number;
|
|
51
|
+
/**
|
|
52
|
+
* Analyze adjacent frame pairs to compute information gain scores (G(t)).
|
|
53
|
+
* Processes frames in batches for memory efficiency.
|
|
54
|
+
*
|
|
55
|
+
* Pipeline:
|
|
56
|
+
* 1. AKAZE Feature Set Difference
|
|
57
|
+
* 2. DBSCAN Spatial Clustering
|
|
58
|
+
* 3. Spatio-temporal IoU Tracking
|
|
59
|
+
* 4. G(t) Information Gain Scoring
|
|
60
|
+
*/
|
|
61
|
+
export declare function analyzeFrames(ctx: ProcessContext): Promise<AnalysisResult>;
|
|
62
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { DBSCANResult } from '../types/index.js';
|
|
2
|
+
export interface Point2D {
|
|
3
|
+
x: number;
|
|
4
|
+
y: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* DBSCAN clustering with resolution-independent eps.
|
|
8
|
+
* eps = alpha * sqrt(width^2 + height^2)
|
|
9
|
+
*/
|
|
10
|
+
export declare function dbscan(points: Point2D[], imageWidth: number, imageHeight: number, alpha?: number, minPts?: number): DBSCANResult;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { FrameNode, ProcessContext } from '../types/index.js';
|
|
2
|
+
export interface FFprobeMetadata {
|
|
3
|
+
format?: {
|
|
4
|
+
format_name?: string;
|
|
5
|
+
duration?: string;
|
|
6
|
+
};
|
|
7
|
+
streams?: Array<{
|
|
8
|
+
codec_type?: string;
|
|
9
|
+
}>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Extract frames from video/GIF using FFmpeg.
|
|
13
|
+
* Always uses FPS-based extraction. For long videos, FPS is automatically
|
|
14
|
+
* reduced to stay within maxFrames budget.
|
|
15
|
+
*/
|
|
16
|
+
export declare function extractFrames(ctx: ProcessContext): Promise<FrameNode[]>;
|
|
17
|
+
export declare function getVideoMetadata(inputPath: string): Promise<FFprobeMetadata>;
|
|
18
|
+
/**
|
|
19
|
+
* Extract frames from a specific time range of a video using FFmpeg.
|
|
20
|
+
* Uses input seeking (-ss before -i) for fast seek + -t for duration.
|
|
21
|
+
*
|
|
22
|
+
* @param inputPath - Path to the video file
|
|
23
|
+
* @param outputDir - Directory to write extracted frames
|
|
24
|
+
* @param fps - Frames per second for extraction
|
|
25
|
+
* @param scale - Height scale for vision analysis
|
|
26
|
+
* @param startTime - Start time in seconds
|
|
27
|
+
* @param duration - Duration in seconds to extract
|
|
28
|
+
* @returns Array of FrameNode with segment-local timestamps (starting from 0)
|
|
29
|
+
*/
|
|
30
|
+
export declare function extractFramesForRange(inputPath: string, outputDir: string, fps: number, scale: number, startTime: number, duration: number): Promise<FrameNode[]>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { runPipeline } from './orchestrator.js';
|
|
2
|
+
export { analyzeFrames, computeIoU, computeInformationGain, } from './analyzer.js';
|
|
3
|
+
export { extractFrames } from './extractor.js';
|
|
4
|
+
export { pruneTo, pruneByThreshold, pruneByThresholdWithCap, suppressConsecutiveRuns, } from './pruner.js';
|
|
5
|
+
export { dbscan } from './dbscan.js';
|
|
6
|
+
export type { Point2D } from './dbscan.js';
|
|
7
|
+
export { resolveInput, resolveOptions } from './input-resolver.js';
|
|
8
|
+
export { shouldSegment, computeSegmentPlan, processSegment, mergeSegmentFrames, runSegmentedPipeline, } from './segmenter.js';
|
|
9
|
+
export { createWorkspace, createSegmentWorkspace, cleanupWorkspace, finalizeOutput, readFramesAsBuffers, writeInputBuffer, writeInputFrames, } from './workspace.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { FrameNode, ResolvedOptions, SieveOptions } from '../types/index.js';
|
|
2
|
+
export declare function resolveOptions(options: SieveOptions): ResolvedOptions;
|
|
3
|
+
/**
|
|
4
|
+
* Resolve the input source to a list of FrameNode[].
|
|
5
|
+
*
|
|
6
|
+
* - 'file' mode: validate file exists and delegate to extractor (caller's responsibility)
|
|
7
|
+
* - 'buffer' mode: write buffer as temp video file, return path via FrameNode trick (empty list)
|
|
8
|
+
* - 'frames' mode: write frame buffers as JPGs, return FrameNode[]
|
|
9
|
+
*/
|
|
10
|
+
export declare function resolveInput(options: SieveOptions, workspacePath: string): Promise<{
|
|
11
|
+
frames: FrameNode[];
|
|
12
|
+
resolvedInputPath?: string;
|
|
13
|
+
}>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { FrameNode, ScoreEdge } from '../types/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Edge-aware greedy merge with re-linking — O(N log N).
|
|
4
|
+
*
|
|
5
|
+
* 1. Build a doubly-linked list of frames
|
|
6
|
+
* 2. Insert all edges into a min-heap
|
|
7
|
+
* 3. Pop the lowest-score edge (most similar pair)
|
|
8
|
+
* 4. Remove the later frame (tgtId), re-link neighbors
|
|
9
|
+
* 5. Push synthetic edge with score = max(left, right)
|
|
10
|
+
* 6. Repeat until surviving count === targetCount
|
|
11
|
+
* 7. First and last frames are never removed (boundary preservation)
|
|
12
|
+
*
|
|
13
|
+
* Stale heap entries (involving removed frames) are lazily skipped on pop.
|
|
14
|
+
*/
|
|
15
|
+
export declare function pruneTo(graph: ScoreEdge[], frames: FrameNode[], targetCount: number): Set<number>;
|
|
16
|
+
/**
|
|
17
|
+
* Non-Maximum Suppression (NMS) for consecutive edge runs.
|
|
18
|
+
*
|
|
19
|
+
* Consecutive edges share overlapping frames (edge i: frame i->i+1,
|
|
20
|
+
* edge i+1: frame i+1->i+2), so consecutive passing edges indicate
|
|
21
|
+
* the same visual transition region. This function groups consecutive
|
|
22
|
+
* passing edge indices into "runs" and keeps all distinct peaks per run.
|
|
23
|
+
*
|
|
24
|
+
* Multi-peak detection: within each run, strict local maxima (score higher
|
|
25
|
+
* than both neighbors) are identified. Each local maximum represents a
|
|
26
|
+
* distinct visual transition. If no strict local maxima exist (plateau or
|
|
27
|
+
* monotonic sequence), the global peak of the run is selected as fallback.
|
|
28
|
+
*
|
|
29
|
+
* Single-element runs are unaffected (isolated transitions preserved).
|
|
30
|
+
*
|
|
31
|
+
* @param graph - full ScoreEdge array (for targetId lookup)
|
|
32
|
+
* @param passingIndices - edge indices that passed threshold filtering (sorted ascending)
|
|
33
|
+
* @param normalizedScores - normalized score array (same length as graph)
|
|
34
|
+
* @returns Set of targetIds to add to surviving set (one or more per run)
|
|
35
|
+
*/
|
|
36
|
+
export declare function suppressConsecutiveRuns(graph: ScoreEdge[], passingIndices: number[], normalizedScores: number[]): Set<number>;
|
|
37
|
+
/**
|
|
38
|
+
* Threshold-based pruning with NMS -- O(N).
|
|
39
|
+
*
|
|
40
|
+
* 1. Scores are normalized to [0, 1] via percentile normalization.
|
|
41
|
+
* 2. Edges with normalized score >= threshold are collected.
|
|
42
|
+
* 3. Non-Maximum Suppression groups consecutive passing edges and keeps
|
|
43
|
+
* only the peak per run, preventing near-duplicate frame selection
|
|
44
|
+
* from a single visual transition.
|
|
45
|
+
*
|
|
46
|
+
* First and last frames are always preserved (boundary protection).
|
|
47
|
+
*/
|
|
48
|
+
export declare function pruneByThreshold(graph: ScoreEdge[], frames: FrameNode[], threshold: number): Set<number>;
|
|
49
|
+
/**
|
|
50
|
+
* Combined threshold + count pruning -- 2-stage pipeline.
|
|
51
|
+
*
|
|
52
|
+
* Stage 1: pruneByThreshold -- keep all frames with normalized score >= threshold
|
|
53
|
+
* Stage 2: if result exceeds maxCount, rebuild subgraph with synthetic edges
|
|
54
|
+
* (min-score over each gap) and apply pruneTo on the surviving subset
|
|
55
|
+
*
|
|
56
|
+
* Edge reconstruction: for consecutive survivors A, B with removed frames
|
|
57
|
+
* [x1, x2, ...] between them, the synthetic edge score is:
|
|
58
|
+
* min(score(A->x1), score(x1->x2), ..., score(xN->B))
|
|
59
|
+
* This preserves the "weakest link" semantics.
|
|
60
|
+
*/
|
|
61
|
+
export declare function pruneByThresholdWithCap(graph: ScoreEdge[], frames: FrameNode[], threshold: number, maxCount: number): Set<number>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ProgressPhase, SieveInput, SieveOptionsBase, SieveResult } from '../types/index.js';
|
|
2
|
+
export type SieveWorkerOptions = Omit<SieveOptionsBase, 'onProgress'> & SieveInput;
|
|
3
|
+
/**
|
|
4
|
+
* Run the pipeline, choosing the best execution strategy:
|
|
5
|
+
*
|
|
6
|
+
* - Production (bundled .mjs): Worker thread — spinner never freezes
|
|
7
|
+
* - Dev mode (tsx .ts): Main thread — simpler, spinner may stutter during CPU work
|
|
8
|
+
*/
|
|
9
|
+
export declare function runPipelineInWorker(options: SieveWorkerOptions, onProgress: (phase: ProgressPhase, percent: number) => void): Promise<SieveResult>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { AnimationMetadata, FrameNode, ResolvedOptions, ScoreEdge, SegmentPlan, SegmentResult, SieveOptions, SieveResult } from '../types/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Determine whether segmentation should be used.
|
|
4
|
+
* Returns false for frames mode and GIF files.
|
|
5
|
+
* Actual duration check happens inside runSegmentedPipeline after metadata fetch.
|
|
6
|
+
*/
|
|
7
|
+
export declare function shouldSegment(resolvedOptions: ResolvedOptions, originalOptions: SieveOptions): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Compute segment boundaries with overlap, frame allocation, and effectiveFps.
|
|
10
|
+
* Pure function — no I/O.
|
|
11
|
+
*
|
|
12
|
+
* - effectiveFps is uniform across all segments
|
|
13
|
+
* - Overlap: 1 frame at each internal boundary
|
|
14
|
+
* - allocatedFrames total <= maxFrames (last segment adjusted if needed)
|
|
15
|
+
*/
|
|
16
|
+
export declare function computeSegmentPlan(totalDuration: number, maxSegmentDuration: number, maxFrames: number, fps: number): SegmentPlan[];
|
|
17
|
+
/**
|
|
18
|
+
* Merge multiple segment results into a single unified frame/edge/animation set.
|
|
19
|
+
* - Timestamps adjusted using extractStartTime (Section 18 note 1)
|
|
20
|
+
* - Overlap frames deduplicated by threshold 1/(effectiveFps*2) (Section 18 note 5)
|
|
21
|
+
* - Global IDs reassigned after dedup
|
|
22
|
+
* - Duplicate edges keep higher score
|
|
23
|
+
*/
|
|
24
|
+
export declare function mergeSegmentFrames(segmentResults: SegmentResult[]): {
|
|
25
|
+
frames: FrameNode[];
|
|
26
|
+
edges: ScoreEdge[];
|
|
27
|
+
animations: AnimationMetadata[];
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Extract frames for a single segment and analyze them.
|
|
31
|
+
* Each segment uses an isolated workspace directory.
|
|
32
|
+
*/
|
|
33
|
+
export declare function processSegment(inputPath: string, segment: SegmentPlan, workspacePath: string, resolvedOptions: ResolvedOptions, onProgress: (percent: number) => void): Promise<SegmentResult>;
|
|
34
|
+
/**
|
|
35
|
+
* Full segmented pipeline: metadata → plan → parallel extract+analyze → merge → prune → finalize.
|
|
36
|
+
* Called from runPipeline when shouldSegment() returns true.
|
|
37
|
+
*/
|
|
38
|
+
export declare function runSegmentedPipeline(options: SieveOptions, resolvedOptions: ResolvedOptions): Promise<SieveResult>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { FrameNode, ProcessContext } from '../types/index.js';
|
|
2
|
+
export declare function createWorkspace(sessionId: string): Promise<string>;
|
|
3
|
+
export declare function finalizeOutput(ctx: ProcessContext, selectedFrames: FrameNode[]): Promise<string[]>;
|
|
4
|
+
export declare function createSegmentWorkspace(parentWorkspacePath: string, segmentIndex: number): Promise<string>;
|
|
5
|
+
export declare function cleanupWorkspace(workspacePath: string): Promise<void>;
|
|
6
|
+
/**
|
|
7
|
+
* Remove stale workspace directories left by previous interrupted runs.
|
|
8
|
+
* Only deletes directories older than 1 hour to avoid removing active workspaces.
|
|
9
|
+
*/
|
|
10
|
+
export declare function cleanupStaleWorkspaces(): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Write a video buffer to a temp file in the workspace and return the path.
|
|
13
|
+
* Used by 'buffer' input mode.
|
|
14
|
+
*/
|
|
15
|
+
export declare function writeInputBuffer(buffer: Buffer, workspacePath: string): Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Write an array of frame Buffers as JPG files and return FrameNode[].
|
|
18
|
+
* Used by 'frames' input mode.
|
|
19
|
+
*/
|
|
20
|
+
export declare function writeInputFrames(frames: Buffer[], workspacePath: string): Promise<FrameNode[]>;
|
|
21
|
+
/**
|
|
22
|
+
* Read selected FrameNode files as Buffers with JPEG compression.
|
|
23
|
+
* Used to return output buffers in 'buffer' and 'frames' modes.
|
|
24
|
+
*/
|
|
25
|
+
export declare function readFramesAsBuffers(frameNodes: FrameNode[], quality: number): Promise<Buffer[]>;
|