@mevdragon/vidfarm-devcli 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/demo/dist/app.js +81 -53
- package/dist/src/app.js +1010 -8
- package/dist/src/cli.js +390 -53
- package/dist/src/config.js +6 -0
- package/dist/src/devcli/clip-store.js +335 -0
- package/dist/src/devcli/clips.js +803 -0
- package/dist/src/devcli/telemetry.js +236 -0
- package/dist/src/frontend/homepage-view.js +5 -0
- package/dist/src/frontend/template-editor-chat.js +212 -0
- package/dist/src/services/clip-curation/cost.js +112 -0
- package/dist/src/services/clip-curation/ffmpeg.js +258 -0
- package/dist/src/services/clip-curation/gemini.js +342 -0
- package/dist/src/services/clip-curation/index.js +15 -0
- package/dist/src/services/clip-curation/presets.js +20 -0
- package/dist/src/services/clip-curation/presets.v1.json +59 -0
- package/dist/src/services/clip-curation/query.js +163 -0
- package/dist/src/services/clip-curation/refine.js +136 -0
- package/dist/src/services/clip-curation/scan.js +137 -0
- package/dist/src/services/clip-curation/taxonomy.js +71 -0
- package/dist/src/services/clip-curation/taxonomy.v1.json +90 -0
- package/dist/src/services/clip-curation/types.js +7 -0
- package/dist/src/services/clip-records.js +209 -0
- package/dist/src/services/clip-search.js +47 -0
- package/dist/src/services/clip-vectors.js +125 -0
- package/dist/src/services/hyperframes.js +369 -9
- package/dist/src/services/scene-annotations.js +32 -0
- package/package.json +5 -1
- package/public/assets/homepage-client-app.js +17 -17
- package/public/assets/page-runtime-client-app.js +31 -31
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1",
|
|
3
|
+
"presets": [
|
|
4
|
+
{
|
|
5
|
+
"preset_id": "preset_funny_reactions",
|
|
6
|
+
"name": "funny reaction gifs",
|
|
7
|
+
"description": "Short single-subject reactions facing the camera — the raw material for reaction cutaways.",
|
|
8
|
+
"criteria": {
|
|
9
|
+
"emotion": ["surprised", "amused", "deadpan", "exasperated"],
|
|
10
|
+
"subject": ["person", "single_speaker"],
|
|
11
|
+
"composition": ["close_up", "medium_shot", "centered"],
|
|
12
|
+
"max_duration_sec": 2.5,
|
|
13
|
+
"has_subject": true,
|
|
14
|
+
"semantic_text": "funny reaction, person reacting to the camera"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"preset_id": "preset_broll_transitions",
|
|
19
|
+
"name": "b-roll transitions",
|
|
20
|
+
"description": "Quiet low-motion cutaways with no talking — good glue between beats.",
|
|
21
|
+
"criteria": {
|
|
22
|
+
"motion": ["static", "low"],
|
|
23
|
+
"max_duration_sec": 1.5,
|
|
24
|
+
"has_dialogue": false,
|
|
25
|
+
"semantic_text": "short b-roll transition, no dialogue"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"preset_id": "preset_establishing_shots",
|
|
30
|
+
"name": "establishing shots",
|
|
31
|
+
"description": "Wide scene-setters with no clear subject.",
|
|
32
|
+
"criteria": {
|
|
33
|
+
"composition": ["wide_shot"],
|
|
34
|
+
"has_subject": false,
|
|
35
|
+
"semantic_text": "wide establishing shot of a location"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"preset_id": "preset_typing_hands",
|
|
40
|
+
"name": "typing hands",
|
|
41
|
+
"description": "Close-ups of hands on a keyboard — classic productivity b-roll.",
|
|
42
|
+
"criteria": {
|
|
43
|
+
"action": ["typing", "hands_on_keyboard"],
|
|
44
|
+
"composition": ["close_up"],
|
|
45
|
+
"semantic_text": "close up of hands typing on a keyboard"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"preset_id": "preset_money_shots",
|
|
50
|
+
"name": "money shots",
|
|
51
|
+
"description": "High-energy hero shots — the payoff frames.",
|
|
52
|
+
"criteria": {
|
|
53
|
+
"composition": ["close_up", "medium_shot"],
|
|
54
|
+
"energy": ["high"],
|
|
55
|
+
"semantic_text": "high energy hero product money shot"
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Structured filter + hybrid semantic re-rank. Same logic on devcli and cloud
|
|
2
|
+
// so a preset or NL query returns the same clips regardless of target.
|
|
3
|
+
//
|
|
4
|
+
// Search flow (handoff): structured filter FIRST (cheap, exact), then semantic
|
|
5
|
+
// re-rank the survivors by embedding similarity. If the structured filter
|
|
6
|
+
// matches nothing but the query carries semantic intent, we relax to a
|
|
7
|
+
// semantic-only pass so natural-language queries still return their best guess.
|
|
8
|
+
const DIALOGUE_AUDIO = new Set(["dialogue", "voiceover"]);
|
|
9
|
+
/** How many categorical facets `criteria` constrains (used for the match-ratio soft score). */
|
|
10
|
+
function constraintCount(criteria) {
|
|
11
|
+
let n = 0;
|
|
12
|
+
for (const key of ["subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"]) {
|
|
13
|
+
if (Array.isArray(criteria[key]) && criteria[key].length)
|
|
14
|
+
n++;
|
|
15
|
+
}
|
|
16
|
+
if (criteria.min_duration_sec != null || criteria.max_duration_sec != null)
|
|
17
|
+
n++;
|
|
18
|
+
if (criteria.has_dialogue != null)
|
|
19
|
+
n++;
|
|
20
|
+
if (criteria.has_subject != null)
|
|
21
|
+
n++;
|
|
22
|
+
return n;
|
|
23
|
+
}
|
|
24
|
+
function hasSubject(tags) {
|
|
25
|
+
return tags.subject.length > 0 && !(tags.subject.length === 1 && tags.subject[0] === "no_subject");
|
|
26
|
+
}
|
|
27
|
+
function hasDialogue(tags) {
|
|
28
|
+
return tags.audio_type.some((a) => DIALOGUE_AUDIO.has(a)) || Boolean(tags.transcript.trim());
|
|
29
|
+
}
|
|
30
|
+
/** Hard structured filter: does this clip satisfy every constraint in `criteria`? */
|
|
31
|
+
export function matchesCriteria(clip, criteria) {
|
|
32
|
+
const t = clip.tags;
|
|
33
|
+
const anyOf = (have, want) => !want || want.length === 0 || want.some((w) => have.includes(w));
|
|
34
|
+
if (!anyOf(t.subject, criteria.subject))
|
|
35
|
+
return false;
|
|
36
|
+
if (!anyOf(t.action, criteria.action))
|
|
37
|
+
return false;
|
|
38
|
+
if (!anyOf(t.emotion, criteria.emotion))
|
|
39
|
+
return false;
|
|
40
|
+
if (!anyOf(t.setting, criteria.setting))
|
|
41
|
+
return false;
|
|
42
|
+
if (!anyOf(t.composition, criteria.composition))
|
|
43
|
+
return false;
|
|
44
|
+
if (!anyOf(t.motion, criteria.motion))
|
|
45
|
+
return false;
|
|
46
|
+
if (!anyOf(t.audio_type, criteria.audio_type))
|
|
47
|
+
return false;
|
|
48
|
+
if (criteria.energy && criteria.energy.length && !criteria.energy.includes(t.energy))
|
|
49
|
+
return false;
|
|
50
|
+
if (criteria.min_duration_sec != null && clip.duration_sec < criteria.min_duration_sec)
|
|
51
|
+
return false;
|
|
52
|
+
if (criteria.max_duration_sec != null && clip.duration_sec > criteria.max_duration_sec)
|
|
53
|
+
return false;
|
|
54
|
+
if (criteria.has_dialogue === true && !hasDialogue(t))
|
|
55
|
+
return false;
|
|
56
|
+
if (criteria.has_dialogue === false && hasDialogue(t))
|
|
57
|
+
return false;
|
|
58
|
+
if (criteria.has_subject === true && !hasSubject(t))
|
|
59
|
+
return false;
|
|
60
|
+
if (criteria.has_subject === false && hasSubject(t))
|
|
61
|
+
return false;
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
/** Fraction of constrained facets this clip satisfies (0..1); 1 when there are no constraints. */
|
|
65
|
+
export function structuredMatchRatio(clip, criteria) {
|
|
66
|
+
const total = constraintCount(criteria);
|
|
67
|
+
if (total === 0)
|
|
68
|
+
return 1;
|
|
69
|
+
const t = clip.tags;
|
|
70
|
+
let hit = 0;
|
|
71
|
+
const anyOf = (have, want) => Boolean(want && want.length && want.some((w) => have.includes(w)));
|
|
72
|
+
if (anyOf(t.subject, criteria.subject))
|
|
73
|
+
hit++;
|
|
74
|
+
if (anyOf(t.action, criteria.action))
|
|
75
|
+
hit++;
|
|
76
|
+
if (anyOf(t.emotion, criteria.emotion))
|
|
77
|
+
hit++;
|
|
78
|
+
if (anyOf(t.setting, criteria.setting))
|
|
79
|
+
hit++;
|
|
80
|
+
if (anyOf(t.composition, criteria.composition))
|
|
81
|
+
hit++;
|
|
82
|
+
if (anyOf(t.motion, criteria.motion))
|
|
83
|
+
hit++;
|
|
84
|
+
if (anyOf(t.audio_type, criteria.audio_type))
|
|
85
|
+
hit++;
|
|
86
|
+
if (criteria.energy && criteria.energy.length && criteria.energy.includes(t.energy))
|
|
87
|
+
hit++;
|
|
88
|
+
if ((criteria.min_duration_sec != null || criteria.max_duration_sec != null) &&
|
|
89
|
+
(criteria.min_duration_sec == null || clip.duration_sec >= criteria.min_duration_sec) &&
|
|
90
|
+
(criteria.max_duration_sec == null || clip.duration_sec <= criteria.max_duration_sec))
|
|
91
|
+
hit++;
|
|
92
|
+
if (criteria.has_dialogue != null && hasDialogue(t) === criteria.has_dialogue)
|
|
93
|
+
hit++;
|
|
94
|
+
if (criteria.has_subject != null && hasSubject(t) === criteria.has_subject)
|
|
95
|
+
hit++;
|
|
96
|
+
return hit / total;
|
|
97
|
+
}
|
|
98
|
+
/** Cosine similarity of two vectors (safe on unnormalized input). */
|
|
99
|
+
export function cosineSimilarity(a, b) {
|
|
100
|
+
if (!a || !b || a.length === 0 || a.length !== b.length)
|
|
101
|
+
return 0;
|
|
102
|
+
let dot = 0;
|
|
103
|
+
let na = 0;
|
|
104
|
+
let nb = 0;
|
|
105
|
+
for (let i = 0; i < a.length; i++) {
|
|
106
|
+
dot += a[i] * b[i];
|
|
107
|
+
na += a[i] * a[i];
|
|
108
|
+
nb += b[i] * b[i];
|
|
109
|
+
}
|
|
110
|
+
if (na === 0 || nb === 0)
|
|
111
|
+
return 0;
|
|
112
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Run the two-stage search over an in-memory clip set. Cloud can push the
|
|
116
|
+
* structured part into DynamoDB and the semantic part into a vector index; this
|
|
117
|
+
* pure function is the reference implementation devcli uses directly.
|
|
118
|
+
*/
|
|
119
|
+
export function searchClips(clips, opts) {
|
|
120
|
+
const { criteria, queryEmbedding } = opts;
|
|
121
|
+
const limit = opts.limit ?? 20;
|
|
122
|
+
const relax = opts.relaxWhenEmpty ?? true;
|
|
123
|
+
let candidates = clips.filter((c) => matchesCriteria(c, criteria));
|
|
124
|
+
let relaxed = false;
|
|
125
|
+
if (candidates.length === 0 && relax && (queryEmbedding || criteria.semantic_text)) {
|
|
126
|
+
candidates = clips.slice();
|
|
127
|
+
relaxed = true;
|
|
128
|
+
}
|
|
129
|
+
const hits = candidates.map((clip) => scoreClip(clip, criteria, queryEmbedding, relaxed));
|
|
130
|
+
return rankHits(hits, limit);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Score a single clip against a query. `relaxed` = the structured filter was
|
|
134
|
+
* bypassed (semantic-only pass), so factor in how well it still matches the
|
|
135
|
+
* criteria; when not relaxed the clip already passed the hard filter (ratio 1).
|
|
136
|
+
* Exported so the devcli SQLite store can reuse the exact scoring after doing
|
|
137
|
+
* the structured filter / KNN prefilter in SQL.
|
|
138
|
+
*/
|
|
139
|
+
export function scoreClip(clip, criteria, queryEmbedding, relaxed) {
|
|
140
|
+
const similarity = queryEmbedding ? cosineSimilarity(clip.embedding, queryEmbedding) : undefined;
|
|
141
|
+
const ratio = relaxed ? structuredMatchRatio(clip, criteria) : 1;
|
|
142
|
+
let score;
|
|
143
|
+
if (similarity != null) {
|
|
144
|
+
// Blend: mostly semantic, with a structured-match boost. Normalize cosine
|
|
145
|
+
// (-1..1) into 0..1 so scores read sensibly.
|
|
146
|
+
const sim01 = (similarity + 1) / 2;
|
|
147
|
+
score = 0.8 * sim01 + 0.2 * ratio;
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
score = ratio;
|
|
151
|
+
}
|
|
152
|
+
return { clip, score, similarity };
|
|
153
|
+
}
|
|
154
|
+
/** Sort hits by score (newer clips break ties) and take the top `limit`. */
|
|
155
|
+
export function rankHits(hits, limit) {
|
|
156
|
+
hits.sort((a, b) => {
|
|
157
|
+
if (b.score !== a.score)
|
|
158
|
+
return b.score - a.score;
|
|
159
|
+
return (b.clip.created_at || "").localeCompare(a.clip.created_at || "");
|
|
160
|
+
});
|
|
161
|
+
return hits.slice(0, limit);
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=query.js.map
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Prompt-guided segmentation. ffmpeg gives us candidate scene cuts by visual
|
|
2
|
+
// change; this optional pass lets a user's guidance prompt actually reshape HOW
|
|
3
|
+
// the video is clipped — one multimodal LLM call sees a keyframe per candidate
|
|
4
|
+
// scene and decides, per scene, keep/drop (is it relevant to the goal?) and
|
|
5
|
+
// merge-with-previous (does it continue the prior kept scene as one clip?). We
|
|
6
|
+
// then rebuild the scene list from those decisions. Falls back to the raw scenes
|
|
7
|
+
// on any mismatch/error so a scan never breaks.
|
|
8
|
+
import { mkdirSync, rmSync } from "node:fs";
|
|
9
|
+
import { readFile } from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { generateObject } from "ai";
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
import { extractKeyframes } from "./ffmpeg.js";
|
|
14
|
+
// Sending one small keyframe per scene; cap the montage so a single call stays
|
|
15
|
+
// affordable/latency-bounded. Longer videos keep their raw segmentation (logged).
|
|
16
|
+
const MAX_SCENES_FOR_REFINE = 60;
|
|
17
|
+
export async function refineScenesWithGuidance(input) {
|
|
18
|
+
const { client, videoPath, scenes, guidancePrompt } = input;
|
|
19
|
+
if (!guidancePrompt.trim() || scenes.length <= 1) {
|
|
20
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0 };
|
|
21
|
+
}
|
|
22
|
+
if (scenes.length > MAX_SCENES_FOR_REFINE) {
|
|
23
|
+
return {
|
|
24
|
+
scenes,
|
|
25
|
+
refined: false,
|
|
26
|
+
kept: scenes.length,
|
|
27
|
+
dropped: 0,
|
|
28
|
+
merged: 0,
|
|
29
|
+
note: `too many scenes (${scenes.length} > ${MAX_SCENES_FOR_REFINE}) for guided refinement — kept raw segmentation`
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const framesDir = path.join(input.workDir, "refine-frames");
|
|
33
|
+
mkdirSync(framesDir, { recursive: true });
|
|
34
|
+
try {
|
|
35
|
+
// One small keyframe per candidate scene, in timeline order.
|
|
36
|
+
const frames = [];
|
|
37
|
+
for (const scene of scenes) {
|
|
38
|
+
const [p] = await extractKeyframes({ videoPath, scene, outDir: framesDir, count: 1, width: 200 });
|
|
39
|
+
if (p)
|
|
40
|
+
frames.push({ index: scene.index, path: p });
|
|
41
|
+
}
|
|
42
|
+
if (frames.length !== scenes.length) {
|
|
43
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "keyframe extraction incomplete" };
|
|
44
|
+
}
|
|
45
|
+
const decisions = await askForDecisions(client, guidancePrompt, scenes, frames);
|
|
46
|
+
if (!decisions || decisions.length === 0) {
|
|
47
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "model returned no plan" };
|
|
48
|
+
}
|
|
49
|
+
return rebuildScenes(scenes, decisions);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "refinement failed; using raw scenes" };
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
try {
|
|
56
|
+
rmSync(framesDir, { recursive: true, force: true });
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* ignore */
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async function askForDecisions(client, guidancePrompt, scenes, frames) {
|
|
64
|
+
const images = await Promise.all(frames.map((f) => readFile(f.path)));
|
|
65
|
+
const listing = scenes
|
|
66
|
+
.map((s, i) => `Scene ${i}: ${s.start_time_sec.toFixed(1)}s–${s.end_time_sec.toFixed(1)}s (${s.duration_sec.toFixed(1)}s)`)
|
|
67
|
+
.join("\n");
|
|
68
|
+
const prompt = [
|
|
69
|
+
`You are curating clips from a long video for this goal: "${guidancePrompt.trim()}".`,
|
|
70
|
+
`There are ${scenes.length} candidate scenes, shown as ${scenes.length} keyframes below in timeline order (one per scene).`,
|
|
71
|
+
listing,
|
|
72
|
+
"",
|
|
73
|
+
"For EACH scene (in order) decide:",
|
|
74
|
+
"- keep: true if the scene is relevant/useful for the goal, false to drop it.",
|
|
75
|
+
"- merge_with_previous: true if this scene continues the previous KEPT scene and they should be ONE clip (e.g. a single continuous action split by a minor cut). Otherwise false.",
|
|
76
|
+
"",
|
|
77
|
+
`Return exactly ${scenes.length} decisions, in the same order as the scenes. Be selective — dropping off-topic scenes and merging fragmented ones is the point.`
|
|
78
|
+
].join("\n");
|
|
79
|
+
// No fixed-length constraint — Gemini's structured-output schema translation
|
|
80
|
+
// rejects minItems==maxItems; we align by index instead (rebuildScenes).
|
|
81
|
+
const schema = z.object({
|
|
82
|
+
decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
|
|
83
|
+
});
|
|
84
|
+
try {
|
|
85
|
+
const { object } = await generateObject({
|
|
86
|
+
model: client.taggingModel,
|
|
87
|
+
schema,
|
|
88
|
+
maxOutputTokens: Math.max(1000, scenes.length * 40),
|
|
89
|
+
messages: [
|
|
90
|
+
{
|
|
91
|
+
role: "user",
|
|
92
|
+
content: [{ type: "text", text: prompt }, ...images.map((image) => ({ type: "image", image }))]
|
|
93
|
+
}
|
|
94
|
+
]
|
|
95
|
+
});
|
|
96
|
+
return object.decisions;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
if (process.env.VIDFARM_DEBUG_LOGS === "true") {
|
|
100
|
+
// eslint-disable-next-line no-console
|
|
101
|
+
console.warn(`[clips] refine call failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function rebuildScenes(scenes, decisions) {
|
|
107
|
+
const out = [];
|
|
108
|
+
let dropped = 0;
|
|
109
|
+
let merged = 0;
|
|
110
|
+
for (let i = 0; i < scenes.length; i++) {
|
|
111
|
+
// Align by index; a missing decision (short array) defaults to keep.
|
|
112
|
+
const d = decisions[i] ?? { keep: true, merge_with_previous: false };
|
|
113
|
+
if (!d.keep) {
|
|
114
|
+
dropped++;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (d.merge_with_previous && out.length > 0) {
|
|
118
|
+
// Extend the previous kept clip to cover this scene.
|
|
119
|
+
const prev = out[out.length - 1];
|
|
120
|
+
prev.end_time_sec = scenes[i].end_time_sec;
|
|
121
|
+
prev.duration_sec = round3(prev.end_time_sec - prev.start_time_sec);
|
|
122
|
+
merged++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
out.push({ ...scenes[i], index: out.length });
|
|
126
|
+
}
|
|
127
|
+
// Never return an empty plan — if the model dropped everything, keep the raw scenes.
|
|
128
|
+
if (out.length === 0) {
|
|
129
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "model dropped every scene; using raw scenes" };
|
|
130
|
+
}
|
|
131
|
+
return { scenes: out, refined: true, kept: out.length, dropped, merged };
|
|
132
|
+
}
|
|
133
|
+
function round3(n) {
|
|
134
|
+
return Math.round(n * 1000) / 1000;
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=refine.js.map
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// Scan orchestration shared by devcli and cloud. `processSceneToClip` turns ONE
|
|
2
|
+
// detected scene into a full Clip record (extract clip/keyframes/thumb/audio →
|
|
3
|
+
// Gemini tag+transcribe → embed). devcli calls it in a bounded pool; a cloud
|
|
4
|
+
// Step Functions Map state calls it once per scene. Persistence is the caller's
|
|
5
|
+
// job (SQLite vs DynamoDB+S3) — this stays storage-agnostic.
|
|
6
|
+
import { mkdirSync } from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { createIdV7 } from "../../lib/ids.js";
|
|
9
|
+
import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
|
|
10
|
+
import { buildClipEmbeddingText } from "./gemini.js";
|
|
11
|
+
import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
|
|
12
|
+
import { refineScenesWithGuidance } from "./refine.js";
|
|
13
|
+
/** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
|
|
14
|
+
export async function processSceneToClip(input) {
|
|
15
|
+
const { ctx, videoPath, scene } = input;
|
|
16
|
+
const framesDir = path.join(ctx.workDir, "frames");
|
|
17
|
+
const audioDir = path.join(ctx.workDir, "audio");
|
|
18
|
+
mkdirSync(ctx.clipsDir, { recursive: true });
|
|
19
|
+
mkdirSync(ctx.thumbsDir, { recursive: true });
|
|
20
|
+
const clipId = createIdV7("clip");
|
|
21
|
+
// 1. Keyframes for visual tagging (small, low-res).
|
|
22
|
+
const keyframePaths = await extractKeyframes({
|
|
23
|
+
videoPath,
|
|
24
|
+
scene,
|
|
25
|
+
outDir: framesDir,
|
|
26
|
+
count: ctx.framesPerScene ?? 2
|
|
27
|
+
});
|
|
28
|
+
// 2. Per-scene audio for ASR (one combined tag+transcribe call).
|
|
29
|
+
let audioPath;
|
|
30
|
+
if (ctx.includeAudio ?? true) {
|
|
31
|
+
const a = await extractSceneAudio({
|
|
32
|
+
videoPath,
|
|
33
|
+
scene,
|
|
34
|
+
outPath: path.join(audioDir, `scene-${scene.index}.mp3`)
|
|
35
|
+
});
|
|
36
|
+
audioPath = a ?? undefined;
|
|
37
|
+
}
|
|
38
|
+
// 3. Tag (+ transcription when audio present on Gemini), biased by the optional guidance prompt.
|
|
39
|
+
const tagResult = await ctx.client.tagScene({
|
|
40
|
+
keyframePaths,
|
|
41
|
+
transcript: "",
|
|
42
|
+
audioPath,
|
|
43
|
+
scene,
|
|
44
|
+
guidancePrompt: ctx.guidancePrompt
|
|
45
|
+
});
|
|
46
|
+
// 4. Cut the clip + thumbnail.
|
|
47
|
+
const clipLocal = path.join(ctx.clipsDir, `${clipId}.mp4`);
|
|
48
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clipId}.jpg`);
|
|
49
|
+
const clipOk = await extractClip({ videoPath, scene, outPath: clipLocal, reencode: ctx.reencodeClips !== false });
|
|
50
|
+
if (!clipOk)
|
|
51
|
+
throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s`);
|
|
52
|
+
await extractThumbnail({ videoPath, scene, outPath: thumbLocal });
|
|
53
|
+
// 5. Embed the document text for semantic search (skipped when the client can't embed).
|
|
54
|
+
let embedding;
|
|
55
|
+
if (ctx.client.canEmbed) {
|
|
56
|
+
const embedText = buildClipEmbeddingText(tagResult.tags, tagResult.description);
|
|
57
|
+
[embedding] = await ctx.client.embedDocuments([embedText]);
|
|
58
|
+
}
|
|
59
|
+
const clip = {
|
|
60
|
+
clip_id: clipId,
|
|
61
|
+
source_video_id: input.sourceVideoId,
|
|
62
|
+
source_filename: input.sourceFilename,
|
|
63
|
+
start_time_sec: scene.start_time_sec,
|
|
64
|
+
end_time_sec: scene.end_time_sec,
|
|
65
|
+
duration_sec: scene.duration_sec,
|
|
66
|
+
file_path: ctx.clipStoredPath ? ctx.clipStoredPath(clipId, clipLocal) : clipLocal,
|
|
67
|
+
thumbnail_path: ctx.thumbStoredPath ? ctx.thumbStoredPath(clipId, thumbLocal) : thumbLocal,
|
|
68
|
+
tags: tagResult.tags,
|
|
69
|
+
description: tagResult.description,
|
|
70
|
+
embedding,
|
|
71
|
+
taxonomy_version: ctx.client.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION,
|
|
72
|
+
tier: ctx.client.tier,
|
|
73
|
+
provider: ctx.client.provider,
|
|
74
|
+
embedding_model: ctx.client.embeddingModelId ?? undefined,
|
|
75
|
+
owner_id: input.ownerId,
|
|
76
|
+
created_at: new Date().toISOString()
|
|
77
|
+
};
|
|
78
|
+
return clip;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Run the full local scan pipeline for one video with a bounded concurrency
|
|
82
|
+
* pool. (Cloud replaces this loop with a Step Functions Map over the scenes.)
|
|
83
|
+
*/
|
|
84
|
+
export async function scanVideo(opts) {
|
|
85
|
+
const probe = opts.probe ?? (await probeVideo(opts.videoPath));
|
|
86
|
+
let scenes = opts.scenes ??
|
|
87
|
+
(await detectScenes(opts.videoPath, {
|
|
88
|
+
durationSec: probe.duration_sec,
|
|
89
|
+
...opts.sceneOptions
|
|
90
|
+
}));
|
|
91
|
+
// Prompt-guided segmentation: let the guidance reshape which scenes become
|
|
92
|
+
// clips (keep/drop/merge) before we spend tagging tokens on them.
|
|
93
|
+
if (opts.ctx.guidancePrompt?.trim() && opts.ctx.refineSegmentation !== false) {
|
|
94
|
+
const refined = await refineScenesWithGuidance({
|
|
95
|
+
client: opts.ctx.client,
|
|
96
|
+
videoPath: opts.videoPath,
|
|
97
|
+
scenes,
|
|
98
|
+
guidancePrompt: opts.ctx.guidancePrompt,
|
|
99
|
+
workDir: opts.ctx.workDir
|
|
100
|
+
});
|
|
101
|
+
scenes = refined.scenes;
|
|
102
|
+
opts.onRefine?.(refined);
|
|
103
|
+
}
|
|
104
|
+
const sourceVideoId = opts.sourceVideoId ?? createIdV7("srcvid");
|
|
105
|
+
const sourceFilename = opts.sourceFilename ?? path.basename(opts.videoPath);
|
|
106
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
107
|
+
const clips = [];
|
|
108
|
+
let cursor = 0;
|
|
109
|
+
let completed = 0;
|
|
110
|
+
async function worker() {
|
|
111
|
+
while (cursor < scenes.length) {
|
|
112
|
+
const idx = cursor++;
|
|
113
|
+
const scene = scenes[idx];
|
|
114
|
+
try {
|
|
115
|
+
const clip = await processSceneToClip({
|
|
116
|
+
ctx: opts.ctx,
|
|
117
|
+
videoPath: opts.videoPath,
|
|
118
|
+
scene,
|
|
119
|
+
sourceVideoId,
|
|
120
|
+
sourceFilename,
|
|
121
|
+
ownerId: opts.ownerId
|
|
122
|
+
});
|
|
123
|
+
clips.push(clip);
|
|
124
|
+
completed++;
|
|
125
|
+
await opts.onClip?.(clip, completed, scenes.length);
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
opts.onError?.(scene, error);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, scenes.length) }, () => worker()));
|
|
133
|
+
// Preserve timeline order for a stable library.
|
|
134
|
+
clips.sort((a, b) => a.start_time_sec - b.start_time_sec);
|
|
135
|
+
return { source_video_id: sourceVideoId, probe, scenes, clips };
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=scan.js.map
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Versioned tag taxonomy loader. The vocabulary lives in taxonomy.v1.json so it
|
|
2
|
+
// can evolve independently; clips record the version they were tagged with
|
|
3
|
+
// (Clip.taxonomy_version) so old clips stay valid when the vocab grows.
|
|
4
|
+
import taxonomyV1 from "./taxonomy.v1.json" with { type: "json" };
|
|
5
|
+
const REGISTRY = {
|
|
6
|
+
v1: taxonomyV1
|
|
7
|
+
};
|
|
8
|
+
export const ACTIVE_TAXONOMY_VERSION = "v1";
|
|
9
|
+
/** Get a taxonomy by version (defaults to the active one). */
|
|
10
|
+
export function getTaxonomy(version = ACTIVE_TAXONOMY_VERSION) {
|
|
11
|
+
const found = REGISTRY[version];
|
|
12
|
+
if (!found) {
|
|
13
|
+
throw new Error(`Unknown taxonomy version "${version}" (have: ${Object.keys(REGISTRY).join(", ")})`);
|
|
14
|
+
}
|
|
15
|
+
return found;
|
|
16
|
+
}
|
|
17
|
+
/** The multi-valued categorical fields (everything except `energy`, which is single). */
|
|
18
|
+
export const MULTI_VALUE_CATEGORIES = [
|
|
19
|
+
"subject",
|
|
20
|
+
"action",
|
|
21
|
+
"emotion",
|
|
22
|
+
"setting",
|
|
23
|
+
"composition",
|
|
24
|
+
"motion",
|
|
25
|
+
"audio_type"
|
|
26
|
+
];
|
|
27
|
+
/**
|
|
28
|
+
* Clamp an LLM-produced tag object to the controlled vocabulary: drop any value
|
|
29
|
+
* not in the taxonomy so a hallucinated tag can never poison the index. Returns
|
|
30
|
+
* a fully-populated ClipTags-shaped object (missing categories become []).
|
|
31
|
+
*/
|
|
32
|
+
export function sanitizeTagValues(raw, version = ACTIVE_TAXONOMY_VERSION) {
|
|
33
|
+
const tax = getTaxonomy(version);
|
|
34
|
+
const cats = tax.categories;
|
|
35
|
+
const clampList = (values, allowed) => {
|
|
36
|
+
if (!Array.isArray(values))
|
|
37
|
+
return [];
|
|
38
|
+
const set = new Set(allowed);
|
|
39
|
+
const out = [];
|
|
40
|
+
for (const v of values) {
|
|
41
|
+
const s = String(v).trim().toLowerCase().replace(/\s+/g, "_");
|
|
42
|
+
if (set.has(s) && !out.includes(s))
|
|
43
|
+
out.push(s);
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
};
|
|
47
|
+
const clampOne = (value, allowed, fallback) => {
|
|
48
|
+
const s = String(value ?? "").trim().toLowerCase().replace(/\s+/g, "_");
|
|
49
|
+
return allowed.includes(s) ? s : fallback;
|
|
50
|
+
};
|
|
51
|
+
return {
|
|
52
|
+
subject: clampList(raw.subject, cats.subject),
|
|
53
|
+
action: clampList(raw.action, cats.action),
|
|
54
|
+
emotion: clampList(raw.emotion, cats.emotion),
|
|
55
|
+
setting: clampList(raw.setting, cats.setting),
|
|
56
|
+
composition: clampList(raw.composition, cats.composition),
|
|
57
|
+
motion: clampList(raw.motion, cats.motion),
|
|
58
|
+
energy: clampOne(raw.energy, cats.energy, "medium"),
|
|
59
|
+
audio_type: clampList(raw.audio_type, cats.audio_type)
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/** A compact, prompt-friendly rendering of the vocabulary to inline in the tagging prompt. */
|
|
63
|
+
export function taxonomyForPrompt(version = ACTIVE_TAXONOMY_VERSION) {
|
|
64
|
+
const tax = getTaxonomy(version);
|
|
65
|
+
const lines = [];
|
|
66
|
+
for (const [cat, values] of Object.entries(tax.categories)) {
|
|
67
|
+
lines.push(`- ${cat}: ${values.join(", ")}`);
|
|
68
|
+
}
|
|
69
|
+
return lines.join("\n");
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=taxonomy.js.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1",
|
|
3
|
+
"notes": "Controlled vocabulary for VidFarm clip curation. Start small (~50 tags); expand based on real usage. Clips store the version they were tagged against so the vocab can evolve without breaking existing clips.",
|
|
4
|
+
"categories": {
|
|
5
|
+
"subject": [
|
|
6
|
+
"person",
|
|
7
|
+
"single_speaker",
|
|
8
|
+
"multiple_people",
|
|
9
|
+
"hands",
|
|
10
|
+
"product",
|
|
11
|
+
"screen_ui",
|
|
12
|
+
"text_on_screen",
|
|
13
|
+
"animal",
|
|
14
|
+
"food",
|
|
15
|
+
"vehicle",
|
|
16
|
+
"no_subject"
|
|
17
|
+
],
|
|
18
|
+
"action": [
|
|
19
|
+
"talking",
|
|
20
|
+
"laughing",
|
|
21
|
+
"looking_at_camera",
|
|
22
|
+
"reacting",
|
|
23
|
+
"pointing",
|
|
24
|
+
"typing",
|
|
25
|
+
"hands_on_keyboard",
|
|
26
|
+
"walking",
|
|
27
|
+
"eating",
|
|
28
|
+
"holding_product",
|
|
29
|
+
"unboxing",
|
|
30
|
+
"gesturing",
|
|
31
|
+
"still"
|
|
32
|
+
],
|
|
33
|
+
"emotion": [
|
|
34
|
+
"amused",
|
|
35
|
+
"surprised",
|
|
36
|
+
"excited",
|
|
37
|
+
"confused",
|
|
38
|
+
"deadpan",
|
|
39
|
+
"exasperated",
|
|
40
|
+
"serious",
|
|
41
|
+
"neutral",
|
|
42
|
+
"sad",
|
|
43
|
+
"angry"
|
|
44
|
+
],
|
|
45
|
+
"setting": [
|
|
46
|
+
"indoor",
|
|
47
|
+
"outdoor",
|
|
48
|
+
"office",
|
|
49
|
+
"home",
|
|
50
|
+
"studio",
|
|
51
|
+
"kitchen",
|
|
52
|
+
"car",
|
|
53
|
+
"nature",
|
|
54
|
+
"urban",
|
|
55
|
+
"abstract_or_graphic"
|
|
56
|
+
],
|
|
57
|
+
"composition": [
|
|
58
|
+
"close_up",
|
|
59
|
+
"medium_shot",
|
|
60
|
+
"wide_shot",
|
|
61
|
+
"centered",
|
|
62
|
+
"off_center",
|
|
63
|
+
"over_the_shoulder",
|
|
64
|
+
"top_down",
|
|
65
|
+
"product_shot"
|
|
66
|
+
],
|
|
67
|
+
"motion": [
|
|
68
|
+
"static",
|
|
69
|
+
"low",
|
|
70
|
+
"medium",
|
|
71
|
+
"high",
|
|
72
|
+
"camera_shake",
|
|
73
|
+
"pan_or_tilt",
|
|
74
|
+
"zoom"
|
|
75
|
+
],
|
|
76
|
+
"energy": [
|
|
77
|
+
"low",
|
|
78
|
+
"medium",
|
|
79
|
+
"high"
|
|
80
|
+
],
|
|
81
|
+
"audio_type": [
|
|
82
|
+
"dialogue",
|
|
83
|
+
"voiceover",
|
|
84
|
+
"music",
|
|
85
|
+
"sfx",
|
|
86
|
+
"ambient",
|
|
87
|
+
"silence"
|
|
88
|
+
]
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Shared data model for VidFarm clip curation — the "third pillar" library
|
|
2
|
+
// (viral DNA + HTML formats + CLIPS). One scanned scene becomes one Clip
|
|
3
|
+
// record. This shape is IDENTICAL across the two deployment targets
|
|
4
|
+
// (devcli/local SQLite and AWS DynamoDB) so users can move between local and
|
|
5
|
+
// cloud without lock-in. See drafts/clip-hunting/vidfarm_clip_curation_handoff.md.
|
|
6
|
+
export {};
|
|
7
|
+
//# sourceMappingURL=types.js.map
|