@mevdragon/vidfarm-devcli 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/vidfarm-media/SKILL.md +43 -6
- package/SKILL.director.md +37 -23
- package/SKILL.platform.md +6 -6
- package/demo/dist/app.js +69 -69
- package/demo/dist/favicon.ico +0 -0
- package/dist/src/app.js +1832 -213
- package/dist/src/cli.js +238 -17
- package/dist/src/devcli/clip-store.js +29 -2
- package/dist/src/devcli/clips.js +116 -73
- package/dist/src/devcli/composition-edit.js +262 -0
- package/dist/src/devcli/timeline-edit.js +283 -0
- package/dist/src/editor-chat.js +29 -6
- package/dist/src/frontend/discover-client.js +130 -0
- package/dist/src/frontend/discover-store.js +23 -0
- package/dist/src/frontend/file-directory.js +744 -0
- package/dist/src/frontend/template-editor-chat.js +22 -19
- package/dist/src/landing-page.js +24 -7
- package/dist/src/page-shell.js +25 -1
- package/dist/src/reskin/agency-page.js +214 -170
- package/dist/src/reskin/calendar-page.js +503 -187
- package/dist/src/reskin/chat-page.js +739 -299
- package/dist/src/reskin/discover-page.js +1450 -279
- package/dist/src/reskin/document.js +1392 -16
- package/dist/src/reskin/help-page.js +139 -100
- package/dist/src/reskin/index-page.js +1 -1
- package/dist/src/reskin/inpaint-page.js +547 -0
- package/dist/src/reskin/job-runs-page.js +355 -127
- package/dist/src/reskin/library-page.js +1188 -317
- package/dist/src/reskin/login-page.js +124 -87
- package/dist/src/reskin/pricing-page.js +197 -166
- package/dist/src/reskin/settings-page.js +566 -187
- package/dist/src/reskin/theme.js +434 -17
- package/dist/src/services/clip-curation/gemini.js +5 -0
- package/dist/src/services/clip-curation/hunt.js +79 -1
- package/dist/src/services/clip-curation/index.js +2 -1
- package/dist/src/services/clip-curation/local-agent.js +4 -3
- package/dist/src/services/clip-curation/media-select.js +85 -0
- package/dist/src/services/clip-curation/query.js +5 -1
- package/dist/src/services/clip-curation/refine.js +50 -20
- package/dist/src/services/clip-curation/scan.js +10 -3
- package/dist/src/services/clip-curation/taxonomy.js +3 -1
- package/dist/src/services/clip-curation/taxonomy.v1.json +13 -1
- package/dist/src/services/clip-records.js +14 -1
- package/dist/src/services/clip-search.js +43 -13
- package/dist/src/services/file-directory.js +114 -0
- package/dist/src/services/storage.js +24 -1
- package/dist/src/services/upstream.js +5 -5
- package/dist/src/template-editor-shell.js +16 -2
- package/package.json +1 -1
- package/public/assets/discover-client-app.js +1 -0
- package/public/assets/file-directory-app.js +2 -0
- package/public/assets/homepage-client-app.js +12 -12
- package/public/assets/page-runtime-client-app.js +24 -24
- package/src/assets/favicon.ico +0 -0
- package/src/assets/logo-vidfarm.png +0 -0
|
@@ -6,6 +6,66 @@
|
|
|
6
6
|
// NEVER GhostCut on the long-form source). Shared by devcli (local), the Hono
|
|
7
7
|
// API, and the cloud scan Lambda so every surface parses and normalizes
|
|
8
8
|
// identically. All helpers are pure/deterministic — no AI calls here.
|
|
9
|
+
// ── Unspecified-prompt defaults ─────────────────────────────────────────────
|
|
10
|
+
// When a user imports a source video without saying what they want, we bias
|
|
11
|
+
// toward the short-form product default: a handful of DISTINCT, vertical,
|
|
12
|
+
// text-free 5–15s clips rather than every raw cut. These fill only the gaps —
|
|
13
|
+
// any field the user (or their prompt) sets always wins.
|
|
14
|
+
/** Default soft clip length when unspecified: ~10s, 5–15s band. */
|
|
15
|
+
export const DEFAULT_CLIP_DURATION_BAND = { min_sec: 5, max_sec: 15, target_sec: 10 };
|
|
16
|
+
/** Default mining density when neither max_clips nor clips_per_10min is set. */
|
|
17
|
+
export const DEFAULT_CLIPS_PER_10_MIN = 10;
|
|
18
|
+
/** Default output aspect when unspecified (vertical / shorts). */
|
|
19
|
+
export const DEFAULT_CLIP_ASPECT = "9:16";
|
|
20
|
+
/**
|
|
21
|
+
* Fill the unspecified-prompt defaults on a normalized hunt spec: a 5–15s
|
|
22
|
+
* duration band, vertical 9:16 crop, and avoid-text ON. Only fills gaps — an
|
|
23
|
+
* explicit user choice (including avoid_text:false) is preserved.
|
|
24
|
+
*/
|
|
25
|
+
export function applyHuntSpecDefaults(spec) {
|
|
26
|
+
const out = { ...spec };
|
|
27
|
+
if (!out.duration_band)
|
|
28
|
+
out.duration_band = { ...DEFAULT_CLIP_DURATION_BAND };
|
|
29
|
+
if (out.avoid_text === undefined)
|
|
30
|
+
out.avoid_text = true;
|
|
31
|
+
if (!out.aspect)
|
|
32
|
+
out.aspect = DEFAULT_CLIP_ASPECT;
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The number of clips to aim for given the (effective, windowed) source
|
|
37
|
+
* duration. A hard max_clips wins; otherwise density (default 10 per 10 min ≈
|
|
38
|
+
* one clip per minute) scales with length. Always ≥ 1.
|
|
39
|
+
*/
|
|
40
|
+
export function resolveTargetClipCount(effectiveDurationSec, spec) {
|
|
41
|
+
if (spec.max_clips && spec.max_clips > 0)
|
|
42
|
+
return Math.max(1, Math.round(spec.max_clips));
|
|
43
|
+
const per10 = spec.clips_per_10min && spec.clips_per_10min > 0 ? spec.clips_per_10min : DEFAULT_CLIPS_PER_10_MIN;
|
|
44
|
+
const tenMinBlocks = Math.max(0, effectiveDurationSec) / 600;
|
|
45
|
+
return Math.max(1, Math.round(tenMinBlocks * per10));
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Deterministic count cap / uniqueness backstop: when more scenes survived than
|
|
49
|
+
* the target, keep an EVENLY-SPACED subset across the timeline. Spread-out
|
|
50
|
+
* scenes are far less likely to be near-duplicates of each other than adjacent
|
|
51
|
+
* ones, so this doubles as a cheap "distinct clips" filter when the AI refine
|
|
52
|
+
* pass is skipped (>60 scenes) or over-returns. Re-indexes the survivors.
|
|
53
|
+
*/
|
|
54
|
+
export function capScenesEvenlyToCount(scenes, target) {
|
|
55
|
+
if (!Number.isFinite(target) || target <= 0 || scenes.length <= target)
|
|
56
|
+
return scenes;
|
|
57
|
+
const step = scenes.length / target;
|
|
58
|
+
const picked = [];
|
|
59
|
+
const seen = new Set();
|
|
60
|
+
for (let i = 0; i < target; i++) {
|
|
61
|
+
const idx = Math.min(scenes.length - 1, Math.floor(i * step + step / 2));
|
|
62
|
+
if (seen.has(idx))
|
|
63
|
+
continue;
|
|
64
|
+
seen.add(idx);
|
|
65
|
+
picked.push(scenes[idx]);
|
|
66
|
+
}
|
|
67
|
+
return picked.map((s, i) => ({ ...s, index: i }));
|
|
68
|
+
}
|
|
9
69
|
// ── Duration band ───────────────────────────────────────────────────────────
|
|
10
70
|
/**
|
|
11
71
|
* Map a spoken target ("10 sec", "30 sec", "60 sec", "120 sec") to a soft
|
|
@@ -242,7 +302,7 @@ export function buildEffectiveGuidance(input) {
|
|
|
242
302
|
*/
|
|
243
303
|
export function parseClipHuntPrompt(prompt) {
|
|
244
304
|
const text = (prompt ?? "").trim();
|
|
245
|
-
const result = { windows: [], target_duration_sec: null, aspect: null, avoid_text: false };
|
|
305
|
+
const result = { windows: [], target_duration_sec: null, aspect: null, avoid_text: false, max_clips: null };
|
|
246
306
|
if (!text)
|
|
247
307
|
return result;
|
|
248
308
|
const tc = "\\d{1,2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?";
|
|
@@ -277,6 +337,14 @@ export function parseClipHuntPrompt(prompt) {
|
|
|
277
337
|
/(without|no)\s+(?:any\s+)?(captions?|subtitles?|(?:on[- ]?screen\s+)?text)/i.test(text) ||
|
|
278
338
|
/\b(caption|text)-free\b/i.test(text) ||
|
|
279
339
|
/no\s+text\s+on\s+screen/i.test(text);
|
|
340
|
+
// "10 clips" / "give me 20 raws" / "top 5 cuts" → a clip-count target.
|
|
341
|
+
const countMatch = /(\d{1,3})\s*(?:clips?|raws?|shots?|cuts?|segments?)\b/i.exec(text) ??
|
|
342
|
+
/\b(?:top|best|give me|find|get|pull|grab)\s+(\d{1,3})\b/i.exec(text);
|
|
343
|
+
if (countMatch) {
|
|
344
|
+
const n = Number(countMatch[1]);
|
|
345
|
+
if (Number.isFinite(n) && n > 0)
|
|
346
|
+
result.max_clips = Math.min(500, Math.round(n));
|
|
347
|
+
}
|
|
280
348
|
return result;
|
|
281
349
|
}
|
|
282
350
|
// ── Wire normalization (API / SFN / tool payloads) ──────────────────────────
|
|
@@ -311,8 +379,18 @@ export function normalizeHuntSpec(raw) {
|
|
|
311
379
|
spec.aspect = aspect;
|
|
312
380
|
if (rec.crop_focus != null)
|
|
313
381
|
spec.crop_focus = normalizeCropFocus(rec.crop_focus);
|
|
382
|
+
// Explicit true/false both honored so a caller can opt OUT of the avoid-text
|
|
383
|
+
// default; only an absent field falls through to applyHuntSpecDefaults.
|
|
314
384
|
if (rec.avoid_text === true || rec.no_text === true || rec.no_captions === true)
|
|
315
385
|
spec.avoid_text = true;
|
|
386
|
+
else if (rec.avoid_text === false || rec.no_text === false || rec.allow_text === true)
|
|
387
|
+
spec.avoid_text = false;
|
|
388
|
+
const maxClips = Number(rec.max_clips ?? rec.clip_count ?? rec.target_clip_count);
|
|
389
|
+
if (Number.isFinite(maxClips) && maxClips > 0)
|
|
390
|
+
spec.max_clips = Math.min(500, Math.round(maxClips));
|
|
391
|
+
const perTen = Number(rec.clips_per_10min ?? rec.clips_per_10_min ?? rec.density);
|
|
392
|
+
if (Number.isFinite(perTen) && perTen > 0)
|
|
393
|
+
spec.clips_per_10min = Math.min(200, perTen);
|
|
316
394
|
if (typeof rec.source_url === "string" && rec.source_url.trim())
|
|
317
395
|
spec.source_url = rec.source_url.trim();
|
|
318
396
|
return spec;
|
|
@@ -6,12 +6,13 @@ export { ACTIVE_TAXONOMY_VERSION, MULTI_VALUE_CATEGORIES, getTaxonomy, sanitizeT
|
|
|
6
6
|
export { BUILTIN_PRESETS, findBuiltinPreset } from "./presets.js";
|
|
7
7
|
export { ClipModelClient, ClipGeminiClient, ClipEmbeddingClient, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_MODELS, TAG_MODEL_BY_TIER, buildClipEmbeddingText, buildQueryPrompt, buildTaggingPrompt, normalize } from "./gemini.js";
|
|
8
8
|
export { LocalAgentClipClient, detectLocalAgent, extractJsonObject } from "./local-agent.js";
|
|
9
|
-
export { AVOID_TEXT_GUIDANCE, buildEffectiveGuidance, cropFilterFor, effectiveHuntDurationSec, fitScenesToDurationBand, normalizeAspect, normalizeCropFocus, normalizeHuntSpec, normalizeWindows, parseClipHuntPrompt, parseTimecode, parseTimeRanges, resolveDurationBand } from "./hunt.js";
|
|
9
|
+
export { AVOID_TEXT_GUIDANCE, DEFAULT_CLIP_ASPECT, DEFAULT_CLIP_DURATION_BAND, DEFAULT_CLIPS_PER_10_MIN, applyHuntSpecDefaults, buildEffectiveGuidance, capScenesEvenlyToCount, cropFilterFor, effectiveHuntDurationSec, fitScenesToDurationBand, normalizeAspect, normalizeCropFocus, normalizeHuntSpec, normalizeWindows, parseClipHuntPrompt, parseTimecode, parseTimeRanges, resolveDurationBand, resolveTargetClipCount } from "./hunt.js";
|
|
10
10
|
export { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, hasFfmpeg, probeVideo, resolveFfmpeg, resolveFfprobe } from "./ffmpeg.js";
|
|
11
11
|
export { cosineSimilarity, matchesCriteria, rankHits, scoreClip, searchClips, structuredMatchRatio } from "./query.js";
|
|
12
12
|
export { estimateScanCostFromDuration, estimateScanCostFromScenes, formatCostEstimate } from "./cost.js";
|
|
13
13
|
export { processSceneToClip, scanVideo } from "./scan.js";
|
|
14
14
|
export { refineScenesWithGuidance } from "./refine.js";
|
|
15
|
+
export { estimateMediaHeight, pickBestVideoMedia } from "./media-select.js";
|
|
15
16
|
// Re-export ClipPreset-related helpers already covered by ./presets and ./types
|
|
16
17
|
// via the wildcard exports above; nothing extra needed here.
|
|
17
18
|
//# sourceMappingURL=index.js.map
|
|
@@ -67,7 +67,8 @@ export class LocalAgentClipClient {
|
|
|
67
67
|
...input.keyframePaths.map((p) => `- ${p}`),
|
|
68
68
|
"",
|
|
69
69
|
"Respond with ONLY the JSON object (no prose, no markdown fence) with keys:",
|
|
70
|
-
`subject, action, emotion, setting, composition, motion (string arrays), energy (string), audio_type (string array), transcript (string), description (string)
|
|
70
|
+
`content_type, subject, action, emotion, setting, composition, motion (string arrays), energy (string), audio_type (string array), transcript (string), description (string).`,
|
|
71
|
+
`content_type is the KIND of shot — pick 1-2 of: talking_head, b_roll, product_shot, screen_recording, demo, reaction, interview, establishing, lifestyle, text_graphic.`
|
|
71
72
|
].join("\n");
|
|
72
73
|
try {
|
|
73
74
|
const raw = await this.runAgent(prompt);
|
|
@@ -75,7 +76,7 @@ export class LocalAgentClipClient {
|
|
|
75
76
|
if (!parsed)
|
|
76
77
|
throw new Error("local agent returned no JSON");
|
|
77
78
|
const withDefaults = {
|
|
78
|
-
subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
|
|
79
|
+
content_type: [], subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
|
|
79
80
|
energy: "medium", audio_type: [], transcript: "", description: "",
|
|
80
81
|
...parsed
|
|
81
82
|
};
|
|
@@ -92,7 +93,7 @@ export class LocalAgentClipClient {
|
|
|
92
93
|
console.warn(`[clips] tag failed (local-agent/${this.kind}) @${input.scene.start_time_sec.toFixed(1)}s: ${message}`);
|
|
93
94
|
return {
|
|
94
95
|
tags: {
|
|
95
|
-
subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
|
|
96
|
+
content_type: [], subject: [], action: [], emotion: [], setting: [], composition: [], motion: [],
|
|
96
97
|
energy: "medium",
|
|
97
98
|
audio_type: input.transcript ? ["dialogue"] : ["silence"],
|
|
98
99
|
transcript: input.transcript || ""
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// Resolution-aware media selection for the RapidAPI social-video downloader.
|
|
2
|
+
// The lookup returns several renditions per source (e.g. 360p / 720p / 1080p,
|
|
3
|
+
// sometimes audio-only or HLS playlists). The clip library's quality is capped
|
|
4
|
+
// by whatever source we ingest, so we must pick the HIGHEST-resolution playable
|
|
5
|
+
// MP4 — not just the first one. Shared by the clip-scan Lambda (infra/lambda)
|
|
6
|
+
// and the in-process serve/import path (src/app.ts) so both stay in lock-step.
|
|
7
|
+
/** True when a candidate is a directly-downloadable MP4 (not HLS/DASH/audio). */
|
|
8
|
+
function isPlayableMp4(m) {
|
|
9
|
+
if (!m.url)
|
|
10
|
+
return false;
|
|
11
|
+
if (m.videoAvailable === false)
|
|
12
|
+
return false;
|
|
13
|
+
if (m.type && !/video/i.test(m.type))
|
|
14
|
+
return false;
|
|
15
|
+
// HLS/DASH playlists can't be fetched as a single file — skip them.
|
|
16
|
+
if (/\.(m3u8|mpd)(\?|$)/i.test(m.url))
|
|
17
|
+
return false;
|
|
18
|
+
const ext = (m.extension ?? "").toLowerCase();
|
|
19
|
+
return ext === "mp4" || /\.mp4(\?|$)/i.test(m.url);
|
|
20
|
+
}
|
|
21
|
+
/** Any playable video (mp4 preferred, but a non-HLS fallback still beats nothing). */
|
|
22
|
+
function isPlayableVideo(m) {
|
|
23
|
+
if (!m.url)
|
|
24
|
+
return false;
|
|
25
|
+
if (m.videoAvailable === false)
|
|
26
|
+
return false;
|
|
27
|
+
if (m.type && !/video/i.test(m.type))
|
|
28
|
+
return false;
|
|
29
|
+
if (/\.(m3u8|mpd)(\?|$)/i.test(m.url))
|
|
30
|
+
return false;
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Estimate a rendition's vertical resolution (used as the ranking key). Reads an
|
|
35
|
+
* explicit height first, then parses the quality/label string. Returns 0 when
|
|
36
|
+
* nothing is parseable so labelled renditions always outrank unknown ones.
|
|
37
|
+
*/
|
|
38
|
+
export function estimateMediaHeight(m) {
|
|
39
|
+
const h = Number(m.height);
|
|
40
|
+
if (Number.isFinite(h) && h > 0)
|
|
41
|
+
return h;
|
|
42
|
+
const w = Number(m.width);
|
|
43
|
+
// Assume ~16:9 when only width is known (a coarse but monotonic proxy).
|
|
44
|
+
if (Number.isFinite(w) && w > 0)
|
|
45
|
+
return Math.round((w * 9) / 16);
|
|
46
|
+
const label = `${m.quality ?? ""} ${m.label ?? ""}`.toLowerCase();
|
|
47
|
+
// "1080p", "720 p", "2160p60", "1440P" → the leading pixel-height number.
|
|
48
|
+
const pMatch = label.match(/(\d{3,4})\s*p/);
|
|
49
|
+
if (pMatch)
|
|
50
|
+
return Number(pMatch[1]);
|
|
51
|
+
// "4k"/"2k" shorthand.
|
|
52
|
+
if (/\b4k|uhd\b/.test(label))
|
|
53
|
+
return 2160;
|
|
54
|
+
if (/\b2k\b/.test(label))
|
|
55
|
+
return 1440;
|
|
56
|
+
// Named tiers.
|
|
57
|
+
if (/full\s*hd|fhd/.test(label))
|
|
58
|
+
return 1080;
|
|
59
|
+
if (/\bhd\b/.test(label))
|
|
60
|
+
return 720;
|
|
61
|
+
if (/\bsd\b|low|small/.test(label))
|
|
62
|
+
return 360;
|
|
63
|
+
// A bare number (some providers give just "1080" / "720").
|
|
64
|
+
const nMatch = label.match(/\b(\d{3,4})\b/);
|
|
65
|
+
if (nMatch)
|
|
66
|
+
return Number(nMatch[1]);
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Pick the best rendition to ingest: the highest-resolution playable MP4, with a
|
|
71
|
+
* non-MP4 playable video as a last resort, else null. Ranking is stable so ties
|
|
72
|
+
* keep the provider's own ordering (usually its preferred rendition first).
|
|
73
|
+
*/
|
|
74
|
+
export function pickBestVideoMedia(medias) {
|
|
75
|
+
const list = Array.isArray(medias) ? medias : [];
|
|
76
|
+
const mp4s = list.filter(isPlayableMp4);
|
|
77
|
+
const pool = mp4s.length ? mp4s : list.filter(isPlayableVideo);
|
|
78
|
+
if (!pool.length)
|
|
79
|
+
return null;
|
|
80
|
+
// Decorate-sort-undecorate to keep it a stable sort by descending height.
|
|
81
|
+
return pool
|
|
82
|
+
.map((m, index) => ({ m, index, h: estimateMediaHeight(m) }))
|
|
83
|
+
.sort((a, b) => b.h - a.h || a.index - b.index)[0].m;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=media-select.js.map
|
|
@@ -9,7 +9,7 @@ const DIALOGUE_AUDIO = new Set(["dialogue", "voiceover"]);
|
|
|
9
9
|
/** How many categorical facets `criteria` constrains (used for the match-ratio soft score). */
|
|
10
10
|
function constraintCount(criteria) {
|
|
11
11
|
let n = 0;
|
|
12
|
-
for (const key of ["subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"]) {
|
|
12
|
+
for (const key of ["content_type", "subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"]) {
|
|
13
13
|
if (Array.isArray(criteria[key]) && criteria[key].length)
|
|
14
14
|
n++;
|
|
15
15
|
}
|
|
@@ -31,6 +31,8 @@ function hasDialogue(tags) {
|
|
|
31
31
|
export function matchesCriteria(clip, criteria) {
|
|
32
32
|
const t = clip.tags;
|
|
33
33
|
const anyOf = (have, want) => !want || want.length === 0 || want.some((w) => have.includes(w));
|
|
34
|
+
if (!anyOf(t.content_type ?? [], criteria.content_type))
|
|
35
|
+
return false;
|
|
34
36
|
if (!anyOf(t.subject, criteria.subject))
|
|
35
37
|
return false;
|
|
36
38
|
if (!anyOf(t.action, criteria.action))
|
|
@@ -69,6 +71,8 @@ export function structuredMatchRatio(clip, criteria) {
|
|
|
69
71
|
const t = clip.tags;
|
|
70
72
|
let hit = 0;
|
|
71
73
|
const anyOf = (have, want) => Boolean(want && want.length && want.some((w) => have.includes(w)));
|
|
74
|
+
if (anyOf(t.content_type ?? [], criteria.content_type))
|
|
75
|
+
hit++;
|
|
72
76
|
if (anyOf(t.subject, criteria.subject))
|
|
73
77
|
hit++;
|
|
74
78
|
if (anyOf(t.action, criteria.action))
|
|
@@ -8,39 +8,63 @@
|
|
|
8
8
|
import { mkdirSync, rmSync } from "node:fs";
|
|
9
9
|
import path from "node:path";
|
|
10
10
|
import { extractKeyframes } from "./ffmpeg.js";
|
|
11
|
-
//
|
|
12
|
-
// affordable/latency-bounded. Longer videos
|
|
13
|
-
|
|
11
|
+
// One small keyframe per scene; cap the montage per LLM call so a single request
|
|
12
|
+
// stays affordable/latency-bounded. Longer videos are refined in consecutive
|
|
13
|
+
// batches of this size (see MAX_REFINE_BATCHES) rather than skipping guidance.
|
|
14
|
+
const MAX_SCENES_PER_BATCH = 60;
|
|
15
|
+
// Bound total cost on very long sources: past this many batches we keep the raw
|
|
16
|
+
// segmentation for the whole video (logged) instead of firing dozens of calls.
|
|
17
|
+
const MAX_REFINE_BATCHES = 6;
|
|
14
18
|
export async function refineScenesWithGuidance(input) {
|
|
15
19
|
const { client, videoPath, scenes, guidancePrompt } = input;
|
|
16
20
|
if (!guidancePrompt.trim() || scenes.length <= 1) {
|
|
17
21
|
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0 };
|
|
18
22
|
}
|
|
19
|
-
|
|
23
|
+
const batchCount = Math.ceil(scenes.length / MAX_SCENES_PER_BATCH);
|
|
24
|
+
if (batchCount > MAX_REFINE_BATCHES) {
|
|
20
25
|
return {
|
|
21
26
|
scenes,
|
|
22
27
|
refined: false,
|
|
23
28
|
kept: scenes.length,
|
|
24
29
|
dropped: 0,
|
|
25
30
|
merged: 0,
|
|
26
|
-
note: `too many scenes (${scenes.length} > ${
|
|
31
|
+
note: `too many scenes (${scenes.length} > ${MAX_SCENES_PER_BATCH * MAX_REFINE_BATCHES}) for guided refinement — kept raw segmentation`
|
|
27
32
|
};
|
|
28
33
|
}
|
|
29
34
|
const framesDir = path.join(input.workDir, "refine-frames");
|
|
30
35
|
mkdirSync(framesDir, { recursive: true });
|
|
31
36
|
try {
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
// Decide every scene, batching the LLM calls so a long source still gets
|
|
38
|
+
// guided keep/drop/merge (instead of falling back to raw cuts). Decisions
|
|
39
|
+
// are concatenated in timeline order so merge_with_previous still works
|
|
40
|
+
// across a batch boundary in rebuildScenes.
|
|
41
|
+
const keepDefault = () => ({ keep: true, merge_with_previous: false });
|
|
42
|
+
const decisions = [];
|
|
43
|
+
for (let b = 0; b < batchCount; b++) {
|
|
44
|
+
const batch = scenes.slice(b * MAX_SCENES_PER_BATCH, (b + 1) * MAX_SCENES_PER_BATCH);
|
|
45
|
+
// One small keyframe per candidate scene in this batch, in timeline order.
|
|
46
|
+
const frames = [];
|
|
47
|
+
for (const scene of batch) {
|
|
48
|
+
const [p] = await extractKeyframes({ videoPath, scene, outDir: framesDir, count: 1, width: 200 });
|
|
49
|
+
if (p)
|
|
50
|
+
frames.push({ index: scene.index, path: p });
|
|
51
|
+
}
|
|
52
|
+
// Couldn't frame this batch — keep its scenes rather than dropping blind.
|
|
53
|
+
if (frames.length !== batch.length) {
|
|
54
|
+
for (let i = 0; i < batch.length; i++)
|
|
55
|
+
decisions.push(keepDefault());
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
// Divide the overall clip target across batches by their share of scenes.
|
|
59
|
+
const batchTarget = input.targetClipCount && input.targetClipCount > 0
|
|
60
|
+
? Math.max(1, Math.round((input.targetClipCount * batch.length) / scenes.length))
|
|
61
|
+
: undefined;
|
|
62
|
+
const batchDecisions = await askForDecisions(client, guidancePrompt, batch, frames, batchTarget);
|
|
63
|
+
for (let i = 0; i < batch.length; i++) {
|
|
64
|
+
decisions.push(batchDecisions?.[i] ?? keepDefault());
|
|
65
|
+
}
|
|
38
66
|
}
|
|
39
|
-
if (
|
|
40
|
-
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "keyframe extraction incomplete" };
|
|
41
|
-
}
|
|
42
|
-
const decisions = await askForDecisions(client, guidancePrompt, scenes, frames);
|
|
43
|
-
if (!decisions || decisions.length === 0) {
|
|
67
|
+
if (decisions.length === 0) {
|
|
44
68
|
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "model returned no plan" };
|
|
45
69
|
}
|
|
46
70
|
return rebuildScenes(scenes, decisions);
|
|
@@ -57,20 +81,26 @@ export async function refineScenesWithGuidance(input) {
|
|
|
57
81
|
}
|
|
58
82
|
}
|
|
59
83
|
}
|
|
60
|
-
async function askForDecisions(client, guidancePrompt, scenes, frames) {
|
|
84
|
+
async function askForDecisions(client, guidancePrompt, scenes, frames, targetClipCount) {
|
|
61
85
|
const listing = scenes
|
|
62
86
|
.map((s, i) => `Scene ${i}: ${s.start_time_sec.toFixed(1)}s–${s.end_time_sec.toFixed(1)}s (${s.duration_sec.toFixed(1)}s)`)
|
|
63
87
|
.join("\n");
|
|
88
|
+
const target = targetClipCount && targetClipCount > 0 ? Math.round(targetClipCount) : null;
|
|
64
89
|
const prompt = [
|
|
65
|
-
`You are curating clips from a long video for this goal: "${guidancePrompt.trim()}".`,
|
|
90
|
+
`You are curating short clips from a long video for this exact goal: "${guidancePrompt.trim()}".`,
|
|
66
91
|
`There are ${scenes.length} candidate scenes, shown as ${scenes.length} keyframes below in timeline order (one per scene).`,
|
|
67
92
|
listing,
|
|
68
93
|
"",
|
|
69
94
|
"For EACH scene (in order) decide:",
|
|
70
|
-
"- keep: true if the scene
|
|
95
|
+
"- keep: true ONLY if the scene clearly and directly serves the goal above; when in doubt, drop it (keep:false). Follow the goal STRICTLY — do not keep loosely-related or generic filler.",
|
|
71
96
|
"- merge_with_previous: true if this scene continues the previous KEPT scene and they should be ONE clip (e.g. a single continuous action split by a minor cut). Otherwise false.",
|
|
72
97
|
"",
|
|
73
|
-
|
|
98
|
+
"UNIQUENESS (important): the output must be DISTINCT clips — no near-duplicates. If several scenes show essentially the same shot (same subject, framing, action, and setting), KEEP ONLY THE SINGLE BEST ONE and set keep:false for the visually-redundant rest. Prefer variety across the kept clips.",
|
|
99
|
+
target
|
|
100
|
+
? `Aim for roughly ${target} kept clip${target === 1 ? "" : "s"} total (a few more or fewer is fine) — keep the strongest, most distinct scenes and drop the weakest/most repetitive.`
|
|
101
|
+
: "Be selective — dropping off-topic and duplicate scenes and merging fragmented ones is the point.",
|
|
102
|
+
"",
|
|
103
|
+
`Return exactly ${scenes.length} decisions, in the same order as the scenes.`
|
|
74
104
|
].join("\n");
|
|
75
105
|
// The client owns the model call (API providers via structured output, the
|
|
76
106
|
// local devcli agent via CLI) — refine only builds the prompt and rebuilds
|
|
@@ -9,7 +9,7 @@ import { createIdV7 } from "../../lib/ids.js";
|
|
|
9
9
|
import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
|
|
10
10
|
import { buildClipEmbeddingText } from "./gemini.js";
|
|
11
11
|
import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
|
|
12
|
-
import { cropFilterFor, fitScenesToDurationBand, normalizeWindows } from "./hunt.js";
|
|
12
|
+
import { capScenesEvenlyToCount, cropFilterFor, fitScenesToDurationBand, normalizeWindows } from "./hunt.js";
|
|
13
13
|
import { refineScenesWithGuidance } from "./refine.js";
|
|
14
14
|
/** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
|
|
15
15
|
export async function processSceneToClip(input) {
|
|
@@ -114,20 +114,27 @@ export async function scanVideo(opts) {
|
|
|
114
114
|
...opts.sceneOptions
|
|
115
115
|
}), band);
|
|
116
116
|
// Prompt-guided segmentation: let the guidance reshape which scenes become
|
|
117
|
-
// clips (keep/drop/merge) before we spend tagging tokens on them.
|
|
117
|
+
// clips (keep/drop/merge/dedup) before we spend tagging tokens on them.
|
|
118
118
|
if (opts.ctx.guidancePrompt?.trim() && opts.ctx.refineSegmentation !== false) {
|
|
119
119
|
const refined = await refineScenesWithGuidance({
|
|
120
120
|
client: opts.ctx.client,
|
|
121
121
|
videoPath: opts.videoPath,
|
|
122
122
|
scenes,
|
|
123
123
|
guidancePrompt: opts.ctx.guidancePrompt,
|
|
124
|
-
workDir: opts.ctx.workDir
|
|
124
|
+
workDir: opts.ctx.workDir,
|
|
125
|
+
targetClipCount: opts.targetClipCount
|
|
125
126
|
});
|
|
126
127
|
// Refine merges can exceed the band max — re-enforce it (fit never merges
|
|
127
128
|
// across the gaps refine's drops created, only contiguous spans).
|
|
128
129
|
scenes = fitScenesToDurationBand(refined.scenes, band);
|
|
129
130
|
opts.onRefine?.(refined);
|
|
130
131
|
}
|
|
132
|
+
// Deterministic count/uniqueness cap: if more scenes survived than the target
|
|
133
|
+
// (refine skipped >60 scenes, or the model kept more than asked), keep an
|
|
134
|
+
// evenly-spaced — hence less duplicative — subset.
|
|
135
|
+
if (opts.targetClipCount && opts.targetClipCount > 0) {
|
|
136
|
+
scenes = capScenesEvenlyToCount(scenes, opts.targetClipCount);
|
|
137
|
+
}
|
|
131
138
|
const sourceVideoId = opts.sourceVideoId ?? createIdV7("srcvid");
|
|
132
139
|
const sourceFilename = opts.sourceFilename ?? path.basename(opts.videoPath);
|
|
133
140
|
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
@@ -16,6 +16,7 @@ export function getTaxonomy(version = ACTIVE_TAXONOMY_VERSION) {
|
|
|
16
16
|
}
|
|
17
17
|
/** The multi-valued categorical fields (everything except `energy`, which is single). */
|
|
18
18
|
export const MULTI_VALUE_CATEGORIES = [
|
|
19
|
+
"content_type",
|
|
19
20
|
"subject",
|
|
20
21
|
"action",
|
|
21
22
|
"emotion",
|
|
@@ -27,7 +28,7 @@ export const MULTI_VALUE_CATEGORIES = [
|
|
|
27
28
|
/**
|
|
28
29
|
* Clamp an LLM-produced tag object to the controlled vocabulary: drop any value
|
|
29
30
|
* not in the taxonomy so a hallucinated tag can never poison the index. Returns
|
|
30
|
-
* a fully-populated
|
|
31
|
+
* a fully-populated RawTags-shaped object (missing categories become []).
|
|
31
32
|
*/
|
|
32
33
|
export function sanitizeTagValues(raw, version = ACTIVE_TAXONOMY_VERSION) {
|
|
33
34
|
const tax = getTaxonomy(version);
|
|
@@ -49,6 +50,7 @@ export function sanitizeTagValues(raw, version = ACTIVE_TAXONOMY_VERSION) {
|
|
|
49
50
|
return allowed.includes(s) ? s : fallback;
|
|
50
51
|
};
|
|
51
52
|
return {
|
|
53
|
+
content_type: clampList(raw.content_type, cats.content_type),
|
|
52
54
|
subject: clampList(raw.subject, cats.subject),
|
|
53
55
|
action: clampList(raw.action, cats.action),
|
|
54
56
|
emotion: clampList(raw.emotion, cats.emotion),
|
|
@@ -1,7 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": "v1",
|
|
3
|
-
"notes": "Controlled vocabulary for VidFarm clip curation. Start small (~50 tags); expand based on real usage. Clips store the version they were tagged against so the vocab can evolve without breaking existing clips.",
|
|
3
|
+
"notes": "Controlled vocabulary for VidFarm clip curation. Start small (~50 tags); expand based on real usage. Clips store the version they were tagged against so the vocab can evolve without breaking existing clips. content_type was added additively to v1 (2026-07-08) — clips tagged before then simply carry no content_type until re-scanned.",
|
|
4
4
|
"categories": {
|
|
5
|
+
"content_type": [
|
|
6
|
+
"talking_head",
|
|
7
|
+
"b_roll",
|
|
8
|
+
"product_shot",
|
|
9
|
+
"screen_recording",
|
|
10
|
+
"demo",
|
|
11
|
+
"reaction",
|
|
12
|
+
"interview",
|
|
13
|
+
"establishing",
|
|
14
|
+
"lifestyle",
|
|
15
|
+
"text_graphic"
|
|
16
|
+
],
|
|
5
17
|
"subject": [
|
|
6
18
|
"person",
|
|
7
19
|
"single_speaker",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// entity_type tag, local-dynamo shim in `serve` mode). Same Clip shape as the
|
|
4
4
|
// devcli SQLite store so users move between local and cloud without lock-in.
|
|
5
5
|
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
|
|
6
|
-
import { DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, QueryCommand } from "@aws-sdk/lib-dynamodb";
|
|
6
|
+
import { DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, QueryCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb";
|
|
7
7
|
import { config } from "../config.js";
|
|
8
8
|
import { createLocalDynamoClient } from "./local-dynamo.js";
|
|
9
9
|
import { BUILTIN_PRESETS } from "./clip-curation/index.js";
|
|
@@ -102,6 +102,19 @@ export class ClipRecordsService {
|
|
|
102
102
|
async deleteClip(owner, clipId) {
|
|
103
103
|
await dynamodb.send(new DeleteCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: clipSk(clipId) } }));
|
|
104
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* Move a clip between virtual Raws folders. Partial update (doesn't rewrite
|
|
107
|
+
* the embedding vector). `folderPath` is stored normalized; "" = raws root.
|
|
108
|
+
*/
|
|
109
|
+
async updateClipFolder(owner, clipId, folderPath) {
|
|
110
|
+
await dynamodb.send(new UpdateCommand({
|
|
111
|
+
TableName: requireMainTableName(),
|
|
112
|
+
Key: { pk: clipPk(owner), sk: clipSk(clipId) },
|
|
113
|
+
UpdateExpression: "SET folder_path = :fp",
|
|
114
|
+
ConditionExpression: "attribute_exists(pk)",
|
|
115
|
+
ExpressionAttributeValues: { ":fp": folderPath }
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
105
118
|
// ── Presets ──────────────────────────────────────────────────────────────
|
|
106
119
|
async listPresets(owner) {
|
|
107
120
|
const res = await dynamodb.send(new QueryCommand({
|
|
@@ -8,21 +8,51 @@ const vectorIndex = resolveClipVectorIndex((owner) => clipRecords.listClipsWithE
|
|
|
8
8
|
export async function searchUserClips(owner, input) {
|
|
9
9
|
const limit = input.limit ?? 20;
|
|
10
10
|
const all = await clipRecords.listClips(owner, { limit: 5000 });
|
|
11
|
-
|
|
12
|
-
if (
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
return rankHits(hits, limit);
|
|
11
|
+
// ── No semantic signal → pure structured (hard) filter. ──────────────────
|
|
12
|
+
if (!input.queryEmbedding) {
|
|
13
|
+
const structured = all.filter((c) => matchesCriteria(c, input.criteria));
|
|
14
|
+
return rankHits(structured.map((c) => scoreClip(c, input.criteria, undefined, false)), limit);
|
|
16
15
|
}
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
16
|
+
// ── Semantic query → SEMANTIC-FIRST hybrid. ──────────────────────────────
|
|
17
|
+
// The old flow hard-AND-filtered by the (often auto-derived) categorical
|
|
18
|
+
// criteria BEFORE looking at embeddings, so the true nearest-neighbor clips
|
|
19
|
+
// were silently excluded whenever the NL→criteria step over-constrained —
|
|
20
|
+
// the "search doesn't work" symptom. Now: retrieve the nearest neighbors over
|
|
21
|
+
// the WHOLE library from the vector index, and only UNION in exact structured
|
|
22
|
+
// matches when the criteria actually constrains something. Everything is
|
|
23
|
+
// scored with the soft structured boost (`relaxed`), so an exact facet match
|
|
24
|
+
// floats a clip up without ever gating a strong semantic match out.
|
|
25
|
+
const knn = await vectorIndex.query(owner, input.queryEmbedding, Math.max(limit * 5, 50));
|
|
26
|
+
const byId = new Map(all.map((c) => [c.clip_id, c]));
|
|
27
|
+
const pool = new Map();
|
|
28
|
+
for (const h of knn) {
|
|
29
|
+
const c = byId.get(h.clip_id);
|
|
30
|
+
if (c)
|
|
31
|
+
pool.set(c.clip_id, c);
|
|
24
32
|
}
|
|
25
|
-
|
|
33
|
+
if (hasStructuredConstraints(input.criteria)) {
|
|
34
|
+
for (const c of all)
|
|
35
|
+
if (matchesCriteria(c, input.criteria))
|
|
36
|
+
pool.set(c.clip_id, c);
|
|
37
|
+
}
|
|
38
|
+
// Fallback: an empty vector index (e.g. clips scanned without an embedding
|
|
39
|
+
// provider) yields no neighbors — degrade to the whole library so keyword /
|
|
40
|
+
// structured scoring still returns something rather than nothing.
|
|
41
|
+
const candidates = pool.size > 0 ? [...pool.values()] : all;
|
|
42
|
+
const hits = candidates.map((c) => scoreClip(c, input.criteria, input.queryEmbedding, true));
|
|
43
|
+
return rankHits(hits, limit);
|
|
44
|
+
}
|
|
45
|
+
/** True when the criteria constrains any categorical/duration/boolean facet (not just free-text). */
|
|
46
|
+
function hasStructuredConstraints(criteria) {
|
|
47
|
+
for (const [key, value] of Object.entries(criteria)) {
|
|
48
|
+
if (key === "semantic_text")
|
|
49
|
+
continue;
|
|
50
|
+
if (value == null)
|
|
51
|
+
continue;
|
|
52
|
+
if (Array.isArray(value) ? value.length > 0 : true)
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
return false;
|
|
26
56
|
}
|
|
27
57
|
export { vectorIndex };
|
|
28
58
|
/**
|