@mevdragon/vidfarm-devcli 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/demo/dist/app.js +49 -49
- package/dist/src/app.js +863 -2
- package/dist/src/cli.js +59 -3
- package/dist/src/config.js +6 -0
- package/dist/src/devcli/clip-store.js +335 -0
- package/dist/src/devcli/clips.js +803 -0
- package/dist/src/frontend/homepage-view.js +5 -0
- package/dist/src/services/clip-curation/cost.js +112 -0
- package/dist/src/services/clip-curation/ffmpeg.js +258 -0
- package/dist/src/services/clip-curation/gemini.js +342 -0
- package/dist/src/services/clip-curation/index.js +15 -0
- package/dist/src/services/clip-curation/presets.js +20 -0
- package/dist/src/services/clip-curation/presets.v1.json +59 -0
- package/dist/src/services/clip-curation/query.js +163 -0
- package/dist/src/services/clip-curation/refine.js +136 -0
- package/dist/src/services/clip-curation/scan.js +137 -0
- package/dist/src/services/clip-curation/taxonomy.js +71 -0
- package/dist/src/services/clip-curation/taxonomy.v1.json +90 -0
- package/dist/src/services/clip-curation/types.js +7 -0
- package/dist/src/services/clip-records.js +209 -0
- package/dist/src/services/clip-search.js +47 -0
- package/dist/src/services/clip-vectors.js +125 -0
- package/dist/src/services/hyperframes.js +347 -0
- package/dist/src/services/scene-annotations.js +32 -0
- package/package.json +5 -1
- package/public/assets/homepage-client-app.js +17 -17
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Prompt-guided segmentation. ffmpeg gives us candidate scene cuts by visual
|
|
2
|
+
// change; this optional pass lets a user's guidance prompt actually reshape HOW
|
|
3
|
+
// the video is clipped — one multimodal LLM call sees a keyframe per candidate
|
|
4
|
+
// scene and decides, per scene, keep/drop (is it relevant to the goal?) and
|
|
5
|
+
// merge-with-previous (does it continue the prior kept scene as one clip?). We
|
|
6
|
+
// then rebuild the scene list from those decisions. Falls back to the raw scenes
|
|
7
|
+
// on any mismatch/error so a scan never breaks.
|
|
8
|
+
import { mkdirSync, rmSync } from "node:fs";
|
|
9
|
+
import { readFile } from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { generateObject } from "ai";
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
import { extractKeyframes } from "./ffmpeg.js";
|
|
14
|
+
// Sending one small keyframe per scene; cap the montage so a single call stays
|
|
15
|
+
// affordable/latency-bounded. Longer videos keep their raw segmentation (logged).
|
|
16
|
+
const MAX_SCENES_FOR_REFINE = 60;
|
|
17
|
+
export async function refineScenesWithGuidance(input) {
|
|
18
|
+
const { client, videoPath, scenes, guidancePrompt } = input;
|
|
19
|
+
if (!guidancePrompt.trim() || scenes.length <= 1) {
|
|
20
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0 };
|
|
21
|
+
}
|
|
22
|
+
if (scenes.length > MAX_SCENES_FOR_REFINE) {
|
|
23
|
+
return {
|
|
24
|
+
scenes,
|
|
25
|
+
refined: false,
|
|
26
|
+
kept: scenes.length,
|
|
27
|
+
dropped: 0,
|
|
28
|
+
merged: 0,
|
|
29
|
+
note: `too many scenes (${scenes.length} > ${MAX_SCENES_FOR_REFINE}) for guided refinement — kept raw segmentation`
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const framesDir = path.join(input.workDir, "refine-frames");
|
|
33
|
+
mkdirSync(framesDir, { recursive: true });
|
|
34
|
+
try {
|
|
35
|
+
// One small keyframe per candidate scene, in timeline order.
|
|
36
|
+
const frames = [];
|
|
37
|
+
for (const scene of scenes) {
|
|
38
|
+
const [p] = await extractKeyframes({ videoPath, scene, outDir: framesDir, count: 1, width: 200 });
|
|
39
|
+
if (p)
|
|
40
|
+
frames.push({ index: scene.index, path: p });
|
|
41
|
+
}
|
|
42
|
+
if (frames.length !== scenes.length) {
|
|
43
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "keyframe extraction incomplete" };
|
|
44
|
+
}
|
|
45
|
+
const decisions = await askForDecisions(client, guidancePrompt, scenes, frames);
|
|
46
|
+
if (!decisions || decisions.length === 0) {
|
|
47
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "model returned no plan" };
|
|
48
|
+
}
|
|
49
|
+
return rebuildScenes(scenes, decisions);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "refinement failed; using raw scenes" };
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
try {
|
|
56
|
+
rmSync(framesDir, { recursive: true, force: true });
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* ignore */
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async function askForDecisions(client, guidancePrompt, scenes, frames) {
|
|
64
|
+
const images = await Promise.all(frames.map((f) => readFile(f.path)));
|
|
65
|
+
const listing = scenes
|
|
66
|
+
.map((s, i) => `Scene ${i}: ${s.start_time_sec.toFixed(1)}s–${s.end_time_sec.toFixed(1)}s (${s.duration_sec.toFixed(1)}s)`)
|
|
67
|
+
.join("\n");
|
|
68
|
+
const prompt = [
|
|
69
|
+
`You are curating clips from a long video for this goal: "${guidancePrompt.trim()}".`,
|
|
70
|
+
`There are ${scenes.length} candidate scenes, shown as ${scenes.length} keyframes below in timeline order (one per scene).`,
|
|
71
|
+
listing,
|
|
72
|
+
"",
|
|
73
|
+
"For EACH scene (in order) decide:",
|
|
74
|
+
"- keep: true if the scene is relevant/useful for the goal, false to drop it.",
|
|
75
|
+
"- merge_with_previous: true if this scene continues the previous KEPT scene and they should be ONE clip (e.g. a single continuous action split by a minor cut). Otherwise false.",
|
|
76
|
+
"",
|
|
77
|
+
`Return exactly ${scenes.length} decisions, in the same order as the scenes. Be selective — dropping off-topic scenes and merging fragmented ones is the point.`
|
|
78
|
+
].join("\n");
|
|
79
|
+
// No fixed-length constraint — Gemini's structured-output schema translation
|
|
80
|
+
// rejects minItems==maxItems; we align by index instead (rebuildScenes).
|
|
81
|
+
const schema = z.object({
|
|
82
|
+
decisions: z.array(z.object({ keep: z.boolean(), merge_with_previous: z.boolean() }))
|
|
83
|
+
});
|
|
84
|
+
try {
|
|
85
|
+
const { object } = await generateObject({
|
|
86
|
+
model: client.taggingModel,
|
|
87
|
+
schema,
|
|
88
|
+
maxOutputTokens: Math.max(1000, scenes.length * 40),
|
|
89
|
+
messages: [
|
|
90
|
+
{
|
|
91
|
+
role: "user",
|
|
92
|
+
content: [{ type: "text", text: prompt }, ...images.map((image) => ({ type: "image", image }))]
|
|
93
|
+
}
|
|
94
|
+
]
|
|
95
|
+
});
|
|
96
|
+
return object.decisions;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
if (process.env.VIDFARM_DEBUG_LOGS === "true") {
|
|
100
|
+
// eslint-disable-next-line no-console
|
|
101
|
+
console.warn(`[clips] refine call failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
102
|
+
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function rebuildScenes(scenes, decisions) {
|
|
107
|
+
const out = [];
|
|
108
|
+
let dropped = 0;
|
|
109
|
+
let merged = 0;
|
|
110
|
+
for (let i = 0; i < scenes.length; i++) {
|
|
111
|
+
// Align by index; a missing decision (short array) defaults to keep.
|
|
112
|
+
const d = decisions[i] ?? { keep: true, merge_with_previous: false };
|
|
113
|
+
if (!d.keep) {
|
|
114
|
+
dropped++;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (d.merge_with_previous && out.length > 0) {
|
|
118
|
+
// Extend the previous kept clip to cover this scene.
|
|
119
|
+
const prev = out[out.length - 1];
|
|
120
|
+
prev.end_time_sec = scenes[i].end_time_sec;
|
|
121
|
+
prev.duration_sec = round3(prev.end_time_sec - prev.start_time_sec);
|
|
122
|
+
merged++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
out.push({ ...scenes[i], index: out.length });
|
|
126
|
+
}
|
|
127
|
+
// Never return an empty plan — if the model dropped everything, keep the raw scenes.
|
|
128
|
+
if (out.length === 0) {
|
|
129
|
+
return { scenes, refined: false, kept: scenes.length, dropped: 0, merged: 0, note: "model dropped every scene; using raw scenes" };
|
|
130
|
+
}
|
|
131
|
+
return { scenes: out, refined: true, kept: out.length, dropped, merged };
|
|
132
|
+
}
|
|
133
|
+
function round3(n) {
|
|
134
|
+
return Math.round(n * 1000) / 1000;
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=refine.js.map
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// Scan orchestration shared by devcli and cloud. `processSceneToClip` turns ONE
|
|
2
|
+
// detected scene into a full Clip record (extract clip/keyframes/thumb/audio →
|
|
3
|
+
// Gemini tag+transcribe → embed). devcli calls it in a bounded pool; a cloud
|
|
4
|
+
// Step Functions Map state calls it once per scene. Persistence is the caller's
|
|
5
|
+
// job (SQLite vs DynamoDB+S3) — this stays storage-agnostic.
|
|
6
|
+
import { mkdirSync } from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { createIdV7 } from "../../lib/ids.js";
|
|
9
|
+
import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
|
|
10
|
+
import { buildClipEmbeddingText } from "./gemini.js";
|
|
11
|
+
import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
|
|
12
|
+
import { refineScenesWithGuidance } from "./refine.js";
|
|
13
|
+
/** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
|
|
14
|
+
export async function processSceneToClip(input) {
|
|
15
|
+
const { ctx, videoPath, scene } = input;
|
|
16
|
+
const framesDir = path.join(ctx.workDir, "frames");
|
|
17
|
+
const audioDir = path.join(ctx.workDir, "audio");
|
|
18
|
+
mkdirSync(ctx.clipsDir, { recursive: true });
|
|
19
|
+
mkdirSync(ctx.thumbsDir, { recursive: true });
|
|
20
|
+
const clipId = createIdV7("clip");
|
|
21
|
+
// 1. Keyframes for visual tagging (small, low-res).
|
|
22
|
+
const keyframePaths = await extractKeyframes({
|
|
23
|
+
videoPath,
|
|
24
|
+
scene,
|
|
25
|
+
outDir: framesDir,
|
|
26
|
+
count: ctx.framesPerScene ?? 2
|
|
27
|
+
});
|
|
28
|
+
// 2. Per-scene audio for ASR (one combined tag+transcribe call).
|
|
29
|
+
let audioPath;
|
|
30
|
+
if (ctx.includeAudio ?? true) {
|
|
31
|
+
const a = await extractSceneAudio({
|
|
32
|
+
videoPath,
|
|
33
|
+
scene,
|
|
34
|
+
outPath: path.join(audioDir, `scene-${scene.index}.mp3`)
|
|
35
|
+
});
|
|
36
|
+
audioPath = a ?? undefined;
|
|
37
|
+
}
|
|
38
|
+
// 3. Tag (+ transcription when audio present on Gemini), biased by the optional guidance prompt.
|
|
39
|
+
const tagResult = await ctx.client.tagScene({
|
|
40
|
+
keyframePaths,
|
|
41
|
+
transcript: "",
|
|
42
|
+
audioPath,
|
|
43
|
+
scene,
|
|
44
|
+
guidancePrompt: ctx.guidancePrompt
|
|
45
|
+
});
|
|
46
|
+
// 4. Cut the clip + thumbnail.
|
|
47
|
+
const clipLocal = path.join(ctx.clipsDir, `${clipId}.mp4`);
|
|
48
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clipId}.jpg`);
|
|
49
|
+
const clipOk = await extractClip({ videoPath, scene, outPath: clipLocal, reencode: ctx.reencodeClips !== false });
|
|
50
|
+
if (!clipOk)
|
|
51
|
+
throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s`);
|
|
52
|
+
await extractThumbnail({ videoPath, scene, outPath: thumbLocal });
|
|
53
|
+
// 5. Embed the document text for semantic search (skipped when the client can't embed).
|
|
54
|
+
let embedding;
|
|
55
|
+
if (ctx.client.canEmbed) {
|
|
56
|
+
const embedText = buildClipEmbeddingText(tagResult.tags, tagResult.description);
|
|
57
|
+
[embedding] = await ctx.client.embedDocuments([embedText]);
|
|
58
|
+
}
|
|
59
|
+
const clip = {
|
|
60
|
+
clip_id: clipId,
|
|
61
|
+
source_video_id: input.sourceVideoId,
|
|
62
|
+
source_filename: input.sourceFilename,
|
|
63
|
+
start_time_sec: scene.start_time_sec,
|
|
64
|
+
end_time_sec: scene.end_time_sec,
|
|
65
|
+
duration_sec: scene.duration_sec,
|
|
66
|
+
file_path: ctx.clipStoredPath ? ctx.clipStoredPath(clipId, clipLocal) : clipLocal,
|
|
67
|
+
thumbnail_path: ctx.thumbStoredPath ? ctx.thumbStoredPath(clipId, thumbLocal) : thumbLocal,
|
|
68
|
+
tags: tagResult.tags,
|
|
69
|
+
description: tagResult.description,
|
|
70
|
+
embedding,
|
|
71
|
+
taxonomy_version: ctx.client.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION,
|
|
72
|
+
tier: ctx.client.tier,
|
|
73
|
+
provider: ctx.client.provider,
|
|
74
|
+
embedding_model: ctx.client.embeddingModelId ?? undefined,
|
|
75
|
+
owner_id: input.ownerId,
|
|
76
|
+
created_at: new Date().toISOString()
|
|
77
|
+
};
|
|
78
|
+
return clip;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Run the full local scan pipeline for one video with a bounded concurrency
|
|
82
|
+
* pool. (Cloud replaces this loop with a Step Functions Map over the scenes.)
|
|
83
|
+
*/
|
|
84
|
+
export async function scanVideo(opts) {
|
|
85
|
+
const probe = opts.probe ?? (await probeVideo(opts.videoPath));
|
|
86
|
+
let scenes = opts.scenes ??
|
|
87
|
+
(await detectScenes(opts.videoPath, {
|
|
88
|
+
durationSec: probe.duration_sec,
|
|
89
|
+
...opts.sceneOptions
|
|
90
|
+
}));
|
|
91
|
+
// Prompt-guided segmentation: let the guidance reshape which scenes become
|
|
92
|
+
// clips (keep/drop/merge) before we spend tagging tokens on them.
|
|
93
|
+
if (opts.ctx.guidancePrompt?.trim() && opts.ctx.refineSegmentation !== false) {
|
|
94
|
+
const refined = await refineScenesWithGuidance({
|
|
95
|
+
client: opts.ctx.client,
|
|
96
|
+
videoPath: opts.videoPath,
|
|
97
|
+
scenes,
|
|
98
|
+
guidancePrompt: opts.ctx.guidancePrompt,
|
|
99
|
+
workDir: opts.ctx.workDir
|
|
100
|
+
});
|
|
101
|
+
scenes = refined.scenes;
|
|
102
|
+
opts.onRefine?.(refined);
|
|
103
|
+
}
|
|
104
|
+
const sourceVideoId = opts.sourceVideoId ?? createIdV7("srcvid");
|
|
105
|
+
const sourceFilename = opts.sourceFilename ?? path.basename(opts.videoPath);
|
|
106
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
107
|
+
const clips = [];
|
|
108
|
+
let cursor = 0;
|
|
109
|
+
let completed = 0;
|
|
110
|
+
async function worker() {
|
|
111
|
+
while (cursor < scenes.length) {
|
|
112
|
+
const idx = cursor++;
|
|
113
|
+
const scene = scenes[idx];
|
|
114
|
+
try {
|
|
115
|
+
const clip = await processSceneToClip({
|
|
116
|
+
ctx: opts.ctx,
|
|
117
|
+
videoPath: opts.videoPath,
|
|
118
|
+
scene,
|
|
119
|
+
sourceVideoId,
|
|
120
|
+
sourceFilename,
|
|
121
|
+
ownerId: opts.ownerId
|
|
122
|
+
});
|
|
123
|
+
clips.push(clip);
|
|
124
|
+
completed++;
|
|
125
|
+
await opts.onClip?.(clip, completed, scenes.length);
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
opts.onError?.(scene, error);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, scenes.length) }, () => worker()));
|
|
133
|
+
// Preserve timeline order for a stable library.
|
|
134
|
+
clips.sort((a, b) => a.start_time_sec - b.start_time_sec);
|
|
135
|
+
return { source_video_id: sourceVideoId, probe, scenes, clips };
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=scan.js.map
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Versioned tag taxonomy loader. The vocabulary lives in taxonomy.v1.json so it
|
|
2
|
+
// can evolve independently; clips record the version they were tagged with
|
|
3
|
+
// (Clip.taxonomy_version) so old clips stay valid when the vocab grows.
|
|
4
|
+
import taxonomyV1 from "./taxonomy.v1.json" with { type: "json" };
|
|
5
|
+
const REGISTRY = {
|
|
6
|
+
v1: taxonomyV1
|
|
7
|
+
};
|
|
8
|
+
export const ACTIVE_TAXONOMY_VERSION = "v1";
|
|
9
|
+
/** Get a taxonomy by version (defaults to the active one). */
|
|
10
|
+
export function getTaxonomy(version = ACTIVE_TAXONOMY_VERSION) {
|
|
11
|
+
const found = REGISTRY[version];
|
|
12
|
+
if (!found) {
|
|
13
|
+
throw new Error(`Unknown taxonomy version "${version}" (have: ${Object.keys(REGISTRY).join(", ")})`);
|
|
14
|
+
}
|
|
15
|
+
return found;
|
|
16
|
+
}
|
|
17
|
+
/** The multi-valued categorical fields (everything except `energy`, which is single). */
|
|
18
|
+
export const MULTI_VALUE_CATEGORIES = [
|
|
19
|
+
"subject",
|
|
20
|
+
"action",
|
|
21
|
+
"emotion",
|
|
22
|
+
"setting",
|
|
23
|
+
"composition",
|
|
24
|
+
"motion",
|
|
25
|
+
"audio_type"
|
|
26
|
+
];
|
|
27
|
+
/**
|
|
28
|
+
* Clamp an LLM-produced tag object to the controlled vocabulary: drop any value
|
|
29
|
+
* not in the taxonomy so a hallucinated tag can never poison the index. Returns
|
|
30
|
+
* a fully-populated ClipTags-shaped object (missing categories become []).
|
|
31
|
+
*/
|
|
32
|
+
export function sanitizeTagValues(raw, version = ACTIVE_TAXONOMY_VERSION) {
|
|
33
|
+
const tax = getTaxonomy(version);
|
|
34
|
+
const cats = tax.categories;
|
|
35
|
+
const clampList = (values, allowed) => {
|
|
36
|
+
if (!Array.isArray(values))
|
|
37
|
+
return [];
|
|
38
|
+
const set = new Set(allowed);
|
|
39
|
+
const out = [];
|
|
40
|
+
for (const v of values) {
|
|
41
|
+
const s = String(v).trim().toLowerCase().replace(/\s+/g, "_");
|
|
42
|
+
if (set.has(s) && !out.includes(s))
|
|
43
|
+
out.push(s);
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
};
|
|
47
|
+
const clampOne = (value, allowed, fallback) => {
|
|
48
|
+
const s = String(value ?? "").trim().toLowerCase().replace(/\s+/g, "_");
|
|
49
|
+
return allowed.includes(s) ? s : fallback;
|
|
50
|
+
};
|
|
51
|
+
return {
|
|
52
|
+
subject: clampList(raw.subject, cats.subject),
|
|
53
|
+
action: clampList(raw.action, cats.action),
|
|
54
|
+
emotion: clampList(raw.emotion, cats.emotion),
|
|
55
|
+
setting: clampList(raw.setting, cats.setting),
|
|
56
|
+
composition: clampList(raw.composition, cats.composition),
|
|
57
|
+
motion: clampList(raw.motion, cats.motion),
|
|
58
|
+
energy: clampOne(raw.energy, cats.energy, "medium"),
|
|
59
|
+
audio_type: clampList(raw.audio_type, cats.audio_type)
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/** A compact, prompt-friendly rendering of the vocabulary to inline in the tagging prompt. */
|
|
63
|
+
export function taxonomyForPrompt(version = ACTIVE_TAXONOMY_VERSION) {
|
|
64
|
+
const tax = getTaxonomy(version);
|
|
65
|
+
const lines = [];
|
|
66
|
+
for (const [cat, values] of Object.entries(tax.categories)) {
|
|
67
|
+
lines.push(`- ${cat}: ${values.join(", ")}`);
|
|
68
|
+
}
|
|
69
|
+
return lines.join("\n");
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=taxonomy.js.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1",
|
|
3
|
+
"notes": "Controlled vocabulary for VidFarm clip curation. Start small (~50 tags); expand based on real usage. Clips store the version they were tagged against so the vocab can evolve without breaking existing clips.",
|
|
4
|
+
"categories": {
|
|
5
|
+
"subject": [
|
|
6
|
+
"person",
|
|
7
|
+
"single_speaker",
|
|
8
|
+
"multiple_people",
|
|
9
|
+
"hands",
|
|
10
|
+
"product",
|
|
11
|
+
"screen_ui",
|
|
12
|
+
"text_on_screen",
|
|
13
|
+
"animal",
|
|
14
|
+
"food",
|
|
15
|
+
"vehicle",
|
|
16
|
+
"no_subject"
|
|
17
|
+
],
|
|
18
|
+
"action": [
|
|
19
|
+
"talking",
|
|
20
|
+
"laughing",
|
|
21
|
+
"looking_at_camera",
|
|
22
|
+
"reacting",
|
|
23
|
+
"pointing",
|
|
24
|
+
"typing",
|
|
25
|
+
"hands_on_keyboard",
|
|
26
|
+
"walking",
|
|
27
|
+
"eating",
|
|
28
|
+
"holding_product",
|
|
29
|
+
"unboxing",
|
|
30
|
+
"gesturing",
|
|
31
|
+
"still"
|
|
32
|
+
],
|
|
33
|
+
"emotion": [
|
|
34
|
+
"amused",
|
|
35
|
+
"surprised",
|
|
36
|
+
"excited",
|
|
37
|
+
"confused",
|
|
38
|
+
"deadpan",
|
|
39
|
+
"exasperated",
|
|
40
|
+
"serious",
|
|
41
|
+
"neutral",
|
|
42
|
+
"sad",
|
|
43
|
+
"angry"
|
|
44
|
+
],
|
|
45
|
+
"setting": [
|
|
46
|
+
"indoor",
|
|
47
|
+
"outdoor",
|
|
48
|
+
"office",
|
|
49
|
+
"home",
|
|
50
|
+
"studio",
|
|
51
|
+
"kitchen",
|
|
52
|
+
"car",
|
|
53
|
+
"nature",
|
|
54
|
+
"urban",
|
|
55
|
+
"abstract_or_graphic"
|
|
56
|
+
],
|
|
57
|
+
"composition": [
|
|
58
|
+
"close_up",
|
|
59
|
+
"medium_shot",
|
|
60
|
+
"wide_shot",
|
|
61
|
+
"centered",
|
|
62
|
+
"off_center",
|
|
63
|
+
"over_the_shoulder",
|
|
64
|
+
"top_down",
|
|
65
|
+
"product_shot"
|
|
66
|
+
],
|
|
67
|
+
"motion": [
|
|
68
|
+
"static",
|
|
69
|
+
"low",
|
|
70
|
+
"medium",
|
|
71
|
+
"high",
|
|
72
|
+
"camera_shake",
|
|
73
|
+
"pan_or_tilt",
|
|
74
|
+
"zoom"
|
|
75
|
+
],
|
|
76
|
+
"energy": [
|
|
77
|
+
"low",
|
|
78
|
+
"medium",
|
|
79
|
+
"high"
|
|
80
|
+
],
|
|
81
|
+
"audio_type": [
|
|
82
|
+
"dialogue",
|
|
83
|
+
"voiceover",
|
|
84
|
+
"music",
|
|
85
|
+
"sfx",
|
|
86
|
+
"ambient",
|
|
87
|
+
"silence"
|
|
88
|
+
]
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Shared data model for VidFarm clip curation — the "third pillar" library
|
|
2
|
+
// (viral DNA + HTML formats + CLIPS). One scanned scene becomes one Clip
|
|
3
|
+
// record. This shape is IDENTICAL across the two deployment targets
|
|
4
|
+
// (devcli/local SQLite and AWS DynamoDB) so users can move between local and
|
|
5
|
+
// cloud without lock-in. See drafts/clip-hunting/vidfarm_clip_curation_handoff.md.
|
|
6
|
+
export {};
|
|
7
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// Cloud clip records — DynamoDB single-table store for clip curation, mirroring
|
|
2
|
+
// the serverless-records / serverless-jobs conventions (pk/sk + gsi1 overload,
|
|
3
|
+
// entity_type tag, local-dynamo shim in `serve` mode). Same Clip shape as the
|
|
4
|
+
// devcli SQLite store so users move between local and cloud without lock-in.
|
|
5
|
+
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
|
|
6
|
+
import { DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, QueryCommand } from "@aws-sdk/lib-dynamodb";
|
|
7
|
+
import { config } from "../config.js";
|
|
8
|
+
import { createLocalDynamoClient } from "./local-dynamo.js";
|
|
9
|
+
import { BUILTIN_PRESETS } from "./clip-curation/index.js";
|
|
10
|
+
// Same client construction as serverless-records.ts: disk shim locally, real
|
|
11
|
+
// client when deployed. Call sites are identical either way.
|
|
12
|
+
const dynamodb = config.RECORDS_DRIVER === "local"
|
|
13
|
+
? createLocalDynamoClient()
|
|
14
|
+
: DynamoDBDocumentClient.from(new DynamoDBClient({}), {
|
|
15
|
+
marshallOptions: { removeUndefinedValues: true }
|
|
16
|
+
});
|
|
17
|
+
function requireMainTableName() {
|
|
18
|
+
if (config.RECORDS_DRIVER === "local") {
|
|
19
|
+
return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME || "vidfarm-local-main";
|
|
20
|
+
}
|
|
21
|
+
if (!config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME) {
|
|
22
|
+
throw new Error("Clip record store requires VIDFARM_SERVERLESS_MAIN_TABLE_NAME.");
|
|
23
|
+
}
|
|
24
|
+
return config.VIDFARM_SERVERLESS_MAIN_TABLE_NAME;
|
|
25
|
+
}
|
|
26
|
+
const ENTITY_CLIP = "clip_curation_clip";
|
|
27
|
+
const ENTITY_PRESET = "clip_curation_preset";
|
|
28
|
+
const ENTITY_SOURCE = "clip_curation_source";
|
|
29
|
+
const ENTITY_SCAN = "clip_curation_scan";
|
|
30
|
+
// Key scheme (one owner partition, sk-prefixed by entity; gsi1 lists by recency).
|
|
31
|
+
const clipPk = (owner) => `USER#${owner}`;
|
|
32
|
+
const clipSk = (clipId) => `CLIP#${clipId}`;
|
|
33
|
+
const clipListPk = (owner) => `USER#${owner}#CLIPS`;
|
|
34
|
+
const clipSourceListPk = (owner, sourceId) => `USER#${owner}#CLIPS#SRC#${sourceId}`;
|
|
35
|
+
const presetSk = (presetId) => `CLIPPRESET#${presetId}`;
|
|
36
|
+
const presetListPk = (owner) => `USER#${owner}#CLIPPRESETS`;
|
|
37
|
+
const sourceSk = (sourceId) => `CLIPSRC#${sourceId}`;
|
|
38
|
+
const sourceListPk = (owner) => `USER#${owner}#CLIPSRCS`;
|
|
39
|
+
const scanSk = (scanId) => `CLIPSCAN#${scanId}`;
|
|
40
|
+
export class ClipRecordsService {
|
|
41
|
+
// ── Clips ────────────────────────────────────────────────────────────────
|
|
42
|
+
async putClip(clip) {
|
|
43
|
+
const owner = requireOwner(clip.owner_id);
|
|
44
|
+
await dynamodb.send(new PutCommand({
|
|
45
|
+
TableName: requireMainTableName(),
|
|
46
|
+
Item: {
|
|
47
|
+
pk: clipPk(owner),
|
|
48
|
+
sk: clipSk(clip.clip_id),
|
|
49
|
+
entity_type: ENTITY_CLIP,
|
|
50
|
+
gsi1pk: clipListPk(owner),
|
|
51
|
+
gsi1sk: `${clip.created_at}#${clip.clip_id}`,
|
|
52
|
+
gsi2pk: clipSourceListPk(owner, clip.source_video_id),
|
|
53
|
+
gsi2sk: `${clip.start_time_sec.toFixed(3)}#${clip.clip_id}`,
|
|
54
|
+
...clip
|
|
55
|
+
}
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
async getClip(owner, clipId) {
|
|
59
|
+
const res = await dynamodb.send(new GetCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: clipSk(clipId) } }));
|
|
60
|
+
return res.Item ? stripKeys(res.Item) : null;
|
|
61
|
+
}
|
|
62
|
+
/** List a user's clips (newest first), optionally scoped to a source video. */
|
|
63
|
+
async listClips(owner, opts = {}) {
|
|
64
|
+
const limit = opts.limit ?? 200;
|
|
65
|
+
if (opts.sourceVideoId) {
|
|
66
|
+
const res = await dynamodb.send(new QueryCommand({
|
|
67
|
+
TableName: requireMainTableName(),
|
|
68
|
+
IndexName: "gsi2",
|
|
69
|
+
KeyConditionExpression: "gsi2pk = :pk",
|
|
70
|
+
ExpressionAttributeValues: { ":pk": clipSourceListPk(owner, opts.sourceVideoId) },
|
|
71
|
+
Limit: limit
|
|
72
|
+
}));
|
|
73
|
+
return (res.Items ?? []).map(stripKeys);
|
|
74
|
+
}
|
|
75
|
+
const res = await dynamodb.send(new QueryCommand({
|
|
76
|
+
TableName: requireMainTableName(),
|
|
77
|
+
IndexName: "gsi1",
|
|
78
|
+
KeyConditionExpression: "gsi1pk = :pk",
|
|
79
|
+
ExpressionAttributeValues: { ":pk": clipListPk(owner) },
|
|
80
|
+
ScanIndexForward: false,
|
|
81
|
+
Limit: limit
|
|
82
|
+
}));
|
|
83
|
+
return (res.Items ?? []).map(stripKeys);
|
|
84
|
+
}
|
|
85
|
+
/** All embeddings for a user's clips — feeds the brute-force vector search. */
|
|
86
|
+
async listClipsWithEmbeddings(owner) {
|
|
87
|
+
const clips = await this.listClips(owner, { limit: 5000 });
|
|
88
|
+
return clips.filter((c) => Array.isArray(c.embedding) && c.embedding.length > 0);
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The embedding model the library was built with (from the newest embedded
|
|
92
|
+
* clip). A match query must be embedded with the SAME model to be comparable.
|
|
93
|
+
*/
|
|
94
|
+
async getLibraryEmbeddingModel(owner) {
|
|
95
|
+
const clips = await this.listClips(owner, { limit: 20 });
|
|
96
|
+
for (const c of clips) {
|
|
97
|
+
if (c.embedding_model && Array.isArray(c.embedding) && c.embedding.length > 0)
|
|
98
|
+
return c.embedding_model;
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
async deleteClip(owner, clipId) {
|
|
103
|
+
await dynamodb.send(new DeleteCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: clipSk(clipId) } }));
|
|
104
|
+
}
|
|
105
|
+
// ── Presets ──────────────────────────────────────────────────────────────
|
|
106
|
+
async listPresets(owner) {
|
|
107
|
+
const res = await dynamodb.send(new QueryCommand({
|
|
108
|
+
TableName: requireMainTableName(),
|
|
109
|
+
KeyConditionExpression: "pk = :pk AND begins_with(sk, :sk)",
|
|
110
|
+
ExpressionAttributeValues: { ":pk": clipPk(owner), ":sk": "CLIPPRESET#" }
|
|
111
|
+
}));
|
|
112
|
+
const saved = (res.Items ?? []).map((i) => stripKeys(i));
|
|
113
|
+
return [...BUILTIN_PRESETS, ...saved];
|
|
114
|
+
}
|
|
115
|
+
async putPreset(owner, preset) {
|
|
116
|
+
await dynamodb.send(new PutCommand({
|
|
117
|
+
TableName: requireMainTableName(),
|
|
118
|
+
Item: {
|
|
119
|
+
pk: clipPk(owner),
|
|
120
|
+
sk: presetSk(preset.preset_id),
|
|
121
|
+
entity_type: ENTITY_PRESET,
|
|
122
|
+
gsi1pk: presetListPk(owner),
|
|
123
|
+
gsi1sk: preset.created_at,
|
|
124
|
+
...preset,
|
|
125
|
+
owner_id: owner
|
|
126
|
+
}
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
async deletePreset(owner, presetId) {
|
|
130
|
+
await dynamodb.send(new DeleteCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: presetSk(presetId) } }));
|
|
131
|
+
}
|
|
132
|
+
// ── Source videos ──────────────────────────────────────────────────────────
|
|
133
|
+
async putSource(record) {
|
|
134
|
+
await dynamodb.send(new PutCommand({
|
|
135
|
+
TableName: requireMainTableName(),
|
|
136
|
+
Item: {
|
|
137
|
+
pk: clipPk(record.owner_id),
|
|
138
|
+
sk: sourceSk(record.source_video_id),
|
|
139
|
+
entity_type: ENTITY_SOURCE,
|
|
140
|
+
gsi1pk: sourceListPk(record.owner_id),
|
|
141
|
+
gsi1sk: record.created_at,
|
|
142
|
+
...record
|
|
143
|
+
}
|
|
144
|
+
}));
|
|
145
|
+
}
|
|
146
|
+
async getSource(owner, sourceId) {
|
|
147
|
+
const res = await dynamodb.send(new GetCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: sourceSk(sourceId) } }));
|
|
148
|
+
return res.Item ? stripKeys(res.Item) : null;
|
|
149
|
+
}
|
|
150
|
+
async listSources(owner) {
|
|
151
|
+
const res = await dynamodb.send(new QueryCommand({
|
|
152
|
+
TableName: requireMainTableName(),
|
|
153
|
+
IndexName: "gsi1",
|
|
154
|
+
KeyConditionExpression: "gsi1pk = :pk",
|
|
155
|
+
ExpressionAttributeValues: { ":pk": sourceListPk(owner) },
|
|
156
|
+
ScanIndexForward: false
|
|
157
|
+
}));
|
|
158
|
+
return (res.Items ?? []).map((i) => stripKeys(i));
|
|
159
|
+
}
|
|
160
|
+
// ── Scan jobs ──────────────────────────────────────────────────────────────
|
|
161
|
+
async putScan(record) {
|
|
162
|
+
await dynamodb.send(new PutCommand({
|
|
163
|
+
TableName: requireMainTableName(),
|
|
164
|
+
Item: {
|
|
165
|
+
pk: clipPk(record.owner_id),
|
|
166
|
+
sk: scanSk(record.scan_id),
|
|
167
|
+
entity_type: ENTITY_SCAN,
|
|
168
|
+
...record
|
|
169
|
+
}
|
|
170
|
+
}));
|
|
171
|
+
}
|
|
172
|
+
async getScan(owner, scanId) {
|
|
173
|
+
const res = await dynamodb.send(new GetCommand({ TableName: requireMainTableName(), Key: { pk: clipPk(owner), sk: scanSk(scanId) } }));
|
|
174
|
+
return res.Item ? stripKeys(res.Item) : null;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
export const clipRecords = new ClipRecordsService();
|
|
178
|
+
/** Build a user-saved preset record from raw criteria. */
|
|
179
|
+
export function makeUserPreset(input) {
|
|
180
|
+
return {
|
|
181
|
+
preset_id: `preset_${Date.now().toString(36)}_${Math.floor(Math.random() * 1e6).toString(36)}`,
|
|
182
|
+
name: input.name,
|
|
183
|
+
description: input.description,
|
|
184
|
+
criteria: input.criteria,
|
|
185
|
+
builtin: false,
|
|
186
|
+
owner_id: input.owner,
|
|
187
|
+
created_at: new Date().toISOString()
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
function requireOwner(owner) {
|
|
191
|
+
if (!owner)
|
|
192
|
+
throw new Error("Clip records require an owner_id.");
|
|
193
|
+
return owner;
|
|
194
|
+
}
|
|
195
|
+
// Drop the DynamoDB key/index attributes so callers get back a clean record.
|
|
196
|
+
function stripKeys(item) {
|
|
197
|
+
const { pk, sk, entity_type, gsi1pk, gsi1sk, gsi2pk, gsi2sk, gsi3pk, gsi3sk, ...rest } = item;
|
|
198
|
+
void pk;
|
|
199
|
+
void sk;
|
|
200
|
+
void entity_type;
|
|
201
|
+
void gsi1pk;
|
|
202
|
+
void gsi1sk;
|
|
203
|
+
void gsi2pk;
|
|
204
|
+
void gsi2sk;
|
|
205
|
+
void gsi3pk;
|
|
206
|
+
void gsi3sk;
|
|
207
|
+
return rest;
|
|
208
|
+
}
|
|
209
|
+
//# sourceMappingURL=clip-records.js.map
|