@mevdragon/vidfarm-devcli 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.director.md +43 -10
- package/SKILL.platform.md +5 -3
- package/demo/dist/app.js +107 -57
- package/dist/src/app.js +1115 -17
- package/dist/src/cli.js +304 -43
- package/dist/src/composition-runtime.js +26 -0
- package/dist/src/config.js +6 -0
- package/dist/src/devcli/clip-store.js +335 -0
- package/dist/src/devcli/clips.js +803 -0
- package/dist/src/devcli/composition-edit.js +12 -0
- package/dist/src/editor-chat.js +1 -0
- package/dist/src/frontend/homepage-client.js +61 -6
- package/dist/src/frontend/homepage-view.js +22 -4
- package/dist/src/homepage.js +46 -0
- package/dist/src/hyperframes/composition.js +87 -0
- package/dist/src/primitive-registry.js +136 -0
- package/dist/src/services/billing.js +1 -0
- package/dist/src/services/clip-curation/cost.js +112 -0
- package/dist/src/services/clip-curation/ffmpeg.js +258 -0
- package/dist/src/services/clip-curation/gemini.js +342 -0
- package/dist/src/services/clip-curation/index.js +15 -0
- package/dist/src/services/clip-curation/presets.js +20 -0
- package/dist/src/services/clip-curation/presets.v1.json +59 -0
- package/dist/src/services/clip-curation/query.js +163 -0
- package/dist/src/services/clip-curation/refine.js +136 -0
- package/dist/src/services/clip-curation/scan.js +137 -0
- package/dist/src/services/clip-curation/taxonomy.js +71 -0
- package/dist/src/services/clip-curation/taxonomy.v1.json +90 -0
- package/dist/src/services/clip-curation/types.js +7 -0
- package/dist/src/services/clip-records.js +209 -0
- package/dist/src/services/clip-search.js +47 -0
- package/dist/src/services/clip-vectors.js +125 -0
- package/dist/src/services/hyperframes.js +373 -1
- package/dist/src/services/scene-annotations.js +32 -0
- package/dist/src/services/storage.js +6 -0
- package/package.json +5 -1
- package/public/assets/homepage-client-app.js +18 -18
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
// Model client shared by devcli + cloud. Multi-provider: the TAGGING model
|
|
2
|
+
// (vision + optional audio ASR) is swappable across gemini / openai / openrouter
|
|
3
|
+
// so users pick their cost/quality tradeoff; EMBEDDINGS ride on a resolvable
|
|
4
|
+
// embedding provider (gemini or openai — OpenRouter has no /embeddings) so a
|
|
5
|
+
// library stays in ONE vector space. BYOK: the caller passes provider keys.
|
|
6
|
+
//
|
|
7
|
+
// One place for the prompt templates so every provider produces byte-identical
|
|
8
|
+
// clip metadata (same taxonomy, same schema).
|
|
9
|
+
import { readFile } from "node:fs/promises";
|
|
10
|
+
import { embedMany, generateObject } from "ai";
|
|
11
|
+
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
12
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
13
|
+
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
14
|
+
import { z } from "zod";
|
|
15
|
+
import { ACTIVE_TAXONOMY_VERSION, getTaxonomy, sanitizeTagValues, taxonomyForPrompt } from "./taxonomy.js";
|
|
16
|
+
// Tag model per provider/tier. Overridable via env so pinned models can move
|
|
17
|
+
// forward without a code change. Handoff: gemini flash-lite (cheap, low media_resolution).
|
|
18
|
+
function resolveTagModel(provider, tier) {
|
|
19
|
+
switch (provider) {
|
|
20
|
+
case "openai":
|
|
21
|
+
return process.env.VIDFARM_CLIP_TAG_MODEL_OPENAI ?? "gpt-5.4-nano";
|
|
22
|
+
case "openrouter":
|
|
23
|
+
return process.env.VIDFARM_CLIP_TAG_MODEL_OPENROUTER ?? "google/gemini-2.5-flash";
|
|
24
|
+
case "gemini":
|
|
25
|
+
default:
|
|
26
|
+
return tier === "flash"
|
|
27
|
+
? process.env.VIDFARM_CLIP_TAG_MODEL_FLASH ?? "gemini-2.5-flash"
|
|
28
|
+
: process.env.VIDFARM_CLIP_TAG_MODEL_LITE ?? "gemini-2.5-flash-lite";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export const EMBEDDING_MODELS = {
|
|
32
|
+
gemini: process.env.VIDFARM_CLIP_EMBEDDING_MODEL ?? "gemini-embedding-001",
|
|
33
|
+
openai: process.env.VIDFARM_CLIP_EMBEDDING_MODEL_OPENAI ?? "text-embedding-3-small"
|
|
34
|
+
};
|
|
35
|
+
// gemini-embedding-001 and text-embedding-3-small both support reducing to 768
|
|
36
|
+
// dims, so every provider's vectors fit the same fixed-width index. Cosine is
|
|
37
|
+
// dimension-agnostic; still, vectors from DIFFERENT models are NOT comparable —
|
|
38
|
+
// a library should stay on one embedding model (callers store embedding_model
|
|
39
|
+
// and warn on mismatch).
|
|
40
|
+
export const EMBEDDING_DIM = Number(process.env.VIDFARM_CLIP_EMBEDDING_DIM ?? 768);
|
|
41
|
+
export class ClipModelClient {
|
|
42
|
+
provider;
|
|
43
|
+
tier;
|
|
44
|
+
taxonomyVersion;
|
|
45
|
+
tagModelId;
|
|
46
|
+
embeddingProvider;
|
|
47
|
+
embeddingModelId;
|
|
48
|
+
tagModel;
|
|
49
|
+
embedModel;
|
|
50
|
+
constructor(opts) {
|
|
51
|
+
if (!opts.apiKey)
|
|
52
|
+
throw new Error(`A ${opts.provider ?? "gemini"} API key is required.`);
|
|
53
|
+
this.provider = opts.provider ?? "gemini";
|
|
54
|
+
this.tier = opts.tier ?? "flash-lite";
|
|
55
|
+
this.taxonomyVersion = opts.taxonomyVersion ?? ACTIVE_TAXONOMY_VERSION;
|
|
56
|
+
this.tagModelId = resolveTagModel(this.provider, this.tier);
|
|
57
|
+
this.tagModel = buildLanguageModel(this.provider, opts.apiKey, this.tagModelId, opts.publicBaseUrl);
|
|
58
|
+
const embedding = resolveEmbeddingConfig(this.provider, opts.apiKey, opts.embedding);
|
|
59
|
+
if (embedding) {
|
|
60
|
+
this.embeddingProvider = embedding.provider;
|
|
61
|
+
this.embeddingModelId = EMBEDDING_MODELS[embedding.provider];
|
|
62
|
+
this.embedModel = buildEmbeddingModel(embedding.provider, embedding.apiKey, this.embeddingModelId);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
this.embeddingProvider = null;
|
|
66
|
+
this.embeddingModelId = null;
|
|
67
|
+
this.embedModel = null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** True when this client can produce embeddings (semantic search available). */
|
|
71
|
+
get canEmbed() {
|
|
72
|
+
return this.embedModel !== null;
|
|
73
|
+
}
|
|
74
|
+
/** The underlying tagging LanguageModel — used by the guided-segmentation pass. */
|
|
75
|
+
get taggingModel() {
|
|
76
|
+
return this.tagModel;
|
|
77
|
+
}
|
|
78
|
+
tagSchema() {
|
|
79
|
+
const cats = getTaxonomy(this.taxonomyVersion).categories;
|
|
80
|
+
const enumArray = (values) => z.array(z.enum(values)).default([]);
|
|
81
|
+
return z.object({
|
|
82
|
+
subject: enumArray(cats.subject),
|
|
83
|
+
action: enumArray(cats.action),
|
|
84
|
+
emotion: enumArray(cats.emotion),
|
|
85
|
+
setting: enumArray(cats.setting),
|
|
86
|
+
composition: enumArray(cats.composition),
|
|
87
|
+
motion: enumArray(cats.motion),
|
|
88
|
+
energy: z.enum(cats.energy),
|
|
89
|
+
audio_type: enumArray(cats.audio_type),
|
|
90
|
+
transcript: z.string().default(""),
|
|
91
|
+
description: z.string()
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Tag one scene from its keyframes (+ audio on Gemini, which transcribes in
|
|
96
|
+
* the same call). OpenAI/OpenRouter get frames only — their chat models don't
|
|
97
|
+
* take raw audio, so transcript stays empty unless supplied. Degrades to an
|
|
98
|
+
* empty-but-valid tag set on error so one bad scene never kills a scan.
|
|
99
|
+
*/
|
|
100
|
+
async tagScene(input) {
|
|
101
|
+
const supportsAudio = this.provider === "gemini";
|
|
102
|
+
const hasAudio = supportsAudio && Boolean(input.audioPath);
|
|
103
|
+
const prompt = buildTaggingPrompt(input.transcript, this.taxonomyVersion, hasAudio, input.guidancePrompt);
|
|
104
|
+
const images = await Promise.all(input.keyframePaths.map((p) => readFile(p)));
|
|
105
|
+
const audio = hasAudio && input.audioPath ? await readFile(input.audioPath).catch(() => null) : null;
|
|
106
|
+
try {
|
|
107
|
+
const { object } = await generateObject({
|
|
108
|
+
model: this.tagModel,
|
|
109
|
+
schema: this.tagSchema(),
|
|
110
|
+
maxOutputTokens: this.provider === "openai" ? 1500 : 500,
|
|
111
|
+
providerOptions: this.taggingProviderOptions(),
|
|
112
|
+
messages: [
|
|
113
|
+
{
|
|
114
|
+
role: "user",
|
|
115
|
+
content: [
|
|
116
|
+
{ type: "text", text: prompt },
|
|
117
|
+
...images.map((image) => this.imagePart(image)),
|
|
118
|
+
...(audio ? [{ type: "file", data: audio, mediaType: "audio/mpeg" }] : [])
|
|
119
|
+
]
|
|
120
|
+
}
|
|
121
|
+
]
|
|
122
|
+
});
|
|
123
|
+
const sanitized = sanitizeTagValues(object, this.taxonomyVersion);
|
|
124
|
+
const tags = {
|
|
125
|
+
...sanitized,
|
|
126
|
+
transcript: (input.transcript || object.transcript || "").trim()
|
|
127
|
+
};
|
|
128
|
+
return { tags, description: String(object.description ?? "").trim() };
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
132
|
+
// eslint-disable-next-line no-console
|
|
133
|
+
console.warn(`[clips] tag failed (${this.provider}) @${input.scene.start_time_sec.toFixed(1)}s: ${message}`);
|
|
134
|
+
return {
|
|
135
|
+
tags: {
|
|
136
|
+
subject: [],
|
|
137
|
+
action: [],
|
|
138
|
+
emotion: [],
|
|
139
|
+
setting: [],
|
|
140
|
+
composition: [],
|
|
141
|
+
motion: [],
|
|
142
|
+
energy: "medium",
|
|
143
|
+
audio_type: input.transcript ? ["dialogue"] : ["silence"],
|
|
144
|
+
transcript: input.transcript || ""
|
|
145
|
+
},
|
|
146
|
+
description: input.transcript ? input.transcript.slice(0, 120) : ""
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
taggingProviderOptions() {
|
|
151
|
+
if (this.provider === "gemini") {
|
|
152
|
+
return { google: { mediaResolution: "MEDIA_RESOLUTION_LOW", structuredOutputs: true } };
|
|
153
|
+
}
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
imagePart(image) {
|
|
157
|
+
// Low image detail on OpenAI keeps the token cost near the Gemini low-res target.
|
|
158
|
+
if (this.provider === "openai") {
|
|
159
|
+
return { type: "image", image, providerOptions: { openai: { imageDetail: "low" } } };
|
|
160
|
+
}
|
|
161
|
+
return { type: "image", image };
|
|
162
|
+
}
|
|
163
|
+
async embedDocuments(texts) {
|
|
164
|
+
return this.embed(texts, "RETRIEVAL_DOCUMENT");
|
|
165
|
+
}
|
|
166
|
+
async embedQuery(text) {
|
|
167
|
+
const [vec] = await this.embed([text], "RETRIEVAL_QUERY");
|
|
168
|
+
return vec;
|
|
169
|
+
}
|
|
170
|
+
/** Batch-embed many retrieval queries (e.g. one per scene annotation) in a single call. */
|
|
171
|
+
async embedQueries(texts) {
|
|
172
|
+
return this.embed(texts, "RETRIEVAL_QUERY");
|
|
173
|
+
}
|
|
174
|
+
async embed(texts, taskType) {
|
|
175
|
+
if (texts.length === 0 || !this.embedModel)
|
|
176
|
+
return [];
|
|
177
|
+
const providerOptions = this.embeddingProvider === "gemini"
|
|
178
|
+
? { google: { taskType, outputDimensionality: EMBEDDING_DIM } }
|
|
179
|
+
: { openai: { dimensions: EMBEDDING_DIM } };
|
|
180
|
+
const { embeddings } = await embedMany({
|
|
181
|
+
model: this.embedModel,
|
|
182
|
+
values: texts.map((t) => (t.trim() ? t.trim() : " ")),
|
|
183
|
+
providerOptions
|
|
184
|
+
});
|
|
185
|
+
return embeddings.map((e) => normalize(e));
|
|
186
|
+
}
|
|
187
|
+
/** Convert a natural-language query into a structured SplicingCriteria (categoricals constrained to the taxonomy). */
|
|
188
|
+
async naturalLanguageToCriteria(query) {
|
|
189
|
+
const cats = getTaxonomy(this.taxonomyVersion).categories;
|
|
190
|
+
const opt = (values) => z.array(z.enum(values)).optional();
|
|
191
|
+
const schema = z.object({
|
|
192
|
+
subject: opt(cats.subject),
|
|
193
|
+
action: opt(cats.action),
|
|
194
|
+
emotion: opt(cats.emotion),
|
|
195
|
+
setting: opt(cats.setting),
|
|
196
|
+
composition: opt(cats.composition),
|
|
197
|
+
motion: opt(cats.motion),
|
|
198
|
+
audio_type: opt(cats.audio_type),
|
|
199
|
+
energy: z.array(z.enum(cats.energy)).optional(),
|
|
200
|
+
min_duration_sec: z.number().optional(),
|
|
201
|
+
max_duration_sec: z.number().optional(),
|
|
202
|
+
has_dialogue: z.boolean().optional(),
|
|
203
|
+
has_subject: z.boolean().optional()
|
|
204
|
+
});
|
|
205
|
+
try {
|
|
206
|
+
const { object } = await generateObject({
|
|
207
|
+
model: this.tagModel,
|
|
208
|
+
schema,
|
|
209
|
+
maxOutputTokens: this.provider === "openai" ? 1200 : 300,
|
|
210
|
+
prompt: buildQueryPrompt(query, this.taxonomyVersion)
|
|
211
|
+
});
|
|
212
|
+
return { ...pruneUndefined(object), semantic_text: query };
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return { semantic_text: query };
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/** Back-compat alias — the client used to be Gemini-only. */
|
|
220
|
+
export const ClipGeminiClient = ClipModelClient;
|
|
221
|
+
function buildLanguageModel(provider, apiKey, modelId, publicBaseUrl) {
|
|
222
|
+
switch (provider) {
|
|
223
|
+
case "openai":
|
|
224
|
+
return createOpenAI({ apiKey })(modelId);
|
|
225
|
+
case "openrouter":
|
|
226
|
+
return createOpenAICompatible({
|
|
227
|
+
name: "openrouter",
|
|
228
|
+
apiKey,
|
|
229
|
+
baseURL: "https://openrouter.ai/api/v1",
|
|
230
|
+
headers: { "HTTP-Referer": publicBaseUrl ?? "https://vidfarm.cc", "X-Title": "Vidfarm Clips" }
|
|
231
|
+
}).chatModel(modelId);
|
|
232
|
+
case "gemini":
|
|
233
|
+
default:
|
|
234
|
+
return createGoogleGenerativeAI({ apiKey })(modelId);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
function buildEmbeddingModel(provider, apiKey, modelId) {
|
|
238
|
+
return provider === "openai"
|
|
239
|
+
? createOpenAI({ apiKey }).textEmbeddingModel(modelId)
|
|
240
|
+
: createGoogleGenerativeAI({ apiKey }).textEmbeddingModel(modelId);
|
|
241
|
+
}
|
|
242
|
+
function resolveEmbeddingConfig(provider, apiKey, explicit) {
|
|
243
|
+
if (explicit === null)
|
|
244
|
+
return null; // caller explicitly disabled embeddings
|
|
245
|
+
if (explicit)
|
|
246
|
+
return explicit;
|
|
247
|
+
if (provider === "gemini")
|
|
248
|
+
return { provider: "gemini", apiKey };
|
|
249
|
+
if (provider === "openai")
|
|
250
|
+
return { provider: "openai", apiKey };
|
|
251
|
+
return null; // openrouter: no embeddings unless an explicit config is given
|
|
252
|
+
}
|
|
253
|
+
/** The document text embedded for a clip: description + transcript + salient tags. */
|
|
254
|
+
export function buildClipEmbeddingText(tags, description) {
|
|
255
|
+
const parts = [];
|
|
256
|
+
if (description)
|
|
257
|
+
parts.push(description);
|
|
258
|
+
if (tags.transcript)
|
|
259
|
+
parts.push(`Transcript: ${tags.transcript}`);
|
|
260
|
+
const tagBits = [
|
|
261
|
+
...tags.subject,
|
|
262
|
+
...tags.action,
|
|
263
|
+
...tags.emotion,
|
|
264
|
+
...tags.setting,
|
|
265
|
+
...tags.composition,
|
|
266
|
+
...tags.motion,
|
|
267
|
+
...tags.audio_type,
|
|
268
|
+
`energy:${tags.energy}`
|
|
269
|
+
];
|
|
270
|
+
if (tagBits.length)
|
|
271
|
+
parts.push(tagBits.join(", "));
|
|
272
|
+
return parts.join("\n").trim() || "video clip";
|
|
273
|
+
}
|
|
274
|
+
function buildTaggingPrompt(transcript, version, hasAudio = false, guidancePrompt) {
|
|
275
|
+
const speech = transcript?.trim()
|
|
276
|
+
? `The scene's spoken audio (ASR transcript) is: "${transcript.trim()}"`
|
|
277
|
+
: hasAudio
|
|
278
|
+
? `An audio clip of the scene is attached — transcribe any speech into the transcript field (use "" if there is no speech).`
|
|
279
|
+
: `There is no clear speech in this scene (no dialogue).`;
|
|
280
|
+
const guidance = guidancePrompt?.trim()
|
|
281
|
+
? `\nThe user is curating clips for this goal: "${guidancePrompt.trim()}". Bias the description and tags toward what matters for that goal (stay accurate — never invent tags not visible in the scene).\n`
|
|
282
|
+
: "";
|
|
283
|
+
return [
|
|
284
|
+
"You are tagging one short video scene for a searchable clip library.",
|
|
285
|
+
"You are given 1-3 keyframes from the scene and its audio transcript.",
|
|
286
|
+
speech,
|
|
287
|
+
guidance,
|
|
288
|
+
"",
|
|
289
|
+
"Return JSON matching the schema. Use ONLY values from this controlled vocabulary:",
|
|
290
|
+
taxonomyForPrompt(version),
|
|
291
|
+
"",
|
|
292
|
+
"Rules:",
|
|
293
|
+
"- Pick only tags you are confident about from the keyframes/audio. Omit a category (empty list) rather than guess.",
|
|
294
|
+
"- If there is no clear person/subject, use subject: [\"no_subject\"].",
|
|
295
|
+
"- If there is no speech, set audio_type appropriately (e.g. music/ambient/silence) and leave transcript empty.",
|
|
296
|
+
"- `energy` is a single value: low | medium | high.",
|
|
297
|
+
"- `transcript` should echo the spoken words verbatim (or \"\" if none).",
|
|
298
|
+
"- `description`: one short factual sentence (<= 20 words) describing what happens — this drives semantic search, so be concrete.",
|
|
299
|
+
"Keep the whole response under ~200 tokens."
|
|
300
|
+
].join("\n");
|
|
301
|
+
}
|
|
302
|
+
function buildQueryPrompt(query, version) {
|
|
303
|
+
return [
|
|
304
|
+
"Convert this clip-search request into a structured filter.",
|
|
305
|
+
`Request: "${query}"`,
|
|
306
|
+
"",
|
|
307
|
+
"Use ONLY values from this controlled vocabulary (omit a field if the request doesn't constrain it):",
|
|
308
|
+
taxonomyForPrompt(version),
|
|
309
|
+
"",
|
|
310
|
+
"Only include fields the request clearly implies. Prefer fewer, high-precision constraints.",
|
|
311
|
+
"Set has_dialogue/has_subject only when the request is explicit about speech or a subject.",
|
|
312
|
+
"Durations are in seconds."
|
|
313
|
+
].join("\n");
|
|
314
|
+
}
|
|
315
|
+
function pruneUndefined(obj) {
|
|
316
|
+
const out = {};
|
|
317
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
318
|
+
if (v === undefined)
|
|
319
|
+
continue;
|
|
320
|
+
if (Array.isArray(v) && v.length === 0)
|
|
321
|
+
continue;
|
|
322
|
+
out[k] = v;
|
|
323
|
+
}
|
|
324
|
+
return out;
|
|
325
|
+
}
|
|
326
|
+
/** L2-normalize a vector so cosine == dot product (and reduced-dim embeddings compare correctly). */
|
|
327
|
+
export function normalize(vec) {
|
|
328
|
+
let sum = 0;
|
|
329
|
+
for (const v of vec)
|
|
330
|
+
sum += v * v;
|
|
331
|
+
const norm = Math.sqrt(sum);
|
|
332
|
+
if (norm === 0)
|
|
333
|
+
return vec;
|
|
334
|
+
return vec.map((v) => v / norm);
|
|
335
|
+
}
|
|
336
|
+
// Legacy export kept for callers that referenced the old constant.
|
|
337
|
+
export const TAG_MODEL_BY_TIER = {
|
|
338
|
+
"flash-lite": resolveTagModel("gemini", "flash-lite"),
|
|
339
|
+
flash: resolveTagModel("gemini", "flash")
|
|
340
|
+
};
|
|
341
|
+
export const EMBEDDING_MODEL = EMBEDDING_MODELS.gemini;
|
|
342
|
+
//# sourceMappingURL=gemini.js.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// vidfarm clip-curation core — the shared "third pillar" library consumed by
|
|
2
|
+
// devcli (local), the Hono API (src/app.ts), and the cloud scan Lambdas. Build
|
|
3
|
+
// once, use in both targets (handoff: Shared Components).
|
|
4
|
+
export * from "./types.js";
|
|
5
|
+
export { ACTIVE_TAXONOMY_VERSION, MULTI_VALUE_CATEGORIES, getTaxonomy, sanitizeTagValues, taxonomyForPrompt } from "./taxonomy.js";
|
|
6
|
+
export { BUILTIN_PRESETS, findBuiltinPreset } from "./presets.js";
|
|
7
|
+
export { ClipModelClient, ClipGeminiClient, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_MODELS, TAG_MODEL_BY_TIER, buildClipEmbeddingText, normalize } from "./gemini.js";
|
|
8
|
+
export { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, hasFfmpeg, probeVideo, resolveFfmpeg, resolveFfprobe } from "./ffmpeg.js";
|
|
9
|
+
export { cosineSimilarity, matchesCriteria, rankHits, scoreClip, searchClips, structuredMatchRatio } from "./query.js";
|
|
10
|
+
export { estimateScanCostFromDuration, estimateScanCostFromScenes, formatCostEstimate } from "./cost.js";
|
|
11
|
+
export { processSceneToClip, scanVideo } from "./scan.js";
|
|
12
|
+
export { refineScenesWithGuidance } from "./refine.js";
|
|
13
|
+
// Re-export ClipPreset-related helpers already covered by ./presets and ./types
|
|
14
|
+
// via the wildcard exports above; nothing extra needed here.
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Starter preset criteria library. Presets are just saved SplicingCriteria
|
|
2
|
+
// (see types.ts). The shipped ones are `builtin: true`; users save their own
|
|
3
|
+
// with `builtin: false` (persisted per-store in devcli/cloud).
|
|
4
|
+
import presetsV1 from "./presets.v1.json" with { type: "json" };
|
|
5
|
+
const SEED = presetsV1;
|
|
6
|
+
/** The shipped built-in presets. Frozen so callers can't mutate the shared array. */
|
|
7
|
+
export const BUILTIN_PRESETS = SEED.presets.map((p) => ({
|
|
8
|
+
preset_id: p.preset_id,
|
|
9
|
+
name: p.name,
|
|
10
|
+
description: p.description,
|
|
11
|
+
criteria: p.criteria,
|
|
12
|
+
builtin: true,
|
|
13
|
+
created_at: "1970-01-01T00:00:00.000Z"
|
|
14
|
+
}));
|
|
15
|
+
/** Look up a built-in preset by its id or (case-insensitive) name. */
|
|
16
|
+
export function findBuiltinPreset(nameOrId) {
|
|
17
|
+
const needle = nameOrId.trim().toLowerCase();
|
|
18
|
+
return BUILTIN_PRESETS.find((p) => p.preset_id.toLowerCase() === needle || p.name.toLowerCase() === needle);
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=presets.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "v1",
|
|
3
|
+
"presets": [
|
|
4
|
+
{
|
|
5
|
+
"preset_id": "preset_funny_reactions",
|
|
6
|
+
"name": "funny reaction gifs",
|
|
7
|
+
"description": "Short single-subject reactions facing the camera — the raw material for reaction cutaways.",
|
|
8
|
+
"criteria": {
|
|
9
|
+
"emotion": ["surprised", "amused", "deadpan", "exasperated"],
|
|
10
|
+
"subject": ["person", "single_speaker"],
|
|
11
|
+
"composition": ["close_up", "medium_shot", "centered"],
|
|
12
|
+
"max_duration_sec": 2.5,
|
|
13
|
+
"has_subject": true,
|
|
14
|
+
"semantic_text": "funny reaction, person reacting to the camera"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"preset_id": "preset_broll_transitions",
|
|
19
|
+
"name": "b-roll transitions",
|
|
20
|
+
"description": "Quiet low-motion cutaways with no talking — good glue between beats.",
|
|
21
|
+
"criteria": {
|
|
22
|
+
"motion": ["static", "low"],
|
|
23
|
+
"max_duration_sec": 1.5,
|
|
24
|
+
"has_dialogue": false,
|
|
25
|
+
"semantic_text": "short b-roll transition, no dialogue"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"preset_id": "preset_establishing_shots",
|
|
30
|
+
"name": "establishing shots",
|
|
31
|
+
"description": "Wide scene-setters with no clear subject.",
|
|
32
|
+
"criteria": {
|
|
33
|
+
"composition": ["wide_shot"],
|
|
34
|
+
"has_subject": false,
|
|
35
|
+
"semantic_text": "wide establishing shot of a location"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"preset_id": "preset_typing_hands",
|
|
40
|
+
"name": "typing hands",
|
|
41
|
+
"description": "Close-ups of hands on a keyboard — classic productivity b-roll.",
|
|
42
|
+
"criteria": {
|
|
43
|
+
"action": ["typing", "hands_on_keyboard"],
|
|
44
|
+
"composition": ["close_up"],
|
|
45
|
+
"semantic_text": "close up of hands typing on a keyboard"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"preset_id": "preset_money_shots",
|
|
50
|
+
"name": "money shots",
|
|
51
|
+
"description": "High-energy hero shots — the payoff frames.",
|
|
52
|
+
"criteria": {
|
|
53
|
+
"composition": ["close_up", "medium_shot"],
|
|
54
|
+
"energy": ["high"],
|
|
55
|
+
"semantic_text": "high energy hero product money shot"
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Structured filter + hybrid semantic re-rank. Same logic on devcli and cloud
|
|
2
|
+
// so a preset or NL query returns the same clips regardless of target.
|
|
3
|
+
//
|
|
4
|
+
// Search flow (handoff): structured filter FIRST (cheap, exact), then semantic
|
|
5
|
+
// re-rank the survivors by embedding similarity. If the structured filter
|
|
6
|
+
// matches nothing but the query carries semantic intent, we relax to a
|
|
7
|
+
// semantic-only pass so natural-language queries still return their best guess.
|
|
8
|
+
const DIALOGUE_AUDIO = new Set(["dialogue", "voiceover"]);
|
|
9
|
+
/** How many categorical facets `criteria` constrains (used for the match-ratio soft score). */
|
|
10
|
+
function constraintCount(criteria) {
|
|
11
|
+
let n = 0;
|
|
12
|
+
for (const key of ["subject", "action", "emotion", "setting", "composition", "motion", "audio_type", "energy"]) {
|
|
13
|
+
if (Array.isArray(criteria[key]) && criteria[key].length)
|
|
14
|
+
n++;
|
|
15
|
+
}
|
|
16
|
+
if (criteria.min_duration_sec != null || criteria.max_duration_sec != null)
|
|
17
|
+
n++;
|
|
18
|
+
if (criteria.has_dialogue != null)
|
|
19
|
+
n++;
|
|
20
|
+
if (criteria.has_subject != null)
|
|
21
|
+
n++;
|
|
22
|
+
return n;
|
|
23
|
+
}
|
|
24
|
+
function hasSubject(tags) {
|
|
25
|
+
return tags.subject.length > 0 && !(tags.subject.length === 1 && tags.subject[0] === "no_subject");
|
|
26
|
+
}
|
|
27
|
+
function hasDialogue(tags) {
|
|
28
|
+
return tags.audio_type.some((a) => DIALOGUE_AUDIO.has(a)) || Boolean(tags.transcript.trim());
|
|
29
|
+
}
|
|
30
|
+
/** Hard structured filter: does this clip satisfy every constraint in `criteria`? */
|
|
31
|
+
export function matchesCriteria(clip, criteria) {
|
|
32
|
+
const t = clip.tags;
|
|
33
|
+
const anyOf = (have, want) => !want || want.length === 0 || want.some((w) => have.includes(w));
|
|
34
|
+
if (!anyOf(t.subject, criteria.subject))
|
|
35
|
+
return false;
|
|
36
|
+
if (!anyOf(t.action, criteria.action))
|
|
37
|
+
return false;
|
|
38
|
+
if (!anyOf(t.emotion, criteria.emotion))
|
|
39
|
+
return false;
|
|
40
|
+
if (!anyOf(t.setting, criteria.setting))
|
|
41
|
+
return false;
|
|
42
|
+
if (!anyOf(t.composition, criteria.composition))
|
|
43
|
+
return false;
|
|
44
|
+
if (!anyOf(t.motion, criteria.motion))
|
|
45
|
+
return false;
|
|
46
|
+
if (!anyOf(t.audio_type, criteria.audio_type))
|
|
47
|
+
return false;
|
|
48
|
+
if (criteria.energy && criteria.energy.length && !criteria.energy.includes(t.energy))
|
|
49
|
+
return false;
|
|
50
|
+
if (criteria.min_duration_sec != null && clip.duration_sec < criteria.min_duration_sec)
|
|
51
|
+
return false;
|
|
52
|
+
if (criteria.max_duration_sec != null && clip.duration_sec > criteria.max_duration_sec)
|
|
53
|
+
return false;
|
|
54
|
+
if (criteria.has_dialogue === true && !hasDialogue(t))
|
|
55
|
+
return false;
|
|
56
|
+
if (criteria.has_dialogue === false && hasDialogue(t))
|
|
57
|
+
return false;
|
|
58
|
+
if (criteria.has_subject === true && !hasSubject(t))
|
|
59
|
+
return false;
|
|
60
|
+
if (criteria.has_subject === false && hasSubject(t))
|
|
61
|
+
return false;
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
/** Fraction of constrained facets this clip satisfies (0..1); 1 when there are no constraints. */
|
|
65
|
+
export function structuredMatchRatio(clip, criteria) {
|
|
66
|
+
const total = constraintCount(criteria);
|
|
67
|
+
if (total === 0)
|
|
68
|
+
return 1;
|
|
69
|
+
const t = clip.tags;
|
|
70
|
+
let hit = 0;
|
|
71
|
+
const anyOf = (have, want) => Boolean(want && want.length && want.some((w) => have.includes(w)));
|
|
72
|
+
if (anyOf(t.subject, criteria.subject))
|
|
73
|
+
hit++;
|
|
74
|
+
if (anyOf(t.action, criteria.action))
|
|
75
|
+
hit++;
|
|
76
|
+
if (anyOf(t.emotion, criteria.emotion))
|
|
77
|
+
hit++;
|
|
78
|
+
if (anyOf(t.setting, criteria.setting))
|
|
79
|
+
hit++;
|
|
80
|
+
if (anyOf(t.composition, criteria.composition))
|
|
81
|
+
hit++;
|
|
82
|
+
if (anyOf(t.motion, criteria.motion))
|
|
83
|
+
hit++;
|
|
84
|
+
if (anyOf(t.audio_type, criteria.audio_type))
|
|
85
|
+
hit++;
|
|
86
|
+
if (criteria.energy && criteria.energy.length && criteria.energy.includes(t.energy))
|
|
87
|
+
hit++;
|
|
88
|
+
if ((criteria.min_duration_sec != null || criteria.max_duration_sec != null) &&
|
|
89
|
+
(criteria.min_duration_sec == null || clip.duration_sec >= criteria.min_duration_sec) &&
|
|
90
|
+
(criteria.max_duration_sec == null || clip.duration_sec <= criteria.max_duration_sec))
|
|
91
|
+
hit++;
|
|
92
|
+
if (criteria.has_dialogue != null && hasDialogue(t) === criteria.has_dialogue)
|
|
93
|
+
hit++;
|
|
94
|
+
if (criteria.has_subject != null && hasSubject(t) === criteria.has_subject)
|
|
95
|
+
hit++;
|
|
96
|
+
return hit / total;
|
|
97
|
+
}
|
|
98
|
+
/** Cosine similarity of two vectors (safe on unnormalized input). */
|
|
99
|
+
export function cosineSimilarity(a, b) {
|
|
100
|
+
if (!a || !b || a.length === 0 || a.length !== b.length)
|
|
101
|
+
return 0;
|
|
102
|
+
let dot = 0;
|
|
103
|
+
let na = 0;
|
|
104
|
+
let nb = 0;
|
|
105
|
+
for (let i = 0; i < a.length; i++) {
|
|
106
|
+
dot += a[i] * b[i];
|
|
107
|
+
na += a[i] * a[i];
|
|
108
|
+
nb += b[i] * b[i];
|
|
109
|
+
}
|
|
110
|
+
if (na === 0 || nb === 0)
|
|
111
|
+
return 0;
|
|
112
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Run the two-stage search over an in-memory clip set. Cloud can push the
|
|
116
|
+
* structured part into DynamoDB and the semantic part into a vector index; this
|
|
117
|
+
* pure function is the reference implementation devcli uses directly.
|
|
118
|
+
*/
|
|
119
|
+
export function searchClips(clips, opts) {
|
|
120
|
+
const { criteria, queryEmbedding } = opts;
|
|
121
|
+
const limit = opts.limit ?? 20;
|
|
122
|
+
const relax = opts.relaxWhenEmpty ?? true;
|
|
123
|
+
let candidates = clips.filter((c) => matchesCriteria(c, criteria));
|
|
124
|
+
let relaxed = false;
|
|
125
|
+
if (candidates.length === 0 && relax && (queryEmbedding || criteria.semantic_text)) {
|
|
126
|
+
candidates = clips.slice();
|
|
127
|
+
relaxed = true;
|
|
128
|
+
}
|
|
129
|
+
const hits = candidates.map((clip) => scoreClip(clip, criteria, queryEmbedding, relaxed));
|
|
130
|
+
return rankHits(hits, limit);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Score a single clip against a query. `relaxed` = the structured filter was
|
|
134
|
+
* bypassed (semantic-only pass), so factor in how well it still matches the
|
|
135
|
+
* criteria; when not relaxed the clip already passed the hard filter (ratio 1).
|
|
136
|
+
* Exported so the devcli SQLite store can reuse the exact scoring after doing
|
|
137
|
+
* the structured filter / KNN prefilter in SQL.
|
|
138
|
+
*/
|
|
139
|
+
export function scoreClip(clip, criteria, queryEmbedding, relaxed) {
|
|
140
|
+
const similarity = queryEmbedding ? cosineSimilarity(clip.embedding, queryEmbedding) : undefined;
|
|
141
|
+
const ratio = relaxed ? structuredMatchRatio(clip, criteria) : 1;
|
|
142
|
+
let score;
|
|
143
|
+
if (similarity != null) {
|
|
144
|
+
// Blend: mostly semantic, with a structured-match boost. Normalize cosine
|
|
145
|
+
// (-1..1) into 0..1 so scores read sensibly.
|
|
146
|
+
const sim01 = (similarity + 1) / 2;
|
|
147
|
+
score = 0.8 * sim01 + 0.2 * ratio;
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
score = ratio;
|
|
151
|
+
}
|
|
152
|
+
return { clip, score, similarity };
|
|
153
|
+
}
|
|
154
|
+
/** Sort hits by score (newer clips break ties) and take the top `limit`. */
|
|
155
|
+
export function rankHits(hits, limit) {
|
|
156
|
+
hits.sort((a, b) => {
|
|
157
|
+
if (b.score !== a.score)
|
|
158
|
+
return b.score - a.score;
|
|
159
|
+
return (b.clip.created_at || "").localeCompare(a.clip.created_at || "");
|
|
160
|
+
});
|
|
161
|
+
return hits.slice(0, limit);
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=query.js.map
|