@mevdragon/vidfarm-devcli 0.20.4 → 0.20.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.director.md +64 -3
- package/demo/dist/app.js +50 -50
- package/dist/src/app.js +3 -2
- package/dist/src/cli.js +183 -3
- package/dist/src/editor-chat.js +75 -3
- package/dist/src/services/hyperframes.js +195 -27
- package/package.json +2 -2
package/dist/src/app.js
CHANGED
|
@@ -55,7 +55,7 @@ import { resolveRootFileCandidates } from "./lib/package-root.js";
|
|
|
55
55
|
import { stripTrackingParams } from "./lib/url-clean.js";
|
|
56
56
|
import { addSeconds, nowIso } from "./lib/time.js";
|
|
57
57
|
import { templateRegistry } from "./registry.js";
|
|
58
|
-
import { buildAsyncJobHandoffText, buildEditorChatGuardrailNotice, buildEditorChatSystemInstruction, collectVerifiedHttpJobIds, compactEditorChatModelMessages, createLoadSkillTool, createStreamingJobIdRedactor, defaultEditorChatModel, EDITOR_CHAT_PROVIDER_PRIORITY, extractEditorChatCompositionBrief, formatEditorChatCurrentDateTime, lintReplaceCompositionAction, readAsyncJobHandoff, toEditorChatModelMessages } from "./editor-chat.js";
|
|
58
|
+
import { buildAsyncJobHandoffText, buildEditorChatGuardrailNotice, buildEditorChatSystemInstruction, collectVerifiedHttpJobIds, compactEditorChatModelMessages, createLoadSkillTool, createStreamingJobIdRedactor, defaultEditorChatModel, EDITOR_CHAT_PROVIDER_PRIORITY, extractEditorChatCompositionBrief, extractEditorChatLazyPromptInstruction, formatEditorChatCurrentDateTime, lintReplaceCompositionAction, readAsyncJobHandoff, toEditorChatModelMessages } from "./editor-chat.js";
|
|
59
59
|
import { apiCallHistoryStore } from "./services/api-call-history.js";
|
|
60
60
|
import { AuthService } from "./services/auth.js";
|
|
61
61
|
import { BillingService } from "./services/billing.js";
|
|
@@ -5003,7 +5003,8 @@ async function streamEditorChatAgent(input, send) {
|
|
|
5003
5003
|
template: input.template,
|
|
5004
5004
|
cachedContext: input.cachedContext ?? null,
|
|
5005
5005
|
currentDateTime: formatEditorChatCurrentDateTime(input.clientContext) || `ISO timestamp: ${nowIso()}`,
|
|
5006
|
-
compositionBrief: extractEditorChatCompositionBrief(rawMessages)
|
|
5006
|
+
compositionBrief: extractEditorChatCompositionBrief(rawMessages),
|
|
5007
|
+
lazyPromptInstruction: extractEditorChatLazyPromptInstruction(rawMessages)
|
|
5007
5008
|
});
|
|
5008
5009
|
const preparedMessages = compactEditorChatModelMessages(toEditorChatModelMessages(rawMessages));
|
|
5009
5010
|
const tools = input.template
|
package/dist/src/cli.js
CHANGED
|
@@ -114,8 +114,10 @@ Make a video in one command (orchestrates ingest → decompose → fork):
|
|
|
114
114
|
Composition lifecycle (fork → decompose → snapshot → render):
|
|
115
115
|
fork <template_id> Fork a template into an editable composition → POST /api/v1/compositions
|
|
116
116
|
pull <forkId> Sync composition.html/json + video-context → GET .../compositions/:forkId/{composition.html,...}
|
|
117
|
-
+ cast.json to disk,
|
|
118
|
-
(
|
|
117
|
+
+ cast.json to disk, write a local agent brief
|
|
118
|
+
(.harness/context.json + .harness/agent-guide.md),
|
|
119
|
+
then print
|
|
120
|
+
timeline gaps (blank space) and scene/layer keys.
|
|
119
121
|
--dir <path> Local dir (default: .vidfarm/<forkId>)
|
|
120
122
|
--refetch Overwrite local copies with the cloud version
|
|
121
123
|
decompose <forkId> Split the fork's source into scenes (smart) → POST .../compositions/:forkId/auto-decompose
|
|
@@ -1376,6 +1378,171 @@ async function fetchCompositionFiles(input) {
|
|
|
1376
1378
|
console.log(`[vidfarm] fetched ${filename} (${body.length} bytes)`);
|
|
1377
1379
|
}
|
|
1378
1380
|
}
|
|
1381
|
+
function readJsonFile(filePath) {
|
|
1382
|
+
if (!existsSync(filePath))
|
|
1383
|
+
return null;
|
|
1384
|
+
try {
|
|
1385
|
+
const parsed = JSON.parse(readFileSync(filePath, "utf8"));
|
|
1386
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
1387
|
+
}
|
|
1388
|
+
catch {
|
|
1389
|
+
return null;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
function readJsonString(value) {
|
|
1393
|
+
return typeof value === "string" ? value.trim() : "";
|
|
1394
|
+
}
|
|
1395
|
+
function readJsonStringList(value, max = 8) {
|
|
1396
|
+
if (!Array.isArray(value))
|
|
1397
|
+
return [];
|
|
1398
|
+
const out = [];
|
|
1399
|
+
for (const entry of value) {
|
|
1400
|
+
const text = readJsonString(entry);
|
|
1401
|
+
if (text && !out.includes(text))
|
|
1402
|
+
out.push(text);
|
|
1403
|
+
if (out.length >= max)
|
|
1404
|
+
break;
|
|
1405
|
+
}
|
|
1406
|
+
return out;
|
|
1407
|
+
}
|
|
1408
|
+
function writeLocalAgentHarnessFiles(dir) {
|
|
1409
|
+
const videoContext = readJsonFile(path.join(dir, "video-context.json"));
|
|
1410
|
+
const editorHarness = readJsonFile(path.join(dir, "editor-harness.json"));
|
|
1411
|
+
const sceneAnnotations = readJsonFile(path.join(dir, "scene-annotations.json"));
|
|
1412
|
+
const cast = readJsonFile(path.join(dir, "cast.json"));
|
|
1413
|
+
const viralDna = videoContext && readJsonString(videoContext.status) === "ready" && videoContext.viral_dna && typeof videoContext.viral_dna === "object"
|
|
1414
|
+
? videoContext.viral_dna
|
|
1415
|
+
: null;
|
|
1416
|
+
const harness = editorHarness && readJsonString(editorHarness.status) === "ready" && editorHarness.harness && typeof editorHarness.harness === "object"
|
|
1417
|
+
? editorHarness.harness
|
|
1418
|
+
: null;
|
|
1419
|
+
const annotations = sceneAnnotations && readJsonString(sceneAnnotations.status) === "done" && Array.isArray(sceneAnnotations.annotations)
|
|
1420
|
+
? sceneAnnotations.annotations
|
|
1421
|
+
: [];
|
|
1422
|
+
const castMembers = cast && Array.isArray(cast.cast) ? cast.cast : [];
|
|
1423
|
+
const harnessDir = path.join(dir, ".harness");
|
|
1424
|
+
mkdirSync(harnessDir, { recursive: true });
|
|
1425
|
+
const contextPath = path.join(harnessDir, "context.json");
|
|
1426
|
+
const instructionsPath = path.join(harnessDir, "agent-guide.md");
|
|
1427
|
+
const bundle = {
|
|
1428
|
+
generated_by: "vidfarm-devcli",
|
|
1429
|
+
generated_at: new Date().toISOString(),
|
|
1430
|
+
source_title: readJsonString(videoContext?.summary) || null,
|
|
1431
|
+
composition_context: viralDna,
|
|
1432
|
+
static_vs_pivot: viralDna?.static_vs_pivot ?? null,
|
|
1433
|
+
editor_harness: harness,
|
|
1434
|
+
scenes: Array.isArray(videoContext?.scenes) ? videoContext?.scenes : [],
|
|
1435
|
+
scene_annotations: annotations,
|
|
1436
|
+
cast: castMembers,
|
|
1437
|
+
transcript: videoContext?.transcript ?? null
|
|
1438
|
+
};
|
|
1439
|
+
writeFileSync(contextPath, JSON.stringify(bundle, null, 2));
|
|
1440
|
+
const emotionalPunch = viralDna?.emotional_punch && typeof viralDna.emotional_punch === "object"
|
|
1441
|
+
? viralDna.emotional_punch
|
|
1442
|
+
: null;
|
|
1443
|
+
const importantScenes = readJsonStringList(harness?.important_scenes, 6);
|
|
1444
|
+
const editingBias = readJsonStringList(harness?.editing_bias, 6);
|
|
1445
|
+
const doList = readJsonStringList(harness?.do, 5);
|
|
1446
|
+
const dontList = readJsonStringList(harness?.dont, 5);
|
|
1447
|
+
const preserve = readJsonStringList(emotionalPunch?.preserve, 5);
|
|
1448
|
+
const staticVsPivot = viralDna?.static_vs_pivot && typeof viralDna.static_vs_pivot === "object"
|
|
1449
|
+
? viralDna.static_vs_pivot
|
|
1450
|
+
: null;
|
|
1451
|
+
const sceneReplacement = staticVsPivot?.scene_replacement && typeof staticVsPivot.scene_replacement === "object"
|
|
1452
|
+
? staticVsPivot.scene_replacement
|
|
1453
|
+
: null;
|
|
1454
|
+
const narrationPivot = staticVsPivot?.narration && typeof staticVsPivot.narration === "object"
|
|
1455
|
+
? staticVsPivot.narration
|
|
1456
|
+
: null;
|
|
1457
|
+
const musicPivot = staticVsPivot?.music && typeof staticVsPivot.music === "object"
|
|
1458
|
+
? staticVsPivot.music
|
|
1459
|
+
: null;
|
|
1460
|
+
const captionsPivot = staticVsPivot?.captions && typeof staticVsPivot.captions === "object"
|
|
1461
|
+
? staticVsPivot.captions
|
|
1462
|
+
: null;
|
|
1463
|
+
const visualRebuild = staticVsPivot?.visual_rebuild && typeof staticVsPivot.visual_rebuild === "object"
|
|
1464
|
+
? staticVsPivot.visual_rebuild
|
|
1465
|
+
: null;
|
|
1466
|
+
const viralPivot = staticVsPivot?.viral_dna && typeof staticVsPivot.viral_dna === "object"
|
|
1467
|
+
? staticVsPivot.viral_dna
|
|
1468
|
+
: null;
|
|
1469
|
+
const mustPreserve = annotations
|
|
1470
|
+
.flatMap((entry) => readJsonStringList(entry.must_preserve, 3).map((item) => {
|
|
1471
|
+
const slug = readJsonString(entry.scene_slug);
|
|
1472
|
+
return slug ? `${slug}: ${item}` : item;
|
|
1473
|
+
}))
|
|
1474
|
+
.slice(0, 8);
|
|
1475
|
+
const agentMd = `# Vidfarm Local Harness
|
|
1476
|
+
|
|
1477
|
+
Read \`.harness/context.json\` before making substantial edits. This fork was pulled from Vidfarm with decompose context; do not freestyle the rewrite from \`composition.html\` alone.
|
|
1478
|
+
|
|
1479
|
+
Files to use:
|
|
1480
|
+
- \`video-context.json\`: transcript, scene descriptions, and viral DNA.
|
|
1481
|
+
- \`editor-harness.json\`: the technical editing brief under \`harness\` (pace, typography, audio, scene roles).
|
|
1482
|
+
- \`scene-annotations.json\`: per-scene replacement and regeneration DNA.
|
|
1483
|
+
- \`.harness/context.json\`: merged agent-facing snapshot generated by \`vidfarm pull\`.
|
|
1484
|
+
- \`.harness/agent-guide.md\`: Vidfarm-specific local scripting guidance generated by \`vidfarm pull\`.
|
|
1485
|
+
|
|
1486
|
+
Rules:
|
|
1487
|
+
- Treat the viral DNA and editor harness as hard defaults unless the user explicitly overrides them.
|
|
1488
|
+
- If you rewrite text, make it fit the existing performance and joke mechanic. Do not replace a format-specific bit with generic product-benefit copy.
|
|
1489
|
+
- Preserve the emotional punch first: tone, mechanism, contrast, audio/visual tension, delivery, and preserve beats.
|
|
1490
|
+
- If the user gives a lazy prompt like "recreate this for my business" with little detail, use static_vs_pivot as the first-pass execution plan instead of interrogating them for every choice.
|
|
1491
|
+
- Protect any harness scene marked critical/must_keep and the must_preserve notes from scene annotations.
|
|
1492
|
+
- When swapping visuals, match both the literal scene DNA and the narrative purpose of the beat.
|
|
1493
|
+
- For replacement graphics, screenshots, or still-like scenes, prefer AI image generation plus Ken Burns before paying for AI video unless static_vs_pivot says motion footage is load-bearing.
|
|
1494
|
+
- If narration must be customized, default to premium ElevenLabs first, then the user's own ElevenLabs path, then BYOK OpenAI/Gemini/OpenRouter. If captions or scenes were timed to the old VO, retime them to the new narration.
|
|
1495
|
+
|
|
1496
|
+
Current format snapshot:
|
|
1497
|
+
- Trend tagline: ${readJsonString(viralDna?.trend_tagline) || "unknown"}
|
|
1498
|
+
- Hook: ${readJsonString(viralDna?.hook) || "unknown"}
|
|
1499
|
+
- Payoff: ${readJsonString(viralDna?.payoff) || "unknown"}
|
|
1500
|
+
- Emotional core: ${readJsonString(emotionalPunch?.core_emotion) || "unknown"}
|
|
1501
|
+
- Tone: ${readJsonString(emotionalPunch?.tone) || "unknown"}
|
|
1502
|
+
- Mechanism: ${readJsonString(emotionalPunch?.mechanism) || "unknown"}
|
|
1503
|
+
- Contrast: ${readJsonString(emotionalPunch?.contrast) || "unknown"}
|
|
1504
|
+
- Audio/visual tension: ${readJsonString(emotionalPunch?.audio_visual_tension) || "unknown"}
|
|
1505
|
+
- Delivery: ${readJsonString(emotionalPunch?.delivery) || "unknown"}
|
|
1506
|
+
- Harness one-liner: ${readJsonString(harness?.one_liner) || "unknown"}
|
|
1507
|
+
|
|
1508
|
+
Static vs pivot:
|
|
1509
|
+
- Summary: ${readJsonString(staticVsPivot?.summary) || "unknown"}
|
|
1510
|
+
- Scene replacement: ${readJsonString(sceneReplacement?.overall) || "unknown"}${readJsonString(sceneReplacement?.reason) ? ` — ${readJsonString(sceneReplacement?.reason)}` : ""}
|
|
1511
|
+
- Narration: ${readJsonString(narrationPivot?.overall) || "unknown"}${readJsonString(narrationPivot?.reason) ? ` — ${readJsonString(narrationPivot?.reason)}` : ""}
|
|
1512
|
+
- Music: ${readJsonString(musicPivot?.overall) || "unknown"}${readJsonString(musicPivot?.volume_guidance) ? ` — ${readJsonString(musicPivot?.volume_guidance)}` : ""}
|
|
1513
|
+
- Captions: ${readJsonString(captionsPivot?.overall) || "unknown"}${readJsonString(captionsPivot?.animation_dependency) ? ` — ${readJsonString(captionsPivot?.animation_dependency)}` : ""}
|
|
1514
|
+
- Visual rebuild: ${readJsonString(visualRebuild?.default_strategy) || "unknown"}${readJsonString(visualRebuild?.reason) ? ` — ${readJsonString(visualRebuild?.reason)}` : ""}
|
|
1515
|
+
|
|
1516
|
+
Priority preserve beats:
|
|
1517
|
+
${preserve.length ? preserve.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1518
|
+
|
|
1519
|
+
Static-vs-pivot must-keep:
|
|
1520
|
+
${readJsonStringList(viralPivot?.must_keep, 6).length ? readJsonStringList(viralPivot?.must_keep, 6).map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1521
|
+
|
|
1522
|
+
Static-vs-pivot flexible:
|
|
1523
|
+
${readJsonStringList(viralPivot?.flexible, 6).length ? readJsonStringList(viralPivot?.flexible, 6).map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1524
|
+
|
|
1525
|
+
Static-vs-pivot load-bearing scenes:
|
|
1526
|
+
${readJsonStringList(sceneReplacement?.load_bearing_scenes, 6).length ? readJsonStringList(sceneReplacement?.load_bearing_scenes, 6).map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1527
|
+
|
|
1528
|
+
Important scenes:
|
|
1529
|
+
${importantScenes.length ? importantScenes.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1530
|
+
|
|
1531
|
+
Scene-level must-preserve cues:
|
|
1532
|
+
${mustPreserve.length ? mustPreserve.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1533
|
+
|
|
1534
|
+
Editing bias:
|
|
1535
|
+
${editingBias.length ? editingBias.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1536
|
+
|
|
1537
|
+
Do:
|
|
1538
|
+
${doList.length ? doList.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1539
|
+
|
|
1540
|
+
Don't:
|
|
1541
|
+
${dontList.length ? dontList.map((item) => `- ${item}`).join("\n") : "- none recorded"}
|
|
1542
|
+
`;
|
|
1543
|
+
writeFileSync(instructionsPath, agentMd);
|
|
1544
|
+
return { contextPath, instructionsPath };
|
|
1545
|
+
}
|
|
1379
1546
|
async function fetchVideoAssets(input) {
|
|
1380
1547
|
const refs = readSourceRefs(input.dir);
|
|
1381
1548
|
if (!refs.compositionSourceUrl && !refs.originalSourceUrl) {
|
|
@@ -4439,12 +4606,24 @@ async function runPullCommand(argv) {
|
|
|
4439
4606
|
if (!existsSync(htmlPath))
|
|
4440
4607
|
throw new Error(`pull did not fetch composition.html for ${forkId} (check credentials / fork id).`);
|
|
4441
4608
|
const info = inspectComposition(readFileSync(htmlPath, "utf8"));
|
|
4609
|
+
const harnessFiles = writeLocalAgentHarnessFiles(dir);
|
|
4442
4610
|
const hasContext = existsSync(path.join(dir, "video-context.json"));
|
|
4443
4611
|
const hasCast = existsSync(path.join(dir, "cast.json"));
|
|
4444
4612
|
const hasAnnotations = existsSync(path.join(dir, "scene-annotations.json"));
|
|
4445
4613
|
const hasHarness = existsSync(path.join(dir, "editor-harness.json"));
|
|
4446
4614
|
if (ctx.json) {
|
|
4447
|
-
printJson({
|
|
4615
|
+
printJson({
|
|
4616
|
+
ok: true,
|
|
4617
|
+
fork_id: forkId,
|
|
4618
|
+
dir,
|
|
4619
|
+
has_video_context: hasContext,
|
|
4620
|
+
has_cast: hasCast,
|
|
4621
|
+
has_scene_annotations: hasAnnotations,
|
|
4622
|
+
has_editor_harness: hasHarness,
|
|
4623
|
+
harness_context_path: harnessFiles.contextPath,
|
|
4624
|
+
harness_instructions_path: harnessFiles.instructionsPath,
|
|
4625
|
+
...info
|
|
4626
|
+
});
|
|
4448
4627
|
return;
|
|
4449
4628
|
}
|
|
4450
4629
|
console.log(`${GREEN}Pulled ${forkId} → ${dir}${RESET}`);
|
|
@@ -4456,6 +4635,7 @@ async function runPullCommand(argv) {
|
|
|
4456
4635
|
console.log(` ${layer.key} ${DIM}${layer.kind ?? "?"} ${layer.start}-${(layer.start + layer.duration).toFixed(2)}s track${layer.track}${flags ? ` [${flags}]` : ""}${layer.slug ? ` slug=${layer.slug}` : ""}${RESET}`);
|
|
4457
4636
|
}
|
|
4458
4637
|
console.log(`${DIM}Grounding: ${hasContext ? "video-context.json ✓" : "video-context.json ✗ (run `vidfarm decompose`)"}, ${hasCast ? "cast.json ✓" : "cast.json ✗"}, ${hasAnnotations ? "scene-annotations.json ✓" : "scene-annotations.json ✗"}, ${hasHarness ? "editor-harness.json ✓ (style brief under .harness)" : "editor-harness.json ✗"}.${RESET}`);
|
|
4638
|
+
console.log(`${DIM}Agent brief: ${path.relative(dir, harnessFiles.instructionsPath) || path.basename(harnessFiles.instructionsPath)}; merged context: ${path.relative(dir, harnessFiles.contextPath)}.${RESET}`);
|
|
4459
4639
|
console.log(`${DIM}Next: vidfarm generate video --prompt "..." --aspect-ratio ${info.aspect_ratio ?? "9:16"} --place ${dir} --at <gap start>|--replace <layer_key>${RESET}`);
|
|
4460
4640
|
}
|
|
4461
4641
|
async function runVisibilityCommand(argv) {
|
package/dist/src/editor-chat.js
CHANGED
|
@@ -50,7 +50,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
|
|
|
50
50
|
"IMAGE UTILITY PRIMITIVES beyond generate/edit are also first-class here. POST /api/v1/primitives/images/inpaint with body { tracer, payload: { source_image_url, mask_url, instruction } } does a MASKED edit — reach for it when only a region of an image should change and a mask is available. POST /api/v1/primitives/images/render-html with body { tracer, payload: { html, css?, width?, height?, background_color? } } rasterizes HTML/CSS into a still image — use it for title cards, quote cards, data callouts, or any designed graphic you want as a timeline image. POST /api/v1/primitives/images/remove-background with body { tracer, payload: { source_image_url } } is the AI-matting cut-out for ARBITRARY photos with a messy background; it BILLS THE WALLET (RapidAPI). POST /api/v1/primitives/images/remove-background-greenscreen with body { tracer, payload: { source_image_url, key_color?, tolerance?, softness?, despill? } } is the CHEAP local chroma-key alternative — when the source already sits on a FLAT solid background (green screen, or any key_color like #FFFFFF for white), prefer this over the RapidAPI route. MOST IMPORTANT — POST /api/v1/primitives/images/create-overlay with body { tracer, payload: { prompt, key_color?, aspect_ratio?, prompt_attachments? } } is the go-to CREATE MEDIA OVERLAY primitive: it generates an AI image on a forced flat key-color background (BYOK image keys) and chroma-keys it out in ONE job, returning a ready-to-composite transparent PNG (result.primary_file_url). This is exactly how you mint 'Vox-style' floating illustrations, props, icons, and cut-out characters to add_layer as an animated overlay — reach for it FREQUENTLY instead of a flat still whenever a scene wants a transparent graphic element floating over the video. It bills a small wallet fee plus the BYOK image-gen cost.",
|
|
51
51
|
"If the user asks to generate a new AI video from text or reference images, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, input_references?, duration? } }. AI video duration is seconds; use duration: 4 for a 4-second video and never use duration_ms on /videos/generate. input_references must be direct image asset URLs, not webpage or social post URLs. Use frame_images for exact first/last-frame image-to-video.",
|
|
52
52
|
"If the user asks to render HTML/design into an image, call POST /api/v1/primitives/images/render-html. If the user asks to turn existing media URLs into a slideshow/video, call POST /api/v1/primitives/videos/render-slides. If the user asks to dedupe, camouflage, or apply small zoom/tilt/rotate/filter/tint transforms to an image or video for reuse, call POST /api/v1/primitives/media/dedupe with body { tracer, payload: { source_media_url, media_type?, effects?: { zoom?, tilt?, rotate?, saturation?, speed?, horizontal_flip?, contrast?, brightness?, hue_rotate?, blur? }, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Primitive media utility routes are also allowed directly here, including /videos/trim, /videos/remove-captions, /videos/extract-audio, /videos/probe, /videos/extract-frame, /videos/mute, /videos/replace-audio, /videos/concat, /audio/trim, /audio/probe, /audio/concat, and /audio/normalize. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or a YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver), aspect crops every clip, and avoid_text prefers text-free scenes via scene SELECTION (never caption removal on the source). The hunt is async (202 + scan_id): poll GET /clips/scan/:scanId, then browse GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys. To erase captions from a finished HUNTED clip, pass that short mp4 to /videos/remove-captions afterwards. When the user instead wants ONE EXACT subrange saved as a raw — 'clip 0:30 to 0:45 into my raws', 'save that 10-second bit as a raw', a precise hand-picked in/out — do NOT run an AI hunt: call POST /raws/clip-range with body { source_url | preview_url | temp_file_id, start_sec, end_sec, tracer?, folder_path?, name? }. It trims EXACTLY [start_sec, end_sec] and lands a single raw (compute-only, no AI, no BYOK key). This is the same operation as the web /tools/clipper page and the devcli `vidfarm clipper` command; use it for surgical single clips and reserve /clips/scan for open-ended multi-clip hunts.",
|
|
53
|
-
"SPEECH PRIMITIVES (TTS/STT) are first-class here. VOICEOVER / NARRATION: when the user wants a script, hook, caption, or dub read aloud as AI audio, call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format? } }
|
|
53
|
+
"SPEECH PRIMITIVES (TTS/STT) are first-class here. VOICEOVER / NARRATION: when the user wants a script, hook, caption, or dub read aloud as AI audio, call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions?, use_wallet_credits?, provider?, model?, output_format? } }. `use_wallet_credits` defaults to true, which routes to premium ElevenLabs on the platform path; keep that default for the best narration unless there is a reason to avoid wallet spend. Set `use_wallet_credits:false` only when you intentionally want the user's own ElevenLabs key or a BYOK OpenAI/Gemini/OpenRouter path. ElevenLabs voices use a voice_id (list them with GET /api/v1/primitives/audio/voices); OpenAI/Gemini voices use their normal provider presets. instructions is a free-form voice-STYLE prompt (tone, pacing, accent, emotion, persona; e.g. 'calm warm bedtime narrator' or 'excited sports announcer, fast paced'). The finished job returns a durable audio URL — place it on the timeline with editor_action add_layer (kind=audio, src=that URL), typically starting where the narration should begin. TRANSCRIPTION: when the user wants a transcript, subtitles/captions from speech, a translation source, or to know who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language?, prompt? } } — source_url may be a VIDEO or AUDIO URL (video audio is demuxed automatically; pick the fork's source via GET /api/v1/compositions/:forkId/remove-video-captions first when transcribing the current composition). One job returns BOTH formats: a simple subtitle version (plain transcript text + timed SRT cues, stored as transcript.txt/transcript.srt) and an advanced multi-speaker version (speaker-attributed timed segments in output.segments and transcript.json) for multi-narration sources like podcasts and interviews. diarize defaults to true and speaker attribution needs a Gemini key — openai/openrouter keys degrade to a plain single-speaker transcript. For transcript-grounded caption work on THIS fork, the read-only video_context tool may already contain the decompose-time transcript; prefer it before running a new billable-key transcription, and use /audio/transcribe for other URLs, multi-speaker attribution, or SRT output. REGENERATE / REVOICE: when the user wants the SAME speaker to say DIFFERENT words (reword the narration, fix a line, change an offer or CTA, redub in the original voice), call POST /api/v1/primitives/audio/regenerate-speech with body { tracer, payload: { source_url (video or audio; video audio is demuxed), rewrite_instruction (how to reword the original transcript) OR text (the exact new script — one of the two is required), voice?, instructions?, provider?, model?, language?, output_format? } }. One job listens to the source, profiles the speaker's voice, rewords the script, and returns a durable audio URL spoken by the CLOSEST PRESET voice with a style prompt matched to the speaker (result fields: audioUrl, script, original_transcript, voice_profile, voice_match). Voice profiling needs a saved Gemini key — with only openai/openrouter keys the job degrades to transcript-only rewording with a default or explicitly passed voice and voice_match 'none'. IMPORTANT HONESTY RULE: OpenAI and Gemini APIs cannot clone voice identity (preset voices only), so always tell the user the regenerated audio APPROXIMATES the original voice rather than clones it. To swap the regenerated audio into the composition, mute the original span (/videos/mute or volume/muted on the layer) and place the new audio with editor_action add_layer (kind=audio, src=audioUrl). RECONCILE LENGTH: reworded narration is rarely exactly the same length as the original, so after placing it do NOT assume the timeline still lines up — read the new audio layer's duration (or /audio/probe it) and re-sync: adjust the muted original span, and when captions or scenes were timed to the narration, re-run set_captions (or ripple_edit / set_layer_timing) so words and visuals track the new audio instead of drifting out of sync.",
|
|
54
54
|
"MUSIC / BGM — HONESTY RULE: there is NO music-generation capability in this editor. You CANNOT create a background-music track from a prompt, and you must NEVER fabricate one by duplicating the narration/speech audio onto a second 'music' layer — that just doubles the voice and is not music. Real background music can ONLY come from: (1) a track the user already owns — browse_files search across /files and /raws for their audio (mp3/wav/m4a/aac) — or (2) a music file/URL the user explicitly provides. If neither exists, tell the user plainly that you can't generate music here and offer to search their files or use a track they upload; do NOT invent a music layer or claim you added music. Only once you have a REAL music URL, add_layer kind=audio at a low volume (e.g. 0.1-0.2) under the narration.",
|
|
55
55
|
"NATIVE MULTI-TRACK AUDIO — the timeline mixes UNLIMITED simultaneous audio layers. Each audio layer sits on its own track and carries its own data-volume (0-2, default 1), and the runtime mixes them together with per-track volume honored IDENTICALLY in the live preview and the exported MP4 (the render does a real ffmpeg amix of every audio layer). So overlaying sound is native: you do NOT need a pre-mixed file. Put narration/voiceover on one audio layer at full level (~1.0), a music bed on a SEPARATE audio layer at low level (~0.1-0.2), and any SFX on their own layers — each via editor_action add_layer (kind=audio, src, plus its own start/duration/track/volume), or tune an existing audio/video layer's level with set_layer_media (volume, muted). Different layers at different volumes is the normal, supported case — never collapse multiple sounds into one track when they should be independently adjustable.",
|
|
56
56
|
"SPLIT COMBINED AUDIO WHEN RECREATING — the most important use of multi-track audio: when the user is recreating/replicating a template whose ORIGINAL had music + narration baked TOGETHER into a single audio track, do NOT reproduce it as one combined bed. Rebuild it as TWO independent audio tracks — a fresh narration track (generate the new script via /api/v1/primitives/audio/speech, or same-speaker reword via /api/v1/primitives/audio/regenerate-speech) at ~1.0 on one track, and a separate REAL music track at ~0.1-0.2 on another track — then mute or remove the original combined source-audio layer so the old voice doesn't play under the new one. This gives the user independent voice and music volume afterwards, and is the elegant workaround for AI TTS models being unable to emit narration+music in one file: compose the mix on the timeline instead. HONESTY: you cannot un-mix / stem-separate the original's baked-in combined audio — the 2-track version is BUILT from a fresh narration track PLUS a real music file (owned / user-provided / found via browse_files), never by faking a music layer or duplicating the voice track (see the MUSIC/BGM honesty rule).",
|
|
@@ -88,7 +88,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
|
|
|
88
88
|
"FORKING IS AUTOMATIC — you never manually 'create a fork' before editing. The editor_context reports the project state so you know what you're holding: composition_origin ('inspiration' = a decomposed catalog template opened as a structural reference — the common case; 'raw' = the user's own new project), has_saved_fork (whether edits are currently persisting to a real editable copy), and is_new_project. When has_saved_fork is false on a REAL template (a template_… id, a catalog reference, or any decomposed composition), just start editing — the FIRST mutating editor_action mints the user their OWN private editable fork and every edit after it persists normally; do NOT ask permission to fork, wait for the user to fork, or announce 'forking'. The ONE exception is is_new_project:true (composition_id \"original\"): that is a BLANK project with no template behind it yet, so there is nothing to fork and editor_action edits will NOT persist. In that state don't promise saves — mint a real composition first: on a blank project the create_video tool IS available (call it to build a brand-new video from the user's idea via mode=generate, or to recreate a pasted TikTok/YouTube/Instagram/X URL via mode=replicate — it runs the ingest→decompose→fork pipeline and drops an 'Open in editor' link), or point the user at a fitting existing template (GET /discover/feed?q=<their idea>). Once a real composition is open, edit it normally. A save_status:\"error\" / is_read_only immediately after editing a brand-new project means the auto-fork itself failed — say so plainly instead of claiming the edit saved. The ONLY time you fork ON PURPOSE is when the user EXPLICITLY asks to branch / duplicate / 'save as' / 'make a copy' / 'try a variation' / 'keep this version and start another' — then call editor_action action_type=fork_composition with fork_from='current' (branch a new copy that keeps all current edits) or fork_from='default' (start a fresh copy from the original template, discarding the current edits). This mints a NEW fork and opens it in the editor; it is the same 'New fork' action available in the editor's ⋯ menu. Never call fork_composition just to begin editing — that is what the automatic first-edit fork is for.",
|
|
89
89
|
"AGENTIC VIDEO EDITING — THE THREE AXES (your core mental model for this editor). Almost every /editor session is the user taking a template, fork, or project and RE-WORKING it, and a re-work only ever touches three independent axes: SCENES (the video/image clips that carry the visuals), AUDIO (narration/voiceover, music, SFX), and TEXT (on-screen captions, titles, overlays). On each axis the intent sits somewhere on a SWAP↔REPLACE spectrum, and the three axes in one request can sit at DIFFERENT points — read the request and place each axis before you act. SWAP (light, cheap, the common case) keeps the existing structure and changes content IN PLACE: rewrite a caption/text layer (set_layer_text / set_captions), swap one clip's media for another of the same kind while keeping its timing and geometry (set_layer_media), or re-voice the narration (regenerate-speech, then mute the original span and add_layer the new audio). Many sessions are ONLY a text swap — that is a two-minute job, so just do it and confirm. REPLACE (heavy) throws out that axis's content and rebuilds it: restructure the scenes outright (new clip count / new beats via remove_layer + add_layer / generate_layer, or one replace_composition_html), lay down a brand-new audio bed, or rewrite every caption. The HARDEST end is a full re-theme where the ONLY thing preserved is the VIRAL DNA — the hook shape, pacing, scene-count rhythm, and transition/caption style — while every scene, every word, and the audio are all replaced for the user's new subject. Name your plan back to the user in these terms (e.g. 'I'll SWAP the captions and REPLACE the scenes'), then execute axis by axis. Be proactive at the heavy end: propose the whole re-work and drive it scene by scene instead of waiting to be micro-managed one layer at a time. This is first-class agentic editing — you are expected to carry a total transformation, not just nudge single layers.",
|
|
90
90
|
"EDITOR HARNESS — YOUR STYLE-EXECUTION BRIEF (read it before any nontrivial edit). editor_context.editor_harness is the decompose pass's TECHNICAL EDITING DIRECTION: how to edit so the result recreates the source's STYLE. It is the 'how to edit like this' companion to composition_context's viral DNA ('why it works'). When present it carries: one_liner (the style in a sentence); pacing (cut_rhythm fast|medium|slow|varied, avg_scene_seconds, cuts_per_10s, energy_curve); typography (caption_style, placement, font_character, emphasis, text_density); broll (reliance, sourcing, shot_kinds, cadence); transitions (default, intro, outro, usage); audio (voiceover, music, sfx, captions_from); emotional (target_feeling, tone, comedic_timing, intonation, vibe_anchors — HOW to edit so the FEELING survives); scenes[] (each a beat's role + importance + must_keep + edit_bias); important_scenes; editing_bias; do; dont. USE IT to pick concrete moves: map typography.caption_style straight to a set_captions caption_style preset (karaoke/word-pop/spotlight/stack-up/neon/bounce); map transitions.default/intro/outro to a set_transitions call; honor pacing (keep cut rhythm / avg_scene_seconds when adding or splitting scenes); follow broll.reliance + sourcing to decide whether to HUNT raws (/clips/scan) vs generate_layer and at what cadence; lay audio per audio.*; treat emotional.* as a HARD constraint — the vibe, the joke, and the intonation are the first things a remix flattens, so preserve emotional.comedic_timing (the held beat / pause / hard cut that sells the joke), keep the delivery cadence in emotional.intonation when you rewrite or re-voice narration, and honor every one of emotional.vibe_anchors when you re-cut or re-time; and above all PROTECT the beats in important_scenes and any scene with must_keep:true or importance 'critical' (swapping their subject is fine, but preserve their timing, role, and caption cadence — that is where the style lives). Treat editing_bias/do/dont as hard constraints for this composition. The harness sets defaults, not a straitjacket — an explicit user instruction always wins. The local devcli agent reads the SAME brief from editor-harness.json (under `.harness`); the web editor gets it inline here. If editor_harness is absent, the fork wasn't decomposed with the harness pass — fall back to composition_context + the video_context tool.",
|
|
91
|
-
"RE-THEME / REUSE A TEMPLATE AS A REFERENCE — the single most common editor intent. Users open a template to keep its VIRAL DNA (structure, scene count, pacing, hook shape, transitions, caption style) while changing the SUBJECT to their own topic — e.g. 'make this book-recap template about \"The Richest Man in Babylon\"', 'do this one for my coffee brand'. Treat it as a first-class flow: PRESERVE the DNA and format (read composition_context and, when you need the scene beats/arc, the video_context tool) and REPLACE the content — rewrite every on-screen text/caption layer to the new subject IN THE TEMPLATE'S VOICE (set_layer_text / set_captions, per the format rule above), regenerate the narration
|
|
91
|
+
"RE-THEME / REUSE A TEMPLATE AS A REFERENCE — the single most common editor intent. Users open a template to keep its VIRAL DNA (structure, scene count, pacing, hook shape, transitions, caption style) while changing the SUBJECT to their own topic — e.g. 'make this book-recap template about \"The Richest Man in Babylon\"', 'do this one for my coffee brand'. Treat it as a first-class flow: PRESERVE the DNA and format (read composition_context and, when you need the scene beats/arc, the video_context tool) and REPLACE the content — rewrite every on-screen text/caption layer to the new subject IN THE TEMPLATE'S VOICE (set_layer_text / set_captions, per the format rule above), regenerate the narration only when composition_context.static_vs_pivot.narration says the original words no longer make sense, and swap the visuals according to composition_context.static_vs_pivot.scene_replacement and visual_rebuild (prefer image generation plus Ken Burns for graphic/still scenes, and reserve AI video or hunted footage for load-bearing motion scenes). Keep the clip timings, transitions, and caption animation intact unless static_vs_pivot says the new narration/captions need re-timing. You don't ask permission to fork — the first edit auto-forks it into the user's own copy (see FORKING IS AUTOMATIC). Before rewriting, PROPOSE a short plan and ask ONLY what you can't infer: the specifics of the new subject (e.g. which of the book's ideas to feature — offer to pick the strongest N to match the scene count), whether to keep the current narrator voice, any brand assets/images they want used (else you'll generate them), and a target length if it differs from the template. Don't interrogate — one concise proposal plus a couple of questions, then execute scene by scene. Example opener for the book case: 'I'll keep this template's N-scene structure, pacing and caption style and re-theme it to The Richest Man in Babylon — rewriting the narration around the book's core money lessons and swapping the visuals to match. Any specific lessons you want featured (or I'll pick the strongest N)? Keep the current narrator voice? Have a cover image or should I generate the visuals?'",
|
|
92
92
|
"FUEL A SCENE REPLACE WITH RAW CLIPS (don't default to expensive AI video). A heavy REPLACE of the SCENES axis needs footage, and you have three sources in cost order: (1) the user's own library — browse_files search across /raws and /files for clips that already fit the new scenes; (2) HUNTING new raws out of a long-form source — the cheap way to get REAL footage; (3) AI generation (generate_layer) — the expensive last resort, only for scenes no real clip can cover. When a scene replace needs footage the user could plausibly supply and their library doesn't already have it, PROACTIVELY offer to mine it: if they have or name a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL, call POST /clips/scan (alias /raws/scan) with body { tracer, source_url|temp_file_id|attachment_id, prompt: <what the new scenes need>, aspect: <the editor_context aspect_ratio>, target_duration_sec?, avoid_text?, ranges? } — an async hunt (202 + scan_id; bills AWS compute only, its BYOK AI tagging is free) that cuts the source into short, tagged, searchable raws. Poll GET /clips/scan/:scanId, then browse GET /raws/feed?source=<source_video_id> or POST /raws/search { query } and drop the picks onto the timeline (set_layer_media to swap a clip in place keeping timing+geometry, add_layer for net-new scenes). Hunting real clips is dramatically cheaper than AI-generating every scene and usually more authentic, so reach for it FIRST when a heavy scene REPLACE needs footage — reserve generate_layer for the gaps real footage cannot fill. If the user describes a big scene re-work but hasn't given you footage, ASK them for a source video to hunt (or point them at their /raws library) BEFORE reaching for AI generation.",
|
|
93
93
|
"CRITICAL — verify your edits actually landed. An editor_action tool result only echoes the arguments you sent; it is NOT proof the change was applied or saved. The real outcome shows up in the NEXT <editor_context>. Before telling the user an edit succeeded, check that block: (1) if it contains save_status:\"error\" (or is_read_only:true / a save_error message), the composition is NOT being saved — do NOT claim any edit worked; tell the user exactly what save_error says (e.g. the video is read-only from this link and they should open their own copy to edit) and stop making mutating editor_action calls until it clears; (2) if it contains recent_action_results, each entry with ok:false is an edit from your previous turn that FAILED (read its error) — acknowledge the failure and retry or explain it rather than repeating a false success; (3) otherwise confirm the intended change is reflected in the layers (e.g. the new text on the target layer) before saying it's done. Never report a timeline edit as successful without this check.",
|
|
94
94
|
"editor_context also carries the canvas shape and free time: composition_width, composition_height, and aspect_ratio (e.g. '9:16') describe the output frame — when generating media to place on this timeline, request the matching aspect_ratio on /videos/generate and /images/generate so it fills the canvas without letterboxing or wrong cropping. timeline_gaps is a precomputed array of {start, end, duration} spans (seconds) where NO visual layer is on screen — i.e. the literal blank/black moments. When the user says something like 'at 00:05:52 there's blank space, add X', map their timestamp (interpret mm:ss or seconds sensibly) to the covering entry in timeline_gaps, set the new layer's start to that gap's start (or the user's exact time if inside the gap) and clamp its duration to fit the gap unless the user asks otherwise. If the requested time is NOT inside any timeline_gaps entry, tell the user it already has content there and offer to overlay on a higher track or replace the existing layer instead of silently colliding.",
|
|
@@ -117,7 +117,7 @@ export function buildTemplateEditorChatSystemPrompt(input) {
|
|
|
117
117
|
"KNOWLEDGE ON DEMAND (load_skill): deeper authoring knowledge ships as vidfarm skill packs you can read mid-conversation with the load_skill tool — YOUR OWN capability map for this editor (editor-capabilities: the editor_action verb catalog, editor_context fields, forking rules, the three-axes SWAP↔REPLACE re-theme model, and fueling a scenes replace with raws — load it before a full re-theme, a multi-scene rebuild, or when you need an exact verb/param), the composition HTML contract (hyperframes-core), motion/animation craft and transition doctrine (hyperframes-animation), seek-safe keyframe patterns (hyperframes-keyframes), creative direction/palettes/typography (hyperframes-creative), caption identities (embedded-captions), media/audio resolution (vidfarm-media), and end-to-end workflow playbooks (product-launch-video, faceless-explainer, website-to-video, general-video, motion-graphics, slideshow, talking-head-recut). Call load_skill with the pack name to read its SKILL.md; it lists reference files you can then load by relative path (e.g. file='references/step-4-vo.md'). Use it when a request needs craft beyond the built-in preset vocabulary — designing a whole composition via replace_composition_html, picking motion or transition language for a style, or following a named workflow. NOTE: hyperframes-animation and hyperframes-keyframes describe BOTH script-driven adapters (anime.js/GSAP/Lottie/Three.js) AND script-free CSS techniques — in the web editor apply only the CSS @keyframes / declarative-preset half (the JS adapters are stripped on save; see the web-editor motion rule), and reserve the adapter half for local devcli work. Load only what the task needs, never bulk-load packs, and never paste large skill content back to the user.",
|
|
118
118
|
"replace_composition_html is validated before it reaches the editor: when the HTML violates the composition contract (missing data-composition-id, invalid clip timing, same-track overlaps, unknown preset names, media without src) the tool result comes back with rejected=true and lint_errors, and NOTHING is applied — fix the reported issues and call it again. lint_warnings are advisory and the action still applies.",
|
|
119
119
|
"MOTION IN THE WEB EDITOR IS DECLARATIVE / CSS ONLY — no author <script>. This editor strips every <script> tag on save (stored-XSS defense), so replace_composition_html that contains a <script> is REJECTED (rejected=true), not applied. That means the JavaScript runtime animation adapters — anime.js, GSAP, Lottie, Three.js, TypeGPU, WAAPI-driven code — are NOT available here; they are a local-devcli-only capability. Author all motion for the web editor with the durable, script-free vocabulary: (1) built-in preset actions — set_layer_media ken_burns (still-image pan/zoom), set_transitions / transition + transition_out (scene entrances/exits), set_captions / caption_animation (word-by-word captions); and (2) plain CSS — @keyframes rules in a <style> block plus an inline animation on the layer element (the renderer scrubs and bakes CSS @keyframes deterministically into the final MP4, exactly like the preview). Never try to drive motion with anime.js/GSAP/etc. in the web editor, and never claim you added a scripted animation here. If the user explicitly wants a JS-adapter animation (a Lottie file, a GSAP timeline, a Three.js scene), tell them that is a local devcli / hyperframes-CLI workflow and offer the CSS/preset equivalent for the web editor instead.",
|
|
120
|
-
"TEMPLATE FORMAT — MATCH IT WHEN WRITING TEXT: the COMPOSITION BRIEF (stable template context in the system prompt) carries composition_context — WHAT THIS TEMPLATE IS: its format/genre and narrative arc (trend_tagline, hook, retention, payoff, preserve, avoid) plus emotional_punch (core_emotion, tone, arc, peak_moment, mechanism, humor, delivery, preserve) — the FEELING the format delivers and what makes it land. ALWAYS read it before writing or adjusting ANY captions, text layers, or on-screen copy, and make your copy fit the template's format and voice — do NOT default to generic product/app ad copy. emotional_punch is the part a remix most often flattens: if the format is a joke, keep emotional_punch.humor's mechanic (rebuild the SAME joke around the new subject, don't drop it for a feature bullet); match emotional_punch.tone and emotional_punch.delivery so intonation, comedic timing, and vibe survive; and never trade the peak_moment's payoff for a flat product plug. The format dictates the caption style: a text-message / DM / iMessage conversation template's captions must read like short back-and-forth chat messages between people (in-character, conversational), a 'story time' / talking-head template reads like first-person spoken narration, a listicle reads like punchy numbered items, a fake-news / headline template reads like a news chyron, etc. `preserve` usually names format cues that must stay (e.g. the messaging-bubble layout); `avoid` names what to keep out. When the template's format is a conversation, skit, or roleplay, keep the captions in that voice and only weave the product in the way that format allows (subtly, in-dialogue) rather than replacing the messages with feature bullets. If the brief's composition_context is thin or you are unsure what the video actually shows, call video_context (scene visual descriptions + transcript) to ground the format BEFORE writing captions.",
|
|
120
|
+
"TEMPLATE FORMAT — MATCH IT WHEN WRITING TEXT: the COMPOSITION BRIEF (stable template context in the system prompt) carries composition_context — WHAT THIS TEMPLATE IS: its format/genre and narrative arc (trend_tagline, hook, retention, payoff, preserve, avoid) plus emotional_punch (core_emotion, tone, arc, peak_moment, mechanism, contrast, humor, audio_visual_tension, delivery, preserve) — the FEELING the format delivers and what makes it land. ALWAYS read it before writing or adjusting ANY captions, text layers, or on-screen copy, and make your copy fit the template's format and voice — do NOT default to generic product/app ad copy. emotional_punch is the part a remix most often flattens: if the format is a joke, keep emotional_punch.humor's mechanic (rebuild the SAME joke around the new subject, don't drop it for a feature bullet); preserve emotional_punch.contrast when the bit depends on a mismatch (for example tough soundtrack vs delicate visuals, swagger vs mundane setting); use emotional_punch.audio_visual_tension when audio and visuals intentionally play against each other; match emotional_punch.tone and emotional_punch.delivery so intonation, comedic timing, and vibe survive; and never trade the peak_moment's payoff for a flat product plug. The format dictates the caption style: a text-message / DM / iMessage conversation template's captions must read like short back-and-forth chat messages between people (in-character, conversational), a 'story time' / talking-head template reads like first-person spoken narration, a listicle reads like punchy numbered items, a fake-news / headline template reads like a news chyron, etc. `preserve` usually names format cues that must stay (e.g. the messaging-bubble layout); `avoid` names what to keep out. When the template's format is a conversation, skit, or roleplay, keep the captions in that voice and only weave the product in the way that format allows (subtly, in-dialogue) rather than replacing the messages with feature bullets. If the brief's composition_context is thin or you are unsure what the video actually shows, call video_context (scene visual descriptions + transcript) to ground the format BEFORE writing captions.",
|
|
121
121
|
"FILE DIRECTORY: the user's files live in ONE navigable directory with three canonical roots, each with nested subfolders and a copyable path per file: (1) /files — durable 'My Files' assets they keep and reuse (videos mp4/mov/webm, images png/jpg/jpeg/gif/webp/svg, audio mp3/wav/m4a/aac, documents pdf/md/txt/csv), vector-searchable via metadata notes; (2) /temp — scratch space auto-foldered by date (YYYY-MM-DD) and auto-deleted after 30 days; (3) /raws — the reusable clip / source-footage library, foldered by source (e.g. /raws/<source-slug>/). A file's canonical path looks like /raws/product-demos/clip.mp4 or /files/acme-skincare/logo.png. Use the browse_files tool to navigate, search, read, write, and reorganize it: action=list with path (e.g. path='/', '/raws', '/files/brand-assets') enumerates that folder's subfolders + files as canonical items (path, name, kind, view_url); action=search with a plain-language query finds files by MEANING across roots — it combines semantic vector match, substring, and absolute-path match (scope with path='/raws' for footage, '/files' for durable assets, '/' for everything; force one signal with mode=semantic|substring|path). Prefer search over paging list when the library is large or the user references an asset vaguely. When you omit path entirely, list/search now default to '/' (the whole directory) so /raws and /temp are never hidden — but pass path='/raws' to stay in footage. For /raws you can add content_type to filter by EXACT shot kind (talking_head, b_roll, product_shot, screen_recording, demo, reaction, interview, establishing, lifestyle, text_graphic; comma-separate to OR them) — this is an exact tag filter, distinct from the semantic query. For a big folder, list returns next_offset; pass it back as offset to page deeper (with limit up to 500). action=read with a file_id (copied from a prior list/search) fetches one file; action=write with file_name + content (or source_url) SAVES a text/media file into /files; action=annotate with file_id + notes attaches vector-embedded metadata to a /files entry; action=move with file_id + a target path (e.g. path='/raws/demos') reorganizes a raw between /raws folders; action=rename with path + new_name (+ file_id for a file) renames a /files or /temp file or folder in place — use it to keep the directory tidy (e.g. fix a character folder or file name). When the user references their own footage, brand assets, logos, music, a brief, a script, or raw clips ('use my product photo', 'the video I uploaded', 'my brand logo', 'that clip of the founder'), browse_files search or list first to find it rather than asking them to re-upload or paste a URL. For text files (md/txt/csv/srt/vtt/json), read returns the full text_content inline so you can read a brief or script directly. For images/video/audio/pdf, read returns only a durable view_url — use that URL as media: drop it into the timeline via editor_action add_layer/set_layer_media, or pass it into primitive routes (images/edit source_image_url, videos/generate input_references, videos/download). Never invent file ids, paths, or view URLs — always list or search to discover the real ones first.",
|
|
122
122
|
"SAVING CONTEXT TO MY FILES: browse_files action=write persists durable, reusable context the user (and future chats/agents) can read back. write saves text files (md/txt/csv/json/srt/vtt) via content, and IMPORTS media into the library via source_url (a durable URL from a finished primitive job or existing asset) — use source_url to persist a generated asset the user will want again (a character sprite card, a recurring background, a logo variant) instead of leaving it stranded in a job result. Use write to capture the Getting Started artifacts as Markdown: a product About.md (basic offer/product context) or a deeper Interview.md, plus awareness-levels.md, persuasive-angles.md, and ad-hooks.md distilled from the matching brainstorm outputs. When a brainstorm job returns strategy the user wants to keep, offer to save it (or save it) as the corresponding .md so it is not lost when the chat ends. Namescope every write under the right product/offer folder (see MY FILES IS MULTI-OFFER) — e.g. folder_path='acme-skincare', file_name='About.md'. Pass notes on write (or annotate afterwards) for any asset worth finding again: notes describe what the file is, who/what it depicts, and when to use it, and they are vector-embedded so action=search finds the file by meaning in future sessions. Before overwriting an existing file of the same name in the same folder, confirm with the user (writing the same name replaces it).",
|
|
123
123
|
"RECURRING CHARACTERS ARE FIRST-CLASS: mascots, spokespeople, avatars, and product characters that must look the same across videos live in a dedicated, browsable home — the /files/characters/ folder, one subfolder per character keyed by a URL-safe SLUG (lowercase, hyphenated), e.g. /files/characters/zara/. Each character is a TRIO of files in that folder: (1) <character_id>.json — the machine-readable MANIFEST, named after the character's id which is 'character_'+slug (e.g. slug 'zara' → id 'character_zara' → file character_zara.json; the id ALREADY carries the 'character_' prefix, so never double it to character_character_zara.json): { id:'character_<slug>', slug, name, role, appearance (face/build/hair/skin), wardrobe, palette (signature colors), voice (tone/accent/energy), do, dont, sprite_card_path:'/files/characters/<slug>/character_sprite_card.png', about_path:'/files/characters/<slug>/character_about.md', created_at }; (2) character_sprite_card.png — ONE reference-sheet image (full body front/side/back plus a face close-up on a neutral background); (3) character_about.md — the prose the .json summarizes, for richer wording when prompting. AWARENESS: when a user refers to 'our mascot', 'the same character', 'the fox', or names a character, FIRST browse_files list path='/files/characters' (or search) to see who already exists and read that character's manifest + about — never re-imagine a saved character from memory; that is how characters drift off-model. CONSISTENCY: on every generation featuring the character, pass the sprite card's view_url as the reference input (prompt_attachments for images/generate + images/edit, input_references for videos/generate and generate_layer) and lift wording from the manifest/about into the prompt.",
|
|
@@ -191,6 +191,9 @@ export function buildEditorChatSystemInstruction(input) {
|
|
|
191
191
|
if (input.compositionBrief?.trim()) {
|
|
192
192
|
sections.push(input.compositionBrief.trim());
|
|
193
193
|
}
|
|
194
|
+
if (input.lazyPromptInstruction?.trim()) {
|
|
195
|
+
sections.push(input.lazyPromptInstruction.trim());
|
|
196
|
+
}
|
|
194
197
|
if (input.currentDateTime?.trim()) {
|
|
195
198
|
sections.push([
|
|
196
199
|
"Current datetime for scheduling:",
|
|
@@ -216,6 +219,75 @@ export function formatEditorChatCurrentDateTime(clientContext) {
|
|
|
216
219
|
}
|
|
217
220
|
return lines.join("\n");
|
|
218
221
|
}
|
|
222
|
+
function stripEditorContextBlock(text) {
|
|
223
|
+
return text.replace(EDITOR_CONTEXT_BLOCK_RE, "").trim();
|
|
224
|
+
}
|
|
225
|
+
function latestUserMessageText(messages) {
|
|
226
|
+
if (!Array.isArray(messages)) {
|
|
227
|
+
return "";
|
|
228
|
+
}
|
|
229
|
+
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
|
230
|
+
const entry = messages[i];
|
|
231
|
+
if (!entry || typeof entry !== "object" || entry.role !== "user") {
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
return stripEditorContextBlock(readRawMessageText(entry));
|
|
235
|
+
}
|
|
236
|
+
return "";
|
|
237
|
+
}
|
|
238
|
+
function isLazyRethemePrompt(text) {
|
|
239
|
+
const normalized = text.trim().toLowerCase();
|
|
240
|
+
if (!normalized) {
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
const isShort = normalized.length <= 220;
|
|
244
|
+
const likelyRetheme = [
|
|
245
|
+
"recreate this",
|
|
246
|
+
"recreate that",
|
|
247
|
+
"do this for",
|
|
248
|
+
"make this for",
|
|
249
|
+
"redo this for",
|
|
250
|
+
"use this for",
|
|
251
|
+
"turn this into",
|
|
252
|
+
"for my business",
|
|
253
|
+
"for my brand",
|
|
254
|
+
"for my company",
|
|
255
|
+
"for my product",
|
|
256
|
+
"for my app",
|
|
257
|
+
"for my store"
|
|
258
|
+
].some((needle) => normalized.includes(needle));
|
|
259
|
+
const detailed = [
|
|
260
|
+
"scene",
|
|
261
|
+
"script",
|
|
262
|
+
"voice",
|
|
263
|
+
"narration",
|
|
264
|
+
"caption",
|
|
265
|
+
"captions",
|
|
266
|
+
"music",
|
|
267
|
+
"shot",
|
|
268
|
+
"b-roll",
|
|
269
|
+
"asset",
|
|
270
|
+
"logo",
|
|
271
|
+
"exactly",
|
|
272
|
+
"specific"
|
|
273
|
+
].filter((needle) => normalized.includes(needle)).length >= 3 || normalized.split(/\s+/).length > 40;
|
|
274
|
+
return isShort && likelyRetheme && !detailed;
|
|
275
|
+
}
|
|
276
|
+
export function extractEditorChatLazyPromptInstruction(messages) {
|
|
277
|
+
const latest = latestUserMessageText(messages);
|
|
278
|
+
if (!isLazyRethemePrompt(latest)) {
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
return [
|
|
282
|
+
"LAZY RETHEME MODE — the latest user prompt is a thin 'recreate this for my business/brand' request with little detail, so do NOT bounce the user into a long interview.",
|
|
283
|
+
"Default to a proactive re-theme plan grounded in composition_context, editor_harness, video_context, and especially viral_dna.static_vs_pivot. Treat static_vs_pivot as the source of truth for what must stay versus what can pivot.",
|
|
284
|
+
"For replacement visuals, prefer the cheapest faithful path first: when scenes are graphics, screenshots, cards, stills, or can plausibly read as motion from a still, generate IMAGES and apply Ken Burns rather than paying for AI video. Only escalate to AI video or hunted footage when static_vs_pivot says the scene truly needs moving footage.",
|
|
285
|
+
"Narration decision: if static_vs_pivot.narration says the original audio is load-bearing, keep it. If the spoken words are source-specific and static_vs_pivot says customize_script, regenerate narration and then re-sync dependent scenes/captions. Prefer premium ElevenLabs narration on the platform path first, then the customer's own ElevenLabs path if they have one, then BYOK OpenAI/Gemini/OpenRouter as fallback.",
|
|
286
|
+
"If narration is customized, also inspect static_vs_pivot.music and decide whether the original music must stay, be recreated under the new voice, or is irrelevant. Preserve the intended voice/music balance with native multi-track audio rather than one baked combined track.",
|
|
287
|
+
"If static_vs_pivot.captions says timings or animations depend on the spoken words, recreate the animated captions against the NEW narration timing rather than reusing stale cue timing.",
|
|
288
|
+
"Be explicit about scene replacement: replace every scene static_vs_pivot.scene_replacement marks as replaceable, but preserve every load-bearing or must-keep scene it names. Do not flatten the viral DNA by replacing scenes that are the core proof, payoff, or joke mechanic."
|
|
289
|
+
].join("\n");
|
|
290
|
+
}
|
|
219
291
|
function tryParseUrl(value) {
|
|
220
292
|
try {
|
|
221
293
|
return new URL(value);
|