@mevdragon/vidfarm-devcli 0.20.6 → 0.20.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/.agents/skills/editor-capabilities/SKILL.md +15 -1
  2. package/.agents/skills/vidfarm-director/SKILL.md +106 -0
  3. package/.agents/skills/vidfarm-director/recipes/find-and-fork-template.md +16 -0
  4. package/.agents/skills/vidfarm-director/recipes/local-edit-render-approve.md +13 -0
  5. package/.agents/skills/vidfarm-director/recipes/onboard-a-new-director.md +13 -0
  6. package/.agents/skills/vidfarm-director/recipes/retheme-template.md +17 -0
  7. package/.agents/skills/vidfarm-director/references/assets-and-sourcing.md +117 -0
  8. package/.agents/skills/vidfarm-director/references/automation-and-local-dev.md +245 -0
  9. package/.agents/skills/vidfarm-director/references/core-workflows.md +273 -0
  10. package/.agents/skills/vidfarm-director/references/editor-workflows.md +365 -0
  11. package/.agents/skills/vidfarm-director/references/onboarding.md +28 -0
  12. package/.agents/skills/vidfarm-director/references/primitives.md +308 -0
  13. package/SKILL.director.md +517 -250
  14. package/dist/src/cli.js +563 -99
  15. package/dist/src/devcli/clips.js +29 -12
  16. package/dist/src/devcli/doctor.js +2 -2
  17. package/dist/src/devcli/local-backend.js +46 -2
  18. package/dist/src/services/clip-curation/cost.js +5 -1
  19. package/dist/src/services/clip-curation/gemini.js +14 -2
  20. package/dist/src/services/clip-curation/hunt.js +56 -0
  21. package/dist/src/services/clip-curation/index.js +1 -1
  22. package/dist/src/services/clip-curation/scan.js +5 -3
  23. package/package.json +30 -10
  24. package/SKILL.platform.md +0 -432
  25. package/demo/README.md +0 -28
  26. package/demo/dist/app.css +0 -1
  27. package/demo/dist/app.js +0 -1850
  28. package/demo/dist/chunks/chunk-DXB73IDG.js +0 -1
  29. package/demo/dist/chunks/chunk-S7OWAJDS.js +0 -36
  30. package/demo/dist/chunks/chunk-VTIBZ6AN.js +0 -1
  31. package/demo/dist/chunks/dist-ADSJKBVE.js +0 -332
  32. package/demo/dist/chunks/domEditingLayers-VZMLL4AP-SGHWPND4.js +0 -1
  33. package/demo/dist/chunks/hyperframes-player-XB65TCD6.js +0 -425
  34. package/demo/dist/chunks/lib-XAQ37YOE.js +0 -1
  35. package/demo/dist/chunks/src-TJ2QYA4U.js +0 -207
  36. package/demo/dist/favicon.ico +0 -0
  37. package/demo/dist/icons/timeline/audio.svg +0 -7
  38. package/demo/dist/icons/timeline/captions.svg +0 -5
  39. package/demo/dist/icons/timeline/composition.svg +0 -12
  40. package/demo/dist/icons/timeline/image.svg +0 -18
  41. package/demo/dist/icons/timeline/music.svg +0 -10
  42. package/demo/dist/icons/timeline/text.svg +0 -3
  43. package/demo/dist/index.html +0 -15
  44. package/dist/src/account-pages-legacy.js +0 -9396
  45. package/dist/src/account-pages.js +0 -61
  46. package/dist/src/app.js +0 -21603
  47. package/dist/src/composition-runtime.js +0 -1053
  48. package/dist/src/config.js +0 -217
  49. package/dist/src/context.js +0 -447
  50. package/dist/src/dev-app-legacy.js +0 -739
  51. package/dist/src/dev-app.js +0 -6
  52. package/dist/src/devcli/migrate-local.js +0 -140
  53. package/dist/src/devcli/sync.js +0 -368
  54. package/dist/src/domain.js +0 -5
  55. package/dist/src/editor-chat-history.js +0 -82
  56. package/dist/src/editor-chat.js +0 -828
  57. package/dist/src/editor-dark-theme.js +0 -1128
  58. package/dist/src/frontend/debug.js +0 -71
  59. package/dist/src/frontend/discover-client.js +0 -130
  60. package/dist/src/frontend/discover-store.js +0 -23
  61. package/dist/src/frontend/file-directory.js +0 -1018
  62. package/dist/src/frontend/flockposter-cache-store.js +0 -124
  63. package/dist/src/frontend/homepage-client.js +0 -446
  64. package/dist/src/frontend/homepage-shared.js +0 -201
  65. package/dist/src/frontend/homepage-store.js +0 -66
  66. package/dist/src/frontend/homepage-view.js +0 -705
  67. package/dist/src/frontend/page-runtime-client.js +0 -132
  68. package/dist/src/frontend/page-runtime-store.js +0 -9
  69. package/dist/src/frontend/sentry.js +0 -42
  70. package/dist/src/frontend/template-editor-chat.js +0 -4181
  71. package/dist/src/help-page.js +0 -346
  72. package/dist/src/homepage.js +0 -1458
  73. package/dist/src/index.js +0 -16
  74. package/dist/src/instrument.js +0 -30
  75. package/dist/src/landing-page.js +0 -384
  76. package/dist/src/page-runtime.js +0 -2
  77. package/dist/src/page-shell.js +0 -1452
  78. package/dist/src/primitive-context.js +0 -416
  79. package/dist/src/primitive-registry.js +0 -3940
  80. package/dist/src/primitive-sdk.js +0 -4
  81. package/dist/src/primitives/hyperframes-media.js +0 -108
  82. package/dist/src/react-page-shell.js +0 -35
  83. package/dist/src/ready-post-schedule-component.js +0 -1540
  84. package/dist/src/registry.js +0 -296
  85. package/dist/src/reskin/agency-page.js +0 -299
  86. package/dist/src/reskin/calendar-page.js +0 -568
  87. package/dist/src/reskin/chat-page.js +0 -942
  88. package/dist/src/reskin/discover-page.js +0 -1788
  89. package/dist/src/reskin/document.js +0 -1587
  90. package/dist/src/reskin/help-page.js +0 -357
  91. package/dist/src/reskin/index-page.js +0 -62
  92. package/dist/src/reskin/inpaint-clipper-page.js +0 -890
  93. package/dist/src/reskin/inpaint-page.js +0 -2554
  94. package/dist/src/reskin/inpaint-video-page.js +0 -1339
  95. package/dist/src/reskin/job-runs-page.js +0 -477
  96. package/dist/src/reskin/library-page.js +0 -1634
  97. package/dist/src/reskin/login-page.js +0 -262
  98. package/dist/src/reskin/portfolio-page.js +0 -687
  99. package/dist/src/reskin/pricing-page.js +0 -390
  100. package/dist/src/reskin/settings-page.js +0 -732
  101. package/dist/src/reskin/theme.js +0 -711
  102. package/dist/src/runtime.js +0 -35
  103. package/dist/src/services/api-call-history.js +0 -249
  104. package/dist/src/services/auth.js +0 -152
  105. package/dist/src/services/billing-pricing.js +0 -39
  106. package/dist/src/services/billing.js +0 -241
  107. package/dist/src/services/cast.js +0 -127
  108. package/dist/src/services/chat-threads.js +0 -92
  109. package/dist/src/services/clip-records.js +0 -250
  110. package/dist/src/services/clip-search.js +0 -77
  111. package/dist/src/services/clip-vectors.js +0 -125
  112. package/dist/src/services/composition-sanitize.js +0 -124
  113. package/dist/src/services/composition-watch.js +0 -79
  114. package/dist/src/services/elevenlabs.js +0 -222
  115. package/dist/src/services/file-directory.js +0 -117
  116. package/dist/src/services/fork-access.js +0 -93
  117. package/dist/src/services/fork-manifest.js +0 -42
  118. package/dist/src/services/ghostcut.js +0 -179
  119. package/dist/src/services/hyperframes.js +0 -3654
  120. package/dist/src/services/job-capacity.js +0 -14
  121. package/dist/src/services/job-logs.js +0 -197
  122. package/dist/src/services/jobs.js +0 -136
  123. package/dist/src/services/local-dynamo.js +0 -0
  124. package/dist/src/services/media-processing.js +0 -766
  125. package/dist/src/services/primitive-media-lambda.js +0 -280
  126. package/dist/src/services/providers.js +0 -2748
  127. package/dist/src/services/rate-limits.js +0 -262
  128. package/dist/src/services/scene-annotations.js +0 -32
  129. package/dist/src/services/serverless-auth.js +0 -382
  130. package/dist/src/services/serverless-jobs.js +0 -1084
  131. package/dist/src/services/serverless-provider-keys.js +0 -409
  132. package/dist/src/services/serverless-records.js +0 -1515
  133. package/dist/src/services/serverless-template-configs.js +0 -75
  134. package/dist/src/services/storage.js +0 -461
  135. package/dist/src/services/swipe-customize.js +0 -437
  136. package/dist/src/services/template-certification.js +0 -413
  137. package/dist/src/services/template-loader.js +0 -99
  138. package/dist/src/services/template-runtime-bundles.js +0 -217
  139. package/dist/src/services/template-sources.js +0 -1017
  140. package/dist/src/services/upstream.js +0 -248
  141. package/dist/src/services/video-normalization.js +0 -2
  142. package/dist/src/services/webhooks.js +0 -62
  143. package/dist/src/template-editor-pages.js +0 -2576
  144. package/dist/src/template-editor-shell.js +0 -2893
  145. package/dist/src/template-sdk.js +0 -4
  146. package/dist/src/worker.js +0 -17
  147. package/public/assets/discover-client-app.js +0 -1
  148. package/public/assets/file-directory-app.js +0 -3
  149. package/public/assets/homepage-app.js +0 -54
  150. package/public/assets/homepage-client-app.js +0 -80
  151. package/public/assets/page-runtime-client-app.js +0 -94
  152. package/public/assets/placeholders/scene-placeholder.png +0 -0
  153. package/src/assets/SELLING_AWARENESS_STAGES.md +0 -579
  154. package/src/assets/SELLING_WITH_HOOKS.md +0 -377
  155. package/src/assets/SELLING_WITH_VSLS.md +0 -606
  156. package/src/assets/favicon.ico +0 -0
  157. package/src/assets/logo-vidfarm.png +0 -0
@@ -1,3654 +0,0 @@
1
- import { spawn } from "node:child_process";
2
- import { randomUUID } from "node:crypto";
3
- import { createWriteStream } from "node:fs";
4
- import { cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
5
- import os from "node:os";
6
- import path from "node:path";
7
- import { pipeline } from "node:stream/promises";
8
- import { CloudFormationClient, DescribeStacksCommand } from "@aws-sdk/client-cloudformation";
9
- import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
10
- import { DescribeExecutionCommand, ExecutionDoesNotExist, SFNClient } from "@aws-sdk/client-sfn";
11
- import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
12
- import ffmpegStatic from "ffmpeg-static";
13
- import ffprobeStatic from "ffprobe-static";
14
- import { parseHTML } from "linkedom";
15
- import { config } from "../config.js";
16
- import { devErrorFields, devLog } from "../lib/dev-log.js";
17
- import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, LEGACY_TRANSITION_CSS_V1, TRANSITION_ADJACENCY_EPSILON, TRANSITION_CSS, TRANSITION_OUT_PRESETS, TRANSITION_PRESETS, TRANSITION_STYLE_MARKER, buildHyperframeCompositionHtml, clampTransitionDuration, transitionOutDelaySeconds } from "../hyperframes/composition.js";
18
- import { applyMarkupUsd } from "./billing-pricing.js";
19
- // The scene-annotation pass reuses the clip library's controlled vocabulary and
20
- // embedding-text recipe verbatim so a source scene's clip-match query is
21
- // symmetric with how a candidate clip was tagged/embedded. This is the ONE
22
- // coupling to the (parallel) clip-curation work — a plain leaf import; the
23
- // retrieval logic itself (search over these criteria) is merged in later.
24
- import { buildClipEmbeddingText, sanitizeTagValues, taxonomyForPrompt } from "./clip-curation/index.js";
25
- /** Thrown by the primitive render wrapper when a distributed render is still in
26
- * flight. The job runner catches it and parks the job as `waiting_for_render`
27
- * so Step Functions waits + re-invokes — no single Lambda holds the render open
28
- * for its 15-minute lifetime. */
29
- export class RenderInProgressError extends Error {
30
- executionArn;
31
- renderId;
32
- waitSeconds;
33
- progressFraction;
34
- constructor(input) {
35
- super(`HyperFrames render ${input.renderId} still in progress`);
36
- this.name = "RenderInProgressError";
37
- this.executionArn = input.executionArn;
38
- this.renderId = input.renderId;
39
- this.waitSeconds = input.waitSeconds;
40
- this.progressFraction = input.progressFraction;
41
- }
42
- }
43
- let stackCache = null;
44
- const SMART_DECOMPOSE_TEMP_PREFIX = "vidfarm-smart-decompose-";
45
- const SMART_DECOMPOSE_TEMP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
46
- async function sweepStaleSmartDecomposeTempDirs() {
47
- const tmpRoot = os.tmpdir();
48
- let entries;
49
- try {
50
- entries = await readdir(tmpRoot);
51
- }
52
- catch {
53
- return;
54
- }
55
- const now = Date.now();
56
- await Promise.all(entries.map(async (name) => {
57
- if (!name.startsWith(SMART_DECOMPOSE_TEMP_PREFIX))
58
- return;
59
- const full = path.join(tmpRoot, name);
60
- try {
61
- const info = await stat(full);
62
- if (!info.isDirectory())
63
- return;
64
- if (now - info.mtimeMs < SMART_DECOMPOSE_TEMP_MAX_AGE_MS)
65
- return;
66
- await rm(full, { recursive: true, force: true });
67
- }
68
- catch {
69
- /* ignore */
70
- }
71
- }));
72
- }
73
- sweepStaleSmartDecomposeTempDirs().catch((err) => {
74
- console.error("smart decompose temp sweep failed", err);
75
- });
76
- // True when no cloud HyperFrames render stack is wired (env or explicit
77
- // CloudFormation outputs). On a `vidfarm serve` box this is the normal state,
78
- // so render should run in-process rather than fail. `VIDFARM_LOCAL_RENDER`
79
- // forces local even when a cloud stack IS configured (useful for offline dev).
80
- function shouldRenderLocally() {
81
- if (config.VIDFARM_LOCAL_RENDER)
82
- return true;
83
- const hasCloudStack = Boolean((config.HYPERFRAMES_RENDER_STATE_MACHINE_ARN || config.HYPERFRAMES_STATE_MACHINE_ARN) &&
84
- (config.HYPERFRAMES_RENDER_BUCKET_NAME || config.HYPERFRAMES_LAMBDA_BUCKET_NAME));
85
- return !hasCloudStack;
86
- }
87
- // The bundled producer/aws-lambda libraries embed upstream-vendor doc links in
88
- // some error messages (font resolution, large-variables). Those URLs surface in
89
- // job logs and the export modal, so rewrite them to vidfarm's docs before the
90
- // message leaves the render layer.
91
- function scrubVendorDocLinks(text) {
92
- return text.replace(/https?:\/\/[a-z0-9.-]*heygen\.com[^\s"')\]]*/gi, "https://vidfarm.cc/skill/vidfarm-director");
93
- }
94
- export class HyperframesService {
95
- async render(input) {
96
- const compositionHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
97
- if (!compositionHtml.includes("data-composition-id=")) {
98
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
99
- }
100
- if (shouldRenderLocally()) {
101
- return this.renderLocal({ ...input, compositionHtml });
102
- }
103
- const prep = await this.prepareCloudRender(input, compositionHtml);
104
- const handle = await this.dispatchCloudRender(prep, input);
105
- if (input.wait === false) {
106
- devLog("hyperframes.render.dispatched_async", {
107
- composition_id: prep.metadata.compositionId,
108
- render_id: prep.renderId,
109
- execution_arn: handle.executionArn
110
- });
111
- return {
112
- ...handle,
113
- outputUrl: null,
114
- outputS3Uri: handle.outputS3Uri,
115
- endedAt: null,
116
- metadata: {
117
- mode: "hyperframes_lambda",
118
- status: "RUNNING",
119
- stack: prep.stack,
120
- composition: prep.metadata
121
- }
122
- };
123
- }
124
- const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
125
- const startedAtMs = Date.now();
126
- const timeoutMs = input.timeoutMs ?? 12 * 60 * 1000;
127
- const pollIntervalMs = input.pollIntervalMs ?? 5000;
128
- let pollCount = 0;
129
- let lastLoggedStatus = null;
130
- for (;;) {
131
- const progress = await getRenderProgress({
132
- executionArn: handle.executionArn,
133
- region: prep.stack.region,
134
- defaultMemorySizeMb: prep.stack.lambdaMemoryMb
135
- });
136
- pollCount += 1;
137
- if (progress.status !== lastLoggedStatus) {
138
- devLog("hyperframes.render.progress", {
139
- composition_id: prep.metadata.compositionId,
140
- render_id: prep.renderId,
141
- status: progress.status,
142
- poll_count: pollCount,
143
- elapsed_ms: Date.now() - startedAtMs,
144
- frames_rendered: progress.framesRendered,
145
- total_frames: progress.totalFrames
146
- });
147
- lastLoggedStatus = progress.status;
148
- }
149
- if (progress.status === "SUCCEEDED") {
150
- devLog("hyperframes.render.succeeded", {
151
- composition_id: prep.metadata.compositionId,
152
- render_id: prep.renderId,
153
- duration_ms: Date.now() - startedAtMs,
154
- output_s3_uri: progress.outputFile?.s3Uri || handle.outputS3Uri,
155
- frames_rendered: progress.framesRendered,
156
- total_frames: progress.totalFrames,
157
- lambdas_invoked: progress.lambdasInvoked,
158
- cost_usd: progress.costs?.accruedSoFarUsd
159
- });
160
- return this.buildCloudRenderResult(handle, progress, prep.metadata, prep.stack);
161
- }
162
- if (progress.fatalErrorEncountered || ["FAILED", "TIMED_OUT", "ABORTED"].includes(progress.status)) {
163
- devLog("hyperframes.render.failed", {
164
- composition_id: prep.metadata.compositionId,
165
- render_id: prep.renderId,
166
- status: progress.status,
167
- duration_ms: Date.now() - startedAtMs,
168
- errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
169
- }, "error");
170
- throw new Error(`HyperFrames render ${handle.renderId} failed: ${scrubVendorDocLinks(JSON.stringify(progress.errors))}`);
171
- }
172
- if (Date.now() - startedAtMs > timeoutMs) {
173
- devLog("hyperframes.render.timeout", {
174
- composition_id: prep.metadata.compositionId,
175
- render_id: prep.renderId,
176
- timeout_ms: timeoutMs,
177
- last_status: progress.status
178
- }, "error");
179
- throw new Error(`Timed out waiting for HyperFrames render ${handle.renderId}.`);
180
- }
181
- await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
182
- }
183
- }
184
- /**
185
- * One non-blocking step of a distributed render, keyed by a stable
186
- * `executionName`. The first call stages + dispatches the render to the
187
- * HyperFrames render state machine and returns `running` immediately; later
188
- * calls (from a re-invoked job runner) DescribeExecution the SAME execution
189
- * and either keep waiting, finalize on SUCCEEDED, or report failure. This is
190
- * what keeps a 40-minute render off any single Lambda's 15-minute clock.
191
- */
192
- async renderStep(input) {
193
- const compositionHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
194
- if (!compositionHtml.includes("data-composition-id=")) {
195
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
196
- }
197
- // Local/dev has no Step Functions — render in-process, blocking, and finalize
198
- // in one step so the devcli and local backend behave exactly as before.
199
- if (shouldRenderLocally()) {
200
- try {
201
- const result = await this.renderLocal({ ...input, compositionHtml });
202
- return { state: "succeeded", ...result };
203
- }
204
- catch (error) {
205
- return {
206
- state: "failed",
207
- renderId: input.executionName,
208
- executionArn: null,
209
- message: error instanceof Error ? error.message : String(error)
210
- };
211
- }
212
- }
213
- const stack = await resolveHyperframesStack();
214
- const executionArn = executionArnFromName(stack.stateMachineArn, input.executionName);
215
- const existingStatus = await describeRenderExecutionStatus(executionArn, stack.region);
216
- if (existingStatus === null) {
217
- // Never dispatched — stage + kick off the render, then park.
218
- const prep = await this.prepareCloudRender(input, compositionHtml, input.executionName);
219
- const handle = await this.dispatchCloudRender(prep, input);
220
- devLog("hyperframes.render.dispatched_async", {
221
- composition_id: prep.metadata.compositionId,
222
- render_id: prep.renderId,
223
- execution_arn: handle.executionArn
224
- });
225
- return { state: "running", executionArn: handle.executionArn, renderId: handle.renderId, progressFraction: 0.1 };
226
- }
227
- if (existingStatus === "RUNNING" || existingStatus === "PENDING_REDRIVE") {
228
- // Cheap path: DescribeExecution already told us it's still going. Avoid the
229
- // heavier GetExecutionHistory pagination that getRenderProgress performs.
230
- return { state: "running", executionArn, renderId: input.executionName, progressFraction: 0.5 };
231
- }
232
- // Terminal (SUCCEEDED / FAILED / TIMED_OUT / ABORTED) — read the full snapshot
233
- // for output location, frames, and cost metadata.
234
- const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
235
- const progress = await getRenderProgress({
236
- executionArn,
237
- region: stack.region,
238
- defaultMemorySizeMb: stack.lambdaMemoryMb
239
- });
240
- if (progress.status === "SUCCEEDED") {
241
- const outputS3Uri = progress.outputFile?.s3Uri || null;
242
- if (!outputS3Uri) {
243
- return {
244
- state: "failed",
245
- renderId: input.executionName,
246
- executionArn,
247
- message: `HyperFrames render ${input.executionName} reported SUCCEEDED without an output file.`
248
- };
249
- }
250
- const metadata = extractCompositionMetadata(compositionHtml, input);
251
- const handleShape = {
252
- renderId: input.executionName,
253
- executionArn,
254
- bucketName: stack.bucketName,
255
- stateMachineArn: stack.stateMachineArn,
256
- outputS3Uri,
257
- // Not carried across re-invocations; unused by downstream consumers,
258
- // which read outputS3Uri / outputUrl / renderId / metadata only.
259
- projectS3Uri: "",
260
- startedAt: progress.startedAt
261
- };
262
- const result = await this.buildCloudRenderResult(handleShape, progress, metadata, stack);
263
- devLog("hyperframes.render.succeeded", {
264
- composition_id: metadata.compositionId,
265
- render_id: input.executionName,
266
- output_s3_uri: outputS3Uri,
267
- frames_rendered: progress.framesRendered,
268
- total_frames: progress.totalFrames,
269
- lambdas_invoked: progress.lambdasInvoked,
270
- cost_usd: progress.costs?.accruedSoFarUsd
271
- });
272
- return { state: "succeeded", ...result };
273
- }
274
- devLog("hyperframes.render.failed", {
275
- render_id: input.executionName,
276
- status: progress.status,
277
- errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
278
- }, "error");
279
- return {
280
- state: "failed",
281
- renderId: input.executionName,
282
- executionArn,
283
- message: `HyperFrames render ${input.executionName} ${progress.status}: ${scrubVendorDocLinks(JSON.stringify(progress.errors))}`
284
- };
285
- }
286
- async prepareCloudRender(input, compositionHtml, executionName) {
287
- const metadata = extractCompositionMetadata(compositionHtml, input);
288
- const stack = await resolveHyperframesStack();
289
- devLog("hyperframes.render.staging", {
290
- composition_id: metadata.compositionId,
291
- width: metadata.width,
292
- height: metadata.height,
293
- fps: metadata.fps,
294
- stack_name: stack.stackName,
295
- region: stack.region
296
- });
297
- const projectDir = await stageProject({
298
- compositionHtml,
299
- compositionId: metadata.compositionId,
300
- projectFiles: input.projectFiles,
301
- copyPaths: input.copyPaths
302
- });
303
- const timestamp = Date.now().toString(36);
304
- const renderId = executionName || input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
305
- const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
306
- return { metadata, stack, projectDir, renderId, outputKey };
307
- }
308
- async dispatchCloudRender(prep, input) {
309
- devLog("hyperframes.render.dispatch", {
310
- composition_id: prep.metadata.compositionId,
311
- render_id: prep.renderId,
312
- output_key: prep.outputKey,
313
- chunk_size: input.chunkSize ?? 240,
314
- max_parallel_chunks: input.maxParallelChunks ?? 8
315
- });
316
- const { renderToLambda } = await import("@hyperframes/aws-lambda/sdk");
317
- return renderToLambda({
318
- projectDir: prep.projectDir,
319
- bucketName: prep.stack.bucketName,
320
- stateMachineArn: prep.stack.stateMachineArn,
321
- region: prep.stack.region,
322
- executionName: prep.renderId,
323
- outputKey: prep.outputKey,
324
- config: {
325
- fps: normalizeRenderFps(prep.metadata.fps),
326
- width: prep.metadata.width,
327
- height: prep.metadata.height,
328
- format: input.format || "mp4",
329
- codec: "h264",
330
- quality: "standard",
331
- chunkSize: input.chunkSize ?? 240,
332
- maxParallelChunks: input.maxParallelChunks ?? 8,
333
- runtimeCap: "lambda"
334
- }
335
- });
336
- }
337
- async buildCloudRenderResult(handle, progress, metadata, stack) {
338
- const outputS3Uri = progress.outputFile?.s3Uri || handle.outputS3Uri;
339
- return {
340
- ...handle,
341
- outputS3Uri,
342
- outputUrl: outputS3Uri ? await signedS3Url(outputS3Uri, stack.region) : null,
343
- endedAt: progress.endedAt,
344
- metadata: {
345
- mode: "hyperframes_lambda",
346
- status: progress.status,
347
- stack,
348
- composition: metadata,
349
- progress,
350
- costs: progress.costs,
351
- estimatedCostUsd: progress.costs.accruedSoFarUsd,
352
- chargeUsd: applyMarkupUsd(progress.costs.accruedSoFarUsd),
353
- costDisplay: progress.costs.displayCost,
354
- lambdasInvoked: progress.lambdasInvoked,
355
- framesRendered: progress.framesRendered,
356
- totalFrames: progress.totalFrames
357
- }
358
- };
359
- }
360
- // In-process render: stage the project, then drive HyperFrames' single-machine
361
- // orchestrator (headless Chromium → per-frame capture → ffmpeg encode) to write
362
- // the MP4 straight to local disk storage. No AWS, no Lambda fan-out. The result
363
- // is shaped like the cloud render result so the `hyperframes_render` primitive
364
- // and the publish/polling flow consume it unchanged.
365
- async renderLocal(input) {
366
- const rawHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
367
- if (!rawHtml.includes("data-composition-id=")) {
368
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
369
- }
370
- // libx264 (yuv420p) requires even width/height. The in-process renderer sizes
371
- // the canvas from the composition's data-width/data-height, so round any odd
372
- // dimension down to even here — the cloud path sidesteps this by rendering at
373
- // explicit even asset dimensions.
374
- const compositionHtml = forceEvenCompositionDimensions(rawHtml);
375
- const metadata = extractCompositionMetadata(compositionHtml, input);
376
- const projectDir = await stageProject({
377
- compositionHtml,
378
- compositionId: metadata.compositionId,
379
- projectFiles: input.projectFiles,
380
- copyPaths: input.copyPaths
381
- });
382
- const timestamp = Date.now().toString(36);
383
- const renderId = input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
384
- const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
385
- const outputLocation = path.join(config.VIDFARM_DATA_DIR, "storage", outputKey);
386
- await mkdir(path.dirname(outputLocation), { recursive: true });
387
- const startedAt = new Date().toISOString();
388
- const startedAtMs = Date.now();
389
- devLog("hyperframes.render_local.start", {
390
- composition_id: metadata.compositionId,
391
- render_id: renderId,
392
- width: metadata.width,
393
- height: metadata.height,
394
- fps: metadata.fps,
395
- output_key: outputKey
396
- });
397
- const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
398
- const job = createRenderJob({
399
- fps: normalizeRenderFps(metadata.fps),
400
- quality: "standard",
401
- format: input.format || "mp4"
402
- });
403
- let lastStatus = null;
404
- try {
405
- await executeRenderJob(job, projectDir, outputLocation, (progressJob, message) => {
406
- if (progressJob.status !== lastStatus) {
407
- devLog("hyperframes.render_local.progress", {
408
- render_id: renderId,
409
- status: progressJob.status,
410
- message
411
- });
412
- lastStatus = progressJob.status;
413
- }
414
- });
415
- }
416
- catch (error) {
417
- throw new Error(scrubVendorDocLinks(error instanceof Error ? error.message : String(error)), { cause: error });
418
- }
419
- // Single-segment encoded key — matches the `/storage/:key` route which
420
- // decodeURIComponent's one param (slashes ride through as %2F).
421
- const outputUrl = `${config.PUBLIC_BASE_URL}/storage/${encodeURIComponent(outputKey)}`;
422
- const durationMs = Date.now() - startedAtMs;
423
- devLog("hyperframes.render_local.succeeded", {
424
- composition_id: metadata.compositionId,
425
- render_id: renderId,
426
- duration_ms: durationMs,
427
- output_location: outputLocation,
428
- output_url: outputUrl
429
- });
430
- return {
431
- renderId,
432
- outputUrl,
433
- outputS3Uri: null,
434
- executionArn: `local:${renderId}`,
435
- bucketName: "",
436
- stateMachineArn: "",
437
- projectS3Uri: `file://${projectDir}`,
438
- siteId: null,
439
- startedAt,
440
- endedAt: new Date().toISOString(),
441
- metadata: {
442
- mode: "hyperframes_local",
443
- status: "SUCCEEDED",
444
- composition: metadata,
445
- outputLocation,
446
- durationMs,
447
- // Local render burns the operator's own CPU — no per-frame Lambda cost.
448
- chargeUsd: 0,
449
- estimatedCostUsd: 0,
450
- costDisplay: "$0.00 (local render)"
451
- }
452
- };
453
- }
454
- // In-process STILL render: render one frame of a composition to a PNG/JPEG/WEBP
455
- // image on local disk. Uses the same headless-Chromium capture as renderLocal
456
- // via HyperFrames' png-sequence format, then transcodes to the requested still
457
- // format with ffmpeg. Replaces the Remotion `renderStill` path.
458
- async renderStill(input) {
459
- const compositionHtml = forceEvenCompositionDimensions(normalizePublishHtml(input.compositionHtml));
460
- if (!compositionHtml.includes("data-composition-id=")) {
461
- throw new Error("HyperFrames still render requires composition HTML with data-composition-id.");
462
- }
463
- const metadata = extractCompositionMetadata(compositionHtml, { compositionId: input.compositionId });
464
- const projectDir = await stageProject({
465
- compositionHtml,
466
- compositionId: metadata.compositionId,
467
- projectFiles: input.projectFiles,
468
- copyPaths: input.copyPaths
469
- });
470
- const timestamp = Date.now().toString(36);
471
- const renderId = `vfimg-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
472
- const framesDir = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects", `${renderId}-frames`);
473
- await mkdir(framesDir, { recursive: true });
474
- devLog("hyperframes.render_still.start", {
475
- composition_id: metadata.compositionId,
476
- render_id: renderId,
477
- width: metadata.width,
478
- height: metadata.height
479
- });
480
- const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
481
- // png-sequence writes outputPath as a DIRECTORY of frame_000001.png … files.
482
- const job = createRenderJob({ fps: 30, quality: "high", format: "png-sequence" });
483
- try {
484
- try {
485
- await executeRenderJob(job, projectDir, framesDir);
486
- }
487
- catch (error) {
488
- throw new Error(scrubVendorDocLinks(error instanceof Error ? error.message : String(error)), { cause: error });
489
- }
490
- const frameFiles = (await readdir(framesDir))
491
- .filter((name) => name.toLowerCase().endsWith(".png"))
492
- .sort();
493
- if (!frameFiles.length) {
494
- throw new Error("HyperFrames still render produced no frames.");
495
- }
496
- const firstFramePath = path.join(framesDir, frameFiles[0]);
497
- const format = input.imageFormat ?? "png";
498
- const extension = format === "jpeg" ? "jpg" : format;
499
- const outputKey = input.outputKey
500
- || `renders/vidfarm/stills/${metadata.compositionId}/${renderId}/output.${extension}`;
501
- const outputLocation = path.join(config.VIDFARM_DATA_DIR, "storage", outputKey);
502
- await mkdir(path.dirname(outputLocation), { recursive: true });
503
- if (format === "png") {
504
- await cp(firstFramePath, outputLocation, { force: true });
505
- }
506
- else {
507
- // Transcode the captured PNG frame to the requested still format.
508
- await runBinary(ffmpegBin, [
509
- "-y",
510
- "-i",
511
- firstFramePath,
512
- ...(format === "jpeg" ? ["-q:v", "3"] : []),
513
- outputLocation
514
- ]);
515
- }
516
- const outputUrl = `${config.PUBLIC_BASE_URL}/storage/${encodeURIComponent(outputKey)}`;
517
- devLog("hyperframes.render_still.succeeded", {
518
- composition_id: metadata.compositionId,
519
- render_id: renderId,
520
- image_format: format,
521
- output_location: outputLocation
522
- });
523
- return {
524
- renderId,
525
- outputUrl,
526
- outputLocation,
527
- metadata: {
528
- mode: "hyperframes_local_still",
529
- compositionId: metadata.compositionId,
530
- width: metadata.width,
531
- height: metadata.height,
532
- imageFormat: format,
533
- outputKey
534
- }
535
- };
536
- }
537
- finally {
538
- await rm(framesDir, { recursive: true, force: true });
539
- }
540
- }
541
- async readRenderOutput(outputS3Uri, region) {
542
- const parsed = parseS3Uri(outputS3Uri);
543
- if (!parsed) {
544
- throw new Error(`Invalid HyperFrames output S3 URI: ${outputS3Uri}`);
545
- }
546
- const s3 = new S3Client({ region: region || config.HYPERFRAMES_RENDER_REGION || config.AWS_REGION });
547
- const response = await s3.send(new GetObjectCommand({ Bucket: parsed.bucket, Key: parsed.key }));
548
- if (!response.Body) {
549
- throw new Error(`HyperFrames output is empty: ${outputS3Uri}`);
550
- }
551
- return {
552
- bytes: Buffer.from(await response.Body.transformToByteArray()),
553
- contentType: response.ContentType || "video/mp4"
554
- };
555
- }
556
- }
557
- // Convert a "raw" composition (source video only, no timeline scenes) into a
558
- // scene-scoped composition by chopping the source video into ~target-second
559
- // chunks and inserting timeline-proxy divs the editor can drag. This is a
560
- // deterministic time-based split; smarter AI-frame-based decomposition can
561
- // replace it later without changing the shape of the produced HTML.
562
- export function decomposeUndecomposedComposition(html, options) {
563
- const { document } = parseHTML(html);
564
- const root = document.querySelector("[data-composition-id]");
565
- if (!root)
566
- return { ok: false, error: "Composition root not found." };
567
- const existingProxies = document.querySelectorAll('[data-vf-timeline-proxy="video"]');
568
- if (existingProxies.length > 0) {
569
- return { ok: false, error: "Composition already has timeline scenes." };
570
- }
571
- const backingVideo = document.querySelector("[data-vf-playback-source]") ?? document.querySelector("video");
572
- const sourceUrl = (backingVideo?.getAttribute("data-src") || backingVideo?.getAttribute("src") || root.getAttribute("data-source-video") || "").trim();
573
- if (!sourceUrl)
574
- return { ok: false, error: "No source video URL found on the composition." };
575
- const durationSeconds = Number.parseFloat(root.getAttribute("data-duration") ?? "");
576
- if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) {
577
- return { ok: false, error: "Composition data-duration is missing — cannot compute scenes." };
578
- }
579
- const targetSeconds = options.targetSceneSeconds;
580
- const preferredCount = options.requestedScenes ?? Math.max(1, Math.round(durationSeconds / targetSeconds));
581
- const sceneCount = Math.min(24, Math.max(1, preferredCount));
582
- const rawScene = durationSeconds / sceneCount;
583
- for (let index = 0; index < sceneCount; index += 1) {
584
- const start = index === 0 ? 0 : Number((index * rawScene).toFixed(3));
585
- const end = index === sceneCount - 1 ? durationSeconds : Number(((index + 1) * rawScene).toFixed(3));
586
- const duration = Number(Math.max(0.1, end - start).toFixed(3));
587
- const proxy = document.createElement("div");
588
- const proxyId = shortElementId();
589
- proxy.id = proxyId;
590
- proxy.setAttribute("data-hf-id", proxyId);
591
- proxy.setAttribute("data-layer-mode", "both");
592
- proxy.setAttribute("data-layer-kind", "video");
593
- proxy.setAttribute("data-viral-note", `Auto-decomposed scene ${index + 1} of ${sceneCount}.`);
594
- proxy.setAttribute("data-start", String(start));
595
- proxy.setAttribute("data-duration", String(duration));
596
- proxy.setAttribute("data-end", String(Number((start + duration).toFixed(3))));
597
- proxy.setAttribute("data-track-index", "0");
598
- proxy.setAttribute("data-label", `Scene ${index + 1}`);
599
- proxy.setAttribute("data-media-start", String(start));
600
- proxy.setAttribute("data-playback-start", String(start));
601
- proxy.setAttribute("data-vf-timeline-proxy", "video");
602
- proxy.setAttribute("data-src", sourceUrl);
603
- proxy.setAttribute("style", "position:absolute;left:0%;top:0%;width:1%;height:1%;z-index:0;display:none;pointer-events:none;");
604
- proxy.className = "clip";
605
- root.append(proxy);
606
- }
607
- return {
608
- ok: true,
609
- html: `<!doctype html>\n${document.documentElement.outerHTML}`,
610
- sceneCount,
611
- durationSeconds
612
- };
613
- }
614
- // ==========================================================================
615
- // Smart video decomposition — samples frames from the source, ships them
616
- // to Gemini vision alongside a strict JSON schema request, and returns the
617
- // scene bounds + text captions found in the footage.
618
- // ==========================================================================
619
- const ffmpegBin = typeof ffmpegStatic === "string" ? ffmpegStatic : ffmpegStatic.path;
620
- const ffprobeBin = typeof ffprobeStatic.path === "string"
621
- ? ffprobeStatic.path
622
- : ffprobeStatic;
623
- // The in-process HyperFrames producer (renderLocal / renderStill) shells out to
624
- // ffmpeg + ffprobe, resolving them from HYPERFRAMES_FFMPEG_PATH / HYPERFRAMES_FFPROBE_PATH,
625
- // then bare `ffmpeg`/`ffprobe` on PATH. The api Lambda image ships no ffmpeg on PATH, so
626
- // point the producer at the bundled ffmpeg-static / ffprobe-static binaries (already
627
- // resolved above) — otherwise render fails with "ffprobe not found. Please install FFmpeg."
628
- if (ffmpegBin && !process.env.HYPERFRAMES_FFMPEG_PATH?.trim()) {
629
- process.env.HYPERFRAMES_FFMPEG_PATH = ffmpegBin;
630
- }
631
- if (ffprobeBin && !process.env.HYPERFRAMES_FFPROBE_PATH?.trim()) {
632
- process.env.HYPERFRAMES_FFPROBE_PATH = ffprobeBin;
633
- }
634
- // The ARCHETYPE / production style of a source video. Detected once per
635
- // decompose and used two ways: (a) stamped onto the template metadata so the
636
- // catalog + clip matcher can filter by format, and (b) to CONDITION the
637
- // per-scene clip annotations, because what counts as a scene's "literal DNA"
638
- // differs by format — an AI fruit drama is defined by its stylized character +
639
- // action, a UGC trend by the creator's face/gesture/authenticity, an
640
- // educational series by the board/diagram/screen, live action by real-world
641
- // subjects + cinematography. Controlled + extendable; ambiguous videos fall back
642
- // to "other".
643
- export const CONTENT_FORMATS = [
644
- "ai_fruit_drama", // AI-generated surreal food/object dramas ("fruit drama")
645
- "ai_generative_skit", // other AI-generated character skits / dramas
646
- "ugc_trend", // phone-shot creator-to-camera trend/challenge
647
- "talking_head", // single speaker addressing camera (advice/opinion)
648
- "educational_series", // explainer / how-to / lecture / listicle
649
- "live_action", // real-world filmed scenes, actors, cinematography
650
- "product_demo", // hands-on showcase / unboxing / review
651
- "street_interview", // vox-pop / man-on-the-street
652
- "reaction_commentary", // reacting to / narrating other footage
653
- "cinematic_broll", // moody establishing / atmosphere / montage
654
- "screen_recording", // app/screen capture, tutorials, gameplay
655
- "animation_motion_graphics", // 2D/3D animation, kinetic typography
656
- "other"
657
- ];
658
- export const DEFAULT_CONTENT_FORMAT = {
659
- format: "other",
660
- label: "Unclassified",
661
- confidence: 0,
662
- rationale: "",
663
- secondary: []
664
- };
665
- const RUN_BINARY_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
666
- function runBinary(bin, args) {
667
- return new Promise((resolve, reject) => {
668
- const proc = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
669
- let stdout = "";
670
- let stderr = "";
671
- let stdoutBytes = 0;
672
- let stderrBytes = 0;
673
- let settled = false;
674
- let killedForOverflow = null;
675
- const settle = (fn) => {
676
- if (settled)
677
- return;
678
- settled = true;
679
- fn();
680
- };
681
- const killForOverflow = (stream) => {
682
- if (killedForOverflow || settled)
683
- return;
684
- killedForOverflow = stream;
685
- try {
686
- proc.kill("SIGKILL");
687
- }
688
- catch { /* ignore */ }
689
- settle(() => reject(new Error(`${path.basename(bin)} ${stream} exceeded ${RUN_BINARY_MAX_BUFFER_BYTES} bytes; killed with SIGKILL.`)));
690
- };
691
- proc.stdout.on("data", (chunk) => {
692
- stdoutBytes += chunk.length;
693
- if (stdoutBytes > RUN_BINARY_MAX_BUFFER_BYTES) {
694
- killForOverflow("stdout");
695
- return;
696
- }
697
- stdout += chunk.toString();
698
- });
699
- proc.stderr.on("data", (chunk) => {
700
- stderrBytes += chunk.length;
701
- if (stderrBytes > RUN_BINARY_MAX_BUFFER_BYTES) {
702
- killForOverflow("stderr");
703
- return;
704
- }
705
- stderr += chunk.toString();
706
- });
707
- proc.on("error", (err) => {
708
- settle(() => reject(err));
709
- });
710
- proc.on("close", (code) => {
711
- if (killedForOverflow)
712
- return;
713
- settle(() => {
714
- if (code === 0)
715
- resolve(stdout);
716
- else
717
- reject(new Error(`${path.basename(bin)} exited ${code}: ${stderr || stdout}`));
718
- });
719
- });
720
- });
721
- }
722
- async function downloadVideoTo(url, dest) {
723
- const response = await fetch(url);
724
- if (!response.ok || !response.body)
725
- throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
726
- await pipeline(response.body, createWriteStream(dest));
727
- }
728
- async function ffprobeVideo(filePath) {
729
- const output = await runBinary(ffprobeBin, [
730
- "-v", "error",
731
- "-show_entries", "format=duration:stream=width,height",
732
- "-of", "json",
733
- filePath
734
- ]);
735
- const parsed = JSON.parse(output || "{}");
736
- const stream = (parsed.streams || []).find((s) => s.width && s.height) || {};
737
- return {
738
- width: Number(stream.width || 0),
739
- height: Number(stream.height || 0),
740
- durationSeconds: Number(parsed.format?.duration || 0)
741
- };
742
- }
743
- // Detect hard scene cuts deterministically with ffmpeg's frame-difference
744
- // scene score, so scene-layer boundaries are grounded in real pixel-level cuts
745
- // instead of the vision model eyeballing sparse frames and guessing. This is
746
- // the scene-layer analogue of the OCR pass that grounds caption timing.
747
- //
748
- // We run `select='gt(scene,THRESH)'` (score 0..1; higher = bigger visual jump)
749
- // and dump the selected frames' metadata to a temp file via `metadata=print`.
750
- // `-f null -` discards the decoded video so stdout stays tiny. Tunable via
751
- // SCENE_CUT_THRESHOLD (default 0.30). Never throws — cuts are an optional hint,
752
- // so any failure returns [] and decompose falls back to model-only boundaries.
753
- async function detectSceneCuts(videoPath, durationSeconds, tempDir) {
754
- const thresholdRaw = Number(process.env.SCENE_CUT_THRESHOLD ?? "0.30");
755
- const threshold = Number.isFinite(thresholdRaw) && thresholdRaw > 0 && thresholdRaw < 1 ? thresholdRaw : 0.30;
756
- const metaPath = path.join(tempDir, "scene-cuts.txt");
757
- try {
758
- await runBinary(ffmpegBin, [
759
- "-y",
760
- "-i", videoPath,
761
- "-vf", `select='gt(scene,${threshold})',metadata=print:file=${metaPath}`,
762
- "-an",
763
- "-f", "null",
764
- "-"
765
- ]);
766
- }
767
- catch (err) {
768
- console.error("scene-cut detection ffmpeg failed", err instanceof Error ? err.message : err);
769
- return [];
770
- }
771
- let raw;
772
- try {
773
- raw = await readFile(metaPath, "utf8");
774
- }
775
- catch {
776
- // No selected frames (e.g. single continuous take) → file may be absent.
777
- return [];
778
- }
779
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : Infinity;
780
- const cuts = [];
781
- const re = /pts_time:([0-9]+\.?[0-9]*)/g;
782
- let match;
783
- while ((match = re.exec(raw)) !== null) {
784
- const t = Number(match[1]);
785
- // Ignore a cut at t≈0 (that's just the first frame, not a boundary) and
786
- // anything past the probed duration. Collapse cuts within 0.4s of an
787
- // already-recorded one so micro-flicker doesn't over-segment the timeline.
788
- if (!Number.isFinite(t) || t < 0.4 || t >= safeDuration - 0.1)
789
- continue;
790
- if (cuts.length > 0 && t - cuts[cuts.length - 1] < 0.4)
791
- continue;
792
- cuts.push(Number(t.toFixed(3)));
793
- }
794
- return cuts;
795
- }
796
- // Sample frames for the vision pass. Vision no longer owns caption timing —
797
- // that's handled deterministically by the dense OCR pass (extractOcrFrames +
798
- // snapCaptionTimingsToOcr). Vision only needs enough coverage to identify
799
- // which captions exist, their text, style, and rough scene boundaries. Keep
800
- // the cadence around 1 fps with a low ceiling to control token cost. Frames
801
- // stay at 960px wide so small overlay text is legible for the LLM.
802
- //
803
- // Uses a single ffmpeg invocation with `-vf fps=…` — the previous per-frame
804
- // spawn loop occasionally left individual output files unwritten when the
805
- // seek raced a demux edge case, producing ENOENT reads. Batched extraction
806
- // is also ~10× faster.
807
- async function sampleFramesAsBase64(videoPath, durationSeconds, tempDir) {
808
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : 5;
809
- const MIN_FRAMES = 8;
810
- // Denser + sharper than before (was 14 / 960px / q3 / 1.0fps). More frames
811
- // catch more scene transitions and short-lived captions; higher resolution +
812
- // lower JPEG quantization keep thin overlay glyphs legible for the model.
813
- const MAX_FRAMES = 18;
814
- const targetPerSecond = 1.2;
815
- const desired = Math.round(safeDuration * targetPerSecond);
816
- const frameCount = Math.min(MAX_FRAMES, Math.max(MIN_FRAMES, desired));
817
- if (frameCount <= 0)
818
- return [];
819
- const effectiveInterval = safeDuration / frameCount;
820
- const outDir = path.join(tempDir, "vision-frames");
821
- await mkdir(outDir, { recursive: true });
822
- const outPattern = path.join(outDir, "frame-%04d.jpg");
823
- const fps = 1 / effectiveInterval;
824
- await runBinary(ffmpegBin, [
825
- "-y",
826
- "-i", videoPath,
827
- "-vf", `fps=${fps.toFixed(6)},scale=1120:-2`,
828
- "-q:v", "2",
829
- "-vframes", String(frameCount),
830
- outPattern
831
- ]);
832
- const results = [];
833
- for (let i = 0; i < frameCount; i += 1) {
834
- const framePath = path.join(outDir, `frame-${String(i + 1).padStart(4, "0")}.jpg`);
835
- let buf;
836
- try {
837
- buf = await readFile(framePath);
838
- }
839
- catch {
840
- // ffmpeg may emit fewer frames than requested for very short/edge-case
841
- // videos; skip missing frames instead of throwing so the vision pass
842
- // still runs with whatever we got.
843
- continue;
844
- }
845
- const timestamp = Math.min(safeDuration - 0.01, Math.max(0.01, i * effectiveInterval + effectiveInterval / 2));
846
- results.push({ timestamp: Number(timestamp.toFixed(3)), data: buf.toString("base64") });
847
- }
848
- return results;
849
- }
850
- // Extract frames at a dense fixed interval for OCR — this is what actually
851
- // grounds caption start/end timing. Interval defaults to 0.5s; tunable via
852
- // OCR_FRAME_INTERVAL_SECONDS. Cap total frames so a 5-minute video doesn't
853
- // fan out into hundreds of OCR calls.
854
- //
855
- // IMPORTANT: uses a SINGLE ffmpeg invocation with `-vf fps=…` to write all
856
- // frames in one pass. The previous per-frame ffmpeg spawn loop was ~1s per
857
- // frame and pushed 30s videos past the 60s CloudFront timeout on their own.
858
- async function extractDenseOcrFrames(videoPath, durationSeconds, tempDir) {
859
- const intervalRaw = Number(process.env.OCR_FRAME_INTERVAL_SECONDS ?? "0.5");
860
- const interval = Number.isFinite(intervalRaw) && intervalRaw > 0 ? Math.max(0.1, intervalRaw) : 0.5;
861
- const maxFramesRaw = Number(process.env.OCR_MAX_FRAMES ?? "60");
862
- const maxFrames = Number.isFinite(maxFramesRaw) && maxFramesRaw > 0 ? Math.floor(maxFramesRaw) : 60;
863
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : 5;
864
- const totalDesired = Math.max(1, Math.floor(safeDuration / interval));
865
- const totalFrames = Math.min(maxFrames, totalDesired);
866
- if (totalFrames <= 0)
867
- return [];
868
- const effectiveInterval = safeDuration / totalFrames;
869
- const outDir = path.join(tempDir, "ocr-frames");
870
- await mkdir(outDir, { recursive: true });
871
- const outPattern = path.join(outDir, "ocr-%04d.jpg");
872
- const fps = 1 / effectiveInterval;
873
- await runBinary(ffmpegBin, [
874
- "-y",
875
- "-i", videoPath,
876
- // fps filter samples at the target rate; scale keeps the frame small
877
- // enough that token cost stays down while text stays readable.
878
- "-vf", `fps=${fps.toFixed(6)},scale=640:-2`,
879
- "-q:v", "5",
880
- "-vframes", String(totalFrames),
881
- outPattern
882
- ]);
883
- const results = [];
884
- for (let i = 0; i < totalFrames; i += 1) {
885
- const framePath = path.join(outDir, `ocr-${String(i + 1).padStart(4, "0")}.jpg`);
886
- let buf;
887
- try {
888
- buf = await readFile(framePath);
889
- }
890
- catch {
891
- // ffmpeg may emit fewer frames than requested for very short videos
892
- // (rounding). Stop cleanly instead of throwing.
893
- break;
894
- }
895
- // Frame N (1-indexed) captures roughly at the midpoint of its interval.
896
- const timestamp = Math.min(safeDuration - 0.01, Math.max(0.01, i * effectiveInterval + effectiveInterval / 2));
897
- results.push({ timestamp: Number(timestamp.toFixed(3)), data: buf.toString("base64") });
898
- }
899
- return results;
900
- }
901
- // Prompt the vision model to return ONLY the overlay text visible in each
902
- // frame, verbatim, no interpretation. This is intentionally narrow so the
903
- // model behaves as an OCR engine rather than doing scene analysis (which
904
- // the sparse vision pass already handles).
905
- function buildOcrBatchPrompt(frames) {
906
- const frameList = frames
907
- .map((f, i) => ` ${i + 1}. frame @ ${f.timestamp.toFixed(2)}s`)
908
- .join("\n");
909
- return `You are an OCR engine. For each of the following ${frames.length} video frames, return every piece of overlay/caption text that appears in the image. Ignore any text that is part of the background scenery (signs, packaging, phone UIs, watermarks). Only report captions/subtitles/CTAs that were added on top of the video.
910
-
911
- Return ONLY strictly valid JSON with this schema:
912
- {
913
- "frames": [
914
- { "index": 1, "timestamp": <seconds>, "texts": ["exact text line 1", "exact text line 2"] },
915
- ...
916
- ]
917
- }
918
-
919
- Rules:
920
- - One entry per frame, in the same order given below.
921
- - "texts" is an array of text lines exactly as they appear (preserve case, punctuation, emoji). Split multi-line captions into one string per visible line.
922
- - If a frame has no overlay text, "texts" MUST be [].
923
- - Do NOT paraphrase, translate, or "clean up" text. Verbatim only.
924
- - Do NOT hallucinate text that isn't clearly visible.
925
-
926
- FRAMES (in order):
927
- ${frameList}`;
928
- }
929
- // Run the dense frames through the same vision fallback chain that
930
- // smartDecomposeVideo uses. Batches ~8 frames per call to bound token cost
931
- // per request while keeping request count low. Any batch that fails is
932
- // dropped — downstream code falls back to vision-provided timing for
933
- // captions that don't get an OCR match, so partial OCR is still useful.
934
- async function runOcrPass(input) {
935
- // Larger batches + higher concurrency keep the total OCR wall-time under
936
- // the 60s CloudFront read timeout for typical 28s vertical videos (56
937
- // dense frames). Gemini handles 12 image inputs per call fine and the
938
- // extra parallelism means 56 frames finish in ~1 round of 5 concurrent
939
- // batches instead of 2 rounds of 3.
940
- const BATCH_SIZE = 12;
941
- const CONCURRENCY = 5;
942
- const batches = [];
943
- for (let i = 0; i < input.frames.length; i += BATCH_SIZE) {
944
- batches.push(input.frames.slice(i, i + BATCH_SIZE));
945
- }
946
- const results = [];
947
- for (let i = 0; i < batches.length; i += CONCURRENCY) {
948
- const slice = batches.slice(i, i + CONCURRENCY);
949
- const settled = await Promise.allSettled(slice.map(async (batch) => {
950
- const prompt = buildOcrBatchPrompt(batch);
951
- const attempt = await callVisionWithFallback({ prompt, frames: batch, keys: input.keys });
952
- const record = (attempt.result && typeof attempt.result === "object") ? attempt.result : {};
953
- const framesRaw = Array.isArray(record.frames) ? record.frames : [];
954
- const parsed = [];
955
- for (let idx = 0; idx < batch.length; idx += 1) {
956
- const entry = framesRaw[idx];
957
- const timestamp = batch[idx].timestamp;
958
- const textsRaw = entry && Array.isArray(entry.texts) ? entry.texts : [];
959
- const texts = textsRaw
960
- .map((t) => (typeof t === "string" ? t : ""))
961
- .filter((t) => t.trim().length > 0);
962
- parsed.push({ timestamp, texts });
963
- }
964
- return parsed;
965
- }));
966
- for (const s of settled) {
967
- if (s.status === "fulfilled") {
968
- results.push(...s.value);
969
- }
970
- else {
971
- console.error("smart decompose OCR batch failed", s.reason instanceof Error ? s.reason.message : s.reason);
972
- }
973
- }
974
- }
975
- results.sort((a, b) => a.timestamp - b.timestamp);
976
- return results;
977
- }
978
- function normalizeForOcrMatch(text) {
979
- return text
980
- .toLowerCase()
981
- .replace(/[\u2018\u2019\u201C\u201D]/g, "'")
982
- .replace(/[^\p{Letter}\p{Number}\s]/gu, " ")
983
- .replace(/\s+/g, " ")
984
- .trim();
985
- }
986
- // Overlap ratio: what fraction of the caption's significant words appear in
987
- // the frame's concatenated OCR text? Stopwords-free would be nicer but for
988
- // short-form video captions ("single", "have kids", "buy my course") the
989
- // full word list is fine and simpler.
990
- function captionOverlapWithFrame(captionText, frame) {
991
- const captionWords = normalizeForOcrMatch(captionText).split(" ").filter((w) => w.length >= 2);
992
- if (captionWords.length === 0)
993
- return 0;
994
- const frameText = " " + normalizeForOcrMatch(frame.texts.join(" ")) + " ";
995
- if (frameText.trim().length === 0)
996
- return 0;
997
- let hits = 0;
998
- for (const word of captionWords) {
999
- if (frameText.includes(" " + word + " "))
1000
- hits += 1;
1001
- }
1002
- return hits / captionWords.length;
1003
- }
1004
- // For each vision-detected caption, find the contiguous run of OCR frames
1005
- // where the caption's text is present, and snap start/duration to that run.
1006
- // If no OCR match is confident enough, keep the vision-provided timing
1007
- // (graceful degradation).
1008
- function snapCaptionTimingsToOcr(input) {
1009
- if (input.ocrFrames.length === 0) {
1010
- return { captions: input.captions, snappedCount: 0 };
1011
- }
1012
- const MATCH_THRESHOLD = 0.6;
1013
- // Half the median inter-frame gap — used to extend duration past the last
1014
- // matching frame since the caption presumably stayed on-screen partway
1015
- // into the next (non-matching) frame.
1016
- const gaps = [];
1017
- for (let i = 1; i < input.ocrFrames.length; i += 1) {
1018
- gaps.push(input.ocrFrames[i].timestamp - input.ocrFrames[i - 1].timestamp);
1019
- }
1020
- const sortedGaps = gaps.slice().sort((a, b) => a - b);
1021
- const medianGap = sortedGaps.length > 0 ? sortedGaps[Math.floor(sortedGaps.length / 2)] : 0.5;
1022
- const halfGap = Math.max(0.05, medianGap / 2);
1023
- let snappedCount = 0;
1024
- const snapped = input.captions.map((caption) => {
1025
- const scores = input.ocrFrames.map((frame) => captionOverlapWithFrame(caption.text, frame));
1026
- const matchIndexes = scores
1027
- .map((s, i) => ({ s, i }))
1028
- .filter((x) => x.s >= MATCH_THRESHOLD)
1029
- .map((x) => x.i);
1030
- if (matchIndexes.length === 0) {
1031
- return caption;
1032
- }
1033
- // Pick the contiguous run whose midpoint is closest to the vision-
1034
- // provided midpoint. This handles the case of a repeated caption
1035
- // (same text shown twice with a gap in between) by choosing the
1036
- // occurrence the vision pass was pointing at.
1037
- const runs = [];
1038
- let runStart = matchIndexes[0];
1039
- let prev = matchIndexes[0];
1040
- for (let i = 1; i < matchIndexes.length; i += 1) {
1041
- const idx = matchIndexes[i];
1042
- if (idx - prev <= 2) {
1043
- prev = idx;
1044
- }
1045
- else {
1046
- runs.push({ start: runStart, end: prev });
1047
- runStart = idx;
1048
- prev = idx;
1049
- }
1050
- }
1051
- runs.push({ start: runStart, end: prev });
1052
- const visionMid = caption.start + caption.duration / 2;
1053
- const bestRun = runs.reduce((best, r) => {
1054
- const runMid = (input.ocrFrames[r.start].timestamp + input.ocrFrames[r.end].timestamp) / 2;
1055
- const bestMid = (input.ocrFrames[best.start].timestamp + input.ocrFrames[best.end].timestamp) / 2;
1056
- return Math.abs(runMid - visionMid) < Math.abs(bestMid - visionMid) ? r : best;
1057
- }, runs[0]);
1058
- const firstTs = input.ocrFrames[bestRun.start].timestamp;
1059
- const lastTs = input.ocrFrames[bestRun.end].timestamp;
1060
- const newStart = Math.max(0, firstTs - halfGap);
1061
- const newEnd = Math.min(input.durationSeconds, lastTs + halfGap);
1062
- const newDuration = Math.max(0.1, newEnd - newStart);
1063
- snappedCount += 1;
1064
- return {
1065
- ...caption,
1066
- start: Number(newStart.toFixed(3)),
1067
- duration: Number(newDuration.toFixed(3))
1068
- };
1069
- });
1070
- return { captions: snapped, snappedCount };
1071
- }
1072
- // Snap vision-provided scene boundaries onto ffmpeg-detected hard cuts, the
1073
- // scene-layer analogue of snapCaptionTimingsToOcr. Each scene "start" that
1074
- // falls within TOLERANCE of a detected cut is pulled exactly onto that cut;
1075
- // durations are then recomputed so scenes stay contiguous and still cover the
1076
- // full duration (start[0] forced to 0, last scene extended to the end). If no
1077
- // cuts were detected the scenes are returned unchanged.
1078
- function snapScenesToCuts(input) {
1079
- if (input.cuts.length === 0 || input.scenes.length === 0) {
1080
- return { scenes: input.scenes, snappedCount: 0 };
1081
- }
1082
- const TOLERANCE = 0.6;
1083
- const sorted = [...input.scenes].sort((a, b) => a.start - b.start);
1084
- let snappedCount = 0;
1085
- const snappedStarts = sorted.map((scene, index) => {
1086
- if (index === 0)
1087
- return 0; // first scene always starts at 0
1088
- let nearest = scene.start;
1089
- let bestDelta = TOLERANCE;
1090
- for (const cut of input.cuts) {
1091
- const delta = Math.abs(cut - scene.start);
1092
- if (delta < bestDelta) {
1093
- bestDelta = delta;
1094
- nearest = cut;
1095
- }
1096
- }
1097
- if (nearest !== scene.start)
1098
- snappedCount += 1;
1099
- return nearest;
1100
- });
1101
- const rebuilt = sorted.map((scene, index) => {
1102
- const start = Number(snappedStarts[index].toFixed(3));
1103
- const nextStart = index + 1 < snappedStarts.length ? snappedStarts[index + 1] : input.durationSeconds;
1104
- const duration = Math.max(0.1, Number((nextStart - start).toFixed(3)));
1105
- return { ...scene, start, duration };
1106
- }).filter((s) => s.duration > 0.05);
1107
- return { scenes: rebuilt, snappedCount };
1108
- }
1109
- // Product-AGNOSTIC placement-surface prompt. Deliberately does NOT know the
1110
- // user's product — it enumerates reusable slots where any product could later
1111
- // be inserted. The product-SPECIFIC pitch stays the on-demand
1112
- // brainstorm_product_placement primitive, which these surfaces can seed.
1113
- function buildPlacementSurfacePrompt(input) {
1114
- const frameListing = input.frameTimestamps.length > 0
1115
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
1116
- : "";
1117
- const sceneSlugList = input.sceneSlugs.length > 0
1118
- ? `\nSCENE SLUGS (link each surface to the closest one via "scene_slug"): ${input.sceneSlugs.join(", ")}\n`
1119
- : "";
1120
- return `You are Vidfarm's native-advertising product-placement scout. You are studying a short-form vertical video (TikTok/Reel) to find natural PLACEMENT SURFACES — moments and spots where a product could later be inserted so it feels native, not bolted on.
1121
-
1122
- CRITICAL: You do NOT know the product yet. Do NOT invent a specific product. Describe the SLOT itself (what is physically/attention-wise available), and list the KINDS of products that would fit. Think of this as reusable metadata that a later, product-specific pass will draw from.
1123
-
1124
- Video duration: ${input.durationSeconds.toFixed(2)} seconds.
1125
- ${frameListing}${sceneSlugList}
1126
- Return strictly valid JSON matching this schema:
1127
- {
1128
- "surfaces": [{
1129
- "slug": string,
1130
- "timestamp": string,
1131
- "start": number,
1132
- "scene_slug": string,
1133
- "surface_type": string,
1134
- "surface_note": string,
1135
- "best_for": [string, ...],
1136
- "why_it_works": string
1137
- }]
1138
- }
1139
-
1140
- RULES:
1141
- - Identify 3-8 distinct surfaces, ordered strongest to weakest. Each must reference a real moment you actually see in the frames.
1142
- - "slug": short snake_case role label, e.g. "desk_prop_slot", "phone_screen_swap", "hand_held_reveal", "b_roll_cutaway", "end_card_cta". Unique.
1143
- - "timestamp": human range like "0:04" or "0:12-0:18". "start": the numeric seconds (float) where the surface begins — use it for editor targeting.
1144
- - "scene_slug": the closest scene slug from the list above (or "" if none given).
1145
- - "surface_type": one of prop | set_dressing | screen_ui | verbal_mention | caption_overlay | sticker_lower_third | b_roll_cutaway | apparel | before_after_demo | end_card.
1146
- - "surface_note": describe the SLOT product-agnostically — what is physically there and how long it holds (e.g. "empty desk surface faces camera, unobstructed for ~2s", "subject holds phone screen toward camera", "blank wall behind subject at chest height").
1147
- - "best_for": 2-4 product/offer CATEGORIES that would fit this slot naturally (e.g. "beverage/snack brand", "SaaS dashboard", "skincare bottle", "mobile app UI"). Categories, never a specific brand.
1148
- - "why_it_works": the attention/emotion/story logic that makes a placement here land.
1149
- - Bias toward surfaces that ride existing attention or story beats. If the video genuinely has no natural surface, return "surfaces": []. Never fabricate.`;
1150
- }
1151
- function normalizePlacementSurfaces(raw) {
1152
- const record = (raw && typeof raw === "object") ? raw : {};
1153
- const surfacesRaw = Array.isArray(record.surfaces) ? record.surfaces : [];
1154
- const usedSlugs = new Set();
1155
- return surfacesRaw
1156
- .map((s, index) => {
1157
- const slug = dedupeSlug(normalizeSlug(typeof s.slug === "string" ? s.slug : `surface_${index + 1}`) || `surface_${index + 1}`, usedSlugs);
1158
- const bestForRaw = Array.isArray(s.best_for) ? s.best_for : [];
1159
- const best_for = bestForRaw
1160
- .map((b) => (typeof b === "string" ? b.trim() : ""))
1161
- .filter((b) => b.length > 0)
1162
- .slice(0, 6);
1163
- return {
1164
- slug,
1165
- timestamp: typeof s.timestamp === "string" ? s.timestamp : "",
1166
- start: Number.isFinite(Number(s.start)) ? Number(Number(s.start).toFixed(3)) : 0,
1167
- scene_slug: typeof s.scene_slug === "string" ? normalizeSlug(s.scene_slug) : "",
1168
- surface_type: typeof s.surface_type === "string" ? s.surface_type : "prop",
1169
- surface_note: typeof s.surface_note === "string" ? s.surface_note : "",
1170
- best_for,
1171
- why_it_works: typeof s.why_it_works === "string" ? s.why_it_works : ""
1172
- };
1173
- })
1174
- .filter((s) => s.surface_note.trim().length > 0);
1175
- }
1176
- // Isolated placement-surface vision call. Reuses the SAME sparse frames already
1177
- // sampled for the main pass — no extra ffmpeg — and runs as its own request so
1178
- // it can never dilute the caption/scene call's attention budget. Non-fatal:
1179
- // callers wrap in .catch(() => []).
1180
- async function runPlacementSurfacePass(input) {
1181
- if (input.frames.length === 0)
1182
- return [];
1183
- const prompt = buildPlacementSurfacePrompt({
1184
- durationSeconds: input.durationSeconds,
1185
- frameTimestamps: input.frames.map((f) => f.timestamp),
1186
- sceneSlugs: input.sceneSlugs
1187
- });
1188
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
1189
- return normalizePlacementSurfaces(attempt.result);
1190
- }
1191
- // ---------------------------------------------------------------------------
1192
- // Scene-annotation pass — clip-replacement DNA.
1193
- //
1194
- // A dependent vision pass (runs AFTER scenes are finalized, so annotations line
1195
- // up 1:1 by scene slug) that (1) classifies the whole video's content FORMAT and
1196
- // (2) for each scene emits the literal + viral DNA needed to find a REPLACEMENT
1197
- // clip from the clip library. It reuses the sparse frames already sampled for
1198
- // the main pass (no extra ffmpeg) and the clip library's own controlled tag
1199
- // vocabulary + embedding recipe, so a scene's clip-match query is symmetric with
1200
- // how a candidate clip was tagged/embedded. Non-fatal: callers wrap in .catch().
1201
- // ---------------------------------------------------------------------------
1202
- // Vocabulary for a scene's role in the retention arc. Constrained in the prompt,
1203
- // clamped here; free-form-ish (unknown → "context") so it never blocks output.
1204
- const SCENE_ROLE_VOCAB = [
1205
- "hook", "build", "reveal", "payoff", "transition", "cta", "proof", "context", "punchline", "loop"
1206
- ];
1207
- function asPlainRecord(value) {
1208
- return value && typeof value === "object" && !Array.isArray(value) ? value : {};
1209
- }
1210
- function readAnnotationString(value) {
1211
- return typeof value === "string" ? value.trim() : "";
1212
- }
1213
- function readAnnotationStringList(value, max) {
1214
- if (!Array.isArray(value))
1215
- return [];
1216
- const out = [];
1217
- for (const v of value) {
1218
- const s = typeof v === "string" ? v.trim() : "";
1219
- if (s && !out.includes(s))
1220
- out.push(s);
1221
- if (out.length >= max)
1222
- break;
1223
- }
1224
- return out;
1225
- }
1226
- function humanizeContentFormat(format) {
1227
- return format
1228
- .split("_")
1229
- .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
1230
- .join(" ");
1231
- }
1232
- function normalizeContentFormat(raw) {
1233
- const rec = asPlainRecord(raw);
1234
- const slug = normalizeSlug(readAnnotationString(rec.format));
1235
- const format = CONTENT_FORMATS.includes(slug)
1236
- ? slug
1237
- : "other";
1238
- const secondaryRaw = Array.isArray(rec.secondary) ? rec.secondary : [];
1239
- const secondary = Array.from(new Set(secondaryRaw
1240
- .map((s) => normalizeSlug(readAnnotationString(s)))
1241
- .filter((s) => CONTENT_FORMATS.includes(s) && s !== format))).slice(0, 3);
1242
- const conf = Number(rec.confidence);
1243
- const label = readAnnotationString(rec.label);
1244
- return {
1245
- format,
1246
- label: label || humanizeContentFormat(format),
1247
- confidence: Number.isFinite(conf) ? Math.max(0, Math.min(1, conf)) : 0,
1248
- rationale: readAnnotationString(rec.rationale),
1249
- secondary
1250
- };
1251
- }
1252
- function normalizeSceneRole(value) {
1253
- const slug = normalizeSlug(readAnnotationString(value));
1254
- return SCENE_ROLE_VOCAB.includes(slug) ? slug : "context";
1255
- }
1256
- function normalizeReplaceability(value) {
1257
- const s = readAnnotationString(value).toLowerCase();
1258
- return s === "high" || s === "low" ? s : "medium";
1259
- }
1260
- // Deterministic per-scene spoken text pulled from the transcript segments that
1261
- // overlap the scene window — more reliable than asking the vision model to
1262
- // re-transcribe, and keeps `tags.transcript` grounded in the audio pass.
1263
- function transcriptForScene(transcript, scene) {
1264
- if (!transcript || !Array.isArray(transcript.segments))
1265
- return "";
1266
- const end = scene.start + scene.duration;
1267
- return transcript.segments
1268
- .filter((seg) => seg.end > scene.start && seg.start < end)
1269
- .map((seg) => seg.text.trim())
1270
- .filter((t) => t.length > 0)
1271
- .join(" ")
1272
- .trim();
1273
- }
1274
- // Compose the retrieval query text. Leads with the clip-doc recipe (description +
1275
- // tags) so it lands near literally-similar clip embeddings, then appends the
1276
- // viral purpose so the vector also reflects the beat's INTENT — a replacement
1277
- // must match both axes.
1278
- function buildSceneEmbeddingText(input) {
1279
- const literalDoc = buildClipEmbeddingText(input.tags, input.literal);
1280
- const purpose = [
1281
- input.scene_role ? `Role: ${input.scene_role}` : "",
1282
- input.narrative_purpose,
1283
- input.why_it_works
1284
- ].filter((p) => p && p.length > 0).join(". ");
1285
- return purpose ? `${literalDoc}\nPurpose: ${purpose}` : literalDoc;
1286
- }
1287
- // Turn a scene's literal tags into a ready-to-run SplicingCriteria for the clip
1288
- // library. Tag constraints are the high-precision structured filter; the
1289
- // duration floor lets candidates be trimmed to fit; has_subject/has_dialogue are
1290
- // derived from the tags. semantic_text carries the embedding query for re-rank.
1291
- function buildSceneMatchCriteria(tags, scene, semanticText) {
1292
- const criteria = { semantic_text: semanticText };
1293
- if (tags.subject.length)
1294
- criteria.subject = tags.subject;
1295
- if (tags.action.length)
1296
- criteria.action = tags.action;
1297
- if (tags.emotion.length)
1298
- criteria.emotion = tags.emotion;
1299
- if (tags.setting.length)
1300
- criteria.setting = tags.setting;
1301
- if (tags.composition.length)
1302
- criteria.composition = tags.composition;
1303
- if (tags.motion.length)
1304
- criteria.motion = tags.motion;
1305
- if (tags.audio_type.length)
1306
- criteria.audio_type = tags.audio_type;
1307
- if (tags.energy)
1308
- criteria.energy = [tags.energy];
1309
- if (Number.isFinite(scene.duration) && scene.duration > 0) {
1310
- criteria.min_duration_sec = Number(Math.max(0, scene.duration * 0.6).toFixed(2));
1311
- }
1312
- const hasRealSubject = tags.subject.some((s) => s !== "no_subject");
1313
- if (hasRealSubject)
1314
- criteria.has_subject = true;
1315
- else if (tags.subject.includes("no_subject"))
1316
- criteria.has_subject = false;
1317
- if (tags.audio_type.includes("dialogue"))
1318
- criteria.has_dialogue = true;
1319
- return criteria;
1320
- }
1321
- function buildSceneAnnotationPrompt(input) {
1322
- const frameListing = input.frameTimestamps.length > 0
1323
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
1324
- : "";
1325
- const sceneListing = input.scenes.map((s) => (` - scene_slug="${s.slug}" @ ${s.start.toFixed(2)}s for ${s.duration.toFixed(2)}s — ${s.label}: ${s.description || "(no description)"}${s.viral_note ? ` [viral note: ${s.viral_note}]` : ""}`)).join("\n");
1326
- const dna = input.viralDna;
1327
- const dnaBlock = `Video-level viral DNA (context for each scene's PURPOSE):\n hook: ${dna.hook || "—"}\n retention: ${dna.retention || "—"}\n payoff: ${dna.payoff || "—"}\n preserve: ${(dna.preserve || []).join("; ") || "—"}\n avoid: ${(dna.avoid || []).join("; ") || "—"}`;
1328
- const userGuidance = input.userPrompt && input.userPrompt.trim().length > 0
1329
- ? `\nUSER GUIDANCE (highest priority):\n"""\n${input.userPrompt.trim()}\n"""\n`
1330
- : "";
1331
- return `You are Vidfarm's scene annotator. For EACH scene of this short-form vertical video, write an annotation rich enough to do TWO things: (A) find a REPLACEMENT clip from a searchable clip library — a different piece of footage that could stand in for this scene without breaking the video; and (B) let an AI agent RE-CREATE this scene from scratch with an image/video generator when no library clip fits.
1332
-
1333
- Both goals hang on capturing the same TWO axes:
1334
- 1) LITERAL DNA — what is physically on screen (subject, action, setting, shot, motion, mood). A candidate or generated clip must look and feel like this.
1335
- 2) VIRAL / NARRATIVE DNA — the ROLE this beat plays in why the video works (the hook, the reveal, the payoff, a transition...). A replacement or regeneration must serve the same purpose in the story, or the swap kills retention.
1336
- ${userGuidance}
1337
- FIRST classify the whole video's FORMAT (its production archetype) — what counts as "literal DNA" differs by format. Pick the single best "format" and list any close seconds in "secondary". Allowed formats:
1338
- ${CONTENT_FORMATS.join(", ")}
1339
- Format guidance: ai_fruit_drama / ai_generative_skit = AI-generated surreal character dramas (e.g. talking fruit); ugc_trend = phone-shot creator-to-camera trend; talking_head = single speaker giving advice; educational_series = explainer / how-to / listicle; live_action = real filmed scenes / actors; product_demo = hands-on showcase / unboxing; street_interview = man-on-the-street; reaction_commentary = reacting to other footage; cinematic_broll = moody montage / atmosphere; screen_recording = app / screen capture; animation_motion_graphics = animation / kinetic text.
1340
-
1341
- Video duration: ${input.durationSeconds.toFixed(2)}s. One-line summary: ${input.summary || "(none)"}.
1342
- ${dnaBlock}
1343
- ${frameListing}
1344
- SCENES to annotate (return EXACTLY one annotation per scene_slug below, in this order):
1345
- ${sceneListing}
1346
-
1347
- For the LITERAL "tags", use ONLY values from this controlled vocabulary — it is the SAME vocabulary the clip library is tagged with, so your tags must line up with how a candidate clip would be tagged:
1348
- ${taxonomyForPrompt()}
1349
-
1350
- Return strictly valid JSON matching this schema:
1351
- {
1352
- "content_format": { "format": string, "label": string, "confidence": number, "rationale": string, "secondary": [string, ...] },
1353
- "scenes": [{
1354
- "scene_slug": string,
1355
- "literal": string,
1356
- "tags": { "subject": [string], "action": [string], "emotion": [string], "setting": [string], "composition": [string], "motion": [string], "energy": "low"|"medium"|"high", "audio_type": [string], "transcript": string },
1357
- "scene_role": string,
1358
- "narrative_purpose": string,
1359
- "why_it_works": string,
1360
- "must_preserve": [string, ...],
1361
- "can_vary": [string, ...],
1362
- "replaceability": "high"|"medium"|"low",
1363
- "replaceability_reason": string,
1364
- "recreation_prompt": string,
1365
- "recreation_notes": [string, ...]
1366
- }]
1367
- }
1368
-
1369
- RULES:
1370
- - Output one scene object per scene_slug listed above, using the EXACT scene_slug strings. Do not invent, merge, split, or reorder scenes.
1371
- - "literal": one concrete, factual sentence describing exactly what is on screen in this scene (subject + action + setting + shot). This is the primary match key — be specific and visual, not interpretive.
1372
- - "tags": pick only vocabulary values you are confident about from the frames; omit a category (empty list) rather than guess. "energy" is a single value. "transcript" = words spoken in this scene verbatim, or "" if none.
1373
- - "scene_role": one of ${SCENE_ROLE_VOCAB.join(", ")} — the function this beat serves in the retention arc.
1374
- - "narrative_purpose" / "why_it_works": the viral DNA — WHY this beat exists and how it holds attention. A replacement clip must reproduce this effect, not just the look.
1375
- - "must_preserve": 1-4 things a replacement MUST keep (e.g. "subject facing camera", "surprised reaction on the beat drop", "product clearly visible"). "can_vary": 1-4 things free to change (specific person, exact background, color grade).
1376
- - "replaceability": how safely this scene can be swapped from generic library clips. "low" for scenes carrying unique baked-in text / a specific face / a spoken payoff; "high" for generic b-roll / cutaways / establishing shots. Explain in "replaceability_reason".
1377
- - "recreation_prompt": a self-contained generative brief an AI agent could paste into a text-to-image/video model to REGENERATE this exact beat — describe subject, action, setting, framing, camera move, lighting, mood, art style, and pacing in one rich paragraph. Write it in the idiom of the detected format (e.g. an ai_fruit_drama prompt reads like a stylized 3D-character render brief; a live_action prompt reads like a cinematography direction). Do NOT reference "this video" or timestamps — it must stand alone.
1378
- - "recreation_notes": 2-5 short discrete production directives that back up the recreation_prompt (e.g. "9:16 vertical", "handheld push-in", "soft key light from left", "on-screen text: 'wait for it'", "cut on the beat at ~1.2s", "dialogue VO: <verbatim line>"). Include any exact on-screen text or spoken line that must be reproduced.
1379
- - Match your vocabulary and framing to the detected format above.`;
1380
- }
1381
- async function runSceneAnnotationPass(input) {
1382
- if (input.frames.length === 0 || input.scenes.length === 0) {
1383
- return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null };
1384
- }
1385
- const prompt = buildSceneAnnotationPrompt({
1386
- durationSeconds: input.durationSeconds,
1387
- frameTimestamps: input.frames.map((f) => f.timestamp),
1388
- scenes: input.scenes,
1389
- viralDna: input.viralDna,
1390
- summary: input.summary,
1391
- userPrompt: input.userPrompt ?? null
1392
- });
1393
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
1394
- const record = asPlainRecord(attempt.result);
1395
- const contentFormat = normalizeContentFormat(record.content_format);
1396
- const rawScenes = Array.isArray(record.scenes) ? record.scenes : [];
1397
- const bySlug = new Map();
1398
- for (const r of rawScenes) {
1399
- const rec = asPlainRecord(r);
1400
- const slug = normalizeSlug(readAnnotationString(rec.scene_slug));
1401
- if (slug && !bySlug.has(slug))
1402
- bySlug.set(slug, rec);
1403
- }
1404
- // Iterate the FINALIZED scenes (not the model output) so every scene gets
1405
- // exactly one annotation aligned by slug, even if the model dropped, renamed,
1406
- // or duplicated one. Missing fields degrade to the scene's own description.
1407
- const annotations = input.scenes.map((scene) => {
1408
- const raw = bySlug.get(scene.slug) ?? {};
1409
- const tags = {
1410
- ...sanitizeTagValues(asPlainRecord(raw.tags)),
1411
- transcript: transcriptForScene(input.transcript, scene) || readAnnotationString(asPlainRecord(raw.tags).transcript)
1412
- };
1413
- const literal = readAnnotationString(raw.literal) || scene.description || scene.label;
1414
- const scene_role = normalizeSceneRole(raw.scene_role);
1415
- const narrative_purpose = readAnnotationString(raw.narrative_purpose) || scene.viral_note;
1416
- const why_it_works = readAnnotationString(raw.why_it_works);
1417
- const embedding_text = buildSceneEmbeddingText({ literal, tags, scene_role, narrative_purpose, why_it_works });
1418
- return {
1419
- scene_slug: scene.slug,
1420
- start: scene.start,
1421
- duration: scene.duration,
1422
- content_format: contentFormat.format,
1423
- literal,
1424
- tags,
1425
- scene_role,
1426
- narrative_purpose,
1427
- why_it_works,
1428
- must_preserve: readAnnotationStringList(raw.must_preserve, 4),
1429
- can_vary: readAnnotationStringList(raw.can_vary, 4),
1430
- replaceability: normalizeReplaceability(raw.replaceability),
1431
- replaceability_reason: readAnnotationString(raw.replaceability_reason),
1432
- embedding_text,
1433
- match_criteria: buildSceneMatchCriteria(tags, scene, embedding_text),
1434
- // Fall back to the literal + purpose if the model omitted a recreation
1435
- // brief, so an agent always has *something* to regenerate from.
1436
- recreation_prompt: readAnnotationString(raw.recreation_prompt)
1437
- || [literal, narrative_purpose].filter((p) => p.length > 0).join(" "),
1438
- recreation_notes: readAnnotationStringList(raw.recreation_notes, 6)
1439
- };
1440
- });
1441
- return { contentFormat, annotations, provider: attempt.provider, model: attempt.model };
1442
- }
1443
- // Standalone entry point for the ASYNC scene-annotation pass, invoked once per
1444
- // /scene-annotations-poll. Mirrors analyzeCastMembers: it re-downloads the
1445
- // (caption-free) source, re-samples the same sparse frames the main decompose
1446
- // used, and runs the annotation vision call against the ALREADY-finalized scenes
1447
- // (passed in from the persisted smart-decompose snapshot) so annotations align
1448
- // 1:1 by slug. Kept out of the synchronous decompose so that request stays under
1449
- // the origin timeout.
1450
- export async function annotateScenesFromSource(input) {
1451
- if (input.scenes.length === 0) {
1452
- return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null, durationSeconds: 0 };
1453
- }
1454
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
1455
- const sourcePath = path.join(tempDir, "source.mp4");
1456
- const startedAt = Date.now();
1457
- devLog("hyperframes.scene_annotations.start", { source_url: input.sourceUrl, scene_count: input.scenes.length });
1458
- try {
1459
- await downloadVideoTo(input.sourceUrl, sourcePath);
1460
- const meta = await ffprobeVideo(sourcePath);
1461
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
1462
- throw new Error("Could not read video duration");
1463
- }
1464
- const frames = await sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir);
1465
- const outcome = await runSceneAnnotationPass({
1466
- frames,
1467
- keys: input.keys,
1468
- durationSeconds: meta.durationSeconds,
1469
- scenes: input.scenes,
1470
- viralDna: input.viralDna,
1471
- summary: input.summary,
1472
- transcript: input.transcript,
1473
- userPrompt: input.userPrompt ?? null
1474
- });
1475
- devLog("hyperframes.scene_annotations.completed", {
1476
- provider: outcome.provider,
1477
- model: outcome.model,
1478
- duration_ms: Date.now() - startedAt,
1479
- content_format: outcome.contentFormat.format,
1480
- annotation_count: outcome.annotations.length
1481
- });
1482
- return { ...outcome, durationSeconds: meta.durationSeconds };
1483
- }
1484
- catch (error) {
1485
- devLog("hyperframes.scene_annotations.failed", {
1486
- source_url: input.sourceUrl,
1487
- duration_ms: Date.now() - startedAt,
1488
- ...devErrorFields(error)
1489
- }, "error");
1490
- throw error;
1491
- }
1492
- finally {
1493
- await rm(tempDir, { recursive: true, force: true });
1494
- }
1495
- }
1496
- // The frames handed to the vision passes are SILENT stills — but a short-form
1497
- // video's emotional punch is often carried by its AUDIO (the trending sound, a
1498
- // music-bed drop, vocal intonation, a punchline's delivery). This digest folds
1499
- // the transcript pass into a compact block so the viral-DNA and editor-harness
1500
- // prompts can reason about the sound, not just the pixels. Music-only clips
1501
- // (no speech) still get a note so the model infers a trending-sound bed rather
1502
- // than guessing there is silence.
1503
- function buildAudioDigestForPrompt(transcript) {
1504
- if (!transcript) {
1505
- return `\nAUDIO TRACK: none detected, or transcription unavailable. Judge emotional punch from the visuals, on-screen text, and pacing alone — and do not fabricate spoken lines, music, or a trending sound you cannot infer.\n`;
1506
- }
1507
- const text = (transcript.text || "").trim();
1508
- const clipped = text.length > 1200 ? `${text.slice(0, 1200)} …` : text;
1509
- const segs = transcript.segments.slice(0, 24)
1510
- .map((s) => ` [${s.start.toFixed(1)}-${s.end.toFixed(1)}s] ${s.text.trim()}`)
1511
- .filter((line) => line.trim().length > 0)
1512
- .join("\n");
1513
- return `\nAUDIO TRANSCRIPT — the video's spoken/sung audio. The SOUND is a primary driver of emotional punch: read the words, the intonation and cadence they imply, WHERE the delivery lands (the pause before a punchline, an exclamation, a beat-synced line), and infer any music / trending-sound bed even when there are no lyrics.
1514
- - language: ${transcript.language ?? "unknown"}
1515
- - full text: ${clipped || "(no speech transcribed — likely a music-only or trending-sound bed; treat the audio as a music/sound-driven format and reason about the beat/drop, not dialogue)"}${segs ? `\n- timed lines:\n${segs}` : ""}
1516
- `;
1517
- }
1518
- function buildSmartDecomposePrompt(input) {
1519
- const sceneCutHint = input.sceneCuts && input.sceneCuts.length > 0
1520
- ? `\nDETECTED SCENE CUTS (authoritative — ffmpeg measured hard visual cuts at these timestamps, in seconds):\n${input.sceneCuts.map((t) => ` cut @ ${t.toFixed(2)}s`).join("\n")}\n\nThese are ground truth for where the video visually changes. Align scene boundaries to these cuts: each detected cut should begin a new scene (a scene "start" should equal a detected cut time, within ~0.15s), and do NOT place a scene boundary where there is no detected cut unless the frames clearly show a change the detector missed. If two adjacent cuts are very close and belong to the same beat, you may merge them into one scene, but never exceed 6 scenes.\n`
1521
- : "";
1522
- const userGuidance = input.userPrompt && input.userPrompt.trim().length > 0
1523
- ? `\nUSER GUIDANCE (highest priority — pay special attention to this when deciding scenes, captions, and viral_dna):\n"""\n${input.userPrompt.trim()}\n"""\n`
1524
- : "";
1525
- const trendTaglineHint = input.trendTaglineHint && input.trendTaglineHint.trim().length > 0
1526
- ? `\nTREND TAGLINE (caller-supplied, use verbatim as viral_dna.trend_tagline — do NOT rephrase):\n"""\n${input.trendTaglineHint.trim()}\n"""\n`
1527
- : "";
1528
- const frameListing = input.frameTimestamps.length > 0
1529
- ? `\nFRAMES SUPPLIED (in order — each image is captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n\nUse the frame timestamps to determine caption "start" and "duration". Cross-reference consecutive frames: a caption whose text is identical across frames N and M was on-screen from at least the timestamp of frame N through the timestamp of frame M (extend "duration" accordingly). A caption present in only one frame is short-lived — set duration to roughly (next_frame_timestamp - this_frame_timestamp) or 0.8s, whichever is smaller.\n`
1530
- : "";
1531
- const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
1532
- return `You are decomposing a short-form vertical video (TikTok/Reel) into editable HyperFrames timeline layers.
1533
- ${userGuidance}${trendTaglineHint}
1534
- Video metadata:
1535
- - duration: ${input.durationSeconds.toFixed(3)} seconds
1536
- - dimensions: ${input.width} x ${input.height} (9:16 vertical, canvas is 100% x 100%)
1537
- ${frameListing}${sceneCutHint}${audioDigest}
1538
-
1539
- Return strictly valid JSON matching this schema:
1540
- {
1541
- "summary": string,
1542
- "viral_dna": {
1543
- "trend_tagline": string,
1544
- "hook": string,
1545
- "retention": string,
1546
- "payoff": string,
1547
- "emotional_punch": {
1548
- "core_emotion": string,
1549
- "tone": string,
1550
- "arc": string,
1551
- "peak_moment": string,
1552
- "mechanism": string,
1553
- "contrast": string,
1554
- "humor": string,
1555
- "audio_visual_tension": string,
1556
- "delivery": string,
1557
- "preserve": [string, ...]
1558
- },
1559
- "static_vs_pivot": {
1560
- "summary": string,
1561
- "scene_replacement": {
1562
- "overall": "replace_most" | "replace_some" | "keep_most" | "keep_all",
1563
- "reason": string,
1564
- "must_keep": [string, ...],
1565
- "should_replace": [string, ...],
1566
- "load_bearing_scenes": [string, ...]
1567
- },
1568
- "narration": {
1569
- "overall": "keep_original" | "customize_script" | "optional",
1570
- "reason": string,
1571
- "same_timing_required": boolean,
1572
- "use_premium_tts_when_customizing": boolean
1573
- },
1574
- "music": {
1575
- "overall": "keep_original" | "recreate_under_narration" | "optional" | "none",
1576
- "reason": string,
1577
- "volume_guidance": string
1578
- },
1579
- "captions": {
1580
- "overall": "reuse_existing" | "retime_to_new_narration" | "optional" | "none",
1581
- "reason": string,
1582
- "animation_dependency": string
1583
- },
1584
- "visual_rebuild": {
1585
- "default_strategy": "ai_images_with_ken_burns" | "real_footage" | "mixed",
1586
- "reason": string
1587
- },
1588
- "viral_dna": {
1589
- "must_keep": [string, ...],
1590
- "flexible": [string, ...],
1591
- "do_not_change": [string, ...]
1592
- }
1593
- },
1594
- "preserve": [string, ...],
1595
- "avoid": [string, ...],
1596
- "promotions": [string, ...],
1597
- "keywords": [string, ...]
1598
- },
1599
- "scenes": [{
1600
- "slug": string,
1601
- "start": number,
1602
- "duration": number,
1603
- "label": string,
1604
- "description": string,
1605
- "viral_note": string
1606
- }],
1607
- "captions": [{
1608
- "slug": string,
1609
- "text": string,
1610
- "start": number,
1611
- "duration": number,
1612
- "x": number,
1613
- "y": number,
1614
- "width": number,
1615
- "height": number,
1616
- "font_size_percent": number,
1617
- "font_size": number,
1618
- "color": string,
1619
- "background": string,
1620
- "background_style": "plain" | "outline" | "highlight-solid" | "highlight-translucent",
1621
- "font_family": string,
1622
- "font_weight": number,
1623
- "viral_note": string
1624
- }]
1625
- }
1626
-
1627
- VIRAL_DNA RULES:
1628
- - "trend_tagline": a short (3-8 word) semantic identifier for the underlying trend/format itself, NOT this specific video's topic. Think of it as the label a scroller would use to categorize the format (e.g., "green-screen news debunk", "day-in-the-life sped-up montage", "best-at-what-cost destination reveal", "pinned-comment dance dare"). If the caller pre-supplied a trend_tagline (see USER GUIDANCE), preserve it verbatim; otherwise generate one.
1629
- - "hook": the 1-sentence tactic used in the first 1-2 seconds to stop the scroll (e.g., "Pinned comment screenshot dares the viewer to keep watching").
1630
- - "retention": how the middle of the video keeps attention (visual cuts, pattern loops, escalating stakes).
1631
- - "payoff": what the viewer earns at the end (reveal, punchline, transformation).
1632
- - "emotional_punch": the FEELING this format delivers and exactly what makes it land — the part remixes most often flatten. hook/retention/payoff are the mechanical structure; this is the vibe, the joke, and the intonation that make a viewer actually FEEL something. Fill EVERY sub-field:
1633
- - "core_emotion": the dominant feeling a viewer walks away with (e.g. "delighted surprise", "righteous validation", "warm nostalgia", "second-hand cringe", "smug satisfaction").
1634
- - "tone": the register / vibe in 1-3 words (comedic, deadpan, earnest, dramatic, wholesome, chaotic, satirical, hype, ...).
1635
- - "arc": how the feeling MOVES across the runtime (e.g. "curiosity → mounting tension → relief-laugh", "flat setup → sudden absurd turn").
1636
- - "peak_moment": the single beat that lands the emotional hit — the punchline, the reveal, the gut-punch, the beat-drop — and roughly when.
1637
- - "mechanism": WHY it lands — the technique doing the work (comedic timing, irony, subversion of expectation, escalation, deadpan intonation, relatability, a trending-sound drop synced to the visual). Weigh the AUDIO here: on TikTok/Reels the trending sound or music bed is frequently the single biggest driver of the emotional hit, so if a recognizable trending sound, a beat drop, or a music swell is doing the work, name it as the mechanism.
1638
- - "contrast": the specific mismatch / juxtaposition that creates the feeling. This is CRITICAL for irony- or sass-driven formats: identify the actual tension, not just the visible gesture. Examples: "hard, macho soundtrack over a dainty office-laptop scene", "overconfident delivery about a trivial preference", "luxury attitude applied to a mundane workflow". If there is no real mismatch, write "none".
1639
- - "humor": if it's funny, state what the JOKE actually is and why it works, concretely enough that a remixer can rebuild the same joke with new subject matter; write "none" if the format is not comedic. Focus on the underlying bit, not surface choreography.
1640
- - "audio_visual_tension": describe how the SOUND and VISUALS interact emotionally: whether the soundtrack / vocal cadence reinforces the visuals, undercuts them, or creates a deliberate mismatch. Name the exact pairing if that's the joke or swagger engine; write "none" only if audio contributes nothing.
1641
- - "delivery": the performance cues to preserve so the vibe survives — speech intonation / cadence (from the AUDIO TRANSCRIPT), the pause held before the punchline, a facial beat or reaction, and especially the SOUND: the music / trending-audio bed, where the beat drop or lyric hit lands relative to the cut, a music swell, an sfx sting, the deadpan-then-cut. This is what keeps intonation, comedic timing, and the audio-driven vibe from being lost in a remix.
1642
- - "preserve": 2-4 emotional beats a remix MUST keep to reproduce the same feeling (e.g. "keep the beat drop landing exactly on the reveal cut", "hold the silent beat before the reveal", "keep the flat unbothered delivery", "the payoff must undercut the buildup, not confirm it").
1643
- Use BOTH the frames and the AUDIO TRANSCRIPT block above. Never leave emotional_punch empty — infer the feeling from the frames, captions, and pacing even when there is no audio, and when the audio is music-only, treat the sound itself as the emotional engine. Do NOT over-index on a single gesture or hand movement if the real emotional engine is the character attitude, the soundtrack choice, the setting mismatch, or the audio-vs-visual contrast.
1644
- - "static_vs_pivot": this is the load-bearing reuse guidance for lazy prompts like "please recreate this video for my business". Make it operational, not vague.
1645
- - "summary": one sentence on what can change versus what must stay.
1646
- - "scene_replacement": decide whether most scenes should be replaced or only some, why, which scenes are load-bearing, and which scenes are the first candidates to swap.
1647
- - "narration": decide whether the original spoken audio can stay as-is (for example because it is a generic or trending sound) or whether it must be customized because the words are source-specific. Set "customize_script" whenever the spoken words would stop making sense after a subject swap. If narration is timing-sensitive, set same_timing_required true.
1648
- - "music": decide whether original music should stay, be recreated under a new narration, or is irrelevant. Include simple mix guidance like "keep low under VO" or "music is the main viral engine".
1649
- - "captions": decide whether existing caption timing/animation can be reused, whether they must be re-timed to new narration, or whether captions are not important. Call out if the animation is tightly coupled to the spoken words.
1650
- - "visual_rebuild": default strategy for replacement visuals. When the source mainly uses graphics, screenshots, product cards, or still-like scenes that do not need expensive motion video, prefer "ai_images_with_ken_burns" and explain that slow pan/zoom is enough.
1651
- - "viral_dna": separate format elements into "must_keep", "flexible", and "do_not_change" so an agent can preserve the viral structure while swapping subject matter.
1652
- - "preserve": 3-5 concrete elements a remixer MUST keep to preserve the format (e.g., "Pinned comment stays on-screen the whole video", "Wedding dance payoff shot").
1653
- - "avoid": 3-5 things that would break the format if changed (e.g., "Do not remove the on-screen text overlay", "Do not slow the cuts below 1.5s").
1654
- - "promotions": 3-5 categories of product / service / offer this exact format could be reused to promote, WITHOUT changing the beat structure or hook mechanic. Each item is a concrete pitchable use-case, not a genre. Examples: "SaaS free-trial signup (before/after workflow)", "Fitness challenge (7-day transformation)", "Local restaurant grand-opening (line-out-the-door proof)", "Ebook / course launch (result-first payoff)", "Real-estate open-house (walk-through reveal)". Anchor each promotion to the same viral tactic captured in "hook" — do not invent unrelated verticals.
1655
- - "keywords": 6-12 lowercase search terms that make this format discoverable when a marketer describes their product/offer in their own words. Include the niches, target audiences, product categories, outcomes, and emotions this format naturally sells to — plus obvious synonyms a user might type. Think about what a customer would search: e.g. for a before/after fitness format include "weight loss", "fitness", "gym", "transformation", "before after", "health app", "diet", "body goals", "wellness". Single words or 2-3 word phrases, no punctuation, no duplicates. These are for matching only, so bias toward the words a customer would use, not jargon.
1656
- - Use short imperative fragments in preserve/avoid arrays. Never leave empty — infer from the frames if unstated.
1657
-
1658
- SCENE RULES:
1659
- - 2 to 6 scenes covering the FULL duration; contiguous (scene[i].start = scene[i-1].start + scene[i-1].duration); first scene starts at 0.0; last scene ends at ${input.durationSeconds.toFixed(3)}.
1660
- - Every scene MUST include a "description": 1-3 plain sentences literally describing what is happening on screen during that scene — setting, subjects, their actions, camera movement/framing, and any notable visual changes (cuts, zooms, transitions, product shots). Describe only what a viewer literally sees in the frames; do NOT repeat viral_note commentary, marketing analysis, or audio content here.
1661
-
1662
- SLUG RULES:
1663
- - Every scene AND every caption MUST have a "slug": short snake_case (2-4 words) describing its ROLE in the viral structure. Unique within scenes; unique within captions. Examples: "hook_reveal", "problem_setup", "payoff_context", "cta_follow", "caption_hook_dare".
1664
-
1665
- CAPTION RULES (CRITICAL — this is the primary use of vision on this task):
1666
- - Inspect EVERY supplied frame. Enumerate EVERY distinct visible text overlay across all frames — hooks, dialog subtitles, hashtags, usernames, handles, brand tags, CTAs, sticker text, arrow labels, price tags, meme text, comment-screenshot text, product names — anything rendered as text on the pixels.
1667
- - One caption = one visually distinct text block. If two lines share the same visual treatment and appear together as one overlay, keep them as ONE caption with "\\n" between the lines. If two blocks look different (different font, color, position, or independent timing), they are SEPARATE captions.
1668
- - Preserve text VERBATIM. Copy spelling, punctuation, casing, emojis, and line breaks exactly as they appear. Do NOT paraphrase, normalize casing, translate, or "clean up" spacing. If a caption contains a typo, keep the typo.
1669
- - The SAME text overlay that persists across multiple sampled frames is ONE caption with a longer duration — do NOT emit one caption per frame. Merge by (identical text + roughly same on-screen position + same visual style).
1670
- - If a caption changes text mid-clip (e.g. rolling subtitle updates), emit each version as a separate caption with its own tight start/duration.
1671
- - Position numbers (x, y, width, height) must reflect where the text ACTUALLY sits in the frame — measure the bounding box, don't guess. Vertical (9:16) captions almost never span x=0..100; hooks near y=8-18, dialog near y=60-72, CTAs near y=80-88.
1672
- - If no visible text overlays exist across ANY frame, return "captions": []. NEVER invent text. NEVER hallucinate captions from audio or context.
1673
- - Prioritize completeness. Missing an on-screen caption is a worse failure than emitting an extra minor one.
1674
-
1675
- FONT_FAMILY (choose ONE, match visual appearance):
1676
- - "TikTok Sans" — clean sans-serif (DEFAULT if unsure). Rounded, modern.
1677
- - "Montserrat" — geometric bold sans. Common on IG/TikTok bold captions.
1678
- - "Source Code Pro" — monospaced. For code/terminal aesthetic.
1679
- - "Yesteryear" — cursive script. For quotes / romantic vibes.
1680
- - "Georgia" — serif. For editorial / news look.
1681
- - "Abel" — condensed sans. Newsletter/headline vibe.
1682
- - Return exactly one of the above six strings. If unsure, return "TikTok Sans".
1683
-
1684
- FONT_WEIGHT (integer):
1685
- - 400 for normal weight
1686
- - 600 for semibold (typical TikTok caption look)
1687
- - 700 for bold (heavy captions)
1688
- - 800 or 900 for ultra-black display text
1689
-
1690
- FONT_SIZE — return TWO numbers, both required. The renderer uses font_size_percent to survive canvas resizes.
1691
- - "font_size_percent" (float 1.0–15.0): the caption glyph height as a percent of canvas height (${input.height || 1280}px). This is the SOURCE OF TRUTH. Measure a single-line uppercase-height letter (like an "M" or "H") in the actual frame and divide by canvas height, times 100.
1692
- - Tiny UI / hashtag row: 1.2–2.0
1693
- - Standard TikTok caption (dialog line): 2.0–3.2
1694
- - Standard hook caption (readable but not screaming): 3.2–5.0
1695
- - Bold hook overlay filling a wide band: 5.0–8.0
1696
- - Giant kinetic display / meme text: 8.0–12.0
1697
- - "font_size" (integer pixels): compute as round(font_size_percent / 100 * ${input.height || 1280}). Include it for redundancy.
1698
- - Do NOT default to 24. Measure the actual on-screen glyph height. When unsure between two ranges, pick the one closest to what you actually measure — do not over-inflate.
1699
- - Give the SAME caption the SAME font_size — do not vary between instances.
1700
-
1701
- COLOR (hex "#rrggbb"):
1702
- - Pure white: "#ffffff" (most common TikTok captions)
1703
- - Pure black: "#000000"
1704
- - Bright yellow (highlight text): "#ffe600"
1705
- - Match the actual on-screen color as closely as you can. Never return names like "white"; always hex.
1706
-
1707
- BACKGROUND_STYLE (choose ONE):
1708
- - "plain" — no visible pill/outline. Just letters over the video. (Most common.)
1709
- - "outline" — thin dark outline around each glyph (for legibility over busy video).
1710
- - "highlight-solid" — a solid colored pill / block behind the text (typical TikTok white-on-black caption card).
1711
- - "highlight-translucent" — semi-transparent pill.
1712
- - BACKGROUND ("#rrggbb") is the pill / outline color. For "plain" this is unused but still return a valid hex like "#000000".
1713
-
1714
- POSITION (percent 0-100, floats OK):
1715
- - x, y = top-left corner of the caption's bounding box relative to the canvas.
1716
- - width, height = its bounding box size.
1717
- - TikTok baked captions are usually centered horizontally: x ≈ 5-10, width ≈ 80-90.
1718
- - Vertical placement: hook captions near top (y ≈ 8-18); dialog/subtitle mid-lower (y ≈ 60-72); brand tag / cta bottom (y ≈ 80-88).
1719
- - Height should roughly match font_size and line count.
1720
-
1721
- TIMING RULES:
1722
- - start, duration are seconds (floats). start >= 0. start + duration <= ${input.durationSeconds.toFixed(3)}.
1723
-
1724
- OUTPUT: pure JSON, no prose, no markdown fences.`;
1725
- }
1726
- const ALLOWED_FONTS = [
1727
- "TikTok Sans", "Montserrat", "Source Code Pro", "Yesteryear", "Georgia", "Abel"
1728
- ];
1729
- const ALLOWED_BG_STYLES = [
1730
- "plain", "outline", "highlight-solid", "highlight-translucent"
1731
- ];
1732
- function normalizeFontFamily(value) {
1733
- if (typeof value !== "string")
1734
- return "TikTok Sans";
1735
- const trimmed = value.trim();
1736
- const match = ALLOWED_FONTS.find((font) => font.toLowerCase() === trimmed.toLowerCase());
1737
- return match ?? "TikTok Sans";
1738
- }
1739
- function normalizeHexColor(value, fallback) {
1740
- if (typeof value !== "string")
1741
- return fallback;
1742
- const trimmed = value.trim();
1743
- if (/^#[0-9a-fA-F]{6}$/.test(trimmed))
1744
- return trimmed.toLowerCase();
1745
- if (/^#[0-9a-fA-F]{3}$/.test(trimmed)) {
1746
- const [, r, g, b] = /^#([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])$/.exec(trimmed) ?? [];
1747
- return `#${r}${r}${g}${g}${b}${b}`.toLowerCase();
1748
- }
1749
- return fallback;
1750
- }
1751
- function normalizeBackgroundStyle(value) {
1752
- if (typeof value !== "string")
1753
- return "plain";
1754
- const lowered = value.trim().toLowerCase();
1755
- const match = ALLOWED_BG_STYLES.find((style) => style === lowered);
1756
- return match ?? "plain";
1757
- }
1758
- // Convert a #rrggbb hex to an rgba(...) string with the given alpha. Falls back
1759
- // to a solid dark backing if the hex is malformed so the caption still stays
1760
- // legible.
1761
- function hexToRgba(hex, alpha) {
1762
- const match = /^#([0-9a-f]{6})$/i.exec(String(hex ?? "").trim());
1763
- if (!match)
1764
- return `rgba(0,0,0,${alpha})`;
1765
- const value = match[1];
1766
- const r = parseInt(value.slice(0, 2), 16);
1767
- const g = parseInt(value.slice(2, 4), 16);
1768
- const b = parseInt(value.slice(4, 6), 16);
1769
- return `rgba(${r},${g},${b},${alpha})`;
1770
- }
1771
- // Builds the inline-span CSS for a caption based on its background_style so
1772
- // captions actually render with the visual treatment the vision LLM extracted
1773
- // ("outline" gets a real text stroke; "highlight-*" gets a colored pill).
1774
- // Without this the span just had `color:#fff` and everything looked washed
1775
- // out over the video.
1776
- function captionInlineTextStyle(caption) {
1777
- const parts = [`color:${caption.color}`];
1778
- const strokeColor = caption.background && /^#[0-9a-f]{6}$/i.test(caption.background)
1779
- ? caption.background
1780
- : "#000000";
1781
- // Outline width scales with font size so small captions still get a
1782
- // visible edge. Clamp so it never disappears (1px) and never eats the
1783
- // glyph interior (5px).
1784
- const strokePx = Math.max(1, Math.min(5, Math.round(caption.font_size * 0.06)));
1785
- switch (caption.background_style) {
1786
- case "outline":
1787
- parts.push(`-webkit-text-stroke:${strokePx}px ${strokeColor}`, `paint-order:stroke fill`,
1788
- // text-shadow fallback for renderers that ignore paint-order.
1789
- `text-shadow:0 0 ${strokePx}px ${strokeColor}, ${strokePx}px ${strokePx}px 0 ${strokeColor}, -${strokePx}px ${strokePx}px 0 ${strokeColor}, ${strokePx}px -${strokePx}px 0 ${strokeColor}, -${strokePx}px -${strokePx}px 0 ${strokeColor}`);
1790
- break;
1791
- case "highlight-solid":
1792
- parts.push(`background:${strokeColor}`, `padding:0.18em 0.5em`, `border-radius:0.35em`, `box-decoration-break:clone`, `-webkit-box-decoration-break:clone`);
1793
- break;
1794
- case "highlight-translucent":
1795
- parts.push(`background:${hexToRgba(strokeColor, 0.55)}`, `padding:0.18em 0.5em`, `border-radius:0.35em`, `box-decoration-break:clone`, `-webkit-box-decoration-break:clone`);
1796
- break;
1797
- case "plain":
1798
- default:
1799
- // Even "plain" captions need SOME shadow over busy TikTok footage or
1800
- // white-on-white becomes invisible. Use a subtle drop shadow that
1801
- // doesn't read as an intentional outline.
1802
- parts.push(`text-shadow:0 1px 2px rgba(0,0,0,0.55), 0 0 3px rgba(0,0,0,0.35)`);
1803
- break;
1804
- }
1805
- return parts.join(";");
1806
- }
1807
- function normalizeFontWeight(value) {
1808
- const parsed = Number(value);
1809
- if (!Number.isFinite(parsed))
1810
- return 600;
1811
- const clamped = Math.max(100, Math.min(900, Math.round(parsed / 100) * 100));
1812
- return clamped;
1813
- }
1814
- function normalizeFontSize(value, canvasHeight) {
1815
- const parsed = Number(value);
1816
- const heightRef = canvasHeight > 0 ? canvasHeight : 1280;
1817
- if (!Number.isFinite(parsed) || parsed <= 0)
1818
- return Math.round(heightRef * 0.028);
1819
- return Math.max(10, Math.min(160, Math.round(parsed)));
1820
- }
1821
- function resolveFontSize(input) {
1822
- const heightRef = input.canvasHeight > 0 ? input.canvasHeight : 1280;
1823
- const percent = Number(input.fontSizePercent);
1824
- if (Number.isFinite(percent) && percent > 0) {
1825
- const clampedPct = Math.max(0.8, Math.min(15, percent));
1826
- return Math.max(10, Math.min(160, Math.round((clampedPct / 100) * heightRef)));
1827
- }
1828
- return normalizeFontSize(input.fontSize, input.canvasHeight);
1829
- }
1830
- const DEFAULT_EMOTIONAL_PUNCH = {
1831
- core_emotion: "",
1832
- tone: "",
1833
- arc: "",
1834
- peak_moment: "",
1835
- mechanism: "",
1836
- contrast: "",
1837
- humor: "",
1838
- audio_visual_tension: "",
1839
- delivery: "",
1840
- preserve: []
1841
- };
1842
- export const DEFAULT_VIRAL_DNA = {
1843
- trend_tagline: "",
1844
- hook: "",
1845
- retention: "",
1846
- payoff: "",
1847
- emotional_punch: { ...DEFAULT_EMOTIONAL_PUNCH },
1848
- static_vs_pivot: {
1849
- summary: "",
1850
- scene_replacement: { overall: "replace_some", reason: "", must_keep: [], should_replace: [], load_bearing_scenes: [] },
1851
- narration: { overall: "optional", reason: "", same_timing_required: false, use_premium_tts_when_customizing: true },
1852
- music: { overall: "optional", reason: "", volume_guidance: "" },
1853
- captions: { overall: "optional", reason: "", animation_dependency: "" },
1854
- visual_rebuild: { default_strategy: "mixed", reason: "" },
1855
- viral_dna: { must_keep: [], flexible: [], do_not_change: [] }
1856
- },
1857
- preserve: [],
1858
- avoid: [],
1859
- promotions: [],
1860
- keywords: []
1861
- };
1862
- function normalizeEmotionalPunch(value) {
1863
- if (!value || typeof value !== "object")
1864
- return { ...DEFAULT_EMOTIONAL_PUNCH };
1865
- const record = value;
1866
- const asString = (v) => typeof v === "string" ? v.trim() : "";
1867
- const asStringArray = (v) => Array.isArray(v)
1868
- ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
1869
- : [];
1870
- return {
1871
- core_emotion: asString(record.core_emotion ?? record.coreEmotion),
1872
- tone: asString(record.tone),
1873
- arc: asString(record.arc ?? record.emotional_arc),
1874
- peak_moment: asString(record.peak_moment ?? record.peakMoment),
1875
- mechanism: asString(record.mechanism),
1876
- contrast: asString(record.contrast),
1877
- humor: asString(record.humor),
1878
- audio_visual_tension: asString(record.audio_visual_tension ?? record.audioVisualTension),
1879
- delivery: asString(record.delivery),
1880
- preserve: asStringArray(record.preserve).slice(0, 4)
1881
- };
1882
- }
1883
- function normalizeStaticVsPivot(value) {
1884
- const fallback = DEFAULT_VIRAL_DNA.static_vs_pivot;
1885
- if (!value || typeof value !== "object") {
1886
- return {
1887
- ...fallback,
1888
- scene_replacement: { ...fallback.scene_replacement },
1889
- narration: { ...fallback.narration },
1890
- music: { ...fallback.music },
1891
- captions: { ...fallback.captions },
1892
- visual_rebuild: { ...fallback.visual_rebuild },
1893
- viral_dna: { ...fallback.viral_dna }
1894
- };
1895
- }
1896
- const record = value;
1897
- const asString = (v) => typeof v === "string" ? v.trim() : "";
1898
- const asStringArray = (v) => Array.isArray(v)
1899
- ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
1900
- : [];
1901
- const sceneReplacement = record.scene_replacement && typeof record.scene_replacement === "object"
1902
- ? record.scene_replacement
1903
- : {};
1904
- const narration = record.narration && typeof record.narration === "object"
1905
- ? record.narration
1906
- : {};
1907
- const music = record.music && typeof record.music === "object"
1908
- ? record.music
1909
- : {};
1910
- const captions = record.captions && typeof record.captions === "object"
1911
- ? record.captions
1912
- : {};
1913
- const visualRebuild = record.visual_rebuild && typeof record.visual_rebuild === "object"
1914
- ? record.visual_rebuild
1915
- : {};
1916
- const viralDna = record.viral_dna && typeof record.viral_dna === "object"
1917
- ? record.viral_dna
1918
- : {};
1919
- const oneOf = (v, allowed, fallbackValue) => {
1920
- const text = asString(v);
1921
- return allowed.includes(text) ? text : fallbackValue;
1922
- };
1923
- return {
1924
- summary: asString(record.summary),
1925
- scene_replacement: {
1926
- overall: oneOf(sceneReplacement.overall, ["replace_most", "replace_some", "keep_most", "keep_all"], "replace_some"),
1927
- reason: asString(sceneReplacement.reason),
1928
- must_keep: asStringArray(sceneReplacement.must_keep),
1929
- should_replace: asStringArray(sceneReplacement.should_replace),
1930
- load_bearing_scenes: asStringArray(sceneReplacement.load_bearing_scenes)
1931
- },
1932
- narration: {
1933
- overall: oneOf(narration.overall, ["keep_original", "customize_script", "optional"], "optional"),
1934
- reason: asString(narration.reason),
1935
- same_timing_required: narration.same_timing_required === true,
1936
- use_premium_tts_when_customizing: narration.use_premium_tts_when_customizing !== false
1937
- },
1938
- music: {
1939
- overall: oneOf(music.overall, ["keep_original", "recreate_under_narration", "optional", "none"], "optional"),
1940
- reason: asString(music.reason),
1941
- volume_guidance: asString(music.volume_guidance)
1942
- },
1943
- captions: {
1944
- overall: oneOf(captions.overall, ["reuse_existing", "retime_to_new_narration", "optional", "none"], "optional"),
1945
- reason: asString(captions.reason),
1946
- animation_dependency: asString(captions.animation_dependency)
1947
- },
1948
- visual_rebuild: {
1949
- default_strategy: oneOf(visualRebuild.default_strategy, ["ai_images_with_ken_burns", "real_footage", "mixed"], "mixed"),
1950
- reason: asString(visualRebuild.reason)
1951
- },
1952
- viral_dna: {
1953
- must_keep: asStringArray(viralDna.must_keep),
1954
- flexible: asStringArray(viralDna.flexible),
1955
- do_not_change: asStringArray(viralDna.do_not_change)
1956
- }
1957
- };
1958
- }
1959
- function normalizeViralDna(value) {
1960
- if (!value || typeof value !== "object")
1961
- return DEFAULT_VIRAL_DNA;
1962
- const record = value;
1963
- const asString = (v) => typeof v === "string" ? v.trim() : "";
1964
- const asStringArray = (v) => Array.isArray(v)
1965
- ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
1966
- : [];
1967
- return {
1968
- trend_tagline: asString(record.trend_tagline ?? record.trendTagline),
1969
- hook: asString(record.hook),
1970
- retention: asString(record.retention),
1971
- payoff: asString(record.payoff),
1972
- emotional_punch: normalizeEmotionalPunch(record.emotional_punch ?? record.emotionalPunch),
1973
- static_vs_pivot: normalizeStaticVsPivot(record.static_vs_pivot ?? record.staticVsPivot),
1974
- preserve: asStringArray(record.preserve),
1975
- avoid: asStringArray(record.avoid),
1976
- promotions: asStringArray(record.promotions),
1977
- keywords: asStringArray(record.keywords).map((item) => item.toLowerCase())
1978
- };
1979
- }
1980
- // ---------------------------------------------------------------------------
1981
- // Editor-harness pass — technical editing direction (the "how to edit like
1982
- // this" companion to viral DNA's "why it works"). Isolated vision call in the
1983
- // decompose burst (never the caption/scene call) so it can't dilute
1984
- // timeline-layer quality; reuses the SAME sparse frames already sampled + the
1985
- // ffmpeg cut timings for grounded pacing. Non-fatal: callers wrap in .catch().
1986
- // ---------------------------------------------------------------------------
1987
- export const DEFAULT_EDITOR_HARNESS = {
1988
- format: "other",
1989
- one_liner: "",
1990
- aspect_ratio: "",
1991
- pacing: { cut_rhythm: "medium", avg_scene_seconds: 0, cuts_per_10s: 0, energy_curve: "" },
1992
- typography: { caption_style: "", placement: "", font_character: "", emphasis: "", text_density: "none" },
1993
- broll: { reliance: "none", sourcing: "", shot_kinds: [], cadence: "" },
1994
- transitions: { default: "cut", intro: "", outro: "", usage: "" },
1995
- audio: { voiceover: "", music: "", sfx: "", captions_from: "" },
1996
- emotional: { target_feeling: "", tone: "", comedic_timing: "", intonation: "", vibe_anchors: [] },
1997
- scenes: [],
1998
- important_scenes: [],
1999
- editing_bias: [],
2000
- do: [],
2001
- dont: []
2002
- };
2003
- const HARNESS_CUT_RHYTHMS = ["fast", "medium", "slow", "varied"];
2004
- const HARNESS_TEXT_DENSITY = ["heavy", "moderate", "minimal", "none"];
2005
- const HARNESS_BROLL_RELIANCE = ["heavy", "moderate", "light", "none"];
2006
- const HARNESS_SCENE_IMPORTANCE = ["critical", "high", "normal"];
2007
- function buildEditorHarnessPrompt(input) {
2008
- const frameListing = input.frameTimestamps.length > 0
2009
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
2010
- : "";
2011
- const cutListing = input.sceneCuts.length > 0
2012
- ? `\nDETECTED HARD CUTS (seconds, from the editing timeline — use these to ground pacing/cut_rhythm/cuts_per_10s/avg_scene_seconds): ${input.sceneCuts.map((c) => c.toFixed(2)).join(", ")}\n`
2013
- : "\n(No hard cuts detected — likely a single continuous take or slow crossfades. Judge pacing from the frames.)\n";
2014
- const promptHint = input.userPrompt && input.userPrompt.trim().length > 0
2015
- ? `\nUSER GOAL (bias the harness toward recreating THIS intent): ${input.userPrompt.trim()}\n`
2016
- : "";
2017
- const roleVocab = SCENE_ROLE_VOCAB.join(" | ");
2018
- const formatVocab = CONTENT_FORMATS.join(" | ");
2019
- const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
2020
- const emotional = input.viralDna?.emotional_punch;
2021
- const emotionalSnapshot = emotional && [
2022
- emotional.core_emotion,
2023
- emotional.tone,
2024
- emotional.arc,
2025
- emotional.peak_moment,
2026
- emotional.mechanism,
2027
- emotional.contrast,
2028
- emotional.humor,
2029
- emotional.audio_visual_tension,
2030
- emotional.delivery
2031
- ].some((value) => value && value.trim().length > 0)
2032
- ? `\nVIRAL DNA FEELING SNAPSHOT (operationalize this in the harness; do NOT drift from it):\n- core_emotion: ${emotional.core_emotion || "—"}\n- tone: ${emotional.tone || "—"}\n- arc: ${emotional.arc || "—"}\n- peak_moment: ${emotional.peak_moment || "—"}\n- mechanism: ${emotional.mechanism || "—"}\n- contrast: ${emotional.contrast || "—"}\n- humor: ${emotional.humor || "—"}\n- audio_visual_tension: ${emotional.audio_visual_tension || "—"}\n- delivery: ${emotional.delivery || "—"}\n- preserve: ${(emotional.preserve || []).join("; ") || "—"}\n`
2033
- : "";
2034
- return `You are Vidfarm's EDITING DIRECTOR. You are studying a short-form video to write an EDITOR HARNESS — a compact, technical brief that lets another AI editor recreate this video's STYLE (not its exact content) in one clean pass on a timeline editor (scenes, captions, audio, transitions, keyframes).
2035
-
2036
- This is DIFFERENT from "viral DNA" (why the video works / what to preserve). The harness is HOW to edit: pace, typography, b-roll usage, transitions, audio bed, and the role each scene plays. Be concrete and directive — every field should be something an editor can act on, not vibes.
2037
-
2038
- Video duration: ${input.durationSeconds.toFixed(2)} seconds.
2039
- ${frameListing}${cutListing}${audioDigest}${emotionalSnapshot}${promptHint}
2040
- Return strictly valid JSON matching this schema:
2041
- {
2042
- "format": string, // one of: ${formatVocab}
2043
- "one_liner": string, // ONE sentence: how to edit to match this style
2044
- "aspect_ratio": string, // "9:16" | "1:1" | "16:9" (infer from the frames)
2045
- "pacing": {
2046
- "cut_rhythm": "fast" | "medium" | "slow" | "varied",
2047
- "avg_scene_seconds": number, // typical shot length in seconds
2048
- "cuts_per_10s": number, // ground on the detected cuts + duration
2049
- "energy_curve": string // e.g. "front-loaded hook then steady"
2050
- },
2051
- "typography": {
2052
- "caption_style": string, // karaoke | word-pop | spotlight | stack-up | neon | bounce | none | free-form label
2053
- "placement": string, // e.g. "center-lower" | "top" | "full-bleed" | "none"
2054
- "font_character": string, // e.g. "bold condensed all-caps sans"
2055
- "emphasis": string, // e.g. "active-word highlight in yellow" | "none"
2056
- "text_density": "heavy" | "moderate" | "minimal" | "none"
2057
- },
2058
- "broll": {
2059
- "reliance": "heavy" | "moderate" | "light" | "none",
2060
- "sourcing": string, // generated | stock | screen-recording | talking-head | mixed
2061
- "shot_kinds": [string, ...], // e.g. b_roll, product_shot, screen_recording, reaction, establishing, text_graphic
2062
- "cadence": string // e.g. "new visual ~every 2s under VO"
2063
- },
2064
- "transitions": {
2065
- "default": string, // junction at most cuts: cut | crossfade | whip | slide | zoom | dissolve | ...
2066
- "intro": string, // first-clip entrance
2067
- "outro": string, // last-clip exit
2068
- "usage": string // one line on how transitions are used
2069
- },
2070
- "audio": {
2071
- "voiceover": string, // e.g. "first-person narration" | "none"
2072
- "music": string, // e.g. "upbeat lofi bed, low volume" | "none"
2073
- "sfx": string, // e.g. "whoosh on cuts" | "none"
2074
- "captions_from": string // "voiceover" | "on-screen text" | "none"
2075
- },
2076
- "emotional": {
2077
- "target_feeling": string, // the emotion the edit must reproduce (the vibe/joke that makes it land)
2078
- "tone": string, // register to hold: comedic | deadpan | earnest | dramatic | wholesome | hype | ...
2079
- "comedic_timing": string, // where the beat/pause/cut sells the joke; "none" if not comedic
2080
- "intonation": string, // vocal delivery/cadence to preserve on VO or on-cam speech; "none" if silent
2081
- "vibe_anchors": [string, ...] // 2-4 concrete edit moves that carry the emotion
2082
- },
2083
- "scenes": [{
2084
- "role": string, // ${roleVocab}
2085
- "timestamp": string, // human range, e.g. "0:00-0:03"
2086
- "start": number, // numeric seconds where the beat begins
2087
- "importance": "critical" | "high" | "normal",
2088
- "must_keep": boolean, // true if swapping/removing this beat kills the style
2089
- "note": string, // what happens in this beat
2090
- "edit_bias": string // how to edit this beat while staying on-style
2091
- }],
2092
- "important_scenes": [string, ...],// roles or timestamps that carry the virality — protect when editing
2093
- "editing_bias": [string, ...], // 3-6 top directives an editor MUST follow to stay on-style
2094
- "do": [string, ...], // 2-5 concrete DOs
2095
- "dont": [string, ...] // 2-5 concrete DON'Ts
2096
- }
2097
-
2098
- RULES:
2099
- - Ground pacing numbers on the detected cuts + duration when cuts are supplied; otherwise estimate from the frames. Round avg_scene_seconds and cuts_per_10s to at most 2 decimals.
2100
- - "scenes": 3-8 entries covering the whole runtime in order, each tied to a real moment you see. Reserve importance "critical" for the 1-3 beats that actually carry the style/retention (usually the hook + payoff).
2101
- - Prefer the caption_style / transition / shot_kind vocabulary above so the editor can map straight to presets; a free-form label is fine when nothing fits.
2102
- - "emotional": this is HOW to edit so the FEELING survives — the vibe, the joke, the intonation, and the SOUND are the first things a remix flattens, so make these directive. Use the AUDIO TRANSCRIPT block: on TikTok/Reels the trending sound / music bed is often the biggest carrier of the feeling. "target_feeling" = the emotion the recreated edit must produce; "tone" = the register to hold; "comedic_timing" = the exact editing move that sells the joke (the beat held before a cut, the deadpan-then-punch, the sfx sting or beat-drop on the button word) or "none"; "intonation" = the vocal delivery/cadence to preserve on any VO or on-cam speech (or "none" if silent) — flag it if the speech must be re-voiced in the SAME cadence; "vibe_anchors" = 2-4 concrete edit choices that carry the emotion, and include the audio ones (e.g. "keep the trending sound and land the beat drop on the reveal cut", "swell the music on the payoff", "hold a full beat of silence before the reveal cut", "cut hard on the punch word"). When the viral-DNA snapshot above names a contrast or audio_visual_tension, convert that into actionable edit moves instead of drifting to superficial gestures. Infer the feeling from frames + pacing even without audio, and when the audio is music-only treat the sound as the emotional engine; never leave it empty.
2103
- - Keep every string tight and directive. Never invent audio you cannot infer — say "none"/"unknown" instead.
2104
- - Output ONLY the JSON object.`;
2105
- }
2106
- function normalizeEditorHarness(raw) {
2107
- const rec = asPlainRecord(raw);
2108
- const oneOf = (value, allowed, fallback) => {
2109
- const s = normalizeSlug(readAnnotationString(value));
2110
- return allowed.includes(s) ? s : fallback;
2111
- };
2112
- const num = (value) => {
2113
- const n = Number(value);
2114
- return Number.isFinite(n) && n >= 0 ? Number(n.toFixed(2)) : 0;
2115
- };
2116
- const formatSlug = normalizeSlug(readAnnotationString(rec.format));
2117
- const format = CONTENT_FORMATS.includes(formatSlug)
2118
- ? formatSlug
2119
- : "other";
2120
- const pacing = asPlainRecord(rec.pacing);
2121
- const typography = asPlainRecord(rec.typography);
2122
- const broll = asPlainRecord(rec.broll);
2123
- const transitions = asPlainRecord(rec.transitions);
2124
- const audio = asPlainRecord(rec.audio);
2125
- const emotional = asPlainRecord(rec.emotional);
2126
- const scenesRaw = Array.isArray(rec.scenes) ? rec.scenes : [];
2127
- const scenes = scenesRaw
2128
- .map((s) => {
2129
- const sr = asPlainRecord(s);
2130
- return {
2131
- role: normalizeSlug(readAnnotationString(sr.role)) || "context",
2132
- timestamp: readAnnotationString(sr.timestamp),
2133
- start: num(sr.start),
2134
- importance: oneOf(sr.importance, HARNESS_SCENE_IMPORTANCE, "normal"),
2135
- must_keep: sr.must_keep === true,
2136
- note: readAnnotationString(sr.note),
2137
- edit_bias: readAnnotationString(sr.edit_bias)
2138
- };
2139
- })
2140
- .filter((s) => s.note.length > 0 || s.edit_bias.length > 0)
2141
- .slice(0, 12);
2142
- return {
2143
- format,
2144
- one_liner: readAnnotationString(rec.one_liner),
2145
- aspect_ratio: readAnnotationString(rec.aspect_ratio),
2146
- pacing: {
2147
- cut_rhythm: oneOf(pacing.cut_rhythm, HARNESS_CUT_RHYTHMS, "medium"),
2148
- avg_scene_seconds: num(pacing.avg_scene_seconds),
2149
- cuts_per_10s: num(pacing.cuts_per_10s),
2150
- energy_curve: readAnnotationString(pacing.energy_curve)
2151
- },
2152
- typography: {
2153
- caption_style: readAnnotationString(typography.caption_style),
2154
- placement: readAnnotationString(typography.placement),
2155
- font_character: readAnnotationString(typography.font_character),
2156
- emphasis: readAnnotationString(typography.emphasis),
2157
- text_density: oneOf(typography.text_density, HARNESS_TEXT_DENSITY, "none")
2158
- },
2159
- broll: {
2160
- reliance: oneOf(broll.reliance, HARNESS_BROLL_RELIANCE, "none"),
2161
- sourcing: readAnnotationString(broll.sourcing),
2162
- shot_kinds: readAnnotationStringList(broll.shot_kinds, 8),
2163
- cadence: readAnnotationString(broll.cadence)
2164
- },
2165
- transitions: {
2166
- default: readAnnotationString(transitions.default) || "cut",
2167
- intro: readAnnotationString(transitions.intro),
2168
- outro: readAnnotationString(transitions.outro),
2169
- usage: readAnnotationString(transitions.usage)
2170
- },
2171
- audio: {
2172
- voiceover: readAnnotationString(audio.voiceover),
2173
- music: readAnnotationString(audio.music),
2174
- sfx: readAnnotationString(audio.sfx),
2175
- captions_from: readAnnotationString(audio.captions_from)
2176
- },
2177
- emotional: {
2178
- target_feeling: readAnnotationString(emotional.target_feeling),
2179
- tone: readAnnotationString(emotional.tone),
2180
- comedic_timing: readAnnotationString(emotional.comedic_timing),
2181
- intonation: readAnnotationString(emotional.intonation),
2182
- vibe_anchors: readAnnotationStringList(emotional.vibe_anchors, 4)
2183
- },
2184
- scenes,
2185
- important_scenes: readAnnotationStringList(rec.important_scenes, 8),
2186
- editing_bias: readAnnotationStringList(rec.editing_bias, 8),
2187
- do: readAnnotationStringList(rec.do, 6),
2188
- dont: readAnnotationStringList(rec.dont, 6)
2189
- };
2190
- }
2191
- async function runEditorHarnessPass(input) {
2192
- if (input.frames.length === 0)
2193
- return DEFAULT_EDITOR_HARNESS;
2194
- const prompt = buildEditorHarnessPrompt({
2195
- durationSeconds: input.durationSeconds,
2196
- frameTimestamps: input.frames.map((f) => f.timestamp),
2197
- sceneCuts: input.sceneCuts,
2198
- userPrompt: input.userPrompt ?? null,
2199
- transcript: input.transcript ?? null,
2200
- viralDna: input.viralDna ?? null
2201
- });
2202
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
2203
- return normalizeEditorHarness(attempt.result);
2204
- }
2205
- function shortElementId() {
2206
- return `element_${randomUUID().slice(0, 8)}`;
2207
- }
2208
- function extractJson(text) {
2209
- const trimmed = text.trim();
2210
- const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
2211
- const candidate = fenced?.[1] ?? trimmed;
2212
- try {
2213
- return JSON.parse(candidate);
2214
- }
2215
- catch {
2216
- const objectMatch = /\{[\s\S]*\}/.exec(candidate);
2217
- if (objectMatch)
2218
- return JSON.parse(objectMatch[0]);
2219
- throw new Error("Gemini did not return valid JSON");
2220
- }
2221
- }
2222
- async function callGeminiVision(input) {
2223
- const parts = [{ text: input.prompt }];
2224
- for (const frame of input.frames) {
2225
- parts.push({ text: `Frame captured at ${frame.timestamp.toFixed(2)}s` });
2226
- parts.push({ inlineData: { mimeType: "image/jpeg", data: frame.data } });
2227
- }
2228
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent?key=${encodeURIComponent(input.apiKey)}`, {
2229
- method: "POST",
2230
- headers: { "content-type": "application/json" },
2231
- body: JSON.stringify({
2232
- contents: [{ role: "user", parts }],
2233
- generationConfig: {
2234
- responseMimeType: "application/json",
2235
- temperature: 0.15,
2236
- // Bumped from Gemini's 8k default so long caption arrays (10+ text
2237
- // overlays × per-caption position/style metadata) don't truncate
2238
- // mid-JSON and blow up the parser.
2239
- maxOutputTokens: 16384
2240
- }
2241
- })
2242
- });
2243
- const raw = await response.text();
2244
- if (!response.ok) {
2245
- let message = raw;
2246
- try {
2247
- message = JSON.parse(raw).error?.message ?? raw;
2248
- }
2249
- catch { }
2250
- throw new Error(`Gemini responded ${response.status}: ${message}`);
2251
- }
2252
- const parsed = JSON.parse(raw);
2253
- const text = parsed.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
2254
- return extractJson(text);
2255
- }
2256
- async function callOpenAIVision(input) {
2257
- const content = [{ type: "text", text: input.prompt }];
2258
- for (const frame of input.frames) {
2259
- content.push({ type: "text", text: `Frame captured at ${frame.timestamp.toFixed(2)}s` });
2260
- content.push({
2261
- type: "image_url",
2262
- image_url: { url: `data:image/jpeg;base64,${frame.data}` }
2263
- });
2264
- }
2265
- // OpenRouter proxies many models and still accepts the classic Chat
2266
- // Completions params. Native OpenAI's current models (gpt-5.x / o-series)
2267
- // rejected `max_tokens` (they require `max_completion_tokens`) and only allow
2268
- // the default temperature, so sending `max_tokens`/`temperature:0.15` 400s
2269
- // every call. Branch the body by endpoint.
2270
- const isOpenRouter = Boolean(input.endpoint && input.endpoint.includes("openrouter"));
2271
- // Give the JSON response room to breathe so a video with 12+ text overlays
2272
- // doesn't get truncated (same rationale as Gemini's maxOutputTokens).
2273
- const OUTPUT_TOKEN_BUDGET = 16384;
2274
- const body = {
2275
- model: input.model,
2276
- response_format: { type: "json_object" },
2277
- messages: [{ role: "user", content }]
2278
- };
2279
- if (isOpenRouter) {
2280
- body.temperature = 0.15;
2281
- body.max_tokens = OUTPUT_TOKEN_BUDGET;
2282
- }
2283
- else {
2284
- body.max_completion_tokens = OUTPUT_TOKEN_BUDGET;
2285
- }
2286
- const response = await fetch(input.endpoint ?? "https://api.openai.com/v1/chat/completions", {
2287
- method: "POST",
2288
- headers: {
2289
- "content-type": "application/json",
2290
- "authorization": `Bearer ${input.apiKey}`,
2291
- ...(input.extraHeaders ?? {})
2292
- },
2293
- body: JSON.stringify(body)
2294
- });
2295
- const raw = await response.text();
2296
- if (!response.ok) {
2297
- let message = raw;
2298
- try {
2299
- message = JSON.parse(raw).error?.message ?? raw;
2300
- }
2301
- catch { }
2302
- throw new Error(`${input.endpoint?.includes("openrouter") ? "OpenRouter" : "OpenAI"} responded ${response.status}: ${message}`);
2303
- }
2304
- const parsed = JSON.parse(raw);
2305
- const text = parsed.choices?.[0]?.message?.content ?? "";
2306
- return extractJson(text);
2307
- }
2308
- async function callVisionWithFallback(input) {
2309
- const attempts = [];
2310
- if (input.keys.gemini) {
2311
- const model = process.env.GEMINI_MODEL ?? "gemini-3.5-flash";
2312
- attempts.push({
2313
- provider: "gemini",
2314
- model,
2315
- run: () => callGeminiVision({ apiKey: input.keys.gemini, model, prompt: input.prompt, frames: input.frames })
2316
- });
2317
- }
2318
- if (input.keys.openai) {
2319
- const model = process.env.OPENAI_VISION_MODEL ?? "gpt-5.5-2026-04-23";
2320
- attempts.push({
2321
- provider: "openai",
2322
- model,
2323
- run: () => callOpenAIVision({ apiKey: input.keys.openai, model, prompt: input.prompt, frames: input.frames })
2324
- });
2325
- }
2326
- if (input.keys.openrouter) {
2327
- const model = process.env.OPENROUTER_VISION_MODEL ?? "google/gemini-3.5-flash";
2328
- attempts.push({
2329
- provider: "openrouter",
2330
- model,
2331
- run: () => callOpenAIVision({
2332
- apiKey: input.keys.openrouter,
2333
- model,
2334
- prompt: input.prompt,
2335
- frames: input.frames,
2336
- endpoint: "https://openrouter.ai/api/v1/chat/completions",
2337
- extraHeaders: {
2338
- "HTTP-Referer": "https://staging.vidfarm.cc",
2339
- "X-Title": "Vidfarm smart decompose"
2340
- }
2341
- })
2342
- });
2343
- }
2344
- if (attempts.length === 0) {
2345
- throw new Error("No vision provider key available (GEMINI_API_KEY, OPENAI_API_KEY, or OPENROUTER_API_KEY).");
2346
- }
2347
- const errors = [];
2348
- for (const attempt of attempts) {
2349
- try {
2350
- const result = await attempt.run();
2351
- return { result, provider: attempt.provider, model: attempt.model };
2352
- }
2353
- catch (error) {
2354
- const message = error instanceof Error ? error.message : String(error);
2355
- errors.push(`${attempt.provider}(${attempt.model}): ${message}`);
2356
- console.error(`smart decompose ${attempt.provider} failed`, message);
2357
- }
2358
- }
2359
- throw new Error(`All vision providers failed. ${errors.join(" | ")}`);
2360
- }
2361
- // ==========================================================================
2362
- // Audio transcript pass — demux the source's audio track to mono 16kHz WAV
2363
- // (codec-free, accepted by every STT provider) and transcribe it verbatim
2364
- // with timestamped segments. Runs concurrently with the vision + OCR passes
2365
- // and is non-fatal: a video with no audio track, or a transcription failure,
2366
- // yields transcript: null instead of failing the decompose.
2367
- // ==========================================================================
2368
- async function extractAudioAsWav(videoPath, tempDir) {
2369
- const probeOutput = await runBinary(ffprobeBin, [
2370
- "-v", "error",
2371
- "-select_streams", "a",
2372
- "-show_entries", "stream=codec_type",
2373
- "-of", "json",
2374
- videoPath
2375
- ]);
2376
- const parsed = JSON.parse(probeOutput || "{}");
2377
- if (!Array.isArray(parsed.streams) || parsed.streams.length === 0)
2378
- return null;
2379
- const audioPath = path.join(tempDir, "audio.wav");
2380
- await runBinary(ffmpegBin, [
2381
- "-y",
2382
- "-i", videoPath,
2383
- "-vn",
2384
- "-ac", "1",
2385
- "-ar", "16000",
2386
- "-c:a", "pcm_s16le",
2387
- audioPath
2388
- ]);
2389
- return audioPath;
2390
- }
2391
- function buildTranscriptionPrompt(durationSeconds) {
2392
- return `Transcribe the spoken audio in this ${durationSeconds.toFixed(1)}-second clip VERBATIM.
2393
-
2394
- Return strictly valid JSON matching this schema:
2395
- {
2396
- "language": string,
2397
- "segments": [{ "start": number, "end": number, "text": string }]
2398
- }
2399
-
2400
- RULES:
2401
- - Transcribe every spoken word exactly as heard — keep filler words, slang, and repetitions. Do not paraphrase, translate, or clean up grammar.
2402
- - "language" is the BCP-47 code of the dominant spoken language (e.g. "en", "es").
2403
- - Split into short segments at natural phrase boundaries; "start"/"end" are seconds within the clip (start >= 0, end <= ${durationSeconds.toFixed(3)}, end > start).
2404
- - Only transcribe speech and sung lyrics. Ignore background music without lyrics and sound effects.
2405
- - If there is no speech at all, return { "language": null, "segments": [] }.
2406
-
2407
- OUTPUT: pure JSON, no prose, no markdown fences.`;
2408
- }
2409
- function normalizeTranscriptSegments(raw, durationSeconds) {
2410
- if (!Array.isArray(raw))
2411
- return [];
2412
- return raw
2413
- .map((entry) => {
2414
- const record = (entry && typeof entry === "object") ? entry : {};
2415
- const start = Number(record.start ?? NaN);
2416
- const end = Number(record.end ?? NaN);
2417
- const text = typeof record.text === "string" ? record.text.trim() : "";
2418
- return { start, end, text };
2419
- })
2420
- .filter((s) => s.text.length > 0 && Number.isFinite(s.start) && Number.isFinite(s.end) && s.end > s.start)
2421
- .map((s) => ({
2422
- start: Number(Math.max(0, s.start).toFixed(3)),
2423
- end: Number(Math.min(durationSeconds > 0 ? durationSeconds : s.end, s.end).toFixed(3)),
2424
- text: s.text
2425
- }))
2426
- .sort((a, b) => a.start - b.start);
2427
- }
2428
- async function callGeminiTranscription(input) {
2429
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent?key=${encodeURIComponent(input.apiKey)}`, {
2430
- method: "POST",
2431
- headers: { "content-type": "application/json" },
2432
- body: JSON.stringify({
2433
- contents: [{
2434
- role: "user",
2435
- parts: [
2436
- { text: buildTranscriptionPrompt(input.durationSeconds) },
2437
- { inlineData: { mimeType: "audio/wav", data: input.audioBase64 } }
2438
- ]
2439
- }],
2440
- generationConfig: {
2441
- responseMimeType: "application/json",
2442
- temperature: 0,
2443
- maxOutputTokens: 16384
2444
- }
2445
- })
2446
- });
2447
- const raw = await response.text();
2448
- if (!response.ok) {
2449
- let message = raw;
2450
- try {
2451
- message = JSON.parse(raw).error?.message ?? raw;
2452
- }
2453
- catch { }
2454
- throw new Error(`Gemini responded ${response.status}: ${message}`);
2455
- }
2456
- const parsed = JSON.parse(raw);
2457
- const text = parsed.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
2458
- const record = (extractJson(text) ?? {});
2459
- return {
2460
- language: typeof record.language === "string" && record.language.trim() ? record.language.trim() : null,
2461
- segments: normalizeTranscriptSegments(record.segments, input.durationSeconds)
2462
- };
2463
- }
2464
- async function callOpenAITranscription(input) {
2465
- const audioBuffer = await readFile(input.audioPath);
2466
- // Only whisper-1 supports verbose_json (per-segment timestamps). Newer
2467
- // gpt-4o-*-transcribe models return plain json; collapse to one segment.
2468
- const wantsSegments = input.model === "whisper-1";
2469
- const form = new FormData();
2470
- form.append("file", new Blob([new Uint8Array(audioBuffer)], { type: "audio/wav" }), "audio.wav");
2471
- form.append("model", input.model);
2472
- form.append("response_format", wantsSegments ? "verbose_json" : "json");
2473
- const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
2474
- method: "POST",
2475
- headers: { "authorization": `Bearer ${input.apiKey}` },
2476
- body: form
2477
- });
2478
- const raw = await response.text();
2479
- if (!response.ok) {
2480
- let message = raw;
2481
- try {
2482
- message = JSON.parse(raw).error?.message ?? raw;
2483
- }
2484
- catch { }
2485
- throw new Error(`OpenAI responded ${response.status}: ${message}`);
2486
- }
2487
- const parsed = JSON.parse(raw);
2488
- const segments = wantsSegments && Array.isArray(parsed.segments)
2489
- ? normalizeTranscriptSegments(parsed.segments, input.durationSeconds)
2490
- : [];
2491
- if (segments.length === 0 && typeof parsed.text === "string" && parsed.text.trim().length > 0) {
2492
- segments.push({ start: 0, end: Number(input.durationSeconds.toFixed(3)), text: parsed.text.trim() });
2493
- }
2494
- return {
2495
- language: typeof parsed.language === "string" && parsed.language.trim() ? parsed.language.trim() : null,
2496
- segments
2497
- };
2498
- }
2499
- async function transcribeAudioWithFallback(input) {
2500
- const attempts = [];
2501
- if (input.keys.gemini) {
2502
- const model = process.env.GEMINI_TRANSCRIBE_MODEL ?? process.env.GEMINI_MODEL ?? "gemini-3.5-flash";
2503
- attempts.push({
2504
- provider: "gemini",
2505
- model,
2506
- run: async () => {
2507
- const audioBase64 = (await readFile(input.audioPath)).toString("base64");
2508
- return callGeminiTranscription({ apiKey: input.keys.gemini, model, audioBase64, durationSeconds: input.durationSeconds });
2509
- }
2510
- });
2511
- }
2512
- if (input.keys.openai) {
2513
- const model = process.env.OPENAI_TRANSCRIBE_MODEL ?? "whisper-1";
2514
- attempts.push({
2515
- provider: "openai",
2516
- model,
2517
- run: () => callOpenAITranscription({ apiKey: input.keys.openai, model, audioPath: input.audioPath, durationSeconds: input.durationSeconds })
2518
- });
2519
- }
2520
- if (attempts.length === 0)
2521
- return null;
2522
- const errors = [];
2523
- for (const attempt of attempts) {
2524
- try {
2525
- const result = await attempt.run();
2526
- return {
2527
- text: result.segments.map((s) => s.text).join(" ").trim(),
2528
- language: result.language,
2529
- segments: result.segments,
2530
- provider: attempt.provider,
2531
- model: attempt.model
2532
- };
2533
- }
2534
- catch (error) {
2535
- const message = error instanceof Error ? error.message : String(error);
2536
- errors.push(`${attempt.provider}(${attempt.model}): ${message}`);
2537
- console.error(`smart decompose transcription ${attempt.provider} failed`, message);
2538
- }
2539
- }
2540
- throw new Error(`All transcription providers failed. ${errors.join(" | ")}`);
2541
- }
2542
- export async function smartDecomposeVideo(input) {
2543
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
2544
- const sourcePath = path.join(tempDir, "source.mp4");
2545
- const startedAt = Date.now();
2546
- devLog("hyperframes.smart_decompose.start", {
2547
- source_url: input.sourceUrl,
2548
- has_user_prompt: Boolean(input.userPrompt),
2549
- providers_available: {
2550
- gemini: Boolean(input.keys.gemini),
2551
- openai: Boolean(input.keys.openai),
2552
- openrouter: Boolean(input.keys.openrouter)
2553
- }
2554
- });
2555
- try {
2556
- await downloadVideoTo(input.sourceUrl, sourcePath);
2557
- const meta = await ffprobeVideo(sourcePath);
2558
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
2559
- throw new Error("Could not read video duration");
2560
- }
2561
- devLog("hyperframes.smart_decompose.probed", {
2562
- duration_seconds: meta.durationSeconds,
2563
- width: meta.width,
2564
- height: meta.height
2565
- });
2566
- // Sparse frames for the vision pass (scenes, captions, style, viral_dna)
2567
- // and dense frames for the OCR pass (deterministic caption timing).
2568
- // Phase 1 — deterministic ffmpeg/whisper work up front. Transcription is
2569
- // pulled INTO this phase (chained off the audio extract) so the transcript
2570
- // is ready BEFORE the vision + harness passes run: emotional punch on
2571
- // short-form video is frequently carried by the AUDIO (a trending sound, a
2572
- // beat drop, vocal intonation), and the silent frames alone can't capture
2573
- // it. The transcript overlaps with frame-sampling/OCR/cut-detection here
2574
- // instead of with the vision call, so this stays roughly latency-neutral.
2575
- const [frames, ocrRawFrames, transcript, sceneCuts] = await Promise.all([
2576
- sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir),
2577
- extractDenseOcrFrames(sourcePath, meta.durationSeconds, tempDir),
2578
- extractAudioAsWav(sourcePath, tempDir)
2579
- .then((audioPath) => audioPath
2580
- ? transcribeAudioWithFallback({ audioPath, durationSeconds: meta.durationSeconds, keys: input.keys })
2581
- : null)
2582
- .catch((err) => {
2583
- console.error("smart decompose audio extract/transcription failed", err instanceof Error ? err.message : err);
2584
- return null;
2585
- }),
2586
- detectSceneCuts(sourcePath, meta.durationSeconds, tempDir).catch((err) => {
2587
- console.error("smart decompose scene-cut detection failed", err instanceof Error ? err.message : err);
2588
- return [];
2589
- })
2590
- ]);
2591
- devLog("hyperframes.smart_decompose.frames_sampled", {
2592
- frame_count: frames.length,
2593
- ocr_frame_count: ocrRawFrames.length,
2594
- has_audio_track: Boolean(transcript),
2595
- scene_cut_count: sceneCuts.length
2596
- });
2597
- const prompt = buildSmartDecomposePrompt({
2598
- durationSeconds: meta.durationSeconds,
2599
- width: meta.width,
2600
- height: meta.height,
2601
- userPrompt: input.userPrompt ?? null,
2602
- trendTaglineHint: input.trendTaglineHint ?? null,
2603
- frameTimestamps: frames.map((f) => f.timestamp),
2604
- sceneCuts,
2605
- transcript
2606
- });
2607
- // Phase 2 — vision, OCR, and product-placement run concurrently, all now
2608
- // transcript-aware. OCR and placement failures are non-fatal. The
2609
- // placement call is DELIBERATELY separate from the caption/scene vision
2610
- // call so it can never dilute timeline-layer quality — it reuses the same
2611
- // sparse frames (no extra ffmpeg) plus the transcript from phase 1.
2612
- const [attempt, ocrFrames, placementSurfaces] = await Promise.all([
2613
- callVisionWithFallback({
2614
- prompt,
2615
- frames,
2616
- keys: input.keys
2617
- }),
2618
- runOcrPass({ frames: ocrRawFrames, keys: input.keys }).catch((err) => {
2619
- console.error("smart decompose OCR pass failed", err instanceof Error ? err.message : err);
2620
- return [];
2621
- }),
2622
- runPlacementSurfacePass({
2623
- frames,
2624
- keys: input.keys,
2625
- durationSeconds: meta.durationSeconds,
2626
- sceneSlugs: []
2627
- }).catch((err) => {
2628
- console.error("smart decompose placement pass failed", err instanceof Error ? err.message : err);
2629
- return [];
2630
- })
2631
- ]);
2632
- devLog("hyperframes.smart_decompose.ocr_completed", {
2633
- frames_with_text: ocrFrames.filter((f) => f.texts.length > 0).length,
2634
- total_ocr_frames: ocrFrames.length
2635
- });
2636
- devLog("hyperframes.smart_decompose.transcript_completed", {
2637
- has_transcript: Boolean(transcript),
2638
- segment_count: transcript?.segments.length ?? 0,
2639
- language: transcript?.language ?? null,
2640
- provider: transcript?.provider ?? null
2641
- });
2642
- const raw = attempt.result;
2643
- const record = (raw && typeof raw === "object") ? raw : {};
2644
- const scenesRaw = Array.isArray(record.scenes) ? record.scenes : [];
2645
- const captionsRaw = Array.isArray(record.captions) ? record.captions : [];
2646
- const usedSceneSlugs = new Set();
2647
- const scenes = scenesRaw
2648
- .map((scene, index) => {
2649
- const slug = dedupeSlug(normalizeSlug(typeof scene.slug === "string" ? scene.slug : `scene_${index + 1}`) || `scene_${index + 1}`, usedSceneSlugs);
2650
- return {
2651
- slug,
2652
- start: Number(scene.start ?? 0),
2653
- duration: Number(scene.duration ?? 0),
2654
- label: typeof scene.label === "string" ? scene.label : `Scene ${index + 1}`,
2655
- description: typeof scene.description === "string" ? scene.description : "",
2656
- viral_note: typeof scene.viral_note === "string" ? scene.viral_note : ""
2657
- };
2658
- })
2659
- .filter((s) => Number.isFinite(s.start) && Number.isFinite(s.duration) && s.duration > 0);
2660
- const usedCaptionSlugs = new Set();
2661
- const captions = captionsRaw
2662
- .map((c, index) => {
2663
- const slug = dedupeSlug(normalizeSlug(typeof c.slug === "string" ? c.slug : `caption_${index + 1}`) || `caption_${index + 1}`, usedCaptionSlugs);
2664
- return {
2665
- id: `caption_${index + 1}`,
2666
- slug,
2667
- text: typeof c.text === "string" ? c.text : "",
2668
- start: Number(c.start ?? 0),
2669
- duration: Number(c.duration ?? 0),
2670
- x: Number(c.x ?? 8),
2671
- y: Number(c.y ?? 68),
2672
- width: Number(c.width ?? 84),
2673
- height: Number(c.height ?? 12),
2674
- font_size: resolveFontSize({ fontSize: c.font_size, fontSizePercent: c.font_size_percent, canvasHeight: meta.height }),
2675
- color: normalizeHexColor(c.color, "#ffffff"),
2676
- background: normalizeHexColor(c.background, "#000000"),
2677
- background_style: normalizeBackgroundStyle(c.background_style),
2678
- font_family: normalizeFontFamily(c.font_family),
2679
- font_weight: normalizeFontWeight(c.font_weight),
2680
- viral_note: typeof c.viral_note === "string" ? c.viral_note : undefined
2681
- };
2682
- })
2683
- .filter((c) => c.text.trim().length > 0 && Number.isFinite(c.start) && Number.isFinite(c.duration) && c.duration > 0);
2684
- const snapped = snapCaptionTimingsToOcr({
2685
- captions,
2686
- ocrFrames,
2687
- durationSeconds: meta.durationSeconds
2688
- });
2689
- const finalCaptions = snapped.captions;
2690
- devLog("hyperframes.smart_decompose.captions_snapped", {
2691
- total_captions: captions.length,
2692
- snapped_count: snapped.snappedCount
2693
- });
2694
- // Ground scene boundaries on the ffmpeg-detected hard cuts (no-op if none).
2695
- const snappedScenes = snapScenesToCuts({ scenes, cuts: sceneCuts, durationSeconds: meta.durationSeconds });
2696
- const finalScenes = snappedScenes.scenes;
2697
- devLog("hyperframes.smart_decompose.scenes_snapped", {
2698
- total_scenes: scenes.length,
2699
- snapped_count: snappedScenes.snappedCount,
2700
- scene_cut_count: sceneCuts.length
2701
- });
2702
- // Link each product-placement surface to the scene its start falls within,
2703
- // deterministically from the finalized scene ranges (the placement call ran
2704
- // before scenes existed, so it couldn't know the slugs).
2705
- const finalPlacementSurfaces = placementSurfaces.map((surface) => {
2706
- const owning = finalScenes.find((s) => surface.start >= s.start && surface.start < s.start + s.duration);
2707
- return owning ? { ...surface, scene_slug: owning.slug } : surface;
2708
- });
2709
- const viralDna = normalizeViralDna(record.viral_dna);
2710
- // Caller-supplied trend_tagline always wins over whatever the LLM emits so
2711
- // the user's chosen tagline round-trips even if the model tries to rephrase.
2712
- if (input.trendTaglineHint && input.trendTaglineHint.trim().length > 0) {
2713
- viralDna.trend_tagline = input.trendTaglineHint.trim();
2714
- }
2715
- // Editor harness — technical editing direction. Run AFTER viral DNA is
2716
- // normalized so the harness can operationalize the exact emotional punch
2717
- // instead of guessing independently and drifting from the decompose result.
2718
- const editorHarness = await runEditorHarnessPass({
2719
- frames,
2720
- keys: input.keys,
2721
- durationSeconds: meta.durationSeconds,
2722
- sceneCuts,
2723
- userPrompt: input.userPrompt ?? null,
2724
- transcript,
2725
- viralDna
2726
- }).catch((err) => {
2727
- console.error("smart decompose editor-harness pass failed", err instanceof Error ? err.message : err);
2728
- return DEFAULT_EDITOR_HARNESS;
2729
- });
2730
- // NOTE: the per-scene clip-annotation + content-format pass is deliberately
2731
- // NOT run here — it is a separate vision call that would risk the ~60s origin
2732
- // timeout on longer videos. /auto-decompose seeds scene-annotations.json
2733
- // "pending" and /scene-annotations-poll drives annotateScenesFromSource
2734
- // against these finalized scenes (persisted in smart-decompose.json).
2735
- devLog("hyperframes.smart_decompose.completed", {
2736
- provider: attempt.provider,
2737
- model: attempt.model,
2738
- duration_ms: Date.now() - startedAt,
2739
- scene_count: finalScenes.length,
2740
- caption_count: finalCaptions.length,
2741
- captions_snapped: snapped.snappedCount,
2742
- scenes_snapped: snappedScenes.snappedCount,
2743
- placement_surface_count: finalPlacementSurfaces.length,
2744
- editor_harness_format: editorHarness.format,
2745
- editor_harness_scene_count: editorHarness.scenes.length
2746
- });
2747
- return {
2748
- summary: typeof record.summary === "string" ? record.summary : "",
2749
- viralDna,
2750
- placementSurfaces: finalPlacementSurfaces,
2751
- editorHarness,
2752
- transcript,
2753
- durationSeconds: meta.durationSeconds,
2754
- width: meta.width,
2755
- height: meta.height,
2756
- scenes: finalScenes,
2757
- captions: finalCaptions,
2758
- provider: attempt.provider,
2759
- model: attempt.model
2760
- };
2761
- }
2762
- catch (error) {
2763
- devLog("hyperframes.smart_decompose.failed", {
2764
- source_url: input.sourceUrl,
2765
- duration_ms: Date.now() - startedAt,
2766
- ...devErrorFields(error)
2767
- }, "error");
2768
- throw error;
2769
- }
2770
- finally {
2771
- await rm(tempDir, { recursive: true, force: true });
2772
- }
2773
- }
2774
- export function buildSmartDecomposedHtml(input) {
2775
- const { compositionId, sourceUrl, title, result } = input;
2776
- const totalDuration = Number(result.durationSeconds.toFixed(3));
2777
- const scenes = result.scenes.length > 0
2778
- ? result.scenes
2779
- : [{ slug: "scene_1", start: 0, duration: totalDuration, label: "Scene 1", description: "", viral_note: "" }];
2780
- // Assign each caption to the lowest track index (starting at 2 — track 0
2781
- // is video, track 1 is audio) where it doesn't temporally overlap any
2782
- // caption already on that track. Enforces the "no two clips on the same
2783
- // track can overlap in time" rule. Multiple stacked labels (like a
2784
- // staircase of "single/talking/dating/marriage/have kids" all shown for
2785
- // the full duration) end up on tracks 2,3,4,5,6 respectively.
2786
- const CAPTION_TRACK_START = 2;
2787
- const captionTrackClipRanges = [];
2788
- const assignCaptionTrackIndex = (start, end) => {
2789
- for (let i = 0; i < captionTrackClipRanges.length; i += 1) {
2790
- if (!captionTrackClipRanges[i].some((r) => start < r.end && r.start < end)) {
2791
- captionTrackClipRanges[i].push({ start, end });
2792
- return CAPTION_TRACK_START + i;
2793
- }
2794
- }
2795
- captionTrackClipRanges.push([{ start, end }]);
2796
- return CAPTION_TRACK_START + captionTrackClipRanges.length - 1;
2797
- };
2798
- const sceneEls = scenes.map((scene, index) => {
2799
- const start = Math.max(0, Math.min(totalDuration - 0.1, Number(scene.start.toFixed(3))));
2800
- const duration = Math.max(0.1, Number(scene.duration.toFixed(3)));
2801
- const end = Math.min(totalDuration, start + duration);
2802
- const id = shortElementId();
2803
- const sceneSlug = scene.slug || `scene_${index + 1}`;
2804
- return `<div class="clip" id="${id}" data-hf-id="${id}" data-hf-slug="${escapeAttr(sceneSlug)}" data-layer-mode="both" data-layer-kind="video" data-viral-note="${escapeAttr(scene.viral_note ?? "")}" data-start="${start}" data-duration="${Number(duration.toFixed(3))}" data-end="${Number(end.toFixed(3))}" data-track-index="0" data-label="${escapeAttr(scene.label)}" data-media-start="${start}" data-playback-start="${start}" data-vf-timeline-proxy="video" data-src="${escapeAttr(sourceUrl)}" style="position:absolute;left:0%;top:0%;width:1%;height:1%;z-index:0;display:none;pointer-events:none;"></div>`;
2805
- }).join("\n ");
2806
- // Sort captions by start time so the track-assignment greedy fills lower
2807
- // tracks first with earlier captions.
2808
- const sortedCaptions = [...result.captions].sort((a, b) => (a.start - b.start) || (a.duration - b.duration));
2809
- const captionEls = sortedCaptions.map((caption, index) => {
2810
- const start = Math.max(0, Math.min(totalDuration - 0.1, Number(caption.start.toFixed(3))));
2811
- const duration = Math.max(0.1, Math.min(totalDuration - start, Number(caption.duration.toFixed(3))));
2812
- const end = start + duration;
2813
- const trackIndex = assignCaptionTrackIndex(start, end);
2814
- const id = shortElementId();
2815
- const captionSlug = caption.slug || `caption_${index + 1}`;
2816
- const clampedX = clampPercent(caption.x, 0, 95);
2817
- const clampedY = clampPercent(caption.y, 0, 95);
2818
- const clampedW = clampPercent(caption.width, 5, 100 - clampedX);
2819
- const clampedH = clampPercent(caption.height, 3, 100 - clampedY);
2820
- const escapedText = escapeHtml(caption.text);
2821
- const innerStyle = captionInlineTextStyle(caption);
2822
- return `<div class="clip" id="${id}" data-hf-id="${id}" data-hf-slug="${escapeAttr(captionSlug)}" data-layer-mode="publish" data-layer-kind="caption" data-viral-note="${escapeAttr(caption.viral_note ?? "")}" data-start="${start}" data-duration="${Number(duration.toFixed(3))}" data-end="${Number(end.toFixed(3))}" data-track-index="${trackIndex}" data-label="${escapeAttr(caption.text.slice(0, 40))}" data-text-background-style="${caption.background_style}" data-text-background-color="${escapeAttr(caption.background)}" data-font-family="${escapeAttr(caption.font_family)}" style="position:absolute;left:${clampedX}%;top:${clampedY}%;width:${clampedW}%;height:${clampedH}%;z-index:${trackIndex};display:flex;align-items:center;justify-content:center;padding:3px;box-sizing:border-box;text-align:center;color:${caption.color};background:transparent;font-family:'${caption.font_family}', 'TikTok Sans', sans-serif;font-weight:${caption.font_weight};font-size:${caption.font_size}px;line-height:1.2;overflow:visible;"><span data-vf-text-lines="true"><span data-vf-text-inline="true" style="${innerStyle}">${escapedText}</span></span></div>`;
2823
- }).join("\n ");
2824
- const audioId = shortElementId();
2825
- const audioEl = `<audio class="clip" id="${audioId}" data-hf-id="${audioId}" data-hf-slug="source_audio" data-layer-mode="both" data-layer-kind="audio" data-viral-note="Original TikTok audio track." data-start="0" data-duration="${totalDuration}" data-end="${totalDuration}" data-track-index="1" data-label="Original audio" data-media-start="0" data-playback-start="0" data-timeline-role="music" data-volume="1" src="${escapeAttr(sourceUrl)}" preload="metadata"></audio>`;
2826
- const viralDnaJson = escapeAttr(JSON.stringify(result.viralDna ?? DEFAULT_VIRAL_DNA));
2827
- const editorHarnessJson = escapeAttr(JSON.stringify(result.editorHarness ?? DEFAULT_EDITOR_HARNESS));
2828
- return `<!doctype html>
2829
- <html lang="en">
2830
- <head>
2831
- <meta charset="UTF-8" />
2832
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
2833
- <style>
2834
- @import url("https://fonts.googleapis.com/css2?family=Abel&family=Montserrat:wght@600;700&family=Source+Code+Pro:wght@700&family=TikTok+Sans:wght@400;600;700;800;900&family=Yesteryear&display=swap");
2835
- html, body { width:100%; height:100%; margin:0; overflow:hidden; background:#050604; }
2836
- [data-composition-id] { position:relative; width:100vw; height:100vh; overflow:hidden; background:#050604; }
2837
- audio { display:none; }
2838
- </style>
2839
- </head>
2840
- <body>
2841
- <div data-composition-id="${escapeAttr(compositionId)}" data-source-video="${escapeAttr(sourceUrl)}" data-width="${result.width || 720}" data-height="${result.height || 1280}" data-duration="${totalDuration}" data-source-title="${escapeAttr(title)}" data-viral-dna="${viralDnaJson}" data-editor-harness="${editorHarnessJson}">
2842
- <video data-vf-playback-source="true" src="${escapeAttr(sourceUrl)}" muted playsinline preload="auto" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;object-fit:cover;background:#050604"></video>
2843
- ${sceneEls}
2844
- ${audioEl}
2845
- ${captionEls}
2846
- </div>
2847
- </body>
2848
- </html>`;
2849
- }
2850
- export function buildSmartDecomposedData(input) {
2851
- const { sourceUrl, result } = input;
2852
- const durationSeconds = Number(result.durationSeconds.toFixed(3));
2853
- return {
2854
- summary: result.summary,
2855
- scenes: result.scenes.map((scene) => ({
2856
- slug: scene.slug,
2857
- start: scene.start,
2858
- duration: scene.duration,
2859
- label: scene.label,
2860
- description: scene.description,
2861
- viral_note: scene.viral_note
2862
- })),
2863
- captions: result.captions.map((caption) => ({
2864
- slug: caption.slug,
2865
- text: caption.text,
2866
- start: caption.start,
2867
- duration: caption.duration,
2868
- x: caption.x,
2869
- y: caption.y,
2870
- width: caption.width,
2871
- height: caption.height,
2872
- color: caption.color,
2873
- background: caption.background,
2874
- background_style: caption.background_style,
2875
- font_family: caption.font_family,
2876
- font_size: caption.font_size,
2877
- font_weight: caption.font_weight,
2878
- viral_note: caption.viral_note
2879
- })),
2880
- viralDna: result.viralDna,
2881
- width: result.width,
2882
- height: result.height,
2883
- durationSeconds,
2884
- source_assets: [{
2885
- type: "video",
2886
- url: sourceUrl,
2887
- width: result.width,
2888
- height: result.height,
2889
- duration: durationSeconds
2890
- }]
2891
- };
2892
- }
2893
- // The public placeholder tile every blank scene starts on — a gray diagonal-
2894
- // striped 720×1280 image with a dashed text box (uploaded once by
2895
- // scripts/upload-scene-placeholder.ts). Hardcoded to the prod bucket URL (a
2896
- // public object that resolves from any stage) so the blank composition renders
2897
- // identically everywhere. Mirrors SCENE_PLACEHOLDER_IMAGE_URL in the SPA editor.
2898
- export const SCENE_PLACEHOLDER_IMAGE_URL = "https://vidfarmprodstack-vidfarmbucket335ee12f-0vsvtd5earqy.s3.us-east-1.amazonaws.com/primitives/placeholders/scene-placeholder.png";
2899
- const BLANK_PROJECT_SCENE_COUNT = 3;
2900
- const BLANK_PROJECT_SCENE_DURATION = 4;
2901
- // The canonical, reusable "new blank project" composition. This is the single
2902
- // composition.html every fresh /editor/original/fork/new opens on, and the base
2903
- // every blank project forks from (POST /api/v1/compositions seeds a new fork by
2904
- // copying this). It mirrors blankProjectSemanticLayers() in the SPA editor: a
2905
- // continuous placeholder image track (3 butt-cut 4s scenes on track 0) plus a
2906
- // "Scene 1/2/3" caption on track 3, giving the user an editable 12s scaffold
2907
- // instead of an empty canvas. Like the other builders it emits clips only — the
2908
- // runtime is injected on serve by rewriteHyperframesDraftCompositionHtml, and z
2909
- // is normalized to track index on the way out.
2910
- export function buildBlankCompositionHtml(compositionId) {
2911
- const totalDuration = BLANK_PROJECT_SCENE_COUNT * BLANK_PROJECT_SCENE_DURATION;
2912
- const sceneEls = [];
2913
- const captionEls = [];
2914
- for (let index = 0; index < BLANK_PROJECT_SCENE_COUNT; index += 1) {
2915
- const sceneNumber = index + 1;
2916
- const start = index * BLANK_PROJECT_SCENE_DURATION;
2917
- const end = start + BLANK_PROJECT_SCENE_DURATION;
2918
- // Placeholder image scene — full canvas, track 0. Together the three form
2919
- // one continuous 12s media track the user replaces with real footage.
2920
- const imageId = `scene-${sceneNumber}-image`;
2921
- sceneEls.push(`<img class="clip" id="${imageId}" data-hf-id="${imageId}" data-hf-slug="scene_${sceneNumber}" data-layer-mode="both" data-layer-kind="image" data-viral-note="Placeholder scene backdrop — replace with your own media." data-start="${start}" data-duration="${BLANK_PROJECT_SCENE_DURATION}" data-end="${end}" data-track-index="0" data-label="Scene ${sceneNumber}" src="${escapeAttr(SCENE_PLACEHOLDER_IMAGE_URL)}" alt="" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;opacity:1;border-radius:0px;overflow:hidden;object-fit:cover;" />`);
2922
- // "Scene N" label sitting in the tile's blank text box — white on a solid
2923
- // black highlight so it reads over both the placeholder and swapped-in media.
2924
- const caption = {
2925
- slug: `scene_${sceneNumber}_label`,
2926
- text: `Scene ${sceneNumber}`,
2927
- start,
2928
- duration: BLANK_PROJECT_SCENE_DURATION,
2929
- x: 13.5,
2930
- y: 44,
2931
- width: 73,
2932
- height: 12,
2933
- font_size: 34,
2934
- color: "#ffffff",
2935
- background: "#000000",
2936
- background_style: "highlight-solid",
2937
- font_family: "TikTok Sans",
2938
- font_weight: 700,
2939
- viral_note: "Placeholder scene label — edit or delete once you add real media."
2940
- };
2941
- const captionId = `scene-${sceneNumber}-caption`;
2942
- const innerStyle = captionInlineTextStyle(caption);
2943
- captionEls.push(`<div class="clip" id="${captionId}" data-hf-id="${captionId}" data-hf-slug="${escapeAttr(caption.slug)}" data-layer-mode="publish" data-layer-kind="caption" data-viral-note="${escapeAttr(caption.viral_note ?? "")}" data-start="${start}" data-duration="${BLANK_PROJECT_SCENE_DURATION}" data-end="${end}" data-track-index="3" data-label="${escapeAttr(caption.text)}" data-text-background-style="${caption.background_style}" data-text-background-color="${escapeAttr(caption.background)}" data-font-family="${escapeAttr(caption.font_family)}" style="position:absolute;left:${caption.x}%;top:${caption.y}%;width:${caption.width}%;height:${caption.height}%;z-index:3;display:flex;align-items:center;justify-content:center;padding:3px;box-sizing:border-box;text-align:center;color:${caption.color};background:transparent;font-family:'${caption.font_family}', 'TikTok Sans', sans-serif;font-weight:${caption.font_weight};font-size:${caption.font_size}px;line-height:1.2;overflow:visible;"><span data-vf-text-lines="true"><span data-vf-text-inline="true" style="${innerStyle}">${escapeHtml(caption.text)}</span></span></div>`);
2944
- }
2945
- // data-vf-origin="raw": a brand-new project, so the editor never auto-prompts
2946
- // the Decompose modal (there is no source video to decompose).
2947
- return `<!doctype html>
2948
- <html lang="en">
2949
- <head>
2950
- <meta charset="UTF-8" />
2951
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
2952
- <style>
2953
- @import url("https://fonts.googleapis.com/css2?family=Abel&family=Montserrat:wght@600;700&family=Source+Code+Pro:wght@700&family=TikTok+Sans:wght@400;600;700;800;900&family=Yesteryear&display=swap");
2954
- html, body { width:100%; height:100%; margin:0; overflow:hidden; background:#050604; }
2955
- [data-composition-id] { position:relative; width:100vw; height:100vh; overflow:hidden; background:#050604; }
2956
- </style>
2957
- </head>
2958
- <body>
2959
- <div data-composition-id="${escapeAttr(compositionId)}" data-vf-origin="raw" data-width="720" data-height="1280" data-duration="${totalDuration}" data-source-title="Untitled project">
2960
- ${sceneEls.join("\n ")}
2961
- ${captionEls.join("\n ")}
2962
- </div>
2963
- </body>
2964
- </html>`;
2965
- }
2966
- function normalizeSlug(value) {
2967
- return String(value ?? "")
2968
- .toLowerCase()
2969
- .replace(/[^a-z0-9]+/g, "_")
2970
- .replace(/^_+|_+$/g, "")
2971
- .slice(0, 64);
2972
- }
2973
- function dedupeSlug(base, used) {
2974
- const seed = base || "element";
2975
- if (!used.has(seed)) {
2976
- used.add(seed);
2977
- return seed;
2978
- }
2979
- for (let index = 2; index < 999; index += 1) {
2980
- const candidate = `${seed}_${index}`;
2981
- if (!used.has(candidate)) {
2982
- used.add(candidate);
2983
- return candidate;
2984
- }
2985
- }
2986
- const fallback = `${seed}_${Date.now().toString(36)}`;
2987
- used.add(fallback);
2988
- return fallback;
2989
- }
2990
- function escapeAttr(value) {
2991
- return String(value ?? "").replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
2992
- }
2993
- function escapeHtml(value) {
2994
- return String(value ?? "").replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
2995
- }
2996
- function clampPercent(value, min, max) {
2997
- if (!Number.isFinite(value))
2998
- return min;
2999
- return Math.max(min, Math.min(max, value));
3000
- }
3001
- export function stampCompositionIdentity(html, input) {
3002
- if (!input.forkId)
3003
- return html;
3004
- const composed = `${input.forkId}_v${input.version}`;
3005
- const { document } = parseHTML(html);
3006
- const root = document.querySelector("[data-composition-id]");
3007
- if (!root)
3008
- return html;
3009
- root.setAttribute("data-composition-id", composed);
3010
- root.setAttribute("data-fork-id", input.forkId);
3011
- root.setAttribute("data-fork-version", String(input.version));
3012
- return `<!doctype html>\n${document.documentElement.outerHTML}`;
3013
- }
3014
- // True when the union of canvas-filling intervals leaves any hole ≥ minGap within
3015
- // [0, total] — a leading gap, a hole between clips, or a trailing gap. Mirrors the
3016
- // editor-side computeCompositionGaps intent (see src/devcli/composition-edit.ts).
3017
- function compositionHasCoverageGap(intervals, total, minGap = 0.2) {
3018
- if (!Number.isFinite(total) || total <= 0)
3019
- return false;
3020
- const sorted = intervals.filter(([start, end]) => end > start).sort((a, b) => a[0] - b[0]);
3021
- let cursor = 0;
3022
- for (const [start, end] of sorted) {
3023
- if (start > cursor + minGap)
3024
- return true;
3025
- cursor = Math.max(cursor, end);
3026
- }
3027
- return cursor < total - minGap;
3028
- }
3029
- // Scene transitions: the INCOMING clip carries data-transition and its vf-tr-*
3030
- // animation plays over the clip's first data-transition-duration seconds — the
3031
- // render engine's CSS adapter scrubs that on its own. What the render engine
3032
- // canNOT do is keep the OUTGOING clip on screen beneath it (visibility is
3033
- // derived strictly from data-start/data-duration), so materialize the overlap
3034
- // here: extend the previous same-track butt-cut clip's data-duration by the
3035
- // transition length. The granted extension is recorded in data-transition-hold
3036
- // so the pass is idempotent and self-healing — every run first restores each
3037
- // clip's nominal duration (duration - hold), then re-grants from the current
3038
- // set of [data-transition] clips (so removed/retimed transitions shrink their
3039
- // neighbor back). Stored drafts stay butt-cut; only publish/render output
3040
- // carries the overlap. The preview runtime applies the same window live.
3041
- function materializeTransitionOverlaps(document) {
3042
- const upsertStyleVar = (el, name, value) => {
3043
- const styleValue = el.getAttribute("style") || "";
3044
- const decl = `${name}:${value}`;
3045
- const pattern = new RegExp(`(^|;)\\s*${name.replace(/[.*+?^${}()|[\]\\-]/g, "\\$&")}\\s*:[^;]*`, "i");
3046
- const nextStyle = pattern.test(styleValue)
3047
- ? styleValue.replace(pattern, (_match, lead) => `${lead}${decl}`)
3048
- : styleValue.length === 0 || styleValue.endsWith(";")
3049
- ? `${styleValue}${decl}`
3050
- : `${styleValue};${decl}`;
3051
- if (nextStyle !== styleValue)
3052
- el.setAttribute("style", nextStyle);
3053
- };
3054
- const clips = [];
3055
- for (const el of Array.from(document.querySelectorAll("[data-start]"))) {
3056
- if (el.hasAttribute("data-composition-id"))
3057
- continue;
3058
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
3059
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
3060
- if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0)
3061
- continue;
3062
- const priorHold = Number.parseFloat(el.getAttribute("data-transition-hold") || "");
3063
- const hold = Number.isFinite(priorHold) && priorHold > 0 ? Math.min(priorHold, duration) : 0;
3064
- clips.push({
3065
- el,
3066
- track: Number.parseInt(el.getAttribute("data-track-index") || "0", 10) || 0,
3067
- start,
3068
- nominal: duration - hold,
3069
- hold: 0
3070
- });
3071
- }
3072
- if (!clips.length)
3073
- return;
3074
- for (const clip of clips) {
3075
- const transitionEl = clip.el;
3076
- const tagName = transitionEl.tagName?.toLowerCase?.() ?? "";
3077
- // Exit transitions: validate/clamp here; the tail-positioning delay var is
3078
- // re-derived in the final loop below once holds are known (the animation
3079
- // must end at the clip's rendered end, nominal + hold).
3080
- const outPreset = transitionEl.getAttribute("data-transition-out");
3081
- if (outPreset) {
3082
- if (!TRANSITION_OUT_PRESETS.includes(outPreset) || tagName === "audio") {
3083
- transitionEl.removeAttribute("data-transition-out");
3084
- transitionEl.removeAttribute("data-transition-out-duration");
3085
- }
3086
- else {
3087
- const outDuration = Math.min(clampTransitionDuration(transitionEl.getAttribute("data-transition-out-duration")), clip.nominal);
3088
- const outDurationValue = Number(outDuration.toFixed(3));
3089
- transitionEl.setAttribute("data-transition-out-duration", String(outDurationValue));
3090
- upsertStyleVar(transitionEl, "--vf-tr-out-dur", `${outDurationValue}s`);
3091
- }
3092
- }
3093
- const preset = transitionEl.getAttribute("data-transition");
3094
- if (!preset)
3095
- continue;
3096
- if (!TRANSITION_PRESETS.includes(preset)) {
3097
- // Unknown preset would render as a permanently-paused animation with no
3098
- // keyframes — harmless — but drop it so agents get clean state back.
3099
- transitionEl.removeAttribute("data-transition");
3100
- transitionEl.removeAttribute("data-transition-duration");
3101
- continue;
3102
- }
3103
- if (tagName === "audio") {
3104
- transitionEl.removeAttribute("data-transition");
3105
- transitionEl.removeAttribute("data-transition-duration");
3106
- continue;
3107
- }
3108
- // The animation cannot outlast its own clip.
3109
- const duration = Math.min(clampTransitionDuration(transitionEl.getAttribute("data-transition-duration")), clip.nominal);
3110
- const durationValue = Number(duration.toFixed(3));
3111
- transitionEl.setAttribute("data-transition-duration", String(durationValue));
3112
- // Keep the inline var (which drives animation-duration) in lockstep, same
3113
- // as the --vf-kb-dur re-derivation above.
3114
- upsertStyleVar(transitionEl, "--vf-tr-dur", `${durationValue}s`);
3115
- // Grant the hold to the clip this one cuts away from: same track, nominal
3116
- // end at (within epsilon of) this clip's start.
3117
- for (const candidate of clips) {
3118
- if (candidate === clip || candidate.track !== clip.track)
3119
- continue;
3120
- if (Math.abs(candidate.start + candidate.nominal - clip.start) > TRANSITION_ADJACENCY_EPSILON)
3121
- continue;
3122
- const candidateTag = candidate.el.tagName?.toLowerCase?.() ?? "";
3123
- if (candidateTag === "audio")
3124
- continue;
3125
- candidate.hold = Math.max(candidate.hold, duration);
3126
- }
3127
- }
3128
- for (const clip of clips) {
3129
- const nextDuration = Number((clip.nominal + clip.hold).toFixed(3));
3130
- const currentDuration = Number.parseFloat(clip.el.getAttribute("data-duration") || "");
3131
- if (currentDuration !== nextDuration) {
3132
- clip.el.setAttribute("data-duration", String(nextDuration));
3133
- }
3134
- if (clip.el.hasAttribute("data-end")) {
3135
- clip.el.setAttribute("data-end", String(Number((clip.start + nextDuration).toFixed(3))));
3136
- }
3137
- if (clip.hold > 0) {
3138
- clip.el.setAttribute("data-transition-hold", String(Number(clip.hold.toFixed(3))));
3139
- }
3140
- else {
3141
- clip.el.removeAttribute("data-transition-hold");
3142
- }
3143
- // Position the exit animation at the clip's rendered tail. Timeline edits
3144
- // touch data-duration only, so this re-derivation (like --vf-kb-dur) is
3145
- // what keeps stored delays honest after retimes.
3146
- if (clip.el.getAttribute("data-transition-out")) {
3147
- const outDuration = clampTransitionDuration(clip.el.getAttribute("data-transition-out-duration"));
3148
- upsertStyleVar(clip.el, "--vf-tr-out-delay", `${transitionOutDelaySeconds(nextDuration, outDuration)}s`);
3149
- }
3150
- }
3151
- }
3152
- export function normalizePublishHtml(html) {
3153
- const { document } = parseHTML(html);
3154
- // Coverage gaps: any interval of the timeline with no active canvas-filling
3155
- // layer renders as the page background (#050604 — near-black), which the user
3156
- // experiences as the video "going black" mid-composition (e.g. a scene that
3157
- // was shortened/deleted or a replacement clip shorter than its slot). The
3158
- // full-length backing source video (data-vf-playback-source) is the natural
3159
- // fallback — every scene proxy is cut from that same source — but it is
3160
- // normally dropped at publish. When a gap exists we instead retain it as a
3161
- // timed bottom base layer so uncovered intervals show the original footage
3162
- // rather than black; with no gap we drop it (avoids decoding a redundant
3163
- // full-length video on every render).
3164
- const compositionRoot = document.querySelector("[data-composition-id]");
3165
- const totalDuration = compositionRoot
3166
- ? Number.parseFloat(compositionRoot.getAttribute("data-duration") || "")
3167
- : NaN;
3168
- const coverageIntervals = [];
3169
- for (const el of Array.from(document.querySelectorAll('[data-vf-timeline-proxy="video"], video[data-start], img[data-start]'))) {
3170
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
3171
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
3172
- if (Number.isFinite(start) && Number.isFinite(duration) && duration > 0) {
3173
- coverageIntervals.push([start, start + duration]);
3174
- }
3175
- }
3176
- const retainBackingVideo = Number.isFinite(totalDuration) &&
3177
- totalDuration > 0 &&
3178
- compositionHasCoverageGap(coverageIntervals, totalDuration);
3179
- for (const backingVideo of Array.from(document.querySelectorAll("[data-vf-playback-source]"))) {
3180
- if (!retainBackingVideo) {
3181
- backingVideo.remove();
3182
- continue;
3183
- }
3184
- // Retime the backing video into a full-timeline base layer the render engine
3185
- // seeks per frame (video[data-start]), pinned below every proxy (z-index:-1).
3186
- backingVideo.removeAttribute("data-vf-playback-source");
3187
- backingVideo.setAttribute("data-start", "0");
3188
- backingVideo.setAttribute("data-duration", String(totalDuration));
3189
- backingVideo.setAttribute("data-media-start", "0");
3190
- backingVideo.setAttribute("muted", "");
3191
- backingVideo.setAttribute("playsinline", "");
3192
- backingVideo.setAttribute("preload", "auto");
3193
- backingVideo.setAttribute("style", [
3194
- "position:absolute",
3195
- "left:0%",
3196
- "top:0%",
3197
- "width:100%",
3198
- "height:100%",
3199
- "z-index:-1",
3200
- "object-fit:cover",
3201
- "background:#050604"
3202
- ].join(";"));
3203
- }
3204
- for (const proxy of Array.from(document.querySelectorAll('[data-vf-timeline-proxy="video"]'))) {
3205
- const video = document.createElement("video");
3206
- const source = proxy.getAttribute("data-src") || proxy.getAttribute("src") || "";
3207
- for (const attr of Array.from(proxy.attributes)) {
3208
- if (attr.name === "data-vf-timeline-proxy" || attr.name === "data-src")
3209
- continue;
3210
- video.setAttribute(attr.name, attr.value);
3211
- }
3212
- video.setAttribute("src", source);
3213
- video.setAttribute("muted", "");
3214
- video.setAttribute("playsinline", "");
3215
- video.setAttribute("preload", "auto");
3216
- const trackIndex = Number.parseInt(proxy.getAttribute("data-track-index") ?? "0", 10) || 0;
3217
- video.setAttribute("style", [
3218
- "position:absolute",
3219
- "left:0%",
3220
- "top:0%",
3221
- "width:100%",
3222
- "height:100%",
3223
- `z-index:${trackIndex}`,
3224
- "object-fit:cover",
3225
- "background:#050604"
3226
- ].join(";"));
3227
- proxy.replaceWith(video);
3228
- }
3229
- // Enforce z-index = data-track-index for every timed layer so visual stacking
3230
- // strictly follows the timeline layer index. No arbitrary inline z-index survives.
3231
- for (const el of Array.from(document.querySelectorAll("[data-track-index]"))) {
3232
- const trackAttr = el.getAttribute("data-track-index");
3233
- if (!trackAttr)
3234
- continue;
3235
- const trackIndex = Number.parseInt(trackAttr, 10);
3236
- if (!Number.isFinite(trackIndex))
3237
- continue;
3238
- const html = el;
3239
- if (typeof html.style === "undefined")
3240
- continue;
3241
- html.style.zIndex = String(trackIndex);
3242
- }
3243
- // HyperFrames contract: visible timed elements (div/img) carry class="clip" —
3244
- // the upstream runtime keeps clip-less timed elements visible for the whole
3245
- // composition. video/audio deliberately stay bare (framework-managed), so the
3246
- // class is STRIPPED there (the editor's add-layer paths set it universally).
3247
- // Only promote layers with a valid timing window: a clip-less element with a
3248
- // missing/invalid data-duration historically stayed visible full-length, and
3249
- // promoting it would silently hide it on re-renders of older compositions.
3250
- for (const el of Array.from(document.querySelectorAll("[data-start]"))) {
3251
- if (el.hasAttribute("data-composition-id"))
3252
- continue;
3253
- const tag = el.tagName.toLowerCase();
3254
- if (tag === "video" || tag === "audio") {
3255
- el.classList.remove("clip");
3256
- if (!el.getAttribute("class"))
3257
- el.removeAttribute("class");
3258
- continue;
3259
- }
3260
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
3261
- const clipDuration = Number.parseFloat(el.getAttribute("data-duration") || "");
3262
- if (!Number.isFinite(start) || start < 0 || !Number.isFinite(clipDuration) || clipDuration <= 0)
3263
- continue;
3264
- if (!el.classList.contains("clip")) {
3265
- el.classList.add("clip");
3266
- }
3267
- }
3268
- // Ken Burns motion must span exactly the clip: timeline edits touch
3269
- // data-duration only, so re-derive the inline --vf-kb-dur (which drives
3270
- // animation-duration) from data-duration on every [data-kenburns] layer.
3271
- // Also make sure the shared keyframes stylesheet travels with compositions
3272
- // stored before the builder started embedding it.
3273
- const kenBurnsLayers = Array.from(document.querySelectorAll("[data-kenburns]"));
3274
- for (const el of kenBurnsLayers) {
3275
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
3276
- if (!Number.isFinite(duration) || duration <= 0)
3277
- continue;
3278
- const styleValue = el.getAttribute("style") || "";
3279
- const durDecl = `--vf-kb-dur:${Number(duration.toFixed(3))}s`;
3280
- const nextStyle = /(^|;)\s*--vf-kb-dur\s*:[^;]*/i.test(styleValue)
3281
- ? styleValue.replace(/(^|;)\s*--vf-kb-dur\s*:[^;]*/i, (_match, lead) => `${lead}${durDecl}`)
3282
- : styleValue.length === 0 || styleValue.endsWith(";")
3283
- ? `${styleValue}${durDecl}`
3284
- : `${styleValue};${durDecl}`;
3285
- if (nextStyle !== styleValue)
3286
- el.setAttribute("style", nextStyle);
3287
- }
3288
- if (kenBurnsLayers.length > 0 && !html.includes(KEN_BURNS_STYLE_MARKER)) {
3289
- const styleEl = document.createElement("style");
3290
- styleEl.textContent = KEN_BURNS_CSS;
3291
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3292
- }
3293
- // Animated captions: word windows are stored relative to the layer start
3294
- // (inline animation-delay/duration on each [data-cap-word]), so retimes need
3295
- // no re-derivation — but the shared vf-cap-* keyframes stylesheet must
3296
- // travel with compositions stored before the builder embedded it, or every
3297
- // word renders in its resting state.
3298
- if (document.querySelector("[data-caption-animation]") && !html.includes(CAPTION_STYLE_MARKER)) {
3299
- const styleEl = document.createElement("style");
3300
- styleEl.textContent = CAPTION_ANIM_CSS;
3301
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3302
- }
3303
- materializeTransitionOverlaps(document);
3304
- if (document.querySelector("[data-transition], [data-transition-out]") && !html.includes(TRANSITION_STYLE_MARKER)) {
3305
- // Compositions stored before V2 carry the V1 block inline: upgrade it in
3306
- // place (exact string swap) instead of stacking a second stylesheet. If the
3307
- // legacy block drifted, append after it — later source order wins the
3308
- // equal-specificity animation-name battle, so V2 still takes effect.
3309
- let upgradedLegacy = false;
3310
- if (html.includes("__VF_TRANSITIONS_V1__")) {
3311
- for (const styleEl of Array.from(document.querySelectorAll("style"))) {
3312
- const text = styleEl.textContent || "";
3313
- if (!text.includes("__VF_TRANSITIONS_V1__"))
3314
- continue;
3315
- styleEl.textContent = text.includes(LEGACY_TRANSITION_CSS_V1)
3316
- ? text.split(LEGACY_TRANSITION_CSS_V1).join(TRANSITION_CSS)
3317
- : `${text}\n${TRANSITION_CSS}`;
3318
- upgradedLegacy = true;
3319
- }
3320
- }
3321
- if (!upgradedLegacy) {
3322
- const styleEl = document.createElement("style");
3323
- styleEl.textContent = TRANSITION_CSS;
3324
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3325
- }
3326
- }
3327
- // Editor sessions occasionally leave inline style attributes with double
3328
- // quotes inside CSS values (font-family: "TikTok Sans"). When serialized into
3329
- // an HTML attribute, those quotes become &quot; entities. The render lambda
3330
- // resolves font-family with a regex over raw HTML and misreads &quot; as a
3331
- // literal font name, exploding with FontFetchError. Normalize to single
3332
- // quotes so serialization stays entity-free.
3333
- for (const el of Array.from(document.querySelectorAll("[style]"))) {
3334
- const styleValue = el.getAttribute("style");
3335
- if (!styleValue || styleValue.indexOf('"') === -1)
3336
- continue;
3337
- el.setAttribute("style", styleValue.replace(/"/g, "'"));
3338
- }
3339
- return `<!doctype html>\n${document.documentElement.outerHTML}`;
3340
- }
3341
- /**
3342
- * Reconstruct a Step Functions execution ARN from the render state machine ARN
3343
- * plus a deterministic execution name, so a re-invoked job runner can poll the
3344
- * SAME render without persisting the ARN handed back at dispatch time.
3345
- * arn:...:stateMachine:NAME -> arn:...:execution:NAME:EXEC_NAME
3346
- */
3347
- function executionArnFromName(stateMachineArn, executionName) {
3348
- const marker = ":stateMachine:";
3349
- const idx = stateMachineArn.indexOf(marker);
3350
- if (idx === -1) {
3351
- throw new Error(`Unexpected HyperFrames render state machine ARN: ${stateMachineArn}`);
3352
- }
3353
- const prefix = stateMachineArn.slice(0, idx);
3354
- const machineName = stateMachineArn.slice(idx + marker.length);
3355
- return `${prefix}:execution:${machineName}:${executionName}`;
3356
- }
3357
- /**
3358
- * DescribeExecution for a render execution. Returns the Step Functions status,
3359
- * or `null` when the execution does not exist yet (never dispatched) — the
3360
- * signal renderStep() uses to decide between dispatch and resume.
3361
- */
3362
- async function describeRenderExecutionStatus(executionArn, region) {
3363
- const client = new SFNClient({ region });
3364
- try {
3365
- const response = await client.send(new DescribeExecutionCommand({ executionArn }));
3366
- return response.status ?? "RUNNING";
3367
- }
3368
- catch (error) {
3369
- if (error instanceof ExecutionDoesNotExist) {
3370
- return null;
3371
- }
3372
- const name = error instanceof Error ? error.name : "";
3373
- if (name === "ExecutionDoesNotExist") {
3374
- return null;
3375
- }
3376
- throw error;
3377
- }
3378
- }
3379
- async function resolveHyperframesStack() {
3380
- const region = config.HYPERFRAMES_RENDER_REGION || config.AWS_REGION || "us-east-1";
3381
- const lambdaMemoryMb = positiveInteger(config.HYPERFRAMES_RENDER_MEMORY_MB, 3072);
3382
- const envStack = {
3383
- stackName: config.HYPERFRAMES_LAMBDA_STACK_NAME || "hyperframes-vidfarm-prod",
3384
- bucketName: config.HYPERFRAMES_RENDER_BUCKET_NAME || config.HYPERFRAMES_LAMBDA_BUCKET_NAME || "",
3385
- stateMachineArn: config.HYPERFRAMES_RENDER_STATE_MACHINE_ARN || config.HYPERFRAMES_STATE_MACHINE_ARN || "",
3386
- region,
3387
- lambdaMemoryMb
3388
- };
3389
- if (envStack.bucketName && envStack.stateMachineArn) {
3390
- return envStack;
3391
- }
3392
- if (stackCache && stackCache.expiresAt > Date.now()) {
3393
- return stackCache.value;
3394
- }
3395
- const stackName = envStack.stackName;
3396
- const cloudFormation = new CloudFormationClient({ region });
3397
- const result = await cloudFormation.send(new DescribeStacksCommand({ StackName: stackName }));
3398
- const stack = result.Stacks?.[0];
3399
- if (!stack) {
3400
- throw new Error(`HyperFrames Lambda stack not found: ${stackName}`);
3401
- }
3402
- const outputs = stack.Outputs || [];
3403
- const value = {
3404
- stackName,
3405
- bucketName: stackOutput(outputs, "RenderBucketName"),
3406
- stateMachineArn: stackOutput(outputs, "RenderStateMachineArn"),
3407
- functionArn: stackOutput(outputs, "RenderFunctionArn"),
3408
- region: stackOutput(outputs, "RenderRegion") || region,
3409
- lambdaMemoryMb: positiveInteger(stackOutput(outputs, "RenderMemoryMb"), lambdaMemoryMb)
3410
- };
3411
- if (!value.bucketName || !value.stateMachineArn) {
3412
- throw new Error(`HyperFrames Lambda stack ${stackName} is missing render outputs.`);
3413
- }
3414
- stackCache = { value, expiresAt: Date.now() + 60_000 };
3415
- return value;
3416
- }
3417
- async function stageProject(input) {
3418
- const root = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects");
3419
- const projectDir = path.join(root, `${Date.now().toString(36)}-${safeIdentifier(input.compositionId)}-${randomUUID().slice(0, 8)}`);
3420
- await rm(projectDir, { recursive: true, force: true });
3421
- await mkdir(projectDir, { recursive: true });
3422
- await writeFile(path.join(projectDir, "index.html"), input.compositionHtml, "utf8");
3423
- for (const file of input.projectFiles || []) {
3424
- const target = safeProjectPath(projectDir, file.path);
3425
- await mkdir(path.dirname(target), { recursive: true });
3426
- await writeFile(target, file.contents);
3427
- }
3428
- for (const copyPath of input.copyPaths || []) {
3429
- const target = safeProjectPath(projectDir, copyPath.to);
3430
- await mkdir(path.dirname(target), { recursive: true });
3431
- await cp(copyPath.from, target, { recursive: true, force: true });
3432
- }
3433
- return projectDir;
3434
- }
3435
- function extractCompositionMetadata(html, input) {
3436
- const compositionId = safeIdentifier(input.compositionId || htmlAttribute(html, "data-composition-id") || input.composition?.compositionId || "composition");
3437
- const width = positiveInteger(input.width || htmlAttribute(html, "data-width") || input.composition?.width, 720);
3438
- const height = positiveInteger(input.height || htmlAttribute(html, "data-height") || input.composition?.height, 1280);
3439
- const fps = positiveInteger(input.fps || rootTagAttribute(html, "data-fps") || 30, 30);
3440
- const duration = Number.parseFloat(htmlAttribute(html, "data-duration") || String(input.composition?.duration || ""));
3441
- return {
3442
- compositionId,
3443
- title: input.title || htmlAttribute(html, "data-source-video") || compositionId,
3444
- width,
3445
- height,
3446
- fps,
3447
- duration: Number.isFinite(duration) && duration > 0 ? duration : null
3448
- };
3449
- }
3450
- function safeProjectPath(root, relativePath) {
3451
- const target = path.resolve(root, relativePath);
3452
- const resolvedRoot = path.resolve(root);
3453
- if (target !== resolvedRoot && !target.startsWith(`${resolvedRoot}${path.sep}`)) {
3454
- throw new Error(`Invalid HyperFrames project path: ${relativePath}`);
3455
- }
3456
- return target;
3457
- }
3458
- // Round the composition's data-width / data-height down to the nearest even
3459
- // integer so libx264 (yuv420p) accepts the frame size. Only rewrites values
3460
- // that are actually odd, leaving well-formed even compositions untouched.
3461
- function forceEvenCompositionDimensions(html) {
3462
- return html.replace(/\bdata-(width|height)=(["'])(\d+)\2/gi, (match, dim, quote, value) => {
3463
- const n = Number.parseInt(value, 10);
3464
- if (!Number.isFinite(n) || n % 2 === 0)
3465
- return match;
3466
- return `data-${dim}=${quote}${n - 1}${quote}`;
3467
- });
3468
- }
3469
- // Reads an attribute from the SAME opening tag that carries
3470
- // data-composition-id (the composition root). htmlAttribute matches the first
3471
- // occurrence anywhere in the document, which nested sub-compositions or
3472
- // embedded snippets could hijack — fps must come from the root only.
3473
- function rootTagAttribute(html, name) {
3474
- const rootTag = html.match(/<[a-zA-Z][^>]*\bdata-composition-id\s*=[^>]*>/);
3475
- if (!rootTag) {
3476
- return "";
3477
- }
3478
- const value = rootTag[0].match(new RegExp(`\\b${name}\\s*=\\s*["']([^"']*)["']`, "i"));
3479
- return value ? value[1] : "";
3480
- }
3481
- function htmlAttribute(html, name) {
3482
- const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3483
- const match = html.match(new RegExp(`${escaped}=(["'])(.*?)\\1`, "i"));
3484
- return match?.[2] || "";
3485
- }
3486
- function stackOutput(outputs, key) {
3487
- return outputs.find((output) => output.OutputKey === key)?.OutputValue
3488
- || outputs.find((output) => String(output.OutputKey || "").endsWith(key))?.OutputValue
3489
- || "";
3490
- }
3491
- function parseS3Uri(uri) {
3492
- const match = String(uri || "").match(/^s3:\/\/([^/]+)\/(.+)$/);
3493
- return match ? { bucket: match[1], key: match[2] } : null;
3494
- }
3495
- async function signedS3Url(s3Uri, region) {
3496
- const parsed = parseS3Uri(s3Uri);
3497
- if (!parsed)
3498
- return null;
3499
- const client = new S3Client({ region });
3500
- return getSignedUrl(client, new GetObjectCommand({ Bucket: parsed.bucket, Key: parsed.key }), { expiresIn: 3600 });
3501
- }
3502
- function safeIdentifier(value) {
3503
- return String(value || "composition")
3504
- .toLowerCase()
3505
- .replace(/[^a-z0-9_-]+/g, "-")
3506
- .replace(/^-+|-+$/g, "")
3507
- .slice(0, 64) || "composition";
3508
- }
3509
- function positiveInteger(value, fallback) {
3510
- const next = Math.round(Number(value));
3511
- return Number.isFinite(next) && next > 0 ? next : fallback;
3512
- }
3513
- function normalizeRenderFps(value) {
3514
- if (value <= 24)
3515
- return 24;
3516
- if (value > 45)
3517
- return 60;
3518
- return 30;
3519
- }
3520
- function buildCastAnalysisPrompt(input) {
3521
- const summaryBlock = input.summary && input.summary.trim()
3522
- ? `\nVIDEO SUMMARY (from the first decompose pass):\n"""\n${input.summary.trim()}\n"""\n`
3523
- : "";
3524
- const sceneBlock = input.scenes && input.scenes.length > 0
3525
- ? `\nSCENES (slug — description, from the first pass; use these exact slugs in "appears_in"):\n${input.scenes
3526
- .map((s) => ` ${s.slug}: ${(s.description ?? "").slice(0, 240)}`)
3527
- .join("\n")}\n`
3528
- : "";
3529
- const frameBlock = input.frameTimestamps.length > 0
3530
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp): ${input.frameTimestamps
3531
- .map((t) => `${t.toFixed(2)}s`)
3532
- .join(", ")}\nPick "best_frame_seconds" from one of these exact timestamps — the frame where that person is largest, most frontal, and least occluded (ideally alone).\n`
3533
- : "";
3534
- return `You are identifying the recurring CAST (the people/characters) of a short-form vertical video so they can be reused and regenerated consistently in later edits.
3535
- ${summaryBlock}${sceneBlock}
3536
- Video metadata:
3537
- - duration: ${input.durationSeconds.toFixed(3)} seconds
3538
- - dimensions: ${input.width} x ${input.height} (canvas is 100% x 100%)
3539
- ${frameBlock}
3540
- Identify each DISTINCT person who appears. Merge the same person across scenes into ONE entry — do not create a new entry per scene. Ignore incidental crowd/background extras who never recur unless they are visually significant. If the video has no people (product-only, screen-recording, etc.), return an empty "cast" array.
3541
-
3542
- For each person, describe them in enough concrete detail (face, hair, skin tone, build, age range, distinguishing features, wardrobe) that an image model could regenerate the SAME person from the description alone. Give a stable snake_case "slug" (e.g. "hero_girl", "male_narrator"), a short "name"/label, a "role" of "main" | "secondary" | "background", the list of scene slugs they "appears_in", the single "best_frame_seconds" timestamp where they're cleanest, a normalized bounding box "bbox" {x,y,width,height} in 0..1 coordinates locating that person within that frame (null if you cannot localize), and "is_primary": true for the single most central recurring subject (false for the rest).
3543
-
3544
- Return strictly valid JSON:
3545
- {
3546
- "cast": [{
3547
- "slug": string,
3548
- "name": string,
3549
- "role": "main" | "secondary" | "background",
3550
- "description": string,
3551
- "wardrobe": string,
3552
- "appears_in": [string, ...],
3553
- "best_frame_seconds": number,
3554
- "bbox": { "x": number, "y": number, "width": number, "height": number } | null,
3555
- "is_primary": boolean
3556
- }]
3557
- }`;
3558
- }
3559
- function clamp01(value) {
3560
- const n = Number(value);
3561
- if (!Number.isFinite(n))
3562
- return 0;
3563
- return Math.min(1, Math.max(0, n));
3564
- }
3565
- function normalizeCastMembers(raw, durationSeconds) {
3566
- const root = raw && typeof raw === "object" ? raw : {};
3567
- const list = Array.isArray(root.cast)
3568
- ? root.cast
3569
- : Array.isArray(root.members)
3570
- ? root.members
3571
- : [];
3572
- const usedSlugs = new Set();
3573
- const members = [];
3574
- for (const entry of list) {
3575
- if (!entry || typeof entry !== "object")
3576
- continue;
3577
- const e = entry;
3578
- const description = typeof e.description === "string" ? e.description.trim() : "";
3579
- if (!description)
3580
- continue;
3581
- let slug = safeIdentifier(typeof e.slug === "string" && e.slug.trim() ? e.slug : (typeof e.name === "string" ? e.name : `cast-${members.length + 1}`));
3582
- while (usedSlugs.has(slug))
3583
- slug = `${slug}-${members.length + 1}`;
3584
- usedSlugs.add(slug);
3585
- const bboxRaw = e.bbox && typeof e.bbox === "object" ? e.bbox : null;
3586
- const bbox = bboxRaw
3587
- ? {
3588
- x: clamp01(bboxRaw.x),
3589
- y: clamp01(bboxRaw.y),
3590
- width: clamp01(bboxRaw.width ?? bboxRaw.w),
3591
- height: clamp01(bboxRaw.height ?? bboxRaw.h)
3592
- }
3593
- : null;
3594
- const best = Number(e.best_frame_seconds ?? e.best_frame ?? 0);
3595
- members.push({
3596
- slug,
3597
- name: typeof e.name === "string" && e.name.trim() ? e.name.trim().slice(0, 120) : slug,
3598
- role: typeof e.role === "string" && e.role.trim() ? e.role.trim().toLowerCase().slice(0, 40) : "secondary",
3599
- description: description.slice(0, 2000),
3600
- wardrobe: typeof e.wardrobe === "string" ? e.wardrobe.trim().slice(0, 600) : "",
3601
- appears_in: Array.isArray(e.appears_in)
3602
- ? e.appears_in.map((s) => (typeof s === "string" ? s.trim() : "")).filter(Boolean).slice(0, 40)
3603
- : [],
3604
- best_frame_seconds: Number.isFinite(best) && best > 0
3605
- ? Math.min(Math.max(0, best), Math.max(0, durationSeconds - 0.05))
3606
- : 0,
3607
- bbox: bbox && (bbox.width > 0 || bbox.height > 0) ? bbox : null,
3608
- is_primary: e.is_primary === true
3609
- });
3610
- }
3611
- // Guarantee exactly one primary when there's at least one member.
3612
- if (members.length > 0 && !members.some((m) => m.is_primary)) {
3613
- members[0].is_primary = true;
3614
- }
3615
- return members;
3616
- }
3617
- export async function analyzeCastMembers(input) {
3618
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
3619
- const sourcePath = path.join(tempDir, "source.mp4");
3620
- try {
3621
- await downloadVideoTo(input.sourceUrl, sourcePath);
3622
- const meta = await ffprobeVideo(sourcePath);
3623
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
3624
- throw new Error("Could not read video duration for cast analysis");
3625
- }
3626
- const frames = await sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir);
3627
- const prompt = buildCastAnalysisPrompt({
3628
- durationSeconds: meta.durationSeconds,
3629
- width: meta.width,
3630
- height: meta.height,
3631
- summary: input.summary ?? null,
3632
- scenes: input.scenes ?? [],
3633
- frameTimestamps: frames.map((f) => f.timestamp)
3634
- });
3635
- const attempt = await callVisionWithFallback({ prompt, frames, keys: input.keys });
3636
- devLog("hyperframes.cast_analyze.done", {
3637
- source_url: input.sourceUrl,
3638
- provider: attempt.provider,
3639
- model: attempt.model
3640
- });
3641
- return {
3642
- members: normalizeCastMembers(attempt.result, meta.durationSeconds),
3643
- provider: attempt.provider,
3644
- model: attempt.model,
3645
- durationSeconds: meta.durationSeconds,
3646
- width: meta.width,
3647
- height: meta.height
3648
- };
3649
- }
3650
- finally {
3651
- await rm(tempDir, { recursive: true, force: true }).catch(() => { });
3652
- }
3653
- }
3654
- //# sourceMappingURL=hyperframes.js.map