@mevdragon/vidfarm-devcli 0.20.5 → 0.20.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/.agents/skills/editor-capabilities/SKILL.md +15 -1
  2. package/.agents/skills/vidfarm-director/SKILL.md +106 -0
  3. package/.agents/skills/vidfarm-director/recipes/find-and-fork-template.md +16 -0
  4. package/.agents/skills/vidfarm-director/recipes/local-edit-render-approve.md +13 -0
  5. package/.agents/skills/vidfarm-director/recipes/onboard-a-new-director.md +13 -0
  6. package/.agents/skills/vidfarm-director/recipes/retheme-template.md +17 -0
  7. package/.agents/skills/vidfarm-director/references/assets-and-sourcing.md +111 -0
  8. package/.agents/skills/vidfarm-director/references/automation-and-local-dev.md +243 -0
  9. package/.agents/skills/vidfarm-director/references/core-workflows.md +273 -0
  10. package/.agents/skills/vidfarm-director/references/editor-workflows.md +365 -0
  11. package/.agents/skills/vidfarm-director/references/onboarding.md +28 -0
  12. package/.agents/skills/vidfarm-director/references/primitives.md +265 -0
  13. package/SKILL.director.md +521 -261
  14. package/dist/src/cli.js +404 -96
  15. package/dist/src/devcli/clips.js +15 -10
  16. package/dist/src/devcli/doctor.js +2 -2
  17. package/dist/src/devcli/local-backend.js +46 -2
  18. package/dist/src/services/clip-curation/cost.js +5 -1
  19. package/dist/src/services/clip-curation/gemini.js +10 -0
  20. package/package.json +30 -10
  21. package/SKILL.platform.md +0 -432
  22. package/demo/README.md +0 -28
  23. package/demo/dist/app.css +0 -1
  24. package/demo/dist/app.js +0 -1850
  25. package/demo/dist/chunks/chunk-DXB73IDG.js +0 -1
  26. package/demo/dist/chunks/chunk-S7OWAJDS.js +0 -36
  27. package/demo/dist/chunks/chunk-VTIBZ6AN.js +0 -1
  28. package/demo/dist/chunks/dist-ADSJKBVE.js +0 -332
  29. package/demo/dist/chunks/domEditingLayers-VZMLL4AP-SGHWPND4.js +0 -1
  30. package/demo/dist/chunks/hyperframes-player-XB65TCD6.js +0 -425
  31. package/demo/dist/chunks/lib-XAQ37YOE.js +0 -1
  32. package/demo/dist/chunks/src-TJ2QYA4U.js +0 -207
  33. package/demo/dist/favicon.ico +0 -0
  34. package/demo/dist/icons/timeline/audio.svg +0 -7
  35. package/demo/dist/icons/timeline/captions.svg +0 -5
  36. package/demo/dist/icons/timeline/composition.svg +0 -12
  37. package/demo/dist/icons/timeline/image.svg +0 -18
  38. package/demo/dist/icons/timeline/music.svg +0 -10
  39. package/demo/dist/icons/timeline/text.svg +0 -3
  40. package/demo/dist/index.html +0 -15
  41. package/dist/src/account-pages-legacy.js +0 -9396
  42. package/dist/src/account-pages.js +0 -61
  43. package/dist/src/app.js +0 -21602
  44. package/dist/src/composition-runtime.js +0 -1053
  45. package/dist/src/config.js +0 -217
  46. package/dist/src/context.js +0 -447
  47. package/dist/src/dev-app-legacy.js +0 -739
  48. package/dist/src/dev-app.js +0 -6
  49. package/dist/src/devcli/migrate-local.js +0 -140
  50. package/dist/src/devcli/sync.js +0 -368
  51. package/dist/src/domain.js +0 -5
  52. package/dist/src/editor-chat-history.js +0 -82
  53. package/dist/src/editor-chat.js +0 -756
  54. package/dist/src/editor-dark-theme.js +0 -1128
  55. package/dist/src/frontend/debug.js +0 -71
  56. package/dist/src/frontend/discover-client.js +0 -130
  57. package/dist/src/frontend/discover-store.js +0 -23
  58. package/dist/src/frontend/file-directory.js +0 -1018
  59. package/dist/src/frontend/flockposter-cache-store.js +0 -124
  60. package/dist/src/frontend/homepage-client.js +0 -446
  61. package/dist/src/frontend/homepage-shared.js +0 -201
  62. package/dist/src/frontend/homepage-store.js +0 -66
  63. package/dist/src/frontend/homepage-view.js +0 -705
  64. package/dist/src/frontend/page-runtime-client.js +0 -132
  65. package/dist/src/frontend/page-runtime-store.js +0 -9
  66. package/dist/src/frontend/sentry.js +0 -42
  67. package/dist/src/frontend/template-editor-chat.js +0 -4181
  68. package/dist/src/help-page.js +0 -346
  69. package/dist/src/homepage.js +0 -1458
  70. package/dist/src/index.js +0 -16
  71. package/dist/src/instrument.js +0 -30
  72. package/dist/src/landing-page.js +0 -384
  73. package/dist/src/page-runtime.js +0 -2
  74. package/dist/src/page-shell.js +0 -1452
  75. package/dist/src/primitive-context.js +0 -416
  76. package/dist/src/primitive-registry.js +0 -3940
  77. package/dist/src/primitive-sdk.js +0 -4
  78. package/dist/src/primitives/hyperframes-media.js +0 -108
  79. package/dist/src/react-page-shell.js +0 -35
  80. package/dist/src/ready-post-schedule-component.js +0 -1540
  81. package/dist/src/registry.js +0 -296
  82. package/dist/src/reskin/agency-page.js +0 -299
  83. package/dist/src/reskin/calendar-page.js +0 -568
  84. package/dist/src/reskin/chat-page.js +0 -942
  85. package/dist/src/reskin/discover-page.js +0 -1788
  86. package/dist/src/reskin/document.js +0 -1587
  87. package/dist/src/reskin/help-page.js +0 -357
  88. package/dist/src/reskin/index-page.js +0 -62
  89. package/dist/src/reskin/inpaint-clipper-page.js +0 -890
  90. package/dist/src/reskin/inpaint-page.js +0 -2554
  91. package/dist/src/reskin/inpaint-video-page.js +0 -1339
  92. package/dist/src/reskin/job-runs-page.js +0 -477
  93. package/dist/src/reskin/library-page.js +0 -1634
  94. package/dist/src/reskin/login-page.js +0 -262
  95. package/dist/src/reskin/portfolio-page.js +0 -687
  96. package/dist/src/reskin/pricing-page.js +0 -390
  97. package/dist/src/reskin/settings-page.js +0 -732
  98. package/dist/src/reskin/theme.js +0 -711
  99. package/dist/src/runtime.js +0 -35
  100. package/dist/src/services/api-call-history.js +0 -249
  101. package/dist/src/services/auth.js +0 -152
  102. package/dist/src/services/billing-pricing.js +0 -39
  103. package/dist/src/services/billing.js +0 -241
  104. package/dist/src/services/cast.js +0 -127
  105. package/dist/src/services/chat-threads.js +0 -92
  106. package/dist/src/services/clip-records.js +0 -250
  107. package/dist/src/services/clip-search.js +0 -77
  108. package/dist/src/services/clip-vectors.js +0 -125
  109. package/dist/src/services/composition-sanitize.js +0 -124
  110. package/dist/src/services/composition-watch.js +0 -79
  111. package/dist/src/services/elevenlabs.js +0 -222
  112. package/dist/src/services/file-directory.js +0 -117
  113. package/dist/src/services/fork-access.js +0 -93
  114. package/dist/src/services/fork-manifest.js +0 -42
  115. package/dist/src/services/ghostcut.js +0 -179
  116. package/dist/src/services/hyperframes.js +0 -3517
  117. package/dist/src/services/job-capacity.js +0 -14
  118. package/dist/src/services/job-logs.js +0 -197
  119. package/dist/src/services/jobs.js +0 -136
  120. package/dist/src/services/local-dynamo.js +0 -0
  121. package/dist/src/services/media-processing.js +0 -766
  122. package/dist/src/services/primitive-media-lambda.js +0 -280
  123. package/dist/src/services/providers.js +0 -2748
  124. package/dist/src/services/rate-limits.js +0 -262
  125. package/dist/src/services/scene-annotations.js +0 -32
  126. package/dist/src/services/serverless-auth.js +0 -382
  127. package/dist/src/services/serverless-jobs.js +0 -1084
  128. package/dist/src/services/serverless-provider-keys.js +0 -409
  129. package/dist/src/services/serverless-records.js +0 -1515
  130. package/dist/src/services/serverless-template-configs.js +0 -75
  131. package/dist/src/services/storage.js +0 -461
  132. package/dist/src/services/swipe-customize.js +0 -437
  133. package/dist/src/services/template-certification.js +0 -413
  134. package/dist/src/services/template-loader.js +0 -99
  135. package/dist/src/services/template-runtime-bundles.js +0 -217
  136. package/dist/src/services/template-sources.js +0 -1017
  137. package/dist/src/services/upstream.js +0 -248
  138. package/dist/src/services/video-normalization.js +0 -2
  139. package/dist/src/services/webhooks.js +0 -62
  140. package/dist/src/template-editor-pages.js +0 -2576
  141. package/dist/src/template-editor-shell.js +0 -2893
  142. package/dist/src/template-sdk.js +0 -4
  143. package/dist/src/worker.js +0 -17
  144. package/public/assets/discover-client-app.js +0 -1
  145. package/public/assets/file-directory-app.js +0 -3
  146. package/public/assets/homepage-app.js +0 -54
  147. package/public/assets/homepage-client-app.js +0 -80
  148. package/public/assets/page-runtime-client-app.js +0 -94
  149. package/public/assets/placeholders/scene-placeholder.png +0 -0
  150. package/src/assets/SELLING_AWARENESS_STAGES.md +0 -579
  151. package/src/assets/SELLING_WITH_HOOKS.md +0 -377
  152. package/src/assets/SELLING_WITH_VSLS.md +0 -606
  153. package/src/assets/favicon.ico +0 -0
  154. package/src/assets/logo-vidfarm.png +0 -0
@@ -1,3517 +0,0 @@
1
- import { spawn } from "node:child_process";
2
- import { randomUUID } from "node:crypto";
3
- import { createWriteStream } from "node:fs";
4
- import { cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
5
- import os from "node:os";
6
- import path from "node:path";
7
- import { pipeline } from "node:stream/promises";
8
- import { CloudFormationClient, DescribeStacksCommand } from "@aws-sdk/client-cloudformation";
9
- import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
10
- import { DescribeExecutionCommand, ExecutionDoesNotExist, SFNClient } from "@aws-sdk/client-sfn";
11
- import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
12
- import ffmpegStatic from "ffmpeg-static";
13
- import ffprobeStatic from "ffprobe-static";
14
- import { parseHTML } from "linkedom";
15
- import { config } from "../config.js";
16
- import { devErrorFields, devLog } from "../lib/dev-log.js";
17
- import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, LEGACY_TRANSITION_CSS_V1, TRANSITION_ADJACENCY_EPSILON, TRANSITION_CSS, TRANSITION_OUT_PRESETS, TRANSITION_PRESETS, TRANSITION_STYLE_MARKER, buildHyperframeCompositionHtml, clampTransitionDuration, transitionOutDelaySeconds } from "../hyperframes/composition.js";
18
- import { applyMarkupUsd } from "./billing-pricing.js";
19
- // The scene-annotation pass reuses the clip library's controlled vocabulary and
20
- // embedding-text recipe verbatim so a source scene's clip-match query is
21
- // symmetric with how a candidate clip was tagged/embedded. This is the ONE
22
- // coupling to the (parallel) clip-curation work — a plain leaf import; the
23
- // retrieval logic itself (search over these criteria) is merged in later.
24
- import { buildClipEmbeddingText, sanitizeTagValues, taxonomyForPrompt } from "./clip-curation/index.js";
25
- /** Thrown by the primitive render wrapper when a distributed render is still in
26
- * flight. The job runner catches it and parks the job as `waiting_for_render`
27
- * so Step Functions waits + re-invokes — no single Lambda holds the render open
28
- * for its 15-minute lifetime. */
29
- export class RenderInProgressError extends Error {
30
- executionArn;
31
- renderId;
32
- waitSeconds;
33
- progressFraction;
34
- constructor(input) {
35
- super(`HyperFrames render ${input.renderId} still in progress`);
36
- this.name = "RenderInProgressError";
37
- this.executionArn = input.executionArn;
38
- this.renderId = input.renderId;
39
- this.waitSeconds = input.waitSeconds;
40
- this.progressFraction = input.progressFraction;
41
- }
42
- }
43
- let stackCache = null;
44
- const SMART_DECOMPOSE_TEMP_PREFIX = "vidfarm-smart-decompose-";
45
- const SMART_DECOMPOSE_TEMP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
46
- async function sweepStaleSmartDecomposeTempDirs() {
47
- const tmpRoot = os.tmpdir();
48
- let entries;
49
- try {
50
- entries = await readdir(tmpRoot);
51
- }
52
- catch {
53
- return;
54
- }
55
- const now = Date.now();
56
- await Promise.all(entries.map(async (name) => {
57
- if (!name.startsWith(SMART_DECOMPOSE_TEMP_PREFIX))
58
- return;
59
- const full = path.join(tmpRoot, name);
60
- try {
61
- const info = await stat(full);
62
- if (!info.isDirectory())
63
- return;
64
- if (now - info.mtimeMs < SMART_DECOMPOSE_TEMP_MAX_AGE_MS)
65
- return;
66
- await rm(full, { recursive: true, force: true });
67
- }
68
- catch {
69
- /* ignore */
70
- }
71
- }));
72
- }
73
- sweepStaleSmartDecomposeTempDirs().catch((err) => {
74
- console.error("smart decompose temp sweep failed", err);
75
- });
76
- // True when no cloud HyperFrames render stack is wired (env or explicit
77
- // CloudFormation outputs). On a `vidfarm serve` box this is the normal state,
78
- // so render should run in-process rather than fail. `VIDFARM_LOCAL_RENDER`
79
- // forces local even when a cloud stack IS configured (useful for offline dev).
80
- function shouldRenderLocally() {
81
- if (config.VIDFARM_LOCAL_RENDER)
82
- return true;
83
- const hasCloudStack = Boolean((config.HYPERFRAMES_RENDER_STATE_MACHINE_ARN || config.HYPERFRAMES_STATE_MACHINE_ARN) &&
84
- (config.HYPERFRAMES_RENDER_BUCKET_NAME || config.HYPERFRAMES_LAMBDA_BUCKET_NAME));
85
- return !hasCloudStack;
86
- }
87
- // The bundled producer/aws-lambda libraries embed upstream-vendor doc links in
88
- // some error messages (font resolution, large-variables). Those URLs surface in
89
- // job logs and the export modal, so rewrite them to vidfarm's docs before the
90
- // message leaves the render layer.
91
- function scrubVendorDocLinks(text) {
92
- return text.replace(/https?:\/\/[a-z0-9.-]*heygen\.com[^\s"')\]]*/gi, "https://vidfarm.cc/skill/vidfarm-director");
93
- }
94
- export class HyperframesService {
95
- async render(input) {
96
- const compositionHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
97
- if (!compositionHtml.includes("data-composition-id=")) {
98
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
99
- }
100
- if (shouldRenderLocally()) {
101
- return this.renderLocal({ ...input, compositionHtml });
102
- }
103
- const prep = await this.prepareCloudRender(input, compositionHtml);
104
- const handle = await this.dispatchCloudRender(prep, input);
105
- if (input.wait === false) {
106
- devLog("hyperframes.render.dispatched_async", {
107
- composition_id: prep.metadata.compositionId,
108
- render_id: prep.renderId,
109
- execution_arn: handle.executionArn
110
- });
111
- return {
112
- ...handle,
113
- outputUrl: null,
114
- outputS3Uri: handle.outputS3Uri,
115
- endedAt: null,
116
- metadata: {
117
- mode: "hyperframes_lambda",
118
- status: "RUNNING",
119
- stack: prep.stack,
120
- composition: prep.metadata
121
- }
122
- };
123
- }
124
- const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
125
- const startedAtMs = Date.now();
126
- const timeoutMs = input.timeoutMs ?? 12 * 60 * 1000;
127
- const pollIntervalMs = input.pollIntervalMs ?? 5000;
128
- let pollCount = 0;
129
- let lastLoggedStatus = null;
130
- for (;;) {
131
- const progress = await getRenderProgress({
132
- executionArn: handle.executionArn,
133
- region: prep.stack.region,
134
- defaultMemorySizeMb: prep.stack.lambdaMemoryMb
135
- });
136
- pollCount += 1;
137
- if (progress.status !== lastLoggedStatus) {
138
- devLog("hyperframes.render.progress", {
139
- composition_id: prep.metadata.compositionId,
140
- render_id: prep.renderId,
141
- status: progress.status,
142
- poll_count: pollCount,
143
- elapsed_ms: Date.now() - startedAtMs,
144
- frames_rendered: progress.framesRendered,
145
- total_frames: progress.totalFrames
146
- });
147
- lastLoggedStatus = progress.status;
148
- }
149
- if (progress.status === "SUCCEEDED") {
150
- devLog("hyperframes.render.succeeded", {
151
- composition_id: prep.metadata.compositionId,
152
- render_id: prep.renderId,
153
- duration_ms: Date.now() - startedAtMs,
154
- output_s3_uri: progress.outputFile?.s3Uri || handle.outputS3Uri,
155
- frames_rendered: progress.framesRendered,
156
- total_frames: progress.totalFrames,
157
- lambdas_invoked: progress.lambdasInvoked,
158
- cost_usd: progress.costs?.accruedSoFarUsd
159
- });
160
- return this.buildCloudRenderResult(handle, progress, prep.metadata, prep.stack);
161
- }
162
- if (progress.fatalErrorEncountered || ["FAILED", "TIMED_OUT", "ABORTED"].includes(progress.status)) {
163
- devLog("hyperframes.render.failed", {
164
- composition_id: prep.metadata.compositionId,
165
- render_id: prep.renderId,
166
- status: progress.status,
167
- duration_ms: Date.now() - startedAtMs,
168
- errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
169
- }, "error");
170
- throw new Error(`HyperFrames render ${handle.renderId} failed: ${scrubVendorDocLinks(JSON.stringify(progress.errors))}`);
171
- }
172
- if (Date.now() - startedAtMs > timeoutMs) {
173
- devLog("hyperframes.render.timeout", {
174
- composition_id: prep.metadata.compositionId,
175
- render_id: prep.renderId,
176
- timeout_ms: timeoutMs,
177
- last_status: progress.status
178
- }, "error");
179
- throw new Error(`Timed out waiting for HyperFrames render ${handle.renderId}.`);
180
- }
181
- await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
182
- }
183
- }
184
- /**
185
- * One non-blocking step of a distributed render, keyed by a stable
186
- * `executionName`. The first call stages + dispatches the render to the
187
- * HyperFrames render state machine and returns `running` immediately; later
188
- * calls (from a re-invoked job runner) DescribeExecution the SAME execution
189
- * and either keep waiting, finalize on SUCCEEDED, or report failure. This is
190
- * what keeps a 40-minute render off any single Lambda's 15-minute clock.
191
- */
192
- async renderStep(input) {
193
- const compositionHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
194
- if (!compositionHtml.includes("data-composition-id=")) {
195
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
196
- }
197
- // Local/dev has no Step Functions — render in-process, blocking, and finalize
198
- // in one step so the devcli and local backend behave exactly as before.
199
- if (shouldRenderLocally()) {
200
- try {
201
- const result = await this.renderLocal({ ...input, compositionHtml });
202
- return { state: "succeeded", ...result };
203
- }
204
- catch (error) {
205
- return {
206
- state: "failed",
207
- renderId: input.executionName,
208
- executionArn: null,
209
- message: error instanceof Error ? error.message : String(error)
210
- };
211
- }
212
- }
213
- const stack = await resolveHyperframesStack();
214
- const executionArn = executionArnFromName(stack.stateMachineArn, input.executionName);
215
- const existingStatus = await describeRenderExecutionStatus(executionArn, stack.region);
216
- if (existingStatus === null) {
217
- // Never dispatched — stage + kick off the render, then park.
218
- const prep = await this.prepareCloudRender(input, compositionHtml, input.executionName);
219
- const handle = await this.dispatchCloudRender(prep, input);
220
- devLog("hyperframes.render.dispatched_async", {
221
- composition_id: prep.metadata.compositionId,
222
- render_id: prep.renderId,
223
- execution_arn: handle.executionArn
224
- });
225
- return { state: "running", executionArn: handle.executionArn, renderId: handle.renderId, progressFraction: 0.1 };
226
- }
227
- if (existingStatus === "RUNNING" || existingStatus === "PENDING_REDRIVE") {
228
- // Cheap path: DescribeExecution already told us it's still going. Avoid the
229
- // heavier GetExecutionHistory pagination that getRenderProgress performs.
230
- return { state: "running", executionArn, renderId: input.executionName, progressFraction: 0.5 };
231
- }
232
- // Terminal (SUCCEEDED / FAILED / TIMED_OUT / ABORTED) — read the full snapshot
233
- // for output location, frames, and cost metadata.
234
- const { getRenderProgress } = await import("@hyperframes/aws-lambda/sdk");
235
- const progress = await getRenderProgress({
236
- executionArn,
237
- region: stack.region,
238
- defaultMemorySizeMb: stack.lambdaMemoryMb
239
- });
240
- if (progress.status === "SUCCEEDED") {
241
- const outputS3Uri = progress.outputFile?.s3Uri || null;
242
- if (!outputS3Uri) {
243
- return {
244
- state: "failed",
245
- renderId: input.executionName,
246
- executionArn,
247
- message: `HyperFrames render ${input.executionName} reported SUCCEEDED without an output file.`
248
- };
249
- }
250
- const metadata = extractCompositionMetadata(compositionHtml, input);
251
- const handleShape = {
252
- renderId: input.executionName,
253
- executionArn,
254
- bucketName: stack.bucketName,
255
- stateMachineArn: stack.stateMachineArn,
256
- outputS3Uri,
257
- // Not carried across re-invocations; unused by downstream consumers,
258
- // which read outputS3Uri / outputUrl / renderId / metadata only.
259
- projectS3Uri: "",
260
- startedAt: progress.startedAt
261
- };
262
- const result = await this.buildCloudRenderResult(handleShape, progress, metadata, stack);
263
- devLog("hyperframes.render.succeeded", {
264
- composition_id: metadata.compositionId,
265
- render_id: input.executionName,
266
- output_s3_uri: outputS3Uri,
267
- frames_rendered: progress.framesRendered,
268
- total_frames: progress.totalFrames,
269
- lambdas_invoked: progress.lambdasInvoked,
270
- cost_usd: progress.costs?.accruedSoFarUsd
271
- });
272
- return { state: "succeeded", ...result };
273
- }
274
- devLog("hyperframes.render.failed", {
275
- render_id: input.executionName,
276
- status: progress.status,
277
- errors: Array.isArray(progress.errors) ? progress.errors.slice(0, 3) : progress.errors
278
- }, "error");
279
- return {
280
- state: "failed",
281
- renderId: input.executionName,
282
- executionArn,
283
- message: `HyperFrames render ${input.executionName} ${progress.status}: ${scrubVendorDocLinks(JSON.stringify(progress.errors))}`
284
- };
285
- }
286
- async prepareCloudRender(input, compositionHtml, executionName) {
287
- const metadata = extractCompositionMetadata(compositionHtml, input);
288
- const stack = await resolveHyperframesStack();
289
- devLog("hyperframes.render.staging", {
290
- composition_id: metadata.compositionId,
291
- width: metadata.width,
292
- height: metadata.height,
293
- fps: metadata.fps,
294
- stack_name: stack.stackName,
295
- region: stack.region
296
- });
297
- const projectDir = await stageProject({
298
- compositionHtml,
299
- compositionId: metadata.compositionId,
300
- projectFiles: input.projectFiles,
301
- copyPaths: input.copyPaths
302
- });
303
- const timestamp = Date.now().toString(36);
304
- const renderId = executionName || input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
305
- const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
306
- return { metadata, stack, projectDir, renderId, outputKey };
307
- }
308
- async dispatchCloudRender(prep, input) {
309
- devLog("hyperframes.render.dispatch", {
310
- composition_id: prep.metadata.compositionId,
311
- render_id: prep.renderId,
312
- output_key: prep.outputKey,
313
- chunk_size: input.chunkSize ?? 240,
314
- max_parallel_chunks: input.maxParallelChunks ?? 8
315
- });
316
- const { renderToLambda } = await import("@hyperframes/aws-lambda/sdk");
317
- return renderToLambda({
318
- projectDir: prep.projectDir,
319
- bucketName: prep.stack.bucketName,
320
- stateMachineArn: prep.stack.stateMachineArn,
321
- region: prep.stack.region,
322
- executionName: prep.renderId,
323
- outputKey: prep.outputKey,
324
- config: {
325
- fps: normalizeRenderFps(prep.metadata.fps),
326
- width: prep.metadata.width,
327
- height: prep.metadata.height,
328
- format: input.format || "mp4",
329
- codec: "h264",
330
- quality: "standard",
331
- chunkSize: input.chunkSize ?? 240,
332
- maxParallelChunks: input.maxParallelChunks ?? 8,
333
- runtimeCap: "lambda"
334
- }
335
- });
336
- }
337
- async buildCloudRenderResult(handle, progress, metadata, stack) {
338
- const outputS3Uri = progress.outputFile?.s3Uri || handle.outputS3Uri;
339
- return {
340
- ...handle,
341
- outputS3Uri,
342
- outputUrl: outputS3Uri ? await signedS3Url(outputS3Uri, stack.region) : null,
343
- endedAt: progress.endedAt,
344
- metadata: {
345
- mode: "hyperframes_lambda",
346
- status: progress.status,
347
- stack,
348
- composition: metadata,
349
- progress,
350
- costs: progress.costs,
351
- estimatedCostUsd: progress.costs.accruedSoFarUsd,
352
- chargeUsd: applyMarkupUsd(progress.costs.accruedSoFarUsd),
353
- costDisplay: progress.costs.displayCost,
354
- lambdasInvoked: progress.lambdasInvoked,
355
- framesRendered: progress.framesRendered,
356
- totalFrames: progress.totalFrames
357
- }
358
- };
359
- }
360
- // In-process render: stage the project, then drive HyperFrames' single-machine
361
- // orchestrator (headless Chromium → per-frame capture → ffmpeg encode) to write
362
- // the MP4 straight to local disk storage. No AWS, no Lambda fan-out. The result
363
- // is shaped like the cloud render result so the `hyperframes_render` primitive
364
- // and the publish/polling flow consume it unchanged.
365
- async renderLocal(input) {
366
- const rawHtml = normalizePublishHtml(input.compositionHtml || (input.composition ? buildHyperframeCompositionHtml(input.composition) : ""));
367
- if (!rawHtml.includes("data-composition-id=")) {
368
- throw new Error("HyperFrames render requires composition HTML with data-composition-id.");
369
- }
370
- // libx264 (yuv420p) requires even width/height. The in-process renderer sizes
371
- // the canvas from the composition's data-width/data-height, so round any odd
372
- // dimension down to even here — the cloud path sidesteps this by rendering at
373
- // explicit even asset dimensions.
374
- const compositionHtml = forceEvenCompositionDimensions(rawHtml);
375
- const metadata = extractCompositionMetadata(compositionHtml, input);
376
- const projectDir = await stageProject({
377
- compositionHtml,
378
- compositionId: metadata.compositionId,
379
- projectFiles: input.projectFiles,
380
- copyPaths: input.copyPaths
381
- });
382
- const timestamp = Date.now().toString(36);
383
- const renderId = input.executionName || `vf-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
384
- const outputKey = input.outputKey || `renders/vidfarm/${metadata.compositionId}/${renderId}/output.mp4`;
385
- const outputLocation = path.join(config.VIDFARM_DATA_DIR, "storage", outputKey);
386
- await mkdir(path.dirname(outputLocation), { recursive: true });
387
- const startedAt = new Date().toISOString();
388
- const startedAtMs = Date.now();
389
- devLog("hyperframes.render_local.start", {
390
- composition_id: metadata.compositionId,
391
- render_id: renderId,
392
- width: metadata.width,
393
- height: metadata.height,
394
- fps: metadata.fps,
395
- output_key: outputKey
396
- });
397
- const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
398
- const job = createRenderJob({
399
- fps: normalizeRenderFps(metadata.fps),
400
- quality: "standard",
401
- format: input.format || "mp4"
402
- });
403
- let lastStatus = null;
404
- try {
405
- await executeRenderJob(job, projectDir, outputLocation, (progressJob, message) => {
406
- if (progressJob.status !== lastStatus) {
407
- devLog("hyperframes.render_local.progress", {
408
- render_id: renderId,
409
- status: progressJob.status,
410
- message
411
- });
412
- lastStatus = progressJob.status;
413
- }
414
- });
415
- }
416
- catch (error) {
417
- throw new Error(scrubVendorDocLinks(error instanceof Error ? error.message : String(error)), { cause: error });
418
- }
419
- // Single-segment encoded key — matches the `/storage/:key` route which
420
- // decodeURIComponent's one param (slashes ride through as %2F).
421
- const outputUrl = `${config.PUBLIC_BASE_URL}/storage/${encodeURIComponent(outputKey)}`;
422
- const durationMs = Date.now() - startedAtMs;
423
- devLog("hyperframes.render_local.succeeded", {
424
- composition_id: metadata.compositionId,
425
- render_id: renderId,
426
- duration_ms: durationMs,
427
- output_location: outputLocation,
428
- output_url: outputUrl
429
- });
430
- return {
431
- renderId,
432
- outputUrl,
433
- outputS3Uri: null,
434
- executionArn: `local:${renderId}`,
435
- bucketName: "",
436
- stateMachineArn: "",
437
- projectS3Uri: `file://${projectDir}`,
438
- siteId: null,
439
- startedAt,
440
- endedAt: new Date().toISOString(),
441
- metadata: {
442
- mode: "hyperframes_local",
443
- status: "SUCCEEDED",
444
- composition: metadata,
445
- outputLocation,
446
- durationMs,
447
- // Local render burns the operator's own CPU — no per-frame Lambda cost.
448
- chargeUsd: 0,
449
- estimatedCostUsd: 0,
450
- costDisplay: "$0.00 (local render)"
451
- }
452
- };
453
- }
454
- // In-process STILL render: render one frame of a composition to a PNG/JPEG/WEBP
455
- // image on local disk. Uses the same headless-Chromium capture as renderLocal
456
- // via HyperFrames' png-sequence format, then transcodes to the requested still
457
- // format with ffmpeg. Replaces the Remotion `renderStill` path.
458
- async renderStill(input) {
459
- const compositionHtml = forceEvenCompositionDimensions(normalizePublishHtml(input.compositionHtml));
460
- if (!compositionHtml.includes("data-composition-id=")) {
461
- throw new Error("HyperFrames still render requires composition HTML with data-composition-id.");
462
- }
463
- const metadata = extractCompositionMetadata(compositionHtml, { compositionId: input.compositionId });
464
- const projectDir = await stageProject({
465
- compositionHtml,
466
- compositionId: metadata.compositionId,
467
- projectFiles: input.projectFiles,
468
- copyPaths: input.copyPaths
469
- });
470
- const timestamp = Date.now().toString(36);
471
- const renderId = `vfimg-${metadata.compositionId.slice(0, 40)}-${timestamp}-${randomUUID().slice(0, 8)}`;
472
- const framesDir = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects", `${renderId}-frames`);
473
- await mkdir(framesDir, { recursive: true });
474
- devLog("hyperframes.render_still.start", {
475
- composition_id: metadata.compositionId,
476
- render_id: renderId,
477
- width: metadata.width,
478
- height: metadata.height
479
- });
480
- const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
481
- // png-sequence writes outputPath as a DIRECTORY of frame_000001.png … files.
482
- const job = createRenderJob({ fps: 30, quality: "high", format: "png-sequence" });
483
- try {
484
- try {
485
- await executeRenderJob(job, projectDir, framesDir);
486
- }
487
- catch (error) {
488
- throw new Error(scrubVendorDocLinks(error instanceof Error ? error.message : String(error)), { cause: error });
489
- }
490
- const frameFiles = (await readdir(framesDir))
491
- .filter((name) => name.toLowerCase().endsWith(".png"))
492
- .sort();
493
- if (!frameFiles.length) {
494
- throw new Error("HyperFrames still render produced no frames.");
495
- }
496
- const firstFramePath = path.join(framesDir, frameFiles[0]);
497
- const format = input.imageFormat ?? "png";
498
- const extension = format === "jpeg" ? "jpg" : format;
499
- const outputKey = input.outputKey
500
- || `renders/vidfarm/stills/${metadata.compositionId}/${renderId}/output.${extension}`;
501
- const outputLocation = path.join(config.VIDFARM_DATA_DIR, "storage", outputKey);
502
- await mkdir(path.dirname(outputLocation), { recursive: true });
503
- if (format === "png") {
504
- await cp(firstFramePath, outputLocation, { force: true });
505
- }
506
- else {
507
- // Transcode the captured PNG frame to the requested still format.
508
- await runBinary(ffmpegBin, [
509
- "-y",
510
- "-i",
511
- firstFramePath,
512
- ...(format === "jpeg" ? ["-q:v", "3"] : []),
513
- outputLocation
514
- ]);
515
- }
516
- const outputUrl = `${config.PUBLIC_BASE_URL}/storage/${encodeURIComponent(outputKey)}`;
517
- devLog("hyperframes.render_still.succeeded", {
518
- composition_id: metadata.compositionId,
519
- render_id: renderId,
520
- image_format: format,
521
- output_location: outputLocation
522
- });
523
- return {
524
- renderId,
525
- outputUrl,
526
- outputLocation,
527
- metadata: {
528
- mode: "hyperframes_local_still",
529
- compositionId: metadata.compositionId,
530
- width: metadata.width,
531
- height: metadata.height,
532
- imageFormat: format,
533
- outputKey
534
- }
535
- };
536
- }
537
- finally {
538
- await rm(framesDir, { recursive: true, force: true });
539
- }
540
- }
541
- async readRenderOutput(outputS3Uri, region) {
542
- const parsed = parseS3Uri(outputS3Uri);
543
- if (!parsed) {
544
- throw new Error(`Invalid HyperFrames output S3 URI: ${outputS3Uri}`);
545
- }
546
- const s3 = new S3Client({ region: region || config.HYPERFRAMES_RENDER_REGION || config.AWS_REGION });
547
- const response = await s3.send(new GetObjectCommand({ Bucket: parsed.bucket, Key: parsed.key }));
548
- if (!response.Body) {
549
- throw new Error(`HyperFrames output is empty: ${outputS3Uri}`);
550
- }
551
- return {
552
- bytes: Buffer.from(await response.Body.transformToByteArray()),
553
- contentType: response.ContentType || "video/mp4"
554
- };
555
- }
556
- }
557
- // Convert a "raw" composition (source video only, no timeline scenes) into a
558
- // scene-scoped composition by chopping the source video into ~target-second
559
- // chunks and inserting timeline-proxy divs the editor can drag. This is a
560
- // deterministic time-based split; smarter AI-frame-based decomposition can
561
- // replace it later without changing the shape of the produced HTML.
562
- export function decomposeUndecomposedComposition(html, options) {
563
- const { document } = parseHTML(html);
564
- const root = document.querySelector("[data-composition-id]");
565
- if (!root)
566
- return { ok: false, error: "Composition root not found." };
567
- const existingProxies = document.querySelectorAll('[data-vf-timeline-proxy="video"]');
568
- if (existingProxies.length > 0) {
569
- return { ok: false, error: "Composition already has timeline scenes." };
570
- }
571
- const backingVideo = document.querySelector("[data-vf-playback-source]") ?? document.querySelector("video");
572
- const sourceUrl = (backingVideo?.getAttribute("data-src") || backingVideo?.getAttribute("src") || root.getAttribute("data-source-video") || "").trim();
573
- if (!sourceUrl)
574
- return { ok: false, error: "No source video URL found on the composition." };
575
- const durationSeconds = Number.parseFloat(root.getAttribute("data-duration") ?? "");
576
- if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) {
577
- return { ok: false, error: "Composition data-duration is missing — cannot compute scenes." };
578
- }
579
- const targetSeconds = options.targetSceneSeconds;
580
- const preferredCount = options.requestedScenes ?? Math.max(1, Math.round(durationSeconds / targetSeconds));
581
- const sceneCount = Math.min(24, Math.max(1, preferredCount));
582
- const rawScene = durationSeconds / sceneCount;
583
- for (let index = 0; index < sceneCount; index += 1) {
584
- const start = index === 0 ? 0 : Number((index * rawScene).toFixed(3));
585
- const end = index === sceneCount - 1 ? durationSeconds : Number(((index + 1) * rawScene).toFixed(3));
586
- const duration = Number(Math.max(0.1, end - start).toFixed(3));
587
- const proxy = document.createElement("div");
588
- const proxyId = shortElementId();
589
- proxy.id = proxyId;
590
- proxy.setAttribute("data-hf-id", proxyId);
591
- proxy.setAttribute("data-layer-mode", "both");
592
- proxy.setAttribute("data-layer-kind", "video");
593
- proxy.setAttribute("data-viral-note", `Auto-decomposed scene ${index + 1} of ${sceneCount}.`);
594
- proxy.setAttribute("data-start", String(start));
595
- proxy.setAttribute("data-duration", String(duration));
596
- proxy.setAttribute("data-end", String(Number((start + duration).toFixed(3))));
597
- proxy.setAttribute("data-track-index", "0");
598
- proxy.setAttribute("data-label", `Scene ${index + 1}`);
599
- proxy.setAttribute("data-media-start", String(start));
600
- proxy.setAttribute("data-playback-start", String(start));
601
- proxy.setAttribute("data-vf-timeline-proxy", "video");
602
- proxy.setAttribute("data-src", sourceUrl);
603
- proxy.setAttribute("style", "position:absolute;left:0%;top:0%;width:1%;height:1%;z-index:0;display:none;pointer-events:none;");
604
- proxy.className = "clip";
605
- root.append(proxy);
606
- }
607
- return {
608
- ok: true,
609
- html: `<!doctype html>\n${document.documentElement.outerHTML}`,
610
- sceneCount,
611
- durationSeconds
612
- };
613
- }
614
- // ==========================================================================
615
- // Smart video decomposition — samples frames from the source, ships them
616
- // to Gemini vision alongside a strict JSON schema request, and returns the
617
- // scene bounds + text captions found in the footage.
618
- // ==========================================================================
619
- const ffmpegBin = typeof ffmpegStatic === "string" ? ffmpegStatic : ffmpegStatic.path;
620
- const ffprobeBin = typeof ffprobeStatic.path === "string"
621
- ? ffprobeStatic.path
622
- : ffprobeStatic;
623
- // The in-process HyperFrames producer (renderLocal / renderStill) shells out to
624
- // ffmpeg + ffprobe, resolving them from HYPERFRAMES_FFMPEG_PATH / HYPERFRAMES_FFPROBE_PATH,
625
- // then bare `ffmpeg`/`ffprobe` on PATH. The api Lambda image ships no ffmpeg on PATH, so
626
- // point the producer at the bundled ffmpeg-static / ffprobe-static binaries (already
627
- // resolved above) — otherwise render fails with "ffprobe not found. Please install FFmpeg."
628
- if (ffmpegBin && !process.env.HYPERFRAMES_FFMPEG_PATH?.trim()) {
629
- process.env.HYPERFRAMES_FFMPEG_PATH = ffmpegBin;
630
- }
631
- if (ffprobeBin && !process.env.HYPERFRAMES_FFPROBE_PATH?.trim()) {
632
- process.env.HYPERFRAMES_FFPROBE_PATH = ffprobeBin;
633
- }
634
- // The ARCHETYPE / production style of a source video. Detected once per
635
- // decompose and used two ways: (a) stamped onto the template metadata so the
636
- // catalog + clip matcher can filter by format, and (b) to CONDITION the
637
- // per-scene clip annotations, because what counts as a scene's "literal DNA"
638
- // differs by format — an AI fruit drama is defined by its stylized character +
639
- // action, a UGC trend by the creator's face/gesture/authenticity, an
640
- // educational series by the board/diagram/screen, live action by real-world
641
- // subjects + cinematography. Controlled + extendable; ambiguous videos fall back
642
- // to "other".
643
- export const CONTENT_FORMATS = [
644
- "ai_fruit_drama", // AI-generated surreal food/object dramas ("fruit drama")
645
- "ai_generative_skit", // other AI-generated character skits / dramas
646
- "ugc_trend", // phone-shot creator-to-camera trend/challenge
647
- "talking_head", // single speaker addressing camera (advice/opinion)
648
- "educational_series", // explainer / how-to / lecture / listicle
649
- "live_action", // real-world filmed scenes, actors, cinematography
650
- "product_demo", // hands-on showcase / unboxing / review
651
- "street_interview", // vox-pop / man-on-the-street
652
- "reaction_commentary", // reacting to / narrating other footage
653
- "cinematic_broll", // moody establishing / atmosphere / montage
654
- "screen_recording", // app/screen capture, tutorials, gameplay
655
- "animation_motion_graphics", // 2D/3D animation, kinetic typography
656
- "other"
657
- ];
658
- export const DEFAULT_CONTENT_FORMAT = {
659
- format: "other",
660
- label: "Unclassified",
661
- confidence: 0,
662
- rationale: "",
663
- secondary: []
664
- };
665
- const RUN_BINARY_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
666
- function runBinary(bin, args) {
667
- return new Promise((resolve, reject) => {
668
- const proc = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
669
- let stdout = "";
670
- let stderr = "";
671
- let stdoutBytes = 0;
672
- let stderrBytes = 0;
673
- let settled = false;
674
- let killedForOverflow = null;
675
- const settle = (fn) => {
676
- if (settled)
677
- return;
678
- settled = true;
679
- fn();
680
- };
681
- const killForOverflow = (stream) => {
682
- if (killedForOverflow || settled)
683
- return;
684
- killedForOverflow = stream;
685
- try {
686
- proc.kill("SIGKILL");
687
- }
688
- catch { /* ignore */ }
689
- settle(() => reject(new Error(`${path.basename(bin)} ${stream} exceeded ${RUN_BINARY_MAX_BUFFER_BYTES} bytes; killed with SIGKILL.`)));
690
- };
691
- proc.stdout.on("data", (chunk) => {
692
- stdoutBytes += chunk.length;
693
- if (stdoutBytes > RUN_BINARY_MAX_BUFFER_BYTES) {
694
- killForOverflow("stdout");
695
- return;
696
- }
697
- stdout += chunk.toString();
698
- });
699
- proc.stderr.on("data", (chunk) => {
700
- stderrBytes += chunk.length;
701
- if (stderrBytes > RUN_BINARY_MAX_BUFFER_BYTES) {
702
- killForOverflow("stderr");
703
- return;
704
- }
705
- stderr += chunk.toString();
706
- });
707
- proc.on("error", (err) => {
708
- settle(() => reject(err));
709
- });
710
- proc.on("close", (code) => {
711
- if (killedForOverflow)
712
- return;
713
- settle(() => {
714
- if (code === 0)
715
- resolve(stdout);
716
- else
717
- reject(new Error(`${path.basename(bin)} exited ${code}: ${stderr || stdout}`));
718
- });
719
- });
720
- });
721
- }
722
- async function downloadVideoTo(url, dest) {
723
- const response = await fetch(url);
724
- if (!response.ok || !response.body)
725
- throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
726
- await pipeline(response.body, createWriteStream(dest));
727
- }
728
- async function ffprobeVideo(filePath) {
729
- const output = await runBinary(ffprobeBin, [
730
- "-v", "error",
731
- "-show_entries", "format=duration:stream=width,height",
732
- "-of", "json",
733
- filePath
734
- ]);
735
- const parsed = JSON.parse(output || "{}");
736
- const stream = (parsed.streams || []).find((s) => s.width && s.height) || {};
737
- return {
738
- width: Number(stream.width || 0),
739
- height: Number(stream.height || 0),
740
- durationSeconds: Number(parsed.format?.duration || 0)
741
- };
742
- }
743
- // Detect hard scene cuts deterministically with ffmpeg's frame-difference
744
- // scene score, so scene-layer boundaries are grounded in real pixel-level cuts
745
- // instead of the vision model eyeballing sparse frames and guessing. This is
746
- // the scene-layer analogue of the OCR pass that grounds caption timing.
747
- //
748
- // We run `select='gt(scene,THRESH)'` (score 0..1; higher = bigger visual jump)
749
- // and dump the selected frames' metadata to a temp file via `metadata=print`.
750
- // `-f null -` discards the decoded video so stdout stays tiny. Tunable via
751
- // SCENE_CUT_THRESHOLD (default 0.30). Never throws — cuts are an optional hint,
752
- // so any failure returns [] and decompose falls back to model-only boundaries.
753
- async function detectSceneCuts(videoPath, durationSeconds, tempDir) {
754
- const thresholdRaw = Number(process.env.SCENE_CUT_THRESHOLD ?? "0.30");
755
- const threshold = Number.isFinite(thresholdRaw) && thresholdRaw > 0 && thresholdRaw < 1 ? thresholdRaw : 0.30;
756
- const metaPath = path.join(tempDir, "scene-cuts.txt");
757
- try {
758
- await runBinary(ffmpegBin, [
759
- "-y",
760
- "-i", videoPath,
761
- "-vf", `select='gt(scene,${threshold})',metadata=print:file=${metaPath}`,
762
- "-an",
763
- "-f", "null",
764
- "-"
765
- ]);
766
- }
767
- catch (err) {
768
- console.error("scene-cut detection ffmpeg failed", err instanceof Error ? err.message : err);
769
- return [];
770
- }
771
- let raw;
772
- try {
773
- raw = await readFile(metaPath, "utf8");
774
- }
775
- catch {
776
- // No selected frames (e.g. single continuous take) → file may be absent.
777
- return [];
778
- }
779
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : Infinity;
780
- const cuts = [];
781
- const re = /pts_time:([0-9]+\.?[0-9]*)/g;
782
- let match;
783
- while ((match = re.exec(raw)) !== null) {
784
- const t = Number(match[1]);
785
- // Ignore a cut at t≈0 (that's just the first frame, not a boundary) and
786
- // anything past the probed duration. Collapse cuts within 0.4s of an
787
- // already-recorded one so micro-flicker doesn't over-segment the timeline.
788
- if (!Number.isFinite(t) || t < 0.4 || t >= safeDuration - 0.1)
789
- continue;
790
- if (cuts.length > 0 && t - cuts[cuts.length - 1] < 0.4)
791
- continue;
792
- cuts.push(Number(t.toFixed(3)));
793
- }
794
- return cuts;
795
- }
796
- // Sample frames for the vision pass. Vision no longer owns caption timing —
797
- // that's handled deterministically by the dense OCR pass (extractOcrFrames +
798
- // snapCaptionTimingsToOcr). Vision only needs enough coverage to identify
799
- // which captions exist, their text, style, and rough scene boundaries. Keep
800
- // the cadence around 1 fps with a low ceiling to control token cost. Frames
801
- // stay at 960px wide so small overlay text is legible for the LLM.
802
- //
803
- // Uses a single ffmpeg invocation with `-vf fps=…` — the previous per-frame
804
- // spawn loop occasionally left individual output files unwritten when the
805
- // seek raced a demux edge case, producing ENOENT reads. Batched extraction
806
- // is also ~10× faster.
807
- async function sampleFramesAsBase64(videoPath, durationSeconds, tempDir) {
808
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : 5;
809
- const MIN_FRAMES = 8;
810
- // Denser + sharper than before (was 14 / 960px / q3 / 1.0fps). More frames
811
- // catch more scene transitions and short-lived captions; higher resolution +
812
- // lower JPEG quantization keep thin overlay glyphs legible for the model.
813
- const MAX_FRAMES = 18;
814
- const targetPerSecond = 1.2;
815
- const desired = Math.round(safeDuration * targetPerSecond);
816
- const frameCount = Math.min(MAX_FRAMES, Math.max(MIN_FRAMES, desired));
817
- if (frameCount <= 0)
818
- return [];
819
- const effectiveInterval = safeDuration / frameCount;
820
- const outDir = path.join(tempDir, "vision-frames");
821
- await mkdir(outDir, { recursive: true });
822
- const outPattern = path.join(outDir, "frame-%04d.jpg");
823
- const fps = 1 / effectiveInterval;
824
- await runBinary(ffmpegBin, [
825
- "-y",
826
- "-i", videoPath,
827
- "-vf", `fps=${fps.toFixed(6)},scale=1120:-2`,
828
- "-q:v", "2",
829
- "-vframes", String(frameCount),
830
- outPattern
831
- ]);
832
- const results = [];
833
- for (let i = 0; i < frameCount; i += 1) {
834
- const framePath = path.join(outDir, `frame-${String(i + 1).padStart(4, "0")}.jpg`);
835
- let buf;
836
- try {
837
- buf = await readFile(framePath);
838
- }
839
- catch {
840
- // ffmpeg may emit fewer frames than requested for very short/edge-case
841
- // videos; skip missing frames instead of throwing so the vision pass
842
- // still runs with whatever we got.
843
- continue;
844
- }
845
- const timestamp = Math.min(safeDuration - 0.01, Math.max(0.01, i * effectiveInterval + effectiveInterval / 2));
846
- results.push({ timestamp: Number(timestamp.toFixed(3)), data: buf.toString("base64") });
847
- }
848
- return results;
849
- }
850
- // Extract frames at a dense fixed interval for OCR — this is what actually
851
- // grounds caption start/end timing. Interval defaults to 0.5s; tunable via
852
- // OCR_FRAME_INTERVAL_SECONDS. Cap total frames so a 5-minute video doesn't
853
- // fan out into hundreds of OCR calls.
854
- //
855
- // IMPORTANT: uses a SINGLE ffmpeg invocation with `-vf fps=…` to write all
856
- // frames in one pass. The previous per-frame ffmpeg spawn loop was ~1s per
857
- // frame and pushed 30s videos past the 60s CloudFront timeout on their own.
858
- async function extractDenseOcrFrames(videoPath, durationSeconds, tempDir) {
859
- const intervalRaw = Number(process.env.OCR_FRAME_INTERVAL_SECONDS ?? "0.5");
860
- const interval = Number.isFinite(intervalRaw) && intervalRaw > 0 ? Math.max(0.1, intervalRaw) : 0.5;
861
- const maxFramesRaw = Number(process.env.OCR_MAX_FRAMES ?? "60");
862
- const maxFrames = Number.isFinite(maxFramesRaw) && maxFramesRaw > 0 ? Math.floor(maxFramesRaw) : 60;
863
- const safeDuration = Number.isFinite(durationSeconds) && durationSeconds > 0 ? durationSeconds : 5;
864
- const totalDesired = Math.max(1, Math.floor(safeDuration / interval));
865
- const totalFrames = Math.min(maxFrames, totalDesired);
866
- if (totalFrames <= 0)
867
- return [];
868
- const effectiveInterval = safeDuration / totalFrames;
869
- const outDir = path.join(tempDir, "ocr-frames");
870
- await mkdir(outDir, { recursive: true });
871
- const outPattern = path.join(outDir, "ocr-%04d.jpg");
872
- const fps = 1 / effectiveInterval;
873
- await runBinary(ffmpegBin, [
874
- "-y",
875
- "-i", videoPath,
876
- // fps filter samples at the target rate; scale keeps the frame small
877
- // enough that token cost stays down while text stays readable.
878
- "-vf", `fps=${fps.toFixed(6)},scale=640:-2`,
879
- "-q:v", "5",
880
- "-vframes", String(totalFrames),
881
- outPattern
882
- ]);
883
- const results = [];
884
- for (let i = 0; i < totalFrames; i += 1) {
885
- const framePath = path.join(outDir, `ocr-${String(i + 1).padStart(4, "0")}.jpg`);
886
- let buf;
887
- try {
888
- buf = await readFile(framePath);
889
- }
890
- catch {
891
- // ffmpeg may emit fewer frames than requested for very short videos
892
- // (rounding). Stop cleanly instead of throwing.
893
- break;
894
- }
895
- // Frame N (1-indexed) captures roughly at the midpoint of its interval.
896
- const timestamp = Math.min(safeDuration - 0.01, Math.max(0.01, i * effectiveInterval + effectiveInterval / 2));
897
- results.push({ timestamp: Number(timestamp.toFixed(3)), data: buf.toString("base64") });
898
- }
899
- return results;
900
- }
901
- // Prompt the vision model to return ONLY the overlay text visible in each
902
- // frame, verbatim, no interpretation. This is intentionally narrow so the
903
- // model behaves as an OCR engine rather than doing scene analysis (which
904
- // the sparse vision pass already handles).
905
- function buildOcrBatchPrompt(frames) {
906
- const frameList = frames
907
- .map((f, i) => ` ${i + 1}. frame @ ${f.timestamp.toFixed(2)}s`)
908
- .join("\n");
909
- return `You are an OCR engine. For each of the following ${frames.length} video frames, return every piece of overlay/caption text that appears in the image. Ignore any text that is part of the background scenery (signs, packaging, phone UIs, watermarks). Only report captions/subtitles/CTAs that were added on top of the video.
910
-
911
- Return ONLY strictly valid JSON with this schema:
912
- {
913
- "frames": [
914
- { "index": 1, "timestamp": <seconds>, "texts": ["exact text line 1", "exact text line 2"] },
915
- ...
916
- ]
917
- }
918
-
919
- Rules:
920
- - One entry per frame, in the same order given below.
921
- - "texts" is an array of text lines exactly as they appear (preserve case, punctuation, emoji). Split multi-line captions into one string per visible line.
922
- - If a frame has no overlay text, "texts" MUST be [].
923
- - Do NOT paraphrase, translate, or "clean up" text. Verbatim only.
924
- - Do NOT hallucinate text that isn't clearly visible.
925
-
926
- FRAMES (in order):
927
- ${frameList}`;
928
- }
929
- // Run the dense frames through the same vision fallback chain that
930
- // smartDecomposeVideo uses. Batches ~8 frames per call to bound token cost
931
- // per request while keeping request count low. Any batch that fails is
932
- // dropped — downstream code falls back to vision-provided timing for
933
- // captions that don't get an OCR match, so partial OCR is still useful.
934
- async function runOcrPass(input) {
935
- // Larger batches + higher concurrency keep the total OCR wall-time under
936
- // the 60s CloudFront read timeout for typical 28s vertical videos (56
937
- // dense frames). Gemini handles 12 image inputs per call fine and the
938
- // extra parallelism means 56 frames finish in ~1 round of 5 concurrent
939
- // batches instead of 2 rounds of 3.
940
- const BATCH_SIZE = 12;
941
- const CONCURRENCY = 5;
942
- const batches = [];
943
- for (let i = 0; i < input.frames.length; i += BATCH_SIZE) {
944
- batches.push(input.frames.slice(i, i + BATCH_SIZE));
945
- }
946
- const results = [];
947
- for (let i = 0; i < batches.length; i += CONCURRENCY) {
948
- const slice = batches.slice(i, i + CONCURRENCY);
949
- const settled = await Promise.allSettled(slice.map(async (batch) => {
950
- const prompt = buildOcrBatchPrompt(batch);
951
- const attempt = await callVisionWithFallback({ prompt, frames: batch, keys: input.keys });
952
- const record = (attempt.result && typeof attempt.result === "object") ? attempt.result : {};
953
- const framesRaw = Array.isArray(record.frames) ? record.frames : [];
954
- const parsed = [];
955
- for (let idx = 0; idx < batch.length; idx += 1) {
956
- const entry = framesRaw[idx];
957
- const timestamp = batch[idx].timestamp;
958
- const textsRaw = entry && Array.isArray(entry.texts) ? entry.texts : [];
959
- const texts = textsRaw
960
- .map((t) => (typeof t === "string" ? t : ""))
961
- .filter((t) => t.trim().length > 0);
962
- parsed.push({ timestamp, texts });
963
- }
964
- return parsed;
965
- }));
966
- for (const s of settled) {
967
- if (s.status === "fulfilled") {
968
- results.push(...s.value);
969
- }
970
- else {
971
- console.error("smart decompose OCR batch failed", s.reason instanceof Error ? s.reason.message : s.reason);
972
- }
973
- }
974
- }
975
- results.sort((a, b) => a.timestamp - b.timestamp);
976
- return results;
977
- }
978
- function normalizeForOcrMatch(text) {
979
- return text
980
- .toLowerCase()
981
- .replace(/[\u2018\u2019\u201C\u201D]/g, "'")
982
- .replace(/[^\p{Letter}\p{Number}\s]/gu, " ")
983
- .replace(/\s+/g, " ")
984
- .trim();
985
- }
986
- // Overlap ratio: what fraction of the caption's significant words appear in
987
- // the frame's concatenated OCR text? Stopwords-free would be nicer but for
988
- // short-form video captions ("single", "have kids", "buy my course") the
989
- // full word list is fine and simpler.
990
- function captionOverlapWithFrame(captionText, frame) {
991
- const captionWords = normalizeForOcrMatch(captionText).split(" ").filter((w) => w.length >= 2);
992
- if (captionWords.length === 0)
993
- return 0;
994
- const frameText = " " + normalizeForOcrMatch(frame.texts.join(" ")) + " ";
995
- if (frameText.trim().length === 0)
996
- return 0;
997
- let hits = 0;
998
- for (const word of captionWords) {
999
- if (frameText.includes(" " + word + " "))
1000
- hits += 1;
1001
- }
1002
- return hits / captionWords.length;
1003
- }
1004
- // For each vision-detected caption, find the contiguous run of OCR frames
1005
- // where the caption's text is present, and snap start/duration to that run.
1006
- // If no OCR match is confident enough, keep the vision-provided timing
1007
- // (graceful degradation).
1008
- function snapCaptionTimingsToOcr(input) {
1009
- if (input.ocrFrames.length === 0) {
1010
- return { captions: input.captions, snappedCount: 0 };
1011
- }
1012
- const MATCH_THRESHOLD = 0.6;
1013
- // Half the median inter-frame gap — used to extend duration past the last
1014
- // matching frame since the caption presumably stayed on-screen partway
1015
- // into the next (non-matching) frame.
1016
- const gaps = [];
1017
- for (let i = 1; i < input.ocrFrames.length; i += 1) {
1018
- gaps.push(input.ocrFrames[i].timestamp - input.ocrFrames[i - 1].timestamp);
1019
- }
1020
- const sortedGaps = gaps.slice().sort((a, b) => a - b);
1021
- const medianGap = sortedGaps.length > 0 ? sortedGaps[Math.floor(sortedGaps.length / 2)] : 0.5;
1022
- const halfGap = Math.max(0.05, medianGap / 2);
1023
- let snappedCount = 0;
1024
- const snapped = input.captions.map((caption) => {
1025
- const scores = input.ocrFrames.map((frame) => captionOverlapWithFrame(caption.text, frame));
1026
- const matchIndexes = scores
1027
- .map((s, i) => ({ s, i }))
1028
- .filter((x) => x.s >= MATCH_THRESHOLD)
1029
- .map((x) => x.i);
1030
- if (matchIndexes.length === 0) {
1031
- return caption;
1032
- }
1033
- // Pick the contiguous run whose midpoint is closest to the vision-
1034
- // provided midpoint. This handles the case of a repeated caption
1035
- // (same text shown twice with a gap in between) by choosing the
1036
- // occurrence the vision pass was pointing at.
1037
- const runs = [];
1038
- let runStart = matchIndexes[0];
1039
- let prev = matchIndexes[0];
1040
- for (let i = 1; i < matchIndexes.length; i += 1) {
1041
- const idx = matchIndexes[i];
1042
- if (idx - prev <= 2) {
1043
- prev = idx;
1044
- }
1045
- else {
1046
- runs.push({ start: runStart, end: prev });
1047
- runStart = idx;
1048
- prev = idx;
1049
- }
1050
- }
1051
- runs.push({ start: runStart, end: prev });
1052
- const visionMid = caption.start + caption.duration / 2;
1053
- const bestRun = runs.reduce((best, r) => {
1054
- const runMid = (input.ocrFrames[r.start].timestamp + input.ocrFrames[r.end].timestamp) / 2;
1055
- const bestMid = (input.ocrFrames[best.start].timestamp + input.ocrFrames[best.end].timestamp) / 2;
1056
- return Math.abs(runMid - visionMid) < Math.abs(bestMid - visionMid) ? r : best;
1057
- }, runs[0]);
1058
- const firstTs = input.ocrFrames[bestRun.start].timestamp;
1059
- const lastTs = input.ocrFrames[bestRun.end].timestamp;
1060
- const newStart = Math.max(0, firstTs - halfGap);
1061
- const newEnd = Math.min(input.durationSeconds, lastTs + halfGap);
1062
- const newDuration = Math.max(0.1, newEnd - newStart);
1063
- snappedCount += 1;
1064
- return {
1065
- ...caption,
1066
- start: Number(newStart.toFixed(3)),
1067
- duration: Number(newDuration.toFixed(3))
1068
- };
1069
- });
1070
- return { captions: snapped, snappedCount };
1071
- }
1072
- // Snap vision-provided scene boundaries onto ffmpeg-detected hard cuts, the
1073
- // scene-layer analogue of snapCaptionTimingsToOcr. Each scene "start" that
1074
- // falls within TOLERANCE of a detected cut is pulled exactly onto that cut;
1075
- // durations are then recomputed so scenes stay contiguous and still cover the
1076
- // full duration (start[0] forced to 0, last scene extended to the end). If no
1077
- // cuts were detected the scenes are returned unchanged.
1078
- function snapScenesToCuts(input) {
1079
- if (input.cuts.length === 0 || input.scenes.length === 0) {
1080
- return { scenes: input.scenes, snappedCount: 0 };
1081
- }
1082
- const TOLERANCE = 0.6;
1083
- const sorted = [...input.scenes].sort((a, b) => a.start - b.start);
1084
- let snappedCount = 0;
1085
- const snappedStarts = sorted.map((scene, index) => {
1086
- if (index === 0)
1087
- return 0; // first scene always starts at 0
1088
- let nearest = scene.start;
1089
- let bestDelta = TOLERANCE;
1090
- for (const cut of input.cuts) {
1091
- const delta = Math.abs(cut - scene.start);
1092
- if (delta < bestDelta) {
1093
- bestDelta = delta;
1094
- nearest = cut;
1095
- }
1096
- }
1097
- if (nearest !== scene.start)
1098
- snappedCount += 1;
1099
- return nearest;
1100
- });
1101
- const rebuilt = sorted.map((scene, index) => {
1102
- const start = Number(snappedStarts[index].toFixed(3));
1103
- const nextStart = index + 1 < snappedStarts.length ? snappedStarts[index + 1] : input.durationSeconds;
1104
- const duration = Math.max(0.1, Number((nextStart - start).toFixed(3)));
1105
- return { ...scene, start, duration };
1106
- }).filter((s) => s.duration > 0.05);
1107
- return { scenes: rebuilt, snappedCount };
1108
- }
1109
- // Product-AGNOSTIC placement-surface prompt. Deliberately does NOT know the
1110
- // user's product — it enumerates reusable slots where any product could later
1111
- // be inserted. The product-SPECIFIC pitch stays the on-demand
1112
- // brainstorm_product_placement primitive, which these surfaces can seed.
1113
- function buildPlacementSurfacePrompt(input) {
1114
- const frameListing = input.frameTimestamps.length > 0
1115
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
1116
- : "";
1117
- const sceneSlugList = input.sceneSlugs.length > 0
1118
- ? `\nSCENE SLUGS (link each surface to the closest one via "scene_slug"): ${input.sceneSlugs.join(", ")}\n`
1119
- : "";
1120
- return `You are Vidfarm's native-advertising product-placement scout. You are studying a short-form vertical video (TikTok/Reel) to find natural PLACEMENT SURFACES — moments and spots where a product could later be inserted so it feels native, not bolted on.
1121
-
1122
- CRITICAL: You do NOT know the product yet. Do NOT invent a specific product. Describe the SLOT itself (what is physically/attention-wise available), and list the KINDS of products that would fit. Think of this as reusable metadata that a later, product-specific pass will draw from.
1123
-
1124
- Video duration: ${input.durationSeconds.toFixed(2)} seconds.
1125
- ${frameListing}${sceneSlugList}
1126
- Return strictly valid JSON matching this schema:
1127
- {
1128
- "surfaces": [{
1129
- "slug": string,
1130
- "timestamp": string,
1131
- "start": number,
1132
- "scene_slug": string,
1133
- "surface_type": string,
1134
- "surface_note": string,
1135
- "best_for": [string, ...],
1136
- "why_it_works": string
1137
- }]
1138
- }
1139
-
1140
- RULES:
1141
- - Identify 3-8 distinct surfaces, ordered strongest to weakest. Each must reference a real moment you actually see in the frames.
1142
- - "slug": short snake_case role label, e.g. "desk_prop_slot", "phone_screen_swap", "hand_held_reveal", "b_roll_cutaway", "end_card_cta". Unique.
1143
- - "timestamp": human range like "0:04" or "0:12-0:18". "start": the numeric seconds (float) where the surface begins — use it for editor targeting.
1144
- - "scene_slug": the closest scene slug from the list above (or "" if none given).
1145
- - "surface_type": one of prop | set_dressing | screen_ui | verbal_mention | caption_overlay | sticker_lower_third | b_roll_cutaway | apparel | before_after_demo | end_card.
1146
- - "surface_note": describe the SLOT product-agnostically — what is physically there and how long it holds (e.g. "empty desk surface faces camera, unobstructed for ~2s", "subject holds phone screen toward camera", "blank wall behind subject at chest height").
1147
- - "best_for": 2-4 product/offer CATEGORIES that would fit this slot naturally (e.g. "beverage/snack brand", "SaaS dashboard", "skincare bottle", "mobile app UI"). Categories, never a specific brand.
1148
- - "why_it_works": the attention/emotion/story logic that makes a placement here land.
1149
- - Bias toward surfaces that ride existing attention or story beats. If the video genuinely has no natural surface, return "surfaces": []. Never fabricate.`;
1150
- }
1151
- function normalizePlacementSurfaces(raw) {
1152
- const record = (raw && typeof raw === "object") ? raw : {};
1153
- const surfacesRaw = Array.isArray(record.surfaces) ? record.surfaces : [];
1154
- const usedSlugs = new Set();
1155
- return surfacesRaw
1156
- .map((s, index) => {
1157
- const slug = dedupeSlug(normalizeSlug(typeof s.slug === "string" ? s.slug : `surface_${index + 1}`) || `surface_${index + 1}`, usedSlugs);
1158
- const bestForRaw = Array.isArray(s.best_for) ? s.best_for : [];
1159
- const best_for = bestForRaw
1160
- .map((b) => (typeof b === "string" ? b.trim() : ""))
1161
- .filter((b) => b.length > 0)
1162
- .slice(0, 6);
1163
- return {
1164
- slug,
1165
- timestamp: typeof s.timestamp === "string" ? s.timestamp : "",
1166
- start: Number.isFinite(Number(s.start)) ? Number(Number(s.start).toFixed(3)) : 0,
1167
- scene_slug: typeof s.scene_slug === "string" ? normalizeSlug(s.scene_slug) : "",
1168
- surface_type: typeof s.surface_type === "string" ? s.surface_type : "prop",
1169
- surface_note: typeof s.surface_note === "string" ? s.surface_note : "",
1170
- best_for,
1171
- why_it_works: typeof s.why_it_works === "string" ? s.why_it_works : ""
1172
- };
1173
- })
1174
- .filter((s) => s.surface_note.trim().length > 0);
1175
- }
1176
- // Isolated placement-surface vision call. Reuses the SAME sparse frames already
1177
- // sampled for the main pass — no extra ffmpeg — and runs as its own request so
1178
- // it can never dilute the caption/scene call's attention budget. Non-fatal:
1179
- // callers wrap in .catch(() => []).
1180
- async function runPlacementSurfacePass(input) {
1181
- if (input.frames.length === 0)
1182
- return [];
1183
- const prompt = buildPlacementSurfacePrompt({
1184
- durationSeconds: input.durationSeconds,
1185
- frameTimestamps: input.frames.map((f) => f.timestamp),
1186
- sceneSlugs: input.sceneSlugs
1187
- });
1188
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
1189
- return normalizePlacementSurfaces(attempt.result);
1190
- }
1191
- // ---------------------------------------------------------------------------
1192
- // Scene-annotation pass — clip-replacement DNA.
1193
- //
1194
- // A dependent vision pass (runs AFTER scenes are finalized, so annotations line
1195
- // up 1:1 by scene slug) that (1) classifies the whole video's content FORMAT and
1196
- // (2) for each scene emits the literal + viral DNA needed to find a REPLACEMENT
1197
- // clip from the clip library. It reuses the sparse frames already sampled for
1198
- // the main pass (no extra ffmpeg) and the clip library's own controlled tag
1199
- // vocabulary + embedding recipe, so a scene's clip-match query is symmetric with
1200
- // how a candidate clip was tagged/embedded. Non-fatal: callers wrap in .catch().
1201
- // ---------------------------------------------------------------------------
1202
- // Vocabulary for a scene's role in the retention arc. Constrained in the prompt,
1203
- // clamped here; free-form-ish (unknown → "context") so it never blocks output.
1204
- const SCENE_ROLE_VOCAB = [
1205
- "hook", "build", "reveal", "payoff", "transition", "cta", "proof", "context", "punchline", "loop"
1206
- ];
1207
- function asPlainRecord(value) {
1208
- return value && typeof value === "object" && !Array.isArray(value) ? value : {};
1209
- }
1210
- function readAnnotationString(value) {
1211
- return typeof value === "string" ? value.trim() : "";
1212
- }
1213
- function readAnnotationStringList(value, max) {
1214
- if (!Array.isArray(value))
1215
- return [];
1216
- const out = [];
1217
- for (const v of value) {
1218
- const s = typeof v === "string" ? v.trim() : "";
1219
- if (s && !out.includes(s))
1220
- out.push(s);
1221
- if (out.length >= max)
1222
- break;
1223
- }
1224
- return out;
1225
- }
1226
- function humanizeContentFormat(format) {
1227
- return format
1228
- .split("_")
1229
- .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
1230
- .join(" ");
1231
- }
1232
- function normalizeContentFormat(raw) {
1233
- const rec = asPlainRecord(raw);
1234
- const slug = normalizeSlug(readAnnotationString(rec.format));
1235
- const format = CONTENT_FORMATS.includes(slug)
1236
- ? slug
1237
- : "other";
1238
- const secondaryRaw = Array.isArray(rec.secondary) ? rec.secondary : [];
1239
- const secondary = Array.from(new Set(secondaryRaw
1240
- .map((s) => normalizeSlug(readAnnotationString(s)))
1241
- .filter((s) => CONTENT_FORMATS.includes(s) && s !== format))).slice(0, 3);
1242
- const conf = Number(rec.confidence);
1243
- const label = readAnnotationString(rec.label);
1244
- return {
1245
- format,
1246
- label: label || humanizeContentFormat(format),
1247
- confidence: Number.isFinite(conf) ? Math.max(0, Math.min(1, conf)) : 0,
1248
- rationale: readAnnotationString(rec.rationale),
1249
- secondary
1250
- };
1251
- }
1252
- function normalizeSceneRole(value) {
1253
- const slug = normalizeSlug(readAnnotationString(value));
1254
- return SCENE_ROLE_VOCAB.includes(slug) ? slug : "context";
1255
- }
1256
- function normalizeReplaceability(value) {
1257
- const s = readAnnotationString(value).toLowerCase();
1258
- return s === "high" || s === "low" ? s : "medium";
1259
- }
1260
- // Deterministic per-scene spoken text pulled from the transcript segments that
1261
- // overlap the scene window — more reliable than asking the vision model to
1262
- // re-transcribe, and keeps `tags.transcript` grounded in the audio pass.
1263
- function transcriptForScene(transcript, scene) {
1264
- if (!transcript || !Array.isArray(transcript.segments))
1265
- return "";
1266
- const end = scene.start + scene.duration;
1267
- return transcript.segments
1268
- .filter((seg) => seg.end > scene.start && seg.start < end)
1269
- .map((seg) => seg.text.trim())
1270
- .filter((t) => t.length > 0)
1271
- .join(" ")
1272
- .trim();
1273
- }
1274
- // Compose the retrieval query text. Leads with the clip-doc recipe (description +
1275
- // tags) so it lands near literally-similar clip embeddings, then appends the
1276
- // viral purpose so the vector also reflects the beat's INTENT — a replacement
1277
- // must match both axes.
1278
- function buildSceneEmbeddingText(input) {
1279
- const literalDoc = buildClipEmbeddingText(input.tags, input.literal);
1280
- const purpose = [
1281
- input.scene_role ? `Role: ${input.scene_role}` : "",
1282
- input.narrative_purpose,
1283
- input.why_it_works
1284
- ].filter((p) => p && p.length > 0).join(". ");
1285
- return purpose ? `${literalDoc}\nPurpose: ${purpose}` : literalDoc;
1286
- }
1287
- // Turn a scene's literal tags into a ready-to-run SplicingCriteria for the clip
1288
- // library. Tag constraints are the high-precision structured filter; the
1289
- // duration floor lets candidates be trimmed to fit; has_subject/has_dialogue are
1290
- // derived from the tags. semantic_text carries the embedding query for re-rank.
1291
- function buildSceneMatchCriteria(tags, scene, semanticText) {
1292
- const criteria = { semantic_text: semanticText };
1293
- if (tags.subject.length)
1294
- criteria.subject = tags.subject;
1295
- if (tags.action.length)
1296
- criteria.action = tags.action;
1297
- if (tags.emotion.length)
1298
- criteria.emotion = tags.emotion;
1299
- if (tags.setting.length)
1300
- criteria.setting = tags.setting;
1301
- if (tags.composition.length)
1302
- criteria.composition = tags.composition;
1303
- if (tags.motion.length)
1304
- criteria.motion = tags.motion;
1305
- if (tags.audio_type.length)
1306
- criteria.audio_type = tags.audio_type;
1307
- if (tags.energy)
1308
- criteria.energy = [tags.energy];
1309
- if (Number.isFinite(scene.duration) && scene.duration > 0) {
1310
- criteria.min_duration_sec = Number(Math.max(0, scene.duration * 0.6).toFixed(2));
1311
- }
1312
- const hasRealSubject = tags.subject.some((s) => s !== "no_subject");
1313
- if (hasRealSubject)
1314
- criteria.has_subject = true;
1315
- else if (tags.subject.includes("no_subject"))
1316
- criteria.has_subject = false;
1317
- if (tags.audio_type.includes("dialogue"))
1318
- criteria.has_dialogue = true;
1319
- return criteria;
1320
- }
1321
- function buildSceneAnnotationPrompt(input) {
1322
- const frameListing = input.frameTimestamps.length > 0
1323
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
1324
- : "";
1325
- const sceneListing = input.scenes.map((s) => (` - scene_slug="${s.slug}" @ ${s.start.toFixed(2)}s for ${s.duration.toFixed(2)}s — ${s.label}: ${s.description || "(no description)"}${s.viral_note ? ` [viral note: ${s.viral_note}]` : ""}`)).join("\n");
1326
- const dna = input.viralDna;
1327
- const dnaBlock = `Video-level viral DNA (context for each scene's PURPOSE):\n hook: ${dna.hook || "—"}\n retention: ${dna.retention || "—"}\n payoff: ${dna.payoff || "—"}\n preserve: ${(dna.preserve || []).join("; ") || "—"}\n avoid: ${(dna.avoid || []).join("; ") || "—"}`;
1328
- const userGuidance = input.userPrompt && input.userPrompt.trim().length > 0
1329
- ? `\nUSER GUIDANCE (highest priority):\n"""\n${input.userPrompt.trim()}\n"""\n`
1330
- : "";
1331
- return `You are Vidfarm's scene annotator. For EACH scene of this short-form vertical video, write an annotation rich enough to do TWO things: (A) find a REPLACEMENT clip from a searchable clip library — a different piece of footage that could stand in for this scene without breaking the video; and (B) let an AI agent RE-CREATE this scene from scratch with an image/video generator when no library clip fits.
1332
-
1333
- Both goals hang on capturing the same TWO axes:
1334
- 1) LITERAL DNA — what is physically on screen (subject, action, setting, shot, motion, mood). A candidate or generated clip must look and feel like this.
1335
- 2) VIRAL / NARRATIVE DNA — the ROLE this beat plays in why the video works (the hook, the reveal, the payoff, a transition...). A replacement or regeneration must serve the same purpose in the story, or the swap kills retention.
1336
- ${userGuidance}
1337
- FIRST classify the whole video's FORMAT (its production archetype) — what counts as "literal DNA" differs by format. Pick the single best "format" and list any close seconds in "secondary". Allowed formats:
1338
- ${CONTENT_FORMATS.join(", ")}
1339
- Format guidance: ai_fruit_drama / ai_generative_skit = AI-generated surreal character dramas (e.g. talking fruit); ugc_trend = phone-shot creator-to-camera trend; talking_head = single speaker giving advice; educational_series = explainer / how-to / listicle; live_action = real filmed scenes / actors; product_demo = hands-on showcase / unboxing; street_interview = man-on-the-street; reaction_commentary = reacting to other footage; cinematic_broll = moody montage / atmosphere; screen_recording = app / screen capture; animation_motion_graphics = animation / kinetic text.
1340
-
1341
- Video duration: ${input.durationSeconds.toFixed(2)}s. One-line summary: ${input.summary || "(none)"}.
1342
- ${dnaBlock}
1343
- ${frameListing}
1344
- SCENES to annotate (return EXACTLY one annotation per scene_slug below, in this order):
1345
- ${sceneListing}
1346
-
1347
- For the LITERAL "tags", use ONLY values from this controlled vocabulary — it is the SAME vocabulary the clip library is tagged with, so your tags must line up with how a candidate clip would be tagged:
1348
- ${taxonomyForPrompt()}
1349
-
1350
- Return strictly valid JSON matching this schema:
1351
- {
1352
- "content_format": { "format": string, "label": string, "confidence": number, "rationale": string, "secondary": [string, ...] },
1353
- "scenes": [{
1354
- "scene_slug": string,
1355
- "literal": string,
1356
- "tags": { "subject": [string], "action": [string], "emotion": [string], "setting": [string], "composition": [string], "motion": [string], "energy": "low"|"medium"|"high", "audio_type": [string], "transcript": string },
1357
- "scene_role": string,
1358
- "narrative_purpose": string,
1359
- "why_it_works": string,
1360
- "must_preserve": [string, ...],
1361
- "can_vary": [string, ...],
1362
- "replaceability": "high"|"medium"|"low",
1363
- "replaceability_reason": string,
1364
- "recreation_prompt": string,
1365
- "recreation_notes": [string, ...]
1366
- }]
1367
- }
1368
-
1369
- RULES:
1370
- - Output one scene object per scene_slug listed above, using the EXACT scene_slug strings. Do not invent, merge, split, or reorder scenes.
1371
- - "literal": one concrete, factual sentence describing exactly what is on screen in this scene (subject + action + setting + shot). This is the primary match key — be specific and visual, not interpretive.
1372
- - "tags": pick only vocabulary values you are confident about from the frames; omit a category (empty list) rather than guess. "energy" is a single value. "transcript" = words spoken in this scene verbatim, or "" if none.
1373
- - "scene_role": one of ${SCENE_ROLE_VOCAB.join(", ")} — the function this beat serves in the retention arc.
1374
- - "narrative_purpose" / "why_it_works": the viral DNA — WHY this beat exists and how it holds attention. A replacement clip must reproduce this effect, not just the look.
1375
- - "must_preserve": 1-4 things a replacement MUST keep (e.g. "subject facing camera", "surprised reaction on the beat drop", "product clearly visible"). "can_vary": 1-4 things free to change (specific person, exact background, color grade).
1376
- - "replaceability": how safely this scene can be swapped from generic library clips. "low" for scenes carrying unique baked-in text / a specific face / a spoken payoff; "high" for generic b-roll / cutaways / establishing shots. Explain in "replaceability_reason".
1377
- - "recreation_prompt": a self-contained generative brief an AI agent could paste into a text-to-image/video model to REGENERATE this exact beat — describe subject, action, setting, framing, camera move, lighting, mood, art style, and pacing in one rich paragraph. Write it in the idiom of the detected format (e.g. an ai_fruit_drama prompt reads like a stylized 3D-character render brief; a live_action prompt reads like a cinematography direction). Do NOT reference "this video" or timestamps — it must stand alone.
1378
- - "recreation_notes": 2-5 short discrete production directives that back up the recreation_prompt (e.g. "9:16 vertical", "handheld push-in", "soft key light from left", "on-screen text: 'wait for it'", "cut on the beat at ~1.2s", "dialogue VO: <verbatim line>"). Include any exact on-screen text or spoken line that must be reproduced.
1379
- - Match your vocabulary and framing to the detected format above.`;
1380
- }
1381
- async function runSceneAnnotationPass(input) {
1382
- if (input.frames.length === 0 || input.scenes.length === 0) {
1383
- return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null };
1384
- }
1385
- const prompt = buildSceneAnnotationPrompt({
1386
- durationSeconds: input.durationSeconds,
1387
- frameTimestamps: input.frames.map((f) => f.timestamp),
1388
- scenes: input.scenes,
1389
- viralDna: input.viralDna,
1390
- summary: input.summary,
1391
- userPrompt: input.userPrompt ?? null
1392
- });
1393
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
1394
- const record = asPlainRecord(attempt.result);
1395
- const contentFormat = normalizeContentFormat(record.content_format);
1396
- const rawScenes = Array.isArray(record.scenes) ? record.scenes : [];
1397
- const bySlug = new Map();
1398
- for (const r of rawScenes) {
1399
- const rec = asPlainRecord(r);
1400
- const slug = normalizeSlug(readAnnotationString(rec.scene_slug));
1401
- if (slug && !bySlug.has(slug))
1402
- bySlug.set(slug, rec);
1403
- }
1404
- // Iterate the FINALIZED scenes (not the model output) so every scene gets
1405
- // exactly one annotation aligned by slug, even if the model dropped, renamed,
1406
- // or duplicated one. Missing fields degrade to the scene's own description.
1407
- const annotations = input.scenes.map((scene) => {
1408
- const raw = bySlug.get(scene.slug) ?? {};
1409
- const tags = {
1410
- ...sanitizeTagValues(asPlainRecord(raw.tags)),
1411
- transcript: transcriptForScene(input.transcript, scene) || readAnnotationString(asPlainRecord(raw.tags).transcript)
1412
- };
1413
- const literal = readAnnotationString(raw.literal) || scene.description || scene.label;
1414
- const scene_role = normalizeSceneRole(raw.scene_role);
1415
- const narrative_purpose = readAnnotationString(raw.narrative_purpose) || scene.viral_note;
1416
- const why_it_works = readAnnotationString(raw.why_it_works);
1417
- const embedding_text = buildSceneEmbeddingText({ literal, tags, scene_role, narrative_purpose, why_it_works });
1418
- return {
1419
- scene_slug: scene.slug,
1420
- start: scene.start,
1421
- duration: scene.duration,
1422
- content_format: contentFormat.format,
1423
- literal,
1424
- tags,
1425
- scene_role,
1426
- narrative_purpose,
1427
- why_it_works,
1428
- must_preserve: readAnnotationStringList(raw.must_preserve, 4),
1429
- can_vary: readAnnotationStringList(raw.can_vary, 4),
1430
- replaceability: normalizeReplaceability(raw.replaceability),
1431
- replaceability_reason: readAnnotationString(raw.replaceability_reason),
1432
- embedding_text,
1433
- match_criteria: buildSceneMatchCriteria(tags, scene, embedding_text),
1434
- // Fall back to the literal + purpose if the model omitted a recreation
1435
- // brief, so an agent always has *something* to regenerate from.
1436
- recreation_prompt: readAnnotationString(raw.recreation_prompt)
1437
- || [literal, narrative_purpose].filter((p) => p.length > 0).join(" "),
1438
- recreation_notes: readAnnotationStringList(raw.recreation_notes, 6)
1439
- };
1440
- });
1441
- return { contentFormat, annotations, provider: attempt.provider, model: attempt.model };
1442
- }
1443
- // Standalone entry point for the ASYNC scene-annotation pass, invoked once per
1444
- // /scene-annotations-poll. Mirrors analyzeCastMembers: it re-downloads the
1445
- // (caption-free) source, re-samples the same sparse frames the main decompose
1446
- // used, and runs the annotation vision call against the ALREADY-finalized scenes
1447
- // (passed in from the persisted smart-decompose snapshot) so annotations align
1448
- // 1:1 by slug. Kept out of the synchronous decompose so that request stays under
1449
- // the origin timeout.
1450
- export async function annotateScenesFromSource(input) {
1451
- if (input.scenes.length === 0) {
1452
- return { contentFormat: DEFAULT_CONTENT_FORMAT, annotations: [], provider: null, model: null, durationSeconds: 0 };
1453
- }
1454
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
1455
- const sourcePath = path.join(tempDir, "source.mp4");
1456
- const startedAt = Date.now();
1457
- devLog("hyperframes.scene_annotations.start", { source_url: input.sourceUrl, scene_count: input.scenes.length });
1458
- try {
1459
- await downloadVideoTo(input.sourceUrl, sourcePath);
1460
- const meta = await ffprobeVideo(sourcePath);
1461
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
1462
- throw new Error("Could not read video duration");
1463
- }
1464
- const frames = await sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir);
1465
- const outcome = await runSceneAnnotationPass({
1466
- frames,
1467
- keys: input.keys,
1468
- durationSeconds: meta.durationSeconds,
1469
- scenes: input.scenes,
1470
- viralDna: input.viralDna,
1471
- summary: input.summary,
1472
- transcript: input.transcript,
1473
- userPrompt: input.userPrompt ?? null
1474
- });
1475
- devLog("hyperframes.scene_annotations.completed", {
1476
- provider: outcome.provider,
1477
- model: outcome.model,
1478
- duration_ms: Date.now() - startedAt,
1479
- content_format: outcome.contentFormat.format,
1480
- annotation_count: outcome.annotations.length
1481
- });
1482
- return { ...outcome, durationSeconds: meta.durationSeconds };
1483
- }
1484
- catch (error) {
1485
- devLog("hyperframes.scene_annotations.failed", {
1486
- source_url: input.sourceUrl,
1487
- duration_ms: Date.now() - startedAt,
1488
- ...devErrorFields(error)
1489
- }, "error");
1490
- throw error;
1491
- }
1492
- finally {
1493
- await rm(tempDir, { recursive: true, force: true });
1494
- }
1495
- }
1496
- // The frames handed to the vision passes are SILENT stills — but a short-form
1497
- // video's emotional punch is often carried by its AUDIO (the trending sound, a
1498
- // music-bed drop, vocal intonation, a punchline's delivery). This digest folds
1499
- // the transcript pass into a compact block so the viral-DNA and editor-harness
1500
- // prompts can reason about the sound, not just the pixels. Music-only clips
1501
- // (no speech) still get a note so the model infers a trending-sound bed rather
1502
- // than guessing there is silence.
1503
- function buildAudioDigestForPrompt(transcript) {
1504
- if (!transcript) {
1505
- return `\nAUDIO TRACK: none detected, or transcription unavailable. Judge emotional punch from the visuals, on-screen text, and pacing alone — and do not fabricate spoken lines, music, or a trending sound you cannot infer.\n`;
1506
- }
1507
- const text = (transcript.text || "").trim();
1508
- const clipped = text.length > 1200 ? `${text.slice(0, 1200)} …` : text;
1509
- const segs = transcript.segments.slice(0, 24)
1510
- .map((s) => ` [${s.start.toFixed(1)}-${s.end.toFixed(1)}s] ${s.text.trim()}`)
1511
- .filter((line) => line.trim().length > 0)
1512
- .join("\n");
1513
- return `\nAUDIO TRANSCRIPT — the video's spoken/sung audio. The SOUND is a primary driver of emotional punch: read the words, the intonation and cadence they imply, WHERE the delivery lands (the pause before a punchline, an exclamation, a beat-synced line), and infer any music / trending-sound bed even when there are no lyrics.
1514
- - language: ${transcript.language ?? "unknown"}
1515
- - full text: ${clipped || "(no speech transcribed — likely a music-only or trending-sound bed; treat the audio as a music/sound-driven format and reason about the beat/drop, not dialogue)"}${segs ? `\n- timed lines:\n${segs}` : ""}
1516
- `;
1517
- }
1518
- function buildSmartDecomposePrompt(input) {
1519
- const sceneCutHint = input.sceneCuts && input.sceneCuts.length > 0
1520
- ? `\nDETECTED SCENE CUTS (authoritative — ffmpeg measured hard visual cuts at these timestamps, in seconds):\n${input.sceneCuts.map((t) => ` cut @ ${t.toFixed(2)}s`).join("\n")}\n\nThese are ground truth for where the video visually changes. Align scene boundaries to these cuts: each detected cut should begin a new scene (a scene "start" should equal a detected cut time, within ~0.15s), and do NOT place a scene boundary where there is no detected cut unless the frames clearly show a change the detector missed. If two adjacent cuts are very close and belong to the same beat, you may merge them into one scene, but never exceed 6 scenes.\n`
1521
- : "";
1522
- const userGuidance = input.userPrompt && input.userPrompt.trim().length > 0
1523
- ? `\nUSER GUIDANCE (highest priority — pay special attention to this when deciding scenes, captions, and viral_dna):\n"""\n${input.userPrompt.trim()}\n"""\n`
1524
- : "";
1525
- const trendTaglineHint = input.trendTaglineHint && input.trendTaglineHint.trim().length > 0
1526
- ? `\nTREND TAGLINE (caller-supplied, use verbatim as viral_dna.trend_tagline — do NOT rephrase):\n"""\n${input.trendTaglineHint.trim()}\n"""\n`
1527
- : "";
1528
- const frameListing = input.frameTimestamps.length > 0
1529
- ? `\nFRAMES SUPPLIED (in order — each image is captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n\nUse the frame timestamps to determine caption "start" and "duration". Cross-reference consecutive frames: a caption whose text is identical across frames N and M was on-screen from at least the timestamp of frame N through the timestamp of frame M (extend "duration" accordingly). A caption present in only one frame is short-lived — set duration to roughly (next_frame_timestamp - this_frame_timestamp) or 0.8s, whichever is smaller.\n`
1530
- : "";
1531
- const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
1532
- return `You are decomposing a short-form vertical video (TikTok/Reel) into editable HyperFrames timeline layers.
1533
- ${userGuidance}${trendTaglineHint}
1534
- Video metadata:
1535
- - duration: ${input.durationSeconds.toFixed(3)} seconds
1536
- - dimensions: ${input.width} x ${input.height} (9:16 vertical, canvas is 100% x 100%)
1537
- ${frameListing}${sceneCutHint}${audioDigest}
1538
-
1539
- Return strictly valid JSON matching this schema:
1540
- {
1541
- "summary": string,
1542
- "viral_dna": {
1543
- "trend_tagline": string,
1544
- "hook": string,
1545
- "retention": string,
1546
- "payoff": string,
1547
- "emotional_punch": {
1548
- "core_emotion": string,
1549
- "tone": string,
1550
- "arc": string,
1551
- "peak_moment": string,
1552
- "mechanism": string,
1553
- "contrast": string,
1554
- "humor": string,
1555
- "audio_visual_tension": string,
1556
- "delivery": string,
1557
- "preserve": [string, ...]
1558
- },
1559
- "preserve": [string, ...],
1560
- "avoid": [string, ...],
1561
- "promotions": [string, ...],
1562
- "keywords": [string, ...]
1563
- },
1564
- "scenes": [{
1565
- "slug": string,
1566
- "start": number,
1567
- "duration": number,
1568
- "label": string,
1569
- "description": string,
1570
- "viral_note": string
1571
- }],
1572
- "captions": [{
1573
- "slug": string,
1574
- "text": string,
1575
- "start": number,
1576
- "duration": number,
1577
- "x": number,
1578
- "y": number,
1579
- "width": number,
1580
- "height": number,
1581
- "font_size_percent": number,
1582
- "font_size": number,
1583
- "color": string,
1584
- "background": string,
1585
- "background_style": "plain" | "outline" | "highlight-solid" | "highlight-translucent",
1586
- "font_family": string,
1587
- "font_weight": number,
1588
- "viral_note": string
1589
- }]
1590
- }
1591
-
1592
- VIRAL_DNA RULES:
1593
- - "trend_tagline": a short (3-8 word) semantic identifier for the underlying trend/format itself, NOT this specific video's topic. Think of it as the label a scroller would use to categorize the format (e.g., "green-screen news debunk", "day-in-the-life sped-up montage", "best-at-what-cost destination reveal", "pinned-comment dance dare"). If the caller pre-supplied a trend_tagline (see USER GUIDANCE), preserve it verbatim; otherwise generate one.
1594
- - "hook": the 1-sentence tactic used in the first 1-2 seconds to stop the scroll (e.g., "Pinned comment screenshot dares the viewer to keep watching").
1595
- - "retention": how the middle of the video keeps attention (visual cuts, pattern loops, escalating stakes).
1596
- - "payoff": what the viewer earns at the end (reveal, punchline, transformation).
1597
- - "emotional_punch": the FEELING this format delivers and exactly what makes it land — the part remixes most often flatten. hook/retention/payoff are the mechanical structure; this is the vibe, the joke, and the intonation that make a viewer actually FEEL something. Fill EVERY sub-field:
1598
- - "core_emotion": the dominant feeling a viewer walks away with (e.g. "delighted surprise", "righteous validation", "warm nostalgia", "second-hand cringe", "smug satisfaction").
1599
- - "tone": the register / vibe in 1-3 words (comedic, deadpan, earnest, dramatic, wholesome, chaotic, satirical, hype, ...).
1600
- - "arc": how the feeling MOVES across the runtime (e.g. "curiosity → mounting tension → relief-laugh", "flat setup → sudden absurd turn").
1601
- - "peak_moment": the single beat that lands the emotional hit — the punchline, the reveal, the gut-punch, the beat-drop — and roughly when.
1602
- - "mechanism": WHY it lands — the technique doing the work (comedic timing, irony, subversion of expectation, escalation, deadpan intonation, relatability, a trending-sound drop synced to the visual). Weigh the AUDIO here: on TikTok/Reels the trending sound or music bed is frequently the single biggest driver of the emotional hit, so if a recognizable trending sound, a beat drop, or a music swell is doing the work, name it as the mechanism.
1603
- - "contrast": the specific mismatch / juxtaposition that creates the feeling. This is CRITICAL for irony- or sass-driven formats: identify the actual tension, not just the visible gesture. Examples: "hard, macho soundtrack over a dainty office-laptop scene", "overconfident delivery about a trivial preference", "luxury attitude applied to a mundane workflow". If there is no real mismatch, write "none".
1604
- - "humor": if it's funny, state what the JOKE actually is and why it works, concretely enough that a remixer can rebuild the same joke with new subject matter; write "none" if the format is not comedic. Focus on the underlying bit, not surface choreography.
1605
- - "audio_visual_tension": describe how the SOUND and VISUALS interact emotionally: whether the soundtrack / vocal cadence reinforces the visuals, undercuts them, or creates a deliberate mismatch. Name the exact pairing if that's the joke or swagger engine; write "none" only if audio contributes nothing.
1606
- - "delivery": the performance cues to preserve so the vibe survives — speech intonation / cadence (from the AUDIO TRANSCRIPT), the pause held before the punchline, a facial beat or reaction, and especially the SOUND: the music / trending-audio bed, where the beat drop or lyric hit lands relative to the cut, a music swell, an sfx sting, the deadpan-then-cut. This is what keeps intonation, comedic timing, and the audio-driven vibe from being lost in a remix.
1607
- - "preserve": 2-4 emotional beats a remix MUST keep to reproduce the same feeling (e.g. "keep the beat drop landing exactly on the reveal cut", "hold the silent beat before the reveal", "keep the flat unbothered delivery", "the payoff must undercut the buildup, not confirm it").
1608
- Use BOTH the frames and the AUDIO TRANSCRIPT block above. Never leave emotional_punch empty — infer the feeling from the frames, captions, and pacing even when there is no audio, and when the audio is music-only, treat the sound itself as the emotional engine. Do NOT over-index on a single gesture or hand movement if the real emotional engine is the character attitude, the soundtrack choice, the setting mismatch, or the audio-vs-visual contrast.
1609
- - "preserve": 3-5 concrete elements a remixer MUST keep to preserve the format (e.g., "Pinned comment stays on-screen the whole video", "Wedding dance payoff shot").
1610
- - "avoid": 3-5 things that would break the format if changed (e.g., "Do not remove the on-screen text overlay", "Do not slow the cuts below 1.5s").
1611
- - "promotions": 3-5 categories of product / service / offer this exact format could be reused to promote, WITHOUT changing the beat structure or hook mechanic. Each item is a concrete pitchable use-case, not a genre. Examples: "SaaS free-trial signup (before/after workflow)", "Fitness challenge (7-day transformation)", "Local restaurant grand-opening (line-out-the-door proof)", "Ebook / course launch (result-first payoff)", "Real-estate open-house (walk-through reveal)". Anchor each promotion to the same viral tactic captured in "hook" — do not invent unrelated verticals.
1612
- - "keywords": 6-12 lowercase search terms that make this format discoverable when a marketer describes their product/offer in their own words. Include the niches, target audiences, product categories, outcomes, and emotions this format naturally sells to — plus obvious synonyms a user might type. Think about what a customer would search: e.g. for a before/after fitness format include "weight loss", "fitness", "gym", "transformation", "before after", "health app", "diet", "body goals", "wellness". Single words or 2-3 word phrases, no punctuation, no duplicates. These are for matching only, so bias toward the words a customer would use, not jargon.
1613
- - Use short imperative fragments in preserve/avoid arrays. Never leave empty — infer from the frames if unstated.
1614
-
1615
- SCENE RULES:
1616
- - 2 to 6 scenes covering the FULL duration; contiguous (scene[i].start = scene[i-1].start + scene[i-1].duration); first scene starts at 0.0; last scene ends at ${input.durationSeconds.toFixed(3)}.
1617
- - Every scene MUST include a "description": 1-3 plain sentences literally describing what is happening on screen during that scene — setting, subjects, their actions, camera movement/framing, and any notable visual changes (cuts, zooms, transitions, product shots). Describe only what a viewer literally sees in the frames; do NOT repeat viral_note commentary, marketing analysis, or audio content here.
1618
-
1619
- SLUG RULES:
1620
- - Every scene AND every caption MUST have a "slug": short snake_case (2-4 words) describing its ROLE in the viral structure. Unique within scenes; unique within captions. Examples: "hook_reveal", "problem_setup", "payoff_context", "cta_follow", "caption_hook_dare".
1621
-
1622
- CAPTION RULES (CRITICAL — this is the primary use of vision on this task):
1623
- - Inspect EVERY supplied frame. Enumerate EVERY distinct visible text overlay across all frames — hooks, dialog subtitles, hashtags, usernames, handles, brand tags, CTAs, sticker text, arrow labels, price tags, meme text, comment-screenshot text, product names — anything rendered as text on the pixels.
1624
- - One caption = one visually distinct text block. If two lines share the same visual treatment and appear together as one overlay, keep them as ONE caption with "\\n" between the lines. If two blocks look different (different font, color, position, or independent timing), they are SEPARATE captions.
1625
- - Preserve text VERBATIM. Copy spelling, punctuation, casing, emojis, and line breaks exactly as they appear. Do NOT paraphrase, normalize casing, translate, or "clean up" spacing. If a caption contains a typo, keep the typo.
1626
- - The SAME text overlay that persists across multiple sampled frames is ONE caption with a longer duration — do NOT emit one caption per frame. Merge by (identical text + roughly same on-screen position + same visual style).
1627
- - If a caption changes text mid-clip (e.g. rolling subtitle updates), emit each version as a separate caption with its own tight start/duration.
1628
- - Position numbers (x, y, width, height) must reflect where the text ACTUALLY sits in the frame — measure the bounding box, don't guess. Vertical (9:16) captions almost never span x=0..100; hooks near y=8-18, dialog near y=60-72, CTAs near y=80-88.
1629
- - If no visible text overlays exist across ANY frame, return "captions": []. NEVER invent text. NEVER hallucinate captions from audio or context.
1630
- - Prioritize completeness. Missing an on-screen caption is a worse failure than emitting an extra minor one.
1631
-
1632
- FONT_FAMILY (choose ONE, match visual appearance):
1633
- - "TikTok Sans" — clean sans-serif (DEFAULT if unsure). Rounded, modern.
1634
- - "Montserrat" — geometric bold sans. Common on IG/TikTok bold captions.
1635
- - "Source Code Pro" — monospaced. For code/terminal aesthetic.
1636
- - "Yesteryear" — cursive script. For quotes / romantic vibes.
1637
- - "Georgia" — serif. For editorial / news look.
1638
- - "Abel" — condensed sans. Newsletter/headline vibe.
1639
- - Return exactly one of the above six strings. If unsure, return "TikTok Sans".
1640
-
1641
- FONT_WEIGHT (integer):
1642
- - 400 for normal weight
1643
- - 600 for semibold (typical TikTok caption look)
1644
- - 700 for bold (heavy captions)
1645
- - 800 or 900 for ultra-black display text
1646
-
1647
- FONT_SIZE — return TWO numbers, both required. The renderer uses font_size_percent to survive canvas resizes.
1648
- - "font_size_percent" (float 1.0–15.0): the caption glyph height as a percent of canvas height (${input.height || 1280}px). This is the SOURCE OF TRUTH. Measure a single-line uppercase-height letter (like an "M" or "H") in the actual frame and divide by canvas height, times 100.
1649
- - Tiny UI / hashtag row: 1.2–2.0
1650
- - Standard TikTok caption (dialog line): 2.0–3.2
1651
- - Standard hook caption (readable but not screaming): 3.2–5.0
1652
- - Bold hook overlay filling a wide band: 5.0–8.0
1653
- - Giant kinetic display / meme text: 8.0–12.0
1654
- - "font_size" (integer pixels): compute as round(font_size_percent / 100 * ${input.height || 1280}). Include it for redundancy.
1655
- - Do NOT default to 24. Measure the actual on-screen glyph height. When unsure between two ranges, pick the one closest to what you actually measure — do not over-inflate.
1656
- - Give the SAME caption the SAME font_size — do not vary between instances.
1657
-
1658
- COLOR (hex "#rrggbb"):
1659
- - Pure white: "#ffffff" (most common TikTok captions)
1660
- - Pure black: "#000000"
1661
- - Bright yellow (highlight text): "#ffe600"
1662
- - Match the actual on-screen color as closely as you can. Never return names like "white"; always hex.
1663
-
1664
- BACKGROUND_STYLE (choose ONE):
1665
- - "plain" — no visible pill/outline. Just letters over the video. (Most common.)
1666
- - "outline" — thin dark outline around each glyph (for legibility over busy video).
1667
- - "highlight-solid" — a solid colored pill / block behind the text (typical TikTok white-on-black caption card).
1668
- - "highlight-translucent" — semi-transparent pill.
1669
- - BACKGROUND ("#rrggbb") is the pill / outline color. For "plain" this is unused but still return a valid hex like "#000000".
1670
-
1671
- POSITION (percent 0-100, floats OK):
1672
- - x, y = top-left corner of the caption's bounding box relative to the canvas.
1673
- - width, height = its bounding box size.
1674
- - TikTok baked captions are usually centered horizontally: x ≈ 5-10, width ≈ 80-90.
1675
- - Vertical placement: hook captions near top (y ≈ 8-18); dialog/subtitle mid-lower (y ≈ 60-72); brand tag / cta bottom (y ≈ 80-88).
1676
- - Height should roughly match font_size and line count.
1677
-
1678
- TIMING RULES:
1679
- - start, duration are seconds (floats). start >= 0. start + duration <= ${input.durationSeconds.toFixed(3)}.
1680
-
1681
- OUTPUT: pure JSON, no prose, no markdown fences.`;
1682
- }
1683
- const ALLOWED_FONTS = [
1684
- "TikTok Sans", "Montserrat", "Source Code Pro", "Yesteryear", "Georgia", "Abel"
1685
- ];
1686
- const ALLOWED_BG_STYLES = [
1687
- "plain", "outline", "highlight-solid", "highlight-translucent"
1688
- ];
1689
- function normalizeFontFamily(value) {
1690
- if (typeof value !== "string")
1691
- return "TikTok Sans";
1692
- const trimmed = value.trim();
1693
- const match = ALLOWED_FONTS.find((font) => font.toLowerCase() === trimmed.toLowerCase());
1694
- return match ?? "TikTok Sans";
1695
- }
1696
- function normalizeHexColor(value, fallback) {
1697
- if (typeof value !== "string")
1698
- return fallback;
1699
- const trimmed = value.trim();
1700
- if (/^#[0-9a-fA-F]{6}$/.test(trimmed))
1701
- return trimmed.toLowerCase();
1702
- if (/^#[0-9a-fA-F]{3}$/.test(trimmed)) {
1703
- const [, r, g, b] = /^#([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])$/.exec(trimmed) ?? [];
1704
- return `#${r}${r}${g}${g}${b}${b}`.toLowerCase();
1705
- }
1706
- return fallback;
1707
- }
1708
- function normalizeBackgroundStyle(value) {
1709
- if (typeof value !== "string")
1710
- return "plain";
1711
- const lowered = value.trim().toLowerCase();
1712
- const match = ALLOWED_BG_STYLES.find((style) => style === lowered);
1713
- return match ?? "plain";
1714
- }
1715
- // Convert a #rrggbb hex to an rgba(...) string with the given alpha. Falls back
1716
- // to a solid dark backing if the hex is malformed so the caption still stays
1717
- // legible.
1718
- function hexToRgba(hex, alpha) {
1719
- const match = /^#([0-9a-f]{6})$/i.exec(String(hex ?? "").trim());
1720
- if (!match)
1721
- return `rgba(0,0,0,${alpha})`;
1722
- const value = match[1];
1723
- const r = parseInt(value.slice(0, 2), 16);
1724
- const g = parseInt(value.slice(2, 4), 16);
1725
- const b = parseInt(value.slice(4, 6), 16);
1726
- return `rgba(${r},${g},${b},${alpha})`;
1727
- }
1728
- // Builds the inline-span CSS for a caption based on its background_style so
1729
- // captions actually render with the visual treatment the vision LLM extracted
1730
- // ("outline" gets a real text stroke; "highlight-*" gets a colored pill).
1731
- // Without this the span just had `color:#fff` and everything looked washed
1732
- // out over the video.
1733
- function captionInlineTextStyle(caption) {
1734
- const parts = [`color:${caption.color}`];
1735
- const strokeColor = caption.background && /^#[0-9a-f]{6}$/i.test(caption.background)
1736
- ? caption.background
1737
- : "#000000";
1738
- // Outline width scales with font size so small captions still get a
1739
- // visible edge. Clamp so it never disappears (1px) and never eats the
1740
- // glyph interior (5px).
1741
- const strokePx = Math.max(1, Math.min(5, Math.round(caption.font_size * 0.06)));
1742
- switch (caption.background_style) {
1743
- case "outline":
1744
- parts.push(`-webkit-text-stroke:${strokePx}px ${strokeColor}`, `paint-order:stroke fill`,
1745
- // text-shadow fallback for renderers that ignore paint-order.
1746
- `text-shadow:0 0 ${strokePx}px ${strokeColor}, ${strokePx}px ${strokePx}px 0 ${strokeColor}, -${strokePx}px ${strokePx}px 0 ${strokeColor}, ${strokePx}px -${strokePx}px 0 ${strokeColor}, -${strokePx}px -${strokePx}px 0 ${strokeColor}`);
1747
- break;
1748
- case "highlight-solid":
1749
- parts.push(`background:${strokeColor}`, `padding:0.18em 0.5em`, `border-radius:0.35em`, `box-decoration-break:clone`, `-webkit-box-decoration-break:clone`);
1750
- break;
1751
- case "highlight-translucent":
1752
- parts.push(`background:${hexToRgba(strokeColor, 0.55)}`, `padding:0.18em 0.5em`, `border-radius:0.35em`, `box-decoration-break:clone`, `-webkit-box-decoration-break:clone`);
1753
- break;
1754
- case "plain":
1755
- default:
1756
- // Even "plain" captions need SOME shadow over busy TikTok footage or
1757
- // white-on-white becomes invisible. Use a subtle drop shadow that
1758
- // doesn't read as an intentional outline.
1759
- parts.push(`text-shadow:0 1px 2px rgba(0,0,0,0.55), 0 0 3px rgba(0,0,0,0.35)`);
1760
- break;
1761
- }
1762
- return parts.join(";");
1763
- }
1764
- function normalizeFontWeight(value) {
1765
- const parsed = Number(value);
1766
- if (!Number.isFinite(parsed))
1767
- return 600;
1768
- const clamped = Math.max(100, Math.min(900, Math.round(parsed / 100) * 100));
1769
- return clamped;
1770
- }
1771
- function normalizeFontSize(value, canvasHeight) {
1772
- const parsed = Number(value);
1773
- const heightRef = canvasHeight > 0 ? canvasHeight : 1280;
1774
- if (!Number.isFinite(parsed) || parsed <= 0)
1775
- return Math.round(heightRef * 0.028);
1776
- return Math.max(10, Math.min(160, Math.round(parsed)));
1777
- }
1778
- function resolveFontSize(input) {
1779
- const heightRef = input.canvasHeight > 0 ? input.canvasHeight : 1280;
1780
- const percent = Number(input.fontSizePercent);
1781
- if (Number.isFinite(percent) && percent > 0) {
1782
- const clampedPct = Math.max(0.8, Math.min(15, percent));
1783
- return Math.max(10, Math.min(160, Math.round((clampedPct / 100) * heightRef)));
1784
- }
1785
- return normalizeFontSize(input.fontSize, input.canvasHeight);
1786
- }
1787
- const DEFAULT_EMOTIONAL_PUNCH = {
1788
- core_emotion: "",
1789
- tone: "",
1790
- arc: "",
1791
- peak_moment: "",
1792
- mechanism: "",
1793
- contrast: "",
1794
- humor: "",
1795
- audio_visual_tension: "",
1796
- delivery: "",
1797
- preserve: []
1798
- };
1799
- export const DEFAULT_VIRAL_DNA = {
1800
- trend_tagline: "", hook: "", retention: "", payoff: "", emotional_punch: { ...DEFAULT_EMOTIONAL_PUNCH }, preserve: [], avoid: [], promotions: [], keywords: []
1801
- };
1802
- function normalizeEmotionalPunch(value) {
1803
- if (!value || typeof value !== "object")
1804
- return { ...DEFAULT_EMOTIONAL_PUNCH };
1805
- const record = value;
1806
- const asString = (v) => typeof v === "string" ? v.trim() : "";
1807
- const asStringArray = (v) => Array.isArray(v)
1808
- ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
1809
- : [];
1810
- return {
1811
- core_emotion: asString(record.core_emotion ?? record.coreEmotion),
1812
- tone: asString(record.tone),
1813
- arc: asString(record.arc ?? record.emotional_arc),
1814
- peak_moment: asString(record.peak_moment ?? record.peakMoment),
1815
- mechanism: asString(record.mechanism),
1816
- contrast: asString(record.contrast),
1817
- humor: asString(record.humor),
1818
- audio_visual_tension: asString(record.audio_visual_tension ?? record.audioVisualTension),
1819
- delivery: asString(record.delivery),
1820
- preserve: asStringArray(record.preserve).slice(0, 4)
1821
- };
1822
- }
1823
- function normalizeViralDna(value) {
1824
- if (!value || typeof value !== "object")
1825
- return DEFAULT_VIRAL_DNA;
1826
- const record = value;
1827
- const asString = (v) => typeof v === "string" ? v.trim() : "";
1828
- const asStringArray = (v) => Array.isArray(v)
1829
- ? v.map((item) => (typeof item === "string" ? item.trim() : "")).filter((item) => item.length > 0)
1830
- : [];
1831
- return {
1832
- trend_tagline: asString(record.trend_tagline ?? record.trendTagline),
1833
- hook: asString(record.hook),
1834
- retention: asString(record.retention),
1835
- payoff: asString(record.payoff),
1836
- emotional_punch: normalizeEmotionalPunch(record.emotional_punch ?? record.emotionalPunch),
1837
- preserve: asStringArray(record.preserve),
1838
- avoid: asStringArray(record.avoid),
1839
- promotions: asStringArray(record.promotions),
1840
- keywords: asStringArray(record.keywords).map((item) => item.toLowerCase())
1841
- };
1842
- }
1843
- // ---------------------------------------------------------------------------
1844
- // Editor-harness pass — technical editing direction (the "how to edit like
1845
- // this" companion to viral DNA's "why it works"). Isolated vision call in the
1846
- // decompose burst (never the caption/scene call) so it can't dilute
1847
- // timeline-layer quality; reuses the SAME sparse frames already sampled + the
1848
- // ffmpeg cut timings for grounded pacing. Non-fatal: callers wrap in .catch().
1849
- // ---------------------------------------------------------------------------
1850
- export const DEFAULT_EDITOR_HARNESS = {
1851
- format: "other",
1852
- one_liner: "",
1853
- aspect_ratio: "",
1854
- pacing: { cut_rhythm: "medium", avg_scene_seconds: 0, cuts_per_10s: 0, energy_curve: "" },
1855
- typography: { caption_style: "", placement: "", font_character: "", emphasis: "", text_density: "none" },
1856
- broll: { reliance: "none", sourcing: "", shot_kinds: [], cadence: "" },
1857
- transitions: { default: "cut", intro: "", outro: "", usage: "" },
1858
- audio: { voiceover: "", music: "", sfx: "", captions_from: "" },
1859
- emotional: { target_feeling: "", tone: "", comedic_timing: "", intonation: "", vibe_anchors: [] },
1860
- scenes: [],
1861
- important_scenes: [],
1862
- editing_bias: [],
1863
- do: [],
1864
- dont: []
1865
- };
1866
- const HARNESS_CUT_RHYTHMS = ["fast", "medium", "slow", "varied"];
1867
- const HARNESS_TEXT_DENSITY = ["heavy", "moderate", "minimal", "none"];
1868
- const HARNESS_BROLL_RELIANCE = ["heavy", "moderate", "light", "none"];
1869
- const HARNESS_SCENE_IMPORTANCE = ["critical", "high", "normal"];
1870
- function buildEditorHarnessPrompt(input) {
1871
- const frameListing = input.frameTimestamps.length > 0
1872
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp):\n${input.frameTimestamps.map((t, i) => ` frame ${i + 1} @ ${t.toFixed(2)}s`).join("\n")}\n`
1873
- : "";
1874
- const cutListing = input.sceneCuts.length > 0
1875
- ? `\nDETECTED HARD CUTS (seconds, from the editing timeline — use these to ground pacing/cut_rhythm/cuts_per_10s/avg_scene_seconds): ${input.sceneCuts.map((c) => c.toFixed(2)).join(", ")}\n`
1876
- : "\n(No hard cuts detected — likely a single continuous take or slow crossfades. Judge pacing from the frames.)\n";
1877
- const promptHint = input.userPrompt && input.userPrompt.trim().length > 0
1878
- ? `\nUSER GOAL (bias the harness toward recreating THIS intent): ${input.userPrompt.trim()}\n`
1879
- : "";
1880
- const roleVocab = SCENE_ROLE_VOCAB.join(" | ");
1881
- const formatVocab = CONTENT_FORMATS.join(" | ");
1882
- const audioDigest = buildAudioDigestForPrompt(input.transcript ?? null);
1883
- const emotional = input.viralDna?.emotional_punch;
1884
- const emotionalSnapshot = emotional && [
1885
- emotional.core_emotion,
1886
- emotional.tone,
1887
- emotional.arc,
1888
- emotional.peak_moment,
1889
- emotional.mechanism,
1890
- emotional.contrast,
1891
- emotional.humor,
1892
- emotional.audio_visual_tension,
1893
- emotional.delivery
1894
- ].some((value) => value && value.trim().length > 0)
1895
- ? `\nVIRAL DNA FEELING SNAPSHOT (operationalize this in the harness; do NOT drift from it):\n- core_emotion: ${emotional.core_emotion || "—"}\n- tone: ${emotional.tone || "—"}\n- arc: ${emotional.arc || "—"}\n- peak_moment: ${emotional.peak_moment || "—"}\n- mechanism: ${emotional.mechanism || "—"}\n- contrast: ${emotional.contrast || "—"}\n- humor: ${emotional.humor || "—"}\n- audio_visual_tension: ${emotional.audio_visual_tension || "—"}\n- delivery: ${emotional.delivery || "—"}\n- preserve: ${(emotional.preserve || []).join("; ") || "—"}\n`
1896
- : "";
1897
- return `You are Vidfarm's EDITING DIRECTOR. You are studying a short-form video to write an EDITOR HARNESS — a compact, technical brief that lets another AI editor recreate this video's STYLE (not its exact content) in one clean pass on a timeline editor (scenes, captions, audio, transitions, keyframes).
1898
-
1899
- This is DIFFERENT from "viral DNA" (why the video works / what to preserve). The harness is HOW to edit: pace, typography, b-roll usage, transitions, audio bed, and the role each scene plays. Be concrete and directive — every field should be something an editor can act on, not vibes.
1900
-
1901
- Video duration: ${input.durationSeconds.toFixed(2)} seconds.
1902
- ${frameListing}${cutListing}${audioDigest}${emotionalSnapshot}${promptHint}
1903
- Return strictly valid JSON matching this schema:
1904
- {
1905
- "format": string, // one of: ${formatVocab}
1906
- "one_liner": string, // ONE sentence: how to edit to match this style
1907
- "aspect_ratio": string, // "9:16" | "1:1" | "16:9" (infer from the frames)
1908
- "pacing": {
1909
- "cut_rhythm": "fast" | "medium" | "slow" | "varied",
1910
- "avg_scene_seconds": number, // typical shot length in seconds
1911
- "cuts_per_10s": number, // ground on the detected cuts + duration
1912
- "energy_curve": string // e.g. "front-loaded hook then steady"
1913
- },
1914
- "typography": {
1915
- "caption_style": string, // karaoke | word-pop | spotlight | stack-up | neon | bounce | none | free-form label
1916
- "placement": string, // e.g. "center-lower" | "top" | "full-bleed" | "none"
1917
- "font_character": string, // e.g. "bold condensed all-caps sans"
1918
- "emphasis": string, // e.g. "active-word highlight in yellow" | "none"
1919
- "text_density": "heavy" | "moderate" | "minimal" | "none"
1920
- },
1921
- "broll": {
1922
- "reliance": "heavy" | "moderate" | "light" | "none",
1923
- "sourcing": string, // generated | stock | screen-recording | talking-head | mixed
1924
- "shot_kinds": [string, ...], // e.g. b_roll, product_shot, screen_recording, reaction, establishing, text_graphic
1925
- "cadence": string // e.g. "new visual ~every 2s under VO"
1926
- },
1927
- "transitions": {
1928
- "default": string, // junction at most cuts: cut | crossfade | whip | slide | zoom | dissolve | ...
1929
- "intro": string, // first-clip entrance
1930
- "outro": string, // last-clip exit
1931
- "usage": string // one line on how transitions are used
1932
- },
1933
- "audio": {
1934
- "voiceover": string, // e.g. "first-person narration" | "none"
1935
- "music": string, // e.g. "upbeat lofi bed, low volume" | "none"
1936
- "sfx": string, // e.g. "whoosh on cuts" | "none"
1937
- "captions_from": string // "voiceover" | "on-screen text" | "none"
1938
- },
1939
- "emotional": {
1940
- "target_feeling": string, // the emotion the edit must reproduce (the vibe/joke that makes it land)
1941
- "tone": string, // register to hold: comedic | deadpan | earnest | dramatic | wholesome | hype | ...
1942
- "comedic_timing": string, // where the beat/pause/cut sells the joke; "none" if not comedic
1943
- "intonation": string, // vocal delivery/cadence to preserve on VO or on-cam speech; "none" if silent
1944
- "vibe_anchors": [string, ...] // 2-4 concrete edit moves that carry the emotion
1945
- },
1946
- "scenes": [{
1947
- "role": string, // ${roleVocab}
1948
- "timestamp": string, // human range, e.g. "0:00-0:03"
1949
- "start": number, // numeric seconds where the beat begins
1950
- "importance": "critical" | "high" | "normal",
1951
- "must_keep": boolean, // true if swapping/removing this beat kills the style
1952
- "note": string, // what happens in this beat
1953
- "edit_bias": string // how to edit this beat while staying on-style
1954
- }],
1955
- "important_scenes": [string, ...],// roles or timestamps that carry the virality — protect when editing
1956
- "editing_bias": [string, ...], // 3-6 top directives an editor MUST follow to stay on-style
1957
- "do": [string, ...], // 2-5 concrete DOs
1958
- "dont": [string, ...] // 2-5 concrete DON'Ts
1959
- }
1960
-
1961
- RULES:
1962
- - Ground pacing numbers on the detected cuts + duration when cuts are supplied; otherwise estimate from the frames. Round avg_scene_seconds and cuts_per_10s to at most 2 decimals.
1963
- - "scenes": 3-8 entries covering the whole runtime in order, each tied to a real moment you see. Reserve importance "critical" for the 1-3 beats that actually carry the style/retention (usually the hook + payoff).
1964
- - Prefer the caption_style / transition / shot_kind vocabulary above so the editor can map straight to presets; a free-form label is fine when nothing fits.
1965
- - "emotional": this is HOW to edit so the FEELING survives — the vibe, the joke, the intonation, and the SOUND are the first things a remix flattens, so make these directive. Use the AUDIO TRANSCRIPT block: on TikTok/Reels the trending sound / music bed is often the biggest carrier of the feeling. "target_feeling" = the emotion the recreated edit must produce; "tone" = the register to hold; "comedic_timing" = the exact editing move that sells the joke (the beat held before a cut, the deadpan-then-punch, the sfx sting or beat-drop on the button word) or "none"; "intonation" = the vocal delivery/cadence to preserve on any VO or on-cam speech (or "none" if silent) — flag it if the speech must be re-voiced in the SAME cadence; "vibe_anchors" = 2-4 concrete edit choices that carry the emotion, and include the audio ones (e.g. "keep the trending sound and land the beat drop on the reveal cut", "swell the music on the payoff", "hold a full beat of silence before the reveal cut", "cut hard on the punch word"). When the viral-DNA snapshot above names a contrast or audio_visual_tension, convert that into actionable edit moves instead of drifting to superficial gestures. Infer the feeling from frames + pacing even without audio, and when the audio is music-only treat the sound as the emotional engine; never leave it empty.
1966
- - Keep every string tight and directive. Never invent audio you cannot infer — say "none"/"unknown" instead.
1967
- - Output ONLY the JSON object.`;
1968
- }
1969
- function normalizeEditorHarness(raw) {
1970
- const rec = asPlainRecord(raw);
1971
- const oneOf = (value, allowed, fallback) => {
1972
- const s = normalizeSlug(readAnnotationString(value));
1973
- return allowed.includes(s) ? s : fallback;
1974
- };
1975
- const num = (value) => {
1976
- const n = Number(value);
1977
- return Number.isFinite(n) && n >= 0 ? Number(n.toFixed(2)) : 0;
1978
- };
1979
- const formatSlug = normalizeSlug(readAnnotationString(rec.format));
1980
- const format = CONTENT_FORMATS.includes(formatSlug)
1981
- ? formatSlug
1982
- : "other";
1983
- const pacing = asPlainRecord(rec.pacing);
1984
- const typography = asPlainRecord(rec.typography);
1985
- const broll = asPlainRecord(rec.broll);
1986
- const transitions = asPlainRecord(rec.transitions);
1987
- const audio = asPlainRecord(rec.audio);
1988
- const emotional = asPlainRecord(rec.emotional);
1989
- const scenesRaw = Array.isArray(rec.scenes) ? rec.scenes : [];
1990
- const scenes = scenesRaw
1991
- .map((s) => {
1992
- const sr = asPlainRecord(s);
1993
- return {
1994
- role: normalizeSlug(readAnnotationString(sr.role)) || "context",
1995
- timestamp: readAnnotationString(sr.timestamp),
1996
- start: num(sr.start),
1997
- importance: oneOf(sr.importance, HARNESS_SCENE_IMPORTANCE, "normal"),
1998
- must_keep: sr.must_keep === true,
1999
- note: readAnnotationString(sr.note),
2000
- edit_bias: readAnnotationString(sr.edit_bias)
2001
- };
2002
- })
2003
- .filter((s) => s.note.length > 0 || s.edit_bias.length > 0)
2004
- .slice(0, 12);
2005
- return {
2006
- format,
2007
- one_liner: readAnnotationString(rec.one_liner),
2008
- aspect_ratio: readAnnotationString(rec.aspect_ratio),
2009
- pacing: {
2010
- cut_rhythm: oneOf(pacing.cut_rhythm, HARNESS_CUT_RHYTHMS, "medium"),
2011
- avg_scene_seconds: num(pacing.avg_scene_seconds),
2012
- cuts_per_10s: num(pacing.cuts_per_10s),
2013
- energy_curve: readAnnotationString(pacing.energy_curve)
2014
- },
2015
- typography: {
2016
- caption_style: readAnnotationString(typography.caption_style),
2017
- placement: readAnnotationString(typography.placement),
2018
- font_character: readAnnotationString(typography.font_character),
2019
- emphasis: readAnnotationString(typography.emphasis),
2020
- text_density: oneOf(typography.text_density, HARNESS_TEXT_DENSITY, "none")
2021
- },
2022
- broll: {
2023
- reliance: oneOf(broll.reliance, HARNESS_BROLL_RELIANCE, "none"),
2024
- sourcing: readAnnotationString(broll.sourcing),
2025
- shot_kinds: readAnnotationStringList(broll.shot_kinds, 8),
2026
- cadence: readAnnotationString(broll.cadence)
2027
- },
2028
- transitions: {
2029
- default: readAnnotationString(transitions.default) || "cut",
2030
- intro: readAnnotationString(transitions.intro),
2031
- outro: readAnnotationString(transitions.outro),
2032
- usage: readAnnotationString(transitions.usage)
2033
- },
2034
- audio: {
2035
- voiceover: readAnnotationString(audio.voiceover),
2036
- music: readAnnotationString(audio.music),
2037
- sfx: readAnnotationString(audio.sfx),
2038
- captions_from: readAnnotationString(audio.captions_from)
2039
- },
2040
- emotional: {
2041
- target_feeling: readAnnotationString(emotional.target_feeling),
2042
- tone: readAnnotationString(emotional.tone),
2043
- comedic_timing: readAnnotationString(emotional.comedic_timing),
2044
- intonation: readAnnotationString(emotional.intonation),
2045
- vibe_anchors: readAnnotationStringList(emotional.vibe_anchors, 4)
2046
- },
2047
- scenes,
2048
- important_scenes: readAnnotationStringList(rec.important_scenes, 8),
2049
- editing_bias: readAnnotationStringList(rec.editing_bias, 8),
2050
- do: readAnnotationStringList(rec.do, 6),
2051
- dont: readAnnotationStringList(rec.dont, 6)
2052
- };
2053
- }
2054
- async function runEditorHarnessPass(input) {
2055
- if (input.frames.length === 0)
2056
- return DEFAULT_EDITOR_HARNESS;
2057
- const prompt = buildEditorHarnessPrompt({
2058
- durationSeconds: input.durationSeconds,
2059
- frameTimestamps: input.frames.map((f) => f.timestamp),
2060
- sceneCuts: input.sceneCuts,
2061
- userPrompt: input.userPrompt ?? null,
2062
- transcript: input.transcript ?? null,
2063
- viralDna: input.viralDna ?? null
2064
- });
2065
- const attempt = await callVisionWithFallback({ prompt, frames: input.frames, keys: input.keys });
2066
- return normalizeEditorHarness(attempt.result);
2067
- }
2068
- function shortElementId() {
2069
- return `element_${randomUUID().slice(0, 8)}`;
2070
- }
2071
- function extractJson(text) {
2072
- const trimmed = text.trim();
2073
- const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
2074
- const candidate = fenced?.[1] ?? trimmed;
2075
- try {
2076
- return JSON.parse(candidate);
2077
- }
2078
- catch {
2079
- const objectMatch = /\{[\s\S]*\}/.exec(candidate);
2080
- if (objectMatch)
2081
- return JSON.parse(objectMatch[0]);
2082
- throw new Error("Gemini did not return valid JSON");
2083
- }
2084
- }
2085
- async function callGeminiVision(input) {
2086
- const parts = [{ text: input.prompt }];
2087
- for (const frame of input.frames) {
2088
- parts.push({ text: `Frame captured at ${frame.timestamp.toFixed(2)}s` });
2089
- parts.push({ inlineData: { mimeType: "image/jpeg", data: frame.data } });
2090
- }
2091
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent?key=${encodeURIComponent(input.apiKey)}`, {
2092
- method: "POST",
2093
- headers: { "content-type": "application/json" },
2094
- body: JSON.stringify({
2095
- contents: [{ role: "user", parts }],
2096
- generationConfig: {
2097
- responseMimeType: "application/json",
2098
- temperature: 0.15,
2099
- // Bumped from Gemini's 8k default so long caption arrays (10+ text
2100
- // overlays × per-caption position/style metadata) don't truncate
2101
- // mid-JSON and blow up the parser.
2102
- maxOutputTokens: 16384
2103
- }
2104
- })
2105
- });
2106
- const raw = await response.text();
2107
- if (!response.ok) {
2108
- let message = raw;
2109
- try {
2110
- message = JSON.parse(raw).error?.message ?? raw;
2111
- }
2112
- catch { }
2113
- throw new Error(`Gemini responded ${response.status}: ${message}`);
2114
- }
2115
- const parsed = JSON.parse(raw);
2116
- const text = parsed.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
2117
- return extractJson(text);
2118
- }
2119
- async function callOpenAIVision(input) {
2120
- const content = [{ type: "text", text: input.prompt }];
2121
- for (const frame of input.frames) {
2122
- content.push({ type: "text", text: `Frame captured at ${frame.timestamp.toFixed(2)}s` });
2123
- content.push({
2124
- type: "image_url",
2125
- image_url: { url: `data:image/jpeg;base64,${frame.data}` }
2126
- });
2127
- }
2128
- // OpenRouter proxies many models and still accepts the classic Chat
2129
- // Completions params. Native OpenAI's current models (gpt-5.x / o-series)
2130
- // rejected `max_tokens` (they require `max_completion_tokens`) and only allow
2131
- // the default temperature, so sending `max_tokens`/`temperature:0.15` 400s
2132
- // every call. Branch the body by endpoint.
2133
- const isOpenRouter = Boolean(input.endpoint && input.endpoint.includes("openrouter"));
2134
- // Give the JSON response room to breathe so a video with 12+ text overlays
2135
- // doesn't get truncated (same rationale as Gemini's maxOutputTokens).
2136
- const OUTPUT_TOKEN_BUDGET = 16384;
2137
- const body = {
2138
- model: input.model,
2139
- response_format: { type: "json_object" },
2140
- messages: [{ role: "user", content }]
2141
- };
2142
- if (isOpenRouter) {
2143
- body.temperature = 0.15;
2144
- body.max_tokens = OUTPUT_TOKEN_BUDGET;
2145
- }
2146
- else {
2147
- body.max_completion_tokens = OUTPUT_TOKEN_BUDGET;
2148
- }
2149
- const response = await fetch(input.endpoint ?? "https://api.openai.com/v1/chat/completions", {
2150
- method: "POST",
2151
- headers: {
2152
- "content-type": "application/json",
2153
- "authorization": `Bearer ${input.apiKey}`,
2154
- ...(input.extraHeaders ?? {})
2155
- },
2156
- body: JSON.stringify(body)
2157
- });
2158
- const raw = await response.text();
2159
- if (!response.ok) {
2160
- let message = raw;
2161
- try {
2162
- message = JSON.parse(raw).error?.message ?? raw;
2163
- }
2164
- catch { }
2165
- throw new Error(`${input.endpoint?.includes("openrouter") ? "OpenRouter" : "OpenAI"} responded ${response.status}: ${message}`);
2166
- }
2167
- const parsed = JSON.parse(raw);
2168
- const text = parsed.choices?.[0]?.message?.content ?? "";
2169
- return extractJson(text);
2170
- }
2171
- async function callVisionWithFallback(input) {
2172
- const attempts = [];
2173
- if (input.keys.gemini) {
2174
- const model = process.env.GEMINI_MODEL ?? "gemini-3.5-flash";
2175
- attempts.push({
2176
- provider: "gemini",
2177
- model,
2178
- run: () => callGeminiVision({ apiKey: input.keys.gemini, model, prompt: input.prompt, frames: input.frames })
2179
- });
2180
- }
2181
- if (input.keys.openai) {
2182
- const model = process.env.OPENAI_VISION_MODEL ?? "gpt-5.5-2026-04-23";
2183
- attempts.push({
2184
- provider: "openai",
2185
- model,
2186
- run: () => callOpenAIVision({ apiKey: input.keys.openai, model, prompt: input.prompt, frames: input.frames })
2187
- });
2188
- }
2189
- if (input.keys.openrouter) {
2190
- const model = process.env.OPENROUTER_VISION_MODEL ?? "google/gemini-3.5-flash";
2191
- attempts.push({
2192
- provider: "openrouter",
2193
- model,
2194
- run: () => callOpenAIVision({
2195
- apiKey: input.keys.openrouter,
2196
- model,
2197
- prompt: input.prompt,
2198
- frames: input.frames,
2199
- endpoint: "https://openrouter.ai/api/v1/chat/completions",
2200
- extraHeaders: {
2201
- "HTTP-Referer": "https://staging.vidfarm.cc",
2202
- "X-Title": "Vidfarm smart decompose"
2203
- }
2204
- })
2205
- });
2206
- }
2207
- if (attempts.length === 0) {
2208
- throw new Error("No vision provider key available (GEMINI_API_KEY, OPENAI_API_KEY, or OPENROUTER_API_KEY).");
2209
- }
2210
- const errors = [];
2211
- for (const attempt of attempts) {
2212
- try {
2213
- const result = await attempt.run();
2214
- return { result, provider: attempt.provider, model: attempt.model };
2215
- }
2216
- catch (error) {
2217
- const message = error instanceof Error ? error.message : String(error);
2218
- errors.push(`${attempt.provider}(${attempt.model}): ${message}`);
2219
- console.error(`smart decompose ${attempt.provider} failed`, message);
2220
- }
2221
- }
2222
- throw new Error(`All vision providers failed. ${errors.join(" | ")}`);
2223
- }
2224
- // ==========================================================================
2225
- // Audio transcript pass — demux the source's audio track to mono 16kHz WAV
2226
- // (codec-free, accepted by every STT provider) and transcribe it verbatim
2227
- // with timestamped segments. Runs concurrently with the vision + OCR passes
2228
- // and is non-fatal: a video with no audio track, or a transcription failure,
2229
- // yields transcript: null instead of failing the decompose.
2230
- // ==========================================================================
2231
- async function extractAudioAsWav(videoPath, tempDir) {
2232
- const probeOutput = await runBinary(ffprobeBin, [
2233
- "-v", "error",
2234
- "-select_streams", "a",
2235
- "-show_entries", "stream=codec_type",
2236
- "-of", "json",
2237
- videoPath
2238
- ]);
2239
- const parsed = JSON.parse(probeOutput || "{}");
2240
- if (!Array.isArray(parsed.streams) || parsed.streams.length === 0)
2241
- return null;
2242
- const audioPath = path.join(tempDir, "audio.wav");
2243
- await runBinary(ffmpegBin, [
2244
- "-y",
2245
- "-i", videoPath,
2246
- "-vn",
2247
- "-ac", "1",
2248
- "-ar", "16000",
2249
- "-c:a", "pcm_s16le",
2250
- audioPath
2251
- ]);
2252
- return audioPath;
2253
- }
2254
- function buildTranscriptionPrompt(durationSeconds) {
2255
- return `Transcribe the spoken audio in this ${durationSeconds.toFixed(1)}-second clip VERBATIM.
2256
-
2257
- Return strictly valid JSON matching this schema:
2258
- {
2259
- "language": string,
2260
- "segments": [{ "start": number, "end": number, "text": string }]
2261
- }
2262
-
2263
- RULES:
2264
- - Transcribe every spoken word exactly as heard — keep filler words, slang, and repetitions. Do not paraphrase, translate, or clean up grammar.
2265
- - "language" is the BCP-47 code of the dominant spoken language (e.g. "en", "es").
2266
- - Split into short segments at natural phrase boundaries; "start"/"end" are seconds within the clip (start >= 0, end <= ${durationSeconds.toFixed(3)}, end > start).
2267
- - Only transcribe speech and sung lyrics. Ignore background music without lyrics and sound effects.
2268
- - If there is no speech at all, return { "language": null, "segments": [] }.
2269
-
2270
- OUTPUT: pure JSON, no prose, no markdown fences.`;
2271
- }
2272
- function normalizeTranscriptSegments(raw, durationSeconds) {
2273
- if (!Array.isArray(raw))
2274
- return [];
2275
- return raw
2276
- .map((entry) => {
2277
- const record = (entry && typeof entry === "object") ? entry : {};
2278
- const start = Number(record.start ?? NaN);
2279
- const end = Number(record.end ?? NaN);
2280
- const text = typeof record.text === "string" ? record.text.trim() : "";
2281
- return { start, end, text };
2282
- })
2283
- .filter((s) => s.text.length > 0 && Number.isFinite(s.start) && Number.isFinite(s.end) && s.end > s.start)
2284
- .map((s) => ({
2285
- start: Number(Math.max(0, s.start).toFixed(3)),
2286
- end: Number(Math.min(durationSeconds > 0 ? durationSeconds : s.end, s.end).toFixed(3)),
2287
- text: s.text
2288
- }))
2289
- .sort((a, b) => a.start - b.start);
2290
- }
2291
- async function callGeminiTranscription(input) {
2292
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(input.model)}:generateContent?key=${encodeURIComponent(input.apiKey)}`, {
2293
- method: "POST",
2294
- headers: { "content-type": "application/json" },
2295
- body: JSON.stringify({
2296
- contents: [{
2297
- role: "user",
2298
- parts: [
2299
- { text: buildTranscriptionPrompt(input.durationSeconds) },
2300
- { inlineData: { mimeType: "audio/wav", data: input.audioBase64 } }
2301
- ]
2302
- }],
2303
- generationConfig: {
2304
- responseMimeType: "application/json",
2305
- temperature: 0,
2306
- maxOutputTokens: 16384
2307
- }
2308
- })
2309
- });
2310
- const raw = await response.text();
2311
- if (!response.ok) {
2312
- let message = raw;
2313
- try {
2314
- message = JSON.parse(raw).error?.message ?? raw;
2315
- }
2316
- catch { }
2317
- throw new Error(`Gemini responded ${response.status}: ${message}`);
2318
- }
2319
- const parsed = JSON.parse(raw);
2320
- const text = parsed.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
2321
- const record = (extractJson(text) ?? {});
2322
- return {
2323
- language: typeof record.language === "string" && record.language.trim() ? record.language.trim() : null,
2324
- segments: normalizeTranscriptSegments(record.segments, input.durationSeconds)
2325
- };
2326
- }
2327
- async function callOpenAITranscription(input) {
2328
- const audioBuffer = await readFile(input.audioPath);
2329
- // Only whisper-1 supports verbose_json (per-segment timestamps). Newer
2330
- // gpt-4o-*-transcribe models return plain json; collapse to one segment.
2331
- const wantsSegments = input.model === "whisper-1";
2332
- const form = new FormData();
2333
- form.append("file", new Blob([new Uint8Array(audioBuffer)], { type: "audio/wav" }), "audio.wav");
2334
- form.append("model", input.model);
2335
- form.append("response_format", wantsSegments ? "verbose_json" : "json");
2336
- const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
2337
- method: "POST",
2338
- headers: { "authorization": `Bearer ${input.apiKey}` },
2339
- body: form
2340
- });
2341
- const raw = await response.text();
2342
- if (!response.ok) {
2343
- let message = raw;
2344
- try {
2345
- message = JSON.parse(raw).error?.message ?? raw;
2346
- }
2347
- catch { }
2348
- throw new Error(`OpenAI responded ${response.status}: ${message}`);
2349
- }
2350
- const parsed = JSON.parse(raw);
2351
- const segments = wantsSegments && Array.isArray(parsed.segments)
2352
- ? normalizeTranscriptSegments(parsed.segments, input.durationSeconds)
2353
- : [];
2354
- if (segments.length === 0 && typeof parsed.text === "string" && parsed.text.trim().length > 0) {
2355
- segments.push({ start: 0, end: Number(input.durationSeconds.toFixed(3)), text: parsed.text.trim() });
2356
- }
2357
- return {
2358
- language: typeof parsed.language === "string" && parsed.language.trim() ? parsed.language.trim() : null,
2359
- segments
2360
- };
2361
- }
2362
- async function transcribeAudioWithFallback(input) {
2363
- const attempts = [];
2364
- if (input.keys.gemini) {
2365
- const model = process.env.GEMINI_TRANSCRIBE_MODEL ?? process.env.GEMINI_MODEL ?? "gemini-3.5-flash";
2366
- attempts.push({
2367
- provider: "gemini",
2368
- model,
2369
- run: async () => {
2370
- const audioBase64 = (await readFile(input.audioPath)).toString("base64");
2371
- return callGeminiTranscription({ apiKey: input.keys.gemini, model, audioBase64, durationSeconds: input.durationSeconds });
2372
- }
2373
- });
2374
- }
2375
- if (input.keys.openai) {
2376
- const model = process.env.OPENAI_TRANSCRIBE_MODEL ?? "whisper-1";
2377
- attempts.push({
2378
- provider: "openai",
2379
- model,
2380
- run: () => callOpenAITranscription({ apiKey: input.keys.openai, model, audioPath: input.audioPath, durationSeconds: input.durationSeconds })
2381
- });
2382
- }
2383
- if (attempts.length === 0)
2384
- return null;
2385
- const errors = [];
2386
- for (const attempt of attempts) {
2387
- try {
2388
- const result = await attempt.run();
2389
- return {
2390
- text: result.segments.map((s) => s.text).join(" ").trim(),
2391
- language: result.language,
2392
- segments: result.segments,
2393
- provider: attempt.provider,
2394
- model: attempt.model
2395
- };
2396
- }
2397
- catch (error) {
2398
- const message = error instanceof Error ? error.message : String(error);
2399
- errors.push(`${attempt.provider}(${attempt.model}): ${message}`);
2400
- console.error(`smart decompose transcription ${attempt.provider} failed`, message);
2401
- }
2402
- }
2403
- throw new Error(`All transcription providers failed. ${errors.join(" | ")}`);
2404
- }
2405
- export async function smartDecomposeVideo(input) {
2406
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
2407
- const sourcePath = path.join(tempDir, "source.mp4");
2408
- const startedAt = Date.now();
2409
- devLog("hyperframes.smart_decompose.start", {
2410
- source_url: input.sourceUrl,
2411
- has_user_prompt: Boolean(input.userPrompt),
2412
- providers_available: {
2413
- gemini: Boolean(input.keys.gemini),
2414
- openai: Boolean(input.keys.openai),
2415
- openrouter: Boolean(input.keys.openrouter)
2416
- }
2417
- });
2418
- try {
2419
- await downloadVideoTo(input.sourceUrl, sourcePath);
2420
- const meta = await ffprobeVideo(sourcePath);
2421
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
2422
- throw new Error("Could not read video duration");
2423
- }
2424
- devLog("hyperframes.smart_decompose.probed", {
2425
- duration_seconds: meta.durationSeconds,
2426
- width: meta.width,
2427
- height: meta.height
2428
- });
2429
- // Sparse frames for the vision pass (scenes, captions, style, viral_dna)
2430
- // and dense frames for the OCR pass (deterministic caption timing).
2431
- // Phase 1 — deterministic ffmpeg/whisper work up front. Transcription is
2432
- // pulled INTO this phase (chained off the audio extract) so the transcript
2433
- // is ready BEFORE the vision + harness passes run: emotional punch on
2434
- // short-form video is frequently carried by the AUDIO (a trending sound, a
2435
- // beat drop, vocal intonation), and the silent frames alone can't capture
2436
- // it. The transcript overlaps with frame-sampling/OCR/cut-detection here
2437
- // instead of with the vision call, so this stays roughly latency-neutral.
2438
- const [frames, ocrRawFrames, transcript, sceneCuts] = await Promise.all([
2439
- sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir),
2440
- extractDenseOcrFrames(sourcePath, meta.durationSeconds, tempDir),
2441
- extractAudioAsWav(sourcePath, tempDir)
2442
- .then((audioPath) => audioPath
2443
- ? transcribeAudioWithFallback({ audioPath, durationSeconds: meta.durationSeconds, keys: input.keys })
2444
- : null)
2445
- .catch((err) => {
2446
- console.error("smart decompose audio extract/transcription failed", err instanceof Error ? err.message : err);
2447
- return null;
2448
- }),
2449
- detectSceneCuts(sourcePath, meta.durationSeconds, tempDir).catch((err) => {
2450
- console.error("smart decompose scene-cut detection failed", err instanceof Error ? err.message : err);
2451
- return [];
2452
- })
2453
- ]);
2454
- devLog("hyperframes.smart_decompose.frames_sampled", {
2455
- frame_count: frames.length,
2456
- ocr_frame_count: ocrRawFrames.length,
2457
- has_audio_track: Boolean(transcript),
2458
- scene_cut_count: sceneCuts.length
2459
- });
2460
- const prompt = buildSmartDecomposePrompt({
2461
- durationSeconds: meta.durationSeconds,
2462
- width: meta.width,
2463
- height: meta.height,
2464
- userPrompt: input.userPrompt ?? null,
2465
- trendTaglineHint: input.trendTaglineHint ?? null,
2466
- frameTimestamps: frames.map((f) => f.timestamp),
2467
- sceneCuts,
2468
- transcript
2469
- });
2470
- // Phase 2 — vision, OCR, and product-placement run concurrently, all now
2471
- // transcript-aware. OCR and placement failures are non-fatal. The
2472
- // placement call is DELIBERATELY separate from the caption/scene vision
2473
- // call so it can never dilute timeline-layer quality — it reuses the same
2474
- // sparse frames (no extra ffmpeg) plus the transcript from phase 1.
2475
- const [attempt, ocrFrames, placementSurfaces] = await Promise.all([
2476
- callVisionWithFallback({
2477
- prompt,
2478
- frames,
2479
- keys: input.keys
2480
- }),
2481
- runOcrPass({ frames: ocrRawFrames, keys: input.keys }).catch((err) => {
2482
- console.error("smart decompose OCR pass failed", err instanceof Error ? err.message : err);
2483
- return [];
2484
- }),
2485
- runPlacementSurfacePass({
2486
- frames,
2487
- keys: input.keys,
2488
- durationSeconds: meta.durationSeconds,
2489
- sceneSlugs: []
2490
- }).catch((err) => {
2491
- console.error("smart decompose placement pass failed", err instanceof Error ? err.message : err);
2492
- return [];
2493
- })
2494
- ]);
2495
- devLog("hyperframes.smart_decompose.ocr_completed", {
2496
- frames_with_text: ocrFrames.filter((f) => f.texts.length > 0).length,
2497
- total_ocr_frames: ocrFrames.length
2498
- });
2499
- devLog("hyperframes.smart_decompose.transcript_completed", {
2500
- has_transcript: Boolean(transcript),
2501
- segment_count: transcript?.segments.length ?? 0,
2502
- language: transcript?.language ?? null,
2503
- provider: transcript?.provider ?? null
2504
- });
2505
- const raw = attempt.result;
2506
- const record = (raw && typeof raw === "object") ? raw : {};
2507
- const scenesRaw = Array.isArray(record.scenes) ? record.scenes : [];
2508
- const captionsRaw = Array.isArray(record.captions) ? record.captions : [];
2509
- const usedSceneSlugs = new Set();
2510
- const scenes = scenesRaw
2511
- .map((scene, index) => {
2512
- const slug = dedupeSlug(normalizeSlug(typeof scene.slug === "string" ? scene.slug : `scene_${index + 1}`) || `scene_${index + 1}`, usedSceneSlugs);
2513
- return {
2514
- slug,
2515
- start: Number(scene.start ?? 0),
2516
- duration: Number(scene.duration ?? 0),
2517
- label: typeof scene.label === "string" ? scene.label : `Scene ${index + 1}`,
2518
- description: typeof scene.description === "string" ? scene.description : "",
2519
- viral_note: typeof scene.viral_note === "string" ? scene.viral_note : ""
2520
- };
2521
- })
2522
- .filter((s) => Number.isFinite(s.start) && Number.isFinite(s.duration) && s.duration > 0);
2523
- const usedCaptionSlugs = new Set();
2524
- const captions = captionsRaw
2525
- .map((c, index) => {
2526
- const slug = dedupeSlug(normalizeSlug(typeof c.slug === "string" ? c.slug : `caption_${index + 1}`) || `caption_${index + 1}`, usedCaptionSlugs);
2527
- return {
2528
- id: `caption_${index + 1}`,
2529
- slug,
2530
- text: typeof c.text === "string" ? c.text : "",
2531
- start: Number(c.start ?? 0),
2532
- duration: Number(c.duration ?? 0),
2533
- x: Number(c.x ?? 8),
2534
- y: Number(c.y ?? 68),
2535
- width: Number(c.width ?? 84),
2536
- height: Number(c.height ?? 12),
2537
- font_size: resolveFontSize({ fontSize: c.font_size, fontSizePercent: c.font_size_percent, canvasHeight: meta.height }),
2538
- color: normalizeHexColor(c.color, "#ffffff"),
2539
- background: normalizeHexColor(c.background, "#000000"),
2540
- background_style: normalizeBackgroundStyle(c.background_style),
2541
- font_family: normalizeFontFamily(c.font_family),
2542
- font_weight: normalizeFontWeight(c.font_weight),
2543
- viral_note: typeof c.viral_note === "string" ? c.viral_note : undefined
2544
- };
2545
- })
2546
- .filter((c) => c.text.trim().length > 0 && Number.isFinite(c.start) && Number.isFinite(c.duration) && c.duration > 0);
2547
- const snapped = snapCaptionTimingsToOcr({
2548
- captions,
2549
- ocrFrames,
2550
- durationSeconds: meta.durationSeconds
2551
- });
2552
- const finalCaptions = snapped.captions;
2553
- devLog("hyperframes.smart_decompose.captions_snapped", {
2554
- total_captions: captions.length,
2555
- snapped_count: snapped.snappedCount
2556
- });
2557
- // Ground scene boundaries on the ffmpeg-detected hard cuts (no-op if none).
2558
- const snappedScenes = snapScenesToCuts({ scenes, cuts: sceneCuts, durationSeconds: meta.durationSeconds });
2559
- const finalScenes = snappedScenes.scenes;
2560
- devLog("hyperframes.smart_decompose.scenes_snapped", {
2561
- total_scenes: scenes.length,
2562
- snapped_count: snappedScenes.snappedCount,
2563
- scene_cut_count: sceneCuts.length
2564
- });
2565
- // Link each product-placement surface to the scene its start falls within,
2566
- // deterministically from the finalized scene ranges (the placement call ran
2567
- // before scenes existed, so it couldn't know the slugs).
2568
- const finalPlacementSurfaces = placementSurfaces.map((surface) => {
2569
- const owning = finalScenes.find((s) => surface.start >= s.start && surface.start < s.start + s.duration);
2570
- return owning ? { ...surface, scene_slug: owning.slug } : surface;
2571
- });
2572
- const viralDna = normalizeViralDna(record.viral_dna);
2573
- // Caller-supplied trend_tagline always wins over whatever the LLM emits so
2574
- // the user's chosen tagline round-trips even if the model tries to rephrase.
2575
- if (input.trendTaglineHint && input.trendTaglineHint.trim().length > 0) {
2576
- viralDna.trend_tagline = input.trendTaglineHint.trim();
2577
- }
2578
- // Editor harness — technical editing direction. Run AFTER viral DNA is
2579
- // normalized so the harness can operationalize the exact emotional punch
2580
- // instead of guessing independently and drifting from the decompose result.
2581
- const editorHarness = await runEditorHarnessPass({
2582
- frames,
2583
- keys: input.keys,
2584
- durationSeconds: meta.durationSeconds,
2585
- sceneCuts,
2586
- userPrompt: input.userPrompt ?? null,
2587
- transcript,
2588
- viralDna
2589
- }).catch((err) => {
2590
- console.error("smart decompose editor-harness pass failed", err instanceof Error ? err.message : err);
2591
- return DEFAULT_EDITOR_HARNESS;
2592
- });
2593
- // NOTE: the per-scene clip-annotation + content-format pass is deliberately
2594
- // NOT run here — it is a separate vision call that would risk the ~60s origin
2595
- // timeout on longer videos. /auto-decompose seeds scene-annotations.json
2596
- // "pending" and /scene-annotations-poll drives annotateScenesFromSource
2597
- // against these finalized scenes (persisted in smart-decompose.json).
2598
- devLog("hyperframes.smart_decompose.completed", {
2599
- provider: attempt.provider,
2600
- model: attempt.model,
2601
- duration_ms: Date.now() - startedAt,
2602
- scene_count: finalScenes.length,
2603
- caption_count: finalCaptions.length,
2604
- captions_snapped: snapped.snappedCount,
2605
- scenes_snapped: snappedScenes.snappedCount,
2606
- placement_surface_count: finalPlacementSurfaces.length,
2607
- editor_harness_format: editorHarness.format,
2608
- editor_harness_scene_count: editorHarness.scenes.length
2609
- });
2610
- return {
2611
- summary: typeof record.summary === "string" ? record.summary : "",
2612
- viralDna,
2613
- placementSurfaces: finalPlacementSurfaces,
2614
- editorHarness,
2615
- transcript,
2616
- durationSeconds: meta.durationSeconds,
2617
- width: meta.width,
2618
- height: meta.height,
2619
- scenes: finalScenes,
2620
- captions: finalCaptions,
2621
- provider: attempt.provider,
2622
- model: attempt.model
2623
- };
2624
- }
2625
- catch (error) {
2626
- devLog("hyperframes.smart_decompose.failed", {
2627
- source_url: input.sourceUrl,
2628
- duration_ms: Date.now() - startedAt,
2629
- ...devErrorFields(error)
2630
- }, "error");
2631
- throw error;
2632
- }
2633
- finally {
2634
- await rm(tempDir, { recursive: true, force: true });
2635
- }
2636
- }
2637
- export function buildSmartDecomposedHtml(input) {
2638
- const { compositionId, sourceUrl, title, result } = input;
2639
- const totalDuration = Number(result.durationSeconds.toFixed(3));
2640
- const scenes = result.scenes.length > 0
2641
- ? result.scenes
2642
- : [{ slug: "scene_1", start: 0, duration: totalDuration, label: "Scene 1", description: "", viral_note: "" }];
2643
- // Assign each caption to the lowest track index (starting at 2 — track 0
2644
- // is video, track 1 is audio) where it doesn't temporally overlap any
2645
- // caption already on that track. Enforces the "no two clips on the same
2646
- // track can overlap in time" rule. Multiple stacked labels (like a
2647
- // staircase of "single/talking/dating/marriage/have kids" all shown for
2648
- // the full duration) end up on tracks 2,3,4,5,6 respectively.
2649
- const CAPTION_TRACK_START = 2;
2650
- const captionTrackClipRanges = [];
2651
- const assignCaptionTrackIndex = (start, end) => {
2652
- for (let i = 0; i < captionTrackClipRanges.length; i += 1) {
2653
- if (!captionTrackClipRanges[i].some((r) => start < r.end && r.start < end)) {
2654
- captionTrackClipRanges[i].push({ start, end });
2655
- return CAPTION_TRACK_START + i;
2656
- }
2657
- }
2658
- captionTrackClipRanges.push([{ start, end }]);
2659
- return CAPTION_TRACK_START + captionTrackClipRanges.length - 1;
2660
- };
2661
- const sceneEls = scenes.map((scene, index) => {
2662
- const start = Math.max(0, Math.min(totalDuration - 0.1, Number(scene.start.toFixed(3))));
2663
- const duration = Math.max(0.1, Number(scene.duration.toFixed(3)));
2664
- const end = Math.min(totalDuration, start + duration);
2665
- const id = shortElementId();
2666
- const sceneSlug = scene.slug || `scene_${index + 1}`;
2667
- return `<div class="clip" id="${id}" data-hf-id="${id}" data-hf-slug="${escapeAttr(sceneSlug)}" data-layer-mode="both" data-layer-kind="video" data-viral-note="${escapeAttr(scene.viral_note ?? "")}" data-start="${start}" data-duration="${Number(duration.toFixed(3))}" data-end="${Number(end.toFixed(3))}" data-track-index="0" data-label="${escapeAttr(scene.label)}" data-media-start="${start}" data-playback-start="${start}" data-vf-timeline-proxy="video" data-src="${escapeAttr(sourceUrl)}" style="position:absolute;left:0%;top:0%;width:1%;height:1%;z-index:0;display:none;pointer-events:none;"></div>`;
2668
- }).join("\n ");
2669
- // Sort captions by start time so the track-assignment greedy fills lower
2670
- // tracks first with earlier captions.
2671
- const sortedCaptions = [...result.captions].sort((a, b) => (a.start - b.start) || (a.duration - b.duration));
2672
- const captionEls = sortedCaptions.map((caption, index) => {
2673
- const start = Math.max(0, Math.min(totalDuration - 0.1, Number(caption.start.toFixed(3))));
2674
- const duration = Math.max(0.1, Math.min(totalDuration - start, Number(caption.duration.toFixed(3))));
2675
- const end = start + duration;
2676
- const trackIndex = assignCaptionTrackIndex(start, end);
2677
- const id = shortElementId();
2678
- const captionSlug = caption.slug || `caption_${index + 1}`;
2679
- const clampedX = clampPercent(caption.x, 0, 95);
2680
- const clampedY = clampPercent(caption.y, 0, 95);
2681
- const clampedW = clampPercent(caption.width, 5, 100 - clampedX);
2682
- const clampedH = clampPercent(caption.height, 3, 100 - clampedY);
2683
- const escapedText = escapeHtml(caption.text);
2684
- const innerStyle = captionInlineTextStyle(caption);
2685
- return `<div class="clip" id="${id}" data-hf-id="${id}" data-hf-slug="${escapeAttr(captionSlug)}" data-layer-mode="publish" data-layer-kind="caption" data-viral-note="${escapeAttr(caption.viral_note ?? "")}" data-start="${start}" data-duration="${Number(duration.toFixed(3))}" data-end="${Number(end.toFixed(3))}" data-track-index="${trackIndex}" data-label="${escapeAttr(caption.text.slice(0, 40))}" data-text-background-style="${caption.background_style}" data-text-background-color="${escapeAttr(caption.background)}" data-font-family="${escapeAttr(caption.font_family)}" style="position:absolute;left:${clampedX}%;top:${clampedY}%;width:${clampedW}%;height:${clampedH}%;z-index:${trackIndex};display:flex;align-items:center;justify-content:center;padding:3px;box-sizing:border-box;text-align:center;color:${caption.color};background:transparent;font-family:'${caption.font_family}', 'TikTok Sans', sans-serif;font-weight:${caption.font_weight};font-size:${caption.font_size}px;line-height:1.2;overflow:visible;"><span data-vf-text-lines="true"><span data-vf-text-inline="true" style="${innerStyle}">${escapedText}</span></span></div>`;
2686
- }).join("\n ");
2687
- const audioId = shortElementId();
2688
- const audioEl = `<audio class="clip" id="${audioId}" data-hf-id="${audioId}" data-hf-slug="source_audio" data-layer-mode="both" data-layer-kind="audio" data-viral-note="Original TikTok audio track." data-start="0" data-duration="${totalDuration}" data-end="${totalDuration}" data-track-index="1" data-label="Original audio" data-media-start="0" data-playback-start="0" data-timeline-role="music" data-volume="1" src="${escapeAttr(sourceUrl)}" preload="metadata"></audio>`;
2689
- const viralDnaJson = escapeAttr(JSON.stringify(result.viralDna ?? DEFAULT_VIRAL_DNA));
2690
- const editorHarnessJson = escapeAttr(JSON.stringify(result.editorHarness ?? DEFAULT_EDITOR_HARNESS));
2691
- return `<!doctype html>
2692
- <html lang="en">
2693
- <head>
2694
- <meta charset="UTF-8" />
2695
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
2696
- <style>
2697
- @import url("https://fonts.googleapis.com/css2?family=Abel&family=Montserrat:wght@600;700&family=Source+Code+Pro:wght@700&family=TikTok+Sans:wght@400;600;700;800;900&family=Yesteryear&display=swap");
2698
- html, body { width:100%; height:100%; margin:0; overflow:hidden; background:#050604; }
2699
- [data-composition-id] { position:relative; width:100vw; height:100vh; overflow:hidden; background:#050604; }
2700
- audio { display:none; }
2701
- </style>
2702
- </head>
2703
- <body>
2704
- <div data-composition-id="${escapeAttr(compositionId)}" data-source-video="${escapeAttr(sourceUrl)}" data-width="${result.width || 720}" data-height="${result.height || 1280}" data-duration="${totalDuration}" data-source-title="${escapeAttr(title)}" data-viral-dna="${viralDnaJson}" data-editor-harness="${editorHarnessJson}">
2705
- <video data-vf-playback-source="true" src="${escapeAttr(sourceUrl)}" muted playsinline preload="auto" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;object-fit:cover;background:#050604"></video>
2706
- ${sceneEls}
2707
- ${audioEl}
2708
- ${captionEls}
2709
- </div>
2710
- </body>
2711
- </html>`;
2712
- }
2713
- export function buildSmartDecomposedData(input) {
2714
- const { sourceUrl, result } = input;
2715
- const durationSeconds = Number(result.durationSeconds.toFixed(3));
2716
- return {
2717
- summary: result.summary,
2718
- scenes: result.scenes.map((scene) => ({
2719
- slug: scene.slug,
2720
- start: scene.start,
2721
- duration: scene.duration,
2722
- label: scene.label,
2723
- description: scene.description,
2724
- viral_note: scene.viral_note
2725
- })),
2726
- captions: result.captions.map((caption) => ({
2727
- slug: caption.slug,
2728
- text: caption.text,
2729
- start: caption.start,
2730
- duration: caption.duration,
2731
- x: caption.x,
2732
- y: caption.y,
2733
- width: caption.width,
2734
- height: caption.height,
2735
- color: caption.color,
2736
- background: caption.background,
2737
- background_style: caption.background_style,
2738
- font_family: caption.font_family,
2739
- font_size: caption.font_size,
2740
- font_weight: caption.font_weight,
2741
- viral_note: caption.viral_note
2742
- })),
2743
- viralDna: result.viralDna,
2744
- width: result.width,
2745
- height: result.height,
2746
- durationSeconds,
2747
- source_assets: [{
2748
- type: "video",
2749
- url: sourceUrl,
2750
- width: result.width,
2751
- height: result.height,
2752
- duration: durationSeconds
2753
- }]
2754
- };
2755
- }
2756
- // The public placeholder tile every blank scene starts on — a gray diagonal-
2757
- // striped 720×1280 image with a dashed text box (uploaded once by
2758
- // scripts/upload-scene-placeholder.ts). Hardcoded to the prod bucket URL (a
2759
- // public object that resolves from any stage) so the blank composition renders
2760
- // identically everywhere. Mirrors SCENE_PLACEHOLDER_IMAGE_URL in the SPA editor.
2761
- export const SCENE_PLACEHOLDER_IMAGE_URL = "https://vidfarmprodstack-vidfarmbucket335ee12f-0vsvtd5earqy.s3.us-east-1.amazonaws.com/primitives/placeholders/scene-placeholder.png";
2762
- const BLANK_PROJECT_SCENE_COUNT = 3;
2763
- const BLANK_PROJECT_SCENE_DURATION = 4;
2764
- // The canonical, reusable "new blank project" composition. This is the single
2765
- // composition.html every fresh /editor/original/fork/new opens on, and the base
2766
- // every blank project forks from (POST /api/v1/compositions seeds a new fork by
2767
- // copying this). It mirrors blankProjectSemanticLayers() in the SPA editor: a
2768
- // continuous placeholder image track (3 butt-cut 4s scenes on track 0) plus a
2769
- // "Scene 1/2/3" caption on track 3, giving the user an editable 12s scaffold
2770
- // instead of an empty canvas. Like the other builders it emits clips only — the
2771
- // runtime is injected on serve by rewriteHyperframesDraftCompositionHtml, and z
2772
- // is normalized to track index on the way out.
2773
- export function buildBlankCompositionHtml(compositionId) {
2774
- const totalDuration = BLANK_PROJECT_SCENE_COUNT * BLANK_PROJECT_SCENE_DURATION;
2775
- const sceneEls = [];
2776
- const captionEls = [];
2777
- for (let index = 0; index < BLANK_PROJECT_SCENE_COUNT; index += 1) {
2778
- const sceneNumber = index + 1;
2779
- const start = index * BLANK_PROJECT_SCENE_DURATION;
2780
- const end = start + BLANK_PROJECT_SCENE_DURATION;
2781
- // Placeholder image scene — full canvas, track 0. Together the three form
2782
- // one continuous 12s media track the user replaces with real footage.
2783
- const imageId = `scene-${sceneNumber}-image`;
2784
- sceneEls.push(`<img class="clip" id="${imageId}" data-hf-id="${imageId}" data-hf-slug="scene_${sceneNumber}" data-layer-mode="both" data-layer-kind="image" data-viral-note="Placeholder scene backdrop — replace with your own media." data-start="${start}" data-duration="${BLANK_PROJECT_SCENE_DURATION}" data-end="${end}" data-track-index="0" data-label="Scene ${sceneNumber}" src="${escapeAttr(SCENE_PLACEHOLDER_IMAGE_URL)}" alt="" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;opacity:1;border-radius:0px;overflow:hidden;object-fit:cover;" />`);
2785
- // "Scene N" label sitting in the tile's blank text box — white on a solid
2786
- // black highlight so it reads over both the placeholder and swapped-in media.
2787
- const caption = {
2788
- slug: `scene_${sceneNumber}_label`,
2789
- text: `Scene ${sceneNumber}`,
2790
- start,
2791
- duration: BLANK_PROJECT_SCENE_DURATION,
2792
- x: 13.5,
2793
- y: 44,
2794
- width: 73,
2795
- height: 12,
2796
- font_size: 34,
2797
- color: "#ffffff",
2798
- background: "#000000",
2799
- background_style: "highlight-solid",
2800
- font_family: "TikTok Sans",
2801
- font_weight: 700,
2802
- viral_note: "Placeholder scene label — edit or delete once you add real media."
2803
- };
2804
- const captionId = `scene-${sceneNumber}-caption`;
2805
- const innerStyle = captionInlineTextStyle(caption);
2806
- captionEls.push(`<div class="clip" id="${captionId}" data-hf-id="${captionId}" data-hf-slug="${escapeAttr(caption.slug)}" data-layer-mode="publish" data-layer-kind="caption" data-viral-note="${escapeAttr(caption.viral_note ?? "")}" data-start="${start}" data-duration="${BLANK_PROJECT_SCENE_DURATION}" data-end="${end}" data-track-index="3" data-label="${escapeAttr(caption.text)}" data-text-background-style="${caption.background_style}" data-text-background-color="${escapeAttr(caption.background)}" data-font-family="${escapeAttr(caption.font_family)}" style="position:absolute;left:${caption.x}%;top:${caption.y}%;width:${caption.width}%;height:${caption.height}%;z-index:3;display:flex;align-items:center;justify-content:center;padding:3px;box-sizing:border-box;text-align:center;color:${caption.color};background:transparent;font-family:'${caption.font_family}', 'TikTok Sans', sans-serif;font-weight:${caption.font_weight};font-size:${caption.font_size}px;line-height:1.2;overflow:visible;"><span data-vf-text-lines="true"><span data-vf-text-inline="true" style="${innerStyle}">${escapeHtml(caption.text)}</span></span></div>`);
2807
- }
2808
- // data-vf-origin="raw": a brand-new project, so the editor never auto-prompts
2809
- // the Decompose modal (there is no source video to decompose).
2810
- return `<!doctype html>
2811
- <html lang="en">
2812
- <head>
2813
- <meta charset="UTF-8" />
2814
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
2815
- <style>
2816
- @import url("https://fonts.googleapis.com/css2?family=Abel&family=Montserrat:wght@600;700&family=Source+Code+Pro:wght@700&family=TikTok+Sans:wght@400;600;700;800;900&family=Yesteryear&display=swap");
2817
- html, body { width:100%; height:100%; margin:0; overflow:hidden; background:#050604; }
2818
- [data-composition-id] { position:relative; width:100vw; height:100vh; overflow:hidden; background:#050604; }
2819
- </style>
2820
- </head>
2821
- <body>
2822
- <div data-composition-id="${escapeAttr(compositionId)}" data-vf-origin="raw" data-width="720" data-height="1280" data-duration="${totalDuration}" data-source-title="Untitled project">
2823
- ${sceneEls.join("\n ")}
2824
- ${captionEls.join("\n ")}
2825
- </div>
2826
- </body>
2827
- </html>`;
2828
- }
2829
- function normalizeSlug(value) {
2830
- return String(value ?? "")
2831
- .toLowerCase()
2832
- .replace(/[^a-z0-9]+/g, "_")
2833
- .replace(/^_+|_+$/g, "")
2834
- .slice(0, 64);
2835
- }
2836
- function dedupeSlug(base, used) {
2837
- const seed = base || "element";
2838
- if (!used.has(seed)) {
2839
- used.add(seed);
2840
- return seed;
2841
- }
2842
- for (let index = 2; index < 999; index += 1) {
2843
- const candidate = `${seed}_${index}`;
2844
- if (!used.has(candidate)) {
2845
- used.add(candidate);
2846
- return candidate;
2847
- }
2848
- }
2849
- const fallback = `${seed}_${Date.now().toString(36)}`;
2850
- used.add(fallback);
2851
- return fallback;
2852
- }
2853
- function escapeAttr(value) {
2854
- return String(value ?? "").replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
2855
- }
2856
- function escapeHtml(value) {
2857
- return String(value ?? "").replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
2858
- }
2859
- function clampPercent(value, min, max) {
2860
- if (!Number.isFinite(value))
2861
- return min;
2862
- return Math.max(min, Math.min(max, value));
2863
- }
2864
- export function stampCompositionIdentity(html, input) {
2865
- if (!input.forkId)
2866
- return html;
2867
- const composed = `${input.forkId}_v${input.version}`;
2868
- const { document } = parseHTML(html);
2869
- const root = document.querySelector("[data-composition-id]");
2870
- if (!root)
2871
- return html;
2872
- root.setAttribute("data-composition-id", composed);
2873
- root.setAttribute("data-fork-id", input.forkId);
2874
- root.setAttribute("data-fork-version", String(input.version));
2875
- return `<!doctype html>\n${document.documentElement.outerHTML}`;
2876
- }
2877
- // True when the union of canvas-filling intervals leaves any hole ≥ minGap within
2878
- // [0, total] — a leading gap, a hole between clips, or a trailing gap. Mirrors the
2879
- // editor-side computeCompositionGaps intent (see src/devcli/composition-edit.ts).
2880
- function compositionHasCoverageGap(intervals, total, minGap = 0.2) {
2881
- if (!Number.isFinite(total) || total <= 0)
2882
- return false;
2883
- const sorted = intervals.filter(([start, end]) => end > start).sort((a, b) => a[0] - b[0]);
2884
- let cursor = 0;
2885
- for (const [start, end] of sorted) {
2886
- if (start > cursor + minGap)
2887
- return true;
2888
- cursor = Math.max(cursor, end);
2889
- }
2890
- return cursor < total - minGap;
2891
- }
2892
- // Scene transitions: the INCOMING clip carries data-transition and its vf-tr-*
2893
- // animation plays over the clip's first data-transition-duration seconds — the
2894
- // render engine's CSS adapter scrubs that on its own. What the render engine
2895
- // canNOT do is keep the OUTGOING clip on screen beneath it (visibility is
2896
- // derived strictly from data-start/data-duration), so materialize the overlap
2897
- // here: extend the previous same-track butt-cut clip's data-duration by the
2898
- // transition length. The granted extension is recorded in data-transition-hold
2899
- // so the pass is idempotent and self-healing — every run first restores each
2900
- // clip's nominal duration (duration - hold), then re-grants from the current
2901
- // set of [data-transition] clips (so removed/retimed transitions shrink their
2902
- // neighbor back). Stored drafts stay butt-cut; only publish/render output
2903
- // carries the overlap. The preview runtime applies the same window live.
2904
- function materializeTransitionOverlaps(document) {
2905
- const upsertStyleVar = (el, name, value) => {
2906
- const styleValue = el.getAttribute("style") || "";
2907
- const decl = `${name}:${value}`;
2908
- const pattern = new RegExp(`(^|;)\\s*${name.replace(/[.*+?^${}()|[\]\\-]/g, "\\$&")}\\s*:[^;]*`, "i");
2909
- const nextStyle = pattern.test(styleValue)
2910
- ? styleValue.replace(pattern, (_match, lead) => `${lead}${decl}`)
2911
- : styleValue.length === 0 || styleValue.endsWith(";")
2912
- ? `${styleValue}${decl}`
2913
- : `${styleValue};${decl}`;
2914
- if (nextStyle !== styleValue)
2915
- el.setAttribute("style", nextStyle);
2916
- };
2917
- const clips = [];
2918
- for (const el of Array.from(document.querySelectorAll("[data-start]"))) {
2919
- if (el.hasAttribute("data-composition-id"))
2920
- continue;
2921
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
2922
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
2923
- if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0)
2924
- continue;
2925
- const priorHold = Number.parseFloat(el.getAttribute("data-transition-hold") || "");
2926
- const hold = Number.isFinite(priorHold) && priorHold > 0 ? Math.min(priorHold, duration) : 0;
2927
- clips.push({
2928
- el,
2929
- track: Number.parseInt(el.getAttribute("data-track-index") || "0", 10) || 0,
2930
- start,
2931
- nominal: duration - hold,
2932
- hold: 0
2933
- });
2934
- }
2935
- if (!clips.length)
2936
- return;
2937
- for (const clip of clips) {
2938
- const transitionEl = clip.el;
2939
- const tagName = transitionEl.tagName?.toLowerCase?.() ?? "";
2940
- // Exit transitions: validate/clamp here; the tail-positioning delay var is
2941
- // re-derived in the final loop below once holds are known (the animation
2942
- // must end at the clip's rendered end, nominal + hold).
2943
- const outPreset = transitionEl.getAttribute("data-transition-out");
2944
- if (outPreset) {
2945
- if (!TRANSITION_OUT_PRESETS.includes(outPreset) || tagName === "audio") {
2946
- transitionEl.removeAttribute("data-transition-out");
2947
- transitionEl.removeAttribute("data-transition-out-duration");
2948
- }
2949
- else {
2950
- const outDuration = Math.min(clampTransitionDuration(transitionEl.getAttribute("data-transition-out-duration")), clip.nominal);
2951
- const outDurationValue = Number(outDuration.toFixed(3));
2952
- transitionEl.setAttribute("data-transition-out-duration", String(outDurationValue));
2953
- upsertStyleVar(transitionEl, "--vf-tr-out-dur", `${outDurationValue}s`);
2954
- }
2955
- }
2956
- const preset = transitionEl.getAttribute("data-transition");
2957
- if (!preset)
2958
- continue;
2959
- if (!TRANSITION_PRESETS.includes(preset)) {
2960
- // Unknown preset would render as a permanently-paused animation with no
2961
- // keyframes — harmless — but drop it so agents get clean state back.
2962
- transitionEl.removeAttribute("data-transition");
2963
- transitionEl.removeAttribute("data-transition-duration");
2964
- continue;
2965
- }
2966
- if (tagName === "audio") {
2967
- transitionEl.removeAttribute("data-transition");
2968
- transitionEl.removeAttribute("data-transition-duration");
2969
- continue;
2970
- }
2971
- // The animation cannot outlast its own clip.
2972
- const duration = Math.min(clampTransitionDuration(transitionEl.getAttribute("data-transition-duration")), clip.nominal);
2973
- const durationValue = Number(duration.toFixed(3));
2974
- transitionEl.setAttribute("data-transition-duration", String(durationValue));
2975
- // Keep the inline var (which drives animation-duration) in lockstep, same
2976
- // as the --vf-kb-dur re-derivation above.
2977
- upsertStyleVar(transitionEl, "--vf-tr-dur", `${durationValue}s`);
2978
- // Grant the hold to the clip this one cuts away from: same track, nominal
2979
- // end at (within epsilon of) this clip's start.
2980
- for (const candidate of clips) {
2981
- if (candidate === clip || candidate.track !== clip.track)
2982
- continue;
2983
- if (Math.abs(candidate.start + candidate.nominal - clip.start) > TRANSITION_ADJACENCY_EPSILON)
2984
- continue;
2985
- const candidateTag = candidate.el.tagName?.toLowerCase?.() ?? "";
2986
- if (candidateTag === "audio")
2987
- continue;
2988
- candidate.hold = Math.max(candidate.hold, duration);
2989
- }
2990
- }
2991
- for (const clip of clips) {
2992
- const nextDuration = Number((clip.nominal + clip.hold).toFixed(3));
2993
- const currentDuration = Number.parseFloat(clip.el.getAttribute("data-duration") || "");
2994
- if (currentDuration !== nextDuration) {
2995
- clip.el.setAttribute("data-duration", String(nextDuration));
2996
- }
2997
- if (clip.el.hasAttribute("data-end")) {
2998
- clip.el.setAttribute("data-end", String(Number((clip.start + nextDuration).toFixed(3))));
2999
- }
3000
- if (clip.hold > 0) {
3001
- clip.el.setAttribute("data-transition-hold", String(Number(clip.hold.toFixed(3))));
3002
- }
3003
- else {
3004
- clip.el.removeAttribute("data-transition-hold");
3005
- }
3006
- // Position the exit animation at the clip's rendered tail. Timeline edits
3007
- // touch data-duration only, so this re-derivation (like --vf-kb-dur) is
3008
- // what keeps stored delays honest after retimes.
3009
- if (clip.el.getAttribute("data-transition-out")) {
3010
- const outDuration = clampTransitionDuration(clip.el.getAttribute("data-transition-out-duration"));
3011
- upsertStyleVar(clip.el, "--vf-tr-out-delay", `${transitionOutDelaySeconds(nextDuration, outDuration)}s`);
3012
- }
3013
- }
3014
- }
3015
- export function normalizePublishHtml(html) {
3016
- const { document } = parseHTML(html);
3017
- // Coverage gaps: any interval of the timeline with no active canvas-filling
3018
- // layer renders as the page background (#050604 — near-black), which the user
3019
- // experiences as the video "going black" mid-composition (e.g. a scene that
3020
- // was shortened/deleted or a replacement clip shorter than its slot). The
3021
- // full-length backing source video (data-vf-playback-source) is the natural
3022
- // fallback — every scene proxy is cut from that same source — but it is
3023
- // normally dropped at publish. When a gap exists we instead retain it as a
3024
- // timed bottom base layer so uncovered intervals show the original footage
3025
- // rather than black; with no gap we drop it (avoids decoding a redundant
3026
- // full-length video on every render).
3027
- const compositionRoot = document.querySelector("[data-composition-id]");
3028
- const totalDuration = compositionRoot
3029
- ? Number.parseFloat(compositionRoot.getAttribute("data-duration") || "")
3030
- : NaN;
3031
- const coverageIntervals = [];
3032
- for (const el of Array.from(document.querySelectorAll('[data-vf-timeline-proxy="video"], video[data-start], img[data-start]'))) {
3033
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
3034
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
3035
- if (Number.isFinite(start) && Number.isFinite(duration) && duration > 0) {
3036
- coverageIntervals.push([start, start + duration]);
3037
- }
3038
- }
3039
- const retainBackingVideo = Number.isFinite(totalDuration) &&
3040
- totalDuration > 0 &&
3041
- compositionHasCoverageGap(coverageIntervals, totalDuration);
3042
- for (const backingVideo of Array.from(document.querySelectorAll("[data-vf-playback-source]"))) {
3043
- if (!retainBackingVideo) {
3044
- backingVideo.remove();
3045
- continue;
3046
- }
3047
- // Retime the backing video into a full-timeline base layer the render engine
3048
- // seeks per frame (video[data-start]), pinned below every proxy (z-index:-1).
3049
- backingVideo.removeAttribute("data-vf-playback-source");
3050
- backingVideo.setAttribute("data-start", "0");
3051
- backingVideo.setAttribute("data-duration", String(totalDuration));
3052
- backingVideo.setAttribute("data-media-start", "0");
3053
- backingVideo.setAttribute("muted", "");
3054
- backingVideo.setAttribute("playsinline", "");
3055
- backingVideo.setAttribute("preload", "auto");
3056
- backingVideo.setAttribute("style", [
3057
- "position:absolute",
3058
- "left:0%",
3059
- "top:0%",
3060
- "width:100%",
3061
- "height:100%",
3062
- "z-index:-1",
3063
- "object-fit:cover",
3064
- "background:#050604"
3065
- ].join(";"));
3066
- }
3067
- for (const proxy of Array.from(document.querySelectorAll('[data-vf-timeline-proxy="video"]'))) {
3068
- const video = document.createElement("video");
3069
- const source = proxy.getAttribute("data-src") || proxy.getAttribute("src") || "";
3070
- for (const attr of Array.from(proxy.attributes)) {
3071
- if (attr.name === "data-vf-timeline-proxy" || attr.name === "data-src")
3072
- continue;
3073
- video.setAttribute(attr.name, attr.value);
3074
- }
3075
- video.setAttribute("src", source);
3076
- video.setAttribute("muted", "");
3077
- video.setAttribute("playsinline", "");
3078
- video.setAttribute("preload", "auto");
3079
- const trackIndex = Number.parseInt(proxy.getAttribute("data-track-index") ?? "0", 10) || 0;
3080
- video.setAttribute("style", [
3081
- "position:absolute",
3082
- "left:0%",
3083
- "top:0%",
3084
- "width:100%",
3085
- "height:100%",
3086
- `z-index:${trackIndex}`,
3087
- "object-fit:cover",
3088
- "background:#050604"
3089
- ].join(";"));
3090
- proxy.replaceWith(video);
3091
- }
3092
- // Enforce z-index = data-track-index for every timed layer so visual stacking
3093
- // strictly follows the timeline layer index. No arbitrary inline z-index survives.
3094
- for (const el of Array.from(document.querySelectorAll("[data-track-index]"))) {
3095
- const trackAttr = el.getAttribute("data-track-index");
3096
- if (!trackAttr)
3097
- continue;
3098
- const trackIndex = Number.parseInt(trackAttr, 10);
3099
- if (!Number.isFinite(trackIndex))
3100
- continue;
3101
- const html = el;
3102
- if (typeof html.style === "undefined")
3103
- continue;
3104
- html.style.zIndex = String(trackIndex);
3105
- }
3106
- // HyperFrames contract: visible timed elements (div/img) carry class="clip" —
3107
- // the upstream runtime keeps clip-less timed elements visible for the whole
3108
- // composition. video/audio deliberately stay bare (framework-managed), so the
3109
- // class is STRIPPED there (the editor's add-layer paths set it universally).
3110
- // Only promote layers with a valid timing window: a clip-less element with a
3111
- // missing/invalid data-duration historically stayed visible full-length, and
3112
- // promoting it would silently hide it on re-renders of older compositions.
3113
- for (const el of Array.from(document.querySelectorAll("[data-start]"))) {
3114
- if (el.hasAttribute("data-composition-id"))
3115
- continue;
3116
- const tag = el.tagName.toLowerCase();
3117
- if (tag === "video" || tag === "audio") {
3118
- el.classList.remove("clip");
3119
- if (!el.getAttribute("class"))
3120
- el.removeAttribute("class");
3121
- continue;
3122
- }
3123
- const start = Number.parseFloat(el.getAttribute("data-start") || "");
3124
- const clipDuration = Number.parseFloat(el.getAttribute("data-duration") || "");
3125
- if (!Number.isFinite(start) || start < 0 || !Number.isFinite(clipDuration) || clipDuration <= 0)
3126
- continue;
3127
- if (!el.classList.contains("clip")) {
3128
- el.classList.add("clip");
3129
- }
3130
- }
3131
- // Ken Burns motion must span exactly the clip: timeline edits touch
3132
- // data-duration only, so re-derive the inline --vf-kb-dur (which drives
3133
- // animation-duration) from data-duration on every [data-kenburns] layer.
3134
- // Also make sure the shared keyframes stylesheet travels with compositions
3135
- // stored before the builder started embedding it.
3136
- const kenBurnsLayers = Array.from(document.querySelectorAll("[data-kenburns]"));
3137
- for (const el of kenBurnsLayers) {
3138
- const duration = Number.parseFloat(el.getAttribute("data-duration") || "");
3139
- if (!Number.isFinite(duration) || duration <= 0)
3140
- continue;
3141
- const styleValue = el.getAttribute("style") || "";
3142
- const durDecl = `--vf-kb-dur:${Number(duration.toFixed(3))}s`;
3143
- const nextStyle = /(^|;)\s*--vf-kb-dur\s*:[^;]*/i.test(styleValue)
3144
- ? styleValue.replace(/(^|;)\s*--vf-kb-dur\s*:[^;]*/i, (_match, lead) => `${lead}${durDecl}`)
3145
- : styleValue.length === 0 || styleValue.endsWith(";")
3146
- ? `${styleValue}${durDecl}`
3147
- : `${styleValue};${durDecl}`;
3148
- if (nextStyle !== styleValue)
3149
- el.setAttribute("style", nextStyle);
3150
- }
3151
- if (kenBurnsLayers.length > 0 && !html.includes(KEN_BURNS_STYLE_MARKER)) {
3152
- const styleEl = document.createElement("style");
3153
- styleEl.textContent = KEN_BURNS_CSS;
3154
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3155
- }
3156
- // Animated captions: word windows are stored relative to the layer start
3157
- // (inline animation-delay/duration on each [data-cap-word]), so retimes need
3158
- // no re-derivation — but the shared vf-cap-* keyframes stylesheet must
3159
- // travel with compositions stored before the builder embedded it, or every
3160
- // word renders in its resting state.
3161
- if (document.querySelector("[data-caption-animation]") && !html.includes(CAPTION_STYLE_MARKER)) {
3162
- const styleEl = document.createElement("style");
3163
- styleEl.textContent = CAPTION_ANIM_CSS;
3164
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3165
- }
3166
- materializeTransitionOverlaps(document);
3167
- if (document.querySelector("[data-transition], [data-transition-out]") && !html.includes(TRANSITION_STYLE_MARKER)) {
3168
- // Compositions stored before V2 carry the V1 block inline: upgrade it in
3169
- // place (exact string swap) instead of stacking a second stylesheet. If the
3170
- // legacy block drifted, append after it — later source order wins the
3171
- // equal-specificity animation-name battle, so V2 still takes effect.
3172
- let upgradedLegacy = false;
3173
- if (html.includes("__VF_TRANSITIONS_V1__")) {
3174
- for (const styleEl of Array.from(document.querySelectorAll("style"))) {
3175
- const text = styleEl.textContent || "";
3176
- if (!text.includes("__VF_TRANSITIONS_V1__"))
3177
- continue;
3178
- styleEl.textContent = text.includes(LEGACY_TRANSITION_CSS_V1)
3179
- ? text.split(LEGACY_TRANSITION_CSS_V1).join(TRANSITION_CSS)
3180
- : `${text}\n${TRANSITION_CSS}`;
3181
- upgradedLegacy = true;
3182
- }
3183
- }
3184
- if (!upgradedLegacy) {
3185
- const styleEl = document.createElement("style");
3186
- styleEl.textContent = TRANSITION_CSS;
3187
- (document.head ?? document.documentElement)?.appendChild(styleEl);
3188
- }
3189
- }
3190
- // Editor sessions occasionally leave inline style attributes with double
3191
- // quotes inside CSS values (font-family: "TikTok Sans"). When serialized into
3192
- // an HTML attribute, those quotes become &quot; entities. The render lambda
3193
- // resolves font-family with a regex over raw HTML and misreads &quot; as a
3194
- // literal font name, exploding with FontFetchError. Normalize to single
3195
- // quotes so serialization stays entity-free.
3196
- for (const el of Array.from(document.querySelectorAll("[style]"))) {
3197
- const styleValue = el.getAttribute("style");
3198
- if (!styleValue || styleValue.indexOf('"') === -1)
3199
- continue;
3200
- el.setAttribute("style", styleValue.replace(/"/g, "'"));
3201
- }
3202
- return `<!doctype html>\n${document.documentElement.outerHTML}`;
3203
- }
3204
- /**
3205
- * Reconstruct a Step Functions execution ARN from the render state machine ARN
3206
- * plus a deterministic execution name, so a re-invoked job runner can poll the
3207
- * SAME render without persisting the ARN handed back at dispatch time.
3208
- * arn:...:stateMachine:NAME -> arn:...:execution:NAME:EXEC_NAME
3209
- */
3210
- function executionArnFromName(stateMachineArn, executionName) {
3211
- const marker = ":stateMachine:";
3212
- const idx = stateMachineArn.indexOf(marker);
3213
- if (idx === -1) {
3214
- throw new Error(`Unexpected HyperFrames render state machine ARN: ${stateMachineArn}`);
3215
- }
3216
- const prefix = stateMachineArn.slice(0, idx);
3217
- const machineName = stateMachineArn.slice(idx + marker.length);
3218
- return `${prefix}:execution:${machineName}:${executionName}`;
3219
- }
3220
- /**
3221
- * DescribeExecution for a render execution. Returns the Step Functions status,
3222
- * or `null` when the execution does not exist yet (never dispatched) — the
3223
- * signal renderStep() uses to decide between dispatch and resume.
3224
- */
3225
- async function describeRenderExecutionStatus(executionArn, region) {
3226
- const client = new SFNClient({ region });
3227
- try {
3228
- const response = await client.send(new DescribeExecutionCommand({ executionArn }));
3229
- return response.status ?? "RUNNING";
3230
- }
3231
- catch (error) {
3232
- if (error instanceof ExecutionDoesNotExist) {
3233
- return null;
3234
- }
3235
- const name = error instanceof Error ? error.name : "";
3236
- if (name === "ExecutionDoesNotExist") {
3237
- return null;
3238
- }
3239
- throw error;
3240
- }
3241
- }
3242
- async function resolveHyperframesStack() {
3243
- const region = config.HYPERFRAMES_RENDER_REGION || config.AWS_REGION || "us-east-1";
3244
- const lambdaMemoryMb = positiveInteger(config.HYPERFRAMES_RENDER_MEMORY_MB, 3072);
3245
- const envStack = {
3246
- stackName: config.HYPERFRAMES_LAMBDA_STACK_NAME || "hyperframes-vidfarm-prod",
3247
- bucketName: config.HYPERFRAMES_RENDER_BUCKET_NAME || config.HYPERFRAMES_LAMBDA_BUCKET_NAME || "",
3248
- stateMachineArn: config.HYPERFRAMES_RENDER_STATE_MACHINE_ARN || config.HYPERFRAMES_STATE_MACHINE_ARN || "",
3249
- region,
3250
- lambdaMemoryMb
3251
- };
3252
- if (envStack.bucketName && envStack.stateMachineArn) {
3253
- return envStack;
3254
- }
3255
- if (stackCache && stackCache.expiresAt > Date.now()) {
3256
- return stackCache.value;
3257
- }
3258
- const stackName = envStack.stackName;
3259
- const cloudFormation = new CloudFormationClient({ region });
3260
- const result = await cloudFormation.send(new DescribeStacksCommand({ StackName: stackName }));
3261
- const stack = result.Stacks?.[0];
3262
- if (!stack) {
3263
- throw new Error(`HyperFrames Lambda stack not found: ${stackName}`);
3264
- }
3265
- const outputs = stack.Outputs || [];
3266
- const value = {
3267
- stackName,
3268
- bucketName: stackOutput(outputs, "RenderBucketName"),
3269
- stateMachineArn: stackOutput(outputs, "RenderStateMachineArn"),
3270
- functionArn: stackOutput(outputs, "RenderFunctionArn"),
3271
- region: stackOutput(outputs, "RenderRegion") || region,
3272
- lambdaMemoryMb: positiveInteger(stackOutput(outputs, "RenderMemoryMb"), lambdaMemoryMb)
3273
- };
3274
- if (!value.bucketName || !value.stateMachineArn) {
3275
- throw new Error(`HyperFrames Lambda stack ${stackName} is missing render outputs.`);
3276
- }
3277
- stackCache = { value, expiresAt: Date.now() + 60_000 };
3278
- return value;
3279
- }
3280
- async function stageProject(input) {
3281
- const root = path.join(config.VIDFARM_DATA_DIR, "vidfarm-projects");
3282
- const projectDir = path.join(root, `${Date.now().toString(36)}-${safeIdentifier(input.compositionId)}-${randomUUID().slice(0, 8)}`);
3283
- await rm(projectDir, { recursive: true, force: true });
3284
- await mkdir(projectDir, { recursive: true });
3285
- await writeFile(path.join(projectDir, "index.html"), input.compositionHtml, "utf8");
3286
- for (const file of input.projectFiles || []) {
3287
- const target = safeProjectPath(projectDir, file.path);
3288
- await mkdir(path.dirname(target), { recursive: true });
3289
- await writeFile(target, file.contents);
3290
- }
3291
- for (const copyPath of input.copyPaths || []) {
3292
- const target = safeProjectPath(projectDir, copyPath.to);
3293
- await mkdir(path.dirname(target), { recursive: true });
3294
- await cp(copyPath.from, target, { recursive: true, force: true });
3295
- }
3296
- return projectDir;
3297
- }
3298
- function extractCompositionMetadata(html, input) {
3299
- const compositionId = safeIdentifier(input.compositionId || htmlAttribute(html, "data-composition-id") || input.composition?.compositionId || "composition");
3300
- const width = positiveInteger(input.width || htmlAttribute(html, "data-width") || input.composition?.width, 720);
3301
- const height = positiveInteger(input.height || htmlAttribute(html, "data-height") || input.composition?.height, 1280);
3302
- const fps = positiveInteger(input.fps || rootTagAttribute(html, "data-fps") || 30, 30);
3303
- const duration = Number.parseFloat(htmlAttribute(html, "data-duration") || String(input.composition?.duration || ""));
3304
- return {
3305
- compositionId,
3306
- title: input.title || htmlAttribute(html, "data-source-video") || compositionId,
3307
- width,
3308
- height,
3309
- fps,
3310
- duration: Number.isFinite(duration) && duration > 0 ? duration : null
3311
- };
3312
- }
3313
- function safeProjectPath(root, relativePath) {
3314
- const target = path.resolve(root, relativePath);
3315
- const resolvedRoot = path.resolve(root);
3316
- if (target !== resolvedRoot && !target.startsWith(`${resolvedRoot}${path.sep}`)) {
3317
- throw new Error(`Invalid HyperFrames project path: ${relativePath}`);
3318
- }
3319
- return target;
3320
- }
3321
- // Round the composition's data-width / data-height down to the nearest even
3322
- // integer so libx264 (yuv420p) accepts the frame size. Only rewrites values
3323
- // that are actually odd, leaving well-formed even compositions untouched.
3324
- function forceEvenCompositionDimensions(html) {
3325
- return html.replace(/\bdata-(width|height)=(["'])(\d+)\2/gi, (match, dim, quote, value) => {
3326
- const n = Number.parseInt(value, 10);
3327
- if (!Number.isFinite(n) || n % 2 === 0)
3328
- return match;
3329
- return `data-${dim}=${quote}${n - 1}${quote}`;
3330
- });
3331
- }
3332
- // Reads an attribute from the SAME opening tag that carries
3333
- // data-composition-id (the composition root). htmlAttribute matches the first
3334
- // occurrence anywhere in the document, which nested sub-compositions or
3335
- // embedded snippets could hijack — fps must come from the root only.
3336
- function rootTagAttribute(html, name) {
3337
- const rootTag = html.match(/<[a-zA-Z][^>]*\bdata-composition-id\s*=[^>]*>/);
3338
- if (!rootTag) {
3339
- return "";
3340
- }
3341
- const value = rootTag[0].match(new RegExp(`\\b${name}\\s*=\\s*["']([^"']*)["']`, "i"));
3342
- return value ? value[1] : "";
3343
- }
3344
- function htmlAttribute(html, name) {
3345
- const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3346
- const match = html.match(new RegExp(`${escaped}=(["'])(.*?)\\1`, "i"));
3347
- return match?.[2] || "";
3348
- }
3349
- function stackOutput(outputs, key) {
3350
- return outputs.find((output) => output.OutputKey === key)?.OutputValue
3351
- || outputs.find((output) => String(output.OutputKey || "").endsWith(key))?.OutputValue
3352
- || "";
3353
- }
3354
- function parseS3Uri(uri) {
3355
- const match = String(uri || "").match(/^s3:\/\/([^/]+)\/(.+)$/);
3356
- return match ? { bucket: match[1], key: match[2] } : null;
3357
- }
3358
- async function signedS3Url(s3Uri, region) {
3359
- const parsed = parseS3Uri(s3Uri);
3360
- if (!parsed)
3361
- return null;
3362
- const client = new S3Client({ region });
3363
- return getSignedUrl(client, new GetObjectCommand({ Bucket: parsed.bucket, Key: parsed.key }), { expiresIn: 3600 });
3364
- }
3365
- function safeIdentifier(value) {
3366
- return String(value || "composition")
3367
- .toLowerCase()
3368
- .replace(/[^a-z0-9_-]+/g, "-")
3369
- .replace(/^-+|-+$/g, "")
3370
- .slice(0, 64) || "composition";
3371
- }
3372
- function positiveInteger(value, fallback) {
3373
- const next = Math.round(Number(value));
3374
- return Number.isFinite(next) && next > 0 ? next : fallback;
3375
- }
3376
- function normalizeRenderFps(value) {
3377
- if (value <= 24)
3378
- return 24;
3379
- if (value > 45)
3380
- return 60;
3381
- return 30;
3382
- }
3383
- function buildCastAnalysisPrompt(input) {
3384
- const summaryBlock = input.summary && input.summary.trim()
3385
- ? `\nVIDEO SUMMARY (from the first decompose pass):\n"""\n${input.summary.trim()}\n"""\n`
3386
- : "";
3387
- const sceneBlock = input.scenes && input.scenes.length > 0
3388
- ? `\nSCENES (slug — description, from the first pass; use these exact slugs in "appears_in"):\n${input.scenes
3389
- .map((s) => ` ${s.slug}: ${(s.description ?? "").slice(0, 240)}`)
3390
- .join("\n")}\n`
3391
- : "";
3392
- const frameBlock = input.frameTimestamps.length > 0
3393
- ? `\nFRAMES SUPPLIED (in order, each captioned with its capture timestamp): ${input.frameTimestamps
3394
- .map((t) => `${t.toFixed(2)}s`)
3395
- .join(", ")}\nPick "best_frame_seconds" from one of these exact timestamps — the frame where that person is largest, most frontal, and least occluded (ideally alone).\n`
3396
- : "";
3397
- return `You are identifying the recurring CAST (the people/characters) of a short-form vertical video so they can be reused and regenerated consistently in later edits.
3398
- ${summaryBlock}${sceneBlock}
3399
- Video metadata:
3400
- - duration: ${input.durationSeconds.toFixed(3)} seconds
3401
- - dimensions: ${input.width} x ${input.height} (canvas is 100% x 100%)
3402
- ${frameBlock}
3403
- Identify each DISTINCT person who appears. Merge the same person across scenes into ONE entry — do not create a new entry per scene. Ignore incidental crowd/background extras who never recur unless they are visually significant. If the video has no people (product-only, screen-recording, etc.), return an empty "cast" array.
3404
-
3405
- For each person, describe them in enough concrete detail (face, hair, skin tone, build, age range, distinguishing features, wardrobe) that an image model could regenerate the SAME person from the description alone. Give a stable snake_case "slug" (e.g. "hero_girl", "male_narrator"), a short "name"/label, a "role" of "main" | "secondary" | "background", the list of scene slugs they "appears_in", the single "best_frame_seconds" timestamp where they're cleanest, a normalized bounding box "bbox" {x,y,width,height} in 0..1 coordinates locating that person within that frame (null if you cannot localize), and "is_primary": true for the single most central recurring subject (false for the rest).
3406
-
3407
- Return strictly valid JSON:
3408
- {
3409
- "cast": [{
3410
- "slug": string,
3411
- "name": string,
3412
- "role": "main" | "secondary" | "background",
3413
- "description": string,
3414
- "wardrobe": string,
3415
- "appears_in": [string, ...],
3416
- "best_frame_seconds": number,
3417
- "bbox": { "x": number, "y": number, "width": number, "height": number } | null,
3418
- "is_primary": boolean
3419
- }]
3420
- }`;
3421
- }
3422
- function clamp01(value) {
3423
- const n = Number(value);
3424
- if (!Number.isFinite(n))
3425
- return 0;
3426
- return Math.min(1, Math.max(0, n));
3427
- }
3428
- function normalizeCastMembers(raw, durationSeconds) {
3429
- const root = raw && typeof raw === "object" ? raw : {};
3430
- const list = Array.isArray(root.cast)
3431
- ? root.cast
3432
- : Array.isArray(root.members)
3433
- ? root.members
3434
- : [];
3435
- const usedSlugs = new Set();
3436
- const members = [];
3437
- for (const entry of list) {
3438
- if (!entry || typeof entry !== "object")
3439
- continue;
3440
- const e = entry;
3441
- const description = typeof e.description === "string" ? e.description.trim() : "";
3442
- if (!description)
3443
- continue;
3444
- let slug = safeIdentifier(typeof e.slug === "string" && e.slug.trim() ? e.slug : (typeof e.name === "string" ? e.name : `cast-${members.length + 1}`));
3445
- while (usedSlugs.has(slug))
3446
- slug = `${slug}-${members.length + 1}`;
3447
- usedSlugs.add(slug);
3448
- const bboxRaw = e.bbox && typeof e.bbox === "object" ? e.bbox : null;
3449
- const bbox = bboxRaw
3450
- ? {
3451
- x: clamp01(bboxRaw.x),
3452
- y: clamp01(bboxRaw.y),
3453
- width: clamp01(bboxRaw.width ?? bboxRaw.w),
3454
- height: clamp01(bboxRaw.height ?? bboxRaw.h)
3455
- }
3456
- : null;
3457
- const best = Number(e.best_frame_seconds ?? e.best_frame ?? 0);
3458
- members.push({
3459
- slug,
3460
- name: typeof e.name === "string" && e.name.trim() ? e.name.trim().slice(0, 120) : slug,
3461
- role: typeof e.role === "string" && e.role.trim() ? e.role.trim().toLowerCase().slice(0, 40) : "secondary",
3462
- description: description.slice(0, 2000),
3463
- wardrobe: typeof e.wardrobe === "string" ? e.wardrobe.trim().slice(0, 600) : "",
3464
- appears_in: Array.isArray(e.appears_in)
3465
- ? e.appears_in.map((s) => (typeof s === "string" ? s.trim() : "")).filter(Boolean).slice(0, 40)
3466
- : [],
3467
- best_frame_seconds: Number.isFinite(best) && best > 0
3468
- ? Math.min(Math.max(0, best), Math.max(0, durationSeconds - 0.05))
3469
- : 0,
3470
- bbox: bbox && (bbox.width > 0 || bbox.height > 0) ? bbox : null,
3471
- is_primary: e.is_primary === true
3472
- });
3473
- }
3474
- // Guarantee exactly one primary when there's at least one member.
3475
- if (members.length > 0 && !members.some((m) => m.is_primary)) {
3476
- members[0].is_primary = true;
3477
- }
3478
- return members;
3479
- }
3480
- export async function analyzeCastMembers(input) {
3481
- const tempDir = await mkdtemp(path.join(os.tmpdir(), SMART_DECOMPOSE_TEMP_PREFIX));
3482
- const sourcePath = path.join(tempDir, "source.mp4");
3483
- try {
3484
- await downloadVideoTo(input.sourceUrl, sourcePath);
3485
- const meta = await ffprobeVideo(sourcePath);
3486
- if (!Number.isFinite(meta.durationSeconds) || meta.durationSeconds <= 0) {
3487
- throw new Error("Could not read video duration for cast analysis");
3488
- }
3489
- const frames = await sampleFramesAsBase64(sourcePath, meta.durationSeconds, tempDir);
3490
- const prompt = buildCastAnalysisPrompt({
3491
- durationSeconds: meta.durationSeconds,
3492
- width: meta.width,
3493
- height: meta.height,
3494
- summary: input.summary ?? null,
3495
- scenes: input.scenes ?? [],
3496
- frameTimestamps: frames.map((f) => f.timestamp)
3497
- });
3498
- const attempt = await callVisionWithFallback({ prompt, frames, keys: input.keys });
3499
- devLog("hyperframes.cast_analyze.done", {
3500
- source_url: input.sourceUrl,
3501
- provider: attempt.provider,
3502
- model: attempt.model
3503
- });
3504
- return {
3505
- members: normalizeCastMembers(attempt.result, meta.durationSeconds),
3506
- provider: attempt.provider,
3507
- model: attempt.model,
3508
- durationSeconds: meta.durationSeconds,
3509
- width: meta.width,
3510
- height: meta.height
3511
- };
3512
- }
3513
- finally {
3514
- await rm(tempDir, { recursive: true, force: true }).catch(() => { });
3515
- }
3516
- }
3517
- //# sourceMappingURL=hyperframes.js.map