@mevdragon/vidfarm-devcli 0.20.6 → 0.20.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/editor-capabilities/SKILL.md +15 -1
- package/.agents/skills/vidfarm-director/SKILL.md +106 -0
- package/.agents/skills/vidfarm-director/recipes/find-and-fork-template.md +16 -0
- package/.agents/skills/vidfarm-director/recipes/local-edit-render-approve.md +13 -0
- package/.agents/skills/vidfarm-director/recipes/onboard-a-new-director.md +13 -0
- package/.agents/skills/vidfarm-director/recipes/retheme-template.md +17 -0
- package/.agents/skills/vidfarm-director/references/assets-and-sourcing.md +111 -0
- package/.agents/skills/vidfarm-director/references/automation-and-local-dev.md +243 -0
- package/.agents/skills/vidfarm-director/references/core-workflows.md +273 -0
- package/.agents/skills/vidfarm-director/references/editor-workflows.md +365 -0
- package/.agents/skills/vidfarm-director/references/onboarding.md +28 -0
- package/.agents/skills/vidfarm-director/references/primitives.md +265 -0
- package/SKILL.director.md +476 -260
- package/dist/src/cli.js +362 -96
- package/dist/src/devcli/clips.js +15 -10
- package/dist/src/devcli/doctor.js +2 -2
- package/dist/src/devcli/local-backend.js +46 -2
- package/dist/src/services/clip-curation/cost.js +5 -1
- package/dist/src/services/clip-curation/gemini.js +10 -0
- package/package.json +30 -10
- package/SKILL.platform.md +0 -432
- package/demo/README.md +0 -28
- package/demo/dist/app.css +0 -1
- package/demo/dist/app.js +0 -1850
- package/demo/dist/chunks/chunk-DXB73IDG.js +0 -1
- package/demo/dist/chunks/chunk-S7OWAJDS.js +0 -36
- package/demo/dist/chunks/chunk-VTIBZ6AN.js +0 -1
- package/demo/dist/chunks/dist-ADSJKBVE.js +0 -332
- package/demo/dist/chunks/domEditingLayers-VZMLL4AP-SGHWPND4.js +0 -1
- package/demo/dist/chunks/hyperframes-player-XB65TCD6.js +0 -425
- package/demo/dist/chunks/lib-XAQ37YOE.js +0 -1
- package/demo/dist/chunks/src-TJ2QYA4U.js +0 -207
- package/demo/dist/favicon.ico +0 -0
- package/demo/dist/icons/timeline/audio.svg +0 -7
- package/demo/dist/icons/timeline/captions.svg +0 -5
- package/demo/dist/icons/timeline/composition.svg +0 -12
- package/demo/dist/icons/timeline/image.svg +0 -18
- package/demo/dist/icons/timeline/music.svg +0 -10
- package/demo/dist/icons/timeline/text.svg +0 -3
- package/demo/dist/index.html +0 -15
- package/dist/src/account-pages-legacy.js +0 -9396
- package/dist/src/account-pages.js +0 -61
- package/dist/src/app.js +0 -21603
- package/dist/src/composition-runtime.js +0 -1053
- package/dist/src/config.js +0 -217
- package/dist/src/context.js +0 -447
- package/dist/src/dev-app-legacy.js +0 -739
- package/dist/src/dev-app.js +0 -6
- package/dist/src/devcli/migrate-local.js +0 -140
- package/dist/src/devcli/sync.js +0 -368
- package/dist/src/domain.js +0 -5
- package/dist/src/editor-chat-history.js +0 -82
- package/dist/src/editor-chat.js +0 -828
- package/dist/src/editor-dark-theme.js +0 -1128
- package/dist/src/frontend/debug.js +0 -71
- package/dist/src/frontend/discover-client.js +0 -130
- package/dist/src/frontend/discover-store.js +0 -23
- package/dist/src/frontend/file-directory.js +0 -1018
- package/dist/src/frontend/flockposter-cache-store.js +0 -124
- package/dist/src/frontend/homepage-client.js +0 -446
- package/dist/src/frontend/homepage-shared.js +0 -201
- package/dist/src/frontend/homepage-store.js +0 -66
- package/dist/src/frontend/homepage-view.js +0 -705
- package/dist/src/frontend/page-runtime-client.js +0 -132
- package/dist/src/frontend/page-runtime-store.js +0 -9
- package/dist/src/frontend/sentry.js +0 -42
- package/dist/src/frontend/template-editor-chat.js +0 -4181
- package/dist/src/help-page.js +0 -346
- package/dist/src/homepage.js +0 -1458
- package/dist/src/index.js +0 -16
- package/dist/src/instrument.js +0 -30
- package/dist/src/landing-page.js +0 -384
- package/dist/src/page-runtime.js +0 -2
- package/dist/src/page-shell.js +0 -1452
- package/dist/src/primitive-context.js +0 -416
- package/dist/src/primitive-registry.js +0 -3940
- package/dist/src/primitive-sdk.js +0 -4
- package/dist/src/primitives/hyperframes-media.js +0 -108
- package/dist/src/react-page-shell.js +0 -35
- package/dist/src/ready-post-schedule-component.js +0 -1540
- package/dist/src/registry.js +0 -296
- package/dist/src/reskin/agency-page.js +0 -299
- package/dist/src/reskin/calendar-page.js +0 -568
- package/dist/src/reskin/chat-page.js +0 -942
- package/dist/src/reskin/discover-page.js +0 -1788
- package/dist/src/reskin/document.js +0 -1587
- package/dist/src/reskin/help-page.js +0 -357
- package/dist/src/reskin/index-page.js +0 -62
- package/dist/src/reskin/inpaint-clipper-page.js +0 -890
- package/dist/src/reskin/inpaint-page.js +0 -2554
- package/dist/src/reskin/inpaint-video-page.js +0 -1339
- package/dist/src/reskin/job-runs-page.js +0 -477
- package/dist/src/reskin/library-page.js +0 -1634
- package/dist/src/reskin/login-page.js +0 -262
- package/dist/src/reskin/portfolio-page.js +0 -687
- package/dist/src/reskin/pricing-page.js +0 -390
- package/dist/src/reskin/settings-page.js +0 -732
- package/dist/src/reskin/theme.js +0 -711
- package/dist/src/runtime.js +0 -35
- package/dist/src/services/api-call-history.js +0 -249
- package/dist/src/services/auth.js +0 -152
- package/dist/src/services/billing-pricing.js +0 -39
- package/dist/src/services/billing.js +0 -241
- package/dist/src/services/cast.js +0 -127
- package/dist/src/services/chat-threads.js +0 -92
- package/dist/src/services/clip-records.js +0 -250
- package/dist/src/services/clip-search.js +0 -77
- package/dist/src/services/clip-vectors.js +0 -125
- package/dist/src/services/composition-sanitize.js +0 -124
- package/dist/src/services/composition-watch.js +0 -79
- package/dist/src/services/elevenlabs.js +0 -222
- package/dist/src/services/file-directory.js +0 -117
- package/dist/src/services/fork-access.js +0 -93
- package/dist/src/services/fork-manifest.js +0 -42
- package/dist/src/services/ghostcut.js +0 -179
- package/dist/src/services/hyperframes.js +0 -3654
- package/dist/src/services/job-capacity.js +0 -14
- package/dist/src/services/job-logs.js +0 -197
- package/dist/src/services/jobs.js +0 -136
- package/dist/src/services/local-dynamo.js +0 -0
- package/dist/src/services/media-processing.js +0 -766
- package/dist/src/services/primitive-media-lambda.js +0 -280
- package/dist/src/services/providers.js +0 -2748
- package/dist/src/services/rate-limits.js +0 -262
- package/dist/src/services/scene-annotations.js +0 -32
- package/dist/src/services/serverless-auth.js +0 -382
- package/dist/src/services/serverless-jobs.js +0 -1084
- package/dist/src/services/serverless-provider-keys.js +0 -409
- package/dist/src/services/serverless-records.js +0 -1515
- package/dist/src/services/serverless-template-configs.js +0 -75
- package/dist/src/services/storage.js +0 -461
- package/dist/src/services/swipe-customize.js +0 -437
- package/dist/src/services/template-certification.js +0 -413
- package/dist/src/services/template-loader.js +0 -99
- package/dist/src/services/template-runtime-bundles.js +0 -217
- package/dist/src/services/template-sources.js +0 -1017
- package/dist/src/services/upstream.js +0 -248
- package/dist/src/services/video-normalization.js +0 -2
- package/dist/src/services/webhooks.js +0 -62
- package/dist/src/template-editor-pages.js +0 -2576
- package/dist/src/template-editor-shell.js +0 -2893
- package/dist/src/template-sdk.js +0 -4
- package/dist/src/worker.js +0 -17
- package/public/assets/discover-client-app.js +0 -1
- package/public/assets/file-directory-app.js +0 -3
- package/public/assets/homepage-app.js +0 -54
- package/public/assets/homepage-client-app.js +0 -80
- package/public/assets/page-runtime-client-app.js +0 -94
- package/public/assets/placeholders/scene-placeholder.png +0 -0
- package/src/assets/SELLING_AWARENESS_STAGES.md +0 -579
- package/src/assets/SELLING_WITH_HOOKS.md +0 -377
- package/src/assets/SELLING_WITH_VSLS.md +0 -606
- package/src/assets/favicon.ico +0 -0
- package/src/assets/logo-vidfarm.png +0 -0
|
@@ -1,3940 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
-
import sharp from "sharp";
|
|
5
|
-
import { z } from "zod";
|
|
6
|
-
import { config } from "./config.js";
|
|
7
|
-
import { definePrimitive } from "./primitive-sdk.js";
|
|
8
|
-
import { PrimitiveMediaLambdaService } from "./services/primitive-media-lambda.js";
|
|
9
|
-
import { estimateGhostcutCostUsd, isGhostcutConfigured, pollStatus as pollGhostcutStatus, submitSubtitleRemoval } from "./services/ghostcut.js";
|
|
10
|
-
import { buildSrtFromSegments, defaultSpeechModelFor, GEMINI_TTS_VOICES, inferSpeechProviderForVoice, OPENAI_TTS_VOICES, speechVoiceFamilyFor, speechVoiceMismatch } from "./services/speech.js";
|
|
11
|
-
import { buildCaptionCues, cuesFromText } from "./services/captions.js";
|
|
12
|
-
import { CAPTION_ANIMATIONS, CAPTION_STYLE_PRESETS, captionStylePresetById } from "./hyperframes/composition.js";
|
|
13
|
-
import { buildHtmlImageCompositionHtml, buildMediaDedupeCompositionHtml, DEDUPE_DEFAULT_EFFECTS } from "./primitives/hyperframes-media.js";
|
|
14
|
-
const imageProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
|
|
15
|
-
const videoProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
|
|
16
|
-
const textProviderSchema = z.enum(["openai", "gemini", "openrouter", "perplexity"]);
|
|
17
|
-
const outputFormatSchema = z.enum(["png", "jpeg", "webp"]);
|
|
18
|
-
const aspectRatioSchema = z.enum(["1:1", "4:3", "3:4", "16:9", "9:16"]).optional();
|
|
19
|
-
const videoAspectRatioSchema = z.enum(["1:1", "4:3", "3:4", "16:9", "9:16", "21:9", "9:21"]).optional();
|
|
20
|
-
const imageSizeSchema = z.enum(["1K", "2K", "4K"]).optional();
|
|
21
|
-
const brainstormCountSchema = z.number().int().min(1).max(50).default(12);
|
|
22
|
-
const generateImagePayloadSchema = z.object({
|
|
23
|
-
provider: imageProviderSchema.optional(),
|
|
24
|
-
model: z.string().min(1).optional(),
|
|
25
|
-
prompt: z.string().min(3).max(8000),
|
|
26
|
-
prompt_attachments: z.array(z.string().url()).max(16).default([]),
|
|
27
|
-
aspect_ratio: aspectRatioSchema,
|
|
28
|
-
size: z.string().min(1).max(40).optional(),
|
|
29
|
-
image_size: imageSizeSchema,
|
|
30
|
-
output_format: outputFormatSchema.default("png")
|
|
31
|
-
});
|
|
32
|
-
const editImagePayloadSchema = z.object({
|
|
33
|
-
provider: imageProviderSchema.optional(),
|
|
34
|
-
model: z.string().min(1).optional(),
|
|
35
|
-
source_image_url: z.string().url(),
|
|
36
|
-
instruction: z.string().min(3).max(8000),
|
|
37
|
-
reference_attachments: z.array(z.string().url()).max(15).default([]),
|
|
38
|
-
aspect_ratio: aspectRatioSchema,
|
|
39
|
-
size: z.string().min(1).max(40).optional(),
|
|
40
|
-
image_size: imageSizeSchema,
|
|
41
|
-
output_format: outputFormatSchema.default("png")
|
|
42
|
-
});
|
|
43
|
-
const inpaintImagePayloadSchema = z.object({
|
|
44
|
-
provider: imageProviderSchema.optional(),
|
|
45
|
-
model: z.string().min(1).optional(),
|
|
46
|
-
source_image_url: z.string().url(),
|
|
47
|
-
mask_url: z.string().url(),
|
|
48
|
-
instruction: z.string().min(3).max(8000),
|
|
49
|
-
reference_attachments: z.array(z.string().url()).max(15).default([]),
|
|
50
|
-
aspect_ratio: aspectRatioSchema,
|
|
51
|
-
size: z.string().min(1).max(40).optional(),
|
|
52
|
-
image_size: imageSizeSchema,
|
|
53
|
-
output_format: outputFormatSchema.default("png")
|
|
54
|
-
});
|
|
55
|
-
const videoFrameImageSchema = z.preprocess((raw) => {
|
|
56
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
57
|
-
return raw;
|
|
58
|
-
const frame = raw;
|
|
59
|
-
return {
|
|
60
|
-
...frame,
|
|
61
|
-
image_url: frame.image_url ?? frame.imageUrl,
|
|
62
|
-
frame_type: frame.frame_type ?? frame.frameType
|
|
63
|
-
};
|
|
64
|
-
}, z.object({
|
|
65
|
-
image_url: z.string().url(),
|
|
66
|
-
frame_type: z.enum(["first_frame", "last_frame"]).default("first_frame")
|
|
67
|
-
}));
|
|
68
|
-
const generateVideoPayloadSchema = z.preprocess((raw) => {
|
|
69
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
70
|
-
return raw;
|
|
71
|
-
const payload = raw;
|
|
72
|
-
const explicitSeconds = payload.duration ??
|
|
73
|
-
payload.duration_seconds ??
|
|
74
|
-
payload.durationSeconds ??
|
|
75
|
-
payload.seconds ??
|
|
76
|
-
payload.video_length_seconds ??
|
|
77
|
-
payload.videoLengthSeconds;
|
|
78
|
-
const milliseconds = payload.duration_ms ?? payload.durationMs ?? payload.video_length_ms ?? payload.videoLengthMs;
|
|
79
|
-
return {
|
|
80
|
-
...payload,
|
|
81
|
-
duration: normalizeAiVideoDurationSeconds(explicitSeconds, milliseconds)
|
|
82
|
-
};
|
|
83
|
-
}, z.object({
|
|
84
|
-
provider: videoProviderSchema.optional(),
|
|
85
|
-
model: z.string().min(1).optional(),
|
|
86
|
-
prompt: z.string().min(3).max(8000),
|
|
87
|
-
duration: z.number().int().min(1).max(30).optional(),
|
|
88
|
-
resolution: z.enum(["480p", "720p", "1080p", "1K", "2K", "4K"]).optional(),
|
|
89
|
-
aspect_ratio: videoAspectRatioSchema,
|
|
90
|
-
size: z.string().min(1).max(40).optional(),
|
|
91
|
-
generate_audio: z.boolean().optional(),
|
|
92
|
-
frame_images: z.array(videoFrameImageSchema).max(2).default([]),
|
|
93
|
-
input_references: z.array(z.string().url()).max(8).default([]),
|
|
94
|
-
seed: z.number().int().optional(),
|
|
95
|
-
provider_options: z.record(z.string(), z.unknown()).optional(),
|
|
96
|
-
output_format: z.enum(["mp4"]).default("mp4")
|
|
97
|
-
}));
|
|
98
|
-
const removeBackgroundImagePayloadSchema = z.preprocess((raw) => {
|
|
99
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
100
|
-
return raw;
|
|
101
|
-
const payload = raw;
|
|
102
|
-
return {
|
|
103
|
-
...payload,
|
|
104
|
-
source_image_url: payload.source_image_url ??
|
|
105
|
-
payload.image_url ??
|
|
106
|
-
payload.imageUrl ??
|
|
107
|
-
payload.url
|
|
108
|
-
};
|
|
109
|
-
}, z.object({
|
|
110
|
-
source_image_url: z.string().url(),
|
|
111
|
-
output_format: z.enum(["png", "jpeg", "webp"]).default("webp")
|
|
112
|
-
}));
|
|
113
|
-
// Transparent-capable output only (jpeg has no alpha) for the chroma-key routes.
|
|
114
|
-
const alphaOutputFormatSchema = z.enum(["png", "webp"]).default("png");
|
|
115
|
-
const greenscreenRemoveBackgroundPayloadSchema = z.preprocess((raw) => {
|
|
116
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
117
|
-
return raw;
|
|
118
|
-
const p = raw;
|
|
119
|
-
return {
|
|
120
|
-
...p,
|
|
121
|
-
source_image_url: p.source_image_url ?? p.image_url ?? p.imageUrl ?? p.url,
|
|
122
|
-
key_color: p.key_color ?? p.keyColor ?? p.background_color ?? p.backgroundColor ?? p.color,
|
|
123
|
-
tolerance: p.tolerance ?? p.threshold,
|
|
124
|
-
softness: p.softness ?? p.feather,
|
|
125
|
-
despill: p.despill ?? p.spill_suppression ?? p.spillSuppression
|
|
126
|
-
};
|
|
127
|
-
}, z.object({
|
|
128
|
-
source_image_url: z.string().url(),
|
|
129
|
-
// The background color to key out. Green by default; pass white/#FFFFFF,
|
|
130
|
-
// blue, or any hex/rgb/named color to key a different flat background.
|
|
131
|
-
key_color: z.string().min(1).max(64).default("#00FF00"),
|
|
132
|
-
// Base similarity radius (0..1). Larger = more aggressive removal.
|
|
133
|
-
tolerance: z.coerce.number().min(0).max(1).default(0.3),
|
|
134
|
-
// Feather band beyond `tolerance` for soft anti-aliased edges (0..1).
|
|
135
|
-
softness: z.coerce.number().min(0).max(1).default(0.1),
|
|
136
|
-
// Suppress residual key-color fringe on the subject's edges.
|
|
137
|
-
despill: z.boolean().default(true),
|
|
138
|
-
output_format: alphaOutputFormatSchema
|
|
139
|
-
}));
|
|
140
|
-
const mediaOverlayPayloadSchema = z.preprocess((raw) => {
|
|
141
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
142
|
-
return raw;
|
|
143
|
-
const p = raw;
|
|
144
|
-
return {
|
|
145
|
-
...p,
|
|
146
|
-
key_color: p.key_color ?? p.keyColor ?? p.background_color ?? p.backgroundColor,
|
|
147
|
-
tolerance: p.tolerance ?? p.threshold,
|
|
148
|
-
softness: p.softness ?? p.feather,
|
|
149
|
-
despill: p.despill ?? p.spill_suppression ?? p.spillSuppression,
|
|
150
|
-
prompt_attachments: p.prompt_attachments ?? p.promptAttachments ?? p.reference_attachments
|
|
151
|
-
};
|
|
152
|
-
}, z.object({
|
|
153
|
-
provider: imageProviderSchema.optional(),
|
|
154
|
-
model: z.string().min(1).optional(),
|
|
155
|
-
prompt: z.string().min(3).max(8000),
|
|
156
|
-
prompt_attachments: z.array(z.string().url()).max(16).default([]),
|
|
157
|
-
aspect_ratio: aspectRatioSchema,
|
|
158
|
-
image_size: imageSizeSchema,
|
|
159
|
-
key_color: z.string().min(1).max(64).default("#00FF00"),
|
|
160
|
-
tolerance: z.coerce.number().min(0).max(1).default(0.3),
|
|
161
|
-
softness: z.coerce.number().min(0).max(1).default(0.1),
|
|
162
|
-
despill: z.boolean().default(true),
|
|
163
|
-
output_format: alphaOutputFormatSchema
|
|
164
|
-
}));
|
|
165
|
-
const renderHtmlImagePayloadSchema = z.object({
|
|
166
|
-
width: z.number().int().min(16).max(4096).default(1080),
|
|
167
|
-
height: z.number().int().min(16).max(4096).default(1920),
|
|
168
|
-
html: z.string().min(1).max(200_000),
|
|
169
|
-
css: z.string().max(200_000).optional(),
|
|
170
|
-
background_color: z.string().min(1).max(64).default("#ffffff"),
|
|
171
|
-
output_format: outputFormatSchema.default("png")
|
|
172
|
-
});
|
|
173
|
-
const mediaTypeSchema = z.enum(["image", "video"]);
|
|
174
|
-
const mediaFitSchema = z.enum(["cover", "contain", "fill"]);
|
|
175
|
-
const renderVideoSlideSchema = z.preprocess((raw) => {
|
|
176
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
177
|
-
return raw;
|
|
178
|
-
const slide = raw;
|
|
179
|
-
return {
|
|
180
|
-
...slide,
|
|
181
|
-
media_url: slide.media_url ?? slide.image_url ?? slide.imageUrl ?? slide.video_url ?? slide.videoUrl,
|
|
182
|
-
duration_ms: slide.duration_ms ?? slide.durationMs,
|
|
183
|
-
media_type: slide.media_type ?? slide.mediaType,
|
|
184
|
-
background_color: slide.background_color ?? slide.backgroundColor
|
|
185
|
-
};
|
|
186
|
-
}, z.object({
|
|
187
|
-
media_url: z.string().url(),
|
|
188
|
-
media_type: mediaTypeSchema.optional(),
|
|
189
|
-
duration_ms: z.number().int().min(100).max(120_000),
|
|
190
|
-
fit: mediaFitSchema.optional(),
|
|
191
|
-
background_color: z.string().min(1).max(64).optional()
|
|
192
|
-
}));
|
|
193
|
-
const renderSlidesVideoPayloadSchema = z.object({
|
|
194
|
-
slides: z.array(renderVideoSlideSchema).min(1).max(100),
|
|
195
|
-
width: z.number().int().min(16).max(4096).default(1080),
|
|
196
|
-
height: z.number().int().min(16).max(4096).default(1920),
|
|
197
|
-
background_color: z.string().min(1).max(64).default("#000000"),
|
|
198
|
-
default_fit: mediaFitSchema.default("cover")
|
|
199
|
-
}).superRefine((value, ctx) => {
|
|
200
|
-
const totalDurationMs = value.slides.reduce((total, slide) => total + slide.duration_ms, 0);
|
|
201
|
-
if (totalDurationMs > 10 * 60 * 1000) {
|
|
202
|
-
ctx.addIssue({
|
|
203
|
-
code: z.ZodIssueCode.custom,
|
|
204
|
-
message: "Total slideshow duration must be 10 minutes or less.",
|
|
205
|
-
path: ["slides"]
|
|
206
|
-
});
|
|
207
|
-
}
|
|
208
|
-
});
|
|
209
|
-
const dedupeMediaFitSchema = z.enum(["cover", "contain", "fill", "none", "scale-down"]);
|
|
210
|
-
const dedupeEffectsSchema = z.preprocess((raw) => {
|
|
211
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
212
|
-
return raw;
|
|
213
|
-
const p = raw;
|
|
214
|
-
return {
|
|
215
|
-
zoom: p.zoom,
|
|
216
|
-
tilt: p.tilt,
|
|
217
|
-
rotate: p.rotate,
|
|
218
|
-
saturation: p.saturation,
|
|
219
|
-
speed: p.speed,
|
|
220
|
-
horizontal_flip: p.horizontal_flip ?? p.horizontalFlip,
|
|
221
|
-
contrast: p.contrast,
|
|
222
|
-
brightness: p.brightness,
|
|
223
|
-
hue_rotate: p.hue_rotate ?? p.hueRotate,
|
|
224
|
-
blur: p.blur
|
|
225
|
-
};
|
|
226
|
-
}, z.object({
|
|
227
|
-
zoom: z.number().min(0.1).max(10).optional(),
|
|
228
|
-
tilt: z.number().min(-90).max(90).optional(),
|
|
229
|
-
rotate: z.number().min(-360).max(360).optional(),
|
|
230
|
-
saturation: z.number().min(0).max(10).optional(),
|
|
231
|
-
speed: z.number().min(0.1).max(10).optional(),
|
|
232
|
-
horizontal_flip: z.boolean().optional(),
|
|
233
|
-
contrast: z.number().min(0).max(10).optional(),
|
|
234
|
-
brightness: z.number().min(0).max(10).optional(),
|
|
235
|
-
hue_rotate: z.number().min(-360).max(360).optional(),
|
|
236
|
-
blur: z.number().min(0).max(50).optional()
|
|
237
|
-
}));
|
|
238
|
-
const mediaDedupePayloadSchema = z.preprocess((raw) => {
|
|
239
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
240
|
-
return raw;
|
|
241
|
-
const p = raw;
|
|
242
|
-
return {
|
|
243
|
-
...p,
|
|
244
|
-
source_media_url: p.source_media_url
|
|
245
|
-
?? p.source_video_url
|
|
246
|
-
?? p.source_image_url
|
|
247
|
-
?? p.media_url
|
|
248
|
-
?? p.mediaUrl
|
|
249
|
-
?? p.url,
|
|
250
|
-
media_type: p.media_type ?? p.mediaType,
|
|
251
|
-
tint_color: p.tint_color ?? p.tintColor,
|
|
252
|
-
tint_opacity: p.tint_opacity ?? p.tintOpacity,
|
|
253
|
-
duration_ms: p.duration_ms ?? p.durationMs,
|
|
254
|
-
fallback_duration_ms: p.fallback_duration_ms ?? p.fallbackDurationMs,
|
|
255
|
-
object_fit: p.object_fit ?? p.objectFit,
|
|
256
|
-
object_position: p.object_position ?? p.objectPosition,
|
|
257
|
-
background_color: p.background_color ?? p.backgroundColor,
|
|
258
|
-
output_format: p.output_format ?? p.outputFormat
|
|
259
|
-
};
|
|
260
|
-
}, z.object({
|
|
261
|
-
source_media_url: z.string().url(),
|
|
262
|
-
media_type: mediaTypeSchema.optional(),
|
|
263
|
-
effects: dedupeEffectsSchema.optional(),
|
|
264
|
-
tint_color: z.string().min(1).max(64).default("#FF8C00"),
|
|
265
|
-
tint_opacity: z.number().min(0).max(1).default(0.08),
|
|
266
|
-
width: z.number().int().min(16).max(4096).default(1080),
|
|
267
|
-
height: z.number().int().min(16).max(4096).default(1920),
|
|
268
|
-
duration_ms: z.number().int().min(100).max(600_000).optional(),
|
|
269
|
-
fallback_duration_ms: z.number().int().min(100).max(600_000).default(5000),
|
|
270
|
-
object_fit: dedupeMediaFitSchema.default("cover"),
|
|
271
|
-
object_position: z.string().min(1).max(64).default("center center"),
|
|
272
|
-
background_color: z.string().min(1).max(64).default("#000000"),
|
|
273
|
-
muted: z.boolean().default(false),
|
|
274
|
-
volume: z.number().min(0).max(2).default(1),
|
|
275
|
-
output_format: outputFormatSchema.default("png")
|
|
276
|
-
}));
|
|
277
|
-
const hyperframeLayerPayloadSchema = z.object({
|
|
278
|
-
id: z.string().min(1).max(120),
|
|
279
|
-
kind: z.enum(["video", "image", "audio", "caption", "text", "html", "shape"]),
|
|
280
|
-
label: z.string().max(200).optional(),
|
|
281
|
-
mode: z.enum(["both", "editor", "publish"]).default("publish"),
|
|
282
|
-
start: z.number().min(0),
|
|
283
|
-
duration: z.number().positive(),
|
|
284
|
-
track: z.number().int().min(0).default(0),
|
|
285
|
-
z: z.number().int().default(0),
|
|
286
|
-
x: z.number().default(0),
|
|
287
|
-
y: z.number().default(0),
|
|
288
|
-
width: z.number().positive().default(100),
|
|
289
|
-
height: z.number().positive().default(100),
|
|
290
|
-
src: z.string().min(1).optional(),
|
|
291
|
-
mediaStart: z.number().min(0).optional(),
|
|
292
|
-
media_start: z.number().min(0).optional(),
|
|
293
|
-
playbackStart: z.number().min(0).optional(),
|
|
294
|
-
playback_start: z.number().min(0).optional(),
|
|
295
|
-
fit: z.enum(["cover", "contain", "fill", "none"]).default("cover"),
|
|
296
|
-
muted: z.boolean().optional(),
|
|
297
|
-
volume: z.number().min(0).max(2).optional(),
|
|
298
|
-
text: z.string().max(20_000).optional(),
|
|
299
|
-
html: z.string().max(200_000).optional(),
|
|
300
|
-
color: z.string().max(80).optional(),
|
|
301
|
-
background: z.string().max(80).optional(),
|
|
302
|
-
textBackgroundStyle: z.enum(["plain", "outline", "highlight-solid", "highlight-translucent", "panel"]).optional(),
|
|
303
|
-
text_background_style: z.enum(["plain", "outline", "highlight-solid", "highlight-translucent", "panel"]).optional(),
|
|
304
|
-
fontFamily: z.string().max(120).optional(),
|
|
305
|
-
font_family: z.string().max(120).optional(),
|
|
306
|
-
fontWeight: z.number().int().min(100).max(1000).optional(),
|
|
307
|
-
font_weight: z.number().int().min(100).max(1000).optional(),
|
|
308
|
-
fontSize: z.number().int().min(4).max(400).optional(),
|
|
309
|
-
font_size: z.number().int().min(4).max(400).optional(),
|
|
310
|
-
lineHeight: z.number().min(0.5).max(4).optional(),
|
|
311
|
-
line_height: z.number().min(0.5).max(4).optional(),
|
|
312
|
-
radius: z.number().min(0).max(999).optional(),
|
|
313
|
-
opacity: z.number().min(0).max(1).optional(),
|
|
314
|
-
viralAlignment: z.string().max(2000).optional(),
|
|
315
|
-
viral_alignment: z.string().max(2000).optional()
|
|
316
|
-
});
|
|
317
|
-
const hyperframesCompositionPayloadSchema = z.object({
|
|
318
|
-
composition_id: z.string().min(1).max(120),
|
|
319
|
-
title: z.string().max(200).optional(),
|
|
320
|
-
source_folder: z.string().max(500).optional(),
|
|
321
|
-
width: z.number().int().min(16).max(4096).default(720),
|
|
322
|
-
height: z.number().int().min(16).max(4096).default(1280),
|
|
323
|
-
fps: z.number().int().min(1).max(120).default(30),
|
|
324
|
-
duration: z.number().positive().optional(),
|
|
325
|
-
duration_seconds: z.number().positive().optional(),
|
|
326
|
-
background_color: z.string().max(80).default("#050604"),
|
|
327
|
-
css: z.string().max(200_000).optional(),
|
|
328
|
-
layers: z.array(hyperframeLayerPayloadSchema).min(1).max(300)
|
|
329
|
-
});
|
|
330
|
-
const hyperframesProjectFilePayloadSchema = z.object({
|
|
331
|
-
path: z.string().min(1).max(500),
|
|
332
|
-
contents: z.string().max(2_000_000).optional(),
|
|
333
|
-
source_path: z.string().min(1).max(500).optional(),
|
|
334
|
-
encoding: z.enum(["utf8", "base64"]).default("utf8"),
|
|
335
|
-
content_type: z.string().max(120).optional()
|
|
336
|
-
}).refine((value) => Boolean(value.contents || value.source_path), {
|
|
337
|
-
message: "project_files entry needs contents or source_path."
|
|
338
|
-
});
|
|
339
|
-
function readHyperframesProjectFileContents(file) {
|
|
340
|
-
if (file.source_path) {
|
|
341
|
-
const normalized = file.source_path.replace(/^\/+/, "").replace(/\\/g, "/");
|
|
342
|
-
if (normalized.includes("..") || !normalized.startsWith("drafts/")) {
|
|
343
|
-
throw new Error(`project_files.source_path must live under drafts/: ${file.source_path}`);
|
|
344
|
-
}
|
|
345
|
-
// The JobRunner lambda chdirs to /tmp/vidfarm-job-runner at startup, so
|
|
346
|
-
// process.cwd() no longer points at the lambda image root. The drafts/
|
|
347
|
-
// tree is baked into the image at LAMBDA_TASK_ROOT (/var/task).
|
|
348
|
-
const root = process.env.LAMBDA_TASK_ROOT ?? process.cwd();
|
|
349
|
-
const resolved = path.resolve(root, normalized);
|
|
350
|
-
return readFileSync(resolved);
|
|
351
|
-
}
|
|
352
|
-
if (file.contents === undefined) {
|
|
353
|
-
throw new Error("project_files entry has neither contents nor source_path");
|
|
354
|
-
}
|
|
355
|
-
return file.encoding === "base64" ? Buffer.from(file.contents, "base64") : file.contents;
|
|
356
|
-
}
|
|
357
|
-
const hyperframesRenderPayloadSchema = z.object({
|
|
358
|
-
composition_html: z.string().min(1).max(2_000_000).optional(),
|
|
359
|
-
composition: hyperframesCompositionPayloadSchema.optional(),
|
|
360
|
-
composition_id: z.string().min(1).max(120).optional(),
|
|
361
|
-
title: z.string().max(200).optional(),
|
|
362
|
-
width: z.number().int().min(16).max(4096).optional(),
|
|
363
|
-
height: z.number().int().min(16).max(4096).optional(),
|
|
364
|
-
fps: z.number().int().min(1).max(120).default(30),
|
|
365
|
-
chunk_size: z.number().int().min(1).max(1800).default(240),
|
|
366
|
-
max_parallel_chunks: z.number().int().min(1).max(40).default(8),
|
|
367
|
-
output_key: z.string().min(1).max(300).default("output.mp4"),
|
|
368
|
-
project_files: z.array(hyperframesProjectFilePayloadSchema).max(100).default([]),
|
|
369
|
-
composition_data: z.record(z.string(), z.unknown()).optional()
|
|
370
|
-
}).refine((value) => Boolean(value.composition_html || value.composition), {
|
|
371
|
-
message: "Provide either composition_html or composition layers.",
|
|
372
|
-
path: ["composition_html"]
|
|
373
|
-
});
|
|
374
|
-
const mediaSourceUrlSchema = z.string().url().refine((value) => !isVidfarmControlPlaneArtifactUrl(value), {
|
|
375
|
-
message: "Pass a direct media URL, such as artifact.public_url, not a Vidfarm job artifacts API endpoint."
|
|
376
|
-
});
|
|
377
|
-
const normalizeVideoPayloadSchema = z.object({
|
|
378
|
-
source_video_url: mediaSourceUrlSchema,
|
|
379
|
-
max_width: z.number().int().min(240).max(4096).default(1080),
|
|
380
|
-
max_height: z.number().int().min(240).max(4096).default(1920),
|
|
381
|
-
crf: z.number().int().min(18).max(35).default(23),
|
|
382
|
-
preset: z.enum(["veryfast", "faster", "fast", "medium"]).default("veryfast"),
|
|
383
|
-
max_video_bitrate_kbps: z.number().int().min(500).max(50_000).default(8000),
|
|
384
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
385
|
-
});
|
|
386
|
-
const trimVideoPayloadSchema = z.object({
|
|
387
|
-
source_video_url: mediaSourceUrlSchema,
|
|
388
|
-
start_ms: z.number().int().min(0).default(0),
|
|
389
|
-
end_ms: z.number().int().min(1).optional(),
|
|
390
|
-
duration_ms: z.number().int().min(1).optional(),
|
|
391
|
-
max_width: z.number().int().min(240).max(4096).optional(),
|
|
392
|
-
max_height: z.number().int().min(240).max(4096).optional(),
|
|
393
|
-
crf: z.number().int().min(18).max(35).default(23),
|
|
394
|
-
preset: z.enum(["veryfast", "faster", "fast", "medium"]).default("veryfast"),
|
|
395
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
396
|
-
}).refine((value) => value.duration_ms != null || value.end_ms != null, {
|
|
397
|
-
message: "Either duration_ms or end_ms is required.",
|
|
398
|
-
path: ["duration_ms"]
|
|
399
|
-
});
|
|
400
|
-
const extractAudioPayloadSchema = z.object({
|
|
401
|
-
source_video_url: mediaSourceUrlSchema,
|
|
402
|
-
output_format: z.enum(["mp3", "wav", "m4a"]).default("mp3"),
|
|
403
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
404
|
-
});
|
|
405
|
-
const trimAudioPayloadSchema = z.object({
|
|
406
|
-
source_audio_url: mediaSourceUrlSchema,
|
|
407
|
-
start_ms: z.number().int().min(0).default(0),
|
|
408
|
-
end_ms: z.number().int().min(1).optional(),
|
|
409
|
-
duration_ms: z.number().int().min(1).optional(),
|
|
410
|
-
output_format: z.enum(["mp3", "wav", "m4a"]).default("mp3"),
|
|
411
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
412
|
-
}).refine((value) => value.duration_ms != null || value.end_ms != null, {
|
|
413
|
-
message: "Either duration_ms or end_ms is required.",
|
|
414
|
-
path: ["duration_ms"]
|
|
415
|
-
});
|
|
416
|
-
const speechProviderSchema = z.enum(["openai", "gemini", "openrouter"]);
|
|
417
|
-
const ttsPayloadSchema = z.preprocess((raw) => {
|
|
418
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
419
|
-
return raw;
|
|
420
|
-
const payload = raw;
|
|
421
|
-
return {
|
|
422
|
-
...payload,
|
|
423
|
-
text: payload.text ?? payload.input ?? payload.script,
|
|
424
|
-
instructions: payload.instructions ?? payload.style ?? payload.style_instructions ?? payload.voice_style
|
|
425
|
-
};
|
|
426
|
-
}, z.object({
|
|
427
|
-
provider: speechProviderSchema.optional(),
|
|
428
|
-
model: z.string().min(1).optional(),
|
|
429
|
-
text: z.string().min(1).max(8000),
|
|
430
|
-
voice: z.string().min(1).max(120).optional(),
|
|
431
|
-
instructions: z.string().min(1).max(4000).optional(),
|
|
432
|
-
output_format: z.enum(["mp3", "wav"]).default("mp3"),
|
|
433
|
-
// Default TRUE: great ElevenLabs narration out of the box on the platform key,
|
|
434
|
-
// billed to the wallet. Set false to use your own ElevenLabs key or a BYOK
|
|
435
|
-
// openai/gemini/openrouter key instead. See voice options via /audio/voices.
|
|
436
|
-
use_wallet_credits: z.boolean().default(true)
|
|
437
|
-
}).superRefine((value, ctx) => {
|
|
438
|
-
// Reject an explicit provider/voice contradiction at enqueue (400) instead
|
|
439
|
-
// of queueing a job that can only fail at the provider.
|
|
440
|
-
if (!value.provider || !value.voice)
|
|
441
|
-
return;
|
|
442
|
-
const mismatch = speechVoiceMismatch({ provider: value.provider, model: value.model, voice: value.voice });
|
|
443
|
-
if (mismatch) {
|
|
444
|
-
ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["voice"], message: mismatch });
|
|
445
|
-
}
|
|
446
|
-
}));
|
|
447
|
-
const sttPayloadSchema = z.preprocess((raw) => {
|
|
448
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
449
|
-
return raw;
|
|
450
|
-
const payload = raw;
|
|
451
|
-
return {
|
|
452
|
-
...payload,
|
|
453
|
-
source_url: payload.source_url ??
|
|
454
|
-
payload.source_video_url ??
|
|
455
|
-
payload.source_audio_url ??
|
|
456
|
-
payload.source_media_url ??
|
|
457
|
-
payload.video_url ??
|
|
458
|
-
payload.audio_url ??
|
|
459
|
-
payload.url
|
|
460
|
-
};
|
|
461
|
-
}, z.object({
|
|
462
|
-
provider: speechProviderSchema.optional(),
|
|
463
|
-
model: z.string().min(1).optional(),
|
|
464
|
-
source_url: mediaSourceUrlSchema,
|
|
465
|
-
media_type: z.enum(["auto", "video", "audio"]).default("auto"),
|
|
466
|
-
diarize: z.boolean().default(true),
|
|
467
|
-
// Word-level timings on each segment (animated captions). Real timestamps
|
|
468
|
-
// come from the openai path (whisper-1); other providers keep segment-level
|
|
469
|
-
// output and downstream callers estimate word windows.
|
|
470
|
-
word_timestamps: z.boolean().default(false),
|
|
471
|
-
language: z.string().min(2).max(16).optional(),
|
|
472
|
-
prompt: z.string().min(1).max(2000).optional(),
|
|
473
|
-
// Default TRUE: platform ElevenLabs Scribe (native diarization + real word
|
|
474
|
-
// timestamps), billed to the wallet. Set false to use your own ElevenLabs
|
|
475
|
-
// key or a BYOK gemini/openai/openrouter key.
|
|
476
|
-
use_wallet_credits: z.boolean().default(true)
|
|
477
|
-
}));
|
|
478
|
-
// Music generation (ElevenLabs). Prompt-based, or a hand-built composition plan.
|
|
479
|
-
const musicPayloadSchema = z.preprocess((raw) => {
|
|
480
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
481
|
-
return raw;
|
|
482
|
-
const payload = raw;
|
|
483
|
-
return {
|
|
484
|
-
...payload,
|
|
485
|
-
prompt: payload.prompt ?? payload.text ?? payload.description,
|
|
486
|
-
music_length_ms: payload.music_length_ms ?? payload.length_ms ?? payload.duration_ms
|
|
487
|
-
};
|
|
488
|
-
}, z.object({
|
|
489
|
-
prompt: z.string().min(1).max(2000).optional(),
|
|
490
|
-
// ElevenLabs music caps at 5 minutes per generation.
|
|
491
|
-
music_length_ms: z.number().int().min(3000).max(300000).default(30000),
|
|
492
|
-
// Optional ElevenLabs composition plan for granular section-by-section control.
|
|
493
|
-
composition_plan: z.record(z.string(), z.any()).optional(),
|
|
494
|
-
model: z.string().min(1).optional(),
|
|
495
|
-
output_format: z.string().min(1).optional(),
|
|
496
|
-
// Default TRUE: platform ElevenLabs key, billed to the wallet. Set false to
|
|
497
|
-
// use your own saved ElevenLabs key (no wallet charge).
|
|
498
|
-
use_wallet_credits: z.boolean().default(true)
|
|
499
|
-
}).refine((value) => Boolean(value.prompt) || Boolean(value.composition_plan), {
|
|
500
|
-
message: "Provide either a prompt or a composition_plan.",
|
|
501
|
-
path: ["prompt"]
|
|
502
|
-
}));
|
|
503
|
-
// Animated captions from speech: STT with word-level timings, paged into
|
|
504
|
-
// ready-to-apply cue layers. Either transcribe a source_url or page plain text.
|
|
505
|
-
const captionsPayloadSchema = z.preprocess((raw) => {
|
|
506
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
507
|
-
return raw;
|
|
508
|
-
const payload = raw;
|
|
509
|
-
return {
|
|
510
|
-
...payload,
|
|
511
|
-
source_url: payload.source_url ??
|
|
512
|
-
payload.source_video_url ??
|
|
513
|
-
payload.source_audio_url ??
|
|
514
|
-
payload.source_media_url ??
|
|
515
|
-
payload.video_url ??
|
|
516
|
-
payload.audio_url ??
|
|
517
|
-
payload.url
|
|
518
|
-
};
|
|
519
|
-
}, z.object({
|
|
520
|
-
provider: speechProviderSchema.optional(),
|
|
521
|
-
model: z.string().min(1).optional(),
|
|
522
|
-
source_url: mediaSourceUrlSchema.optional(),
|
|
523
|
-
media_type: z.enum(["auto", "video", "audio"]).default("auto"),
|
|
524
|
-
/** Skip STT entirely: page this script across the window instead. */
|
|
525
|
-
text: z.string().min(1).max(20_000).optional(),
|
|
526
|
-
language: z.string().min(2).max(16).optional(),
|
|
527
|
-
prompt: z.string().min(1).max(2000).optional(),
|
|
528
|
-
/** Named look: spotlight | karaoke | word-pop | stack-up | neon | bounce. */
|
|
529
|
-
style: z.string().min(1).max(32).optional(),
|
|
530
|
-
animation: z.enum(CAPTION_ANIMATIONS).optional(),
|
|
531
|
-
active_color: z.string().min(1).max(64).optional(),
|
|
532
|
-
highlight_color: z.string().min(1).max(64).optional(),
|
|
533
|
-
uppercase: z.boolean().optional(),
|
|
534
|
-
color: z.string().min(1).max(64).optional(),
|
|
535
|
-
background: z.string().min(1).max(64).optional(),
|
|
536
|
-
background_style: z.enum(["plain", "outline", "highlight-solid", "highlight-translucent"]).optional(),
|
|
537
|
-
font_family: z.string().min(1).max(64).optional(),
|
|
538
|
-
font_size: z.number().int().min(8).max(200).optional(),
|
|
539
|
-
max_words_per_cue: z.number().int().min(1).max(8).default(4),
|
|
540
|
-
/** Shift every cue by this many seconds (where the narration layer starts). */
|
|
541
|
-
offset_sec: z.number().min(0).max(3600).default(0),
|
|
542
|
-
/** Text mode window: where the paged script starts and how long it spans. */
|
|
543
|
-
window_start: z.number().min(0).max(3600).default(0),
|
|
544
|
-
window_duration: z.number().positive().max(3600).optional()
|
|
545
|
-
}).refine((payload) => Boolean(payload.source_url || payload.text), {
|
|
546
|
-
message: "Provide source_url (audio/video to transcribe) or text (script to page)."
|
|
547
|
-
}));
|
|
548
|
-
// Regenerate speech in (approximately) the original speaker's voice: listen to
|
|
549
|
-
// the source, transcribe + profile the voice, reword the script per the
|
|
550
|
-
// caller's instruction (or take an exact new script), then TTS with the
|
|
551
|
-
// closest preset voice + a style prompt matched to the speaker. OpenAI/Gemini
|
|
552
|
-
// APIs cannot clone voice IDENTITY (preset voices only), so the result is an
|
|
553
|
-
// honest approximation, never a clone — the output says so via voice_match.
|
|
554
|
-
const speechRegeneratePayloadSchema = z.preprocess((raw) => {
|
|
555
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
556
|
-
return raw;
|
|
557
|
-
const payload = raw;
|
|
558
|
-
return {
|
|
559
|
-
...payload,
|
|
560
|
-
source_url: payload.source_url ??
|
|
561
|
-
payload.source_video_url ??
|
|
562
|
-
payload.source_audio_url ??
|
|
563
|
-
payload.source_media_url ??
|
|
564
|
-
payload.video_url ??
|
|
565
|
-
payload.audio_url ??
|
|
566
|
-
payload.url,
|
|
567
|
-
text: payload.text ?? payload.script ?? payload.new_text,
|
|
568
|
-
rewrite_instruction: payload.rewrite_instruction ?? payload.instruction ?? payload.rewrite ?? payload.reword,
|
|
569
|
-
instructions: payload.instructions ?? payload.style ?? payload.style_instructions ?? payload.voice_style
|
|
570
|
-
};
|
|
571
|
-
}, z.object({
|
|
572
|
-
provider: speechProviderSchema.optional(),
|
|
573
|
-
model: z.string().min(1).optional(),
|
|
574
|
-
source_url: mediaSourceUrlSchema,
|
|
575
|
-
media_type: z.enum(["auto", "video", "audio"]).default("auto"),
|
|
576
|
-
/** Exact new script to speak. Mutually optional with rewrite_instruction; one is required. */
|
|
577
|
-
text: z.string().min(1).max(8000).optional(),
|
|
578
|
-
/** How to reword the original transcript (e.g. "mention the summer sale instead of the discount code"). */
|
|
579
|
-
rewrite_instruction: z.string().min(1).max(4000).optional(),
|
|
580
|
-
/** Explicit preset override; skips the profiled voice recommendation. */
|
|
581
|
-
voice: z.string().min(1).max(120).optional(),
|
|
582
|
-
/** Extra voice-style guidance appended to the profiled style instructions. */
|
|
583
|
-
instructions: z.string().min(1).max(4000).optional(),
|
|
584
|
-
language: z.string().min(2).max(16).optional(),
|
|
585
|
-
output_format: z.enum(["mp3", "wav"]).default("mp3")
|
|
586
|
-
}).refine((value) => Boolean(value.text || value.rewrite_instruction), {
|
|
587
|
-
message: "Provide text (the exact new script) or rewrite_instruction (how to reword the original transcript)."
|
|
588
|
-
}).superRefine((value, ctx) => {
|
|
589
|
-
if (!value.provider || !value.voice)
|
|
590
|
-
return;
|
|
591
|
-
const mismatch = speechVoiceMismatch({ provider: value.provider, model: value.model, voice: value.voice });
|
|
592
|
-
if (mismatch) {
|
|
593
|
-
ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["voice"], message: mismatch });
|
|
594
|
-
}
|
|
595
|
-
}));
|
|
596
|
-
const probeVideoPayloadSchema = z.object({
|
|
597
|
-
source_video_url: mediaSourceUrlSchema
|
|
598
|
-
});
|
|
599
|
-
const removeVideoCaptionsPayloadSchema = z.object({
|
|
600
|
-
source_video_url: mediaSourceUrlSchema
|
|
601
|
-
});
|
|
602
|
-
const probeAudioPayloadSchema = z.object({
|
|
603
|
-
source_audio_url: mediaSourceUrlSchema
|
|
604
|
-
});
|
|
605
|
-
const extractFramePayloadSchema = z.object({
|
|
606
|
-
source_video_url: mediaSourceUrlSchema,
|
|
607
|
-
at_ms: z.number().int().min(0).default(0),
|
|
608
|
-
output_format: z.enum(["png", "jpeg"]).default("png")
|
|
609
|
-
});
|
|
610
|
-
const muteVideoPayloadSchema = z.object({
|
|
611
|
-
source_video_url: mediaSourceUrlSchema
|
|
612
|
-
});
|
|
613
|
-
const replaceAudioPayloadSchema = z.object({
|
|
614
|
-
source_video_url: mediaSourceUrlSchema,
|
|
615
|
-
source_audio_url: mediaSourceUrlSchema,
|
|
616
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
617
|
-
});
|
|
618
|
-
const concatVideoPayloadSchema = z.object({
|
|
619
|
-
source_video_urls: z.array(mediaSourceUrlSchema).min(1).max(32),
|
|
620
|
-
width: z.number().int().min(16).max(4096).optional(),
|
|
621
|
-
height: z.number().int().min(16).max(4096).optional(),
|
|
622
|
-
fit: mediaFitSchema.default("contain"),
|
|
623
|
-
background_color: z.string().min(1).max(64).default("#000000")
|
|
624
|
-
});
|
|
625
|
-
const concatAudioPayloadSchema = z.object({
|
|
626
|
-
source_audio_urls: z.array(mediaSourceUrlSchema).min(1).max(64),
|
|
627
|
-
output_format: z.enum(["mp3", "wav", "m4a"]).default("mp3"),
|
|
628
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128)
|
|
629
|
-
});
|
|
630
|
-
const normalizeAudioPayloadSchema = z.object({
|
|
631
|
-
source_audio_url: mediaSourceUrlSchema,
|
|
632
|
-
output_format: z.enum(["mp3", "wav", "m4a"]).default("mp3"),
|
|
633
|
-
audio_bitrate_kbps: z.number().int().min(64).max(320).default(128),
|
|
634
|
-
sample_rate_hz: z.number().int().min(8_000).max(96_000).default(48_000)
|
|
635
|
-
});
|
|
636
|
-
const downloadVideoPayloadSchema = z.preprocess((raw) => {
|
|
637
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
638
|
-
return raw;
|
|
639
|
-
const payload = raw;
|
|
640
|
-
return {
|
|
641
|
-
...payload,
|
|
642
|
-
source_url: payload.source_url ?? payload.video_url ?? payload.url,
|
|
643
|
-
save_manifest: payload.save_manifest ?? payload.include_manifest ?? true
|
|
644
|
-
};
|
|
645
|
-
}, z.object({
|
|
646
|
-
source_url: z.string().url(),
|
|
647
|
-
quality: z.enum(["best", "hd", "full_hd"]).default("best"),
|
|
648
|
-
save_manifest: z.boolean().default(true)
|
|
649
|
-
}));
|
|
650
|
-
// Ingest of an already-uploaded video file (Add Template → upload). The URL is
|
|
651
|
-
// server-minted (our own storage), so no social-host restrictions apply.
|
|
652
|
-
const ingestVideoPayloadSchema = z.object({
|
|
653
|
-
source_video_url: z.string().url(),
|
|
654
|
-
title: z.string().min(1).max(300).optional(),
|
|
655
|
-
file_name: z.string().min(1).max(300).optional(),
|
|
656
|
-
save_manifest: z.boolean().default(true)
|
|
657
|
-
});
|
|
658
|
-
const brainstormHooksPayloadSchema = z.preprocess(normalizeOfferDescriptionPayload, z.object({
|
|
659
|
-
offer_description: z.string().min(10).max(12_000),
|
|
660
|
-
count: brainstormCountSchema,
|
|
661
|
-
provider: textProviderSchema.optional(),
|
|
662
|
-
model: z.string().min(1).max(200).optional()
|
|
663
|
-
}));
|
|
664
|
-
const brainstormAwarenessStagesPayloadSchema = z.preprocess(normalizeOfferDescriptionPayload, z.object({
|
|
665
|
-
offer_description: z.string().min(10).max(12_000),
|
|
666
|
-
provider: textProviderSchema.optional(),
|
|
667
|
-
model: z.string().min(1).max(200).optional()
|
|
668
|
-
}));
|
|
669
|
-
const brainstormAnglesPayloadSchema = z.preprocess(normalizeAnglePayload, z.object({
|
|
670
|
-
offer_description: z.string().min(10).max(12_000),
|
|
671
|
-
problem_awareness: z.enum(["problem_unaware", "problem_aware"]),
|
|
672
|
-
solution_awareness: z.enum(["solution_unaware", "solution_aware"]),
|
|
673
|
-
count: brainstormCountSchema,
|
|
674
|
-
provider: textProviderSchema.optional(),
|
|
675
|
-
model: z.string().min(1).max(200).optional()
|
|
676
|
-
}));
|
|
677
|
-
const brainstormColdstartPayloadSchema = z.preprocess(normalizeColdstartPayload, z.object({
|
|
678
|
-
user_message: z.string().min(3).max(12_000),
|
|
679
|
-
count: z.number().int().min(6).max(30).default(12),
|
|
680
|
-
provider: textProviderSchema.optional(),
|
|
681
|
-
model: z.string().min(1).max(200).optional()
|
|
682
|
-
}));
|
|
683
|
-
const brainstormProductPlacementPayloadSchema = z.preprocess(normalizeProductPlacementPayload, z.object({
|
|
684
|
-
source_video_url: z.string().url(),
|
|
685
|
-
offer_description: z.string().min(10).max(12_000),
|
|
686
|
-
count: z.number().int().min(3).max(30).default(8),
|
|
687
|
-
provider: textProviderSchema.optional(),
|
|
688
|
-
model: z.string().min(1).max(200).optional()
|
|
689
|
-
}));
|
|
690
|
-
const brainstormHooksResultSchema = z.object({
|
|
691
|
-
hooks: z.array(z.object({
|
|
692
|
-
hook: z.string().min(1),
|
|
693
|
-
explanation: z.string().min(1)
|
|
694
|
-
})).min(1).max(50)
|
|
695
|
-
});
|
|
696
|
-
const brainstormAnglesResultSchema = z.object({
|
|
697
|
-
angles: z.array(z.object({
|
|
698
|
-
angle: z.string().min(1),
|
|
699
|
-
explanation: z.string().min(1)
|
|
700
|
-
})).min(1).max(50)
|
|
701
|
-
});
|
|
702
|
-
const brainstormColdstartResultSchema = z.object({
|
|
703
|
-
questions: z.array(z.object({
|
|
704
|
-
question: z.string().min(1),
|
|
705
|
-
why_it_matters: z.string().min(1)
|
|
706
|
-
})).min(6).max(30)
|
|
707
|
-
});
|
|
708
|
-
const brainstormProductPlacementResultSchema = z.object({
|
|
709
|
-
opportunities: z.array(z.object({
|
|
710
|
-
timestamp: z.string().min(1),
|
|
711
|
-
scene: z.string().min(1),
|
|
712
|
-
placement_type: z.string().min(1),
|
|
713
|
-
placement_idea: z.string().min(1),
|
|
714
|
-
why_it_works: z.string().min(1)
|
|
715
|
-
})).min(1).max(50)
|
|
716
|
-
});
|
|
717
|
-
const PRIMITIVE_IMAGE_GENERATE_ID = "primitive:image_generate";
|
|
718
|
-
const PRIMITIVE_IMAGE_EDIT_ID = "primitive:image_edit";
|
|
719
|
-
const PRIMITIVE_IMAGE_INPAINT_ID = "primitive:image_inpaint";
|
|
720
|
-
const PRIMITIVE_IMAGE_RENDER_HTML_ID = "primitive:image_render_html";
|
|
721
|
-
const PRIMITIVE_IMAGE_REMOVE_BACKGROUND_ID = "primitive:image_remove_background";
|
|
722
|
-
const PRIMITIVE_IMAGE_REMOVE_BACKGROUND_GREENSCREEN_ID = "primitive:image_remove_background_greenscreen";
|
|
723
|
-
const PRIMITIVE_MEDIA_OVERLAY_ID = "primitive:media_overlay";
|
|
724
|
-
const PRIMITIVE_VIDEO_GENERATE_ID = "primitive:video_generate";
|
|
725
|
-
const PRIMITIVE_VIDEO_RENDER_SLIDES_ID = "primitive:video_render_slides";
|
|
726
|
-
const PRIMITIVE_VIDEO_NORMALIZE_ID = "primitive:video_normalize";
|
|
727
|
-
const PRIMITIVE_VIDEO_DOWNLOAD_ID = "primitive:video_download";
|
|
728
|
-
const PRIMITIVE_VIDEO_INGEST_ID = "primitive:video_ingest";
|
|
729
|
-
const PRIMITIVE_VIDEO_TRIM_ID = "primitive:video_trim";
|
|
730
|
-
const PRIMITIVE_VIDEO_REMOVE_CAPTIONS_ID = "primitive:video_remove_captions";
|
|
731
|
-
const PRIMITIVE_VIDEO_EXTRACT_AUDIO_ID = "primitive:video_extract_audio";
|
|
732
|
-
const PRIMITIVE_AUDIO_TRIM_ID = "primitive:audio_trim";
|
|
733
|
-
const PRIMITIVE_VIDEO_PROBE_ID = "primitive:video_probe";
|
|
734
|
-
const PRIMITIVE_AUDIO_PROBE_ID = "primitive:audio_probe";
|
|
735
|
-
const PRIMITIVE_VIDEO_EXTRACT_FRAME_ID = "primitive:video_extract_frame";
|
|
736
|
-
const PRIMITIVE_VIDEO_MUTE_ID = "primitive:video_mute";
|
|
737
|
-
const PRIMITIVE_VIDEO_REPLACE_AUDIO_ID = "primitive:video_replace_audio";
|
|
738
|
-
const PRIMITIVE_VIDEO_CONCAT_ID = "primitive:video_concat";
|
|
739
|
-
const PRIMITIVE_AUDIO_CONCAT_ID = "primitive:audio_concat";
|
|
740
|
-
const PRIMITIVE_AUDIO_NORMALIZE_ID = "primitive:audio_normalize";
|
|
741
|
-
const PRIMITIVE_HYPERFRAMES_RENDER_ID = "primitive:hyperframes_render";
|
|
742
|
-
const PRIMITIVE_BRAINSTORM_HOOKS_ID = "primitive:brainstorm_hooks";
|
|
743
|
-
const PRIMITIVE_BRAINSTORM_AWARENESS_STAGES_ID = "primitive:brainstorm_awareness_stages";
|
|
744
|
-
const PRIMITIVE_BRAINSTORM_ANGLES_ID = "primitive:brainstorm_angles";
|
|
745
|
-
const PRIMITIVE_BRAINSTORM_COLDSTART_ID = "primitive:brainstorm_coldstart";
|
|
746
|
-
const PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID = "primitive:brainstorm_product_placement";
|
|
747
|
-
const PRIMITIVE_MEDIA_DEDUPE_ID = "primitive:media_dedupe";
|
|
748
|
-
const PRIMITIVE_TTS_ID = "primitive:tts";
|
|
749
|
-
const PRIMITIVE_STT_ID = "primitive:stt";
|
|
750
|
-
const PRIMITIVE_MUSIC_ID = "primitive:music";
|
|
751
|
-
const PRIMITIVE_CAPTIONS_ID = "primitive:captions";
|
|
752
|
-
const PRIMITIVE_SPEECH_REGENERATE_ID = "primitive:speech_regenerate";
|
|
753
|
-
// Gemini inline audio caps out around 20MB; at the 64kbps mono mp3 we extract,
|
|
754
|
-
// that is roughly 40 minutes of speech.
|
|
755
|
-
const MAX_TRANSCRIBE_AUDIO_BYTES = 20 * 1024 * 1024;
|
|
756
|
-
// Keep job results comfortably under the DynamoDB item limit: transcripts
|
|
757
|
-
// longer than this inline budget stay artifact-only (transcript.json).
|
|
758
|
-
const MAX_INLINE_TRANSCRIPT_CHARS = 20_000;
|
|
759
|
-
const primitiveMediaService = new PrimitiveMediaLambdaService();
|
|
760
|
-
const hooksReferenceText = readPrimitiveAsset("SELLING_WITH_HOOKS.md");
|
|
761
|
-
const awarenessReferenceText = readPrimitiveAsset("SELLING_AWARENESS_STAGES.md");
|
|
762
|
-
const imageGeneratePrimitive = definePrimitive({
|
|
763
|
-
id: PRIMITIVE_IMAGE_GENERATE_ID,
|
|
764
|
-
kind: "image_generate",
|
|
765
|
-
operations: {
|
|
766
|
-
run: {
|
|
767
|
-
description: "Generate a new image and persist it as a reusable platform URL.",
|
|
768
|
-
inputSchema: generateImagePayloadSchema,
|
|
769
|
-
workflow: "run"
|
|
770
|
-
}
|
|
771
|
-
},
|
|
772
|
-
jobs: {
|
|
773
|
-
async run(ctx, input) {
|
|
774
|
-
const payload = generateImagePayloadSchema.parse(input);
|
|
775
|
-
ctx.logger.progress(0.12, "Generating primitive image");
|
|
776
|
-
const provider = payload.provider ?? "openai";
|
|
777
|
-
const model = payload.model ?? defaultImageModelForProvider(provider);
|
|
778
|
-
const generated = await ctx.providers.generateImage({
|
|
779
|
-
provider,
|
|
780
|
-
model,
|
|
781
|
-
prompt: payload.prompt,
|
|
782
|
-
promptAttachments: payload.prompt_attachments,
|
|
783
|
-
aspectRatio: payload.aspect_ratio,
|
|
784
|
-
size: payload.size,
|
|
785
|
-
imageSize: payload.image_size
|
|
786
|
-
});
|
|
787
|
-
const normalized = await normalizePrimitiveImageOutput(generated.bytes, payload.output_format);
|
|
788
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
789
|
-
key: `output.${extensionForOutputFormat(payload.output_format)}`,
|
|
790
|
-
bytes: normalized.bytes,
|
|
791
|
-
contentType: normalized.contentType,
|
|
792
|
-
metadata: {
|
|
793
|
-
primitive: "image_generate",
|
|
794
|
-
provider,
|
|
795
|
-
model,
|
|
796
|
-
prompt: payload.prompt,
|
|
797
|
-
revisedPrompt: generated.revisedPrompt
|
|
798
|
-
}
|
|
799
|
-
});
|
|
800
|
-
ctx.logger.progress(1, "Primitive image generation complete");
|
|
801
|
-
return {
|
|
802
|
-
progress: 1,
|
|
803
|
-
output: buildImagePrimitiveOutput({
|
|
804
|
-
fileUrl: stored.url,
|
|
805
|
-
contentType: stored.contentType,
|
|
806
|
-
revisedPrompt: generated.revisedPrompt,
|
|
807
|
-
files: stored.url ? [stored.url] : []
|
|
808
|
-
})
|
|
809
|
-
};
|
|
810
|
-
}
|
|
811
|
-
}
|
|
812
|
-
});
|
|
813
|
-
const imageEditPrimitive = definePrimitive({
|
|
814
|
-
id: PRIMITIVE_IMAGE_EDIT_ID,
|
|
815
|
-
kind: "image_edit",
|
|
816
|
-
operations: {
|
|
817
|
-
run: {
|
|
818
|
-
description: "Edit an existing image into a new reusable platform URL.",
|
|
819
|
-
inputSchema: editImagePayloadSchema,
|
|
820
|
-
workflow: "run"
|
|
821
|
-
}
|
|
822
|
-
},
|
|
823
|
-
jobs: {
|
|
824
|
-
async run(ctx, input) {
|
|
825
|
-
const payload = editImagePayloadSchema.parse(input);
|
|
826
|
-
ctx.logger.progress(0.12, "Editing primitive image");
|
|
827
|
-
const provider = payload.provider ?? "openai";
|
|
828
|
-
const model = payload.model ?? defaultImageModelForProvider(provider);
|
|
829
|
-
const edited = await ctx.providers.editImage({
|
|
830
|
-
provider,
|
|
831
|
-
model,
|
|
832
|
-
sourceImageUrl: payload.source_image_url,
|
|
833
|
-
instruction: payload.instruction,
|
|
834
|
-
referenceAttachments: payload.reference_attachments,
|
|
835
|
-
aspectRatio: payload.aspect_ratio,
|
|
836
|
-
size: payload.size,
|
|
837
|
-
imageSize: payload.image_size
|
|
838
|
-
});
|
|
839
|
-
const normalized = await normalizePrimitiveImageOutput(edited.bytes, payload.output_format);
|
|
840
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
841
|
-
key: `output.${extensionForOutputFormat(payload.output_format)}`,
|
|
842
|
-
bytes: normalized.bytes,
|
|
843
|
-
contentType: normalized.contentType,
|
|
844
|
-
metadata: {
|
|
845
|
-
primitive: "image_edit",
|
|
846
|
-
provider,
|
|
847
|
-
model,
|
|
848
|
-
sourceImageUrl: payload.source_image_url,
|
|
849
|
-
instruction: payload.instruction,
|
|
850
|
-
revisedPrompt: edited.revisedPrompt
|
|
851
|
-
}
|
|
852
|
-
});
|
|
853
|
-
ctx.logger.progress(1, "Primitive image edit complete");
|
|
854
|
-
return {
|
|
855
|
-
progress: 1,
|
|
856
|
-
output: buildImagePrimitiveOutput({
|
|
857
|
-
fileUrl: stored.url,
|
|
858
|
-
contentType: stored.contentType,
|
|
859
|
-
revisedPrompt: edited.revisedPrompt,
|
|
860
|
-
derivedFromUrl: payload.source_image_url,
|
|
861
|
-
files: stored.url ? [stored.url] : []
|
|
862
|
-
})
|
|
863
|
-
};
|
|
864
|
-
}
|
|
865
|
-
}
|
|
866
|
-
});
|
|
867
|
-
const imageInpaintPrimitive = definePrimitive({
|
|
868
|
-
id: PRIMITIVE_IMAGE_INPAINT_ID,
|
|
869
|
-
kind: "image_inpaint",
|
|
870
|
-
operations: {
|
|
871
|
-
run: {
|
|
872
|
-
description: "Edit a masked region of an existing image into a new reusable platform URL.",
|
|
873
|
-
inputSchema: inpaintImagePayloadSchema,
|
|
874
|
-
workflow: "run"
|
|
875
|
-
}
|
|
876
|
-
},
|
|
877
|
-
jobs: {
|
|
878
|
-
async run(ctx, input) {
|
|
879
|
-
const payload = inpaintImagePayloadSchema.parse(input);
|
|
880
|
-
ctx.logger.progress(0.12, "Inpainting primitive image");
|
|
881
|
-
const provider = payload.provider ?? "openai";
|
|
882
|
-
const model = payload.model ?? defaultImageModelForProvider(provider);
|
|
883
|
-
const edited = await ctx.providers.inpaintImage({
|
|
884
|
-
provider,
|
|
885
|
-
model,
|
|
886
|
-
sourceImageUrl: payload.source_image_url,
|
|
887
|
-
maskUrl: payload.mask_url,
|
|
888
|
-
instruction: payload.instruction,
|
|
889
|
-
referenceAttachments: payload.reference_attachments,
|
|
890
|
-
aspectRatio: payload.aspect_ratio,
|
|
891
|
-
size: payload.size,
|
|
892
|
-
imageSize: payload.image_size
|
|
893
|
-
});
|
|
894
|
-
const normalized = await normalizePrimitiveImageOutput(edited.bytes, payload.output_format);
|
|
895
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
896
|
-
key: `output.${extensionForOutputFormat(payload.output_format)}`,
|
|
897
|
-
bytes: normalized.bytes,
|
|
898
|
-
contentType: normalized.contentType,
|
|
899
|
-
metadata: {
|
|
900
|
-
primitive: "image_inpaint",
|
|
901
|
-
provider,
|
|
902
|
-
model,
|
|
903
|
-
sourceImageUrl: payload.source_image_url,
|
|
904
|
-
maskUrl: payload.mask_url,
|
|
905
|
-
instruction: payload.instruction,
|
|
906
|
-
revisedPrompt: edited.revisedPrompt
|
|
907
|
-
}
|
|
908
|
-
});
|
|
909
|
-
ctx.logger.progress(1, "Primitive image inpaint complete");
|
|
910
|
-
return {
|
|
911
|
-
progress: 1,
|
|
912
|
-
output: buildImagePrimitiveOutput({
|
|
913
|
-
fileUrl: stored.url,
|
|
914
|
-
contentType: stored.contentType,
|
|
915
|
-
revisedPrompt: edited.revisedPrompt,
|
|
916
|
-
derivedFromUrl: payload.source_image_url,
|
|
917
|
-
maskUrl: payload.mask_url,
|
|
918
|
-
files: stored.url ? [stored.url] : []
|
|
919
|
-
})
|
|
920
|
-
};
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
});
|
|
924
|
-
const imageRenderHtmlPrimitive = definePrimitive({
|
|
925
|
-
id: PRIMITIVE_IMAGE_RENDER_HTML_ID,
|
|
926
|
-
kind: "image_render_html",
|
|
927
|
-
operations: {
|
|
928
|
-
run: {
|
|
929
|
-
description: "Render an HTML-driven still image into a reusable platform URL.",
|
|
930
|
-
inputSchema: renderHtmlImagePayloadSchema,
|
|
931
|
-
workflow: "run"
|
|
932
|
-
}
|
|
933
|
-
},
|
|
934
|
-
jobs: {
|
|
935
|
-
async run(ctx, input) {
|
|
936
|
-
const payload = renderHtmlImagePayloadSchema.parse(input);
|
|
937
|
-
ctx.logger.progress(0.18, "Rendering primitive HTML image");
|
|
938
|
-
const rendered = await ctx.hyperframes.renderStill({
|
|
939
|
-
compositionHtml: buildHtmlImageCompositionHtml({
|
|
940
|
-
compositionId: "primitive-html-image",
|
|
941
|
-
width: payload.width,
|
|
942
|
-
height: payload.height,
|
|
943
|
-
backgroundColor: payload.background_color,
|
|
944
|
-
html: payload.html,
|
|
945
|
-
css: payload.css ?? ""
|
|
946
|
-
}),
|
|
947
|
-
compositionId: "primitive-html-image",
|
|
948
|
-
imageFormat: payload.output_format,
|
|
949
|
-
outputKey: `output.${extensionForOutputFormat(payload.output_format)}`
|
|
950
|
-
});
|
|
951
|
-
ctx.logger.progress(1, "Primitive HTML image render complete");
|
|
952
|
-
return {
|
|
953
|
-
progress: 1,
|
|
954
|
-
output: {
|
|
955
|
-
files: rendered.outputUrl ? [rendered.outputUrl] : [],
|
|
956
|
-
primary_file_url: rendered.outputUrl,
|
|
957
|
-
image: {
|
|
958
|
-
file_url: rendered.outputUrl,
|
|
959
|
-
content_type: contentTypeForOutputFormat(payload.output_format),
|
|
960
|
-
width: payload.width,
|
|
961
|
-
height: payload.height
|
|
962
|
-
},
|
|
963
|
-
render: rendered
|
|
964
|
-
}
|
|
965
|
-
};
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
});
|
|
969
|
-
const imageRemoveBackgroundPrimitive = definePrimitive({
|
|
970
|
-
id: PRIMITIVE_IMAGE_REMOVE_BACKGROUND_ID,
|
|
971
|
-
kind: "image_remove_background",
|
|
972
|
-
operations: {
|
|
973
|
-
run: {
|
|
974
|
-
description: "Remove the background from a source image URL and store the transparent result as a reusable platform URL.",
|
|
975
|
-
inputSchema: removeBackgroundImagePayloadSchema,
|
|
976
|
-
workflow: "run"
|
|
977
|
-
}
|
|
978
|
-
},
|
|
979
|
-
jobs: {
|
|
980
|
-
async run(ctx, input) {
|
|
981
|
-
const payload = removeBackgroundImagePayloadSchema.parse(input);
|
|
982
|
-
const sourceUrl = payload.source_image_url;
|
|
983
|
-
const sourceHost = safeUrlHost(sourceUrl);
|
|
984
|
-
const apiKey = (config.RAPIDAPI_KEY || "").trim();
|
|
985
|
-
if (!apiKey) {
|
|
986
|
-
throw new Error("RAPIDAPI_KEY is not configured.");
|
|
987
|
-
}
|
|
988
|
-
ctx.logger.progress(0.15, "Requesting remote background removal", {
|
|
989
|
-
sourceUrl,
|
|
990
|
-
sourceHost,
|
|
991
|
-
apiHost: config.RAPIDAPI_REMOVE_BACKGROUND_HOST
|
|
992
|
-
});
|
|
993
|
-
const apiResponse = await fetch(config.RAPIDAPI_REMOVE_BACKGROUND_URL, {
|
|
994
|
-
method: "POST",
|
|
995
|
-
headers: {
|
|
996
|
-
"x-rapidapi-key": apiKey,
|
|
997
|
-
"x-rapidapi-host": config.RAPIDAPI_REMOVE_BACKGROUND_HOST,
|
|
998
|
-
"content-type": "application/x-www-form-urlencoded"
|
|
999
|
-
},
|
|
1000
|
-
body: new URLSearchParams({ image_url: sourceUrl })
|
|
1001
|
-
});
|
|
1002
|
-
if (!apiResponse.ok) {
|
|
1003
|
-
const details = await apiResponse.text().catch(() => "");
|
|
1004
|
-
throw new Error(`Background removal failed with HTTP ${apiResponse.status}${details ? `: ${details.slice(0, 400)}` : ""}`);
|
|
1005
|
-
}
|
|
1006
|
-
const lookup = await apiResponse.json();
|
|
1007
|
-
if (!lookup.url) {
|
|
1008
|
-
throw new Error("Background removal returned no result URL.");
|
|
1009
|
-
}
|
|
1010
|
-
ctx.logger.progress(0.55, "Downloading background-removed image", {
|
|
1011
|
-
remoteUrl: lookup.url
|
|
1012
|
-
});
|
|
1013
|
-
const remoteImage = await fetchRemoteBinary(lookup.url, "image/webp");
|
|
1014
|
-
const normalized = await normalizePrimitiveImageOutput(remoteImage.bytes, payload.output_format);
|
|
1015
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
1016
|
-
key: `output.${extensionForOutputFormat(payload.output_format)}`,
|
|
1017
|
-
bytes: normalized.bytes,
|
|
1018
|
-
contentType: normalized.contentType,
|
|
1019
|
-
metadata: {
|
|
1020
|
-
primitive: "image_remove_background",
|
|
1021
|
-
source_image_url: sourceUrl,
|
|
1022
|
-
source_host: sourceHost,
|
|
1023
|
-
upstream_url: lookup.url,
|
|
1024
|
-
upstream_processing_time_seconds: lookup.processing_time ?? null
|
|
1025
|
-
}
|
|
1026
|
-
});
|
|
1027
|
-
await ctx.billing.record({
|
|
1028
|
-
type: "cpu_estimate",
|
|
1029
|
-
costUsd: config.RAPIDAPI_REMOVE_BACKGROUND_COST_USD,
|
|
1030
|
-
costCenterSlug: "rapidapi_remove_background",
|
|
1031
|
-
idempotencyKey: `rapidapi_remove_background:${sourceUrl}`,
|
|
1032
|
-
occurredAtMs: Date.now(),
|
|
1033
|
-
metadata: {
|
|
1034
|
-
operation: "remove-background",
|
|
1035
|
-
source_image_url: sourceUrl,
|
|
1036
|
-
source_host: sourceHost,
|
|
1037
|
-
upstream_url: lookup.url,
|
|
1038
|
-
output_key: stored.key
|
|
1039
|
-
}
|
|
1040
|
-
});
|
|
1041
|
-
ctx.logger.progress(1, "Background removal complete");
|
|
1042
|
-
return {
|
|
1043
|
-
progress: 1,
|
|
1044
|
-
output: buildImagePrimitiveOutput({
|
|
1045
|
-
fileUrl: stored.url,
|
|
1046
|
-
contentType: stored.contentType,
|
|
1047
|
-
derivedFromUrl: sourceUrl,
|
|
1048
|
-
files: compactUrls([stored.url])
|
|
1049
|
-
})
|
|
1050
|
-
};
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
});
|
|
1054
|
-
const imageRemoveBackgroundGreenscreenPrimitive = definePrimitive({
|
|
1055
|
-
id: PRIMITIVE_IMAGE_REMOVE_BACKGROUND_GREENSCREEN_ID,
|
|
1056
|
-
kind: "image_remove_background_greenscreen",
|
|
1057
|
-
operations: {
|
|
1058
|
-
run: {
|
|
1059
|
-
description: "Cheap local chroma-key background removal. Drops a flat, solid key-color background (green screen by default; pass key_color for white/blue/any flat fill) to a transparent PNG/WebP cut-out using sharp — no third-party API, compute only. Ideal for images you generated on a controlled solid background; for arbitrary photos with a messy background use image_remove_background (RapidAPI matting) instead.",
|
|
1060
|
-
inputSchema: greenscreenRemoveBackgroundPayloadSchema,
|
|
1061
|
-
workflow: "run"
|
|
1062
|
-
}
|
|
1063
|
-
},
|
|
1064
|
-
jobs: {
|
|
1065
|
-
async run(ctx, input) {
|
|
1066
|
-
const payload = greenscreenRemoveBackgroundPayloadSchema.parse(input);
|
|
1067
|
-
const sourceUrl = payload.source_image_url;
|
|
1068
|
-
ctx.logger.progress(0.2, "Downloading source image for chroma key", { sourceUrl });
|
|
1069
|
-
const source = await fetchRemoteBinary(sourceUrl, "image/png");
|
|
1070
|
-
ctx.logger.progress(0.5, "Keying out background", { keyColor: payload.key_color });
|
|
1071
|
-
const keyed = await chromaKeyRemoveBackground(source.bytes, {
|
|
1072
|
-
keyColor: payload.key_color,
|
|
1073
|
-
tolerance: payload.tolerance,
|
|
1074
|
-
softness: payload.softness,
|
|
1075
|
-
despill: payload.despill,
|
|
1076
|
-
outputFormat: payload.output_format
|
|
1077
|
-
});
|
|
1078
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
1079
|
-
key: `output.${keyed.contentType === "image/webp" ? "webp" : "png"}`,
|
|
1080
|
-
bytes: keyed.bytes,
|
|
1081
|
-
contentType: keyed.contentType,
|
|
1082
|
-
metadata: {
|
|
1083
|
-
primitive: "image_remove_background_greenscreen",
|
|
1084
|
-
source_image_url: sourceUrl,
|
|
1085
|
-
key_color: payload.key_color,
|
|
1086
|
-
tolerance: payload.tolerance,
|
|
1087
|
-
softness: payload.softness,
|
|
1088
|
-
despill: payload.despill,
|
|
1089
|
-
keyed_fraction: keyed.keyedFraction
|
|
1090
|
-
}
|
|
1091
|
-
});
|
|
1092
|
-
await ctx.billing.record({
|
|
1093
|
-
type: "cpu_estimate",
|
|
1094
|
-
costUsd: config.GREENSCREEN_CHROMA_KEY_COST_USD,
|
|
1095
|
-
costCenterSlug: "greenscreen_chroma_key",
|
|
1096
|
-
occurredAtMs: Date.now(),
|
|
1097
|
-
metadata: {
|
|
1098
|
-
operation: "remove-background-greenscreen",
|
|
1099
|
-
source_image_url: sourceUrl,
|
|
1100
|
-
key_color: payload.key_color,
|
|
1101
|
-
keyed_fraction: keyed.keyedFraction,
|
|
1102
|
-
output_key: stored.key
|
|
1103
|
-
}
|
|
1104
|
-
});
|
|
1105
|
-
ctx.logger.progress(1, "Chroma-key background removal complete");
|
|
1106
|
-
return {
|
|
1107
|
-
progress: 1,
|
|
1108
|
-
output: {
|
|
1109
|
-
...buildImagePrimitiveOutput({
|
|
1110
|
-
fileUrl: stored.url,
|
|
1111
|
-
contentType: keyed.contentType,
|
|
1112
|
-
derivedFromUrl: sourceUrl,
|
|
1113
|
-
files: compactUrls([stored.url])
|
|
1114
|
-
}),
|
|
1115
|
-
key_color: payload.key_color,
|
|
1116
|
-
keyed_fraction: keyed.keyedFraction
|
|
1117
|
-
}
|
|
1118
|
-
};
|
|
1119
|
-
}
|
|
1120
|
-
}
|
|
1121
|
-
});
|
|
1122
|
-
const mediaOverlayPrimitive = definePrimitive({
|
|
1123
|
-
id: PRIMITIVE_MEDIA_OVERLAY_ID,
|
|
1124
|
-
kind: "media_overlay",
|
|
1125
|
-
operations: {
|
|
1126
|
-
run: {
|
|
1127
|
-
description: "Create a transparent-background media overlay in ONE job: generate an AI image of the subject on a forced flat key-color background (using the caller's own BYOK image keys), then chroma-key that background out to a ready-to-composite transparent PNG/WebP. This is the go-to way to mint 'Vox-style' floating illustrations, props, icons, or cut-out characters to animate over a composition — prefer it over generate-then-manually-remove.",
|
|
1128
|
-
inputSchema: mediaOverlayPayloadSchema,
|
|
1129
|
-
workflow: "run",
|
|
1130
|
-
providerHint: "openai"
|
|
1131
|
-
}
|
|
1132
|
-
},
|
|
1133
|
-
jobs: {
|
|
1134
|
-
async run(ctx, input) {
|
|
1135
|
-
const payload = mediaOverlayPayloadSchema.parse(input);
|
|
1136
|
-
const provider = payload.provider ?? "openai";
|
|
1137
|
-
const model = payload.model ?? defaultImageModelForProvider(provider);
|
|
1138
|
-
const keyRgb = parseKeyColor(payload.key_color);
|
|
1139
|
-
const keyLabel = describeKeyColor(payload.key_color, keyRgb);
|
|
1140
|
-
const augmentedPrompt = buildOverlayGenerationPrompt(payload.prompt, payload.key_color, keyLabel);
|
|
1141
|
-
ctx.logger.progress(0.12, "Generating overlay subject on flat key-color background", { provider, model, keyColor: payload.key_color });
|
|
1142
|
-
const generated = await ctx.providers.generateImage({
|
|
1143
|
-
provider,
|
|
1144
|
-
model,
|
|
1145
|
-
prompt: augmentedPrompt,
|
|
1146
|
-
promptAttachments: payload.prompt_attachments,
|
|
1147
|
-
aspectRatio: payload.aspect_ratio,
|
|
1148
|
-
imageSize: payload.image_size
|
|
1149
|
-
});
|
|
1150
|
-
// Keep the raw key-color frame as an artifact for debugging / re-keying at
|
|
1151
|
-
// different tolerances without paying for a second generation.
|
|
1152
|
-
const normalizedSource = await normalizePrimitiveImageOutput(generated.bytes, "png");
|
|
1153
|
-
const sourceStored = await storePrimitiveImage(ctx, {
|
|
1154
|
-
key: "source-greenscreen.png",
|
|
1155
|
-
bytes: normalizedSource.bytes,
|
|
1156
|
-
contentType: normalizedSource.contentType,
|
|
1157
|
-
metadata: {
|
|
1158
|
-
primitive: "media_overlay",
|
|
1159
|
-
stage: "greenscreen_source",
|
|
1160
|
-
prompt: payload.prompt,
|
|
1161
|
-
key_color: payload.key_color,
|
|
1162
|
-
revised_prompt: generated.revisedPrompt
|
|
1163
|
-
}
|
|
1164
|
-
});
|
|
1165
|
-
ctx.logger.progress(0.62, "Keying out background", { keyColor: payload.key_color });
|
|
1166
|
-
const keyed = await chromaKeyRemoveBackground(generated.bytes, {
|
|
1167
|
-
keyColor: payload.key_color,
|
|
1168
|
-
tolerance: payload.tolerance,
|
|
1169
|
-
softness: payload.softness,
|
|
1170
|
-
despill: payload.despill,
|
|
1171
|
-
outputFormat: payload.output_format
|
|
1172
|
-
});
|
|
1173
|
-
const stored = await storePrimitiveImage(ctx, {
|
|
1174
|
-
key: `output.${keyed.contentType === "image/webp" ? "webp" : "png"}`,
|
|
1175
|
-
bytes: keyed.bytes,
|
|
1176
|
-
contentType: keyed.contentType,
|
|
1177
|
-
metadata: {
|
|
1178
|
-
primitive: "media_overlay",
|
|
1179
|
-
stage: "transparent_overlay",
|
|
1180
|
-
prompt: payload.prompt,
|
|
1181
|
-
provider,
|
|
1182
|
-
model,
|
|
1183
|
-
key_color: payload.key_color,
|
|
1184
|
-
keyed_fraction: keyed.keyedFraction,
|
|
1185
|
-
revised_prompt: generated.revisedPrompt
|
|
1186
|
-
}
|
|
1187
|
-
});
|
|
1188
|
-
await ctx.billing.record({
|
|
1189
|
-
type: "cpu_estimate",
|
|
1190
|
-
costUsd: config.MEDIA_OVERLAY_COST_USD,
|
|
1191
|
-
costCenterSlug: "media_overlay",
|
|
1192
|
-
occurredAtMs: Date.now(),
|
|
1193
|
-
metadata: {
|
|
1194
|
-
operation: "create-media-overlay",
|
|
1195
|
-
prompt: payload.prompt,
|
|
1196
|
-
key_color: payload.key_color,
|
|
1197
|
-
keyed_fraction: keyed.keyedFraction,
|
|
1198
|
-
output_key: stored.key,
|
|
1199
|
-
greenscreen_source_key: sourceStored.key
|
|
1200
|
-
}
|
|
1201
|
-
});
|
|
1202
|
-
ctx.logger.progress(1, "Media overlay ready");
|
|
1203
|
-
return {
|
|
1204
|
-
progress: 1,
|
|
1205
|
-
output: {
|
|
1206
|
-
...buildImagePrimitiveOutput({
|
|
1207
|
-
fileUrl: stored.url,
|
|
1208
|
-
contentType: keyed.contentType,
|
|
1209
|
-
revisedPrompt: generated.revisedPrompt,
|
|
1210
|
-
files: compactUrls([stored.url, sourceStored.url])
|
|
1211
|
-
}),
|
|
1212
|
-
greenscreen_source_url: sourceStored.url,
|
|
1213
|
-
key_color: payload.key_color,
|
|
1214
|
-
keyed_fraction: keyed.keyedFraction
|
|
1215
|
-
}
|
|
1216
|
-
};
|
|
1217
|
-
}
|
|
1218
|
-
}
|
|
1219
|
-
});
|
|
1220
|
-
const videoGeneratePrimitive = definePrimitive({
|
|
1221
|
-
id: PRIMITIVE_VIDEO_GENERATE_ID,
|
|
1222
|
-
kind: "video_generate",
|
|
1223
|
-
operations: {
|
|
1224
|
-
run: {
|
|
1225
|
-
description: "Generate a new AI video from a text prompt and optional reference images.",
|
|
1226
|
-
inputSchema: generateVideoPayloadSchema,
|
|
1227
|
-
workflow: "run",
|
|
1228
|
-
providerHint: "openai"
|
|
1229
|
-
}
|
|
1230
|
-
},
|
|
1231
|
-
jobs: {
|
|
1232
|
-
async run(ctx, input) {
|
|
1233
|
-
const payload = generateVideoPayloadSchema.parse(input);
|
|
1234
|
-
const provider = payload.provider ?? "openai";
|
|
1235
|
-
const model = payload.model ?? defaultVideoModelForProvider(provider);
|
|
1236
|
-
ctx.logger.progress(0.08, "Submitting primitive AI video generation");
|
|
1237
|
-
const generated = await ctx.providers.generateVideo({
|
|
1238
|
-
provider,
|
|
1239
|
-
model,
|
|
1240
|
-
prompt: payload.prompt,
|
|
1241
|
-
duration: payload.duration,
|
|
1242
|
-
resolution: payload.resolution,
|
|
1243
|
-
aspectRatio: payload.aspect_ratio,
|
|
1244
|
-
size: payload.size,
|
|
1245
|
-
generateAudio: payload.generate_audio,
|
|
1246
|
-
frameImages: payload.frame_images.map((frame) => ({
|
|
1247
|
-
imageUrl: frame.image_url,
|
|
1248
|
-
frameType: frame.frame_type
|
|
1249
|
-
})),
|
|
1250
|
-
inputReferences: payload.input_references,
|
|
1251
|
-
seed: payload.seed,
|
|
1252
|
-
providerOptions: payload.provider_options
|
|
1253
|
-
});
|
|
1254
|
-
const stored = await storePrimitiveVideo(ctx, {
|
|
1255
|
-
key: "output.mp4",
|
|
1256
|
-
bytes: generated.bytes,
|
|
1257
|
-
contentType: generated.contentType,
|
|
1258
|
-
metadata: {
|
|
1259
|
-
primitive: "video_generate",
|
|
1260
|
-
provider,
|
|
1261
|
-
model,
|
|
1262
|
-
prompt: payload.prompt,
|
|
1263
|
-
providerJob: generated.providerJob
|
|
1264
|
-
}
|
|
1265
|
-
});
|
|
1266
|
-
await recordPrimitiveProviderBilling(ctx, generated, "video_generate");
|
|
1267
|
-
ctx.logger.progress(1, "Primitive AI video generation complete");
|
|
1268
|
-
return {
|
|
1269
|
-
progress: 1,
|
|
1270
|
-
output: {
|
|
1271
|
-
files: stored.url ? [stored.url] : [],
|
|
1272
|
-
primary_file_url: stored.url,
|
|
1273
|
-
video: {
|
|
1274
|
-
file_url: stored.url,
|
|
1275
|
-
content_type: stored.contentType,
|
|
1276
|
-
provider,
|
|
1277
|
-
model
|
|
1278
|
-
},
|
|
1279
|
-
provider_job: generated.providerJob
|
|
1280
|
-
}
|
|
1281
|
-
};
|
|
1282
|
-
}
|
|
1283
|
-
}
|
|
1284
|
-
});
|
|
1285
|
-
const videoRenderSlidesPrimitive = definePrimitive({
|
|
1286
|
-
id: PRIMITIVE_VIDEO_RENDER_SLIDES_ID,
|
|
1287
|
-
kind: "video_render_slides",
|
|
1288
|
-
operations: {
|
|
1289
|
-
run: {
|
|
1290
|
-
description: "Render image/video slides into a reusable MP4 from [{ media_url, duration_ms }].",
|
|
1291
|
-
inputSchema: renderSlidesVideoPayloadSchema,
|
|
1292
|
-
workflow: "run"
|
|
1293
|
-
}
|
|
1294
|
-
},
|
|
1295
|
-
jobs: {
|
|
1296
|
-
async run(ctx, input) {
|
|
1297
|
-
const payload = renderSlidesVideoPayloadSchema.parse(input);
|
|
1298
|
-
const durationMs = payload.slides.reduce((total, slide) => total + slide.duration_ms, 0);
|
|
1299
|
-
ctx.logger.progress(0.18, "Rendering primitive media slideshow", {
|
|
1300
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
1301
|
-
});
|
|
1302
|
-
const render = await executePrimitiveMediaOperation(ctx, {
|
|
1303
|
-
operation: "video_render_slides",
|
|
1304
|
-
payload: {
|
|
1305
|
-
slides: payload.slides.map((slide) => ({
|
|
1306
|
-
media_url: slide.media_url,
|
|
1307
|
-
media_type: slide.media_type ?? inferMediaType(slide.media_url),
|
|
1308
|
-
duration_ms: slide.duration_ms,
|
|
1309
|
-
fit: slide.fit ?? payload.default_fit,
|
|
1310
|
-
background_color: slide.background_color ?? payload.background_color
|
|
1311
|
-
})),
|
|
1312
|
-
width: payload.width,
|
|
1313
|
-
height: payload.height,
|
|
1314
|
-
background_color: payload.background_color,
|
|
1315
|
-
default_fit: payload.default_fit
|
|
1316
|
-
},
|
|
1317
|
-
outputKey: "output.mp4"
|
|
1318
|
-
});
|
|
1319
|
-
ctx.logger.progress(1, "Primitive media slideshow render complete");
|
|
1320
|
-
return {
|
|
1321
|
-
progress: 1,
|
|
1322
|
-
output: {
|
|
1323
|
-
files: compactUrls([
|
|
1324
|
-
...payload.slides.map((slide) => slide.media_url),
|
|
1325
|
-
render.publicUrl
|
|
1326
|
-
]),
|
|
1327
|
-
primary_file_url: render.publicUrl,
|
|
1328
|
-
video: {
|
|
1329
|
-
file_url: render.publicUrl,
|
|
1330
|
-
content_type: "video/mp4",
|
|
1331
|
-
width: payload.width,
|
|
1332
|
-
height: payload.height,
|
|
1333
|
-
duration_ms: durationMs,
|
|
1334
|
-
slide_count: payload.slides.length
|
|
1335
|
-
},
|
|
1336
|
-
render
|
|
1337
|
-
}
|
|
1338
|
-
};
|
|
1339
|
-
}
|
|
1340
|
-
}
|
|
1341
|
-
});
|
|
1342
|
-
const mediaDedupePrimitive = definePrimitive({
|
|
1343
|
-
id: PRIMITIVE_MEDIA_DEDUPE_ID,
|
|
1344
|
-
kind: "media_dedupe",
|
|
1345
|
-
operations: {
|
|
1346
|
-
run: {
|
|
1347
|
-
description: "Apply dedupe transforms (zoom, tilt, rotate, playback speed, color filter, blur, tint, flip) to any source image or video URL for reuse across compositions.",
|
|
1348
|
-
inputSchema: mediaDedupePayloadSchema,
|
|
1349
|
-
workflow: "run"
|
|
1350
|
-
}
|
|
1351
|
-
},
|
|
1352
|
-
jobs: {
|
|
1353
|
-
async run(ctx, input) {
|
|
1354
|
-
const payload = mediaDedupePayloadSchema.parse(input);
|
|
1355
|
-
const resolvedMediaType = payload.media_type ?? inferMediaType(payload.source_media_url);
|
|
1356
|
-
const durationMs = payload.duration_ms ?? payload.fallback_duration_ms;
|
|
1357
|
-
const dedupeComposition = (mediaType, durationSeconds) => buildMediaDedupeCompositionHtml({
|
|
1358
|
-
compositionId: `primitive-media-dedupe-${mediaType}`,
|
|
1359
|
-
mediaType,
|
|
1360
|
-
sourceMediaUrl: payload.source_media_url,
|
|
1361
|
-
width: payload.width,
|
|
1362
|
-
height: payload.height,
|
|
1363
|
-
durationSeconds,
|
|
1364
|
-
effects: payload.effects,
|
|
1365
|
-
tintColor: payload.tint_color,
|
|
1366
|
-
tintOpacity: payload.tint_opacity,
|
|
1367
|
-
objectFit: payload.object_fit,
|
|
1368
|
-
objectPosition: payload.object_position,
|
|
1369
|
-
backgroundColor: payload.background_color,
|
|
1370
|
-
muted: payload.muted
|
|
1371
|
-
});
|
|
1372
|
-
ctx.logger.progress(0.2, "Rendering primitive media dedupe", {
|
|
1373
|
-
mediaType: resolvedMediaType,
|
|
1374
|
-
width: payload.width,
|
|
1375
|
-
height: payload.height
|
|
1376
|
-
});
|
|
1377
|
-
if (resolvedMediaType === "image") {
|
|
1378
|
-
const rendered = await ctx.hyperframes.renderStill({
|
|
1379
|
-
compositionHtml: dedupeComposition("image", 0.1),
|
|
1380
|
-
compositionId: "primitive-media-dedupe-image",
|
|
1381
|
-
imageFormat: payload.output_format,
|
|
1382
|
-
outputKey: `output.${extensionForOutputFormat(payload.output_format)}`
|
|
1383
|
-
});
|
|
1384
|
-
ctx.logger.progress(1, "Primitive media dedupe still complete");
|
|
1385
|
-
return {
|
|
1386
|
-
progress: 1,
|
|
1387
|
-
output: {
|
|
1388
|
-
files: compactUrls([payload.source_media_url, rendered.outputUrl]),
|
|
1389
|
-
primary_file_url: rendered.outputUrl,
|
|
1390
|
-
image: {
|
|
1391
|
-
file_url: rendered.outputUrl,
|
|
1392
|
-
content_type: contentTypeForOutputFormat(payload.output_format),
|
|
1393
|
-
width: payload.width,
|
|
1394
|
-
height: payload.height,
|
|
1395
|
-
source_url: payload.source_media_url
|
|
1396
|
-
},
|
|
1397
|
-
render: rendered
|
|
1398
|
-
}
|
|
1399
|
-
};
|
|
1400
|
-
}
|
|
1401
|
-
// Speed manifests as a shorter output clip: durationInFrames = (durationMs /
|
|
1402
|
-
// speed) / 1000 * fps, matching the old media-dedupe.tsx frame count.
|
|
1403
|
-
const speed = Math.max(0.1, payload.effects?.speed ?? DEDUPE_DEFAULT_EFFECTS.speed);
|
|
1404
|
-
const outputDurationSeconds = Math.max(0.05, durationMs / speed / 1000);
|
|
1405
|
-
const rendered = await ctx.hyperframes.render({
|
|
1406
|
-
compositionHtml: dedupeComposition("video", outputDurationSeconds),
|
|
1407
|
-
compositionId: "primitive-media-dedupe-video",
|
|
1408
|
-
width: payload.width,
|
|
1409
|
-
height: payload.height,
|
|
1410
|
-
fps: 30
|
|
1411
|
-
});
|
|
1412
|
-
ctx.logger.progress(1, "Primitive media dedupe video complete");
|
|
1413
|
-
return {
|
|
1414
|
-
progress: 1,
|
|
1415
|
-
output: {
|
|
1416
|
-
files: compactUrls([payload.source_media_url, rendered.outputUrl]),
|
|
1417
|
-
primary_file_url: rendered.outputUrl,
|
|
1418
|
-
video: {
|
|
1419
|
-
file_url: rendered.outputUrl,
|
|
1420
|
-
content_type: "video/mp4",
|
|
1421
|
-
width: payload.width,
|
|
1422
|
-
height: payload.height,
|
|
1423
|
-
duration_ms: durationMs,
|
|
1424
|
-
source_url: payload.source_media_url
|
|
1425
|
-
},
|
|
1426
|
-
render: rendered
|
|
1427
|
-
}
|
|
1428
|
-
};
|
|
1429
|
-
}
|
|
1430
|
-
}
|
|
1431
|
-
});
|
|
1432
|
-
const videoNormalizePrimitive = definePrimitive({
|
|
1433
|
-
id: PRIMITIVE_VIDEO_NORMALIZE_ID,
|
|
1434
|
-
kind: "video_normalize",
|
|
1435
|
-
operations: {
|
|
1436
|
-
run: {
|
|
1437
|
-
description: "Normalize and compress a source MP4 into a render-friendly reusable platform MP4.",
|
|
1438
|
-
inputSchema: normalizeVideoPayloadSchema,
|
|
1439
|
-
workflow: "run"
|
|
1440
|
-
}
|
|
1441
|
-
},
|
|
1442
|
-
jobs: {
|
|
1443
|
-
async run(ctx, input) {
|
|
1444
|
-
const payload = normalizeVideoPayloadSchema.parse(input);
|
|
1445
|
-
ctx.logger.progress(0.12, "Normalizing source video", {
|
|
1446
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
1447
|
-
});
|
|
1448
|
-
const normalized = await executePrimitiveMediaOperation(ctx, {
|
|
1449
|
-
operation: "video_normalize",
|
|
1450
|
-
payload: {
|
|
1451
|
-
source_video_url: payload.source_video_url,
|
|
1452
|
-
max_width: payload.max_width,
|
|
1453
|
-
max_height: payload.max_height,
|
|
1454
|
-
crf: payload.crf,
|
|
1455
|
-
preset: payload.preset,
|
|
1456
|
-
max_video_bitrate_kbps: payload.max_video_bitrate_kbps,
|
|
1457
|
-
audio_bitrate_kbps: payload.audio_bitrate_kbps
|
|
1458
|
-
},
|
|
1459
|
-
outputKey: "normalized.mp4"
|
|
1460
|
-
});
|
|
1461
|
-
ctx.logger.progress(1, "Video normalization complete");
|
|
1462
|
-
return {
|
|
1463
|
-
progress: 1,
|
|
1464
|
-
output: {
|
|
1465
|
-
files: compactUrls([payload.source_video_url, normalized.publicUrl]),
|
|
1466
|
-
sourceVideoUrl: payload.source_video_url,
|
|
1467
|
-
normalizedVideoUrl: normalized.publicUrl,
|
|
1468
|
-
primary_file_url: normalized.publicUrl,
|
|
1469
|
-
video: {
|
|
1470
|
-
file_url: normalized.publicUrl,
|
|
1471
|
-
content_type: normalized.contentType,
|
|
1472
|
-
normalization: normalized.metadata
|
|
1473
|
-
}
|
|
1474
|
-
}
|
|
1475
|
-
};
|
|
1476
|
-
}
|
|
1477
|
-
}
|
|
1478
|
-
});
|
|
1479
|
-
const videoDownloadPrimitive = definePrimitive({
|
|
1480
|
-
id: PRIMITIVE_VIDEO_DOWNLOAD_ID,
|
|
1481
|
-
kind: "video_download",
|
|
1482
|
-
operations: {
|
|
1483
|
-
run: {
|
|
1484
|
-
description: "Download a supported social/media post URL into a durable reusable platform MP4.",
|
|
1485
|
-
inputSchema: downloadVideoPayloadSchema,
|
|
1486
|
-
workflow: "run"
|
|
1487
|
-
}
|
|
1488
|
-
},
|
|
1489
|
-
jobs: {
|
|
1490
|
-
async run(ctx, input) {
|
|
1491
|
-
const payload = downloadVideoPayloadSchema.parse(input);
|
|
1492
|
-
const sourceUrl = payload.source_url;
|
|
1493
|
-
const sourceHost = safeUrlHost(sourceUrl);
|
|
1494
|
-
const apiKey = (config.RAPIDAPI_KEY || "").trim();
|
|
1495
|
-
if (!apiKey) {
|
|
1496
|
-
throw new Error("RAPIDAPI_KEY is not configured.");
|
|
1497
|
-
}
|
|
1498
|
-
ctx.logger.progress(0.12, "Requesting remote video download metadata", {
|
|
1499
|
-
sourceUrl,
|
|
1500
|
-
sourceHost,
|
|
1501
|
-
apiHost: config.RAPIDAPI_VIDEO_DOWNLOAD_HOST
|
|
1502
|
-
});
|
|
1503
|
-
const downloadResponse = await fetch(config.RAPIDAPI_VIDEO_DOWNLOAD_URL, {
|
|
1504
|
-
method: "POST",
|
|
1505
|
-
headers: {
|
|
1506
|
-
"x-rapidapi-key": apiKey,
|
|
1507
|
-
"x-rapidapi-host": config.RAPIDAPI_VIDEO_DOWNLOAD_HOST,
|
|
1508
|
-
"content-type": "application/x-www-form-urlencoded"
|
|
1509
|
-
},
|
|
1510
|
-
body: new URLSearchParams({ url: sourceUrl })
|
|
1511
|
-
});
|
|
1512
|
-
if (!downloadResponse.ok) {
|
|
1513
|
-
const details = await downloadResponse.text().catch(() => "");
|
|
1514
|
-
throw new Error(`Video download lookup failed with HTTP ${downloadResponse.status}${details ? `: ${details.slice(0, 400)}` : ""}`);
|
|
1515
|
-
}
|
|
1516
|
-
const lookup = await downloadResponse.json();
|
|
1517
|
-
const selectedMedia = selectDownloadedVideoMediaCandidate(lookup.medias ?? [], payload.quality);
|
|
1518
|
-
if (!selectedMedia?.url) {
|
|
1519
|
-
throw new Error("Video download lookup returned no playable MP4 media URL.");
|
|
1520
|
-
}
|
|
1521
|
-
ctx.logger.progress(0.45, "Downloading source video", {
|
|
1522
|
-
selectedUrl: selectedMedia.url,
|
|
1523
|
-
selectedQuality: selectedMedia.quality ?? null
|
|
1524
|
-
});
|
|
1525
|
-
const remoteVideo = await fetchRemoteBinary(selectedMedia.url, "video/mp4");
|
|
1526
|
-
const contentType = normalizeVideoContentType(remoteVideo.contentType);
|
|
1527
|
-
const extension = videoExtensionForContentType(contentType);
|
|
1528
|
-
const storedVideo = await storePrimitiveVideo(ctx, {
|
|
1529
|
-
key: `source-video.${extension}`,
|
|
1530
|
-
bytes: remoteVideo.bytes,
|
|
1531
|
-
contentType,
|
|
1532
|
-
metadata: {
|
|
1533
|
-
primitive: "video_download",
|
|
1534
|
-
operation: "download-video",
|
|
1535
|
-
source_url: sourceUrl,
|
|
1536
|
-
source_host: sourceHost,
|
|
1537
|
-
selected_media_url: selectedMedia.url,
|
|
1538
|
-
selected_quality: selectedMedia.quality ?? null,
|
|
1539
|
-
upstream_source: lookup.source ?? null
|
|
1540
|
-
}
|
|
1541
|
-
});
|
|
1542
|
-
// Re-host a DURABLE poster so the frontend never depends on the ephemeral
|
|
1543
|
-
// third-party `lookup.thumbnail` URL. Prefer extracting a frame from the
|
|
1544
|
-
// MP4 we just stored (works even when the provider gives no thumbnail);
|
|
1545
|
-
// fall back to re-hosting the provider thumbnail bytes. Best-effort.
|
|
1546
|
-
let durableThumbnailUrl = null;
|
|
1547
|
-
ctx.logger.progress(0.7, "Extracting thumbnail frame");
|
|
1548
|
-
try {
|
|
1549
|
-
const frame = await executePrimitiveMediaOperation(ctx, {
|
|
1550
|
-
operation: "video_extract_frame",
|
|
1551
|
-
payload: { source_video_url: storedVideo.url, at_ms: 1000, output_format: "jpeg" },
|
|
1552
|
-
outputKey: "thumbnail.jpg"
|
|
1553
|
-
});
|
|
1554
|
-
durableThumbnailUrl = frame.publicUrl;
|
|
1555
|
-
}
|
|
1556
|
-
catch (error) {
|
|
1557
|
-
ctx.logger.progress(0.75, "Thumbnail frame extraction skipped", {
|
|
1558
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1559
|
-
});
|
|
1560
|
-
}
|
|
1561
|
-
if (!durableThumbnailUrl && lookup.thumbnail) {
|
|
1562
|
-
try {
|
|
1563
|
-
const remoteThumb = await fetchRemoteBinary(lookup.thumbnail, "image/jpeg");
|
|
1564
|
-
const thumbContentType = remoteThumb.contentType.startsWith("image/") ? remoteThumb.contentType : "image/jpeg";
|
|
1565
|
-
const storedThumb = await storePrimitiveImage(ctx, {
|
|
1566
|
-
key: "thumbnail.jpg",
|
|
1567
|
-
bytes: remoteThumb.bytes,
|
|
1568
|
-
contentType: thumbContentType,
|
|
1569
|
-
metadata: {
|
|
1570
|
-
primitive: "video_download",
|
|
1571
|
-
operation: "download-video",
|
|
1572
|
-
source_url: sourceUrl,
|
|
1573
|
-
rehosted_from: lookup.thumbnail
|
|
1574
|
-
}
|
|
1575
|
-
});
|
|
1576
|
-
durableThumbnailUrl = storedThumb.url;
|
|
1577
|
-
}
|
|
1578
|
-
catch (error) {
|
|
1579
|
-
ctx.logger.progress(0.78, "Thumbnail re-host skipped", {
|
|
1580
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1581
|
-
});
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
const manifest = {
|
|
1585
|
-
primitive: "video_download",
|
|
1586
|
-
operation: "download-video",
|
|
1587
|
-
sourceUrl,
|
|
1588
|
-
sourceHost,
|
|
1589
|
-
title: lookup.title ?? null,
|
|
1590
|
-
// Raw ephemeral third-party thumbnail (kept for provenance) plus the
|
|
1591
|
-
// durable re-hosted poster the finalizer/frontend should actually use.
|
|
1592
|
-
thumbnail: durableThumbnailUrl ?? lookup.thumbnail ?? null,
|
|
1593
|
-
rawThumbnail: lookup.thumbnail ?? null,
|
|
1594
|
-
duration: lookup.duration ?? null,
|
|
1595
|
-
source: lookup.source ?? null,
|
|
1596
|
-
selectedMedia: {
|
|
1597
|
-
url: selectedMedia.url,
|
|
1598
|
-
quality: selectedMedia.quality ?? null,
|
|
1599
|
-
extension: selectedMedia.extension ?? extension,
|
|
1600
|
-
audioAvailable: selectedMedia.audioAvailable ?? null,
|
|
1601
|
-
videoAvailable: selectedMedia.videoAvailable ?? null
|
|
1602
|
-
},
|
|
1603
|
-
videoUrl: storedVideo.url
|
|
1604
|
-
};
|
|
1605
|
-
const manifestArtifact = payload.save_manifest
|
|
1606
|
-
? await ctx.storage.putJson("manifest.json", manifest)
|
|
1607
|
-
: null;
|
|
1608
|
-
await ctx.billing.record({
|
|
1609
|
-
type: "cpu_estimate",
|
|
1610
|
-
costUsd: config.RAPIDAPI_VIDEO_DOWNLOAD_BASE_COST_USD,
|
|
1611
|
-
costCenterSlug: "rapidapi_video_download",
|
|
1612
|
-
idempotencyKey: `rapidapi_video_download:${sourceUrl}`,
|
|
1613
|
-
occurredAtMs: Date.now(),
|
|
1614
|
-
metadata: {
|
|
1615
|
-
operation: "download-video",
|
|
1616
|
-
source_url: sourceUrl,
|
|
1617
|
-
source_host: sourceHost,
|
|
1618
|
-
selected_quality: selectedMedia.quality ?? null
|
|
1619
|
-
}
|
|
1620
|
-
});
|
|
1621
|
-
await ctx.billing.record({
|
|
1622
|
-
type: "cpu_estimate",
|
|
1623
|
-
costUsd: config.RAPIDAPI_VIDEO_DOWNLOAD_LAMBDA_COST_USD,
|
|
1624
|
-
costCenterSlug: "video_download_lambda",
|
|
1625
|
-
idempotencyKey: `video_download_lambda:${sourceUrl}:${storedVideo.key}`,
|
|
1626
|
-
occurredAtMs: Date.now(),
|
|
1627
|
-
metadata: {
|
|
1628
|
-
operation: "download-video",
|
|
1629
|
-
source_url: sourceUrl,
|
|
1630
|
-
source_host: sourceHost,
|
|
1631
|
-
output_key: storedVideo.key,
|
|
1632
|
-
content_type: contentType,
|
|
1633
|
-
bytes: remoteVideo.bytes.length
|
|
1634
|
-
}
|
|
1635
|
-
});
|
|
1636
|
-
ctx.logger.progress(1, "Video download complete", {
|
|
1637
|
-
videoUrl: storedVideo.url,
|
|
1638
|
-
sourceHost,
|
|
1639
|
-
thumbnailUrl: durableThumbnailUrl
|
|
1640
|
-
});
|
|
1641
|
-
return {
|
|
1642
|
-
progress: 1,
|
|
1643
|
-
output: {
|
|
1644
|
-
files: compactUrls([storedVideo.url, durableThumbnailUrl, manifestArtifact?.url]),
|
|
1645
|
-
sourceUrl,
|
|
1646
|
-
sourceHost,
|
|
1647
|
-
videoUrl: storedVideo.url,
|
|
1648
|
-
sourceVideoUrl: storedVideo.url,
|
|
1649
|
-
primary_file_url: storedVideo.url,
|
|
1650
|
-
title: lookup.title ?? null,
|
|
1651
|
-
// Durable re-hosted poster (falls back to the raw provider thumbnail).
|
|
1652
|
-
thumbnailUrl: durableThumbnailUrl ?? lookup.thumbnail ?? null,
|
|
1653
|
-
thumbnail: durableThumbnailUrl ?? lookup.thumbnail ?? null,
|
|
1654
|
-
duration: lookup.duration ?? null,
|
|
1655
|
-
selectedMediaUrl: selectedMedia.url,
|
|
1656
|
-
manifest: manifestArtifact
|
|
1657
|
-
}
|
|
1658
|
-
};
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
});
|
|
1662
|
-
// Upload-based sibling of video_download: the source video already sits in our
|
|
1663
|
-
// storage (uploaded via the Add Template flow), so there is no social lookup —
|
|
1664
|
-
// just mirror it into a durable job artifact, probe duration, and grab a
|
|
1665
|
-
// thumbnail frame. Output shape matches video_download so the inspiration
|
|
1666
|
-
// finalizer treats both identically.
|
|
1667
|
-
const videoIngestPrimitive = definePrimitive({
|
|
1668
|
-
id: PRIMITIVE_VIDEO_INGEST_ID,
|
|
1669
|
-
kind: "video_ingest",
|
|
1670
|
-
operations: {
|
|
1671
|
-
run: {
|
|
1672
|
-
description: "Ingest an uploaded source video file into a durable reusable platform MP4 with probed metadata and a thumbnail.",
|
|
1673
|
-
inputSchema: ingestVideoPayloadSchema,
|
|
1674
|
-
workflow: "run"
|
|
1675
|
-
}
|
|
1676
|
-
},
|
|
1677
|
-
jobs: {
|
|
1678
|
-
async run(ctx, input) {
|
|
1679
|
-
const payload = ingestVideoPayloadSchema.parse(input);
|
|
1680
|
-
const sourceUrl = payload.source_video_url;
|
|
1681
|
-
const sourceHost = safeUrlHost(sourceUrl);
|
|
1682
|
-
ctx.logger.progress(0.12, "Fetching uploaded source video", { sourceUrl, sourceHost });
|
|
1683
|
-
const remoteVideo = await fetchRemoteBinary(sourceUrl, "video/mp4");
|
|
1684
|
-
const contentType = normalizeVideoContentType(remoteVideo.contentType);
|
|
1685
|
-
const extension = videoExtensionForContentType(contentType);
|
|
1686
|
-
const storedVideo = await storePrimitiveVideo(ctx, {
|
|
1687
|
-
key: `source-video.${extension}`,
|
|
1688
|
-
bytes: remoteVideo.bytes,
|
|
1689
|
-
contentType,
|
|
1690
|
-
metadata: {
|
|
1691
|
-
primitive: "video_ingest",
|
|
1692
|
-
operation: "ingest-video",
|
|
1693
|
-
source_url: sourceUrl,
|
|
1694
|
-
source_host: sourceHost,
|
|
1695
|
-
file_name: payload.file_name ?? null
|
|
1696
|
-
}
|
|
1697
|
-
});
|
|
1698
|
-
// Best-effort enrichment: a probe past the duration cap or a failed
|
|
1699
|
-
// thumbnail must not sink the ingest — the card still finalizes and the
|
|
1700
|
-
// template can decompose later.
|
|
1701
|
-
let durationSeconds = null;
|
|
1702
|
-
ctx.logger.progress(0.55, "Probing uploaded video metadata");
|
|
1703
|
-
try {
|
|
1704
|
-
const probed = await executePrimitiveMediaOperation(ctx, {
|
|
1705
|
-
operation: "video_probe",
|
|
1706
|
-
payload: { source_video_url: storedVideo.url },
|
|
1707
|
-
outputKey: "probe.json"
|
|
1708
|
-
});
|
|
1709
|
-
const probedDuration = Number(probed.metadata?.durationSeconds);
|
|
1710
|
-
durationSeconds = Number.isFinite(probedDuration) && probedDuration > 0 ? probedDuration : null;
|
|
1711
|
-
}
|
|
1712
|
-
catch (error) {
|
|
1713
|
-
ctx.logger.progress(0.6, "Video probe skipped", {
|
|
1714
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1715
|
-
});
|
|
1716
|
-
}
|
|
1717
|
-
let thumbnailUrl = null;
|
|
1718
|
-
ctx.logger.progress(0.7, "Extracting thumbnail frame");
|
|
1719
|
-
try {
|
|
1720
|
-
const frame = await executePrimitiveMediaOperation(ctx, {
|
|
1721
|
-
operation: "video_extract_frame",
|
|
1722
|
-
payload: { source_video_url: storedVideo.url, at_ms: 0, output_format: "jpeg" },
|
|
1723
|
-
outputKey: "thumbnail.jpg"
|
|
1724
|
-
});
|
|
1725
|
-
thumbnailUrl = frame.publicUrl;
|
|
1726
|
-
}
|
|
1727
|
-
catch (error) {
|
|
1728
|
-
ctx.logger.progress(0.75, "Thumbnail extraction skipped", {
|
|
1729
|
-
error: error instanceof Error ? error.message : String(error)
|
|
1730
|
-
});
|
|
1731
|
-
}
|
|
1732
|
-
const manifest = {
|
|
1733
|
-
primitive: "video_ingest",
|
|
1734
|
-
operation: "ingest-video",
|
|
1735
|
-
sourceUrl,
|
|
1736
|
-
sourceHost,
|
|
1737
|
-
// Caller-chosen title only — never the file name. Untitled ingests
|
|
1738
|
-
// finalize with title null so the template displays as its id.
|
|
1739
|
-
title: payload.title ?? null,
|
|
1740
|
-
thumbnail: thumbnailUrl,
|
|
1741
|
-
duration: durationSeconds,
|
|
1742
|
-
source: "upload",
|
|
1743
|
-
videoUrl: storedVideo.url
|
|
1744
|
-
};
|
|
1745
|
-
const manifestArtifact = payload.save_manifest
|
|
1746
|
-
? await ctx.storage.putJson("manifest.json", manifest)
|
|
1747
|
-
: null;
|
|
1748
|
-
await ctx.billing.record({
|
|
1749
|
-
type: "cpu_estimate",
|
|
1750
|
-
costUsd: config.RAPIDAPI_VIDEO_DOWNLOAD_LAMBDA_COST_USD,
|
|
1751
|
-
costCenterSlug: "video_ingest_lambda",
|
|
1752
|
-
idempotencyKey: `video_ingest_lambda:${sourceUrl}:${storedVideo.key}`,
|
|
1753
|
-
occurredAtMs: Date.now(),
|
|
1754
|
-
metadata: {
|
|
1755
|
-
operation: "ingest-video",
|
|
1756
|
-
source_url: sourceUrl,
|
|
1757
|
-
source_host: sourceHost,
|
|
1758
|
-
output_key: storedVideo.key,
|
|
1759
|
-
content_type: contentType,
|
|
1760
|
-
bytes: remoteVideo.bytes.length
|
|
1761
|
-
}
|
|
1762
|
-
});
|
|
1763
|
-
ctx.logger.progress(1, "Video ingest complete", {
|
|
1764
|
-
videoUrl: storedVideo.url,
|
|
1765
|
-
durationSeconds,
|
|
1766
|
-
thumbnailUrl
|
|
1767
|
-
});
|
|
1768
|
-
return {
|
|
1769
|
-
progress: 1,
|
|
1770
|
-
output: {
|
|
1771
|
-
files: compactUrls([storedVideo.url, thumbnailUrl, manifestArtifact?.url]),
|
|
1772
|
-
sourceUrl,
|
|
1773
|
-
sourceHost,
|
|
1774
|
-
videoUrl: storedVideo.url,
|
|
1775
|
-
sourceVideoUrl: storedVideo.url,
|
|
1776
|
-
primary_file_url: storedVideo.url,
|
|
1777
|
-
title: manifest.title,
|
|
1778
|
-
thumbnail: thumbnailUrl,
|
|
1779
|
-
duration: durationSeconds,
|
|
1780
|
-
// Manifest CONTENT (not the artifact record) so the inspiration
|
|
1781
|
-
// finalizer can read thumbnail/title/duration directly.
|
|
1782
|
-
manifest
|
|
1783
|
-
}
|
|
1784
|
-
};
|
|
1785
|
-
}
|
|
1786
|
-
}
|
|
1787
|
-
});
|
|
1788
|
-
const videoTrimPrimitive = definePrimitive({
|
|
1789
|
-
id: PRIMITIVE_VIDEO_TRIM_ID,
|
|
1790
|
-
kind: "video_trim",
|
|
1791
|
-
operations: {
|
|
1792
|
-
run: {
|
|
1793
|
-
description: "Trim a source video into a reusable platform MP4.",
|
|
1794
|
-
inputSchema: trimVideoPayloadSchema,
|
|
1795
|
-
workflow: "run"
|
|
1796
|
-
}
|
|
1797
|
-
},
|
|
1798
|
-
jobs: {
|
|
1799
|
-
async run(ctx, input) {
|
|
1800
|
-
const payload = trimVideoPayloadSchema.parse(input);
|
|
1801
|
-
ctx.logger.progress(0.12, "Trimming source video", {
|
|
1802
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
1803
|
-
});
|
|
1804
|
-
const trimmed = await executePrimitiveMediaOperation(ctx, {
|
|
1805
|
-
operation: "video_trim",
|
|
1806
|
-
payload,
|
|
1807
|
-
outputKey: "trimmed.mp4"
|
|
1808
|
-
});
|
|
1809
|
-
ctx.logger.progress(1, "Video trim complete");
|
|
1810
|
-
return {
|
|
1811
|
-
progress: 1,
|
|
1812
|
-
output: {
|
|
1813
|
-
files: compactUrls([payload.source_video_url, trimmed.publicUrl]),
|
|
1814
|
-
sourceVideoUrl: payload.source_video_url,
|
|
1815
|
-
trimmedVideoUrl: trimmed.publicUrl,
|
|
1816
|
-
primary_file_url: trimmed.publicUrl,
|
|
1817
|
-
video: {
|
|
1818
|
-
file_url: trimmed.publicUrl,
|
|
1819
|
-
content_type: trimmed.contentType,
|
|
1820
|
-
trim: trimmed.metadata
|
|
1821
|
-
}
|
|
1822
|
-
}
|
|
1823
|
-
};
|
|
1824
|
-
}
|
|
1825
|
-
}
|
|
1826
|
-
});
|
|
1827
|
-
// GhostCut task polling budget. The job runner lambda has a 15-minute
|
|
1828
|
-
// timeout; probe + submit + mirror take up the rest. GhostCut usually clears
|
|
1829
|
-
// a sub-3-minute clip in a few minutes.
|
|
1830
|
-
const REMOVE_CAPTIONS_POLL_TIMEOUT_MS = 12 * 60 * 1000;
|
|
1831
|
-
const REMOVE_CAPTIONS_POLL_INTERVAL_MS = 5000;
|
|
1832
|
-
// Removes burned-in captions/subtitles/on-screen text from a source video via
|
|
1833
|
-
// the GhostCut inpainting service — the same plumbing auto-decompose uses
|
|
1834
|
-
// through src/services/ghostcut.ts. Unlike the fork-scoped
|
|
1835
|
-
// /remove-video-captions(-poll) composition routes, where the CLIENT drives
|
|
1836
|
-
// completion to keep per-request lambda time bounded, this primitive runs
|
|
1837
|
-
// inside the job runner and polls GhostCut to completion itself.
|
|
1838
|
-
const videoRemoveCaptionsPrimitive = definePrimitive({
|
|
1839
|
-
id: PRIMITIVE_VIDEO_REMOVE_CAPTIONS_ID,
|
|
1840
|
-
kind: "video_remove_captions",
|
|
1841
|
-
operations: {
|
|
1842
|
-
run: {
|
|
1843
|
-
description: "Remove burned-in captions/subtitles/on-screen text from a source video into a reusable caption-free platform MP4.",
|
|
1844
|
-
inputSchema: removeVideoCaptionsPayloadSchema,
|
|
1845
|
-
workflow: "run"
|
|
1846
|
-
}
|
|
1847
|
-
},
|
|
1848
|
-
jobs: {
|
|
1849
|
-
async run(ctx, input) {
|
|
1850
|
-
const payload = removeVideoCaptionsPayloadSchema.parse(input);
|
|
1851
|
-
if (!isGhostcutConfigured()) {
|
|
1852
|
-
throw new Error("Caption removal is unavailable: GHOSTCUT_KEY/GHOSTCUT_SECRET are not configured on this deployment.");
|
|
1853
|
-
}
|
|
1854
|
-
// Probe first: confirms the source actually has a video stream and
|
|
1855
|
-
// captures the duration that prices the GhostCut task (per-30s chunks).
|
|
1856
|
-
ctx.logger.progress(0.05, "Probing source video");
|
|
1857
|
-
const probed = await executePrimitiveMediaOperation(ctx, {
|
|
1858
|
-
operation: "video_probe",
|
|
1859
|
-
payload: { source_video_url: payload.source_video_url },
|
|
1860
|
-
outputKey: "probe.json"
|
|
1861
|
-
});
|
|
1862
|
-
const probeMetadata = (probed.metadata ?? {});
|
|
1863
|
-
if (probeMetadata.hasVideo === false) {
|
|
1864
|
-
throw new Error("source_video_url has no video stream — caption removal only works on videos.");
|
|
1865
|
-
}
|
|
1866
|
-
const probedDurationSeconds = Number(probeMetadata.durationSeconds ?? 0);
|
|
1867
|
-
const durationSeconds = Number.isFinite(probedDurationSeconds) && probedDurationSeconds > 0 ? probedDurationSeconds : 30;
|
|
1868
|
-
// GhostCut is a per-FINISHED-CLIP tool, never a long-form pass. Clip
|
|
1869
|
-
// hunting handles "no captions" as a scene-selection filter (avoid_text);
|
|
1870
|
-
// after the hunt, apply caption removal to individual finished clips.
|
|
1871
|
-
if (config.GHOSTCUT_MAX_DURATION_SEC > 0 && durationSeconds > config.GHOSTCUT_MAX_DURATION_SEC) {
|
|
1872
|
-
throw new Error(`Caption removal is limited to videos up to ${Math.round(config.GHOSTCUT_MAX_DURATION_SEC / 60)} minutes (this source is ${Math.round(durationSeconds / 60)} min). ` +
|
|
1873
|
-
`For long-form video, hunt clips first (POST /clips/scan with avoid_text/no-captions guidance selects text-free scenes), then remove captions from the finished clips you keep.`);
|
|
1874
|
-
}
|
|
1875
|
-
const cost = estimateGhostcutCostUsd(durationSeconds);
|
|
1876
|
-
ctx.logger.progress(0.12, "Submitting caption-removal task", {
|
|
1877
|
-
durationSeconds: Number(durationSeconds.toFixed(3)),
|
|
1878
|
-
estimatedChunks: cost.chunks,
|
|
1879
|
-
estimatedUsd: cost.usd
|
|
1880
|
-
});
|
|
1881
|
-
const submission = await submitSubtitleRemoval(payload.source_video_url);
|
|
1882
|
-
const startedAtMs = Date.now();
|
|
1883
|
-
let cleanedVideoUrl = null;
|
|
1884
|
-
while (cleanedVideoUrl === null) {
|
|
1885
|
-
if (Date.now() - startedAtMs > REMOVE_CAPTIONS_POLL_TIMEOUT_MS) {
|
|
1886
|
-
throw new Error(`GhostCut task ${submission.taskId} did not finish within ${Math.round(REMOVE_CAPTIONS_POLL_TIMEOUT_MS / 60000)} minutes.`);
|
|
1887
|
-
}
|
|
1888
|
-
const status = await pollGhostcutStatus(submission.taskId);
|
|
1889
|
-
if (status.state === "failed") {
|
|
1890
|
-
throw new Error(`GhostCut task ${submission.taskId} failed: ${status.reason}`);
|
|
1891
|
-
}
|
|
1892
|
-
if (status.state === "done") {
|
|
1893
|
-
cleanedVideoUrl = status.videoUrl;
|
|
1894
|
-
break;
|
|
1895
|
-
}
|
|
1896
|
-
const upstreamProgress = Math.max(0, Math.min(100, status.progress));
|
|
1897
|
-
ctx.logger.progress(0.15 + (upstreamProgress / 100) * 0.65, "Removing burned-in captions", {
|
|
1898
|
-
taskId: submission.taskId,
|
|
1899
|
-
upstreamProgress
|
|
1900
|
-
});
|
|
1901
|
-
await new Promise((resolve) => setTimeout(resolve, REMOVE_CAPTIONS_POLL_INTERVAL_MS));
|
|
1902
|
-
}
|
|
1903
|
-
// GhostCut hosts results on a short-lived third-party CDN — mirror the
|
|
1904
|
-
// cleaned MP4 into platform storage so the output URL is durable.
|
|
1905
|
-
ctx.logger.progress(0.85, "Mirroring caption-free video to platform storage");
|
|
1906
|
-
const remoteVideo = await fetchRemoteBinary(cleanedVideoUrl, "video/mp4");
|
|
1907
|
-
const stored = await storePrimitiveVideo(ctx, {
|
|
1908
|
-
key: "captions-removed.mp4",
|
|
1909
|
-
bytes: remoteVideo.bytes,
|
|
1910
|
-
contentType: "video/mp4",
|
|
1911
|
-
metadata: {
|
|
1912
|
-
primitive: "video_remove_captions",
|
|
1913
|
-
operation: "remove-video-captions",
|
|
1914
|
-
source_video_url: payload.source_video_url,
|
|
1915
|
-
ghostcut_task_id: submission.taskId
|
|
1916
|
-
}
|
|
1917
|
-
});
|
|
1918
|
-
await ctx.billing.record({
|
|
1919
|
-
type: "cpu_estimate",
|
|
1920
|
-
costUsd: cost.usd,
|
|
1921
|
-
costCenterSlug: "ghostcut_subtitle_removal",
|
|
1922
|
-
idempotencyKey: `ghostcut_subtitle_removal:${submission.taskId}`,
|
|
1923
|
-
occurredAtMs: Date.now(),
|
|
1924
|
-
metadata: {
|
|
1925
|
-
operation: "remove-video-captions",
|
|
1926
|
-
source_video_url: payload.source_video_url,
|
|
1927
|
-
duration_seconds: Number(durationSeconds.toFixed(3)),
|
|
1928
|
-
chunk_count: cost.chunks,
|
|
1929
|
-
usd_per_chunk: cost.usdPerChunk,
|
|
1930
|
-
chunk_seconds: 30,
|
|
1931
|
-
task_id: submission.taskId
|
|
1932
|
-
}
|
|
1933
|
-
});
|
|
1934
|
-
ctx.logger.progress(1, "Caption removal complete", { videoUrl: stored.url });
|
|
1935
|
-
return {
|
|
1936
|
-
progress: 1,
|
|
1937
|
-
output: {
|
|
1938
|
-
files: compactUrls([payload.source_video_url, stored.url]),
|
|
1939
|
-
sourceVideoUrl: payload.source_video_url,
|
|
1940
|
-
captionsRemovedVideoUrl: stored.url,
|
|
1941
|
-
primary_file_url: stored.url,
|
|
1942
|
-
video: {
|
|
1943
|
-
file_url: stored.url,
|
|
1944
|
-
content_type: stored.contentType,
|
|
1945
|
-
duration_seconds: Number(durationSeconds.toFixed(3)),
|
|
1946
|
-
ghostcut_task_id: submission.taskId
|
|
1947
|
-
}
|
|
1948
|
-
}
|
|
1949
|
-
};
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
1952
|
-
});
|
|
1953
|
-
const videoExtractAudioPrimitive = definePrimitive({
|
|
1954
|
-
id: PRIMITIVE_VIDEO_EXTRACT_AUDIO_ID,
|
|
1955
|
-
kind: "video_extract_audio",
|
|
1956
|
-
operations: {
|
|
1957
|
-
run: {
|
|
1958
|
-
description: "Extract audio from a source video into a reusable platform audio file.",
|
|
1959
|
-
inputSchema: extractAudioPayloadSchema,
|
|
1960
|
-
workflow: "run"
|
|
1961
|
-
}
|
|
1962
|
-
},
|
|
1963
|
-
jobs: {
|
|
1964
|
-
async run(ctx, input) {
|
|
1965
|
-
const payload = extractAudioPayloadSchema.parse(input);
|
|
1966
|
-
ctx.logger.progress(0.12, "Extracting audio from source video", {
|
|
1967
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
1968
|
-
});
|
|
1969
|
-
const extracted = await executePrimitiveMediaOperation(ctx, {
|
|
1970
|
-
operation: "video_extract_audio",
|
|
1971
|
-
payload,
|
|
1972
|
-
outputKey: `audio.${payload.output_format}`
|
|
1973
|
-
});
|
|
1974
|
-
ctx.logger.progress(1, "Video audio extraction complete");
|
|
1975
|
-
return {
|
|
1976
|
-
progress: 1,
|
|
1977
|
-
output: {
|
|
1978
|
-
files: compactUrls([payload.source_video_url, extracted.publicUrl]),
|
|
1979
|
-
sourceVideoUrl: payload.source_video_url,
|
|
1980
|
-
audioUrl: extracted.publicUrl,
|
|
1981
|
-
primary_file_url: extracted.publicUrl,
|
|
1982
|
-
audio: {
|
|
1983
|
-
file_url: extracted.publicUrl,
|
|
1984
|
-
content_type: extracted.contentType,
|
|
1985
|
-
extraction: extracted.metadata
|
|
1986
|
-
}
|
|
1987
|
-
}
|
|
1988
|
-
};
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
});
|
|
1992
|
-
const audioTrimPrimitive = definePrimitive({
|
|
1993
|
-
id: PRIMITIVE_AUDIO_TRIM_ID,
|
|
1994
|
-
kind: "audio_trim",
|
|
1995
|
-
operations: {
|
|
1996
|
-
run: {
|
|
1997
|
-
description: "Trim a source audio file into a reusable platform audio file.",
|
|
1998
|
-
inputSchema: trimAudioPayloadSchema,
|
|
1999
|
-
workflow: "run"
|
|
2000
|
-
}
|
|
2001
|
-
},
|
|
2002
|
-
jobs: {
|
|
2003
|
-
async run(ctx, input) {
|
|
2004
|
-
const payload = trimAudioPayloadSchema.parse(input);
|
|
2005
|
-
ctx.logger.progress(0.12, "Trimming source audio", {
|
|
2006
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
2007
|
-
});
|
|
2008
|
-
const trimmed = await executePrimitiveMediaOperation(ctx, {
|
|
2009
|
-
operation: "audio_trim",
|
|
2010
|
-
payload,
|
|
2011
|
-
outputKey: `trimmed.${payload.output_format}`
|
|
2012
|
-
});
|
|
2013
|
-
ctx.logger.progress(1, "Audio trim complete");
|
|
2014
|
-
return {
|
|
2015
|
-
progress: 1,
|
|
2016
|
-
output: {
|
|
2017
|
-
files: compactUrls([payload.source_audio_url, trimmed.publicUrl]),
|
|
2018
|
-
sourceAudioUrl: payload.source_audio_url,
|
|
2019
|
-
trimmedAudioUrl: trimmed.publicUrl,
|
|
2020
|
-
primary_file_url: trimmed.publicUrl,
|
|
2021
|
-
audio: {
|
|
2022
|
-
file_url: trimmed.publicUrl,
|
|
2023
|
-
content_type: trimmed.contentType,
|
|
2024
|
-
trim: trimmed.metadata
|
|
2025
|
-
}
|
|
2026
|
-
}
|
|
2027
|
-
};
|
|
2028
|
-
}
|
|
2029
|
-
}
|
|
2030
|
-
});
|
|
2031
|
-
const ttsPrimitive = definePrimitive({
|
|
2032
|
-
id: PRIMITIVE_TTS_ID,
|
|
2033
|
-
kind: "tts",
|
|
2034
|
-
operations: {
|
|
2035
|
-
run: {
|
|
2036
|
-
description: "Generate spoken narration audio from text (TTS) with a promptable voice style, persisted as a reusable platform audio URL.",
|
|
2037
|
-
inputSchema: ttsPayloadSchema,
|
|
2038
|
-
workflow: "run"
|
|
2039
|
-
}
|
|
2040
|
-
},
|
|
2041
|
-
jobs: {
|
|
2042
|
-
async run(ctx, input) {
|
|
2043
|
-
const payload = ttsPayloadSchema.parse(input);
|
|
2044
|
-
ctx.logger.progress(0.12, "Generating speech audio");
|
|
2045
|
-
// A bare voice preset implies its provider ("Puck" → gemini) — blindly
|
|
2046
|
-
// defaulting to openai would guarantee a cross-family voice failure.
|
|
2047
|
-
const provider = payload.provider ?? inferSpeechProviderForVoice(payload.voice) ?? "openai";
|
|
2048
|
-
const model = payload.model ?? defaultSpeechModelFor("tts", provider) ?? "";
|
|
2049
|
-
const generated = await ctx.providers.generateSpeech({
|
|
2050
|
-
provider,
|
|
2051
|
-
model,
|
|
2052
|
-
text: payload.text,
|
|
2053
|
-
voice: payload.voice,
|
|
2054
|
-
instructions: payload.instructions,
|
|
2055
|
-
responseFormat: payload.output_format,
|
|
2056
|
-
useWalletCredits: payload.use_wallet_credits
|
|
2057
|
-
});
|
|
2058
|
-
// Route may have resolved to a different provider (e.g. platform ElevenLabs
|
|
2059
|
-
// when use_wallet_credits is on); report what actually ran.
|
|
2060
|
-
const effectiveProvider = generated.effectiveProvider;
|
|
2061
|
-
const effectiveModel = generated.effectiveModel;
|
|
2062
|
-
if (generated.billing) {
|
|
2063
|
-
await ctx.billing.record({
|
|
2064
|
-
type: "ai_generation",
|
|
2065
|
-
costUsd: generated.billing.costUsd,
|
|
2066
|
-
costCenterSlug: generated.billing.costCenterSlug,
|
|
2067
|
-
metadata: { ...generated.billing.metadata, primitive: "tts", voice: payload.voice ?? null }
|
|
2068
|
-
});
|
|
2069
|
-
}
|
|
2070
|
-
// Gemini/ElevenLabs TTS may answer in wav regardless of the requested
|
|
2071
|
-
// format; name the artifact after what actually came back.
|
|
2072
|
-
const extension = generated.contentType.includes("wav") ? "wav" : "mp3";
|
|
2073
|
-
const stored = await ctx.storage.putBuffer(`speech.${extension}`, generated.bytes, {
|
|
2074
|
-
contentType: generated.contentType,
|
|
2075
|
-
kind: "audio",
|
|
2076
|
-
metadata: {
|
|
2077
|
-
primitive: "tts",
|
|
2078
|
-
provider: effectiveProvider,
|
|
2079
|
-
model: effectiveModel,
|
|
2080
|
-
voice: payload.voice ?? null,
|
|
2081
|
-
styleInstructions: payload.instructions ?? null,
|
|
2082
|
-
wallet_billed: Boolean(generated.billing)
|
|
2083
|
-
}
|
|
2084
|
-
});
|
|
2085
|
-
ctx.logger.progress(1, "Speech generation complete");
|
|
2086
|
-
return {
|
|
2087
|
-
progress: 1,
|
|
2088
|
-
output: {
|
|
2089
|
-
files: compactUrls([stored.url]),
|
|
2090
|
-
primary_file_url: stored.url,
|
|
2091
|
-
audioUrl: stored.url,
|
|
2092
|
-
audio: {
|
|
2093
|
-
file_url: stored.url,
|
|
2094
|
-
content_type: generated.contentType,
|
|
2095
|
-
provider: effectiveProvider,
|
|
2096
|
-
model: effectiveModel,
|
|
2097
|
-
voice: payload.voice ?? null,
|
|
2098
|
-
style_instructions: payload.instructions ?? null,
|
|
2099
|
-
text_length: payload.text.length,
|
|
2100
|
-
wallet_billed: Boolean(generated.billing)
|
|
2101
|
-
}
|
|
2102
|
-
}
|
|
2103
|
-
};
|
|
2104
|
-
}
|
|
2105
|
-
}
|
|
2106
|
-
});
|
|
2107
|
-
const musicPrimitive = definePrimitive({
|
|
2108
|
-
id: PRIMITIVE_MUSIC_ID,
|
|
2109
|
-
kind: "music",
|
|
2110
|
-
operations: {
|
|
2111
|
-
run: {
|
|
2112
|
-
description: "Generate music (instrumental or with lyrics) from a prompt or a composition plan via ElevenLabs, persisted as a reusable platform audio URL.",
|
|
2113
|
-
inputSchema: musicPayloadSchema,
|
|
2114
|
-
workflow: "run"
|
|
2115
|
-
}
|
|
2116
|
-
},
|
|
2117
|
-
jobs: {
|
|
2118
|
-
async run(ctx, input) {
|
|
2119
|
-
const payload = musicPayloadSchema.parse(input);
|
|
2120
|
-
ctx.logger.progress(0.12, "Composing music");
|
|
2121
|
-
const generated = await ctx.providers.generateMusic({
|
|
2122
|
-
prompt: payload.prompt,
|
|
2123
|
-
musicLengthMs: payload.music_length_ms,
|
|
2124
|
-
compositionPlan: payload.composition_plan,
|
|
2125
|
-
model: payload.model,
|
|
2126
|
-
outputFormat: payload.output_format,
|
|
2127
|
-
useWalletCredits: payload.use_wallet_credits
|
|
2128
|
-
});
|
|
2129
|
-
if (generated.billing) {
|
|
2130
|
-
await ctx.billing.record({
|
|
2131
|
-
type: "ai_generation",
|
|
2132
|
-
costUsd: generated.billing.costUsd,
|
|
2133
|
-
costCenterSlug: generated.billing.costCenterSlug,
|
|
2134
|
-
metadata: { ...generated.billing.metadata, primitive: "music", prompt: payload.prompt ?? null }
|
|
2135
|
-
});
|
|
2136
|
-
}
|
|
2137
|
-
const stored = await ctx.storage.putBuffer("music.mp3", generated.bytes, {
|
|
2138
|
-
contentType: generated.contentType,
|
|
2139
|
-
kind: "audio",
|
|
2140
|
-
metadata: {
|
|
2141
|
-
primitive: "music",
|
|
2142
|
-
provider: "elevenlabs",
|
|
2143
|
-
music_length_ms: generated.musicLengthMs,
|
|
2144
|
-
wallet_billed: Boolean(generated.billing)
|
|
2145
|
-
}
|
|
2146
|
-
});
|
|
2147
|
-
ctx.logger.progress(1, "Music generation complete");
|
|
2148
|
-
return {
|
|
2149
|
-
progress: 1,
|
|
2150
|
-
output: {
|
|
2151
|
-
files: compactUrls([stored.url]),
|
|
2152
|
-
primary_file_url: stored.url,
|
|
2153
|
-
audioUrl: stored.url,
|
|
2154
|
-
audio: {
|
|
2155
|
-
file_url: stored.url,
|
|
2156
|
-
content_type: generated.contentType,
|
|
2157
|
-
provider: "elevenlabs",
|
|
2158
|
-
music_length_ms: generated.musicLengthMs,
|
|
2159
|
-
prompt: payload.prompt ?? null,
|
|
2160
|
-
wallet_billed: Boolean(generated.billing)
|
|
2161
|
-
}
|
|
2162
|
-
}
|
|
2163
|
-
};
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2166
|
-
});
|
|
2167
|
-
// Shared source→speech-ready-audio loader for the STT-family primitives.
|
|
2168
|
-
// Plain audio URLs pass through; videos (and unknown containers) go through
|
|
2169
|
-
// the ffmpeg seam, which demuxes and normalizes into a small mono mp3 the
|
|
2170
|
-
// speech providers accept.
|
|
2171
|
-
async function loadSpeechAudioForPrimitive(ctx, payload) {
|
|
2172
|
-
const sourceKind = payload.media_type === "auto" ? inferSpeechSourceKind(payload.source_url) : payload.media_type;
|
|
2173
|
-
let audioBytes;
|
|
2174
|
-
let audioContentType;
|
|
2175
|
-
let audioUrl;
|
|
2176
|
-
let extractionMetadata = null;
|
|
2177
|
-
if (sourceKind === "audio") {
|
|
2178
|
-
ctx.logger.progress(0.1, "Fetching source audio");
|
|
2179
|
-
const response = await fetch(payload.source_url);
|
|
2180
|
-
if (!response.ok) {
|
|
2181
|
-
throw new Error(`Could not fetch source audio (${response.status}).`);
|
|
2182
|
-
}
|
|
2183
|
-
audioBytes = new Uint8Array(await response.arrayBuffer());
|
|
2184
|
-
audioContentType = response.headers.get("content-type")?.split(";")[0]?.trim() || "audio/mpeg";
|
|
2185
|
-
audioUrl = payload.source_url;
|
|
2186
|
-
}
|
|
2187
|
-
else {
|
|
2188
|
-
ctx.logger.progress(0.1, "Extracting audio track from source", {
|
|
2189
|
-
executionMode: config.VIDFARM_PRIMITIVE_MEDIA_LAMBDA_URL ? "lambda" : "local_fallback"
|
|
2190
|
-
});
|
|
2191
|
-
const extracted = await executePrimitiveMediaOperation(ctx, {
|
|
2192
|
-
operation: "video_extract_audio",
|
|
2193
|
-
payload: {
|
|
2194
|
-
source_video_url: payload.source_url,
|
|
2195
|
-
output_format: "mp3",
|
|
2196
|
-
audio_bitrate_kbps: 64
|
|
2197
|
-
},
|
|
2198
|
-
outputKey: "source-audio.mp3"
|
|
2199
|
-
});
|
|
2200
|
-
if (!extracted.publicUrl) {
|
|
2201
|
-
throw new Error("Audio extraction produced no readable audio URL.");
|
|
2202
|
-
}
|
|
2203
|
-
const response = await fetch(extracted.publicUrl);
|
|
2204
|
-
if (!response.ok) {
|
|
2205
|
-
throw new Error(`Could not read extracted audio (${response.status}).`);
|
|
2206
|
-
}
|
|
2207
|
-
audioBytes = new Uint8Array(await response.arrayBuffer());
|
|
2208
|
-
audioContentType = extracted.contentType || "audio/mpeg";
|
|
2209
|
-
audioUrl = extracted.publicUrl;
|
|
2210
|
-
extractionMetadata = extracted.metadata ?? null;
|
|
2211
|
-
}
|
|
2212
|
-
if (audioBytes.byteLength > MAX_TRANSCRIBE_AUDIO_BYTES) {
|
|
2213
|
-
const sizeMb = Math.round(audioBytes.byteLength / (1024 * 1024));
|
|
2214
|
-
const maxMb = Math.round(MAX_TRANSCRIBE_AUDIO_BYTES / (1024 * 1024));
|
|
2215
|
-
throw new Error(`Audio is too large to transcribe (${sizeMb}MB > ${maxMb}MB, roughly 40 minutes of speech). Trim the source with /videos/trim or /audio/trim and transcribe in parts.`);
|
|
2216
|
-
}
|
|
2217
|
-
return { audioBytes, audioContentType, audioUrl, extractionMetadata };
|
|
2218
|
-
}
|
|
2219
|
-
const sttPrimitive = definePrimitive({
|
|
2220
|
-
id: PRIMITIVE_STT_ID,
|
|
2221
|
-
kind: "stt",
|
|
2222
|
-
operations: {
|
|
2223
|
-
run: {
|
|
2224
|
-
description: "Transcribe speech from a video or audio URL (STT) into a simple subtitle transcript plus speaker-attributed multi-narration segments.",
|
|
2225
|
-
inputSchema: sttPayloadSchema,
|
|
2226
|
-
workflow: "run"
|
|
2227
|
-
}
|
|
2228
|
-
},
|
|
2229
|
-
jobs: {
|
|
2230
|
-
async run(ctx, input) {
|
|
2231
|
-
const payload = sttPayloadSchema.parse(input);
|
|
2232
|
-
const { audioBytes, audioContentType, extractionMetadata } = await loadSpeechAudioForPrimitive(ctx, payload);
|
|
2233
|
-
ctx.logger.progress(0.45, "Transcribing speech");
|
|
2234
|
-
const requestedProvider = payload.provider ?? (payload.word_timestamps ? "openai" : "gemini");
|
|
2235
|
-
const requestedModel = payload.model ?? defaultSpeechModelFor("stt", requestedProvider) ?? "";
|
|
2236
|
-
const transcription = await ctx.providers.transcribeSpeech({
|
|
2237
|
-
provider: requestedProvider,
|
|
2238
|
-
model: requestedModel,
|
|
2239
|
-
audio: audioBytes,
|
|
2240
|
-
contentType: audioContentType,
|
|
2241
|
-
prompt: payload.prompt,
|
|
2242
|
-
language: payload.language,
|
|
2243
|
-
diarize: payload.diarize,
|
|
2244
|
-
wordTimestamps: payload.word_timestamps,
|
|
2245
|
-
useWalletCredits: payload.use_wallet_credits
|
|
2246
|
-
});
|
|
2247
|
-
const provider = transcription.effectiveProvider;
|
|
2248
|
-
const model = transcription.effectiveModel;
|
|
2249
|
-
if (transcription.billing) {
|
|
2250
|
-
await ctx.billing.record({
|
|
2251
|
-
type: "ai_generation",
|
|
2252
|
-
costUsd: transcription.billing.costUsd,
|
|
2253
|
-
costCenterSlug: transcription.billing.costCenterSlug,
|
|
2254
|
-
metadata: { ...transcription.billing.metadata, primitive: "stt", source_url: payload.source_url }
|
|
2255
|
-
});
|
|
2256
|
-
}
|
|
2257
|
-
ctx.logger.progress(0.85, "Storing transcript artifacts");
|
|
2258
|
-
const speakers = [...new Set(transcription.segments.map((segment) => segment.speaker))];
|
|
2259
|
-
const srt = buildSrtFromSegments(transcription.segments);
|
|
2260
|
-
const storedText = await ctx.storage.putText("transcript.txt", transcription.text || "");
|
|
2261
|
-
const storedSrt = srt ? await ctx.storage.putText("transcript.srt", srt, "application/x-subrip; charset=utf-8") : null;
|
|
2262
|
-
const storedJson = await ctx.storage.putJson("transcript.json", {
|
|
2263
|
-
source_url: payload.source_url,
|
|
2264
|
-
provider,
|
|
2265
|
-
model,
|
|
2266
|
-
language: transcription.language,
|
|
2267
|
-
diarization: transcription.diarization,
|
|
2268
|
-
text: transcription.text,
|
|
2269
|
-
speakers,
|
|
2270
|
-
segments: transcription.segments,
|
|
2271
|
-
...(extractionMetadata ? { extraction: extractionMetadata } : {})
|
|
2272
|
-
});
|
|
2273
|
-
const inlineOk = (transcription.text?.length ?? 0) <= MAX_INLINE_TRANSCRIPT_CHARS;
|
|
2274
|
-
ctx.logger.progress(1, "Speech transcription complete");
|
|
2275
|
-
return {
|
|
2276
|
-
progress: 1,
|
|
2277
|
-
output: {
|
|
2278
|
-
files: compactUrls([storedJson.url, storedSrt?.url, storedText.url]),
|
|
2279
|
-
primary_file_url: storedJson.url,
|
|
2280
|
-
sourceUrl: payload.source_url,
|
|
2281
|
-
text: inlineOk ? transcription.text : transcription.text.slice(0, MAX_INLINE_TRANSCRIPT_CHARS),
|
|
2282
|
-
text_truncated: !inlineOk,
|
|
2283
|
-
language: transcription.language,
|
|
2284
|
-
diarization: transcription.diarization,
|
|
2285
|
-
// Format 1 — SIMPLE SUBTITLE VERSION: plain transcript + SRT cues (no speakers).
|
|
2286
|
-
subtitles: {
|
|
2287
|
-
srt: inlineOk ? srt : null,
|
|
2288
|
-
srt_url: storedSrt?.url ?? null,
|
|
2289
|
-
text_url: storedText.url
|
|
2290
|
-
},
|
|
2291
|
-
// Format 2 — ADVANCED MULTI-SPEAKER VERSION: speaker-attributed timed segments.
|
|
2292
|
-
speakers,
|
|
2293
|
-
segments: inlineOk ? transcription.segments : undefined,
|
|
2294
|
-
transcript: {
|
|
2295
|
-
json_url: storedJson.url,
|
|
2296
|
-
content_type: "application/json"
|
|
2297
|
-
}
|
|
2298
|
-
}
|
|
2299
|
-
};
|
|
2300
|
-
}
|
|
2301
|
-
}
|
|
2302
|
-
});
|
|
2303
|
-
// Speech → animated captions convenience primitive. One job: transcribe the
|
|
2304
|
-
// narration WITH word-level timings (openai/whisper-1 real timestamps, other
|
|
2305
|
-
// providers estimated), page it into CapCut-style cues, and return the cue
|
|
2306
|
-
// list shaped EXACTLY for the editor's `editor_action set_captions` — the
|
|
2307
|
-
// output even carries a ready `set_captions_action` object so an agent can
|
|
2308
|
-
// pipe it through without any cue math. Composition-agnostic: it never
|
|
2309
|
-
// mutates a fork; POST /api/v1/compositions/:forkId/captions is the one-step
|
|
2310
|
-
// variant that applies server-side.
|
|
2311
|
-
const captionsPrimitive = definePrimitive({
|
|
2312
|
-
id: PRIMITIVE_CAPTIONS_ID,
|
|
2313
|
-
kind: "captions",
|
|
2314
|
-
operations: {
|
|
2315
|
-
run: {
|
|
2316
|
-
description: "Transcribe narration (or page a script) into animated word-by-word caption cues ready for the editor's set_captions action.",
|
|
2317
|
-
inputSchema: captionsPayloadSchema,
|
|
2318
|
-
workflow: "run"
|
|
2319
|
-
}
|
|
2320
|
-
},
|
|
2321
|
-
jobs: {
|
|
2322
|
-
async run(ctx, input) {
|
|
2323
|
-
const payload = captionsPayloadSchema.parse(input);
|
|
2324
|
-
const preset = payload.style
|
|
2325
|
-
? captionStylePresetById(payload.style)
|
|
2326
|
-
: payload.animation
|
|
2327
|
-
? null
|
|
2328
|
-
: CAPTION_STYLE_PRESETS[0];
|
|
2329
|
-
if (payload.style && !preset) {
|
|
2330
|
-
throw new Error(`Unknown caption style "${payload.style}". Use one of: ${CAPTION_STYLE_PRESETS.map((entry) => entry.id).join(", ")} — or pass animation directly.`);
|
|
2331
|
-
}
|
|
2332
|
-
const animation = payload.animation ?? preset?.animation ?? "highlight-pop";
|
|
2333
|
-
let cues;
|
|
2334
|
-
let wordTimestampSource = "estimated";
|
|
2335
|
-
let transcriptText = null;
|
|
2336
|
-
let transcriptArtifacts = {};
|
|
2337
|
-
if (payload.text?.trim()) {
|
|
2338
|
-
const tokens = payload.text.trim().split(/\s+/).filter(Boolean);
|
|
2339
|
-
const windowDuration = payload.window_duration ?? Math.max(0.4, tokens.length * 0.32);
|
|
2340
|
-
cues = cuesFromText(payload.text, payload.window_start + payload.offset_sec, windowDuration, {
|
|
2341
|
-
maxWordsPerCue: payload.max_words_per_cue
|
|
2342
|
-
});
|
|
2343
|
-
}
|
|
2344
|
-
else {
|
|
2345
|
-
const { audioBytes, audioContentType } = await loadSpeechAudioForPrimitive(ctx, {
|
|
2346
|
-
source_url: payload.source_url,
|
|
2347
|
-
media_type: payload.media_type
|
|
2348
|
-
});
|
|
2349
|
-
ctx.logger.progress(0.45, "Transcribing speech with word timings");
|
|
2350
|
-
// openai first: whisper-1 is the only provider with REAL word-level
|
|
2351
|
-
// timestamps; providerAttempts falls back to the caller's other keys.
|
|
2352
|
-
const provider = payload.provider ?? "openai";
|
|
2353
|
-
const model = payload.model ?? defaultSpeechModelFor("stt", provider) ?? "";
|
|
2354
|
-
const transcription = await ctx.providers.transcribeSpeech({
|
|
2355
|
-
provider,
|
|
2356
|
-
model,
|
|
2357
|
-
audio: audioBytes,
|
|
2358
|
-
contentType: audioContentType,
|
|
2359
|
-
prompt: payload.prompt,
|
|
2360
|
-
language: payload.language,
|
|
2361
|
-
diarize: false,
|
|
2362
|
-
wordTimestamps: true
|
|
2363
|
-
});
|
|
2364
|
-
if (!transcription.segments.length) {
|
|
2365
|
-
throw new Error("Transcription returned no speech segments — the source may be silent.");
|
|
2366
|
-
}
|
|
2367
|
-
transcriptText = transcription.text || null;
|
|
2368
|
-
wordTimestampSource = transcription.segments.some((segment) => segment.words?.length) ? "provider" : "estimated";
|
|
2369
|
-
ctx.logger.progress(0.7, "Paging transcript into caption cues");
|
|
2370
|
-
cues = buildCaptionCues(transcription.segments, {
|
|
2371
|
-
maxWordsPerCue: payload.max_words_per_cue,
|
|
2372
|
-
offsetSec: payload.offset_sec
|
|
2373
|
-
});
|
|
2374
|
-
const srt = buildSrtFromSegments(transcription.segments);
|
|
2375
|
-
const storedSrt = srt ? await ctx.storage.putText("transcript.srt", srt, "application/x-subrip; charset=utf-8") : null;
|
|
2376
|
-
if (storedSrt?.url)
|
|
2377
|
-
transcriptArtifacts.srt_url = storedSrt.url;
|
|
2378
|
-
}
|
|
2379
|
-
if (!cues.length) {
|
|
2380
|
-
throw new Error("No caption cues produced from the source.");
|
|
2381
|
-
}
|
|
2382
|
-
// Style echo in the exact field names editor_action set_captions takes,
|
|
2383
|
-
// so the agent can spread this straight into the action.
|
|
2384
|
-
const styleFields = {
|
|
2385
|
-
...(preset ? { caption_style: preset.id } : {}),
|
|
2386
|
-
caption_animation: animation,
|
|
2387
|
-
...(payload.active_color ?? preset?.activeColor ? { caption_active_color: payload.active_color ?? preset?.activeColor } : {}),
|
|
2388
|
-
...(payload.highlight_color ?? preset?.highlightColor ? { caption_highlight_color: payload.highlight_color ?? preset?.highlightColor } : {}),
|
|
2389
|
-
...((payload.uppercase ?? preset?.uppercase) !== undefined ? { caption_uppercase: payload.uppercase ?? preset?.uppercase } : {}),
|
|
2390
|
-
...(payload.color ?? preset?.color ? { color: payload.color ?? preset?.color } : {}),
|
|
2391
|
-
...(payload.background ?? preset?.background ? { background: payload.background ?? preset?.background } : {}),
|
|
2392
|
-
...(payload.background_style ?? preset?.textBackgroundStyle ? { background_style: payload.background_style ?? preset?.textBackgroundStyle } : {}),
|
|
2393
|
-
...(payload.font_family ? { font_family: payload.font_family } : {}),
|
|
2394
|
-
...(payload.font_size ? { font_size: payload.font_size } : {})
|
|
2395
|
-
};
|
|
2396
|
-
const setCaptionsAction = {
|
|
2397
|
-
action_type: "set_captions",
|
|
2398
|
-
captions: cues,
|
|
2399
|
-
...styleFields,
|
|
2400
|
-
explanation: "Apply transcribed animated captions."
|
|
2401
|
-
};
|
|
2402
|
-
ctx.logger.progress(0.85, "Storing caption artifacts");
|
|
2403
|
-
const storedCaptions = await ctx.storage.putJson("captions.json", {
|
|
2404
|
-
source_url: payload.source_url ?? null,
|
|
2405
|
-
word_timestamps: wordTimestampSource,
|
|
2406
|
-
style: styleFields,
|
|
2407
|
-
cue_count: cues.length,
|
|
2408
|
-
cues,
|
|
2409
|
-
set_captions_action: setCaptionsAction,
|
|
2410
|
-
...(transcriptText ? { transcript_text: transcriptText } : {})
|
|
2411
|
-
});
|
|
2412
|
-
const cuesInline = JSON.stringify(cues).length <= 60_000;
|
|
2413
|
-
ctx.logger.progress(1, "Caption cues ready");
|
|
2414
|
-
return {
|
|
2415
|
-
progress: 1,
|
|
2416
|
-
output: {
|
|
2417
|
-
files: compactUrls([storedCaptions.url, transcriptArtifacts.srt_url]),
|
|
2418
|
-
primary_file_url: storedCaptions.url,
|
|
2419
|
-
cue_count: cues.length,
|
|
2420
|
-
word_timestamps: wordTimestampSource,
|
|
2421
|
-
...styleFields,
|
|
2422
|
-
// The cue list, ready to pass as editor_action set_captions `captions`.
|
|
2423
|
-
captions: cuesInline ? cues : undefined,
|
|
2424
|
-
captions_truncated: !cuesInline,
|
|
2425
|
-
// Fully-formed editor_action arguments — apply verbatim.
|
|
2426
|
-
set_captions_action: cuesInline ? setCaptionsAction : undefined,
|
|
2427
|
-
captions_json_url: storedCaptions.url,
|
|
2428
|
-
transcript: transcriptArtifacts.srt_url ? { srt_url: transcriptArtifacts.srt_url } : undefined
|
|
2429
|
-
}
|
|
2430
|
-
};
|
|
2431
|
-
}
|
|
2432
|
-
}
|
|
2433
|
-
});
|
|
2434
|
-
// Voice-casting hints for the profiling prompt: the exact preset names each
|
|
2435
|
-
// provider accepts (imported lists are the guard), annotated so an audio-native
|
|
2436
|
-
// model can pick the closest match to what it hears.
|
|
2437
|
-
const OPENAI_VOICE_CASTING_HINTS = "alloy (neutral, balanced), ash (warm, mature male), ballad (calm British male), cedar (deep, steady male), coral (warm, friendly female), echo (confident male), fable (bright British, androgynous), marin (clear, upbeat female), nova (energetic young female), onyx (deep, authoritative male), sage (soft, gentle female), shimmer (crisp, expressive female), verse (versatile, expressive male)";
|
|
2438
|
-
const GEMINI_VOICE_CASTING_HINTS = "Zephyr (bright female), Puck (upbeat male), Charon (informative male), Kore (firm female), Fenrir (excitable male), Leda (youthful female), Orus (firm male), Aoede (breezy female), Callirrhoe (easy-going female), Autonoe (bright female), Enceladus (breathy male), Iapetus (clear male), Umbriel (easy-going male), Algieba (smooth male), Despina (smooth female), Erinome (clear female), Algenib (gravelly male), Rasalgethi (informative male), Laomedeia (upbeat female), Achernar (soft female), Alnilam (firm male), Schedar (even male), Gacrux (mature female), Pulcherrima (forward female), Achird (friendly male), Zubenelgenubi (casual male), Vindemiatrix (gentle female), Sadachbia (lively male), Sadaltager (knowledgeable male), Sulafat (warm female)";
|
|
2439
|
-
// Loose on purpose: a shape miss must degrade to transcript-only, never fail
|
|
2440
|
-
// the whole regeneration job.
|
|
2441
|
-
const speechVoiceProfileResultSchema = z.object({
|
|
2442
|
-
transcript: z.string().optional().default(""),
|
|
2443
|
-
language: z.string().nullish(),
|
|
2444
|
-
voice_profile: z.object({
|
|
2445
|
-
gender: z.string().nullish(),
|
|
2446
|
-
age_range: z.string().nullish(),
|
|
2447
|
-
accent: z.string().nullish(),
|
|
2448
|
-
pace: z.string().nullish(),
|
|
2449
|
-
pitch: z.string().nullish(),
|
|
2450
|
-
energy: z.string().nullish(),
|
|
2451
|
-
tone: z.string().nullish()
|
|
2452
|
-
}).nullish(),
|
|
2453
|
-
style_instructions: z.string().nullish(),
|
|
2454
|
-
recommended_voice: z.object({
|
|
2455
|
-
openai: z.string().nullish(),
|
|
2456
|
-
gemini: z.string().nullish()
|
|
2457
|
-
}).nullish()
|
|
2458
|
-
});
|
|
2459
|
-
const speechRewriteResultSchema = z.object({
|
|
2460
|
-
script: z.string().min(1)
|
|
2461
|
-
});
|
|
2462
|
-
function buildVoiceProfilePrompt(language) {
|
|
2463
|
-
return [
|
|
2464
|
-
"You are a professional voice-casting director. Listen to the attached audio.",
|
|
2465
|
-
"Profile the dominant speaker and transcribe their words verbatim (if several people speak, profile only the dominant speaker and transcribe only their words).",
|
|
2466
|
-
"Return STRICT JSON ONLY with this exact shape:",
|
|
2467
|
-
`{"transcript": "<verbatim words spoken>", "language": "<BCP-47 code or null>", "voice_profile": {"gender": "...", "age_range": "...", "accent": "...", "pace": "...", "pitch": "...", "energy": "...", "tone": "..."}, "style_instructions": "<2-4 sentences>", "recommended_voice": {"openai": "<one name>", "gemini": "<one name>"}}`,
|
|
2468
|
-
"style_instructions must be a dense free-form text-to-speech voice direction that would make a voice actor sound as close as possible to this speaker: accent, pacing, pitch, energy, emotion, attitude, and delivery quirks (pauses, emphasis, breathiness, smile, hesitations).",
|
|
2469
|
-
`recommended_voice.openai must be exactly one name from: ${OPENAI_VOICE_CASTING_HINTS}.`,
|
|
2470
|
-
`recommended_voice.gemini must be exactly one name from: ${GEMINI_VOICE_CASTING_HINTS}.`,
|
|
2471
|
-
"Pick the preset closest to the speaker's gender, age, and timbre.",
|
|
2472
|
-
...(language?.trim() ? [`Expected language: ${language.trim()}.`] : [])
|
|
2473
|
-
].join("\n");
|
|
2474
|
-
}
|
|
2475
|
-
function buildSpeechRewritePrompt(input) {
|
|
2476
|
-
return [
|
|
2477
|
-
"You rewrite spoken-word scripts. Rewrite the original transcript below, following the instruction, as words to be SPOKEN aloud by the same person.",
|
|
2478
|
-
"Rules:",
|
|
2479
|
-
"- Keep roughly the same length and rhythm as the original unless the instruction says otherwise (the new audio should fill a similar duration).",
|
|
2480
|
-
"- Keep the speaker's register, energy, and phrasing style; this must sound like the same person talking.",
|
|
2481
|
-
"- Plain spoken words only: no stage directions, no markdown, no quotes, no speaker labels, no timestamps.",
|
|
2482
|
-
...(input.language?.trim() ? [`- Write the script in this language: ${input.language.trim()}.`] : ["- Write in the same language as the original transcript."]),
|
|
2483
|
-
`Return STRICT JSON ONLY with shape {"script": "..."}.`,
|
|
2484
|
-
"",
|
|
2485
|
-
`Instruction: ${input.instruction}`,
|
|
2486
|
-
"",
|
|
2487
|
-
"Original transcript:",
|
|
2488
|
-
input.transcript
|
|
2489
|
-
].join("\n");
|
|
2490
|
-
}
|
|
2491
|
-
// Voice-matched speech regeneration ("same voice, new words"). Pipeline:
|
|
2492
|
-
// (1) load the source audio (videos are demuxed), (2) profile the speaker with
|
|
2493
|
-
// an audio-native LLM — verbatim transcript + voice profile + closest preset
|
|
2494
|
-
// voices in ONE call (needs a saved Gemini key; without one this degrades to a
|
|
2495
|
-
// plain STT transcript and no voice match), (3) reword the transcript per
|
|
2496
|
-
// rewrite_instruction unless an exact `text` script was given, (4) TTS with
|
|
2497
|
-
// the matched preset + profiled style instructions. Neither OpenAI nor Gemini
|
|
2498
|
-
// exposes voice cloning through their public APIs (preset voices only; actual
|
|
2499
|
-
// cloning is sales-allowlisted at both), so voice_match is always an honest
|
|
2500
|
-
// approximation, never a clone.
|
|
2501
|
-
const speechRegeneratePrimitive = definePrimitive({
|
|
2502
|
-
id: PRIMITIVE_SPEECH_REGENERATE_ID,
|
|
2503
|
-
kind: "speech_regenerate",
|
|
2504
|
-
operations: {
|
|
2505
|
-
run: {
|
|
2506
|
-
description: "Regenerate the speech of a video/audio source as new AI audio that sounds like the original speaker (closest preset voice + matched style direction — an approximation, not a clone), speaking a reworded or replacement script.",
|
|
2507
|
-
inputSchema: speechRegeneratePayloadSchema,
|
|
2508
|
-
workflow: "run"
|
|
2509
|
-
}
|
|
2510
|
-
},
|
|
2511
|
-
jobs: {
|
|
2512
|
-
async run(ctx, input) {
|
|
2513
|
-
const payload = speechRegeneratePayloadSchema.parse(input);
|
|
2514
|
-
const { audioBytes, audioContentType, audioUrl } = await loadSpeechAudioForPrimitive(ctx, payload);
|
|
2515
|
-
// Step 1 — hear the speaker. One Gemini call returns transcript + voice
|
|
2516
|
-
// profile together; a failure here must never sink the job.
|
|
2517
|
-
const available = ctx.providers.listAvailableProviders();
|
|
2518
|
-
let profile = null;
|
|
2519
|
-
if (available.includes("gemini") && audioUrl) {
|
|
2520
|
-
ctx.logger.progress(0.3, "Profiling the original voice");
|
|
2521
|
-
try {
|
|
2522
|
-
const profiled = await runBrainstormJson(ctx, {
|
|
2523
|
-
kind: "speech_voice_profile",
|
|
2524
|
-
provider: "gemini",
|
|
2525
|
-
prompt: buildVoiceProfilePrompt(payload.language),
|
|
2526
|
-
attachments: [audioUrl],
|
|
2527
|
-
resultSchema: speechVoiceProfileResultSchema
|
|
2528
|
-
});
|
|
2529
|
-
profile = profiled.result;
|
|
2530
|
-
}
|
|
2531
|
-
catch (error) {
|
|
2532
|
-
ctx.logger.warn("Voice profiling failed; continuing without a voice match", {
|
|
2533
|
-
error: error instanceof Error ? error.message : String(error)
|
|
2534
|
-
});
|
|
2535
|
-
}
|
|
2536
|
-
}
|
|
2537
|
-
let originalTranscript = profile?.transcript?.trim() || "";
|
|
2538
|
-
if (!originalTranscript && payload.rewrite_instruction) {
|
|
2539
|
-
ctx.logger.progress(0.4, "Transcribing original speech");
|
|
2540
|
-
const sttProvider = ["gemini", "openai", "openrouter"].find((provider) => available.includes(provider)) ?? "gemini";
|
|
2541
|
-
const transcription = await ctx.providers.transcribeSpeech({
|
|
2542
|
-
provider: sttProvider,
|
|
2543
|
-
model: defaultSpeechModelFor("stt", sttProvider) ?? "",
|
|
2544
|
-
audio: audioBytes,
|
|
2545
|
-
contentType: audioContentType,
|
|
2546
|
-
language: payload.language,
|
|
2547
|
-
diarize: false
|
|
2548
|
-
});
|
|
2549
|
-
originalTranscript = transcription.text.trim();
|
|
2550
|
-
if (!originalTranscript) {
|
|
2551
|
-
throw new Error("The source has no transcribable speech to reword. Pass text with the exact script instead.");
|
|
2552
|
-
}
|
|
2553
|
-
}
|
|
2554
|
-
// Step 2 — the words: exact script wins, otherwise reword the transcript.
|
|
2555
|
-
let script = payload.text?.trim() ?? "";
|
|
2556
|
-
if (!script) {
|
|
2557
|
-
ctx.logger.progress(0.5, "Rewording the script");
|
|
2558
|
-
const rewrite = await runBrainstormJson(ctx, {
|
|
2559
|
-
kind: "speech_rewrite",
|
|
2560
|
-
prompt: buildSpeechRewritePrompt({
|
|
2561
|
-
transcript: originalTranscript,
|
|
2562
|
-
instruction: payload.rewrite_instruction,
|
|
2563
|
-
language: payload.language ?? profile?.language ?? null
|
|
2564
|
-
}),
|
|
2565
|
-
resultSchema: speechRewriteResultSchema
|
|
2566
|
-
});
|
|
2567
|
-
script = rewrite.result.script.trim();
|
|
2568
|
-
}
|
|
2569
|
-
if (!script) {
|
|
2570
|
-
throw new Error("No script produced to speak.");
|
|
2571
|
-
}
|
|
2572
|
-
if (script.length > 8000) {
|
|
2573
|
-
throw new Error("The regenerated script is too long for TTS (8000 characters max). Trim the source with /audio/trim or narrow the rewrite_instruction.");
|
|
2574
|
-
}
|
|
2575
|
-
// Step 3 — voice + style. openai first when nothing else decides: its
|
|
2576
|
-
// TTS takes the style `instructions` natively (gemini gets them
|
|
2577
|
-
// prepended into the prompt). A bare payload voice implies its provider.
|
|
2578
|
-
const provider = payload.provider
|
|
2579
|
-
?? inferSpeechProviderForVoice(payload.voice)
|
|
2580
|
-
?? ["openai", "gemini", "openrouter"].find((candidate) => available.includes(candidate))
|
|
2581
|
-
?? "openai";
|
|
2582
|
-
const model = payload.model ?? defaultSpeechModelFor("tts", provider) ?? "";
|
|
2583
|
-
const family = speechVoiceFamilyFor(provider, model);
|
|
2584
|
-
const presets = family === "gemini" ? GEMINI_TTS_VOICES : OPENAI_TTS_VOICES;
|
|
2585
|
-
const recommendedRaw = (family === "gemini" ? profile?.recommended_voice?.gemini : profile?.recommended_voice?.openai)?.trim().toLowerCase();
|
|
2586
|
-
const recommendedVoice = recommendedRaw ? presets.find((preset) => preset.toLowerCase() === recommendedRaw) : undefined;
|
|
2587
|
-
const voice = payload.voice ?? recommendedVoice;
|
|
2588
|
-
const profiledStyle = profile?.style_instructions?.trim();
|
|
2589
|
-
const instructions = [profiledStyle, payload.instructions?.trim()].filter(Boolean).join(" ") || undefined;
|
|
2590
|
-
ctx.logger.progress(0.65, "Generating voice-matched speech");
|
|
2591
|
-
const generated = await ctx.providers.generateSpeech({
|
|
2592
|
-
provider,
|
|
2593
|
-
model,
|
|
2594
|
-
text: script,
|
|
2595
|
-
voice,
|
|
2596
|
-
instructions,
|
|
2597
|
-
responseFormat: payload.output_format
|
|
2598
|
-
});
|
|
2599
|
-
const extension = generated.contentType.includes("wav") ? "wav" : "mp3";
|
|
2600
|
-
const stored = await ctx.storage.putBuffer(`speech.${extension}`, generated.bytes, {
|
|
2601
|
-
contentType: generated.contentType,
|
|
2602
|
-
kind: "audio",
|
|
2603
|
-
metadata: {
|
|
2604
|
-
primitive: "speech_regenerate",
|
|
2605
|
-
provider,
|
|
2606
|
-
model,
|
|
2607
|
-
voice: voice ?? null,
|
|
2608
|
-
styleInstructions: instructions ?? null
|
|
2609
|
-
}
|
|
2610
|
-
});
|
|
2611
|
-
ctx.logger.progress(0.9, "Storing regeneration artifacts");
|
|
2612
|
-
const voiceMatch = profiledStyle || recommendedVoice ? "approximate" : "none";
|
|
2613
|
-
const voiceMatchNote = voiceMatch === "approximate"
|
|
2614
|
-
? "The audio uses the closest preset voice with style instructions matched to the original speaker. OpenAI/Gemini APIs cannot clone voice identity, so this is an approximation, not a clone."
|
|
2615
|
-
: "No voice profile was available (profiling needs a saved Gemini key), so a default or explicitly requested preset voice was used without matching the original speaker.";
|
|
2616
|
-
const storedJson = await ctx.storage.putJson("regenerate-speech.json", {
|
|
2617
|
-
source_url: payload.source_url,
|
|
2618
|
-
provider,
|
|
2619
|
-
model,
|
|
2620
|
-
voice: voice ?? null,
|
|
2621
|
-
style_instructions: instructions ?? null,
|
|
2622
|
-
voice_match: voiceMatch,
|
|
2623
|
-
voice_match_note: voiceMatchNote,
|
|
2624
|
-
voice_profile: profile?.voice_profile ?? null,
|
|
2625
|
-
original_transcript: originalTranscript || null,
|
|
2626
|
-
rewrite_instruction: payload.rewrite_instruction ?? null,
|
|
2627
|
-
script,
|
|
2628
|
-
audio_url: stored.url
|
|
2629
|
-
});
|
|
2630
|
-
ctx.logger.progress(1, "Speech regeneration complete");
|
|
2631
|
-
return {
|
|
2632
|
-
progress: 1,
|
|
2633
|
-
output: {
|
|
2634
|
-
files: compactUrls([stored.url, storedJson.url]),
|
|
2635
|
-
primary_file_url: stored.url,
|
|
2636
|
-
audioUrl: stored.url,
|
|
2637
|
-
script,
|
|
2638
|
-
original_transcript: originalTranscript ? originalTranscript.slice(0, MAX_INLINE_TRANSCRIPT_CHARS) : null,
|
|
2639
|
-
voice_match: voiceMatch,
|
|
2640
|
-
voice_match_note: voiceMatchNote,
|
|
2641
|
-
voice_profile: profile?.voice_profile ?? null,
|
|
2642
|
-
audio: {
|
|
2643
|
-
file_url: stored.url,
|
|
2644
|
-
content_type: generated.contentType,
|
|
2645
|
-
provider,
|
|
2646
|
-
model,
|
|
2647
|
-
voice: voice ?? null,
|
|
2648
|
-
style_instructions: instructions ?? null,
|
|
2649
|
-
text_length: script.length
|
|
2650
|
-
},
|
|
2651
|
-
details_json_url: storedJson.url
|
|
2652
|
-
}
|
|
2653
|
-
};
|
|
2654
|
-
}
|
|
2655
|
-
}
|
|
2656
|
-
});
|
|
2657
|
-
const videoProbePrimitive = definePrimitive({
|
|
2658
|
-
id: PRIMITIVE_VIDEO_PROBE_ID,
|
|
2659
|
-
kind: "video_probe",
|
|
2660
|
-
operations: {
|
|
2661
|
-
run: {
|
|
2662
|
-
description: "Probe a source video and return reusable JSON metadata.",
|
|
2663
|
-
inputSchema: probeVideoPayloadSchema,
|
|
2664
|
-
workflow: "run"
|
|
2665
|
-
}
|
|
2666
|
-
},
|
|
2667
|
-
jobs: {
|
|
2668
|
-
async run(ctx, input) {
|
|
2669
|
-
const payload = probeVideoPayloadSchema.parse(input);
|
|
2670
|
-
const probed = await executePrimitiveMediaOperation(ctx, {
|
|
2671
|
-
operation: "video_probe",
|
|
2672
|
-
payload,
|
|
2673
|
-
outputKey: "probe.json"
|
|
2674
|
-
});
|
|
2675
|
-
return {
|
|
2676
|
-
progress: 1,
|
|
2677
|
-
output: {
|
|
2678
|
-
files: compactUrls([payload.source_video_url, probed.publicUrl]),
|
|
2679
|
-
sourceVideoUrl: payload.source_video_url,
|
|
2680
|
-
metadataUrl: probed.publicUrl,
|
|
2681
|
-
primary_file_url: probed.publicUrl,
|
|
2682
|
-
probe: probed.metadata
|
|
2683
|
-
}
|
|
2684
|
-
};
|
|
2685
|
-
}
|
|
2686
|
-
}
|
|
2687
|
-
});
|
|
2688
|
-
const audioProbePrimitive = definePrimitive({
|
|
2689
|
-
id: PRIMITIVE_AUDIO_PROBE_ID,
|
|
2690
|
-
kind: "audio_probe",
|
|
2691
|
-
operations: {
|
|
2692
|
-
run: {
|
|
2693
|
-
description: "Probe a source audio file and return reusable JSON metadata.",
|
|
2694
|
-
inputSchema: probeAudioPayloadSchema,
|
|
2695
|
-
workflow: "run"
|
|
2696
|
-
}
|
|
2697
|
-
},
|
|
2698
|
-
jobs: {
|
|
2699
|
-
async run(ctx, input) {
|
|
2700
|
-
const payload = probeAudioPayloadSchema.parse(input);
|
|
2701
|
-
const probed = await executePrimitiveMediaOperation(ctx, {
|
|
2702
|
-
operation: "audio_probe",
|
|
2703
|
-
payload,
|
|
2704
|
-
outputKey: "probe.json"
|
|
2705
|
-
});
|
|
2706
|
-
return {
|
|
2707
|
-
progress: 1,
|
|
2708
|
-
output: {
|
|
2709
|
-
files: compactUrls([payload.source_audio_url, probed.publicUrl]),
|
|
2710
|
-
sourceAudioUrl: payload.source_audio_url,
|
|
2711
|
-
metadataUrl: probed.publicUrl,
|
|
2712
|
-
primary_file_url: probed.publicUrl,
|
|
2713
|
-
probe: probed.metadata
|
|
2714
|
-
}
|
|
2715
|
-
};
|
|
2716
|
-
}
|
|
2717
|
-
}
|
|
2718
|
-
});
|
|
2719
|
-
const videoExtractFramePrimitive = definePrimitive({
|
|
2720
|
-
id: PRIMITIVE_VIDEO_EXTRACT_FRAME_ID,
|
|
2721
|
-
kind: "video_extract_frame",
|
|
2722
|
-
operations: {
|
|
2723
|
-
run: {
|
|
2724
|
-
description: "Extract a single frame from a source video into a reusable image file.",
|
|
2725
|
-
inputSchema: extractFramePayloadSchema,
|
|
2726
|
-
workflow: "run"
|
|
2727
|
-
}
|
|
2728
|
-
},
|
|
2729
|
-
jobs: {
|
|
2730
|
-
async run(ctx, input) {
|
|
2731
|
-
const payload = extractFramePayloadSchema.parse(input);
|
|
2732
|
-
const frame = await executePrimitiveMediaOperation(ctx, {
|
|
2733
|
-
operation: "video_extract_frame",
|
|
2734
|
-
payload,
|
|
2735
|
-
outputKey: `frame.${payload.output_format === "jpeg" ? "jpg" : payload.output_format}`
|
|
2736
|
-
});
|
|
2737
|
-
return {
|
|
2738
|
-
progress: 1,
|
|
2739
|
-
output: {
|
|
2740
|
-
files: compactUrls([payload.source_video_url, frame.publicUrl]),
|
|
2741
|
-
sourceVideoUrl: payload.source_video_url,
|
|
2742
|
-
primary_file_url: frame.publicUrl,
|
|
2743
|
-
image: {
|
|
2744
|
-
file_url: frame.publicUrl,
|
|
2745
|
-
content_type: frame.contentType,
|
|
2746
|
-
extracted_from_url: payload.source_video_url
|
|
2747
|
-
},
|
|
2748
|
-
frame: frame.metadata
|
|
2749
|
-
}
|
|
2750
|
-
};
|
|
2751
|
-
}
|
|
2752
|
-
}
|
|
2753
|
-
});
|
|
2754
|
-
const videoMutePrimitive = definePrimitive({
|
|
2755
|
-
id: PRIMITIVE_VIDEO_MUTE_ID,
|
|
2756
|
-
kind: "video_mute",
|
|
2757
|
-
operations: {
|
|
2758
|
-
run: {
|
|
2759
|
-
description: "Remove audio from a source video and return a reusable muted MP4.",
|
|
2760
|
-
inputSchema: muteVideoPayloadSchema,
|
|
2761
|
-
workflow: "run"
|
|
2762
|
-
}
|
|
2763
|
-
},
|
|
2764
|
-
jobs: {
|
|
2765
|
-
async run(ctx, input) {
|
|
2766
|
-
const payload = muteVideoPayloadSchema.parse(input);
|
|
2767
|
-
const muted = await executePrimitiveMediaOperation(ctx, {
|
|
2768
|
-
operation: "video_mute",
|
|
2769
|
-
payload,
|
|
2770
|
-
outputKey: "muted.mp4"
|
|
2771
|
-
});
|
|
2772
|
-
return {
|
|
2773
|
-
progress: 1,
|
|
2774
|
-
output: {
|
|
2775
|
-
files: compactUrls([payload.source_video_url, muted.publicUrl]),
|
|
2776
|
-
sourceVideoUrl: payload.source_video_url,
|
|
2777
|
-
mutedVideoUrl: muted.publicUrl,
|
|
2778
|
-
primary_file_url: muted.publicUrl,
|
|
2779
|
-
video: {
|
|
2780
|
-
file_url: muted.publicUrl,
|
|
2781
|
-
content_type: muted.contentType,
|
|
2782
|
-
mute: muted.metadata
|
|
2783
|
-
}
|
|
2784
|
-
}
|
|
2785
|
-
};
|
|
2786
|
-
}
|
|
2787
|
-
}
|
|
2788
|
-
});
|
|
2789
|
-
const videoReplaceAudioPrimitive = definePrimitive({
|
|
2790
|
-
id: PRIMITIVE_VIDEO_REPLACE_AUDIO_ID,
|
|
2791
|
-
kind: "video_replace_audio",
|
|
2792
|
-
operations: {
|
|
2793
|
-
run: {
|
|
2794
|
-
description: "Replace a source video's audio track with another audio file.",
|
|
2795
|
-
inputSchema: replaceAudioPayloadSchema,
|
|
2796
|
-
workflow: "run"
|
|
2797
|
-
}
|
|
2798
|
-
},
|
|
2799
|
-
jobs: {
|
|
2800
|
-
async run(ctx, input) {
|
|
2801
|
-
const payload = replaceAudioPayloadSchema.parse(input);
|
|
2802
|
-
const replaced = await executePrimitiveMediaOperation(ctx, {
|
|
2803
|
-
operation: "video_replace_audio",
|
|
2804
|
-
payload,
|
|
2805
|
-
outputKey: "replaced-audio.mp4"
|
|
2806
|
-
});
|
|
2807
|
-
return {
|
|
2808
|
-
progress: 1,
|
|
2809
|
-
output: {
|
|
2810
|
-
files: compactUrls([payload.source_video_url, payload.source_audio_url, replaced.publicUrl]),
|
|
2811
|
-
sourceVideoUrl: payload.source_video_url,
|
|
2812
|
-
sourceAudioUrl: payload.source_audio_url,
|
|
2813
|
-
primary_file_url: replaced.publicUrl,
|
|
2814
|
-
video: {
|
|
2815
|
-
file_url: replaced.publicUrl,
|
|
2816
|
-
content_type: replaced.contentType,
|
|
2817
|
-
audio_replacement: replaced.metadata
|
|
2818
|
-
}
|
|
2819
|
-
}
|
|
2820
|
-
};
|
|
2821
|
-
}
|
|
2822
|
-
}
|
|
2823
|
-
});
|
|
2824
|
-
const videoConcatPrimitive = definePrimitive({
|
|
2825
|
-
id: PRIMITIVE_VIDEO_CONCAT_ID,
|
|
2826
|
-
kind: "video_concat",
|
|
2827
|
-
operations: {
|
|
2828
|
-
run: {
|
|
2829
|
-
description: "Concatenate multiple source videos into one reusable MP4.",
|
|
2830
|
-
inputSchema: concatVideoPayloadSchema,
|
|
2831
|
-
workflow: "run"
|
|
2832
|
-
}
|
|
2833
|
-
},
|
|
2834
|
-
jobs: {
|
|
2835
|
-
async run(ctx, input) {
|
|
2836
|
-
const payload = concatVideoPayloadSchema.parse(input);
|
|
2837
|
-
const concatenated = await executePrimitiveMediaOperation(ctx, {
|
|
2838
|
-
operation: "video_concat",
|
|
2839
|
-
payload,
|
|
2840
|
-
outputKey: "concat.mp4"
|
|
2841
|
-
});
|
|
2842
|
-
return {
|
|
2843
|
-
progress: 1,
|
|
2844
|
-
output: {
|
|
2845
|
-
files: compactUrls([...payload.source_video_urls, concatenated.publicUrl]),
|
|
2846
|
-
sourceVideoUrls: payload.source_video_urls,
|
|
2847
|
-
primary_file_url: concatenated.publicUrl,
|
|
2848
|
-
video: {
|
|
2849
|
-
file_url: concatenated.publicUrl,
|
|
2850
|
-
content_type: concatenated.contentType,
|
|
2851
|
-
concat: concatenated.metadata
|
|
2852
|
-
}
|
|
2853
|
-
}
|
|
2854
|
-
};
|
|
2855
|
-
}
|
|
2856
|
-
}
|
|
2857
|
-
});
|
|
2858
|
-
const audioConcatPrimitive = definePrimitive({
|
|
2859
|
-
id: PRIMITIVE_AUDIO_CONCAT_ID,
|
|
2860
|
-
kind: "audio_concat",
|
|
2861
|
-
operations: {
|
|
2862
|
-
run: {
|
|
2863
|
-
description: "Concatenate multiple source audio files into one reusable audio file.",
|
|
2864
|
-
inputSchema: concatAudioPayloadSchema,
|
|
2865
|
-
workflow: "run"
|
|
2866
|
-
}
|
|
2867
|
-
},
|
|
2868
|
-
jobs: {
|
|
2869
|
-
async run(ctx, input) {
|
|
2870
|
-
const payload = concatAudioPayloadSchema.parse(input);
|
|
2871
|
-
const concatenated = await executePrimitiveMediaOperation(ctx, {
|
|
2872
|
-
operation: "audio_concat",
|
|
2873
|
-
payload,
|
|
2874
|
-
outputKey: `concat.${payload.output_format}`
|
|
2875
|
-
});
|
|
2876
|
-
return {
|
|
2877
|
-
progress: 1,
|
|
2878
|
-
output: {
|
|
2879
|
-
files: compactUrls([...payload.source_audio_urls, concatenated.publicUrl]),
|
|
2880
|
-
sourceAudioUrls: payload.source_audio_urls,
|
|
2881
|
-
primary_file_url: concatenated.publicUrl,
|
|
2882
|
-
audio: {
|
|
2883
|
-
file_url: concatenated.publicUrl,
|
|
2884
|
-
content_type: concatenated.contentType,
|
|
2885
|
-
concat: concatenated.metadata
|
|
2886
|
-
}
|
|
2887
|
-
}
|
|
2888
|
-
};
|
|
2889
|
-
}
|
|
2890
|
-
}
|
|
2891
|
-
});
|
|
2892
|
-
const audioNormalizePrimitive = definePrimitive({
|
|
2893
|
-
id: PRIMITIVE_AUDIO_NORMALIZE_ID,
|
|
2894
|
-
kind: "audio_normalize",
|
|
2895
|
-
operations: {
|
|
2896
|
-
run: {
|
|
2897
|
-
description: "Normalize a source audio file into a reusable platform audio file.",
|
|
2898
|
-
inputSchema: normalizeAudioPayloadSchema,
|
|
2899
|
-
workflow: "run"
|
|
2900
|
-
}
|
|
2901
|
-
},
|
|
2902
|
-
jobs: {
|
|
2903
|
-
async run(ctx, input) {
|
|
2904
|
-
const payload = normalizeAudioPayloadSchema.parse(input);
|
|
2905
|
-
const normalized = await executePrimitiveMediaOperation(ctx, {
|
|
2906
|
-
operation: "audio_normalize",
|
|
2907
|
-
payload,
|
|
2908
|
-
outputKey: `normalized.${payload.output_format}`
|
|
2909
|
-
});
|
|
2910
|
-
return {
|
|
2911
|
-
progress: 1,
|
|
2912
|
-
output: {
|
|
2913
|
-
files: compactUrls([payload.source_audio_url, normalized.publicUrl]),
|
|
2914
|
-
sourceAudioUrl: payload.source_audio_url,
|
|
2915
|
-
normalizedAudioUrl: normalized.publicUrl,
|
|
2916
|
-
primary_file_url: normalized.publicUrl,
|
|
2917
|
-
audio: {
|
|
2918
|
-
file_url: normalized.publicUrl,
|
|
2919
|
-
content_type: normalized.contentType,
|
|
2920
|
-
normalization: normalized.metadata
|
|
2921
|
-
}
|
|
2922
|
-
}
|
|
2923
|
-
};
|
|
2924
|
-
}
|
|
2925
|
-
}
|
|
2926
|
-
});
|
|
2927
|
-
const hyperframesRenderPrimitive = definePrimitive({
|
|
2928
|
-
id: PRIMITIVE_HYPERFRAMES_RENDER_ID,
|
|
2929
|
-
kind: "hyperframes_render",
|
|
2930
|
-
operations: {
|
|
2931
|
-
run: {
|
|
2932
|
-
description: "Render a timeline/layer composition to MP4 with production Lambda.",
|
|
2933
|
-
inputSchema: hyperframesRenderPayloadSchema,
|
|
2934
|
-
workflow: "run"
|
|
2935
|
-
}
|
|
2936
|
-
},
|
|
2937
|
-
jobs: {
|
|
2938
|
-
async run(ctx, input) {
|
|
2939
|
-
const payload = hyperframesRenderPayloadSchema.parse(input);
|
|
2940
|
-
ctx.logger.progress(0.08, "Preparing timeline render");
|
|
2941
|
-
const composition = payload.composition ? normalizeHyperframesCompositionPayload(payload.composition) : undefined;
|
|
2942
|
-
const compositionHtml = payload.composition_html;
|
|
2943
|
-
const compositionDataArtifact = payload.composition_data
|
|
2944
|
-
? await ctx.storage.putJson("composition.json", payload.composition_data)
|
|
2945
|
-
: null;
|
|
2946
|
-
const render = await ctx.hyperframes.render({
|
|
2947
|
-
compositionHtml,
|
|
2948
|
-
composition,
|
|
2949
|
-
compositionId: payload.composition_id ?? composition?.compositionId,
|
|
2950
|
-
title: payload.title ?? composition?.title,
|
|
2951
|
-
width: payload.width ?? composition?.width,
|
|
2952
|
-
height: payload.height ?? composition?.height,
|
|
2953
|
-
fps: payload.fps,
|
|
2954
|
-
chunkSize: payload.chunk_size,
|
|
2955
|
-
maxParallelChunks: payload.max_parallel_chunks,
|
|
2956
|
-
outputKey: payload.output_key,
|
|
2957
|
-
projectFiles: payload.project_files.map((file) => ({
|
|
2958
|
-
path: file.path,
|
|
2959
|
-
contents: readHyperframesProjectFileContents(file),
|
|
2960
|
-
contentType: file.content_type
|
|
2961
|
-
}))
|
|
2962
|
-
});
|
|
2963
|
-
ctx.logger.progress(1, "Timeline render complete", {
|
|
2964
|
-
renderId: render.renderId,
|
|
2965
|
-
outputUrl: render.outputUrl,
|
|
2966
|
-
outputS3Uri: render.outputS3Uri
|
|
2967
|
-
});
|
|
2968
|
-
return {
|
|
2969
|
-
progress: 1,
|
|
2970
|
-
output: {
|
|
2971
|
-
files: compactUrls([render.outputUrl, compositionDataArtifact?.url]),
|
|
2972
|
-
primary_file_url: render.outputUrl,
|
|
2973
|
-
video: {
|
|
2974
|
-
file_url: render.outputUrl,
|
|
2975
|
-
content_type: "video/mp4",
|
|
2976
|
-
width: payload.width ?? composition?.width ?? null,
|
|
2977
|
-
height: payload.height ?? composition?.height ?? null,
|
|
2978
|
-
fps: payload.fps
|
|
2979
|
-
},
|
|
2980
|
-
render: {
|
|
2981
|
-
render_id: render.renderId,
|
|
2982
|
-
output_url: render.outputUrl,
|
|
2983
|
-
output_s3_uri: render.outputS3Uri,
|
|
2984
|
-
metadata: render.metadata
|
|
2985
|
-
},
|
|
2986
|
-
hyperframes: {
|
|
2987
|
-
composition_id: payload.composition_id ?? composition?.compositionId ?? null,
|
|
2988
|
-
composition_data_url: compositionDataArtifact?.url ?? null,
|
|
2989
|
-
architecture: "timeline_layers"
|
|
2990
|
-
}
|
|
2991
|
-
}
|
|
2992
|
-
};
|
|
2993
|
-
}
|
|
2994
|
-
}
|
|
2995
|
-
});
|
|
2996
|
-
const brainstormHooksPrimitive = definePrimitive({
|
|
2997
|
-
id: PRIMITIVE_BRAINSTORM_HOOKS_ID,
|
|
2998
|
-
kind: "brainstorm_hooks",
|
|
2999
|
-
operations: {
|
|
3000
|
-
run: {
|
|
3001
|
-
description: "Generate diverse TikTok-native ad hooks plus short explanations for a provided offer.",
|
|
3002
|
-
inputSchema: brainstormHooksPayloadSchema,
|
|
3003
|
-
workflow: "run",
|
|
3004
|
-
providerHint: "openai"
|
|
3005
|
-
}
|
|
3006
|
-
},
|
|
3007
|
-
jobs: {
|
|
3008
|
-
async run(ctx, input) {
|
|
3009
|
-
const payload = brainstormHooksPayloadSchema.parse(input);
|
|
3010
|
-
ctx.logger.progress(0.15, "Brainstorming hooks");
|
|
3011
|
-
const generated = await runBrainstormJson(ctx, {
|
|
3012
|
-
kind: "brainstorm_hooks",
|
|
3013
|
-
provider: payload.provider,
|
|
3014
|
-
model: payload.model,
|
|
3015
|
-
prompt: buildHooksPrompt(payload.offer_description, payload.count),
|
|
3016
|
-
resultSchema: brainstormHooksResultSchema
|
|
3017
|
-
});
|
|
3018
|
-
const promptArtifact = await ctx.storage.putText("prompt.txt", buildHooksPrompt(payload.offer_description, payload.count));
|
|
3019
|
-
const resultArtifact = await ctx.storage.putJson("result.json", generated.result);
|
|
3020
|
-
ctx.logger.progress(1, "Hook brainstorm complete");
|
|
3021
|
-
return {
|
|
3022
|
-
progress: 1,
|
|
3023
|
-
output: {
|
|
3024
|
-
hooks: generated.result.hooks,
|
|
3025
|
-
files: compactUrls([resultArtifact.url, promptArtifact.url]),
|
|
3026
|
-
artifacts: {
|
|
3027
|
-
prompt_url: promptArtifact.url,
|
|
3028
|
-
result_url: resultArtifact.url
|
|
3029
|
-
},
|
|
3030
|
-
usage: generated.usage
|
|
3031
|
-
}
|
|
3032
|
-
};
|
|
3033
|
-
}
|
|
3034
|
-
}
|
|
3035
|
-
});
|
|
3036
|
-
const brainstormAwarenessStagesPrimitive = definePrimitive({
|
|
3037
|
-
id: PRIMITIVE_BRAINSTORM_AWARENESS_STAGES_ID,
|
|
3038
|
-
kind: "brainstorm_awareness_stages",
|
|
3039
|
-
operations: {
|
|
3040
|
-
run: {
|
|
3041
|
-
description: "Explain which awareness stages to target first for an offer and what ad types to create.",
|
|
3042
|
-
inputSchema: brainstormAwarenessStagesPayloadSchema,
|
|
3043
|
-
workflow: "run",
|
|
3044
|
-
providerHint: "openai"
|
|
3045
|
-
}
|
|
3046
|
-
},
|
|
3047
|
-
jobs: {
|
|
3048
|
-
async run(ctx, input) {
|
|
3049
|
-
const payload = brainstormAwarenessStagesPayloadSchema.parse(input);
|
|
3050
|
-
ctx.logger.progress(0.15, "Mapping awareness stages");
|
|
3051
|
-
const prompt = buildAwarenessStagesPrompt(payload.offer_description);
|
|
3052
|
-
const generated = await runBrainstormText(ctx, {
|
|
3053
|
-
kind: "brainstorm_awareness_stages",
|
|
3054
|
-
provider: payload.provider,
|
|
3055
|
-
model: payload.model,
|
|
3056
|
-
prompt,
|
|
3057
|
-
temperature: 0.55
|
|
3058
|
-
});
|
|
3059
|
-
const promptArtifact = await ctx.storage.putText("prompt.txt", prompt);
|
|
3060
|
-
const resultArtifact = await ctx.storage.putText("response.md", generated.text, "text/markdown; charset=utf-8");
|
|
3061
|
-
ctx.logger.progress(1, "Awareness-stage brainstorm complete");
|
|
3062
|
-
return {
|
|
3063
|
-
progress: 1,
|
|
3064
|
-
output: {
|
|
3065
|
-
response: generated.text,
|
|
3066
|
-
files: compactUrls([resultArtifact.url, promptArtifact.url]),
|
|
3067
|
-
artifacts: {
|
|
3068
|
-
prompt_url: promptArtifact.url,
|
|
3069
|
-
response_url: resultArtifact.url
|
|
3070
|
-
},
|
|
3071
|
-
usage: generated.usage
|
|
3072
|
-
}
|
|
3073
|
-
};
|
|
3074
|
-
}
|
|
3075
|
-
}
|
|
3076
|
-
});
|
|
3077
|
-
const brainstormAnglesPrimitive = definePrimitive({
|
|
3078
|
-
id: PRIMITIVE_BRAINSTORM_ANGLES_ID,
|
|
3079
|
-
kind: "brainstorm_angles",
|
|
3080
|
-
operations: {
|
|
3081
|
-
run: {
|
|
3082
|
-
description: "Generate diverse persuasive TikTok advertising angles for an offer at the selected awareness levels.",
|
|
3083
|
-
inputSchema: brainstormAnglesPayloadSchema,
|
|
3084
|
-
workflow: "run",
|
|
3085
|
-
providerHint: "openai"
|
|
3086
|
-
}
|
|
3087
|
-
},
|
|
3088
|
-
jobs: {
|
|
3089
|
-
async run(ctx, input) {
|
|
3090
|
-
const payload = brainstormAnglesPayloadSchema.parse(input);
|
|
3091
|
-
ctx.logger.progress(0.15, "Brainstorming angles");
|
|
3092
|
-
const prompt = buildAnglesPrompt(payload);
|
|
3093
|
-
const generated = await runBrainstormJson(ctx, {
|
|
3094
|
-
kind: "brainstorm_angles",
|
|
3095
|
-
provider: payload.provider,
|
|
3096
|
-
model: payload.model,
|
|
3097
|
-
prompt,
|
|
3098
|
-
resultSchema: brainstormAnglesResultSchema
|
|
3099
|
-
});
|
|
3100
|
-
const promptArtifact = await ctx.storage.putText("prompt.txt", prompt);
|
|
3101
|
-
const resultArtifact = await ctx.storage.putJson("result.json", generated.result);
|
|
3102
|
-
ctx.logger.progress(1, "Angle brainstorm complete");
|
|
3103
|
-
return {
|
|
3104
|
-
progress: 1,
|
|
3105
|
-
output: {
|
|
3106
|
-
angles: generated.result.angles,
|
|
3107
|
-
awareness: {
|
|
3108
|
-
problem_awareness: payload.problem_awareness,
|
|
3109
|
-
solution_awareness: payload.solution_awareness
|
|
3110
|
-
},
|
|
3111
|
-
files: compactUrls([resultArtifact.url, promptArtifact.url]),
|
|
3112
|
-
artifacts: {
|
|
3113
|
-
prompt_url: promptArtifact.url,
|
|
3114
|
-
result_url: resultArtifact.url
|
|
3115
|
-
},
|
|
3116
|
-
usage: generated.usage
|
|
3117
|
-
}
|
|
3118
|
-
};
|
|
3119
|
-
}
|
|
3120
|
-
}
|
|
3121
|
-
});
|
|
3122
|
-
const brainstormColdstartPrimitive = definePrimitive({
|
|
3123
|
-
id: PRIMITIVE_BRAINSTORM_COLDSTART_ID,
|
|
3124
|
-
kind: "brainstorm_coldstart",
|
|
3125
|
-
operations: {
|
|
3126
|
-
run: {
|
|
3127
|
-
description: "Generate a high-signal cold-start questionnaire that helps a customer build a TikTok content strategy from zero.",
|
|
3128
|
-
inputSchema: brainstormColdstartPayloadSchema,
|
|
3129
|
-
workflow: "run",
|
|
3130
|
-
providerHint: "openai"
|
|
3131
|
-
}
|
|
3132
|
-
},
|
|
3133
|
-
jobs: {
|
|
3134
|
-
async run(ctx, input) {
|
|
3135
|
-
const payload = brainstormColdstartPayloadSchema.parse(input);
|
|
3136
|
-
ctx.logger.progress(0.15, "Generating cold-start questions");
|
|
3137
|
-
const prompt = buildColdstartPrompt(payload.user_message, payload.count);
|
|
3138
|
-
const generated = await runBrainstormJson(ctx, {
|
|
3139
|
-
kind: "brainstorm_coldstart",
|
|
3140
|
-
provider: payload.provider,
|
|
3141
|
-
model: payload.model,
|
|
3142
|
-
prompt,
|
|
3143
|
-
resultSchema: brainstormColdstartResultSchema
|
|
3144
|
-
});
|
|
3145
|
-
const promptArtifact = await ctx.storage.putText("prompt.txt", prompt);
|
|
3146
|
-
const resultArtifact = await ctx.storage.putJson("result.json", generated.result);
|
|
3147
|
-
ctx.logger.progress(1, "Cold-start questionnaire complete");
|
|
3148
|
-
return {
|
|
3149
|
-
progress: 1,
|
|
3150
|
-
output: {
|
|
3151
|
-
questions: generated.result.questions,
|
|
3152
|
-
encouragement: "Encourage the user to save their answers in a Google Doc or markdown file and upload that context into Vidfarm for future strategy, scripting, and template runs.",
|
|
3153
|
-
files: compactUrls([resultArtifact.url, promptArtifact.url]),
|
|
3154
|
-
artifacts: {
|
|
3155
|
-
prompt_url: promptArtifact.url,
|
|
3156
|
-
result_url: resultArtifact.url
|
|
3157
|
-
},
|
|
3158
|
-
usage: generated.usage
|
|
3159
|
-
}
|
|
3160
|
-
};
|
|
3161
|
-
}
|
|
3162
|
-
}
|
|
3163
|
-
});
|
|
3164
|
-
const brainstormProductPlacementPrimitive = definePrimitive({
|
|
3165
|
-
id: PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID,
|
|
3166
|
-
kind: "brainstorm_product_placement",
|
|
3167
|
-
operations: {
|
|
3168
|
-
run: {
|
|
3169
|
-
description: "Watch a source video and identify concrete, timestamped product-placement opportunities for a given product/offer.",
|
|
3170
|
-
inputSchema: brainstormProductPlacementPayloadSchema,
|
|
3171
|
-
workflow: "run",
|
|
3172
|
-
providerHint: "gemini"
|
|
3173
|
-
}
|
|
3174
|
-
},
|
|
3175
|
-
jobs: {
|
|
3176
|
-
async run(ctx, input) {
|
|
3177
|
-
const payload = brainstormProductPlacementPayloadSchema.parse(input);
|
|
3178
|
-
ctx.logger.progress(0.15, "Analyzing video for product placement opportunities");
|
|
3179
|
-
const provider = payload.provider ?? preferredVideoTextProvider(ctx.providers.listAvailableProviders());
|
|
3180
|
-
const prompt = buildProductPlacementPrompt(payload.offer_description, payload.count);
|
|
3181
|
-
const generated = await runBrainstormJson(ctx, {
|
|
3182
|
-
kind: "brainstorm_product_placement",
|
|
3183
|
-
provider,
|
|
3184
|
-
model: payload.model,
|
|
3185
|
-
prompt,
|
|
3186
|
-
attachments: [payload.source_video_url],
|
|
3187
|
-
resultSchema: brainstormProductPlacementResultSchema
|
|
3188
|
-
});
|
|
3189
|
-
const promptArtifact = await ctx.storage.putText("prompt.txt", prompt);
|
|
3190
|
-
const resultArtifact = await ctx.storage.putJson("result.json", generated.result);
|
|
3191
|
-
ctx.logger.progress(1, "Product-placement analysis complete");
|
|
3192
|
-
return {
|
|
3193
|
-
progress: 1,
|
|
3194
|
-
output: {
|
|
3195
|
-
opportunities: generated.result.opportunities,
|
|
3196
|
-
source_video_url: payload.source_video_url,
|
|
3197
|
-
files: compactUrls([resultArtifact.url, promptArtifact.url]),
|
|
3198
|
-
artifacts: {
|
|
3199
|
-
prompt_url: promptArtifact.url,
|
|
3200
|
-
result_url: resultArtifact.url
|
|
3201
|
-
},
|
|
3202
|
-
usage: generated.usage
|
|
3203
|
-
}
|
|
3204
|
-
};
|
|
3205
|
-
}
|
|
3206
|
-
}
|
|
3207
|
-
});
|
|
3208
|
-
class PrimitiveRegistry {
|
|
3209
|
-
definitions = new Map([
|
|
3210
|
-
[imageGeneratePrimitive.id, imageGeneratePrimitive],
|
|
3211
|
-
[imageEditPrimitive.id, imageEditPrimitive],
|
|
3212
|
-
[imageInpaintPrimitive.id, imageInpaintPrimitive],
|
|
3213
|
-
[imageRenderHtmlPrimitive.id, imageRenderHtmlPrimitive],
|
|
3214
|
-
[imageRemoveBackgroundPrimitive.id, imageRemoveBackgroundPrimitive],
|
|
3215
|
-
[imageRemoveBackgroundGreenscreenPrimitive.id, imageRemoveBackgroundGreenscreenPrimitive],
|
|
3216
|
-
[mediaOverlayPrimitive.id, mediaOverlayPrimitive],
|
|
3217
|
-
[videoGeneratePrimitive.id, videoGeneratePrimitive],
|
|
3218
|
-
[videoRenderSlidesPrimitive.id, videoRenderSlidesPrimitive],
|
|
3219
|
-
[mediaDedupePrimitive.id, mediaDedupePrimitive],
|
|
3220
|
-
[videoNormalizePrimitive.id, videoNormalizePrimitive],
|
|
3221
|
-
[videoDownloadPrimitive.id, videoDownloadPrimitive],
|
|
3222
|
-
[videoIngestPrimitive.id, videoIngestPrimitive],
|
|
3223
|
-
[videoTrimPrimitive.id, videoTrimPrimitive],
|
|
3224
|
-
[videoRemoveCaptionsPrimitive.id, videoRemoveCaptionsPrimitive],
|
|
3225
|
-
[videoExtractAudioPrimitive.id, videoExtractAudioPrimitive],
|
|
3226
|
-
[audioTrimPrimitive.id, audioTrimPrimitive],
|
|
3227
|
-
[videoProbePrimitive.id, videoProbePrimitive],
|
|
3228
|
-
[audioProbePrimitive.id, audioProbePrimitive],
|
|
3229
|
-
[videoExtractFramePrimitive.id, videoExtractFramePrimitive],
|
|
3230
|
-
[videoMutePrimitive.id, videoMutePrimitive],
|
|
3231
|
-
[videoReplaceAudioPrimitive.id, videoReplaceAudioPrimitive],
|
|
3232
|
-
[videoConcatPrimitive.id, videoConcatPrimitive],
|
|
3233
|
-
[audioConcatPrimitive.id, audioConcatPrimitive],
|
|
3234
|
-
[audioNormalizePrimitive.id, audioNormalizePrimitive],
|
|
3235
|
-
[ttsPrimitive.id, ttsPrimitive],
|
|
3236
|
-
[sttPrimitive.id, sttPrimitive],
|
|
3237
|
-
[musicPrimitive.id, musicPrimitive],
|
|
3238
|
-
[captionsPrimitive.id, captionsPrimitive],
|
|
3239
|
-
[speechRegeneratePrimitive.id, speechRegeneratePrimitive],
|
|
3240
|
-
[hyperframesRenderPrimitive.id, hyperframesRenderPrimitive],
|
|
3241
|
-
[brainstormHooksPrimitive.id, brainstormHooksPrimitive],
|
|
3242
|
-
[brainstormAwarenessStagesPrimitive.id, brainstormAwarenessStagesPrimitive],
|
|
3243
|
-
[brainstormAnglesPrimitive.id, brainstormAnglesPrimitive],
|
|
3244
|
-
[brainstormColdstartPrimitive.id, brainstormColdstartPrimitive],
|
|
3245
|
-
[brainstormProductPlacementPrimitive.id, brainstormProductPlacementPrimitive]
|
|
3246
|
-
]);
|
|
3247
|
-
get(id) {
|
|
3248
|
-
return this.definitions.get(id) ?? null;
|
|
3249
|
-
}
|
|
3250
|
-
getByKind(kind) {
|
|
3251
|
-
return this.list().find((definition) => definition.kind === kind) ?? null;
|
|
3252
|
-
}
|
|
3253
|
-
list() {
|
|
3254
|
-
return [...this.definitions.values()];
|
|
3255
|
-
}
|
|
3256
|
-
}
|
|
3257
|
-
export const primitiveRegistry = new PrimitiveRegistry();
|
|
3258
|
-
function normalizeHyperframesCompositionPayload(payload) {
|
|
3259
|
-
const duration = payload.duration_seconds ?? payload.duration ?? payload.layers.reduce((max, layer) => Math.max(max, layer.start + layer.duration), 0);
|
|
3260
|
-
return {
|
|
3261
|
-
compositionId: payload.composition_id,
|
|
3262
|
-
title: payload.title,
|
|
3263
|
-
sourceFolder: payload.source_folder,
|
|
3264
|
-
width: payload.width,
|
|
3265
|
-
height: payload.height,
|
|
3266
|
-
duration,
|
|
3267
|
-
backgroundColor: payload.background_color,
|
|
3268
|
-
css: payload.css,
|
|
3269
|
-
layers: payload.layers.map((layer) => ({
|
|
3270
|
-
id: layer.id,
|
|
3271
|
-
kind: layer.kind,
|
|
3272
|
-
label: layer.label,
|
|
3273
|
-
mode: layer.mode,
|
|
3274
|
-
start: layer.start,
|
|
3275
|
-
duration: layer.duration,
|
|
3276
|
-
track: layer.track,
|
|
3277
|
-
z: layer.z,
|
|
3278
|
-
x: layer.x,
|
|
3279
|
-
y: layer.y,
|
|
3280
|
-
width: layer.width,
|
|
3281
|
-
height: layer.height,
|
|
3282
|
-
src: layer.src,
|
|
3283
|
-
mediaStart: layer.mediaStart ?? layer.media_start,
|
|
3284
|
-
playbackStart: layer.playbackStart ?? layer.playback_start ?? layer.mediaStart ?? layer.media_start,
|
|
3285
|
-
fit: layer.fit,
|
|
3286
|
-
muted: layer.muted,
|
|
3287
|
-
volume: layer.volume,
|
|
3288
|
-
text: layer.text,
|
|
3289
|
-
html: layer.html,
|
|
3290
|
-
color: layer.color,
|
|
3291
|
-
background: layer.background,
|
|
3292
|
-
textBackgroundStyle: layer.textBackgroundStyle ?? layer.text_background_style,
|
|
3293
|
-
fontFamily: layer.fontFamily ?? layer.font_family,
|
|
3294
|
-
fontWeight: layer.fontWeight ?? layer.font_weight,
|
|
3295
|
-
fontSize: layer.fontSize ?? layer.font_size,
|
|
3296
|
-
lineHeight: layer.lineHeight ?? layer.line_height,
|
|
3297
|
-
radius: layer.radius,
|
|
3298
|
-
opacity: layer.opacity,
|
|
3299
|
-
viralAlignment: layer.viralAlignment ?? layer.viral_alignment
|
|
3300
|
-
}))
|
|
3301
|
-
};
|
|
3302
|
-
}
|
|
3303
|
-
async function storePrimitiveImage(ctx, input) {
|
|
3304
|
-
const stored = await ctx.storage.putBuffer(input.key, input.bytes, {
|
|
3305
|
-
contentType: input.contentType,
|
|
3306
|
-
kind: "image",
|
|
3307
|
-
metadata: input.metadata
|
|
3308
|
-
});
|
|
3309
|
-
return { ...stored, contentType: input.contentType };
|
|
3310
|
-
}
|
|
3311
|
-
async function storePrimitiveVideo(ctx, input) {
|
|
3312
|
-
const stored = await ctx.storage.putBuffer(input.key, input.bytes, {
|
|
3313
|
-
contentType: input.contentType,
|
|
3314
|
-
kind: "video",
|
|
3315
|
-
metadata: input.metadata
|
|
3316
|
-
});
|
|
3317
|
-
return { ...stored, contentType: input.contentType };
|
|
3318
|
-
}
|
|
3319
|
-
function buildImagePrimitiveOutput(input) {
|
|
3320
|
-
return {
|
|
3321
|
-
files: input.files,
|
|
3322
|
-
primary_file_url: input.fileUrl,
|
|
3323
|
-
image: {
|
|
3324
|
-
file_url: input.fileUrl,
|
|
3325
|
-
content_type: input.contentType,
|
|
3326
|
-
...(input.revisedPrompt !== undefined ? { revised_prompt: input.revisedPrompt } : {}),
|
|
3327
|
-
...(input.derivedFromUrl ? { derived_from_url: input.derivedFromUrl } : {}),
|
|
3328
|
-
...(input.maskUrl ? { mask_url: input.maskUrl } : {})
|
|
3329
|
-
}
|
|
3330
|
-
};
|
|
3331
|
-
}
|
|
3332
|
-
function defaultImageModelForProvider(provider) {
|
|
3333
|
-
switch (provider) {
|
|
3334
|
-
case "openai":
|
|
3335
|
-
return "gpt-image-2";
|
|
3336
|
-
case "openrouter":
|
|
3337
|
-
return "google/gemini-2.5-flash-image-preview";
|
|
3338
|
-
default:
|
|
3339
|
-
return "gemini-2.5-flash-image";
|
|
3340
|
-
}
|
|
3341
|
-
}
|
|
3342
|
-
function defaultVideoModelForProvider(provider) {
|
|
3343
|
-
switch (provider) {
|
|
3344
|
-
case "openai":
|
|
3345
|
-
return "sora-2";
|
|
3346
|
-
case "gemini":
|
|
3347
|
-
return "veo-3.0-generate-001";
|
|
3348
|
-
case "openrouter":
|
|
3349
|
-
return "bytedance/seedance-2.0";
|
|
3350
|
-
}
|
|
3351
|
-
}
|
|
3352
|
-
function defaultTextModelForProvider(provider) {
|
|
3353
|
-
switch (provider) {
|
|
3354
|
-
case "gemini":
|
|
3355
|
-
return "gemini-2.5-flash";
|
|
3356
|
-
case "openrouter":
|
|
3357
|
-
return "qwen/qwen3.6-plus";
|
|
3358
|
-
case "perplexity":
|
|
3359
|
-
return "sonar";
|
|
3360
|
-
default:
|
|
3361
|
-
return "gpt-5.4-mini";
|
|
3362
|
-
}
|
|
3363
|
-
}
|
|
3364
|
-
// Extensions that are safe to feed a speech provider without the ffmpeg
|
|
3365
|
-
// normalization pass; everything else (video or unknown) gets demuxed first.
|
|
3366
|
-
function inferSpeechSourceKind(sourceUrl) {
|
|
3367
|
-
return /\.(mp3|m4a|aac|ogg|oga|flac|wav)(\?|#|$)/i.test(sourceUrl) ? "audio" : "video";
|
|
3368
|
-
}
|
|
3369
|
-
function normalizeAiVideoDurationSeconds(explicitSeconds, milliseconds) {
|
|
3370
|
-
const seconds = Number(explicitSeconds);
|
|
3371
|
-
if (Number.isFinite(seconds) && seconds > 0) {
|
|
3372
|
-
return Math.max(1, Math.round(seconds));
|
|
3373
|
-
}
|
|
3374
|
-
const durationMs = Number(milliseconds);
|
|
3375
|
-
if (Number.isFinite(durationMs) && durationMs > 0) {
|
|
3376
|
-
return Math.max(1, Math.ceil(durationMs / 1000));
|
|
3377
|
-
}
|
|
3378
|
-
return undefined;
|
|
3379
|
-
}
|
|
3380
|
-
function extensionForOutputFormat(format) {
|
|
3381
|
-
return format === "jpeg" ? "jpg" : format;
|
|
3382
|
-
}
|
|
3383
|
-
function contentTypeForOutputFormat(format, fallback) {
|
|
3384
|
-
if (format === "jpeg") {
|
|
3385
|
-
return "image/jpeg";
|
|
3386
|
-
}
|
|
3387
|
-
if (format === "webp") {
|
|
3388
|
-
return "image/webp";
|
|
3389
|
-
}
|
|
3390
|
-
return fallback && fallback !== "image/svg+xml" ? fallback : "image/png";
|
|
3391
|
-
}
|
|
3392
|
-
async function normalizePrimitiveImageOutput(bytes, format) {
|
|
3393
|
-
const buffer = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
3394
|
-
if (format === "jpeg") {
|
|
3395
|
-
return {
|
|
3396
|
-
bytes: await sharp(buffer).jpeg({ quality: 92 }).toBuffer(),
|
|
3397
|
-
contentType: "image/jpeg"
|
|
3398
|
-
};
|
|
3399
|
-
}
|
|
3400
|
-
if (format === "webp") {
|
|
3401
|
-
return {
|
|
3402
|
-
bytes: await sharp(buffer).webp({ quality: 92 }).toBuffer(),
|
|
3403
|
-
contentType: "image/webp"
|
|
3404
|
-
};
|
|
3405
|
-
}
|
|
3406
|
-
return {
|
|
3407
|
-
bytes: await sharp(buffer).png().toBuffer(),
|
|
3408
|
-
contentType: "image/png"
|
|
3409
|
-
};
|
|
3410
|
-
}
|
|
3411
|
-
function clampByte(value) {
|
|
3412
|
-
const n = Math.round(Number.isFinite(value) ? value : 0);
|
|
3413
|
-
return Math.max(0, Math.min(255, n));
|
|
3414
|
-
}
|
|
3415
|
-
const NAMED_KEY_COLORS = {
|
|
3416
|
-
green: [0, 255, 0],
|
|
3417
|
-
lime: [0, 255, 0],
|
|
3418
|
-
"chroma-green": [0, 177, 64],
|
|
3419
|
-
"chromakey-green": [0, 177, 64],
|
|
3420
|
-
white: [255, 255, 255],
|
|
3421
|
-
black: [0, 0, 0],
|
|
3422
|
-
blue: [0, 0, 255],
|
|
3423
|
-
"chroma-blue": [0, 71, 187],
|
|
3424
|
-
"chromakey-blue": [0, 71, 187],
|
|
3425
|
-
red: [255, 0, 0],
|
|
3426
|
-
magenta: [255, 0, 255],
|
|
3427
|
-
cyan: [0, 255, 255],
|
|
3428
|
-
gray: [128, 128, 128],
|
|
3429
|
-
grey: [128, 128, 128]
|
|
3430
|
-
};
|
|
3431
|
-
// Parse a key color from hex (#rgb / #rrggbb), rgb()/rgba() triples, or a small
|
|
3432
|
-
// set of named colors. Throws on anything unrecognized so a typo surfaces
|
|
3433
|
-
// instead of silently keying out the wrong color.
|
|
3434
|
-
function parseKeyColor(value) {
|
|
3435
|
-
const raw = value.trim().toLowerCase();
|
|
3436
|
-
const named = NAMED_KEY_COLORS[raw];
|
|
3437
|
-
if (named) {
|
|
3438
|
-
return { r: named[0], g: named[1], b: named[2] };
|
|
3439
|
-
}
|
|
3440
|
-
const rgbMatch = raw.match(/^rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)/);
|
|
3441
|
-
if (rgbMatch) {
|
|
3442
|
-
return { r: clampByte(Number(rgbMatch[1])), g: clampByte(Number(rgbMatch[2])), b: clampByte(Number(rgbMatch[3])) };
|
|
3443
|
-
}
|
|
3444
|
-
let hex = raw.startsWith("#") ? raw.slice(1) : raw;
|
|
3445
|
-
if (/^[0-9a-f]{3}$/.test(hex)) {
|
|
3446
|
-
hex = hex.split("").map((ch) => ch + ch).join("");
|
|
3447
|
-
}
|
|
3448
|
-
if (/^[0-9a-f]{6}$/.test(hex)) {
|
|
3449
|
-
return {
|
|
3450
|
-
r: parseInt(hex.slice(0, 2), 16),
|
|
3451
|
-
g: parseInt(hex.slice(2, 4), 16),
|
|
3452
|
-
b: parseInt(hex.slice(4, 6), 16)
|
|
3453
|
-
};
|
|
3454
|
-
}
|
|
3455
|
-
throw new Error(`Unrecognized key_color "${value}". Use a hex value (e.g. #00FF00), an rgb(...) triple, or a named color (green/white/blue/black).`);
|
|
3456
|
-
}
|
|
3457
|
-
// A plain-English label for the key color so the image-generation prompt can
|
|
3458
|
-
// tell the model exactly which flat background to paint.
|
|
3459
|
-
function describeKeyColor(raw, rgb) {
|
|
3460
|
-
const named = raw.trim().toLowerCase();
|
|
3461
|
-
if (/green|lime/.test(named))
|
|
3462
|
-
return "chroma green";
|
|
3463
|
-
if (/blue/.test(named))
|
|
3464
|
-
return "chroma blue";
|
|
3465
|
-
if (named === "white" || (rgb.r > 240 && rgb.g > 240 && rgb.b > 240))
|
|
3466
|
-
return "solid white";
|
|
3467
|
-
if (named === "black" || (rgb.r < 15 && rgb.g < 15 && rgb.b < 15))
|
|
3468
|
-
return "solid black";
|
|
3469
|
-
if (rgb.g > rgb.r && rgb.g > rgb.b)
|
|
3470
|
-
return "chroma green";
|
|
3471
|
-
if (rgb.b > rgb.r && rgb.b > rgb.g)
|
|
3472
|
-
return "chroma blue";
|
|
3473
|
-
return "flat solid";
|
|
3474
|
-
}
|
|
3475
|
-
function buildOverlayGenerationPrompt(subject, keyHex, keyLabel) {
|
|
3476
|
-
return [
|
|
3477
|
-
subject.trim(),
|
|
3478
|
-
"",
|
|
3479
|
-
`BACKGROUND REQUIREMENT (critical for clean keying): render ONLY the subject, fully in frame, on a completely flat, uniform, evenly-lit ${keyLabel} background (target color ${keyHex}). The background must be one single solid ${keyLabel} fill — absolutely NO gradient, NO vignette, NO cast shadows or reflections on the background, and NO ${keyLabel} tones anywhere on the subject itself (avoid ${keyLabel} clothing, props, or rim light). Do not add any border, frame, text, or watermark. Give the subject clean, crisp edges against the ${keyLabel} so it can be cut out to a transparent PNG.`
|
|
3480
|
-
].join("\n");
|
|
3481
|
-
}
|
|
3482
|
-
// Local chroma-key background removal. Distance-thresholds every pixel against
|
|
3483
|
-
// the key color: within `tolerance` becomes fully transparent, a `softness`
|
|
3484
|
-
// feather band anti-aliases the edge, and (for chromatic keys) a classic
|
|
3485
|
-
// despill clamps the key's dominant channel so no colored fringe survives.
|
|
3486
|
-
async function chromaKeyRemoveBackground(source, options) {
|
|
3487
|
-
const key = parseKeyColor(options.keyColor);
|
|
3488
|
-
const buffer = Buffer.isBuffer(source) ? source : Buffer.from(source);
|
|
3489
|
-
const { data, info } = await sharp(buffer).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
3490
|
-
const channels = info.channels;
|
|
3491
|
-
const maxDist = Math.sqrt(3) * 255;
|
|
3492
|
-
const inner = Math.max(0, options.tolerance) * maxDist;
|
|
3493
|
-
const outer = Math.max(inner + 1e-6, (options.tolerance + Math.max(0, options.softness)) * maxDist);
|
|
3494
|
-
const keyChannels = [key.r, key.g, key.b];
|
|
3495
|
-
const keyChroma = Math.max(...keyChannels) - Math.min(...keyChannels);
|
|
3496
|
-
// Only despill chromatic keys (green/blue/etc.); white/gray/black have no hue
|
|
3497
|
-
// fringe to suppress and clamping them would tint the subject.
|
|
3498
|
-
const doDespill = options.despill && keyChroma >= 40;
|
|
3499
|
-
const dominant = keyChannels.indexOf(Math.max(...keyChannels));
|
|
3500
|
-
let keyedPixels = 0;
|
|
3501
|
-
const totalPixels = info.width * info.height;
|
|
3502
|
-
for (let i = 0; i < data.length; i += channels) {
|
|
3503
|
-
const r = data[i];
|
|
3504
|
-
const g = data[i + 1];
|
|
3505
|
-
const b = data[i + 2];
|
|
3506
|
-
const baseAlpha = channels === 4 ? data[i + 3] : 255;
|
|
3507
|
-
const dr = r - key.r;
|
|
3508
|
-
const dg = g - key.g;
|
|
3509
|
-
const db = b - key.b;
|
|
3510
|
-
const dist = Math.sqrt(dr * dr + dg * dg + db * db);
|
|
3511
|
-
let alpha;
|
|
3512
|
-
if (dist <= inner) {
|
|
3513
|
-
alpha = 0;
|
|
3514
|
-
keyedPixels++;
|
|
3515
|
-
}
|
|
3516
|
-
else if (dist >= outer) {
|
|
3517
|
-
alpha = baseAlpha;
|
|
3518
|
-
}
|
|
3519
|
-
else {
|
|
3520
|
-
const t = (dist - inner) / (outer - inner);
|
|
3521
|
-
alpha = Math.round(baseAlpha * t);
|
|
3522
|
-
}
|
|
3523
|
-
if (doDespill && alpha > 0) {
|
|
3524
|
-
if (dominant === 1)
|
|
3525
|
-
data[i + 1] = Math.min(g, Math.max(r, b));
|
|
3526
|
-
else if (dominant === 0)
|
|
3527
|
-
data[i] = Math.min(r, Math.max(g, b));
|
|
3528
|
-
else
|
|
3529
|
-
data[i + 2] = Math.min(b, Math.max(r, g));
|
|
3530
|
-
}
|
|
3531
|
-
if (channels === 4)
|
|
3532
|
-
data[i + 3] = alpha;
|
|
3533
|
-
}
|
|
3534
|
-
const encoded = sharp(data, { raw: { width: info.width, height: info.height, channels } });
|
|
3535
|
-
const bytes = options.outputFormat === "webp"
|
|
3536
|
-
? await encoded.webp({ quality: 92, alphaQuality: 100 }).toBuffer()
|
|
3537
|
-
: await encoded.png().toBuffer();
|
|
3538
|
-
return {
|
|
3539
|
-
bytes,
|
|
3540
|
-
contentType: options.outputFormat === "webp" ? "image/webp" : "image/png",
|
|
3541
|
-
width: info.width,
|
|
3542
|
-
height: info.height,
|
|
3543
|
-
keyedFraction: totalPixels > 0 ? keyedPixels / totalPixels : 0
|
|
3544
|
-
};
|
|
3545
|
-
}
|
|
3546
|
-
async function recordPrimitiveProviderBilling(ctx, generated, kind) {
|
|
3547
|
-
if (!Number.isFinite(generated.usageCostUsd) || generated.usageCostUsd <= 0) {
|
|
3548
|
-
return;
|
|
3549
|
-
}
|
|
3550
|
-
const providerJob = generated.providerJob ?? {};
|
|
3551
|
-
const providerJobId = providerJob.id ?? providerJob.generation_id;
|
|
3552
|
-
const providerJobIdText = typeof providerJobId === "string" && providerJobId.trim() ? providerJobId.trim() : null;
|
|
3553
|
-
await ctx.billing.record({
|
|
3554
|
-
type: "ai_generation",
|
|
3555
|
-
costUsd: generated.usageCostUsd,
|
|
3556
|
-
idempotencyKey: providerJobIdText ? `primitive-provider:${kind}:${providerJobIdText}` : undefined,
|
|
3557
|
-
occurredAtMs: Date.now(),
|
|
3558
|
-
metadata: {
|
|
3559
|
-
kind,
|
|
3560
|
-
provider_job_id: providerJobIdText,
|
|
3561
|
-
provider_job: providerJob
|
|
3562
|
-
}
|
|
3563
|
-
});
|
|
3564
|
-
}
|
|
3565
|
-
function readPrimitiveAsset(name) {
|
|
3566
|
-
const moduleDir = path.dirname(fileURLToPath(import.meta.url));
|
|
3567
|
-
const candidates = [
|
|
3568
|
-
path.resolve(moduleDir, "assets", name),
|
|
3569
|
-
path.resolve(moduleDir, "..", "src", "assets", name),
|
|
3570
|
-
// Package-root copy (compiled module lives at <root>/dist/src): covers
|
|
3571
|
-
// `vidfarm serve` launched from an arbitrary cwd out of the npm package.
|
|
3572
|
-
path.resolve(moduleDir, "..", "..", "src", "assets", name),
|
|
3573
|
-
path.resolve(process.cwd(), "dist", "src", "assets", name),
|
|
3574
|
-
path.resolve(process.cwd(), "src", "assets", name)
|
|
3575
|
-
];
|
|
3576
|
-
const assetPath = candidates.find((candidate) => existsSync(candidate));
|
|
3577
|
-
if (!assetPath) {
|
|
3578
|
-
throw new Error(`Primitive asset not found: ${name}`);
|
|
3579
|
-
}
|
|
3580
|
-
return readFileSync(assetPath, "utf8");
|
|
3581
|
-
}
|
|
3582
|
-
function normalizeOfferDescriptionPayload(raw) {
|
|
3583
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
3584
|
-
return raw;
|
|
3585
|
-
}
|
|
3586
|
-
const payload = raw;
|
|
3587
|
-
return {
|
|
3588
|
-
...payload,
|
|
3589
|
-
offer_description: payload.offer_description
|
|
3590
|
-
?? payload.offerDescription
|
|
3591
|
-
?? payload.description
|
|
3592
|
-
?? payload.offer
|
|
3593
|
-
?? payload.details,
|
|
3594
|
-
count: payload.count ?? payload.n ?? payload.number_of_hooks ?? payload.number_of_angles ?? payload.question_count
|
|
3595
|
-
};
|
|
3596
|
-
}
|
|
3597
|
-
function normalizeAnglePayload(raw) {
|
|
3598
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
3599
|
-
return raw;
|
|
3600
|
-
}
|
|
3601
|
-
const payload = normalizeOfferDescriptionPayload(raw);
|
|
3602
|
-
return {
|
|
3603
|
-
...payload,
|
|
3604
|
-
problem_awareness: payload.problem_awareness ?? payload.problemAwareness ?? payload.problem_stage ?? payload.problemStage,
|
|
3605
|
-
solution_awareness: payload.solution_awareness ?? payload.solutionAwareness ?? payload.solution_stage ?? payload.solutionStage
|
|
3606
|
-
};
|
|
3607
|
-
}
|
|
3608
|
-
function normalizeColdstartPayload(raw) {
|
|
3609
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
3610
|
-
return raw;
|
|
3611
|
-
}
|
|
3612
|
-
const payload = raw;
|
|
3613
|
-
return {
|
|
3614
|
-
...payload,
|
|
3615
|
-
user_message: payload.user_message
|
|
3616
|
-
?? payload.userMessage
|
|
3617
|
-
?? payload.message
|
|
3618
|
-
?? payload.user_input
|
|
3619
|
-
?? payload.userInput
|
|
3620
|
-
?? payload.offer_description
|
|
3621
|
-
?? payload.offerDescription
|
|
3622
|
-
?? payload.description
|
|
3623
|
-
?? payload.offer
|
|
3624
|
-
?? payload.details,
|
|
3625
|
-
count: payload.count ?? payload.n ?? payload.question_count
|
|
3626
|
-
};
|
|
3627
|
-
}
|
|
3628
|
-
function normalizeProductPlacementPayload(raw) {
|
|
3629
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
3630
|
-
return raw;
|
|
3631
|
-
}
|
|
3632
|
-
const payload = normalizeOfferDescriptionPayload(raw);
|
|
3633
|
-
return {
|
|
3634
|
-
...payload,
|
|
3635
|
-
source_video_url: payload.source_video_url
|
|
3636
|
-
?? payload.sourceVideoUrl
|
|
3637
|
-
?? payload.video_url
|
|
3638
|
-
?? payload.videoUrl
|
|
3639
|
-
?? payload.source_url
|
|
3640
|
-
?? payload.sourceUrl
|
|
3641
|
-
?? payload.url
|
|
3642
|
-
};
|
|
3643
|
-
}
|
|
3644
|
-
function preferredTextProvider(availableProviders) {
|
|
3645
|
-
// Text-only priority — the audio-only elevenlabs provider is never a candidate.
|
|
3646
|
-
const priority = ["openai", "gemini", "openrouter", "perplexity"];
|
|
3647
|
-
return priority.find((provider) => availableProviders.includes(provider)) ?? "openai";
|
|
3648
|
-
}
|
|
3649
|
-
// Product-placement analysis needs a model that can watch video. Gemini has native video
|
|
3650
|
-
// understanding; OpenRouter can route to multimodal models; fall back to whatever is available.
|
|
3651
|
-
function preferredVideoTextProvider(availableProviders) {
|
|
3652
|
-
const priority = ["gemini", "openrouter", "openai", "perplexity"];
|
|
3653
|
-
return priority.find((provider) => availableProviders.includes(provider)) ?? "gemini";
|
|
3654
|
-
}
|
|
3655
|
-
async function runBrainstormText(ctx, input) {
|
|
3656
|
-
const provider = input.provider ?? preferredTextProvider(ctx.providers.listAvailableProviders());
|
|
3657
|
-
const model = input.model ?? defaultTextModelForProvider(provider);
|
|
3658
|
-
const generated = await ctx.providers.generateText({
|
|
3659
|
-
provider,
|
|
3660
|
-
model,
|
|
3661
|
-
prompt: input.prompt,
|
|
3662
|
-
temperature: input.temperature ?? 0.7,
|
|
3663
|
-
responseFormat: input.responseFormat ?? "text",
|
|
3664
|
-
...(input.attachments && input.attachments.length ? { attachments: input.attachments } : {})
|
|
3665
|
-
});
|
|
3666
|
-
await recordPrimitiveProviderBilling(ctx, {
|
|
3667
|
-
usageCostUsd: generated.usage.costUsd,
|
|
3668
|
-
providerJob: {
|
|
3669
|
-
provider,
|
|
3670
|
-
model,
|
|
3671
|
-
usage: generated.usage
|
|
3672
|
-
}
|
|
3673
|
-
}, input.kind);
|
|
3674
|
-
return {
|
|
3675
|
-
provider,
|
|
3676
|
-
model,
|
|
3677
|
-
text: generated.text.trim(),
|
|
3678
|
-
usage: generated.usage
|
|
3679
|
-
};
|
|
3680
|
-
}
|
|
3681
|
-
async function runBrainstormJson(ctx, input) {
|
|
3682
|
-
const generated = await runBrainstormText(ctx, {
|
|
3683
|
-
...input,
|
|
3684
|
-
temperature: 0.35,
|
|
3685
|
-
responseFormat: "json"
|
|
3686
|
-
});
|
|
3687
|
-
const parsed = input.resultSchema.parse(parseJsonFromModelText(generated.text));
|
|
3688
|
-
return {
|
|
3689
|
-
...generated,
|
|
3690
|
-
result: parsed
|
|
3691
|
-
};
|
|
3692
|
-
}
|
|
3693
|
-
function parseJsonFromModelText(text) {
|
|
3694
|
-
const trimmed = text.trim();
|
|
3695
|
-
try {
|
|
3696
|
-
return JSON.parse(trimmed);
|
|
3697
|
-
}
|
|
3698
|
-
catch {
|
|
3699
|
-
const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
3700
|
-
if (fenced?.[1]) {
|
|
3701
|
-
return JSON.parse(fenced[1]);
|
|
3702
|
-
}
|
|
3703
|
-
const firstBrace = trimmed.indexOf("{");
|
|
3704
|
-
const lastBrace = trimmed.lastIndexOf("}");
|
|
3705
|
-
if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
|
|
3706
|
-
return JSON.parse(trimmed.slice(firstBrace, lastBrace + 1));
|
|
3707
|
-
}
|
|
3708
|
-
throw new Error("AI returned invalid JSON for brainstorm primitive output.");
|
|
3709
|
-
}
|
|
3710
|
-
}
|
|
3711
|
-
function buildHooksPrompt(offerDescription, count) {
|
|
3712
|
-
return [
|
|
3713
|
-
"You are Vidfarm's elite paid-social hook strategist.",
|
|
3714
|
-
"Your job is to generate high-upside ad hooks for TikTok-native creative.",
|
|
3715
|
-
"Bias toward hooks that feel native to TikTok, but keep raw persuasive power more important than surface trendiness.",
|
|
3716
|
-
"Max creativity. Do not hold back. Be as diverse as possible while staying relevant to the offer.",
|
|
3717
|
-
"Use many angle families across the set: pain, status, curiosity, gossip, confession, myth-busting, shock, taboo, mechanism, objection, identity, urgency, pattern interrupt, transformation, proof, enemy, mistake, secret, social proof, aspiration, and contrarian frames.",
|
|
3718
|
-
"Each hook should be sharp, testable, and materially different from the others.",
|
|
3719
|
-
`Generate exactly ${count} hooks.`,
|
|
3720
|
-
"Return strict JSON only with shape {\"hooks\":[{\"hook\":\"...\",\"explanation\":\"...\"}]}.",
|
|
3721
|
-
"Explanation should be 1-3 sentences on why the hook works and what angle it is leveraging.",
|
|
3722
|
-
"",
|
|
3723
|
-
"Offer details:",
|
|
3724
|
-
offerDescription.trim(),
|
|
3725
|
-
"",
|
|
3726
|
-
"Reference lessons to apply:",
|
|
3727
|
-
hooksReferenceText
|
|
3728
|
-
].join("\n");
|
|
3729
|
-
}
|
|
3730
|
-
function buildAwarenessStagesPrompt(offerDescription) {
|
|
3731
|
-
return [
|
|
3732
|
-
"You are Vidfarm's direct-response strategist.",
|
|
3733
|
-
"The user is not sure what type of ads to produce.",
|
|
3734
|
-
"Use Eugene Schwartz-style awareness-stage reasoning to recommend what kinds of ads to make first.",
|
|
3735
|
-
"Write a practical response for TikTok-first performance marketing, but keep the advice broadly useful for paid social.",
|
|
3736
|
-
"Do not return JSON. Return a clear markdown string.",
|
|
3737
|
-
"Include:",
|
|
3738
|
-
"- which awareness stages are most promising first",
|
|
3739
|
-
"- why those stages fit this offer",
|
|
3740
|
-
"- what ad types/messages belong in each stage",
|
|
3741
|
-
"- what to test before moving to deeper or broader awareness stages",
|
|
3742
|
-
"- a short next-step recommendation",
|
|
3743
|
-
"",
|
|
3744
|
-
"Offer details:",
|
|
3745
|
-
offerDescription.trim(),
|
|
3746
|
-
"",
|
|
3747
|
-
"Reference lessons to apply:",
|
|
3748
|
-
awarenessReferenceText
|
|
3749
|
-
].join("\n");
|
|
3750
|
-
}
|
|
3751
|
-
function buildAnglesPrompt(payload) {
|
|
3752
|
-
return [
|
|
3753
|
-
"You are Vidfarm's senior direct-response angle strategist.",
|
|
3754
|
-
"Generate persuasive advertising angles for TikTok-native ads.",
|
|
3755
|
-
"Persuasive depth matters more than format gimmicks. The user wants raw winning angles to test.",
|
|
3756
|
-
"Do not mention templates, scenes, or production mechanics. Focus on the strategic angle itself.",
|
|
3757
|
-
"Create major diversity across the set: pain, myths, enemies, mechanisms, shocking truths, confessions, gossip/social proof, objections, hidden costs, status, identity, aspiration, taboo, mistake, urgency, proof, authority, trend hijack, curiosity, and contrarian frames.",
|
|
3758
|
-
"Favor known winning angle archetypes while still making them specific to the offer.",
|
|
3759
|
-
`Generate exactly ${payload.count} angles.`,
|
|
3760
|
-
"Return strict JSON only with shape {\"angles\":[{\"angle\":\"...\",\"explanation\":\"...\"}]}",
|
|
3761
|
-
"Explanation should say why the angle fits this awareness state and what persuasive lever it uses.",
|
|
3762
|
-
"",
|
|
3763
|
-
`Problem awareness: ${payload.problem_awareness}`,
|
|
3764
|
-
`Solution awareness: ${payload.solution_awareness}`,
|
|
3765
|
-
`Derived stage note: ${describeAwarenessSelection(payload.problem_awareness, payload.solution_awareness)}`,
|
|
3766
|
-
"",
|
|
3767
|
-
"Offer details:",
|
|
3768
|
-
payload.offer_description.trim(),
|
|
3769
|
-
"",
|
|
3770
|
-
"Reference lessons to apply from awareness stages:",
|
|
3771
|
-
awarenessReferenceText,
|
|
3772
|
-
"",
|
|
3773
|
-
"Reference lessons to apply from hooks:",
|
|
3774
|
-
hooksReferenceText
|
|
3775
|
-
].join("\n");
|
|
3776
|
-
}
|
|
3777
|
-
function buildColdstartPrompt(userMessage, count) {
|
|
3778
|
-
return [
|
|
3779
|
-
"You are Vidfarm's TikTok growth strategist.",
|
|
3780
|
-
"The customer has no idea where to start.",
|
|
3781
|
-
"Generate the best possible foundational Q&A questions for the customer and AI to discuss so they can build a strong TikTok strategy from zero.",
|
|
3782
|
-
"The goal is to produce questions whose answers can later power awareness-stage selection, angle generation, hooks, scripts, and template choice.",
|
|
3783
|
-
`Generate exactly ${count} questions.`,
|
|
3784
|
-
"You must include questions covering, at minimum:",
|
|
3785
|
-
"- one-line offer",
|
|
3786
|
-
"- one-paragraph offer",
|
|
3787
|
-
"- precise target customer psyche",
|
|
3788
|
-
"- customer pain in their own words",
|
|
3789
|
-
"- awareness levels or uncertainty about them",
|
|
3790
|
-
"Add more strong questions beyond those when useful.",
|
|
3791
|
-
"Return strict JSON only with shape {\"questions\":[{\"question\":\"...\",\"why_it_matters\":\"...\"}]}",
|
|
3792
|
-
"Questions should be phrased so the user can answer them in a Google Doc or markdown file for future use.",
|
|
3793
|
-
"At least one question should explicitly encourage them to save these answers for reuse inside Vidfarm.",
|
|
3794
|
-
"",
|
|
3795
|
-
"User message:",
|
|
3796
|
-
userMessage.trim()
|
|
3797
|
-
].join("\n");
|
|
3798
|
-
}
|
|
3799
|
-
function buildProductPlacementPrompt(offerDescription, count) {
|
|
3800
|
-
return [
|
|
3801
|
-
"You are Vidfarm's native-advertising product-placement strategist.",
|
|
3802
|
-
"You are watching the attached video. Study what is actually on screen: scenes, actions, objects, spoken lines, on-screen text, transitions, and emotional beats.",
|
|
3803
|
-
"Your job is to find the highest-leverage moments to organically place the user's product into THIS video so it feels native rather than bolted on.",
|
|
3804
|
-
"Bias toward placements that ride the existing attention, emotion, or story beat of the moment. Native and believable beats loud and interruptive.",
|
|
3805
|
-
"Consider many placement types: on-screen prop/set dressing, character interaction with the product, verbal mention or callout, text/caption overlay, sticker or lower-third, b-roll cutaway, screen/UI replacement, before/after demo, and end-card CTA.",
|
|
3806
|
-
`Identify exactly ${count} distinct product-placement opportunities, ordered from strongest to weakest.`,
|
|
3807
|
-
"Each opportunity must reference a real, specific moment you actually saw in the video, with an approximate timestamp or range (for example \"0:04\" or \"0:12-0:18\").",
|
|
3808
|
-
"Return strict JSON only with shape {\"opportunities\":[{\"timestamp\":\"...\",\"scene\":\"...\",\"placement_type\":\"...\",\"placement_idea\":\"...\",\"why_it_works\":\"...\"}]}.",
|
|
3809
|
-
"\"scene\" briefly describes what is happening on screen at that moment. \"placement_idea\" is the concrete, actionable way to insert the product. \"why_it_works\" explains the persuasion/attention logic.",
|
|
3810
|
-
"",
|
|
3811
|
-
"Product / offer to place:",
|
|
3812
|
-
offerDescription.trim()
|
|
3813
|
-
].join("\n");
|
|
3814
|
-
}
|
|
3815
|
-
function describeAwarenessSelection(problemAwareness, solutionAwareness) {
|
|
3816
|
-
if (problemAwareness === "problem_unaware" && solutionAwareness === "solution_unaware") {
|
|
3817
|
-
return "Lead with unaware-style pattern interrupts, identity, curiosity, story, and emotionally resonant entry points before surfacing the problem.";
|
|
3818
|
-
}
|
|
3819
|
-
if (problemAwareness === "problem_aware" && solutionAwareness === "solution_unaware") {
|
|
3820
|
-
return "Lead with sharp articulation of the pain and consequences, then reveal a new possible solution path.";
|
|
3821
|
-
}
|
|
3822
|
-
if (problemAwareness === "problem_aware" && solutionAwareness === "solution_aware") {
|
|
3823
|
-
return "Lead with strong solution-category promises, differentiation, mechanism, and proof.";
|
|
3824
|
-
}
|
|
3825
|
-
return "Lead with high-curiosity or identity-first entry points that bridge people from vague discomfort toward the existence of a real solution category.";
|
|
3826
|
-
}
|
|
3827
|
-
async function recordPrimitiveMediaBilling(ctx, input) {
|
|
3828
|
-
if (!Number.isFinite(input.costUsd) || input.costUsd <= 0) {
|
|
3829
|
-
return;
|
|
3830
|
-
}
|
|
3831
|
-
await ctx.billing.record({
|
|
3832
|
-
type: "cpu_estimate",
|
|
3833
|
-
costUsd: input.costUsd,
|
|
3834
|
-
costCenterSlug: input.costCenterSlug,
|
|
3835
|
-
idempotencyKey: input.idempotencyKey,
|
|
3836
|
-
occurredAtMs: Date.now(),
|
|
3837
|
-
metadata: input.metadata
|
|
3838
|
-
});
|
|
3839
|
-
}
|
|
3840
|
-
async function executePrimitiveMediaOperation(ctx, input) {
|
|
3841
|
-
const execution = await primitiveMediaService.execute({
|
|
3842
|
-
operation: input.operation,
|
|
3843
|
-
payload: input.payload,
|
|
3844
|
-
outputKey: ctx.storage.resolveKey(input.outputKey),
|
|
3845
|
-
localOutputKey: input.outputKey,
|
|
3846
|
-
storeLocal: async (key, bytes, contentType) => ctx.storage.putBuffer(key, bytes, {
|
|
3847
|
-
contentType,
|
|
3848
|
-
kind: contentType.startsWith("audio/") ? "audio" : contentType.startsWith("image/") ? "image" : "video",
|
|
3849
|
-
metadata: {
|
|
3850
|
-
primitive_media_operation: input.operation
|
|
3851
|
-
}
|
|
3852
|
-
})
|
|
3853
|
-
});
|
|
3854
|
-
await recordPrimitiveMediaBilling(ctx, {
|
|
3855
|
-
costUsd: execution.billingCostUsd,
|
|
3856
|
-
costCenterSlug: execution.billingCostCenterSlug,
|
|
3857
|
-
idempotencyKey: `${execution.billingCostCenterSlug}:${ctx.customer.id}:${input.operation}:${execution.storageKey}`,
|
|
3858
|
-
metadata: {
|
|
3859
|
-
...execution.billingMetadata,
|
|
3860
|
-
output_key: execution.storageKey
|
|
3861
|
-
}
|
|
3862
|
-
});
|
|
3863
|
-
return execution;
|
|
3864
|
-
}
|
|
3865
|
-
function inferMediaType(mediaUrl) {
|
|
3866
|
-
return /\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(mediaUrl) ? "video" : "image";
|
|
3867
|
-
}
|
|
3868
|
-
function compactUrls(values) {
|
|
3869
|
-
return values.filter((value) => Boolean(value));
|
|
3870
|
-
}
|
|
3871
|
-
function isVidfarmControlPlaneArtifactUrl(value) {
|
|
3872
|
-
try {
|
|
3873
|
-
const url = new URL(value);
|
|
3874
|
-
return /\/api\/v1\/(?:user\/me|templates\/[^/]+|primitives(?:\/[^/]+)?)\/jobs\/[^/]+\/artifacts\/?$/u.test(url.pathname);
|
|
3875
|
-
}
|
|
3876
|
-
catch {
|
|
3877
|
-
return false;
|
|
3878
|
-
}
|
|
3879
|
-
}
|
|
3880
|
-
function selectDownloadedVideoMediaCandidate(medias, qualityPreference) {
|
|
3881
|
-
const playable = medias.filter((media) => typeof media.url === "string"
|
|
3882
|
-
&& media.url.length > 0
|
|
3883
|
-
&& media.videoAvailable !== false
|
|
3884
|
-
&& (media.extension?.toLowerCase() === "mp4" || /\.mp4(\?|#|$)/i.test(media.url)));
|
|
3885
|
-
if (!playable.length)
|
|
3886
|
-
return null;
|
|
3887
|
-
const preferredRank = qualityPreference === "full_hd"
|
|
3888
|
-
? ["full hd", "hd", "best", ""]
|
|
3889
|
-
: qualityPreference === "hd"
|
|
3890
|
-
? ["hd", "full hd", "best", ""]
|
|
3891
|
-
: ["full hd", "hd", "best", ""];
|
|
3892
|
-
const sorted = playable.slice().sort((left, right) => {
|
|
3893
|
-
const leftRank = preferredRank.indexOf(normalizeDownloadedVideoQuality(left.quality));
|
|
3894
|
-
const rightRank = preferredRank.indexOf(normalizeDownloadedVideoQuality(right.quality));
|
|
3895
|
-
const normalizedLeftRank = leftRank === -1 ? preferredRank.length : leftRank;
|
|
3896
|
-
const normalizedRightRank = rightRank === -1 ? preferredRank.length : rightRank;
|
|
3897
|
-
if (normalizedLeftRank !== normalizedRightRank)
|
|
3898
|
-
return normalizedLeftRank - normalizedRightRank;
|
|
3899
|
-
return (right.size ?? 0) - (left.size ?? 0);
|
|
3900
|
-
});
|
|
3901
|
-
return sorted[0] ?? null;
|
|
3902
|
-
}
|
|
3903
|
-
function normalizeDownloadedVideoQuality(value) {
|
|
3904
|
-
return String(value ?? "").trim().toLowerCase().replace(/_/g, " ");
|
|
3905
|
-
}
|
|
3906
|
-
async function fetchRemoteBinary(url, fallbackContentType) {
|
|
3907
|
-
const response = await fetch(url);
|
|
3908
|
-
if (!response.ok) {
|
|
3909
|
-
throw new Error(`Unable to download remote media: HTTP ${response.status}`);
|
|
3910
|
-
}
|
|
3911
|
-
const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() || fallbackContentType;
|
|
3912
|
-
return {
|
|
3913
|
-
bytes: Buffer.from(await response.arrayBuffer()),
|
|
3914
|
-
contentType
|
|
3915
|
-
};
|
|
3916
|
-
}
|
|
3917
|
-
function normalizeVideoContentType(contentType) {
|
|
3918
|
-
const normalized = contentType.toLowerCase();
|
|
3919
|
-
if (normalized === "video/quicktime")
|
|
3920
|
-
return "video/quicktime";
|
|
3921
|
-
if (normalized === "video/webm")
|
|
3922
|
-
return "video/webm";
|
|
3923
|
-
return "video/mp4";
|
|
3924
|
-
}
|
|
3925
|
-
function videoExtensionForContentType(contentType) {
|
|
3926
|
-
if (contentType === "video/quicktime")
|
|
3927
|
-
return "mov";
|
|
3928
|
-
if (contentType === "video/webm")
|
|
3929
|
-
return "webm";
|
|
3930
|
-
return "mp4";
|
|
3931
|
-
}
|
|
3932
|
-
function safeUrlHost(value) {
|
|
3933
|
-
try {
|
|
3934
|
-
return new URL(value).host;
|
|
3935
|
-
}
|
|
3936
|
-
catch {
|
|
3937
|
-
return null;
|
|
3938
|
-
}
|
|
3939
|
-
}
|
|
3940
|
-
//# sourceMappingURL=primitive-registry.js.map
|