@mevdragon/vidfarm-devcli 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.director.md +6 -3
- package/SKILL.platform.md +2 -2
- package/demo/dist/app.css +1 -1
- package/demo/dist/app.js +121 -62
- package/dist/src/app.js +738 -106
- package/dist/src/cli.js +18 -4
- package/dist/src/composition-runtime.js +64 -6
- package/dist/src/devcli/clips.js +5 -0
- package/dist/src/editor-chat.js +1 -1
- package/dist/src/frontend/homepage-view.js +1 -1
- package/dist/src/frontend/template-editor-chat.js +3 -1
- package/dist/src/homepage.js +7 -0
- package/dist/src/primitive-registry.js +237 -47
- package/dist/src/services/providers.js +1 -0
- package/dist/src/services/serverless-records.js +12 -2
- package/dist/src/services/storage.js +12 -0
- package/package.json +1 -1
- package/public/assets/homepage-client-app.js +23 -23
- package/public/assets/page-runtime-client-app.js +1 -1
package/dist/src/app.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import { pipeline as streamPipeline } from "node:stream/promises";
|
|
4
6
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
5
7
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
6
8
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
@@ -45,10 +47,14 @@ import { ServerlessProviderKeyService } from "./services/serverless-provider-key
|
|
|
45
47
|
import { ServerlessTemplateConfigService } from "./services/serverless-template-configs.js";
|
|
46
48
|
import { parseHTML } from "linkedom";
|
|
47
49
|
import { joinStorageKey, StorageService } from "./services/storage.js";
|
|
50
|
+
import { buildCaptionCues, cuesFromText } from "./services/captions.js";
|
|
51
|
+
import { defaultSpeechModelFor as speechModelFor, transcribeSpeechWithKey } from "./services/speech.js";
|
|
52
|
+
import { insertCaptionLayers } from "./devcli/composition-edit.js";
|
|
53
|
+
import { loadSpeechAudioLocally, MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES } from "./devcli/speech.js";
|
|
48
54
|
import { DescribeExecutionCommand, SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
|
|
49
55
|
import { clipRecords, makeUserPreset } from "./services/clip-records.js";
|
|
50
|
-
import { matchScenesToClips, searchUserClips } from "./services/clip-search.js";
|
|
51
|
-
import { BUILTIN_PRESETS, ClipModelClient, effectiveHuntDurationSec, estimateScanCostFromDuration, normalizeHuntSpec, parseClipHuntPrompt, resolveDurationBand } from "./services/clip-curation/index.js";
|
|
56
|
+
import { matchScenesToClips, searchUserClips, vectorIndex } from "./services/clip-search.js";
|
|
57
|
+
import { BUILTIN_PRESETS, buildEffectiveGuidance, ClipModelClient, detectLocalAgent, effectiveHuntDurationSec, estimateScanCostFromDuration, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, probeVideo, resolveDurationBand, scanVideo } from "./services/clip-curation/index.js";
|
|
52
58
|
import { analyzeCastMembers, annotateScenesFromSource, buildSmartDecomposedHtml, HyperframesService, smartDecomposeVideo, stampCompositionIdentity } from "./services/hyperframes.js";
|
|
53
59
|
import { buildPendingSceneAnnotationsState } from "./services/scene-annotations.js";
|
|
54
60
|
import { createPrimitiveJobContext } from "./primitive-context.js";
|
|
@@ -850,6 +856,26 @@ function consumeSettingsFlash(c) {
|
|
|
850
856
|
return null;
|
|
851
857
|
}
|
|
852
858
|
}
|
|
859
|
+
// Fully-local box (`vidfarm serve`): the browser is pre-authed through
|
|
860
|
+
// /auto-login at boot, but a fresh browser profile / cleared cookies / a
|
|
861
|
+
// directly-opened /editor URL carries no session cookie — and every editor
|
|
862
|
+
// POST (Render, chat, uploads) then fails with a 401 "Unauthorized" modal
|
|
863
|
+
// even though the box is a single-user localhost machine whose only customer
|
|
864
|
+
// is the env-bootstrapped one. Fall back to the bootstrap key's customer,
|
|
865
|
+
// the exact identity /auto-login would have granted.
|
|
866
|
+
async function localBoxFallbackCustomer() {
|
|
867
|
+
const isFullyLocalBox = !config.isProduction
|
|
868
|
+
&& config.RECORDS_DRIVER === "local"
|
|
869
|
+
&& config.STORAGE_DRIVER === "local";
|
|
870
|
+
if (!isFullyLocalBox || !config.VIDFARM_API_KEY)
|
|
871
|
+
return null;
|
|
872
|
+
try {
|
|
873
|
+
return await auth.authenticateAsync(config.VIDFARM_API_KEY);
|
|
874
|
+
}
|
|
875
|
+
catch {
|
|
876
|
+
return null;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
853
879
|
async function getBrowserCustomer(c, body) {
|
|
854
880
|
try {
|
|
855
881
|
// Agent-first auth: an explicit vidfarm-api-key header wins over browser
|
|
@@ -881,10 +907,10 @@ async function getBrowserCustomer(c, body) {
|
|
|
881
907
|
return currentCustomer;
|
|
882
908
|
}
|
|
883
909
|
const customers = await listBrowserSessionCustomers(c);
|
|
884
|
-
if (customers.length
|
|
885
|
-
return null;
|
|
910
|
+
if (customers.length === 1) {
|
|
911
|
+
return customers[0] ?? null;
|
|
886
912
|
}
|
|
887
|
-
return
|
|
913
|
+
return await localBoxFallbackCustomer();
|
|
888
914
|
}
|
|
889
915
|
catch {
|
|
890
916
|
return null;
|
|
@@ -2918,10 +2944,19 @@ function isFormFile(value) {
|
|
|
2918
2944
|
&& "arrayBuffer" in value);
|
|
2919
2945
|
}
|
|
2920
2946
|
async function getVisibleApiKey(customerId) {
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2947
|
+
// Prefer the newest key THIS server can validate, not just the newest key.
|
|
2948
|
+
// After an API_KEY_SALT rotation the shared table holds keys hashed under
|
|
2949
|
+
// different salts (deployed stage vs local dev boxes share the staging
|
|
2950
|
+
// table); embedding a key the current server rejects breaks editor chat and
|
|
2951
|
+
// its tool calls with "Invalid API credentials." on every message.
|
|
2952
|
+
const existingKeys = await serverlessRecords.listActiveApiKeysForCustomer(customerId);
|
|
2953
|
+
for (const row of existingKeys) {
|
|
2954
|
+
const rawValue = row?.raw_value ? String(row.raw_value) : null;
|
|
2955
|
+
if (!rawValue)
|
|
2956
|
+
continue;
|
|
2957
|
+
if (String(row.key_hash ?? "") === hashSecret(rawValue + config.API_KEY_SALT)) {
|
|
2958
|
+
return rawValue;
|
|
2959
|
+
}
|
|
2925
2960
|
}
|
|
2926
2961
|
const apiKey = `vf_${createId("key")}`;
|
|
2927
2962
|
await serverlessRecords.insertApiKey({
|
|
@@ -3031,6 +3066,13 @@ function asRecord(value) {
|
|
|
3031
3066
|
function readNonEmptyString(value) {
|
|
3032
3067
|
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
3033
3068
|
}
|
|
3069
|
+
// "raw" = the user's own new project (prompt-generated, own raw clips) — the
|
|
3070
|
+
// editor must not auto-prompt the Decompose modal for these. Anything else
|
|
3071
|
+
// (absent, unknown) reads as "inspiration", matching records that predate the
|
|
3072
|
+
// field.
|
|
3073
|
+
function readProjectOrigin(value) {
|
|
3074
|
+
return value === "raw" ? "raw" : "inspiration";
|
|
3075
|
+
}
|
|
3034
3076
|
function readFiniteNumber(value, fallback = 0) {
|
|
3035
3077
|
const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN;
|
|
3036
3078
|
return Number.isFinite(number) ? number : fallback;
|
|
@@ -3090,7 +3132,8 @@ function buildPrimitiveEditorDocsRoutes() {
|
|
|
3090
3132
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/concat`, summary: "Primitive audio concat job. Body must be { tracer, payload: { source_audio_urls, output_format?, audio_bitrate_kbps? }, webhook_url? }." },
|
|
3091
3133
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/normalize`, summary: "Primitive audio normalization job. Body must be { tracer, payload: { source_audio_url, output_format?, audio_bitrate_kbps?, sample_rate_hz? }, webhook_url? }." },
|
|
3092
3134
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/speech`, summary: "Primitive text-to-speech (TTS) job using saved OpenAI, Gemini, or OpenRouter provider keys. Body must be { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format? }, webhook_url? }. instructions is a free-form voice-style prompt (tone, pacing, accent, emotion, persona — e.g. \"calm, warm bedtime narrator with a slight British accent\"). voice is a provider preset (OpenAI: alloy/ash/coral/…; Gemini: Kore/Puck/Charon/…); provider is inferred from the voice when omitted, and a voice paired with the wrong provider is rejected with a 400. Returns a durable platform audio URL (mp3 or wav). Alias: POST /api/v1/primitives/tts." },
|
|
3093
|
-
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt." },
|
|
3135
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt. Pass word_timestamps: true for word-level timings on each segment (real timestamps on an OpenAI key via whisper-1; other providers stay segment-level)." },
|
|
3136
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/captions`, summary: "ANIMATED CAPTIONS from speech: one job transcribes narration WITH word-level timings and pages it into word-by-word caption cues ready for the editor's set_captions action. Body must be { tracer, payload: { source_url (video or audio; video audio is demuxed) OR text (page a script instead of transcribing), style? (spotlight|karaoke|word-pop|stack-up|neon|bounce), animation?, active_color?, highlight_color?, uppercase?, color?, background?, background_style?, font_family?, font_size?, max_words_per_cue?, offset_sec?, language?, provider?, window_start?/window_duration? (text mode) }, webhook_url? }. The finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments — apply verbatim). Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter fall back to estimated word windows. Alias: POST /api/v1/primitives/captions." },
|
|
3094
3137
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/coldstart`, summary: "Primitive cold-start strategy questionnaire. Body should usually be { tracer, payload: { user_message }, webhook_url? }. user_message can be rough, like \"I don't know where to start\" or a messy description. count is optional only when the user explicitly wants more or fewer questions. offer_description remains accepted as a backward-compatible alias." },
|
|
3095
3138
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, summary: "Primitive awareness-stage recommendation. Body must be { tracer, payload: { offer_description }, webhook_url? }. Use this when the customer is unsure what type of ads to produce." },
|
|
3096
3139
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/angles`, summary: "Primitive ad-angle brainstorm. Body must be { tracer, payload: { offer_description, problem_awareness, solution_awareness }, webhook_url? }. Allowed values: problem_awareness is problem_unaware|problem_aware, solution_awareness is solution_unaware|solution_aware. count is optional only when the user explicitly wants more or fewer angles. Returns diverse persuasive TikTok angles with explanations." },
|
|
@@ -3684,7 +3727,7 @@ function normalizeOperationRequestTracer(tracer) {
|
|
|
3684
3727
|
}
|
|
3685
3728
|
function createTemplateHttpTool(input) {
|
|
3686
3729
|
return tool({
|
|
3687
|
-
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys. To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
3730
|
+
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key, and word_timestamps: true adds word-level timings per segment (real on an OpenAI key via whisper-1). If the user wants ANIMATED word-by-word captions generated from narration, call POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url (narration audio or video URL) OR text (a script to page), style?: \"spotlight\"|\"karaoke\"|\"word-pop\"|\"stack-up\"|\"neon\"|\"bounce\", max_words_per_cue?, offset_sec?, language?, provider? } } — the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments); apply them with the editor_action tool (action_type=set_captions) so the change flows through the editor state. Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter estimate word windows. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, provider?: \"gemini\"|\"openai\"|\"openrouter\", tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys (auto-picked gemini → openai → openrouter unless body.provider names one). To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
3688
3731
|
inputSchema: z.object({
|
|
3689
3732
|
method: z.enum(["GET", "POST"]).describe("HTTP method to use."),
|
|
3690
3733
|
path: z.string().min(1).describe("Absolute path or same-origin URL for an allowed Vidfarm REST route."),
|
|
@@ -4864,7 +4907,8 @@ app.get("/editor/:templateId/composition", async (c) => {
|
|
|
4864
4907
|
compositionId: templateId,
|
|
4865
4908
|
sourceUrl: template.videoUrl,
|
|
4866
4909
|
durationSeconds: template.durationSeconds ?? 12,
|
|
4867
|
-
title: template.title || `Template · ${template.sourceHost}
|
|
4910
|
+
title: template.title || `Template · ${template.sourceHost}`,
|
|
4911
|
+
origin: template.origin
|
|
4868
4912
|
}));
|
|
4869
4913
|
});
|
|
4870
4914
|
app.get("/help", async (c) => renderApprovedHelpPage(c));
|
|
@@ -4976,6 +5020,7 @@ function buildTemplateHomepageEntry(template) {
|
|
|
4976
5020
|
keywords: template.keywords ?? null,
|
|
4977
5021
|
summary: template.searchSummary ?? null,
|
|
4978
5022
|
visibility: template.visibility,
|
|
5023
|
+
origin: template.origin ?? "inspiration",
|
|
4979
5024
|
notes: template.notes,
|
|
4980
5025
|
status: "ready"
|
|
4981
5026
|
};
|
|
@@ -5128,6 +5173,7 @@ app.post("/discover/templates", async (c) => {
|
|
|
5128
5173
|
const explicitTitle = readNonEmptyString(body.title) ?? null;
|
|
5129
5174
|
const tagline = readNonEmptyString(body.tagline) ?? readNonEmptyString(body.trend_tagline) ?? null;
|
|
5130
5175
|
const notes = readNonEmptyString(body.notes) ?? null;
|
|
5176
|
+
const origin = readProjectOrigin(body.origin);
|
|
5131
5177
|
const uploadFileName = readNonEmptyString(uploadRef?.file_name) ?? null;
|
|
5132
5178
|
const primitive = primitiveRegistry.get("primitive:video_ingest");
|
|
5133
5179
|
if (!primitive)
|
|
@@ -5183,6 +5229,7 @@ app.post("/discover/templates", async (c) => {
|
|
|
5183
5229
|
downloadJobId: job.id,
|
|
5184
5230
|
trendTagline: tagline,
|
|
5185
5231
|
visibility: "private",
|
|
5232
|
+
origin,
|
|
5186
5233
|
notes
|
|
5187
5234
|
});
|
|
5188
5235
|
// Pending-card label while the ingest runs: explicit title, else the file
|
|
@@ -5340,6 +5387,7 @@ function buildPendingInspirationHomepageEntry(inspiration) {
|
|
|
5340
5387
|
sourceType: inspiration.sourceHost || "social",
|
|
5341
5388
|
durationSeconds: inspiration.durationSeconds ?? null,
|
|
5342
5389
|
visibility: "private",
|
|
5390
|
+
origin: inspiration.origin ?? "inspiration",
|
|
5343
5391
|
notes: inspiration.notes,
|
|
5344
5392
|
status: inspiration.status === "failed" ? "failed" : "processing"
|
|
5345
5393
|
};
|
|
@@ -5701,7 +5749,8 @@ app.get("/composition/current.html", async (c) => {
|
|
|
5701
5749
|
compositionId,
|
|
5702
5750
|
sourceUrl: template.videoUrl,
|
|
5703
5751
|
durationSeconds: template.durationSeconds ?? 12,
|
|
5704
|
-
title: template.title || `Template · ${template.sourceHost}
|
|
5752
|
+
title: template.title || `Template · ${template.sourceHost}`,
|
|
5753
|
+
origin: template.origin
|
|
5705
5754
|
});
|
|
5706
5755
|
const rewritten = rewriteHyperframesDraftCompositionHtml(rawHtml, `templates/${encodeURIComponent(template.id)}`);
|
|
5707
5756
|
return c.html(rewritten, 200, { "cache-control": "no-store" });
|
|
@@ -6212,7 +6261,8 @@ app.post(COMPOSITIONS_PREFIX, async (c) => {
|
|
|
6212
6261
|
compositionId: fork.id,
|
|
6213
6262
|
sourceUrl: template.videoUrl,
|
|
6214
6263
|
durationSeconds: template.durationSeconds ?? 12,
|
|
6215
|
-
title
|
|
6264
|
+
title,
|
|
6265
|
+
origin: template.origin
|
|
6216
6266
|
});
|
|
6217
6267
|
await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), rawHtml, "text/html; charset=utf-8");
|
|
6218
6268
|
}
|
|
@@ -6351,6 +6401,144 @@ app.put(`${COMPOSITIONS_PREFIX}/:forkId/composition.html`, async (c) => {
|
|
|
6351
6401
|
return forkAccessErrorResponse(c, error);
|
|
6352
6402
|
}
|
|
6353
6403
|
});
|
|
6404
|
+
// ONE-STEP animated captions: transcribe the fork's own narration (word-level
|
|
6405
|
+
// timings when an OpenAI key is saved) and write the animated caption cue
|
|
6406
|
+
// layers straight into the working composition.html. Runs on the caller's raw
|
|
6407
|
+
// provider keys (BYOK — never billed), synchronous (~5-20s of STT). The write
|
|
6408
|
+
// goes through the same sanitize+persist path as the editor's PUT, and is
|
|
6409
|
+
// deliberately NOT watch-suppressed so a local `vidfarm serve` live-morphs the
|
|
6410
|
+
// captions into open editor tabs. When an editor tab is open against the
|
|
6411
|
+
// CLOUD host, prefer the two-part flow instead (POST /api/v1/primitives/
|
|
6412
|
+
// audio/captions → editor_action set_captions) so the mutation flows through
|
|
6413
|
+
// the editor's own state and cannot be clobbered by its next auto-save.
|
|
6414
|
+
app.post(`${COMPOSITIONS_PREFIX}/:forkId/captions`, async (c) => {
|
|
6415
|
+
const customer = await getBrowserCustomer(c);
|
|
6416
|
+
if (!customer)
|
|
6417
|
+
return c.json({ ok: false, error: "Unauthorized" }, 401);
|
|
6418
|
+
try {
|
|
6419
|
+
const access = await assertForkAccess({ forkId: c.req.param("forkId"), customerId: customer.id, shareToken: readShareToken(c) }, "edit");
|
|
6420
|
+
const fork = access.fork;
|
|
6421
|
+
const html = await storage.readText(storage.compositionForkWorkingKey(fork.id, "composition.html"));
|
|
6422
|
+
if (!html)
|
|
6423
|
+
return c.json({ ok: false, error: "Composition missing" }, 404);
|
|
6424
|
+
const body = asRecord(await c.req.json().catch(() => ({}))) ?? {};
|
|
6425
|
+
const str = (key) => (typeof body[key] === "string" && body[key].trim() ? body[key].trim() : undefined);
|
|
6426
|
+
const num = (key) => (typeof body[key] === "number" && Number.isFinite(body[key]) ? body[key] : undefined);
|
|
6427
|
+
const parsed = parseHTML(html);
|
|
6428
|
+
const root = parsed.document.querySelector("[data-composition-id]");
|
|
6429
|
+
const compositionDuration = Number(root?.getAttribute("data-duration")) || 0;
|
|
6430
|
+
const maxWords = Math.max(1, Math.min(8, Math.round(num("max_words_per_cue") ?? 4)));
|
|
6431
|
+
const offsetSec = Math.max(0, num("offset_sec") ?? 0);
|
|
6432
|
+
let cues;
|
|
6433
|
+
let wordTimestampSource = "estimated";
|
|
6434
|
+
let sourceNote;
|
|
6435
|
+
const text = str("text");
|
|
6436
|
+
if (text) {
|
|
6437
|
+
const windowStart = Math.max(0, num("start") ?? 0);
|
|
6438
|
+
const windowDuration = num("duration") ?? Math.max(0.4, compositionDuration - windowStart);
|
|
6439
|
+
cues = cuesFromText(text, windowStart + offsetSec, windowDuration, { maxWordsPerCue: maxWords });
|
|
6440
|
+
sourceNote = "text";
|
|
6441
|
+
}
|
|
6442
|
+
else {
|
|
6443
|
+
// Narration source: explicit override → audio layer → backing playback
|
|
6444
|
+
// video → source video attr → any video layer.
|
|
6445
|
+
const timedNodes = Array.from(parsed.document.querySelectorAll("[data-start]"));
|
|
6446
|
+
const audioLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "audio" && (node.getAttribute("src") || "").trim());
|
|
6447
|
+
const videoLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "video" && (node.getAttribute("src") || "").trim());
|
|
6448
|
+
const backing = parsed.document.querySelector("[data-vf-playback-source]");
|
|
6449
|
+
const sourceUrl = str("audio_url") ?? str("source_url")
|
|
6450
|
+
?? audioLayer?.getAttribute("src")?.trim()
|
|
6451
|
+
?? backing?.getAttribute("src")?.trim()
|
|
6452
|
+
?? backing?.getAttribute("data-src")?.trim()
|
|
6453
|
+
?? root?.getAttribute("data-source-video")?.trim()
|
|
6454
|
+
?? videoLayer?.getAttribute("src")?.trim();
|
|
6455
|
+
if (!sourceUrl || !/^https?:\/\//i.test(sourceUrl)) {
|
|
6456
|
+
return c.json({
|
|
6457
|
+
ok: false,
|
|
6458
|
+
error: "No narration to transcribe — the composition has no audio/video layer with a usable URL. Pass audio_url (or text to page a script)."
|
|
6459
|
+
}, 400);
|
|
6460
|
+
}
|
|
6461
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
6462
|
+
const requested = str("provider");
|
|
6463
|
+
// openai first: whisper-1 is the only STT with real word-level timestamps.
|
|
6464
|
+
const provider = ["openai", "gemini", "openrouter"].find((name) => requested ? name === requested && keys[name] : Boolean(keys[name]));
|
|
6465
|
+
if (!provider) {
|
|
6466
|
+
return c.json({
|
|
6467
|
+
ok: false,
|
|
6468
|
+
error: requested
|
|
6469
|
+
? `No active ${requested} provider key is saved for this account.`
|
|
6470
|
+
: "Captions need an AI provider key for transcription. Add an OpenAI (best: real word timings), Gemini, or OpenRouter key in Settings."
|
|
6471
|
+
}, 400);
|
|
6472
|
+
}
|
|
6473
|
+
const audio = await loadSpeechAudioLocally(sourceUrl);
|
|
6474
|
+
if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
|
|
6475
|
+
return c.json({ ok: false, error: "Narration audio exceeds the ~40-minute transcription limit. Trim it or pass a shorter audio_url." }, 400);
|
|
6476
|
+
}
|
|
6477
|
+
const transcription = await transcribeSpeechWithKey({
|
|
6478
|
+
provider,
|
|
6479
|
+
model: str("model") ?? speechModelFor("stt", provider) ?? "",
|
|
6480
|
+
audio: audio.bytes,
|
|
6481
|
+
contentType: audio.contentType,
|
|
6482
|
+
language: str("language"),
|
|
6483
|
+
diarize: false,
|
|
6484
|
+
wordTimestamps: true,
|
|
6485
|
+
apiKey: keys[provider]
|
|
6486
|
+
});
|
|
6487
|
+
if (!transcription.segments.length) {
|
|
6488
|
+
return c.json({ ok: false, error: "Transcription returned no speech segments — the narration may be silent." }, 502);
|
|
6489
|
+
}
|
|
6490
|
+
wordTimestampSource = transcription.segments.some((segment) => segment.words?.length) ? "provider" : "estimated";
|
|
6491
|
+
cues = buildCaptionCues(transcription.segments, {
|
|
6492
|
+
maxWordsPerCue: maxWords,
|
|
6493
|
+
offsetSec,
|
|
6494
|
+
fallbackDurationSec: compositionDuration || undefined
|
|
6495
|
+
});
|
|
6496
|
+
sourceNote = sourceUrl;
|
|
6497
|
+
}
|
|
6498
|
+
if (compositionDuration > 0) {
|
|
6499
|
+
cues = cues
|
|
6500
|
+
.filter((cue) => cue.start < compositionDuration)
|
|
6501
|
+
.map((cue) => ({ ...cue, duration: Math.min(cue.duration, Math.max(0.1, compositionDuration - cue.start)) }));
|
|
6502
|
+
}
|
|
6503
|
+
if (!cues.length) {
|
|
6504
|
+
return c.json({ ok: false, error: "No caption cues produced." }, 502);
|
|
6505
|
+
}
|
|
6506
|
+
const result = insertCaptionLayers(html, cues, {
|
|
6507
|
+
style: str("style"),
|
|
6508
|
+
animation: str("animation"),
|
|
6509
|
+
activeColor: str("active_color"),
|
|
6510
|
+
highlightColor: str("highlight_color"),
|
|
6511
|
+
uppercase: body.uppercase === true ? true : body.uppercase === false ? false : undefined,
|
|
6512
|
+
color: str("color"),
|
|
6513
|
+
background: str("background"),
|
|
6514
|
+
backgroundStyle: str("background_style"),
|
|
6515
|
+
fontFamily: str("font_family"),
|
|
6516
|
+
fontSize: num("font_size"),
|
|
6517
|
+
track: num("track"),
|
|
6518
|
+
x: num("x"),
|
|
6519
|
+
y: num("y"),
|
|
6520
|
+
width: num("width"),
|
|
6521
|
+
height: num("height")
|
|
6522
|
+
});
|
|
6523
|
+
const { html: sanitized, removed } = sanitizeCompositionHtml(result.html);
|
|
6524
|
+
await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), sanitized, "text/html; charset=utf-8");
|
|
6525
|
+
const updated = await serverlessRecords.updateCompositionFork({ forkId: fork.id, patch: {} });
|
|
6526
|
+
await writeForkManifest(storage, updated ?? fork, { kind: "working" });
|
|
6527
|
+
return c.json({
|
|
6528
|
+
ok: true,
|
|
6529
|
+
cue_count: cues.length,
|
|
6530
|
+
replaced: result.replaced,
|
|
6531
|
+
style: result.styleId,
|
|
6532
|
+
word_timestamps: wordTimestampSource,
|
|
6533
|
+
source: sourceNote,
|
|
6534
|
+
layer_keys: result.layerKeys,
|
|
6535
|
+
sanitized: removed
|
|
6536
|
+
});
|
|
6537
|
+
}
|
|
6538
|
+
catch (error) {
|
|
6539
|
+
return forkAccessErrorResponse(c, error);
|
|
6540
|
+
}
|
|
6541
|
+
});
|
|
6354
6542
|
// Same-origin live-reload stream (local `vidfarm serve` only). Broadcasts when
|
|
6355
6543
|
// an on-disk working composition file changes so open editor tabs live-morph
|
|
6356
6544
|
// the agent's edits. Replaces the old cross-origin dev-serve `/_events`.
|
|
@@ -8650,6 +8838,7 @@ function serializeInspiration(inspiration) {
|
|
|
8650
8838
|
keywords: inspiration.keywords ?? null,
|
|
8651
8839
|
summary: inspiration.searchSummary ?? null,
|
|
8652
8840
|
visibility: inspiration.visibility ?? "public",
|
|
8841
|
+
origin: inspiration.origin ?? "inspiration",
|
|
8653
8842
|
notes: inspiration.notes ?? null,
|
|
8654
8843
|
ingester_customer_id: inspiration.customerId,
|
|
8655
8844
|
error: inspiration.errorMessage,
|
|
@@ -8710,6 +8899,7 @@ async function finalizeInspirationIfReady(inspiration) {
|
|
|
8710
8899
|
songName: ready.songName,
|
|
8711
8900
|
songUrl: ready.songUrl,
|
|
8712
8901
|
visibility: "private",
|
|
8902
|
+
origin: ready.origin ?? "inspiration",
|
|
8713
8903
|
notes: ready.notes
|
|
8714
8904
|
});
|
|
8715
8905
|
ready = (await serverlessRecords.updateInspiration({
|
|
@@ -10808,7 +10998,8 @@ async function ensureInspirationReadyThenFork(input) {
|
|
|
10808
10998
|
videoUrl: inspiration.videoUrl,
|
|
10809
10999
|
durationSeconds: inspiration.durationSeconds,
|
|
10810
11000
|
songName: inspiration.songName,
|
|
10811
|
-
songUrl: inspiration.songUrl
|
|
11001
|
+
songUrl: inspiration.songUrl,
|
|
11002
|
+
origin: inspiration.origin ?? "inspiration"
|
|
10812
11003
|
});
|
|
10813
11004
|
await serverlessRecords.updateInspiration({
|
|
10814
11005
|
inspirationId: inspiration.id,
|
|
@@ -10835,6 +11026,10 @@ function buildRawTikTokCompositionHtml(input) {
|
|
|
10835
11026
|
const duration = Number.isFinite(input.durationSeconds) && input.durationSeconds > 0
|
|
10836
11027
|
? Number(input.durationSeconds.toFixed(3))
|
|
10837
11028
|
: 12;
|
|
11029
|
+
// data-vf-origin travels with the document through fork copies; the editor
|
|
11030
|
+
// reads it to decide whether to auto-prompt the Decompose modal ("raw"
|
|
11031
|
+
// projects never prompt). Absent means "inspiration" (pre-field documents).
|
|
11032
|
+
const origin = input.origin === "raw" ? "raw" : "inspiration";
|
|
10838
11033
|
return `<!doctype html>
|
|
10839
11034
|
<html lang="en">
|
|
10840
11035
|
<head>
|
|
@@ -10847,7 +11042,7 @@ function buildRawTikTokCompositionHtml(input) {
|
|
|
10847
11042
|
</style>
|
|
10848
11043
|
</head>
|
|
10849
11044
|
<body>
|
|
10850
|
-
<div data-composition-id="${escapeAttribute(input.compositionId)}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
|
|
11045
|
+
<div data-composition-id="${escapeAttribute(input.compositionId)}" data-vf-origin="${origin}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
|
|
10851
11046
|
<video id="vf-source-video" data-hf-id="vf-source-video" data-vf-playback-source="true" data-layer-kind="video" data-start="0" data-duration="${duration}" data-track-index="0" data-label="Original video" data-timeline-label="Original video" src="${escapeAttribute(input.sourceUrl)}" muted playsinline preload="auto" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;object-fit:cover;background:#050604"></video>
|
|
10852
11047
|
</div>
|
|
10853
11048
|
</body>
|
|
@@ -13823,7 +14018,8 @@ const AI_PRIMITIVE_IDS = new Set([
|
|
|
13823
14018
|
"primitive:image_inpaint",
|
|
13824
14019
|
"primitive:video_generate",
|
|
13825
14020
|
"primitive:tts",
|
|
13826
|
-
"primitive:stt"
|
|
14021
|
+
"primitive:stt",
|
|
14022
|
+
"primitive:captions"
|
|
13827
14023
|
]);
|
|
13828
14024
|
async function preflightAiProviderKeys(input) {
|
|
13829
14025
|
if (!operationMayUseAiProvider(input)) {
|
|
@@ -14346,6 +14542,9 @@ app.post(`${PRIMITIVES_PREFIX}/audio/speech`, async (c) => createPrimitiveJob(c,
|
|
|
14346
14542
|
app.post(`${PRIMITIVES_PREFIX}/tts`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:tts", operationName: "run" }));
|
|
14347
14543
|
app.post(`${PRIMITIVES_PREFIX}/audio/transcribe`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
14348
14544
|
app.post(`${PRIMITIVES_PREFIX}/stt`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
14545
|
+
// Animated-caption cues from narration (STT + word timings + cue paging).
|
|
14546
|
+
app.post(`${PRIMITIVES_PREFIX}/audio/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
14547
|
+
app.post(`${PRIMITIVES_PREFIX}/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
14349
14548
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/hooks`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_hooks", operationName: "run" }));
|
|
14350
14549
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_awareness_stages", operationName: "run" }));
|
|
14351
14550
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/angles`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_angles", operationName: "run" }));
|
|
@@ -15312,6 +15511,9 @@ function renderClipsPage(input) {
|
|
|
15312
15511
|
.clips-import-head p { font-size: 0.9rem; }
|
|
15313
15512
|
.clips-import-field { display: grid; gap: 6px; }
|
|
15314
15513
|
.clips-import-field textarea { min-height: 96px; border-radius: 18px; }
|
|
15514
|
+
.clips-import-field select { width: 100%; }
|
|
15515
|
+
.clips-import-hint { font-size: 0.78rem; font-weight: 400; color: var(--muted); }
|
|
15516
|
+
.clips-import-hint a { color: inherit; text-decoration: underline; }
|
|
15315
15517
|
.clips-import-note { min-height: 18px; font-size: 0.85rem; color: var(--muted); }
|
|
15316
15518
|
.clips-import-note[data-tone="error"] { color: #9f3d3d; }
|
|
15317
15519
|
.clips-import-note[data-tone="success"] { color: #3f7d55; }
|
|
@@ -15379,6 +15581,20 @@ function renderClipsPage(input) {
|
|
|
15379
15581
|
<label for="importPrompt">What to clip for</label>
|
|
15380
15582
|
<textarea id="importPrompt" name="prompt" placeholder="Clip scenes of people with food, no text on screen, vertical, between 5-10 secs long"></textarea>
|
|
15381
15583
|
</div>
|
|
15584
|
+
<div class="clips-import-field">
|
|
15585
|
+
<label for="importProvider">AI provider</label>
|
|
15586
|
+
<select id="importProvider" name="provider">
|
|
15587
|
+
<option value="">Auto — use my saved keys</option>
|
|
15588
|
+
<option value="gemini">Gemini</option>
|
|
15589
|
+
<option value="openai">OpenAI</option>
|
|
15590
|
+
<option value="openrouter">OpenRouter</option>
|
|
15591
|
+
</select>
|
|
15592
|
+
<div class="clips-import-hint" id="importProviderHint"></div>
|
|
15593
|
+
</div>
|
|
15594
|
+
<div class="clips-import-field">
|
|
15595
|
+
<label for="importTracer">Tracer <span class="clips-import-hint">optional — rolls up this hunt's billing and clips under one id</span></label>
|
|
15596
|
+
<input id="importTracer" name="tracer" type="text" placeholder="my-campaign-001" autocomplete="off" spellcheck="false" />
|
|
15597
|
+
</div>
|
|
15382
15598
|
<div class="clips-import-note" id="importNote"></div>
|
|
15383
15599
|
<div class="clips-import-actions">
|
|
15384
15600
|
<button type="button" class="secondary" id="importCancel">Cancel</button>
|
|
@@ -15468,18 +15684,58 @@ function renderClipsPage(input) {
|
|
|
15468
15684
|
var importForm = document.getElementById('importForm');
|
|
15469
15685
|
var importUrl = document.getElementById('importUrl');
|
|
15470
15686
|
var importPrompt = document.getElementById('importPrompt');
|
|
15687
|
+
var importTracer = document.getElementById('importTracer');
|
|
15688
|
+
var importProvider = document.getElementById('importProvider');
|
|
15689
|
+
var importProviderHint = document.getElementById('importProviderHint');
|
|
15471
15690
|
var importNote = document.getElementById('importNote');
|
|
15472
15691
|
var importCancel = document.getElementById('importCancel');
|
|
15473
15692
|
var importSubmit = document.getElementById('importSubmit');
|
|
15474
15693
|
var scanPoll = null;
|
|
15694
|
+
var scanOptions = null;
|
|
15475
15695
|
|
|
15476
15696
|
function setNote(msg, tone){
|
|
15477
15697
|
if(!importNote) return;
|
|
15478
15698
|
importNote.textContent = msg || '';
|
|
15479
15699
|
if(tone) importNote.setAttribute('data-tone', tone); else importNote.removeAttribute('data-tone');
|
|
15480
15700
|
}
|
|
15701
|
+
function providerName(p){ return p==='gemini'?'Gemini':p==='openai'?'OpenAI':p==='openrouter'?'OpenRouter':p; }
|
|
15702
|
+
function renderProviderOptions(){
|
|
15703
|
+
if(!importProvider || !scanOptions) return;
|
|
15704
|
+
var d = scanOptions;
|
|
15705
|
+
var hasAnyKey = (d.providers||[]).some(function(p){ return p.has_key; });
|
|
15706
|
+
var autoLabel = 'Auto';
|
|
15707
|
+
if(d.default_provider === 'agent') autoLabel = 'Auto — local '+((d.local_agent&&d.local_agent.kind)||'agent')+' CLI (no API key)';
|
|
15708
|
+
else if(d.default_provider) autoLabel = 'Auto — saved '+providerName(d.default_provider)+' key';
|
|
15709
|
+
else if(d.pipeline === 'upstream') autoLabel = 'Auto — linked cloud account';
|
|
15710
|
+
var html = '<option value="">'+esc(autoLabel)+'</option>';
|
|
15711
|
+
if(d.local_agent) html += '<option value="agent">Local agent ('+esc(d.local_agent.kind)+') — your CLI subscription, no API key</option>';
|
|
15712
|
+
(d.providers||[]).forEach(function(p){
|
|
15713
|
+
html += '<option value="'+esc(p.provider)+'"'+(p.has_key?'':' disabled')+'>'+esc(providerName(p.provider))+(p.has_key?' — key saved':' — no key saved')+'</option>';
|
|
15714
|
+
});
|
|
15715
|
+
importProvider.innerHTML = html;
|
|
15716
|
+
if(!importProviderHint) return;
|
|
15717
|
+
if(!hasAnyKey && !d.local_agent && d.pipeline !== 'upstream'){
|
|
15718
|
+
importProviderHint.innerHTML = 'No AI provider available yet — add a Gemini, OpenAI, or OpenRouter key in <a href="/settings" target="_blank" rel="noopener">Settings</a>'
|
|
15719
|
+
+ (d.pipeline === 'cloud' ? '.' : ', or install the Claude Code / Codex CLI for keyless local scans.');
|
|
15720
|
+
} else if(d.pipeline === 'upstream'){
|
|
15721
|
+
importProviderHint.textContent = 'No local key or agent CLI — this hunt will run on your linked cloud account (its saved keys, cloud compute).';
|
|
15722
|
+
} else if(d.pipeline === 'local'){
|
|
15723
|
+
importProviderHint.textContent = d.default_provider === 'agent'
|
|
15724
|
+
? 'Runs on this machine with your local agent CLI — no API key needed. Or pick a saved key.'
|
|
15725
|
+
: 'Runs on this machine with your saved key (BYOK — billed to your own provider account).';
|
|
15726
|
+
} else {
|
|
15727
|
+
importProviderHint.textContent = 'AI tagging runs on your own key (BYOK) — pick a provider or let Auto choose.';
|
|
15728
|
+
}
|
|
15729
|
+
}
|
|
15730
|
+
function loadScanOptions(){
|
|
15731
|
+
fetch('/clips/scan-options', {credentials:'same-origin', headers:{Accept:'application/json'}})
|
|
15732
|
+
.then(function(r){ return r.ok ? r.json() : null; })
|
|
15733
|
+
.then(function(d){ if(d){ scanOptions = d; renderProviderOptions(); } })
|
|
15734
|
+
.catch(function(){});
|
|
15735
|
+
}
|
|
15481
15736
|
function openImport(){
|
|
15482
15737
|
setNote('');
|
|
15738
|
+
if(!scanOptions) loadScanOptions();
|
|
15483
15739
|
if(importDialog && typeof importDialog.showModal === 'function' && !importDialog.open) importDialog.showModal();
|
|
15484
15740
|
if(importUrl) importUrl.focus();
|
|
15485
15741
|
}
|
|
@@ -15522,16 +15778,21 @@ function renderClipsPage(input) {
|
|
|
15522
15778
|
e.preventDefault();
|
|
15523
15779
|
var url = (importUrl && importUrl.value || '').trim();
|
|
15524
15780
|
var prompt = (importPrompt && importPrompt.value || '').trim();
|
|
15781
|
+
var tracer = (importTracer && importTracer.value || '').trim();
|
|
15782
|
+
var provider = (importProvider && importProvider.value || '').trim();
|
|
15525
15783
|
if(!url){ setNote('Enter a video URL.', 'error'); return; }
|
|
15526
15784
|
setNote('Starting scan…');
|
|
15527
15785
|
if(importSubmit) importSubmit.disabled = true;
|
|
15528
|
-
|
|
15786
|
+
var payload = {source_url:url, prompt:prompt};
|
|
15787
|
+
if(tracer) payload.tracer = tracer;
|
|
15788
|
+
if(provider) payload.provider = provider;
|
|
15789
|
+
fetch('/clips/scan', {method:'POST', credentials:'same-origin', headers:{'Content-Type':'application/json', Accept:'application/json'}, body: JSON.stringify(payload)})
|
|
15529
15790
|
.then(function(r){ return r.json().then(function(d){ return {ok:r.ok, status:r.status, data:d}; }); })
|
|
15530
15791
|
.then(function(res){
|
|
15531
15792
|
if(importSubmit) importSubmit.disabled = false;
|
|
15532
15793
|
if(!res.ok){ setNote((res.data && res.data.error) || 'Could not start the scan.', 'error'); return; }
|
|
15533
15794
|
setNote('Scan started — this can take a few minutes.', 'success');
|
|
15534
|
-
statusEl.textContent='Scanning source…';
|
|
15795
|
+
statusEl.textContent='Scanning source…'+(res.data && res.data.tracer ? ' (tracer '+res.data.tracer+')' : '');
|
|
15535
15796
|
closeImport();
|
|
15536
15797
|
if(importForm.reset) importForm.reset();
|
|
15537
15798
|
if(res.data && res.data.scan_id) pollScan(res.data.scan_id);
|
|
@@ -15559,9 +15820,23 @@ async function resolveCustomerProviderKey(customerId, provider) {
|
|
|
15559
15820
|
}
|
|
15560
15821
|
return null;
|
|
15561
15822
|
}
|
|
15562
|
-
//
|
|
15563
|
-
|
|
15564
|
-
|
|
15823
|
+
// The BYOK API providers clip hunts can run on, in auto-pick preference order.
|
|
15824
|
+
const CLIP_API_PROVIDERS = ["gemini", "openai", "openrouter"];
|
|
15825
|
+
const NO_CLIP_PROVIDER_KEY_ERROR = "No AI provider key saved. Add a gemini, openai, or openrouter key (Settings → AI Provider Keys, or POST /api/v1/user/me/provider-keys) and retry.";
|
|
15826
|
+
// An explicitly requested provider — "agent" means the caller's local
|
|
15827
|
+
// claude/codex CLI (vidfarm serve boxes only) — or null for auto-pick.
|
|
15828
|
+
function readClipProviderChoice(value) {
|
|
15829
|
+
if (value === "gemini" || value === "openai" || value === "openrouter")
|
|
15830
|
+
return value;
|
|
15831
|
+
if (value === "agent" || value === "local" || value === "local-agent")
|
|
15832
|
+
return "agent";
|
|
15833
|
+
return null;
|
|
15834
|
+
}
|
|
15835
|
+
// Auto-pick: the first API provider the customer holds an active key for
|
|
15836
|
+
// (gemini → openai → openrouter) — no more hard gemini default.
|
|
15837
|
+
async function pickClipProviderFromKeys(customerId) {
|
|
15838
|
+
const keys = await loadCustomerVisionKeys(customerId);
|
|
15839
|
+
return CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
|
|
15565
15840
|
}
|
|
15566
15841
|
/**
|
|
15567
15842
|
* A client for embedding MATCH queries in the SAME vector space the library was
|
|
@@ -15617,7 +15892,8 @@ function parseClipStorageKey(pathOrUri) {
|
|
|
15617
15892
|
async function clipMediaUrl(pathOrUri) {
|
|
15618
15893
|
if (!pathOrUri)
|
|
15619
15894
|
return null;
|
|
15620
|
-
|
|
15895
|
+
// s3://… (cloud hunts) or a bare clips/thumbs storage key (serve-box local hunts).
|
|
15896
|
+
const key = parseClipStorageKey(pathOrUri) ?? (/^(clips|thumbs)\//.test(pathOrUri) ? pathOrUri : null);
|
|
15621
15897
|
if (key)
|
|
15622
15898
|
return (await storage.getReadUrl(key)) ?? storage.getPublicUrl(key);
|
|
15623
15899
|
return pathOrUri; // non-s3 value (e.g. a local devcli-origin path) — pass through
|
|
@@ -15677,11 +15953,14 @@ app.post(`${CLIPS_PREFIX}/search`, async (c) => {
|
|
|
15677
15953
|
try {
|
|
15678
15954
|
const body = (await c.req.json().catch(() => ({})));
|
|
15679
15955
|
const limit = Math.max(1, Math.min(200, Number(body.limit) || 20));
|
|
15680
|
-
const
|
|
15956
|
+
const requested = readClipProviderChoice(body.provider);
|
|
15681
15957
|
let criteria = body.criteria ?? (body.query ? { semantic_text: body.query } : {});
|
|
15682
15958
|
let queryEmbedding;
|
|
15683
15959
|
if (body.query || criteria.semantic_text) {
|
|
15684
|
-
|
|
15960
|
+
// Any saved key works for NL→criteria + query embedding; no key at all
|
|
15961
|
+
// fail-softs to the structured/keyword pass below.
|
|
15962
|
+
const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
15963
|
+
const built = provider ? await buildCustomerClipClient(customer.id, provider) : { error: NO_CLIP_PROVIDER_KEY_ERROR };
|
|
15685
15964
|
if ("client" in built) {
|
|
15686
15965
|
const client = built.client;
|
|
15687
15966
|
if (body.query && !body.criteria)
|
|
@@ -15837,72 +16116,106 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
|
|
|
15837
16116
|
}
|
|
15838
16117
|
const rawS3Uri = body.raw_s3_uri ??
|
|
15839
16118
|
(resolvedS3Key && config.AWS_S3_BUCKET ? `s3://${config.AWS_S3_BUCKET}/${resolvedS3Key}` : undefined);
|
|
15840
|
-
|
|
16119
|
+
// On a serve box (no bucket) resolvedS3Key is a LOCAL storage key — still a valid source.
|
|
16120
|
+
if (!rawS3Uri && !resolvedS3Key && !huntSpec.source_url) {
|
|
15841
16121
|
return c.json({ error: "Provide source_url (a video URL), temp_file_id/attachment_id of an upload, raw_s3_uri (s3://…), or s3_key of the staged source video." }, 400);
|
|
15842
16122
|
}
|
|
15843
|
-
|
|
15844
|
-
|
|
15845
|
-
try {
|
|
15846
|
-
await assertWalletCanQueueJobs(customer.id);
|
|
15847
|
-
}
|
|
15848
|
-
catch (error) {
|
|
15849
|
-
if (error instanceof InsufficientWalletFundsError) {
|
|
15850
|
-
return insufficientFundsResponse(c, error);
|
|
15851
|
-
}
|
|
15852
|
-
throw error;
|
|
15853
|
-
}
|
|
15854
|
-
const provider = normalizeClipProvider(body.provider);
|
|
15855
|
-
const built = await buildCustomerClipClient(customer.id, provider);
|
|
15856
|
-
if ("error" in built)
|
|
15857
|
-
return c.json({ error: built.error }, 400);
|
|
15858
|
-
const { client, apiKey: tagApiKey, embeddingKey } = built;
|
|
16123
|
+
const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
|
|
16124
|
+
const choice = readClipProviderChoice(body.provider);
|
|
15859
16125
|
const tier = normalizeScanTier(body.tier);
|
|
15860
16126
|
const framesPerScene = Math.max(1, Math.min(3, Number(body.frames_per_scene) || 2));
|
|
15861
|
-
const includeAudio = body.include_audio !== false && provider === "gemini";
|
|
15862
16127
|
const tracer = readNonEmptyString(body.tracer) ?? `clips_import:${customer.id}`;
|
|
15863
16128
|
const now = nowIso();
|
|
15864
16129
|
const scanId = createIdV7("clipscan");
|
|
15865
16130
|
const sourceVideoId = createIdV7("srcvid");
|
|
15866
|
-
// The URL ingest is deferred to the
|
|
15867
|
-
const sourcePointer = rawS3Uri ?? huntSpec.source_url ?? "";
|
|
16131
|
+
// The URL ingest is deferred to the runner, so raw_s3_uri may be empty here.
|
|
16132
|
+
const sourcePointer = rawS3Uri ?? resolvedS3Key ?? huntSpec.source_url ?? "";
|
|
15868
16133
|
const filename = body.filename || (sourcePointer.split("/").pop() || "source.mp4").split("?")[0] || "source.mp4";
|
|
15869
|
-
|
|
15870
|
-
|
|
15871
|
-
|
|
15872
|
-
|
|
15873
|
-
|
|
15874
|
-
|
|
15875
|
-
|
|
15876
|
-
|
|
15877
|
-
? estimateScanCostFromDuration(effectiveDurationSec, {
|
|
16134
|
+
const putScanRecords = async (input) => {
|
|
16135
|
+
await clipRecords.putSource({
|
|
16136
|
+
source_video_id: sourceVideoId,
|
|
16137
|
+
owner_id: customer.id,
|
|
16138
|
+
filename,
|
|
16139
|
+
raw_s3_uri: sourcePointer,
|
|
16140
|
+
duration_sec: body.duration_sec,
|
|
16141
|
+
clip_count: 0,
|
|
15878
16142
|
tier,
|
|
15879
|
-
|
|
15880
|
-
|
|
15881
|
-
|
|
15882
|
-
|
|
15883
|
-
|
|
15884
|
-
|
|
15885
|
-
|
|
15886
|
-
|
|
15887
|
-
|
|
15888
|
-
|
|
15889
|
-
|
|
15890
|
-
|
|
15891
|
-
|
|
15892
|
-
|
|
15893
|
-
|
|
15894
|
-
|
|
15895
|
-
|
|
15896
|
-
|
|
15897
|
-
tier,
|
|
15898
|
-
status: "pending",
|
|
15899
|
-
scan_id: scanId,
|
|
15900
|
-
created_at: now,
|
|
15901
|
-
updated_at: now
|
|
16143
|
+
status: "pending",
|
|
16144
|
+
scan_id: scanId,
|
|
16145
|
+
tracer,
|
|
16146
|
+
created_at: now,
|
|
16147
|
+
updated_at: now
|
|
16148
|
+
});
|
|
16149
|
+
await clipRecords.putScan({
|
|
16150
|
+
scan_id: scanId,
|
|
16151
|
+
owner_id: customer.id,
|
|
16152
|
+
source_video_id: sourceVideoId,
|
|
16153
|
+
status: input.scanStatus,
|
|
16154
|
+
tier,
|
|
16155
|
+
raw_s3_uri: sourcePointer,
|
|
16156
|
+
execution_arn: input.executionArn,
|
|
16157
|
+
tracer,
|
|
16158
|
+
created_at: now,
|
|
16159
|
+
updated_at: now
|
|
16160
|
+
});
|
|
15902
16161
|
};
|
|
15903
|
-
|
|
15904
|
-
|
|
15905
|
-
|
|
16162
|
+
const scanResponse = (input) => c.json({
|
|
16163
|
+
scan_id: scanId,
|
|
16164
|
+
source_video_id: sourceVideoId,
|
|
16165
|
+
status: input.status,
|
|
16166
|
+
execution_arn: input.executionArn ?? null,
|
|
16167
|
+
tracer,
|
|
16168
|
+
hunt_spec: huntSpec,
|
|
16169
|
+
estimate: input.estimate ?? null,
|
|
16170
|
+
compute_estimate: input.computeEstimate ?? null,
|
|
16171
|
+
pipeline: input.pipeline,
|
|
16172
|
+
pipeline_deployed: cloudPipeline
|
|
16173
|
+
}, 202);
|
|
16174
|
+
if (cloudPipeline) {
|
|
16175
|
+
// ── Deployed pipeline (Lambda + Step Functions): BYOK API keys only ──
|
|
16176
|
+
if (choice === "agent") {
|
|
16177
|
+
return c.json({ error: "provider \"agent\" (your local claude/codex CLI) only runs on a local `vidfarm serve` box — the deployed pipeline needs a saved gemini, openai, or openrouter key." }, 400);
|
|
16178
|
+
}
|
|
16179
|
+
// Hunts bill AWS compute post-hoc, so submission is the spend gate (same
|
|
16180
|
+
// policy as job submission).
|
|
16181
|
+
try {
|
|
16182
|
+
await assertWalletCanQueueJobs(customer.id);
|
|
16183
|
+
}
|
|
16184
|
+
catch (error) {
|
|
16185
|
+
if (error instanceof InsufficientWalletFundsError) {
|
|
16186
|
+
return insufficientFundsResponse(c, error);
|
|
16187
|
+
}
|
|
16188
|
+
throw error;
|
|
16189
|
+
}
|
|
16190
|
+
const provider = choice ?? (await pickClipProviderFromKeys(customer.id));
|
|
16191
|
+
if (!provider)
|
|
16192
|
+
return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
|
|
16193
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16194
|
+
if ("error" in built)
|
|
16195
|
+
return c.json({ error: built.error }, 400);
|
|
16196
|
+
const { client, apiKey: tagApiKey, embeddingKey } = built;
|
|
16197
|
+
const includeAudio = body.include_audio !== false && provider === "gemini";
|
|
16198
|
+
// Time-range hunts only pay for the windows: both estimates use the
|
|
16199
|
+
// EFFECTIVE (windowed) duration, and the duration band drives the expected
|
|
16200
|
+
// scene size. The BYOK estimate is the user's own provider spend; the
|
|
16201
|
+
// compute estimate is what the wallet will be billed (AWS only, marked up).
|
|
16202
|
+
const effectiveDurationSec = body.duration_sec && body.duration_sec > 0
|
|
16203
|
+
? effectiveHuntDurationSec(body.duration_sec, huntSpec.windows)
|
|
16204
|
+
: null;
|
|
16205
|
+
const estimate = effectiveDurationSec
|
|
16206
|
+
? estimateScanCostFromDuration(effectiveDurationSec, {
|
|
16207
|
+
tier,
|
|
16208
|
+
provider,
|
|
16209
|
+
tagModelId: client.tagModelId,
|
|
16210
|
+
embedModelId: client.embeddingModelId,
|
|
16211
|
+
framesPerScene,
|
|
16212
|
+
includeAudio,
|
|
16213
|
+
avgSceneSec: huntSpec.duration_band?.target_sec
|
|
16214
|
+
})
|
|
16215
|
+
: null;
|
|
16216
|
+
const computeEstimate = effectiveDurationSec
|
|
16217
|
+
? estimateClipScanComputeUsd(effectiveDurationSec, huntSpec.duration_band?.target_sec)
|
|
16218
|
+
: null;
|
|
15906
16219
|
const res = await clipScanSfn.send(new StartExecutionCommand({
|
|
15907
16220
|
stateMachineArn: config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN,
|
|
15908
16221
|
name: scanId.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
|
|
@@ -15924,43 +16237,358 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
|
|
|
15924
16237
|
tracer
|
|
15925
16238
|
})
|
|
15926
16239
|
}));
|
|
15927
|
-
executionArn
|
|
16240
|
+
await putScanRecords({ scanStatus: "running", executionArn: res.executionArn });
|
|
16241
|
+
return scanResponse({ status: "running", executionArn: res.executionArn, pipeline: "cloud", estimate, computeEstimate });
|
|
16242
|
+
}
|
|
16243
|
+
// ── vidfarm serve (no deployed pipeline): LOCAL-FIRST, cloud as backup ──
|
|
16244
|
+
// Same policy as `vidfarm clips scan`: the user's local claude/codex CLI
|
|
16245
|
+
// subscription evaluates scenes when installed, their saved BYOK keys are
|
|
16246
|
+
// the fallback, and the upstream cloud REST API is the explicit backup.
|
|
16247
|
+
// Local hunts are free (no wallet gate), like local renders.
|
|
16248
|
+
let localSourcePath = null;
|
|
16249
|
+
if (resolvedS3Key) {
|
|
16250
|
+
localSourcePath = storage.getLocalPath(resolvedS3Key);
|
|
16251
|
+
if (!localSourcePath || !existsSync(localSourcePath)) {
|
|
16252
|
+
return c.json({ error: "That upload isn't on this box's local storage — re-upload it or pass source_url." }, 400);
|
|
16253
|
+
}
|
|
16254
|
+
}
|
|
16255
|
+
else if (body.raw_s3_uri) {
|
|
16256
|
+
return c.json({ error: "raw_s3_uri isn't supported on a local box — pass source_url, temp_file_id, or attachment_id." }, 400);
|
|
16257
|
+
}
|
|
16258
|
+
const localAgent = choice === "agent" || choice === null ? detectLocalAgent() : null;
|
|
16259
|
+
if (choice === "agent" && !localAgent) {
|
|
16260
|
+
return c.json({ error: "No local agent CLI found — install Claude Code (`claude`) or Codex (`codex`), or pick gemini/openai/openrouter." }, 400);
|
|
16261
|
+
}
|
|
16262
|
+
let evaluator = null;
|
|
16263
|
+
let evaluatorProvider = null;
|
|
16264
|
+
let evaluatorError = null;
|
|
16265
|
+
if (localAgent) {
|
|
16266
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
16267
|
+
const embedding = keys.gemini
|
|
16268
|
+
? { provider: "gemini", apiKey: keys.gemini }
|
|
16269
|
+
: keys.openai
|
|
16270
|
+
? { provider: "openai", apiKey: keys.openai }
|
|
16271
|
+
: null;
|
|
16272
|
+
evaluator = new LocalAgentClipClient({ agent: localAgent.kind, bin: localAgent.bin, embedding, tier });
|
|
16273
|
+
evaluatorProvider = `agent/${localAgent.kind}`;
|
|
16274
|
+
}
|
|
16275
|
+
else {
|
|
16276
|
+
const provider = (choice !== "agent" ? choice : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
16277
|
+
if (provider) {
|
|
16278
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16279
|
+
if ("error" in built) {
|
|
16280
|
+
if (choice)
|
|
16281
|
+
return c.json({ error: built.error }, 400);
|
|
16282
|
+
evaluatorError = built.error;
|
|
16283
|
+
}
|
|
16284
|
+
else {
|
|
16285
|
+
evaluator = built.client;
|
|
16286
|
+
evaluatorProvider = provider;
|
|
16287
|
+
}
|
|
16288
|
+
}
|
|
16289
|
+
else {
|
|
16290
|
+
evaluatorError = NO_CLIP_PROVIDER_KEY_ERROR;
|
|
16291
|
+
}
|
|
16292
|
+
}
|
|
16293
|
+
const ffmpegOk = await hasFfmpeg();
|
|
16294
|
+
const canIngestUrl = !huntSpec.source_url ||
|
|
16295
|
+
Boolean((config.RAPIDAPI_KEY || "").trim()) ||
|
|
16296
|
+
looksLikeDirectVideoUrl(huntSpec.source_url);
|
|
16297
|
+
if (evaluator && ffmpegOk && (localSourcePath || canIngestUrl)) {
|
|
16298
|
+
const includeAudio = body.include_audio !== false && evaluatorProvider === "gemini";
|
|
16299
|
+
await putScanRecords({ scanStatus: "running" });
|
|
16300
|
+
void runClipScanInProcess({
|
|
16301
|
+
scanId,
|
|
16302
|
+
ownerId: customer.id,
|
|
16303
|
+
sourceVideoId,
|
|
16304
|
+
filename,
|
|
16305
|
+
client: evaluator,
|
|
16306
|
+
localSourcePath,
|
|
16307
|
+
sourceUrl: huntSpec.source_url ?? null,
|
|
16308
|
+
huntSpec,
|
|
16309
|
+
guidancePrompt,
|
|
16310
|
+
framesPerScene,
|
|
16311
|
+
includeAudio
|
|
16312
|
+
}).catch((error) => {
|
|
16313
|
+
devLog("clip_scan.local_run_failed", { scan_id: scanId, ...devErrorFields(error) }, "error");
|
|
16314
|
+
});
|
|
16315
|
+
return scanResponse({ status: "running", pipeline: "local" });
|
|
16316
|
+
}
|
|
16317
|
+
// BACKUP: hand the hunt to the upstream cloud deployment (its saved BYOK
|
|
16318
|
+
// keys run the AI; its account is billed the AWS compute).
|
|
16319
|
+
if (hasUpstreamKey() && huntSpec.source_url) {
|
|
16320
|
+
const upstream = await upstreamJson("/clips/scan", {
|
|
16321
|
+
method: "POST",
|
|
16322
|
+
body: {
|
|
16323
|
+
source_url: huntSpec.source_url,
|
|
16324
|
+
prompt: guidancePrompt,
|
|
16325
|
+
hunt_spec: huntSpec,
|
|
16326
|
+
tracer,
|
|
16327
|
+
...(choice && choice !== "agent" ? { provider: choice } : {})
|
|
16328
|
+
},
|
|
16329
|
+
timeoutMs: 30_000
|
|
16330
|
+
});
|
|
16331
|
+
if (!upstream.ok || !upstream.json?.scan_id) {
|
|
16332
|
+
return c.json({ error: upstream.json?.error ?? `Upstream cloud scan failed to start (HTTP ${upstream.status}).` }, 502);
|
|
16333
|
+
}
|
|
16334
|
+
const marker = `upstream:${upstream.json.scan_id}:${upstream.json.source_video_id ?? ""}`;
|
|
16335
|
+
await putScanRecords({ scanStatus: "running", executionArn: marker });
|
|
16336
|
+
return scanResponse({ status: "running", executionArn: marker, pipeline: "upstream" });
|
|
16337
|
+
}
|
|
16338
|
+
const reasons = [];
|
|
16339
|
+
if (!evaluator)
|
|
16340
|
+
reasons.push(evaluatorError ?? "no scene evaluator available");
|
|
16341
|
+
if (evaluator && !ffmpegOk)
|
|
16342
|
+
reasons.push("ffmpeg was not found on this machine");
|
|
16343
|
+
if (evaluator && ffmpegOk && huntSpec.source_url && !canIngestUrl) {
|
|
16344
|
+
reasons.push("this box can't ingest social URLs (set RAPIDAPI_KEY, or pass a direct .mp4 link / upload the file)");
|
|
15928
16345
|
}
|
|
15929
|
-
const scan = {
|
|
15930
|
-
scan_id: scanId,
|
|
15931
|
-
owner_id: customer.id,
|
|
15932
|
-
source_video_id: sourceVideoId,
|
|
15933
|
-
status: executionArn ? "running" : "queued",
|
|
15934
|
-
tier,
|
|
15935
|
-
raw_s3_uri: sourcePointer,
|
|
15936
|
-
execution_arn: executionArn,
|
|
15937
|
-
tracer,
|
|
15938
|
-
created_at: now,
|
|
15939
|
-
updated_at: now
|
|
15940
|
-
};
|
|
15941
|
-
await clipRecords.putScan(scan);
|
|
15942
16346
|
return c.json({
|
|
15943
|
-
|
|
15944
|
-
|
|
15945
|
-
status: scan.status,
|
|
15946
|
-
execution_arn: executionArn ?? null,
|
|
15947
|
-
tracer,
|
|
15948
|
-
hunt_spec: huntSpec,
|
|
15949
|
-
estimate,
|
|
15950
|
-
compute_estimate: computeEstimate,
|
|
15951
|
-
pipeline_deployed: Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN)
|
|
15952
|
-
}, 202);
|
|
16347
|
+
error: `Can't run this hunt on this box: ${reasons.join("; ")}. Install Claude Code (\`claude\`) or Codex (\`codex\`) for keyless local scans, add a provider key, or link a cloud account (VIDFARM_UPSTREAM_HOST + VIDFARM_UPSTREAM_API_KEY) to use the cloud as backup.`
|
|
16348
|
+
}, 400);
|
|
15953
16349
|
}
|
|
15954
16350
|
catch (error) {
|
|
15955
16351
|
return c.json({ error: error instanceof Error ? error.message : "Failed to start clip scan." }, 400);
|
|
15956
16352
|
}
|
|
15957
16353
|
});
|
|
16354
|
+
// GET /clips/scan-options — what the Import Source modal can run a hunt with:
|
|
16355
|
+
// which BYOK providers have saved keys, whether a local agent CLI is available
|
|
16356
|
+
// (vidfarm serve boxes only), what "auto" resolves to, and where the hunt runs.
|
|
16357
|
+
app.get(`${CLIPS_PREFIX}/scan-options`, async (c) => {
|
|
16358
|
+
const customer = requireCustomer(c);
|
|
16359
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
16360
|
+
const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
|
|
16361
|
+
const localAgent = cloudPipeline ? null : detectLocalAgent();
|
|
16362
|
+
const keyProvider = CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
|
|
16363
|
+
const defaultProvider = localAgent ? "agent" : keyProvider;
|
|
16364
|
+
const ffmpegOk = cloudPipeline ? true : await hasFfmpeg();
|
|
16365
|
+
const pipeline = cloudPipeline
|
|
16366
|
+
? keyProvider ? "cloud" : null
|
|
16367
|
+
: defaultProvider && ffmpegOk ? "local" : hasUpstreamKey() ? "upstream" : null;
|
|
16368
|
+
return c.json({
|
|
16369
|
+
providers: CLIP_API_PROVIDERS.map((p) => ({ provider: p, has_key: Boolean(keys[p]) })),
|
|
16370
|
+
local_agent: localAgent ? { kind: localAgent.kind } : null,
|
|
16371
|
+
default_provider: defaultProvider,
|
|
16372
|
+
pipeline,
|
|
16373
|
+
upstream_backup: !cloudPipeline && hasUpstreamKey()
|
|
16374
|
+
});
|
|
16375
|
+
});
|
|
16376
|
+
function looksLikeDirectVideoUrl(url) {
|
|
16377
|
+
try {
|
|
16378
|
+
return /\.(mp4|mov|m4v|webm|mkv)$/i.test(new URL(url).pathname);
|
|
16379
|
+
}
|
|
16380
|
+
catch {
|
|
16381
|
+
return false;
|
|
16382
|
+
}
|
|
16383
|
+
}
|
|
16384
|
+
// Local URL ingest for serve-box hunts: direct video links download straight;
|
|
16385
|
+
// social URLs (TikTok/YouTube/IG/X) resolve through the same RapidAPI
|
|
16386
|
+
// downloader the video_download primitive uses (needs RAPIDAPI_KEY locally).
|
|
16387
|
+
async function ingestClipSourceUrlLocally(sourceUrl, workDir) {
|
|
16388
|
+
if (!sourceUrl)
|
|
16389
|
+
throw new Error("Clip scan needs a source video.");
|
|
16390
|
+
let mediaUrl = sourceUrl;
|
|
16391
|
+
if (!looksLikeDirectVideoUrl(sourceUrl)) {
|
|
16392
|
+
const apiKey = (config.RAPIDAPI_KEY || "").trim();
|
|
16393
|
+
if (!apiKey) {
|
|
16394
|
+
throw new Error("This box can't ingest social URLs (RAPIDAPI_KEY unset) — pass a direct .mp4 URL or upload the file.");
|
|
16395
|
+
}
|
|
16396
|
+
const lookupRes = await fetch(config.RAPIDAPI_VIDEO_DOWNLOAD_URL, {
|
|
16397
|
+
method: "POST",
|
|
16398
|
+
headers: {
|
|
16399
|
+
"x-rapidapi-key": apiKey,
|
|
16400
|
+
"x-rapidapi-host": config.RAPIDAPI_VIDEO_DOWNLOAD_HOST,
|
|
16401
|
+
"content-type": "application/x-www-form-urlencoded"
|
|
16402
|
+
},
|
|
16403
|
+
body: new URLSearchParams({ url: sourceUrl })
|
|
16404
|
+
});
|
|
16405
|
+
if (!lookupRes.ok)
|
|
16406
|
+
throw new Error(`Video download lookup failed with HTTP ${lookupRes.status}.`);
|
|
16407
|
+
const lookup = (await lookupRes.json());
|
|
16408
|
+
const playable = (lookup.medias ?? []).filter((m) => m.url && m.videoAvailable !== false && (m.type ? /video/i.test(m.type) : true));
|
|
16409
|
+
const chosen = playable.find((m) => (m.extension ?? "").toLowerCase() === "mp4" || /\.mp4(\?|$)/i.test(m.url ?? "")) ?? playable[0];
|
|
16410
|
+
if (!chosen?.url)
|
|
16411
|
+
throw new Error("Video download lookup returned no playable MP4 media URL.");
|
|
16412
|
+
mediaUrl = chosen.url;
|
|
16413
|
+
}
|
|
16414
|
+
const localPath = path.join(workDir, "ingest", "source.mp4");
|
|
16415
|
+
mkdirSync(path.dirname(localPath), { recursive: true });
|
|
16416
|
+
const res = await fetch(mediaUrl);
|
|
16417
|
+
if (!res.ok || !res.body)
|
|
16418
|
+
throw new Error(`Source video download failed with HTTP ${res.status}.`);
|
|
16419
|
+
await streamPipeline(Readable.fromWeb(res.body), createWriteStream(localPath));
|
|
16420
|
+
return localPath;
|
|
16421
|
+
}
|
|
16422
|
+
// In-process clip hunt for `vidfarm serve` boxes (no Step Functions worker).
|
|
16423
|
+
// Mirrors the clip-scan Lambda pipeline — ingest → probe/detect → refine →
|
|
16424
|
+
// per-scene tag/embed → finalize — with local ffmpeg and the resolved
|
|
16425
|
+
// evaluator (local claude/codex CLI or a BYOK key). Free: no wallet billing,
|
|
16426
|
+
// exactly like local renders. Detached; the client polls /clips/scan/:scanId.
|
|
16427
|
+
async function runClipScanInProcess(input) {
|
|
16428
|
+
const workRoot = path.join(config.VIDFARM_DATA_DIR, "clip-scans", input.scanId);
|
|
16429
|
+
const touchScan = async (patch) => {
|
|
16430
|
+
const scan = await clipRecords.getScan(input.ownerId, input.scanId);
|
|
16431
|
+
if (scan)
|
|
16432
|
+
await clipRecords.putScan({ ...scan, ...patch, updated_at: nowIso() });
|
|
16433
|
+
};
|
|
16434
|
+
const touchSource = async (patch) => {
|
|
16435
|
+
const source = await clipRecords.getSource(input.ownerId, input.sourceVideoId);
|
|
16436
|
+
if (source)
|
|
16437
|
+
await clipRecords.putSource({ ...source, ...patch, updated_at: nowIso() });
|
|
16438
|
+
};
|
|
16439
|
+
try {
|
|
16440
|
+
mkdirSync(workRoot, { recursive: true });
|
|
16441
|
+
const videoPath = input.localSourcePath ?? (await ingestClipSourceUrlLocally(input.sourceUrl ?? "", workRoot));
|
|
16442
|
+
const probe = await probeVideo(videoPath);
|
|
16443
|
+
await touchSource({ status: "scanning", duration_sec: probe.duration_sec });
|
|
16444
|
+
const spec = input.huntSpec;
|
|
16445
|
+
const ctx = {
|
|
16446
|
+
client: input.client,
|
|
16447
|
+
workDir: path.join(workRoot, "work"),
|
|
16448
|
+
clipsDir: path.join(workRoot, "clips"),
|
|
16449
|
+
thumbsDir: path.join(workRoot, "thumbs"),
|
|
16450
|
+
framesPerScene: input.framesPerScene,
|
|
16451
|
+
includeAudio: input.includeAudio,
|
|
16452
|
+
guidancePrompt: buildEffectiveGuidance({ contentPrompt: input.guidancePrompt, avoidText: spec.avoid_text }) || undefined,
|
|
16453
|
+
reencodeClips: true,
|
|
16454
|
+
crop: spec.aspect && spec.aspect !== "original"
|
|
16455
|
+
? { aspect: spec.aspect, focus: normalizeCropFocus(spec.crop_focus) }
|
|
16456
|
+
: undefined
|
|
16457
|
+
};
|
|
16458
|
+
let stored = 0;
|
|
16459
|
+
await scanVideo({
|
|
16460
|
+
ctx,
|
|
16461
|
+
videoPath,
|
|
16462
|
+
probe,
|
|
16463
|
+
windows: spec.windows,
|
|
16464
|
+
durationBand: spec.duration_band ?? null,
|
|
16465
|
+
sourceVideoId: input.sourceVideoId,
|
|
16466
|
+
sourceFilename: input.filename,
|
|
16467
|
+
ownerId: input.ownerId,
|
|
16468
|
+
// Local agents run one CLI process per call — keep parallelism modest.
|
|
16469
|
+
concurrency: input.client.provider === "local-agent" ? 2 : 3,
|
|
16470
|
+
onClip: async (clip) => {
|
|
16471
|
+
// Persist through the storage service (local driver) as bare keys —
|
|
16472
|
+
// clipMediaUrl resolves clips/… + thumbs/… keys to fetchable URLs.
|
|
16473
|
+
const clipKey = `clips/${input.ownerId}/${clip.clip_id}.mp4`;
|
|
16474
|
+
const clipLocal = path.join(ctx.clipsDir, `${clip.clip_id}.mp4`);
|
|
16475
|
+
await storage.putBuffer(clipKey, readFileSync(clipLocal), "video/mp4");
|
|
16476
|
+
clip.file_path = clipKey;
|
|
16477
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clip.clip_id}.jpg`);
|
|
16478
|
+
if (existsSync(thumbLocal)) {
|
|
16479
|
+
const thumbKey = `thumbs/${input.ownerId}/${clip.clip_id}.jpg`;
|
|
16480
|
+
await storage.putBuffer(thumbKey, readFileSync(thumbLocal), "image/jpeg");
|
|
16481
|
+
clip.thumbnail_path = thumbKey;
|
|
16482
|
+
}
|
|
16483
|
+
else {
|
|
16484
|
+
clip.thumbnail_path = "";
|
|
16485
|
+
}
|
|
16486
|
+
await clipRecords.putClip(clip);
|
|
16487
|
+
await vectorIndex.upsert(input.ownerId, clip);
|
|
16488
|
+
stored++;
|
|
16489
|
+
},
|
|
16490
|
+
onError: (scene, error) => {
|
|
16491
|
+
devLog("clip_scan.local_scene_failed", { scan_id: input.scanId, scene_start: scene.start_time_sec, ...devErrorFields(error) }, "warn");
|
|
16492
|
+
}
|
|
16493
|
+
});
|
|
16494
|
+
await touchSource({ status: "complete", clip_count: stored });
|
|
16495
|
+
await touchScan({ status: "complete" });
|
|
16496
|
+
}
|
|
16497
|
+
catch (error) {
|
|
16498
|
+
devLog("clip_scan.local_failed", { scan_id: input.scanId, ...devErrorFields(error) }, "error");
|
|
16499
|
+
await touchSource({ status: "failed" }).catch(() => undefined);
|
|
16500
|
+
await touchScan({ status: "failed", error: error instanceof Error ? error.message : String(error) }).catch(() => undefined);
|
|
16501
|
+
}
|
|
16502
|
+
finally {
|
|
16503
|
+
rmSync(workRoot, { recursive: true, force: true });
|
|
16504
|
+
}
|
|
16505
|
+
}
|
|
16506
|
+
// A serve-box hunt handed to the upstream cloud (execution_arn
|
|
16507
|
+
// "upstream:<scan_id>:<source_video_id>"): mirror the upstream status onto the
|
|
16508
|
+
// local records, and when the cloud hunt completes pull the finished clips
|
|
16509
|
+
// down into local storage — the cloud runs the hunt, the box owns the result.
|
|
16510
|
+
const activeUpstreamClipMirrors = new Set();
|
|
16511
|
+
async function reconcileUpstreamClipScan(ownerId, scan) {
|
|
16512
|
+
const [, upstreamScanId, upstreamSourceId = ""] = (scan.execution_arn ?? "").split(":");
|
|
16513
|
+
if (!upstreamScanId)
|
|
16514
|
+
return scan;
|
|
16515
|
+
const res = await upstreamJson(`/clips/scan/${encodeURIComponent(upstreamScanId)}`);
|
|
16516
|
+
if (!res.ok || !res.json)
|
|
16517
|
+
return scan; // transient upstream hiccup — keep polling
|
|
16518
|
+
const status = res.json.source?.status ?? res.json.scan?.status ?? "running";
|
|
16519
|
+
if (status === "failed" || status === "error") {
|
|
16520
|
+
const failed = { ...scan, status: "failed", error: res.json.scan?.error ?? "Upstream cloud hunt failed.", updated_at: nowIso() };
|
|
16521
|
+
await clipRecords.putScan(failed);
|
|
16522
|
+
const source = await clipRecords.getSource(ownerId, scan.source_video_id);
|
|
16523
|
+
if (source && source.status !== "complete") {
|
|
16524
|
+
await clipRecords.putSource({ ...source, status: "failed", updated_at: nowIso() });
|
|
16525
|
+
}
|
|
16526
|
+
return failed;
|
|
16527
|
+
}
|
|
16528
|
+
if (status === "complete" && !activeUpstreamClipMirrors.has(scan.scan_id)) {
|
|
16529
|
+
activeUpstreamClipMirrors.add(scan.scan_id);
|
|
16530
|
+
void mirrorUpstreamClipsToLocal({ ownerId, scan, upstreamSourceId })
|
|
16531
|
+
.catch((error) => devLog("clip_scan.upstream_mirror_failed", { scan_id: scan.scan_id, ...devErrorFields(error) }, "error"))
|
|
16532
|
+
.finally(() => activeUpstreamClipMirrors.delete(scan.scan_id));
|
|
16533
|
+
}
|
|
16534
|
+
return scan; // stays "running" until the mirror lands the clips locally
|
|
16535
|
+
}
|
|
16536
|
+
async function mirrorUpstreamClipsToLocal(input) {
|
|
16537
|
+
const { ownerId, scan } = input;
|
|
16538
|
+
const query = input.upstreamSourceId ? `?source=${encodeURIComponent(input.upstreamSourceId)}&limit=200` : "?limit=200";
|
|
16539
|
+
const feed = await upstreamJson(`/clips/feed${query}`, { timeoutMs: 60_000 });
|
|
16540
|
+
if (!feed.ok || !feed.json)
|
|
16541
|
+
throw new Error(`Upstream clip feed unavailable (HTTP ${feed.status}).`);
|
|
16542
|
+
let stored = 0;
|
|
16543
|
+
for (const raw of feed.json.clips ?? []) {
|
|
16544
|
+
const clipId = typeof raw.clip_id === "string" ? raw.clip_id : null;
|
|
16545
|
+
const viewUrl = typeof raw.view_url === "string" ? raw.view_url : null;
|
|
16546
|
+
if (!clipId || !viewUrl)
|
|
16547
|
+
continue;
|
|
16548
|
+
const clipRes = await fetch(viewUrl);
|
|
16549
|
+
if (!clipRes.ok)
|
|
16550
|
+
continue;
|
|
16551
|
+
const clipKey = `clips/${ownerId}/${clipId}.mp4`;
|
|
16552
|
+
await storage.putBuffer(clipKey, new Uint8Array(await clipRes.arrayBuffer()), "video/mp4");
|
|
16553
|
+
let thumbKey = "";
|
|
16554
|
+
const thumbUrl = typeof raw.thumbnail_url === "string" ? raw.thumbnail_url : null;
|
|
16555
|
+
if (thumbUrl) {
|
|
16556
|
+
const thumbRes = await fetch(thumbUrl);
|
|
16557
|
+
if (thumbRes.ok) {
|
|
16558
|
+
thumbKey = `thumbs/${ownerId}/${clipId}.jpg`;
|
|
16559
|
+
await storage.putBuffer(thumbKey, new Uint8Array(await thumbRes.arrayBuffer()), "image/jpeg");
|
|
16560
|
+
}
|
|
16561
|
+
}
|
|
16562
|
+
const { view_url: _v, thumbnail_url: _t, ...rest } = raw;
|
|
16563
|
+
void _v;
|
|
16564
|
+
void _t;
|
|
16565
|
+
await clipRecords.putClip({
|
|
16566
|
+
...rest,
|
|
16567
|
+
owner_id: ownerId,
|
|
16568
|
+
source_video_id: scan.source_video_id,
|
|
16569
|
+
file_path: clipKey,
|
|
16570
|
+
thumbnail_path: thumbKey
|
|
16571
|
+
});
|
|
16572
|
+
stored++;
|
|
16573
|
+
}
|
|
16574
|
+
const source = await clipRecords.getSource(ownerId, scan.source_video_id);
|
|
16575
|
+
if (source)
|
|
16576
|
+
await clipRecords.putSource({ ...source, status: "complete", clip_count: stored, updated_at: nowIso() });
|
|
16577
|
+
const current = await clipRecords.getScan(ownerId, scan.scan_id);
|
|
16578
|
+
await clipRecords.putScan({ ...(current ?? scan), status: "complete", updated_at: nowIso() });
|
|
16579
|
+
}
|
|
15958
16580
|
// GET /clips/scan/:scanId — poll scan status.
|
|
15959
16581
|
app.get(`${CLIPS_PREFIX}/scan/:scanId`, async (c) => {
|
|
15960
16582
|
const customer = requireCustomer(c);
|
|
15961
16583
|
let scan = await clipRecords.getScan(customer.id, c.req.param("scanId"));
|
|
15962
16584
|
if (!scan)
|
|
15963
16585
|
return c.json({ error: "Scan not found" }, 404);
|
|
16586
|
+
// Serve-box hunts handed to the upstream cloud reconcile against the
|
|
16587
|
+
// upstream REST API instead of Step Functions.
|
|
16588
|
+
if ((scan.status === "running" || scan.status === "queued") && scan.execution_arn?.startsWith("upstream:")) {
|
|
16589
|
+
scan = await reconcileUpstreamClipScan(customer.id, scan);
|
|
16590
|
+
}
|
|
16591
|
+
else
|
|
15964
16592
|
// Lazily reconcile with the Step Functions execution: the pipeline has no
|
|
15965
16593
|
// failure hook back into the scan record, so a crashed execution would
|
|
15966
16594
|
// otherwise leave "running" forever and pollers spin.
|
|
@@ -16006,7 +16634,11 @@ app.post(`${CLIPS_PREFIX}/presets`, async (c) => {
|
|
|
16006
16634
|
return c.json({ error: "name is required." }, 400);
|
|
16007
16635
|
let criteria = body.criteria;
|
|
16008
16636
|
if (!criteria && body.query) {
|
|
16009
|
-
const
|
|
16637
|
+
const requested = readClipProviderChoice(body.provider);
|
|
16638
|
+
const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
16639
|
+
if (!provider)
|
|
16640
|
+
return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
|
|
16641
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16010
16642
|
if ("error" in built)
|
|
16011
16643
|
return c.json({ error: built.error }, 400);
|
|
16012
16644
|
criteria = await built.client.naturalLanguageToCriteria(body.query);
|