@mevdragon/vidfarm-devcli 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/src/app.js CHANGED
@@ -1,6 +1,8 @@
1
1
  import { createHash, randomBytes, randomUUID } from "node:crypto";
2
- import { existsSync, readFileSync, statSync } from "node:fs";
2
+ import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
3
3
  import path from "node:path";
4
+ import { Readable } from "node:stream";
5
+ import { pipeline as streamPipeline } from "node:stream/promises";
4
6
  import { createGoogleGenerativeAI } from "@ai-sdk/google";
5
7
  import { createOpenAI } from "@ai-sdk/openai";
6
8
  import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
@@ -21,7 +23,7 @@ import { writeForkManifest } from "./services/fork-manifest.js";
21
23
  import { renderHelpPage } from "./help-page.js";
22
24
  import { renderHomepage } from "./homepage.js";
23
25
  import { COMPOSITION_RUNTIME_SCRIPT_BODY } from "./composition-runtime.js";
24
- import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, TRANSITION_CSS, TRANSITION_STYLE_MARKER } from "./hyperframes/composition.js";
26
+ import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, TRANSITION_CSS, TRANSITION_STYLE_MARKER, upgradeLegacyTransitionCss } from "./hyperframes/composition.js";
25
27
  import { normalizeCompositionZIndexHtml, sanitizeCompositionHtml } from "./services/composition-sanitize.js";
26
28
  import { applyMarkupUsd } from "./services/billing-pricing.js";
27
29
  import { primitiveRegistry } from "./primitive-registry.js";
@@ -45,10 +47,14 @@ import { ServerlessProviderKeyService } from "./services/serverless-provider-key
45
47
  import { ServerlessTemplateConfigService } from "./services/serverless-template-configs.js";
46
48
  import { parseHTML } from "linkedom";
47
49
  import { joinStorageKey, StorageService } from "./services/storage.js";
50
+ import { buildCaptionCues, cuesFromText } from "./services/captions.js";
51
+ import { defaultSpeechModelFor as speechModelFor, transcribeSpeechWithKey } from "./services/speech.js";
52
+ import { insertCaptionLayers } from "./devcli/composition-edit.js";
53
+ import { loadSpeechAudioLocally, MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES } from "./devcli/speech.js";
48
54
  import { DescribeExecutionCommand, SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
49
55
  import { clipRecords, makeUserPreset } from "./services/clip-records.js";
50
- import { matchScenesToClips, searchUserClips } from "./services/clip-search.js";
51
- import { BUILTIN_PRESETS, ClipModelClient, effectiveHuntDurationSec, estimateScanCostFromDuration, normalizeHuntSpec, parseClipHuntPrompt, resolveDurationBand } from "./services/clip-curation/index.js";
56
+ import { matchScenesToClips, searchUserClips, vectorIndex } from "./services/clip-search.js";
57
+ import { BUILTIN_PRESETS, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, estimateScanCostFromDuration, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, probeVideo, resolveDurationBand, scanVideo } from "./services/clip-curation/index.js";
52
58
  import { analyzeCastMembers, annotateScenesFromSource, buildSmartDecomposedHtml, HyperframesService, smartDecomposeVideo, stampCompositionIdentity } from "./services/hyperframes.js";
53
59
  import { buildPendingSceneAnnotationsState } from "./services/scene-annotations.js";
54
60
  import { createPrimitiveJobContext } from "./primitive-context.js";
@@ -395,7 +401,16 @@ const userAttachmentFinalizeSchema = z.object({
395
401
  content_type: z.string().trim().min(1).max(255),
396
402
  size_bytes: z.coerce.number().int().min(1).max(1024 * 1024 * 1024),
397
403
  storage_key: z.string().min(1).max(2000),
398
- folder_path: z.string().trim().max(500).optional()
404
+ folder_path: z.string().trim().max(500).optional(),
405
+ notes: z.string().trim().max(4000).optional()
406
+ });
407
+ const userAttachmentNotesSchema = z.object({
408
+ notes: z.string().max(4000).nullable().optional()
409
+ });
410
+ const userAttachmentSearchSchema = z.object({
411
+ query: z.string().min(1).max(2000),
412
+ folder_path: z.string().trim().max(500).optional(),
413
+ limit: z.coerce.number().int().min(1).max(100).optional()
399
414
  });
400
415
  const userFileFolderSchema = z.object({
401
416
  folder_path: z.string().trim().min(1).max(500)
@@ -850,6 +865,26 @@ function consumeSettingsFlash(c) {
850
865
  return null;
851
866
  }
852
867
  }
868
+ // Fully-local box (`vidfarm serve`): the browser is pre-authed through
869
+ // /auto-login at boot, but a fresh browser profile / cleared cookies / a
870
+ // directly-opened /editor URL carries no session cookie — and every editor
871
+ // POST (Render, chat, uploads) then fails with a 401 "Unauthorized" modal
872
+ // even though the box is a single-user localhost machine whose only customer
873
+ // is the env-bootstrapped one. Fall back to the bootstrap key's customer,
874
+ // the exact identity /auto-login would have granted.
875
+ async function localBoxFallbackCustomer() {
876
+ const isFullyLocalBox = !config.isProduction
877
+ && config.RECORDS_DRIVER === "local"
878
+ && config.STORAGE_DRIVER === "local";
879
+ if (!isFullyLocalBox || !config.VIDFARM_API_KEY)
880
+ return null;
881
+ try {
882
+ return await auth.authenticateAsync(config.VIDFARM_API_KEY);
883
+ }
884
+ catch {
885
+ return null;
886
+ }
887
+ }
853
888
  async function getBrowserCustomer(c, body) {
854
889
  try {
855
890
  // Agent-first auth: an explicit vidfarm-api-key header wins over browser
@@ -881,10 +916,10 @@ async function getBrowserCustomer(c, body) {
881
916
  return currentCustomer;
882
917
  }
883
918
  const customers = await listBrowserSessionCustomers(c);
884
- if (customers.length !== 1) {
885
- return null;
919
+ if (customers.length === 1) {
920
+ return customers[0] ?? null;
886
921
  }
887
- return customers[0] ?? null;
922
+ return await localBoxFallbackCustomer();
888
923
  }
889
924
  catch {
890
925
  return null;
@@ -2891,6 +2926,9 @@ function serializeUserAttachment(c, attachment) {
2891
2926
  sizeBytes: attachment.sizeBytes,
2892
2927
  createdAt: attachment.createdAt,
2893
2928
  folderPath: attachment.folderPath ?? "",
2929
+ notes: attachment.notes ?? null,
2930
+ // Flag only — the raw vector never ships to the client (mirrors serializeClip).
2931
+ hasNotesEmbedding: Boolean(attachment.notesEmbedding && attachment.notesEmbedding.length > 0),
2894
2932
  viewUrl: attachment.publicUrl || storage.getPublicUrl(attachment.storageKey) || buildUserAttachmentViewUrl(c, attachment.storageKey)
2895
2933
  };
2896
2934
  }
@@ -2918,10 +2956,19 @@ function isFormFile(value) {
2918
2956
  && "arrayBuffer" in value);
2919
2957
  }
2920
2958
  async function getVisibleApiKey(customerId) {
2921
- const existing = await serverlessRecords.getLatestApiKeyForCustomer(customerId);
2922
- const rawValue = existing?.raw_value ? String(existing.raw_value) : null;
2923
- if (rawValue) {
2924
- return rawValue;
2959
+ // Prefer the newest key THIS server can validate, not just the newest key.
2960
+ // After an API_KEY_SALT rotation the shared table holds keys hashed under
2961
+ // different salts (deployed stage vs local dev boxes share the staging
2962
+ // table); embedding a key the current server rejects breaks editor chat and
2963
+ // its tool calls with "Invalid API credentials." on every message.
2964
+ const existingKeys = await serverlessRecords.listActiveApiKeysForCustomer(customerId);
2965
+ for (const row of existingKeys) {
2966
+ const rawValue = row?.raw_value ? String(row.raw_value) : null;
2967
+ if (!rawValue)
2968
+ continue;
2969
+ if (String(row.key_hash ?? "") === hashSecret(rawValue + config.API_KEY_SALT)) {
2970
+ return rawValue;
2971
+ }
2925
2972
  }
2926
2973
  const apiKey = `vf_${createId("key")}`;
2927
2974
  await serverlessRecords.insertApiKey({
@@ -3031,6 +3078,13 @@ function asRecord(value) {
3031
3078
  function readNonEmptyString(value) {
3032
3079
  return typeof value === "string" && value.trim() ? value.trim() : null;
3033
3080
  }
3081
+ // "raw" = the user's own new project (prompt-generated, own raw clips) — the
3082
+ // editor must not auto-prompt the Decompose modal for these. Anything else
3083
+ // (absent, unknown) reads as "inspiration", matching records that predate the
3084
+ // field.
3085
+ function readProjectOrigin(value) {
3086
+ return value === "raw" ? "raw" : "inspiration";
3087
+ }
3034
3088
  function readFiniteNumber(value, fallback = 0) {
3035
3089
  const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN;
3036
3090
  return Number.isFinite(number) ? number : fallback;
@@ -3090,7 +3144,8 @@ function buildPrimitiveEditorDocsRoutes() {
3090
3144
  { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/concat`, summary: "Primitive audio concat job. Body must be { tracer, payload: { source_audio_urls, output_format?, audio_bitrate_kbps? }, webhook_url? }." },
3091
3145
  { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/normalize`, summary: "Primitive audio normalization job. Body must be { tracer, payload: { source_audio_url, output_format?, audio_bitrate_kbps?, sample_rate_hz? }, webhook_url? }." },
3092
3146
  { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/speech`, summary: "Primitive text-to-speech (TTS) job using saved OpenAI, Gemini, or OpenRouter provider keys. Body must be { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format? }, webhook_url? }. instructions is a free-form voice-style prompt (tone, pacing, accent, emotion, persona — e.g. \"calm, warm bedtime narrator with a slight British accent\"). voice is a provider preset (OpenAI: alloy/ash/coral/…; Gemini: Kore/Puck/Charon/…); provider is inferred from the voice when omitted, and a voice paired with the wrong provider is rejected with a 400. Returns a durable platform audio URL (mp3 or wav). Alias: POST /api/v1/primitives/tts." },
3093
- { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt." },
3147
+ { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt. Pass word_timestamps: true for word-level timings on each segment (real timestamps on an OpenAI key via whisper-1; other providers stay segment-level)." },
3148
+ { method: "POST", path: `${PRIMITIVES_PREFIX}/audio/captions`, summary: "ANIMATED CAPTIONS from speech: one job transcribes narration WITH word-level timings and pages it into word-by-word caption cues ready for the editor's set_captions action. Body must be { tracer, payload: { source_url (video or audio; video audio is demuxed) OR text (page a script instead of transcribing), style? (spotlight|karaoke|word-pop|stack-up|neon|bounce), animation?, active_color?, highlight_color?, uppercase?, color?, background?, background_style?, font_family?, font_size?, max_words_per_cue?, offset_sec?, language?, provider?, window_start?/window_duration? (text mode) }, webhook_url? }. The finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments — apply verbatim). Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter fall back to estimated word windows. Alias: POST /api/v1/primitives/captions." },
3094
3149
  { method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/coldstart`, summary: "Primitive cold-start strategy questionnaire. Body should usually be { tracer, payload: { user_message }, webhook_url? }. user_message can be rough, like \"I don't know where to start\" or a messy description. count is optional only when the user explicitly wants more or fewer questions. offer_description remains accepted as a backward-compatible alias." },
3095
3150
  { method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, summary: "Primitive awareness-stage recommendation. Body must be { tracer, payload: { offer_description }, webhook_url? }. Use this when the customer is unsure what type of ads to produce." },
3096
3151
  { method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/angles`, summary: "Primitive ad-angle brainstorm. Body must be { tracer, payload: { offer_description, problem_awareness, solution_awareness }, webhook_url? }. Allowed values: problem_awareness is problem_unaware|problem_aware, solution_awareness is solution_unaware|solution_aware. count is optional only when the user explicitly wants more or fewer angles. Returns diverse persuasive TikTok angles with explanations." },
@@ -3684,7 +3739,7 @@ function normalizeOperationRequestTracer(tracer) {
3684
3739
  }
3685
3740
  function createTemplateHttpTool(input) {
3686
3741
  return tool({
3687
- description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys. To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
3742
+ description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key, and word_timestamps: true adds word-level timings per segment (real on an OpenAI key via whisper-1). If the user wants ANIMATED word-by-word captions generated from narration, call POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url (narration audio or video URL) OR text (a script to page), style?: \"spotlight\"|\"karaoke\"|\"word-pop\"|\"stack-up\"|\"neon\"|\"bounce\", max_words_per_cue?, offset_sec?, language?, provider? } } — the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments); apply them with the editor_action tool (action_type=set_captions) so the change flows through the editor state. Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter estimate word windows. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, provider?: \"gemini\"|\"openai\"|\"openrouter\", tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys (auto-picked gemini → openai → openrouter unless body.provider names one). To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
3688
3743
  inputSchema: z.object({
3689
3744
  method: z.enum(["GET", "POST"]).describe("HTTP method to use."),
3690
3745
  path: z.string().min(1).describe("Absolute path or same-origin URL for an allowed Vidfarm REST route."),
@@ -3932,16 +3987,19 @@ function browseFilesTextContentType(fileName) {
3932
3987
  }
3933
3988
  function createBrowseFilesTool(input) {
3934
3989
  return tool({
3935
- description: "Browse AND write the user's My Files filesystem — their personal library of uploaded assets (mp4, mov, webm, png, jpg, jpeg, gif, webp, svg, mp3, wav, m4a, aac, pdf, md, txt, csv, and more) organized into virtual folders. Use action=list to enumerate files and virtual folders so you can discover what assets exist; pass folder_path to scope the listing to one folder (omit or empty for the root). Use action=read to fetch one file by file_id (preferred; copy it from a prior list result) or by file_name: it returns the file's durable view_url plus metadata, and for text files (md, txt, csv, srt, vtt, json) it also returns the full text_content inline so you can read it. For images, video, audio, and PDFs it returns view_url only — reference that URL directly (drop it into composition layers, or pass it into primitive routes like images/edit source_image_url, videos/generate input_references, or videos/download); you cannot read their raw bytes as text. Use action=write to SAVE a text file into My Files — pass file_name (e.g. About.md), content (the full text), and optional folder_path to namescope it under a product/offer folder. This is how you persist Getting Started context the user can reuse later: a product About.md or Interview.md, and awareness-levels.md / persuasive-angles.md / ad-hooks.md distilled from brainstorm output. write only accepts text files (md, txt, csv, json, srt, vtt); to save generated images/video/audio use the primitive routes and reference their durable URLs. write returns the new file_id and view_url. Always list before you read so you use a real file_id; never invent file ids or view URLs.",
3990
+ description: "Browse AND write the user's My Files filesystem — their personal library of uploaded assets (mp4, mov, webm, png, jpg, jpeg, gif, webp, svg, mp3, wav, m4a, aac, pdf, md, txt, csv, and more) organized into virtual folders. Use action=list to enumerate files and virtual folders so you can discover what assets exist; pass folder_path to scope the listing to one folder (omit or empty for the root). Use action=search to find files by MEANING when you don't know where something lives: pass query (plain language, e.g. 'character reference sheet for the mascot' or 'logo on transparent background') and it matches keywords AND vector-embedded metadata notes across every file's name, folder, and notes — prefer search over paging through list when My Files is large or the user references an asset vaguely. Use action=read to fetch one file by file_id (preferred; copy it from a prior list/search result) or by file_name: it returns the file's durable view_url plus metadata, and for text files (md, txt, csv, srt, vtt, json) it also returns the full text_content inline so you can read it. For images, video, audio, and PDFs it returns view_url only — reference that URL directly (drop it into composition layers, or pass it into primitive routes like images/edit source_image_url, videos/generate input_references, or videos/download); you cannot read their raw bytes as text. Use action=write to SAVE a text file into My Files — pass file_name (e.g. About.md), content (the full text), optional folder_path to namescope it under a product/offer folder, and optional notes. This is how you persist Getting Started context the user can reuse later: a product About.md or Interview.md, and awareness-levels.md / persuasive-angles.md / ad-hooks.md distilled from brainstorm output. write saves text files (md, txt, csv, json, srt, vtt) via content, or IMPORTS any media asset into the library by passing its durable URL as source_url — e.g. save a finished image job's output as character_sprite_card.png so it persists in My Files instead of living only in a job result. write returns the new file_id and view_url. Use action=annotate to set metadata notes on ANY existing file (including binary assets): notes say what the file is, who/what it depicts, and when to use it, and they are vector-embedded so future sessions can search their way back to the asset. Annotate every asset worth finding again — especially character references: keep a character_sprite_card.png plus an about_character.md per recurring character in that character's folder, annotate both, and pass the sprite card's view_url into generation reference inputs (images prompt_attachments / videos input_references) for character consistency. Always list or search before you read so you use a real file_id; never invent file ids or view URLs.",
3936
3991
  inputSchema: z.object({
3937
- action: z.enum(["list", "read", "write"]).describe("list = enumerate files and folders; read = fetch one file's view_url and, for text files, its contents; write = save a text file into My Files."),
3938
- folder_path: z.string().optional().describe("Folder to scope a list to, the folder of the file to read, or the folder to write into. Omit or leave empty for the root."),
3939
- file_id: z.string().optional().describe("Attachment id to read, copied from a prior list result. Preferred over file_name."),
3940
- file_name: z.string().optional().describe("File name to read when you do not have the id, or the name to save under for write (e.g. About.md). For read it is matched within folder_path when one is provided."),
3941
- content: z.string().optional().describe("For write: the full text content of the file to save (e.g. Markdown for About.md). Required for write; ignored otherwise."),
3942
- explanation: z.string().optional().describe("Short reason for the browse or save.")
3992
+ action: z.enum(["list", "read", "write", "annotate", "search"]).describe("list = enumerate files and folders; read = fetch one file's view_url and, for text files, its contents; write = save a text file into My Files; annotate = set the metadata notes on one existing file; search = find files by meaning across names, folders, and notes."),
3993
+ folder_path: z.string().optional().describe("Folder to scope a list or search to, the folder of the file to read, or the folder to write into. Omit or leave empty for the root."),
3994
+ file_id: z.string().optional().describe("Attachment id to read or annotate, copied from a prior list/search result. Preferred over file_name."),
3995
+ file_name: z.string().optional().describe("File name to read/annotate when you do not have the id, or the name to save under for write (e.g. About.md). Matched within folder_path when one is provided."),
3996
+ content: z.string().optional().describe("For write: the full text content of the file to save (e.g. Markdown for About.md). Required for text writes; mutually exclusive with source_url."),
3997
+ source_url: z.string().optional().describe("For write: a durable media URL (from a finished primitive job result or an existing asset) to IMPORT into My Files as file_name — this is how you save a generated character_sprite_card.png or other binary asset into the library. Mutually exclusive with content."),
3998
+ notes: z.string().optional().describe("For annotate (required) or write (optional): metadata notes describing what the file is, who/what it depicts, and when to use it — e.g. 'Sprite card for Zara, our mascot: front/side/back views, teal jacket. Use as the reference image whenever generating Zara.' Notes are vector-embedded so search finds the file by meaning. For annotate, an empty string clears existing notes."),
3999
+ query: z.string().optional().describe("For search: what you're looking for, in plain language. Required for search; ignored otherwise."),
4000
+ explanation: z.string().optional().describe("Short reason for the browse, save, annotation, or search.")
3943
4001
  }),
3944
- execute: async ({ action, folder_path, file_id, file_name, content, explanation }) => {
4002
+ execute: async ({ action, folder_path, file_id, file_name, content, source_url, notes, query, explanation }) => {
3945
4003
  const listUrl = new URL(`${USER_PREFIX}/me/attachments`, config.PUBLIC_BASE_URL);
3946
4004
  const headers = new Headers();
3947
4005
  headers.set("vidfarm-api-key", input.vidfarmApiKey);
@@ -3954,22 +4012,59 @@ function createBrowseFilesTool(input) {
3954
4012
  if (action === "write") {
3955
4013
  const name = (file_name ?? "").trim();
3956
4014
  if (!name) {
3957
- return { action, explanation: explanation ?? null, status: 400, error: "write requires file_name (e.g. About.md)." };
4015
+ return { action, explanation: explanation ?? null, status: 400, error: "write requires file_name (e.g. About.md, or character_sprite_card.png with source_url)." };
3958
4016
  }
3959
- if (typeof content !== "string") {
3960
- return { action, explanation: explanation ?? null, status: 400, error: "write requires content (the full text of the file)." };
4017
+ const importSrc = (source_url ?? "").trim();
4018
+ let fileBlob;
4019
+ if (importSrc) {
4020
+ // Import a durable media URL (e.g. a finished primitive job's output)
4021
+ // into My Files. The api-key header only travels to our own origin.
4022
+ let sourceUrl;
4023
+ try {
4024
+ sourceUrl = new URL(importSrc, config.PUBLIC_BASE_URL);
4025
+ }
4026
+ catch {
4027
+ return { action, explanation: explanation ?? null, status: 400, error: `source_url is not a valid URL: ${importSrc.slice(0, 200)}` };
4028
+ }
4029
+ const sourceHeaders = new Headers();
4030
+ if (sourceUrl.origin === new URL(config.PUBLIC_BASE_URL).origin) {
4031
+ sourceHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
4032
+ }
4033
+ const sourceResponse = await fetch(sourceUrl, {
4034
+ method: "GET",
4035
+ headers: sourceHeaders,
4036
+ signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
4037
+ });
4038
+ if (!sourceResponse.ok) {
4039
+ return { action, explanation: explanation ?? null, status: sourceResponse.status, error: `Could not fetch source_url (${sourceResponse.status}). Use a durable URL from a finished job result or an existing asset.` };
4040
+ }
4041
+ const sourceBytes = await sourceResponse.arrayBuffer();
4042
+ if (sourceBytes.byteLength > 50 * 1024 * 1024) {
4043
+ return { action, explanation: explanation ?? null, status: 400, error: "source_url asset is larger than the 50 MB My Files limit." };
4044
+ }
4045
+ const sourceType = (sourceResponse.headers.get("content-type") ?? "").split(";")[0].trim();
4046
+ fileBlob = new Blob([sourceBytes], sourceType ? { type: sourceType } : undefined);
3961
4047
  }
3962
- if (!isBrowseFilesTextFile(name, "")) {
3963
- return { action, explanation: explanation ?? null, status: 400, error: `write only supports text files (md, txt, csv, json, srt, vtt). "${name}" is not one; save generated media via primitive routes and reference their view_url instead.` };
4048
+ else {
4049
+ if (typeof content !== "string") {
4050
+ return { action, explanation: explanation ?? null, status: 400, error: "write requires content (the full text of the file) or source_url (a durable media URL to import)." };
4051
+ }
4052
+ if (!isBrowseFilesTextFile(name, "")) {
4053
+ return { action, explanation: explanation ?? null, status: 400, error: `write with content only supports text files (md, txt, csv, json, srt, vtt). To save media like "${name}", pass its durable URL as source_url instead.` };
4054
+ }
4055
+ fileBlob = new Blob([content], { type: browseFilesTextContentType(name) });
3964
4056
  }
3965
4057
  const uploadUrl = new URL(`${USER_PREFIX}/me/attachments/upload`, config.PUBLIC_BASE_URL);
3966
4058
  const uploadHeaders = new Headers();
3967
4059
  uploadHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
3968
4060
  const form = new FormData();
3969
- form.append("file", new Blob([content], { type: browseFilesTextContentType(name) }), name);
4061
+ form.append("file", fileBlob, name);
3970
4062
  if (normalizedFolder) {
3971
4063
  form.append("folder_path", normalizedFolder);
3972
4064
  }
4065
+ if (typeof notes === "string" && notes.trim()) {
4066
+ form.append("notes", notes.trim());
4067
+ }
3973
4068
  const writeResponse = await fetch(uploadUrl, {
3974
4069
  method: "POST",
3975
4070
  headers: uploadHeaders,
@@ -4004,11 +4099,65 @@ function createBrowseFilesTool(input) {
4004
4099
  content_type: saved.contentType,
4005
4100
  size_bytes: saved.sizeBytes,
4006
4101
  folder_path: saved.folderPath ?? "",
4102
+ notes: saved.notes ?? null,
4007
4103
  view_url: saved.viewUrl,
4008
4104
  created_at: saved.createdAt
4009
4105
  }
4010
4106
  };
4011
4107
  }
4108
+ if (action === "search") {
4109
+ const q = (query ?? "").trim();
4110
+ if (!q) {
4111
+ return { action, explanation: explanation ?? null, status: 400, error: "search requires query (what you're looking for, in plain language)." };
4112
+ }
4113
+ const searchUrl = new URL(`${USER_PREFIX}/me/attachments/search`, config.PUBLIC_BASE_URL);
4114
+ const searchHeaders = new Headers();
4115
+ searchHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
4116
+ searchHeaders.set("Content-Type", "application/json");
4117
+ const searchResponse = await fetch(searchUrl, {
4118
+ method: "POST",
4119
+ headers: searchHeaders,
4120
+ body: JSON.stringify({ query: q, ...(normalizedFolder ? { folder_path: normalizedFolder } : {}), limit: 20 }),
4121
+ signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
4122
+ });
4123
+ const searchText = await searchResponse.text();
4124
+ let searchJson = null;
4125
+ try {
4126
+ searchJson = searchText ? JSON.parse(searchText) : null;
4127
+ }
4128
+ catch {
4129
+ searchJson = null;
4130
+ }
4131
+ if (!searchResponse.ok || !Array.isArray(searchJson?.results)) {
4132
+ return {
4133
+ action,
4134
+ explanation: explanation ?? null,
4135
+ status: searchResponse.status,
4136
+ error: `My Files search failed (${searchResponse.status}). ${(searchJson?.error ?? searchText).slice(0, 500)}`
4137
+ };
4138
+ }
4139
+ return {
4140
+ action,
4141
+ explanation: explanation ?? null,
4142
+ status: 200,
4143
+ query: q,
4144
+ folder_path: normalizedFolder,
4145
+ semantic: Boolean(searchJson.semantic),
4146
+ result_count: searchJson.results.length,
4147
+ results: searchJson.results.map((file) => ({
4148
+ file_id: file.id,
4149
+ file_name: file.fileName,
4150
+ content_type: file.contentType,
4151
+ size_bytes: file.sizeBytes,
4152
+ folder_path: file.folderPath ?? "",
4153
+ notes: file.notes ?? null,
4154
+ similarity: file.similarity ?? null,
4155
+ keyword_match: Boolean(file.keyword_match),
4156
+ view_url: file.viewUrl,
4157
+ created_at: file.createdAt
4158
+ }))
4159
+ };
4160
+ }
4012
4161
  const listResponse = await fetch(listUrl, {
4013
4162
  method: "GET",
4014
4163
  headers,
@@ -4043,6 +4192,7 @@ function createBrowseFilesTool(input) {
4043
4192
  content_type: file.contentType,
4044
4193
  size_bytes: file.sizeBytes,
4045
4194
  folder_path: file.folderPath ?? "",
4195
+ notes: file.notes ?? null,
4046
4196
  view_url: file.viewUrl,
4047
4197
  created_at: file.createdAt
4048
4198
  }))
@@ -4061,7 +4211,56 @@ function createBrowseFilesTool(input) {
4061
4211
  status: 404,
4062
4212
  error: file_id || file_name
4063
4213
  ? "No matching file in My Files. Call action=list first to copy an exact file_id."
4064
- : "read requires file_id (preferred) or file_name."
4214
+ : `${action} requires file_id (preferred) or file_name.`
4215
+ };
4216
+ }
4217
+ if (action === "annotate") {
4218
+ if (typeof notes !== "string") {
4219
+ return { action, explanation: explanation ?? null, status: 400, error: "annotate requires notes (the metadata to attach; empty string clears)." };
4220
+ }
4221
+ const annotateUrl = new URL(`${USER_PREFIX}/me/attachments/${encodeURIComponent(match.id)}`, config.PUBLIC_BASE_URL);
4222
+ const annotateHeaders = new Headers();
4223
+ annotateHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
4224
+ annotateHeaders.set("Content-Type", "application/json");
4225
+ const annotateResponse = await fetch(annotateUrl, {
4226
+ method: "PATCH",
4227
+ headers: annotateHeaders,
4228
+ body: JSON.stringify({ notes }),
4229
+ signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
4230
+ });
4231
+ const annotateText = await annotateResponse.text();
4232
+ let annotateJson = null;
4233
+ try {
4234
+ annotateJson = annotateText ? JSON.parse(annotateText) : null;
4235
+ }
4236
+ catch {
4237
+ annotateJson = null;
4238
+ }
4239
+ if (!annotateResponse.ok || !annotateJson?.attachment) {
4240
+ return {
4241
+ action,
4242
+ explanation: explanation ?? null,
4243
+ status: annotateResponse.status,
4244
+ error: `Could not annotate file (${annotateResponse.status}). ${(annotateJson?.error ?? annotateText).slice(0, 500)}`
4245
+ };
4246
+ }
4247
+ const annotated = annotateJson.attachment;
4248
+ return {
4249
+ action,
4250
+ explanation: explanation ?? null,
4251
+ status: 200,
4252
+ annotated: true,
4253
+ notes_embedded: Boolean(annotateJson.notes_embedded),
4254
+ file: {
4255
+ file_id: annotated.id,
4256
+ file_name: annotated.fileName,
4257
+ content_type: annotated.contentType,
4258
+ size_bytes: annotated.sizeBytes,
4259
+ folder_path: annotated.folderPath ?? "",
4260
+ notes: annotated.notes ?? null,
4261
+ view_url: annotated.viewUrl,
4262
+ created_at: annotated.createdAt
4263
+ }
4065
4264
  };
4066
4265
  }
4067
4266
  const fileInfo = {
@@ -4070,6 +4269,7 @@ function createBrowseFilesTool(input) {
4070
4269
  content_type: match.contentType,
4071
4270
  size_bytes: match.sizeBytes,
4072
4271
  folder_path: match.folderPath ?? "",
4272
+ notes: match.notes ?? null,
4073
4273
  view_url: match.viewUrl,
4074
4274
  created_at: match.createdAt
4075
4275
  };
@@ -4188,8 +4388,9 @@ function createEditorActionTool() {
4188
4388
  "replace_composition_html",
4189
4389
  "export_composition",
4190
4390
  "generate_layer",
4191
- "set_captions"
4192
- ]).describe("Which mutation to perform. Use generate_layer to AI-generate a video/image and auto-place it on the timeline (fill a gap or replace a scene). Use export_composition to trigger the same Export button available in the editor UI — this queues an AWS Lambda render and the finished MP4 URL will surface in the next editor_context as last_export_url. Use set_captions to lay down a full run of ANIMATED word-by-word captions (TikTok/CapCut style): pass captions[] cues (or text + start/duration to auto-page), plus caption_style; it replaces any existing animated caption layers. To restyle existing captions without changing text/timing, use set_layer_style with caption_* fields on each caption layer."),
4391
+ "set_captions",
4392
+ "set_transitions"
4393
+ ]).describe("Which mutation to perform. Use generate_layer to AI-generate a video/image and auto-place it on the timeline (fill a gap or replace a scene). Use export_composition to trigger the same Export button available in the editor UI — this queues an AWS Lambda render and the finished MP4 URL will surface in the next editor_context as last_export_url. Use set_captions to lay down a full run of ANIMATED word-by-word captions (TikTok/CapCut style): pass captions[] cues (or text + start/duration to auto-page), plus caption_style; it replaces any existing animated caption layers. To restyle existing captions without changing text/timing, use set_layer_style with caption_* fields on each caption layer. Use set_transitions to configure scene transitions across the WHOLE timeline in one call: transition = the junction preset applied at every cut (every scene clip except each track's first; 'none' clears), transition_intro = the first clip's entrance, transition_outro = the last clip's exit."),
4193
4394
  layer_key: z.string().optional().describe("Existing layer key from editor_context (required for remove/set/duplicate/split). MAY be either the element_id (data-hf-id) or the slug (data-hf-slug). For add_layer, optionally provide a stable id (starts with a letter, [A-Za-z0-9_-], <=64 chars) to reuse in later tool calls this turn."),
4194
4395
  slug: z.string().optional().describe("snake_case slug for the layer. On add_layer, seeds data-hf-slug. On set_layer_identity, sets it. Slugs give viral DNA a stable handle for the element."),
4195
4396
  note: z.string().optional().describe("Human-facing note explaining the layer's role, referenced by viral DNA. On add_layer seeds data-hf-note. On set_layer_identity, sets it. Empty string removes."),
@@ -4222,8 +4423,12 @@ function createEditorActionTool() {
4222
4423
  object_position: z.string().optional().describe("CSS object-position for image/video (e.g., center, top, bottom left)."),
4223
4424
  ken_burns: z.enum(["zoom-in", "zoom-out", "pan-left", "pan-right", "pan-up", "pan-down", "zoom-in-left", "zoom-in-right", "none"]).optional().describe("Ken Burns motion for still-image layers: a slow pan/zoom spanning the clip's full duration. Apply with set_layer_media on an existing image, or seed it on add_layer/generate_layer when kind/media_type is image. 'none' removes the effect. When the user says 'animate the images' / 'ken burns', apply it to every still image, varying presets across adjacent clips (zoom-in, pan-left, zoom-out, ...) for a documentary feel."),
4224
4425
  ken_burns_intensity: z.number().optional().describe("Ken Burns strength: extra zoom / pan travel as a fraction of the frame, 0.04-0.5 (default 0.18; ~0.1 subtle, ~0.3 dramatic). Only meaningful alongside ken_burns."),
4225
- transition: z.enum(["crossfade", "fade-black", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("Scene transition INTO this clip at its start, from the previous clip on the same track. Set it on the INCOMING clip (the one after the cut) with set_layer_media, or seed it on add_layer/generate_layer. Works on video and image scene clips; 'none' removes it. The previous butt-cut clip is automatically held on screen beneath the incoming one for the transition window — do not retime anything yourself. When the user asks for 'transitions between scenes' / 'smooth cuts', apply one to EVERY scene clip except the first, varying or matching presets to the template's energy (crossfade/fade-black = calm, slide/wipe = energetic, zoom/circle-open = punchy)."),
4226
- transition_duration: z.number().optional().describe("Transition length in seconds, 0.1-2.5 (default 0.5; ~0.3 snappy, ~1.0 slow cinematic). Only meaningful alongside transition."),
4426
+ transition: z.enum(["crossfade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "whip-left", "whip-right", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("Scene transition INTO this clip at its start, from the previous clip on the same track. Set it on the INCOMING clip (the one after the cut) with set_layer_media, or seed it on add_layer/generate_layer. On set_transitions it is the junction preset applied at EVERY cut. Works on video and image scene clips; 'none' removes it. The previous butt-cut clip is automatically held on screen beneath the incoming one for the transition window — do not retime anything yourself. Preset energy: crossfade/fade-black/fade-white/blur = calm, smoke = dreamy, slide/wipe = energetic, whip/flash = fast-paced social, zoom/circle-open = punchy."),
4427
+ transition_duration: z.number().optional().describe("Transition length in seconds, 0.1-2.5 (default 0.5; ~0.3 snappy, ~1.0 slow cinematic). Only meaningful alongside transition/transition_intro."),
4428
+ transition_out: z.enum(["fade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-close", "none"]).optional().describe("Exit transition OUT of this clip at its end (fade to black, slide away, ...). Set it with set_layer_media on the clip that should animate away, or seed it on add_layer/generate_layer. The animation plays over the clip's last transition_out_duration seconds; no retiming needed. Most cuts only need the incoming clip's transition — use transition_out for ending a video (fade to black), for clips followed by a gap, or for deliberate dip-to-black moments (outgoing fade-black + incoming fade-black)."),
4429
+ transition_out_duration: z.number().optional().describe("Exit transition length in seconds, 0.1-2.5 (default 0.5). Only meaningful alongside transition_out/transition_outro."),
4430
+ transition_intro: z.enum(["crossfade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "whip-left", "whip-right", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("set_transitions only: entrance for the very FIRST scene clip (fade in from black = fade-black). 'none' clears it."),
4431
+ transition_outro: z.enum(["fade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-close", "none"]).optional().describe("set_transitions only: exit for the very LAST scene clip (fade to black = fade-black). 'none' clears it."),
4227
4432
  caption_style: z.enum(["spotlight", "karaoke", "word-pop", "stack-up", "neon", "bounce"]).optional().describe("Animated caption preset look: spotlight (active word gets a highlight pill, bold outline text), karaoke (words fill with color as spoken), word-pop (one word at a time, punchy scale-in), stack-up (words fade/rise in and stay), neon (active word glow pulse), bounce (active word bounces). Applies animation + colors + background treatment in one go; individual caption_*/color/background/background_style fields override specifics."),
4228
4433
  caption_animation: z.enum(["highlight-pop", "karaoke-fill", "word-pop", "cumulative-reveal", "glow-pulse", "bounce-wave", "none"]).optional().describe("Word-by-word caption animation, if you want to set it directly instead of via caption_style. Works on set_captions, add_layer (kind=text), and set_layer_style. 'none' removes the animation and flattens the layer back to static text."),
4229
4434
  caption_active_color: z.string().optional().describe("CSS color the active word takes while it is spoken (karaoke fill color, highlight text color, glow color)."),
@@ -4648,7 +4853,11 @@ function rewriteHyperframesDraftCompositionHtml(html, draftPath) {
4648
4853
  }
4649
4854
  }
4650
4855
  // And the scene-transition keyframes: toggling data-transition on a clip
4651
- // must animate immediately on older stored compositions.
4856
+ // must animate immediately on older stored compositions. Compositions that
4857
+ // embed the V1 block get it swapped for the current one in place; if the
4858
+ // swap misses (drifted block), the marker check below appends V2 after it,
4859
+ // which wins by source order.
4860
+ rewritten = upgradeLegacyTransitionCss(rewritten);
4652
4861
  if (!rewritten.includes(TRANSITION_STYLE_MARKER)) {
4653
4862
  const transitionTag = `<style data-vf-transitions="true">${TRANSITION_CSS}</style>`;
4654
4863
  if (/<\/head>/i.test(rewritten)) {
@@ -4864,7 +5073,8 @@ app.get("/editor/:templateId/composition", async (c) => {
4864
5073
  compositionId: templateId,
4865
5074
  sourceUrl: template.videoUrl,
4866
5075
  durationSeconds: template.durationSeconds ?? 12,
4867
- title: template.title || `Template · ${template.sourceHost}`
5076
+ title: template.title || `Template · ${template.sourceHost}`,
5077
+ origin: template.origin
4868
5078
  }));
4869
5079
  });
4870
5080
  app.get("/help", async (c) => renderApprovedHelpPage(c));
@@ -4976,6 +5186,7 @@ function buildTemplateHomepageEntry(template) {
4976
5186
  keywords: template.keywords ?? null,
4977
5187
  summary: template.searchSummary ?? null,
4978
5188
  visibility: template.visibility,
5189
+ origin: template.origin ?? "inspiration",
4979
5190
  notes: template.notes,
4980
5191
  status: "ready"
4981
5192
  };
@@ -5128,6 +5339,7 @@ app.post("/discover/templates", async (c) => {
5128
5339
  const explicitTitle = readNonEmptyString(body.title) ?? null;
5129
5340
  const tagline = readNonEmptyString(body.tagline) ?? readNonEmptyString(body.trend_tagline) ?? null;
5130
5341
  const notes = readNonEmptyString(body.notes) ?? null;
5342
+ const origin = readProjectOrigin(body.origin);
5131
5343
  const uploadFileName = readNonEmptyString(uploadRef?.file_name) ?? null;
5132
5344
  const primitive = primitiveRegistry.get("primitive:video_ingest");
5133
5345
  if (!primitive)
@@ -5183,6 +5395,7 @@ app.post("/discover/templates", async (c) => {
5183
5395
  downloadJobId: job.id,
5184
5396
  trendTagline: tagline,
5185
5397
  visibility: "private",
5398
+ origin,
5186
5399
  notes
5187
5400
  });
5188
5401
  // Pending-card label while the ingest runs: explicit title, else the file
@@ -5340,6 +5553,7 @@ function buildPendingInspirationHomepageEntry(inspiration) {
5340
5553
  sourceType: inspiration.sourceHost || "social",
5341
5554
  durationSeconds: inspiration.durationSeconds ?? null,
5342
5555
  visibility: "private",
5556
+ origin: inspiration.origin ?? "inspiration",
5343
5557
  notes: inspiration.notes,
5344
5558
  status: inspiration.status === "failed" ? "failed" : "processing"
5345
5559
  };
@@ -5701,7 +5915,8 @@ app.get("/composition/current.html", async (c) => {
5701
5915
  compositionId,
5702
5916
  sourceUrl: template.videoUrl,
5703
5917
  durationSeconds: template.durationSeconds ?? 12,
5704
- title: template.title || `Template · ${template.sourceHost}`
5918
+ title: template.title || `Template · ${template.sourceHost}`,
5919
+ origin: template.origin
5705
5920
  });
5706
5921
  const rewritten = rewriteHyperframesDraftCompositionHtml(rawHtml, `templates/${encodeURIComponent(template.id)}`);
5707
5922
  return c.html(rewritten, 200, { "cache-control": "no-store" });
@@ -6212,7 +6427,8 @@ app.post(COMPOSITIONS_PREFIX, async (c) => {
6212
6427
  compositionId: fork.id,
6213
6428
  sourceUrl: template.videoUrl,
6214
6429
  durationSeconds: template.durationSeconds ?? 12,
6215
- title
6430
+ title,
6431
+ origin: template.origin
6216
6432
  });
6217
6433
  await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), rawHtml, "text/html; charset=utf-8");
6218
6434
  }
@@ -6351,6 +6567,144 @@ app.put(`${COMPOSITIONS_PREFIX}/:forkId/composition.html`, async (c) => {
6351
6567
  return forkAccessErrorResponse(c, error);
6352
6568
  }
6353
6569
  });
6570
+ // ONE-STEP animated captions: transcribe the fork's own narration (word-level
6571
+ // timings when an OpenAI key is saved) and write the animated caption cue
6572
+ // layers straight into the working composition.html. Runs on the caller's raw
6573
+ // provider keys (BYOK — never billed), synchronous (~5-20s of STT). The write
6574
+ // goes through the same sanitize+persist path as the editor's PUT, and is
6575
+ // deliberately NOT watch-suppressed so a local `vidfarm serve` live-morphs the
6576
+ // captions into open editor tabs. When an editor tab is open against the
6577
+ // CLOUD host, prefer the two-part flow instead (POST /api/v1/primitives/
6578
+ // audio/captions → editor_action set_captions) so the mutation flows through
6579
+ // the editor's own state and cannot be clobbered by its next auto-save.
6580
+ app.post(`${COMPOSITIONS_PREFIX}/:forkId/captions`, async (c) => {
6581
+ const customer = await getBrowserCustomer(c);
6582
+ if (!customer)
6583
+ return c.json({ ok: false, error: "Unauthorized" }, 401);
6584
+ try {
6585
+ const access = await assertForkAccess({ forkId: c.req.param("forkId"), customerId: customer.id, shareToken: readShareToken(c) }, "edit");
6586
+ const fork = access.fork;
6587
+ const html = await storage.readText(storage.compositionForkWorkingKey(fork.id, "composition.html"));
6588
+ if (!html)
6589
+ return c.json({ ok: false, error: "Composition missing" }, 404);
6590
+ const body = asRecord(await c.req.json().catch(() => ({}))) ?? {};
6591
+ const str = (key) => (typeof body[key] === "string" && body[key].trim() ? body[key].trim() : undefined);
6592
+ const num = (key) => (typeof body[key] === "number" && Number.isFinite(body[key]) ? body[key] : undefined);
6593
+ const parsed = parseHTML(html);
6594
+ const root = parsed.document.querySelector("[data-composition-id]");
6595
+ const compositionDuration = Number(root?.getAttribute("data-duration")) || 0;
6596
+ const maxWords = Math.max(1, Math.min(8, Math.round(num("max_words_per_cue") ?? 4)));
6597
+ const offsetSec = Math.max(0, num("offset_sec") ?? 0);
6598
+ let cues;
6599
+ let wordTimestampSource = "estimated";
6600
+ let sourceNote;
6601
+ const text = str("text");
6602
+ if (text) {
6603
+ const windowStart = Math.max(0, num("start") ?? 0);
6604
+ const windowDuration = num("duration") ?? Math.max(0.4, compositionDuration - windowStart);
6605
+ cues = cuesFromText(text, windowStart + offsetSec, windowDuration, { maxWordsPerCue: maxWords });
6606
+ sourceNote = "text";
6607
+ }
6608
+ else {
6609
+ // Narration source: explicit override → audio layer → backing playback
6610
+ // video → source video attr → any video layer.
6611
+ const timedNodes = Array.from(parsed.document.querySelectorAll("[data-start]"));
6612
+ const audioLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "audio" && (node.getAttribute("src") || "").trim());
6613
+ const videoLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "video" && (node.getAttribute("src") || "").trim());
6614
+ const backing = parsed.document.querySelector("[data-vf-playback-source]");
6615
+ const sourceUrl = str("audio_url") ?? str("source_url")
6616
+ ?? audioLayer?.getAttribute("src")?.trim()
6617
+ ?? backing?.getAttribute("src")?.trim()
6618
+ ?? backing?.getAttribute("data-src")?.trim()
6619
+ ?? root?.getAttribute("data-source-video")?.trim()
6620
+ ?? videoLayer?.getAttribute("src")?.trim();
6621
+ if (!sourceUrl || !/^https?:\/\//i.test(sourceUrl)) {
6622
+ return c.json({
6623
+ ok: false,
6624
+ error: "No narration to transcribe — the composition has no audio/video layer with a usable URL. Pass audio_url (or text to page a script)."
6625
+ }, 400);
6626
+ }
6627
+ const keys = await loadCustomerVisionKeys(customer.id);
6628
+ const requested = str("provider");
6629
+ // openai first: whisper-1 is the only STT with real word-level timestamps.
6630
+ const provider = ["openai", "gemini", "openrouter"].find((name) => requested ? name === requested && keys[name] : Boolean(keys[name]));
6631
+ if (!provider) {
6632
+ return c.json({
6633
+ ok: false,
6634
+ error: requested
6635
+ ? `No active ${requested} provider key is saved for this account.`
6636
+ : "Captions need an AI provider key for transcription. Add an OpenAI (best: real word timings), Gemini, or OpenRouter key in Settings."
6637
+ }, 400);
6638
+ }
6639
+ const audio = await loadSpeechAudioLocally(sourceUrl);
6640
+ if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
6641
+ return c.json({ ok: false, error: "Narration audio exceeds the ~40-minute transcription limit. Trim it or pass a shorter audio_url." }, 400);
6642
+ }
6643
+ const transcription = await transcribeSpeechWithKey({
6644
+ provider,
6645
+ model: str("model") ?? speechModelFor("stt", provider) ?? "",
6646
+ audio: audio.bytes,
6647
+ contentType: audio.contentType,
6648
+ language: str("language"),
6649
+ diarize: false,
6650
+ wordTimestamps: true,
6651
+ apiKey: keys[provider]
6652
+ });
6653
+ if (!transcription.segments.length) {
6654
+ return c.json({ ok: false, error: "Transcription returned no speech segments — the narration may be silent." }, 502);
6655
+ }
6656
+ wordTimestampSource = transcription.segments.some((segment) => segment.words?.length) ? "provider" : "estimated";
6657
+ cues = buildCaptionCues(transcription.segments, {
6658
+ maxWordsPerCue: maxWords,
6659
+ offsetSec,
6660
+ fallbackDurationSec: compositionDuration || undefined
6661
+ });
6662
+ sourceNote = sourceUrl;
6663
+ }
6664
+ if (compositionDuration > 0) {
6665
+ cues = cues
6666
+ .filter((cue) => cue.start < compositionDuration)
6667
+ .map((cue) => ({ ...cue, duration: Math.min(cue.duration, Math.max(0.1, compositionDuration - cue.start)) }));
6668
+ }
6669
+ if (!cues.length) {
6670
+ return c.json({ ok: false, error: "No caption cues produced." }, 502);
6671
+ }
6672
+ const result = insertCaptionLayers(html, cues, {
6673
+ style: str("style"),
6674
+ animation: str("animation"),
6675
+ activeColor: str("active_color"),
6676
+ highlightColor: str("highlight_color"),
6677
+ uppercase: body.uppercase === true ? true : body.uppercase === false ? false : undefined,
6678
+ color: str("color"),
6679
+ background: str("background"),
6680
+ backgroundStyle: str("background_style"),
6681
+ fontFamily: str("font_family"),
6682
+ fontSize: num("font_size"),
6683
+ track: num("track"),
6684
+ x: num("x"),
6685
+ y: num("y"),
6686
+ width: num("width"),
6687
+ height: num("height")
6688
+ });
6689
+ const { html: sanitized, removed } = sanitizeCompositionHtml(result.html);
6690
+ await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), sanitized, "text/html; charset=utf-8");
6691
+ const updated = await serverlessRecords.updateCompositionFork({ forkId: fork.id, patch: {} });
6692
+ await writeForkManifest(storage, updated ?? fork, { kind: "working" });
6693
+ return c.json({
6694
+ ok: true,
6695
+ cue_count: cues.length,
6696
+ replaced: result.replaced,
6697
+ style: result.styleId,
6698
+ word_timestamps: wordTimestampSource,
6699
+ source: sourceNote,
6700
+ layer_keys: result.layerKeys,
6701
+ sanitized: removed
6702
+ });
6703
+ }
6704
+ catch (error) {
6705
+ return forkAccessErrorResponse(c, error);
6706
+ }
6707
+ });
6354
6708
  // Same-origin live-reload stream (local `vidfarm serve` only). Broadcasts when
6355
6709
  // an on-disk working composition file changes so open editor tabs live-morph
6356
6710
  // the agent's edits. Replaces the old cross-origin dev-serve `/_events`.
@@ -8650,6 +9004,7 @@ function serializeInspiration(inspiration) {
8650
9004
  keywords: inspiration.keywords ?? null,
8651
9005
  summary: inspiration.searchSummary ?? null,
8652
9006
  visibility: inspiration.visibility ?? "public",
9007
+ origin: inspiration.origin ?? "inspiration",
8653
9008
  notes: inspiration.notes ?? null,
8654
9009
  ingester_customer_id: inspiration.customerId,
8655
9010
  error: inspiration.errorMessage,
@@ -8710,6 +9065,7 @@ async function finalizeInspirationIfReady(inspiration) {
8710
9065
  songName: ready.songName,
8711
9066
  songUrl: ready.songUrl,
8712
9067
  visibility: "private",
9068
+ origin: ready.origin ?? "inspiration",
8713
9069
  notes: ready.notes
8714
9070
  });
8715
9071
  ready = (await serverlessRecords.updateInspiration({
@@ -10808,7 +11164,8 @@ async function ensureInspirationReadyThenFork(input) {
10808
11164
  videoUrl: inspiration.videoUrl,
10809
11165
  durationSeconds: inspiration.durationSeconds,
10810
11166
  songName: inspiration.songName,
10811
- songUrl: inspiration.songUrl
11167
+ songUrl: inspiration.songUrl,
11168
+ origin: inspiration.origin ?? "inspiration"
10812
11169
  });
10813
11170
  await serverlessRecords.updateInspiration({
10814
11171
  inspirationId: inspiration.id,
@@ -10835,6 +11192,10 @@ function buildRawTikTokCompositionHtml(input) {
10835
11192
  const duration = Number.isFinite(input.durationSeconds) && input.durationSeconds > 0
10836
11193
  ? Number(input.durationSeconds.toFixed(3))
10837
11194
  : 12;
11195
+ // data-vf-origin travels with the document through fork copies; the editor
11196
+ // reads it to decide whether to auto-prompt the Decompose modal ("raw"
11197
+ // projects never prompt). Absent means "inspiration" (pre-field documents).
11198
+ const origin = input.origin === "raw" ? "raw" : "inspiration";
10838
11199
  return `<!doctype html>
10839
11200
  <html lang="en">
10840
11201
  <head>
@@ -10847,7 +11208,7 @@ function buildRawTikTokCompositionHtml(input) {
10847
11208
  </style>
10848
11209
  </head>
10849
11210
  <body>
10850
- <div data-composition-id="${escapeAttribute(input.compositionId)}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
11211
+ <div data-composition-id="${escapeAttribute(input.compositionId)}" data-vf-origin="${origin}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
10851
11212
  <video id="vf-source-video" data-hf-id="vf-source-video" data-vf-playback-source="true" data-layer-kind="video" data-start="0" data-duration="${duration}" data-track-index="0" data-label="Original video" data-timeline-label="Original video" src="${escapeAttribute(input.sourceUrl)}" muted playsinline preload="auto" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;object-fit:cover;background:#050604"></video>
10852
11213
  </div>
10853
11214
  </body>
@@ -13823,7 +14184,8 @@ const AI_PRIMITIVE_IDS = new Set([
13823
14184
  "primitive:image_inpaint",
13824
14185
  "primitive:video_generate",
13825
14186
  "primitive:tts",
13826
- "primitive:stt"
14187
+ "primitive:stt",
14188
+ "primitive:captions"
13827
14189
  ]);
13828
14190
  async function preflightAiProviderKeys(input) {
13829
14191
  if (!operationMayUseAiProvider(input)) {
@@ -14346,6 +14708,9 @@ app.post(`${PRIMITIVES_PREFIX}/audio/speech`, async (c) => createPrimitiveJob(c,
14346
14708
  app.post(`${PRIMITIVES_PREFIX}/tts`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:tts", operationName: "run" }));
14347
14709
  app.post(`${PRIMITIVES_PREFIX}/audio/transcribe`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
14348
14710
  app.post(`${PRIMITIVES_PREFIX}/stt`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
14711
+ // Animated-caption cues from narration (STT + word timings + cue paging).
14712
+ app.post(`${PRIMITIVES_PREFIX}/audio/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
14713
+ app.post(`${PRIMITIVES_PREFIX}/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
14349
14714
  app.post(`${PRIMITIVES_PREFIX}/brainstorm/hooks`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_hooks", operationName: "run" }));
14350
14715
  app.post(`${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_awareness_stages", operationName: "run" }));
14351
14716
  app.post(`${PRIMITIVES_PREFIX}/brainstorm/angles`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_angles", operationName: "run" }));
@@ -14573,6 +14938,52 @@ app.get(`${USER_PREFIX}/me/generated-media`, async (c) => {
14573
14938
  next_cursor: page.nextCursor
14574
14939
  });
14575
14940
  });
14941
+ // Embedder for My Files notes — the same BYOK seam clip search uses. Prefer the
14942
+ // provider matching the embedding model the file library was built with (from
14943
+ // the newest annotated attachment) so query vectors stay cosine-comparable; a
14944
+ // fresh library takes the first embed-capable key (gemini → openai; openrouter
14945
+ // can't embed). Null (no key) fail-softs notes search to keyword-only.
14946
+ async function buildFileNotesEmbedder(customerId) {
14947
+ const attachments = await serverlessRecords.listUserAttachments(customerId);
14948
+ const libModel = attachments.find((a) => a.notesEmbeddingModel && Array.isArray(a.notesEmbedding) && a.notesEmbedding.length > 0)?.notesEmbeddingModel ?? null;
14949
+ const providers = libModel ? [libModel.startsWith("text-embedding") ? "openai" : "gemini"] : ["gemini", "openai"];
14950
+ for (const provider of providers) {
14951
+ const apiKey = await resolveCustomerProviderKey(customerId, provider);
14952
+ if (!apiKey)
14953
+ continue;
14954
+ const client = new ClipModelClient({ provider, apiKey });
14955
+ if (client.canEmbed)
14956
+ return client;
14957
+ }
14958
+ return null;
14959
+ }
14960
+ // What we embed for a My Files entry: folder + file name + the notes, so
14961
+ // "the sprite card for our mascot" is findable from any phrasing. Fail-soft:
14962
+ // no key or an embed error still saves the notes (keyword-searchable).
14963
+ async function embedAttachmentNotes(customerId, input) {
14964
+ if (!input.notes.trim())
14965
+ return null;
14966
+ try {
14967
+ const embedder = await buildFileNotesEmbedder(customerId);
14968
+ if (!embedder?.embeddingModelId)
14969
+ return null;
14970
+ const text = [input.folderPath, input.fileName, input.notes].filter(Boolean).join("\n");
14971
+ const embedding = await embedder.embedQuery(text);
14972
+ return embedding && embedding.length > 0 ? { embedding, model: embedder.embeddingModelId } : null;
14973
+ }
14974
+ catch (error) {
14975
+ devLog("attachment_notes.embed_failed", devErrorFields(error));
14976
+ return null;
14977
+ }
14978
+ }
14979
+ function attachmentKeywordMatch(attachment, q) {
14980
+ const hay = `${attachment.fileName} ${attachment.folderPath ?? ""} ${attachment.notes ?? ""}`.toLowerCase();
14981
+ return q
14982
+ .toLowerCase()
14983
+ .split(/\s+/)
14984
+ .filter(Boolean)
14985
+ .every((term) => hay.includes(term));
14986
+ }
14576
14987
  app.get(`${USER_PREFIX}/me/attachments`, async (c) => {
14577
14988
  const customer = requireCustomer(c);
14578
14989
  const attachments = (await serverlessRecords.listUserAttachments(customer.id)).map((attachment) => serializeUserAttachment(c, attachment));
@@ -14651,6 +15062,7 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
14651
15062
  const fileName = sanitizeFileName(file.name || "upload.bin");
14652
15063
  const contentType = file.type || inferAttachmentContentType(fileName);
14653
15064
  const folderPath = sanitizeStorageSubpath(String(formData.get("folder_path") ?? ""));
15065
+ const notes = String(formData.get("notes") ?? "").trim().slice(0, 4000) || null;
14654
15066
  if (!isAllowedAttachment(fileName, contentType)) {
14655
15067
  return c.json({ error: `Unsupported attachment type: ${fileName}` }, 400);
14656
15068
  }
@@ -14662,6 +15074,7 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
14662
15074
  const attachmentId = randomUUID();
14663
15075
  const storageKey = storage.userAttachmentKey(customer.id, attachmentId, fileName, folderPath);
14664
15076
  const stored = await storage.putBuffer(storageKey, buffer, contentType);
15077
+ const embedded = notes ? await embedAttachmentNotes(customer.id, { fileName, folderPath, notes }) : null;
14665
15078
  await serverlessRecords.createUserAttachment({
14666
15079
  id: attachmentId,
14667
15080
  customerId: customer.id,
@@ -14670,7 +15083,10 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
14670
15083
  sizeBytes: buffer.byteLength,
14671
15084
  storageKey,
14672
15085
  folderPath,
14673
- publicUrl: stored.url || storage.getPublicUrl(storageKey) || buildUserAttachmentViewUrl(c, storageKey)
15086
+ publicUrl: stored.url || storage.getPublicUrl(storageKey) || buildUserAttachmentViewUrl(c, storageKey),
15087
+ notes,
15088
+ notesEmbedding: embedded?.embedding ?? null,
15089
+ notesEmbeddingModel: embedded?.model ?? null
14674
15090
  });
14675
15091
  const attachment = await serverlessRecords.getUserAttachment(customer.id, attachmentId);
14676
15092
  if (!attachment) {
@@ -14699,6 +15115,8 @@ app.post(`${USER_PREFIX}/me/attachments`, async (c) => {
14699
15115
  if (!isAllowedAttachment(fileName, contentType)) {
14700
15116
  return c.json({ error: `Unsupported attachment type: ${fileName}` }, 400);
14701
15117
  }
15118
+ const notes = body.notes?.trim() || null;
15119
+ const embedded = notes ? await embedAttachmentNotes(customer.id, { fileName, folderPath, notes }) : null;
14702
15120
  await serverlessRecords.createUserAttachment({
14703
15121
  id: body.attachment_id,
14704
15122
  customerId: customer.id,
@@ -14707,7 +15125,10 @@ app.post(`${USER_PREFIX}/me/attachments`, async (c) => {
14707
15125
  sizeBytes: body.size_bytes,
14708
15126
  storageKey: expectedStorageKey,
14709
15127
  folderPath,
14710
- publicUrl: storage.getPublicUrl(expectedStorageKey) || buildUserAttachmentViewUrl(c, expectedStorageKey)
15128
+ publicUrl: storage.getPublicUrl(expectedStorageKey) || buildUserAttachmentViewUrl(c, expectedStorageKey),
15129
+ notes,
15130
+ notesEmbedding: embedded?.embedding ?? null,
15131
+ notesEmbeddingModel: embedded?.model ?? null
14711
15132
  });
14712
15133
  const attachment = await serverlessRecords.getUserAttachment(customer.id, body.attachment_id);
14713
15134
  if (!attachment) {
@@ -14741,6 +15162,89 @@ app.patch(`${USER_PREFIX}/me/attachments/folders`, async (c) => {
14741
15162
  await serverlessRecords.moveUserAttachmentFolder(customer.id, fromFolder, toFolder);
14742
15163
  return c.json({ ok: true, from_folder_path: fromFolder, to_folder_path: toFolder });
14743
15164
  });
15165
+ // PATCH /me/attachments/:attachmentId — set/clear the metadata notes on one My
15166
+ // Files entry and (re)embed them for vector search. Notes are what make assets
15167
+ // findable by MEANING later — e.g. annotate a character_sprite_card.png with
15168
+ // who the character is so /me/attachments/search finds it from any phrasing.
15169
+ // (Registered after the /folders PATCH so "folders" never matches as an id.)
15170
+ app.patch(`${USER_PREFIX}/me/attachments/:attachmentId`, async (c) => {
15171
+ const customer = requireCustomer(c);
15172
+ try {
15173
+ const body = userAttachmentNotesSchema.parse(await c.req.json());
15174
+ const attachment = await serverlessRecords.getUserAttachment(customer.id, c.req.param("attachmentId"));
15175
+ if (!attachment) {
15176
+ return c.json({ error: "Attachment not found." }, 404);
15177
+ }
15178
+ const notes = (body.notes ?? "").trim() || null;
15179
+ const embedded = notes
15180
+ ? await embedAttachmentNotes(customer.id, { fileName: attachment.fileName, folderPath: attachment.folderPath, notes })
15181
+ : null;
15182
+ const updated = await serverlessRecords.updateUserAttachmentNotes(customer.id, attachment.id, {
15183
+ notes,
15184
+ notesEmbedding: embedded?.embedding ?? null,
15185
+ notesEmbeddingModel: embedded?.model ?? null
15186
+ });
15187
+ if (!updated) {
15188
+ return c.json({ error: "Attachment not found." }, 404);
15189
+ }
15190
+ return c.json({ ok: true, attachment: serializeUserAttachment(c, updated), notes_embedded: Boolean(embedded) });
15191
+ }
15192
+ catch (error) {
15193
+ return c.json({ error: error instanceof Error ? error.message : "Unable to update attachment notes." }, 400);
15194
+ }
15195
+ });
15196
+ // POST /me/attachments/search — find My Files entries by meaning, mirroring
15197
+ // /clips/search: keyword filter over name/folder/notes, semantically re-ranked
15198
+ // via the notes embeddings; when nothing keyword-matches, relax to a pure
15199
+ // semantic pass. Embeddings are BYOK — no saved key fail-softs to keyword-only.
15200
+ app.post(`${USER_PREFIX}/me/attachments/search`, async (c) => {
15201
+ const customer = requireCustomer(c);
15202
+ try {
15203
+ const body = userAttachmentSearchSchema.parse(await c.req.json());
15204
+ const query = body.query.trim();
15205
+ if (!query) {
15206
+ return c.json({ error: "Provide a search query." }, 400);
15207
+ }
15208
+ const folderPath = sanitizeStorageSubpath(body.folder_path);
15209
+ const limit = Math.max(1, Math.min(100, body.limit ?? 20));
15210
+ const all = await serverlessRecords.listUserAttachments(customer.id);
15211
+ const scoped = folderPath ? all.filter((a) => (a.folderPath ?? "") === folderPath) : all;
15212
+ let queryEmbedding;
15213
+ try {
15214
+ const embedder = await buildFileNotesEmbedder(customer.id);
15215
+ if (embedder)
15216
+ queryEmbedding = await embedder.embedQuery(query);
15217
+ }
15218
+ catch (error) {
15219
+ devLog("attachment_search.embed_failed", devErrorFields(error));
15220
+ }
15221
+ const keywordIds = new Set(scoped.filter((a) => attachmentKeywordMatch(a, query)).map((a) => a.id));
15222
+ const pool = keywordIds.size > 0 ? scoped.filter((a) => keywordIds.has(a.id)) : scoped;
15223
+ const hits = pool
15224
+ .map((a) => ({
15225
+ attachment: a,
15226
+ keyword: keywordIds.has(a.id),
15227
+ similarity: queryEmbedding && Array.isArray(a.notesEmbedding) && a.notesEmbedding.length > 0
15228
+ ? cosineSimilarity(a.notesEmbedding, queryEmbedding)
15229
+ : undefined
15230
+ }))
15231
+ .filter((h) => h.keyword || (h.similarity ?? 0) > 0)
15232
+ .sort((x, y) => ((y.similarity ?? 0) + (y.keyword ? 0.2 : 0)) - ((x.similarity ?? 0) + (x.keyword ? 0.2 : 0)))
15233
+ .slice(0, limit);
15234
+ return c.json({
15235
+ results: hits.map((h) => ({
15236
+ ...serializeUserAttachment(c, h.attachment),
15237
+ similarity: h.similarity ?? null,
15238
+ keyword_match: h.keyword
15239
+ })),
15240
+ semantic: Boolean(queryEmbedding),
15241
+ folder_path: folderPath || null
15242
+ });
15243
+ }
15244
+ catch (error) {
15245
+ return c.json({ error: error instanceof Error ? error.message : "My Files search failed." }, 400);
15246
+ }
15247
+ });
14744
15248
  app.delete(`${USER_PREFIX}/me/attachments/folders`, async (c) => {
14745
15249
  const customer = requireCustomer(c);
14746
15250
  const body = userFileFolderSchema.parse(await c.req.json());
@@ -15312,25 +15816,39 @@ function renderClipsPage(input) {
15312
15816
  .clips-import-head p { font-size: 0.9rem; }
15313
15817
  .clips-import-field { display: grid; gap: 6px; }
15314
15818
  .clips-import-field textarea { min-height: 96px; border-radius: 18px; }
15819
+ .clips-import-field select { width: 100%; }
15820
+ .clips-import-hint { font-size: 0.78rem; font-weight: 400; color: var(--muted); }
15821
+ .clips-import-hint a { color: inherit; text-decoration: underline; }
15315
15822
  .clips-import-note { min-height: 18px; font-size: 0.85rem; color: var(--muted); }
15316
15823
  .clips-import-note[data-tone="error"] { color: #9f3d3d; }
15317
15824
  .clips-import-note[data-tone="success"] { color: #3f7d55; }
15318
15825
  .clips-import-actions { display: flex; justify-content: flex-end; gap: 10px; }
15319
15826
 
15320
- @media (max-width: 1180px) {
15827
+ @media (min-width: 1025px) and (max-width: 1180px) {
15828
+ /* Rail stacks inline below the content in this band only; at ≤1024px
15829
+ the shared shell turns the rail into the FAB + popup overlay. */
15321
15830
  .clips-frame { padding-left: 0; }
15322
15831
  .editor-right-rail { position: relative; inset: auto; width: 100%; height: 540px; margin-top: 14px; border-radius: 24px; overflow: hidden; }
15323
15832
  .editor-chat-panel { border-right: 0; border-radius: 24px; }
15324
- }
15325
- @media (min-width: 1025px) and (max-width: 1180px) {
15326
15833
  html, body { height: auto; min-height: 100%; overflow: auto; overflow-x: hidden; }
15327
15834
  .app-shell { height: auto; min-height: 100svh; }
15328
15835
  .page-stack, .frame { height: auto; min-height: 0; overflow: visible; }
15329
15836
  .clips-scroll { overflow: visible; padding-right: 0; }
15330
15837
  }
15331
15838
  @media (max-width: 1024px) {
15839
+ .clips-frame { padding-left: 0; }
15332
15840
  .clips-shell { grid-template-rows: auto auto; }
15333
15841
  .clips-scroll { overflow: visible; padding-right: 0; }
15842
+ .clips-toolbar-row { gap: 8px; }
15843
+ .clips-search { flex: 1 1 100%; min-width: 0; }
15844
+ .clips-search input { min-width: 0; font-size: 16px; }
15845
+ .clips-import-form input,
15846
+ .clips-import-form textarea,
15847
+ .clips-import-form select { font-size: 16px; }
15848
+ }
15849
+ @media (max-width: 560px) {
15850
+ .clips-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
15851
+ .clip-body { padding: 10px; }
15334
15852
  }
15335
15853
  `,
15336
15854
  body: `
@@ -15379,6 +15897,20 @@ function renderClipsPage(input) {
15379
15897
  <label for="importPrompt">What to clip for</label>
15380
15898
  <textarea id="importPrompt" name="prompt" placeholder="Clip scenes of people with food, no text on screen, vertical, between 5-10 secs long"></textarea>
15381
15899
  </div>
15900
+ <div class="clips-import-field">
15901
+ <label for="importProvider">AI provider</label>
15902
+ <select id="importProvider" name="provider">
15903
+ <option value="">Auto — use my saved keys</option>
15904
+ <option value="gemini">Gemini</option>
15905
+ <option value="openai">OpenAI</option>
15906
+ <option value="openrouter">OpenRouter</option>
15907
+ </select>
15908
+ <div class="clips-import-hint" id="importProviderHint"></div>
15909
+ </div>
15910
+ <div class="clips-import-field">
15911
+ <label for="importTracer">Tracer <span class="clips-import-hint">optional — rolls up this hunt's billing and clips under one id</span></label>
15912
+ <input id="importTracer" name="tracer" type="text" placeholder="my-campaign-001" autocomplete="off" spellcheck="false" />
15913
+ </div>
15382
15914
  <div class="clips-import-note" id="importNote"></div>
15383
15915
  <div class="clips-import-actions">
15384
15916
  <button type="button" class="secondary" id="importCancel">Cancel</button>
@@ -15468,18 +16000,58 @@ function renderClipsPage(input) {
15468
16000
  var importForm = document.getElementById('importForm');
15469
16001
  var importUrl = document.getElementById('importUrl');
15470
16002
  var importPrompt = document.getElementById('importPrompt');
16003
+ var importTracer = document.getElementById('importTracer');
16004
+ var importProvider = document.getElementById('importProvider');
16005
+ var importProviderHint = document.getElementById('importProviderHint');
15471
16006
  var importNote = document.getElementById('importNote');
15472
16007
  var importCancel = document.getElementById('importCancel');
15473
16008
  var importSubmit = document.getElementById('importSubmit');
15474
16009
  var scanPoll = null;
16010
+ var scanOptions = null;
15475
16011
 
15476
16012
  function setNote(msg, tone){
15477
16013
  if(!importNote) return;
15478
16014
  importNote.textContent = msg || '';
15479
16015
  if(tone) importNote.setAttribute('data-tone', tone); else importNote.removeAttribute('data-tone');
15480
16016
  }
16017
+ function providerName(p){ return p==='gemini'?'Gemini':p==='openai'?'OpenAI':p==='openrouter'?'OpenRouter':p; }
16018
+ function renderProviderOptions(){
16019
+ if(!importProvider || !scanOptions) return;
16020
+ var d = scanOptions;
16021
+ var hasAnyKey = (d.providers||[]).some(function(p){ return p.has_key; });
16022
+ var autoLabel = 'Auto';
16023
+ if(d.default_provider === 'agent') autoLabel = 'Auto — local '+((d.local_agent&&d.local_agent.kind)||'agent')+' CLI (no API key)';
16024
+ else if(d.default_provider) autoLabel = 'Auto — saved '+providerName(d.default_provider)+' key';
16025
+ else if(d.pipeline === 'upstream') autoLabel = 'Auto — linked cloud account';
16026
+ var html = '<option value="">'+esc(autoLabel)+'</option>';
16027
+ if(d.local_agent) html += '<option value="agent">Local agent ('+esc(d.local_agent.kind)+') — your CLI subscription, no API key</option>';
16028
+ (d.providers||[]).forEach(function(p){
16029
+ html += '<option value="'+esc(p.provider)+'"'+(p.has_key?'':' disabled')+'>'+esc(providerName(p.provider))+(p.has_key?' — key saved':' — no key saved')+'</option>';
16030
+ });
16031
+ importProvider.innerHTML = html;
16032
+ if(!importProviderHint) return;
16033
+ if(!hasAnyKey && !d.local_agent && d.pipeline !== 'upstream'){
16034
+ importProviderHint.innerHTML = 'No AI provider available yet — add a Gemini, OpenAI, or OpenRouter key in <a href="/settings" target="_blank" rel="noopener">Settings</a>'
16035
+ + (d.pipeline === 'cloud' ? '.' : ', or install the Claude Code / Codex CLI for keyless local scans.');
16036
+ } else if(d.pipeline === 'upstream'){
16037
+ importProviderHint.textContent = 'No local key or agent CLI — this hunt will run on your linked cloud account (its saved keys, cloud compute).';
16038
+ } else if(d.pipeline === 'local'){
16039
+ importProviderHint.textContent = d.default_provider === 'agent'
16040
+ ? 'Runs on this machine with your local agent CLI — no API key needed. Or pick a saved key.'
16041
+ : 'Runs on this machine with your saved key (BYOK — billed to your own provider account).';
16042
+ } else {
16043
+ importProviderHint.textContent = 'AI tagging runs on your own key (BYOK) — pick a provider or let Auto choose.';
16044
+ }
16045
+ }
16046
+ function loadScanOptions(){
16047
+ fetch('/clips/scan-options', {credentials:'same-origin', headers:{Accept:'application/json'}})
16048
+ .then(function(r){ return r.ok ? r.json() : null; })
16049
+ .then(function(d){ if(d){ scanOptions = d; renderProviderOptions(); } })
16050
+ .catch(function(){});
16051
+ }
15481
16052
  function openImport(){
15482
16053
  setNote('');
16054
+ if(!scanOptions) loadScanOptions();
15483
16055
  if(importDialog && typeof importDialog.showModal === 'function' && !importDialog.open) importDialog.showModal();
15484
16056
  if(importUrl) importUrl.focus();
15485
16057
  }
@@ -15522,16 +16094,21 @@ function renderClipsPage(input) {
15522
16094
  e.preventDefault();
15523
16095
  var url = (importUrl && importUrl.value || '').trim();
15524
16096
  var prompt = (importPrompt && importPrompt.value || '').trim();
16097
+ var tracer = (importTracer && importTracer.value || '').trim();
16098
+ var provider = (importProvider && importProvider.value || '').trim();
15525
16099
  if(!url){ setNote('Enter a video URL.', 'error'); return; }
15526
16100
  setNote('Starting scan…');
15527
16101
  if(importSubmit) importSubmit.disabled = true;
15528
- fetch('/clips/scan', {method:'POST', credentials:'same-origin', headers:{'Content-Type':'application/json', Accept:'application/json'}, body: JSON.stringify({source_url:url, prompt:prompt})})
16102
+ var payload = {source_url:url, prompt:prompt};
16103
+ if(tracer) payload.tracer = tracer;
16104
+ if(provider) payload.provider = provider;
16105
+ fetch('/clips/scan', {method:'POST', credentials:'same-origin', headers:{'Content-Type':'application/json', Accept:'application/json'}, body: JSON.stringify(payload)})
15529
16106
  .then(function(r){ return r.json().then(function(d){ return {ok:r.ok, status:r.status, data:d}; }); })
15530
16107
  .then(function(res){
15531
16108
  if(importSubmit) importSubmit.disabled = false;
15532
16109
  if(!res.ok){ setNote((res.data && res.data.error) || 'Could not start the scan.', 'error'); return; }
15533
16110
  setNote('Scan started — this can take a few minutes.', 'success');
15534
- statusEl.textContent='Scanning source…';
16111
+ statusEl.textContent='Scanning source…'+(res.data && res.data.tracer ? ' (tracer '+res.data.tracer+')' : '');
15535
16112
  closeImport();
15536
16113
  if(importForm.reset) importForm.reset();
15537
16114
  if(res.data && res.data.scan_id) pollScan(res.data.scan_id);
@@ -15559,9 +16136,23 @@ async function resolveCustomerProviderKey(customerId, provider) {
15559
16136
  }
15560
16137
  return null;
15561
16138
  }
15562
- // Normalize a request-supplied provider to the supported set (default gemini).
15563
- function normalizeClipProvider(value) {
15564
- return value === "openai" || value === "openrouter" ? value : "gemini";
16139
+ // The BYOK API providers clip hunts can run on, in auto-pick preference order.
16140
+ const CLIP_API_PROVIDERS = ["gemini", "openai", "openrouter"];
16141
+ const NO_CLIP_PROVIDER_KEY_ERROR = "No AI provider key saved. Add a gemini, openai, or openrouter key (Settings AI Provider Keys, or POST /api/v1/user/me/provider-keys) and retry.";
16142
+ // An explicitly requested provider — "agent" means the caller's local
16143
+ // claude/codex CLI (vidfarm serve boxes only) — or null for auto-pick.
16144
+ function readClipProviderChoice(value) {
16145
+ if (value === "gemini" || value === "openai" || value === "openrouter")
16146
+ return value;
16147
+ if (value === "agent" || value === "local" || value === "local-agent")
16148
+ return "agent";
16149
+ return null;
16150
+ }
16151
+ // Auto-pick: the first API provider the customer holds an active key for
16152
+ // (gemini → openai → openrouter) — no more hard gemini default.
16153
+ async function pickClipProviderFromKeys(customerId) {
16154
+ const keys = await loadCustomerVisionKeys(customerId);
16155
+ return CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
15565
16156
  }
15566
16157
  /**
15567
16158
  * A client for embedding MATCH queries in the SAME vector space the library was
@@ -15617,7 +16208,8 @@ function parseClipStorageKey(pathOrUri) {
15617
16208
  async function clipMediaUrl(pathOrUri) {
15618
16209
  if (!pathOrUri)
15619
16210
  return null;
15620
- const key = parseClipStorageKey(pathOrUri);
16211
+ // s3://… (cloud hunts) or a bare clips/thumbs storage key (serve-box local hunts).
16212
+ const key = parseClipStorageKey(pathOrUri) ?? (/^(clips|thumbs)\//.test(pathOrUri) ? pathOrUri : null);
15621
16213
  if (key)
15622
16214
  return (await storage.getReadUrl(key)) ?? storage.getPublicUrl(key);
15623
16215
  return pathOrUri; // non-s3 value (e.g. a local devcli-origin path) — pass through
@@ -15677,11 +16269,14 @@ app.post(`${CLIPS_PREFIX}/search`, async (c) => {
15677
16269
  try {
15678
16270
  const body = (await c.req.json().catch(() => ({})));
15679
16271
  const limit = Math.max(1, Math.min(200, Number(body.limit) || 20));
15680
- const provider = normalizeClipProvider(body.provider);
16272
+ const requested = readClipProviderChoice(body.provider);
15681
16273
  let criteria = body.criteria ?? (body.query ? { semantic_text: body.query } : {});
15682
16274
  let queryEmbedding;
15683
16275
  if (body.query || criteria.semantic_text) {
15684
- const built = await buildCustomerClipClient(customer.id, provider);
16276
+ // Any saved key works for NL→criteria + query embedding; no key at all
16277
+ // fail-softs to the structured/keyword pass below.
16278
+ const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
16279
+ const built = provider ? await buildCustomerClipClient(customer.id, provider) : { error: NO_CLIP_PROVIDER_KEY_ERROR };
15685
16280
  if ("client" in built) {
15686
16281
  const client = built.client;
15687
16282
  if (body.query && !body.criteria)
@@ -15837,72 +16432,106 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
15837
16432
  }
15838
16433
  const rawS3Uri = body.raw_s3_uri ??
15839
16434
  (resolvedS3Key && config.AWS_S3_BUCKET ? `s3://${config.AWS_S3_BUCKET}/${resolvedS3Key}` : undefined);
15840
- if (!rawS3Uri && !huntSpec.source_url) {
16435
+ // On a serve box (no bucket) resolvedS3Key is a LOCAL storage key — still a valid source.
16436
+ if (!rawS3Uri && !resolvedS3Key && !huntSpec.source_url) {
15841
16437
  return c.json({ error: "Provide source_url (a video URL), temp_file_id/attachment_id of an upload, raw_s3_uri (s3://…), or s3_key of the staged source video." }, 400);
15842
16438
  }
15843
- // Hunts bill AWS compute post-hoc, so submission is the spend gate (same
15844
- // policy as job submission).
15845
- try {
15846
- await assertWalletCanQueueJobs(customer.id);
15847
- }
15848
- catch (error) {
15849
- if (error instanceof InsufficientWalletFundsError) {
15850
- return insufficientFundsResponse(c, error);
15851
- }
15852
- throw error;
15853
- }
15854
- const provider = normalizeClipProvider(body.provider);
15855
- const built = await buildCustomerClipClient(customer.id, provider);
15856
- if ("error" in built)
15857
- return c.json({ error: built.error }, 400);
15858
- const { client, apiKey: tagApiKey, embeddingKey } = built;
16439
+ const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
16440
+ const choice = readClipProviderChoice(body.provider);
15859
16441
  const tier = normalizeScanTier(body.tier);
15860
16442
  const framesPerScene = Math.max(1, Math.min(3, Number(body.frames_per_scene) || 2));
15861
- const includeAudio = body.include_audio !== false && provider === "gemini";
15862
16443
  const tracer = readNonEmptyString(body.tracer) ?? `clips_import:${customer.id}`;
15863
16444
  const now = nowIso();
15864
16445
  const scanId = createIdV7("clipscan");
15865
16446
  const sourceVideoId = createIdV7("srcvid");
15866
- // The URL ingest is deferred to the Lambda, so raw_s3_uri may be empty here.
15867
- const sourcePointer = rawS3Uri ?? huntSpec.source_url ?? "";
16447
+ // The URL ingest is deferred to the runner, so raw_s3_uri may be empty here.
16448
+ const sourcePointer = rawS3Uri ?? resolvedS3Key ?? huntSpec.source_url ?? "";
15868
16449
  const filename = body.filename || (sourcePointer.split("/").pop() || "source.mp4").split("?")[0] || "source.mp4";
15869
- // Time-range hunts only pay for the windows: both estimates use the
15870
- // EFFECTIVE (windowed) duration, and the duration band drives the expected
15871
- // scene size. The BYOK estimate is the user's own provider spend; the
15872
- // compute estimate is what the wallet will be billed (AWS only, marked up).
15873
- const effectiveDurationSec = body.duration_sec && body.duration_sec > 0
15874
- ? effectiveHuntDurationSec(body.duration_sec, huntSpec.windows)
15875
- : null;
15876
- const estimate = effectiveDurationSec
15877
- ? estimateScanCostFromDuration(effectiveDurationSec, {
16450
+ const putScanRecords = async (input) => {
16451
+ await clipRecords.putSource({
16452
+ source_video_id: sourceVideoId,
16453
+ owner_id: customer.id,
16454
+ filename,
16455
+ raw_s3_uri: sourcePointer,
16456
+ duration_sec: body.duration_sec,
16457
+ clip_count: 0,
15878
16458
  tier,
15879
- provider,
15880
- tagModelId: client.tagModelId,
15881
- embedModelId: client.embeddingModelId,
15882
- framesPerScene,
15883
- includeAudio,
15884
- avgSceneSec: huntSpec.duration_band?.target_sec
15885
- })
15886
- : null;
15887
- const computeEstimate = effectiveDurationSec
15888
- ? estimateClipScanComputeUsd(effectiveDurationSec, huntSpec.duration_band?.target_sec)
15889
- : null;
15890
- const source = {
15891
- source_video_id: sourceVideoId,
15892
- owner_id: customer.id,
15893
- filename,
15894
- raw_s3_uri: sourcePointer,
15895
- duration_sec: body.duration_sec,
15896
- clip_count: 0,
15897
- tier,
15898
- status: "pending",
15899
- scan_id: scanId,
15900
- created_at: now,
15901
- updated_at: now
16459
+ status: "pending",
16460
+ scan_id: scanId,
16461
+ tracer,
16462
+ created_at: now,
16463
+ updated_at: now
16464
+ });
16465
+ await clipRecords.putScan({
16466
+ scan_id: scanId,
16467
+ owner_id: customer.id,
16468
+ source_video_id: sourceVideoId,
16469
+ status: input.scanStatus,
16470
+ tier,
16471
+ raw_s3_uri: sourcePointer,
16472
+ execution_arn: input.executionArn,
16473
+ tracer,
16474
+ created_at: now,
16475
+ updated_at: now
16476
+ });
15902
16477
  };
15903
- await clipRecords.putSource(source);
15904
- let executionArn;
15905
- if (config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN) {
16478
+ const scanResponse = (input) => c.json({
16479
+ scan_id: scanId,
16480
+ source_video_id: sourceVideoId,
16481
+ status: input.status,
16482
+ execution_arn: input.executionArn ?? null,
16483
+ tracer,
16484
+ hunt_spec: huntSpec,
16485
+ estimate: input.estimate ?? null,
16486
+ compute_estimate: input.computeEstimate ?? null,
16487
+ pipeline: input.pipeline,
16488
+ pipeline_deployed: cloudPipeline
16489
+ }, 202);
16490
+ if (cloudPipeline) {
16491
+ // ── Deployed pipeline (Lambda + Step Functions): BYOK API keys only ──
16492
+ if (choice === "agent") {
16493
+ return c.json({ error: "provider \"agent\" (your local claude/codex CLI) only runs on a local `vidfarm serve` box — the deployed pipeline needs a saved gemini, openai, or openrouter key." }, 400);
16494
+ }
16495
+ // Hunts bill AWS compute post-hoc, so submission is the spend gate (same
16496
+ // policy as job submission).
16497
+ try {
16498
+ await assertWalletCanQueueJobs(customer.id);
16499
+ }
16500
+ catch (error) {
16501
+ if (error instanceof InsufficientWalletFundsError) {
16502
+ return insufficientFundsResponse(c, error);
16503
+ }
16504
+ throw error;
16505
+ }
16506
+ const provider = choice ?? (await pickClipProviderFromKeys(customer.id));
16507
+ if (!provider)
16508
+ return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
16509
+ const built = await buildCustomerClipClient(customer.id, provider);
16510
+ if ("error" in built)
16511
+ return c.json({ error: built.error }, 400);
16512
+ const { client, apiKey: tagApiKey, embeddingKey } = built;
16513
+ const includeAudio = body.include_audio !== false && provider === "gemini";
16514
+ // Time-range hunts only pay for the windows: both estimates use the
16515
+ // EFFECTIVE (windowed) duration, and the duration band drives the expected
16516
+ // scene size. The BYOK estimate is the user's own provider spend; the
16517
+ // compute estimate is what the wallet will be billed (AWS only, marked up).
16518
+ const effectiveDurationSec = body.duration_sec && body.duration_sec > 0
16519
+ ? effectiveHuntDurationSec(body.duration_sec, huntSpec.windows)
16520
+ : null;
16521
+ const estimate = effectiveDurationSec
16522
+ ? estimateScanCostFromDuration(effectiveDurationSec, {
16523
+ tier,
16524
+ provider,
16525
+ tagModelId: client.tagModelId,
16526
+ embedModelId: client.embeddingModelId,
16527
+ framesPerScene,
16528
+ includeAudio,
16529
+ avgSceneSec: huntSpec.duration_band?.target_sec
16530
+ })
16531
+ : null;
16532
+ const computeEstimate = effectiveDurationSec
16533
+ ? estimateClipScanComputeUsd(effectiveDurationSec, huntSpec.duration_band?.target_sec)
16534
+ : null;
15906
16535
  const res = await clipScanSfn.send(new StartExecutionCommand({
15907
16536
  stateMachineArn: config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN,
15908
16537
  name: scanId.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
@@ -15924,43 +16553,358 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
15924
16553
  tracer
15925
16554
  })
15926
16555
  }));
15927
- executionArn = res.executionArn;
16556
+ await putScanRecords({ scanStatus: "running", executionArn: res.executionArn });
16557
+ return scanResponse({ status: "running", executionArn: res.executionArn, pipeline: "cloud", estimate, computeEstimate });
16558
+ }
16559
+ // ── vidfarm serve (no deployed pipeline): LOCAL-FIRST, cloud as backup ──
16560
+ // Same policy as `vidfarm clips scan`: the user's local claude/codex CLI
16561
+ // subscription evaluates scenes when installed, their saved BYOK keys are
16562
+ // the fallback, and the upstream cloud REST API is the explicit backup.
16563
+ // Local hunts are free (no wallet gate), like local renders.
16564
+ let localSourcePath = null;
16565
+ if (resolvedS3Key) {
16566
+ localSourcePath = storage.getLocalPath(resolvedS3Key);
16567
+ if (!localSourcePath || !existsSync(localSourcePath)) {
16568
+ return c.json({ error: "That upload isn't on this box's local storage — re-upload it or pass source_url." }, 400);
16569
+ }
16570
+ }
16571
+ else if (body.raw_s3_uri) {
16572
+ return c.json({ error: "raw_s3_uri isn't supported on a local box — pass source_url, temp_file_id, or attachment_id." }, 400);
16573
+ }
16574
+ const localAgent = choice === "agent" || choice === null ? detectLocalAgent() : null;
16575
+ if (choice === "agent" && !localAgent) {
16576
+ return c.json({ error: "No local agent CLI found — install Claude Code (`claude`) or Codex (`codex`), or pick gemini/openai/openrouter." }, 400);
16577
+ }
16578
+ let evaluator = null;
16579
+ let evaluatorProvider = null;
16580
+ let evaluatorError = null;
16581
+ if (localAgent) {
16582
+ const keys = await loadCustomerVisionKeys(customer.id);
16583
+ const embedding = keys.gemini
16584
+ ? { provider: "gemini", apiKey: keys.gemini }
16585
+ : keys.openai
16586
+ ? { provider: "openai", apiKey: keys.openai }
16587
+ : null;
16588
+ evaluator = new LocalAgentClipClient({ agent: localAgent.kind, bin: localAgent.bin, embedding, tier });
16589
+ evaluatorProvider = `agent/${localAgent.kind}`;
16590
+ }
16591
+ else {
16592
+ const provider = (choice !== "agent" ? choice : null) ?? (await pickClipProviderFromKeys(customer.id));
16593
+ if (provider) {
16594
+ const built = await buildCustomerClipClient(customer.id, provider);
16595
+ if ("error" in built) {
16596
+ if (choice)
16597
+ return c.json({ error: built.error }, 400);
16598
+ evaluatorError = built.error;
16599
+ }
16600
+ else {
16601
+ evaluator = built.client;
16602
+ evaluatorProvider = provider;
16603
+ }
16604
+ }
16605
+ else {
16606
+ evaluatorError = NO_CLIP_PROVIDER_KEY_ERROR;
16607
+ }
16608
+ }
16609
+ const ffmpegOk = await hasFfmpeg();
16610
+ const canIngestUrl = !huntSpec.source_url ||
16611
+ Boolean((config.RAPIDAPI_KEY || "").trim()) ||
16612
+ looksLikeDirectVideoUrl(huntSpec.source_url);
16613
+ if (evaluator && ffmpegOk && (localSourcePath || canIngestUrl)) {
16614
+ const includeAudio = body.include_audio !== false && evaluatorProvider === "gemini";
16615
+ await putScanRecords({ scanStatus: "running" });
16616
+ void runClipScanInProcess({
16617
+ scanId,
16618
+ ownerId: customer.id,
16619
+ sourceVideoId,
16620
+ filename,
16621
+ client: evaluator,
16622
+ localSourcePath,
16623
+ sourceUrl: huntSpec.source_url ?? null,
16624
+ huntSpec,
16625
+ guidancePrompt,
16626
+ framesPerScene,
16627
+ includeAudio
16628
+ }).catch((error) => {
16629
+ devLog("clip_scan.local_run_failed", { scan_id: scanId, ...devErrorFields(error) }, "error");
16630
+ });
16631
+ return scanResponse({ status: "running", pipeline: "local" });
16632
+ }
16633
+ // BACKUP: hand the hunt to the upstream cloud deployment (its saved BYOK
16634
+ // keys run the AI; its account is billed the AWS compute).
16635
+ if (hasUpstreamKey() && huntSpec.source_url) {
16636
+ const upstream = await upstreamJson("/clips/scan", {
16637
+ method: "POST",
16638
+ body: {
16639
+ source_url: huntSpec.source_url,
16640
+ prompt: guidancePrompt,
16641
+ hunt_spec: huntSpec,
16642
+ tracer,
16643
+ ...(choice && choice !== "agent" ? { provider: choice } : {})
16644
+ },
16645
+ timeoutMs: 30_000
16646
+ });
16647
+ if (!upstream.ok || !upstream.json?.scan_id) {
16648
+ return c.json({ error: upstream.json?.error ?? `Upstream cloud scan failed to start (HTTP ${upstream.status}).` }, 502);
16649
+ }
16650
+ const marker = `upstream:${upstream.json.scan_id}:${upstream.json.source_video_id ?? ""}`;
16651
+ await putScanRecords({ scanStatus: "running", executionArn: marker });
16652
+ return scanResponse({ status: "running", executionArn: marker, pipeline: "upstream" });
16653
+ }
16654
+ const reasons = [];
16655
+ if (!evaluator)
16656
+ reasons.push(evaluatorError ?? "no scene evaluator available");
16657
+ if (evaluator && !ffmpegOk)
16658
+ reasons.push("ffmpeg was not found on this machine");
16659
+ if (evaluator && ffmpegOk && huntSpec.source_url && !canIngestUrl) {
16660
+ reasons.push("this box can't ingest social URLs (set RAPIDAPI_KEY, or pass a direct .mp4 link / upload the file)");
15928
16661
  }
15929
- const scan = {
15930
- scan_id: scanId,
15931
- owner_id: customer.id,
15932
- source_video_id: sourceVideoId,
15933
- status: executionArn ? "running" : "queued",
15934
- tier,
15935
- raw_s3_uri: sourcePointer,
15936
- execution_arn: executionArn,
15937
- tracer,
15938
- created_at: now,
15939
- updated_at: now
15940
- };
15941
- await clipRecords.putScan(scan);
15942
16662
  return c.json({
15943
- scan_id: scanId,
15944
- source_video_id: sourceVideoId,
15945
- status: scan.status,
15946
- execution_arn: executionArn ?? null,
15947
- tracer,
15948
- hunt_spec: huntSpec,
15949
- estimate,
15950
- compute_estimate: computeEstimate,
15951
- pipeline_deployed: Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN)
15952
- }, 202);
16663
+ error: `Can't run this hunt on this box: ${reasons.join("; ")}. Install Claude Code (\`claude\`) or Codex (\`codex\`) for keyless local scans, add a provider key, or link a cloud account (VIDFARM_UPSTREAM_HOST + VIDFARM_UPSTREAM_API_KEY) to use the cloud as backup.`
16664
+ }, 400);
15953
16665
  }
15954
16666
  catch (error) {
15955
16667
  return c.json({ error: error instanceof Error ? error.message : "Failed to start clip scan." }, 400);
15956
16668
  }
15957
16669
  });
16670
+ // GET /clips/scan-options — what the Import Source modal can run a hunt with:
16671
+ // which BYOK providers have saved keys, whether a local agent CLI is available
16672
+ // (vidfarm serve boxes only), what "auto" resolves to, and where the hunt runs.
16673
+ app.get(`${CLIPS_PREFIX}/scan-options`, async (c) => {
16674
+ const customer = requireCustomer(c);
16675
+ const keys = await loadCustomerVisionKeys(customer.id);
16676
+ const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
16677
+ const localAgent = cloudPipeline ? null : detectLocalAgent();
16678
+ const keyProvider = CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
16679
+ const defaultProvider = localAgent ? "agent" : keyProvider;
16680
+ const ffmpegOk = cloudPipeline ? true : await hasFfmpeg();
16681
+ const pipeline = cloudPipeline
16682
+ ? keyProvider ? "cloud" : null
16683
+ : defaultProvider && ffmpegOk ? "local" : hasUpstreamKey() ? "upstream" : null;
16684
+ return c.json({
16685
+ providers: CLIP_API_PROVIDERS.map((p) => ({ provider: p, has_key: Boolean(keys[p]) })),
16686
+ local_agent: localAgent ? { kind: localAgent.kind } : null,
16687
+ default_provider: defaultProvider,
16688
+ pipeline,
16689
+ upstream_backup: !cloudPipeline && hasUpstreamKey()
16690
+ });
16691
+ });
16692
+ function looksLikeDirectVideoUrl(url) {
16693
+ try {
16694
+ return /\.(mp4|mov|m4v|webm|mkv)$/i.test(new URL(url).pathname);
16695
+ }
16696
+ catch {
16697
+ return false;
16698
+ }
16699
+ }
16700
+ // Local URL ingest for serve-box hunts: direct video links download straight;
16701
+ // social URLs (TikTok/YouTube/IG/X) resolve through the same RapidAPI
16702
+ // downloader the video_download primitive uses (needs RAPIDAPI_KEY locally).
16703
+ async function ingestClipSourceUrlLocally(sourceUrl, workDir) {
16704
+ if (!sourceUrl)
16705
+ throw new Error("Clip scan needs a source video.");
16706
+ let mediaUrl = sourceUrl;
16707
+ if (!looksLikeDirectVideoUrl(sourceUrl)) {
16708
+ const apiKey = (config.RAPIDAPI_KEY || "").trim();
16709
+ if (!apiKey) {
16710
+ throw new Error("This box can't ingest social URLs (RAPIDAPI_KEY unset) — pass a direct .mp4 URL or upload the file.");
16711
+ }
16712
+ const lookupRes = await fetch(config.RAPIDAPI_VIDEO_DOWNLOAD_URL, {
16713
+ method: "POST",
16714
+ headers: {
16715
+ "x-rapidapi-key": apiKey,
16716
+ "x-rapidapi-host": config.RAPIDAPI_VIDEO_DOWNLOAD_HOST,
16717
+ "content-type": "application/x-www-form-urlencoded"
16718
+ },
16719
+ body: new URLSearchParams({ url: sourceUrl })
16720
+ });
16721
+ if (!lookupRes.ok)
16722
+ throw new Error(`Video download lookup failed with HTTP ${lookupRes.status}.`);
16723
+ const lookup = (await lookupRes.json());
16724
+ const playable = (lookup.medias ?? []).filter((m) => m.url && m.videoAvailable !== false && (m.type ? /video/i.test(m.type) : true));
16725
+ const chosen = playable.find((m) => (m.extension ?? "").toLowerCase() === "mp4" || /\.mp4(\?|$)/i.test(m.url ?? "")) ?? playable[0];
16726
+ if (!chosen?.url)
16727
+ throw new Error("Video download lookup returned no playable MP4 media URL.");
16728
+ mediaUrl = chosen.url;
16729
+ }
16730
+ const localPath = path.join(workDir, "ingest", "source.mp4");
16731
+ mkdirSync(path.dirname(localPath), { recursive: true });
16732
+ const res = await fetch(mediaUrl);
16733
+ if (!res.ok || !res.body)
16734
+ throw new Error(`Source video download failed with HTTP ${res.status}.`);
16735
+ await streamPipeline(Readable.fromWeb(res.body), createWriteStream(localPath));
16736
+ return localPath;
16737
+ }
16738
+ // In-process clip hunt for `vidfarm serve` boxes (no Step Functions worker).
16739
+ // Mirrors the clip-scan Lambda pipeline — ingest → probe/detect → refine →
16740
+ // per-scene tag/embed → finalize — with local ffmpeg and the resolved
16741
+ // evaluator (local claude/codex CLI or a BYOK key). Free: no wallet billing,
16742
+ // exactly like local renders. Detached; the client polls /clips/scan/:scanId.
16743
+ async function runClipScanInProcess(input) {
16744
+ const workRoot = path.join(config.VIDFARM_DATA_DIR, "clip-scans", input.scanId);
16745
+ const touchScan = async (patch) => {
16746
+ const scan = await clipRecords.getScan(input.ownerId, input.scanId);
16747
+ if (scan)
16748
+ await clipRecords.putScan({ ...scan, ...patch, updated_at: nowIso() });
16749
+ };
16750
+ const touchSource = async (patch) => {
16751
+ const source = await clipRecords.getSource(input.ownerId, input.sourceVideoId);
16752
+ if (source)
16753
+ await clipRecords.putSource({ ...source, ...patch, updated_at: nowIso() });
16754
+ };
16755
+ try {
16756
+ mkdirSync(workRoot, { recursive: true });
16757
+ const videoPath = input.localSourcePath ?? (await ingestClipSourceUrlLocally(input.sourceUrl ?? "", workRoot));
16758
+ const probe = await probeVideo(videoPath);
16759
+ await touchSource({ status: "scanning", duration_sec: probe.duration_sec });
16760
+ const spec = input.huntSpec;
16761
+ const ctx = {
16762
+ client: input.client,
16763
+ workDir: path.join(workRoot, "work"),
16764
+ clipsDir: path.join(workRoot, "clips"),
16765
+ thumbsDir: path.join(workRoot, "thumbs"),
16766
+ framesPerScene: input.framesPerScene,
16767
+ includeAudio: input.includeAudio,
16768
+ guidancePrompt: buildEffectiveGuidance({ contentPrompt: input.guidancePrompt, avoidText: spec.avoid_text }) || undefined,
16769
+ reencodeClips: true,
16770
+ crop: spec.aspect && spec.aspect !== "original"
16771
+ ? { aspect: spec.aspect, focus: normalizeCropFocus(spec.crop_focus) }
16772
+ : undefined
16773
+ };
16774
+ let stored = 0;
16775
+ await scanVideo({
16776
+ ctx,
16777
+ videoPath,
16778
+ probe,
16779
+ windows: spec.windows,
16780
+ durationBand: spec.duration_band ?? null,
16781
+ sourceVideoId: input.sourceVideoId,
16782
+ sourceFilename: input.filename,
16783
+ ownerId: input.ownerId,
16784
+ // Local agents run one CLI process per call — keep parallelism modest.
16785
+ concurrency: input.client.provider === "local-agent" ? 2 : 3,
16786
+ onClip: async (clip) => {
16787
+ // Persist through the storage service (local driver) as bare keys —
16788
+ // clipMediaUrl resolves clips/… + thumbs/… keys to fetchable URLs.
16789
+ const clipKey = `clips/${input.ownerId}/${clip.clip_id}.mp4`;
16790
+ const clipLocal = path.join(ctx.clipsDir, `${clip.clip_id}.mp4`);
16791
+ await storage.putBuffer(clipKey, readFileSync(clipLocal), "video/mp4");
16792
+ clip.file_path = clipKey;
16793
+ const thumbLocal = path.join(ctx.thumbsDir, `${clip.clip_id}.jpg`);
16794
+ if (existsSync(thumbLocal)) {
16795
+ const thumbKey = `thumbs/${input.ownerId}/${clip.clip_id}.jpg`;
16796
+ await storage.putBuffer(thumbKey, readFileSync(thumbLocal), "image/jpeg");
16797
+ clip.thumbnail_path = thumbKey;
16798
+ }
16799
+ else {
16800
+ clip.thumbnail_path = "";
16801
+ }
16802
+ await clipRecords.putClip(clip);
16803
+ await vectorIndex.upsert(input.ownerId, clip);
16804
+ stored++;
16805
+ },
16806
+ onError: (scene, error) => {
16807
+ devLog("clip_scan.local_scene_failed", { scan_id: input.scanId, scene_start: scene.start_time_sec, ...devErrorFields(error) }, "warn");
16808
+ }
16809
+ });
16810
+ await touchSource({ status: "complete", clip_count: stored });
16811
+ await touchScan({ status: "complete" });
16812
+ }
16813
+ catch (error) {
16814
+ devLog("clip_scan.local_failed", { scan_id: input.scanId, ...devErrorFields(error) }, "error");
16815
+ await touchSource({ status: "failed" }).catch(() => undefined);
16816
+ await touchScan({ status: "failed", error: error instanceof Error ? error.message : String(error) }).catch(() => undefined);
16817
+ }
16818
+ finally {
16819
+ rmSync(workRoot, { recursive: true, force: true });
16820
+ }
16821
+ }
16822
+ // A serve-box hunt handed to the upstream cloud (execution_arn
16823
+ // "upstream:<scan_id>:<source_video_id>"): mirror the upstream status onto the
16824
+ // local records, and when the cloud hunt completes pull the finished clips
16825
+ // down into local storage — the cloud runs the hunt, the box owns the result.
16826
+ const activeUpstreamClipMirrors = new Set();
16827
+ async function reconcileUpstreamClipScan(ownerId, scan) {
16828
+ const [, upstreamScanId, upstreamSourceId = ""] = (scan.execution_arn ?? "").split(":");
16829
+ if (!upstreamScanId)
16830
+ return scan;
16831
+ const res = await upstreamJson(`/clips/scan/${encodeURIComponent(upstreamScanId)}`);
16832
+ if (!res.ok || !res.json)
16833
+ return scan; // transient upstream hiccup — keep polling
16834
+ const status = res.json.source?.status ?? res.json.scan?.status ?? "running";
16835
+ if (status === "failed" || status === "error") {
16836
+ const failed = { ...scan, status: "failed", error: res.json.scan?.error ?? "Upstream cloud hunt failed.", updated_at: nowIso() };
16837
+ await clipRecords.putScan(failed);
16838
+ const source = await clipRecords.getSource(ownerId, scan.source_video_id);
16839
+ if (source && source.status !== "complete") {
16840
+ await clipRecords.putSource({ ...source, status: "failed", updated_at: nowIso() });
16841
+ }
16842
+ return failed;
16843
+ }
16844
+ if (status === "complete" && !activeUpstreamClipMirrors.has(scan.scan_id)) {
16845
+ activeUpstreamClipMirrors.add(scan.scan_id);
16846
+ void mirrorUpstreamClipsToLocal({ ownerId, scan, upstreamSourceId })
16847
+ .catch((error) => devLog("clip_scan.upstream_mirror_failed", { scan_id: scan.scan_id, ...devErrorFields(error) }, "error"))
16848
+ .finally(() => activeUpstreamClipMirrors.delete(scan.scan_id));
16849
+ }
16850
+ return scan; // stays "running" until the mirror lands the clips locally
16851
+ }
16852
+ async function mirrorUpstreamClipsToLocal(input) {
16853
+ const { ownerId, scan } = input;
16854
+ const query = input.upstreamSourceId ? `?source=${encodeURIComponent(input.upstreamSourceId)}&limit=200` : "?limit=200";
16855
+ const feed = await upstreamJson(`/clips/feed${query}`, { timeoutMs: 60_000 });
16856
+ if (!feed.ok || !feed.json)
16857
+ throw new Error(`Upstream clip feed unavailable (HTTP ${feed.status}).`);
16858
+ let stored = 0;
16859
+ for (const raw of feed.json.clips ?? []) {
16860
+ const clipId = typeof raw.clip_id === "string" ? raw.clip_id : null;
16861
+ const viewUrl = typeof raw.view_url === "string" ? raw.view_url : null;
16862
+ if (!clipId || !viewUrl)
16863
+ continue;
16864
+ const clipRes = await fetch(viewUrl);
16865
+ if (!clipRes.ok)
16866
+ continue;
16867
+ const clipKey = `clips/${ownerId}/${clipId}.mp4`;
16868
+ await storage.putBuffer(clipKey, new Uint8Array(await clipRes.arrayBuffer()), "video/mp4");
16869
+ let thumbKey = "";
16870
+ const thumbUrl = typeof raw.thumbnail_url === "string" ? raw.thumbnail_url : null;
16871
+ if (thumbUrl) {
16872
+ const thumbRes = await fetch(thumbUrl);
16873
+ if (thumbRes.ok) {
16874
+ thumbKey = `thumbs/${ownerId}/${clipId}.jpg`;
16875
+ await storage.putBuffer(thumbKey, new Uint8Array(await thumbRes.arrayBuffer()), "image/jpeg");
16876
+ }
16877
+ }
16878
+ const { view_url: _v, thumbnail_url: _t, ...rest } = raw;
16879
+ void _v;
16880
+ void _t;
16881
+ await clipRecords.putClip({
16882
+ ...rest,
16883
+ owner_id: ownerId,
16884
+ source_video_id: scan.source_video_id,
16885
+ file_path: clipKey,
16886
+ thumbnail_path: thumbKey
16887
+ });
16888
+ stored++;
16889
+ }
16890
+ const source = await clipRecords.getSource(ownerId, scan.source_video_id);
16891
+ if (source)
16892
+ await clipRecords.putSource({ ...source, status: "complete", clip_count: stored, updated_at: nowIso() });
16893
+ const current = await clipRecords.getScan(ownerId, scan.scan_id);
16894
+ await clipRecords.putScan({ ...(current ?? scan), status: "complete", updated_at: nowIso() });
16895
+ }
15958
16896
  // GET /clips/scan/:scanId — poll scan status.
15959
16897
  app.get(`${CLIPS_PREFIX}/scan/:scanId`, async (c) => {
15960
16898
  const customer = requireCustomer(c);
15961
16899
  let scan = await clipRecords.getScan(customer.id, c.req.param("scanId"));
15962
16900
  if (!scan)
15963
16901
  return c.json({ error: "Scan not found" }, 404);
16902
+ // Serve-box hunts handed to the upstream cloud reconcile against the
16903
+ // upstream REST API instead of Step Functions.
16904
+ if ((scan.status === "running" || scan.status === "queued") && scan.execution_arn?.startsWith("upstream:")) {
16905
+ scan = await reconcileUpstreamClipScan(customer.id, scan);
16906
+ }
16907
+ else
15964
16908
  // Lazily reconcile with the Step Functions execution: the pipeline has no
15965
16909
  // failure hook back into the scan record, so a crashed execution would
15966
16910
  // otherwise leave "running" forever and pollers spin.
@@ -16006,7 +16950,11 @@ app.post(`${CLIPS_PREFIX}/presets`, async (c) => {
16006
16950
  return c.json({ error: "name is required." }, 400);
16007
16951
  let criteria = body.criteria;
16008
16952
  if (!criteria && body.query) {
16009
- const built = await buildCustomerClipClient(customer.id, normalizeClipProvider(body.provider));
16953
+ const requested = readClipProviderChoice(body.provider);
16954
+ const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
16955
+ if (!provider)
16956
+ return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
16957
+ const built = await buildCustomerClipClient(customer.id, provider);
16010
16958
  if ("error" in built)
16011
16959
  return c.json({ error: built.error }, 400);
16012
16960
  criteria = await built.client.naturalLanguageToCriteria(body.query);