@mevdragon/vidfarm-devcli 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.director.md +29 -7
- package/SKILL.platform.md +4 -4
- package/demo/dist/app.css +1 -1
- package/demo/dist/app.js +254 -80
- package/dist/src/account-pages-legacy.js +9 -14
- package/dist/src/app.js +1081 -133
- package/dist/src/cli.js +107 -9
- package/dist/src/composition-runtime.js +88 -9
- package/dist/src/devcli/clips.js +5 -0
- package/dist/src/devcli/composition-edit.js +176 -8
- package/dist/src/devcli/transitions.js +205 -0
- package/dist/src/editor-chat.js +6 -5
- package/dist/src/frontend/homepage-view.js +1 -1
- package/dist/src/frontend/template-editor-chat.js +9 -6
- package/dist/src/homepage.js +9 -48
- package/dist/src/hyperframes/composition.js +183 -2
- package/dist/src/primitive-registry.js +237 -47
- package/dist/src/services/hyperframes.js +60 -15
- package/dist/src/services/providers.js +1 -0
- package/dist/src/services/serverless-records.js +19 -2
- package/dist/src/services/storage.js +12 -0
- package/dist/src/template-editor-shell.js +87 -48
- package/package.json +1 -1
- package/public/assets/homepage-client-app.js +23 -23
- package/public/assets/page-runtime-client-app.js +31 -31
package/dist/src/app.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import { pipeline as streamPipeline } from "node:stream/promises";
|
|
4
6
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
|
5
7
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
6
8
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
@@ -21,7 +23,7 @@ import { writeForkManifest } from "./services/fork-manifest.js";
|
|
|
21
23
|
import { renderHelpPage } from "./help-page.js";
|
|
22
24
|
import { renderHomepage } from "./homepage.js";
|
|
23
25
|
import { COMPOSITION_RUNTIME_SCRIPT_BODY } from "./composition-runtime.js";
|
|
24
|
-
import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, TRANSITION_CSS, TRANSITION_STYLE_MARKER } from "./hyperframes/composition.js";
|
|
26
|
+
import { CAPTION_ANIM_CSS, CAPTION_STYLE_MARKER, KEN_BURNS_CSS, KEN_BURNS_STYLE_MARKER, TRANSITION_CSS, TRANSITION_STYLE_MARKER, upgradeLegacyTransitionCss } from "./hyperframes/composition.js";
|
|
25
27
|
import { normalizeCompositionZIndexHtml, sanitizeCompositionHtml } from "./services/composition-sanitize.js";
|
|
26
28
|
import { applyMarkupUsd } from "./services/billing-pricing.js";
|
|
27
29
|
import { primitiveRegistry } from "./primitive-registry.js";
|
|
@@ -45,10 +47,14 @@ import { ServerlessProviderKeyService } from "./services/serverless-provider-key
|
|
|
45
47
|
import { ServerlessTemplateConfigService } from "./services/serverless-template-configs.js";
|
|
46
48
|
import { parseHTML } from "linkedom";
|
|
47
49
|
import { joinStorageKey, StorageService } from "./services/storage.js";
|
|
50
|
+
import { buildCaptionCues, cuesFromText } from "./services/captions.js";
|
|
51
|
+
import { defaultSpeechModelFor as speechModelFor, transcribeSpeechWithKey } from "./services/speech.js";
|
|
52
|
+
import { insertCaptionLayers } from "./devcli/composition-edit.js";
|
|
53
|
+
import { loadSpeechAudioLocally, MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES } from "./devcli/speech.js";
|
|
48
54
|
import { DescribeExecutionCommand, SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
|
|
49
55
|
import { clipRecords, makeUserPreset } from "./services/clip-records.js";
|
|
50
|
-
import { matchScenesToClips, searchUserClips } from "./services/clip-search.js";
|
|
51
|
-
import { BUILTIN_PRESETS, ClipModelClient, effectiveHuntDurationSec, estimateScanCostFromDuration, normalizeHuntSpec, parseClipHuntPrompt, resolveDurationBand } from "./services/clip-curation/index.js";
|
|
56
|
+
import { matchScenesToClips, searchUserClips, vectorIndex } from "./services/clip-search.js";
|
|
57
|
+
import { BUILTIN_PRESETS, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, estimateScanCostFromDuration, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, probeVideo, resolveDurationBand, scanVideo } from "./services/clip-curation/index.js";
|
|
52
58
|
import { analyzeCastMembers, annotateScenesFromSource, buildSmartDecomposedHtml, HyperframesService, smartDecomposeVideo, stampCompositionIdentity } from "./services/hyperframes.js";
|
|
53
59
|
import { buildPendingSceneAnnotationsState } from "./services/scene-annotations.js";
|
|
54
60
|
import { createPrimitiveJobContext } from "./primitive-context.js";
|
|
@@ -395,7 +401,16 @@ const userAttachmentFinalizeSchema = z.object({
|
|
|
395
401
|
content_type: z.string().trim().min(1).max(255),
|
|
396
402
|
size_bytes: z.coerce.number().int().min(1).max(1024 * 1024 * 1024),
|
|
397
403
|
storage_key: z.string().min(1).max(2000),
|
|
398
|
-
folder_path: z.string().trim().max(500).optional()
|
|
404
|
+
folder_path: z.string().trim().max(500).optional(),
|
|
405
|
+
notes: z.string().trim().max(4000).optional()
|
|
406
|
+
});
|
|
407
|
+
const userAttachmentNotesSchema = z.object({
|
|
408
|
+
notes: z.string().max(4000).nullable().optional()
|
|
409
|
+
});
|
|
410
|
+
const userAttachmentSearchSchema = z.object({
|
|
411
|
+
query: z.string().min(1).max(2000),
|
|
412
|
+
folder_path: z.string().trim().max(500).optional(),
|
|
413
|
+
limit: z.coerce.number().int().min(1).max(100).optional()
|
|
399
414
|
});
|
|
400
415
|
const userFileFolderSchema = z.object({
|
|
401
416
|
folder_path: z.string().trim().min(1).max(500)
|
|
@@ -850,6 +865,26 @@ function consumeSettingsFlash(c) {
|
|
|
850
865
|
return null;
|
|
851
866
|
}
|
|
852
867
|
}
|
|
868
|
+
// Fully-local box (`vidfarm serve`): the browser is pre-authed through
|
|
869
|
+
// /auto-login at boot, but a fresh browser profile / cleared cookies / a
|
|
870
|
+
// directly-opened /editor URL carries no session cookie — and every editor
|
|
871
|
+
// POST (Render, chat, uploads) then fails with a 401 "Unauthorized" modal
|
|
872
|
+
// even though the box is a single-user localhost machine whose only customer
|
|
873
|
+
// is the env-bootstrapped one. Fall back to the bootstrap key's customer,
|
|
874
|
+
// the exact identity /auto-login would have granted.
|
|
875
|
+
async function localBoxFallbackCustomer() {
|
|
876
|
+
const isFullyLocalBox = !config.isProduction
|
|
877
|
+
&& config.RECORDS_DRIVER === "local"
|
|
878
|
+
&& config.STORAGE_DRIVER === "local";
|
|
879
|
+
if (!isFullyLocalBox || !config.VIDFARM_API_KEY)
|
|
880
|
+
return null;
|
|
881
|
+
try {
|
|
882
|
+
return await auth.authenticateAsync(config.VIDFARM_API_KEY);
|
|
883
|
+
}
|
|
884
|
+
catch {
|
|
885
|
+
return null;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
853
888
|
async function getBrowserCustomer(c, body) {
|
|
854
889
|
try {
|
|
855
890
|
// Agent-first auth: an explicit vidfarm-api-key header wins over browser
|
|
@@ -881,10 +916,10 @@ async function getBrowserCustomer(c, body) {
|
|
|
881
916
|
return currentCustomer;
|
|
882
917
|
}
|
|
883
918
|
const customers = await listBrowserSessionCustomers(c);
|
|
884
|
-
if (customers.length
|
|
885
|
-
return null;
|
|
919
|
+
if (customers.length === 1) {
|
|
920
|
+
return customers[0] ?? null;
|
|
886
921
|
}
|
|
887
|
-
return
|
|
922
|
+
return await localBoxFallbackCustomer();
|
|
888
923
|
}
|
|
889
924
|
catch {
|
|
890
925
|
return null;
|
|
@@ -2891,6 +2926,9 @@ function serializeUserAttachment(c, attachment) {
|
|
|
2891
2926
|
sizeBytes: attachment.sizeBytes,
|
|
2892
2927
|
createdAt: attachment.createdAt,
|
|
2893
2928
|
folderPath: attachment.folderPath ?? "",
|
|
2929
|
+
notes: attachment.notes ?? null,
|
|
2930
|
+
// Flag only — the raw vector never ships to the client (mirrors serializeClip).
|
|
2931
|
+
hasNotesEmbedding: Boolean(attachment.notesEmbedding && attachment.notesEmbedding.length > 0),
|
|
2894
2932
|
viewUrl: attachment.publicUrl || storage.getPublicUrl(attachment.storageKey) || buildUserAttachmentViewUrl(c, attachment.storageKey)
|
|
2895
2933
|
};
|
|
2896
2934
|
}
|
|
@@ -2918,10 +2956,19 @@ function isFormFile(value) {
|
|
|
2918
2956
|
&& "arrayBuffer" in value);
|
|
2919
2957
|
}
|
|
2920
2958
|
async function getVisibleApiKey(customerId) {
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2959
|
+
// Prefer the newest key THIS server can validate, not just the newest key.
|
|
2960
|
+
// After an API_KEY_SALT rotation the shared table holds keys hashed under
|
|
2961
|
+
// different salts (deployed stage vs local dev boxes share the staging
|
|
2962
|
+
// table); embedding a key the current server rejects breaks editor chat and
|
|
2963
|
+
// its tool calls with "Invalid API credentials." on every message.
|
|
2964
|
+
const existingKeys = await serverlessRecords.listActiveApiKeysForCustomer(customerId);
|
|
2965
|
+
for (const row of existingKeys) {
|
|
2966
|
+
const rawValue = row?.raw_value ? String(row.raw_value) : null;
|
|
2967
|
+
if (!rawValue)
|
|
2968
|
+
continue;
|
|
2969
|
+
if (String(row.key_hash ?? "") === hashSecret(rawValue + config.API_KEY_SALT)) {
|
|
2970
|
+
return rawValue;
|
|
2971
|
+
}
|
|
2925
2972
|
}
|
|
2926
2973
|
const apiKey = `vf_${createId("key")}`;
|
|
2927
2974
|
await serverlessRecords.insertApiKey({
|
|
@@ -3031,6 +3078,13 @@ function asRecord(value) {
|
|
|
3031
3078
|
function readNonEmptyString(value) {
|
|
3032
3079
|
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
3033
3080
|
}
|
|
3081
|
+
// "raw" = the user's own new project (prompt-generated, own raw clips) — the
|
|
3082
|
+
// editor must not auto-prompt the Decompose modal for these. Anything else
|
|
3083
|
+
// (absent, unknown) reads as "inspiration", matching records that predate the
|
|
3084
|
+
// field.
|
|
3085
|
+
function readProjectOrigin(value) {
|
|
3086
|
+
return value === "raw" ? "raw" : "inspiration";
|
|
3087
|
+
}
|
|
3034
3088
|
function readFiniteNumber(value, fallback = 0) {
|
|
3035
3089
|
const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN;
|
|
3036
3090
|
return Number.isFinite(number) ? number : fallback;
|
|
@@ -3090,7 +3144,8 @@ function buildPrimitiveEditorDocsRoutes() {
|
|
|
3090
3144
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/concat`, summary: "Primitive audio concat job. Body must be { tracer, payload: { source_audio_urls, output_format?, audio_bitrate_kbps? }, webhook_url? }." },
|
|
3091
3145
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/normalize`, summary: "Primitive audio normalization job. Body must be { tracer, payload: { source_audio_url, output_format?, audio_bitrate_kbps?, sample_rate_hz? }, webhook_url? }." },
|
|
3092
3146
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/speech`, summary: "Primitive text-to-speech (TTS) job using saved OpenAI, Gemini, or OpenRouter provider keys. Body must be { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format? }, webhook_url? }. instructions is a free-form voice-style prompt (tone, pacing, accent, emotion, persona — e.g. \"calm, warm bedtime narrator with a slight British accent\"). voice is a provider preset (OpenAI: alloy/ash/coral/…; Gemini: Kore/Puck/Charon/…); provider is inferred from the voice when omitted, and a voice paired with the wrong provider is rejected with a 400. Returns a durable platform audio URL (mp3 or wav). Alias: POST /api/v1/primitives/tts." },
|
|
3093
|
-
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt." },
|
|
3147
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT) job using saved provider keys. Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, language?, prompt?, provider?, model? }, webhook_url? }. Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT cues, no speakers) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration), plus transcript.json/.srt/.txt artifacts. diarize defaults to true; speaker attribution needs a Gemini key (openai/openrouter degrade to a plain single-speaker transcript). Sources over ~40 minutes of speech must be trimmed first. Alias: POST /api/v1/primitives/stt. Pass word_timestamps: true for word-level timings on each segment (real timestamps on an OpenAI key via whisper-1; other providers stay segment-level)." },
|
|
3148
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/captions`, summary: "ANIMATED CAPTIONS from speech: one job transcribes narration WITH word-level timings and pages it into word-by-word caption cues ready for the editor's set_captions action. Body must be { tracer, payload: { source_url (video or audio; video audio is demuxed) OR text (page a script instead of transcribing), style? (spotlight|karaoke|word-pop|stack-up|neon|bounce), animation?, active_color?, highlight_color?, uppercase?, color?, background?, background_style?, font_family?, font_size?, max_words_per_cue?, offset_sec?, language?, provider?, window_start?/window_duration? (text mode) }, webhook_url? }. The finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments — apply verbatim). Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter fall back to estimated word windows. Alias: POST /api/v1/primitives/captions." },
|
|
3094
3149
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/coldstart`, summary: "Primitive cold-start strategy questionnaire. Body should usually be { tracer, payload: { user_message }, webhook_url? }. user_message can be rough, like \"I don't know where to start\" or a messy description. count is optional only when the user explicitly wants more or fewer questions. offer_description remains accepted as a backward-compatible alias." },
|
|
3095
3150
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, summary: "Primitive awareness-stage recommendation. Body must be { tracer, payload: { offer_description }, webhook_url? }. Use this when the customer is unsure what type of ads to produce." },
|
|
3096
3151
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/angles`, summary: "Primitive ad-angle brainstorm. Body must be { tracer, payload: { offer_description, problem_awareness, solution_awareness }, webhook_url? }. Allowed values: problem_awareness is problem_unaware|problem_aware, solution_awareness is solution_unaware|solution_aware. count is optional only when the user explicitly wants more or fewer angles. Returns diverse persuasive TikTok angles with explanations." },
|
|
@@ -3684,7 +3739,7 @@ function normalizeOperationRequestTracer(tracer) {
|
|
|
3684
3739
|
}
|
|
3685
3740
|
function createTemplateHttpTool(input) {
|
|
3686
3741
|
return tool({
|
|
3687
|
-
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys. To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
3742
|
+
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key, and word_timestamps: true adds word-level timings per segment (real on an OpenAI key via whisper-1). If the user wants ANIMATED word-by-word captions generated from narration, call POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url (narration audio or video URL) OR text (a script to page), style?: \"spotlight\"|\"karaoke\"|\"word-pop\"|\"stack-up\"|\"neon\"|\"bounce\", max_words_per_cue?, offset_sec?, language?, provider? } } — the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments); apply them with the editor_action tool (action_type=set_captions) so the change flows through the editor state. Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter estimate word windows. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, provider?: \"gemini\"|\"openai\"|\"openrouter\", tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys (auto-picked gemini → openai → openrouter unless body.provider names one). To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
3688
3743
|
inputSchema: z.object({
|
|
3689
3744
|
method: z.enum(["GET", "POST"]).describe("HTTP method to use."),
|
|
3690
3745
|
path: z.string().min(1).describe("Absolute path or same-origin URL for an allowed Vidfarm REST route."),
|
|
@@ -3932,16 +3987,19 @@ function browseFilesTextContentType(fileName) {
|
|
|
3932
3987
|
}
|
|
3933
3988
|
function createBrowseFilesTool(input) {
|
|
3934
3989
|
return tool({
|
|
3935
|
-
description: "Browse AND write the user's My Files filesystem — their personal library of uploaded assets (mp4, mov, webm, png, jpg, jpeg, gif, webp, svg, mp3, wav, m4a, aac, pdf, md, txt, csv, and more) organized into virtual folders. Use action=list to enumerate files and virtual folders so you can discover what assets exist; pass folder_path to scope the listing to one folder (omit or empty for the root). Use action=read to fetch one file by file_id (preferred; copy it from a prior list result) or by file_name: it returns the file's durable view_url plus metadata, and for text files (md, txt, csv, srt, vtt, json) it also returns the full text_content inline so you can read it. For images, video, audio, and PDFs it returns view_url only — reference that URL directly (drop it into composition layers, or pass it into primitive routes like images/edit source_image_url, videos/generate input_references, or videos/download); you cannot read their raw bytes as text. Use action=write to SAVE a text file into My Files — pass file_name (e.g. About.md), content (the full text),
|
|
3990
|
+
description: "Browse AND write the user's My Files filesystem — their personal library of uploaded assets (mp4, mov, webm, png, jpg, jpeg, gif, webp, svg, mp3, wav, m4a, aac, pdf, md, txt, csv, and more) organized into virtual folders. Use action=list to enumerate files and virtual folders so you can discover what assets exist; pass folder_path to scope the listing to one folder (omit or empty for the root). Use action=search to find files by MEANING when you don't know where something lives: pass query (plain language, e.g. 'character reference sheet for the mascot' or 'logo on transparent background') and it matches keywords AND vector-embedded metadata notes across every file's name, folder, and notes — prefer search over paging through list when My Files is large or the user references an asset vaguely. Use action=read to fetch one file by file_id (preferred; copy it from a prior list/search result) or by file_name: it returns the file's durable view_url plus metadata, and for text files (md, txt, csv, srt, vtt, json) it also returns the full text_content inline so you can read it. For images, video, audio, and PDFs it returns view_url only — reference that URL directly (drop it into composition layers, or pass it into primitive routes like images/edit source_image_url, videos/generate input_references, or videos/download); you cannot read their raw bytes as text. Use action=write to SAVE a text file into My Files — pass file_name (e.g. About.md), content (the full text), optional folder_path to namescope it under a product/offer folder, and optional notes. This is how you persist Getting Started context the user can reuse later: a product About.md or Interview.md, and awareness-levels.md / persuasive-angles.md / ad-hooks.md distilled from brainstorm output. write saves text files (md, txt, csv, json, srt, vtt) via content, or IMPORTS any media asset into the library by passing its durable URL as source_url — e.g. save a finished image job's output as character_sprite_card.png so it persists in My Files instead of living only in a job result. write returns the new file_id and view_url. Use action=annotate to set metadata notes on ANY existing file (including binary assets): notes say what the file is, who/what it depicts, and when to use it, and they are vector-embedded so future sessions can search their way back to the asset. Annotate every asset worth finding again — especially character references: keep a character_sprite_card.png plus an about_character.md per recurring character in that character's folder, annotate both, and pass the sprite card's view_url into generation reference inputs (images prompt_attachments / videos input_references) for character consistency. Always list or search before you read so you use a real file_id; never invent file ids or view URLs.",
|
|
3936
3991
|
inputSchema: z.object({
|
|
3937
|
-
action: z.enum(["list", "read", "write"]).describe("list = enumerate files and folders; read = fetch one file's view_url and, for text files, its contents; write = save a text file into My Files."),
|
|
3938
|
-
folder_path: z.string().optional().describe("Folder to scope a list to, the folder of the file to read, or the folder to write into. Omit or leave empty for the root."),
|
|
3939
|
-
file_id: z.string().optional().describe("Attachment id to read, copied from a prior list result. Preferred over file_name."),
|
|
3940
|
-
file_name: z.string().optional().describe("File name to read when you do not have the id, or the name to save under for write (e.g. About.md).
|
|
3941
|
-
content: z.string().optional().describe("For write: the full text content of the file to save (e.g. Markdown for About.md). Required for
|
|
3942
|
-
|
|
3992
|
+
action: z.enum(["list", "read", "write", "annotate", "search"]).describe("list = enumerate files and folders; read = fetch one file's view_url and, for text files, its contents; write = save a text file into My Files; annotate = set the metadata notes on one existing file; search = find files by meaning across names, folders, and notes."),
|
|
3993
|
+
folder_path: z.string().optional().describe("Folder to scope a list or search to, the folder of the file to read, or the folder to write into. Omit or leave empty for the root."),
|
|
3994
|
+
file_id: z.string().optional().describe("Attachment id to read or annotate, copied from a prior list/search result. Preferred over file_name."),
|
|
3995
|
+
file_name: z.string().optional().describe("File name to read/annotate when you do not have the id, or the name to save under for write (e.g. About.md). Matched within folder_path when one is provided."),
|
|
3996
|
+
content: z.string().optional().describe("For write: the full text content of the file to save (e.g. Markdown for About.md). Required for text writes; mutually exclusive with source_url."),
|
|
3997
|
+
source_url: z.string().optional().describe("For write: a durable media URL (from a finished primitive job result or an existing asset) to IMPORT into My Files as file_name — this is how you save a generated character_sprite_card.png or other binary asset into the library. Mutually exclusive with content."),
|
|
3998
|
+
notes: z.string().optional().describe("For annotate (required) or write (optional): metadata notes describing what the file is, who/what it depicts, and when to use it — e.g. 'Sprite card for Zara, our mascot: front/side/back views, teal jacket. Use as the reference image whenever generating Zara.' Notes are vector-embedded so search finds the file by meaning. For annotate, an empty string clears existing notes."),
|
|
3999
|
+
query: z.string().optional().describe("For search: what you're looking for, in plain language. Required for search; ignored otherwise."),
|
|
4000
|
+
explanation: z.string().optional().describe("Short reason for the browse, save, annotation, or search.")
|
|
3943
4001
|
}),
|
|
3944
|
-
execute: async ({ action, folder_path, file_id, file_name, content, explanation }) => {
|
|
4002
|
+
execute: async ({ action, folder_path, file_id, file_name, content, source_url, notes, query, explanation }) => {
|
|
3945
4003
|
const listUrl = new URL(`${USER_PREFIX}/me/attachments`, config.PUBLIC_BASE_URL);
|
|
3946
4004
|
const headers = new Headers();
|
|
3947
4005
|
headers.set("vidfarm-api-key", input.vidfarmApiKey);
|
|
@@ -3954,22 +4012,59 @@ function createBrowseFilesTool(input) {
|
|
|
3954
4012
|
if (action === "write") {
|
|
3955
4013
|
const name = (file_name ?? "").trim();
|
|
3956
4014
|
if (!name) {
|
|
3957
|
-
return { action, explanation: explanation ?? null, status: 400, error: "write requires file_name (e.g. About.md)." };
|
|
4015
|
+
return { action, explanation: explanation ?? null, status: 400, error: "write requires file_name (e.g. About.md, or character_sprite_card.png with source_url)." };
|
|
3958
4016
|
}
|
|
3959
|
-
|
|
3960
|
-
|
|
4017
|
+
const importSrc = (source_url ?? "").trim();
|
|
4018
|
+
let fileBlob;
|
|
4019
|
+
if (importSrc) {
|
|
4020
|
+
// Import a durable media URL (e.g. a finished primitive job's output)
|
|
4021
|
+
// into My Files. The api-key header only travels to our own origin.
|
|
4022
|
+
let sourceUrl;
|
|
4023
|
+
try {
|
|
4024
|
+
sourceUrl = new URL(importSrc, config.PUBLIC_BASE_URL);
|
|
4025
|
+
}
|
|
4026
|
+
catch {
|
|
4027
|
+
return { action, explanation: explanation ?? null, status: 400, error: `source_url is not a valid URL: ${importSrc.slice(0, 200)}` };
|
|
4028
|
+
}
|
|
4029
|
+
const sourceHeaders = new Headers();
|
|
4030
|
+
if (sourceUrl.origin === new URL(config.PUBLIC_BASE_URL).origin) {
|
|
4031
|
+
sourceHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
|
|
4032
|
+
}
|
|
4033
|
+
const sourceResponse = await fetch(sourceUrl, {
|
|
4034
|
+
method: "GET",
|
|
4035
|
+
headers: sourceHeaders,
|
|
4036
|
+
signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
|
|
4037
|
+
});
|
|
4038
|
+
if (!sourceResponse.ok) {
|
|
4039
|
+
return { action, explanation: explanation ?? null, status: sourceResponse.status, error: `Could not fetch source_url (${sourceResponse.status}). Use a durable URL from a finished job result or an existing asset.` };
|
|
4040
|
+
}
|
|
4041
|
+
const sourceBytes = await sourceResponse.arrayBuffer();
|
|
4042
|
+
if (sourceBytes.byteLength > 50 * 1024 * 1024) {
|
|
4043
|
+
return { action, explanation: explanation ?? null, status: 400, error: "source_url asset is larger than the 50 MB My Files limit." };
|
|
4044
|
+
}
|
|
4045
|
+
const sourceType = (sourceResponse.headers.get("content-type") ?? "").split(";")[0].trim();
|
|
4046
|
+
fileBlob = new Blob([sourceBytes], sourceType ? { type: sourceType } : undefined);
|
|
3961
4047
|
}
|
|
3962
|
-
|
|
3963
|
-
|
|
4048
|
+
else {
|
|
4049
|
+
if (typeof content !== "string") {
|
|
4050
|
+
return { action, explanation: explanation ?? null, status: 400, error: "write requires content (the full text of the file) or source_url (a durable media URL to import)." };
|
|
4051
|
+
}
|
|
4052
|
+
if (!isBrowseFilesTextFile(name, "")) {
|
|
4053
|
+
return { action, explanation: explanation ?? null, status: 400, error: `write with content only supports text files (md, txt, csv, json, srt, vtt). To save media like "${name}", pass its durable URL as source_url instead.` };
|
|
4054
|
+
}
|
|
4055
|
+
fileBlob = new Blob([content], { type: browseFilesTextContentType(name) });
|
|
3964
4056
|
}
|
|
3965
4057
|
const uploadUrl = new URL(`${USER_PREFIX}/me/attachments/upload`, config.PUBLIC_BASE_URL);
|
|
3966
4058
|
const uploadHeaders = new Headers();
|
|
3967
4059
|
uploadHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
|
|
3968
4060
|
const form = new FormData();
|
|
3969
|
-
form.append("file",
|
|
4061
|
+
form.append("file", fileBlob, name);
|
|
3970
4062
|
if (normalizedFolder) {
|
|
3971
4063
|
form.append("folder_path", normalizedFolder);
|
|
3972
4064
|
}
|
|
4065
|
+
if (typeof notes === "string" && notes.trim()) {
|
|
4066
|
+
form.append("notes", notes.trim());
|
|
4067
|
+
}
|
|
3973
4068
|
const writeResponse = await fetch(uploadUrl, {
|
|
3974
4069
|
method: "POST",
|
|
3975
4070
|
headers: uploadHeaders,
|
|
@@ -4004,11 +4099,65 @@ function createBrowseFilesTool(input) {
|
|
|
4004
4099
|
content_type: saved.contentType,
|
|
4005
4100
|
size_bytes: saved.sizeBytes,
|
|
4006
4101
|
folder_path: saved.folderPath ?? "",
|
|
4102
|
+
notes: saved.notes ?? null,
|
|
4007
4103
|
view_url: saved.viewUrl,
|
|
4008
4104
|
created_at: saved.createdAt
|
|
4009
4105
|
}
|
|
4010
4106
|
};
|
|
4011
4107
|
}
|
|
4108
|
+
if (action === "search") {
|
|
4109
|
+
const q = (query ?? "").trim();
|
|
4110
|
+
if (!q) {
|
|
4111
|
+
return { action, explanation: explanation ?? null, status: 400, error: "search requires query (what you're looking for, in plain language)." };
|
|
4112
|
+
}
|
|
4113
|
+
const searchUrl = new URL(`${USER_PREFIX}/me/attachments/search`, config.PUBLIC_BASE_URL);
|
|
4114
|
+
const searchHeaders = new Headers();
|
|
4115
|
+
searchHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
|
|
4116
|
+
searchHeaders.set("Content-Type", "application/json");
|
|
4117
|
+
const searchResponse = await fetch(searchUrl, {
|
|
4118
|
+
method: "POST",
|
|
4119
|
+
headers: searchHeaders,
|
|
4120
|
+
body: JSON.stringify({ query: q, ...(normalizedFolder ? { folder_path: normalizedFolder } : {}), limit: 20 }),
|
|
4121
|
+
signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
|
|
4122
|
+
});
|
|
4123
|
+
const searchText = await searchResponse.text();
|
|
4124
|
+
let searchJson = null;
|
|
4125
|
+
try {
|
|
4126
|
+
searchJson = searchText ? JSON.parse(searchText) : null;
|
|
4127
|
+
}
|
|
4128
|
+
catch {
|
|
4129
|
+
searchJson = null;
|
|
4130
|
+
}
|
|
4131
|
+
if (!searchResponse.ok || !Array.isArray(searchJson?.results)) {
|
|
4132
|
+
return {
|
|
4133
|
+
action,
|
|
4134
|
+
explanation: explanation ?? null,
|
|
4135
|
+
status: searchResponse.status,
|
|
4136
|
+
error: `My Files search failed (${searchResponse.status}). ${(searchJson?.error ?? searchText).slice(0, 500)}`
|
|
4137
|
+
};
|
|
4138
|
+
}
|
|
4139
|
+
return {
|
|
4140
|
+
action,
|
|
4141
|
+
explanation: explanation ?? null,
|
|
4142
|
+
status: 200,
|
|
4143
|
+
query: q,
|
|
4144
|
+
folder_path: normalizedFolder,
|
|
4145
|
+
semantic: Boolean(searchJson.semantic),
|
|
4146
|
+
result_count: searchJson.results.length,
|
|
4147
|
+
results: searchJson.results.map((file) => ({
|
|
4148
|
+
file_id: file.id,
|
|
4149
|
+
file_name: file.fileName,
|
|
4150
|
+
content_type: file.contentType,
|
|
4151
|
+
size_bytes: file.sizeBytes,
|
|
4152
|
+
folder_path: file.folderPath ?? "",
|
|
4153
|
+
notes: file.notes ?? null,
|
|
4154
|
+
similarity: file.similarity ?? null,
|
|
4155
|
+
keyword_match: Boolean(file.keyword_match),
|
|
4156
|
+
view_url: file.viewUrl,
|
|
4157
|
+
created_at: file.createdAt
|
|
4158
|
+
}))
|
|
4159
|
+
};
|
|
4160
|
+
}
|
|
4012
4161
|
const listResponse = await fetch(listUrl, {
|
|
4013
4162
|
method: "GET",
|
|
4014
4163
|
headers,
|
|
@@ -4043,6 +4192,7 @@ function createBrowseFilesTool(input) {
|
|
|
4043
4192
|
content_type: file.contentType,
|
|
4044
4193
|
size_bytes: file.sizeBytes,
|
|
4045
4194
|
folder_path: file.folderPath ?? "",
|
|
4195
|
+
notes: file.notes ?? null,
|
|
4046
4196
|
view_url: file.viewUrl,
|
|
4047
4197
|
created_at: file.createdAt
|
|
4048
4198
|
}))
|
|
@@ -4061,7 +4211,56 @@ function createBrowseFilesTool(input) {
|
|
|
4061
4211
|
status: 404,
|
|
4062
4212
|
error: file_id || file_name
|
|
4063
4213
|
? "No matching file in My Files. Call action=list first to copy an exact file_id."
|
|
4064
|
-
:
|
|
4214
|
+
: `${action} requires file_id (preferred) or file_name.`
|
|
4215
|
+
};
|
|
4216
|
+
}
|
|
4217
|
+
if (action === "annotate") {
|
|
4218
|
+
if (typeof notes !== "string") {
|
|
4219
|
+
return { action, explanation: explanation ?? null, status: 400, error: "annotate requires notes (the metadata to attach; empty string clears)." };
|
|
4220
|
+
}
|
|
4221
|
+
const annotateUrl = new URL(`${USER_PREFIX}/me/attachments/${encodeURIComponent(match.id)}`, config.PUBLIC_BASE_URL);
|
|
4222
|
+
const annotateHeaders = new Headers();
|
|
4223
|
+
annotateHeaders.set("vidfarm-api-key", input.vidfarmApiKey);
|
|
4224
|
+
annotateHeaders.set("Content-Type", "application/json");
|
|
4225
|
+
const annotateResponse = await fetch(annotateUrl, {
|
|
4226
|
+
method: "PATCH",
|
|
4227
|
+
headers: annotateHeaders,
|
|
4228
|
+
body: JSON.stringify({ notes }),
|
|
4229
|
+
signal: mergeAbortSignals(input.abortSignal, timeoutController.signal)
|
|
4230
|
+
});
|
|
4231
|
+
const annotateText = await annotateResponse.text();
|
|
4232
|
+
let annotateJson = null;
|
|
4233
|
+
try {
|
|
4234
|
+
annotateJson = annotateText ? JSON.parse(annotateText) : null;
|
|
4235
|
+
}
|
|
4236
|
+
catch {
|
|
4237
|
+
annotateJson = null;
|
|
4238
|
+
}
|
|
4239
|
+
if (!annotateResponse.ok || !annotateJson?.attachment) {
|
|
4240
|
+
return {
|
|
4241
|
+
action,
|
|
4242
|
+
explanation: explanation ?? null,
|
|
4243
|
+
status: annotateResponse.status,
|
|
4244
|
+
error: `Could not annotate file (${annotateResponse.status}). ${(annotateJson?.error ?? annotateText).slice(0, 500)}`
|
|
4245
|
+
};
|
|
4246
|
+
}
|
|
4247
|
+
const annotated = annotateJson.attachment;
|
|
4248
|
+
return {
|
|
4249
|
+
action,
|
|
4250
|
+
explanation: explanation ?? null,
|
|
4251
|
+
status: 200,
|
|
4252
|
+
annotated: true,
|
|
4253
|
+
notes_embedded: Boolean(annotateJson.notes_embedded),
|
|
4254
|
+
file: {
|
|
4255
|
+
file_id: annotated.id,
|
|
4256
|
+
file_name: annotated.fileName,
|
|
4257
|
+
content_type: annotated.contentType,
|
|
4258
|
+
size_bytes: annotated.sizeBytes,
|
|
4259
|
+
folder_path: annotated.folderPath ?? "",
|
|
4260
|
+
notes: annotated.notes ?? null,
|
|
4261
|
+
view_url: annotated.viewUrl,
|
|
4262
|
+
created_at: annotated.createdAt
|
|
4263
|
+
}
|
|
4065
4264
|
};
|
|
4066
4265
|
}
|
|
4067
4266
|
const fileInfo = {
|
|
@@ -4070,6 +4269,7 @@ function createBrowseFilesTool(input) {
|
|
|
4070
4269
|
content_type: match.contentType,
|
|
4071
4270
|
size_bytes: match.sizeBytes,
|
|
4072
4271
|
folder_path: match.folderPath ?? "",
|
|
4272
|
+
notes: match.notes ?? null,
|
|
4073
4273
|
view_url: match.viewUrl,
|
|
4074
4274
|
created_at: match.createdAt
|
|
4075
4275
|
};
|
|
@@ -4188,8 +4388,9 @@ function createEditorActionTool() {
|
|
|
4188
4388
|
"replace_composition_html",
|
|
4189
4389
|
"export_composition",
|
|
4190
4390
|
"generate_layer",
|
|
4191
|
-
"set_captions"
|
|
4192
|
-
|
|
4391
|
+
"set_captions",
|
|
4392
|
+
"set_transitions"
|
|
4393
|
+
]).describe("Which mutation to perform. Use generate_layer to AI-generate a video/image and auto-place it on the timeline (fill a gap or replace a scene). Use export_composition to trigger the same Export button available in the editor UI — this queues an AWS Lambda render and the finished MP4 URL will surface in the next editor_context as last_export_url. Use set_captions to lay down a full run of ANIMATED word-by-word captions (TikTok/CapCut style): pass captions[] cues (or text + start/duration to auto-page), plus caption_style; it replaces any existing animated caption layers. To restyle existing captions without changing text/timing, use set_layer_style with caption_* fields on each caption layer. Use set_transitions to configure scene transitions across the WHOLE timeline in one call: transition = the junction preset applied at every cut (every scene clip except each track's first; 'none' clears), transition_intro = the first clip's entrance, transition_outro = the last clip's exit."),
|
|
4193
4394
|
layer_key: z.string().optional().describe("Existing layer key from editor_context (required for remove/set/duplicate/split). MAY be either the element_id (data-hf-id) or the slug (data-hf-slug). For add_layer, optionally provide a stable id (starts with a letter, [A-Za-z0-9_-], <=64 chars) to reuse in later tool calls this turn."),
|
|
4194
4395
|
slug: z.string().optional().describe("snake_case slug for the layer. On add_layer, seeds data-hf-slug. On set_layer_identity, sets it. Slugs give viral DNA a stable handle for the element."),
|
|
4195
4396
|
note: z.string().optional().describe("Human-facing note explaining the layer's role, referenced by viral DNA. On add_layer seeds data-hf-note. On set_layer_identity, sets it. Empty string removes."),
|
|
@@ -4222,8 +4423,12 @@ function createEditorActionTool() {
|
|
|
4222
4423
|
object_position: z.string().optional().describe("CSS object-position for image/video (e.g., center, top, bottom left)."),
|
|
4223
4424
|
ken_burns: z.enum(["zoom-in", "zoom-out", "pan-left", "pan-right", "pan-up", "pan-down", "zoom-in-left", "zoom-in-right", "none"]).optional().describe("Ken Burns motion for still-image layers: a slow pan/zoom spanning the clip's full duration. Apply with set_layer_media on an existing image, or seed it on add_layer/generate_layer when kind/media_type is image. 'none' removes the effect. When the user says 'animate the images' / 'ken burns', apply it to every still image, varying presets across adjacent clips (zoom-in, pan-left, zoom-out, ...) for a documentary feel."),
|
|
4224
4425
|
ken_burns_intensity: z.number().optional().describe("Ken Burns strength: extra zoom / pan travel as a fraction of the frame, 0.04-0.5 (default 0.18; ~0.1 subtle, ~0.3 dramatic). Only meaningful alongside ken_burns."),
|
|
4225
|
-
transition: z.enum(["crossfade", "fade-black", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("Scene transition INTO this clip at its start, from the previous clip on the same track. Set it on the INCOMING clip (the one after the cut) with set_layer_media, or seed it on add_layer/generate_layer. Works on video and image scene clips; 'none' removes it. The previous butt-cut clip is automatically held on screen beneath the incoming one for the transition window — do not retime anything yourself.
|
|
4226
|
-
transition_duration: z.number().optional().describe("Transition length in seconds, 0.1-2.5 (default 0.5; ~0.3 snappy, ~1.0 slow cinematic). Only meaningful alongside transition."),
|
|
4426
|
+
transition: z.enum(["crossfade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "whip-left", "whip-right", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("Scene transition INTO this clip at its start, from the previous clip on the same track. Set it on the INCOMING clip (the one after the cut) with set_layer_media, or seed it on add_layer/generate_layer. On set_transitions it is the junction preset applied at EVERY cut. Works on video and image scene clips; 'none' removes it. The previous butt-cut clip is automatically held on screen beneath the incoming one for the transition window — do not retime anything yourself. Preset energy: crossfade/fade-black/fade-white/blur = calm, smoke = dreamy, slide/wipe = energetic, whip/flash = fast-paced social, zoom/circle-open = punchy."),
|
|
4427
|
+
transition_duration: z.number().optional().describe("Transition length in seconds, 0.1-2.5 (default 0.5; ~0.3 snappy, ~1.0 slow cinematic). Only meaningful alongside transition/transition_intro."),
|
|
4428
|
+
transition_out: z.enum(["fade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-close", "none"]).optional().describe("Exit transition OUT of this clip at its end (fade to black, slide away, ...). Set it with set_layer_media on the clip that should animate away, or seed it on add_layer/generate_layer. The animation plays over the clip's last transition_out_duration seconds; no retiming needed. Most cuts only need the incoming clip's transition — use transition_out for ending a video (fade to black), for clips followed by a gap, or for deliberate dip-to-black moments (outgoing fade-black + incoming fade-black)."),
|
|
4429
|
+
transition_out_duration: z.number().optional().describe("Exit transition length in seconds, 0.1-2.5 (default 0.5). Only meaningful alongside transition_out/transition_outro."),
|
|
4430
|
+
transition_intro: z.enum(["crossfade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "whip-left", "whip-right", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-open", "none"]).optional().describe("set_transitions only: entrance for the very FIRST scene clip (fade in from black = fade-black). 'none' clears it."),
|
|
4431
|
+
transition_outro: z.enum(["fade", "fade-black", "fade-white", "flash", "smoke", "blur", "slide-left", "slide-right", "slide-up", "slide-down", "wipe-left", "wipe-right", "wipe-up", "wipe-down", "zoom-in", "zoom-out", "circle-close", "none"]).optional().describe("set_transitions only: exit for the very LAST scene clip (fade to black = fade-black). 'none' clears it."),
|
|
4227
4432
|
caption_style: z.enum(["spotlight", "karaoke", "word-pop", "stack-up", "neon", "bounce"]).optional().describe("Animated caption preset look: spotlight (active word gets a highlight pill, bold outline text), karaoke (words fill with color as spoken), word-pop (one word at a time, punchy scale-in), stack-up (words fade/rise in and stay), neon (active word glow pulse), bounce (active word bounces). Applies animation + colors + background treatment in one go; individual caption_*/color/background/background_style fields override specifics."),
|
|
4228
4433
|
caption_animation: z.enum(["highlight-pop", "karaoke-fill", "word-pop", "cumulative-reveal", "glow-pulse", "bounce-wave", "none"]).optional().describe("Word-by-word caption animation, if you want to set it directly instead of via caption_style. Works on set_captions, add_layer (kind=text), and set_layer_style. 'none' removes the animation and flattens the layer back to static text."),
|
|
4229
4434
|
caption_active_color: z.string().optional().describe("CSS color the active word takes while it is spoken (karaoke fill color, highlight text color, glow color)."),
|
|
@@ -4648,7 +4853,11 @@ function rewriteHyperframesDraftCompositionHtml(html, draftPath) {
|
|
|
4648
4853
|
}
|
|
4649
4854
|
}
|
|
4650
4855
|
// And the scene-transition keyframes: toggling data-transition on a clip
|
|
4651
|
-
// must animate immediately on older stored compositions.
|
|
4856
|
+
// must animate immediately on older stored compositions. Compositions that
|
|
4857
|
+
// embed the V1 block get it swapped for the current one in place; if the
|
|
4858
|
+
// swap misses (drifted block), the marker check below appends V2 after it,
|
|
4859
|
+
// which wins by source order.
|
|
4860
|
+
rewritten = upgradeLegacyTransitionCss(rewritten);
|
|
4652
4861
|
if (!rewritten.includes(TRANSITION_STYLE_MARKER)) {
|
|
4653
4862
|
const transitionTag = `<style data-vf-transitions="true">${TRANSITION_CSS}</style>`;
|
|
4654
4863
|
if (/<\/head>/i.test(rewritten)) {
|
|
@@ -4864,7 +5073,8 @@ app.get("/editor/:templateId/composition", async (c) => {
|
|
|
4864
5073
|
compositionId: templateId,
|
|
4865
5074
|
sourceUrl: template.videoUrl,
|
|
4866
5075
|
durationSeconds: template.durationSeconds ?? 12,
|
|
4867
|
-
title: template.title || `Template · ${template.sourceHost}
|
|
5076
|
+
title: template.title || `Template · ${template.sourceHost}`,
|
|
5077
|
+
origin: template.origin
|
|
4868
5078
|
}));
|
|
4869
5079
|
});
|
|
4870
5080
|
app.get("/help", async (c) => renderApprovedHelpPage(c));
|
|
@@ -4976,6 +5186,7 @@ function buildTemplateHomepageEntry(template) {
|
|
|
4976
5186
|
keywords: template.keywords ?? null,
|
|
4977
5187
|
summary: template.searchSummary ?? null,
|
|
4978
5188
|
visibility: template.visibility,
|
|
5189
|
+
origin: template.origin ?? "inspiration",
|
|
4979
5190
|
notes: template.notes,
|
|
4980
5191
|
status: "ready"
|
|
4981
5192
|
};
|
|
@@ -5128,6 +5339,7 @@ app.post("/discover/templates", async (c) => {
|
|
|
5128
5339
|
const explicitTitle = readNonEmptyString(body.title) ?? null;
|
|
5129
5340
|
const tagline = readNonEmptyString(body.tagline) ?? readNonEmptyString(body.trend_tagline) ?? null;
|
|
5130
5341
|
const notes = readNonEmptyString(body.notes) ?? null;
|
|
5342
|
+
const origin = readProjectOrigin(body.origin);
|
|
5131
5343
|
const uploadFileName = readNonEmptyString(uploadRef?.file_name) ?? null;
|
|
5132
5344
|
const primitive = primitiveRegistry.get("primitive:video_ingest");
|
|
5133
5345
|
if (!primitive)
|
|
@@ -5183,6 +5395,7 @@ app.post("/discover/templates", async (c) => {
|
|
|
5183
5395
|
downloadJobId: job.id,
|
|
5184
5396
|
trendTagline: tagline,
|
|
5185
5397
|
visibility: "private",
|
|
5398
|
+
origin,
|
|
5186
5399
|
notes
|
|
5187
5400
|
});
|
|
5188
5401
|
// Pending-card label while the ingest runs: explicit title, else the file
|
|
@@ -5340,6 +5553,7 @@ function buildPendingInspirationHomepageEntry(inspiration) {
|
|
|
5340
5553
|
sourceType: inspiration.sourceHost || "social",
|
|
5341
5554
|
durationSeconds: inspiration.durationSeconds ?? null,
|
|
5342
5555
|
visibility: "private",
|
|
5556
|
+
origin: inspiration.origin ?? "inspiration",
|
|
5343
5557
|
notes: inspiration.notes,
|
|
5344
5558
|
status: inspiration.status === "failed" ? "failed" : "processing"
|
|
5345
5559
|
};
|
|
@@ -5701,7 +5915,8 @@ app.get("/composition/current.html", async (c) => {
|
|
|
5701
5915
|
compositionId,
|
|
5702
5916
|
sourceUrl: template.videoUrl,
|
|
5703
5917
|
durationSeconds: template.durationSeconds ?? 12,
|
|
5704
|
-
title: template.title || `Template · ${template.sourceHost}
|
|
5918
|
+
title: template.title || `Template · ${template.sourceHost}`,
|
|
5919
|
+
origin: template.origin
|
|
5705
5920
|
});
|
|
5706
5921
|
const rewritten = rewriteHyperframesDraftCompositionHtml(rawHtml, `templates/${encodeURIComponent(template.id)}`);
|
|
5707
5922
|
return c.html(rewritten, 200, { "cache-control": "no-store" });
|
|
@@ -6212,7 +6427,8 @@ app.post(COMPOSITIONS_PREFIX, async (c) => {
|
|
|
6212
6427
|
compositionId: fork.id,
|
|
6213
6428
|
sourceUrl: template.videoUrl,
|
|
6214
6429
|
durationSeconds: template.durationSeconds ?? 12,
|
|
6215
|
-
title
|
|
6430
|
+
title,
|
|
6431
|
+
origin: template.origin
|
|
6216
6432
|
});
|
|
6217
6433
|
await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), rawHtml, "text/html; charset=utf-8");
|
|
6218
6434
|
}
|
|
@@ -6351,6 +6567,144 @@ app.put(`${COMPOSITIONS_PREFIX}/:forkId/composition.html`, async (c) => {
|
|
|
6351
6567
|
return forkAccessErrorResponse(c, error);
|
|
6352
6568
|
}
|
|
6353
6569
|
});
|
|
6570
|
+
// ONE-STEP animated captions: transcribe the fork's own narration (word-level
|
|
6571
|
+
// timings when an OpenAI key is saved) and write the animated caption cue
|
|
6572
|
+
// layers straight into the working composition.html. Runs on the caller's raw
|
|
6573
|
+
// provider keys (BYOK — never billed), synchronous (~5-20s of STT). The write
|
|
6574
|
+
// goes through the same sanitize+persist path as the editor's PUT, and is
|
|
6575
|
+
// deliberately NOT watch-suppressed so a local `vidfarm serve` live-morphs the
|
|
6576
|
+
// captions into open editor tabs. When an editor tab is open against the
|
|
6577
|
+
// CLOUD host, prefer the two-part flow instead (POST /api/v1/primitives/
|
|
6578
|
+
// audio/captions → editor_action set_captions) so the mutation flows through
|
|
6579
|
+
// the editor's own state and cannot be clobbered by its next auto-save.
|
|
6580
|
+
app.post(`${COMPOSITIONS_PREFIX}/:forkId/captions`, async (c) => {
|
|
6581
|
+
const customer = await getBrowserCustomer(c);
|
|
6582
|
+
if (!customer)
|
|
6583
|
+
return c.json({ ok: false, error: "Unauthorized" }, 401);
|
|
6584
|
+
try {
|
|
6585
|
+
const access = await assertForkAccess({ forkId: c.req.param("forkId"), customerId: customer.id, shareToken: readShareToken(c) }, "edit");
|
|
6586
|
+
const fork = access.fork;
|
|
6587
|
+
const html = await storage.readText(storage.compositionForkWorkingKey(fork.id, "composition.html"));
|
|
6588
|
+
if (!html)
|
|
6589
|
+
return c.json({ ok: false, error: "Composition missing" }, 404);
|
|
6590
|
+
const body = asRecord(await c.req.json().catch(() => ({}))) ?? {};
|
|
6591
|
+
const str = (key) => (typeof body[key] === "string" && body[key].trim() ? body[key].trim() : undefined);
|
|
6592
|
+
const num = (key) => (typeof body[key] === "number" && Number.isFinite(body[key]) ? body[key] : undefined);
|
|
6593
|
+
const parsed = parseHTML(html);
|
|
6594
|
+
const root = parsed.document.querySelector("[data-composition-id]");
|
|
6595
|
+
const compositionDuration = Number(root?.getAttribute("data-duration")) || 0;
|
|
6596
|
+
const maxWords = Math.max(1, Math.min(8, Math.round(num("max_words_per_cue") ?? 4)));
|
|
6597
|
+
const offsetSec = Math.max(0, num("offset_sec") ?? 0);
|
|
6598
|
+
let cues;
|
|
6599
|
+
let wordTimestampSource = "estimated";
|
|
6600
|
+
let sourceNote;
|
|
6601
|
+
const text = str("text");
|
|
6602
|
+
if (text) {
|
|
6603
|
+
const windowStart = Math.max(0, num("start") ?? 0);
|
|
6604
|
+
const windowDuration = num("duration") ?? Math.max(0.4, compositionDuration - windowStart);
|
|
6605
|
+
cues = cuesFromText(text, windowStart + offsetSec, windowDuration, { maxWordsPerCue: maxWords });
|
|
6606
|
+
sourceNote = "text";
|
|
6607
|
+
}
|
|
6608
|
+
else {
|
|
6609
|
+
// Narration source: explicit override → audio layer → backing playback
|
|
6610
|
+
// video → source video attr → any video layer.
|
|
6611
|
+
const timedNodes = Array.from(parsed.document.querySelectorAll("[data-start]"));
|
|
6612
|
+
const audioLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "audio" && (node.getAttribute("src") || "").trim());
|
|
6613
|
+
const videoLayer = timedNodes.find((node) => node.getAttribute("data-layer-kind") === "video" && (node.getAttribute("src") || "").trim());
|
|
6614
|
+
const backing = parsed.document.querySelector("[data-vf-playback-source]");
|
|
6615
|
+
const sourceUrl = str("audio_url") ?? str("source_url")
|
|
6616
|
+
?? audioLayer?.getAttribute("src")?.trim()
|
|
6617
|
+
?? backing?.getAttribute("src")?.trim()
|
|
6618
|
+
?? backing?.getAttribute("data-src")?.trim()
|
|
6619
|
+
?? root?.getAttribute("data-source-video")?.trim()
|
|
6620
|
+
?? videoLayer?.getAttribute("src")?.trim();
|
|
6621
|
+
if (!sourceUrl || !/^https?:\/\//i.test(sourceUrl)) {
|
|
6622
|
+
return c.json({
|
|
6623
|
+
ok: false,
|
|
6624
|
+
error: "No narration to transcribe — the composition has no audio/video layer with a usable URL. Pass audio_url (or text to page a script)."
|
|
6625
|
+
}, 400);
|
|
6626
|
+
}
|
|
6627
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
6628
|
+
const requested = str("provider");
|
|
6629
|
+
// openai first: whisper-1 is the only STT with real word-level timestamps.
|
|
6630
|
+
const provider = ["openai", "gemini", "openrouter"].find((name) => requested ? name === requested && keys[name] : Boolean(keys[name]));
|
|
6631
|
+
if (!provider) {
|
|
6632
|
+
return c.json({
|
|
6633
|
+
ok: false,
|
|
6634
|
+
error: requested
|
|
6635
|
+
? `No active ${requested} provider key is saved for this account.`
|
|
6636
|
+
: "Captions need an AI provider key for transcription. Add an OpenAI (best: real word timings), Gemini, or OpenRouter key in Settings."
|
|
6637
|
+
}, 400);
|
|
6638
|
+
}
|
|
6639
|
+
const audio = await loadSpeechAudioLocally(sourceUrl);
|
|
6640
|
+
if (audio.bytes.byteLength > MAX_LOCAL_TRANSCRIBE_AUDIO_BYTES) {
|
|
6641
|
+
return c.json({ ok: false, error: "Narration audio exceeds the ~40-minute transcription limit. Trim it or pass a shorter audio_url." }, 400);
|
|
6642
|
+
}
|
|
6643
|
+
const transcription = await transcribeSpeechWithKey({
|
|
6644
|
+
provider,
|
|
6645
|
+
model: str("model") ?? speechModelFor("stt", provider) ?? "",
|
|
6646
|
+
audio: audio.bytes,
|
|
6647
|
+
contentType: audio.contentType,
|
|
6648
|
+
language: str("language"),
|
|
6649
|
+
diarize: false,
|
|
6650
|
+
wordTimestamps: true,
|
|
6651
|
+
apiKey: keys[provider]
|
|
6652
|
+
});
|
|
6653
|
+
if (!transcription.segments.length) {
|
|
6654
|
+
return c.json({ ok: false, error: "Transcription returned no speech segments — the narration may be silent." }, 502);
|
|
6655
|
+
}
|
|
6656
|
+
wordTimestampSource = transcription.segments.some((segment) => segment.words?.length) ? "provider" : "estimated";
|
|
6657
|
+
cues = buildCaptionCues(transcription.segments, {
|
|
6658
|
+
maxWordsPerCue: maxWords,
|
|
6659
|
+
offsetSec,
|
|
6660
|
+
fallbackDurationSec: compositionDuration || undefined
|
|
6661
|
+
});
|
|
6662
|
+
sourceNote = sourceUrl;
|
|
6663
|
+
}
|
|
6664
|
+
if (compositionDuration > 0) {
|
|
6665
|
+
cues = cues
|
|
6666
|
+
.filter((cue) => cue.start < compositionDuration)
|
|
6667
|
+
.map((cue) => ({ ...cue, duration: Math.min(cue.duration, Math.max(0.1, compositionDuration - cue.start)) }));
|
|
6668
|
+
}
|
|
6669
|
+
if (!cues.length) {
|
|
6670
|
+
return c.json({ ok: false, error: "No caption cues produced." }, 502);
|
|
6671
|
+
}
|
|
6672
|
+
const result = insertCaptionLayers(html, cues, {
|
|
6673
|
+
style: str("style"),
|
|
6674
|
+
animation: str("animation"),
|
|
6675
|
+
activeColor: str("active_color"),
|
|
6676
|
+
highlightColor: str("highlight_color"),
|
|
6677
|
+
uppercase: body.uppercase === true ? true : body.uppercase === false ? false : undefined,
|
|
6678
|
+
color: str("color"),
|
|
6679
|
+
background: str("background"),
|
|
6680
|
+
backgroundStyle: str("background_style"),
|
|
6681
|
+
fontFamily: str("font_family"),
|
|
6682
|
+
fontSize: num("font_size"),
|
|
6683
|
+
track: num("track"),
|
|
6684
|
+
x: num("x"),
|
|
6685
|
+
y: num("y"),
|
|
6686
|
+
width: num("width"),
|
|
6687
|
+
height: num("height")
|
|
6688
|
+
});
|
|
6689
|
+
const { html: sanitized, removed } = sanitizeCompositionHtml(result.html);
|
|
6690
|
+
await storage.putText(storage.compositionForkWorkingKey(fork.id, "composition.html"), sanitized, "text/html; charset=utf-8");
|
|
6691
|
+
const updated = await serverlessRecords.updateCompositionFork({ forkId: fork.id, patch: {} });
|
|
6692
|
+
await writeForkManifest(storage, updated ?? fork, { kind: "working" });
|
|
6693
|
+
return c.json({
|
|
6694
|
+
ok: true,
|
|
6695
|
+
cue_count: cues.length,
|
|
6696
|
+
replaced: result.replaced,
|
|
6697
|
+
style: result.styleId,
|
|
6698
|
+
word_timestamps: wordTimestampSource,
|
|
6699
|
+
source: sourceNote,
|
|
6700
|
+
layer_keys: result.layerKeys,
|
|
6701
|
+
sanitized: removed
|
|
6702
|
+
});
|
|
6703
|
+
}
|
|
6704
|
+
catch (error) {
|
|
6705
|
+
return forkAccessErrorResponse(c, error);
|
|
6706
|
+
}
|
|
6707
|
+
});
|
|
6354
6708
|
// Same-origin live-reload stream (local `vidfarm serve` only). Broadcasts when
|
|
6355
6709
|
// an on-disk working composition file changes so open editor tabs live-morph
|
|
6356
6710
|
// the agent's edits. Replaces the old cross-origin dev-serve `/_events`.
|
|
@@ -8650,6 +9004,7 @@ function serializeInspiration(inspiration) {
|
|
|
8650
9004
|
keywords: inspiration.keywords ?? null,
|
|
8651
9005
|
summary: inspiration.searchSummary ?? null,
|
|
8652
9006
|
visibility: inspiration.visibility ?? "public",
|
|
9007
|
+
origin: inspiration.origin ?? "inspiration",
|
|
8653
9008
|
notes: inspiration.notes ?? null,
|
|
8654
9009
|
ingester_customer_id: inspiration.customerId,
|
|
8655
9010
|
error: inspiration.errorMessage,
|
|
@@ -8710,6 +9065,7 @@ async function finalizeInspirationIfReady(inspiration) {
|
|
|
8710
9065
|
songName: ready.songName,
|
|
8711
9066
|
songUrl: ready.songUrl,
|
|
8712
9067
|
visibility: "private",
|
|
9068
|
+
origin: ready.origin ?? "inspiration",
|
|
8713
9069
|
notes: ready.notes
|
|
8714
9070
|
});
|
|
8715
9071
|
ready = (await serverlessRecords.updateInspiration({
|
|
@@ -10808,7 +11164,8 @@ async function ensureInspirationReadyThenFork(input) {
|
|
|
10808
11164
|
videoUrl: inspiration.videoUrl,
|
|
10809
11165
|
durationSeconds: inspiration.durationSeconds,
|
|
10810
11166
|
songName: inspiration.songName,
|
|
10811
|
-
songUrl: inspiration.songUrl
|
|
11167
|
+
songUrl: inspiration.songUrl,
|
|
11168
|
+
origin: inspiration.origin ?? "inspiration"
|
|
10812
11169
|
});
|
|
10813
11170
|
await serverlessRecords.updateInspiration({
|
|
10814
11171
|
inspirationId: inspiration.id,
|
|
@@ -10835,6 +11192,10 @@ function buildRawTikTokCompositionHtml(input) {
|
|
|
10835
11192
|
const duration = Number.isFinite(input.durationSeconds) && input.durationSeconds > 0
|
|
10836
11193
|
? Number(input.durationSeconds.toFixed(3))
|
|
10837
11194
|
: 12;
|
|
11195
|
+
// data-vf-origin travels with the document through fork copies; the editor
|
|
11196
|
+
// reads it to decide whether to auto-prompt the Decompose modal ("raw"
|
|
11197
|
+
// projects never prompt). Absent means "inspiration" (pre-field documents).
|
|
11198
|
+
const origin = input.origin === "raw" ? "raw" : "inspiration";
|
|
10838
11199
|
return `<!doctype html>
|
|
10839
11200
|
<html lang="en">
|
|
10840
11201
|
<head>
|
|
@@ -10847,7 +11208,7 @@ function buildRawTikTokCompositionHtml(input) {
|
|
|
10847
11208
|
</style>
|
|
10848
11209
|
</head>
|
|
10849
11210
|
<body>
|
|
10850
|
-
<div data-composition-id="${escapeAttribute(input.compositionId)}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
|
|
11211
|
+
<div data-composition-id="${escapeAttribute(input.compositionId)}" data-vf-origin="${origin}" data-source-video="${escapeAttribute(input.sourceUrl)}" data-width="720" data-height="1280" data-duration="${duration}" data-source-title="${escapeAttribute(input.title)}">
|
|
10851
11212
|
<video id="vf-source-video" data-hf-id="vf-source-video" data-vf-playback-source="true" data-layer-kind="video" data-start="0" data-duration="${duration}" data-track-index="0" data-label="Original video" data-timeline-label="Original video" src="${escapeAttribute(input.sourceUrl)}" muted playsinline preload="auto" style="position:absolute;left:0%;top:0%;width:100%;height:100%;z-index:0;object-fit:cover;background:#050604"></video>
|
|
10852
11213
|
</div>
|
|
10853
11214
|
</body>
|
|
@@ -13823,7 +14184,8 @@ const AI_PRIMITIVE_IDS = new Set([
|
|
|
13823
14184
|
"primitive:image_inpaint",
|
|
13824
14185
|
"primitive:video_generate",
|
|
13825
14186
|
"primitive:tts",
|
|
13826
|
-
"primitive:stt"
|
|
14187
|
+
"primitive:stt",
|
|
14188
|
+
"primitive:captions"
|
|
13827
14189
|
]);
|
|
13828
14190
|
async function preflightAiProviderKeys(input) {
|
|
13829
14191
|
if (!operationMayUseAiProvider(input)) {
|
|
@@ -14346,6 +14708,9 @@ app.post(`${PRIMITIVES_PREFIX}/audio/speech`, async (c) => createPrimitiveJob(c,
|
|
|
14346
14708
|
app.post(`${PRIMITIVES_PREFIX}/tts`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:tts", operationName: "run" }));
|
|
14347
14709
|
app.post(`${PRIMITIVES_PREFIX}/audio/transcribe`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
14348
14710
|
app.post(`${PRIMITIVES_PREFIX}/stt`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
14711
|
+
// Animated-caption cues from narration (STT + word timings + cue paging).
|
|
14712
|
+
app.post(`${PRIMITIVES_PREFIX}/audio/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
14713
|
+
app.post(`${PRIMITIVES_PREFIX}/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
14349
14714
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/hooks`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_hooks", operationName: "run" }));
|
|
14350
14715
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/awareness_stages`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_awareness_stages", operationName: "run" }));
|
|
14351
14716
|
app.post(`${PRIMITIVES_PREFIX}/brainstorm/angles`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:brainstorm_angles", operationName: "run" }));
|
|
@@ -14573,6 +14938,52 @@ app.get(`${USER_PREFIX}/me/generated-media`, async (c) => {
|
|
|
14573
14938
|
next_cursor: page.nextCursor
|
|
14574
14939
|
});
|
|
14575
14940
|
});
|
|
14941
|
+
// Embedder for My Files notes — the same BYOK seam clip search uses. Prefer the
|
|
14942
|
+
// provider matching the embedding model the file library was built with (from
|
|
14943
|
+
// the newest annotated attachment) so query vectors stay cosine-comparable; a
|
|
14944
|
+
// fresh library takes the first embed-capable key (gemini → openai; openrouter
|
|
14945
|
+
// can't embed). Null (no key) fail-softs notes search to keyword-only.
|
|
14946
|
+
async function buildFileNotesEmbedder(customerId) {
|
|
14947
|
+
const attachments = await serverlessRecords.listUserAttachments(customerId);
|
|
14948
|
+
const libModel = attachments.find((a) => a.notesEmbeddingModel && Array.isArray(a.notesEmbedding) && a.notesEmbedding.length > 0)?.notesEmbeddingModel ?? null;
|
|
14949
|
+
const providers = libModel ? [libModel.startsWith("text-embedding") ? "openai" : "gemini"] : ["gemini", "openai"];
|
|
14950
|
+
for (const provider of providers) {
|
|
14951
|
+
const apiKey = await resolveCustomerProviderKey(customerId, provider);
|
|
14952
|
+
if (!apiKey)
|
|
14953
|
+
continue;
|
|
14954
|
+
const client = new ClipModelClient({ provider, apiKey });
|
|
14955
|
+
if (client.canEmbed)
|
|
14956
|
+
return client;
|
|
14957
|
+
}
|
|
14958
|
+
return null;
|
|
14959
|
+
}
|
|
14960
|
+
// What we embed for a My Files entry: folder + file name + the notes, so
|
|
14961
|
+
// "the sprite card for our mascot" is findable from any phrasing. Fail-soft:
|
|
14962
|
+
// no key or an embed error still saves the notes (keyword-searchable).
|
|
14963
|
+
async function embedAttachmentNotes(customerId, input) {
|
|
14964
|
+
if (!input.notes.trim())
|
|
14965
|
+
return null;
|
|
14966
|
+
try {
|
|
14967
|
+
const embedder = await buildFileNotesEmbedder(customerId);
|
|
14968
|
+
if (!embedder?.embeddingModelId)
|
|
14969
|
+
return null;
|
|
14970
|
+
const text = [input.folderPath, input.fileName, input.notes].filter(Boolean).join("\n");
|
|
14971
|
+
const embedding = await embedder.embedQuery(text);
|
|
14972
|
+
return embedding && embedding.length > 0 ? { embedding, model: embedder.embeddingModelId } : null;
|
|
14973
|
+
}
|
|
14974
|
+
catch (error) {
|
|
14975
|
+
devLog("attachment_notes.embed_failed", devErrorFields(error));
|
|
14976
|
+
return null;
|
|
14977
|
+
}
|
|
14978
|
+
}
|
|
14979
|
+
function attachmentKeywordMatch(attachment, q) {
|
|
14980
|
+
const hay = `${attachment.fileName} ${attachment.folderPath ?? ""} ${attachment.notes ?? ""}`.toLowerCase();
|
|
14981
|
+
return q
|
|
14982
|
+
.toLowerCase()
|
|
14983
|
+
.split(/\s+/)
|
|
14984
|
+
.filter(Boolean)
|
|
14985
|
+
.every((term) => hay.includes(term));
|
|
14986
|
+
}
|
|
14576
14987
|
app.get(`${USER_PREFIX}/me/attachments`, async (c) => {
|
|
14577
14988
|
const customer = requireCustomer(c);
|
|
14578
14989
|
const attachments = (await serverlessRecords.listUserAttachments(customer.id)).map((attachment) => serializeUserAttachment(c, attachment));
|
|
@@ -14651,6 +15062,7 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
|
|
|
14651
15062
|
const fileName = sanitizeFileName(file.name || "upload.bin");
|
|
14652
15063
|
const contentType = file.type || inferAttachmentContentType(fileName);
|
|
14653
15064
|
const folderPath = sanitizeStorageSubpath(String(formData.get("folder_path") ?? ""));
|
|
15065
|
+
const notes = String(formData.get("notes") ?? "").trim().slice(0, 4000) || null;
|
|
14654
15066
|
if (!isAllowedAttachment(fileName, contentType)) {
|
|
14655
15067
|
return c.json({ error: `Unsupported attachment type: ${fileName}` }, 400);
|
|
14656
15068
|
}
|
|
@@ -14662,6 +15074,7 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
|
|
|
14662
15074
|
const attachmentId = randomUUID();
|
|
14663
15075
|
const storageKey = storage.userAttachmentKey(customer.id, attachmentId, fileName, folderPath);
|
|
14664
15076
|
const stored = await storage.putBuffer(storageKey, buffer, contentType);
|
|
15077
|
+
const embedded = notes ? await embedAttachmentNotes(customer.id, { fileName, folderPath, notes }) : null;
|
|
14665
15078
|
await serverlessRecords.createUserAttachment({
|
|
14666
15079
|
id: attachmentId,
|
|
14667
15080
|
customerId: customer.id,
|
|
@@ -14670,7 +15083,10 @@ app.post(`${USER_PREFIX}/me/attachments/upload`, async (c) => {
|
|
|
14670
15083
|
sizeBytes: buffer.byteLength,
|
|
14671
15084
|
storageKey,
|
|
14672
15085
|
folderPath,
|
|
14673
|
-
publicUrl: stored.url || storage.getPublicUrl(storageKey) || buildUserAttachmentViewUrl(c, storageKey)
|
|
15086
|
+
publicUrl: stored.url || storage.getPublicUrl(storageKey) || buildUserAttachmentViewUrl(c, storageKey),
|
|
15087
|
+
notes,
|
|
15088
|
+
notesEmbedding: embedded?.embedding ?? null,
|
|
15089
|
+
notesEmbeddingModel: embedded?.model ?? null
|
|
14674
15090
|
});
|
|
14675
15091
|
const attachment = await serverlessRecords.getUserAttachment(customer.id, attachmentId);
|
|
14676
15092
|
if (!attachment) {
|
|
@@ -14699,6 +15115,8 @@ app.post(`${USER_PREFIX}/me/attachments`, async (c) => {
|
|
|
14699
15115
|
if (!isAllowedAttachment(fileName, contentType)) {
|
|
14700
15116
|
return c.json({ error: `Unsupported attachment type: ${fileName}` }, 400);
|
|
14701
15117
|
}
|
|
15118
|
+
const notes = body.notes?.trim() || null;
|
|
15119
|
+
const embedded = notes ? await embedAttachmentNotes(customer.id, { fileName, folderPath, notes }) : null;
|
|
14702
15120
|
await serverlessRecords.createUserAttachment({
|
|
14703
15121
|
id: body.attachment_id,
|
|
14704
15122
|
customerId: customer.id,
|
|
@@ -14707,7 +15125,10 @@ app.post(`${USER_PREFIX}/me/attachments`, async (c) => {
|
|
|
14707
15125
|
sizeBytes: body.size_bytes,
|
|
14708
15126
|
storageKey: expectedStorageKey,
|
|
14709
15127
|
folderPath,
|
|
14710
|
-
publicUrl: storage.getPublicUrl(expectedStorageKey) || buildUserAttachmentViewUrl(c, expectedStorageKey)
|
|
15128
|
+
publicUrl: storage.getPublicUrl(expectedStorageKey) || buildUserAttachmentViewUrl(c, expectedStorageKey),
|
|
15129
|
+
notes,
|
|
15130
|
+
notesEmbedding: embedded?.embedding ?? null,
|
|
15131
|
+
notesEmbeddingModel: embedded?.model ?? null
|
|
14711
15132
|
});
|
|
14712
15133
|
const attachment = await serverlessRecords.getUserAttachment(customer.id, body.attachment_id);
|
|
14713
15134
|
if (!attachment) {
|
|
@@ -14741,6 +15162,89 @@ app.patch(`${USER_PREFIX}/me/attachments/folders`, async (c) => {
|
|
|
14741
15162
|
await serverlessRecords.moveUserAttachmentFolder(customer.id, fromFolder, toFolder);
|
|
14742
15163
|
return c.json({ ok: true, from_folder_path: fromFolder, to_folder_path: toFolder });
|
|
14743
15164
|
});
|
|
15165
|
+
// PATCH /me/attachments/:attachmentId — set/clear the metadata notes on one My
|
|
15166
|
+
// Files entry and (re)embed them for vector search. Notes are what make assets
|
|
15167
|
+
// findable by MEANING later — e.g. annotate a character_sprite_card.png with
|
|
15168
|
+
// who the character is so /me/attachments/search finds it from any phrasing.
|
|
15169
|
+
// (Registered after the /folders PATCH so "folders" never matches as an id.)
|
|
15170
|
+
app.patch(`${USER_PREFIX}/me/attachments/:attachmentId`, async (c) => {
|
|
15171
|
+
const customer = requireCustomer(c);
|
|
15172
|
+
try {
|
|
15173
|
+
const body = userAttachmentNotesSchema.parse(await c.req.json());
|
|
15174
|
+
const attachment = await serverlessRecords.getUserAttachment(customer.id, c.req.param("attachmentId"));
|
|
15175
|
+
if (!attachment) {
|
|
15176
|
+
return c.json({ error: "Attachment not found." }, 404);
|
|
15177
|
+
}
|
|
15178
|
+
const notes = (body.notes ?? "").trim() || null;
|
|
15179
|
+
const embedded = notes
|
|
15180
|
+
? await embedAttachmentNotes(customer.id, { fileName: attachment.fileName, folderPath: attachment.folderPath, notes })
|
|
15181
|
+
: null;
|
|
15182
|
+
const updated = await serverlessRecords.updateUserAttachmentNotes(customer.id, attachment.id, {
|
|
15183
|
+
notes,
|
|
15184
|
+
notesEmbedding: embedded?.embedding ?? null,
|
|
15185
|
+
notesEmbeddingModel: embedded?.model ?? null
|
|
15186
|
+
});
|
|
15187
|
+
if (!updated) {
|
|
15188
|
+
return c.json({ error: "Attachment not found." }, 404);
|
|
15189
|
+
}
|
|
15190
|
+
return c.json({ ok: true, attachment: serializeUserAttachment(c, updated), notes_embedded: Boolean(embedded) });
|
|
15191
|
+
}
|
|
15192
|
+
catch (error) {
|
|
15193
|
+
return c.json({ error: error instanceof Error ? error.message : "Unable to update attachment notes." }, 400);
|
|
15194
|
+
}
|
|
15195
|
+
});
|
|
15196
|
+
// POST /me/attachments/search — find My Files entries by meaning, mirroring
|
|
15197
|
+
// /clips/search: keyword filter over name/folder/notes, semantically re-ranked
|
|
15198
|
+
// via the notes embeddings; when nothing keyword-matches, relax to a pure
|
|
15199
|
+
// semantic pass. Embeddings are BYOK — no saved key fail-softs to keyword-only.
|
|
15200
|
+
app.post(`${USER_PREFIX}/me/attachments/search`, async (c) => {
|
|
15201
|
+
const customer = requireCustomer(c);
|
|
15202
|
+
try {
|
|
15203
|
+
const body = userAttachmentSearchSchema.parse(await c.req.json());
|
|
15204
|
+
const query = body.query.trim();
|
|
15205
|
+
if (!query) {
|
|
15206
|
+
return c.json({ error: "Provide a search query." }, 400);
|
|
15207
|
+
}
|
|
15208
|
+
const folderPath = sanitizeStorageSubpath(body.folder_path);
|
|
15209
|
+
const limit = Math.max(1, Math.min(100, body.limit ?? 20));
|
|
15210
|
+
const all = await serverlessRecords.listUserAttachments(customer.id);
|
|
15211
|
+
const scoped = folderPath ? all.filter((a) => (a.folderPath ?? "") === folderPath) : all;
|
|
15212
|
+
let queryEmbedding;
|
|
15213
|
+
try {
|
|
15214
|
+
const embedder = await buildFileNotesEmbedder(customer.id);
|
|
15215
|
+
if (embedder)
|
|
15216
|
+
queryEmbedding = await embedder.embedQuery(query);
|
|
15217
|
+
}
|
|
15218
|
+
catch (error) {
|
|
15219
|
+
devLog("attachment_search.embed_failed", devErrorFields(error));
|
|
15220
|
+
}
|
|
15221
|
+
const keywordIds = new Set(scoped.filter((a) => attachmentKeywordMatch(a, query)).map((a) => a.id));
|
|
15222
|
+
const pool = keywordIds.size > 0 ? scoped.filter((a) => keywordIds.has(a.id)) : scoped;
|
|
15223
|
+
const hits = pool
|
|
15224
|
+
.map((a) => ({
|
|
15225
|
+
attachment: a,
|
|
15226
|
+
keyword: keywordIds.has(a.id),
|
|
15227
|
+
similarity: queryEmbedding && Array.isArray(a.notesEmbedding) && a.notesEmbedding.length > 0
|
|
15228
|
+
? cosineSimilarity(a.notesEmbedding, queryEmbedding)
|
|
15229
|
+
: undefined
|
|
15230
|
+
}))
|
|
15231
|
+
.filter((h) => h.keyword || (h.similarity ?? 0) > 0)
|
|
15232
|
+
.sort((x, y) => ((y.similarity ?? 0) + (y.keyword ? 0.2 : 0)) - ((x.similarity ?? 0) + (x.keyword ? 0.2 : 0)))
|
|
15233
|
+
.slice(0, limit);
|
|
15234
|
+
return c.json({
|
|
15235
|
+
results: hits.map((h) => ({
|
|
15236
|
+
...serializeUserAttachment(c, h.attachment),
|
|
15237
|
+
similarity: h.similarity ?? null,
|
|
15238
|
+
keyword_match: h.keyword
|
|
15239
|
+
})),
|
|
15240
|
+
semantic: Boolean(queryEmbedding),
|
|
15241
|
+
folder_path: folderPath || null
|
|
15242
|
+
});
|
|
15243
|
+
}
|
|
15244
|
+
catch (error) {
|
|
15245
|
+
return c.json({ error: error instanceof Error ? error.message : "My Files search failed." }, 400);
|
|
15246
|
+
}
|
|
15247
|
+
});
|
|
14744
15248
|
app.delete(`${USER_PREFIX}/me/attachments/folders`, async (c) => {
|
|
14745
15249
|
const customer = requireCustomer(c);
|
|
14746
15250
|
const body = userFileFolderSchema.parse(await c.req.json());
|
|
@@ -15312,25 +15816,39 @@ function renderClipsPage(input) {
|
|
|
15312
15816
|
.clips-import-head p { font-size: 0.9rem; }
|
|
15313
15817
|
.clips-import-field { display: grid; gap: 6px; }
|
|
15314
15818
|
.clips-import-field textarea { min-height: 96px; border-radius: 18px; }
|
|
15819
|
+
.clips-import-field select { width: 100%; }
|
|
15820
|
+
.clips-import-hint { font-size: 0.78rem; font-weight: 400; color: var(--muted); }
|
|
15821
|
+
.clips-import-hint a { color: inherit; text-decoration: underline; }
|
|
15315
15822
|
.clips-import-note { min-height: 18px; font-size: 0.85rem; color: var(--muted); }
|
|
15316
15823
|
.clips-import-note[data-tone="error"] { color: #9f3d3d; }
|
|
15317
15824
|
.clips-import-note[data-tone="success"] { color: #3f7d55; }
|
|
15318
15825
|
.clips-import-actions { display: flex; justify-content: flex-end; gap: 10px; }
|
|
15319
15826
|
|
|
15320
|
-
@media (max-width: 1180px) {
|
|
15827
|
+
@media (min-width: 1025px) and (max-width: 1180px) {
|
|
15828
|
+
/* Rail stacks inline below the content in this band only; at ≤1024px
|
|
15829
|
+
the shared shell turns the rail into the FAB + popup overlay. */
|
|
15321
15830
|
.clips-frame { padding-left: 0; }
|
|
15322
15831
|
.editor-right-rail { position: relative; inset: auto; width: 100%; height: 540px; margin-top: 14px; border-radius: 24px; overflow: hidden; }
|
|
15323
15832
|
.editor-chat-panel { border-right: 0; border-radius: 24px; }
|
|
15324
|
-
}
|
|
15325
|
-
@media (min-width: 1025px) and (max-width: 1180px) {
|
|
15326
15833
|
html, body { height: auto; min-height: 100%; overflow: auto; overflow-x: hidden; }
|
|
15327
15834
|
.app-shell { height: auto; min-height: 100svh; }
|
|
15328
15835
|
.page-stack, .frame { height: auto; min-height: 0; overflow: visible; }
|
|
15329
15836
|
.clips-scroll { overflow: visible; padding-right: 0; }
|
|
15330
15837
|
}
|
|
15331
15838
|
@media (max-width: 1024px) {
|
|
15839
|
+
.clips-frame { padding-left: 0; }
|
|
15332
15840
|
.clips-shell { grid-template-rows: auto auto; }
|
|
15333
15841
|
.clips-scroll { overflow: visible; padding-right: 0; }
|
|
15842
|
+
.clips-toolbar-row { gap: 8px; }
|
|
15843
|
+
.clips-search { flex: 1 1 100%; min-width: 0; }
|
|
15844
|
+
.clips-search input { min-width: 0; font-size: 16px; }
|
|
15845
|
+
.clips-import-form input,
|
|
15846
|
+
.clips-import-form textarea,
|
|
15847
|
+
.clips-import-form select { font-size: 16px; }
|
|
15848
|
+
}
|
|
15849
|
+
@media (max-width: 560px) {
|
|
15850
|
+
.clips-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
|
|
15851
|
+
.clip-body { padding: 10px; }
|
|
15334
15852
|
}
|
|
15335
15853
|
`,
|
|
15336
15854
|
body: `
|
|
@@ -15379,6 +15897,20 @@ function renderClipsPage(input) {
|
|
|
15379
15897
|
<label for="importPrompt">What to clip for</label>
|
|
15380
15898
|
<textarea id="importPrompt" name="prompt" placeholder="Clip scenes of people with food, no text on screen, vertical, between 5-10 secs long"></textarea>
|
|
15381
15899
|
</div>
|
|
15900
|
+
<div class="clips-import-field">
|
|
15901
|
+
<label for="importProvider">AI provider</label>
|
|
15902
|
+
<select id="importProvider" name="provider">
|
|
15903
|
+
<option value="">Auto — use my saved keys</option>
|
|
15904
|
+
<option value="gemini">Gemini</option>
|
|
15905
|
+
<option value="openai">OpenAI</option>
|
|
15906
|
+
<option value="openrouter">OpenRouter</option>
|
|
15907
|
+
</select>
|
|
15908
|
+
<div class="clips-import-hint" id="importProviderHint"></div>
|
|
15909
|
+
</div>
|
|
15910
|
+
<div class="clips-import-field">
|
|
15911
|
+
<label for="importTracer">Tracer <span class="clips-import-hint">optional — rolls up this hunt's billing and clips under one id</span></label>
|
|
15912
|
+
<input id="importTracer" name="tracer" type="text" placeholder="my-campaign-001" autocomplete="off" spellcheck="false" />
|
|
15913
|
+
</div>
|
|
15382
15914
|
<div class="clips-import-note" id="importNote"></div>
|
|
15383
15915
|
<div class="clips-import-actions">
|
|
15384
15916
|
<button type="button" class="secondary" id="importCancel">Cancel</button>
|
|
@@ -15468,18 +16000,58 @@ function renderClipsPage(input) {
|
|
|
15468
16000
|
var importForm = document.getElementById('importForm');
|
|
15469
16001
|
var importUrl = document.getElementById('importUrl');
|
|
15470
16002
|
var importPrompt = document.getElementById('importPrompt');
|
|
16003
|
+
var importTracer = document.getElementById('importTracer');
|
|
16004
|
+
var importProvider = document.getElementById('importProvider');
|
|
16005
|
+
var importProviderHint = document.getElementById('importProviderHint');
|
|
15471
16006
|
var importNote = document.getElementById('importNote');
|
|
15472
16007
|
var importCancel = document.getElementById('importCancel');
|
|
15473
16008
|
var importSubmit = document.getElementById('importSubmit');
|
|
15474
16009
|
var scanPoll = null;
|
|
16010
|
+
var scanOptions = null;
|
|
15475
16011
|
|
|
15476
16012
|
function setNote(msg, tone){
|
|
15477
16013
|
if(!importNote) return;
|
|
15478
16014
|
importNote.textContent = msg || '';
|
|
15479
16015
|
if(tone) importNote.setAttribute('data-tone', tone); else importNote.removeAttribute('data-tone');
|
|
15480
16016
|
}
|
|
16017
|
+
function providerName(p){ return p==='gemini'?'Gemini':p==='openai'?'OpenAI':p==='openrouter'?'OpenRouter':p; }
|
|
16018
|
+
function renderProviderOptions(){
|
|
16019
|
+
if(!importProvider || !scanOptions) return;
|
|
16020
|
+
var d = scanOptions;
|
|
16021
|
+
var hasAnyKey = (d.providers||[]).some(function(p){ return p.has_key; });
|
|
16022
|
+
var autoLabel = 'Auto';
|
|
16023
|
+
if(d.default_provider === 'agent') autoLabel = 'Auto — local '+((d.local_agent&&d.local_agent.kind)||'agent')+' CLI (no API key)';
|
|
16024
|
+
else if(d.default_provider) autoLabel = 'Auto — saved '+providerName(d.default_provider)+' key';
|
|
16025
|
+
else if(d.pipeline === 'upstream') autoLabel = 'Auto — linked cloud account';
|
|
16026
|
+
var html = '<option value="">'+esc(autoLabel)+'</option>';
|
|
16027
|
+
if(d.local_agent) html += '<option value="agent">Local agent ('+esc(d.local_agent.kind)+') — your CLI subscription, no API key</option>';
|
|
16028
|
+
(d.providers||[]).forEach(function(p){
|
|
16029
|
+
html += '<option value="'+esc(p.provider)+'"'+(p.has_key?'':' disabled')+'>'+esc(providerName(p.provider))+(p.has_key?' — key saved':' — no key saved')+'</option>';
|
|
16030
|
+
});
|
|
16031
|
+
importProvider.innerHTML = html;
|
|
16032
|
+
if(!importProviderHint) return;
|
|
16033
|
+
if(!hasAnyKey && !d.local_agent && d.pipeline !== 'upstream'){
|
|
16034
|
+
importProviderHint.innerHTML = 'No AI provider available yet — add a Gemini, OpenAI, or OpenRouter key in <a href="/settings" target="_blank" rel="noopener">Settings</a>'
|
|
16035
|
+
+ (d.pipeline === 'cloud' ? '.' : ', or install the Claude Code / Codex CLI for keyless local scans.');
|
|
16036
|
+
} else if(d.pipeline === 'upstream'){
|
|
16037
|
+
importProviderHint.textContent = 'No local key or agent CLI — this hunt will run on your linked cloud account (its saved keys, cloud compute).';
|
|
16038
|
+
} else if(d.pipeline === 'local'){
|
|
16039
|
+
importProviderHint.textContent = d.default_provider === 'agent'
|
|
16040
|
+
? 'Runs on this machine with your local agent CLI — no API key needed. Or pick a saved key.'
|
|
16041
|
+
: 'Runs on this machine with your saved key (BYOK — billed to your own provider account).';
|
|
16042
|
+
} else {
|
|
16043
|
+
importProviderHint.textContent = 'AI tagging runs on your own key (BYOK) — pick a provider or let Auto choose.';
|
|
16044
|
+
}
|
|
16045
|
+
}
|
|
16046
|
+
function loadScanOptions(){
|
|
16047
|
+
fetch('/clips/scan-options', {credentials:'same-origin', headers:{Accept:'application/json'}})
|
|
16048
|
+
.then(function(r){ return r.ok ? r.json() : null; })
|
|
16049
|
+
.then(function(d){ if(d){ scanOptions = d; renderProviderOptions(); } })
|
|
16050
|
+
.catch(function(){});
|
|
16051
|
+
}
|
|
15481
16052
|
function openImport(){
|
|
15482
16053
|
setNote('');
|
|
16054
|
+
if(!scanOptions) loadScanOptions();
|
|
15483
16055
|
if(importDialog && typeof importDialog.showModal === 'function' && !importDialog.open) importDialog.showModal();
|
|
15484
16056
|
if(importUrl) importUrl.focus();
|
|
15485
16057
|
}
|
|
@@ -15522,16 +16094,21 @@ function renderClipsPage(input) {
|
|
|
15522
16094
|
e.preventDefault();
|
|
15523
16095
|
var url = (importUrl && importUrl.value || '').trim();
|
|
15524
16096
|
var prompt = (importPrompt && importPrompt.value || '').trim();
|
|
16097
|
+
var tracer = (importTracer && importTracer.value || '').trim();
|
|
16098
|
+
var provider = (importProvider && importProvider.value || '').trim();
|
|
15525
16099
|
if(!url){ setNote('Enter a video URL.', 'error'); return; }
|
|
15526
16100
|
setNote('Starting scan…');
|
|
15527
16101
|
if(importSubmit) importSubmit.disabled = true;
|
|
15528
|
-
|
|
16102
|
+
var payload = {source_url:url, prompt:prompt};
|
|
16103
|
+
if(tracer) payload.tracer = tracer;
|
|
16104
|
+
if(provider) payload.provider = provider;
|
|
16105
|
+
fetch('/clips/scan', {method:'POST', credentials:'same-origin', headers:{'Content-Type':'application/json', Accept:'application/json'}, body: JSON.stringify(payload)})
|
|
15529
16106
|
.then(function(r){ return r.json().then(function(d){ return {ok:r.ok, status:r.status, data:d}; }); })
|
|
15530
16107
|
.then(function(res){
|
|
15531
16108
|
if(importSubmit) importSubmit.disabled = false;
|
|
15532
16109
|
if(!res.ok){ setNote((res.data && res.data.error) || 'Could not start the scan.', 'error'); return; }
|
|
15533
16110
|
setNote('Scan started — this can take a few minutes.', 'success');
|
|
15534
|
-
statusEl.textContent='Scanning source…';
|
|
16111
|
+
statusEl.textContent='Scanning source…'+(res.data && res.data.tracer ? ' (tracer '+res.data.tracer+')' : '');
|
|
15535
16112
|
closeImport();
|
|
15536
16113
|
if(importForm.reset) importForm.reset();
|
|
15537
16114
|
if(res.data && res.data.scan_id) pollScan(res.data.scan_id);
|
|
@@ -15559,9 +16136,23 @@ async function resolveCustomerProviderKey(customerId, provider) {
|
|
|
15559
16136
|
}
|
|
15560
16137
|
return null;
|
|
15561
16138
|
}
|
|
15562
|
-
//
|
|
15563
|
-
|
|
15564
|
-
|
|
16139
|
+
// The BYOK API providers clip hunts can run on, in auto-pick preference order.
|
|
16140
|
+
const CLIP_API_PROVIDERS = ["gemini", "openai", "openrouter"];
|
|
16141
|
+
const NO_CLIP_PROVIDER_KEY_ERROR = "No AI provider key saved. Add a gemini, openai, or openrouter key (Settings → AI Provider Keys, or POST /api/v1/user/me/provider-keys) and retry.";
|
|
16142
|
+
// An explicitly requested provider — "agent" means the caller's local
|
|
16143
|
+
// claude/codex CLI (vidfarm serve boxes only) — or null for auto-pick.
|
|
16144
|
+
function readClipProviderChoice(value) {
|
|
16145
|
+
if (value === "gemini" || value === "openai" || value === "openrouter")
|
|
16146
|
+
return value;
|
|
16147
|
+
if (value === "agent" || value === "local" || value === "local-agent")
|
|
16148
|
+
return "agent";
|
|
16149
|
+
return null;
|
|
16150
|
+
}
|
|
16151
|
+
// Auto-pick: the first API provider the customer holds an active key for
|
|
16152
|
+
// (gemini → openai → openrouter) — no more hard gemini default.
|
|
16153
|
+
async function pickClipProviderFromKeys(customerId) {
|
|
16154
|
+
const keys = await loadCustomerVisionKeys(customerId);
|
|
16155
|
+
return CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
|
|
15565
16156
|
}
|
|
15566
16157
|
/**
|
|
15567
16158
|
* A client for embedding MATCH queries in the SAME vector space the library was
|
|
@@ -15617,7 +16208,8 @@ function parseClipStorageKey(pathOrUri) {
|
|
|
15617
16208
|
async function clipMediaUrl(pathOrUri) {
|
|
15618
16209
|
if (!pathOrUri)
|
|
15619
16210
|
return null;
|
|
15620
|
-
|
|
16211
|
+
// s3://… (cloud hunts) or a bare clips/thumbs storage key (serve-box local hunts).
|
|
16212
|
+
const key = parseClipStorageKey(pathOrUri) ?? (/^(clips|thumbs)\//.test(pathOrUri) ? pathOrUri : null);
|
|
15621
16213
|
if (key)
|
|
15622
16214
|
return (await storage.getReadUrl(key)) ?? storage.getPublicUrl(key);
|
|
15623
16215
|
return pathOrUri; // non-s3 value (e.g. a local devcli-origin path) — pass through
|
|
@@ -15677,11 +16269,14 @@ app.post(`${CLIPS_PREFIX}/search`, async (c) => {
|
|
|
15677
16269
|
try {
|
|
15678
16270
|
const body = (await c.req.json().catch(() => ({})));
|
|
15679
16271
|
const limit = Math.max(1, Math.min(200, Number(body.limit) || 20));
|
|
15680
|
-
const
|
|
16272
|
+
const requested = readClipProviderChoice(body.provider);
|
|
15681
16273
|
let criteria = body.criteria ?? (body.query ? { semantic_text: body.query } : {});
|
|
15682
16274
|
let queryEmbedding;
|
|
15683
16275
|
if (body.query || criteria.semantic_text) {
|
|
15684
|
-
|
|
16276
|
+
// Any saved key works for NL→criteria + query embedding; no key at all
|
|
16277
|
+
// fail-softs to the structured/keyword pass below.
|
|
16278
|
+
const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
16279
|
+
const built = provider ? await buildCustomerClipClient(customer.id, provider) : { error: NO_CLIP_PROVIDER_KEY_ERROR };
|
|
15685
16280
|
if ("client" in built) {
|
|
15686
16281
|
const client = built.client;
|
|
15687
16282
|
if (body.query && !body.criteria)
|
|
@@ -15837,72 +16432,106 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
|
|
|
15837
16432
|
}
|
|
15838
16433
|
const rawS3Uri = body.raw_s3_uri ??
|
|
15839
16434
|
(resolvedS3Key && config.AWS_S3_BUCKET ? `s3://${config.AWS_S3_BUCKET}/${resolvedS3Key}` : undefined);
|
|
15840
|
-
|
|
16435
|
+
// On a serve box (no bucket) resolvedS3Key is a LOCAL storage key — still a valid source.
|
|
16436
|
+
if (!rawS3Uri && !resolvedS3Key && !huntSpec.source_url) {
|
|
15841
16437
|
return c.json({ error: "Provide source_url (a video URL), temp_file_id/attachment_id of an upload, raw_s3_uri (s3://…), or s3_key of the staged source video." }, 400);
|
|
15842
16438
|
}
|
|
15843
|
-
|
|
15844
|
-
|
|
15845
|
-
try {
|
|
15846
|
-
await assertWalletCanQueueJobs(customer.id);
|
|
15847
|
-
}
|
|
15848
|
-
catch (error) {
|
|
15849
|
-
if (error instanceof InsufficientWalletFundsError) {
|
|
15850
|
-
return insufficientFundsResponse(c, error);
|
|
15851
|
-
}
|
|
15852
|
-
throw error;
|
|
15853
|
-
}
|
|
15854
|
-
const provider = normalizeClipProvider(body.provider);
|
|
15855
|
-
const built = await buildCustomerClipClient(customer.id, provider);
|
|
15856
|
-
if ("error" in built)
|
|
15857
|
-
return c.json({ error: built.error }, 400);
|
|
15858
|
-
const { client, apiKey: tagApiKey, embeddingKey } = built;
|
|
16439
|
+
const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
|
|
16440
|
+
const choice = readClipProviderChoice(body.provider);
|
|
15859
16441
|
const tier = normalizeScanTier(body.tier);
|
|
15860
16442
|
const framesPerScene = Math.max(1, Math.min(3, Number(body.frames_per_scene) || 2));
|
|
15861
|
-
const includeAudio = body.include_audio !== false && provider === "gemini";
|
|
15862
16443
|
const tracer = readNonEmptyString(body.tracer) ?? `clips_import:${customer.id}`;
|
|
15863
16444
|
const now = nowIso();
|
|
15864
16445
|
const scanId = createIdV7("clipscan");
|
|
15865
16446
|
const sourceVideoId = createIdV7("srcvid");
|
|
15866
|
-
// The URL ingest is deferred to the
|
|
15867
|
-
const sourcePointer = rawS3Uri ?? huntSpec.source_url ?? "";
|
|
16447
|
+
// The URL ingest is deferred to the runner, so raw_s3_uri may be empty here.
|
|
16448
|
+
const sourcePointer = rawS3Uri ?? resolvedS3Key ?? huntSpec.source_url ?? "";
|
|
15868
16449
|
const filename = body.filename || (sourcePointer.split("/").pop() || "source.mp4").split("?")[0] || "source.mp4";
|
|
15869
|
-
|
|
15870
|
-
|
|
15871
|
-
|
|
15872
|
-
|
|
15873
|
-
|
|
15874
|
-
|
|
15875
|
-
|
|
15876
|
-
|
|
15877
|
-
? estimateScanCostFromDuration(effectiveDurationSec, {
|
|
16450
|
+
const putScanRecords = async (input) => {
|
|
16451
|
+
await clipRecords.putSource({
|
|
16452
|
+
source_video_id: sourceVideoId,
|
|
16453
|
+
owner_id: customer.id,
|
|
16454
|
+
filename,
|
|
16455
|
+
raw_s3_uri: sourcePointer,
|
|
16456
|
+
duration_sec: body.duration_sec,
|
|
16457
|
+
clip_count: 0,
|
|
15878
16458
|
tier,
|
|
15879
|
-
|
|
15880
|
-
|
|
15881
|
-
|
|
15882
|
-
|
|
15883
|
-
|
|
15884
|
-
|
|
15885
|
-
|
|
15886
|
-
|
|
15887
|
-
|
|
15888
|
-
|
|
15889
|
-
|
|
15890
|
-
|
|
15891
|
-
|
|
15892
|
-
|
|
15893
|
-
|
|
15894
|
-
|
|
15895
|
-
|
|
15896
|
-
|
|
15897
|
-
tier,
|
|
15898
|
-
status: "pending",
|
|
15899
|
-
scan_id: scanId,
|
|
15900
|
-
created_at: now,
|
|
15901
|
-
updated_at: now
|
|
16459
|
+
status: "pending",
|
|
16460
|
+
scan_id: scanId,
|
|
16461
|
+
tracer,
|
|
16462
|
+
created_at: now,
|
|
16463
|
+
updated_at: now
|
|
16464
|
+
});
|
|
16465
|
+
await clipRecords.putScan({
|
|
16466
|
+
scan_id: scanId,
|
|
16467
|
+
owner_id: customer.id,
|
|
16468
|
+
source_video_id: sourceVideoId,
|
|
16469
|
+
status: input.scanStatus,
|
|
16470
|
+
tier,
|
|
16471
|
+
raw_s3_uri: sourcePointer,
|
|
16472
|
+
execution_arn: input.executionArn,
|
|
16473
|
+
tracer,
|
|
16474
|
+
created_at: now,
|
|
16475
|
+
updated_at: now
|
|
16476
|
+
});
|
|
15902
16477
|
};
|
|
15903
|
-
|
|
15904
|
-
|
|
15905
|
-
|
|
16478
|
+
const scanResponse = (input) => c.json({
|
|
16479
|
+
scan_id: scanId,
|
|
16480
|
+
source_video_id: sourceVideoId,
|
|
16481
|
+
status: input.status,
|
|
16482
|
+
execution_arn: input.executionArn ?? null,
|
|
16483
|
+
tracer,
|
|
16484
|
+
hunt_spec: huntSpec,
|
|
16485
|
+
estimate: input.estimate ?? null,
|
|
16486
|
+
compute_estimate: input.computeEstimate ?? null,
|
|
16487
|
+
pipeline: input.pipeline,
|
|
16488
|
+
pipeline_deployed: cloudPipeline
|
|
16489
|
+
}, 202);
|
|
16490
|
+
if (cloudPipeline) {
|
|
16491
|
+
// ── Deployed pipeline (Lambda + Step Functions): BYOK API keys only ──
|
|
16492
|
+
if (choice === "agent") {
|
|
16493
|
+
return c.json({ error: "provider \"agent\" (your local claude/codex CLI) only runs on a local `vidfarm serve` box — the deployed pipeline needs a saved gemini, openai, or openrouter key." }, 400);
|
|
16494
|
+
}
|
|
16495
|
+
// Hunts bill AWS compute post-hoc, so submission is the spend gate (same
|
|
16496
|
+
// policy as job submission).
|
|
16497
|
+
try {
|
|
16498
|
+
await assertWalletCanQueueJobs(customer.id);
|
|
16499
|
+
}
|
|
16500
|
+
catch (error) {
|
|
16501
|
+
if (error instanceof InsufficientWalletFundsError) {
|
|
16502
|
+
return insufficientFundsResponse(c, error);
|
|
16503
|
+
}
|
|
16504
|
+
throw error;
|
|
16505
|
+
}
|
|
16506
|
+
const provider = choice ?? (await pickClipProviderFromKeys(customer.id));
|
|
16507
|
+
if (!provider)
|
|
16508
|
+
return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
|
|
16509
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16510
|
+
if ("error" in built)
|
|
16511
|
+
return c.json({ error: built.error }, 400);
|
|
16512
|
+
const { client, apiKey: tagApiKey, embeddingKey } = built;
|
|
16513
|
+
const includeAudio = body.include_audio !== false && provider === "gemini";
|
|
16514
|
+
// Time-range hunts only pay for the windows: both estimates use the
|
|
16515
|
+
// EFFECTIVE (windowed) duration, and the duration band drives the expected
|
|
16516
|
+
// scene size. The BYOK estimate is the user's own provider spend; the
|
|
16517
|
+
// compute estimate is what the wallet will be billed (AWS only, marked up).
|
|
16518
|
+
const effectiveDurationSec = body.duration_sec && body.duration_sec > 0
|
|
16519
|
+
? effectiveHuntDurationSec(body.duration_sec, huntSpec.windows)
|
|
16520
|
+
: null;
|
|
16521
|
+
const estimate = effectiveDurationSec
|
|
16522
|
+
? estimateScanCostFromDuration(effectiveDurationSec, {
|
|
16523
|
+
tier,
|
|
16524
|
+
provider,
|
|
16525
|
+
tagModelId: client.tagModelId,
|
|
16526
|
+
embedModelId: client.embeddingModelId,
|
|
16527
|
+
framesPerScene,
|
|
16528
|
+
includeAudio,
|
|
16529
|
+
avgSceneSec: huntSpec.duration_band?.target_sec
|
|
16530
|
+
})
|
|
16531
|
+
: null;
|
|
16532
|
+
const computeEstimate = effectiveDurationSec
|
|
16533
|
+
? estimateClipScanComputeUsd(effectiveDurationSec, huntSpec.duration_band?.target_sec)
|
|
16534
|
+
: null;
|
|
15906
16535
|
const res = await clipScanSfn.send(new StartExecutionCommand({
|
|
15907
16536
|
stateMachineArn: config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN,
|
|
15908
16537
|
name: scanId.replace(/[^A-Za-z0-9-_]/g, "-").slice(0, 80),
|
|
@@ -15924,43 +16553,358 @@ app.post(`${CLIPS_PREFIX}/scan`, async (c) => {
|
|
|
15924
16553
|
tracer
|
|
15925
16554
|
})
|
|
15926
16555
|
}));
|
|
15927
|
-
executionArn
|
|
16556
|
+
await putScanRecords({ scanStatus: "running", executionArn: res.executionArn });
|
|
16557
|
+
return scanResponse({ status: "running", executionArn: res.executionArn, pipeline: "cloud", estimate, computeEstimate });
|
|
16558
|
+
}
|
|
16559
|
+
// ── vidfarm serve (no deployed pipeline): LOCAL-FIRST, cloud as backup ──
|
|
16560
|
+
// Same policy as `vidfarm clips scan`: the user's local claude/codex CLI
|
|
16561
|
+
// subscription evaluates scenes when installed, their saved BYOK keys are
|
|
16562
|
+
// the fallback, and the upstream cloud REST API is the explicit backup.
|
|
16563
|
+
// Local hunts are free (no wallet gate), like local renders.
|
|
16564
|
+
let localSourcePath = null;
|
|
16565
|
+
if (resolvedS3Key) {
|
|
16566
|
+
localSourcePath = storage.getLocalPath(resolvedS3Key);
|
|
16567
|
+
if (!localSourcePath || !existsSync(localSourcePath)) {
|
|
16568
|
+
return c.json({ error: "That upload isn't on this box's local storage — re-upload it or pass source_url." }, 400);
|
|
16569
|
+
}
|
|
16570
|
+
}
|
|
16571
|
+
else if (body.raw_s3_uri) {
|
|
16572
|
+
return c.json({ error: "raw_s3_uri isn't supported on a local box — pass source_url, temp_file_id, or attachment_id." }, 400);
|
|
16573
|
+
}
|
|
16574
|
+
const localAgent = choice === "agent" || choice === null ? detectLocalAgent() : null;
|
|
16575
|
+
if (choice === "agent" && !localAgent) {
|
|
16576
|
+
return c.json({ error: "No local agent CLI found — install Claude Code (`claude`) or Codex (`codex`), or pick gemini/openai/openrouter." }, 400);
|
|
16577
|
+
}
|
|
16578
|
+
let evaluator = null;
|
|
16579
|
+
let evaluatorProvider = null;
|
|
16580
|
+
let evaluatorError = null;
|
|
16581
|
+
if (localAgent) {
|
|
16582
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
16583
|
+
const embedding = keys.gemini
|
|
16584
|
+
? { provider: "gemini", apiKey: keys.gemini }
|
|
16585
|
+
: keys.openai
|
|
16586
|
+
? { provider: "openai", apiKey: keys.openai }
|
|
16587
|
+
: null;
|
|
16588
|
+
evaluator = new LocalAgentClipClient({ agent: localAgent.kind, bin: localAgent.bin, embedding, tier });
|
|
16589
|
+
evaluatorProvider = `agent/${localAgent.kind}`;
|
|
16590
|
+
}
|
|
16591
|
+
else {
|
|
16592
|
+
const provider = (choice !== "agent" ? choice : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
16593
|
+
if (provider) {
|
|
16594
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16595
|
+
if ("error" in built) {
|
|
16596
|
+
if (choice)
|
|
16597
|
+
return c.json({ error: built.error }, 400);
|
|
16598
|
+
evaluatorError = built.error;
|
|
16599
|
+
}
|
|
16600
|
+
else {
|
|
16601
|
+
evaluator = built.client;
|
|
16602
|
+
evaluatorProvider = provider;
|
|
16603
|
+
}
|
|
16604
|
+
}
|
|
16605
|
+
else {
|
|
16606
|
+
evaluatorError = NO_CLIP_PROVIDER_KEY_ERROR;
|
|
16607
|
+
}
|
|
16608
|
+
}
|
|
16609
|
+
const ffmpegOk = await hasFfmpeg();
|
|
16610
|
+
const canIngestUrl = !huntSpec.source_url ||
|
|
16611
|
+
Boolean((config.RAPIDAPI_KEY || "").trim()) ||
|
|
16612
|
+
looksLikeDirectVideoUrl(huntSpec.source_url);
|
|
16613
|
+
if (evaluator && ffmpegOk && (localSourcePath || canIngestUrl)) {
|
|
16614
|
+
const includeAudio = body.include_audio !== false && evaluatorProvider === "gemini";
|
|
16615
|
+
await putScanRecords({ scanStatus: "running" });
|
|
16616
|
+
void runClipScanInProcess({
|
|
16617
|
+
scanId,
|
|
16618
|
+
ownerId: customer.id,
|
|
16619
|
+
sourceVideoId,
|
|
16620
|
+
filename,
|
|
16621
|
+
client: evaluator,
|
|
16622
|
+
localSourcePath,
|
|
16623
|
+
sourceUrl: huntSpec.source_url ?? null,
|
|
16624
|
+
huntSpec,
|
|
16625
|
+
guidancePrompt,
|
|
16626
|
+
framesPerScene,
|
|
16627
|
+
includeAudio
|
|
16628
|
+
}).catch((error) => {
|
|
16629
|
+
devLog("clip_scan.local_run_failed", { scan_id: scanId, ...devErrorFields(error) }, "error");
|
|
16630
|
+
});
|
|
16631
|
+
return scanResponse({ status: "running", pipeline: "local" });
|
|
16632
|
+
}
|
|
16633
|
+
// BACKUP: hand the hunt to the upstream cloud deployment (its saved BYOK
|
|
16634
|
+
// keys run the AI; its account is billed the AWS compute).
|
|
16635
|
+
if (hasUpstreamKey() && huntSpec.source_url) {
|
|
16636
|
+
const upstream = await upstreamJson("/clips/scan", {
|
|
16637
|
+
method: "POST",
|
|
16638
|
+
body: {
|
|
16639
|
+
source_url: huntSpec.source_url,
|
|
16640
|
+
prompt: guidancePrompt,
|
|
16641
|
+
hunt_spec: huntSpec,
|
|
16642
|
+
tracer,
|
|
16643
|
+
...(choice && choice !== "agent" ? { provider: choice } : {})
|
|
16644
|
+
},
|
|
16645
|
+
timeoutMs: 30_000
|
|
16646
|
+
});
|
|
16647
|
+
if (!upstream.ok || !upstream.json?.scan_id) {
|
|
16648
|
+
return c.json({ error: upstream.json?.error ?? `Upstream cloud scan failed to start (HTTP ${upstream.status}).` }, 502);
|
|
16649
|
+
}
|
|
16650
|
+
const marker = `upstream:${upstream.json.scan_id}:${upstream.json.source_video_id ?? ""}`;
|
|
16651
|
+
await putScanRecords({ scanStatus: "running", executionArn: marker });
|
|
16652
|
+
return scanResponse({ status: "running", executionArn: marker, pipeline: "upstream" });
|
|
16653
|
+
}
|
|
16654
|
+
const reasons = [];
|
|
16655
|
+
if (!evaluator)
|
|
16656
|
+
reasons.push(evaluatorError ?? "no scene evaluator available");
|
|
16657
|
+
if (evaluator && !ffmpegOk)
|
|
16658
|
+
reasons.push("ffmpeg was not found on this machine");
|
|
16659
|
+
if (evaluator && ffmpegOk && huntSpec.source_url && !canIngestUrl) {
|
|
16660
|
+
reasons.push("this box can't ingest social URLs (set RAPIDAPI_KEY, or pass a direct .mp4 link / upload the file)");
|
|
15928
16661
|
}
|
|
15929
|
-
const scan = {
|
|
15930
|
-
scan_id: scanId,
|
|
15931
|
-
owner_id: customer.id,
|
|
15932
|
-
source_video_id: sourceVideoId,
|
|
15933
|
-
status: executionArn ? "running" : "queued",
|
|
15934
|
-
tier,
|
|
15935
|
-
raw_s3_uri: sourcePointer,
|
|
15936
|
-
execution_arn: executionArn,
|
|
15937
|
-
tracer,
|
|
15938
|
-
created_at: now,
|
|
15939
|
-
updated_at: now
|
|
15940
|
-
};
|
|
15941
|
-
await clipRecords.putScan(scan);
|
|
15942
16662
|
return c.json({
|
|
15943
|
-
|
|
15944
|
-
|
|
15945
|
-
status: scan.status,
|
|
15946
|
-
execution_arn: executionArn ?? null,
|
|
15947
|
-
tracer,
|
|
15948
|
-
hunt_spec: huntSpec,
|
|
15949
|
-
estimate,
|
|
15950
|
-
compute_estimate: computeEstimate,
|
|
15951
|
-
pipeline_deployed: Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN)
|
|
15952
|
-
}, 202);
|
|
16663
|
+
error: `Can't run this hunt on this box: ${reasons.join("; ")}. Install Claude Code (\`claude\`) or Codex (\`codex\`) for keyless local scans, add a provider key, or link a cloud account (VIDFARM_UPSTREAM_HOST + VIDFARM_UPSTREAM_API_KEY) to use the cloud as backup.`
|
|
16664
|
+
}, 400);
|
|
15953
16665
|
}
|
|
15954
16666
|
catch (error) {
|
|
15955
16667
|
return c.json({ error: error instanceof Error ? error.message : "Failed to start clip scan." }, 400);
|
|
15956
16668
|
}
|
|
15957
16669
|
});
|
|
16670
|
+
// GET /clips/scan-options — what the Import Source modal can run a hunt with:
|
|
16671
|
+
// which BYOK providers have saved keys, whether a local agent CLI is available
|
|
16672
|
+
// (vidfarm serve boxes only), what "auto" resolves to, and where the hunt runs.
|
|
16673
|
+
app.get(`${CLIPS_PREFIX}/scan-options`, async (c) => {
|
|
16674
|
+
const customer = requireCustomer(c);
|
|
16675
|
+
const keys = await loadCustomerVisionKeys(customer.id);
|
|
16676
|
+
const cloudPipeline = Boolean(config.VIDFARM_CLIP_SCAN_STATE_MACHINE_ARN);
|
|
16677
|
+
const localAgent = cloudPipeline ? null : detectLocalAgent();
|
|
16678
|
+
const keyProvider = CLIP_API_PROVIDERS.find((p) => keys[p]) ?? null;
|
|
16679
|
+
const defaultProvider = localAgent ? "agent" : keyProvider;
|
|
16680
|
+
const ffmpegOk = cloudPipeline ? true : await hasFfmpeg();
|
|
16681
|
+
const pipeline = cloudPipeline
|
|
16682
|
+
? keyProvider ? "cloud" : null
|
|
16683
|
+
: defaultProvider && ffmpegOk ? "local" : hasUpstreamKey() ? "upstream" : null;
|
|
16684
|
+
return c.json({
|
|
16685
|
+
providers: CLIP_API_PROVIDERS.map((p) => ({ provider: p, has_key: Boolean(keys[p]) })),
|
|
16686
|
+
local_agent: localAgent ? { kind: localAgent.kind } : null,
|
|
16687
|
+
default_provider: defaultProvider,
|
|
16688
|
+
pipeline,
|
|
16689
|
+
upstream_backup: !cloudPipeline && hasUpstreamKey()
|
|
16690
|
+
});
|
|
16691
|
+
});
|
|
16692
|
+
function looksLikeDirectVideoUrl(url) {
|
|
16693
|
+
try {
|
|
16694
|
+
return /\.(mp4|mov|m4v|webm|mkv)$/i.test(new URL(url).pathname);
|
|
16695
|
+
}
|
|
16696
|
+
catch {
|
|
16697
|
+
return false;
|
|
16698
|
+
}
|
|
16699
|
+
}
|
|
16700
|
+
// Local URL ingest for serve-box hunts: direct video links download straight;
|
|
16701
|
+
// social URLs (TikTok/YouTube/IG/X) resolve through the same RapidAPI
|
|
16702
|
+
// downloader the video_download primitive uses (needs RAPIDAPI_KEY locally).
|
|
16703
|
+
async function ingestClipSourceUrlLocally(sourceUrl, workDir) {
|
|
16704
|
+
if (!sourceUrl)
|
|
16705
|
+
throw new Error("Clip scan needs a source video.");
|
|
16706
|
+
let mediaUrl = sourceUrl;
|
|
16707
|
+
if (!looksLikeDirectVideoUrl(sourceUrl)) {
|
|
16708
|
+
const apiKey = (config.RAPIDAPI_KEY || "").trim();
|
|
16709
|
+
if (!apiKey) {
|
|
16710
|
+
throw new Error("This box can't ingest social URLs (RAPIDAPI_KEY unset) — pass a direct .mp4 URL or upload the file.");
|
|
16711
|
+
}
|
|
16712
|
+
const lookupRes = await fetch(config.RAPIDAPI_VIDEO_DOWNLOAD_URL, {
|
|
16713
|
+
method: "POST",
|
|
16714
|
+
headers: {
|
|
16715
|
+
"x-rapidapi-key": apiKey,
|
|
16716
|
+
"x-rapidapi-host": config.RAPIDAPI_VIDEO_DOWNLOAD_HOST,
|
|
16717
|
+
"content-type": "application/x-www-form-urlencoded"
|
|
16718
|
+
},
|
|
16719
|
+
body: new URLSearchParams({ url: sourceUrl })
|
|
16720
|
+
});
|
|
16721
|
+
if (!lookupRes.ok)
|
|
16722
|
+
throw new Error(`Video download lookup failed with HTTP ${lookupRes.status}.`);
|
|
16723
|
+
const lookup = (await lookupRes.json());
|
|
16724
|
+
const playable = (lookup.medias ?? []).filter((m) => m.url && m.videoAvailable !== false && (m.type ? /video/i.test(m.type) : true));
|
|
16725
|
+
const chosen = playable.find((m) => (m.extension ?? "").toLowerCase() === "mp4" || /\.mp4(\?|$)/i.test(m.url ?? "")) ?? playable[0];
|
|
16726
|
+
if (!chosen?.url)
|
|
16727
|
+
throw new Error("Video download lookup returned no playable MP4 media URL.");
|
|
16728
|
+
mediaUrl = chosen.url;
|
|
16729
|
+
}
|
|
16730
|
+
const localPath = path.join(workDir, "ingest", "source.mp4");
|
|
16731
|
+
mkdirSync(path.dirname(localPath), { recursive: true });
|
|
16732
|
+
const res = await fetch(mediaUrl);
|
|
16733
|
+
if (!res.ok || !res.body)
|
|
16734
|
+
throw new Error(`Source video download failed with HTTP ${res.status}.`);
|
|
16735
|
+
await streamPipeline(Readable.fromWeb(res.body), createWriteStream(localPath));
|
|
16736
|
+
return localPath;
|
|
16737
|
+
}
|
|
16738
|
+
// In-process clip hunt for `vidfarm serve` boxes (no Step Functions worker).
|
|
16739
|
+
// Mirrors the clip-scan Lambda pipeline — ingest → probe/detect → refine →
|
|
16740
|
+
// per-scene tag/embed → finalize — with local ffmpeg and the resolved
|
|
16741
|
+
// evaluator (local claude/codex CLI or a BYOK key). Free: no wallet billing,
|
|
16742
|
+
// exactly like local renders. Detached; the client polls /clips/scan/:scanId.
|
|
16743
|
+
async function runClipScanInProcess(input) {
|
|
16744
|
+
const workRoot = path.join(config.VIDFARM_DATA_DIR, "clip-scans", input.scanId);
|
|
16745
|
+
const touchScan = async (patch) => {
|
|
16746
|
+
const scan = await clipRecords.getScan(input.ownerId, input.scanId);
|
|
16747
|
+
if (scan)
|
|
16748
|
+
await clipRecords.putScan({ ...scan, ...patch, updated_at: nowIso() });
|
|
16749
|
+
};
|
|
16750
|
+
const touchSource = async (patch) => {
|
|
16751
|
+
const source = await clipRecords.getSource(input.ownerId, input.sourceVideoId);
|
|
16752
|
+
if (source)
|
|
16753
|
+
await clipRecords.putSource({ ...source, ...patch, updated_at: nowIso() });
|
|
16754
|
+
};
|
|
16755
|
+
try {
|
|
16756
|
+
mkdirSync(workRoot, { recursive: true });
|
|
16757
|
+
const videoPath = input.localSourcePath ?? (await ingestClipSourceUrlLocally(input.sourceUrl ?? "", workRoot));
|
|
16758
|
+
const probe = await probeVideo(videoPath);
|
|
16759
|
+
await touchSource({ status: "scanning", duration_sec: probe.duration_sec });
|
|
16760
|
+
const spec = input.huntSpec;
|
|
16761
|
+
const ctx = {
|
|
16762
|
+
client: input.client,
|
|
16763
|
+
workDir: path.join(workRoot, "work"),
|
|
16764
|
+
clipsDir: path.join(workRoot, "clips"),
|
|
16765
|
+
thumbsDir: path.join(workRoot, "thumbs"),
|
|
16766
|
+
framesPerScene: input.framesPerScene,
|
|
16767
|
+
includeAudio: input.includeAudio,
|
|
16768
|
+
guidancePrompt: buildEffectiveGuidance({ contentPrompt: input.guidancePrompt, avoidText: spec.avoid_text }) || undefined,
|
|
16769
|
+
reencodeClips: true,
|
|
16770
|
+
crop: spec.aspect && spec.aspect !== "original"
|
|
16771
|
+
? { aspect: spec.aspect, focus: normalizeCropFocus(spec.crop_focus) }
|
|
16772
|
+
: undefined
|
|
16773
|
+
};
|
|
16774
|
+
let stored = 0;
|
|
16775
|
+
await scanVideo({
|
|
16776
|
+
ctx,
|
|
16777
|
+
videoPath,
|
|
16778
|
+
probe,
|
|
16779
|
+
windows: spec.windows,
|
|
16780
|
+
durationBand: spec.duration_band ?? null,
|
|
16781
|
+
sourceVideoId: input.sourceVideoId,
|
|
16782
|
+
sourceFilename: input.filename,
|
|
16783
|
+
ownerId: input.ownerId,
|
|
16784
|
+
// Local agents run one CLI process per call — keep parallelism modest.
|
|
16785
|
+
concurrency: input.client.provider === "local-agent" ? 2 : 3,
|
|
16786
|
+
onClip: async (clip) => {
|
|
16787
|
+
// Persist through the storage service (local driver) as bare keys —
|
|
16788
|
+
// clipMediaUrl resolves clips/… + thumbs/… keys to fetchable URLs.
|
|
16789
|
+
const clipKey = `clips/${input.ownerId}/${clip.clip_id}.mp4`;
|
|
16790
|
+
const clipLocal = path.join(ctx.clipsDir, `${clip.clip_id}.mp4`);
|
|
16791
|
+
await storage.putBuffer(clipKey, readFileSync(clipLocal), "video/mp4");
|
|
16792
|
+
clip.file_path = clipKey;
|
|
16793
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clip.clip_id}.jpg`);
|
|
16794
|
+
if (existsSync(thumbLocal)) {
|
|
16795
|
+
const thumbKey = `thumbs/${input.ownerId}/${clip.clip_id}.jpg`;
|
|
16796
|
+
await storage.putBuffer(thumbKey, readFileSync(thumbLocal), "image/jpeg");
|
|
16797
|
+
clip.thumbnail_path = thumbKey;
|
|
16798
|
+
}
|
|
16799
|
+
else {
|
|
16800
|
+
clip.thumbnail_path = "";
|
|
16801
|
+
}
|
|
16802
|
+
await clipRecords.putClip(clip);
|
|
16803
|
+
await vectorIndex.upsert(input.ownerId, clip);
|
|
16804
|
+
stored++;
|
|
16805
|
+
},
|
|
16806
|
+
onError: (scene, error) => {
|
|
16807
|
+
devLog("clip_scan.local_scene_failed", { scan_id: input.scanId, scene_start: scene.start_time_sec, ...devErrorFields(error) }, "warn");
|
|
16808
|
+
}
|
|
16809
|
+
});
|
|
16810
|
+
await touchSource({ status: "complete", clip_count: stored });
|
|
16811
|
+
await touchScan({ status: "complete" });
|
|
16812
|
+
}
|
|
16813
|
+
catch (error) {
|
|
16814
|
+
devLog("clip_scan.local_failed", { scan_id: input.scanId, ...devErrorFields(error) }, "error");
|
|
16815
|
+
await touchSource({ status: "failed" }).catch(() => undefined);
|
|
16816
|
+
await touchScan({ status: "failed", error: error instanceof Error ? error.message : String(error) }).catch(() => undefined);
|
|
16817
|
+
}
|
|
16818
|
+
finally {
|
|
16819
|
+
rmSync(workRoot, { recursive: true, force: true });
|
|
16820
|
+
}
|
|
16821
|
+
}
|
|
16822
|
+
// A serve-box hunt handed to the upstream cloud (execution_arn
|
|
16823
|
+
// "upstream:<scan_id>:<source_video_id>"): mirror the upstream status onto the
|
|
16824
|
+
// local records, and when the cloud hunt completes pull the finished clips
|
|
16825
|
+
// down into local storage — the cloud runs the hunt, the box owns the result.
|
|
16826
|
+
const activeUpstreamClipMirrors = new Set();
|
|
16827
|
+
async function reconcileUpstreamClipScan(ownerId, scan) {
|
|
16828
|
+
const [, upstreamScanId, upstreamSourceId = ""] = (scan.execution_arn ?? "").split(":");
|
|
16829
|
+
if (!upstreamScanId)
|
|
16830
|
+
return scan;
|
|
16831
|
+
const res = await upstreamJson(`/clips/scan/${encodeURIComponent(upstreamScanId)}`);
|
|
16832
|
+
if (!res.ok || !res.json)
|
|
16833
|
+
return scan; // transient upstream hiccup — keep polling
|
|
16834
|
+
const status = res.json.source?.status ?? res.json.scan?.status ?? "running";
|
|
16835
|
+
if (status === "failed" || status === "error") {
|
|
16836
|
+
const failed = { ...scan, status: "failed", error: res.json.scan?.error ?? "Upstream cloud hunt failed.", updated_at: nowIso() };
|
|
16837
|
+
await clipRecords.putScan(failed);
|
|
16838
|
+
const source = await clipRecords.getSource(ownerId, scan.source_video_id);
|
|
16839
|
+
if (source && source.status !== "complete") {
|
|
16840
|
+
await clipRecords.putSource({ ...source, status: "failed", updated_at: nowIso() });
|
|
16841
|
+
}
|
|
16842
|
+
return failed;
|
|
16843
|
+
}
|
|
16844
|
+
if (status === "complete" && !activeUpstreamClipMirrors.has(scan.scan_id)) {
|
|
16845
|
+
activeUpstreamClipMirrors.add(scan.scan_id);
|
|
16846
|
+
void mirrorUpstreamClipsToLocal({ ownerId, scan, upstreamSourceId })
|
|
16847
|
+
.catch((error) => devLog("clip_scan.upstream_mirror_failed", { scan_id: scan.scan_id, ...devErrorFields(error) }, "error"))
|
|
16848
|
+
.finally(() => activeUpstreamClipMirrors.delete(scan.scan_id));
|
|
16849
|
+
}
|
|
16850
|
+
return scan; // stays "running" until the mirror lands the clips locally
|
|
16851
|
+
}
|
|
16852
|
+
async function mirrorUpstreamClipsToLocal(input) {
|
|
16853
|
+
const { ownerId, scan } = input;
|
|
16854
|
+
const query = input.upstreamSourceId ? `?source=${encodeURIComponent(input.upstreamSourceId)}&limit=200` : "?limit=200";
|
|
16855
|
+
const feed = await upstreamJson(`/clips/feed${query}`, { timeoutMs: 60_000 });
|
|
16856
|
+
if (!feed.ok || !feed.json)
|
|
16857
|
+
throw new Error(`Upstream clip feed unavailable (HTTP ${feed.status}).`);
|
|
16858
|
+
let stored = 0;
|
|
16859
|
+
for (const raw of feed.json.clips ?? []) {
|
|
16860
|
+
const clipId = typeof raw.clip_id === "string" ? raw.clip_id : null;
|
|
16861
|
+
const viewUrl = typeof raw.view_url === "string" ? raw.view_url : null;
|
|
16862
|
+
if (!clipId || !viewUrl)
|
|
16863
|
+
continue;
|
|
16864
|
+
const clipRes = await fetch(viewUrl);
|
|
16865
|
+
if (!clipRes.ok)
|
|
16866
|
+
continue;
|
|
16867
|
+
const clipKey = `clips/${ownerId}/${clipId}.mp4`;
|
|
16868
|
+
await storage.putBuffer(clipKey, new Uint8Array(await clipRes.arrayBuffer()), "video/mp4");
|
|
16869
|
+
let thumbKey = "";
|
|
16870
|
+
const thumbUrl = typeof raw.thumbnail_url === "string" ? raw.thumbnail_url : null;
|
|
16871
|
+
if (thumbUrl) {
|
|
16872
|
+
const thumbRes = await fetch(thumbUrl);
|
|
16873
|
+
if (thumbRes.ok) {
|
|
16874
|
+
thumbKey = `thumbs/${ownerId}/${clipId}.jpg`;
|
|
16875
|
+
await storage.putBuffer(thumbKey, new Uint8Array(await thumbRes.arrayBuffer()), "image/jpeg");
|
|
16876
|
+
}
|
|
16877
|
+
}
|
|
16878
|
+
const { view_url: _v, thumbnail_url: _t, ...rest } = raw;
|
|
16879
|
+
void _v;
|
|
16880
|
+
void _t;
|
|
16881
|
+
await clipRecords.putClip({
|
|
16882
|
+
...rest,
|
|
16883
|
+
owner_id: ownerId,
|
|
16884
|
+
source_video_id: scan.source_video_id,
|
|
16885
|
+
file_path: clipKey,
|
|
16886
|
+
thumbnail_path: thumbKey
|
|
16887
|
+
});
|
|
16888
|
+
stored++;
|
|
16889
|
+
}
|
|
16890
|
+
const source = await clipRecords.getSource(ownerId, scan.source_video_id);
|
|
16891
|
+
if (source)
|
|
16892
|
+
await clipRecords.putSource({ ...source, status: "complete", clip_count: stored, updated_at: nowIso() });
|
|
16893
|
+
const current = await clipRecords.getScan(ownerId, scan.scan_id);
|
|
16894
|
+
await clipRecords.putScan({ ...(current ?? scan), status: "complete", updated_at: nowIso() });
|
|
16895
|
+
}
|
|
15958
16896
|
// GET /clips/scan/:scanId — poll scan status.
|
|
15959
16897
|
app.get(`${CLIPS_PREFIX}/scan/:scanId`, async (c) => {
|
|
15960
16898
|
const customer = requireCustomer(c);
|
|
15961
16899
|
let scan = await clipRecords.getScan(customer.id, c.req.param("scanId"));
|
|
15962
16900
|
if (!scan)
|
|
15963
16901
|
return c.json({ error: "Scan not found" }, 404);
|
|
16902
|
+
// Serve-box hunts handed to the upstream cloud reconcile against the
|
|
16903
|
+
// upstream REST API instead of Step Functions.
|
|
16904
|
+
if ((scan.status === "running" || scan.status === "queued") && scan.execution_arn?.startsWith("upstream:")) {
|
|
16905
|
+
scan = await reconcileUpstreamClipScan(customer.id, scan);
|
|
16906
|
+
}
|
|
16907
|
+
else
|
|
15964
16908
|
// Lazily reconcile with the Step Functions execution: the pipeline has no
|
|
15965
16909
|
// failure hook back into the scan record, so a crashed execution would
|
|
15966
16910
|
// otherwise leave "running" forever and pollers spin.
|
|
@@ -16006,7 +16950,11 @@ app.post(`${CLIPS_PREFIX}/presets`, async (c) => {
|
|
|
16006
16950
|
return c.json({ error: "name is required." }, 400);
|
|
16007
16951
|
let criteria = body.criteria;
|
|
16008
16952
|
if (!criteria && body.query) {
|
|
16009
|
-
const
|
|
16953
|
+
const requested = readClipProviderChoice(body.provider);
|
|
16954
|
+
const provider = (requested !== "agent" ? requested : null) ?? (await pickClipProviderFromKeys(customer.id));
|
|
16955
|
+
if (!provider)
|
|
16956
|
+
return c.json({ error: NO_CLIP_PROVIDER_KEY_ERROR }, 400);
|
|
16957
|
+
const built = await buildCustomerClipClient(customer.id, provider);
|
|
16010
16958
|
if ("error" in built)
|
|
16011
16959
|
return c.json({ error: built.error }, 400);
|
|
16012
16960
|
criteria = await built.client.naturalLanguageToCriteria(body.query);
|