@mevdragon/vidfarm-devcli 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/music/SKILL.md +416 -0
- package/.agents/skills/music/references/api_reference.md +519 -0
- package/.agents/skills/music/references/installation.md +65 -0
- package/.agents/skills/text-to-speech/SKILL.md +226 -0
- package/.agents/skills/text-to-speech/references/installation.md +90 -0
- package/.agents/skills/text-to-speech/references/streaming.md +307 -0
- package/.agents/skills/text-to-speech/references/voice-settings.md +115 -0
- package/.agents/skills/vidfarm-media/SKILL.md +25 -11
- package/.agents/skills/vidfarm-media/references/tts.md +41 -3
- package/SKILL.director.md +31 -13
- package/SKILL.platform.md +2 -2
- package/dist/src/app.js +129 -41
- package/dist/src/cli.js +162 -5
- package/dist/src/config.js +12 -0
- package/dist/src/devcli/clips.js +7 -2
- package/dist/src/domain.js +3 -0
- package/dist/src/editor-chat.js +4 -0
- package/dist/src/primitive-context.js +14 -0
- package/dist/src/primitive-registry.js +140 -18
- package/dist/src/reskin/chat-page.js +1 -1
- package/dist/src/reskin/inpaint-clipper-page.js +446 -205
- package/dist/src/reskin/library-page.js +7 -1
- package/dist/src/reskin/portfolio-page.js +9 -9
- package/dist/src/reskin/settings-page.js +4 -4
- package/dist/src/reskin/theme.js +1 -0
- package/dist/src/services/billing.js +5 -0
- package/dist/src/services/clip-curation/ffmpeg.js +48 -0
- package/dist/src/services/clip-curation/hunt.js +2 -0
- package/dist/src/services/clip-curation/index.js +1 -1
- package/dist/src/services/clip-curation/scan.js +29 -16
- package/dist/src/services/elevenlabs.js +222 -0
- package/dist/src/services/providers.js +216 -2
- package/dist/src/services/swipe-customize.js +5 -2
- package/package.json +1 -1
package/dist/src/app.js
CHANGED
|
@@ -63,6 +63,7 @@ import { ActiveJobLimitExceededError, JobsService } from "./services/jobs.js";
|
|
|
63
63
|
import { ProviderLeaseTimeoutError, ProviderService } from "./services/providers.js";
|
|
64
64
|
import { RateLimitExceededError, RateLimitService } from "./services/rate-limits.js";
|
|
65
65
|
import { ServerlessProviderKeyService } from "./services/serverless-provider-keys.js";
|
|
66
|
+
import { DEFAULT_ELEVENLABS_VOICE_ID, ELEVENLABS_VOICE_LIBRARY_URL } from "./services/elevenlabs.js";
|
|
66
67
|
import { ServerlessTemplateConfigService } from "./services/serverless-template-configs.js";
|
|
67
68
|
import { parseHTML } from "linkedom";
|
|
68
69
|
import { joinStorageKey, storage as sharedStorage } from "./services/storage.js";
|
|
@@ -74,7 +75,7 @@ import { DescribeExecutionCommand, SFNClient, StartExecutionCommand } from "@aws
|
|
|
74
75
|
import { clipRecords, makeUserPreset } from "./services/clip-records.js";
|
|
75
76
|
import { matchScenesToClips, searchUserClips, vectorIndex } from "./services/clip-search.js";
|
|
76
77
|
import { ATTACHMENTS_FOLDER_SCOPE, buildDirectoryPath, DIRECTORY_ROOTS, DIRECTORY_ROOT_KEYS, folderItem, immediateChildFolders, normalizeFolderPath, parseDirectoryPath } from "./services/file-directory.js";
|
|
77
|
-
import { applyHuntSpecDefaults, BUILTIN_PRESETS, buildClipEmbeddingText, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, ACTIVE_TAXONOMY_VERSION, estimateScanCostFromDuration, extractClip, extractThumbnail, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, pickBestVideoMedia, probeVideo, resolveDurationBand, resolveTargetClipCount, scanVideo } from "./services/clip-curation/index.js";
|
|
78
|
+
import { applyHuntSpecDefaults, BUILTIN_PRESETS, buildClipEmbeddingText, buildEffectiveGuidance, ClipModelClient, cosineSimilarity, detectLocalAgent, effectiveHuntDurationSec, ACTIVE_TAXONOMY_VERSION, estimateScanCostFromDuration, extractAudioClip, extractClip, extractThumbnail, extractWaveformThumbnail, hasFfmpeg, LocalAgentClipClient, normalizeCropFocus, normalizeHuntSpec, parseClipHuntPrompt, pickBestVideoMedia, probeVideo, resolveDurationBand, resolveTargetClipCount, scanVideo } from "./services/clip-curation/index.js";
|
|
78
79
|
import { analyzeCastMembers, annotateScenesFromSource, buildBlankCompositionHtml, buildSmartDecomposedHtml, DEFAULT_EDITOR_HARNESS, HyperframesService, smartDecomposeVideo, stampCompositionIdentity } from "./services/hyperframes.js";
|
|
79
80
|
import { buildPendingSceneAnnotationsState } from "./services/scene-annotations.js";
|
|
80
81
|
import { createPrimitiveJobContext } from "./primitive-context.js";
|
|
@@ -381,12 +382,12 @@ const agencyPromoteSchema = z.object({
|
|
|
381
382
|
message: "Provide customer_id or email."
|
|
382
383
|
});
|
|
383
384
|
const providerKeySchema = z.object({
|
|
384
|
-
provider: z.enum(["openai", "gemini", "openrouter", "perplexity"]),
|
|
385
|
+
provider: z.enum(["openai", "gemini", "openrouter", "perplexity", "elevenlabs"]),
|
|
385
386
|
label: z.string().optional(),
|
|
386
387
|
secret: z.string().min(8)
|
|
387
388
|
});
|
|
388
389
|
const settingsProviderKeyFormSchema = z.object({
|
|
389
|
-
provider: z.enum(["openai", "gemini", "openrouter", "perplexity"]),
|
|
390
|
+
provider: z.enum(["openai", "gemini", "openrouter", "perplexity", "elevenlabs"]),
|
|
390
391
|
label: z.string().trim().optional(),
|
|
391
392
|
secret: z.string().min(8)
|
|
392
393
|
});
|
|
@@ -3073,6 +3074,7 @@ function temporaryFileToDirectoryItem(c, f) {
|
|
|
3073
3074
|
// Takes the OUTPUT of serializeClip (view/thumbnail URLs already resolved).
|
|
3074
3075
|
function clipSerializedToDirectoryItem(s) {
|
|
3075
3076
|
const folderPath = normalizeFolderPath(s.folder_path);
|
|
3077
|
+
const isAudio = s.media_kind === "audio";
|
|
3076
3078
|
const name = s.source_filename || s.description || "clip";
|
|
3077
3079
|
const duration = s.duration_sec;
|
|
3078
3080
|
return {
|
|
@@ -3082,11 +3084,14 @@ function clipSerializedToDirectoryItem(s) {
|
|
|
3082
3084
|
name,
|
|
3083
3085
|
kind: "file",
|
|
3084
3086
|
id: s.clip_id,
|
|
3085
|
-
contentType
|
|
3087
|
+
// Audio-only raws (.m4a) must advertise an audio contentType so the editor's
|
|
3088
|
+
// media picker (mediaKindFromRef) auto-places them as an AUDIO track rather
|
|
3089
|
+
// than a <video> layer. The picker checks contentType before extension.
|
|
3090
|
+
contentType: isAudio ? "audio/mp4" : "video/mp4",
|
|
3086
3091
|
viewUrl: s.view_url || undefined,
|
|
3087
3092
|
thumbUrl: s.thumbnail_url || undefined,
|
|
3088
3093
|
durationSec: typeof duration === "number" ? duration : undefined,
|
|
3089
|
-
meta: { description: s.description, tags: s.tags, created_at: s.created_at, source_video_id: s.source_video_id }
|
|
3094
|
+
meta: { description: s.description, tags: s.tags, created_at: s.created_at, source_video_id: s.source_video_id, media_kind: s.media_kind }
|
|
3090
3095
|
};
|
|
3091
3096
|
}
|
|
3092
3097
|
// An approved ready-post as a canonical directory file. view/thumb come from the
|
|
@@ -3498,8 +3503,10 @@ function buildPrimitiveEditorDocsRoutes() {
|
|
|
3498
3503
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/probe`, summary: "Primitive audio probe job. Body must be { tracer, payload: { source_audio_url }, webhook_url? }. Returns durable JSON metadata." },
|
|
3499
3504
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/concat`, summary: "Primitive audio concat job. Body must be { tracer, payload: { source_audio_urls, output_format?, audio_bitrate_kbps? }, webhook_url? }." },
|
|
3500
3505
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/normalize`, summary: "Primitive audio normalization job. Body must be { tracer, payload: { source_audio_url, output_format?, audio_bitrate_kbps?, sample_rate_hz? }, webhook_url? }." },
|
|
3501
|
-
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/speech`, summary: "Primitive text-to-speech (TTS)
|
|
3502
|
-
{ method: "
|
|
3506
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/speech`, summary: "Primitive text-to-speech (TTS). Body must be { tracer, payload: { text, voice?, instructions?, provider?, model?, output_format?, use_wallet_credits? }, webhook_url? }. use_wallet_credits DEFAULTS TRUE → best-quality ElevenLabs narration on the platform key, billed to the wallet; set false to use your own saved ElevenLabs key or a BYOK OpenAI/Gemini/OpenRouter key instead. voice on the ElevenLabs path is a voice_id (or friendly name george/sarah/…); browse voices via GET /api/v1/primitives/audio/voices and pass the chosen voice_id. On the BYOK path voice is a provider preset (OpenAI: alloy/ash/…; Gemini: Kore/Puck/…). instructions is a free-form voice-style prompt (tone/pacing/accent/emotion). Returns a durable platform audio URL. Alias: POST /api/v1/primitives/tts." },
|
|
3507
|
+
{ method: "GET", path: `${PRIMITIVES_PREFIX}/audio/voices`, summary: "List ElevenLabs voices for TTS. Query use_wallet_credits (default true) lists the platform account's voices; use_wallet_credits=false lists the voices on the customer's own saved ElevenLabs key. Returns { scope, default_voice_id, voice_library_url, voices: [{ voice_id, name, category, labels, description, preview_url }] }. Pick a voice_id and pass it as `voice` to /audio/speech. Browse more at the returned voice_library_url." },
|
|
3508
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/music/generate`, summary: "Primitive MUSIC generation (ElevenLabs): instrumental tracks, songs with lyrics, background beds, jingles. Body must be { tracer, payload: { prompt, music_length_ms? (default 30000, max 300000 = 5 min), composition_plan? (ElevenLabs plan for section-by-section control), model?, use_wallet_credits? }, webhook_url? }. use_wallet_credits DEFAULTS TRUE → platform ElevenLabs key billed to the wallet; set false to use your own saved ElevenLabs key. Returns a durable platform audio URL (mp3). Alias: POST /api/v1/primitives/music." },
|
|
3509
|
+
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/transcribe`, summary: "Primitive speech-to-text (STT). Accepts VIDEO or AUDIO sources; video audio is demuxed automatically. Body must be { tracer, payload: { source_url (aliases: source_video_url, source_audio_url, url), diarize?, word_timestamps?, language?, prompt?, provider?, model?, use_wallet_credits? }, webhook_url? }. use_wallet_credits DEFAULTS TRUE → ElevenLabs Scribe on the platform key (native diarization + real word timestamps), billed to the wallet; set false to use your own ElevenLabs key or a BYOK gemini/openai/openrouter key (gemini labels speakers, openai/whisper-1 gives real word timings). Returns TWO formats in one job: a simple subtitle transcript (plain text + timed SRT) and speaker-attributed timed segments, plus transcript.json/.srt/.txt. Sources over ~40 minutes must be trimmed first. Alias: POST /api/v1/primitives/stt." },
|
|
3503
3510
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/captions`, summary: "ANIMATED CAPTIONS from speech: one job transcribes narration WITH word-level timings and pages it into word-by-word caption cues ready for the editor's set_captions action. Body must be { tracer, payload: { source_url (video or audio; video audio is demuxed) OR text (page a script instead of transcribing), style? (spotlight|karaoke|word-pop|stack-up|neon|bounce), animation?, active_color?, highlight_color?, uppercase?, color?, background?, background_style?, font_family?, font_size?, max_words_per_cue?, offset_sec?, language?, provider?, window_start?/window_duration? (text mode) }, webhook_url? }. The finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments — apply verbatim). Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter fall back to estimated word windows. Alias: POST /api/v1/primitives/captions." },
|
|
3504
3511
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/audio/regenerate-speech`, summary: "REGENERATE SPEECH IN THE ORIGINAL SPEAKER'S VOICE (approximate match, not a clone): one job listens to a video/audio source, profiles the speaker's voice (needs a saved Gemini key; other keys degrade to transcript-only with no voice match), rewords the transcript per your instruction (or takes an exact replacement script), and returns new TTS audio using the closest preset voice + a style prompt matched to the speaker. Body must be { tracer, payload: { source_url (video or audio; video audio is demuxed; aliases source_video_url/source_audio_url/url), rewrite_instruction? (how to reword, e.g. \"mention the summer sale instead of the discount code\") OR text? (exact new script — one of the two is required), voice? (explicit preset override), instructions? (extra style guidance), provider?, model?, language?, output_format? }, webhook_url? }. Result carries audioUrl (durable, place with add_layer), script, original_transcript, voice_profile, and voice_match: \"approximate\"|\"none\" — OpenAI/Gemini APIs cannot clone voice identity, so always tell the user the voice is a matched approximation. Alias: POST /api/v1/primitives/regenerate-speech." },
|
|
3505
3512
|
{ method: "POST", path: `${PRIMITIVES_PREFIX}/brainstorm/coldstart`, summary: "Primitive cold-start strategy questionnaire. Body should usually be { tracer, payload: { user_message }, webhook_url? }. user_message can be rough, like \"I don't know where to start\" or a messy description. count is optional only when the user explicitly wants more or fewer questions. offer_description remains accepted as a backward-compatible alias." },
|
|
@@ -3640,7 +3647,10 @@ async function normalizeEditorChatProviderRows(customerId) {
|
|
|
3640
3647
|
secret: readStoredSecret(String(entry.secret))
|
|
3641
3648
|
}))
|
|
3642
3649
|
.filter((entry) => entry.status === "active")
|
|
3643
|
-
.filter((entry) => entry.secret.trim().length > 0)
|
|
3650
|
+
.filter((entry) => entry.secret.trim().length > 0)
|
|
3651
|
+
// ElevenLabs is an audio-only key — never a qualified text/image/video
|
|
3652
|
+
// provider, so it must not surface as an editor-chat / media provider.
|
|
3653
|
+
.filter((entry) => entry.provider !== "elevenlabs");
|
|
3644
3654
|
}
|
|
3645
3655
|
async function chooseEditorChatProvider(customerId, preferredProvider) {
|
|
3646
3656
|
const rows = await normalizeEditorChatProviderRows(customerId);
|
|
@@ -3728,6 +3738,9 @@ function createEditorChatModel(input) {
|
|
|
3728
3738
|
apiKey: input.apiKey,
|
|
3729
3739
|
baseURL: "https://api.perplexity.ai"
|
|
3730
3740
|
}).chatModel(input.model);
|
|
3741
|
+
default:
|
|
3742
|
+
// Audio-only providers (elevenlabs) are never editor-chat models.
|
|
3743
|
+
throw new Error(`Editor chat does not support the ${input.provider} provider.`);
|
|
3731
3744
|
}
|
|
3732
3745
|
}
|
|
3733
3746
|
function buildMockEditorChatText(input) {
|
|
@@ -4108,7 +4121,7 @@ function normalizeOperationRequestTracer(tracer) {
|
|
|
4108
4121
|
}
|
|
4109
4122
|
function createTemplateHttpTool(input) {
|
|
4110
4123
|
return tool({
|
|
4111
|
-
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech) and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions? } } — instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key, and word_timestamps: true adds word-level timings per segment (real on an OpenAI key via whisper-1). If the user wants ANIMATED word-by-word captions generated from narration, call POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url (narration audio or video URL) OR text (a script to page), style?: \"spotlight\"|\"karaoke\"|\"word-pop\"|\"stack-up\"|\"neon\"|\"bounce\", max_words_per_cue?, offset_sec?, language?, provider? } } — the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments); apply them with the editor_action tool (action_type=set_captions) so the change flows through the editor state. Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter estimate word windows. If the user wants the SAME speaker to say DIFFERENT words — reword, revoice, redub, or regenerate existing narration so it sounds like the original voice — call POST /api/v1/primitives/audio/regenerate-speech with body { tracer, payload: { source_url (video or audio; video audio is demuxed), rewrite_instruction (how to reword the original transcript) OR text (the exact new script), voice?, instructions?, provider?, language?, output_format? } } — the job listens to the source, profiles the speaker's voice (needs a saved Gemini key; other keys degrade to transcript-only with no voice match), rewords the script, and returns a durable audio URL spoken by the closest PRESET voice with a style prompt matched to the speaker. OpenAI/Gemini APIs cannot clone voice identity, so always present regenerate-speech output as an approximation of the original voice, never a clone; place the finished audio with editor_action add_layer, typically after muting the original span with /videos/mute or swapping tracks with /videos/replace-audio. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, provider?: \"gemini\"|\"openai\"|\"openrouter\", tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys (auto-picked gemini → openai → openrouter unless body.provider names one). To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
4124
|
+
description: "Call exactly one Vidfarm REST API route and inspect the response before deciding the next call. Use template routes for metadata, skill, config, operations, jobs, logs, and cancel routes. Use primitive routes under /api/v1/primitives for reusable image generation, image editing, inpainting, still rendering, social/media video download, AI video generation, video slide rendering, burned-in caption removal, video trim/extract/probe/mute/concat operations, audio trim/probe/concat/normalize operations, text-to-speech narration (audio/speech), music generation (music/generate), voice listing (audio/voices), and speech-to-text transcription (audio/transcribe) routes, brainstorm strategy routes, and primitive job inspection; these primitive routes are allowed directly from editor chat even when they are not specific to the current template. Primitive POST bodies must use { tracer, payload: { ...primitive fields... }, webhook_url? }; do not put prompt, source_image_url, instruction, slides, html, or other primitive fields at the top level. Never say primitive image generation, image edit, inpaint, HTML render, video download, AI video generation, caption removal, trim, probe, extract-audio, concat, mute, video render, text-to-speech, speech transcription, or brainstorm routes are unavailable here when the route exists. If the user asks to generate a new image similar to an attachment, call POST /api/v1/primitives/images/generate with body { tracer, payload: { prompt, prompt_attachments: [exact attachment URL] } }. If the user asks to revise the attached image itself, call POST /api/v1/primitives/images/edit with body { tracer, payload: { source_image_url, instruction } }. If the user asks to download a social/media post into a durable MP4, call POST /api/v1/primitives/videos/download with body { tracer, payload: { source_url|video_url|url, quality?: \"best\"|\"hd\"|\"full_hd\" } }. If the user asks to generate a new AI video, call POST /api/v1/primitives/videos/generate with body { tracer, payload: { prompt, provider?: \"openai\"|\"gemini\"|\"openrouter\", duration?: seconds, input_references?: [direct image asset URL], frame_images?: [{ image_url, frame_type }] } }; never send webpage or social post URLs in input_references. duration is seconds, not milliseconds, and duration_ms belongs only to /videos/render-slides slides. Default video models are openai/sora-2, gemini/veo-3.0-generate-001, and openrouter/bytedance/seedance-2.0. Use POST /api/v1/primitives/videos/render-slides only to assemble existing media URLs into an MP4 slideshow. Use POST /api/v1/primitives/media/dedupe to apply subtle dedupe transforms (zoom/tilt/rotate/saturation/speed/horizontal_flip/contrast/brightness/hue_rotate/blur/tint) to any image or video URL, with body { tracer, payload: { source_media_url, media_type?, effects?, tint_color?, tint_opacity?, width?, height?, duration_ms?, object_fit?, background_color?, output_format? }, webhook_url? }. Use POST /api/v1/primitives/videos/trim for clipping, /videos/extract-audio for demuxing, /videos/probe or /audio/probe for metadata-only JSON outputs, /videos/extract-frame for stills, /videos/mute to remove audio, /videos/replace-audio to swap tracks, /videos/concat for sequential muted MP4 joins, /audio/trim for clipped audio, /audio/concat for stitched audio, and /audio/normalize for leveled/re-encoded audio. If the user asks for AI voiceover, narration, or dub audio from text (a script, hook, or caption read aloud), call POST /api/v1/primitives/audio/speech with body { tracer, payload: { text, voice?, instructions?, use_wallet_credits? } } — use_wallet_credits DEFAULTS TRUE, which routes to high-quality ElevenLabs on the platform key billed to the user's wallet; keep it true by default and only set it false to use the user's own ElevenLabs key or a BYOK OpenAI/Gemini/OpenRouter key (to save wallet credits). Default to a sensible voice and TELL the user they can pick from many ElevenLabs voices — call GET /api/v1/primitives/audio/voices (add ?use_wallet_credits=false to list voices on their own ElevenLabs key) to list voice_id + name + preview_url, then pass the chosen voice_id as `voice`. instructions is a free-form voice-style prompt like \"excited sports announcer, fast paced\" and the job returns a durable audio URL you can place with add_layer. If the user asks for MUSIC — a background track, beat, jingle, song, or score — call POST /api/v1/primitives/music/generate with body { tracer, payload: { prompt, music_length_ms?, use_wallet_credits? } } (use_wallet_credits DEFAULTS TRUE = platform ElevenLabs billed to the wallet; false = the user's own ElevenLabs key). music_length_ms defaults 30000 and caps at 300000 (5 min); the job returns a durable mp3 you place as its own <audio> layer (keep music ~0.1–0.2 volume under narration). If the user asks to transcribe a video or audio, get subtitles/captions from speech, or identify who says what, call POST /api/v1/primitives/audio/transcribe with body { tracer, payload: { source_url, diarize?, language? } } — it accepts video or audio URLs (video audio is demuxed automatically) and one job returns BOTH a simple subtitle transcript (plain text + timed SRT cues) and an advanced multi-speaker version (speaker-attributed timed segments for multi-narration); speaker attribution needs a Gemini key, and word_timestamps: true adds word-level timings per segment (real on an OpenAI key via whisper-1). If the user wants ANIMATED word-by-word captions generated from narration, call POST /api/v1/primitives/audio/captions with body { tracer, payload: { source_url (narration audio or video URL) OR text (a script to page), style?: \"spotlight\"|\"karaoke\"|\"word-pop\"|\"stack-up\"|\"neon\"|\"bounce\", max_words_per_cue?, offset_sec?, language?, provider? } } — the finished job's result carries captions (the cue array) and set_captions_action (fully-formed editor_action arguments); apply them with the editor_action tool (action_type=set_captions) so the change flows through the editor state. Real word timestamps need an OpenAI key (whisper-1, tried first); gemini/openrouter estimate word windows. If the user wants the SAME speaker to say DIFFERENT words — reword, revoice, redub, or regenerate existing narration so it sounds like the original voice — call POST /api/v1/primitives/audio/regenerate-speech with body { tracer, payload: { source_url (video or audio; video audio is demuxed), rewrite_instruction (how to reword the original transcript) OR text (the exact new script), voice?, instructions?, provider?, language?, output_format? } } — the job listens to the source, profiles the speaker's voice (needs a saved Gemini key; other keys degrade to transcript-only with no voice match), rewords the script, and returns a durable audio URL spoken by the closest PRESET voice with a style prompt matched to the speaker. OpenAI/Gemini APIs cannot clone voice identity, so always present regenerate-speech output as an approximation of the original voice, never a clone; place the finished audio with editor_action add_layer, typically after muting the original span with /videos/mute or swapping tracks with /videos/replace-audio. If the user asks to remove burned-in captions, subtitles, or on-screen text from a video, call POST /api/v1/primitives/videos/remove-captions with body { tracer, payload: { source_video_url } } — an async job that returns a durable caption-free MP4 and bills about $0.10 per 30 seconds of source video. Caption removal is capped at ~15 minutes of source and must NEVER be used on long-form video: when the user wants short clips out of a long video (a podcast, stream VOD, webinar, or any YouTube/TikTok/IG/X URL), call POST /clips/scan instead with body { source_url | temp_file_id | s3_key, prompt, ranges?: [\"MM:SS-MM:SS\"], target_duration_sec?, aspect?: \"9:16\"|\"16:9\"|\"4:3\"|\"1:1\", avoid_text?, provider?: \"gemini\"|\"openai\"|\"openrouter\", tracer? } — target_duration_sec is a SOFT range (30 → 20-40s), ranges restrict the hunt to those source windows (big cost saver on long videos), aspect crops every clip, and avoid_text prefers scenes WITHOUT on-screen text via scene SELECTION (never caption removal on the source). The hunt is async: it returns scan_id immediately; poll GET /clips/scan/:scanId, then browse results via GET /clips/feed?source=<source_video_id> or POST /clips/search. Hunts bill AWS compute only — the AI tagging runs on the user's own saved provider keys (auto-picked gemini → openai → openrouter unless body.provider names one). To remove captions from a finished HUNTED clip, pass that clip's short mp4 to /videos/remove-captions afterwards. If the user explicitly asks for hooks, angles, awareness-stage advice, or a cold-start strategy questionnaire, call the matching brainstorm route immediately instead of brainstorming in chat from memory. Use POST /api/v1/primitives/brainstorm/coldstart when the customer is starting from zero, /brainstorm/awareness_stages when they do not know what ad types to make, /brainstorm/angles when they want persuasive angles, and /brainstorm/hooks when they want many openings. Product placement is a first-class brainstorm primitive like angles and hooks: when the user wants to identify product-placement opportunities in a video or repurpose a video into a native ad for their product, call POST /api/v1/primitives/brainstorm/product_placement with body { tracer, payload: { source_video_url, offer_description }, webhook_url? } — it watches the video and returns timestamped native placement opportunities and prefers a Gemini key. You may also analyze an attached video directly to answer product-placement questions conversationally instead of the primitive when the user just wants a quick take. For brainstorm routes, do not send count by default; only include count when the user explicitly asks for a specific number of hooks, angles, questions, or opportunities. When brainstorm jobs finish and the response includes a JSON artifact, tell the user to open and review that JSON in the chat UI because it contains the structured output. When the user asks for content ideas or says they do not know where to start, prefer the sequence coldstart -> awareness_stages -> angles -> hooks, then use the output to choose templates. For template operation POST bodies, use top-level template_preview_media=\"auto\" by default, \"include\" when the user wants the template to use catalog preview media as references where supported, and \"exclude\" when the user asks not to use preview media as references. Templates decide which generation steps honor that policy. If a POST returns a queued job, do not poll it in this same response; continue with any remaining requested POST calls, then summarize the queued job ids and tracers. If you know a job_id but not the template or primitive, use GET /api/v1/user/me/jobs/:jobId or its /logs and /artifacts subroutes instead of paging through job lists. Use GET /api/v1/compositions/:forkId/remove-video-captions to look up both the original source video URL and the subtitle-removed mirrored URL for the current fork before feeding a video URL into a primitive route; this is a read-only, non-billing status check — never confuse it with POST /remove-video-captions-poll (which advances the fork's decompose-time subtitle-removal job and can bill). Use POST /api/v1/approved/posts with title, caption, pinned_comment, media URLs, and tracer to approve a finished output into a preview/share page. Use POST /api/v1/approved/posts/:postId/schedules with destination_type, destination_id, scheduled_at, timezone, and optional settings/additional_notes to schedule an approved post. When a rendered video was built from slideshow frame images, include both versions in one media array: final MP4 first with kind=\"video\" and role=\"primary\", then every finished captioned slide frame image in order with kind=\"image\" and role=\"slide\". Prefer create-slideshow result.renderVideoInput.slides[].imageUrl or result.slides[].frameImageUrl for carousel media; do not use backgroundImageUrl, manifests, thumbnails, or a partial files array as the full slide set. If the user supplies exact media file URLs for approval, use those URLs as the backup source of truth, preserve their order, infer kind when needed, and assign primary/video and slide/image roles correctly. If the user asks for N distinct generated outputs, make N sequential POST calls rather than one combined POST and do not stop after the first queued job.",
|
|
4112
4125
|
inputSchema: z.object({
|
|
4113
4126
|
method: z.enum(["GET", "POST"]).describe("HTTP method to use."),
|
|
4114
4127
|
path: z.string().min(1).describe("Absolute path or same-origin URL for an allowed Vidfarm REST route."),
|
|
@@ -16883,15 +16896,41 @@ const AI_PRIMITIVE_IDS = new Set([
|
|
|
16883
16896
|
"primitive:video_generate",
|
|
16884
16897
|
"primitive:tts",
|
|
16885
16898
|
"primitive:stt",
|
|
16899
|
+
"primitive:music",
|
|
16886
16900
|
"primitive:captions",
|
|
16887
16901
|
"primitive:speech_regenerate"
|
|
16888
16902
|
]);
|
|
16903
|
+
// Audio primitives that ElevenLabs can serve — either the platform key
|
|
16904
|
+
// (use_wallet_credits) or the customer's own ElevenLabs key. TTS/STT can also
|
|
16905
|
+
// fall back to a BYOK openai/gemini/openrouter key; music cannot.
|
|
16906
|
+
const ELEVENLABS_AUDIO_PRIMITIVE_IDS = new Set(["primitive:tts", "primitive:stt", "primitive:music"]);
|
|
16889
16907
|
async function preflightAiProviderKeys(input) {
|
|
16890
16908
|
if (!operationMayUseAiProvider(input)) {
|
|
16891
16909
|
return null;
|
|
16892
16910
|
}
|
|
16893
|
-
const activeProviders = (await
|
|
16911
|
+
const activeProviders = (await listProviderKeysForCustomer(input.customerId))
|
|
16912
|
+
.filter((entry) => String(entry.status) === "active")
|
|
16913
|
+
.map((entry) => String(entry.provider));
|
|
16894
16914
|
const mediaProviders = activeProviders.filter((provider) => MEDIA_AI_PROVIDER_TYPES.has(provider));
|
|
16915
|
+
// Audio primitives run on ElevenLabs: allow when the platform key is
|
|
16916
|
+
// configured, the customer saved their own ElevenLabs key, or (TTS/STT only)
|
|
16917
|
+
// they hold a BYOK media key. Only block when none of those can serve it.
|
|
16918
|
+
if (ELEVENLABS_AUDIO_PRIMITIVE_IDS.has(input.templateId)) {
|
|
16919
|
+
const hasPlatform = Boolean(config.ELEVENLABS_API_KEY?.trim());
|
|
16920
|
+
const hasUserElevenLabs = activeProviders.includes("elevenlabs");
|
|
16921
|
+
const byokEligible = input.templateId !== "primitive:music";
|
|
16922
|
+
if (hasPlatform || hasUserElevenLabs || (byokEligible && mediaProviders.length > 0)) {
|
|
16923
|
+
return null;
|
|
16924
|
+
}
|
|
16925
|
+
return {
|
|
16926
|
+
error: "No audio provider is available for this customer.",
|
|
16927
|
+
type: "provider_key_required",
|
|
16928
|
+
detail: input.templateId === "primitive:music"
|
|
16929
|
+
? "Music generation needs ElevenLabs. Enable use_wallet_credits (platform key) or save your own ElevenLabs key in Settings → AI keys."
|
|
16930
|
+
: "This audio operation needs ElevenLabs (use_wallet_credits / your own ElevenLabs key) or a BYOK OpenAI/Gemini/OpenRouter key.",
|
|
16931
|
+
configured_providers: [...new Set(activeProviders)]
|
|
16932
|
+
};
|
|
16933
|
+
}
|
|
16895
16934
|
if (mediaProviders.length) {
|
|
16896
16935
|
return null;
|
|
16897
16936
|
}
|
|
@@ -17415,6 +17454,29 @@ app.post(`${PRIMITIVES_PREFIX}/audio/speech`, async (c) => createPrimitiveJob(c,
|
|
|
17415
17454
|
app.post(`${PRIMITIVES_PREFIX}/tts`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:tts", operationName: "run" }));
|
|
17416
17455
|
app.post(`${PRIMITIVES_PREFIX}/audio/transcribe`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
17417
17456
|
app.post(`${PRIMITIVES_PREFIX}/stt`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:stt", operationName: "run" }));
|
|
17457
|
+
// Music generation (ElevenLabs). use_wallet_credits defaults true (platform key
|
|
17458
|
+
// + wallet). Canonical /music/generate plus a flat /music alias.
|
|
17459
|
+
app.post(`${PRIMITIVES_PREFIX}/music/generate`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:music", operationName: "run" }));
|
|
17460
|
+
app.post(`${PRIMITIVES_PREFIX}/music`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:music", operationName: "run" }));
|
|
17461
|
+
// List ElevenLabs voices for TTS. Synchronous (not a job). Scope is the platform
|
|
17462
|
+
// account when use_wallet_credits is true/absent, else the customer's own key.
|
|
17463
|
+
app.get(`${PRIMITIVES_PREFIX}/audio/voices`, async (c) => {
|
|
17464
|
+
const customer = requireCustomer(c);
|
|
17465
|
+
const useWalletParam = (c.req.query("use_wallet_credits") ?? "").toLowerCase();
|
|
17466
|
+
const useWalletCredits = useWalletParam === "false" || useWalletParam === "0" ? false : true;
|
|
17467
|
+
try {
|
|
17468
|
+
const { scope, voices } = await providers.listElevenLabsVoices({ customerId: customer.id, useWalletCredits });
|
|
17469
|
+
return c.json({
|
|
17470
|
+
scope,
|
|
17471
|
+
voice_library_url: ELEVENLABS_VOICE_LIBRARY_URL,
|
|
17472
|
+
default_voice_id: DEFAULT_ELEVENLABS_VOICE_ID,
|
|
17473
|
+
voices
|
|
17474
|
+
});
|
|
17475
|
+
}
|
|
17476
|
+
catch (error) {
|
|
17477
|
+
return c.json({ error: error instanceof Error ? error.message : "Unable to list ElevenLabs voices." }, 400);
|
|
17478
|
+
}
|
|
17479
|
+
});
|
|
17418
17480
|
// Animated-caption cues from narration (STT + word timings + cue paging).
|
|
17419
17481
|
app.post(`${PRIMITIVES_PREFIX}/audio/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
17420
17482
|
app.post(`${PRIMITIVES_PREFIX}/captions`, async (c) => createPrimitiveJob(c, { primitiveId: "primitive:captions", operationName: "run" }));
|
|
@@ -19726,6 +19788,10 @@ app.post(`${RAWS_PREFIX}/scan`, async (c) => {
|
|
|
19726
19788
|
// backup) so mined clips land in `/raws/<folder>` without a pipeline change.
|
|
19727
19789
|
if (folderOverride)
|
|
19728
19790
|
huntSpec.folder_path = folderOverride;
|
|
19791
|
+
// Audio-only mining ("Clip audio only"): each mined clip is cut to an .m4a
|
|
19792
|
+
// (no video). Rides the opaque hunt_spec end-to-end like folder_path.
|
|
19793
|
+
if (body.audio_only === true)
|
|
19794
|
+
huntSpec.audio_only = true;
|
|
19729
19795
|
// Fill the unspecified-prompt mining defaults (5–15s clips, vertical 9:16,
|
|
19730
19796
|
// avoid on-screen text). Explicit user/prompt choices above already won;
|
|
19731
19797
|
// this only fills the gaps. Skipped for import_only (no mining happens).
|
|
@@ -20352,30 +20418,39 @@ async function importSubrangeAsRaw(input) {
|
|
|
20352
20418
|
const end = Math.max(start + 0.05, rawEnd);
|
|
20353
20419
|
const clipDurationSec = Number((end - start).toFixed(3));
|
|
20354
20420
|
await touchSource({ status: "scanning", duration_sec: sourceDurationSec });
|
|
20421
|
+
const audioOnly = Boolean(input.audioOnly);
|
|
20422
|
+
const scene = { index: 0, start_time_sec: start, end_time_sec: end, duration_sec: clipDurationSec };
|
|
20355
20423
|
const clipId = createIdV7("clip");
|
|
20356
|
-
const
|
|
20357
|
-
const
|
|
20358
|
-
const
|
|
20359
|
-
|
|
20360
|
-
|
|
20361
|
-
|
|
20362
|
-
|
|
20363
|
-
|
|
20364
|
-
|
|
20365
|
-
|
|
20424
|
+
const clipExt = audioOnly ? "m4a" : "mp4";
|
|
20425
|
+
const clipKey = `clips/${input.ownerId}/${clipId}.${clipExt}`;
|
|
20426
|
+
const localClip = path.join(workRoot, `${clipId}.${clipExt}`);
|
|
20427
|
+
if (audioOnly) {
|
|
20428
|
+
const trimmed = await extractAudioClip({ videoPath, scene, outPath: localClip });
|
|
20429
|
+
if (!trimmed || !existsSync(localClip))
|
|
20430
|
+
throw new Error("Could not extract audio from that subrange (the source may have no audio track).");
|
|
20431
|
+
await storage.putFile(clipKey, localClip, "audio/mp4");
|
|
20432
|
+
}
|
|
20433
|
+
else {
|
|
20434
|
+
const trimmed = await extractClip({ videoPath, scene, outPath: localClip });
|
|
20435
|
+
if (!trimmed || !existsSync(localClip))
|
|
20436
|
+
throw new Error("Could not trim that subrange.");
|
|
20437
|
+
await storage.putFile(clipKey, localClip, "video/mp4");
|
|
20438
|
+
}
|
|
20366
20439
|
let thumbKey = "";
|
|
20367
|
-
const thumbLocal = path.join(workRoot, `${clipId}
|
|
20368
|
-
const
|
|
20369
|
-
|
|
20370
|
-
|
|
20371
|
-
|
|
20372
|
-
|
|
20440
|
+
const thumbLocal = path.join(workRoot, `${clipId}.${audioOnly ? "png" : "jpg"}`);
|
|
20441
|
+
const thumbScene = { index: 0, start_time_sec: 0, end_time_sec: clipDurationSec, duration_sec: clipDurationSec };
|
|
20442
|
+
// Audio raws have no video frame — plot a waveform PNG (from the trimmed audio)
|
|
20443
|
+
// so the raws gallery still shows something recognizable.
|
|
20444
|
+
const wroteThumb = audioOnly
|
|
20445
|
+
? await extractWaveformThumbnail({ videoPath: localClip, scene: thumbScene, outPath: thumbLocal }).catch(() => false)
|
|
20446
|
+
: await extractThumbnail({ videoPath: localClip, scene: thumbScene, outPath: thumbLocal }).catch(() => false);
|
|
20373
20447
|
if (wroteThumb && existsSync(thumbLocal)) {
|
|
20374
|
-
thumbKey = `thumbs/${input.ownerId}/${clipId}
|
|
20375
|
-
await storage.putFile(thumbKey, thumbLocal, "image/jpeg");
|
|
20448
|
+
thumbKey = `thumbs/${input.ownerId}/${clipId}.${audioOnly ? "png" : "jpg"}`;
|
|
20449
|
+
await storage.putFile(thumbKey, thumbLocal, audioOnly ? "image/png" : "image/jpeg");
|
|
20376
20450
|
}
|
|
20377
20451
|
const baseName = (input.name || ingestedTitle || input.filename || "clip").trim() || "clip";
|
|
20378
|
-
const
|
|
20452
|
+
const notes = (input.notes || "").trim();
|
|
20453
|
+
const description = `${baseName}${audioOnly ? " (audio)" : ""} (${formatClockRange(start)}–${formatClockRange(end)})`;
|
|
20379
20454
|
const folderPath = input.folderPath
|
|
20380
20455
|
? normalizeFolderPath(input.folderPath)
|
|
20381
20456
|
: normalizeFolderPath(String(baseName).toLowerCase().replace(/\.[a-z0-9]+$/i, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60));
|
|
@@ -20388,19 +20463,21 @@ async function importSubrangeAsRaw(input) {
|
|
|
20388
20463
|
duration_sec: clipDurationSec,
|
|
20389
20464
|
file_path: clipKey,
|
|
20390
20465
|
thumbnail_path: thumbKey,
|
|
20391
|
-
tags: { content_type: [], subject: [], action: [], emotion: [], setting: [], composition: [], motion: [], energy: "", audio_type: [], transcript: "" },
|
|
20466
|
+
tags: { content_type: [], subject: [], action: [], emotion: [], setting: [], composition: [], motion: [], energy: "", audio_type: audioOnly ? ["music"] : [], transcript: "" },
|
|
20392
20467
|
description,
|
|
20393
20468
|
taxonomy_version: ACTIVE_TAXONOMY_VERSION,
|
|
20394
20469
|
tier: "flash-lite",
|
|
20395
20470
|
tracer: input.tracer,
|
|
20396
20471
|
folder_path: folderPath,
|
|
20472
|
+
media_kind: audioOnly ? "audio" : "video",
|
|
20397
20473
|
owner_id: input.ownerId,
|
|
20398
20474
|
created_at: nowIso()
|
|
20399
20475
|
};
|
|
20400
20476
|
try {
|
|
20401
20477
|
const embedder = await buildClipLibraryEmbedder(input.ownerId);
|
|
20402
20478
|
if (embedder?.embeddingModelId) {
|
|
20403
|
-
const
|
|
20479
|
+
const embedText = buildClipEmbeddingText(clip.tags, clip.description) + (notes ? `\n${notes}` : "");
|
|
20480
|
+
const [vector] = await embedder.embedDocuments([embedText]);
|
|
20404
20481
|
if (vector?.length) {
|
|
20405
20482
|
clip.embedding = vector;
|
|
20406
20483
|
clip.embedding_model = embedder.embeddingModelId;
|
|
@@ -20414,7 +20491,7 @@ async function importSubrangeAsRaw(input) {
|
|
|
20414
20491
|
await vectorIndex.upsert(input.ownerId, clip).catch(() => undefined);
|
|
20415
20492
|
await touchSource({ status: "complete", clip_count: 1, filename: baseName });
|
|
20416
20493
|
await touchScan({ status: "complete" });
|
|
20417
|
-
return { clipId, clipKey, thumbKey, durationSec: clipDurationSec, sourceStartSec: start, sourceEndSec: end, description, folderPath };
|
|
20494
|
+
return { clipId, clipKey, thumbKey, durationSec: clipDurationSec, sourceStartSec: start, sourceEndSec: end, description, folderPath, mediaKind: audioOnly ? "audio" : "video" };
|
|
20418
20495
|
}
|
|
20419
20496
|
catch (error) {
|
|
20420
20497
|
devLog("clip_range.failed", { scan_id: input.scanId, ...devErrorFields(error) }, "error");
|
|
@@ -20470,6 +20547,7 @@ app.post(`${RAWS_PREFIX}/clip-range`, async (c) => {
|
|
|
20470
20547
|
const customer = requireCustomer(c);
|
|
20471
20548
|
try {
|
|
20472
20549
|
const body = (await c.req.json().catch(() => ({})));
|
|
20550
|
+
const audioOnly = Boolean(body.audio_only ?? body.audioOnly);
|
|
20473
20551
|
const sourceUrl = readNonEmptyString(body.source_url) ?? readNonEmptyString(body.video_url) ?? readNonEmptyString(body.url);
|
|
20474
20552
|
const mediaUrl = readNonEmptyString(body.preview_url) ?? readNonEmptyString(body.media_url);
|
|
20475
20553
|
// Resolve an uploaded source (temp file / My Files attachment) to a local key.
|
|
@@ -20504,7 +20582,11 @@ app.post(`${RAWS_PREFIX}/clip-range`, async (c) => {
|
|
|
20504
20582
|
return c.json({ error: "Provide end_sec (or duration via end_ms)." }, 400);
|
|
20505
20583
|
if (endSec <= startSec)
|
|
20506
20584
|
return c.json({ error: "end_sec must be greater than start_sec." }, 400);
|
|
20507
|
-
|
|
20585
|
+
// Tracer doubles as the session id: prefer an explicit tracer/session_id,
|
|
20586
|
+
// else mint a unique per-clip session so clips never collide in one shared
|
|
20587
|
+
// bucket (a `clipper:<customer>` catch-all would merge every ad-hoc clip).
|
|
20588
|
+
const tracer = readNonEmptyString(body.tracer) ?? readNonEmptyString(body.session_id) ?? `clipper-${createIdV7("clip").split("_").pop()}`;
|
|
20589
|
+
const notes = readNonEmptyString(body.notes) ?? null;
|
|
20508
20590
|
const folderOverride = slugifyRawFolder(body.folder_path) ?? slugifyRawFolder(tracer);
|
|
20509
20591
|
const now = nowIso();
|
|
20510
20592
|
const scanId = createIdV7("clipscan");
|
|
@@ -20527,7 +20609,9 @@ app.post(`${RAWS_PREFIX}/clip-range`, async (c) => {
|
|
|
20527
20609
|
sourceUrl: sourceUrl ?? null,
|
|
20528
20610
|
startSec,
|
|
20529
20611
|
endSec,
|
|
20530
|
-
name: readNonEmptyString(body.name) ?? null
|
|
20612
|
+
name: readNonEmptyString(body.name) ?? null,
|
|
20613
|
+
notes,
|
|
20614
|
+
audioOnly
|
|
20531
20615
|
});
|
|
20532
20616
|
}
|
|
20533
20617
|
catch (error) {
|
|
@@ -20545,6 +20629,7 @@ app.post(`${RAWS_PREFIX}/clip-range`, async (c) => {
|
|
|
20545
20629
|
duration_sec: result.durationSec,
|
|
20546
20630
|
source_start_sec: result.sourceStartSec,
|
|
20547
20631
|
source_end_sec: result.sourceEndSec,
|
|
20632
|
+
media_kind: result.mediaKind,
|
|
20548
20633
|
description: result.description,
|
|
20549
20634
|
view_url: viewUrl ?? `/clips/${result.clipId}/download`,
|
|
20550
20635
|
thumbnail_url: thumbnailUrl
|
|
@@ -20659,6 +20744,7 @@ async function runClipScanInProcess(input) {
|
|
|
20659
20744
|
includeAudio: input.includeAudio,
|
|
20660
20745
|
guidancePrompt: buildEffectiveGuidance({ contentPrompt: input.guidancePrompt, avoidText: spec.avoid_text }) || undefined,
|
|
20661
20746
|
reencodeClips: true,
|
|
20747
|
+
audioOnly: spec.audio_only === true,
|
|
20662
20748
|
crop: spec.aspect && spec.aspect !== "original"
|
|
20663
20749
|
? { aspect: spec.aspect, focus: normalizeCropFocus(spec.crop_focus) }
|
|
20664
20750
|
: undefined
|
|
@@ -20696,15 +20782,17 @@ async function runClipScanInProcess(input) {
|
|
|
20696
20782
|
clip.folder_path = folder;
|
|
20697
20783
|
}
|
|
20698
20784
|
// Persist through the storage service (local driver) as bare keys —
|
|
20699
|
-
// clipMediaUrl resolves clips/… + thumbs/… keys to fetchable URLs.
|
|
20700
|
-
|
|
20701
|
-
const
|
|
20702
|
-
|
|
20785
|
+
// clipMediaUrl resolves clips/… + thumbs/… keys to fetchable URLs. Audio-
|
|
20786
|
+
// only hunts wrote an .m4a clip + .png waveform thumbnail (see scan.ts).
|
|
20787
|
+
const audioClip = clip.media_kind === "audio";
|
|
20788
|
+
const clipKey = `clips/${input.ownerId}/${clip.clip_id}.${audioClip ? "m4a" : "mp4"}`;
|
|
20789
|
+
const clipLocal = path.join(ctx.clipsDir, `${clip.clip_id}.${audioClip ? "m4a" : "mp4"}`);
|
|
20790
|
+
await storage.putBuffer(clipKey, readFileSync(clipLocal), audioClip ? "audio/mp4" : "video/mp4");
|
|
20703
20791
|
clip.file_path = clipKey;
|
|
20704
|
-
const thumbLocal = path.join(ctx.thumbsDir, `${clip.clip_id}
|
|
20792
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clip.clip_id}.${audioClip ? "png" : "jpg"}`);
|
|
20705
20793
|
if (existsSync(thumbLocal)) {
|
|
20706
|
-
const thumbKey = `thumbs/${input.ownerId}/${clip.clip_id}
|
|
20707
|
-
await storage.putBuffer(thumbKey, readFileSync(thumbLocal), "image/jpeg");
|
|
20794
|
+
const thumbKey = `thumbs/${input.ownerId}/${clip.clip_id}.${audioClip ? "png" : "jpg"}`;
|
|
20795
|
+
await storage.putBuffer(thumbKey, readFileSync(thumbLocal), audioClip ? "image/png" : "image/jpeg");
|
|
20708
20796
|
clip.thumbnail_path = thumbKey;
|
|
20709
20797
|
}
|
|
20710
20798
|
else {
|