@mevdragon/vidfarm-devcli 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/music/SKILL.md +416 -0
- package/.agents/skills/music/references/api_reference.md +519 -0
- package/.agents/skills/music/references/installation.md +65 -0
- package/.agents/skills/text-to-speech/SKILL.md +226 -0
- package/.agents/skills/text-to-speech/references/installation.md +90 -0
- package/.agents/skills/text-to-speech/references/streaming.md +307 -0
- package/.agents/skills/text-to-speech/references/voice-settings.md +115 -0
- package/.agents/skills/vidfarm-media/SKILL.md +25 -11
- package/.agents/skills/vidfarm-media/references/tts.md +41 -3
- package/SKILL.director.md +31 -13
- package/SKILL.platform.md +2 -2
- package/dist/src/app.js +129 -41
- package/dist/src/cli.js +162 -5
- package/dist/src/config.js +12 -0
- package/dist/src/devcli/clips.js +7 -2
- package/dist/src/domain.js +3 -0
- package/dist/src/editor-chat.js +4 -0
- package/dist/src/primitive-context.js +14 -0
- package/dist/src/primitive-registry.js +140 -18
- package/dist/src/reskin/chat-page.js +1 -1
- package/dist/src/reskin/inpaint-clipper-page.js +446 -205
- package/dist/src/reskin/library-page.js +7 -1
- package/dist/src/reskin/portfolio-page.js +9 -9
- package/dist/src/reskin/settings-page.js +4 -4
- package/dist/src/reskin/theme.js +1 -0
- package/dist/src/services/billing.js +5 -0
- package/dist/src/services/clip-curation/ffmpeg.js +48 -0
- package/dist/src/services/clip-curation/hunt.js +2 -0
- package/dist/src/services/clip-curation/index.js +1 -1
- package/dist/src/services/clip-curation/scan.js +29 -16
- package/dist/src/services/elevenlabs.js +222 -0
- package/dist/src/services/providers.js +216 -2
- package/dist/src/services/swipe-customize.js +5 -2
- package/package.json +1 -1
package/dist/src/cli.js
CHANGED
|
@@ -285,6 +285,8 @@ Raws (the third library — mine long-form video into a reusable raws store):
|
|
|
285
285
|
--tracer <name> Group this session's clips (also the default folder)
|
|
286
286
|
--folder <name> Save the clip into /raws/<name>
|
|
287
287
|
--name <text> Human name for the clip
|
|
288
|
+
--audio-only Save just the audio of the subrange as an .m4a raw
|
|
289
|
+
(needs --start/--end; e.g. lift a copyright-free track)
|
|
288
290
|
(multi-clip: re-run with a new range + same --tracer)
|
|
289
291
|
|
|
290
292
|
Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit backup:
|
|
@@ -300,7 +302,19 @@ Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit bac
|
|
|
300
302
|
--engine local KEYLESS local Kokoro-82M engine (free, no account) — fixed voice
|
|
301
303
|
presets (af_heart, am_adam, bf_emma, …), no --style. When NO
|
|
302
304
|
provider key is found the default byok engine auto-falls back here.
|
|
303
|
-
--cloud BACKUP:
|
|
305
|
+
--cloud BACKUP: platform job → POST /api/v1/primitives/audio/speech (+ poll).
|
|
306
|
+
Defaults to ElevenLabs on the platform key (wallet-billed); --voice
|
|
307
|
+
takes an ElevenLabs voice_id (see: vidfarm voices).
|
|
308
|
+
--own-key (--cloud) use your own ElevenLabs / BYOK key instead of the wallet
|
|
309
|
+
--list-voices List ElevenLabs voices and exit (same as: vidfarm voices)
|
|
310
|
+
music "<prompt>" Prompt → music track (ElevenLabs). CLOUD-ONLY (no local engine)
|
|
311
|
+
--length <sec> | --length-ms <ms> Duration (default 30s, max 5 min)
|
|
312
|
+
--out <file> Output mp3 path (default music-<id>.mp3)
|
|
313
|
+
--own-key Use your own ElevenLabs key instead of the platform key + wallet
|
|
314
|
+
(default: platform ElevenLabs, wallet-billed) → POST /api/v1/primitives/music/generate
|
|
315
|
+
voices List ElevenLabs voices for TTS (voice_id + name + labels).
|
|
316
|
+
--own-key List voices on your own saved ElevenLabs key (default: platform account)
|
|
317
|
+
--limit <n> | --json Show more / raw JSON
|
|
304
318
|
stt <file|url> Video or audio → transcript (alias: transcribe). LOCAL: local ffmpeg demux +
|
|
305
319
|
Returns BOTH formats: the simple subtitle your provider key (gemini labels
|
|
306
320
|
version (plain text + timed SRT cues) and speakers; openai/openrouter give
|
|
@@ -315,8 +329,11 @@ Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit bac
|
|
|
315
329
|
timestamps (words[] in the .json/--json output), no speaker labels.
|
|
316
330
|
When NO provider key is found the default byok engine auto-falls
|
|
317
331
|
back here. --model then picks the whisper model (tiny.en…large-v3).
|
|
318
|
-
--cloud BACKUP:
|
|
332
|
+
--cloud BACKUP: platform job → POST /api/v1/primitives/audio/transcribe (+ poll).
|
|
333
|
+
Defaults to ElevenLabs Scribe on the platform key (wallet-billed;
|
|
334
|
+
native diarization + real word timings).
|
|
319
335
|
(a local file uploads to your temp folder first — 30-day TTL)
|
|
336
|
+
--own-key (--cloud) use your own ElevenLabs / BYOK key instead of the wallet
|
|
320
337
|
|
|
321
338
|
Animated captions (word-by-word TikTok/CapCut styles on a pulled/served composition):
|
|
322
339
|
captions generate <dir> Transcribe the narration & lay down animated LOCAL: local ffmpeg demux +
|
|
@@ -586,6 +603,12 @@ async function main() {
|
|
|
586
603
|
case "tts":
|
|
587
604
|
await runTtsCommand(rest);
|
|
588
605
|
return;
|
|
606
|
+
case "music":
|
|
607
|
+
await runMusicCommand(rest);
|
|
608
|
+
return;
|
|
609
|
+
case "voices":
|
|
610
|
+
await runVoicesCommand(rest);
|
|
611
|
+
return;
|
|
589
612
|
case "stt":
|
|
590
613
|
case "transcribe":
|
|
591
614
|
await runSttCommand(rest);
|
|
@@ -3045,10 +3068,17 @@ async function runTtsCommand(argv) {
|
|
|
3045
3068
|
"gemini-key": { type: "string" },
|
|
3046
3069
|
"openai-key": { type: "string" },
|
|
3047
3070
|
"openrouter-key": { type: "string" },
|
|
3071
|
+
"own-key": { type: "boolean", default: false },
|
|
3072
|
+
"list-voices": { type: "boolean", default: false },
|
|
3048
3073
|
"no-wait": { type: "boolean", default: false },
|
|
3049
3074
|
tracer: { type: "string" }
|
|
3050
3075
|
}
|
|
3051
3076
|
});
|
|
3077
|
+
// --list-voices: browse the ElevenLabs voice catalog and exit (cloud call).
|
|
3078
|
+
if (parsed.values["list-voices"]) {
|
|
3079
|
+
await runVoicesCommand(parsed.values["own-key"] ? ["--own-key"] : []);
|
|
3080
|
+
return;
|
|
3081
|
+
}
|
|
3052
3082
|
const text = (parsed.values.text ?? parsed.positionals.join(" ")).trim();
|
|
3053
3083
|
if (!text)
|
|
3054
3084
|
throw new Error('tts requires text: vidfarm tts "Welcome back to the channel" --style "calm, warm narrator".');
|
|
@@ -3118,7 +3148,9 @@ async function runTtsCommand(argv) {
|
|
|
3118
3148
|
}
|
|
3119
3149
|
return;
|
|
3120
3150
|
}
|
|
3121
|
-
// BACKUP: the platform primitive route (
|
|
3151
|
+
// BACKUP: the platform primitive route (async job). Defaults to high-quality
|
|
3152
|
+
// ElevenLabs on the platform key, billed to the wallet (use_wallet_credits).
|
|
3153
|
+
// --own-key flips to your own saved ElevenLabs / BYOK provider key instead.
|
|
3122
3154
|
const ctx = commonContext(parsed.values);
|
|
3123
3155
|
const payload = { text, output_format: format };
|
|
3124
3156
|
if (voice)
|
|
@@ -3129,6 +3161,11 @@ async function runTtsCommand(argv) {
|
|
|
3129
3161
|
payload.provider = parsed.values.provider;
|
|
3130
3162
|
if (parsed.values.model)
|
|
3131
3163
|
payload.model = parsed.values.model;
|
|
3164
|
+
if (parsed.values["own-key"])
|
|
3165
|
+
payload.use_wallet_credits = false;
|
|
3166
|
+
if (!ctx.json && !parsed.values["own-key"]) {
|
|
3167
|
+
console.log(`${DIM}Using ElevenLabs on the platform key (wallet-billed). Browse voices: vidfarm voices — then pass --voice <voice_id>. Add --own-key to use your own key.${RESET}`);
|
|
3168
|
+
}
|
|
3132
3169
|
const tracer = parsed.values.tracer ?? `devcli-tts-${Date.now().toString(36)}`;
|
|
3133
3170
|
const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/speech", auth: ctx.auth, body: { tracer, payload } });
|
|
3134
3171
|
assertApiOk(submit, "tts");
|
|
@@ -3290,6 +3327,112 @@ function emitSttResult(input) {
|
|
|
3290
3327
|
console.log(`${GREEN}Wrote ${dest}${RESET}`);
|
|
3291
3328
|
}
|
|
3292
3329
|
}
|
|
3330
|
+
// Music generation (ElevenLabs) — cloud-only (no local engine). Defaults to the
|
|
3331
|
+
// platform key billed to the wallet; --own-key uses your saved ElevenLabs key.
|
|
3332
|
+
async function runMusicCommand(argv) {
|
|
3333
|
+
const parsed = parseArgs({
|
|
3334
|
+
args: argv,
|
|
3335
|
+
allowPositionals: true,
|
|
3336
|
+
options: {
|
|
3337
|
+
...commonOptions(),
|
|
3338
|
+
prompt: { type: "string" },
|
|
3339
|
+
"length-ms": { type: "string" },
|
|
3340
|
+
length: { type: "string" },
|
|
3341
|
+
model: { type: "string" },
|
|
3342
|
+
out: { type: "string" },
|
|
3343
|
+
"own-key": { type: "boolean", default: false },
|
|
3344
|
+
"no-wait": { type: "boolean", default: false },
|
|
3345
|
+
tracer: { type: "string" }
|
|
3346
|
+
}
|
|
3347
|
+
});
|
|
3348
|
+
const prompt = (parsed.values.prompt ?? parsed.positionals.join(" ")).trim();
|
|
3349
|
+
if (!prompt)
|
|
3350
|
+
throw new Error('music requires a prompt: vidfarm music "upbeat lo-fi hip hop beat with jazzy piano" --length 30');
|
|
3351
|
+
const ctx = commonContext(parsed.values);
|
|
3352
|
+
let musicLengthMs;
|
|
3353
|
+
const lengthMs = parsed.values["length-ms"]?.trim();
|
|
3354
|
+
const lengthSec = parsed.values.length?.trim();
|
|
3355
|
+
if (lengthMs)
|
|
3356
|
+
musicLengthMs = Math.round(Number(lengthMs));
|
|
3357
|
+
else if (lengthSec)
|
|
3358
|
+
musicLengthMs = Math.round(Number(lengthSec) * 1000);
|
|
3359
|
+
const payload = { prompt };
|
|
3360
|
+
if (musicLengthMs && Number.isFinite(musicLengthMs))
|
|
3361
|
+
payload.music_length_ms = musicLengthMs;
|
|
3362
|
+
if (parsed.values.model)
|
|
3363
|
+
payload.model = parsed.values.model;
|
|
3364
|
+
if (parsed.values["own-key"])
|
|
3365
|
+
payload.use_wallet_credits = false;
|
|
3366
|
+
if (!ctx.json && !parsed.values["own-key"]) {
|
|
3367
|
+
console.log(`${DIM}Composing with ElevenLabs on the platform key (wallet-billed). Add --own-key to use your own ElevenLabs key.${RESET}`);
|
|
3368
|
+
}
|
|
3369
|
+
const tracer = parsed.values.tracer ?? `devcli-music-${Date.now().toString(36)}`;
|
|
3370
|
+
const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/music/generate", auth: ctx.auth, body: { tracer, payload } });
|
|
3371
|
+
assertApiOk(submit, "music");
|
|
3372
|
+
const jobId = submit.json?.job_id;
|
|
3373
|
+
if (parsed.values["no-wait"] || !jobId) {
|
|
3374
|
+
if (!ctx.json && jobId)
|
|
3375
|
+
console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
|
|
3376
|
+
emitResult(submit, ctx.json);
|
|
3377
|
+
return;
|
|
3378
|
+
}
|
|
3379
|
+
if (!ctx.json)
|
|
3380
|
+
console.log(`${DIM}Generating music (${jobId})… polling every 5s.${RESET}`);
|
|
3381
|
+
const job = await pollPrimitiveJob(ctx, jobId);
|
|
3382
|
+
const mediaUrl = resolveJobMediaUrl(job);
|
|
3383
|
+
if (!mediaUrl) {
|
|
3384
|
+
if (ctx.json) {
|
|
3385
|
+
printJson({ ok: false, job_id: jobId, status: String(job?.status ?? ""), job });
|
|
3386
|
+
}
|
|
3387
|
+
else {
|
|
3388
|
+
console.log(`${RED}Music ${String(job?.status ?? "did not finish")} — no audio URL.${RESET}`);
|
|
3389
|
+
printJson(job);
|
|
3390
|
+
}
|
|
3391
|
+
process.exitCode = 1;
|
|
3392
|
+
return;
|
|
3393
|
+
}
|
|
3394
|
+
const outPath = path.resolve(process.cwd(), parsed.values.out ?? `music-${jobId}.mp3`);
|
|
3395
|
+
await downloadUrlToFile(mediaUrl, outPath);
|
|
3396
|
+
if (ctx.json) {
|
|
3397
|
+
printJson({ ok: true, job_id: jobId, tracer, audio_url: mediaUrl, out: outPath });
|
|
3398
|
+
}
|
|
3399
|
+
else {
|
|
3400
|
+
console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${mediaUrl})${RESET}`);
|
|
3401
|
+
console.log(`${DIM}Place it as its own <audio> layer under narration (~0.1–0.2 volume).${RESET}`);
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
// List ElevenLabs voices for TTS. --own-key lists the voices on your own saved
|
|
3405
|
+
// ElevenLabs key; default lists the platform account's voices.
|
|
3406
|
+
async function runVoicesCommand(argv) {
|
|
3407
|
+
const parsed = parseArgs({
|
|
3408
|
+
args: argv,
|
|
3409
|
+
allowPositionals: true,
|
|
3410
|
+
options: {
|
|
3411
|
+
...commonOptions(),
|
|
3412
|
+
"own-key": { type: "boolean", default: false },
|
|
3413
|
+
limit: { type: "string" }
|
|
3414
|
+
}
|
|
3415
|
+
});
|
|
3416
|
+
const ctx = commonContext(parsed.values);
|
|
3417
|
+
const query = parsed.values["own-key"] ? "?use_wallet_credits=false" : "";
|
|
3418
|
+
const res = await apiRequest({ method: "GET", host: ctx.host, path: `/api/v1/primitives/audio/voices${query}`, auth: ctx.auth });
|
|
3419
|
+
assertApiOk(res, "voices");
|
|
3420
|
+
const body = (res.json ?? {});
|
|
3421
|
+
const voices = Array.isArray(body.voices) ? body.voices : [];
|
|
3422
|
+
if (ctx.json) {
|
|
3423
|
+
printJson(body);
|
|
3424
|
+
return;
|
|
3425
|
+
}
|
|
3426
|
+
console.log(`${DIM}${voices.length} ElevenLabs voices (${body.scope ?? "platform"} account). Browse more: ${body.voice_library_url ?? "https://elevenlabs.io/app/voice-library"}${RESET}`);
|
|
3427
|
+
const limit = Number(parsed.values.limit ?? "40") || 40;
|
|
3428
|
+
for (const voice of voices.slice(0, limit)) {
|
|
3429
|
+
const labels = voice.labels && typeof voice.labels === "object" ? Object.values(voice.labels).join(", ") : "";
|
|
3430
|
+
console.log(`${GREEN}${voice.voice_id}${RESET} ${voice.name}${labels ? ` ${DIM}(${labels})${RESET}` : ""}`);
|
|
3431
|
+
}
|
|
3432
|
+
if (voices.length > limit)
|
|
3433
|
+
console.log(`${DIM}… ${voices.length - limit} more (raise with --limit or use --json).${RESET}`);
|
|
3434
|
+
console.log(`${DIM}Use one: vidfarm tts "Hello" --cloud --voice <voice_id>${RESET}`);
|
|
3435
|
+
}
|
|
3293
3436
|
async function runSttCommand(argv) {
|
|
3294
3437
|
const parsed = parseArgs({
|
|
3295
3438
|
args: argv,
|
|
@@ -3307,6 +3450,7 @@ async function runSttCommand(argv) {
|
|
|
3307
3450
|
"gemini-key": { type: "string" },
|
|
3308
3451
|
"openai-key": { type: "string" },
|
|
3309
3452
|
"openrouter-key": { type: "string" },
|
|
3453
|
+
"own-key": { type: "boolean", default: false },
|
|
3310
3454
|
"no-wait": { type: "boolean", default: false },
|
|
3311
3455
|
tracer: { type: "string" },
|
|
3312
3456
|
folder: { type: "string" }
|
|
@@ -3405,6 +3549,11 @@ async function runSttCommand(argv) {
|
|
|
3405
3549
|
payload.provider = parsed.values.provider;
|
|
3406
3550
|
if (parsed.values.model)
|
|
3407
3551
|
payload.model = parsed.values.model;
|
|
3552
|
+
// Cloud STT defaults to ElevenLabs Scribe on the platform key (native
|
|
3553
|
+
// diarization + real word timings), wallet-billed. --own-key uses your own
|
|
3554
|
+
// ElevenLabs / BYOK key instead.
|
|
3555
|
+
if (parsed.values["own-key"])
|
|
3556
|
+
payload.use_wallet_credits = false;
|
|
3408
3557
|
const tracer = parsed.values.tracer ?? `devcli-stt-${Date.now().toString(36)}`;
|
|
3409
3558
|
const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/transcribe", auth: ctx.auth, body: { tracer, payload } });
|
|
3410
3559
|
assertApiOk(submit, "stt");
|
|
@@ -3953,7 +4102,8 @@ async function runClipperCommand(argv) {
|
|
|
3953
4102
|
out: { type: "string" },
|
|
3954
4103
|
tracer: { type: "string" },
|
|
3955
4104
|
folder: { type: "string" },
|
|
3956
|
-
name: { type: "string" }
|
|
4105
|
+
name: { type: "string" },
|
|
4106
|
+
"audio-only": { type: "boolean", default: false }
|
|
3957
4107
|
}
|
|
3958
4108
|
});
|
|
3959
4109
|
const source = parsed.positionals[0];
|
|
@@ -3970,6 +4120,10 @@ async function runClipperCommand(argv) {
|
|
|
3970
4120
|
if (!wholeVideo && (!hasStart || !hasEnd)) {
|
|
3971
4121
|
throw new Error("clipper needs BOTH --start and --end for an exact subrange, or NEITHER to import the whole video.");
|
|
3972
4122
|
}
|
|
4123
|
+
const audioOnly = Boolean(parsed.values["audio-only"]);
|
|
4124
|
+
if (audioOnly && wholeVideo) {
|
|
4125
|
+
throw new Error("--audio-only needs an exact subrange (--start <t> --end <t>). Whole-video import keeps the video track.");
|
|
4126
|
+
}
|
|
3973
4127
|
const body = {};
|
|
3974
4128
|
if (parsed.values.tracer)
|
|
3975
4129
|
body.tracer = String(parsed.values.tracer);
|
|
@@ -3977,6 +4131,8 @@ async function runClipperCommand(argv) {
|
|
|
3977
4131
|
body.folder_path = String(parsed.values.folder);
|
|
3978
4132
|
if (parsed.values.name)
|
|
3979
4133
|
body.name = String(parsed.values.name);
|
|
4134
|
+
if (audioOnly)
|
|
4135
|
+
body.audio_only = true;
|
|
3980
4136
|
// Resolve the source into the body once: a URL rides as source_url; a local
|
|
3981
4137
|
// file is staged into the temp store and referenced by temp_file_id.
|
|
3982
4138
|
if (/^https?:\/\//i.test(source)) {
|
|
@@ -4037,7 +4193,8 @@ async function runClipperCommand(argv) {
|
|
|
4037
4193
|
}
|
|
4038
4194
|
const j = res.json;
|
|
4039
4195
|
const dest = j?.folder_path ? `/raws/${j.folder_path}` : "your raws library";
|
|
4040
|
-
|
|
4196
|
+
const kindWord = (j?.media_kind === "audio" || audioOnly) ? "audio" : "clip";
|
|
4197
|
+
console.log(`${GREEN}Clipped ${kindWord} ${(j?.duration_sec ?? (endSec - startSec)).toFixed?.(2) ?? j?.duration_sec}s → ${dest} (clip ${j?.clip_id ?? "?"}).${RESET}`);
|
|
4041
4198
|
if (j?.view_url)
|
|
4042
4199
|
console.log(`${DIM}Preview: ${new URL(j.view_url, ctx.host).toString()}${RESET}`);
|
|
4043
4200
|
console.log(`${DIM}Grab another range from the same source with the same --tracer/--folder.${RESET}`);
|
package/dist/src/config.js
CHANGED
|
@@ -66,6 +66,18 @@ const schema = z.object({
|
|
|
66
66
|
OPENROUTER_API_KEY: z.string().optional(),
|
|
67
67
|
GEMINI_API_KEY: z.string().optional(),
|
|
68
68
|
PERPLEXITY_API_KEY: z.string().optional(),
|
|
69
|
+
// Platform ElevenLabs key — the fallback used for music / TTS / STT primitives
|
|
70
|
+
// when the caller opts into `use_wallet_credits` (the default) or has no usable
|
|
71
|
+
// BYOK audio key. Usage on this key is billed to the customer's vidfarm wallet.
|
|
72
|
+
ELEVENLABS_API_KEY: z.string().optional(),
|
|
73
|
+
// Base provider costs for ElevenLabs primitives billed to the wallet (the
|
|
74
|
+
// billing lambda applies the standard markup on top). Defaults track published
|
|
75
|
+
// ElevenLabs list pricing: TTS multilingual v2/v3 $0.10 / 1K chars (flash/turbo
|
|
76
|
+
// is $0.05), Music $0.15 / minute, Scribe STT $0.22 / hour ≈ $0.003667 / min.
|
|
77
|
+
// Override per-environment as pricing shifts.
|
|
78
|
+
ELEVENLABS_MUSIC_USD_PER_MIN: z.coerce.number().min(0).default(0.15),
|
|
79
|
+
ELEVENLABS_TTS_USD_PER_1K_CHARS: z.coerce.number().min(0).default(0.10),
|
|
80
|
+
ELEVENLABS_STT_USD_PER_MIN: z.coerce.number().min(0).default(0.003667),
|
|
69
81
|
GHOSTCUT_KEY: z.string().optional(),
|
|
70
82
|
GHOSTCUT_SECRET: z.string().optional(),
|
|
71
83
|
GHOSTCUT_USD_PER_30S: z.coerce.number().min(0).default(0.10),
|
package/dist/src/devcli/clips.js
CHANGED
|
@@ -44,6 +44,7 @@ export const CLIPS_HELP = `vidfarm raws — build & search a local raws library
|
|
|
44
44
|
--yes Skip the >$1 confirmation prompt
|
|
45
45
|
--frames <n> Keyframes per scene for tagging (1-3, default 2)
|
|
46
46
|
--no-audio Skip per-scene audio (Gemini only; no transcripts; cheaper)
|
|
47
|
+
--audio-only Save each mined clip as an audio-only .m4a (no video); waveform thumb
|
|
47
48
|
--concurrency <n> Parallel scenes (default 3; agent default 2)
|
|
48
49
|
--threshold <0..1> Scene-cut sensitivity (default 0.3; lower = more cuts)
|
|
49
50
|
--min-scene <sec> Merge scenes shorter than this (default 0.6)
|
|
@@ -133,6 +134,7 @@ async function runScan(argv) {
|
|
|
133
134
|
"allow-text": { type: "boolean" },
|
|
134
135
|
"max-clips": { type: "string" },
|
|
135
136
|
"no-refine": { type: "boolean" },
|
|
137
|
+
"audio-only": { type: "boolean" },
|
|
136
138
|
cloud: { type: "boolean" },
|
|
137
139
|
url: { type: "string" },
|
|
138
140
|
tracer: { type: "string" },
|
|
@@ -197,7 +199,8 @@ async function runScan(argv) {
|
|
|
197
199
|
aspect,
|
|
198
200
|
cropFocus,
|
|
199
201
|
avoidText,
|
|
200
|
-
maxClips
|
|
202
|
+
maxClips,
|
|
203
|
+
audioOnly: Boolean(values["audio-only"])
|
|
201
204
|
});
|
|
202
205
|
}
|
|
203
206
|
const videoArg = positionals[0];
|
|
@@ -336,6 +339,7 @@ async function runScan(argv) {
|
|
|
336
339
|
guidancePrompt: guidance,
|
|
337
340
|
refineSegmentation: !values["no-refine"],
|
|
338
341
|
reencodeClips: true,
|
|
342
|
+
audioOnly: Boolean(values["audio-only"]),
|
|
339
343
|
crop: aspect && aspect !== "original" ? { aspect, focus: cropFocus } : undefined
|
|
340
344
|
};
|
|
341
345
|
console.log(`[raws] scanning ${scenes.length} scenes (${provider}/${activeClient.tagModelId}, ${concurrency}x) …`);
|
|
@@ -466,7 +470,8 @@ async function runScanCloud(input) {
|
|
|
466
470
|
// Send explicitly (incl. false) so --allow-text can override the
|
|
467
471
|
// server-side avoid-text default.
|
|
468
472
|
avoid_text: input.avoidText,
|
|
469
|
-
...(input.maxClips ? { max_clips: input.maxClips } : {})
|
|
473
|
+
...(input.maxClips ? { max_clips: input.maxClips } : {}),
|
|
474
|
+
...(input.audioOnly ? { audio_only: true } : {})
|
|
470
475
|
}
|
|
471
476
|
})
|
|
472
477
|
});
|
package/dist/src/domain.js
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
+
// Providers that count as a qualified AI key for text/image/video generation
|
|
2
|
+
// (everything except the audio-only elevenlabs key).
|
|
3
|
+
export const QUALIFIED_AI_PROVIDER_TYPES = ["openai", "gemini", "openrouter", "perplexity"];
|
|
1
4
|
export const CUSTOMER_PLAN_TIERS = ["client", "standalone", "agency"];
|
|
2
5
|
//# sourceMappingURL=domain.js.map
|
package/dist/src/editor-chat.js
CHANGED
|
@@ -21,6 +21,10 @@ export function defaultEditorChatModel(provider) {
|
|
|
21
21
|
return "qwen/qwen3.6-plus";
|
|
22
22
|
case "perplexity":
|
|
23
23
|
return "sonar";
|
|
24
|
+
default:
|
|
25
|
+
// Non-text providers (e.g. audio-only elevenlabs) never reach editor chat;
|
|
26
|
+
// return a safe text default rather than undefined.
|
|
27
|
+
return "gpt-5.4-mini";
|
|
24
28
|
}
|
|
25
29
|
}
|
|
26
30
|
export function buildTemplateEditorChatSystemPrompt(input) {
|
|
@@ -198,6 +198,20 @@ export function createPrimitiveJobContext(input) {
|
|
|
198
198
|
workerId: input.workerId,
|
|
199
199
|
...request
|
|
200
200
|
});
|
|
201
|
+
},
|
|
202
|
+
async generateMusic(request) {
|
|
203
|
+
return input.providers.generateMusic({
|
|
204
|
+
customerId: input.customer.id,
|
|
205
|
+
jobId: input.job.id,
|
|
206
|
+
workerId: input.workerId,
|
|
207
|
+
...request
|
|
208
|
+
});
|
|
209
|
+
},
|
|
210
|
+
async listElevenLabsVoices(request) {
|
|
211
|
+
return input.providers.listElevenLabsVoices({
|
|
212
|
+
customerId: input.customer.id,
|
|
213
|
+
...request
|
|
214
|
+
});
|
|
201
215
|
}
|
|
202
216
|
},
|
|
203
217
|
hyperframes: {
|
|
@@ -429,7 +429,11 @@ const ttsPayloadSchema = z.preprocess((raw) => {
|
|
|
429
429
|
text: z.string().min(1).max(8000),
|
|
430
430
|
voice: z.string().min(1).max(120).optional(),
|
|
431
431
|
instructions: z.string().min(1).max(4000).optional(),
|
|
432
|
-
output_format: z.enum(["mp3", "wav"]).default("mp3")
|
|
432
|
+
output_format: z.enum(["mp3", "wav"]).default("mp3"),
|
|
433
|
+
// Default TRUE: great ElevenLabs narration out of the box on the platform key,
|
|
434
|
+
// billed to the wallet. Set false to use your own ElevenLabs key or a BYOK
|
|
435
|
+
// openai/gemini/openrouter key instead. See voice options via /audio/voices.
|
|
436
|
+
use_wallet_credits: z.boolean().default(true)
|
|
433
437
|
}).superRefine((value, ctx) => {
|
|
434
438
|
// Reject an explicit provider/voice contradiction at enqueue (400) instead
|
|
435
439
|
// of queueing a job that can only fail at the provider.
|
|
@@ -465,7 +469,36 @@ const sttPayloadSchema = z.preprocess((raw) => {
|
|
|
465
469
|
// output and downstream callers estimate word windows.
|
|
466
470
|
word_timestamps: z.boolean().default(false),
|
|
467
471
|
language: z.string().min(2).max(16).optional(),
|
|
468
|
-
prompt: z.string().min(1).max(2000).optional()
|
|
472
|
+
prompt: z.string().min(1).max(2000).optional(),
|
|
473
|
+
// Default TRUE: platform ElevenLabs Scribe (native diarization + real word
|
|
474
|
+
// timestamps), billed to the wallet. Set false to use your own ElevenLabs
|
|
475
|
+
// key or a BYOK gemini/openai/openrouter key.
|
|
476
|
+
use_wallet_credits: z.boolean().default(true)
|
|
477
|
+
}));
|
|
478
|
+
// Music generation (ElevenLabs). Prompt-based, or a hand-built composition plan.
|
|
479
|
+
const musicPayloadSchema = z.preprocess((raw) => {
|
|
480
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
481
|
+
return raw;
|
|
482
|
+
const payload = raw;
|
|
483
|
+
return {
|
|
484
|
+
...payload,
|
|
485
|
+
prompt: payload.prompt ?? payload.text ?? payload.description,
|
|
486
|
+
music_length_ms: payload.music_length_ms ?? payload.length_ms ?? payload.duration_ms
|
|
487
|
+
};
|
|
488
|
+
}, z.object({
|
|
489
|
+
prompt: z.string().min(1).max(2000).optional(),
|
|
490
|
+
// ElevenLabs music caps at 5 minutes per generation.
|
|
491
|
+
music_length_ms: z.number().int().min(3000).max(300000).default(30000),
|
|
492
|
+
// Optional ElevenLabs composition plan for granular section-by-section control.
|
|
493
|
+
composition_plan: z.record(z.string(), z.any()).optional(),
|
|
494
|
+
model: z.string().min(1).optional(),
|
|
495
|
+
output_format: z.string().min(1).optional(),
|
|
496
|
+
// Default TRUE: platform ElevenLabs key, billed to the wallet. Set false to
|
|
497
|
+
// use your own saved ElevenLabs key (no wallet charge).
|
|
498
|
+
use_wallet_credits: z.boolean().default(true)
|
|
499
|
+
}).refine((value) => Boolean(value.prompt) || Boolean(value.composition_plan), {
|
|
500
|
+
message: "Provide either a prompt or a composition_plan.",
|
|
501
|
+
path: ["prompt"]
|
|
469
502
|
}));
|
|
470
503
|
// Animated captions from speech: STT with word-level timings, paged into
|
|
471
504
|
// ready-to-apply cue layers. Either transcribe a source_url or page plain text.
|
|
@@ -714,6 +747,7 @@ const PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID = "primitive:brainstorm_product_
|
|
|
714
747
|
const PRIMITIVE_MEDIA_DEDUPE_ID = "primitive:media_dedupe";
|
|
715
748
|
const PRIMITIVE_TTS_ID = "primitive:tts";
|
|
716
749
|
const PRIMITIVE_STT_ID = "primitive:stt";
|
|
750
|
+
const PRIMITIVE_MUSIC_ID = "primitive:music";
|
|
717
751
|
const PRIMITIVE_CAPTIONS_ID = "primitive:captions";
|
|
718
752
|
const PRIMITIVE_SPEECH_REGENERATE_ID = "primitive:speech_regenerate";
|
|
719
753
|
// Gemini inline audio caps out around 20MB; at the 64kbps mono mp3 we extract,
|
|
@@ -2018,20 +2052,34 @@ const ttsPrimitive = definePrimitive({
|
|
|
2018
2052
|
text: payload.text,
|
|
2019
2053
|
voice: payload.voice,
|
|
2020
2054
|
instructions: payload.instructions,
|
|
2021
|
-
responseFormat: payload.output_format
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
//
|
|
2055
|
+
responseFormat: payload.output_format,
|
|
2056
|
+
useWalletCredits: payload.use_wallet_credits
|
|
2057
|
+
});
|
|
2058
|
+
// Route may have resolved to a different provider (e.g. platform ElevenLabs
|
|
2059
|
+
// when use_wallet_credits is on); report what actually ran.
|
|
2060
|
+
const effectiveProvider = generated.effectiveProvider;
|
|
2061
|
+
const effectiveModel = generated.effectiveModel;
|
|
2062
|
+
if (generated.billing) {
|
|
2063
|
+
await ctx.billing.record({
|
|
2064
|
+
type: "ai_generation",
|
|
2065
|
+
costUsd: generated.billing.costUsd,
|
|
2066
|
+
costCenterSlug: generated.billing.costCenterSlug,
|
|
2067
|
+
metadata: { ...generated.billing.metadata, primitive: "tts", voice: payload.voice ?? null }
|
|
2068
|
+
});
|
|
2069
|
+
}
|
|
2070
|
+
// Gemini/ElevenLabs TTS may answer in wav regardless of the requested
|
|
2071
|
+
// format; name the artifact after what actually came back.
|
|
2025
2072
|
const extension = generated.contentType.includes("wav") ? "wav" : "mp3";
|
|
2026
2073
|
const stored = await ctx.storage.putBuffer(`speech.${extension}`, generated.bytes, {
|
|
2027
2074
|
contentType: generated.contentType,
|
|
2028
2075
|
kind: "audio",
|
|
2029
2076
|
metadata: {
|
|
2030
2077
|
primitive: "tts",
|
|
2031
|
-
provider,
|
|
2032
|
-
model,
|
|
2078
|
+
provider: effectiveProvider,
|
|
2079
|
+
model: effectiveModel,
|
|
2033
2080
|
voice: payload.voice ?? null,
|
|
2034
|
-
styleInstructions: payload.instructions ?? null
|
|
2081
|
+
styleInstructions: payload.instructions ?? null,
|
|
2082
|
+
wallet_billed: Boolean(generated.billing)
|
|
2035
2083
|
}
|
|
2036
2084
|
});
|
|
2037
2085
|
ctx.logger.progress(1, "Speech generation complete");
|
|
@@ -2044,11 +2092,72 @@ const ttsPrimitive = definePrimitive({
|
|
|
2044
2092
|
audio: {
|
|
2045
2093
|
file_url: stored.url,
|
|
2046
2094
|
content_type: generated.contentType,
|
|
2047
|
-
provider,
|
|
2048
|
-
model,
|
|
2095
|
+
provider: effectiveProvider,
|
|
2096
|
+
model: effectiveModel,
|
|
2049
2097
|
voice: payload.voice ?? null,
|
|
2050
2098
|
style_instructions: payload.instructions ?? null,
|
|
2051
|
-
text_length: payload.text.length
|
|
2099
|
+
text_length: payload.text.length,
|
|
2100
|
+
wallet_billed: Boolean(generated.billing)
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
};
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
});
|
|
2107
|
+
const musicPrimitive = definePrimitive({
|
|
2108
|
+
id: PRIMITIVE_MUSIC_ID,
|
|
2109
|
+
kind: "music",
|
|
2110
|
+
operations: {
|
|
2111
|
+
run: {
|
|
2112
|
+
description: "Generate music (instrumental or with lyrics) from a prompt or a composition plan via ElevenLabs, persisted as a reusable platform audio URL.",
|
|
2113
|
+
inputSchema: musicPayloadSchema,
|
|
2114
|
+
workflow: "run"
|
|
2115
|
+
}
|
|
2116
|
+
},
|
|
2117
|
+
jobs: {
|
|
2118
|
+
async run(ctx, input) {
|
|
2119
|
+
const payload = musicPayloadSchema.parse(input);
|
|
2120
|
+
ctx.logger.progress(0.12, "Composing music");
|
|
2121
|
+
const generated = await ctx.providers.generateMusic({
|
|
2122
|
+
prompt: payload.prompt,
|
|
2123
|
+
musicLengthMs: payload.music_length_ms,
|
|
2124
|
+
compositionPlan: payload.composition_plan,
|
|
2125
|
+
model: payload.model,
|
|
2126
|
+
outputFormat: payload.output_format,
|
|
2127
|
+
useWalletCredits: payload.use_wallet_credits
|
|
2128
|
+
});
|
|
2129
|
+
if (generated.billing) {
|
|
2130
|
+
await ctx.billing.record({
|
|
2131
|
+
type: "ai_generation",
|
|
2132
|
+
costUsd: generated.billing.costUsd,
|
|
2133
|
+
costCenterSlug: generated.billing.costCenterSlug,
|
|
2134
|
+
metadata: { ...generated.billing.metadata, primitive: "music", prompt: payload.prompt ?? null }
|
|
2135
|
+
});
|
|
2136
|
+
}
|
|
2137
|
+
const stored = await ctx.storage.putBuffer("music.mp3", generated.bytes, {
|
|
2138
|
+
contentType: generated.contentType,
|
|
2139
|
+
kind: "audio",
|
|
2140
|
+
metadata: {
|
|
2141
|
+
primitive: "music",
|
|
2142
|
+
provider: "elevenlabs",
|
|
2143
|
+
music_length_ms: generated.musicLengthMs,
|
|
2144
|
+
wallet_billed: Boolean(generated.billing)
|
|
2145
|
+
}
|
|
2146
|
+
});
|
|
2147
|
+
ctx.logger.progress(1, "Music generation complete");
|
|
2148
|
+
return {
|
|
2149
|
+
progress: 1,
|
|
2150
|
+
output: {
|
|
2151
|
+
files: compactUrls([stored.url]),
|
|
2152
|
+
primary_file_url: stored.url,
|
|
2153
|
+
audioUrl: stored.url,
|
|
2154
|
+
audio: {
|
|
2155
|
+
file_url: stored.url,
|
|
2156
|
+
content_type: generated.contentType,
|
|
2157
|
+
provider: "elevenlabs",
|
|
2158
|
+
music_length_ms: generated.musicLengthMs,
|
|
2159
|
+
prompt: payload.prompt ?? null,
|
|
2160
|
+
wallet_billed: Boolean(generated.billing)
|
|
2052
2161
|
}
|
|
2053
2162
|
}
|
|
2054
2163
|
};
|
|
@@ -2122,18 +2231,29 @@ const sttPrimitive = definePrimitive({
|
|
|
2122
2231
|
const payload = sttPayloadSchema.parse(input);
|
|
2123
2232
|
const { audioBytes, audioContentType, extractionMetadata } = await loadSpeechAudioForPrimitive(ctx, payload);
|
|
2124
2233
|
ctx.logger.progress(0.45, "Transcribing speech");
|
|
2125
|
-
const
|
|
2126
|
-
const
|
|
2234
|
+
const requestedProvider = payload.provider ?? (payload.word_timestamps ? "openai" : "gemini");
|
|
2235
|
+
const requestedModel = payload.model ?? defaultSpeechModelFor("stt", requestedProvider) ?? "";
|
|
2127
2236
|
const transcription = await ctx.providers.transcribeSpeech({
|
|
2128
|
-
provider,
|
|
2129
|
-
model,
|
|
2237
|
+
provider: requestedProvider,
|
|
2238
|
+
model: requestedModel,
|
|
2130
2239
|
audio: audioBytes,
|
|
2131
2240
|
contentType: audioContentType,
|
|
2132
2241
|
prompt: payload.prompt,
|
|
2133
2242
|
language: payload.language,
|
|
2134
2243
|
diarize: payload.diarize,
|
|
2135
|
-
wordTimestamps: payload.word_timestamps
|
|
2136
|
-
|
|
2244
|
+
wordTimestamps: payload.word_timestamps,
|
|
2245
|
+
useWalletCredits: payload.use_wallet_credits
|
|
2246
|
+
});
|
|
2247
|
+
const provider = transcription.effectiveProvider;
|
|
2248
|
+
const model = transcription.effectiveModel;
|
|
2249
|
+
if (transcription.billing) {
|
|
2250
|
+
await ctx.billing.record({
|
|
2251
|
+
type: "ai_generation",
|
|
2252
|
+
costUsd: transcription.billing.costUsd,
|
|
2253
|
+
costCenterSlug: transcription.billing.costCenterSlug,
|
|
2254
|
+
metadata: { ...transcription.billing.metadata, primitive: "stt", source_url: payload.source_url }
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2137
2257
|
ctx.logger.progress(0.85, "Storing transcript artifacts");
|
|
2138
2258
|
const speakers = [...new Set(transcription.segments.map((segment) => segment.speaker))];
|
|
2139
2259
|
const srt = buildSrtFromSegments(transcription.segments);
|
|
@@ -3114,6 +3234,7 @@ class PrimitiveRegistry {
|
|
|
3114
3234
|
[audioNormalizePrimitive.id, audioNormalizePrimitive],
|
|
3115
3235
|
[ttsPrimitive.id, ttsPrimitive],
|
|
3116
3236
|
[sttPrimitive.id, sttPrimitive],
|
|
3237
|
+
[musicPrimitive.id, musicPrimitive],
|
|
3117
3238
|
[captionsPrimitive.id, captionsPrimitive],
|
|
3118
3239
|
[speechRegeneratePrimitive.id, speechRegeneratePrimitive],
|
|
3119
3240
|
[hyperframesRenderPrimitive.id, hyperframesRenderPrimitive],
|
|
@@ -3521,6 +3642,7 @@ function normalizeProductPlacementPayload(raw) {
|
|
|
3521
3642
|
};
|
|
3522
3643
|
}
|
|
3523
3644
|
function preferredTextProvider(availableProviders) {
|
|
3645
|
+
// Text-only priority — the audio-only elevenlabs provider is never a candidate.
|
|
3524
3646
|
const priority = ["openai", "gemini", "openrouter", "perplexity"];
|
|
3525
3647
|
return priority.find((provider) => availableProviders.includes(provider)) ?? "openai";
|
|
3526
3648
|
}
|
|
@@ -59,8 +59,8 @@ export function renderReskinChat(input = DEFAULT_INPUT, opts = {}) {
|
|
|
59
59
|
<select class="rk-chat-mode-select" id="rk-chat-mode">
|
|
60
60
|
<option value="/chat" selected>Chat</option>
|
|
61
61
|
<option value="/tools/image">Create Image</option>
|
|
62
|
-
<option value="/tools/clipper">Clip a Video</option>
|
|
63
62
|
<option value="/editor/original/fork/new">Create Video</option>
|
|
63
|
+
<option value="/tools/clipper">Clip Media</option>
|
|
64
64
|
</select>
|
|
65
65
|
<span class="rk-chat-mode-chev" aria-hidden="true">▾</span>
|
|
66
66
|
</label>
|