@mevdragon/vidfarm-devcli 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/.agents/skills/music/SKILL.md +416 -0
  2. package/.agents/skills/music/references/api_reference.md +519 -0
  3. package/.agents/skills/music/references/installation.md +65 -0
  4. package/.agents/skills/text-to-speech/SKILL.md +226 -0
  5. package/.agents/skills/text-to-speech/references/installation.md +90 -0
  6. package/.agents/skills/text-to-speech/references/streaming.md +307 -0
  7. package/.agents/skills/text-to-speech/references/voice-settings.md +115 -0
  8. package/.agents/skills/vidfarm-media/SKILL.md +25 -11
  9. package/.agents/skills/vidfarm-media/references/tts.md +41 -3
  10. package/SKILL.director.md +31 -13
  11. package/SKILL.platform.md +2 -2
  12. package/dist/src/app.js +129 -41
  13. package/dist/src/cli.js +162 -5
  14. package/dist/src/config.js +12 -0
  15. package/dist/src/devcli/clips.js +7 -2
  16. package/dist/src/domain.js +3 -0
  17. package/dist/src/editor-chat.js +4 -0
  18. package/dist/src/primitive-context.js +14 -0
  19. package/dist/src/primitive-registry.js +140 -18
  20. package/dist/src/reskin/chat-page.js +1 -1
  21. package/dist/src/reskin/inpaint-clipper-page.js +446 -205
  22. package/dist/src/reskin/library-page.js +7 -1
  23. package/dist/src/reskin/portfolio-page.js +9 -9
  24. package/dist/src/reskin/settings-page.js +4 -4
  25. package/dist/src/reskin/theme.js +1 -0
  26. package/dist/src/services/billing.js +5 -0
  27. package/dist/src/services/clip-curation/ffmpeg.js +48 -0
  28. package/dist/src/services/clip-curation/hunt.js +2 -0
  29. package/dist/src/services/clip-curation/index.js +1 -1
  30. package/dist/src/services/clip-curation/scan.js +29 -16
  31. package/dist/src/services/elevenlabs.js +222 -0
  32. package/dist/src/services/providers.js +216 -2
  33. package/dist/src/services/swipe-customize.js +5 -2
  34. package/package.json +1 -1
package/dist/src/cli.js CHANGED
@@ -285,6 +285,8 @@ Raws (the third library — mine long-form video into a reusable raws store):
285
285
  --tracer <name> Group this session's clips (also the default folder)
286
286
  --folder <name> Save the clip into /raws/<name>
287
287
  --name <text> Human name for the clip
288
+ --audio-only Save just the audio of the subrange as an .m4a raw
289
+ (needs --start/--end; e.g. lift a copyright-free track)
288
290
  (multi-clip: re-run with a new range + same --tracer)
289
291
 
290
292
  Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit backup:
@@ -300,7 +302,19 @@ Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit bac
300
302
  --engine local KEYLESS local Kokoro-82M engine (free, no account) — fixed voice
301
303
  presets (af_heart, am_adam, bf_emma, …), no --style. When NO
302
304
  provider key is found the default byok engine auto-falls back here.
303
- --cloud BACKUP: saved-key platform job → POST /api/v1/primitives/audio/speech (+ poll)
305
+ --cloud BACKUP: platform job → POST /api/v1/primitives/audio/speech (+ poll).
306
+ Defaults to ElevenLabs on the platform key (wallet-billed); --voice
307
+ takes an ElevenLabs voice_id (see: vidfarm voices).
308
+ --own-key (--cloud) use your own ElevenLabs / BYOK key instead of the wallet
309
+ --list-voices List ElevenLabs voices and exit (same as: vidfarm voices)
310
+ music "<prompt>" Prompt → music track (ElevenLabs). CLOUD-ONLY (no local engine)
311
+ --length <sec> | --length-ms <ms> Duration (default 30s, max 5 min)
312
+ --out <file> Output mp3 path (default music-<id>.mp3)
313
+ --own-key Use your own ElevenLabs key instead of the platform key + wallet
314
+ (default: platform ElevenLabs, wallet-billed) → POST /api/v1/primitives/music/generate
315
+ voices List ElevenLabs voices for TTS (voice_id + name + labels).
316
+ --own-key List voices on your own saved ElevenLabs key (default: platform account)
317
+ --limit <n> | --json Show more / raw JSON
304
318
  stt <file|url> Video or audio → transcript (alias: transcribe). LOCAL: local ffmpeg demux +
305
319
  Returns BOTH formats: the simple subtitle your provider key (gemini labels
306
320
  version (plain text + timed SRT cues) and speakers; openai/openrouter give
@@ -315,8 +329,11 @@ Speech (TTS/STT) — LOCAL-FIRST on your own AI key; --cloud is the explicit bac
315
329
  timestamps (words[] in the .json/--json output), no speaker labels.
316
330
  When NO provider key is found the default byok engine auto-falls
317
331
  back here. --model then picks the whisper model (tiny.en…large-v3).
318
- --cloud BACKUP: saved-key platform job → POST /api/v1/primitives/audio/transcribe (+ poll)
332
+ --cloud BACKUP: platform job → POST /api/v1/primitives/audio/transcribe (+ poll).
333
+ Defaults to ElevenLabs Scribe on the platform key (wallet-billed;
334
+ native diarization + real word timings).
319
335
  (a local file uploads to your temp folder first — 30-day TTL)
336
+ --own-key (--cloud) use your own ElevenLabs / BYOK key instead of the wallet
320
337
 
321
338
  Animated captions (word-by-word TikTok/CapCut styles on a pulled/served composition):
322
339
  captions generate <dir> Transcribe the narration & lay down animated LOCAL: local ffmpeg demux +
@@ -586,6 +603,12 @@ async function main() {
586
603
  case "tts":
587
604
  await runTtsCommand(rest);
588
605
  return;
606
+ case "music":
607
+ await runMusicCommand(rest);
608
+ return;
609
+ case "voices":
610
+ await runVoicesCommand(rest);
611
+ return;
589
612
  case "stt":
590
613
  case "transcribe":
591
614
  await runSttCommand(rest);
@@ -3045,10 +3068,17 @@ async function runTtsCommand(argv) {
3045
3068
  "gemini-key": { type: "string" },
3046
3069
  "openai-key": { type: "string" },
3047
3070
  "openrouter-key": { type: "string" },
3071
+ "own-key": { type: "boolean", default: false },
3072
+ "list-voices": { type: "boolean", default: false },
3048
3073
  "no-wait": { type: "boolean", default: false },
3049
3074
  tracer: { type: "string" }
3050
3075
  }
3051
3076
  });
3077
+ // --list-voices: browse the ElevenLabs voice catalog and exit (cloud call).
3078
+ if (parsed.values["list-voices"]) {
3079
+ await runVoicesCommand(parsed.values["own-key"] ? ["--own-key"] : []);
3080
+ return;
3081
+ }
3052
3082
  const text = (parsed.values.text ?? parsed.positionals.join(" ")).trim();
3053
3083
  if (!text)
3054
3084
  throw new Error('tts requires text: vidfarm tts "Welcome back to the channel" --style "calm, warm narrator".');
@@ -3118,7 +3148,9 @@ async function runTtsCommand(argv) {
3118
3148
  }
3119
3149
  return;
3120
3150
  }
3121
- // BACKUP: the platform primitive route (saved provider keys, async job).
3151
+ // BACKUP: the platform primitive route (async job). Defaults to high-quality
3152
+ // ElevenLabs on the platform key, billed to the wallet (use_wallet_credits).
3153
+ // --own-key flips to your own saved ElevenLabs / BYOK provider key instead.
3122
3154
  const ctx = commonContext(parsed.values);
3123
3155
  const payload = { text, output_format: format };
3124
3156
  if (voice)
@@ -3129,6 +3161,11 @@ async function runTtsCommand(argv) {
3129
3161
  payload.provider = parsed.values.provider;
3130
3162
  if (parsed.values.model)
3131
3163
  payload.model = parsed.values.model;
3164
+ if (parsed.values["own-key"])
3165
+ payload.use_wallet_credits = false;
3166
+ if (!ctx.json && !parsed.values["own-key"]) {
3167
+ console.log(`${DIM}Using ElevenLabs on the platform key (wallet-billed). Browse voices: vidfarm voices — then pass --voice <voice_id>. Add --own-key to use your own key.${RESET}`);
3168
+ }
3132
3169
  const tracer = parsed.values.tracer ?? `devcli-tts-${Date.now().toString(36)}`;
3133
3170
  const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/speech", auth: ctx.auth, body: { tracer, payload } });
3134
3171
  assertApiOk(submit, "tts");
@@ -3290,6 +3327,112 @@ function emitSttResult(input) {
3290
3327
  console.log(`${GREEN}Wrote ${dest}${RESET}`);
3291
3328
  }
3292
3329
  }
3330
+ // Music generation (ElevenLabs) — cloud-only (no local engine). Defaults to the
3331
+ // platform key billed to the wallet; --own-key uses your saved ElevenLabs key.
3332
+ async function runMusicCommand(argv) {
3333
+ const parsed = parseArgs({
3334
+ args: argv,
3335
+ allowPositionals: true,
3336
+ options: {
3337
+ ...commonOptions(),
3338
+ prompt: { type: "string" },
3339
+ "length-ms": { type: "string" },
3340
+ length: { type: "string" },
3341
+ model: { type: "string" },
3342
+ out: { type: "string" },
3343
+ "own-key": { type: "boolean", default: false },
3344
+ "no-wait": { type: "boolean", default: false },
3345
+ tracer: { type: "string" }
3346
+ }
3347
+ });
3348
+ const prompt = (parsed.values.prompt ?? parsed.positionals.join(" ")).trim();
3349
+ if (!prompt)
3350
+ throw new Error('music requires a prompt: vidfarm music "upbeat lo-fi hip hop beat with jazzy piano" --length 30');
3351
+ const ctx = commonContext(parsed.values);
3352
+ let musicLengthMs;
3353
+ const lengthMs = parsed.values["length-ms"]?.trim();
3354
+ const lengthSec = parsed.values.length?.trim();
3355
+ if (lengthMs)
3356
+ musicLengthMs = Math.round(Number(lengthMs));
3357
+ else if (lengthSec)
3358
+ musicLengthMs = Math.round(Number(lengthSec) * 1000);
3359
+ const payload = { prompt };
3360
+ if (musicLengthMs && Number.isFinite(musicLengthMs))
3361
+ payload.music_length_ms = musicLengthMs;
3362
+ if (parsed.values.model)
3363
+ payload.model = parsed.values.model;
3364
+ if (parsed.values["own-key"])
3365
+ payload.use_wallet_credits = false;
3366
+ if (!ctx.json && !parsed.values["own-key"]) {
3367
+ console.log(`${DIM}Composing with ElevenLabs on the platform key (wallet-billed). Add --own-key to use your own ElevenLabs key.${RESET}`);
3368
+ }
3369
+ const tracer = parsed.values.tracer ?? `devcli-music-${Date.now().toString(36)}`;
3370
+ const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/music/generate", auth: ctx.auth, body: { tracer, payload } });
3371
+ assertApiOk(submit, "music");
3372
+ const jobId = submit.json?.job_id;
3373
+ if (parsed.values["no-wait"] || !jobId) {
3374
+ if (!ctx.json && jobId)
3375
+ console.log(`${DIM}Queued ${jobId} (tracer ${tracer}). Poll: vidfarm api GET /api/v1/user/me/jobs/${jobId} — or drop --no-wait to auto-poll.${RESET}`);
3376
+ emitResult(submit, ctx.json);
3377
+ return;
3378
+ }
3379
+ if (!ctx.json)
3380
+ console.log(`${DIM}Generating music (${jobId})… polling every 5s.${RESET}`);
3381
+ const job = await pollPrimitiveJob(ctx, jobId);
3382
+ const mediaUrl = resolveJobMediaUrl(job);
3383
+ if (!mediaUrl) {
3384
+ if (ctx.json) {
3385
+ printJson({ ok: false, job_id: jobId, status: String(job?.status ?? ""), job });
3386
+ }
3387
+ else {
3388
+ console.log(`${RED}Music ${String(job?.status ?? "did not finish")} — no audio URL.${RESET}`);
3389
+ printJson(job);
3390
+ }
3391
+ process.exitCode = 1;
3392
+ return;
3393
+ }
3394
+ const outPath = path.resolve(process.cwd(), parsed.values.out ?? `music-${jobId}.mp3`);
3395
+ await downloadUrlToFile(mediaUrl, outPath);
3396
+ if (ctx.json) {
3397
+ printJson({ ok: true, job_id: jobId, tracer, audio_url: mediaUrl, out: outPath });
3398
+ }
3399
+ else {
3400
+ console.log(`${GREEN}Wrote ${outPath}${RESET} ${DIM}(${mediaUrl})${RESET}`);
3401
+ console.log(`${DIM}Place it as its own <audio> layer under narration (~0.1–0.2 volume).${RESET}`);
3402
+ }
3403
+ }
3404
+ // List ElevenLabs voices for TTS. --own-key lists the voices on your own saved
3405
+ // ElevenLabs key; default lists the platform account's voices.
3406
+ async function runVoicesCommand(argv) {
3407
+ const parsed = parseArgs({
3408
+ args: argv,
3409
+ allowPositionals: true,
3410
+ options: {
3411
+ ...commonOptions(),
3412
+ "own-key": { type: "boolean", default: false },
3413
+ limit: { type: "string" }
3414
+ }
3415
+ });
3416
+ const ctx = commonContext(parsed.values);
3417
+ const query = parsed.values["own-key"] ? "?use_wallet_credits=false" : "";
3418
+ const res = await apiRequest({ method: "GET", host: ctx.host, path: `/api/v1/primitives/audio/voices${query}`, auth: ctx.auth });
3419
+ assertApiOk(res, "voices");
3420
+ const body = (res.json ?? {});
3421
+ const voices = Array.isArray(body.voices) ? body.voices : [];
3422
+ if (ctx.json) {
3423
+ printJson(body);
3424
+ return;
3425
+ }
3426
+ console.log(`${DIM}${voices.length} ElevenLabs voices (${body.scope ?? "platform"} account). Browse more: ${body.voice_library_url ?? "https://elevenlabs.io/app/voice-library"}${RESET}`);
3427
+ const limit = Number(parsed.values.limit ?? "40") || 40;
3428
+ for (const voice of voices.slice(0, limit)) {
3429
+ const labels = voice.labels && typeof voice.labels === "object" ? Object.values(voice.labels).join(", ") : "";
3430
+ console.log(`${GREEN}${voice.voice_id}${RESET} ${voice.name}${labels ? ` ${DIM}(${labels})${RESET}` : ""}`);
3431
+ }
3432
+ if (voices.length > limit)
3433
+ console.log(`${DIM}… ${voices.length - limit} more (raise with --limit or use --json).${RESET}`);
3434
+ console.log(`${DIM}Use one: vidfarm tts "Hello" --cloud --voice <voice_id>${RESET}`);
3435
+ }
3293
3436
  async function runSttCommand(argv) {
3294
3437
  const parsed = parseArgs({
3295
3438
  args: argv,
@@ -3307,6 +3450,7 @@ async function runSttCommand(argv) {
3307
3450
  "gemini-key": { type: "string" },
3308
3451
  "openai-key": { type: "string" },
3309
3452
  "openrouter-key": { type: "string" },
3453
+ "own-key": { type: "boolean", default: false },
3310
3454
  "no-wait": { type: "boolean", default: false },
3311
3455
  tracer: { type: "string" },
3312
3456
  folder: { type: "string" }
@@ -3405,6 +3549,11 @@ async function runSttCommand(argv) {
3405
3549
  payload.provider = parsed.values.provider;
3406
3550
  if (parsed.values.model)
3407
3551
  payload.model = parsed.values.model;
3552
+ // Cloud STT defaults to ElevenLabs Scribe on the platform key (native
3553
+ // diarization + real word timings), wallet-billed. --own-key uses your own
3554
+ // ElevenLabs / BYOK key instead.
3555
+ if (parsed.values["own-key"])
3556
+ payload.use_wallet_credits = false;
3408
3557
  const tracer = parsed.values.tracer ?? `devcli-stt-${Date.now().toString(36)}`;
3409
3558
  const submit = await apiRequest({ method: "POST", host: ctx.host, path: "/api/v1/primitives/audio/transcribe", auth: ctx.auth, body: { tracer, payload } });
3410
3559
  assertApiOk(submit, "stt");
@@ -3953,7 +4102,8 @@ async function runClipperCommand(argv) {
3953
4102
  out: { type: "string" },
3954
4103
  tracer: { type: "string" },
3955
4104
  folder: { type: "string" },
3956
- name: { type: "string" }
4105
+ name: { type: "string" },
4106
+ "audio-only": { type: "boolean", default: false }
3957
4107
  }
3958
4108
  });
3959
4109
  const source = parsed.positionals[0];
@@ -3970,6 +4120,10 @@ async function runClipperCommand(argv) {
3970
4120
  if (!wholeVideo && (!hasStart || !hasEnd)) {
3971
4121
  throw new Error("clipper needs BOTH --start and --end for an exact subrange, or NEITHER to import the whole video.");
3972
4122
  }
4123
+ const audioOnly = Boolean(parsed.values["audio-only"]);
4124
+ if (audioOnly && wholeVideo) {
4125
+ throw new Error("--audio-only needs an exact subrange (--start <t> --end <t>). Whole-video import keeps the video track.");
4126
+ }
3973
4127
  const body = {};
3974
4128
  if (parsed.values.tracer)
3975
4129
  body.tracer = String(parsed.values.tracer);
@@ -3977,6 +4131,8 @@ async function runClipperCommand(argv) {
3977
4131
  body.folder_path = String(parsed.values.folder);
3978
4132
  if (parsed.values.name)
3979
4133
  body.name = String(parsed.values.name);
4134
+ if (audioOnly)
4135
+ body.audio_only = true;
3980
4136
  // Resolve the source into the body once: a URL rides as source_url; a local
3981
4137
  // file is staged into the temp store and referenced by temp_file_id.
3982
4138
  if (/^https?:\/\//i.test(source)) {
@@ -4037,7 +4193,8 @@ async function runClipperCommand(argv) {
4037
4193
  }
4038
4194
  const j = res.json;
4039
4195
  const dest = j?.folder_path ? `/raws/${j.folder_path}` : "your raws library";
4040
- console.log(`${GREEN}Clipped ${(j?.duration_sec ?? (endSec - startSec)).toFixed?.(2) ?? j?.duration_sec}s ${dest} (clip ${j?.clip_id ?? "?"}).${RESET}`);
4196
+ const kindWord = (j?.media_kind === "audio" || audioOnly) ? "audio" : "clip";
4197
+ console.log(`${GREEN}Clipped ${kindWord} ${(j?.duration_sec ?? (endSec - startSec)).toFixed?.(2) ?? j?.duration_sec}s → ${dest} (clip ${j?.clip_id ?? "?"}).${RESET}`);
4041
4198
  if (j?.view_url)
4042
4199
  console.log(`${DIM}Preview: ${new URL(j.view_url, ctx.host).toString()}${RESET}`);
4043
4200
  console.log(`${DIM}Grab another range from the same source with the same --tracer/--folder.${RESET}`);
@@ -66,6 +66,18 @@ const schema = z.object({
66
66
  OPENROUTER_API_KEY: z.string().optional(),
67
67
  GEMINI_API_KEY: z.string().optional(),
68
68
  PERPLEXITY_API_KEY: z.string().optional(),
69
+ // Platform ElevenLabs key — the fallback used for music / TTS / STT primitives
70
+ // when the caller opts into `use_wallet_credits` (the default) or has no usable
71
+ // BYOK audio key. Usage on this key is billed to the customer's vidfarm wallet.
72
+ ELEVENLABS_API_KEY: z.string().optional(),
73
+ // Base provider costs for ElevenLabs primitives billed to the wallet (the
74
+ // billing lambda applies the standard markup on top). Defaults track published
75
+ // ElevenLabs list pricing: TTS multilingual v2/v3 $0.10 / 1K chars (flash/turbo
76
+ // is $0.05), Music $0.15 / minute, Scribe STT $0.22 / hour ≈ $0.003667 / min.
77
+ // Override per-environment as pricing shifts.
78
+ ELEVENLABS_MUSIC_USD_PER_MIN: z.coerce.number().min(0).default(0.15),
79
+ ELEVENLABS_TTS_USD_PER_1K_CHARS: z.coerce.number().min(0).default(0.10),
80
+ ELEVENLABS_STT_USD_PER_MIN: z.coerce.number().min(0).default(0.003667),
69
81
  GHOSTCUT_KEY: z.string().optional(),
70
82
  GHOSTCUT_SECRET: z.string().optional(),
71
83
  GHOSTCUT_USD_PER_30S: z.coerce.number().min(0).default(0.10),
@@ -44,6 +44,7 @@ export const CLIPS_HELP = `vidfarm raws — build & search a local raws library
44
44
  --yes Skip the >$1 confirmation prompt
45
45
  --frames <n> Keyframes per scene for tagging (1-3, default 2)
46
46
  --no-audio Skip per-scene audio (Gemini only; no transcripts; cheaper)
47
+ --audio-only Save each mined clip as an audio-only .m4a (no video); waveform thumb
47
48
  --concurrency <n> Parallel scenes (default 3; agent default 2)
48
49
  --threshold <0..1> Scene-cut sensitivity (default 0.3; lower = more cuts)
49
50
  --min-scene <sec> Merge scenes shorter than this (default 0.6)
@@ -133,6 +134,7 @@ async function runScan(argv) {
133
134
  "allow-text": { type: "boolean" },
134
135
  "max-clips": { type: "string" },
135
136
  "no-refine": { type: "boolean" },
137
+ "audio-only": { type: "boolean" },
136
138
  cloud: { type: "boolean" },
137
139
  url: { type: "string" },
138
140
  tracer: { type: "string" },
@@ -197,7 +199,8 @@ async function runScan(argv) {
197
199
  aspect,
198
200
  cropFocus,
199
201
  avoidText,
200
- maxClips
202
+ maxClips,
203
+ audioOnly: Boolean(values["audio-only"])
201
204
  });
202
205
  }
203
206
  const videoArg = positionals[0];
@@ -336,6 +339,7 @@ async function runScan(argv) {
336
339
  guidancePrompt: guidance,
337
340
  refineSegmentation: !values["no-refine"],
338
341
  reencodeClips: true,
342
+ audioOnly: Boolean(values["audio-only"]),
339
343
  crop: aspect && aspect !== "original" ? { aspect, focus: cropFocus } : undefined
340
344
  };
341
345
  console.log(`[raws] scanning ${scenes.length} scenes (${provider}/${activeClient.tagModelId}, ${concurrency}x) …`);
@@ -466,7 +470,8 @@ async function runScanCloud(input) {
466
470
  // Send explicitly (incl. false) so --allow-text can override the
467
471
  // server-side avoid-text default.
468
472
  avoid_text: input.avoidText,
469
- ...(input.maxClips ? { max_clips: input.maxClips } : {})
473
+ ...(input.maxClips ? { max_clips: input.maxClips } : {}),
474
+ ...(input.audioOnly ? { audio_only: true } : {})
470
475
  }
471
476
  })
472
477
  });
@@ -1,2 +1,5 @@
1
+ // Providers that count as a qualified AI key for text/image/video generation
2
+ // (everything except the audio-only elevenlabs key).
3
+ export const QUALIFIED_AI_PROVIDER_TYPES = ["openai", "gemini", "openrouter", "perplexity"];
1
4
  export const CUSTOMER_PLAN_TIERS = ["client", "standalone", "agency"];
2
5
  //# sourceMappingURL=domain.js.map
@@ -21,6 +21,10 @@ export function defaultEditorChatModel(provider) {
21
21
  return "qwen/qwen3.6-plus";
22
22
  case "perplexity":
23
23
  return "sonar";
24
+ default:
25
+ // Non-text providers (e.g. audio-only elevenlabs) never reach editor chat;
26
+ // return a safe text default rather than undefined.
27
+ return "gpt-5.4-mini";
24
28
  }
25
29
  }
26
30
  export function buildTemplateEditorChatSystemPrompt(input) {
@@ -198,6 +198,20 @@ export function createPrimitiveJobContext(input) {
198
198
  workerId: input.workerId,
199
199
  ...request
200
200
  });
201
+ },
202
+ async generateMusic(request) {
203
+ return input.providers.generateMusic({
204
+ customerId: input.customer.id,
205
+ jobId: input.job.id,
206
+ workerId: input.workerId,
207
+ ...request
208
+ });
209
+ },
210
+ async listElevenLabsVoices(request) {
211
+ return input.providers.listElevenLabsVoices({
212
+ customerId: input.customer.id,
213
+ ...request
214
+ });
201
215
  }
202
216
  },
203
217
  hyperframes: {
@@ -429,7 +429,11 @@ const ttsPayloadSchema = z.preprocess((raw) => {
429
429
  text: z.string().min(1).max(8000),
430
430
  voice: z.string().min(1).max(120).optional(),
431
431
  instructions: z.string().min(1).max(4000).optional(),
432
- output_format: z.enum(["mp3", "wav"]).default("mp3")
432
+ output_format: z.enum(["mp3", "wav"]).default("mp3"),
433
+ // Default TRUE: great ElevenLabs narration out of the box on the platform key,
434
+ // billed to the wallet. Set false to use your own ElevenLabs key or a BYOK
435
+ // openai/gemini/openrouter key instead. See voice options via /audio/voices.
436
+ use_wallet_credits: z.boolean().default(true)
433
437
  }).superRefine((value, ctx) => {
434
438
  // Reject an explicit provider/voice contradiction at enqueue (400) instead
435
439
  // of queueing a job that can only fail at the provider.
@@ -465,7 +469,36 @@ const sttPayloadSchema = z.preprocess((raw) => {
465
469
  // output and downstream callers estimate word windows.
466
470
  word_timestamps: z.boolean().default(false),
467
471
  language: z.string().min(2).max(16).optional(),
468
- prompt: z.string().min(1).max(2000).optional()
472
+ prompt: z.string().min(1).max(2000).optional(),
473
+ // Default TRUE: platform ElevenLabs Scribe (native diarization + real word
474
+ // timestamps), billed to the wallet. Set false to use your own ElevenLabs
475
+ // key or a BYOK gemini/openai/openrouter key.
476
+ use_wallet_credits: z.boolean().default(true)
477
+ }));
478
+ // Music generation (ElevenLabs). Prompt-based, or a hand-built composition plan.
479
+ const musicPayloadSchema = z.preprocess((raw) => {
480
+ if (!raw || typeof raw !== "object" || Array.isArray(raw))
481
+ return raw;
482
+ const payload = raw;
483
+ return {
484
+ ...payload,
485
+ prompt: payload.prompt ?? payload.text ?? payload.description,
486
+ music_length_ms: payload.music_length_ms ?? payload.length_ms ?? payload.duration_ms
487
+ };
488
+ }, z.object({
489
+ prompt: z.string().min(1).max(2000).optional(),
490
+ // ElevenLabs music caps at 5 minutes per generation.
491
+ music_length_ms: z.number().int().min(3000).max(300000).default(30000),
492
+ // Optional ElevenLabs composition plan for granular section-by-section control.
493
+ composition_plan: z.record(z.string(), z.any()).optional(),
494
+ model: z.string().min(1).optional(),
495
+ output_format: z.string().min(1).optional(),
496
+ // Default TRUE: platform ElevenLabs key, billed to the wallet. Set false to
497
+ // use your own saved ElevenLabs key (no wallet charge).
498
+ use_wallet_credits: z.boolean().default(true)
499
+ }).refine((value) => Boolean(value.prompt) || Boolean(value.composition_plan), {
500
+ message: "Provide either a prompt or a composition_plan.",
501
+ path: ["prompt"]
469
502
  }));
470
503
  // Animated captions from speech: STT with word-level timings, paged into
471
504
  // ready-to-apply cue layers. Either transcribe a source_url or page plain text.
@@ -714,6 +747,7 @@ const PRIMITIVE_BRAINSTORM_PRODUCT_PLACEMENT_ID = "primitive:brainstorm_product_
714
747
  const PRIMITIVE_MEDIA_DEDUPE_ID = "primitive:media_dedupe";
715
748
  const PRIMITIVE_TTS_ID = "primitive:tts";
716
749
  const PRIMITIVE_STT_ID = "primitive:stt";
750
+ const PRIMITIVE_MUSIC_ID = "primitive:music";
717
751
  const PRIMITIVE_CAPTIONS_ID = "primitive:captions";
718
752
  const PRIMITIVE_SPEECH_REGENERATE_ID = "primitive:speech_regenerate";
719
753
  // Gemini inline audio caps out around 20MB; at the 64kbps mono mp3 we extract,
@@ -2018,20 +2052,34 @@ const ttsPrimitive = definePrimitive({
2018
2052
  text: payload.text,
2019
2053
  voice: payload.voice,
2020
2054
  instructions: payload.instructions,
2021
- responseFormat: payload.output_format
2022
- });
2023
- // Gemini TTS answers in wav regardless of the requested format; name the
2024
- // artifact after what actually came back.
2055
+ responseFormat: payload.output_format,
2056
+ useWalletCredits: payload.use_wallet_credits
2057
+ });
2058
+ // Route may have resolved to a different provider (e.g. platform ElevenLabs
2059
+ // when use_wallet_credits is on); report what actually ran.
2060
+ const effectiveProvider = generated.effectiveProvider;
2061
+ const effectiveModel = generated.effectiveModel;
2062
+ if (generated.billing) {
2063
+ await ctx.billing.record({
2064
+ type: "ai_generation",
2065
+ costUsd: generated.billing.costUsd,
2066
+ costCenterSlug: generated.billing.costCenterSlug,
2067
+ metadata: { ...generated.billing.metadata, primitive: "tts", voice: payload.voice ?? null }
2068
+ });
2069
+ }
2070
+ // Gemini/ElevenLabs TTS may answer in wav regardless of the requested
2071
+ // format; name the artifact after what actually came back.
2025
2072
  const extension = generated.contentType.includes("wav") ? "wav" : "mp3";
2026
2073
  const stored = await ctx.storage.putBuffer(`speech.${extension}`, generated.bytes, {
2027
2074
  contentType: generated.contentType,
2028
2075
  kind: "audio",
2029
2076
  metadata: {
2030
2077
  primitive: "tts",
2031
- provider,
2032
- model,
2078
+ provider: effectiveProvider,
2079
+ model: effectiveModel,
2033
2080
  voice: payload.voice ?? null,
2034
- styleInstructions: payload.instructions ?? null
2081
+ styleInstructions: payload.instructions ?? null,
2082
+ wallet_billed: Boolean(generated.billing)
2035
2083
  }
2036
2084
  });
2037
2085
  ctx.logger.progress(1, "Speech generation complete");
@@ -2044,11 +2092,72 @@ const ttsPrimitive = definePrimitive({
2044
2092
  audio: {
2045
2093
  file_url: stored.url,
2046
2094
  content_type: generated.contentType,
2047
- provider,
2048
- model,
2095
+ provider: effectiveProvider,
2096
+ model: effectiveModel,
2049
2097
  voice: payload.voice ?? null,
2050
2098
  style_instructions: payload.instructions ?? null,
2051
- text_length: payload.text.length
2099
+ text_length: payload.text.length,
2100
+ wallet_billed: Boolean(generated.billing)
2101
+ }
2102
+ }
2103
+ };
2104
+ }
2105
+ }
2106
+ });
2107
+ const musicPrimitive = definePrimitive({
2108
+ id: PRIMITIVE_MUSIC_ID,
2109
+ kind: "music",
2110
+ operations: {
2111
+ run: {
2112
+ description: "Generate music (instrumental or with lyrics) from a prompt or a composition plan via ElevenLabs, persisted as a reusable platform audio URL.",
2113
+ inputSchema: musicPayloadSchema,
2114
+ workflow: "run"
2115
+ }
2116
+ },
2117
+ jobs: {
2118
+ async run(ctx, input) {
2119
+ const payload = musicPayloadSchema.parse(input);
2120
+ ctx.logger.progress(0.12, "Composing music");
2121
+ const generated = await ctx.providers.generateMusic({
2122
+ prompt: payload.prompt,
2123
+ musicLengthMs: payload.music_length_ms,
2124
+ compositionPlan: payload.composition_plan,
2125
+ model: payload.model,
2126
+ outputFormat: payload.output_format,
2127
+ useWalletCredits: payload.use_wallet_credits
2128
+ });
2129
+ if (generated.billing) {
2130
+ await ctx.billing.record({
2131
+ type: "ai_generation",
2132
+ costUsd: generated.billing.costUsd,
2133
+ costCenterSlug: generated.billing.costCenterSlug,
2134
+ metadata: { ...generated.billing.metadata, primitive: "music", prompt: payload.prompt ?? null }
2135
+ });
2136
+ }
2137
+ const stored = await ctx.storage.putBuffer("music.mp3", generated.bytes, {
2138
+ contentType: generated.contentType,
2139
+ kind: "audio",
2140
+ metadata: {
2141
+ primitive: "music",
2142
+ provider: "elevenlabs",
2143
+ music_length_ms: generated.musicLengthMs,
2144
+ wallet_billed: Boolean(generated.billing)
2145
+ }
2146
+ });
2147
+ ctx.logger.progress(1, "Music generation complete");
2148
+ return {
2149
+ progress: 1,
2150
+ output: {
2151
+ files: compactUrls([stored.url]),
2152
+ primary_file_url: stored.url,
2153
+ audioUrl: stored.url,
2154
+ audio: {
2155
+ file_url: stored.url,
2156
+ content_type: generated.contentType,
2157
+ provider: "elevenlabs",
2158
+ music_length_ms: generated.musicLengthMs,
2159
+ prompt: payload.prompt ?? null,
2160
+ wallet_billed: Boolean(generated.billing)
2052
2161
  }
2053
2162
  }
2054
2163
  };
@@ -2122,18 +2231,29 @@ const sttPrimitive = definePrimitive({
2122
2231
  const payload = sttPayloadSchema.parse(input);
2123
2232
  const { audioBytes, audioContentType, extractionMetadata } = await loadSpeechAudioForPrimitive(ctx, payload);
2124
2233
  ctx.logger.progress(0.45, "Transcribing speech");
2125
- const provider = payload.provider ?? (payload.word_timestamps ? "openai" : "gemini");
2126
- const model = payload.model ?? defaultSpeechModelFor("stt", provider) ?? "";
2234
+ const requestedProvider = payload.provider ?? (payload.word_timestamps ? "openai" : "gemini");
2235
+ const requestedModel = payload.model ?? defaultSpeechModelFor("stt", requestedProvider) ?? "";
2127
2236
  const transcription = await ctx.providers.transcribeSpeech({
2128
- provider,
2129
- model,
2237
+ provider: requestedProvider,
2238
+ model: requestedModel,
2130
2239
  audio: audioBytes,
2131
2240
  contentType: audioContentType,
2132
2241
  prompt: payload.prompt,
2133
2242
  language: payload.language,
2134
2243
  diarize: payload.diarize,
2135
- wordTimestamps: payload.word_timestamps
2136
- });
2244
+ wordTimestamps: payload.word_timestamps,
2245
+ useWalletCredits: payload.use_wallet_credits
2246
+ });
2247
+ const provider = transcription.effectiveProvider;
2248
+ const model = transcription.effectiveModel;
2249
+ if (transcription.billing) {
2250
+ await ctx.billing.record({
2251
+ type: "ai_generation",
2252
+ costUsd: transcription.billing.costUsd,
2253
+ costCenterSlug: transcription.billing.costCenterSlug,
2254
+ metadata: { ...transcription.billing.metadata, primitive: "stt", source_url: payload.source_url }
2255
+ });
2256
+ }
2137
2257
  ctx.logger.progress(0.85, "Storing transcript artifacts");
2138
2258
  const speakers = [...new Set(transcription.segments.map((segment) => segment.speaker))];
2139
2259
  const srt = buildSrtFromSegments(transcription.segments);
@@ -3114,6 +3234,7 @@ class PrimitiveRegistry {
3114
3234
  [audioNormalizePrimitive.id, audioNormalizePrimitive],
3115
3235
  [ttsPrimitive.id, ttsPrimitive],
3116
3236
  [sttPrimitive.id, sttPrimitive],
3237
+ [musicPrimitive.id, musicPrimitive],
3117
3238
  [captionsPrimitive.id, captionsPrimitive],
3118
3239
  [speechRegeneratePrimitive.id, speechRegeneratePrimitive],
3119
3240
  [hyperframesRenderPrimitive.id, hyperframesRenderPrimitive],
@@ -3521,6 +3642,7 @@ function normalizeProductPlacementPayload(raw) {
3521
3642
  };
3522
3643
  }
3523
3644
  function preferredTextProvider(availableProviders) {
3645
+ // Text-only priority — the audio-only elevenlabs provider is never a candidate.
3524
3646
  const priority = ["openai", "gemini", "openrouter", "perplexity"];
3525
3647
  return priority.find((provider) => availableProviders.includes(provider)) ?? "openai";
3526
3648
  }
@@ -59,8 +59,8 @@ export function renderReskinChat(input = DEFAULT_INPUT, opts = {}) {
59
59
  <select class="rk-chat-mode-select" id="rk-chat-mode">
60
60
  <option value="/chat" selected>Chat</option>
61
61
  <option value="/tools/image">Create Image</option>
62
- <option value="/tools/clipper">Clip a Video</option>
63
62
  <option value="/editor/original/fork/new">Create Video</option>
63
+ <option value="/tools/clipper">Clip Media</option>
64
64
  </select>
65
65
  <span class="rk-chat-mode-chev" aria-hidden="true">&#9662;</span>
66
66
  </label>