@mevdragon/vidfarm-devcli 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/music/SKILL.md +416 -0
- package/.agents/skills/music/references/api_reference.md +519 -0
- package/.agents/skills/music/references/installation.md +65 -0
- package/.agents/skills/text-to-speech/SKILL.md +226 -0
- package/.agents/skills/text-to-speech/references/installation.md +90 -0
- package/.agents/skills/text-to-speech/references/streaming.md +307 -0
- package/.agents/skills/text-to-speech/references/voice-settings.md +115 -0
- package/.agents/skills/vidfarm-media/SKILL.md +25 -11
- package/.agents/skills/vidfarm-media/references/tts.md +41 -3
- package/SKILL.director.md +31 -13
- package/SKILL.platform.md +2 -2
- package/dist/src/app.js +129 -41
- package/dist/src/cli.js +162 -5
- package/dist/src/config.js +12 -0
- package/dist/src/devcli/clips.js +7 -2
- package/dist/src/domain.js +3 -0
- package/dist/src/editor-chat.js +4 -0
- package/dist/src/primitive-context.js +14 -0
- package/dist/src/primitive-registry.js +140 -18
- package/dist/src/reskin/chat-page.js +1 -1
- package/dist/src/reskin/inpaint-clipper-page.js +446 -205
- package/dist/src/reskin/library-page.js +7 -1
- package/dist/src/reskin/portfolio-page.js +9 -9
- package/dist/src/reskin/settings-page.js +4 -4
- package/dist/src/reskin/theme.js +1 -0
- package/dist/src/services/billing.js +5 -0
- package/dist/src/services/clip-curation/ffmpeg.js +48 -0
- package/dist/src/services/clip-curation/hunt.js +2 -0
- package/dist/src/services/clip-curation/index.js +1 -1
- package/dist/src/services/clip-curation/scan.js +29 -16
- package/dist/src/services/elevenlabs.js +222 -0
- package/dist/src/services/providers.js +216 -2
- package/dist/src/services/swipe-customize.js +5 -2
- package/package.json +1 -1
|
@@ -350,6 +350,10 @@ const CLIPS_DIALOG = `
|
|
|
350
350
|
<option value="openrouter">OpenRouter</option>
|
|
351
351
|
</select>
|
|
352
352
|
</div>
|
|
353
|
+
<label class="rk-clips-toggle" for="rkScanAudioOnly">
|
|
354
|
+
<input type="checkbox" id="rkScanAudioOnly" name="audio_only" />
|
|
355
|
+
<span><b>Clip audio only</b> — save each mined moment as an audio-only <code>.m4a</code> (no video), e.g. lifting sounds from a copyright-free audio video.</span>
|
|
356
|
+
</label>
|
|
353
357
|
</div>
|
|
354
358
|
<p class="rk-hint rk-clips-dialog-note" id="rkScanNote">We'll pull the whole video into your raws library.</p>
|
|
355
359
|
<div class="rk-clips-dialog-actions">
|
|
@@ -1182,6 +1186,7 @@ const CLIPS_SCRIPT = `
|
|
|
1182
1186
|
var scanNote=root.getElementById('rkScanNote');
|
|
1183
1187
|
var scanSubmit=root.getElementById('rkClipsScanSubmit');
|
|
1184
1188
|
var scanClip=root.getElementById('rkScanClip');
|
|
1189
|
+
var scanAudioOnly=root.getElementById('rkScanAudioOnly');
|
|
1185
1190
|
var scanAdvanced=root.getElementById('rkClipsAdvanced');
|
|
1186
1191
|
var IMPORT_NOTE="We'll open the Clipper so you can preview the video and save an exact subrange (or the whole thing) to your raws.";
|
|
1187
1192
|
var CLIP_NOTE='AI tagging runs on your own key (BYOK) — vidfarm only bills the clip-scan compute.';
|
|
@@ -1239,6 +1244,7 @@ const CLIPS_SCRIPT = `
|
|
|
1239
1244
|
var clip=!!(scanClip&&scanClip.checked);
|
|
1240
1245
|
var prompt=clip?(scanPrompt&&scanPrompt.value||'').trim():'';
|
|
1241
1246
|
var provider=clip?(scanProvider&&scanProvider.value||'').trim():'';
|
|
1247
|
+
var audioOnly=clip&&!!(scanAudioOnly&&scanAudioOnly.checked);
|
|
1242
1248
|
if(!url){ setNote('Enter a video URL.','error'); return; }
|
|
1243
1249
|
// Whole-video (non-AI) path now opens the Clipper so the user can pick an
|
|
1244
1250
|
// exact subrange (or save the whole thing) with a live video preview.
|
|
@@ -1253,7 +1259,7 @@ const CLIPS_SCRIPT = `
|
|
|
1253
1259
|
if(scanSubmit){ scanSubmit.disabled=true; scanSubmit.classList.add('is-loading'); }
|
|
1254
1260
|
|
|
1255
1261
|
var payload={source_url:url};
|
|
1256
|
-
if(clip){ payload.prompt=prompt; if(provider)payload.provider=provider; } else { payload.import_only=true; }
|
|
1262
|
+
if(clip){ payload.prompt=prompt; if(provider)payload.provider=provider; if(audioOnly)payload.audio_only=true; } else { payload.import_only=true; }
|
|
1257
1263
|
if(tracer)payload.tracer=tracer;
|
|
1258
1264
|
|
|
1259
1265
|
// Fire the request now; hold it and only wire the result AFTER the modal
|
|
@@ -38,19 +38,19 @@ const SAMPLE = {
|
|
|
38
38
|
export const PORTFOLIOS = {
|
|
39
39
|
"sunny-studio": {
|
|
40
40
|
slug: "sunny-studio",
|
|
41
|
-
title: "
|
|
42
|
-
tagline: "A
|
|
43
|
-
about: "
|
|
41
|
+
title: "Andrea Villanueva — Short-form that sells",
|
|
42
|
+
tagline: "A freelance short-form editor turning brands into scroll-stopping video. 40+ ads shipped, 12M+ organic views.",
|
|
43
|
+
about: "I script, edit, and caption high-retention vertical video for DTC and SaaS brands — solo, hands-on, and fast. Every cut below started as a rough brief and shipped in under a week. Hover to preview, tap any piece to watch it full-screen.",
|
|
44
44
|
author: {
|
|
45
|
-
name: "
|
|
46
|
-
role: "
|
|
45
|
+
name: "Andrea Villanueva",
|
|
46
|
+
role: "Freelance short-form editor · Private hire",
|
|
47
47
|
avatarUrl: PROFILE_PIC,
|
|
48
|
-
bio: "Ex-agency editor now
|
|
49
|
-
location: "
|
|
48
|
+
bio: "Ex-agency editor now freelancing solo. I obsess over the first 3 seconds, punchy pacing, and captions that carry the story even on mute.",
|
|
49
|
+
location: "Philippines · working worldwide",
|
|
50
50
|
availableForHire: true,
|
|
51
51
|
stats: [
|
|
52
52
|
{ label: "Videos shipped", value: "140+" },
|
|
53
|
-
{ label: "Starting from", value: "$
|
|
53
|
+
{ label: "Starting from", value: "$3/hr" },
|
|
54
54
|
{ label: "Client rating", value: "4.9/5" }
|
|
55
55
|
],
|
|
56
56
|
certifications: [
|
|
@@ -267,7 +267,7 @@ export function renderReskinPortfolio(slug, input = {}) {
|
|
|
267
267
|
if (!data)
|
|
268
268
|
return null;
|
|
269
269
|
const { author, videos } = data;
|
|
270
|
-
const contactLabel = author.contactLabel || "
|
|
270
|
+
const contactLabel = author.contactLabel || "Get 1st Video Free";
|
|
271
271
|
const contactHref = author.contactUrl || CTA_URL;
|
|
272
272
|
const statPills = (author.stats && author.stats.length)
|
|
273
273
|
? `<div class="pf-stats">${author.stats
|
|
@@ -278,7 +278,7 @@ export function renderReskinSettings(input) {
|
|
|
278
278
|
const keyRows = input.providerKeys.length
|
|
279
279
|
? input.providerKeys.map((k, i) => {
|
|
280
280
|
const id = `rk-pk-${i}`;
|
|
281
|
-
const tint = k.provider === "openai" ? "rk-tile-green" : k.provider === "gemini" ? "rk-tile-sky" : "rk-tile-violet";
|
|
281
|
+
const tint = k.provider === "openai" ? "rk-tile-green" : k.provider === "gemini" ? "rk-tile-sky" : k.provider === "elevenlabs" ? "rk-tile-amber" : "rk-tile-violet";
|
|
282
282
|
const status = k.status || "active";
|
|
283
283
|
const lastUsed = k.last_used_at ? `used ${rkEscape(fmtDate(k.last_used_at))}` : "never used";
|
|
284
284
|
return `<div class="rk-item">
|
|
@@ -318,8 +318,8 @@ export function renderReskinSettings(input) {
|
|
|
318
318
|
<div class="rk-card">
|
|
319
319
|
<div class="rk-card-head">
|
|
320
320
|
<span class="rk-eyebrow">AI keys</span>
|
|
321
|
-
<h2>Bring your own
|
|
322
|
-
<p class="rk-muted">Stored per workspace.
|
|
321
|
+
<h2>Bring your own AI Keys</h2>
|
|
322
|
+
<p class="rk-muted">Stored per workspace. OpenAI, Gemini, or OpenRouter power clip tagging, recaption, image/video generation, and Swipe — BYOK, never billed by vidfarm. <b>ElevenLabs is audio-only</b> (music, voiceover, transcription) and does <b>not</b> count as a qualified AI key — you still need one of the others for the core features. With no ElevenLabs key, music & speech run on vidfarm's key and bill your wallet (<code>use_wallet_credits</code>).</p>
|
|
323
323
|
</div>
|
|
324
324
|
<div class="rk-list rk-set-keys">${keyRows}</div>
|
|
325
325
|
<form class="rk-set-addkey" method="post" action="/settings/provider-keys">
|
|
@@ -328,7 +328,7 @@ export function renderReskinSettings(input) {
|
|
|
328
328
|
<div class="rk-field">
|
|
329
329
|
<label class="rk-label" for="rk-prov">Provider</label>
|
|
330
330
|
<select class="rk-select" id="rk-prov" name="provider">
|
|
331
|
-
<option value="openai">OpenAI</option><option value="gemini">Gemini</option><option value="openrouter">OpenRouter</option><option value="
|
|
331
|
+
<option value="openai">OpenAI</option><option value="gemini">Gemini</option><option value="openrouter">OpenRouter</option><option value="elevenlabs">ElevenLabs (audio only)</option>
|
|
332
332
|
</select>
|
|
333
333
|
</div>
|
|
334
334
|
<div class="rk-field">
|
package/dist/src/reskin/theme.js
CHANGED
|
@@ -635,6 +635,7 @@ export const RESKIN_CSS = `
|
|
|
635
635
|
.rk-tile-sky{background:var(--rk-sky-tint);color:var(--rk-sky)}
|
|
636
636
|
.rk-tile-violet{background:var(--rk-violet-tint);color:var(--rk-violet)}
|
|
637
637
|
.rk-tile-green{background:var(--rk-green-tint);color:#16a34a}
|
|
638
|
+
.rk-tile-amber{background:var(--rk-gold-tint2);color:var(--rk-gold-700)}
|
|
638
639
|
.rk-tile-coral{background:var(--rk-coral-tint);color:var(--rk-coral)}
|
|
639
640
|
.rk-tile-ink{background:var(--rk-n-100);color:var(--rk-ink)}
|
|
640
641
|
|
|
@@ -168,6 +168,11 @@ function resolveCostCenterSlug(input) {
|
|
|
168
168
|
// primitive (BYOK image gen + chroma key): platform compute/convenience.
|
|
169
169
|
|| explicitSlug === "greenscreen_chroma_key"
|
|
170
170
|
|| explicitSlug === "media_overlay"
|
|
171
|
+
// Platform ElevenLabs audio (music / TTS / STT) run on the wallet when the
|
|
172
|
+
// caller opts into use_wallet_credits (the audio-primitive default).
|
|
173
|
+
|| explicitSlug === "elevenlabs_music"
|
|
174
|
+
|| explicitSlug === "elevenlabs_tts"
|
|
175
|
+
|| explicitSlug === "elevenlabs_stt"
|
|
171
176
|
|| explicitSlug === "primitive_media_lambda"
|
|
172
177
|
|| explicitSlug === "job_runner_lambda"
|
|
173
178
|
|| explicitSlug === "step_functions_standard"
|
|
@@ -256,6 +256,54 @@ export async function extractClip(input) {
|
|
|
256
256
|
}
|
|
257
257
|
return ok;
|
|
258
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Trim EXACTLY [start, start+dur] of the source's AUDIO into a standalone
|
|
261
|
+
* `.m4a` (AAC) file — the audio-only twin of {@link extractClip}. The video
|
|
262
|
+
* track is dropped (`-vn`) so the result is a small, editor-placeable audio raw
|
|
263
|
+
* (e.g. clipping the soundtrack out of a copyright-free YouTube video). Returns
|
|
264
|
+
* false if the source has no audio track (ffmpeg exits non-zero with no output).
|
|
265
|
+
*/
|
|
266
|
+
export async function extractAudioClip(input) {
|
|
267
|
+
mkdirSync(path.dirname(input.outPath), { recursive: true });
|
|
268
|
+
const ffmpeg = await resolveFfmpeg();
|
|
269
|
+
const { start_time_sec: start, duration_sec: dur } = input.scene;
|
|
270
|
+
const { code, stderr } = await run(ffmpeg, [
|
|
271
|
+
"-y",
|
|
272
|
+
"-ss", start.toFixed(3),
|
|
273
|
+
"-i", input.videoPath,
|
|
274
|
+
"-t", dur.toFixed(3),
|
|
275
|
+
"-vn",
|
|
276
|
+
"-c:a", "aac", "-b:a", "192k",
|
|
277
|
+
"-movflags", "+faststart",
|
|
278
|
+
input.outPath
|
|
279
|
+
]);
|
|
280
|
+
const ok = code === 0 && existsSync(input.outPath);
|
|
281
|
+
if (!ok)
|
|
282
|
+
input.onError?.(`ffmpeg exit ${code}: ${stderr.slice(-500).replace(/\s+/g, " ").trim()}`);
|
|
283
|
+
return ok;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* A static waveform PNG for an audio clip — its "thumbnail" so an audio-only raw
|
|
287
|
+
* still shows something in the raws gallery (there is no video frame to grab).
|
|
288
|
+
* Best-effort; returns false if the source has no audio to plot.
|
|
289
|
+
*/
|
|
290
|
+
export async function extractWaveformThumbnail(input) {
|
|
291
|
+
mkdirSync(path.dirname(input.outPath), { recursive: true });
|
|
292
|
+
const ffmpeg = await resolveFfmpeg();
|
|
293
|
+
const { start_time_sec: start, duration_sec: dur } = input.scene;
|
|
294
|
+
const w = input.width ?? 320;
|
|
295
|
+
const h = input.height ?? 200;
|
|
296
|
+
const { code } = await run(ffmpeg, [
|
|
297
|
+
"-y",
|
|
298
|
+
"-ss", start.toFixed(3),
|
|
299
|
+
"-i", input.videoPath,
|
|
300
|
+
"-t", dur.toFixed(3),
|
|
301
|
+
"-filter_complex", `showwavespic=s=${w}x${h}:colors=#e2a63a`,
|
|
302
|
+
"-frames:v", "1",
|
|
303
|
+
input.outPath
|
|
304
|
+
]);
|
|
305
|
+
return code === 0 && existsSync(input.outPath);
|
|
306
|
+
}
|
|
259
307
|
/** Single representative thumbnail for the scene. */
|
|
260
308
|
export async function extractThumbnail(input) {
|
|
261
309
|
mkdirSync(path.dirname(input.outPath), { recursive: true });
|
|
@@ -395,6 +395,8 @@ export function normalizeHuntSpec(raw) {
|
|
|
395
395
|
spec.source_url = rec.source_url.trim();
|
|
396
396
|
if (typeof rec.folder_path === "string" && rec.folder_path.trim())
|
|
397
397
|
spec.folder_path = rec.folder_path.trim().slice(0, 500);
|
|
398
|
+
if (rec.audio_only === true || rec.audioOnly === true)
|
|
399
|
+
spec.audio_only = true;
|
|
398
400
|
return spec;
|
|
399
401
|
}
|
|
400
402
|
// ── misc ────────────────────────────────────────────────────────────────────
|
|
@@ -7,7 +7,7 @@ export { BUILTIN_PRESETS, findBuiltinPreset } from "./presets.js";
|
|
|
7
7
|
export { ClipModelClient, ClipGeminiClient, ClipEmbeddingClient, EMBEDDING_DIM, EMBEDDING_MODEL, EMBEDDING_MODELS, TAG_MODEL_BY_TIER, buildClipEmbeddingText, buildQueryPrompt, buildTaggingPrompt, normalize } from "./gemini.js";
|
|
8
8
|
export { LocalAgentClipClient, detectLocalAgent, extractJsonObject } from "./local-agent.js";
|
|
9
9
|
export { AVOID_TEXT_GUIDANCE, DEFAULT_CLIP_ASPECT, DEFAULT_CLIP_DURATION_BAND, DEFAULT_CLIPS_PER_10_MIN, applyHuntSpecDefaults, buildEffectiveGuidance, capScenesEvenlyToCount, cropFilterFor, effectiveHuntDurationSec, fitScenesToDurationBand, normalizeAspect, normalizeCropFocus, normalizeHuntSpec, normalizeWindows, parseClipHuntPrompt, parseTimecode, parseTimeRanges, resolveDurationBand, resolveTargetClipCount } from "./hunt.js";
|
|
10
|
-
export { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, hasFfmpeg, probeVideo, resolveFfmpeg, resolveFfprobe } from "./ffmpeg.js";
|
|
10
|
+
export { detectScenes, extractAudioClip, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, extractWaveformThumbnail, hasFfmpeg, probeVideo, resolveFfmpeg, resolveFfprobe } from "./ffmpeg.js";
|
|
11
11
|
export { cosineSimilarity, matchesCriteria, rankHits, scoreClip, searchClips, structuredMatchRatio } from "./query.js";
|
|
12
12
|
export { estimateScanCostFromDuration, estimateScanCostFromScenes, formatCostEstimate } from "./cost.js";
|
|
13
13
|
export { processSceneToClip, scanVideo } from "./scan.js";
|
|
@@ -8,7 +8,7 @@ import path from "node:path";
|
|
|
8
8
|
import { createIdV7 } from "../../lib/ids.js";
|
|
9
9
|
import { ACTIVE_TAXONOMY_VERSION } from "./taxonomy.js";
|
|
10
10
|
import { buildClipEmbeddingText } from "./gemini.js";
|
|
11
|
-
import { detectScenes, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, probeVideo } from "./ffmpeg.js";
|
|
11
|
+
import { detectScenes, extractAudioClip, extractClip, extractKeyframes, extractSceneAudio, extractThumbnail, extractWaveformThumbnail, probeVideo } from "./ffmpeg.js";
|
|
12
12
|
import { capScenesEvenlyToCount, cropFilterFor, fitScenesToDurationBand, normalizeWindows } from "./hunt.js";
|
|
13
13
|
import { refineScenesWithGuidance } from "./refine.js";
|
|
14
14
|
/** Turn one scene into a Clip. Throws only on unrecoverable extraction failure. */
|
|
@@ -46,22 +46,34 @@ export async function processSceneToClip(input) {
|
|
|
46
46
|
scene,
|
|
47
47
|
guidancePrompt: ctx.guidancePrompt
|
|
48
48
|
});
|
|
49
|
-
// 4. Cut the clip + thumbnail
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
// 4. Cut the clip + thumbnail. Audio-only hunts emit an .m4a (no video) with a
|
|
50
|
+
// waveform PNG thumbnail; ordinary hunts emit an .mp4 cropped to the target
|
|
51
|
+
// aspect when requested.
|
|
52
|
+
const audioOnly = ctx.audioOnly === true;
|
|
53
|
+
const clipLocal = path.join(ctx.clipsDir, `${clipId}.${audioOnly ? "m4a" : "mp4"}`);
|
|
54
|
+
const thumbLocal = path.join(ctx.thumbsDir, `${clipId}.${audioOnly ? "png" : "jpg"}`);
|
|
52
55
|
let extractDetail = "";
|
|
53
|
-
|
|
54
|
-
videoPath,
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
56
|
+
if (audioOnly) {
|
|
57
|
+
const audioOk = await extractAudioClip({ videoPath, scene, outPath: clipLocal, onError: (detail) => (extractDetail = detail) });
|
|
58
|
+
if (!audioOk) {
|
|
59
|
+
throw new Error(`ffmpeg failed to extract audio for scene @${scene.start_time_sec}s${extractDetail ? ` — ${extractDetail}` : ""}`);
|
|
60
|
+
}
|
|
61
|
+
await extractWaveformThumbnail({ videoPath, scene, outPath: thumbLocal }).catch(() => false);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
const clipOk = await extractClip({
|
|
65
|
+
videoPath,
|
|
66
|
+
scene,
|
|
67
|
+
outPath: clipLocal,
|
|
68
|
+
reencode: ctx.reencodeClips !== false,
|
|
69
|
+
cropFilter,
|
|
70
|
+
onError: (detail) => (extractDetail = detail)
|
|
71
|
+
});
|
|
72
|
+
if (!clipOk) {
|
|
73
|
+
throw new Error(`ffmpeg failed to extract clip for scene @${scene.start_time_sec}s${extractDetail ? ` — ${extractDetail}` : ""}`);
|
|
74
|
+
}
|
|
75
|
+
await extractThumbnail({ videoPath, scene, outPath: thumbLocal, cropFilter });
|
|
63
76
|
}
|
|
64
|
-
await extractThumbnail({ videoPath, scene, outPath: thumbLocal, cropFilter });
|
|
65
77
|
// 5. Embed the document text for semantic search (skipped when the client can't embed).
|
|
66
78
|
let embedding;
|
|
67
79
|
if (ctx.client.canEmbed) {
|
|
@@ -84,7 +96,8 @@ export async function processSceneToClip(input) {
|
|
|
84
96
|
tier: ctx.client.tier,
|
|
85
97
|
provider: ctx.client.provider,
|
|
86
98
|
embedding_model: ctx.client.embeddingModelId ?? undefined,
|
|
87
|
-
aspect: cropFilter && ctx.crop ? ctx.crop.aspect : undefined,
|
|
99
|
+
aspect: !audioOnly && cropFilter && ctx.crop ? ctx.crop.aspect : undefined,
|
|
100
|
+
media_kind: audioOnly ? "audio" : undefined,
|
|
88
101
|
owner_id: input.ownerId,
|
|
89
102
|
created_at: new Date().toISOString()
|
|
90
103
|
};
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
// ElevenLabs audio provider (music, TTS, STT) — shared by the ProviderService
|
|
2
|
+
// wallet/BYOK router and the primitive handlers. ElevenLabs is a SPECIAL
|
|
3
|
+
// audio-only provider: it never counts as a qualified text/image/video key.
|
|
4
|
+
// Two ways a call reaches here:
|
|
5
|
+
// 1. Platform key (config.ELEVENLABS_API_KEY) + wallet billing — the default
|
|
6
|
+
// "use_wallet_credits" path, so good speech/music work with zero setup.
|
|
7
|
+
// 2. The customer's own saved ElevenLabs key (use_wallet_credits: false) — no
|
|
8
|
+
// wallet charge; identity/voices are scoped to THEIR ElevenLabs account.
|
|
9
|
+
//
|
|
10
|
+
// Keep the raw request/response shapes here — one implementation for cloud and
|
|
11
|
+
// devcli, mirroring services/speech.ts for the openai/gemini/openrouter family.
|
|
12
|
+
import { ProviderAuthError, rateLimitOrQuotaError } from "./provider-errors.js";
|
|
13
|
+
import { pcm16MonoToWav } from "./speech.js";
|
|
14
|
+
const ELEVENLABS_BASE_URL = "https://api.elevenlabs.io/v1";
|
|
15
|
+
// Sensible defaults so a bare `music`/`tts` call just works.
|
|
16
|
+
export const DEFAULT_ELEVENLABS_TTS_MODEL = "eleven_multilingual_v2";
|
|
17
|
+
export const DEFAULT_ELEVENLABS_STT_MODEL = "scribe_v1";
|
|
18
|
+
export const DEFAULT_ELEVENLABS_MUSIC_MODEL = "music_v2";
|
|
19
|
+
// George — a warm, versatile narrative voice. Callers can override with any
|
|
20
|
+
// voice_id from `elevenLabsListVoices` (platform or their own account).
|
|
21
|
+
export const DEFAULT_ELEVENLABS_VOICE_ID = "JBFqnCBsd6RMkjVDRZzb";
|
|
22
|
+
export const ELEVENLABS_VOICE_LIBRARY_URL = "https://elevenlabs.io/app/voice-library";
|
|
23
|
+
// A tiny friendly-name → voice_id map so agents/users can say "sarah" instead of
|
|
24
|
+
// an opaque id. Unknown values are passed through untouched (treated as a
|
|
25
|
+
// voice_id), and the real catalog is always `elevenLabsListVoices`.
|
|
26
|
+
const ELEVENLABS_VOICE_PRESETS = {
|
|
27
|
+
george: "JBFqnCBsd6RMkjVDRZzb",
|
|
28
|
+
sarah: "EXAVITQu4vr4xnSDxMaL",
|
|
29
|
+
daniel: "onwK4e9ZLuTAKqWW03F9",
|
|
30
|
+
charlotte: "XB0fDUnXU5powFXDhCwa",
|
|
31
|
+
rachel: "21m00Tcm4TlvDq8ikWAM"
|
|
32
|
+
};
|
|
33
|
+
/** Resolve a caller-supplied voice (friendly preset name or raw id) to a voice_id. */
|
|
34
|
+
export function resolveElevenLabsVoiceId(voice) {
|
|
35
|
+
const raw = voice?.trim();
|
|
36
|
+
if (!raw) {
|
|
37
|
+
return DEFAULT_ELEVENLABS_VOICE_ID;
|
|
38
|
+
}
|
|
39
|
+
return ELEVENLABS_VOICE_PRESETS[raw.toLowerCase()] ?? raw;
|
|
40
|
+
}
|
|
41
|
+
function authHeaders(apiKey) {
|
|
42
|
+
return { "xi-api-key": apiKey };
|
|
43
|
+
}
|
|
44
|
+
async function throwForStatus(provider, operation, response) {
|
|
45
|
+
if (response.status === 401 || response.status === 403) {
|
|
46
|
+
throw new ProviderAuthError("elevenlabs authentication failed");
|
|
47
|
+
}
|
|
48
|
+
if (response.status === 429) {
|
|
49
|
+
throw await rateLimitOrQuotaError(provider, response);
|
|
50
|
+
}
|
|
51
|
+
const details = await response.text().catch(() => "");
|
|
52
|
+
throw new Error(`elevenlabs ${operation} returned ${response.status}${details ? `: ${details.slice(0, 400)}` : ""}`);
|
|
53
|
+
}
|
|
54
|
+
/** Cheap credential probe — the ElevenLabs equivalent of a chat-completion validate. */
|
|
55
|
+
export async function validateElevenLabsKey(apiKey) {
|
|
56
|
+
const response = await fetch(`${ELEVENLABS_BASE_URL}/user`, { headers: authHeaders(apiKey) });
|
|
57
|
+
if (response.ok) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
await throwForStatus("elevenlabs", "key validation", response);
|
|
61
|
+
}
|
|
62
|
+
/** List voices available to whichever account owns `apiKey` (platform or the user's). */
|
|
63
|
+
export async function elevenLabsListVoices(apiKey) {
|
|
64
|
+
const response = await fetch(`${ELEVENLABS_BASE_URL}/voices`, { headers: authHeaders(apiKey) });
|
|
65
|
+
if (!response.ok) {
|
|
66
|
+
await throwForStatus("elevenlabs", "list voices", response);
|
|
67
|
+
}
|
|
68
|
+
const data = (await response.json());
|
|
69
|
+
return (data.voices ?? []).map((voice) => ({
|
|
70
|
+
voice_id: String(voice.voice_id ?? ""),
|
|
71
|
+
name: String(voice.name ?? "Unnamed"),
|
|
72
|
+
category: typeof voice.category === "string" ? voice.category : null,
|
|
73
|
+
description: typeof voice.description === "string" && voice.description.trim() ? voice.description.trim() : null,
|
|
74
|
+
labels: voice.labels && typeof voice.labels === "object"
|
|
75
|
+
? Object.fromEntries(Object.entries(voice.labels).map(([key, value]) => [key, String(value)]))
|
|
76
|
+
: {},
|
|
77
|
+
preview_url: typeof voice.preview_url === "string" ? voice.preview_url : null
|
|
78
|
+
})).filter((voice) => voice.voice_id);
|
|
79
|
+
}
|
|
80
|
+
/** ElevenLabs text-to-speech. `voice` is a voice_id (or a friendly preset name). */
|
|
81
|
+
export async function elevenLabsTextToSpeech(input) {
|
|
82
|
+
const voiceId = resolveElevenLabsVoiceId(input.voice);
|
|
83
|
+
const model = input.model?.trim() || DEFAULT_ELEVENLABS_TTS_MODEL;
|
|
84
|
+
const wantWav = input.responseFormat === "wav";
|
|
85
|
+
// ElevenLabs has no direct "wav" output — request raw PCM and wrap it in a WAV
|
|
86
|
+
// header, mirroring the gemini/openrouter PCM bridge in services/speech.ts.
|
|
87
|
+
const outputFormat = wantWav ? "pcm_44100" : "mp3_44100_128";
|
|
88
|
+
const voiceSettings = {};
|
|
89
|
+
if (input.stability != null)
|
|
90
|
+
voiceSettings.stability = input.stability;
|
|
91
|
+
if (input.similarityBoost != null)
|
|
92
|
+
voiceSettings.similarity_boost = input.similarityBoost;
|
|
93
|
+
if (input.style != null)
|
|
94
|
+
voiceSettings.style = input.style;
|
|
95
|
+
if (input.speed != null)
|
|
96
|
+
voiceSettings.speed = input.speed;
|
|
97
|
+
const response = await fetch(`${ELEVENLABS_BASE_URL}/text-to-speech/${encodeURIComponent(voiceId)}?output_format=${outputFormat}`, {
|
|
98
|
+
method: "POST",
|
|
99
|
+
headers: { ...authHeaders(input.apiKey), "Content-Type": "application/json" },
|
|
100
|
+
body: JSON.stringify({
|
|
101
|
+
text: input.text,
|
|
102
|
+
model_id: model,
|
|
103
|
+
...(input.languageCode ? { language_code: input.languageCode } : {}),
|
|
104
|
+
...(Object.keys(voiceSettings).length ? { voice_settings: voiceSettings } : {})
|
|
105
|
+
})
|
|
106
|
+
});
|
|
107
|
+
if (!response.ok) {
|
|
108
|
+
await throwForStatus("elevenlabs", "text-to-speech", response);
|
|
109
|
+
}
|
|
110
|
+
const raw = Buffer.from(await response.arrayBuffer());
|
|
111
|
+
const bytes = wantWav ? pcm16MonoToWav(raw, 44100) : raw;
|
|
112
|
+
return {
|
|
113
|
+
bytes,
|
|
114
|
+
contentType: wantWav ? "audio/wav" : "audio/mpeg",
|
|
115
|
+
characterCount: input.text.length,
|
|
116
|
+
usage: { inputTokens: input.text.length, outputTokens: 0, costUsd: 0 }
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/** ElevenLabs music generation from a prompt or a hand-built composition plan. */
|
|
120
|
+
export async function elevenLabsMusicCompose(input) {
|
|
121
|
+
const model = input.model?.trim() || DEFAULT_ELEVENLABS_MUSIC_MODEL;
|
|
122
|
+
const outputFormat = input.outputFormat?.trim() || "mp3_44100_128";
|
|
123
|
+
const body = { model_id: model };
|
|
124
|
+
if (input.compositionPlan) {
|
|
125
|
+
body.composition_plan = input.compositionPlan;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
body.prompt = input.prompt;
|
|
129
|
+
body.music_length_ms = input.musicLengthMs ?? 30000;
|
|
130
|
+
}
|
|
131
|
+
const response = await fetch(`${ELEVENLABS_BASE_URL}/music?output_format=${encodeURIComponent(outputFormat)}`, {
|
|
132
|
+
method: "POST",
|
|
133
|
+
headers: { ...authHeaders(input.apiKey), "Content-Type": "application/json" },
|
|
134
|
+
body: JSON.stringify(body)
|
|
135
|
+
});
|
|
136
|
+
if (!response.ok) {
|
|
137
|
+
await throwForStatus("elevenlabs", "music generation", response);
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
bytes: Buffer.from(await response.arrayBuffer()),
|
|
141
|
+
contentType: "audio/mpeg",
|
|
142
|
+
musicLengthMs: input.musicLengthMs ?? 30000
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
/** ElevenLabs speech-to-text (Scribe) — native diarization + word timestamps. */
|
|
146
|
+
export async function elevenLabsSpeechToText(input) {
|
|
147
|
+
const model = input.model?.trim() || DEFAULT_ELEVENLABS_STT_MODEL;
|
|
148
|
+
const form = new FormData();
|
|
149
|
+
form.set("file", new Blob([Buffer.from(input.audio)], { type: input.contentType ?? "audio/mpeg" }), "audio");
|
|
150
|
+
form.set("model_id", model);
|
|
151
|
+
form.set("diarize", String(input.diarize !== false));
|
|
152
|
+
form.set("timestamps_granularity", "word");
|
|
153
|
+
if (input.languageCode) {
|
|
154
|
+
form.set("language_code", input.languageCode);
|
|
155
|
+
}
|
|
156
|
+
if (input.numSpeakers != null) {
|
|
157
|
+
form.set("num_speakers", String(input.numSpeakers));
|
|
158
|
+
}
|
|
159
|
+
const response = await fetch(`${ELEVENLABS_BASE_URL}/speech-to-text`, {
|
|
160
|
+
method: "POST",
|
|
161
|
+
headers: authHeaders(input.apiKey),
|
|
162
|
+
body: form
|
|
163
|
+
});
|
|
164
|
+
if (!response.ok) {
|
|
165
|
+
await throwForStatus("elevenlabs", "speech-to-text", response);
|
|
166
|
+
}
|
|
167
|
+
const data = (await response.json());
|
|
168
|
+
const words = Array.isArray(data.words) ? data.words : [];
|
|
169
|
+
const segments = groupElevenLabsWordsIntoSegments(words);
|
|
170
|
+
const speakers = new Set(segments.map((segment) => segment.speaker));
|
|
171
|
+
const audioDurationSec = words.reduce((max, word) => Math.max(max, Number(word.end ?? 0)), 0) || null;
|
|
172
|
+
return {
|
|
173
|
+
text: String(data.text ?? "").trim(),
|
|
174
|
+
language: typeof data.language_code === "string" && data.language_code.trim() ? data.language_code.trim() : null,
|
|
175
|
+
segments,
|
|
176
|
+
diarization: speakers.size > 1 ? "diarized" : "none",
|
|
177
|
+
usage: { inputTokens: 0, outputTokens: 0, costUsd: 0 },
|
|
178
|
+
audioDurationSec
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
// Fold ElevenLabs' flat word stream (each tagged with a speaker_id) into
|
|
182
|
+
// speaker-turn segments carrying word-level timings — the shape STT callers and
|
|
183
|
+
// the caption builder already consume.
|
|
184
|
+
function groupElevenLabsWordsIntoSegments(words) {
|
|
185
|
+
const segments = [];
|
|
186
|
+
let current = null;
|
|
187
|
+
let currentWords = [];
|
|
188
|
+
const flush = () => {
|
|
189
|
+
if (current) {
|
|
190
|
+
current.text = current.text.trim();
|
|
191
|
+
if (currentWords.length) {
|
|
192
|
+
current.words = currentWords;
|
|
193
|
+
}
|
|
194
|
+
if (current.text) {
|
|
195
|
+
segments.push(current);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
current = null;
|
|
199
|
+
currentWords = [];
|
|
200
|
+
};
|
|
201
|
+
for (const word of words) {
|
|
202
|
+
if (word.type && word.type !== "word" && word.type !== "spacing") {
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const speaker = word.speaker_id?.trim() || "speaker_1";
|
|
206
|
+
const text = String(word.text ?? "");
|
|
207
|
+
if (!current || current.speaker !== speaker) {
|
|
208
|
+
flush();
|
|
209
|
+
current = { speaker, start_sec: Number(word.start ?? 0), end_sec: Number(word.end ?? 0), text: "" };
|
|
210
|
+
currentWords = [];
|
|
211
|
+
}
|
|
212
|
+
current.text += text;
|
|
213
|
+
current.end_sec = Number(word.end ?? current.end_sec ?? 0);
|
|
214
|
+
const trimmed = text.trim();
|
|
215
|
+
if (trimmed && Number.isFinite(Number(word.start)) && Number.isFinite(Number(word.end))) {
|
|
216
|
+
currentWords.push({ text: trimmed, start_sec: Number(word.start), end_sec: Number(word.end) });
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
flush();
|
|
220
|
+
return segments;
|
|
221
|
+
}
|
|
222
|
+
//# sourceMappingURL=elevenlabs.js.map
|