@mevdragon/vidfarm-devcli 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GETTING_STARTED.developers.md +25 -0
- package/README.md +44 -3
- package/SKILL.developer.md +147 -11
- package/dist/src/account-pages.js +1 -1
- package/dist/src/app.js +380 -13
- package/dist/src/cli.js +246 -59
- package/dist/src/config.js +4 -5
- package/dist/src/context.js +16 -0
- package/dist/src/db.js +67 -13
- package/dist/src/homepage.js +33 -1
- package/dist/src/lib/template-paths.js +10 -4
- package/dist/src/registry.js +9 -2
- package/dist/src/services/auth.js +2 -6
- package/dist/src/services/job-logs.js +4 -4
- package/dist/src/services/providers.js +470 -0
- package/dist/src/services/storage.js +13 -0
- package/dist/src/services/template-certification.js +16 -2
- package/dist/src/services/template-loader.js +17 -1
- package/dist/src/services/template-sources.js +338 -17
- package/{templates/template_0000 → dist/templates/vidfarm_template_0000}/src/remotion/Root.js +1 -0
- package/dist/templates/{template_0000 → vidfarm_template_0000}/src/template.js +9 -3
- package/package.json +4 -3
- package/templates/{template_0000 → vidfarm_template_0000}/README.md +23 -0
- package/templates/{template_0000 → vidfarm_template_0000}/package.json +0 -1
- package/templates/{template_0000 → vidfarm_template_0000}/src/sdk.ts +18 -0
- package/templates/{template_0000 → vidfarm_template_0000}/src/template.js +9 -3
- package/templates/{template_0000 → vidfarm_template_0000}/src/template.ts +10 -3
- package/templates/vidfarm_template_0000/tmp/solobacterium-moorei-slideshow.request.json +31 -0
- /package/dist/templates/{template_0000 → vidfarm_template_0000}/src/lib/images.js +0 -0
- /package/dist/templates/{template_0000 → vidfarm_template_0000}/src/remotion/index.js +0 -0
- /package/dist/templates/{template_0000 → vidfarm_template_0000}/src/sdk.js +0 -0
- /package/dist/templates/{template_0000 → vidfarm_template_0000}/src/style-options.js +0 -0
- /package/dist/templates/{template_0000 → vidfarm_template_0000}/src/template-dna.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/SKILL.md +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/Abel-Regular.ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/DMSerifDisplay-Regular.ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/Montserrat[wght].ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/SourceCodePro[wght].ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/TikTokSans-SemiBold.ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/assets/Yesteryear-Regular.ttf +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/composition.json +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/research/preview/.gitkeep +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/research/source_notes.md +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/lib/images.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/lib/images.ts +0 -0
- /package/{dist/templates/template_0000 → templates/vidfarm_template_0000}/src/remotion/Root.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/remotion/Root.tsx +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/remotion/index.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/remotion/index.tsx +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/sdk.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/style-options.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/style-options.ts +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/template-dna.js +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/src/template-dna.ts +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/template.config.json +0 -0
- /package/templates/{template_0000 → vidfarm_template_0000}/tsconfig.json +0 -0
package/dist/src/homepage.js
CHANGED
|
@@ -35,7 +35,7 @@ export function renderHomepage(input) {
|
|
|
35
35
|
const search = `${template.templateId} ${template.title} ${template.slugId}`.toLowerCase();
|
|
36
36
|
const media = template.previewUrl
|
|
37
37
|
? isVideoPreview(template.previewUrl)
|
|
38
|
-
? `<video class="preview" src="${escapeHtml(template.previewUrl)}" preload="
|
|
38
|
+
? `<video class="preview" data-lazy-video data-src="${escapeHtml(template.previewUrl)}" preload="metadata" muted playsinline controls></video>`
|
|
39
39
|
: `<img class="preview" src="${escapeHtml(template.previewUrl)}" alt="${escapeHtml(template.title)} preview" loading="lazy" decoding="async" />`
|
|
40
40
|
: `<div class="preview preview-empty">No preview</div>`;
|
|
41
41
|
return `
|
|
@@ -393,6 +393,7 @@ export function renderHomepage(input) {
|
|
|
393
393
|
const rows = Array.from(document.querySelectorAll(".row"));
|
|
394
394
|
const count = document.getElementById("count");
|
|
395
395
|
const emptyState = document.getElementById("empty-state");
|
|
396
|
+
const lazyVideos = Array.from(document.querySelectorAll("[data-lazy-video]"));
|
|
396
397
|
|
|
397
398
|
const updateFilter = () => {
|
|
398
399
|
const query = (searchInput.value || "").trim().toLowerCase();
|
|
@@ -411,6 +412,37 @@ export function renderHomepage(input) {
|
|
|
411
412
|
|
|
412
413
|
searchInput.addEventListener("input", updateFilter);
|
|
413
414
|
|
|
415
|
+
const loadVideo = (video) => {
|
|
416
|
+
const src = video.getAttribute("data-src");
|
|
417
|
+
if (!src || video.getAttribute("src")) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
video.setAttribute("src", src);
|
|
421
|
+
video.load();
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
if ("IntersectionObserver" in window) {
|
|
425
|
+
const observer = new IntersectionObserver((entries) => {
|
|
426
|
+
for (const entry of entries) {
|
|
427
|
+
if (!entry.isIntersecting) {
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
loadVideo(entry.target);
|
|
431
|
+
observer.unobserve(entry.target);
|
|
432
|
+
}
|
|
433
|
+
}, {
|
|
434
|
+
rootMargin: "240px 0px"
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
for (const video of lazyVideos) {
|
|
438
|
+
observer.observe(video);
|
|
439
|
+
}
|
|
440
|
+
} else {
|
|
441
|
+
for (const video of lazyVideos) {
|
|
442
|
+
loadVideo(video);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
414
446
|
for (const button of document.querySelectorAll("[data-copy-text]")) {
|
|
415
447
|
button.addEventListener("click", async () => {
|
|
416
448
|
const text = button.getAttribute("data-copy-text") || "";
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
export const TEMPLATE_FOLDER_PREFIX = "vidfarm_template_";
|
|
3
|
+
export const LEGACY_TEMPLATE_FOLDER_PREFIX = "template_";
|
|
3
4
|
export function assertTemplateFolderNameHasPrefix(folderName) {
|
|
4
|
-
if (!folderName.startsWith(TEMPLATE_FOLDER_PREFIX)) {
|
|
5
|
-
throw new Error(`Template folder name must start with ${TEMPLATE_FOLDER_PREFIX}. Received: ${folderName}`);
|
|
5
|
+
if (!folderName.startsWith(TEMPLATE_FOLDER_PREFIX) && !folderName.startsWith(LEGACY_TEMPLATE_FOLDER_PREFIX)) {
|
|
6
|
+
throw new Error(`Template folder name must start with ${TEMPLATE_FOLDER_PREFIX} or ${LEGACY_TEMPLATE_FOLDER_PREFIX}. Received: ${folderName}`);
|
|
6
7
|
}
|
|
7
8
|
}
|
|
8
9
|
export function deriveTemplateRootDirFromModulePath(templateModulePath) {
|
|
9
10
|
const normalizedModulePath = templateModulePath.replace(/\\/g, "/");
|
|
10
11
|
const templateSourceDir = path.posix.dirname(normalizedModulePath);
|
|
11
12
|
const templateRootDir = path.posix.dirname(templateSourceDir);
|
|
13
|
+
const sourceDirName = path.posix.basename(templateSourceDir);
|
|
14
|
+
if (sourceDirName !== "src") {
|
|
15
|
+
throw new Error("template_module_path must point at a TypeScript entrypoint inside a src/ folder, such as src/template.ts or templates/vidfarm_template_example/src/template.ts.");
|
|
16
|
+
}
|
|
12
17
|
if (templateRootDir === "." || templateRootDir === "") {
|
|
13
|
-
|
|
18
|
+
return "";
|
|
14
19
|
}
|
|
15
20
|
const folderName = path.posix.basename(templateRootDir);
|
|
16
21
|
assertTemplateFolderNameHasPrefix(folderName);
|
|
17
22
|
return templateRootDir;
|
|
18
23
|
}
|
|
19
24
|
export function defaultSkillPathForTemplateModule(templateModulePath) {
|
|
20
|
-
|
|
25
|
+
const templateRootDir = deriveTemplateRootDirFromModulePath(templateModulePath);
|
|
26
|
+
return templateRootDir ? path.posix.join(templateRootDir, "SKILL.md") : "SKILL.md";
|
|
21
27
|
}
|
package/dist/src/registry.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { database } from "./db.js";
|
|
2
|
-
import { template0000Definition } from "../templates/
|
|
2
|
+
import { template0000Definition } from "../templates/vidfarm_template_0000/src/template.js";
|
|
3
3
|
import { loadTemplateFromModule } from "./services/template-loader.js";
|
|
4
4
|
import { TemplateCertificationService } from "./services/template-certification.js";
|
|
5
5
|
class TemplateRegistry {
|
|
@@ -42,7 +42,14 @@ class TemplateRegistry {
|
|
|
42
42
|
return this.initPromise;
|
|
43
43
|
}
|
|
44
44
|
list() {
|
|
45
|
-
|
|
45
|
+
const merged = new Map();
|
|
46
|
+
for (const template of this.localTemplates.values()) {
|
|
47
|
+
merged.set(template.id, template);
|
|
48
|
+
}
|
|
49
|
+
for (const template of this.runtimeTemplates.values()) {
|
|
50
|
+
merged.set(template.id, template);
|
|
51
|
+
}
|
|
52
|
+
return [...merged.values()].sort((a, b) => a.id.localeCompare(b.id));
|
|
46
53
|
}
|
|
47
54
|
get(templateId) {
|
|
48
55
|
return this.findTemplate(templateId, this.runtimeTemplates) ?? this.findTemplate(templateId, this.localTemplates);
|
|
@@ -54,9 +54,7 @@ export class AuthService {
|
|
|
54
54
|
email,
|
|
55
55
|
name: name ?? existing?.name ?? null,
|
|
56
56
|
defaultWebhookUrl: existing?.defaultWebhookUrl ?? null,
|
|
57
|
-
isDeveloper: existing?.isDeveloper
|
|
58
|
-
|| config.adminEmails.includes(normalizedEmail)
|
|
59
|
-
|| config.developerEmails.includes(normalizedEmail),
|
|
57
|
+
isDeveloper: existing?.isDeveloper || config.adminEmails.includes(normalizedEmail),
|
|
60
58
|
isPaidPlan: existing?.isPaidPlan ?? false
|
|
61
59
|
});
|
|
62
60
|
const rawApiKey = `vf_${createId("key")}`;
|
|
@@ -134,9 +132,7 @@ export class AuthService {
|
|
|
134
132
|
email: normalizedEmail,
|
|
135
133
|
name: input.name ?? existing?.name ?? null,
|
|
136
134
|
defaultWebhookUrl: existing?.defaultWebhookUrl ?? null,
|
|
137
|
-
isDeveloper: existing?.isDeveloper
|
|
138
|
-
|| config.adminEmails.includes(normalizedEmail)
|
|
139
|
-
|| config.developerEmails.includes(normalizedEmail),
|
|
135
|
+
isDeveloper: existing?.isDeveloper || config.adminEmails.includes(normalizedEmail),
|
|
140
136
|
isPaidPlan: true,
|
|
141
137
|
passwordHash: hashPassword(input.password)
|
|
142
138
|
});
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { StorageService } from "./storage.js";
|
|
2
2
|
const JOB_LOG_PREFIX = "job-logs";
|
|
3
|
-
const JOB_LOG_CHUNK_INTERVAL_MS =
|
|
3
|
+
const JOB_LOG_CHUNK_INTERVAL_MS = 60000;
|
|
4
4
|
const JOB_LOG_CHUNK_MAX_BYTES = 1024 * 1024;
|
|
5
|
-
const JOB_LOG_FLUSH_INTERVAL_MS =
|
|
5
|
+
const JOB_LOG_FLUSH_INTERVAL_MS = 5000;
|
|
6
6
|
export class JobLogStore {
|
|
7
7
|
storage = new StorageService();
|
|
8
8
|
activeChunks = new Map();
|
|
@@ -81,7 +81,7 @@ export class JobLogStore {
|
|
|
81
81
|
const minMs = Math.min(startMs, endMs);
|
|
82
82
|
const maxMs = Math.max(startMs, endMs);
|
|
83
83
|
const minuteStarts = [];
|
|
84
|
-
for (let cursor = minMs; cursor <= maxMs; cursor +=
|
|
84
|
+
for (let cursor = minMs; cursor <= maxMs; cursor += 60000) {
|
|
85
85
|
minuteStarts.push(cursor);
|
|
86
86
|
}
|
|
87
87
|
return minuteStarts;
|
|
@@ -155,7 +155,7 @@ function serializedEventSize(event) {
|
|
|
155
155
|
return Buffer.byteLength(`${JSON.stringify(event)}\n`, "utf8");
|
|
156
156
|
}
|
|
157
157
|
function floorToMinute(timestampMs) {
|
|
158
|
-
return Math.floor(timestampMs /
|
|
158
|
+
return Math.floor(timestampMs / 60000) * 60000;
|
|
159
159
|
}
|
|
160
160
|
function parseChunk(content) {
|
|
161
161
|
if (!content) {
|
|
@@ -15,6 +15,16 @@ export class ProviderRateLimitError extends Error {
|
|
|
15
15
|
this.retryAfterSeconds = retryAfterSeconds;
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
|
+
const SUPPORTED_TTS_MODELS = {
|
|
19
|
+
openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
|
|
20
|
+
gemini: new Set(["gemini-3.1-flash-tts-preview"]),
|
|
21
|
+
openai: new Set(["gpt-4o-mini-tts-2025-12-15"])
|
|
22
|
+
};
|
|
23
|
+
const SUPPORTED_STT_MODELS = {
|
|
24
|
+
openrouter: new Set(["openai/gpt-4o-mini-transcribe"]),
|
|
25
|
+
gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
|
|
26
|
+
openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
|
|
27
|
+
};
|
|
18
28
|
function readStoredProviderSecret(value) {
|
|
19
29
|
try {
|
|
20
30
|
return decryptString(value, config.ENCRYPTION_SECRET);
|
|
@@ -129,6 +139,103 @@ export class ProviderService {
|
|
|
129
139
|
await this.releaseLease(lease);
|
|
130
140
|
}
|
|
131
141
|
}
|
|
142
|
+
async generateSpeech(input) {
|
|
143
|
+
assertSupportedSpeechModel("tts", input.provider, input.model);
|
|
144
|
+
const lease = await this.leaseCustomerKey({
|
|
145
|
+
customerId: input.customerId,
|
|
146
|
+
provider: input.provider,
|
|
147
|
+
jobId: input.jobId,
|
|
148
|
+
workerId: input.workerId
|
|
149
|
+
});
|
|
150
|
+
try {
|
|
151
|
+
if (config.mockProviders) {
|
|
152
|
+
const contentType = inferSpeechContentType(input.responseFormat ?? "mp3");
|
|
153
|
+
database.recordProviderKeyUsage({
|
|
154
|
+
id: createId("usage"),
|
|
155
|
+
keyId: lease.keyId,
|
|
156
|
+
jobId: input.jobId,
|
|
157
|
+
provider: input.provider,
|
|
158
|
+
model: input.model,
|
|
159
|
+
eventType: "success",
|
|
160
|
+
inputTokens: 0,
|
|
161
|
+
outputTokens: 0,
|
|
162
|
+
costUsd: 0.01,
|
|
163
|
+
metadata: { mock: true, type: "speech_generation" }
|
|
164
|
+
});
|
|
165
|
+
database.touchProviderKey(lease.keyId);
|
|
166
|
+
return {
|
|
167
|
+
bytes: Buffer.from(`mock speech:${input.text.slice(0, 120)}`, "utf8"),
|
|
168
|
+
contentType,
|
|
169
|
+
usage: {
|
|
170
|
+
inputTokens: 0,
|
|
171
|
+
outputTokens: 0,
|
|
172
|
+
costUsd: 0.01
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
const result = input.provider === "gemini"
|
|
177
|
+
? await this.callGeminiSpeechGeneration({
|
|
178
|
+
model: input.model,
|
|
179
|
+
text: input.text,
|
|
180
|
+
voice: input.voice,
|
|
181
|
+
instructions: input.instructions,
|
|
182
|
+
apiKey: lease.secret
|
|
183
|
+
})
|
|
184
|
+
: await this.callOpenAICompatibleSpeechGeneration({
|
|
185
|
+
provider: input.provider === "openrouter" ? "openrouter" : "openai",
|
|
186
|
+
model: input.model,
|
|
187
|
+
text: input.text,
|
|
188
|
+
voice: input.voice,
|
|
189
|
+
instructions: input.instructions,
|
|
190
|
+
responseFormat: input.responseFormat,
|
|
191
|
+
apiKey: lease.secret
|
|
192
|
+
});
|
|
193
|
+
database.recordProviderKeyUsage({
|
|
194
|
+
id: createId("usage"),
|
|
195
|
+
keyId: lease.keyId,
|
|
196
|
+
jobId: input.jobId,
|
|
197
|
+
provider: input.provider,
|
|
198
|
+
model: input.model,
|
|
199
|
+
eventType: "success",
|
|
200
|
+
inputTokens: result.usage.inputTokens,
|
|
201
|
+
outputTokens: result.usage.outputTokens,
|
|
202
|
+
costUsd: result.usage.costUsd,
|
|
203
|
+
metadata: { type: "speech_generation" }
|
|
204
|
+
});
|
|
205
|
+
database.touchProviderKey(lease.keyId);
|
|
206
|
+
return result;
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
if (error instanceof ProviderRateLimitError) {
|
|
210
|
+
database.recordProviderKeyUsage({
|
|
211
|
+
id: createId("usage"),
|
|
212
|
+
keyId: lease.keyId,
|
|
213
|
+
jobId: input.jobId,
|
|
214
|
+
provider: input.provider,
|
|
215
|
+
model: input.model,
|
|
216
|
+
eventType: "rate_limit",
|
|
217
|
+
metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_generation" }
|
|
218
|
+
});
|
|
219
|
+
database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
|
|
220
|
+
}
|
|
221
|
+
else if (error instanceof ProviderAuthError) {
|
|
222
|
+
database.recordProviderKeyUsage({
|
|
223
|
+
id: createId("usage"),
|
|
224
|
+
keyId: lease.keyId,
|
|
225
|
+
jobId: input.jobId,
|
|
226
|
+
provider: input.provider,
|
|
227
|
+
model: input.model,
|
|
228
|
+
eventType: "auth_error",
|
|
229
|
+
metadata: { type: "speech_generation" }
|
|
230
|
+
});
|
|
231
|
+
database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
|
|
232
|
+
}
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
finally {
|
|
236
|
+
await this.releaseLease(lease);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
132
239
|
async generateImage(input) {
|
|
133
240
|
const lease = await this.leaseCustomerKey({
|
|
134
241
|
customerId: input.customerId,
|
|
@@ -265,6 +372,110 @@ export class ProviderService {
|
|
|
265
372
|
await this.releaseLease(lease);
|
|
266
373
|
}
|
|
267
374
|
}
|
|
375
|
+
async transcribeSpeech(input) {
|
|
376
|
+
assertSupportedSpeechModel("stt", input.provider, input.model);
|
|
377
|
+
const lease = await this.leaseCustomerKey({
|
|
378
|
+
customerId: input.customerId,
|
|
379
|
+
provider: input.provider,
|
|
380
|
+
jobId: input.jobId,
|
|
381
|
+
workerId: input.workerId
|
|
382
|
+
});
|
|
383
|
+
try {
|
|
384
|
+
if (config.mockProviders) {
|
|
385
|
+
database.recordProviderKeyUsage({
|
|
386
|
+
id: createId("usage"),
|
|
387
|
+
keyId: lease.keyId,
|
|
388
|
+
jobId: input.jobId,
|
|
389
|
+
provider: input.provider,
|
|
390
|
+
model: input.model,
|
|
391
|
+
eventType: "success",
|
|
392
|
+
inputTokens: 0,
|
|
393
|
+
outputTokens: 0,
|
|
394
|
+
costUsd: 0.01,
|
|
395
|
+
metadata: { mock: true, type: "speech_transcription" }
|
|
396
|
+
});
|
|
397
|
+
database.touchProviderKey(lease.keyId);
|
|
398
|
+
return {
|
|
399
|
+
text: "Mock transcription output.",
|
|
400
|
+
usage: {
|
|
401
|
+
inputTokens: 0,
|
|
402
|
+
outputTokens: 0,
|
|
403
|
+
costUsd: 0.01
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
const result = input.provider === "gemini"
|
|
408
|
+
? await this.callGeminiSpeechTranscription({
|
|
409
|
+
model: input.model,
|
|
410
|
+
audio: input.audio,
|
|
411
|
+
contentType: input.contentType,
|
|
412
|
+
prompt: input.prompt,
|
|
413
|
+
language: input.language,
|
|
414
|
+
apiKey: lease.secret
|
|
415
|
+
})
|
|
416
|
+
: input.provider === "openrouter"
|
|
417
|
+
? await this.callOpenRouterSpeechTranscription({
|
|
418
|
+
model: input.model,
|
|
419
|
+
audio: input.audio,
|
|
420
|
+
contentType: input.contentType,
|
|
421
|
+
prompt: input.prompt,
|
|
422
|
+
language: input.language,
|
|
423
|
+
apiKey: lease.secret
|
|
424
|
+
})
|
|
425
|
+
: await this.callOpenAISpeechTranscription({
|
|
426
|
+
model: input.model,
|
|
427
|
+
audio: input.audio,
|
|
428
|
+
contentType: input.contentType,
|
|
429
|
+
prompt: input.prompt,
|
|
430
|
+
language: input.language,
|
|
431
|
+
apiKey: lease.secret
|
|
432
|
+
});
|
|
433
|
+
database.recordProviderKeyUsage({
|
|
434
|
+
id: createId("usage"),
|
|
435
|
+
keyId: lease.keyId,
|
|
436
|
+
jobId: input.jobId,
|
|
437
|
+
provider: input.provider,
|
|
438
|
+
model: input.model,
|
|
439
|
+
eventType: "success",
|
|
440
|
+
inputTokens: result.usage.inputTokens,
|
|
441
|
+
outputTokens: result.usage.outputTokens,
|
|
442
|
+
costUsd: result.usage.costUsd,
|
|
443
|
+
metadata: { type: "speech_transcription" }
|
|
444
|
+
});
|
|
445
|
+
database.touchProviderKey(lease.keyId);
|
|
446
|
+
return result;
|
|
447
|
+
}
|
|
448
|
+
catch (error) {
|
|
449
|
+
if (error instanceof ProviderRateLimitError) {
|
|
450
|
+
database.recordProviderKeyUsage({
|
|
451
|
+
id: createId("usage"),
|
|
452
|
+
keyId: lease.keyId,
|
|
453
|
+
jobId: input.jobId,
|
|
454
|
+
provider: input.provider,
|
|
455
|
+
model: input.model,
|
|
456
|
+
eventType: "rate_limit",
|
|
457
|
+
metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_transcription" }
|
|
458
|
+
});
|
|
459
|
+
database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
|
|
460
|
+
}
|
|
461
|
+
else if (error instanceof ProviderAuthError) {
|
|
462
|
+
database.recordProviderKeyUsage({
|
|
463
|
+
id: createId("usage"),
|
|
464
|
+
keyId: lease.keyId,
|
|
465
|
+
jobId: input.jobId,
|
|
466
|
+
provider: input.provider,
|
|
467
|
+
model: input.model,
|
|
468
|
+
eventType: "auth_error",
|
|
469
|
+
metadata: { type: "speech_transcription" }
|
|
470
|
+
});
|
|
471
|
+
database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
|
|
472
|
+
}
|
|
473
|
+
throw error;
|
|
474
|
+
}
|
|
475
|
+
finally {
|
|
476
|
+
await this.releaseLease(lease);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
268
479
|
async callOpenAIImageGeneration(input) {
|
|
269
480
|
if (input.promptAttachments?.length) {
|
|
270
481
|
throw new Error("OpenAI image generation attachments are not implemented in this provider path. Use Gemini for reference attachments.");
|
|
@@ -355,6 +566,94 @@ export class ProviderService {
|
|
|
355
566
|
revisedPrompt: null
|
|
356
567
|
};
|
|
357
568
|
}
|
|
569
|
+
async callOpenAICompatibleSpeechGeneration(input) {
|
|
570
|
+
const responseFormat = input.responseFormat ?? "mp3";
|
|
571
|
+
const endpoint = input.provider === "openrouter"
|
|
572
|
+
? "https://openrouter.ai/api/v1/audio/speech"
|
|
573
|
+
: "https://api.openai.com/v1/audio/speech";
|
|
574
|
+
const response = await fetch(endpoint, {
|
|
575
|
+
method: "POST",
|
|
576
|
+
headers: {
|
|
577
|
+
"Content-Type": "application/json",
|
|
578
|
+
Authorization: `Bearer ${input.apiKey}`
|
|
579
|
+
},
|
|
580
|
+
body: JSON.stringify({
|
|
581
|
+
model: input.model,
|
|
582
|
+
input: input.text,
|
|
583
|
+
voice: input.voice ?? "alloy",
|
|
584
|
+
instructions: input.instructions,
|
|
585
|
+
response_format: responseFormat
|
|
586
|
+
})
|
|
587
|
+
});
|
|
588
|
+
if (response.status === 401 || response.status === 403) {
|
|
589
|
+
throw new ProviderAuthError(`${input.provider} authentication failed`);
|
|
590
|
+
}
|
|
591
|
+
if (response.status === 429) {
|
|
592
|
+
throw new ProviderRateLimitError(`${input.provider} rate limited`);
|
|
593
|
+
}
|
|
594
|
+
if (!response.ok) {
|
|
595
|
+
const details = await response.text();
|
|
596
|
+
throw new Error(`${input.provider} speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
597
|
+
}
|
|
598
|
+
return {
|
|
599
|
+
bytes: Buffer.from(await response.arrayBuffer()),
|
|
600
|
+
contentType: response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat),
|
|
601
|
+
usage: {
|
|
602
|
+
inputTokens: 0,
|
|
603
|
+
outputTokens: 0,
|
|
604
|
+
costUsd: 0
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
async callGeminiSpeechGeneration(input) {
|
|
609
|
+
const prompt = input.instructions?.trim()
|
|
610
|
+
? `${input.instructions.trim()}\n\nSpeak exactly the following text:\n${input.text}`
|
|
611
|
+
: input.text;
|
|
612
|
+
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
|
|
613
|
+
method: "POST",
|
|
614
|
+
headers: {
|
|
615
|
+
"Content-Type": "application/json"
|
|
616
|
+
},
|
|
617
|
+
body: JSON.stringify({
|
|
618
|
+
contents: [{ parts: [{ text: prompt }] }],
|
|
619
|
+
generationConfig: {
|
|
620
|
+
responseModalities: ["AUDIO"],
|
|
621
|
+
speechConfig: {
|
|
622
|
+
voiceConfig: {
|
|
623
|
+
prebuiltVoiceConfig: {
|
|
624
|
+
voiceName: input.voice ?? "Kore"
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
})
|
|
630
|
+
});
|
|
631
|
+
if (response.status === 401 || response.status === 403) {
|
|
632
|
+
throw new ProviderAuthError("gemini authentication failed");
|
|
633
|
+
}
|
|
634
|
+
if (response.status === 429) {
|
|
635
|
+
throw new ProviderRateLimitError("gemini rate limited");
|
|
636
|
+
}
|
|
637
|
+
if (!response.ok) {
|
|
638
|
+
const details = await response.text();
|
|
639
|
+
throw new Error(`gemini speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
640
|
+
}
|
|
641
|
+
const data = await response.json();
|
|
642
|
+
const part = data.candidates?.[0]?.content?.parts?.find((item) => item.inlineData?.data || item.inline_data?.data);
|
|
643
|
+
const encoded = part?.inlineData?.data ?? part?.inline_data?.data;
|
|
644
|
+
if (!encoded) {
|
|
645
|
+
throw new Error("gemini speech generation returned no audio payload");
|
|
646
|
+
}
|
|
647
|
+
return {
|
|
648
|
+
bytes: Buffer.from(encoded, "base64"),
|
|
649
|
+
contentType: part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000",
|
|
650
|
+
usage: {
|
|
651
|
+
inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
|
|
652
|
+
outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
|
|
653
|
+
costUsd: 0
|
|
654
|
+
}
|
|
655
|
+
};
|
|
656
|
+
}
|
|
358
657
|
async callOpenAICompatibleLayoutAnalysis(input) {
|
|
359
658
|
const endpoint = input.provider === "openrouter"
|
|
360
659
|
? "https://openrouter.ai/api/v1/chat/completions"
|
|
@@ -458,6 +757,128 @@ export class ProviderService {
|
|
|
458
757
|
const text = data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "";
|
|
459
758
|
return String(text);
|
|
460
759
|
}
|
|
760
|
+
async callOpenAISpeechTranscription(input) {
|
|
761
|
+
const contentType = input.contentType ?? "audio/mpeg";
|
|
762
|
+
const form = new FormData();
|
|
763
|
+
form.set("file", new Blob([Buffer.from(input.audio)], { type: contentType }), `audio.${audioExtensionFromContentType(contentType)}`);
|
|
764
|
+
form.set("model", input.model);
|
|
765
|
+
form.set("response_format", "json");
|
|
766
|
+
if (input.prompt) {
|
|
767
|
+
form.set("prompt", input.prompt);
|
|
768
|
+
}
|
|
769
|
+
if (input.language) {
|
|
770
|
+
form.set("language", input.language);
|
|
771
|
+
}
|
|
772
|
+
const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
|
|
773
|
+
method: "POST",
|
|
774
|
+
headers: {
|
|
775
|
+
Authorization: `Bearer ${input.apiKey}`
|
|
776
|
+
},
|
|
777
|
+
body: form
|
|
778
|
+
});
|
|
779
|
+
if (response.status === 401 || response.status === 403) {
|
|
780
|
+
throw new ProviderAuthError("openai authentication failed");
|
|
781
|
+
}
|
|
782
|
+
if (response.status === 429) {
|
|
783
|
+
throw new ProviderRateLimitError("openai rate limited");
|
|
784
|
+
}
|
|
785
|
+
if (!response.ok) {
|
|
786
|
+
const details = await response.text();
|
|
787
|
+
throw new Error(`openai transcription returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
788
|
+
}
|
|
789
|
+
const data = await response.json();
|
|
790
|
+
return {
|
|
791
|
+
text: String(data.text ?? ""),
|
|
792
|
+
usage: {
|
|
793
|
+
inputTokens: Number(data.usage?.input_tokens ?? 0),
|
|
794
|
+
outputTokens: Number(data.usage?.output_tokens ?? 0),
|
|
795
|
+
costUsd: Number(data.usage?.cost ?? 0)
|
|
796
|
+
}
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
async callOpenRouterSpeechTranscription(input) {
|
|
800
|
+
const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
|
|
801
|
+
method: "POST",
|
|
802
|
+
headers: {
|
|
803
|
+
"Content-Type": "application/json",
|
|
804
|
+
Authorization: `Bearer ${input.apiKey}`
|
|
805
|
+
},
|
|
806
|
+
body: JSON.stringify({
|
|
807
|
+
model: input.model,
|
|
808
|
+
input_audio: {
|
|
809
|
+
data: Buffer.from(input.audio).toString("base64"),
|
|
810
|
+
format: audioFormatFromContentType(input.contentType)
|
|
811
|
+
},
|
|
812
|
+
...(input.prompt ? { prompt: input.prompt } : {}),
|
|
813
|
+
...(input.language ? { language: input.language } : {})
|
|
814
|
+
})
|
|
815
|
+
});
|
|
816
|
+
if (response.status === 401 || response.status === 403) {
|
|
817
|
+
throw new ProviderAuthError("openrouter authentication failed");
|
|
818
|
+
}
|
|
819
|
+
if (response.status === 429) {
|
|
820
|
+
throw new ProviderRateLimitError("openrouter rate limited");
|
|
821
|
+
}
|
|
822
|
+
if (!response.ok) {
|
|
823
|
+
const details = await response.text();
|
|
824
|
+
throw new Error(`openrouter transcription returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
825
|
+
}
|
|
826
|
+
const data = await response.json();
|
|
827
|
+
return {
|
|
828
|
+
text: String(data.text ?? ""),
|
|
829
|
+
usage: {
|
|
830
|
+
inputTokens: Number(data.usage?.input_tokens ?? data.usage?.prompt_tokens ?? 0),
|
|
831
|
+
outputTokens: Number(data.usage?.output_tokens ?? data.usage?.completion_tokens ?? 0),
|
|
832
|
+
costUsd: Number(data.usage?.cost ?? 0)
|
|
833
|
+
}
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
async callGeminiSpeechTranscription(input) {
|
|
837
|
+
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${input.model}:generateContent?key=${input.apiKey}`, {
|
|
838
|
+
method: "POST",
|
|
839
|
+
headers: {
|
|
840
|
+
"Content-Type": "application/json"
|
|
841
|
+
},
|
|
842
|
+
body: JSON.stringify({
|
|
843
|
+
contents: [{
|
|
844
|
+
parts: [
|
|
845
|
+
{
|
|
846
|
+
inline_data: {
|
|
847
|
+
mime_type: input.contentType ?? "audio/mpeg",
|
|
848
|
+
data: Buffer.from(input.audio).toString("base64")
|
|
849
|
+
}
|
|
850
|
+
},
|
|
851
|
+
{
|
|
852
|
+
text: buildGeminiTranscriptionPrompt(input.prompt, input.language)
|
|
853
|
+
}
|
|
854
|
+
]
|
|
855
|
+
}],
|
|
856
|
+
generationConfig: {
|
|
857
|
+
temperature: 0,
|
|
858
|
+
responseMimeType: "text/plain"
|
|
859
|
+
}
|
|
860
|
+
})
|
|
861
|
+
});
|
|
862
|
+
if (response.status === 401 || response.status === 403) {
|
|
863
|
+
throw new ProviderAuthError("gemini authentication failed");
|
|
864
|
+
}
|
|
865
|
+
if (response.status === 429) {
|
|
866
|
+
throw new ProviderRateLimitError("gemini rate limited");
|
|
867
|
+
}
|
|
868
|
+
if (!response.ok) {
|
|
869
|
+
const details = await response.text();
|
|
870
|
+
throw new Error(`gemini transcription returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
871
|
+
}
|
|
872
|
+
const data = await response.json();
|
|
873
|
+
return {
|
|
874
|
+
text: extractGeminiText(data),
|
|
875
|
+
usage: {
|
|
876
|
+
inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
|
|
877
|
+
outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
|
|
878
|
+
costUsd: 0
|
|
879
|
+
}
|
|
880
|
+
};
|
|
881
|
+
}
|
|
461
882
|
async callProvider(input) {
|
|
462
883
|
if (input.provider === "gemini") {
|
|
463
884
|
return this.callGemini({
|
|
@@ -543,6 +964,55 @@ export class ProviderService {
|
|
|
543
964
|
function clamp(value, min, max) {
|
|
544
965
|
return Math.max(min, Math.min(max, value));
|
|
545
966
|
}
|
|
967
|
+
function assertSupportedSpeechModel(kind, provider, model) {
|
|
968
|
+
const supported = kind === "tts" ? SUPPORTED_TTS_MODELS[provider] : SUPPORTED_STT_MODELS[provider];
|
|
969
|
+
if (!supported?.has(model)) {
|
|
970
|
+
throw new Error(`Unsupported ${kind.toUpperCase()} model for provider ${provider}: ${model}`);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
function inferSpeechContentType(format) {
|
|
974
|
+
return format === "wav" ? "audio/wav" : "audio/mpeg";
|
|
975
|
+
}
|
|
976
|
+
function audioFormatFromContentType(contentType) {
|
|
977
|
+
const normalized = contentType?.split(";")[0]?.trim().toLowerCase();
|
|
978
|
+
switch (normalized) {
|
|
979
|
+
case "audio/wav":
|
|
980
|
+
case "audio/x-wav":
|
|
981
|
+
return "wav";
|
|
982
|
+
case "audio/mp4":
|
|
983
|
+
case "audio/m4a":
|
|
984
|
+
return "mp4";
|
|
985
|
+
case "audio/webm":
|
|
986
|
+
return "webm";
|
|
987
|
+
case "audio/ogg":
|
|
988
|
+
return "ogg";
|
|
989
|
+
default:
|
|
990
|
+
return "mp3";
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
function audioExtensionFromContentType(contentType) {
|
|
994
|
+
switch (audioFormatFromContentType(contentType)) {
|
|
995
|
+
case "wav":
|
|
996
|
+
return "wav";
|
|
997
|
+
case "mp4":
|
|
998
|
+
return "m4a";
|
|
999
|
+
case "webm":
|
|
1000
|
+
return "webm";
|
|
1001
|
+
case "ogg":
|
|
1002
|
+
return "ogg";
|
|
1003
|
+
default:
|
|
1004
|
+
return "mp3";
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
function buildGeminiTranscriptionPrompt(prompt, language) {
|
|
1008
|
+
const basePrompt = prompt?.trim() || "Transcribe this audio verbatim. Return plain text only.";
|
|
1009
|
+
return language?.trim()
|
|
1010
|
+
? `${basePrompt}\n\nExpected language: ${language.trim()}.`
|
|
1011
|
+
: basePrompt;
|
|
1012
|
+
}
|
|
1013
|
+
function extractGeminiText(data) {
|
|
1014
|
+
return String(data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? "").join("\n") ?? "");
|
|
1015
|
+
}
|
|
546
1016
|
function supportsGeminiImageSize(model) {
|
|
547
1017
|
return model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview";
|
|
548
1018
|
}
|