@mevdragon/vidfarm-devcli 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -0
- package/GETTING_STARTED.developers.md +1 -26
- package/README.md +3 -0
- package/SKILL.developer.md +419 -8
- package/auto-create-templates/AUTO_CREATE_TEMPLATES.md +475 -0
- package/auto-create-templates/extractor-system-prompt.md +113 -0
- package/auto-create-templates/production-graph.schema.json +276 -0
- package/auto-create-templates/runbook.md +140 -0
- package/auto-create-templates/template-plan.schema.json +230 -0
- package/auto-create-templates/template-planner-prompt.md +75 -0
- package/dist/src/account-pages-legacy.js +9064 -0
- package/dist/src/account-pages.js +41 -620
- package/dist/src/app.js +6627 -244
- package/dist/src/cli.js +386 -26
- package/dist/src/config.js +12 -2
- package/dist/src/context.js +35 -35
- package/dist/src/db.js +1016 -53
- package/dist/src/dev-app-legacy.js +693 -0
- package/dist/src/dev-app.js +4 -904
- package/dist/src/domain.js +1 -1
- package/dist/src/editor-chat-history.js +72 -0
- package/dist/src/editor-chat.js +202 -0
- package/dist/src/frontend/flockposter-cache-store.js +116 -0
- package/dist/src/frontend/homepage-client.js +114 -0
- package/dist/src/frontend/homepage-shared.js +3 -0
- package/dist/src/frontend/homepage-store.js +27 -0
- package/dist/src/frontend/homepage-view.js +147 -0
- package/dist/src/frontend/page-runtime-client.js +100 -0
- package/dist/src/frontend/page-runtime-store.js +8 -0
- package/dist/src/frontend/template-editor-chat.js +2261 -0
- package/dist/src/homepage.js +695 -371
- package/dist/src/lib/template-style-options.js +95 -15
- package/dist/src/page-runtime.js +1 -0
- package/dist/src/page-shell.js +941 -0
- package/dist/src/primitive-context.js +163 -0
- package/dist/src/primitive-registry.js +340 -0
- package/dist/src/primitive-sdk.js +3 -0
- package/dist/src/primitives/remotion/html-image.js +28 -0
- package/dist/src/react-page-shell.js +34 -0
- package/dist/src/ready-post-schedule-component.js +1514 -0
- package/dist/src/registry.js +36 -17
- package/dist/src/runtime.js +6 -1
- package/dist/src/services/api-call-history.js +245 -0
- package/dist/src/services/auth.js +25 -3
- package/dist/src/services/billing.js +248 -9
- package/dist/src/services/chat-threads.js +88 -0
- package/dist/src/services/job-logs.js +14 -7
- package/dist/src/services/jobs.js +13 -2
- package/dist/src/services/providers.js +552 -133
- package/dist/src/services/rate-limits.js +236 -0
- package/dist/src/services/remotion.js +143 -16
- package/dist/src/services/storage.js +10 -8
- package/dist/src/services/template-certification.js +11 -2
- package/dist/src/services/template-loader.js +3 -1
- package/dist/src/services/template-sources.js +68 -3
- package/dist/src/template-editor-pages.js +2309 -0
- package/dist/src/template-editor-shell.js +1507 -0
- package/dist/src/worker.js +120 -22
- package/package.json +27 -4
- package/public/assets/homepage-app.js +54 -0
- package/public/assets/homepage-client-app.js +54 -0
- package/public/assets/page-runtime-client-app.js +68 -0
- package/dist/templates/vidfarm_template_0000/src/lib/images.js +0 -202
- package/dist/templates/vidfarm_template_0000/src/remotion/Root.js +0 -34
- package/dist/templates/vidfarm_template_0000/src/remotion/index.js +0 -3
- package/dist/templates/vidfarm_template_0000/src/sdk.js +0 -3
- package/dist/templates/vidfarm_template_0000/src/style-options.js +0 -51
- package/dist/templates/vidfarm_template_0000/src/template-dna.js +0 -9
- package/dist/templates/vidfarm_template_0000/src/template.js +0 -1447
- package/templates/vidfarm_template_0000/README.md +0 -100
- package/templates/vidfarm_template_0000/SKILL.md +0 -225
- package/templates/vidfarm_template_0000/assets/Abel-Regular.ttf +0 -0
- package/templates/vidfarm_template_0000/assets/DMSerifDisplay-Regular.ttf +0 -0
- package/templates/vidfarm_template_0000/assets/Montserrat[wght].ttf +0 -0
- package/templates/vidfarm_template_0000/assets/SourceCodePro[wght].ttf +0 -0
- package/templates/vidfarm_template_0000/assets/TikTokSans-SemiBold.ttf +0 -0
- package/templates/vidfarm_template_0000/assets/Yesteryear-Regular.ttf +0 -0
- package/templates/vidfarm_template_0000/composition.json +0 -11
- package/templates/vidfarm_template_0000/package.json +0 -27
- package/templates/vidfarm_template_0000/research/preview/.gitkeep +0 -1
- package/templates/vidfarm_template_0000/research/source_notes.md +0 -7
- package/templates/vidfarm_template_0000/src/lib/images.js +0 -202
- package/templates/vidfarm_template_0000/src/lib/images.ts +0 -241
- package/templates/vidfarm_template_0000/src/remotion/Root.js +0 -33
- package/templates/vidfarm_template_0000/src/remotion/Root.tsx +0 -75
- package/templates/vidfarm_template_0000/src/remotion/index.js +0 -3
- package/templates/vidfarm_template_0000/src/remotion/index.tsx +0 -4
- package/templates/vidfarm_template_0000/src/sdk.js +0 -3
- package/templates/vidfarm_template_0000/src/sdk.ts +0 -140
- package/templates/vidfarm_template_0000/src/style-options.js +0 -51
- package/templates/vidfarm_template_0000/src/style-options.ts +0 -60
- package/templates/vidfarm_template_0000/src/template-dna.js +0 -9
- package/templates/vidfarm_template_0000/src/template-dna.ts +0 -15
- package/templates/vidfarm_template_0000/src/template.js +0 -1447
- package/templates/vidfarm_template_0000/src/template.ts +0 -2049
- package/templates/vidfarm_template_0000/template.config.json +0 -21
- package/templates/vidfarm_template_0000/tmp/solobacterium-moorei-slideshow.request.json +0 -31
- package/templates/vidfarm_template_0000/tsconfig.json +0 -19
|
@@ -8,6 +8,13 @@ export class ProviderKeyUnavailableError extends Error {
|
|
|
8
8
|
}
|
|
9
9
|
export class ProviderAuthError extends Error {
|
|
10
10
|
}
|
|
11
|
+
export class ProviderLeaseTimeoutError extends Error {
|
|
12
|
+
retryAfterSeconds;
|
|
13
|
+
constructor(message, retryAfterSeconds = 5) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.retryAfterSeconds = retryAfterSeconds;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
11
18
|
export class ProviderRateLimitError extends Error {
|
|
12
19
|
retryAfterSeconds;
|
|
13
20
|
constructor(message, retryAfterSeconds = 60) {
|
|
@@ -17,7 +24,7 @@ export class ProviderRateLimitError extends Error {
|
|
|
17
24
|
}
|
|
18
25
|
const SUPPORTED_TTS_MODELS = {
|
|
19
26
|
openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
|
|
20
|
-
gemini: new Set(["gemini-3.1-flash-tts-preview"]),
|
|
27
|
+
gemini: new Set(["gemini-3.1-flash-tts-preview", "gemini-2.5-flash-preview-tts"]),
|
|
21
28
|
openai: new Set(["gpt-4o-mini-tts-2025-12-15"])
|
|
22
29
|
};
|
|
23
30
|
const SUPPORTED_STT_MODELS = {
|
|
@@ -25,6 +32,12 @@ const SUPPORTED_STT_MODELS = {
|
|
|
25
32
|
gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
|
|
26
33
|
openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
|
|
27
34
|
};
|
|
35
|
+
const PROVIDER_KEY_VALIDATION_MODELS = {
|
|
36
|
+
openai: "gpt-4.1-mini",
|
|
37
|
+
gemini: "gemini-3.1-flash-lite-preview",
|
|
38
|
+
openrouter: "google/gemini-3.1-flash-lite-preview",
|
|
39
|
+
perplexity: "sonar"
|
|
40
|
+
};
|
|
28
41
|
function readStoredProviderSecret(value) {
|
|
29
42
|
try {
|
|
30
43
|
return decryptString(value, config.ENCRYPTION_SECRET);
|
|
@@ -33,30 +46,143 @@ function readStoredProviderSecret(value) {
|
|
|
33
46
|
return value;
|
|
34
47
|
}
|
|
35
48
|
}
|
|
49
|
+
function sleep(ms) {
|
|
50
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
51
|
+
}
|
|
52
|
+
function zeroUsageCost(result) {
|
|
53
|
+
return {
|
|
54
|
+
...result,
|
|
55
|
+
usage: {
|
|
56
|
+
...result.usage,
|
|
57
|
+
costUsd: 0
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const DEFAULT_PROVIDER_LEASE_PRIORITY = 100;
|
|
62
|
+
const CHAT_PROVIDER_LEASE_PRIORITY = 0;
|
|
63
|
+
const DEFAULT_PROVIDER_LEASE_TIMEOUT_MS = 120_000;
|
|
64
|
+
const CHAT_PROVIDER_LEASE_TIMEOUT_MS = 45_000;
|
|
65
|
+
const PROVIDER_LEASE_POLL_MS = 250;
|
|
66
|
+
const DEFAULT_PROVIDER_LEASE_SECONDS = 90;
|
|
67
|
+
const CHAT_PROVIDER_LEASE_SECONDS = 300;
|
|
36
68
|
export class ProviderService {
|
|
69
|
+
handleLeasedProviderError(input) {
|
|
70
|
+
if (input.error instanceof ProviderRateLimitError) {
|
|
71
|
+
database.recordProviderKeyUsage({
|
|
72
|
+
id: createId("usage"),
|
|
73
|
+
keyId: input.lease.keyId,
|
|
74
|
+
jobId: input.jobId,
|
|
75
|
+
provider: input.provider,
|
|
76
|
+
model: input.model,
|
|
77
|
+
eventType: "rate_limit",
|
|
78
|
+
metadata: {
|
|
79
|
+
...(input.metadata ?? {}),
|
|
80
|
+
retryAfterSeconds: input.error.retryAfterSeconds
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
database.setProviderKeyCooldown(input.lease.keyId, addSeconds(new Date(), input.error.retryAfterSeconds));
|
|
84
|
+
}
|
|
85
|
+
else if (input.error instanceof ProviderAuthError) {
|
|
86
|
+
database.recordProviderKeyUsage({
|
|
87
|
+
id: createId("usage"),
|
|
88
|
+
keyId: input.lease.keyId,
|
|
89
|
+
jobId: input.jobId,
|
|
90
|
+
provider: input.provider,
|
|
91
|
+
model: input.model,
|
|
92
|
+
eventType: "auth_error",
|
|
93
|
+
metadata: input.metadata ?? {}
|
|
94
|
+
});
|
|
95
|
+
database.setProviderKeyCooldown(input.lease.keyId, null, "invalid", "auth_error");
|
|
96
|
+
}
|
|
97
|
+
throw input.error;
|
|
98
|
+
}
|
|
99
|
+
listAvailableProviders(customerId) {
|
|
100
|
+
const now = Date.now();
|
|
101
|
+
const providers = database.listProviderKeysWithSecrets(customerId)
|
|
102
|
+
.map((entry) => ({
|
|
103
|
+
provider: String(entry.provider),
|
|
104
|
+
status: String(entry.status),
|
|
105
|
+
cooldownUntil: entry.cooldown_until ? String(entry.cooldown_until) : null,
|
|
106
|
+
secret: readStoredProviderSecret(String(entry.secret))
|
|
107
|
+
}))
|
|
108
|
+
.filter((entry) => entry.status === "active")
|
|
109
|
+
.filter((entry) => !entry.cooldownUntil || Date.parse(entry.cooldownUntil) <= now)
|
|
110
|
+
.filter((entry) => entry.secret.trim().length > 0)
|
|
111
|
+
.map((entry) => entry.provider);
|
|
112
|
+
return [...new Set(providers)];
|
|
113
|
+
}
|
|
114
|
+
async validateProviderKey(input) {
|
|
115
|
+
const model = PROVIDER_KEY_VALIDATION_MODELS[input.provider];
|
|
116
|
+
if (!model) {
|
|
117
|
+
throw new Error(`Provider ${input.provider} does not support validation.`);
|
|
118
|
+
}
|
|
119
|
+
if (config.mockProviders) {
|
|
120
|
+
return { provider: input.provider, model, mock: true };
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
await this.callProvider({
|
|
124
|
+
provider: input.provider,
|
|
125
|
+
model,
|
|
126
|
+
prompt: "Reply with OK.",
|
|
127
|
+
temperature: 0,
|
|
128
|
+
apiKey: input.secret
|
|
129
|
+
});
|
|
130
|
+
return { provider: input.provider, model, mock: false };
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
throw new Error(toProviderValidationMessage(input.provider, error));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
37
136
|
async leaseCustomerKey(input) {
|
|
38
137
|
const leaseToken = createId("lease");
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
138
|
+
const waiterId = createId("wait");
|
|
139
|
+
const priority = input.priority ?? DEFAULT_PROVIDER_LEASE_PRIORITY;
|
|
140
|
+
const waitTimeoutMs = input.waitTimeoutMs ?? DEFAULT_PROVIDER_LEASE_TIMEOUT_MS;
|
|
141
|
+
const leaseDurationSeconds = input.leaseDurationSeconds ?? DEFAULT_PROVIDER_LEASE_SECONDS;
|
|
142
|
+
const deadline = Date.now() + waitTimeoutMs;
|
|
143
|
+
try {
|
|
144
|
+
while (Date.now() < deadline) {
|
|
145
|
+
const row = database.acquireProviderKeyLease({
|
|
146
|
+
customerId: input.customerId,
|
|
147
|
+
provider: input.provider,
|
|
148
|
+
waiterId,
|
|
149
|
+
workerId: input.workerId,
|
|
150
|
+
jobId: input.jobId,
|
|
151
|
+
priority,
|
|
152
|
+
leaseToken,
|
|
153
|
+
expiresAt: addSeconds(new Date(), leaseDurationSeconds),
|
|
154
|
+
waiterExpiresAt: addSeconds(new Date(), 10)
|
|
155
|
+
});
|
|
156
|
+
if (row) {
|
|
157
|
+
return {
|
|
158
|
+
keyId: row.keyId,
|
|
159
|
+
leaseToken,
|
|
160
|
+
provider: input.provider,
|
|
161
|
+
secret: readStoredProviderSecret(row.encryptedSecret)
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
await sleep(PROVIDER_LEASE_POLL_MS);
|
|
165
|
+
}
|
|
49
166
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
secret: readStoredProviderSecret(row.encryptedSecret)
|
|
55
|
-
};
|
|
167
|
+
finally {
|
|
168
|
+
database.deleteProviderKeyWaiter(waiterId);
|
|
169
|
+
}
|
|
170
|
+
throw new ProviderLeaseTimeoutError(`No eligible ${input.provider} key became available in time.`);
|
|
56
171
|
}
|
|
57
172
|
async releaseLease(lease) {
|
|
58
173
|
database.releaseProviderKeyLease({ keyId: lease.keyId, leaseToken: lease.leaseToken });
|
|
59
174
|
}
|
|
175
|
+
async leaseEditorChatKey(input) {
|
|
176
|
+
return this.leaseCustomerKey({
|
|
177
|
+
customerId: input.customerId,
|
|
178
|
+
provider: input.provider,
|
|
179
|
+
jobId: input.chatId,
|
|
180
|
+
workerId: `editor-chat:${input.chatId}`,
|
|
181
|
+
priority: CHAT_PROVIDER_LEASE_PRIORITY,
|
|
182
|
+
waitTimeoutMs: CHAT_PROVIDER_LEASE_TIMEOUT_MS,
|
|
183
|
+
leaseDurationSeconds: CHAT_PROVIDER_LEASE_SECONDS
|
|
184
|
+
});
|
|
185
|
+
}
|
|
60
186
|
async generateText(input) {
|
|
61
187
|
const lease = await this.leaseCustomerKey({
|
|
62
188
|
customerId: input.customerId,
|
|
@@ -81,7 +207,6 @@ export class ProviderService {
|
|
|
81
207
|
eventType: "success",
|
|
82
208
|
inputTokens: 180,
|
|
83
209
|
outputTokens: 120,
|
|
84
|
-
costUsd: 0.02,
|
|
85
210
|
metadata: { mock: true }
|
|
86
211
|
});
|
|
87
212
|
database.touchProviderKey(lease.keyId);
|
|
@@ -90,11 +215,11 @@ export class ProviderService {
|
|
|
90
215
|
usage: {
|
|
91
216
|
inputTokens: 180,
|
|
92
217
|
outputTokens: 120,
|
|
93
|
-
costUsd: 0
|
|
218
|
+
costUsd: 0
|
|
94
219
|
}
|
|
95
220
|
};
|
|
96
221
|
}
|
|
97
|
-
const result = await this.callProvider({ ...input, apiKey: lease.secret });
|
|
222
|
+
const result = zeroUsageCost(await this.callProvider({ ...input, apiKey: lease.secret }));
|
|
98
223
|
database.recordProviderKeyUsage({
|
|
99
224
|
id: createId("usage"),
|
|
100
225
|
keyId: lease.keyId,
|
|
@@ -103,37 +228,19 @@ export class ProviderService {
|
|
|
103
228
|
model: input.model,
|
|
104
229
|
eventType: "success",
|
|
105
230
|
inputTokens: result.usage.inputTokens,
|
|
106
|
-
outputTokens: result.usage.outputTokens
|
|
107
|
-
costUsd: result.usage.costUsd
|
|
231
|
+
outputTokens: result.usage.outputTokens
|
|
108
232
|
});
|
|
109
233
|
database.touchProviderKey(lease.keyId);
|
|
110
234
|
return result;
|
|
111
235
|
}
|
|
112
236
|
catch (error) {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
eventType: "rate_limit",
|
|
121
|
-
metadata: { retryAfterSeconds: error.retryAfterSeconds }
|
|
122
|
-
});
|
|
123
|
-
database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
|
|
124
|
-
}
|
|
125
|
-
else if (error instanceof ProviderAuthError) {
|
|
126
|
-
database.recordProviderKeyUsage({
|
|
127
|
-
id: createId("usage"),
|
|
128
|
-
keyId: lease.keyId,
|
|
129
|
-
jobId: input.jobId,
|
|
130
|
-
provider: input.provider,
|
|
131
|
-
model: input.model,
|
|
132
|
-
eventType: "auth_error"
|
|
133
|
-
});
|
|
134
|
-
database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
|
|
135
|
-
}
|
|
136
|
-
throw error;
|
|
237
|
+
this.handleLeasedProviderError({
|
|
238
|
+
error,
|
|
239
|
+
lease,
|
|
240
|
+
jobId: input.jobId,
|
|
241
|
+
provider: input.provider,
|
|
242
|
+
model: input.model
|
|
243
|
+
});
|
|
137
244
|
}
|
|
138
245
|
finally {
|
|
139
246
|
await this.releaseLease(lease);
|
|
@@ -159,7 +266,6 @@ export class ProviderService {
|
|
|
159
266
|
eventType: "success",
|
|
160
267
|
inputTokens: 0,
|
|
161
268
|
outputTokens: 0,
|
|
162
|
-
costUsd: 0.01,
|
|
163
269
|
metadata: { mock: true, type: "speech_generation" }
|
|
164
270
|
});
|
|
165
271
|
database.touchProviderKey(lease.keyId);
|
|
@@ -169,11 +275,11 @@ export class ProviderService {
|
|
|
169
275
|
usage: {
|
|
170
276
|
inputTokens: 0,
|
|
171
277
|
outputTokens: 0,
|
|
172
|
-
costUsd: 0
|
|
278
|
+
costUsd: 0
|
|
173
279
|
}
|
|
174
280
|
};
|
|
175
281
|
}
|
|
176
|
-
const result = input.provider === "gemini"
|
|
282
|
+
const result = zeroUsageCost(input.provider === "gemini"
|
|
177
283
|
? await this.callGeminiSpeechGeneration({
|
|
178
284
|
model: input.model,
|
|
179
285
|
text: input.text,
|
|
@@ -189,7 +295,7 @@ export class ProviderService {
|
|
|
189
295
|
instructions: input.instructions,
|
|
190
296
|
responseFormat: input.responseFormat,
|
|
191
297
|
apiKey: lease.secret
|
|
192
|
-
});
|
|
298
|
+
}));
|
|
193
299
|
database.recordProviderKeyUsage({
|
|
194
300
|
id: createId("usage"),
|
|
195
301
|
keyId: lease.keyId,
|
|
@@ -199,38 +305,20 @@ export class ProviderService {
|
|
|
199
305
|
eventType: "success",
|
|
200
306
|
inputTokens: result.usage.inputTokens,
|
|
201
307
|
outputTokens: result.usage.outputTokens,
|
|
202
|
-
costUsd: result.usage.costUsd,
|
|
203
308
|
metadata: { type: "speech_generation" }
|
|
204
309
|
});
|
|
205
310
|
database.touchProviderKey(lease.keyId);
|
|
206
311
|
return result;
|
|
207
312
|
}
|
|
208
313
|
catch (error) {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_generation" }
|
|
218
|
-
});
|
|
219
|
-
database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
|
|
220
|
-
}
|
|
221
|
-
else if (error instanceof ProviderAuthError) {
|
|
222
|
-
database.recordProviderKeyUsage({
|
|
223
|
-
id: createId("usage"),
|
|
224
|
-
keyId: lease.keyId,
|
|
225
|
-
jobId: input.jobId,
|
|
226
|
-
provider: input.provider,
|
|
227
|
-
model: input.model,
|
|
228
|
-
eventType: "auth_error",
|
|
229
|
-
metadata: { type: "speech_generation" }
|
|
230
|
-
});
|
|
231
|
-
database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
|
|
232
|
-
}
|
|
233
|
-
throw error;
|
|
314
|
+
this.handleLeasedProviderError({
|
|
315
|
+
error,
|
|
316
|
+
lease,
|
|
317
|
+
jobId: input.jobId,
|
|
318
|
+
provider: input.provider,
|
|
319
|
+
model: input.model,
|
|
320
|
+
metadata: { type: "speech_generation" }
|
|
321
|
+
});
|
|
234
322
|
}
|
|
235
323
|
finally {
|
|
236
324
|
await this.releaseLease(lease);
|
|
@@ -255,7 +343,6 @@ export class ProviderService {
|
|
|
255
343
|
eventType: "success",
|
|
256
344
|
inputTokens: 0,
|
|
257
345
|
outputTokens: 0,
|
|
258
|
-
costUsd: 0.04,
|
|
259
346
|
metadata: { mock: true, type: "image_generation" }
|
|
260
347
|
});
|
|
261
348
|
database.touchProviderKey(lease.keyId);
|
|
@@ -274,14 +361,103 @@ export class ProviderService {
|
|
|
274
361
|
imageSize: input.imageSize,
|
|
275
362
|
apiKey: lease.secret
|
|
276
363
|
})
|
|
277
|
-
:
|
|
364
|
+
: input.provider === "openrouter"
|
|
365
|
+
? await this.callOpenRouterImageGeneration({
|
|
366
|
+
model: input.model,
|
|
367
|
+
prompt: input.prompt,
|
|
368
|
+
promptAttachments: input.promptAttachments,
|
|
369
|
+
aspectRatio: input.aspectRatio,
|
|
370
|
+
imageSize: input.imageSize,
|
|
371
|
+
apiKey: lease.secret
|
|
372
|
+
})
|
|
373
|
+
: await this.callOpenAIImageGeneration({
|
|
374
|
+
model: input.model,
|
|
375
|
+
prompt: input.prompt,
|
|
376
|
+
promptAttachments: input.promptAttachments,
|
|
377
|
+
size: input.size,
|
|
378
|
+
aspectRatio: input.aspectRatio,
|
|
379
|
+
apiKey: lease.secret
|
|
380
|
+
});
|
|
381
|
+
database.recordProviderKeyUsage({
|
|
382
|
+
id: createId("usage"),
|
|
383
|
+
keyId: lease.keyId,
|
|
384
|
+
jobId: input.jobId,
|
|
385
|
+
provider: input.provider,
|
|
386
|
+
model: input.model,
|
|
387
|
+
eventType: "success",
|
|
388
|
+
inputTokens: 0,
|
|
389
|
+
outputTokens: 0,
|
|
390
|
+
metadata: { type: "image_generation" }
|
|
391
|
+
});
|
|
392
|
+
database.touchProviderKey(lease.keyId);
|
|
393
|
+
return result;
|
|
394
|
+
}
|
|
395
|
+
catch (error) {
|
|
396
|
+
this.handleLeasedProviderError({
|
|
397
|
+
error,
|
|
398
|
+
lease,
|
|
399
|
+
jobId: input.jobId,
|
|
400
|
+
provider: input.provider,
|
|
401
|
+
model: input.model,
|
|
402
|
+
metadata: { type: "image_generation" }
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
finally {
|
|
406
|
+
await this.releaseLease(lease);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
async editImage(input) {
|
|
410
|
+
const lease = await this.leaseCustomerKey({
|
|
411
|
+
customerId: input.customerId,
|
|
412
|
+
provider: input.provider,
|
|
413
|
+
jobId: input.jobId,
|
|
414
|
+
workerId: input.workerId
|
|
415
|
+
});
|
|
416
|
+
try {
|
|
417
|
+
if (config.mockProviders) {
|
|
418
|
+
const svg = buildMockSlideSvg(`EDIT ${input.instruction}\n${input.sourceImageUrl}`);
|
|
419
|
+
database.recordProviderKeyUsage({
|
|
420
|
+
id: createId("usage"),
|
|
421
|
+
keyId: lease.keyId,
|
|
422
|
+
jobId: input.jobId,
|
|
423
|
+
provider: input.provider,
|
|
278
424
|
model: input.model,
|
|
279
|
-
|
|
280
|
-
|
|
425
|
+
eventType: "success",
|
|
426
|
+
inputTokens: 0,
|
|
427
|
+
outputTokens: 0,
|
|
428
|
+
metadata: { mock: true, type: "image_edit" }
|
|
429
|
+
});
|
|
430
|
+
database.touchProviderKey(lease.keyId);
|
|
431
|
+
return {
|
|
432
|
+
bytes: Buffer.from(svg, "utf8"),
|
|
433
|
+
contentType: "image/svg+xml",
|
|
434
|
+
revisedPrompt: input.instruction
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
const result = input.provider === "openai"
|
|
438
|
+
? await this.callOpenAIImageEdit({
|
|
439
|
+
model: input.model,
|
|
440
|
+
sourceImageUrl: input.sourceImageUrl,
|
|
441
|
+
instruction: input.instruction,
|
|
442
|
+
maskUrl: input.maskUrl,
|
|
281
443
|
size: input.size,
|
|
282
|
-
aspectRatio: input.aspectRatio,
|
|
283
444
|
apiKey: lease.secret
|
|
284
|
-
})
|
|
445
|
+
})
|
|
446
|
+
: input.provider === "perplexity"
|
|
447
|
+
? (() => {
|
|
448
|
+
throw new Error("Perplexity image edit is not supported.");
|
|
449
|
+
})()
|
|
450
|
+
: await this.callAttachmentDrivenImageEdit({
|
|
451
|
+
provider: input.provider,
|
|
452
|
+
model: input.model,
|
|
453
|
+
sourceImageUrl: input.sourceImageUrl,
|
|
454
|
+
instruction: input.instruction,
|
|
455
|
+
referenceAttachments: input.referenceAttachments,
|
|
456
|
+
maskUrl: input.maskUrl,
|
|
457
|
+
aspectRatio: input.aspectRatio,
|
|
458
|
+
imageSize: input.imageSize,
|
|
459
|
+
apiKey: lease.secret
|
|
460
|
+
});
|
|
285
461
|
database.recordProviderKeyUsage({
|
|
286
462
|
id: createId("usage"),
|
|
287
463
|
keyId: lease.keyId,
|
|
@@ -291,16 +467,31 @@ export class ProviderService {
|
|
|
291
467
|
eventType: "success",
|
|
292
468
|
inputTokens: 0,
|
|
293
469
|
outputTokens: 0,
|
|
294
|
-
|
|
295
|
-
metadata: { type: "image_generation" }
|
|
470
|
+
metadata: { type: input.maskUrl ? "image_inpaint" : "image_edit" }
|
|
296
471
|
});
|
|
297
472
|
database.touchProviderKey(lease.keyId);
|
|
298
473
|
return result;
|
|
299
474
|
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
this.handleLeasedProviderError({
|
|
477
|
+
error,
|
|
478
|
+
lease,
|
|
479
|
+
jobId: input.jobId,
|
|
480
|
+
provider: input.provider,
|
|
481
|
+
model: input.model,
|
|
482
|
+
metadata: { type: input.maskUrl ? "image_inpaint" : "image_edit" }
|
|
483
|
+
});
|
|
484
|
+
}
|
|
300
485
|
finally {
|
|
301
486
|
await this.releaseLease(lease);
|
|
302
487
|
}
|
|
303
488
|
}
|
|
489
|
+
async inpaintImage(input) {
|
|
490
|
+
return this.editImage({
|
|
491
|
+
...input,
|
|
492
|
+
maskUrl: input.maskUrl
|
|
493
|
+
});
|
|
494
|
+
}
|
|
304
495
|
async analyzeImageLayout(input) {
|
|
305
496
|
const lease = await this.leaseCustomerKey({
|
|
306
497
|
customerId: input.customerId,
|
|
@@ -319,7 +510,6 @@ export class ProviderService {
|
|
|
319
510
|
eventType: "success",
|
|
320
511
|
inputTokens: 0,
|
|
321
512
|
outputTokens: 0,
|
|
322
|
-
costUsd: 0.005,
|
|
323
513
|
metadata: { mock: true, type: "layout_analysis" }
|
|
324
514
|
});
|
|
325
515
|
database.touchProviderKey(lease.keyId);
|
|
@@ -357,7 +547,6 @@ export class ProviderService {
|
|
|
357
547
|
eventType: "success",
|
|
358
548
|
inputTokens: 0,
|
|
359
549
|
outputTokens: 0,
|
|
360
|
-
costUsd: 0.005,
|
|
361
550
|
metadata: { type: "layout_analysis" }
|
|
362
551
|
});
|
|
363
552
|
database.touchProviderKey(lease.keyId);
|
|
@@ -368,6 +557,16 @@ export class ProviderService {
|
|
|
368
557
|
justification: parsed.justification ?? "Selected to preserve legibility and subject framing."
|
|
369
558
|
};
|
|
370
559
|
}
|
|
560
|
+
catch (error) {
|
|
561
|
+
this.handleLeasedProviderError({
|
|
562
|
+
error,
|
|
563
|
+
lease,
|
|
564
|
+
jobId: input.jobId,
|
|
565
|
+
provider: input.provider,
|
|
566
|
+
model: input.model,
|
|
567
|
+
metadata: { type: "layout_analysis" }
|
|
568
|
+
});
|
|
569
|
+
}
|
|
371
570
|
finally {
|
|
372
571
|
await this.releaseLease(lease);
|
|
373
572
|
}
|
|
@@ -391,7 +590,6 @@ export class ProviderService {
|
|
|
391
590
|
eventType: "success",
|
|
392
591
|
inputTokens: 0,
|
|
393
592
|
outputTokens: 0,
|
|
394
|
-
costUsd: 0.01,
|
|
395
593
|
metadata: { mock: true, type: "speech_transcription" }
|
|
396
594
|
});
|
|
397
595
|
database.touchProviderKey(lease.keyId);
|
|
@@ -400,11 +598,11 @@ export class ProviderService {
|
|
|
400
598
|
usage: {
|
|
401
599
|
inputTokens: 0,
|
|
402
600
|
outputTokens: 0,
|
|
403
|
-
costUsd: 0
|
|
601
|
+
costUsd: 0
|
|
404
602
|
}
|
|
405
603
|
};
|
|
406
604
|
}
|
|
407
|
-
const result = input.provider === "gemini"
|
|
605
|
+
const result = zeroUsageCost(input.provider === "gemini"
|
|
408
606
|
? await this.callGeminiSpeechTranscription({
|
|
409
607
|
model: input.model,
|
|
410
608
|
audio: input.audio,
|
|
@@ -429,7 +627,7 @@ export class ProviderService {
|
|
|
429
627
|
prompt: input.prompt,
|
|
430
628
|
language: input.language,
|
|
431
629
|
apiKey: lease.secret
|
|
432
|
-
});
|
|
630
|
+
}));
|
|
433
631
|
database.recordProviderKeyUsage({
|
|
434
632
|
id: createId("usage"),
|
|
435
633
|
keyId: lease.keyId,
|
|
@@ -439,38 +637,20 @@ export class ProviderService {
|
|
|
439
637
|
eventType: "success",
|
|
440
638
|
inputTokens: result.usage.inputTokens,
|
|
441
639
|
outputTokens: result.usage.outputTokens,
|
|
442
|
-
costUsd: result.usage.costUsd,
|
|
443
640
|
metadata: { type: "speech_transcription" }
|
|
444
641
|
});
|
|
445
642
|
database.touchProviderKey(lease.keyId);
|
|
446
643
|
return result;
|
|
447
644
|
}
|
|
448
645
|
catch (error) {
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_transcription" }
|
|
458
|
-
});
|
|
459
|
-
database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
|
|
460
|
-
}
|
|
461
|
-
else if (error instanceof ProviderAuthError) {
|
|
462
|
-
database.recordProviderKeyUsage({
|
|
463
|
-
id: createId("usage"),
|
|
464
|
-
keyId: lease.keyId,
|
|
465
|
-
jobId: input.jobId,
|
|
466
|
-
provider: input.provider,
|
|
467
|
-
model: input.model,
|
|
468
|
-
eventType: "auth_error",
|
|
469
|
-
metadata: { type: "speech_transcription" }
|
|
470
|
-
});
|
|
471
|
-
database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
|
|
472
|
-
}
|
|
473
|
-
throw error;
|
|
646
|
+
this.handleLeasedProviderError({
|
|
647
|
+
error,
|
|
648
|
+
lease,
|
|
649
|
+
jobId: input.jobId,
|
|
650
|
+
provider: input.provider,
|
|
651
|
+
model: input.model,
|
|
652
|
+
metadata: { type: "speech_transcription" }
|
|
653
|
+
});
|
|
474
654
|
}
|
|
475
655
|
finally {
|
|
476
656
|
await this.releaseLease(lease);
|
|
@@ -519,6 +699,77 @@ export class ProviderService {
|
|
|
519
699
|
revisedPrompt: data.data?.[0]?.revised_prompt ?? null
|
|
520
700
|
};
|
|
521
701
|
}
|
|
702
|
+
async callOpenAIImageEdit(input) {
|
|
703
|
+
const form = new FormData();
|
|
704
|
+
form.set("model", input.model);
|
|
705
|
+
form.set("prompt", input.instruction);
|
|
706
|
+
form.set("size", input.size ?? "1024x1024");
|
|
707
|
+
form.set("image", await remoteImageToBlob(input.sourceImageUrl));
|
|
708
|
+
if (input.maskUrl) {
|
|
709
|
+
form.set("mask", await remoteImageToBlob(input.maskUrl));
|
|
710
|
+
}
|
|
711
|
+
const response = await fetch("https://api.openai.com/v1/images/edits", {
|
|
712
|
+
method: "POST",
|
|
713
|
+
headers: {
|
|
714
|
+
Authorization: `Bearer ${input.apiKey}`
|
|
715
|
+
},
|
|
716
|
+
body: form
|
|
717
|
+
});
|
|
718
|
+
if (response.status === 401) {
|
|
719
|
+
throw new ProviderAuthError("openai authentication failed");
|
|
720
|
+
}
|
|
721
|
+
if (response.status === 429) {
|
|
722
|
+
throw new ProviderRateLimitError("openai rate limited");
|
|
723
|
+
}
|
|
724
|
+
if (!response.ok) {
|
|
725
|
+
const details = await response.text();
|
|
726
|
+
throw new Error(`openai image edit returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
727
|
+
}
|
|
728
|
+
const data = await response.json();
|
|
729
|
+
const encoded = data.data?.[0]?.b64_json;
|
|
730
|
+
if (!encoded) {
|
|
731
|
+
throw new Error("openai image edit returned no image payload");
|
|
732
|
+
}
|
|
733
|
+
return {
|
|
734
|
+
bytes: Buffer.from(encoded, "base64"),
|
|
735
|
+
contentType: "image/png",
|
|
736
|
+
revisedPrompt: data.data?.[0]?.revised_prompt ?? null
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
async callAttachmentDrivenImageEdit(input) {
|
|
740
|
+
const promptAttachments = [
|
|
741
|
+
input.sourceImageUrl,
|
|
742
|
+
...(input.maskUrl ? [input.maskUrl] : []),
|
|
743
|
+
...(input.referenceAttachments ?? [])
|
|
744
|
+
];
|
|
745
|
+
const prompt = [
|
|
746
|
+
input.maskUrl
|
|
747
|
+
? "Edit this image using the provided source image and mask."
|
|
748
|
+
: "Edit this image using the provided source image as the base reference.",
|
|
749
|
+
input.maskUrl
|
|
750
|
+
? "Only change the masked region. Preserve all unmasked regions exactly."
|
|
751
|
+
: "Preserve the overall composition unless the instruction explicitly changes it.",
|
|
752
|
+
`Instruction: ${input.instruction}`,
|
|
753
|
+
"Return one final edited image only."
|
|
754
|
+
].join("\n");
|
|
755
|
+
return input.provider === "gemini"
|
|
756
|
+
? this.callGeminiImageGeneration({
|
|
757
|
+
model: input.model,
|
|
758
|
+
prompt,
|
|
759
|
+
promptAttachments,
|
|
760
|
+
aspectRatio: input.aspectRatio,
|
|
761
|
+
imageSize: input.imageSize,
|
|
762
|
+
apiKey: input.apiKey
|
|
763
|
+
})
|
|
764
|
+
: this.callOpenRouterImageGeneration({
|
|
765
|
+
model: input.model,
|
|
766
|
+
prompt,
|
|
767
|
+
promptAttachments,
|
|
768
|
+
aspectRatio: input.aspectRatio,
|
|
769
|
+
imageSize: input.imageSize,
|
|
770
|
+
apiKey: input.apiKey
|
|
771
|
+
});
|
|
772
|
+
}
|
|
522
773
|
async callGeminiImageGeneration(input) {
|
|
523
774
|
const imageConfig = {};
|
|
524
775
|
if (input.aspectRatio) {
|
|
@@ -566,8 +817,76 @@ export class ProviderService {
|
|
|
566
817
|
revisedPrompt: null
|
|
567
818
|
};
|
|
568
819
|
}
|
|
820
|
+
async callOpenRouterImageGeneration(input) {
|
|
821
|
+
const content = [
|
|
822
|
+
{ type: "text", text: input.prompt },
|
|
823
|
+
...(input.promptAttachments ?? []).map((url) => ({
|
|
824
|
+
type: "image_url",
|
|
825
|
+
image_url: { url }
|
|
826
|
+
}))
|
|
827
|
+
];
|
|
828
|
+
const imageConfig = {};
|
|
829
|
+
if (input.aspectRatio) {
|
|
830
|
+
imageConfig.aspect_ratio = input.aspectRatio;
|
|
831
|
+
}
|
|
832
|
+
if (input.imageSize) {
|
|
833
|
+
imageConfig.image_size = input.imageSize;
|
|
834
|
+
}
|
|
835
|
+
const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
|
836
|
+
method: "POST",
|
|
837
|
+
headers: {
|
|
838
|
+
"Content-Type": "application/json",
|
|
839
|
+
Authorization: `Bearer ${input.apiKey}`
|
|
840
|
+
},
|
|
841
|
+
body: JSON.stringify({
|
|
842
|
+
model: input.model,
|
|
843
|
+
messages: [
|
|
844
|
+
{
|
|
845
|
+
role: "user",
|
|
846
|
+
content
|
|
847
|
+
}
|
|
848
|
+
],
|
|
849
|
+
modalities: ["image", "text"],
|
|
850
|
+
stream: false,
|
|
851
|
+
...(Object.keys(imageConfig).length
|
|
852
|
+
? {
|
|
853
|
+
image_config: imageConfig
|
|
854
|
+
}
|
|
855
|
+
: {})
|
|
856
|
+
})
|
|
857
|
+
});
|
|
858
|
+
if (response.status === 401 || response.status === 403) {
|
|
859
|
+
throw new ProviderAuthError("openrouter authentication failed");
|
|
860
|
+
}
|
|
861
|
+
if (response.status === 429) {
|
|
862
|
+
throw new ProviderRateLimitError("openrouter rate limited");
|
|
863
|
+
}
|
|
864
|
+
if (!response.ok) {
|
|
865
|
+
const details = await response.text();
|
|
866
|
+
throw new Error(`openrouter image generation returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
867
|
+
}
|
|
868
|
+
const data = await response.json();
|
|
869
|
+
const imageUrl = data.choices?.[0]?.message?.images?.[0]?.image_url?.url ??
|
|
870
|
+
data.choices?.[0]?.message?.images?.[0]?.imageUrl?.url;
|
|
871
|
+
if (!imageUrl) {
|
|
872
|
+
throw new Error("openrouter image generation returned no image payload");
|
|
873
|
+
}
|
|
874
|
+
const parsed = decodeInlineDataUrl(imageUrl);
|
|
875
|
+
if (!parsed) {
|
|
876
|
+
throw new Error("openrouter image generation returned an unsupported image payload");
|
|
877
|
+
}
|
|
878
|
+
return {
|
|
879
|
+
bytes: parsed.bytes,
|
|
880
|
+
contentType: parsed.contentType,
|
|
881
|
+
revisedPrompt: data.choices?.[0]?.message?.content ?? null
|
|
882
|
+
};
|
|
883
|
+
}
|
|
569
884
|
async callOpenAICompatibleSpeechGeneration(input) {
|
|
570
|
-
const
|
|
885
|
+
const requestedFormat = input.responseFormat ?? "mp3";
|
|
886
|
+
const usePcmBridge = input.provider === "openrouter" &&
|
|
887
|
+
/gemini/i.test(input.model) &&
|
|
888
|
+
requestedFormat !== "mp3";
|
|
889
|
+
const responseFormat = usePcmBridge ? "pcm" : requestedFormat;
|
|
571
890
|
const endpoint = input.provider === "openrouter"
|
|
572
891
|
? "https://openrouter.ai/api/v1/audio/speech"
|
|
573
892
|
: "https://api.openai.com/v1/audio/speech";
|
|
@@ -595,9 +914,14 @@ export class ProviderService {
|
|
|
595
914
|
const details = await response.text();
|
|
596
915
|
throw new Error(`${input.provider} speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
|
|
597
916
|
}
|
|
917
|
+
const rawBytes = Buffer.from(await response.arrayBuffer());
|
|
918
|
+
const bytes = usePcmBridge ? pcm16MonoToWav(rawBytes, 24000) : rawBytes;
|
|
919
|
+
const contentType = usePcmBridge
|
|
920
|
+
? "audio/wav"
|
|
921
|
+
: response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat);
|
|
598
922
|
return {
|
|
599
|
-
bytes
|
|
600
|
-
contentType
|
|
923
|
+
bytes,
|
|
924
|
+
contentType,
|
|
601
925
|
usage: {
|
|
602
926
|
inputTokens: 0,
|
|
603
927
|
outputTokens: 0,
|
|
@@ -644,9 +968,24 @@ export class ProviderService {
|
|
|
644
968
|
if (!encoded) {
|
|
645
969
|
throw new Error("gemini speech generation returned no audio payload");
|
|
646
970
|
}
|
|
971
|
+
const rawBytes = Buffer.from(encoded, "base64");
|
|
972
|
+
const rawContentType = String(part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000");
|
|
973
|
+
const pcmRateMatch = /audio\/l16;?\s*rate=(\d+)/i.exec(rawContentType);
|
|
974
|
+
const looksLikeWav = isWavBuffer(rawBytes);
|
|
975
|
+
const looksLikeMp3 = isLikelyMp3Buffer(rawBytes);
|
|
976
|
+
const shouldWrapPcm = Boolean(pcmRateMatch ||
|
|
977
|
+
(!looksLikeWav && !looksLikeMp3));
|
|
978
|
+
const bytes = shouldWrapPcm
|
|
979
|
+
? pcm16MonoToWav(rawBytes, Number(pcmRateMatch?.[1] || "24000"))
|
|
980
|
+
: rawBytes;
|
|
981
|
+
const contentType = shouldWrapPcm
|
|
982
|
+
? "audio/wav"
|
|
983
|
+
: looksLikeWav
|
|
984
|
+
? "audio/wav"
|
|
985
|
+
: "audio/mpeg";
|
|
647
986
|
return {
|
|
648
|
-
bytes
|
|
649
|
-
contentType
|
|
987
|
+
bytes,
|
|
988
|
+
contentType,
|
|
650
989
|
usage: {
|
|
651
990
|
inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
|
|
652
991
|
outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
|
|
@@ -964,15 +1303,80 @@ export class ProviderService {
|
|
|
964
1303
|
function clamp(value, min, max) {
|
|
965
1304
|
return Math.max(min, Math.min(max, value));
|
|
966
1305
|
}
|
|
1306
|
+
function summarizeProviderValidationError(error) {
|
|
1307
|
+
const rawMessage = error instanceof Error ? error.message : "Unknown provider error";
|
|
1308
|
+
return rawMessage
|
|
1309
|
+
.replace(/\s+/g, " ")
|
|
1310
|
+
.trim()
|
|
1311
|
+
.slice(0, 280);
|
|
1312
|
+
}
|
|
1313
|
+
function toProviderValidationMessage(provider, error) {
|
|
1314
|
+
const providerLabel = formatProviderLabel(provider);
|
|
1315
|
+
const summary = summarizeProviderValidationError(error);
|
|
1316
|
+
if (!summary) {
|
|
1317
|
+
return `Unable to verify ${providerLabel} API key.`;
|
|
1318
|
+
}
|
|
1319
|
+
if (/401|403|invalid api key|api key not valid|incorrect api key|unauthorized|permission denied|authentication failed/i.test(summary)) {
|
|
1320
|
+
return `${providerLabel} API key was rejected by the provider. ${summary}`;
|
|
1321
|
+
}
|
|
1322
|
+
return `Unable to verify ${providerLabel} API key. ${summary}`;
|
|
1323
|
+
}
|
|
1324
|
+
function formatProviderLabel(provider) {
|
|
1325
|
+
switch (provider) {
|
|
1326
|
+
case "openai":
|
|
1327
|
+
return "OpenAI";
|
|
1328
|
+
case "gemini":
|
|
1329
|
+
return "Gemini";
|
|
1330
|
+
case "openrouter":
|
|
1331
|
+
return "OpenRouter";
|
|
1332
|
+
case "perplexity":
|
|
1333
|
+
return "Perplexity";
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
967
1336
|
function assertSupportedSpeechModel(kind, provider, model) {
|
|
968
|
-
const
|
|
969
|
-
if (!
|
|
970
|
-
throw new Error(`
|
|
1337
|
+
const normalizedModel = model.trim();
|
|
1338
|
+
if (!normalizedModel) {
|
|
1339
|
+
throw new Error(`Missing ${kind.toUpperCase()} model for provider ${provider}.`);
|
|
1340
|
+
}
|
|
1341
|
+
const speechCapableProviders = ["openai", "gemini", "openrouter"];
|
|
1342
|
+
if (!speechCapableProviders.includes(provider)) {
|
|
1343
|
+
throw new Error(`Provider ${provider} does not support ${kind.toUpperCase()} requests.`);
|
|
971
1344
|
}
|
|
972
1345
|
}
|
|
973
1346
|
function inferSpeechContentType(format) {
|
|
974
1347
|
return format === "wav" ? "audio/wav" : "audio/mpeg";
|
|
975
1348
|
}
|
|
1349
|
+
function pcm16MonoToWav(pcmBuffer, sampleRate) {
|
|
1350
|
+
const header = Buffer.alloc(44);
|
|
1351
|
+
header.write("RIFF", 0, 4, "ascii");
|
|
1352
|
+
header.writeUInt32LE(36 + pcmBuffer.byteLength, 4);
|
|
1353
|
+
header.write("WAVE", 8, 4, "ascii");
|
|
1354
|
+
header.write("fmt ", 12, 4, "ascii");
|
|
1355
|
+
header.writeUInt32LE(16, 16);
|
|
1356
|
+
header.writeUInt16LE(1, 20);
|
|
1357
|
+
header.writeUInt16LE(1, 22);
|
|
1358
|
+
header.writeUInt32LE(sampleRate, 24);
|
|
1359
|
+
header.writeUInt32LE(sampleRate * 2, 28);
|
|
1360
|
+
header.writeUInt16LE(2, 32);
|
|
1361
|
+
header.writeUInt16LE(16, 34);
|
|
1362
|
+
header.write("data", 36, 4, "ascii");
|
|
1363
|
+
header.writeUInt32LE(pcmBuffer.byteLength, 40);
|
|
1364
|
+
return Buffer.concat([header, pcmBuffer]);
|
|
1365
|
+
}
|
|
1366
|
+
function isWavBuffer(buffer) {
|
|
1367
|
+
return buffer.byteLength >= 12
|
|
1368
|
+
&& buffer.subarray(0, 4).toString("ascii") === "RIFF"
|
|
1369
|
+
&& buffer.subarray(8, 12).toString("ascii") === "WAVE";
|
|
1370
|
+
}
|
|
1371
|
+
function isLikelyMp3Buffer(buffer) {
|
|
1372
|
+
if (buffer.byteLength < 3) {
|
|
1373
|
+
return false;
|
|
1374
|
+
}
|
|
1375
|
+
if (buffer.subarray(0, 3).toString("ascii") === "ID3") {
|
|
1376
|
+
return true;
|
|
1377
|
+
}
|
|
1378
|
+
return buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0;
|
|
1379
|
+
}
|
|
976
1380
|
function audioFormatFromContentType(contentType) {
|
|
977
1381
|
const normalized = contentType?.split(";")[0]?.trim().toLowerCase();
|
|
978
1382
|
switch (normalized) {
|
|
@@ -1016,21 +1420,26 @@ function extractGeminiText(data) {
|
|
|
1016
1420
|
function supportsGeminiImageSize(model) {
|
|
1017
1421
|
return model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview";
|
|
1018
1422
|
}
|
|
1423
|
+
async function fetchRemoteBytes(input) {
|
|
1424
|
+
const response = await fetch(input.url);
|
|
1425
|
+
if (!response.ok) {
|
|
1426
|
+
throw new Error(`Could not fetch remote media ${input.url}: ${response.status}`);
|
|
1427
|
+
}
|
|
1428
|
+
const mimeType = response.headers.get("content-type")?.split(";")[0]?.trim() || input.fallbackMimeType || "application/octet-stream";
|
|
1429
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
1430
|
+
return { bytes, mimeType };
|
|
1431
|
+
}
|
|
1432
|
+
async function remoteImageToBlob(url) {
|
|
1433
|
+
const { bytes, mimeType } = await fetchRemoteBytes({ url, fallbackMimeType: "image/png" });
|
|
1434
|
+
return new Blob([bytes], { type: mimeType });
|
|
1435
|
+
}
|
|
1019
1436
|
async function buildGeminiPromptParts(prompt, attachments) {
|
|
1020
1437
|
const parts = [];
|
|
1021
1438
|
for (const attachment of attachments) {
|
|
1022
1439
|
if (!/^https?:\/\//i.test(attachment)) {
|
|
1023
1440
|
throw new Error(`Gemini prompt attachments must be public URLs: ${attachment}`);
|
|
1024
1441
|
}
|
|
1025
|
-
const
|
|
1026
|
-
if (!response.ok) {
|
|
1027
|
-
throw new Error(`Could not fetch Gemini prompt attachment ${attachment}: ${response.status}`);
|
|
1028
|
-
}
|
|
1029
|
-
const mimeType = response.headers.get("content-type")?.split(";")[0]?.trim();
|
|
1030
|
-
if (!mimeType) {
|
|
1031
|
-
throw new Error(`Gemini prompt attachment did not return a content type: ${attachment}`);
|
|
1032
|
-
}
|
|
1033
|
-
const bytes = Buffer.from(await response.arrayBuffer());
|
|
1442
|
+
const { bytes, mimeType } = await fetchRemoteBytes({ url: attachment });
|
|
1034
1443
|
parts.push({
|
|
1035
1444
|
inlineData: {
|
|
1036
1445
|
mimeType,
|
|
@@ -1041,6 +1450,16 @@ async function buildGeminiPromptParts(prompt, attachments) {
|
|
|
1041
1450
|
parts.push({ text: prompt });
|
|
1042
1451
|
return parts;
|
|
1043
1452
|
}
|
|
1453
|
+
function decodeInlineDataUrl(value) {
|
|
1454
|
+
const match = value.match(/^data:([^;,]+);base64,(.+)$/);
|
|
1455
|
+
if (!match) {
|
|
1456
|
+
return null;
|
|
1457
|
+
}
|
|
1458
|
+
return {
|
|
1459
|
+
contentType: match[1],
|
|
1460
|
+
bytes: Buffer.from(match[2], "base64")
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1044
1463
|
function buildMockSlideSvg(prompt) {
|
|
1045
1464
|
const escaped = escapeXml(prompt.slice(0, 220));
|
|
1046
1465
|
return `
|