@mevdragon/vidfarm-devcli 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/.env.example +9 -0
  2. package/GETTING_STARTED.developers.md +1 -26
  3. package/README.md +3 -0
  4. package/SKILL.developer.md +419 -8
  5. package/auto-create-templates/AUTO_CREATE_TEMPLATES.md +475 -0
  6. package/auto-create-templates/extractor-system-prompt.md +113 -0
  7. package/auto-create-templates/production-graph.schema.json +276 -0
  8. package/auto-create-templates/runbook.md +140 -0
  9. package/auto-create-templates/template-plan.schema.json +230 -0
  10. package/auto-create-templates/template-planner-prompt.md +75 -0
  11. package/dist/src/account-pages-legacy.js +9064 -0
  12. package/dist/src/account-pages.js +41 -620
  13. package/dist/src/app.js +6627 -244
  14. package/dist/src/cli.js +386 -26
  15. package/dist/src/config.js +12 -2
  16. package/dist/src/context.js +35 -35
  17. package/dist/src/db.js +1016 -53
  18. package/dist/src/dev-app-legacy.js +693 -0
  19. package/dist/src/dev-app.js +4 -904
  20. package/dist/src/domain.js +1 -1
  21. package/dist/src/editor-chat-history.js +72 -0
  22. package/dist/src/editor-chat.js +202 -0
  23. package/dist/src/frontend/flockposter-cache-store.js +116 -0
  24. package/dist/src/frontend/homepage-client.js +114 -0
  25. package/dist/src/frontend/homepage-shared.js +3 -0
  26. package/dist/src/frontend/homepage-store.js +27 -0
  27. package/dist/src/frontend/homepage-view.js +147 -0
  28. package/dist/src/frontend/page-runtime-client.js +100 -0
  29. package/dist/src/frontend/page-runtime-store.js +8 -0
  30. package/dist/src/frontend/template-editor-chat.js +2261 -0
  31. package/dist/src/homepage.js +695 -371
  32. package/dist/src/lib/template-style-options.js +95 -15
  33. package/dist/src/page-runtime.js +1 -0
  34. package/dist/src/page-shell.js +941 -0
  35. package/dist/src/primitive-context.js +163 -0
  36. package/dist/src/primitive-registry.js +340 -0
  37. package/dist/src/primitive-sdk.js +3 -0
  38. package/dist/src/primitives/remotion/html-image.js +28 -0
  39. package/dist/src/react-page-shell.js +34 -0
  40. package/dist/src/ready-post-schedule-component.js +1514 -0
  41. package/dist/src/registry.js +36 -17
  42. package/dist/src/runtime.js +6 -1
  43. package/dist/src/services/api-call-history.js +245 -0
  44. package/dist/src/services/auth.js +25 -3
  45. package/dist/src/services/billing.js +248 -9
  46. package/dist/src/services/chat-threads.js +88 -0
  47. package/dist/src/services/job-logs.js +14 -7
  48. package/dist/src/services/jobs.js +13 -2
  49. package/dist/src/services/providers.js +552 -133
  50. package/dist/src/services/rate-limits.js +236 -0
  51. package/dist/src/services/remotion.js +143 -16
  52. package/dist/src/services/storage.js +10 -8
  53. package/dist/src/services/template-certification.js +11 -2
  54. package/dist/src/services/template-loader.js +3 -1
  55. package/dist/src/services/template-sources.js +68 -3
  56. package/dist/src/template-editor-pages.js +2309 -0
  57. package/dist/src/template-editor-shell.js +1507 -0
  58. package/dist/src/worker.js +120 -22
  59. package/package.json +27 -4
  60. package/public/assets/homepage-app.js +54 -0
  61. package/public/assets/homepage-client-app.js +54 -0
  62. package/public/assets/page-runtime-client-app.js +68 -0
  63. package/dist/templates/vidfarm_template_0000/src/lib/images.js +0 -202
  64. package/dist/templates/vidfarm_template_0000/src/remotion/Root.js +0 -34
  65. package/dist/templates/vidfarm_template_0000/src/remotion/index.js +0 -3
  66. package/dist/templates/vidfarm_template_0000/src/sdk.js +0 -3
  67. package/dist/templates/vidfarm_template_0000/src/style-options.js +0 -51
  68. package/dist/templates/vidfarm_template_0000/src/template-dna.js +0 -9
  69. package/dist/templates/vidfarm_template_0000/src/template.js +0 -1447
  70. package/templates/vidfarm_template_0000/README.md +0 -100
  71. package/templates/vidfarm_template_0000/SKILL.md +0 -225
  72. package/templates/vidfarm_template_0000/assets/Abel-Regular.ttf +0 -0
  73. package/templates/vidfarm_template_0000/assets/DMSerifDisplay-Regular.ttf +0 -0
  74. package/templates/vidfarm_template_0000/assets/Montserrat[wght].ttf +0 -0
  75. package/templates/vidfarm_template_0000/assets/SourceCodePro[wght].ttf +0 -0
  76. package/templates/vidfarm_template_0000/assets/TikTokSans-SemiBold.ttf +0 -0
  77. package/templates/vidfarm_template_0000/assets/Yesteryear-Regular.ttf +0 -0
  78. package/templates/vidfarm_template_0000/composition.json +0 -11
  79. package/templates/vidfarm_template_0000/package.json +0 -27
  80. package/templates/vidfarm_template_0000/research/preview/.gitkeep +0 -1
  81. package/templates/vidfarm_template_0000/research/source_notes.md +0 -7
  82. package/templates/vidfarm_template_0000/src/lib/images.js +0 -202
  83. package/templates/vidfarm_template_0000/src/lib/images.ts +0 -241
  84. package/templates/vidfarm_template_0000/src/remotion/Root.js +0 -33
  85. package/templates/vidfarm_template_0000/src/remotion/Root.tsx +0 -75
  86. package/templates/vidfarm_template_0000/src/remotion/index.js +0 -3
  87. package/templates/vidfarm_template_0000/src/remotion/index.tsx +0 -4
  88. package/templates/vidfarm_template_0000/src/sdk.js +0 -3
  89. package/templates/vidfarm_template_0000/src/sdk.ts +0 -140
  90. package/templates/vidfarm_template_0000/src/style-options.js +0 -51
  91. package/templates/vidfarm_template_0000/src/style-options.ts +0 -60
  92. package/templates/vidfarm_template_0000/src/template-dna.js +0 -9
  93. package/templates/vidfarm_template_0000/src/template-dna.ts +0 -15
  94. package/templates/vidfarm_template_0000/src/template.js +0 -1447
  95. package/templates/vidfarm_template_0000/src/template.ts +0 -2049
  96. package/templates/vidfarm_template_0000/template.config.json +0 -21
  97. package/templates/vidfarm_template_0000/tmp/solobacterium-moorei-slideshow.request.json +0 -31
  98. package/templates/vidfarm_template_0000/tsconfig.json +0 -19
@@ -8,6 +8,13 @@ export class ProviderKeyUnavailableError extends Error {
8
8
  }
9
9
  export class ProviderAuthError extends Error {
10
10
  }
11
+ export class ProviderLeaseTimeoutError extends Error {
12
+ retryAfterSeconds;
13
+ constructor(message, retryAfterSeconds = 5) {
14
+ super(message);
15
+ this.retryAfterSeconds = retryAfterSeconds;
16
+ }
17
+ }
11
18
  export class ProviderRateLimitError extends Error {
12
19
  retryAfterSeconds;
13
20
  constructor(message, retryAfterSeconds = 60) {
@@ -17,7 +24,7 @@ export class ProviderRateLimitError extends Error {
17
24
  }
18
25
  const SUPPORTED_TTS_MODELS = {
19
26
  openrouter: new Set(["google/gemini-3.1-flash-tts-preview"]),
20
- gemini: new Set(["gemini-3.1-flash-tts-preview"]),
27
+ gemini: new Set(["gemini-3.1-flash-tts-preview", "gemini-2.5-flash-preview-tts"]),
21
28
  openai: new Set(["gpt-4o-mini-tts-2025-12-15"])
22
29
  };
23
30
  const SUPPORTED_STT_MODELS = {
@@ -25,6 +32,12 @@ const SUPPORTED_STT_MODELS = {
25
32
  gemini: new Set(["gemini-3.1-flash-lite-preview", "gemini-2.5-flash-lite"]),
26
33
  openai: new Set(["gpt-4o-mini-transcribe-2025-12-15"])
27
34
  };
35
+ const PROVIDER_KEY_VALIDATION_MODELS = {
36
+ openai: "gpt-4.1-mini",
37
+ gemini: "gemini-3.1-flash-lite-preview",
38
+ openrouter: "google/gemini-3.1-flash-lite-preview",
39
+ perplexity: "sonar"
40
+ };
28
41
  function readStoredProviderSecret(value) {
29
42
  try {
30
43
  return decryptString(value, config.ENCRYPTION_SECRET);
@@ -33,30 +46,143 @@ function readStoredProviderSecret(value) {
33
46
  return value;
34
47
  }
35
48
  }
49
+ function sleep(ms) {
50
+ return new Promise((resolve) => setTimeout(resolve, ms));
51
+ }
52
+ function zeroUsageCost(result) {
53
+ return {
54
+ ...result,
55
+ usage: {
56
+ ...result.usage,
57
+ costUsd: 0
58
+ }
59
+ };
60
+ }
61
+ const DEFAULT_PROVIDER_LEASE_PRIORITY = 100;
62
+ const CHAT_PROVIDER_LEASE_PRIORITY = 0;
63
+ const DEFAULT_PROVIDER_LEASE_TIMEOUT_MS = 120_000;
64
+ const CHAT_PROVIDER_LEASE_TIMEOUT_MS = 45_000;
65
+ const PROVIDER_LEASE_POLL_MS = 250;
66
+ const DEFAULT_PROVIDER_LEASE_SECONDS = 90;
67
+ const CHAT_PROVIDER_LEASE_SECONDS = 300;
36
68
  export class ProviderService {
69
+ handleLeasedProviderError(input) {
70
+ if (input.error instanceof ProviderRateLimitError) {
71
+ database.recordProviderKeyUsage({
72
+ id: createId("usage"),
73
+ keyId: input.lease.keyId,
74
+ jobId: input.jobId,
75
+ provider: input.provider,
76
+ model: input.model,
77
+ eventType: "rate_limit",
78
+ metadata: {
79
+ ...(input.metadata ?? {}),
80
+ retryAfterSeconds: input.error.retryAfterSeconds
81
+ }
82
+ });
83
+ database.setProviderKeyCooldown(input.lease.keyId, addSeconds(new Date(), input.error.retryAfterSeconds));
84
+ }
85
+ else if (input.error instanceof ProviderAuthError) {
86
+ database.recordProviderKeyUsage({
87
+ id: createId("usage"),
88
+ keyId: input.lease.keyId,
89
+ jobId: input.jobId,
90
+ provider: input.provider,
91
+ model: input.model,
92
+ eventType: "auth_error",
93
+ metadata: input.metadata ?? {}
94
+ });
95
+ database.setProviderKeyCooldown(input.lease.keyId, null, "invalid", "auth_error");
96
+ }
97
+ throw input.error;
98
+ }
99
+ listAvailableProviders(customerId) {
100
+ const now = Date.now();
101
+ const providers = database.listProviderKeysWithSecrets(customerId)
102
+ .map((entry) => ({
103
+ provider: String(entry.provider),
104
+ status: String(entry.status),
105
+ cooldownUntil: entry.cooldown_until ? String(entry.cooldown_until) : null,
106
+ secret: readStoredProviderSecret(String(entry.secret))
107
+ }))
108
+ .filter((entry) => entry.status === "active")
109
+ .filter((entry) => !entry.cooldownUntil || Date.parse(entry.cooldownUntil) <= now)
110
+ .filter((entry) => entry.secret.trim().length > 0)
111
+ .map((entry) => entry.provider);
112
+ return [...new Set(providers)];
113
+ }
114
+ async validateProviderKey(input) {
115
+ const model = PROVIDER_KEY_VALIDATION_MODELS[input.provider];
116
+ if (!model) {
117
+ throw new Error(`Provider ${input.provider} does not support validation.`);
118
+ }
119
+ if (config.mockProviders) {
120
+ return { provider: input.provider, model, mock: true };
121
+ }
122
+ try {
123
+ await this.callProvider({
124
+ provider: input.provider,
125
+ model,
126
+ prompt: "Reply with OK.",
127
+ temperature: 0,
128
+ apiKey: input.secret
129
+ });
130
+ return { provider: input.provider, model, mock: false };
131
+ }
132
+ catch (error) {
133
+ throw new Error(toProviderValidationMessage(input.provider, error));
134
+ }
135
+ }
37
136
  async leaseCustomerKey(input) {
38
137
  const leaseToken = createId("lease");
39
- const row = database.acquireProviderKeyLease({
40
- customerId: input.customerId,
41
- provider: input.provider,
42
- workerId: input.workerId,
43
- jobId: input.jobId,
44
- leaseToken,
45
- expiresAt: addSeconds(new Date(), 90)
46
- });
47
- if (!row) {
48
- throw new ProviderKeyUnavailableError(`No eligible ${input.provider} key available.`);
138
+ const waiterId = createId("wait");
139
+ const priority = input.priority ?? DEFAULT_PROVIDER_LEASE_PRIORITY;
140
+ const waitTimeoutMs = input.waitTimeoutMs ?? DEFAULT_PROVIDER_LEASE_TIMEOUT_MS;
141
+ const leaseDurationSeconds = input.leaseDurationSeconds ?? DEFAULT_PROVIDER_LEASE_SECONDS;
142
+ const deadline = Date.now() + waitTimeoutMs;
143
+ try {
144
+ while (Date.now() < deadline) {
145
+ const row = database.acquireProviderKeyLease({
146
+ customerId: input.customerId,
147
+ provider: input.provider,
148
+ waiterId,
149
+ workerId: input.workerId,
150
+ jobId: input.jobId,
151
+ priority,
152
+ leaseToken,
153
+ expiresAt: addSeconds(new Date(), leaseDurationSeconds),
154
+ waiterExpiresAt: addSeconds(new Date(), 10)
155
+ });
156
+ if (row) {
157
+ return {
158
+ keyId: row.keyId,
159
+ leaseToken,
160
+ provider: input.provider,
161
+ secret: readStoredProviderSecret(row.encryptedSecret)
162
+ };
163
+ }
164
+ await sleep(PROVIDER_LEASE_POLL_MS);
165
+ }
49
166
  }
50
- return {
51
- keyId: row.keyId,
52
- leaseToken,
53
- provider: input.provider,
54
- secret: readStoredProviderSecret(row.encryptedSecret)
55
- };
167
+ finally {
168
+ database.deleteProviderKeyWaiter(waiterId);
169
+ }
170
+ throw new ProviderLeaseTimeoutError(`No eligible ${input.provider} key became available in time.`);
56
171
  }
57
172
  async releaseLease(lease) {
58
173
  database.releaseProviderKeyLease({ keyId: lease.keyId, leaseToken: lease.leaseToken });
59
174
  }
175
+ async leaseEditorChatKey(input) {
176
+ return this.leaseCustomerKey({
177
+ customerId: input.customerId,
178
+ provider: input.provider,
179
+ jobId: input.chatId,
180
+ workerId: `editor-chat:${input.chatId}`,
181
+ priority: CHAT_PROVIDER_LEASE_PRIORITY,
182
+ waitTimeoutMs: CHAT_PROVIDER_LEASE_TIMEOUT_MS,
183
+ leaseDurationSeconds: CHAT_PROVIDER_LEASE_SECONDS
184
+ });
185
+ }
60
186
  async generateText(input) {
61
187
  const lease = await this.leaseCustomerKey({
62
188
  customerId: input.customerId,
@@ -81,7 +207,6 @@ export class ProviderService {
81
207
  eventType: "success",
82
208
  inputTokens: 180,
83
209
  outputTokens: 120,
84
- costUsd: 0.02,
85
210
  metadata: { mock: true }
86
211
  });
87
212
  database.touchProviderKey(lease.keyId);
@@ -90,11 +215,11 @@ export class ProviderService {
90
215
  usage: {
91
216
  inputTokens: 180,
92
217
  outputTokens: 120,
93
- costUsd: 0.02
218
+ costUsd: 0
94
219
  }
95
220
  };
96
221
  }
97
- const result = await this.callProvider({ ...input, apiKey: lease.secret });
222
+ const result = zeroUsageCost(await this.callProvider({ ...input, apiKey: lease.secret }));
98
223
  database.recordProviderKeyUsage({
99
224
  id: createId("usage"),
100
225
  keyId: lease.keyId,
@@ -103,37 +228,19 @@ export class ProviderService {
103
228
  model: input.model,
104
229
  eventType: "success",
105
230
  inputTokens: result.usage.inputTokens,
106
- outputTokens: result.usage.outputTokens,
107
- costUsd: result.usage.costUsd
231
+ outputTokens: result.usage.outputTokens
108
232
  });
109
233
  database.touchProviderKey(lease.keyId);
110
234
  return result;
111
235
  }
112
236
  catch (error) {
113
- if (error instanceof ProviderRateLimitError) {
114
- database.recordProviderKeyUsage({
115
- id: createId("usage"),
116
- keyId: lease.keyId,
117
- jobId: input.jobId,
118
- provider: input.provider,
119
- model: input.model,
120
- eventType: "rate_limit",
121
- metadata: { retryAfterSeconds: error.retryAfterSeconds }
122
- });
123
- database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
124
- }
125
- else if (error instanceof ProviderAuthError) {
126
- database.recordProviderKeyUsage({
127
- id: createId("usage"),
128
- keyId: lease.keyId,
129
- jobId: input.jobId,
130
- provider: input.provider,
131
- model: input.model,
132
- eventType: "auth_error"
133
- });
134
- database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
135
- }
136
- throw error;
237
+ this.handleLeasedProviderError({
238
+ error,
239
+ lease,
240
+ jobId: input.jobId,
241
+ provider: input.provider,
242
+ model: input.model
243
+ });
137
244
  }
138
245
  finally {
139
246
  await this.releaseLease(lease);
@@ -159,7 +266,6 @@ export class ProviderService {
159
266
  eventType: "success",
160
267
  inputTokens: 0,
161
268
  outputTokens: 0,
162
- costUsd: 0.01,
163
269
  metadata: { mock: true, type: "speech_generation" }
164
270
  });
165
271
  database.touchProviderKey(lease.keyId);
@@ -169,11 +275,11 @@ export class ProviderService {
169
275
  usage: {
170
276
  inputTokens: 0,
171
277
  outputTokens: 0,
172
- costUsd: 0.01
278
+ costUsd: 0
173
279
  }
174
280
  };
175
281
  }
176
- const result = input.provider === "gemini"
282
+ const result = zeroUsageCost(input.provider === "gemini"
177
283
  ? await this.callGeminiSpeechGeneration({
178
284
  model: input.model,
179
285
  text: input.text,
@@ -189,7 +295,7 @@ export class ProviderService {
189
295
  instructions: input.instructions,
190
296
  responseFormat: input.responseFormat,
191
297
  apiKey: lease.secret
192
- });
298
+ }));
193
299
  database.recordProviderKeyUsage({
194
300
  id: createId("usage"),
195
301
  keyId: lease.keyId,
@@ -199,38 +305,20 @@ export class ProviderService {
199
305
  eventType: "success",
200
306
  inputTokens: result.usage.inputTokens,
201
307
  outputTokens: result.usage.outputTokens,
202
- costUsd: result.usage.costUsd,
203
308
  metadata: { type: "speech_generation" }
204
309
  });
205
310
  database.touchProviderKey(lease.keyId);
206
311
  return result;
207
312
  }
208
313
  catch (error) {
209
- if (error instanceof ProviderRateLimitError) {
210
- database.recordProviderKeyUsage({
211
- id: createId("usage"),
212
- keyId: lease.keyId,
213
- jobId: input.jobId,
214
- provider: input.provider,
215
- model: input.model,
216
- eventType: "rate_limit",
217
- metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_generation" }
218
- });
219
- database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
220
- }
221
- else if (error instanceof ProviderAuthError) {
222
- database.recordProviderKeyUsage({
223
- id: createId("usage"),
224
- keyId: lease.keyId,
225
- jobId: input.jobId,
226
- provider: input.provider,
227
- model: input.model,
228
- eventType: "auth_error",
229
- metadata: { type: "speech_generation" }
230
- });
231
- database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
232
- }
233
- throw error;
314
+ this.handleLeasedProviderError({
315
+ error,
316
+ lease,
317
+ jobId: input.jobId,
318
+ provider: input.provider,
319
+ model: input.model,
320
+ metadata: { type: "speech_generation" }
321
+ });
234
322
  }
235
323
  finally {
236
324
  await this.releaseLease(lease);
@@ -255,7 +343,6 @@ export class ProviderService {
255
343
  eventType: "success",
256
344
  inputTokens: 0,
257
345
  outputTokens: 0,
258
- costUsd: 0.04,
259
346
  metadata: { mock: true, type: "image_generation" }
260
347
  });
261
348
  database.touchProviderKey(lease.keyId);
@@ -274,14 +361,103 @@ export class ProviderService {
274
361
  imageSize: input.imageSize,
275
362
  apiKey: lease.secret
276
363
  })
277
- : await this.callOpenAIImageGeneration({
364
+ : input.provider === "openrouter"
365
+ ? await this.callOpenRouterImageGeneration({
366
+ model: input.model,
367
+ prompt: input.prompt,
368
+ promptAttachments: input.promptAttachments,
369
+ aspectRatio: input.aspectRatio,
370
+ imageSize: input.imageSize,
371
+ apiKey: lease.secret
372
+ })
373
+ : await this.callOpenAIImageGeneration({
374
+ model: input.model,
375
+ prompt: input.prompt,
376
+ promptAttachments: input.promptAttachments,
377
+ size: input.size,
378
+ aspectRatio: input.aspectRatio,
379
+ apiKey: lease.secret
380
+ });
381
+ database.recordProviderKeyUsage({
382
+ id: createId("usage"),
383
+ keyId: lease.keyId,
384
+ jobId: input.jobId,
385
+ provider: input.provider,
386
+ model: input.model,
387
+ eventType: "success",
388
+ inputTokens: 0,
389
+ outputTokens: 0,
390
+ metadata: { type: "image_generation" }
391
+ });
392
+ database.touchProviderKey(lease.keyId);
393
+ return result;
394
+ }
395
+ catch (error) {
396
+ this.handleLeasedProviderError({
397
+ error,
398
+ lease,
399
+ jobId: input.jobId,
400
+ provider: input.provider,
401
+ model: input.model,
402
+ metadata: { type: "image_generation" }
403
+ });
404
+ }
405
+ finally {
406
+ await this.releaseLease(lease);
407
+ }
408
+ }
409
+ async editImage(input) {
410
+ const lease = await this.leaseCustomerKey({
411
+ customerId: input.customerId,
412
+ provider: input.provider,
413
+ jobId: input.jobId,
414
+ workerId: input.workerId
415
+ });
416
+ try {
417
+ if (config.mockProviders) {
418
+ const svg = buildMockSlideSvg(`EDIT ${input.instruction}\n${input.sourceImageUrl}`);
419
+ database.recordProviderKeyUsage({
420
+ id: createId("usage"),
421
+ keyId: lease.keyId,
422
+ jobId: input.jobId,
423
+ provider: input.provider,
278
424
  model: input.model,
279
- prompt: input.prompt,
280
- promptAttachments: input.promptAttachments,
425
+ eventType: "success",
426
+ inputTokens: 0,
427
+ outputTokens: 0,
428
+ metadata: { mock: true, type: "image_edit" }
429
+ });
430
+ database.touchProviderKey(lease.keyId);
431
+ return {
432
+ bytes: Buffer.from(svg, "utf8"),
433
+ contentType: "image/svg+xml",
434
+ revisedPrompt: input.instruction
435
+ };
436
+ }
437
+ const result = input.provider === "openai"
438
+ ? await this.callOpenAIImageEdit({
439
+ model: input.model,
440
+ sourceImageUrl: input.sourceImageUrl,
441
+ instruction: input.instruction,
442
+ maskUrl: input.maskUrl,
281
443
  size: input.size,
282
- aspectRatio: input.aspectRatio,
283
444
  apiKey: lease.secret
284
- });
445
+ })
446
+ : input.provider === "perplexity"
447
+ ? (() => {
448
+ throw new Error("Perplexity image edit is not supported.");
449
+ })()
450
+ : await this.callAttachmentDrivenImageEdit({
451
+ provider: input.provider,
452
+ model: input.model,
453
+ sourceImageUrl: input.sourceImageUrl,
454
+ instruction: input.instruction,
455
+ referenceAttachments: input.referenceAttachments,
456
+ maskUrl: input.maskUrl,
457
+ aspectRatio: input.aspectRatio,
458
+ imageSize: input.imageSize,
459
+ apiKey: lease.secret
460
+ });
285
461
  database.recordProviderKeyUsage({
286
462
  id: createId("usage"),
287
463
  keyId: lease.keyId,
@@ -291,16 +467,31 @@ export class ProviderService {
291
467
  eventType: "success",
292
468
  inputTokens: 0,
293
469
  outputTokens: 0,
294
- costUsd: 0.04,
295
- metadata: { type: "image_generation" }
470
+ metadata: { type: input.maskUrl ? "image_inpaint" : "image_edit" }
296
471
  });
297
472
  database.touchProviderKey(lease.keyId);
298
473
  return result;
299
474
  }
475
+ catch (error) {
476
+ this.handleLeasedProviderError({
477
+ error,
478
+ lease,
479
+ jobId: input.jobId,
480
+ provider: input.provider,
481
+ model: input.model,
482
+ metadata: { type: input.maskUrl ? "image_inpaint" : "image_edit" }
483
+ });
484
+ }
300
485
  finally {
301
486
  await this.releaseLease(lease);
302
487
  }
303
488
  }
489
+ async inpaintImage(input) {
490
+ return this.editImage({
491
+ ...input,
492
+ maskUrl: input.maskUrl
493
+ });
494
+ }
304
495
  async analyzeImageLayout(input) {
305
496
  const lease = await this.leaseCustomerKey({
306
497
  customerId: input.customerId,
@@ -319,7 +510,6 @@ export class ProviderService {
319
510
  eventType: "success",
320
511
  inputTokens: 0,
321
512
  outputTokens: 0,
322
- costUsd: 0.005,
323
513
  metadata: { mock: true, type: "layout_analysis" }
324
514
  });
325
515
  database.touchProviderKey(lease.keyId);
@@ -357,7 +547,6 @@ export class ProviderService {
357
547
  eventType: "success",
358
548
  inputTokens: 0,
359
549
  outputTokens: 0,
360
- costUsd: 0.005,
361
550
  metadata: { type: "layout_analysis" }
362
551
  });
363
552
  database.touchProviderKey(lease.keyId);
@@ -368,6 +557,16 @@ export class ProviderService {
368
557
  justification: parsed.justification ?? "Selected to preserve legibility and subject framing."
369
558
  };
370
559
  }
560
+ catch (error) {
561
+ this.handleLeasedProviderError({
562
+ error,
563
+ lease,
564
+ jobId: input.jobId,
565
+ provider: input.provider,
566
+ model: input.model,
567
+ metadata: { type: "layout_analysis" }
568
+ });
569
+ }
371
570
  finally {
372
571
  await this.releaseLease(lease);
373
572
  }
@@ -391,7 +590,6 @@ export class ProviderService {
391
590
  eventType: "success",
392
591
  inputTokens: 0,
393
592
  outputTokens: 0,
394
- costUsd: 0.01,
395
593
  metadata: { mock: true, type: "speech_transcription" }
396
594
  });
397
595
  database.touchProviderKey(lease.keyId);
@@ -400,11 +598,11 @@ export class ProviderService {
400
598
  usage: {
401
599
  inputTokens: 0,
402
600
  outputTokens: 0,
403
- costUsd: 0.01
601
+ costUsd: 0
404
602
  }
405
603
  };
406
604
  }
407
- const result = input.provider === "gemini"
605
+ const result = zeroUsageCost(input.provider === "gemini"
408
606
  ? await this.callGeminiSpeechTranscription({
409
607
  model: input.model,
410
608
  audio: input.audio,
@@ -429,7 +627,7 @@ export class ProviderService {
429
627
  prompt: input.prompt,
430
628
  language: input.language,
431
629
  apiKey: lease.secret
432
- });
630
+ }));
433
631
  database.recordProviderKeyUsage({
434
632
  id: createId("usage"),
435
633
  keyId: lease.keyId,
@@ -439,38 +637,20 @@ export class ProviderService {
439
637
  eventType: "success",
440
638
  inputTokens: result.usage.inputTokens,
441
639
  outputTokens: result.usage.outputTokens,
442
- costUsd: result.usage.costUsd,
443
640
  metadata: { type: "speech_transcription" }
444
641
  });
445
642
  database.touchProviderKey(lease.keyId);
446
643
  return result;
447
644
  }
448
645
  catch (error) {
449
- if (error instanceof ProviderRateLimitError) {
450
- database.recordProviderKeyUsage({
451
- id: createId("usage"),
452
- keyId: lease.keyId,
453
- jobId: input.jobId,
454
- provider: input.provider,
455
- model: input.model,
456
- eventType: "rate_limit",
457
- metadata: { retryAfterSeconds: error.retryAfterSeconds, type: "speech_transcription" }
458
- });
459
- database.setProviderKeyCooldown(lease.keyId, addSeconds(new Date(), error.retryAfterSeconds));
460
- }
461
- else if (error instanceof ProviderAuthError) {
462
- database.recordProviderKeyUsage({
463
- id: createId("usage"),
464
- keyId: lease.keyId,
465
- jobId: input.jobId,
466
- provider: input.provider,
467
- model: input.model,
468
- eventType: "auth_error",
469
- metadata: { type: "speech_transcription" }
470
- });
471
- database.setProviderKeyCooldown(lease.keyId, null, "invalid", "auth_error");
472
- }
473
- throw error;
646
+ this.handleLeasedProviderError({
647
+ error,
648
+ lease,
649
+ jobId: input.jobId,
650
+ provider: input.provider,
651
+ model: input.model,
652
+ metadata: { type: "speech_transcription" }
653
+ });
474
654
  }
475
655
  finally {
476
656
  await this.releaseLease(lease);
@@ -519,6 +699,77 @@ export class ProviderService {
519
699
  revisedPrompt: data.data?.[0]?.revised_prompt ?? null
520
700
  };
521
701
  }
702
+ async callOpenAIImageEdit(input) {
703
+ const form = new FormData();
704
+ form.set("model", input.model);
705
+ form.set("prompt", input.instruction);
706
+ form.set("size", input.size ?? "1024x1024");
707
+ form.set("image", await remoteImageToBlob(input.sourceImageUrl));
708
+ if (input.maskUrl) {
709
+ form.set("mask", await remoteImageToBlob(input.maskUrl));
710
+ }
711
+ const response = await fetch("https://api.openai.com/v1/images/edits", {
712
+ method: "POST",
713
+ headers: {
714
+ Authorization: `Bearer ${input.apiKey}`
715
+ },
716
+ body: form
717
+ });
718
+ if (response.status === 401) {
719
+ throw new ProviderAuthError("openai authentication failed");
720
+ }
721
+ if (response.status === 429) {
722
+ throw new ProviderRateLimitError("openai rate limited");
723
+ }
724
+ if (!response.ok) {
725
+ const details = await response.text();
726
+ throw new Error(`openai image edit returned ${response.status}${details ? `: ${details}` : ""}`);
727
+ }
728
+ const data = await response.json();
729
+ const encoded = data.data?.[0]?.b64_json;
730
+ if (!encoded) {
731
+ throw new Error("openai image edit returned no image payload");
732
+ }
733
+ return {
734
+ bytes: Buffer.from(encoded, "base64"),
735
+ contentType: "image/png",
736
+ revisedPrompt: data.data?.[0]?.revised_prompt ?? null
737
+ };
738
+ }
739
+ async callAttachmentDrivenImageEdit(input) {
740
+ const promptAttachments = [
741
+ input.sourceImageUrl,
742
+ ...(input.maskUrl ? [input.maskUrl] : []),
743
+ ...(input.referenceAttachments ?? [])
744
+ ];
745
+ const prompt = [
746
+ input.maskUrl
747
+ ? "Edit this image using the provided source image and mask."
748
+ : "Edit this image using the provided source image as the base reference.",
749
+ input.maskUrl
750
+ ? "Only change the masked region. Preserve all unmasked regions exactly."
751
+ : "Preserve the overall composition unless the instruction explicitly changes it.",
752
+ `Instruction: ${input.instruction}`,
753
+ "Return one final edited image only."
754
+ ].join("\n");
755
+ return input.provider === "gemini"
756
+ ? this.callGeminiImageGeneration({
757
+ model: input.model,
758
+ prompt,
759
+ promptAttachments,
760
+ aspectRatio: input.aspectRatio,
761
+ imageSize: input.imageSize,
762
+ apiKey: input.apiKey
763
+ })
764
+ : this.callOpenRouterImageGeneration({
765
+ model: input.model,
766
+ prompt,
767
+ promptAttachments,
768
+ aspectRatio: input.aspectRatio,
769
+ imageSize: input.imageSize,
770
+ apiKey: input.apiKey
771
+ });
772
+ }
522
773
  async callGeminiImageGeneration(input) {
523
774
  const imageConfig = {};
524
775
  if (input.aspectRatio) {
@@ -566,8 +817,76 @@ export class ProviderService {
566
817
  revisedPrompt: null
567
818
  };
568
819
  }
820
+ async callOpenRouterImageGeneration(input) {
821
+ const content = [
822
+ { type: "text", text: input.prompt },
823
+ ...(input.promptAttachments ?? []).map((url) => ({
824
+ type: "image_url",
825
+ image_url: { url }
826
+ }))
827
+ ];
828
+ const imageConfig = {};
829
+ if (input.aspectRatio) {
830
+ imageConfig.aspect_ratio = input.aspectRatio;
831
+ }
832
+ if (input.imageSize) {
833
+ imageConfig.image_size = input.imageSize;
834
+ }
835
+ const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
836
+ method: "POST",
837
+ headers: {
838
+ "Content-Type": "application/json",
839
+ Authorization: `Bearer ${input.apiKey}`
840
+ },
841
+ body: JSON.stringify({
842
+ model: input.model,
843
+ messages: [
844
+ {
845
+ role: "user",
846
+ content
847
+ }
848
+ ],
849
+ modalities: ["image", "text"],
850
+ stream: false,
851
+ ...(Object.keys(imageConfig).length
852
+ ? {
853
+ image_config: imageConfig
854
+ }
855
+ : {})
856
+ })
857
+ });
858
+ if (response.status === 401 || response.status === 403) {
859
+ throw new ProviderAuthError("openrouter authentication failed");
860
+ }
861
+ if (response.status === 429) {
862
+ throw new ProviderRateLimitError("openrouter rate limited");
863
+ }
864
+ if (!response.ok) {
865
+ const details = await response.text();
866
+ throw new Error(`openrouter image generation returned ${response.status}${details ? `: ${details}` : ""}`);
867
+ }
868
+ const data = await response.json();
869
+ const imageUrl = data.choices?.[0]?.message?.images?.[0]?.image_url?.url ??
870
+ data.choices?.[0]?.message?.images?.[0]?.imageUrl?.url;
871
+ if (!imageUrl) {
872
+ throw new Error("openrouter image generation returned no image payload");
873
+ }
874
+ const parsed = decodeInlineDataUrl(imageUrl);
875
+ if (!parsed) {
876
+ throw new Error("openrouter image generation returned an unsupported image payload");
877
+ }
878
+ return {
879
+ bytes: parsed.bytes,
880
+ contentType: parsed.contentType,
881
+ revisedPrompt: data.choices?.[0]?.message?.content ?? null
882
+ };
883
+ }
569
884
  async callOpenAICompatibleSpeechGeneration(input) {
570
- const responseFormat = input.responseFormat ?? "mp3";
885
+ const requestedFormat = input.responseFormat ?? "mp3";
886
+ const usePcmBridge = input.provider === "openrouter" &&
887
+ /gemini/i.test(input.model) &&
888
+ requestedFormat !== "mp3";
889
+ const responseFormat = usePcmBridge ? "pcm" : requestedFormat;
571
890
  const endpoint = input.provider === "openrouter"
572
891
  ? "https://openrouter.ai/api/v1/audio/speech"
573
892
  : "https://api.openai.com/v1/audio/speech";
@@ -595,9 +914,14 @@ export class ProviderService {
595
914
  const details = await response.text();
596
915
  throw new Error(`${input.provider} speech generation returned ${response.status}${details ? `: ${details}` : ""}`);
597
916
  }
917
+ const rawBytes = Buffer.from(await response.arrayBuffer());
918
+ const bytes = usePcmBridge ? pcm16MonoToWav(rawBytes, 24000) : rawBytes;
919
+ const contentType = usePcmBridge
920
+ ? "audio/wav"
921
+ : response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat);
598
922
  return {
599
- bytes: Buffer.from(await response.arrayBuffer()),
600
- contentType: response.headers.get("content-type")?.split(";")[0]?.trim() || inferSpeechContentType(responseFormat),
923
+ bytes,
924
+ contentType,
601
925
  usage: {
602
926
  inputTokens: 0,
603
927
  outputTokens: 0,
@@ -644,9 +968,24 @@ export class ProviderService {
644
968
  if (!encoded) {
645
969
  throw new Error("gemini speech generation returned no audio payload");
646
970
  }
971
+ const rawBytes = Buffer.from(encoded, "base64");
972
+ const rawContentType = String(part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000");
973
+ const pcmRateMatch = /audio\/l16;?\s*rate=(\d+)/i.exec(rawContentType);
974
+ const looksLikeWav = isWavBuffer(rawBytes);
975
+ const looksLikeMp3 = isLikelyMp3Buffer(rawBytes);
976
+ const shouldWrapPcm = Boolean(pcmRateMatch ||
977
+ (!looksLikeWav && !looksLikeMp3));
978
+ const bytes = shouldWrapPcm
979
+ ? pcm16MonoToWav(rawBytes, Number(pcmRateMatch?.[1] || "24000"))
980
+ : rawBytes;
981
+ const contentType = shouldWrapPcm
982
+ ? "audio/wav"
983
+ : looksLikeWav
984
+ ? "audio/wav"
985
+ : "audio/mpeg";
647
986
  return {
648
- bytes: Buffer.from(encoded, "base64"),
649
- contentType: part?.inlineData?.mimeType ?? part?.inline_data?.mime_type ?? "audio/L16;rate=24000",
987
+ bytes,
988
+ contentType,
650
989
  usage: {
651
990
  inputTokens: Number(data.usageMetadata?.promptTokenCount ?? 0),
652
991
  outputTokens: Number(data.usageMetadata?.candidatesTokenCount ?? 0),
@@ -964,15 +1303,80 @@ export class ProviderService {
964
1303
  function clamp(value, min, max) {
965
1304
  return Math.max(min, Math.min(max, value));
966
1305
  }
1306
+ function summarizeProviderValidationError(error) {
1307
+ const rawMessage = error instanceof Error ? error.message : "Unknown provider error";
1308
+ return rawMessage
1309
+ .replace(/\s+/g, " ")
1310
+ .trim()
1311
+ .slice(0, 280);
1312
+ }
1313
+ function toProviderValidationMessage(provider, error) {
1314
+ const providerLabel = formatProviderLabel(provider);
1315
+ const summary = summarizeProviderValidationError(error);
1316
+ if (!summary) {
1317
+ return `Unable to verify ${providerLabel} API key.`;
1318
+ }
1319
+ if (/401|403|invalid api key|api key not valid|incorrect api key|unauthorized|permission denied|authentication failed/i.test(summary)) {
1320
+ return `${providerLabel} API key was rejected by the provider. ${summary}`;
1321
+ }
1322
+ return `Unable to verify ${providerLabel} API key. ${summary}`;
1323
+ }
1324
+ function formatProviderLabel(provider) {
1325
+ switch (provider) {
1326
+ case "openai":
1327
+ return "OpenAI";
1328
+ case "gemini":
1329
+ return "Gemini";
1330
+ case "openrouter":
1331
+ return "OpenRouter";
1332
+ case "perplexity":
1333
+ return "Perplexity";
1334
+ }
1335
+ }
967
1336
  function assertSupportedSpeechModel(kind, provider, model) {
968
- const supported = kind === "tts" ? SUPPORTED_TTS_MODELS[provider] : SUPPORTED_STT_MODELS[provider];
969
- if (!supported?.has(model)) {
970
- throw new Error(`Unsupported ${kind.toUpperCase()} model for provider ${provider}: ${model}`);
1337
+ const normalizedModel = model.trim();
1338
+ if (!normalizedModel) {
1339
+ throw new Error(`Missing ${kind.toUpperCase()} model for provider ${provider}.`);
1340
+ }
1341
+ const speechCapableProviders = ["openai", "gemini", "openrouter"];
1342
+ if (!speechCapableProviders.includes(provider)) {
1343
+ throw new Error(`Provider ${provider} does not support ${kind.toUpperCase()} requests.`);
971
1344
  }
972
1345
  }
973
1346
  function inferSpeechContentType(format) {
974
1347
  return format === "wav" ? "audio/wav" : "audio/mpeg";
975
1348
  }
1349
+ function pcm16MonoToWav(pcmBuffer, sampleRate) {
1350
+ const header = Buffer.alloc(44);
1351
+ header.write("RIFF", 0, 4, "ascii");
1352
+ header.writeUInt32LE(36 + pcmBuffer.byteLength, 4);
1353
+ header.write("WAVE", 8, 4, "ascii");
1354
+ header.write("fmt ", 12, 4, "ascii");
1355
+ header.writeUInt32LE(16, 16);
1356
+ header.writeUInt16LE(1, 20);
1357
+ header.writeUInt16LE(1, 22);
1358
+ header.writeUInt32LE(sampleRate, 24);
1359
+ header.writeUInt32LE(sampleRate * 2, 28);
1360
+ header.writeUInt16LE(2, 32);
1361
+ header.writeUInt16LE(16, 34);
1362
+ header.write("data", 36, 4, "ascii");
1363
+ header.writeUInt32LE(pcmBuffer.byteLength, 40);
1364
+ return Buffer.concat([header, pcmBuffer]);
1365
+ }
1366
+ function isWavBuffer(buffer) {
1367
+ return buffer.byteLength >= 12
1368
+ && buffer.subarray(0, 4).toString("ascii") === "RIFF"
1369
+ && buffer.subarray(8, 12).toString("ascii") === "WAVE";
1370
+ }
1371
+ function isLikelyMp3Buffer(buffer) {
1372
+ if (buffer.byteLength < 3) {
1373
+ return false;
1374
+ }
1375
+ if (buffer.subarray(0, 3).toString("ascii") === "ID3") {
1376
+ return true;
1377
+ }
1378
+ return buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0;
1379
+ }
976
1380
  function audioFormatFromContentType(contentType) {
977
1381
  const normalized = contentType?.split(";")[0]?.trim().toLowerCase();
978
1382
  switch (normalized) {
@@ -1016,21 +1420,26 @@ function extractGeminiText(data) {
1016
1420
  function supportsGeminiImageSize(model) {
1017
1421
  return model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview";
1018
1422
  }
1423
+ async function fetchRemoteBytes(input) {
1424
+ const response = await fetch(input.url);
1425
+ if (!response.ok) {
1426
+ throw new Error(`Could not fetch remote media ${input.url}: ${response.status}`);
1427
+ }
1428
+ const mimeType = response.headers.get("content-type")?.split(";")[0]?.trim() || input.fallbackMimeType || "application/octet-stream";
1429
+ const bytes = Buffer.from(await response.arrayBuffer());
1430
+ return { bytes, mimeType };
1431
+ }
1432
+ async function remoteImageToBlob(url) {
1433
+ const { bytes, mimeType } = await fetchRemoteBytes({ url, fallbackMimeType: "image/png" });
1434
+ return new Blob([bytes], { type: mimeType });
1435
+ }
1019
1436
  async function buildGeminiPromptParts(prompt, attachments) {
1020
1437
  const parts = [];
1021
1438
  for (const attachment of attachments) {
1022
1439
  if (!/^https?:\/\//i.test(attachment)) {
1023
1440
  throw new Error(`Gemini prompt attachments must be public URLs: ${attachment}`);
1024
1441
  }
1025
- const response = await fetch(attachment);
1026
- if (!response.ok) {
1027
- throw new Error(`Could not fetch Gemini prompt attachment ${attachment}: ${response.status}`);
1028
- }
1029
- const mimeType = response.headers.get("content-type")?.split(";")[0]?.trim();
1030
- if (!mimeType) {
1031
- throw new Error(`Gemini prompt attachment did not return a content type: ${attachment}`);
1032
- }
1033
- const bytes = Buffer.from(await response.arrayBuffer());
1442
+ const { bytes, mimeType } = await fetchRemoteBytes({ url: attachment });
1034
1443
  parts.push({
1035
1444
  inlineData: {
1036
1445
  mimeType,
@@ -1041,6 +1450,16 @@ async function buildGeminiPromptParts(prompt, attachments) {
1041
1450
  parts.push({ text: prompt });
1042
1451
  return parts;
1043
1452
  }
1453
+ function decodeInlineDataUrl(value) {
1454
+ const match = value.match(/^data:([^;,]+);base64,(.+)$/);
1455
+ if (!match) {
1456
+ return null;
1457
+ }
1458
+ return {
1459
+ contentType: match[1],
1460
+ bytes: Buffer.from(match[2], "base64")
1461
+ };
1462
+ }
1044
1463
  function buildMockSlideSvg(prompt) {
1045
1464
  const escaped = escapeXml(prompt.slice(0, 220));
1046
1465
  return `