@1agh/maude 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/acp/bridge.ts +44 -10
- package/apps/studio/acp/env.ts +23 -0
- package/apps/studio/acp/login-state.ts +263 -0
- package/apps/studio/acp/plugin-bootstrap.ts +31 -25
- package/apps/studio/acp/probe.ts +152 -1
- package/apps/studio/annotations-layer.tsx +396 -46
- package/apps/studio/api.ts +395 -2
- package/apps/studio/bin/_audio-search.mjs +140 -0
- package/apps/studio/bin/_import-asset-pdf-worker.mjs +34 -0
- package/apps/studio/bin/_import-asset.mjs +815 -0
- package/apps/studio/bin/_import-brand.mjs +635 -0
- package/apps/studio/bin/_import-tokens-alias-resolver.mjs +95 -0
- package/apps/studio/bin/_import-tokens.mjs +1201 -0
- package/apps/studio/bin/_transcribe.mjs +419 -0
- package/apps/studio/bin/_transcribe.test.mjs +80 -0
- package/apps/studio/bin/audio-search.sh +28 -0
- package/apps/studio/bin/generate.sh +154 -0
- package/apps/studio/bin/import-asset.sh +34 -0
- package/apps/studio/bin/import-brand.sh +29 -0
- package/apps/studio/bin/import-tokens.sh +33 -0
- package/apps/studio/bin/photo-bg-remove.sh +119 -21
- package/apps/studio/bin/transcribe.sh +39 -0
- package/apps/studio/canvas-edit.ts +197 -2
- package/apps/studio/canvas-lib.tsx +115 -6
- package/apps/studio/client/app.jsx +1954 -685
- package/apps/studio/client/generate-dialog.jsx +352 -0
- package/apps/studio/client/github.js +14 -0
- package/apps/studio/client/inspector-controls.jsx +781 -0
- package/apps/studio/client/panels/BrandUploadPanel.jsx +233 -0
- package/apps/studio/client/panels/ChatPanel.jsx +15 -33
- package/apps/studio/client/panels/IntroVideoDialog.jsx +44 -0
- package/apps/studio/client/panels/OnboardingWizard.jsx +12 -0
- package/apps/studio/client/panels/ReadinessList.jsx +227 -7
- package/apps/studio/client/panels/SettingsPanel.jsx +828 -0
- package/apps/studio/client/panels/SetupChecklist.jsx +223 -0
- package/apps/studio/client/photo-knobs.jsx +167 -172
- package/apps/studio/client/styles/3-shell-maude.css +334 -28
- package/apps/studio/client/styles/4-components.css +223 -0
- package/apps/studio/client/styles/6-acp-chat.css +86 -0
- package/apps/studio/client/tour/quick-setup-tour.js +39 -0
- package/apps/studio/commands/annotation-strokes-command.ts +13 -3
- package/apps/studio/config.schema.json +70 -0
- package/apps/studio/context-menu.tsx +11 -2
- package/apps/studio/context.ts +16 -0
- package/apps/studio/design-setup-readiness.ts +115 -0
- package/apps/studio/dist/client.bundle.js +2097 -2088
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/draw/palette.ts +34 -0
- package/apps/studio/footage/schema.test.ts +92 -0
- package/apps/studio/footage/schema.ts +179 -1
- package/apps/studio/generation/adapters/elevenlabs.ts +370 -0
- package/apps/studio/generation/adapters/gemini.ts +542 -0
- package/apps/studio/generation/adapters/groq.test.ts +80 -0
- package/apps/studio/generation/adapters/groq.ts +247 -0
- package/apps/studio/generation/audio-library.test.ts +95 -0
- package/apps/studio/generation/audio-library.ts +156 -0
- package/apps/studio/generation/captions.test.ts +78 -0
- package/apps/studio/generation/captions.ts +132 -0
- package/apps/studio/generation/download.ts +76 -0
- package/apps/studio/generation/elevenlabs.test.ts +223 -0
- package/apps/studio/generation/gemini.test.ts +350 -0
- package/apps/studio/generation/jobs.test.ts +128 -0
- package/apps/studio/generation/jobs.ts +243 -0
- package/apps/studio/generation/keys.test.ts +75 -0
- package/apps/studio/generation/keys.ts +162 -0
- package/apps/studio/generation/prefs.test.ts +95 -0
- package/apps/studio/generation/prefs.ts +82 -0
- package/apps/studio/generation/registry.ts +88 -0
- package/apps/studio/generation/types.test.ts +123 -0
- package/apps/studio/generation/types.ts +266 -0
- package/apps/studio/generation/whisper-models.test.ts +152 -0
- package/apps/studio/generation/whisper-models.ts +257 -0
- package/apps/studio/generation/whisper-spike-results.md +61 -0
- package/apps/studio/git/service.ts +3 -1
- package/apps/studio/http.ts +867 -9
- package/apps/studio/media/intro.mp4 +0 -0
- package/apps/studio/media-commit-chain.ts +119 -0
- package/apps/studio/paths.ts +10 -0
- package/apps/studio/readiness.ts +108 -20
- package/apps/studio/scaffold-design.ts +1145 -0
- package/apps/studio/server.ts +13 -1
- package/apps/studio/test/_helpers.ts +15 -2
- package/apps/studio/test/acp-bridge.test.ts +34 -1
- package/apps/studio/test/acp-env.test.ts +30 -0
- package/apps/studio/test/acp-plugin-bootstrap.test.ts +23 -57
- package/apps/studio/test/acp-session-plugins.test.ts +72 -1
- package/apps/studio/test/annotation-strokes-command.test.ts +17 -0
- package/apps/studio/test/annotations-layer.test.ts +174 -0
- package/apps/studio/test/canvas-origin-gate.test.ts +37 -0
- package/apps/studio/test/csp-canvas-shell.test.ts +11 -0
- package/apps/studio/test/csrf-write-guard.test.ts +23 -1
- package/apps/studio/test/design-setup-readiness.test.ts +155 -0
- package/apps/studio/test/element-structural-edit.test.ts +108 -0
- package/apps/studio/test/fixtures/fake-claude-auth.mjs +13 -0
- package/apps/studio/test/generate-route.test.ts +106 -0
- package/apps/studio/test/import-asset-browser.test.ts +64 -0
- package/apps/studio/test/import-asset-route.test.ts +71 -0
- package/apps/studio/test/import-asset.test.ts +399 -0
- package/apps/studio/test/import-brand-browser.test.ts +88 -0
- package/apps/studio/test/import-brand-route.test.ts +120 -0
- package/apps/studio/test/import-brand.test.ts +206 -0
- package/apps/studio/test/import-tokens.test.ts +722 -0
- package/apps/studio/test/media-commit-chain.test.ts +210 -0
- package/apps/studio/test/photo-bg-remove-validation.test.ts +71 -0
- package/apps/studio/test/readiness.test.ts +13 -11
- package/apps/studio/test/scaffold-design.test.ts +122 -0
- package/apps/studio/test/tour-overlay.test.tsx +16 -0
- package/apps/studio/tool-palette.tsx +11 -6
- package/apps/studio/ui-prefs.ts +130 -0
- package/apps/studio/use-canvas-media-drop.tsx +4 -2
- package/apps/studio/use-chrome-visibility.tsx +8 -2
- package/apps/studio/whats-new.json +96 -0
- package/apps/studio/ws.ts +5 -0
- package/cli/bin/maude.mjs +6 -6
- package/cli/commands/design.mjs +33 -0
- package/cli/commands/init.mjs +7 -1
- package/cli/lib/gitignore-block.mjs +1 -0
- package/cli/lib/pkg-root.mjs +107 -0
- package/cli/lib/pkg-root.test.mjs +79 -0
- package/cli/lib/reconstruct-toolset.test.mjs +194 -0
- package/cli/lib/update-check.mjs +8 -0
- package/package.json +12 -10
- package/plugins/design/dependencies.json +18 -0
- package/plugins/flow/.claude-plugin/config.schema.json +5 -0
- package/plugins/flow/templates/ai-skeleton/scenario-guide.md +39 -0
- package/plugins/flow/templates/ai-skeleton/scenarios/README.md +2 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
// generation/adapters/gemini.ts — Google "Nano Banana" image generation via the
|
|
2
|
+
// Gemini REST API (DDR-16x, Phase 0). Direct BYOK: the user's own Google key →
|
|
3
|
+
// generativelanguage.googleapis.com directly, no aggregator, no SDK (plain
|
|
4
|
+
// `fetch` in the sidecar per the plan).
|
|
5
|
+
//
|
|
6
|
+
// SYNC shape: `:generateContent` returns the image inline as base64 in ONE round
|
|
7
|
+
// trip, so `submit()` hands back an already-`done` Job — callers never branch on
|
|
8
|
+
// sync/async. The base64 is localized into `assets/<sha8>.png` by the host's
|
|
9
|
+
// `ctx.localize` (download.ts), never referenced inline.
|
|
10
|
+
//
|
|
11
|
+
// The key is injected via AdapterContext at call time (keys.ts) and used ONLY in
|
|
12
|
+
// the `x-goog-api-key` header — never logged, never put in the URL query (so it
|
|
13
|
+
// can't leak via a redirect Location or an error string echoed to the client).
|
|
14
|
+
//
|
|
15
|
+
// Editing (maskless, Nano Banana's strength): pass a `sourceAsset` and the host
|
|
16
|
+
// reads its bytes into an inlineData part alongside the text — the Phase-1 image-
|
|
17
|
+
// editing wiring (Task 1.2) drives that; Phase 0 is text→image only.
|
|
18
|
+
|
|
19
|
+
import type {
|
|
20
|
+
AdapterContext,
|
|
21
|
+
GenAsset,
|
|
22
|
+
GenRequest,
|
|
23
|
+
GenResult,
|
|
24
|
+
Job,
|
|
25
|
+
ModelDescriptor,
|
|
26
|
+
ProviderAdapter,
|
|
27
|
+
ProviderDescriptor,
|
|
28
|
+
} from '../types.ts';
|
|
29
|
+
|
|
30
|
+
const GEMINI_DEFAULT_BASE = 'https://generativelanguage.googleapis.com/v1beta';
|
|
31
|
+
const GEMINI_API_BASE = process.env.MAUDE_GEMINI_API_BASE ?? GEMINI_DEFAULT_BASE;
|
|
32
|
+
// The only host the BYOK key may ever be sent to. `MAUDE_GEMINI_API_BASE` exists
|
|
33
|
+
// so a test can point at a local mock — but per the security fan-out (F2) an
|
|
34
|
+
// env-controlled base is otherwise a key-egress redirect + SSRF, so a non-default
|
|
35
|
+
// base is refused unless MAUDE_GEN_ALLOW_CUSTOM_BASE is explicitly opted in.
|
|
36
|
+
const GEMINI_ALLOWED_HOST = 'generativelanguage.googleapis.com';
|
|
37
|
+
const ALLOW_CUSTOM_BASE = /^(1|true|on|yes)$/i.test(process.env.MAUDE_GEN_ALLOW_CUSTOM_BASE ?? '');
|
|
38
|
+
|
|
39
|
+
// Hard ceiling on the provider RESPONSE the adapter will buffer before decode
|
|
40
|
+
// (security fan-out F1). `res.json()` + `Buffer.from(base64)` would otherwise
|
|
41
|
+
// materialize a multi-GB upstream body in RAM — OOM-crashing the shared server —
|
|
42
|
+
// long before saveAsset's disk cap can fire. Generous for a 4K image, bounded
|
|
43
|
+
// against abuse. Env-overridable for an operator who raises the asset caps.
|
|
44
|
+
const MAX_RESPONSE_BYTES = Math.max(
|
|
45
|
+
1024 * 1024,
|
|
46
|
+
Number(process.env.MAUDE_GEMINI_MAX_RESPONSE_BYTES) || 64 * 1024 * 1024
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Validate the outbound base ONCE at call time (F1/F2 defense-in-depth). Assert
|
|
51
|
+
* https + the fixed host allowlist so a poisoned env can never redirect the
|
|
52
|
+
* key-bearing request to a plaintext scheme or an attacker/internal host.
|
|
53
|
+
*/
|
|
54
|
+
function assertSafeBase(base: string): void {
|
|
55
|
+
let u: URL;
|
|
56
|
+
try {
|
|
57
|
+
u = new URL(base);
|
|
58
|
+
} catch {
|
|
59
|
+
throw new Error('Gemini API base is not a valid URL');
|
|
60
|
+
}
|
|
61
|
+
if (u.protocol !== 'https:') throw new Error('Gemini API base must be https');
|
|
62
|
+
if (u.hostname !== GEMINI_ALLOWED_HOST && !ALLOW_CUSTOM_BASE) {
|
|
63
|
+
throw new Error(
|
|
64
|
+
`Gemini API base host ${u.hostname} is not allowlisted (set MAUDE_GEN_ALLOW_CUSTOM_BASE=1 to override)`
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Read a Response body into text with a hard byte cap (F1) — rejects a body
|
|
70
|
+
* that exceeds the cap by Content-Length hint OR by actual streamed size, so an
|
|
71
|
+
* upstream that omits Content-Length can't slip past. */
|
|
72
|
+
async function readTextCapped(res: Response, maxBytes: number): Promise<string> {
|
|
73
|
+
const declared = Number(res.headers.get('content-length'));
|
|
74
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
75
|
+
throw new Error(`Gemini response too large (${declared} > ${maxBytes} bytes)`);
|
|
76
|
+
}
|
|
77
|
+
const body = res.body;
|
|
78
|
+
if (!body) return await res.text();
|
|
79
|
+
const reader = body.getReader();
|
|
80
|
+
const chunks: Uint8Array[] = [];
|
|
81
|
+
let total = 0;
|
|
82
|
+
for (;;) {
|
|
83
|
+
const { done, value } = await reader.read();
|
|
84
|
+
if (done) break;
|
|
85
|
+
if (value) {
|
|
86
|
+
total += value.byteLength;
|
|
87
|
+
if (total > maxBytes) {
|
|
88
|
+
await reader.cancel().catch(() => {});
|
|
89
|
+
throw new Error(`Gemini response exceeded ${maxBytes} bytes`);
|
|
90
|
+
}
|
|
91
|
+
chunks.push(value);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return new TextDecoder().decode(Buffer.concat(chunks));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Video is far heavier than a JSON image response — a separate, higher cap for
|
|
98
|
+
// the Veo MP4 download (still bounded against a wedged/hostile upstream). F1.
|
|
99
|
+
const MAX_VIDEO_BYTES = Math.max(
|
|
100
|
+
1024 * 1024,
|
|
101
|
+
Number(process.env.MAUDE_VEO_MAX_RESPONSE_BYTES) || 256 * 1024 * 1024
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
/** Read a Response body into bytes with a hard cap (F1) — for the Veo MP4. */
|
|
105
|
+
async function readBytesCapped(res: Response, maxBytes: number): Promise<Uint8Array> {
|
|
106
|
+
const declared = Number(res.headers.get('content-length'));
|
|
107
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
108
|
+
throw new Error(`Veo response too large (${declared} > ${maxBytes} bytes)`);
|
|
109
|
+
}
|
|
110
|
+
const body = res.body;
|
|
111
|
+
if (!body) return new Uint8Array(await res.arrayBuffer());
|
|
112
|
+
const reader = body.getReader();
|
|
113
|
+
const chunks: Uint8Array[] = [];
|
|
114
|
+
let total = 0;
|
|
115
|
+
for (;;) {
|
|
116
|
+
const { done, value } = await reader.read();
|
|
117
|
+
if (done) break;
|
|
118
|
+
if (value) {
|
|
119
|
+
total += value.byteLength;
|
|
120
|
+
if (total > maxBytes) {
|
|
121
|
+
await reader.cancel().catch(() => {});
|
|
122
|
+
throw new Error(`Veo response exceeded ${maxBytes} bytes`);
|
|
123
|
+
}
|
|
124
|
+
chunks.push(value);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return new Uint8Array(Buffer.concat(chunks));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export const GEMINI_DESCRIPTOR: ProviderDescriptor = {
|
|
131
|
+
id: 'gemini',
|
|
132
|
+
label: 'Google Gemini (Nano Banana + Veo)',
|
|
133
|
+
kind: 'cloud',
|
|
134
|
+
auth: 'api-key',
|
|
135
|
+
keychainService: 'com.maude.app.gemini',
|
|
136
|
+
modalities: ['image', 'video'],
|
|
137
|
+
keyUrl: 'https://aistudio.google.com/apikey',
|
|
138
|
+
notes:
|
|
139
|
+
'Nano Banana image generation + Veo video (async, native synced audio + image-to-video). Your Google AI Studio key bills your own Google account. Generated media may carry a SynthID watermark; review Google’s usage terms for commercial use. A video clip takes 1–several minutes.',
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
export const GEMINI_MODELS: ModelDescriptor[] = [
|
|
143
|
+
{
|
|
144
|
+
id: 'gemini-2.5-flash-image',
|
|
145
|
+
label: 'Nano Banana (2.5 Flash Image)',
|
|
146
|
+
modality: 'image',
|
|
147
|
+
aspectRatios: ['1:1', '2:3', '3:2', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9'],
|
|
148
|
+
costNote: '~$0.04/image',
|
|
149
|
+
sync: true,
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
id: 'gemini-3-pro-image-preview',
|
|
153
|
+
label: 'Nano Banana Pro (3 Pro Image)',
|
|
154
|
+
modality: 'image',
|
|
155
|
+
aspectRatios: ['1:1', '2:3', '3:2', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9', '21:9'],
|
|
156
|
+
costNote: 'higher — 1K/2K/4K',
|
|
157
|
+
sync: true,
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: 'veo-3.1-generate-preview',
|
|
161
|
+
label: 'Veo 3.1 (video, native audio)',
|
|
162
|
+
modality: 'video',
|
|
163
|
+
aspectRatios: ['16:9', '9:16'],
|
|
164
|
+
costNote: 'per second — see Google pricing',
|
|
165
|
+
sync: false,
|
|
166
|
+
},
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
const DEFAULT_MODEL = 'gemini-2.5-flash-image';
|
|
170
|
+
const DEFAULT_VIDEO_MODEL = 'veo-3.1-generate-preview';
|
|
171
|
+
// A Veo clip takes 1–several minutes; bound the poll so a stuck operation can't
|
|
172
|
+
// pin a job slot forever. Env-overridable for slower models / long clips.
|
|
173
|
+
const VIDEO_POLL_TIMEOUT_MS = Math.max(
|
|
174
|
+
60_000,
|
|
175
|
+
Number(process.env.MAUDE_VEO_TIMEOUT_MS) || 10 * 60 * 1000
|
|
176
|
+
);
|
|
177
|
+
const VIDEO_POLL_INTERVAL_MS = 10_000;
|
|
178
|
+
/** Poll cadence, read at CALL time so a test can shrink it via env. */
|
|
179
|
+
function videoPollInterval(): number {
|
|
180
|
+
return Math.max(1, Number(process.env.MAUDE_VEO_POLL_INTERVAL_MS) || VIDEO_POLL_INTERVAL_MS);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** The subset of the Gemini generateContent response we read. */
|
|
184
|
+
interface GeminiResponse {
|
|
185
|
+
candidates?: Array<{
|
|
186
|
+
content?: { parts?: Array<{ inlineData?: { mimeType?: string; data?: string } }> };
|
|
187
|
+
finishReason?: string;
|
|
188
|
+
}>;
|
|
189
|
+
promptFeedback?: { blockReason?: string };
|
|
190
|
+
error?: { message?: string };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** The subset of the Veo long-running-operation response we read. Veo's exact
|
|
194
|
+
* shape has shifted across previews, so `extractVideoUri` walks it defensively. */
|
|
195
|
+
interface VeoOperation {
|
|
196
|
+
name?: string;
|
|
197
|
+
done?: boolean;
|
|
198
|
+
error?: { message?: string };
|
|
199
|
+
response?: unknown;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Pull the operation name from a predictLongRunning start response. */
|
|
203
|
+
export function extractOperationName(json: unknown): string | null {
|
|
204
|
+
if (json && typeof json === 'object' && typeof (json as { name?: unknown }).name === 'string')
|
|
205
|
+
return (json as { name: string }).name;
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Walk a done Veo operation for the generated video URI. Veo has returned it at
|
|
211
|
+
* `response.generateVideoResponse.generatedSamples[].video.uri` and at
|
|
212
|
+
* `...generatedVideos[].video.uri` across previews — so recursively find the
|
|
213
|
+
* first `{ uri | fileUri }` string under any `video`/`file` object. Pure +
|
|
214
|
+
* exported so the parse is unit-testable without a live operation.
|
|
215
|
+
*/
|
|
216
|
+
export function extractVideoUri(op: VeoOperation): string | null {
|
|
217
|
+
const seen = new Set<unknown>();
|
|
218
|
+
const walk = (node: unknown): string | null => {
|
|
219
|
+
if (!node || typeof node !== 'object' || seen.has(node)) return null;
|
|
220
|
+
seen.add(node);
|
|
221
|
+
const o = node as Record<string, unknown>;
|
|
222
|
+
for (const key of ['uri', 'fileUri', 'videoUri']) {
|
|
223
|
+
const v = o[key];
|
|
224
|
+
if (typeof v === 'string' && /^https:\/\//.test(v)) return v;
|
|
225
|
+
}
|
|
226
|
+
for (const v of Object.values(o)) {
|
|
227
|
+
if (Array.isArray(v)) {
|
|
228
|
+
for (const item of v) {
|
|
229
|
+
const hit = walk(item);
|
|
230
|
+
if (hit) return hit;
|
|
231
|
+
}
|
|
232
|
+
} else if (v && typeof v === 'object') {
|
|
233
|
+
const hit = walk(v);
|
|
234
|
+
if (hit) return hit;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return null;
|
|
238
|
+
};
|
|
239
|
+
return walk(op.response);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Build the request body — text prompt + optional source image (maskless edit,
|
|
244
|
+
* Nano Banana's strength: the source is read into an `inlineData` part alongside
|
|
245
|
+
* the text) + optional aspect config. Async because the source-asset read is
|
|
246
|
+
* host-provided (ctx.readSourceAsset) so the adapter never touches the FS.
|
|
247
|
+
*/
|
|
248
|
+
async function buildBody(req: GenRequest, ctx: AdapterContext): Promise<unknown> {
|
|
249
|
+
const parts: Array<Record<string, unknown>> = [];
|
|
250
|
+
|
|
251
|
+
// Edit flow (Task 1.2): the source image goes FIRST, then the instruction text
|
|
252
|
+
// (Gemini's documented image-editing order). Requires the host to have wired
|
|
253
|
+
// readSourceAsset; a sourceAsset with no reader is a hard error (never a
|
|
254
|
+
// silent text-only generation that ignores the requested edit).
|
|
255
|
+
if (req.sourceAsset) {
|
|
256
|
+
if (!ctx.readSourceAsset) {
|
|
257
|
+
throw new Error('image editing requires source-asset access, which is not wired');
|
|
258
|
+
}
|
|
259
|
+
const src = await ctx.readSourceAsset(req.sourceAsset);
|
|
260
|
+
if (!src) throw new Error(`source asset not found or unreadable: ${req.sourceAsset}`);
|
|
261
|
+
if (!src.mime.startsWith('image/')) {
|
|
262
|
+
throw new Error(`source asset is not an image (${src.mime}) — cannot edit`);
|
|
263
|
+
}
|
|
264
|
+
parts.push({
|
|
265
|
+
inlineData: { mimeType: src.mime, data: Buffer.from(src.bytes).toString('base64') },
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (req.prompt) parts.push({ text: req.prompt });
|
|
270
|
+
|
|
271
|
+
const body: Record<string, unknown> = {
|
|
272
|
+
contents: [{ role: 'user', parts }],
|
|
273
|
+
generationConfig: {
|
|
274
|
+
responseModalities: ['IMAGE'],
|
|
275
|
+
...(req.aspectRatio ? { imageConfig: { aspectRatio: req.aspectRatio } } : {}),
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
return body;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Resolve the model, defaulting when the request left it blank. */
|
|
282
|
+
function resolveModel(req: GenRequest): string {
|
|
283
|
+
if (req.model && GEMINI_MODELS.some((m) => m.id === req.model)) return req.model;
|
|
284
|
+
return DEFAULT_MODEL;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** A sync Job wrapper — already resolved, so callers never branch on async. */
|
|
288
|
+
function doneJob(id: string, result: Promise<GenResult>): Job {
|
|
289
|
+
let settledStatus: 'done' | 'failed' = 'done';
|
|
290
|
+
const guarded = result.catch((err) => {
|
|
291
|
+
settledStatus = 'failed';
|
|
292
|
+
throw err;
|
|
293
|
+
});
|
|
294
|
+
return {
|
|
295
|
+
id,
|
|
296
|
+
status: () => settledStatus,
|
|
297
|
+
async *events() {
|
|
298
|
+
try {
|
|
299
|
+
await guarded;
|
|
300
|
+
yield { status: 'done' as const };
|
|
301
|
+
} catch (err) {
|
|
302
|
+
yield {
|
|
303
|
+
status: 'failed' as const,
|
|
304
|
+
message: err instanceof Error ? err.message : String(err),
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
},
|
|
308
|
+
result: () => guarded,
|
|
309
|
+
cancel: () => {},
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** An ASYNC Job (Veo video): status starts `running`, `cancel()` aborts the
|
|
314
|
+
* in-flight poll/download via the shared controller. */
|
|
315
|
+
function runningJob(id: string, result: Promise<GenResult>, controller: AbortController): Job {
|
|
316
|
+
let status: 'running' | 'done' | 'failed' = 'running';
|
|
317
|
+
const guarded = result.then(
|
|
318
|
+
(r) => {
|
|
319
|
+
status = 'done';
|
|
320
|
+
return r;
|
|
321
|
+
},
|
|
322
|
+
(err) => {
|
|
323
|
+
status = 'failed';
|
|
324
|
+
throw err;
|
|
325
|
+
}
|
|
326
|
+
);
|
|
327
|
+
return {
|
|
328
|
+
id,
|
|
329
|
+
status: () => status,
|
|
330
|
+
async *events() {
|
|
331
|
+
yield { status: 'running' as const };
|
|
332
|
+
try {
|
|
333
|
+
await guarded;
|
|
334
|
+
yield { status: 'done' as const };
|
|
335
|
+
} catch (err) {
|
|
336
|
+
yield {
|
|
337
|
+
status: 'failed' as const,
|
|
338
|
+
message: err instanceof Error ? err.message : String(err),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
result: () => guarded,
|
|
343
|
+
cancel: () => controller.abort(),
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** Sleep that rejects promptly if the signal aborts (so cancel/timeout is snappy). */
|
|
348
|
+
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
349
|
+
return new Promise((resolve, reject) => {
|
|
350
|
+
if (signal.aborted) return reject(new Error('aborted'));
|
|
351
|
+
const t = setTimeout(resolve, ms);
|
|
352
|
+
signal.addEventListener(
|
|
353
|
+
'abort',
|
|
354
|
+
() => {
|
|
355
|
+
clearTimeout(t);
|
|
356
|
+
reject(new Error('aborted'));
|
|
357
|
+
},
|
|
358
|
+
{ once: true }
|
|
359
|
+
);
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
export function createGeminiAdapter(ctx: AdapterContext): ProviderAdapter {
|
|
364
|
+
async function runOnce(req: GenRequest): Promise<GenResult> {
|
|
365
|
+
if (!ctx.apiKey) {
|
|
366
|
+
throw new Error('no Google Gemini key configured — add one in Settings');
|
|
367
|
+
}
|
|
368
|
+
const model = resolveModel(req);
|
|
369
|
+
const started = Date.now();
|
|
370
|
+
// F2 — validate the outbound base (https + host allowlist) BEFORE the
|
|
371
|
+
// key-bearing request, so a poisoned env can't redirect the BYOK key.
|
|
372
|
+
assertSafeBase(GEMINI_API_BASE);
|
|
373
|
+
const url = `${GEMINI_API_BASE}/models/${encodeURIComponent(model)}:generateContent`;
|
|
374
|
+
|
|
375
|
+
// Build the body first (an edit flow reads the source asset host-side here).
|
|
376
|
+
const requestBody = JSON.stringify(await buildBody(req, ctx));
|
|
377
|
+
|
|
378
|
+
let res: Response;
|
|
379
|
+
try {
|
|
380
|
+
res = await fetch(url, {
|
|
381
|
+
method: 'POST',
|
|
382
|
+
headers: {
|
|
383
|
+
'content-type': 'application/json',
|
|
384
|
+
// Key ONLY in the header — never the query string (redirect/log leak).
|
|
385
|
+
'x-goog-api-key': ctx.apiKey,
|
|
386
|
+
},
|
|
387
|
+
body: requestBody,
|
|
388
|
+
signal: ctx.signal ?? AbortSignal.timeout(120_000),
|
|
389
|
+
});
|
|
390
|
+
} catch (err) {
|
|
391
|
+
throw new Error(`Gemini request failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// F1 — bound the response before decode so an oversized upstream body can't
|
|
395
|
+
// OOM-crash the shared server (saveAsset's cap only guards the disk write,
|
|
396
|
+
// which is downstream of this full in-RAM allocation).
|
|
397
|
+
let json: GeminiResponse | null = null;
|
|
398
|
+
try {
|
|
399
|
+
const text = await readTextCapped(res, MAX_RESPONSE_BYTES);
|
|
400
|
+
json = JSON.parse(text) as GeminiResponse;
|
|
401
|
+
} catch (err) {
|
|
402
|
+
// A too-large body is a hard error; a parse failure on an error status
|
|
403
|
+
// falls through to the status-based message below.
|
|
404
|
+
if (err instanceof Error && /too large|exceeded/.test(err.message)) throw err;
|
|
405
|
+
json = null;
|
|
406
|
+
}
|
|
407
|
+
if (!res.ok) {
|
|
408
|
+
// Never surface the key; Gemini's own error message is safe to relay.
|
|
409
|
+
const msg = json?.error?.message ?? `HTTP ${res.status}`;
|
|
410
|
+
throw new Error(`Gemini error: ${msg}`);
|
|
411
|
+
}
|
|
412
|
+
if (json?.promptFeedback?.blockReason) {
|
|
413
|
+
throw new Error(`Gemini blocked the prompt: ${json.promptFeedback.blockReason}`);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const parts = json?.candidates?.[0]?.content?.parts ?? [];
|
|
417
|
+
const assets: GenAsset[] = [];
|
|
418
|
+
for (const part of parts) {
|
|
419
|
+
const inline = part.inlineData;
|
|
420
|
+
if (inline?.data) {
|
|
421
|
+
assets.push({
|
|
422
|
+
kind: 'image',
|
|
423
|
+
mime: inline.mimeType ?? 'image/png',
|
|
424
|
+
bytes: new Uint8Array(Buffer.from(inline.data, 'base64')),
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if (assets.length === 0) {
|
|
429
|
+
throw new Error('Gemini returned no image (the model may have refused the prompt)');
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
return { assets, usage: { ms: Date.now() - started }, raw: json };
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Veo video (ASYNC): start a long-running operation, poll it to completion,
|
|
437
|
+
* then download the produced MP4 WITH the key (the Veo URI is on Google's host
|
|
438
|
+
* and requires `x-goog-api-key` — so the ADAPTER downloads it, mirroring
|
|
439
|
+
* ElevenLabs history-audio, rather than handing a key-bearing URL to
|
|
440
|
+
* download.ts). Returns the bytes as a normal video GenAsset the host
|
|
441
|
+
* localizes into assets/<sha8>.mp4.
|
|
442
|
+
*/
|
|
443
|
+
async function runVideo(req: GenRequest, signal: AbortSignal): Promise<GenResult> {
|
|
444
|
+
if (!ctx.apiKey) throw new Error('no Google Gemini key configured — add one in Settings');
|
|
445
|
+
if (!req.prompt) throw new Error('video generation requires a prompt');
|
|
446
|
+
assertSafeBase(GEMINI_API_BASE);
|
|
447
|
+
const started = Date.now();
|
|
448
|
+
const model =
|
|
449
|
+
req.model && GEMINI_MODELS.some((m) => m.id === req.model && m.modality === 'video')
|
|
450
|
+
? req.model
|
|
451
|
+
: DEFAULT_VIDEO_MODEL;
|
|
452
|
+
|
|
453
|
+
// Optional image-to-video seed (Task 3.2 seeds a generated still here).
|
|
454
|
+
let image: { mimeType: string; bytesBase64Encoded: string } | undefined;
|
|
455
|
+
if (req.sourceAsset) {
|
|
456
|
+
if (!ctx.readSourceAsset) throw new Error('image-to-video requires source-asset access');
|
|
457
|
+
const src = await ctx.readSourceAsset(req.sourceAsset);
|
|
458
|
+
if (!src) throw new Error(`source asset not found or unreadable: ${req.sourceAsset}`);
|
|
459
|
+
if (!src.mime.startsWith('image/'))
|
|
460
|
+
throw new Error(`i2v seed must be an image (${src.mime})`);
|
|
461
|
+
image = { mimeType: src.mime, bytesBase64Encoded: Buffer.from(src.bytes).toString('base64') };
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const startBody = JSON.stringify({
|
|
465
|
+
instances: [{ prompt: req.prompt, ...(image ? { image } : {}) }],
|
|
466
|
+
parameters: { ...(req.aspectRatio ? { aspectRatio: req.aspectRatio } : {}) },
|
|
467
|
+
});
|
|
468
|
+
const startRes = await fetch(
|
|
469
|
+
`${GEMINI_API_BASE}/models/${encodeURIComponent(model)}:predictLongRunning`,
|
|
470
|
+
{
|
|
471
|
+
method: 'POST',
|
|
472
|
+
headers: { 'content-type': 'application/json', 'x-goog-api-key': ctx.apiKey },
|
|
473
|
+
body: startBody,
|
|
474
|
+
signal,
|
|
475
|
+
}
|
|
476
|
+
);
|
|
477
|
+
const startJson = JSON.parse(await readTextCapped(startRes, MAX_RESPONSE_BYTES));
|
|
478
|
+
if (!startRes.ok)
|
|
479
|
+
throw new Error(`Veo start error: ${startJson?.error?.message ?? `HTTP ${startRes.status}`}`);
|
|
480
|
+
const opName = extractOperationName(startJson);
|
|
481
|
+
if (!opName) throw new Error('Veo did not return an operation name');
|
|
482
|
+
|
|
483
|
+
// Poll the operation (10s cadence) until done or the wall-clock cap.
|
|
484
|
+
const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
|
|
485
|
+
const interval = videoPollInterval();
|
|
486
|
+
let op: VeoOperation | null = null;
|
|
487
|
+
for (;;) {
|
|
488
|
+
await abortableSleep(interval, signal);
|
|
489
|
+
const pollRes = await fetch(`${GEMINI_API_BASE}/${opName}`, {
|
|
490
|
+
headers: { 'x-goog-api-key': ctx.apiKey },
|
|
491
|
+
signal,
|
|
492
|
+
});
|
|
493
|
+
op = JSON.parse(await readTextCapped(pollRes, MAX_RESPONSE_BYTES)) as VeoOperation;
|
|
494
|
+
if (!pollRes.ok)
|
|
495
|
+
throw new Error(`Veo poll error: ${op?.error?.message ?? `HTTP ${pollRes.status}`}`);
|
|
496
|
+
if (op.error) throw new Error(`Veo failed: ${op.error.message ?? 'unknown'}`);
|
|
497
|
+
if (op.done) break;
|
|
498
|
+
if (Date.now() >= deadline) throw new Error('Veo timed out (operation still running)');
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
const uri = extractVideoUri(op);
|
|
502
|
+
if (!uri) throw new Error('Veo operation finished but produced no video URI');
|
|
503
|
+
// Download the MP4 with the key. Re-assert https + the Google host (the URI
|
|
504
|
+
// is provider-issued, but a compromised/rerouted response must not exfiltrate
|
|
505
|
+
// the key to another host).
|
|
506
|
+
const u = new URL(uri);
|
|
507
|
+
if (u.protocol !== 'https:' || u.hostname !== GEMINI_ALLOWED_HOST)
|
|
508
|
+
throw new Error(`Veo video URI host not allowlisted (${u.hostname})`);
|
|
509
|
+
const dlRes = await fetch(uri, { headers: { 'x-goog-api-key': ctx.apiKey }, signal });
|
|
510
|
+
if (!dlRes.ok) throw new Error(`Veo video download failed: HTTP ${dlRes.status}`);
|
|
511
|
+
const bytes = await readBytesCapped(dlRes, MAX_VIDEO_BYTES);
|
|
512
|
+
if (bytes.byteLength === 0) throw new Error('Veo returned an empty video');
|
|
513
|
+
|
|
514
|
+
const asset: GenAsset = {
|
|
515
|
+
kind: 'video',
|
|
516
|
+
mime: dlRes.headers.get('content-type') || 'video/mp4',
|
|
517
|
+
bytes,
|
|
518
|
+
};
|
|
519
|
+
return { assets: [asset], usage: { ms: Date.now() - started }, raw: op };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
return {
|
|
523
|
+
descriptor: GEMINI_DESCRIPTOR,
|
|
524
|
+
async listModels() {
|
|
525
|
+
return GEMINI_MODELS;
|
|
526
|
+
},
|
|
527
|
+
async submit(req: GenRequest): Promise<Job> {
|
|
528
|
+
const id = `gen_${crypto.randomUUID()}`;
|
|
529
|
+
if (req.modality === 'video') {
|
|
530
|
+
// Async provider — a real running Job that polls the operation. cancel()
|
|
531
|
+
// aborts the poll/download; the queue's own timeout also flows in.
|
|
532
|
+
const controller = new AbortController();
|
|
533
|
+
if (ctx.signal)
|
|
534
|
+
ctx.signal.addEventListener('abort', () => controller.abort(), { once: true });
|
|
535
|
+
return runningJob(id, runVideo(req, controller.signal), controller);
|
|
536
|
+
}
|
|
537
|
+
// Sync image — kick off the single call and wrap it in an already-resolving
|
|
538
|
+
// Job so the queue treats it uniformly with async providers.
|
|
539
|
+
return doneJob(id, runOnce(req));
|
|
540
|
+
},
|
|
541
|
+
};
|
|
542
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// groq.test.ts — Groq Whisper STT adapter (Task 2.6). Covers the pure
|
|
2
|
+
// verbose_json → CaptionWord[] normalization (word + segment fallback), the
|
|
3
|
+
// descriptor's transcription-only capability, and the submit guard that rejects
|
|
4
|
+
// a non-transcription modality. The network call itself is a live-key gate.
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from 'bun:test';
|
|
7
|
+
|
|
8
|
+
import { wordsToSrt } from '../captions.ts';
|
|
9
|
+
import type { AdapterContext } from '../types.ts';
|
|
10
|
+
import { createGroqAdapter, GROQ_DESCRIPTOR, GROQ_MODELS, groqVerboseToWords } from './groq.ts';
|
|
11
|
+
|
|
12
|
+
const ctx: AdapterContext = {
|
|
13
|
+
apiKey: 'test-key',
|
|
14
|
+
localize: async () => 'assets/00000000.mp3',
|
|
15
|
+
readSourceAsset: async () => ({ bytes: new Uint8Array([1, 2, 3]), mime: 'audio/mpeg' }),
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
describe('groqVerboseToWords', () => {
|
|
19
|
+
test('maps per-word timings (seconds) and trims text', () => {
|
|
20
|
+
const words = groqVerboseToWords({
|
|
21
|
+
words: [
|
|
22
|
+
{ word: ' Hello', start: 0, end: 0.4 },
|
|
23
|
+
{ word: 'there.', start: 0.4, end: 0.9 },
|
|
24
|
+
{ word: ' ', start: 1, end: 1 }, // blank → dropped
|
|
25
|
+
],
|
|
26
|
+
});
|
|
27
|
+
expect(words).toHaveLength(2);
|
|
28
|
+
expect(words[0]).toEqual({ text: 'Hello', start: 0, end: 0.4 });
|
|
29
|
+
expect(words[1].text).toBe('there.');
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test('falls back to per-segment timings when word granularity is absent', () => {
|
|
33
|
+
const words = groqVerboseToWords({
|
|
34
|
+
segments: [{ text: ' A whole segment ', start: 2, end: 4 }],
|
|
35
|
+
});
|
|
36
|
+
expect(words).toEqual([{ text: 'A whole segment', start: 2, end: 4 }]);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('feeds captions.ts to produce a valid SRT', () => {
|
|
40
|
+
const srt = wordsToSrt(
|
|
41
|
+
groqVerboseToWords({
|
|
42
|
+
words: [
|
|
43
|
+
{ word: 'Hello', start: 0, end: 0.4 },
|
|
44
|
+
{ word: 'world.', start: 0.4, end: 0.9 },
|
|
45
|
+
],
|
|
46
|
+
})
|
|
47
|
+
);
|
|
48
|
+
expect(srt).toContain('00:00:00,000 --> 00:00:00,900');
|
|
49
|
+
expect(srt).toContain('Hello world.');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test('empty input yields no words (never throws)', () => {
|
|
53
|
+
expect(groqVerboseToWords({})).toEqual([]);
|
|
54
|
+
expect(groqVerboseToWords({ words: [] })).toEqual([]);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
describe('GROQ_DESCRIPTOR', () => {
|
|
59
|
+
test('is a cloud, api-key, transcription-only provider', () => {
|
|
60
|
+
expect(GROQ_DESCRIPTOR.id).toBe('groq');
|
|
61
|
+
expect(GROQ_DESCRIPTOR.kind).toBe('cloud');
|
|
62
|
+
expect(GROQ_DESCRIPTOR.auth).toBe('api-key');
|
|
63
|
+
expect(GROQ_DESCRIPTOR.modalities).toEqual(['transcription']);
|
|
64
|
+
expect(GROQ_MODELS.every((m) => m.modality === 'transcription')).toBe(true);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
describe('createGroqAdapter.submit', () => {
|
|
69
|
+
test('rejects a non-transcription modality without a network call', async () => {
|
|
70
|
+
const adapter = createGroqAdapter(ctx);
|
|
71
|
+
const job = await adapter.submit({ modality: 'image', provider: 'groq', prompt: 'x' });
|
|
72
|
+
await expect(job.result()).rejects.toThrow(/only supports transcription/);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('transcription requires a sourceAsset', async () => {
|
|
76
|
+
const adapter = createGroqAdapter({ ...ctx, readSourceAsset: undefined });
|
|
77
|
+
const job = await adapter.submit({ modality: 'transcription', provider: 'groq' });
|
|
78
|
+
await expect(job.result()).rejects.toThrow(/sourceAsset/);
|
|
79
|
+
});
|
|
80
|
+
});
|