@astrofoundry/pi-astro 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/extensions/astro-agents/agents/code-reviewer.md +0 -2
- package/extensions/astro-agents/agents/google-tech-lead.md +0 -2
- package/extensions/astro-agents/agents/spec-writer.md +0 -2
- package/extensions/astro-agents/agents/tester-api.md +0 -2
- package/extensions/astro-agents/agents/tester-ui.md +0 -2
- package/extensions/astro-agents/agents/ui-architect.md +0 -2
- package/extensions/astro-agents/agents/ui-design-system.md +0 -2
- package/extensions/astro-agents/agents/ui-frontend-developer.md +0 -2
- package/extensions/astro-agents/discovery.test.ts +152 -0
- package/extensions/astro-agents/index.test.ts +208 -0
- package/extensions/astro-agents/index.ts +22 -4
- package/extensions/astro-agents/spawn.test.ts +218 -0
- package/extensions/claude-globals/index.test.ts +77 -0
- package/extensions/gemini-image/credentials.test.ts +130 -0
- package/extensions/gemini-image/credentials.ts +53 -0
- package/extensions/gemini-image/index.test.ts +369 -0
- package/extensions/gemini-image/index.ts +313 -0
- package/extensions/gemini-image/models.test.ts +45 -0
- package/extensions/gemini-image/models.ts +50 -0
- package/extensions/gemini-image/pricing.test.ts +95 -0
- package/extensions/gemini-image/pricing.ts +102 -0
- package/extensions/grimoire/index.test.ts +244 -0
- package/extensions/multi-edit/classic.test.ts +274 -0
- package/extensions/multi-edit/classic.ts +435 -0
- package/extensions/multi-edit/diff.test.ts +65 -0
- package/extensions/multi-edit/diff.ts +143 -0
- package/extensions/multi-edit/index.test.ts +170 -0
- package/extensions/multi-edit/index.ts +267 -0
- package/extensions/multi-edit/patch.test.ts +242 -0
- package/extensions/multi-edit/patch.ts +463 -0
- package/extensions/multi-edit/types.ts +53 -0
- package/extensions/multi-edit/workspace.test.ts +165 -0
- package/extensions/multi-edit/workspace.ts +85 -0
- package/package.json +9 -3
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
5
|
+
|
|
6
|
+
const generateContentMock = vi.fn();
|
|
7
|
+
const generateImagesMock = vi.fn();
|
|
8
|
+
|
|
9
|
+
vi.mock("@google/genai", () => {
|
|
10
|
+
class FakeGoogleGenAI {
|
|
11
|
+
models = {
|
|
12
|
+
generateContent: generateContentMock,
|
|
13
|
+
generateImages: generateImagesMock,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
return { GoogleGenAI: FakeGoogleGenAI };
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
type ToolDef = {
|
|
20
|
+
name: string;
|
|
21
|
+
execute: (
|
|
22
|
+
id: string,
|
|
23
|
+
input: Record<string, unknown>,
|
|
24
|
+
signal: AbortSignal | undefined,
|
|
25
|
+
onUpdate: unknown,
|
|
26
|
+
ctx: {
|
|
27
|
+
cwd: string;
|
|
28
|
+
hasUI: boolean;
|
|
29
|
+
ui: {
|
|
30
|
+
notify: ReturnType<typeof vi.fn>;
|
|
31
|
+
confirm: ReturnType<typeof vi.fn>;
|
|
32
|
+
input: ReturnType<typeof vi.fn>;
|
|
33
|
+
};
|
|
34
|
+
},
|
|
35
|
+
) => Promise<{ content: unknown[]; details: Record<string, unknown> }>;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
function makePi(): { tools: ToolDef[]; registerTool: (t: ToolDef) => void } {
|
|
39
|
+
const tools: ToolDef[] = [];
|
|
40
|
+
return { tools, registerTool: (t) => tools.push(t) };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function makeCtx(override: Partial<{ hasUI: boolean; confirm: boolean; inputValue: string | undefined }> = {}): {
|
|
44
|
+
cwd: string;
|
|
45
|
+
hasUI: boolean;
|
|
46
|
+
ui: {
|
|
47
|
+
notify: ReturnType<typeof vi.fn>;
|
|
48
|
+
confirm: ReturnType<typeof vi.fn>;
|
|
49
|
+
input: ReturnType<typeof vi.fn>;
|
|
50
|
+
};
|
|
51
|
+
} {
|
|
52
|
+
const hasUI = override.hasUI ?? true;
|
|
53
|
+
return {
|
|
54
|
+
cwd: "/tmp/cwd",
|
|
55
|
+
hasUI,
|
|
56
|
+
ui: {
|
|
57
|
+
notify: vi.fn(),
|
|
58
|
+
confirm: vi.fn().mockResolvedValue(override.confirm ?? true),
|
|
59
|
+
input: vi.fn().mockResolvedValue(override.inputValue ?? "abc-key-from-paste"),
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
describe("gemini-image extension", () => {
|
|
65
|
+
let fakeHome: string;
|
|
66
|
+
const originalHome = process.env.HOME;
|
|
67
|
+
const originalKey = process.env.GEMINI_API_KEY;
|
|
68
|
+
|
|
69
|
+
beforeEach(() => {
|
|
70
|
+
fakeHome = mkdtempSync(join(tmpdir(), "gem-ext-"));
|
|
71
|
+
process.env.HOME = fakeHome;
|
|
72
|
+
process.env.GEMINI_API_KEY = "test-key";
|
|
73
|
+
vi.resetModules();
|
|
74
|
+
generateContentMock.mockReset();
|
|
75
|
+
generateImagesMock.mockReset();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
afterEach(() => {
|
|
79
|
+
rmSync(fakeHome, { recursive: true, force: true });
|
|
80
|
+
process.env.HOME = originalHome;
|
|
81
|
+
if (originalKey === undefined) delete process.env.GEMINI_API_KEY;
|
|
82
|
+
else process.env.GEMINI_API_KEY = originalKey;
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
async function load(): Promise<{ tool: ToolDef }> {
|
|
86
|
+
const mod = await import("./index.ts");
|
|
87
|
+
const pi = makePi();
|
|
88
|
+
mod.default(pi as unknown as Parameters<typeof mod.default>[0]);
|
|
89
|
+
return { tool: pi.tools[0] };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
it("registers a tool named gemini_image", async () => {
|
|
93
|
+
const { tool } = await load();
|
|
94
|
+
expect(tool.name).toBe("gemini_image");
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("uses default model gemini-2.5-flash-image and calls generateContent", async () => {
|
|
98
|
+
generateContentMock.mockResolvedValue({
|
|
99
|
+
candidates: [
|
|
100
|
+
{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "BASE64DATA" } }] } },
|
|
101
|
+
],
|
|
102
|
+
});
|
|
103
|
+
const { tool } = await load();
|
|
104
|
+
const ctx = makeCtx();
|
|
105
|
+
const res = await tool.execute("t", { prompt: "a cat" }, undefined, undefined, ctx);
|
|
106
|
+
expect(res.details.model).toBe("gemini-2.5-flash-image");
|
|
107
|
+
expect(res.content[0]).toMatchObject({ type: "image", data: "BASE64DATA" });
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("calls Imagen for imagen-4.0-generate-001 with numberOfImages", async () => {
|
|
111
|
+
generateImagesMock.mockResolvedValue({
|
|
112
|
+
generatedImages: [
|
|
113
|
+
{ image: { imageBytes: "IMG1", mimeType: "image/png" } },
|
|
114
|
+
{ image: { imageBytes: "IMG2", mimeType: "image/png" } },
|
|
115
|
+
],
|
|
116
|
+
});
|
|
117
|
+
const { tool } = await load();
|
|
118
|
+
const ctx = makeCtx();
|
|
119
|
+
const res = await tool.execute(
|
|
120
|
+
"t",
|
|
121
|
+
{
|
|
122
|
+
prompt: "landscape",
|
|
123
|
+
model: "imagen-4.0-generate-001",
|
|
124
|
+
number_of_images: 2,
|
|
125
|
+
},
|
|
126
|
+
undefined,
|
|
127
|
+
undefined,
|
|
128
|
+
ctx,
|
|
129
|
+
);
|
|
130
|
+
expect(res.details.images).toBe(2);
|
|
131
|
+
const args = generateImagesMock.mock.calls[0][0];
|
|
132
|
+
expect(args.config.numberOfImages).toBe(2);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("rejects unknown model", async () => {
|
|
136
|
+
const { tool } = await load();
|
|
137
|
+
await expect(
|
|
138
|
+
tool.execute("t", { prompt: "x", model: "bogus-model" }, undefined, undefined, makeCtx()),
|
|
139
|
+
).rejects.toThrow(/Unknown model/);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("rejects invalid aspect_ratio for Imagen (e.g. 21:9)", async () => {
|
|
143
|
+
const { tool } = await load();
|
|
144
|
+
await expect(
|
|
145
|
+
tool.execute(
|
|
146
|
+
"t",
|
|
147
|
+
{ prompt: "x", model: "imagen-4.0-generate-001", aspect_ratio: "21:9" },
|
|
148
|
+
undefined,
|
|
149
|
+
undefined,
|
|
150
|
+
makeCtx(),
|
|
151
|
+
),
|
|
152
|
+
).rejects.toThrow(/aspect_ratio .* Imagen/);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("rejects 4K image_size for Imagen", async () => {
|
|
156
|
+
const { tool } = await load();
|
|
157
|
+
await expect(
|
|
158
|
+
tool.execute(
|
|
159
|
+
"t",
|
|
160
|
+
{ prompt: "x", model: "imagen-4.0-generate-001", image_size: "4K" },
|
|
161
|
+
undefined,
|
|
162
|
+
undefined,
|
|
163
|
+
makeCtx(),
|
|
164
|
+
),
|
|
165
|
+
).rejects.toThrow(/image_size .* Imagen/);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("rejects input_images on Imagen", async () => {
|
|
169
|
+
const { tool } = await load();
|
|
170
|
+
await expect(
|
|
171
|
+
tool.execute(
|
|
172
|
+
"t",
|
|
173
|
+
{
|
|
174
|
+
prompt: "x",
|
|
175
|
+
model: "imagen-4.0-generate-001",
|
|
176
|
+
input_images: ["/tmp/x.png"],
|
|
177
|
+
},
|
|
178
|
+
undefined,
|
|
179
|
+
undefined,
|
|
180
|
+
makeCtx(),
|
|
181
|
+
),
|
|
182
|
+
).rejects.toThrow(/input_images/);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("rejects negative_prompt on Gemini native", async () => {
|
|
186
|
+
const { tool } = await load();
|
|
187
|
+
await expect(
|
|
188
|
+
tool.execute(
|
|
189
|
+
"t",
|
|
190
|
+
{ prompt: "x", negative_prompt: "ugly" },
|
|
191
|
+
undefined,
|
|
192
|
+
undefined,
|
|
193
|
+
makeCtx(),
|
|
194
|
+
),
|
|
195
|
+
).rejects.toThrow(/negative_prompt/);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it("rejects seed on Gemini native", async () => {
|
|
199
|
+
const { tool } = await load();
|
|
200
|
+
await expect(
|
|
201
|
+
tool.execute("t", { prompt: "x", seed: 42 }, undefined, undefined, makeCtx()),
|
|
202
|
+
).rejects.toThrow(/seed/);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
it("rejects number_of_images > 1 on Gemini native", async () => {
|
|
206
|
+
const { tool } = await load();
|
|
207
|
+
await expect(
|
|
208
|
+
tool.execute("t", { prompt: "x", number_of_images: 3 }, undefined, undefined, makeCtx()),
|
|
209
|
+
).rejects.toThrow(/number_of_images/);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it("prompts for API key when none configured and saves if confirmed", async () => {
|
|
213
|
+
delete process.env.GEMINI_API_KEY;
|
|
214
|
+
generateContentMock.mockResolvedValue({
|
|
215
|
+
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "X" } }] } }],
|
|
216
|
+
});
|
|
217
|
+
const { tool } = await load();
|
|
218
|
+
const ctx = makeCtx({ inputValue: "pasted-key" });
|
|
219
|
+
await tool.execute("t", { prompt: "x" }, undefined, undefined, ctx);
|
|
220
|
+
expect(ctx.ui.input).toHaveBeenCalled();
|
|
221
|
+
// Confirm was called for both the save prompt and the cost confirmation
|
|
222
|
+
expect(ctx.ui.confirm.mock.calls.length).toBeGreaterThanOrEqual(1);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it("errors when API key missing and no UI available", async () => {
|
|
226
|
+
delete process.env.GEMINI_API_KEY;
|
|
227
|
+
const { tool } = await load();
|
|
228
|
+
const ctx = makeCtx({ hasUI: false });
|
|
229
|
+
await expect(tool.execute("t", { prompt: "x" }, undefined, undefined, ctx)).rejects.toThrow(/no interactive UI/);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
it("errors when user provides empty key at prompt", async () => {
|
|
233
|
+
delete process.env.GEMINI_API_KEY;
|
|
234
|
+
const { tool } = await load();
|
|
235
|
+
const ctx = makeCtx({ inputValue: " " });
|
|
236
|
+
await expect(tool.execute("t", { prompt: "x" }, undefined, undefined, ctx)).rejects.toThrow(/No API key/);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it("cost confirmation: throws when user rejects", async () => {
|
|
240
|
+
generateContentMock.mockResolvedValue({
|
|
241
|
+
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "X" } }] } }],
|
|
242
|
+
});
|
|
243
|
+
const { tool } = await load();
|
|
244
|
+
const ctx = makeCtx({ confirm: false });
|
|
245
|
+
await expect(tool.execute("t", { prompt: "x" }, undefined, undefined, ctx)).rejects.toThrow(/User cancelled/);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it("skip_confirm bypasses cost prompt", async () => {
|
|
249
|
+
generateContentMock.mockResolvedValue({
|
|
250
|
+
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "X" } }] } }],
|
|
251
|
+
});
|
|
252
|
+
const { tool } = await load();
|
|
253
|
+
const ctx = makeCtx();
|
|
254
|
+
await tool.execute("t", { prompt: "x", skip_confirm: true }, undefined, undefined, ctx);
|
|
255
|
+
// confirm should not be called for cost
|
|
256
|
+
expect(ctx.ui.confirm).not.toHaveBeenCalled();
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it("save_to writes PNG files and reports paths", async () => {
|
|
260
|
+
generateImagesMock.mockResolvedValue({
|
|
261
|
+
generatedImages: [
|
|
262
|
+
{ image: { imageBytes: Buffer.from("fakeimage").toString("base64"), mimeType: "image/png" } },
|
|
263
|
+
],
|
|
264
|
+
});
|
|
265
|
+
const saveDir = mkdtempSync(join(tmpdir(), "gem-save-"));
|
|
266
|
+
const { tool } = await load();
|
|
267
|
+
const ctx = makeCtx();
|
|
268
|
+
const res = await tool.execute(
|
|
269
|
+
"t",
|
|
270
|
+
{
|
|
271
|
+
prompt: "a-landscape",
|
|
272
|
+
model: "imagen-4.0-fast-generate-001",
|
|
273
|
+
save_to: saveDir,
|
|
274
|
+
},
|
|
275
|
+
undefined,
|
|
276
|
+
undefined,
|
|
277
|
+
ctx,
|
|
278
|
+
);
|
|
279
|
+
const paths = res.details.savedPaths as string[];
|
|
280
|
+
expect(paths).toHaveLength(1);
|
|
281
|
+
expect(readdirSync(saveDir)).toHaveLength(1);
|
|
282
|
+
rmSync(saveDir, { recursive: true, force: true });
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it("throws when model returns no images", async () => {
|
|
286
|
+
generateContentMock.mockResolvedValue({ candidates: [] });
|
|
287
|
+
const { tool } = await load();
|
|
288
|
+
await expect(tool.execute("t", { prompt: "x" }, undefined, undefined, makeCtx())).rejects.toThrow(/no images/);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
it("input_images are read from disk and passed as inlineData", async () => {
|
|
292
|
+
const sourceDir = mkdtempSync(join(tmpdir(), "gem-input-"));
|
|
293
|
+
const imgPath = join(sourceDir, "input.png");
|
|
294
|
+
writeFileSync(imgPath, "binary-image");
|
|
295
|
+
generateContentMock.mockResolvedValue({
|
|
296
|
+
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "OUT" } }] } }],
|
|
297
|
+
});
|
|
298
|
+
const { tool } = await load();
|
|
299
|
+
const ctx = makeCtx();
|
|
300
|
+
await tool.execute(
|
|
301
|
+
"t",
|
|
302
|
+
{ prompt: "edit", input_images: [imgPath] },
|
|
303
|
+
undefined,
|
|
304
|
+
undefined,
|
|
305
|
+
ctx,
|
|
306
|
+
);
|
|
307
|
+
const callArgs = generateContentMock.mock.calls[0][0];
|
|
308
|
+
// First content is the text, subsequent are inlineData parts
|
|
309
|
+
expect(callArgs.contents.some((c: Record<string, unknown>) => "inlineData" in c)).toBe(true);
|
|
310
|
+
rmSync(sourceDir, { recursive: true, force: true });
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
it("passes aspect_ratio + image_size into imageConfig for Gemini native", async () => {
|
|
314
|
+
generateContentMock.mockResolvedValue({
|
|
315
|
+
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "X" } }] } }],
|
|
316
|
+
});
|
|
317
|
+
const { tool } = await load();
|
|
318
|
+
await tool.execute(
|
|
319
|
+
"t",
|
|
320
|
+
{ prompt: "x", aspect_ratio: "16:9", image_size: "2K" },
|
|
321
|
+
undefined,
|
|
322
|
+
undefined,
|
|
323
|
+
makeCtx(),
|
|
324
|
+
);
|
|
325
|
+
const cfg = generateContentMock.mock.calls[0][0].config;
|
|
326
|
+
expect(cfg.imageConfig.aspectRatio).toBe("16:9");
|
|
327
|
+
expect(cfg.imageConfig.imageSize).toBe("2K");
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
it("Imagen config: numberOfImages + negative_prompt + seed + person_generation all pass through", async () => {
|
|
331
|
+
generateImagesMock.mockResolvedValue({
|
|
332
|
+
generatedImages: [{ image: { imageBytes: "X", mimeType: "image/png" } }],
|
|
333
|
+
});
|
|
334
|
+
const { tool } = await load();
|
|
335
|
+
await tool.execute(
|
|
336
|
+
"t",
|
|
337
|
+
{
|
|
338
|
+
prompt: "x",
|
|
339
|
+
model: "imagen-4.0-ultra-generate-001",
|
|
340
|
+
number_of_images: 2,
|
|
341
|
+
negative_prompt: "blurry",
|
|
342
|
+
seed: 1337,
|
|
343
|
+
person_generation: "allow_adult",
|
|
344
|
+
},
|
|
345
|
+
undefined,
|
|
346
|
+
undefined,
|
|
347
|
+
makeCtx(),
|
|
348
|
+
);
|
|
349
|
+
const cfg = generateImagesMock.mock.calls[0][0].config;
|
|
350
|
+
expect(cfg.numberOfImages).toBe(2);
|
|
351
|
+
expect(cfg.negativePrompt).toBe("blurry");
|
|
352
|
+
expect(cfg.seed).toBe(1337);
|
|
353
|
+
expect(cfg.personGeneration).toBe("allow_adult");
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
it("rejects invalid Gemini aspect_ratio", async () => {
|
|
357
|
+
const { tool } = await load();
|
|
358
|
+
await expect(
|
|
359
|
+
tool.execute("t", { prompt: "x", aspect_ratio: "17:3" }, undefined, undefined, makeCtx()),
|
|
360
|
+
).rejects.toThrow(/aspect_ratio .* Gemini native/);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it("rejects invalid Gemini image_size", async () => {
|
|
364
|
+
const { tool } = await load();
|
|
365
|
+
await expect(
|
|
366
|
+
tool.execute("t", { prompt: "x", image_size: "8K" as "4K" }, undefined, undefined, makeCtx()),
|
|
367
|
+
).rejects.toThrow(/image_size/);
|
|
368
|
+
});
|
|
369
|
+
});
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
3
|
+
import type { AgentToolResult } from "@mariozechner/pi-agent-core";
|
|
4
|
+
import { GoogleGenAI } from "@google/genai";
|
|
5
|
+
import { StringEnum } from "@mariozechner/pi-ai";
|
|
6
|
+
import { Type } from "typebox";
|
|
7
|
+
import { persistApiKey, resolveExistingApiKey } from "./credentials.ts";
|
|
8
|
+
import {
|
|
9
|
+
ALL_MODELS,
|
|
10
|
+
DEFAULT_MODEL,
|
|
11
|
+
GEMINI_ASPECT_RATIOS,
|
|
12
|
+
GEMINI_IMAGE_SIZES,
|
|
13
|
+
IMAGEN_ASPECT_RATIOS,
|
|
14
|
+
IMAGEN_IMAGE_SIZES,
|
|
15
|
+
PERSON_GENERATION_VALUES,
|
|
16
|
+
isGeminiNative,
|
|
17
|
+
isImagen,
|
|
18
|
+
type ImageModel,
|
|
19
|
+
} from "./models.ts";
|
|
20
|
+
import { estimateCost, formatUsd, hasPricing } from "./pricing.ts";
|
|
21
|
+
|
|
22
|
+
const params = Type.Object({
|
|
23
|
+
prompt: Type.String({ description: "Text prompt describing the image to generate." }),
|
|
24
|
+
model: Type.Optional(
|
|
25
|
+
StringEnum(ALL_MODELS as unknown as readonly [string, ...string[]], {
|
|
26
|
+
description: `Model. Default: ${DEFAULT_MODEL}.`,
|
|
27
|
+
}),
|
|
28
|
+
),
|
|
29
|
+
aspect_ratio: Type.Optional(Type.String({ description: "Aspect ratio; validated per model family." })),
|
|
30
|
+
image_size: Type.Optional(
|
|
31
|
+
StringEnum(["1K", "2K", "4K"] as const, { description: "Output resolution. 4K is Gemini-native only." }),
|
|
32
|
+
),
|
|
33
|
+
number_of_images: Type.Optional(
|
|
34
|
+
Type.Integer({ minimum: 1, maximum: 4, description: "Imagen only. 1-4." }),
|
|
35
|
+
),
|
|
36
|
+
negative_prompt: Type.Optional(Type.String({ description: "Imagen only." })),
|
|
37
|
+
person_generation: Type.Optional(
|
|
38
|
+
StringEnum(PERSON_GENERATION_VALUES as unknown as readonly [string, ...string[]], {
|
|
39
|
+
description: "People policy: dont_allow | allow_adult | allow_all.",
|
|
40
|
+
}),
|
|
41
|
+
),
|
|
42
|
+
seed: Type.Optional(Type.Integer({ description: "Imagen only. Reproducibility seed." })),
|
|
43
|
+
input_images: Type.Optional(
|
|
44
|
+
Type.Array(Type.String(), {
|
|
45
|
+
description: "Gemini native only. Absolute paths to images for editing.",
|
|
46
|
+
}),
|
|
47
|
+
),
|
|
48
|
+
save_to: Type.Optional(Type.String({ description: "Optional directory to write generated PNG(s)." })),
|
|
49
|
+
skip_confirm: Type.Optional(Type.Boolean({ description: "Skip the cost confirmation prompt." })),
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
type Params = {
|
|
53
|
+
prompt: string;
|
|
54
|
+
model?: string;
|
|
55
|
+
aspect_ratio?: string;
|
|
56
|
+
image_size?: "1K" | "2K" | "4K";
|
|
57
|
+
number_of_images?: number;
|
|
58
|
+
negative_prompt?: string;
|
|
59
|
+
person_generation?: (typeof PERSON_GENERATION_VALUES)[number];
|
|
60
|
+
seed?: number;
|
|
61
|
+
input_images?: string[];
|
|
62
|
+
save_to?: string;
|
|
63
|
+
skip_confirm?: boolean;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
async function ensureApiKey(ctx: ExtensionContext): Promise<string> {
|
|
67
|
+
const existing = resolveExistingApiKey();
|
|
68
|
+
if (existing) return existing;
|
|
69
|
+
|
|
70
|
+
if (!ctx.hasUI) {
|
|
71
|
+
throw new Error("Gemini API key is not configured and no interactive UI is available to ask for one.");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const pasted = await ctx.ui.input("Paste your Gemini API key:");
|
|
75
|
+
if (!pasted || !pasted.trim()) {
|
|
76
|
+
throw new Error("No API key provided.");
|
|
77
|
+
}
|
|
78
|
+
const key = pasted.trim();
|
|
79
|
+
|
|
80
|
+
const save = await ctx.ui.confirm("Save for future use?", "The key will be kept locally for later calls.");
|
|
81
|
+
if (save) {
|
|
82
|
+
persistApiKey(key);
|
|
83
|
+
}
|
|
84
|
+
return key;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function validateForModel(model: ImageModel, p: Params): void {
|
|
88
|
+
if (isImagen(model)) {
|
|
89
|
+
if (p.aspect_ratio && !(IMAGEN_ASPECT_RATIOS as readonly string[]).includes(p.aspect_ratio)) {
|
|
90
|
+
throw new Error(
|
|
91
|
+
`aspect_ratio "${p.aspect_ratio}" is not supported for Imagen. Valid: ${IMAGEN_ASPECT_RATIOS.join(", ")}.`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
if (p.image_size && !(IMAGEN_IMAGE_SIZES as readonly string[]).includes(p.image_size)) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`image_size "${p.image_size}" is not supported for Imagen. Valid: ${IMAGEN_IMAGE_SIZES.join(", ")}.`,
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
if (p.input_images && p.input_images.length > 0) {
|
|
100
|
+
throw new Error("Imagen does not support input_images (text-to-image only). Use a Gemini-native model.");
|
|
101
|
+
}
|
|
102
|
+
} else if (isGeminiNative(model)) {
|
|
103
|
+
if (p.aspect_ratio && !(GEMINI_ASPECT_RATIOS as readonly string[]).includes(p.aspect_ratio)) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
`aspect_ratio "${p.aspect_ratio}" is not supported for Gemini native. Valid: ${GEMINI_ASPECT_RATIOS.join(", ")}.`,
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (p.image_size && !(GEMINI_IMAGE_SIZES as readonly string[]).includes(p.image_size)) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`image_size "${p.image_size}" is not supported for Gemini native. Valid: ${GEMINI_IMAGE_SIZES.join(", ")}.`,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
if (p.number_of_images !== undefined && p.number_of_images !== 1) {
|
|
114
|
+
throw new Error("Gemini native produces a single image per call; number_of_images only applies to Imagen.");
|
|
115
|
+
}
|
|
116
|
+
if (p.negative_prompt) {
|
|
117
|
+
throw new Error("negative_prompt is Imagen-only.");
|
|
118
|
+
}
|
|
119
|
+
if (p.seed !== undefined) {
|
|
120
|
+
throw new Error("seed is Imagen-only.");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
interface DecodedImage {
|
|
126
|
+
data: string; // base64
|
|
127
|
+
mimeType: string;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function callGeminiNative(
|
|
131
|
+
ai: GoogleGenAI,
|
|
132
|
+
model: string,
|
|
133
|
+
p: Params,
|
|
134
|
+
signal: AbortSignal | undefined,
|
|
135
|
+
): Promise<DecodedImage[]> {
|
|
136
|
+
const imageConfig: Record<string, string> = {};
|
|
137
|
+
if (p.aspect_ratio) imageConfig.aspectRatio = p.aspect_ratio;
|
|
138
|
+
if (p.image_size) imageConfig.imageSize = p.image_size;
|
|
139
|
+
if (p.person_generation) imageConfig.personGeneration = p.person_generation;
|
|
140
|
+
|
|
141
|
+
const contents: Array<Record<string, unknown>> = [{ text: p.prompt }];
|
|
142
|
+
for (const path of p.input_images ?? []) {
|
|
143
|
+
const buffer = readFileSync(path);
|
|
144
|
+
const mimeType = path.toLowerCase().endsWith(".jpg") || path.toLowerCase().endsWith(".jpeg")
|
|
145
|
+
? "image/jpeg"
|
|
146
|
+
: path.toLowerCase().endsWith(".webp")
|
|
147
|
+
? "image/webp"
|
|
148
|
+
: "image/png";
|
|
149
|
+
contents.push({
|
|
150
|
+
inlineData: {
|
|
151
|
+
mimeType,
|
|
152
|
+
data: buffer.toString("base64"),
|
|
153
|
+
},
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const response = await ai.models.generateContent({
|
|
158
|
+
model,
|
|
159
|
+
contents,
|
|
160
|
+
config: {
|
|
161
|
+
responseModalities: ["Text", "Image"],
|
|
162
|
+
...(Object.keys(imageConfig).length > 0 ? { imageConfig } : {}),
|
|
163
|
+
...(signal ? { abortSignal: signal } : {}),
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
const images: DecodedImage[] = [];
|
|
168
|
+
for (const candidate of response.candidates ?? []) {
|
|
169
|
+
for (const part of candidate.content?.parts ?? []) {
|
|
170
|
+
if (part.inlineData?.data && part.inlineData.mimeType) {
|
|
171
|
+
images.push({ data: part.inlineData.data, mimeType: part.inlineData.mimeType });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return images;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async function callImagen(
|
|
179
|
+
ai: GoogleGenAI,
|
|
180
|
+
model: string,
|
|
181
|
+
p: Params,
|
|
182
|
+
signal: AbortSignal | undefined,
|
|
183
|
+
): Promise<DecodedImage[]> {
|
|
184
|
+
const config: Record<string, unknown> = {};
|
|
185
|
+
if (p.aspect_ratio) config.aspectRatio = p.aspect_ratio;
|
|
186
|
+
if (p.image_size) config.imageSize = p.image_size;
|
|
187
|
+
if (p.number_of_images !== undefined) config.numberOfImages = p.number_of_images;
|
|
188
|
+
if (p.negative_prompt) config.negativePrompt = p.negative_prompt;
|
|
189
|
+
if (p.person_generation) config.personGeneration = p.person_generation;
|
|
190
|
+
if (p.seed !== undefined) config.seed = p.seed;
|
|
191
|
+
if (signal) config.abortSignal = signal;
|
|
192
|
+
|
|
193
|
+
const response = await ai.models.generateImages({
|
|
194
|
+
model,
|
|
195
|
+
prompt: p.prompt,
|
|
196
|
+
...(Object.keys(config).length > 0 ? { config } : {}),
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
const images: DecodedImage[] = [];
|
|
200
|
+
for (const generated of response.generatedImages ?? []) {
|
|
201
|
+
if (generated.image?.imageBytes) {
|
|
202
|
+
images.push({
|
|
203
|
+
data: generated.image.imageBytes,
|
|
204
|
+
mimeType: generated.image.mimeType ?? "image/png",
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return images;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function saveImages(images: DecodedImage[], dir: string, prompt: string): string[] {
|
|
212
|
+
const slug = prompt
|
|
213
|
+
.toLowerCase()
|
|
214
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
215
|
+
.replace(/^-+|-+$/g, "")
|
|
216
|
+
.slice(0, 40) || "image";
|
|
217
|
+
const ts = new Date().toISOString().replace(/[:.]/g, "-");
|
|
218
|
+
const paths: string[] = [];
|
|
219
|
+
for (let i = 0; i < images.length; i++) {
|
|
220
|
+
const ext = images[i].mimeType === "image/jpeg" ? "jpg" : "png";
|
|
221
|
+
const base = images.length > 1 ? `${slug}-${ts}-${i + 1}.${ext}` : `${slug}-${ts}.${ext}`;
|
|
222
|
+
const full = `${dir.replace(/\/$/, "")}/${base}`;
|
|
223
|
+
writeFileSync(full, Buffer.from(images[i].data, "base64"));
|
|
224
|
+
paths.push(full);
|
|
225
|
+
}
|
|
226
|
+
return paths;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export default function geminiImageExtension(pi: ExtensionAPI): void {
|
|
230
|
+
pi.registerTool({
|
|
231
|
+
name: "gemini_image",
|
|
232
|
+
label: "Gemini Image",
|
|
233
|
+
description:
|
|
234
|
+
"Generate or edit images via Google's Gemini (native) and Imagen models. Supports aspect ratios, resolutions, image editing, negative prompts, and seeds.",
|
|
235
|
+
promptGuidelines: [
|
|
236
|
+
"Use gemini-2.5-flash-image (default) for general-purpose generation.",
|
|
237
|
+
"Use gemini-3-pro-image-preview for studio-quality 4K, complex layouts, or precise text rendering.",
|
|
238
|
+
"Use imagen-4.0-*-generate-001 models for batch text-to-image (1-4 per call) or when negative_prompt / seed is needed.",
|
|
239
|
+
"Image editing (input_images) is only supported by Gemini native models.",
|
|
240
|
+
"4K output is only supported by Gemini native models.",
|
|
241
|
+
],
|
|
242
|
+
parameters: params,
|
|
243
|
+
async execute(_toolCallId, input, signal, _onUpdate, ctx) {
|
|
244
|
+
const p = input as Params;
|
|
245
|
+
const model = (p.model as ImageModel | undefined) ?? DEFAULT_MODEL;
|
|
246
|
+
|
|
247
|
+
if (!(ALL_MODELS as readonly string[]).includes(model)) {
|
|
248
|
+
throw new Error(`Unknown model "${model}". Valid: ${ALL_MODELS.join(", ")}.`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
validateForModel(model, p);
|
|
252
|
+
|
|
253
|
+
const apiKey = await ensureApiKey(ctx);
|
|
254
|
+
|
|
255
|
+
const numImages = isImagen(model) ? p.number_of_images ?? 1 : 1;
|
|
256
|
+
|
|
257
|
+
if (!p.skip_confirm && hasPricing(model) && ctx.hasUI) {
|
|
258
|
+
const est = estimateCost({
|
|
259
|
+
model,
|
|
260
|
+
prompt: p.prompt,
|
|
261
|
+
numberOfImages: numImages,
|
|
262
|
+
imageSize: p.image_size,
|
|
263
|
+
});
|
|
264
|
+
const proceed = await ctx.ui.confirm(
|
|
265
|
+
"Confirm image generation",
|
|
266
|
+
`Model: ${model}\nImages: ${numImages}${p.image_size ? `\nSize: ${p.image_size}` : ""}\nEstimated: ~${formatUsd(est.estimatedUsd)} (${est.breakdown})`,
|
|
267
|
+
);
|
|
268
|
+
if (!proceed) {
|
|
269
|
+
throw new Error("User cancelled image generation.");
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const ai = new GoogleGenAI({ apiKey });
|
|
274
|
+
|
|
275
|
+
const images = isImagen(model)
|
|
276
|
+
? await callImagen(ai, model, p, signal)
|
|
277
|
+
: await callGeminiNative(ai, model, p, signal);
|
|
278
|
+
|
|
279
|
+
if (images.length === 0) {
|
|
280
|
+
throw new Error("Model returned no images.");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const savedPaths = p.save_to ? saveImages(images, p.save_to, p.prompt) : [];
|
|
284
|
+
|
|
285
|
+
const summaryLines = [
|
|
286
|
+
`Generated ${images.length} image${images.length > 1 ? "s" : ""} with ${model}.`,
|
|
287
|
+
`Prompt: "${p.prompt}"`,
|
|
288
|
+
];
|
|
289
|
+
if (p.aspect_ratio) summaryLines.push(`Aspect ratio: ${p.aspect_ratio}`);
|
|
290
|
+
if (p.image_size) summaryLines.push(`Size: ${p.image_size}`);
|
|
291
|
+
if (p.negative_prompt) summaryLines.push(`Negative: "${p.negative_prompt}"`);
|
|
292
|
+
if (p.person_generation) summaryLines.push(`People: ${p.person_generation}`);
|
|
293
|
+
if (p.seed !== undefined) summaryLines.push(`Seed: ${p.seed}`);
|
|
294
|
+
if (savedPaths.length > 0) summaryLines.push(`Saved to:\n ${savedPaths.join("\n ")}`);
|
|
295
|
+
|
|
296
|
+
const content: AgentToolResult<unknown>["content"] = images.map((img) => ({
|
|
297
|
+
type: "image",
|
|
298
|
+
data: img.data,
|
|
299
|
+
mimeType: img.mimeType,
|
|
300
|
+
}));
|
|
301
|
+
content.push({ type: "text", text: summaryLines.join("\n") });
|
|
302
|
+
|
|
303
|
+
return {
|
|
304
|
+
content,
|
|
305
|
+
details: {
|
|
306
|
+
model,
|
|
307
|
+
images: images.length,
|
|
308
|
+
savedPaths,
|
|
309
|
+
},
|
|
310
|
+
};
|
|
311
|
+
},
|
|
312
|
+
});
|
|
313
|
+
}
|