@monotykamary/pi-better-openai 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +191 -0
- package/THIRD_PARTY_NOTICES.md +28 -0
- package/index.ts +1328 -0
- package/package.json +94 -0
- package/src/codex-auth.ts +136 -0
- package/src/config.ts +725 -0
- package/src/fast-controller.ts +112 -0
- package/src/footer-layout.ts +97 -0
- package/src/format.ts +104 -0
- package/src/identity.ts +7 -0
- package/src/image.ts +672 -0
- package/src/live/attestation.ts +86 -0
- package/src/live/controller.ts +523 -0
- package/src/live/index.ts +197 -0
- package/src/live/native.ts +109 -0
- package/src/live/protocol.ts +234 -0
- package/src/live/transport.ts +445 -0
- package/src/live/visualizer.ts +269 -0
- package/src/live/voices.ts +21 -0
- package/src/paths.ts +17 -0
- package/src/pet-footer-controller.ts +642 -0
- package/src/pets.ts +983 -0
- package/src/usage-controller.ts +304 -0
- package/src/usage.ts +264 -0
package/src/image.ts
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { extname, isAbsolute, join, resolve, sep } from "node:path";
|
|
5
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
6
|
+
import { Box, Container, Image, Text } from "@earendil-works/pi-tui";
|
|
7
|
+
import sharp from "sharp";
|
|
8
|
+
import type { ResolvedConfig } from "./config.ts";
|
|
9
|
+
import { isRecord } from "./config.ts";
|
|
10
|
+
import {
|
|
11
|
+
extractAccountIdFromJwt,
|
|
12
|
+
getCodexCredentials,
|
|
13
|
+
type CodexCredentialsWithSource,
|
|
14
|
+
} from "./codex-auth.ts";
|
|
15
|
+
import { maskIdentifier, sanitizeDiagnosticError } from "./format.ts";
|
|
16
|
+
import { piAgentDir, resolveUserPath } from "./paths.ts";
|
|
17
|
+
|
|
18
|
+
const OPENAI_IMAGE_TOOL = "openai_image";
|
|
19
|
+
const OPENAI_IMAGE_COMMAND = "openai-image";
|
|
20
|
+
const CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
21
|
+
const DEFAULT_TIMEOUT_MS = 180_000;
|
|
22
|
+
const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024;
|
|
23
|
+
const MAX_IMAGE_INPUTS = 5;
|
|
24
|
+
const MAX_TOTAL_IMAGE_INPUT_BYTES = 50 * 1024 * 1024;
|
|
25
|
+
const SUPPORTED_INPUT_IMAGE_FORMATS = new Set(["png", "jpeg", "jpg", "webp", "gif"]);
|
|
26
|
+
const SSE_EVENT_BOUNDARY = /\r?\n\r?\n/;
|
|
27
|
+
|
|
28
|
+
export const IMAGE_SAVE_MODES = ["none", "project", "global", "custom"] as const;
|
|
29
|
+
export const IMAGE_ACTIONS = ["auto", "generate", "edit"] as const;
|
|
30
|
+
export const IMAGE_OUTPUT_FORMATS = ["png", "jpeg", "webp"] as const;
|
|
31
|
+
|
|
32
|
+
export type ImageSaveMode = (typeof IMAGE_SAVE_MODES)[number];
|
|
33
|
+
export type ImageAction = (typeof IMAGE_ACTIONS)[number];
|
|
34
|
+
export type ImageOutputFormat = (typeof IMAGE_OUTPUT_FORMATS)[number];
|
|
35
|
+
|
|
36
|
+
const TOOL_PARAMS = {
|
|
37
|
+
type: "object",
|
|
38
|
+
properties: {
|
|
39
|
+
prompt: {
|
|
40
|
+
type: "string",
|
|
41
|
+
description:
|
|
42
|
+
"Image generation/editing prompt. Pass the user's wording verbatim unless they explicitly ask you to refine or expand it.",
|
|
43
|
+
},
|
|
44
|
+
action: {
|
|
45
|
+
type: "string",
|
|
46
|
+
enum: IMAGE_ACTIONS,
|
|
47
|
+
description:
|
|
48
|
+
"Whether to generate a new image, edit/reference provided images, or let the model decide.",
|
|
49
|
+
},
|
|
50
|
+
images: {
|
|
51
|
+
type: "array",
|
|
52
|
+
maxItems: MAX_IMAGE_INPUTS,
|
|
53
|
+
items: { type: "string" },
|
|
54
|
+
description: "Local image paths to use as edit targets or references.",
|
|
55
|
+
},
|
|
56
|
+
model: {
|
|
57
|
+
type: "string",
|
|
58
|
+
description:
|
|
59
|
+
"OpenAI Codex model to drive the hosted image_generation tool. Defaults to current openai-codex model or config default.",
|
|
60
|
+
},
|
|
61
|
+
outputFormat: {
|
|
62
|
+
type: "string",
|
|
63
|
+
enum: IMAGE_OUTPUT_FORMATS,
|
|
64
|
+
description: "Generated image format.",
|
|
65
|
+
},
|
|
66
|
+
save: {
|
|
67
|
+
type: "string",
|
|
68
|
+
enum: IMAGE_SAVE_MODES,
|
|
69
|
+
description: "Where to save the generated image.",
|
|
70
|
+
},
|
|
71
|
+
saveDir: { type: "string", description: "Directory to save image when save=custom." },
|
|
72
|
+
},
|
|
73
|
+
required: ["prompt"],
|
|
74
|
+
additionalProperties: false,
|
|
75
|
+
} as const;
|
|
76
|
+
|
|
77
|
+
type ToolParams = {
|
|
78
|
+
prompt: string;
|
|
79
|
+
action?: ImageAction;
|
|
80
|
+
images?: string[];
|
|
81
|
+
model?: string;
|
|
82
|
+
outputFormat?: ImageOutputFormat;
|
|
83
|
+
save?: ImageSaveMode;
|
|
84
|
+
saveDir?: string;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
type CodexImageCredentials = CodexCredentialsWithSource;
|
|
88
|
+
|
|
89
|
+
type ImageInput = {
|
|
90
|
+
path: string;
|
|
91
|
+
data: string;
|
|
92
|
+
mimeType: string;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export type CodexImageResult = {
|
|
96
|
+
id: string;
|
|
97
|
+
status: string;
|
|
98
|
+
prompt: string;
|
|
99
|
+
revisedPrompt?: string;
|
|
100
|
+
data: string;
|
|
101
|
+
mimeType: string;
|
|
102
|
+
savedPath?: string;
|
|
103
|
+
model: string;
|
|
104
|
+
action: ImageAction;
|
|
105
|
+
outputFormat: ImageOutputFormat;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
type ExtractedImageResult = Omit<
|
|
109
|
+
CodexImageResult,
|
|
110
|
+
"prompt" | "savedPath" | "model" | "action" | "outputFormat"
|
|
111
|
+
>;
|
|
112
|
+
|
|
113
|
+
export type ImageGenerationDebug = {
|
|
114
|
+
authFound: boolean;
|
|
115
|
+
authSource?: string;
|
|
116
|
+
accountId?: string;
|
|
117
|
+
endpoint: string;
|
|
118
|
+
defaultModel: string;
|
|
119
|
+
defaultSave: ImageSaveMode;
|
|
120
|
+
enabled: boolean;
|
|
121
|
+
lastStatus?: string;
|
|
122
|
+
lastError?: string;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
async function getCredentials(
|
|
126
|
+
ctx: ExtensionContext,
|
|
127
|
+
signal?: AbortSignal,
|
|
128
|
+
): Promise<CodexImageCredentials> {
|
|
129
|
+
const credentials = await getCodexCredentials(ctx, signal);
|
|
130
|
+
if (credentials) return credentials;
|
|
131
|
+
throw new Error("Missing openai-codex OAuth credentials. Run /login openai-codex.");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function resolveModel(
|
|
135
|
+
params: Pick<ToolParams, "model">,
|
|
136
|
+
ctx: ExtensionContext,
|
|
137
|
+
cfg: ResolvedConfig,
|
|
138
|
+
): string {
|
|
139
|
+
const model = params.model?.trim();
|
|
140
|
+
if (model) return model.includes("/") ? model.split("/").pop() || model : model;
|
|
141
|
+
if (ctx.model?.provider === "openai-codex") return ctx.model.id;
|
|
142
|
+
return cfg.image.defaultModel;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function resolveImageConfig(cfg: ResolvedConfig, params: ToolParams) {
|
|
146
|
+
const action = params.action ?? "auto";
|
|
147
|
+
const outputFormat = params.outputFormat ?? cfg.image.outputFormat;
|
|
148
|
+
const save = params.save ?? cfg.image.defaultSave;
|
|
149
|
+
return { action, outputFormat, save };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function imageMimeType(path: string, outputFormat?: string): string {
|
|
153
|
+
if (outputFormat === "jpeg" || outputFormat === "jpg") return "image/jpeg";
|
|
154
|
+
if (outputFormat === "webp") return "image/webp";
|
|
155
|
+
if (outputFormat === "gif") return "image/gif";
|
|
156
|
+
if (outputFormat === "png") return "image/png";
|
|
157
|
+
const ext = extname(path).toLowerCase();
|
|
158
|
+
if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
|
|
159
|
+
if (ext === ".webp") return "image/webp";
|
|
160
|
+
if (ext === ".gif") return "image/gif";
|
|
161
|
+
return "image/png";
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function extensionForFormat(format: ImageOutputFormat): string {
|
|
165
|
+
return format === "jpeg" ? "jpg" : format;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function isInsideDirectory(root: string, child: string): boolean {
|
|
169
|
+
const normalizedRoot = resolve(root);
|
|
170
|
+
const normalizedChild = resolve(child);
|
|
171
|
+
return (
|
|
172
|
+
normalizedChild !== normalizedRoot && normalizedChild.startsWith(`${normalizedRoot}${sep}`)
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function validateImageInput(
|
|
177
|
+
path: string,
|
|
178
|
+
realWorkspaceRoot: string,
|
|
179
|
+
): Promise<{ mimeType: string; path: string; size: number }> {
|
|
180
|
+
const realInputPath = await realpath(path).catch(() => undefined);
|
|
181
|
+
if (!realInputPath || !isInsideDirectory(realWorkspaceRoot, realInputPath))
|
|
182
|
+
throw new Error(
|
|
183
|
+
`Image input must be a file inside the current workspace: ${displayPath(path)}`,
|
|
184
|
+
);
|
|
185
|
+
|
|
186
|
+
const pathStats = await stat(realInputPath).catch(() => undefined);
|
|
187
|
+
if (!pathStats?.isFile())
|
|
188
|
+
throw new Error(
|
|
189
|
+
`Image input must be a file inside the current workspace: ${displayPath(path)}`,
|
|
190
|
+
);
|
|
191
|
+
if (pathStats.size > MAX_IMAGE_INPUT_BYTES)
|
|
192
|
+
throw new Error(`Image input is too large (max 20 MB): ${displayPath(path)}`);
|
|
193
|
+
|
|
194
|
+
const metadata = await sharp(realInputPath, { animated: false })
|
|
195
|
+
.metadata()
|
|
196
|
+
.catch(() => undefined);
|
|
197
|
+
if (!metadata?.format || !SUPPORTED_INPUT_IMAGE_FORMATS.has(metadata.format))
|
|
198
|
+
throw new Error(`Image input is not a readable image: ${displayPath(path)}`);
|
|
199
|
+
return {
|
|
200
|
+
mimeType: imageMimeType(path, metadata.format),
|
|
201
|
+
path: realInputPath,
|
|
202
|
+
size: pathStats.size,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function readImageInputs(paths: string[] | undefined, cwd: string): Promise<ImageInput[]> {
|
|
207
|
+
const validatedInputs: Array<{ path: string; mimeType: string }> = [];
|
|
208
|
+
const seenPaths = new Set<string>();
|
|
209
|
+
let totalBytes = 0;
|
|
210
|
+
const workspaceRoot = resolve(cwd);
|
|
211
|
+
let realWorkspaceRoot: string | undefined;
|
|
212
|
+
for (const rawPath of paths ?? []) {
|
|
213
|
+
const trimmed = rawPath.trim();
|
|
214
|
+
if (!trimmed) continue;
|
|
215
|
+
const path = isAbsolute(trimmed) ? resolve(trimmed) : resolve(workspaceRoot, trimmed);
|
|
216
|
+
if (!isInsideDirectory(workspaceRoot, path))
|
|
217
|
+
throw new Error(
|
|
218
|
+
`Image input must be a file inside the current workspace: ${displayPath(path)}`,
|
|
219
|
+
);
|
|
220
|
+
realWorkspaceRoot ??= await realpath(workspaceRoot).catch(() => workspaceRoot);
|
|
221
|
+
const input = await validateImageInput(path, realWorkspaceRoot);
|
|
222
|
+
if (seenPaths.has(input.path)) continue;
|
|
223
|
+
if (validatedInputs.length >= MAX_IMAGE_INPUTS)
|
|
224
|
+
throw new Error(`Too many image inputs (max ${MAX_IMAGE_INPUTS}).`);
|
|
225
|
+
totalBytes += input.size;
|
|
226
|
+
if (totalBytes > MAX_TOTAL_IMAGE_INPUT_BYTES)
|
|
227
|
+
throw new Error("Image inputs are too large in total (max 50 MB).");
|
|
228
|
+
seenPaths.add(input.path);
|
|
229
|
+
validatedInputs.push({ path: input.path, mimeType: input.mimeType });
|
|
230
|
+
}
|
|
231
|
+
return Promise.all(
|
|
232
|
+
validatedInputs.map(async (input) => ({
|
|
233
|
+
...input,
|
|
234
|
+
data: (await readFile(input.path)).toString("base64"),
|
|
235
|
+
})),
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function resolveSaveDir(
|
|
240
|
+
mode: ImageSaveMode,
|
|
241
|
+
params: Pick<ToolParams, "saveDir">,
|
|
242
|
+
cwd: string,
|
|
243
|
+
): string | undefined {
|
|
244
|
+
if (mode === "none") return undefined;
|
|
245
|
+
if (mode === "project") return join(cwd, ".pi", "generated-images");
|
|
246
|
+
if (mode === "global") return join(piAgentDir(), "generated-images");
|
|
247
|
+
const dir = params.saveDir?.trim() || process.env.PI_IMAGE_SAVE_DIR?.trim();
|
|
248
|
+
if (!dir) throw new Error("save=custom requires saveDir or PI_IMAGE_SAVE_DIR.");
|
|
249
|
+
return resolveUserPath(dir, cwd);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
async function saveImage(
|
|
253
|
+
data: string,
|
|
254
|
+
format: ImageOutputFormat,
|
|
255
|
+
outputDir: string,
|
|
256
|
+
id: string,
|
|
257
|
+
): Promise<string> {
|
|
258
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
259
|
+
const safeId = id.replace(/[^a-zA-Z0-9_-]/g, "_") || randomUUID().slice(0, 8);
|
|
260
|
+
const path = join(outputDir, `openai-image-${timestamp}-${safeId}.${extensionForFormat(format)}`);
|
|
261
|
+
await mkdir(outputDir, { recursive: true });
|
|
262
|
+
await writeFile(path, Buffer.from(data, "base64"));
|
|
263
|
+
return path;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function buildRequest(
|
|
267
|
+
params: ToolParams,
|
|
268
|
+
model: string,
|
|
269
|
+
cfg: ResolvedConfig,
|
|
270
|
+
images: ImageInput[],
|
|
271
|
+
) {
|
|
272
|
+
const { action, outputFormat } = resolveImageConfig(cfg, params);
|
|
273
|
+
const content: Array<Record<string, unknown>> = [{ type: "input_text", text: params.prompt }];
|
|
274
|
+
for (const image of images) {
|
|
275
|
+
content.push({
|
|
276
|
+
type: "input_image",
|
|
277
|
+
detail: "auto",
|
|
278
|
+
image_url: `data:${image.mimeType};base64,${image.data}`,
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
const tool: Record<string, unknown> = { type: "image_generation", output_format: outputFormat };
|
|
282
|
+
if (action !== "auto") tool.action = action;
|
|
283
|
+
return {
|
|
284
|
+
model,
|
|
285
|
+
instructions: "",
|
|
286
|
+
input: [{ role: "user", content }],
|
|
287
|
+
tools: [tool],
|
|
288
|
+
tool_choice: { type: "image_generation" },
|
|
289
|
+
parallel_tool_calls: false,
|
|
290
|
+
store: false,
|
|
291
|
+
stream: true,
|
|
292
|
+
include: [],
|
|
293
|
+
client_metadata: { "x-codex-installation-id": "pi-better-openai" },
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function dataUrlParts(value: string, fallbackMimeType: string): { data: string; mimeType: string } {
|
|
298
|
+
const match = value.match(/^data:([^;,]+);base64,(.*)$/s);
|
|
299
|
+
if (match)
|
|
300
|
+
return {
|
|
301
|
+
mimeType: match[1] || fallbackMimeType,
|
|
302
|
+
data: (match[2] ?? "").trim(),
|
|
303
|
+
};
|
|
304
|
+
return { data: value.trim(), mimeType: fallbackMimeType };
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function asImageResultItem(
|
|
308
|
+
value: unknown,
|
|
309
|
+
):
|
|
310
|
+
| { id?: string; status?: string; revised_prompt?: string; result?: string; b64_json?: string }
|
|
311
|
+
| undefined {
|
|
312
|
+
if (!isRecord(value) || value.type !== "image_generation_call") return undefined;
|
|
313
|
+
return value as {
|
|
314
|
+
id?: string;
|
|
315
|
+
status?: string;
|
|
316
|
+
revised_prompt?: string;
|
|
317
|
+
result?: string;
|
|
318
|
+
b64_json?: string;
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function isImageContent(
|
|
323
|
+
value: unknown,
|
|
324
|
+
): value is { type: "image"; data: string; mimeType: string } {
|
|
325
|
+
return (
|
|
326
|
+
isRecord(value) &&
|
|
327
|
+
value.type === "image" &&
|
|
328
|
+
typeof value.data === "string" &&
|
|
329
|
+
typeof value.mimeType === "string"
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function extractImageFromEvent(
|
|
334
|
+
event: unknown,
|
|
335
|
+
fallbackMimeType: string,
|
|
336
|
+
): ExtractedImageResult | undefined {
|
|
337
|
+
if (!isRecord(event)) return undefined;
|
|
338
|
+
const item = asImageResultItem(event.item) ?? asImageResultItem(event);
|
|
339
|
+
if (item) {
|
|
340
|
+
const raw =
|
|
341
|
+
typeof item.result === "string" && item.result.trim()
|
|
342
|
+
? item.result
|
|
343
|
+
: typeof item.b64_json === "string"
|
|
344
|
+
? item.b64_json
|
|
345
|
+
: undefined;
|
|
346
|
+
if (!raw) return undefined;
|
|
347
|
+
const { data, mimeType } = dataUrlParts(raw, fallbackMimeType);
|
|
348
|
+
return {
|
|
349
|
+
id: typeof item.id === "string" ? item.id : `ig_${randomUUID().slice(0, 8)}`,
|
|
350
|
+
status: typeof item.status === "string" ? item.status : "completed",
|
|
351
|
+
revisedPrompt: typeof item.revised_prompt === "string" ? item.revised_prompt : undefined,
|
|
352
|
+
data,
|
|
353
|
+
mimeType,
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
const partial =
|
|
357
|
+
typeof event.partial_image_b64 === "string"
|
|
358
|
+
? event.partial_image_b64
|
|
359
|
+
: typeof event.b64_json === "string"
|
|
360
|
+
? event.b64_json
|
|
361
|
+
: undefined;
|
|
362
|
+
if (typeof partial === "string" && partial.trim()) {
|
|
363
|
+
const { data, mimeType } = dataUrlParts(partial, fallbackMimeType);
|
|
364
|
+
return { id: `ig_${randomUUID().slice(0, 8)}`, status: "partial", data, mimeType };
|
|
365
|
+
}
|
|
366
|
+
return undefined;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async function parseSseForImage(
|
|
370
|
+
response: Response,
|
|
371
|
+
fallbackMimeType: string,
|
|
372
|
+
signal?: AbortSignal,
|
|
373
|
+
): Promise<ExtractedImageResult> {
|
|
374
|
+
if (!response.body) throw new Error("No response body from Codex image request.");
|
|
375
|
+
const reader = response.body.getReader();
|
|
376
|
+
const decoder = new TextDecoder();
|
|
377
|
+
let buffer = "";
|
|
378
|
+
try {
|
|
379
|
+
while (true) {
|
|
380
|
+
if (signal?.aborted) throw new Error("Image request was aborted.");
|
|
381
|
+
const { done, value } = await reader.read();
|
|
382
|
+
if (done) break;
|
|
383
|
+
buffer += decoder.decode(value, { stream: true });
|
|
384
|
+
let boundary = SSE_EVENT_BOUNDARY.exec(buffer);
|
|
385
|
+
while (boundary) {
|
|
386
|
+
const idx = boundary.index;
|
|
387
|
+
const chunk = buffer.slice(0, idx);
|
|
388
|
+
buffer = buffer.slice(idx + boundary[0].length);
|
|
389
|
+
const data = chunk
|
|
390
|
+
.split(/\r?\n/)
|
|
391
|
+
.filter((line) => line.startsWith("data:"))
|
|
392
|
+
.map((line) => line.slice(5).trim())
|
|
393
|
+
.join("\n")
|
|
394
|
+
.trim();
|
|
395
|
+
if (data && data !== "[DONE]") {
|
|
396
|
+
let event: unknown;
|
|
397
|
+
try {
|
|
398
|
+
event = JSON.parse(data);
|
|
399
|
+
} catch {
|
|
400
|
+
event = undefined;
|
|
401
|
+
}
|
|
402
|
+
const image = extractImageFromEvent(event, fallbackMimeType);
|
|
403
|
+
if (image?.data && image.status === "completed") {
|
|
404
|
+
await reader.cancel().catch(() => undefined);
|
|
405
|
+
return image;
|
|
406
|
+
}
|
|
407
|
+
if (isRecord(event) && event.type === "response.failed") {
|
|
408
|
+
const error =
|
|
409
|
+
isRecord(event.response) && isRecord(event.response.error)
|
|
410
|
+
? event.response.error
|
|
411
|
+
: undefined;
|
|
412
|
+
const message = sanitizeDiagnosticError(
|
|
413
|
+
typeof error?.message === "string" ? error.message : "Codex image request failed.",
|
|
414
|
+
);
|
|
415
|
+
throw new Error(message);
|
|
416
|
+
}
|
|
417
|
+
if (isRecord(event) && event.type === "error") {
|
|
418
|
+
const message = sanitizeDiagnosticError(
|
|
419
|
+
typeof event.message === "string" ? event.message : JSON.stringify(event),
|
|
420
|
+
);
|
|
421
|
+
throw new Error(`Codex image error: ${message}`);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
boundary = SSE_EVENT_BOUNDARY.exec(buffer);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
} finally {
|
|
428
|
+
reader.releaseLock();
|
|
429
|
+
}
|
|
430
|
+
throw new Error("No completed image_generation_call result returned by Codex.");
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
async function requestCodexImage(
|
|
434
|
+
params: ToolParams,
|
|
435
|
+
ctx: ExtensionContext,
|
|
436
|
+
cfg: ResolvedConfig,
|
|
437
|
+
requestSignal?: AbortSignal,
|
|
438
|
+
): Promise<CodexImageResult> {
|
|
439
|
+
if (!cfg.image.enabled) throw new Error("OpenAI image generation is disabled in config.");
|
|
440
|
+
const cwd = ctx.cwd || process.cwd();
|
|
441
|
+
const model = resolveModel(params, ctx, cfg);
|
|
442
|
+
const { action, outputFormat, save } = resolveImageConfig(cfg, params);
|
|
443
|
+
const saveDir = resolveSaveDir(save, params, cwd);
|
|
444
|
+
const timeoutSignal = AbortSignal.timeout(cfg.image.timeoutMs);
|
|
445
|
+
const baseSignal = requestSignal ?? ctx.signal;
|
|
446
|
+
const signal = baseSignal ? AbortSignal.any([baseSignal, timeoutSignal]) : timeoutSignal;
|
|
447
|
+
const credentials = await getCredentials(ctx, signal);
|
|
448
|
+
const images = await readImageInputs(params.images, cwd);
|
|
449
|
+
const request = buildRequest(params, model, cfg, images);
|
|
450
|
+
const response = await fetch(CODEX_RESPONSES_URL, {
|
|
451
|
+
method: "POST",
|
|
452
|
+
headers: {
|
|
453
|
+
authorization: `Bearer ${credentials.accessToken}`,
|
|
454
|
+
"chatgpt-account-id": credentials.accountId,
|
|
455
|
+
"OpenAI-Beta": "responses=experimental",
|
|
456
|
+
accept: "text/event-stream",
|
|
457
|
+
"content-type": "application/json",
|
|
458
|
+
originator: "codex_cli_rs",
|
|
459
|
+
"User-Agent": "codex_cli_rs/0.0.0 (pi-better-openai)",
|
|
460
|
+
},
|
|
461
|
+
body: JSON.stringify(request),
|
|
462
|
+
signal,
|
|
463
|
+
});
|
|
464
|
+
if (!response.ok) {
|
|
465
|
+
const statusText = response.statusText
|
|
466
|
+
? ` ${sanitizeDiagnosticError(response.statusText, 120)}`
|
|
467
|
+
: "";
|
|
468
|
+
throw new Error(`Codex image request failed (${response.status}${statusText}).`);
|
|
469
|
+
}
|
|
470
|
+
const parsed = await parseSseForImage(
|
|
471
|
+
response,
|
|
472
|
+
imageMimeType(`image.${outputFormat}`, outputFormat),
|
|
473
|
+
signal,
|
|
474
|
+
);
|
|
475
|
+
const savedPath = saveDir
|
|
476
|
+
? await saveImage(parsed.data, outputFormat, saveDir, parsed.id)
|
|
477
|
+
: undefined;
|
|
478
|
+
return { ...parsed, prompt: params.prompt, savedPath, model, action, outputFormat };
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function displayPath(path: string): string {
|
|
482
|
+
const home = homedir();
|
|
483
|
+
if (!home) return path;
|
|
484
|
+
if (path === home) return "~";
|
|
485
|
+
const homePrefix = home.endsWith(sep) ? home : `${home}${sep}`;
|
|
486
|
+
return path.startsWith(homePrefix) ? `~/${path.slice(homePrefix.length)}` : path;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
function resultText(result: CodexImageResult): string {
|
|
490
|
+
const parts = [
|
|
491
|
+
`Generated image using OpenAI image_generation tool via openai-codex/${result.model}.`,
|
|
492
|
+
`Action: ${result.action}.`,
|
|
493
|
+
`Prompt: ${result.prompt}`,
|
|
494
|
+
];
|
|
495
|
+
if (result.revisedPrompt) parts.push(`Revised prompt: ${result.revisedPrompt}`);
|
|
496
|
+
if (result.savedPath) parts.push(`Saved: ${displayPath(result.savedPath)}`);
|
|
497
|
+
return parts.join("\n");
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
export function registerOpenAIImage(
|
|
501
|
+
pi: ExtensionAPI,
|
|
502
|
+
getConfig: (ctx: ExtensionContext) => ResolvedConfig,
|
|
503
|
+
): { getDebug: (ctx: ExtensionContext) => Promise<ImageGenerationDebug> } {
|
|
504
|
+
let lastStatus: string | undefined;
|
|
505
|
+
let lastError: string | undefined;
|
|
506
|
+
|
|
507
|
+
async function generate(
|
|
508
|
+
params: ToolParams,
|
|
509
|
+
ctx: ExtensionContext,
|
|
510
|
+
requestSignal?: AbortSignal,
|
|
511
|
+
): Promise<CodexImageResult> {
|
|
512
|
+
try {
|
|
513
|
+
lastStatus = "requesting";
|
|
514
|
+
lastError = undefined;
|
|
515
|
+
const result = await requestCodexImage(params, ctx, getConfig(ctx), requestSignal);
|
|
516
|
+
lastStatus = `completed (${result.id})`;
|
|
517
|
+
return result;
|
|
518
|
+
} catch (error) {
|
|
519
|
+
lastStatus = "error";
|
|
520
|
+
lastError = sanitizeDiagnosticError(error instanceof Error ? error.message : String(error));
|
|
521
|
+
throw error;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
async function getDebug(ctx: ExtensionContext): Promise<ImageGenerationDebug> {
|
|
526
|
+
const cfg = getConfig(ctx);
|
|
527
|
+
let credentials: CodexImageCredentials | undefined;
|
|
528
|
+
try {
|
|
529
|
+
credentials = await getCredentials(ctx);
|
|
530
|
+
} catch {
|
|
531
|
+
credentials = undefined;
|
|
532
|
+
}
|
|
533
|
+
return {
|
|
534
|
+
authFound: credentials !== undefined,
|
|
535
|
+
authSource: credentials?.source,
|
|
536
|
+
accountId: maskIdentifier(credentials?.accountId),
|
|
537
|
+
endpoint: CODEX_RESPONSES_URL,
|
|
538
|
+
defaultModel: ctx.model?.provider === "openai-codex" ? ctx.model.id : cfg.image.defaultModel,
|
|
539
|
+
defaultSave: cfg.image.defaultSave,
|
|
540
|
+
enabled: cfg.image.enabled,
|
|
541
|
+
lastStatus,
|
|
542
|
+
lastError,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
pi.registerMessageRenderer<CodexImageResult>("openai-image", (message, _options, theme) => {
|
|
547
|
+
const result = message.details;
|
|
548
|
+
const text =
|
|
549
|
+
result && isRecord(result)
|
|
550
|
+
? resultText(result as CodexImageResult)
|
|
551
|
+
: typeof message.content === "string"
|
|
552
|
+
? message.content
|
|
553
|
+
: message.content
|
|
554
|
+
.filter((part) => part.type === "text")
|
|
555
|
+
.map((part) => part.text)
|
|
556
|
+
.join("\n");
|
|
557
|
+
let image: { data: string; mimeType: string; savedPath?: string } | undefined;
|
|
558
|
+
if (
|
|
559
|
+
result &&
|
|
560
|
+
isRecord(result) &&
|
|
561
|
+
typeof result.data === "string" &&
|
|
562
|
+
typeof result.mimeType === "string"
|
|
563
|
+
) {
|
|
564
|
+
image = {
|
|
565
|
+
data: result.data,
|
|
566
|
+
mimeType: result.mimeType,
|
|
567
|
+
savedPath: typeof result.savedPath === "string" ? result.savedPath : undefined,
|
|
568
|
+
};
|
|
569
|
+
} else if (Array.isArray(message.content)) {
|
|
570
|
+
const imagePart = message.content.find(isImageContent);
|
|
571
|
+
if (imagePart) image = { data: imagePart.data, mimeType: imagePart.mimeType };
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
const container = new Container();
|
|
575
|
+
const box = new Box(1, 1, (line) => theme.bg("customMessageBg", line));
|
|
576
|
+
box.addChild(new Text(`${theme.fg("accent", theme.bold("[openai-image]"))}\n\n${text}`, 0, 0));
|
|
577
|
+
if (image) {
|
|
578
|
+
box.addChild(
|
|
579
|
+
new Image(
|
|
580
|
+
image.data,
|
|
581
|
+
image.mimeType,
|
|
582
|
+
{ fallbackColor: (line) => theme.fg("dim", line) },
|
|
583
|
+
{
|
|
584
|
+
maxWidthCells: 80,
|
|
585
|
+
maxHeightCells: 24,
|
|
586
|
+
filename:
|
|
587
|
+
"savedPath" in image && typeof image.savedPath === "string"
|
|
588
|
+
? image.savedPath
|
|
589
|
+
: undefined,
|
|
590
|
+
},
|
|
591
|
+
),
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
container.addChild(box);
|
|
595
|
+
return container;
|
|
596
|
+
});
|
|
597
|
+
|
|
598
|
+
pi.registerCommand(OPENAI_IMAGE_COMMAND, {
|
|
599
|
+
description: "Generate an image with OpenAI Codex image generation",
|
|
600
|
+
handler: async (args, ctx) => {
|
|
601
|
+
const prompt = args.trim();
|
|
602
|
+
if (!prompt) {
|
|
603
|
+
ctx.ui.notify("Usage: /openai-image <prompt>", "error");
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
ctx.ui.notify("Requesting OpenAI image...", "info");
|
|
607
|
+
const result = await generate({ prompt }, ctx);
|
|
608
|
+
pi.sendMessage({
|
|
609
|
+
customType: "openai-image",
|
|
610
|
+
content: [
|
|
611
|
+
{ type: "text", text: resultText(result) },
|
|
612
|
+
{ type: "image", data: result.data, mimeType: result.mimeType },
|
|
613
|
+
],
|
|
614
|
+
display: true,
|
|
615
|
+
details: result,
|
|
616
|
+
});
|
|
617
|
+
},
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
pi.registerTool({
|
|
621
|
+
name: OPENAI_IMAGE_TOOL,
|
|
622
|
+
label: "OpenAI image",
|
|
623
|
+
description:
|
|
624
|
+
"Generate or edit images through OpenAI Codex subscription auth using the hosted image_generation tool. Supports local reference/edit images and saves to the project by default.",
|
|
625
|
+
promptSnippet: "Generate or edit raster images via OpenAI Codex subscription auth.",
|
|
626
|
+
promptGuidelines: [
|
|
627
|
+
"Use openai_image when the user asks to generate or edit a raster image, photo, illustration, mockup, texture, sprite, or bitmap asset.",
|
|
628
|
+
"Pass the user's image prompt verbatim. Do not embellish, rewrite, add camera/style details, or add negative prompt terms unless the user explicitly asks you to refine the prompt.",
|
|
629
|
+
"Use openai_image with images for local reference images or edit targets; save project assets into the workspace when requested.",
|
|
630
|
+
],
|
|
631
|
+
parameters: TOOL_PARAMS,
|
|
632
|
+
async execute(_toolCallId, params: ToolParams, signal, onUpdate, ctx) {
|
|
633
|
+
const cfg = getConfig(ctx);
|
|
634
|
+
const model = resolveModel(params, ctx, cfg);
|
|
635
|
+
onUpdate?.({
|
|
636
|
+
content: [
|
|
637
|
+
{
|
|
638
|
+
type: "text",
|
|
639
|
+
text: `Requesting OpenAI image_generation via openai-codex/${model}...`,
|
|
640
|
+
},
|
|
641
|
+
],
|
|
642
|
+
details: undefined,
|
|
643
|
+
});
|
|
644
|
+
const result = await generate(params, ctx, signal);
|
|
645
|
+
return {
|
|
646
|
+
content: [
|
|
647
|
+
{ type: "text", text: resultText(result) },
|
|
648
|
+
{ type: "image", data: result.data, mimeType: result.mimeType },
|
|
649
|
+
],
|
|
650
|
+
details: result,
|
|
651
|
+
};
|
|
652
|
+
},
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
return { getDebug };
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
export const _imageTest = {
|
|
659
|
+
CODEX_RESPONSES_URL,
|
|
660
|
+
DEFAULT_TIMEOUT_MS,
|
|
661
|
+
OPENAI_IMAGE_TOOL,
|
|
662
|
+
OPENAI_IMAGE_COMMAND,
|
|
663
|
+
MAX_IMAGE_INPUT_BYTES,
|
|
664
|
+
MAX_IMAGE_INPUTS,
|
|
665
|
+
MAX_TOTAL_IMAGE_INPUT_BYTES,
|
|
666
|
+
extractAccountIdFromJwt,
|
|
667
|
+
imageMimeType,
|
|
668
|
+
dataUrlParts,
|
|
669
|
+
extractImageFromEvent,
|
|
670
|
+
displayPath,
|
|
671
|
+
buildRequest,
|
|
672
|
+
};
|