@oai404iao/pi-codex-imagegen 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +39 -7
- package/THIRD_PARTY_NOTICES.md +14 -2
- package/package.json +10 -9
- package/src/background-image-generation.ts +45 -303
- package/src/index.ts +8 -6
- package/src/tools/image-generation/output.ts +29 -0
- package/src/tools/image-generation.ts +80 -159
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -1,8 +1,22 @@
|
|
|
1
1
|
# pi-codex-imagegen
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
See the package guide below for release status; source edits do not publish artifacts.
|
|
4
4
|
|
|
5
|
-
Peer floor
|
|
5
|
+
Peer floor and development target: Pi 0.99.1.
|
|
6
|
+
|
|
7
|
+
Image generation uses standalone Images endpoints and is directly callable,
|
|
8
|
+
including through codemode while active. This does not enable codemode itself.
|
|
9
|
+
Hosted Responses image generation and `directImageApiFallback` have been removed.
|
|
10
|
+
Legacy hosted profiles are disabled with a diagnostic rather than silently routed
|
|
11
|
+
to a different implementation. Historical image outputs still replay and display.
|
|
12
|
+
|
|
13
|
+
Codemode scripts receive `{ path, latestPath?, image: { type, data, mimeType } }`.
|
|
14
|
+
The image data is base64; use `image(result.image)` to forward it to the model.
|
|
15
|
+
Saving and direct model-facing image/text output retain their existing behavior.
|
|
16
|
+
Failed generation rejects the call instead of returning success-shaped data.
|
|
17
|
+
Pi intentionally resolves retained structured data if a `tool_result` hook
|
|
18
|
+
only sets `isError: true`. To reject or redact, that hook must also replace
|
|
19
|
+
`content` without supplying `structuredContent`.
|
|
6
20
|
|
|
7
21
|
`/image-gen` jobs are cancelled when the session is replaced or closed.
|
|
8
22
|
Late authentication/results cannot notify the replacement session or initiate
|
|
@@ -10,13 +24,24 @@ new image writes. Already-started server generation or disk writes cannot be
|
|
|
10
24
|
rolled back; cancellation is not a guarantee of avoiding provider charges.
|
|
11
25
|
|
|
12
26
|
Installs `image_generation`, background image commands, image persistence and
|
|
13
|
-
presentation.
|
|
27
|
+
presentation. Its only workspace dependency is `pi-codex-runtime`, not core,
|
|
28
|
+
web-search or the bundle. Photon supplies image decoding and PNG encoding.
|
|
14
29
|
When global `config.json.imageGeneration` is `false`, none of those generation
|
|
15
|
-
surfaces are registered. The gate also blocks late/manual standalone
|
|
16
|
-
|
|
30
|
+
surfaces are registered. The gate also blocks late/manual standalone execution
|
|
31
|
+
before authentication or network I/O.
|
|
17
32
|
Catalog-supported standalone profiles call Images generation/edit endpoints with
|
|
18
|
-
Pi authentication.
|
|
19
|
-
|
|
33
|
+
Pi authentication. `transparent_background:true` requests transparency;
|
|
34
|
+
omitted or false requests an opaque background. References are sniffed by content,
|
|
35
|
+
including extensionless PNG/JPEG/WebP; GIF/BMP normalize to PNG without resizing.
|
|
36
|
+
All image paths, including background jobs, use
|
|
37
|
+
the selected provider's Pi-resolved credentials, headers and base URL. The
|
|
38
|
+
client never reads a separate `OPENAI_API_KEY` or forces the public OpenAI endpoint.
|
|
39
|
+
Authentication failure never switches accounts or providers.
|
|
40
|
+
`endpoint_config.imageGeneration` limits allowed modes at the authenticated
|
|
41
|
+
endpoint. Rejected/denied standalone jobs do not fall through to hosted jobs.
|
|
42
|
+
Explicit protocol rejection temporarily disables that mode for the session,
|
|
43
|
+
including background jobs. With `codexRequestExtensions:false`, standalone
|
|
44
|
+
requests omit generated Codex image-turn headers.
|
|
20
45
|
|
|
21
46
|
Uses the existing
|
|
22
47
|
`<agentDir>/extensions/pi-codex-minimal-tools/{config,models}.json` settings and
|
|
@@ -31,5 +56,12 @@ See [the package guide](../../docs/codex-packages.md). From repository root:
|
|
|
31
56
|
|
|
32
57
|
```bash
|
|
33
58
|
npm run check -w @oai404iao/pi-codex-imagegen
|
|
59
|
+
npm run test:codex-composition
|
|
34
60
|
npm run test:codex-packages
|
|
35
61
|
```
|
|
62
|
+
|
|
63
|
+
Client/capture/display/background-job regressions are in `tests/`; cross-package
|
|
64
|
+
integration is in root `tests/codex/`. See the
|
|
65
|
+
[shared configuration guide](../pi-codex-runtime/reference/configuration.md) and
|
|
66
|
+
[image-generation example](assets/image-generation.gif). The example is a
|
|
67
|
+
repository asset, not part of the runtime tarball.
|
package/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Source and license notices
|
|
2
2
|
|
|
3
|
-
This
|
|
3
|
+
This package owns image code split from `@oai404iao/pi-codex-minimal-tools`.
|
|
4
4
|
Project material remains Copyright (c) 2026 oai404iao. The composite LICENSE
|
|
5
5
|
and verified Apache license/NOTICE snapshots are retained.
|
|
6
6
|
|
|
@@ -11,7 +11,19 @@ apply-patch compatibility evidence remains pinned to
|
|
|
11
11
|
The unmodified provenance record preserves upstream source identifiers, not
|
|
12
12
|
claims that every referenced source file is shipped in this package.
|
|
13
13
|
|
|
14
|
+
Image request behavior and the runtime-owned namespace declaration now follow
|
|
15
|
+
Codex `5a3140176e668a2f72f3c098490eb7f7052d9d85`, including transparent-background
|
|
16
|
+
requests. The runtime package ships the exact source and canonical-JSON hashes
|
|
17
|
+
in `provenance/openai-codex-5a314017-image-generation.json`.
|
|
18
|
+
The earlier provenance record remains historical evidence.
|
|
19
|
+
|
|
14
20
|
Endpoint clients and presentation code are project implementations. Protocol
|
|
15
21
|
shapes are consumed from pi-codex-runtime rather than copied here.
|
|
16
22
|
|
|
17
|
-
|
|
23
|
+
Reference image normalization uses the separately distributed
|
|
24
|
+
`@silvia-odwyer/photon-node` 0.3.4 dependency (Apache-2.0), retaining original
|
|
25
|
+
dimensions and transparency when encoding GIF/BMP input as PNG. Its package
|
|
26
|
+
includes its own license; no Photon source or WASM binary is vendored here.
|
|
27
|
+
|
|
28
|
+
See `pi-codex-runtime/reference/source-map.md` in the repository for the pinned
|
|
29
|
+
evidence map and `reference/configuration.md` there for image configuration.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oai404iao/pi-codex-imagegen",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Unpublished pi-codex-imagegen workspace for the staged Codex split",
|
|
6
6
|
"type": "module",
|
|
@@ -28,9 +28,9 @@
|
|
|
28
28
|
]
|
|
29
29
|
},
|
|
30
30
|
"peerDependencies": {
|
|
31
|
-
"@earendil-works/pi-ai": ">=0.
|
|
32
|
-
"@earendil-works/pi-coding-agent": ">=0.
|
|
33
|
-
"@earendil-works/pi-tui": ">=0.
|
|
31
|
+
"@earendil-works/pi-ai": ">=0.99.1",
|
|
32
|
+
"@earendil-works/pi-coding-agent": ">=0.99.1",
|
|
33
|
+
"@earendil-works/pi-tui": ">=0.99.1",
|
|
34
34
|
"typebox": "*"
|
|
35
35
|
},
|
|
36
36
|
"peerDependenciesMeta": {
|
|
@@ -45,12 +45,13 @@
|
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@oai404iao/pi-codex-runtime": "0.
|
|
48
|
+
"@oai404iao/pi-codex-runtime": "1.0.0",
|
|
49
|
+
"@silvia-odwyer/photon-node": "0.3.4"
|
|
49
50
|
},
|
|
50
51
|
"devDependencies": {
|
|
51
|
-
"@earendil-works/pi-ai": "0.
|
|
52
|
-
"@earendil-works/pi-coding-agent": "0.
|
|
53
|
-
"@earendil-works/pi-tui": "0.
|
|
52
|
+
"@earendil-works/pi-ai": "0.99.1",
|
|
53
|
+
"@earendil-works/pi-coding-agent": "0.99.1",
|
|
54
|
+
"@earendil-works/pi-tui": "0.99.1",
|
|
54
55
|
"@types/node": "^26.2.0",
|
|
55
56
|
"tsx": "^4.20.6",
|
|
56
57
|
"typebox": "^1.1.24",
|
|
@@ -77,5 +78,5 @@
|
|
|
77
78
|
"engines": {
|
|
78
79
|
"node": ">=22.19.0"
|
|
79
80
|
},
|
|
80
|
-
"gitHead": "
|
|
81
|
+
"gitHead": "12c63eba3998323c7c0e1cf46f61291a20e13bed"
|
|
81
82
|
}
|
|
@@ -1,28 +1,17 @@
|
|
|
1
|
-
import type { Api, Model
|
|
1
|
+
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
2
|
import type { ExtensionAPI, ExtensionCommandContext, Theme } from "@earendil-works/pi-coding-agent";
|
|
3
3
|
import { truncateToWidth, type Component } from "@earendil-works/pi-tui";
|
|
4
|
-
import {
|
|
5
|
-
import { extname, isAbsolute, relative, resolve } from "node:path";
|
|
4
|
+
import { relative } from "node:path";
|
|
6
5
|
import { supportsImageInput, type ModelLike } from "@oai404iao/pi-codex-runtime/internal/capabilities";
|
|
7
|
-
import {
|
|
8
|
-
buildCodexJsonHeaders,
|
|
9
|
-
hasCodexRequestAuth,
|
|
10
|
-
} from "@oai404iao/pi-codex-runtime/internal/codex-http";
|
|
11
6
|
import { createBackgroundImageJobs, panelBranch } from "./background-image-jobs.js";
|
|
12
7
|
import { listResolvedModelProfiles } from "@oai404iao/pi-codex-runtime/internal/model-catalog/catalog";
|
|
13
8
|
import { loadModelSettings } from "@oai404iao/pi-codex-runtime/internal/model-catalog/runtime";
|
|
14
|
-
import { setProviderGeneratedHeader } from "@oai404iao/pi-codex-runtime/internal/provider-headers";
|
|
15
9
|
import { loadSettings } from "@oai404iao/pi-codex-runtime/internal/settings";
|
|
16
10
|
import { standaloneImageGeneration } from "./tools/image-generation.js";
|
|
17
|
-
import {
|
|
18
|
-
buildGeneratedImageDisplayText,
|
|
19
|
-
saveOpenAICodexGeneratedImage,
|
|
20
|
-
} from "./tools/image-generation/storage.js";
|
|
11
|
+
import { buildGeneratedImageDisplayText } from "./tools/image-generation/storage.js";
|
|
21
12
|
import { IMAGE_SAVE_DISPLAY_MESSAGE_TYPE, type SavedGeneratedImage } from "./tools/image-generation/types.js";
|
|
22
13
|
import { projectRoot } from "./utils/images.js";
|
|
23
14
|
|
|
24
|
-
const DEFAULT_CODEX_BASE_URL = "https://chatgpt.com/backend-api";
|
|
25
|
-
const BACKGROUND_IMAGE_INSTRUCTIONS = "Generate or edit images with the hosted image_generation tool. Use the user's prompt and any provided reference images. Return the image_generation_call result.";
|
|
26
15
|
const IMAGE_GEN_ERROR_MESSAGE_TYPE = "codex-image-generation-error";
|
|
27
16
|
|
|
28
17
|
export interface ParsedImageGenCommand {
|
|
@@ -30,20 +19,6 @@ export interface ParsedImageGenCommand {
|
|
|
30
19
|
imagePaths: string[];
|
|
31
20
|
}
|
|
32
21
|
|
|
33
|
-
interface ReferenceImage {
|
|
34
|
-
path: string;
|
|
35
|
-
mimeType: string;
|
|
36
|
-
base64: string;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
interface CodexImageResult {
|
|
40
|
-
id: string;
|
|
41
|
-
result: string;
|
|
42
|
-
outputFormat?: string;
|
|
43
|
-
revisedPrompt?: string;
|
|
44
|
-
imageModel?: string;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
22
|
interface ImageGenerationErrorDetails {
|
|
48
23
|
message: string;
|
|
49
24
|
prompt?: string;
|
|
@@ -57,12 +32,10 @@ interface ModelRegistryLike {
|
|
|
57
32
|
find?: (provider: string, id: string) => unknown;
|
|
58
33
|
}
|
|
59
34
|
|
|
60
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
61
|
-
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
35
|
function isModelLike(value: unknown): value is ModelLike {
|
|
65
|
-
|
|
36
|
+
if (!value || typeof value !== "object") return false;
|
|
37
|
+
const record = value as Record<string, unknown>;
|
|
38
|
+
return typeof record.provider === "string" && (typeof record.id === "string" || typeof record.name === "string");
|
|
66
39
|
}
|
|
67
40
|
|
|
68
41
|
function registryModels(registry: ModelRegistryLike | undefined): ModelLike[] {
|
|
@@ -81,24 +54,22 @@ function registryModels(registry: ModelRegistryLike | undefined): ModelLike[] {
|
|
|
81
54
|
|
|
82
55
|
function isConfiguredImageModel(model: ModelLike | undefined): boolean {
|
|
83
56
|
if (!model || !supportsImageInput(model)) return false;
|
|
84
|
-
const settings = loadModelSettings(model);
|
|
85
|
-
return Boolean(
|
|
86
|
-
settings.modelProfile?.effective.enabled
|
|
87
|
-
&& settings.imageGenerationImplementation,
|
|
88
|
-
);
|
|
57
|
+
const settings = loadModelSettings({ ...model, baseUrl: undefined });
|
|
58
|
+
return Boolean(settings.modelProfile?.effective.enabled && settings.imageGenerationImplementation === "standalone");
|
|
89
59
|
}
|
|
90
60
|
|
|
91
61
|
export function selectCodexImageModel(currentModel: ModelLike | undefined, registry: ModelRegistryLike | undefined): ModelLike | undefined {
|
|
62
|
+
const profile = currentModel && loadModelSettings({ ...currentModel, baseUrl: undefined }).modelProfile;
|
|
63
|
+
if (profile?.removedHostedImageGeneration) {
|
|
64
|
+
throw new Error(`Hosted image generation was removed for ${profile.id}. Change tools.imageGeneration to standalone or false in models.json; no alternate model was selected.`);
|
|
65
|
+
}
|
|
92
66
|
if (isConfiguredImageModel(currentModel)) return currentModel;
|
|
93
67
|
const discovered = registryModels(registry).find(isConfiguredImageModel);
|
|
94
68
|
if (discovered) return discovered;
|
|
95
69
|
for (const profile of listResolvedModelProfiles()) {
|
|
96
|
-
if (!profile.effective.enabled ||
|
|
70
|
+
if (!profile.effective.enabled || profile.effective.tools.imageGeneration !== "standalone") continue;
|
|
97
71
|
const slash = profile.id.indexOf("/");
|
|
98
|
-
const candidate = registry?.find?.(
|
|
99
|
-
profile.id.slice(0, slash),
|
|
100
|
-
profile.id.slice(slash + 1),
|
|
101
|
-
);
|
|
72
|
+
const candidate = registry?.find?.(profile.id.slice(0, slash), profile.id.slice(slash + 1));
|
|
102
73
|
if (isModelLike(candidate) && isConfiguredImageModel(candidate)) return candidate;
|
|
103
74
|
}
|
|
104
75
|
return undefined;
|
|
@@ -142,7 +113,7 @@ function tokenizeArgs(input: string): string[] {
|
|
|
142
113
|
function isSupportedImagePathToken(token: string): boolean {
|
|
143
114
|
const normalized = token.replace(/^file:\/\//, "").replace(/[),.;:]+$/, "");
|
|
144
115
|
if (/^https?:\/\//i.test(normalized) || normalized.startsWith("data:")) return false;
|
|
145
|
-
return /\.(?:png|jpe?g|webp)$/i.test(normalized);
|
|
116
|
+
return /\.(?:png|jpe?g|webp|gif|bmp)$/i.test(normalized);
|
|
146
117
|
}
|
|
147
118
|
|
|
148
119
|
export function parseImageGenCommandArgs(input: string): ParsedImageGenCommand {
|
|
@@ -156,150 +127,6 @@ export function parseImageGenCommandArgs(input: string): ParsedImageGenCommand {
|
|
|
156
127
|
return { prompt: promptParts.join(" ").trim(), imagePaths };
|
|
157
128
|
}
|
|
158
129
|
|
|
159
|
-
function mimeTypeForPath(path: string): string {
|
|
160
|
-
const ext = extname(path).toLowerCase();
|
|
161
|
-
if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
|
|
162
|
-
if (ext === ".webp") return "image/webp";
|
|
163
|
-
if (ext === ".png") return "image/png";
|
|
164
|
-
throw new Error(`Unsupported reference image type: ${path}. Use PNG, JPEG, or WebP.`);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
async function loadReferenceImage(cwd: string, rawPath: string): Promise<ReferenceImage> {
|
|
168
|
-
const path = isAbsolute(rawPath) ? rawPath : resolve(cwd, rawPath);
|
|
169
|
-
const buffer = await readFile(path);
|
|
170
|
-
return { path, mimeType: mimeTypeForPath(path), base64: buffer.toString("base64") };
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
function resolveCodexUrl(baseUrl: string | undefined, options?: { apiKeyMode?: boolean }): string {
|
|
174
|
-
const raw = baseUrl && baseUrl.trim().length > 0 ? baseUrl : DEFAULT_CODEX_BASE_URL;
|
|
175
|
-
const normalized = raw.replace(/\/+$/, "");
|
|
176
|
-
if (options?.apiKeyMode) {
|
|
177
|
-
if (normalized.endsWith("/responses")) return normalized;
|
|
178
|
-
return `${normalized}/responses`;
|
|
179
|
-
}
|
|
180
|
-
if (normalized.endsWith("/codex/responses")) return normalized;
|
|
181
|
-
if (normalized.endsWith("/codex")) return `${normalized}/responses`;
|
|
182
|
-
return `${normalized}/codex/responses`;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
function buildHeaders(
|
|
186
|
-
model: Model<Api>,
|
|
187
|
-
auth: { apiKey?: string; headers?: ProviderHeaders },
|
|
188
|
-
options?: { apiKeyMode?: boolean },
|
|
189
|
-
): Headers {
|
|
190
|
-
const headers = buildCodexJsonHeaders({
|
|
191
|
-
modelHeaders: model.headers,
|
|
192
|
-
auth,
|
|
193
|
-
apiKeyMode: options?.apiKeyMode ?? false,
|
|
194
|
-
});
|
|
195
|
-
setProviderGeneratedHeader(headers, "OpenAI-Beta", "responses=experimental");
|
|
196
|
-
setProviderGeneratedHeader(headers, "accept", "text/event-stream");
|
|
197
|
-
return headers;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
export function buildBackgroundImageRequest(options: {
|
|
201
|
-
prompt: string;
|
|
202
|
-
referenceImages: ReferenceImage[];
|
|
203
|
-
responsesModel: string;
|
|
204
|
-
imageModel: string;
|
|
205
|
-
}): Record<string, unknown> {
|
|
206
|
-
const content: Array<Record<string, unknown>> = [
|
|
207
|
-
{ type: "input_text", text: options.referenceImages.length > 0 ? `Edit the provided image(s): ${options.prompt}` : options.prompt },
|
|
208
|
-
...options.referenceImages.map((image) => ({
|
|
209
|
-
type: "input_image",
|
|
210
|
-
detail: "auto",
|
|
211
|
-
image_url: `data:${image.mimeType};base64,${image.base64}`,
|
|
212
|
-
})),
|
|
213
|
-
];
|
|
214
|
-
return {
|
|
215
|
-
model: options.responsesModel,
|
|
216
|
-
store: false,
|
|
217
|
-
stream: true,
|
|
218
|
-
instructions: BACKGROUND_IMAGE_INSTRUCTIONS,
|
|
219
|
-
input: [{ role: "user", content }],
|
|
220
|
-
tools: [{
|
|
221
|
-
type: "image_generation",
|
|
222
|
-
model: options.imageModel,
|
|
223
|
-
output_format: "png",
|
|
224
|
-
action: options.referenceImages.length > 0 ? "edit" : "generate",
|
|
225
|
-
}],
|
|
226
|
-
tool_choice: { type: "image_generation" },
|
|
227
|
-
};
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
async function* parseSseEvents(response: Response, signal: AbortSignal): AsyncIterable<Record<string, unknown>> {
|
|
231
|
-
if (!response.body) return;
|
|
232
|
-
const reader = response.body.getReader();
|
|
233
|
-
const decoder = new TextDecoder();
|
|
234
|
-
let buffer = "";
|
|
235
|
-
const cancel = () => { void reader.cancel(signal.reason).catch(() => {}); };
|
|
236
|
-
signal.addEventListener("abort", cancel, { once: true });
|
|
237
|
-
if (signal.aborted) cancel();
|
|
238
|
-
try {
|
|
239
|
-
while (true) {
|
|
240
|
-
signal.throwIfAborted();
|
|
241
|
-
const { done, value } = await reader.read();
|
|
242
|
-
signal.throwIfAborted();
|
|
243
|
-
if (done) break;
|
|
244
|
-
buffer += decoder.decode(value, { stream: true });
|
|
245
|
-
let boundary: number;
|
|
246
|
-
while ((boundary = buffer.indexOf("\n\n")) >= 0) {
|
|
247
|
-
const raw = buffer.slice(0, boundary);
|
|
248
|
-
buffer = buffer.slice(boundary + 2);
|
|
249
|
-
for (const line of raw.split(/\r?\n/)) {
|
|
250
|
-
if (!line.startsWith("data:")) continue;
|
|
251
|
-
const data = line.slice(5).trim();
|
|
252
|
-
if (!data || data === "[DONE]") continue;
|
|
253
|
-
yield JSON.parse(data) as Record<string, unknown>;
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
} finally {
|
|
258
|
-
signal.removeEventListener("abort", cancel);
|
|
259
|
-
reader.releaseLock();
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
function collectImageResult(results: CodexImageResult[], item: unknown, fallbackImageModel: string): void {
|
|
264
|
-
if (!item || typeof item !== "object") return;
|
|
265
|
-
const candidate = item as Record<string, unknown>;
|
|
266
|
-
if (candidate.type !== "image_generation_call" || typeof candidate.result !== "string") return;
|
|
267
|
-
const id = typeof candidate.id === "string" ? candidate.id : `ig_${results.length}`;
|
|
268
|
-
if (results.some((result) => result.id === id)) return;
|
|
269
|
-
results.push({
|
|
270
|
-
id,
|
|
271
|
-
result: candidate.result,
|
|
272
|
-
outputFormat: typeof candidate.output_format === "string" ? candidate.output_format : "png",
|
|
273
|
-
revisedPrompt: typeof candidate.revised_prompt === "string" ? candidate.revised_prompt : undefined,
|
|
274
|
-
imageModel: typeof candidate.model === "string" ? candidate.model : fallbackImageModel,
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
function collectResponseText(value: unknown, out: string[]): void {
|
|
279
|
-
if (!value || typeof value !== "object") return;
|
|
280
|
-
const item = value as Record<string, unknown>;
|
|
281
|
-
for (const key of ["text", "refusal", "summary_text", "message"] as const) {
|
|
282
|
-
const text = item[key];
|
|
283
|
-
if (typeof text === "string" && text.trim()) out.push(text.trim());
|
|
284
|
-
}
|
|
285
|
-
for (const key of ["content", "output", "summary"] as const) {
|
|
286
|
-
const child = item[key];
|
|
287
|
-
if (Array.isArray(child)) for (const part of child) collectResponseText(part, out);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
export function summarizeNonImageResponse(response: Record<string, unknown> | undefined): string {
|
|
292
|
-
if (!response) return "No image was returned by Codex.";
|
|
293
|
-
const status = typeof response.status === "string" ? response.status : undefined;
|
|
294
|
-
const error = response.error && typeof response.error === "object" ? response.error as Record<string, unknown> : undefined;
|
|
295
|
-
const errorMessage = typeof error?.message === "string" ? error.message : undefined;
|
|
296
|
-
const texts: string[] = [];
|
|
297
|
-
collectResponseText(response, texts);
|
|
298
|
-
const text = [...new Set(texts)].join(" ").replace(/\s+/g, " ").trim();
|
|
299
|
-
const details = [status && status !== "completed" ? `status ${status}` : undefined, errorMessage, text].filter(Boolean).join(" · ");
|
|
300
|
-
return details ? `No image was returned by Codex: ${details}` : "No image was returned by Codex.";
|
|
301
|
-
}
|
|
302
|
-
|
|
303
130
|
function renderImageGenError(details: ImageGenerationErrorDetails, theme: Theme): Component {
|
|
304
131
|
return {
|
|
305
132
|
invalidate() {},
|
|
@@ -322,108 +149,32 @@ function renderImageGenError(details: ImageGenerationErrorDetails, theme: Theme)
|
|
|
322
149
|
async function runBackgroundImageGeneration(pi: ExtensionAPI, ctx: ExtensionCommandContext, parsed: ParsedImageGenCommand, signal: AbortSignal): Promise<void> {
|
|
323
150
|
signal.throwIfAborted();
|
|
324
151
|
const model = selectCodexImageModel(ctx.model as ModelLike | undefined, ctx.modelRegistry as ModelRegistryLike | undefined) as Model<Api> | undefined;
|
|
325
|
-
if (!model) throw new Error("No image-capable model with an enabled model catalog profile is available.");
|
|
326
|
-
const settings = loadModelSettings(model
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
}, {
|
|
333
|
-
cwd: ctx.cwd,
|
|
334
|
-
model,
|
|
335
|
-
modelRegistry: ctx.modelRegistry,
|
|
336
|
-
}, settings, signal, { callId: "standalone" });
|
|
337
|
-
signal.throwIfAborted();
|
|
338
|
-
const saved = result.details.saved;
|
|
339
|
-
const workspaceRoot = projectRoot(ctx.cwd);
|
|
340
|
-
const relativePath = relative(workspaceRoot, saved.path);
|
|
341
|
-
const latestAbsolutePath = saved.latestPath ?? saved.path;
|
|
342
|
-
const latestRelativePath = relative(workspaceRoot, latestAbsolutePath);
|
|
343
|
-
const savedImage: SavedGeneratedImage = {
|
|
344
|
-
absolutePath: saved.path,
|
|
345
|
-
relativePath: relativePath && !relativePath.startsWith("..") ? relativePath : saved.path,
|
|
346
|
-
latestAbsolutePath,
|
|
347
|
-
latestRelativePath: latestRelativePath && !latestRelativePath.startsWith("..")
|
|
348
|
-
? latestRelativePath
|
|
349
|
-
: latestAbsolutePath,
|
|
350
|
-
responseId: undefined,
|
|
351
|
-
callId: "standalone",
|
|
352
|
-
outputFormat: saved.format,
|
|
353
|
-
imageModel: settings.imageModel,
|
|
354
|
-
revisedPrompt: parsed.prompt,
|
|
355
|
-
};
|
|
356
|
-
pi.sendMessage({
|
|
357
|
-
customType: IMAGE_SAVE_DISPLAY_MESSAGE_TYPE,
|
|
358
|
-
content: [{ type: "text", text: buildGeneratedImageDisplayText(savedImage, { expanded: false }) }],
|
|
359
|
-
display: true,
|
|
360
|
-
details: { savedImages: [savedImage] },
|
|
361
|
-
}, { triggerTurn: false });
|
|
362
|
-
return;
|
|
363
|
-
}
|
|
364
|
-
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
|
|
365
|
-
signal.throwIfAborted();
|
|
366
|
-
if (!auth.ok) throw new Error(auth.error);
|
|
367
|
-
if (!hasCodexRequestAuth({
|
|
368
|
-
modelHeaders: model.headers,
|
|
369
|
-
auth: { apiKey: auth.apiKey, headers: auth.headers },
|
|
370
|
-
})) {
|
|
371
|
-
throw new Error(`No request authentication is configured for ${model.provider}.`);
|
|
372
|
-
}
|
|
373
|
-
const referenceImages = await Promise.all(parsed.imagePaths.map((path) => loadReferenceImage(ctx.cwd, path)));
|
|
374
|
-
signal.throwIfAborted();
|
|
375
|
-
const body = buildBackgroundImageRequest({
|
|
376
|
-
prompt: parsed.prompt,
|
|
377
|
-
referenceImages,
|
|
378
|
-
responsesModel: model.id,
|
|
379
|
-
imageModel: settings.imageModel,
|
|
380
|
-
});
|
|
381
|
-
const response = await fetch(resolveCodexUrl(model.baseUrl, { apiKeyMode: settings.apiKeyMode }), {
|
|
382
|
-
signal,
|
|
383
|
-
method: "POST",
|
|
384
|
-
headers: buildHeaders(model, {
|
|
385
|
-
apiKey: auth.apiKey,
|
|
386
|
-
headers: auth.headers,
|
|
387
|
-
}, { apiKeyMode: settings.apiKeyMode }),
|
|
388
|
-
body: JSON.stringify(body),
|
|
389
|
-
});
|
|
390
|
-
signal.throwIfAborted();
|
|
391
|
-
if (!response.ok) throw new Error(`Codex image generation failed: ${response.status} ${await response.text()}`);
|
|
392
|
-
const results: CodexImageResult[] = [];
|
|
393
|
-
let responseId: string | undefined;
|
|
394
|
-
let lastResponse: Record<string, unknown> | undefined;
|
|
395
|
-
for await (const event of parseSseEvents(response, signal)) {
|
|
396
|
-
if (event.type === "response.created" && event.response && typeof event.response === "object") {
|
|
397
|
-
lastResponse = event.response as Record<string, unknown>;
|
|
398
|
-
const id = (event.response as { id?: unknown }).id;
|
|
399
|
-
if (typeof id === "string") responseId = id;
|
|
400
|
-
}
|
|
401
|
-
if (event.type === "response.output_item.done") collectImageResult(results, event.item, settings.imageModel);
|
|
402
|
-
if ((event.type === "response.completed" || event.type === "response.done") && event.response && typeof event.response === "object") {
|
|
403
|
-
lastResponse = event.response as Record<string, unknown>;
|
|
404
|
-
const responseOutput = (event.response as { output?: unknown }).output;
|
|
405
|
-
if (Array.isArray(responseOutput)) for (const item of responseOutput) collectImageResult(results, item, settings.imageModel);
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
if (results.length === 0) throw new Error(summarizeNonImageResponse(lastResponse));
|
|
409
|
-
const savedImages = [];
|
|
410
|
-
for (const result of results) {
|
|
411
|
-
signal.throwIfAborted();
|
|
412
|
-
savedImages.push(await saveOpenAICodexGeneratedImage(ctx.cwd, {
|
|
413
|
-
responseId,
|
|
414
|
-
callId: result.id,
|
|
415
|
-
result: result.result,
|
|
416
|
-
outputFormat: result.outputFormat,
|
|
417
|
-
imageModel: result.imageModel,
|
|
418
|
-
revisedPrompt: result.revisedPrompt ?? parsed.prompt,
|
|
419
|
-
}));
|
|
420
|
-
}
|
|
152
|
+
if (!model) throw new Error("No image-capable model with an enabled standalone model catalog profile is available.");
|
|
153
|
+
const settings = loadModelSettings({ ...model, baseUrl: undefined }, ctx.cwd);
|
|
154
|
+
const result = await standaloneImageGeneration({
|
|
155
|
+
prompt: parsed.prompt, referenced_image_paths: parsed.imagePaths,
|
|
156
|
+
}, {
|
|
157
|
+
cwd: ctx.cwd, model, modelRegistry: ctx.modelRegistry, sessionManager: ctx.sessionManager,
|
|
158
|
+
}, settings, signal, { callId: "standalone" });
|
|
421
159
|
signal.throwIfAborted();
|
|
160
|
+
const saved = result.details.saved;
|
|
161
|
+
const workspaceRoot = projectRoot(ctx.cwd);
|
|
162
|
+
const relativePath = relative(workspaceRoot, saved.path);
|
|
163
|
+
const latestAbsolutePath = saved.latestPath ?? saved.path;
|
|
164
|
+
const latestRelativePath = relative(workspaceRoot, latestAbsolutePath);
|
|
165
|
+
const savedImage: SavedGeneratedImage = {
|
|
166
|
+
absolutePath: saved.path,
|
|
167
|
+
relativePath: relativePath && !relativePath.startsWith("..") ? relativePath : saved.path,
|
|
168
|
+
latestAbsolutePath,
|
|
169
|
+
latestRelativePath: latestRelativePath && !latestRelativePath.startsWith("..") ? latestRelativePath : latestAbsolutePath,
|
|
170
|
+
responseId: result.details.requestId,
|
|
171
|
+
callId: "standalone", outputFormat: saved.format, imageModel: settings.imageModel,
|
|
172
|
+
revisedPrompt: parsed.prompt,
|
|
173
|
+
};
|
|
422
174
|
pi.sendMessage({
|
|
423
175
|
customType: IMAGE_SAVE_DISPLAY_MESSAGE_TYPE,
|
|
424
|
-
content: [{ type: "text", text: buildGeneratedImageDisplayText(
|
|
425
|
-
display: true,
|
|
426
|
-
details: { savedImages },
|
|
176
|
+
content: [{ type: "text", text: buildGeneratedImageDisplayText(savedImage, { expanded: false }) }],
|
|
177
|
+
display: true, details: { savedImages: [savedImage] },
|
|
427
178
|
}, { triggerTurn: false });
|
|
428
179
|
}
|
|
429
180
|
|
|
@@ -432,23 +183,15 @@ export function registerBackgroundImageGenerationCommand(pi: ExtensionAPI): void
|
|
|
432
183
|
pi.on("session_start", (_event, ctx) => jobs.reset(ctx));
|
|
433
184
|
pi.on("session_shutdown", (_event, ctx) => jobs.reset(ctx));
|
|
434
185
|
pi.registerMessageRenderer<ImageGenerationErrorDetails>(IMAGE_GEN_ERROR_MESSAGE_TYPE, (message, _options, theme) => {
|
|
435
|
-
const rawContent = typeof message.content === "string"
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
.filter((item) => item.type === "text")
|
|
439
|
-
.map((item) => item.text)
|
|
440
|
-
.join("\n");
|
|
441
|
-
const details: ImageGenerationErrorDetails = {
|
|
186
|
+
const rawContent = typeof message.content === "string" ? message.content
|
|
187
|
+
: message.content.filter((item) => item.type === "text").map((item) => item.text).join("\n");
|
|
188
|
+
return renderImageGenError({
|
|
442
189
|
message: message.details?.message ?? rawContent.replace(/^Image generation failed:\s*/i, "") ?? "Image generation failed.",
|
|
443
|
-
prompt: message.details?.prompt,
|
|
444
|
-
|
|
445
|
-
referenceCount: message.details?.referenceCount,
|
|
446
|
-
};
|
|
447
|
-
return renderImageGenError(details, theme);
|
|
190
|
+
prompt: message.details?.prompt, imageModel: message.details?.imageModel, referenceCount: message.details?.referenceCount,
|
|
191
|
+
}, theme);
|
|
448
192
|
});
|
|
449
|
-
|
|
450
193
|
pi.registerCommand("image-gen", {
|
|
451
|
-
description: "Generate or edit an image in the background with the
|
|
194
|
+
description: "Generate or edit an image in the background with the standalone Images API. Usage: /image-gen prompt text [@reference.png]",
|
|
452
195
|
handler: async (args, ctx) => {
|
|
453
196
|
const parsed = parseImageGenCommandArgs(args);
|
|
454
197
|
if (!parsed.prompt) {
|
|
@@ -464,8 +207,7 @@ export function registerBackgroundImageGenerationCommand(pi: ExtensionAPI): void
|
|
|
464
207
|
const message = error instanceof Error ? error.message : String(error);
|
|
465
208
|
pi.sendMessage({
|
|
466
209
|
customType: IMAGE_GEN_ERROR_MESSAGE_TYPE,
|
|
467
|
-
content: `Image generation failed: ${message}`,
|
|
468
|
-
display: true,
|
|
210
|
+
content: `Image generation failed: ${message}`, display: true,
|
|
469
211
|
details: { message, prompt: parsed.prompt, imageModel: settings.imageModel, referenceCount: parsed.imagePaths.length } satisfies ImageGenerationErrorDetails,
|
|
470
212
|
}, { triggerTurn: false });
|
|
471
213
|
})
|
package/src/index.ts
CHANGED
|
@@ -24,12 +24,14 @@ export default function codexImagegen(pi: ExtensionAPI): void {
|
|
|
24
24
|
return { createEventObserver: context => createImageCapture(context, sink) };
|
|
25
25
|
},
|
|
26
26
|
});
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
const definition = createImageGenerationToolDefinition({
|
|
28
|
+
loadSettings: (cwd, model) => loadModelSettings(model, cwd),
|
|
29
|
+
getCurrentTurnId: sessionId => currentCodexTurn(sessionId)?.turnId,
|
|
30
|
+
});
|
|
31
|
+
const parameters = { ...definition.parameters };
|
|
32
|
+
broker.ownedTools.set("image_generation", { parameters, description: definition.description });
|
|
33
|
+
addPackageTool(broker, "image_generation", exposure => {
|
|
34
|
+
pi.registerTool({ ...definition, parameters, exposure } as never);
|
|
33
35
|
});
|
|
34
36
|
registerBackgroundImageGenerationCommand(pi);
|
|
35
37
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { SavedImageInfo } from "../../utils/images.js";
|
|
2
|
+
|
|
3
|
+
export const imageGenerationOutputSchema = {
|
|
4
|
+
type: "object",
|
|
5
|
+
additionalProperties: false,
|
|
6
|
+
required: ["path", "image"],
|
|
7
|
+
properties: {
|
|
8
|
+
path: { type: "string" },
|
|
9
|
+
latestPath: { type: "string" },
|
|
10
|
+
image: {
|
|
11
|
+
type: "object",
|
|
12
|
+
additionalProperties: false,
|
|
13
|
+
required: ["type", "data", "mimeType"],
|
|
14
|
+
properties: {
|
|
15
|
+
type: { type: "string", const: "image" },
|
|
16
|
+
data: { type: "string", description: "Base64 image data. Use image(result.image) to show it to the model." },
|
|
17
|
+
mimeType: { type: "string" },
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export function imageGenerationOutput(saved: SavedImageInfo, base64: string) {
|
|
24
|
+
return {
|
|
25
|
+
path: saved.path,
|
|
26
|
+
...(saved.latestPath ? { latestPath: saved.latestPath } : {}),
|
|
27
|
+
image: { type: "image" as const, data: base64, mimeType: saved.mimeType },
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -1,25 +1,21 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
1
|
import { readFile } from "node:fs/promises";
|
|
3
|
-
import {
|
|
4
|
-
import { buildSessionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { isAbsolute, resolve } from "node:path";
|
|
3
|
+
import { buildSessionContext, detectSupportedImageMimeTypeFromFile, type SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
5
4
|
import { saveBase64Image } from "../utils/images.js";
|
|
5
|
+
import { imageGenerationOutput, imageGenerationOutputSchema } from "./image-generation/output.js";
|
|
6
6
|
import type { CodexMinimalToolsSettings } from "@oai404iao/pi-codex-runtime/internal/settings";
|
|
7
7
|
import type { Api, Model, ProviderHeaders } from "@earendil-works/pi-ai";
|
|
8
|
-
import {
|
|
9
|
-
buildCodexJsonHeaders,
|
|
10
|
-
hasCodexRequestAuth,
|
|
11
|
-
resolveCodexApiEndpoint,
|
|
12
|
-
} from "@oai404iao/pi-codex-runtime/internal/codex-http";
|
|
8
|
+
import { buildCodexJsonHeaders, hasCodexRequestAuth, resolveCodexApiEndpoint, withResolvedAuthBaseUrl } from "@oai404iao/pi-codex-runtime/internal/codex-http";
|
|
13
9
|
import { loadModelSettings, type ResolvedCodexModelSettings } from "@oai404iao/pi-codex-runtime/internal/model-catalog/runtime";
|
|
10
|
+
import { checkEndpointResponse, rememberResolvedEndpoint, requireEndpointCapability } from "@oai404iao/pi-codex-runtime/internal/endpoint-state";
|
|
11
|
+
import { currentCodexTurn, uuidV7 } from "@oai404iao/pi-codex-runtime/internal/codex-wire-identity";
|
|
12
|
+
import { fetchCodexJson } from "@oai404iao/pi-codex-runtime/internal/json-request";
|
|
14
13
|
|
|
15
14
|
export interface ImageGenerationInput {
|
|
16
15
|
prompt?: string;
|
|
17
|
-
referenced_image_paths?: string[];
|
|
18
|
-
num_last_images_to_include?: number;
|
|
19
|
-
|
|
20
|
-
quality?: "low" | "medium" | "high" | "auto";
|
|
21
|
-
background?: "transparent" | "opaque" | "auto";
|
|
22
|
-
output_format?: "png" | "webp" | "jpeg";
|
|
16
|
+
referenced_image_paths?: string[] | null;
|
|
17
|
+
num_last_images_to_include?: number | null;
|
|
18
|
+
transparent_background?: boolean;
|
|
23
19
|
}
|
|
24
20
|
|
|
25
21
|
export const imageGenerationToolSchema = {
|
|
@@ -28,66 +24,28 @@ export const imageGenerationToolSchema = {
|
|
|
28
24
|
properties: {
|
|
29
25
|
prompt: { type: "string", description: "Image generation or editing prompt." },
|
|
30
26
|
referenced_image_paths: {
|
|
31
|
-
type: "array",
|
|
32
|
-
maxItems: 5,
|
|
27
|
+
type: ["array", "null"], maxItems: 5,
|
|
33
28
|
items: { type: "string", minLength: 1 },
|
|
34
|
-
description: "Local PNG, JPEG,
|
|
29
|
+
description: "Local image paths to edit. PNG, JPEG, WebP, GIF and BMP are supported.",
|
|
35
30
|
},
|
|
36
31
|
num_last_images_to_include: {
|
|
37
|
-
type: "integer",
|
|
38
|
-
minimum: 1,
|
|
39
|
-
maximum: 5,
|
|
32
|
+
type: ["integer", "null"], minimum: 1, maximum: 5,
|
|
40
33
|
description: "Use the newest conversation images when one or more targets have no local path.",
|
|
41
34
|
},
|
|
35
|
+
transparent_background: {
|
|
36
|
+
type: "boolean",
|
|
37
|
+
description: "Whether the output should have a transparent background. Defaults to false.",
|
|
38
|
+
},
|
|
42
39
|
},
|
|
43
40
|
required: ["prompt"],
|
|
44
41
|
};
|
|
45
42
|
|
|
46
|
-
async function urlToBase64(url: string, signal?: AbortSignal): Promise<string> {
|
|
47
|
-
const response = await fetch(url, { signal });
|
|
48
|
-
if (!response.ok) throw new Error(`Failed to download generated image: ${response.status} ${await response.text()}`);
|
|
49
|
-
const buffer = Buffer.from(await response.arrayBuffer());
|
|
50
|
-
return buffer.toString("base64");
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export async function directImageGeneration(input: ImageGenerationInput, cwd: string, settings: CodexMinimalToolsSettings, signal?: AbortSignal) {
|
|
54
|
-
if (!settings.imageGeneration) throw new Error("Image generation is disabled by the global imageGeneration setting.");
|
|
55
|
-
if (!settings.directImageApiFallback) throw new Error("Direct Images API fallback is disabled. Use native openai or openai-codex handling, or enable directImageApiFallback.");
|
|
56
|
-
const apiKey = process.env.OPENAI_API_KEY;
|
|
57
|
-
if (!apiKey) throw new Error("OPENAI_API_KEY is required for direct image_generation fallback.");
|
|
58
|
-
if (!input.prompt?.trim()) throw new Error("A prompt is required for direct image_generation fallback.");
|
|
59
|
-
const body: Record<string, unknown> = {
|
|
60
|
-
model: settings.imageModel,
|
|
61
|
-
prompt: input.prompt,
|
|
62
|
-
};
|
|
63
|
-
if (input.size && input.size !== "auto") body.size = input.size;
|
|
64
|
-
if (input.quality && input.quality !== "auto") body.quality = input.quality;
|
|
65
|
-
if (input.background && input.background !== "auto") body.background = input.background;
|
|
66
|
-
if (input.output_format) body.output_format = input.output_format;
|
|
67
|
-
const response = await fetch("https://api.openai.com/v1/images/generations", {
|
|
68
|
-
method: "POST",
|
|
69
|
-
headers: { Authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
|
|
70
|
-
body: JSON.stringify(body),
|
|
71
|
-
signal,
|
|
72
|
-
});
|
|
73
|
-
if (!response.ok) throw new Error(`OpenAI Images API failed: ${response.status} ${await response.text()}`);
|
|
74
|
-
const json = await response.json() as { data?: Array<{ b64_json?: string; url?: string; revised_prompt?: string }> };
|
|
75
|
-
const first = json.data?.[0];
|
|
76
|
-
const base64 = first?.b64_json ?? (first?.url ? await urlToBase64(first.url, signal) : undefined);
|
|
77
|
-
if (!base64) throw new Error("OpenAI Images API returned no image data.");
|
|
78
|
-
const saved = await saveBase64Image({ base64, callId: "direct", cwd, format: input.output_format, responseId: settings.imageModel, settings });
|
|
79
|
-
return {
|
|
80
|
-
content: [{ type: "text", text: `Generated image with ${settings.imageModel}; saved to ${saved.path}${saved.latestPath ? ` (latest: ${saved.latestPath})` : ""}.` }],
|
|
81
|
-
details: { saved, revisedPrompt: first?.revised_prompt, mode: "direct-images-api" },
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
|
|
85
43
|
interface ImageGenerationToolContext {
|
|
86
44
|
cwd: string;
|
|
87
45
|
model?: Model<Api>;
|
|
88
46
|
modelRegistry?: {
|
|
89
47
|
getApiKeyAndHeaders(model: Model<Api>): Promise<
|
|
90
|
-
| { ok: true; apiKey?: string; headers?: ProviderHeaders }
|
|
48
|
+
| { ok: true; apiKey?: string; headers?: ProviderHeaders; baseUrl?: string }
|
|
91
49
|
| { ok: false; error: string }
|
|
92
50
|
>;
|
|
93
51
|
};
|
|
@@ -107,26 +65,27 @@ async function referencedImageUrls(cwd: string, paths: readonly string[]): Promi
|
|
|
107
65
|
return Promise.all(paths.map(async (rawPath) => {
|
|
108
66
|
const normalized = rawPath.replace(/^@/, "");
|
|
109
67
|
const path = isAbsolute(normalized) ? normalized : resolve(cwd, normalized);
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
? "image/png"
|
|
113
|
-
: extension === ".jpg" || extension === ".jpeg"
|
|
114
|
-
? "image/jpeg"
|
|
115
|
-
: extension === ".webp"
|
|
116
|
-
? "image/webp"
|
|
117
|
-
: undefined;
|
|
118
|
-
if (!mimeType) throw new Error(`Unsupported reference image type: ${rawPath}. Use PNG, JPEG, or WebP.`);
|
|
68
|
+
const mimeType = await detectSupportedImageMimeTypeFromFile(path);
|
|
69
|
+
if (!mimeType) throw new Error(`Unsupported reference image type: ${rawPath}. Use PNG, JPEG, WebP, GIF, or BMP.`);
|
|
119
70
|
const data = await readFile(path);
|
|
71
|
+
if (mimeType === "image/gif" || mimeType === "image/bmp") {
|
|
72
|
+
// Codex Original mode normalizes unsupported source formats to PNG without resizing.
|
|
73
|
+
const { PhotonImage } = await import("@silvia-odwyer/photon-node");
|
|
74
|
+
const decoded = PhotonImage.new_from_byteslice(data);
|
|
75
|
+
try {
|
|
76
|
+
return { image_url: `data:image/png;base64,${Buffer.from(decoded.get_bytes()).toString("base64")}` };
|
|
77
|
+
} finally {
|
|
78
|
+
decoded.free();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
120
81
|
return { image_url: `data:${mimeType};base64,${data.toString("base64")}` };
|
|
121
82
|
}));
|
|
122
83
|
}
|
|
123
84
|
|
|
124
|
-
function recentConversationImageUrls(
|
|
125
|
-
ctx
|
|
126
|
-
count
|
|
127
|
-
|
|
128
|
-
if (!ctx.sessionManager) {
|
|
129
|
-
throw new Error("Conversation images are unavailable in this tool context; use referenced_image_paths.");
|
|
85
|
+
function recentConversationImageUrls(ctx: ImageGenerationToolContext, count: number): Array<{ image_url: string }> {
|
|
86
|
+
if (!ctx.sessionManager) throw new Error("Conversation images are unavailable in this tool context; use referenced_image_paths.");
|
|
87
|
+
if (!Number.isSafeInteger(count) || count < 1 || count > 5) {
|
|
88
|
+
throw new Error("num_last_images_to_include must be an integer between 1 and 5.");
|
|
130
89
|
}
|
|
131
90
|
const messages = buildSessionContext(ctx.sessionManager.getBranch()).messages;
|
|
132
91
|
const images: string[] = [];
|
|
@@ -134,31 +93,20 @@ function recentConversationImageUrls(
|
|
|
134
93
|
if (message.role === "user" || message.role === "toolResult") {
|
|
135
94
|
if (!Array.isArray(message.content)) continue;
|
|
136
95
|
for (const item of message.content) {
|
|
137
|
-
if (item.type === "image") {
|
|
138
|
-
images.push(`data:${item.mimeType};base64,${item.data}`);
|
|
139
|
-
}
|
|
96
|
+
if (item.type === "image") images.push(`data:${item.mimeType};base64,${item.data}`);
|
|
140
97
|
}
|
|
141
98
|
continue;
|
|
142
99
|
}
|
|
143
100
|
if (message.role !== "assistant") continue;
|
|
144
101
|
for (const block of message.content as unknown[]) {
|
|
145
102
|
if (!block || typeof block !== "object") continue;
|
|
146
|
-
const candidate = block as {
|
|
147
|
-
|
|
148
|
-
item?: { result?: unknown };
|
|
149
|
-
};
|
|
150
|
-
if (
|
|
151
|
-
candidate.type === "image_generation_call"
|
|
152
|
-
&& typeof candidate.item?.result === "string"
|
|
153
|
-
&& candidate.item.result
|
|
154
|
-
) {
|
|
103
|
+
const candidate = block as { type?: unknown; item?: { result?: unknown } };
|
|
104
|
+
if (candidate.type === "image_generation_call" && typeof candidate.item?.result === "string" && candidate.item.result) {
|
|
155
105
|
images.push(`data:image/png;base64,${candidate.item.result}`);
|
|
156
106
|
}
|
|
157
107
|
}
|
|
158
108
|
}
|
|
159
|
-
if (images.length < count) {
|
|
160
|
-
throw new Error(`Requested the last ${count} conversation images, but only ${images.length} were available.`);
|
|
161
|
-
}
|
|
109
|
+
if (images.length < count) throw new Error(`Requested the last ${count} conversation images, but only ${images.length} were available.`);
|
|
162
110
|
return images.slice(-count).map((image_url) => ({ image_url }));
|
|
163
111
|
}
|
|
164
112
|
|
|
@@ -171,119 +119,92 @@ export async function standaloneImageGeneration(
|
|
|
171
119
|
) {
|
|
172
120
|
signal?.throwIfAborted();
|
|
173
121
|
const callId = invocation.callId ?? "standalone";
|
|
174
|
-
const
|
|
122
|
+
const sessionId = ctx.sessionManager?.getSessionId?.();
|
|
123
|
+
const turnId = invocation.turnId ?? currentCodexTurn(sessionId)?.turnId ?? uuidV7();
|
|
175
124
|
const model = ctx.model;
|
|
176
125
|
if (!model || !ctx.modelRegistry) throw new Error("No active model is available for standalone image generation.");
|
|
177
126
|
if (!settings.enabled) throw new Error("pi-codex-minimal-tools is disabled.");
|
|
178
127
|
if (!settings.imageGeneration) throw new Error("Image generation is disabled by the global setting or current model profile.");
|
|
128
|
+
if (settings.imageGenerationImplementation !== "standalone") throw new Error("Image generation requires a standalone model profile.");
|
|
179
129
|
if (!input.prompt?.trim()) throw new Error("A prompt is required for standalone image generation.");
|
|
180
|
-
if (input.
|
|
130
|
+
if (input.transparent_background !== undefined && typeof input.transparent_background !== "boolean") {
|
|
131
|
+
throw new Error("transparent_background must be a boolean.");
|
|
132
|
+
}
|
|
133
|
+
if (input.num_last_images_to_include != null && input.referenced_image_paths?.length) {
|
|
181
134
|
throw new Error("Provide only one of referenced_image_paths or num_last_images_to_include.");
|
|
182
135
|
}
|
|
136
|
+
// Capture the invocation's images and turn before auth can yield to another turn.
|
|
137
|
+
const images = input.num_last_images_to_include != null
|
|
138
|
+
? recentConversationImageUrls(ctx, input.num_last_images_to_include)
|
|
139
|
+
: await referencedImageUrls(ctx.cwd, input.referenced_image_paths ?? []);
|
|
183
140
|
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
|
|
184
141
|
signal?.throwIfAborted();
|
|
185
142
|
if (!auth.ok) throw new Error(auth.error);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
})) {
|
|
143
|
+
const requestModel = withResolvedAuthBaseUrl(model, auth);
|
|
144
|
+
rememberResolvedEndpoint(model, requestModel, sessionId);
|
|
145
|
+
requireEndpointCapability(settings.endpoint_config, requestModel, sessionId, "imageGeneration.standalone");
|
|
146
|
+
if (!hasCodexRequestAuth({ modelHeaders: model.headers, auth })) {
|
|
190
147
|
throw new Error(`No request authentication for provider: ${model.provider}`);
|
|
191
148
|
}
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
: await referencedImageUrls(ctx.cwd, input.referenced_image_paths ?? []);
|
|
195
|
-
const edit = images.length > 0;
|
|
196
|
-
signal?.throwIfAborted();
|
|
197
|
-
const response = await fetch(
|
|
198
|
-
resolveCodexApiEndpoint(
|
|
199
|
-
model.baseUrl,
|
|
200
|
-
settings.apiKeyMode,
|
|
201
|
-
edit ? "images/edits" : "images/generations",
|
|
202
|
-
),
|
|
149
|
+
const response = await fetchCodexJson(
|
|
150
|
+
resolveCodexApiEndpoint(auth.baseUrl ?? model.baseUrl, settings.responsesEndpoint, images.length ? "images/edits" : "images/generations"),
|
|
203
151
|
{
|
|
204
|
-
method: "POST",
|
|
205
152
|
headers: buildCodexJsonHeaders({
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
extraHeaders: {
|
|
210
|
-
"x-codex-image-turn-id": turnId,
|
|
211
|
-
},
|
|
153
|
+
codexRequestExtensions: settings.codexRequestExtensions,
|
|
154
|
+
modelHeaders: model.headers, auth, endpoint: settings.responsesEndpoint,
|
|
155
|
+
extraHeaders: settings.codexRequestExtensions ? { "x-codex-image-turn-id": turnId } : undefined,
|
|
212
156
|
}),
|
|
213
157
|
body: JSON.stringify({
|
|
214
|
-
model: settings.imageModel,
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
quality: "auto",
|
|
219
|
-
size: "auto",
|
|
158
|
+
model: settings.imageModel, prompt: input.prompt,
|
|
159
|
+
...(images.length ? { images } : {}),
|
|
160
|
+
background: input.transparent_background === true ? "transparent" : "opaque",
|
|
161
|
+
quality: "auto", size: "auto",
|
|
220
162
|
}),
|
|
221
163
|
signal,
|
|
222
164
|
},
|
|
165
|
+
["imageGeneration.standalone"],
|
|
223
166
|
);
|
|
224
167
|
signal?.throwIfAborted();
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
const result = await response.json() as {
|
|
229
|
-
data?: Array<{ b64_json?: string }>;
|
|
230
|
-
};
|
|
168
|
+
await checkEndpointResponse(response, requestModel, sessionId, ["imageGeneration.standalone"], signal);
|
|
169
|
+
const result = await response.json() as { data?: Array<{ b64_json?: string; generation_id?: string }> };
|
|
231
170
|
signal?.throwIfAborted();
|
|
232
171
|
const base64 = result.data?.[0]?.b64_json;
|
|
233
172
|
if (!base64) throw new Error("Standalone image generation returned no image data.");
|
|
234
|
-
const saved = await saveBase64Image({
|
|
235
|
-
base64,
|
|
236
|
-
callId,
|
|
237
|
-
cwd: ctx.cwd,
|
|
238
|
-
format: "png",
|
|
239
|
-
responseId: settings.imageModel,
|
|
240
|
-
settings,
|
|
241
|
-
});
|
|
173
|
+
const saved = await saveBase64Image({ base64, callId, cwd: ctx.cwd, format: "png", responseId: settings.imageModel, settings });
|
|
242
174
|
signal?.throwIfAborted();
|
|
243
175
|
return {
|
|
244
176
|
content: [
|
|
245
177
|
{ type: "image", data: base64, mimeType: "image/png" },
|
|
246
178
|
{ type: "text", text: `Generated image with ${settings.imageModel}; saved to ${saved.path}${saved.latestPath ? ` (latest: ${saved.latestPath})` : ""}.` },
|
|
247
179
|
],
|
|
248
|
-
details: {
|
|
180
|
+
details: {
|
|
181
|
+
saved, mode: "standalone-images-api",
|
|
182
|
+
requestId: response.headers.get("x-codex-imagegen-request-id") ?? undefined,
|
|
183
|
+
generationId: result.data?.[0]?.generation_id,
|
|
184
|
+
},
|
|
185
|
+
structuredContent: imageGenerationOutput(saved, base64),
|
|
249
186
|
};
|
|
250
187
|
}
|
|
251
188
|
|
|
252
189
|
export function createImageGenerationToolDefinition(options: {
|
|
253
190
|
loadSettings?: (cwd: string, model?: Model<Api>) => CodexMinimalToolsSettings | ResolvedCodexModelSettings;
|
|
254
191
|
getCurrentTurnId?: (sessionId: string | undefined) => string | undefined;
|
|
255
|
-
hasProviderRuntime?: () => boolean;
|
|
256
192
|
} = {}) {
|
|
257
193
|
return {
|
|
258
194
|
name: "image_generation",
|
|
259
195
|
label: "Image Generation",
|
|
260
|
-
description: "Generate or edit images using the
|
|
196
|
+
description: "Generate or edit images using the standalone Images API selected by the current model profile. Results are saved under imageOutputDir and mirrored to latest.png.",
|
|
261
197
|
promptSnippet: "Generate or edit images with the implementation selected by the current model profile.",
|
|
262
198
|
parameters: imageGenerationToolSchema,
|
|
199
|
+
outputSchema: imageGenerationOutputSchema,
|
|
263
200
|
async execute(toolCallId: string, params: ImageGenerationInput, signal: AbortSignal | undefined, _onUpdate: unknown, ctx: ImageGenerationToolContext) {
|
|
264
201
|
const cwd = ctx?.cwd ?? process.cwd();
|
|
265
|
-
const
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
}
|
|
272
|
-
if (resolvedSettings.imageGenerationImplementation === "standalone") {
|
|
273
|
-
const sessionId = ctx.sessionManager?.getSessionId?.();
|
|
274
|
-
return standaloneImageGeneration(params, ctx, resolvedSettings, signal, {
|
|
275
|
-
callId: toolCallId,
|
|
276
|
-
turnId: options.getCurrentTurnId?.(sessionId),
|
|
277
|
-
});
|
|
278
|
-
}
|
|
279
|
-
if (settings?.directImageApiFallback) return directImageGeneration(params, cwd, settings, signal);
|
|
280
|
-
if (options.hasProviderRuntime?.() === false) {
|
|
281
|
-
throw new Error("Hosted image_generation requires pi-codex-core. Use a catalog-supported standalone profile or explicitly enable directImageApiFallback.");
|
|
282
|
-
}
|
|
283
|
-
return {
|
|
284
|
-
content: [{ type: "text", text: "image_generation should be handled by the current model profile. If no hosted or standalone implementation is configured, enable directImageApiFallback with OPENAI_API_KEY." }],
|
|
285
|
-
details: { phase: "native-provider", nativeTool: "image_generation" },
|
|
286
|
-
};
|
|
202
|
+
const profileModel = ctx.model ? { ...ctx.model, baseUrl: "" } : undefined;
|
|
203
|
+
const settings = options.loadSettings?.(cwd, profileModel) ?? loadModelSettings(profileModel, cwd);
|
|
204
|
+
const resolved = "modelProfile" in settings ? settings as ResolvedCodexModelSettings : loadModelSettings(profileModel, cwd, settings);
|
|
205
|
+
return standaloneImageGeneration(params, { ...ctx, cwd }, resolved, signal, {
|
|
206
|
+
callId: toolCallId, turnId: options.getCurrentTurnId?.(ctx.sessionManager?.getSessionId?.()),
|
|
207
|
+
});
|
|
287
208
|
},
|
|
288
209
|
};
|
|
289
210
|
}
|