@oai404iao/pi-codex-imagegen 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/LICENSES/Apache-2.0.txt +201 -0
- package/LICENSES/OpenAI-Codex-NOTICE.txt +6 -0
- package/README.md +32 -0
- package/THIRD_PARTY_NOTICES.md +17 -0
- package/package.json +80 -0
- package/provenance/openai-codex-eb9dceba-reserved-tools.json +140 -0
- package/src/background-image-generation.ts +476 -0
- package/src/background-image-jobs.ts +116 -0
- package/src/index.ts +33 -0
- package/src/tools/image-generation/capture.ts +42 -0
- package/src/tools/image-generation/display.ts +73 -0
- package/src/tools/image-generation/preview.ts +79 -0
- package/src/tools/image-generation/storage.ts +185 -0
- package/src/tools/image-generation/types.ts +36 -0
- package/src/tools/image-generation.ts +284 -0
- package/src/utils/images.ts +73 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { makeCachedImagePreview, renderImageGenerationMessage } from "./preview.js";
|
|
3
|
+
import { buildGeneratedImageDisplayText } from "./storage.js";
|
|
4
|
+
import {
|
|
5
|
+
IMAGE_SAVE_DISPLAY_MESSAGE_TYPE,
|
|
6
|
+
type CachedImagePreview,
|
|
7
|
+
type ImageDisplayMessageDetails,
|
|
8
|
+
type PendingActivity,
|
|
9
|
+
type SavedGeneratedImage,
|
|
10
|
+
} from "./types.js";
|
|
11
|
+
|
|
12
|
+
export function createImageDisplay(pi: ExtensionAPI) {
|
|
13
|
+
const pendingActivities: PendingActivity[] = [];
|
|
14
|
+
const imagePreviewCache = new Map<string, CachedImagePreview>();
|
|
15
|
+
let pendingFlushTimer: ReturnType<typeof setTimeout> | undefined;
|
|
16
|
+
let generation = 0;
|
|
17
|
+
|
|
18
|
+
const cancelFlush = () => {
|
|
19
|
+
if (pendingFlushTimer !== undefined) clearTimeout(pendingFlushTimer);
|
|
20
|
+
pendingFlushTimer = undefined;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const flush = () => {
|
|
24
|
+
cancelFlush();
|
|
25
|
+
const activities = pendingActivities.splice(0, pendingActivities.length);
|
|
26
|
+
for (const activity of activities) {
|
|
27
|
+
imagePreviewCache.set(activity.savedImage.absolutePath, makeCachedImagePreview(activity.imageData.data, activity.imageData.mimeType));
|
|
28
|
+
pi.sendMessage(
|
|
29
|
+
{
|
|
30
|
+
customType: IMAGE_SAVE_DISPLAY_MESSAGE_TYPE,
|
|
31
|
+
content: [{ type: "text", text: buildGeneratedImageDisplayText(activity.savedImage, { expanded: false }) }],
|
|
32
|
+
display: true,
|
|
33
|
+
details: { savedImages: [activity.savedImage] } satisfies ImageDisplayMessageDetails,
|
|
34
|
+
},
|
|
35
|
+
{ triggerTurn: false },
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
flush,
|
|
42
|
+
scheduleFlush() {
|
|
43
|
+
if (pendingFlushTimer !== undefined || pendingActivities.length === 0) return;
|
|
44
|
+
pendingFlushTimer = setTimeout(flush, 0);
|
|
45
|
+
},
|
|
46
|
+
clear() {
|
|
47
|
+
generation++;
|
|
48
|
+
cancelFlush();
|
|
49
|
+
pendingActivities.length = 0;
|
|
50
|
+
imagePreviewCache.clear();
|
|
51
|
+
},
|
|
52
|
+
/** Ignore completion callbacks belonging to a replaced/closed session. */
|
|
53
|
+
captureSink() {
|
|
54
|
+
const requestGeneration = generation;
|
|
55
|
+
return (savedImage: SavedGeneratedImage, imageData: { data: string; mimeType: string }) => {
|
|
56
|
+
if (requestGeneration !== generation) return;
|
|
57
|
+
pendingActivities.push({ kind: "image", savedImage, imageData });
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
registerRenderer() {
|
|
61
|
+
pi.registerMessageRenderer<ImageDisplayMessageDetails>(IMAGE_SAVE_DISPLAY_MESSAGE_TYPE, (message, options, theme) => {
|
|
62
|
+
const savedImage = message.details?.savedImages?.[0];
|
|
63
|
+
const textContent = typeof message.content === "string"
|
|
64
|
+
? message.content
|
|
65
|
+
: message.content
|
|
66
|
+
.filter((item) => item.type === "text")
|
|
67
|
+
.map((item) => item.text)
|
|
68
|
+
.join("\n");
|
|
69
|
+
return renderImageGenerationMessage(savedImage, textContent, options, theme, imagePreviewCache);
|
|
70
|
+
});
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { Container, Image, Spacer, Text, getCapabilities, getImageDimensions } from "@earendil-works/pi-tui";
|
|
2
|
+
import { glyphs, treeGlyph } from "@oai404iao/pi-codex-runtime/internal/glyphs";
|
|
3
|
+
import { themeBold, themeFg } from "@oai404iao/pi-codex-runtime/internal/utils/theme";
|
|
4
|
+
import { type CachedImagePreview, type SavedGeneratedImage } from "./types.js";
|
|
5
|
+
|
|
6
|
+
function getNodeFsSync(): { readFileSync(path: string): Buffer } | null {
|
|
7
|
+
if (typeof process === "undefined" || !(process.versions?.node || process.versions?.bun)) {
|
|
8
|
+
return null;
|
|
9
|
+
}
|
|
10
|
+
const builtinProcess = process as typeof process & { getBuiltinModule?: (specifier: string) => unknown };
|
|
11
|
+
if (typeof builtinProcess.getBuiltinModule !== "function") {
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const module = builtinProcess.getBuiltinModule("node:fs") as { readFileSync?: (path: string) => Buffer } | undefined;
|
|
16
|
+
return typeof module?.readFileSync === "function" ? { readFileSync: module.readFileSync } : null;
|
|
17
|
+
} catch {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function makeCachedImagePreview(data: string, mimeType: string, bytes?: number): CachedImagePreview {
|
|
23
|
+
const dimensions = getImageDimensions(data, mimeType) ?? undefined;
|
|
24
|
+
return { data, mimeType, bytes: bytes ?? Buffer.from(data, "base64").byteLength, widthPx: dimensions?.widthPx, heightPx: dimensions?.heightPx };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function loadCachedImagePreview(savedImage: SavedGeneratedImage, imagePreviewCache: Map<string, CachedImagePreview>): CachedImagePreview | undefined {
|
|
28
|
+
const cached = imagePreviewCache.get(savedImage.absolutePath);
|
|
29
|
+
if (cached) return cached;
|
|
30
|
+
const fs = getNodeFsSync();
|
|
31
|
+
if (!fs) return undefined;
|
|
32
|
+
try {
|
|
33
|
+
const buffer = fs.readFileSync(savedImage.absolutePath);
|
|
34
|
+
const data = buffer.toString("base64");
|
|
35
|
+
const mimeType = `image/${savedImage.outputFormat}`;
|
|
36
|
+
const preview = makeCachedImagePreview(data, mimeType, buffer.byteLength);
|
|
37
|
+
imagePreviewCache.set(savedImage.absolutePath, preview);
|
|
38
|
+
return preview;
|
|
39
|
+
} catch {
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function formatImageBytes(bytes: number | undefined): string | undefined {
|
|
45
|
+
if (!Number.isFinite(bytes) || !bytes) return undefined;
|
|
46
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
47
|
+
if (bytes < 1024 * 1024) return `${Math.round(bytes / 102.4) / 10}K`;
|
|
48
|
+
return `${Math.round(bytes / (1024 * 102.4)) / 10}M`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function shouldRenderInlineImage(): { ok: boolean; reason?: string } {
|
|
52
|
+
if (process.env.TMUX) return { ok: false, reason: "inline preview disabled in tmux to avoid overlay/stale image artifacts" };
|
|
53
|
+
const protocol = getCapabilities().images;
|
|
54
|
+
if (!protocol) return { ok: false, reason: "terminal image protocol unavailable" };
|
|
55
|
+
return { ok: true };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function renderImageGenerationMessage(savedImage: SavedGeneratedImage | undefined, messageContent: unknown, options: any, theme: any, imagePreviewCache: Map<string, CachedImagePreview>): Container {
|
|
59
|
+
const container = new Container();
|
|
60
|
+
const preview = savedImage ? loadCachedImagePreview(savedImage, imagePreviewCache) : undefined;
|
|
61
|
+
const type = savedImage?.outputFormat?.toUpperCase() ?? preview?.mimeType?.replace(/^image\//, "").toUpperCase() ?? "IMAGE";
|
|
62
|
+
const dimensions = preview?.widthPx && preview?.heightPx ? `${preview.widthPx}x${preview.heightPx}` : undefined;
|
|
63
|
+
const size = formatImageBytes(preview?.bytes);
|
|
64
|
+
const imageModel = savedImage?.imageModel ? `model ${savedImage.imageModel}` : undefined;
|
|
65
|
+
const meta = [imageModel, type, dimensions, size].filter(Boolean).join(glyphs().dot);
|
|
66
|
+
const label = `${themeFg(theme, "accent", glyphs().bullet)}${themeFg(theme, "text", themeBold(theme, "Image Generation "))}`;
|
|
67
|
+
const pathText = savedImage?.relativePath ?? (typeof messageContent === "string" ? messageContent : "generated image");
|
|
68
|
+
const lines = [`${label}${themeFg(theme, "accent", pathText)}${meta ? themeFg(theme, "dim", `${glyphs().dot}${meta}`) : ""}`];
|
|
69
|
+
if (savedImage?.latestRelativePath) lines.push(`${themeFg(theme, "muted", ` ${treeGlyph("├")}`)}${themeFg(theme, "text", "Latest ")}${themeFg(theme, "accent", savedImage.latestRelativePath)}`);
|
|
70
|
+
if (options?.expanded && savedImage?.revisedPrompt) lines.push(`${themeFg(theme, "muted", ` ${treeGlyph("├")}`)}${themeFg(theme, "text", "Prompt ")}${themeFg(theme, "dim", savedImage.revisedPrompt)}`);
|
|
71
|
+
const inline = shouldRenderInlineImage();
|
|
72
|
+
if (!inline.ok) lines.push(`${themeFg(theme, "muted", ` ${treeGlyph("└")}`)}${themeFg(theme, "warning", inline.reason ?? "inline preview unavailable")}`);
|
|
73
|
+
container.addChild(new Text(lines.join("\n"), 0, 0));
|
|
74
|
+
if (savedImage && preview && inline.ok) {
|
|
75
|
+
container.addChild(new Spacer(1));
|
|
76
|
+
container.addChild(new Image(preview.data, preview.mimeType, { fallbackColor: (text) => themeFg(theme, "dim", text) }, { maxWidthCells: 72, maxHeightCells: options?.expanded ? 24 : 14, filename: savedImage.relativePath }));
|
|
77
|
+
}
|
|
78
|
+
return container;
|
|
79
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { loadSettings } from "@oai404iao/pi-codex-runtime/internal/settings";
|
|
2
|
+
import { saveBase64Image } from "../../utils/images.js";
|
|
3
|
+
import { type SavedGeneratedImage } from "./types.js";
|
|
4
|
+
|
|
5
|
+
const OPENAI_CODEX_IMAGE_DIR = ".pi/openai-codex-images";
|
|
6
|
+
|
|
7
|
+
const OPENAI_CODEX_LATEST_IMAGE_NAME = "latest.png";
|
|
8
|
+
|
|
9
|
+
let fsPromisesPromise: Promise<typeof import("node:fs/promises")> | undefined;
|
|
10
|
+
|
|
11
|
+
const workspaceRootCache = new Map<string, Promise<string>>();
|
|
12
|
+
|
|
13
|
+
const PATH_SEPARATOR = "/";
|
|
14
|
+
|
|
15
|
+
function sanitizeFilePart(value: string | undefined, fallback: string): string {
|
|
16
|
+
const trimmed = (value ?? "").trim();
|
|
17
|
+
if (!trimmed) return fallback;
|
|
18
|
+
return trimmed.replace(/[^a-zA-Z0-9._-]+/g, "-");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function shortenFilePart(value: string | undefined, fallback: string): string {
|
|
22
|
+
const safe = sanitizeFilePart(value, fallback);
|
|
23
|
+
const match = /^([a-zA-Z]+_)(.+)$/.exec(safe);
|
|
24
|
+
const prefix = match?.[1] ?? "";
|
|
25
|
+
const body = match?.[2] ?? safe;
|
|
26
|
+
if (body.length <= 12) return `${prefix}${body}`;
|
|
27
|
+
return `${prefix}${body.slice(0, 8)}-${body.slice(-4)}`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function normalizeImageOutputFormat(value: string | undefined): string {
|
|
31
|
+
const format = (value ?? "png").toLowerCase();
|
|
32
|
+
return format === "png" || format === "jpg" || format === "jpeg" || format === "webp" ? format : "png";
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function normalizePath(value: string): string {
|
|
36
|
+
if (!value) return ".";
|
|
37
|
+
const normalized = value.replace(/\/+/g, PATH_SEPARATOR);
|
|
38
|
+
if (normalized === PATH_SEPARATOR) return normalized;
|
|
39
|
+
return normalized.replace(/\/+$/g, "") || PATH_SEPARATOR;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function joinPaths(...parts: string[]): string {
|
|
43
|
+
if (parts.length === 0) return ".";
|
|
44
|
+
let result = parts[0] ?? "";
|
|
45
|
+
for (let i = 1; i < parts.length; i++) {
|
|
46
|
+
const part = parts[i];
|
|
47
|
+
if (!part) continue;
|
|
48
|
+
if (!result || result.endsWith(PATH_SEPARATOR)) {
|
|
49
|
+
result += part.replace(/^\/+/, "");
|
|
50
|
+
} else {
|
|
51
|
+
result += `${PATH_SEPARATOR}${part.replace(/^\/+/, "")}`;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return normalizePath(result);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function dirnamePath(value: string): string {
|
|
58
|
+
const normalized = normalizePath(value);
|
|
59
|
+
if (normalized === PATH_SEPARATOR) return PATH_SEPARATOR;
|
|
60
|
+
const index = normalized.lastIndexOf(PATH_SEPARATOR);
|
|
61
|
+
if (index < 0) return ".";
|
|
62
|
+
if (index === 0) return PATH_SEPARATOR;
|
|
63
|
+
return normalized.slice(0, index);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function splitPathSegments(value: string): string[] {
|
|
67
|
+
const normalized = normalizePath(value);
|
|
68
|
+
if (normalized === PATH_SEPARATOR) return [];
|
|
69
|
+
return normalized.replace(/^\/+/, "").split(PATH_SEPARATOR).filter(Boolean);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function relativePath(from: string, to: string): string {
|
|
73
|
+
const normalizedFrom = normalizePath(from);
|
|
74
|
+
const normalizedTo = normalizePath(to);
|
|
75
|
+
if (normalizedFrom === normalizedTo) return "";
|
|
76
|
+
const fromSegments = splitPathSegments(normalizedFrom);
|
|
77
|
+
const toSegments = splitPathSegments(normalizedTo);
|
|
78
|
+
let shared = 0;
|
|
79
|
+
while (shared < fromSegments.length && shared < toSegments.length && fromSegments[shared] === toSegments[shared]) {
|
|
80
|
+
shared++;
|
|
81
|
+
}
|
|
82
|
+
const upSegments = new Array(fromSegments.length - shared).fill("..");
|
|
83
|
+
const downSegments = toSegments.slice(shared);
|
|
84
|
+
return [...upSegments, ...downSegments].join(PATH_SEPARATOR);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function getNodeFsPromises(): Promise<typeof import("node:fs/promises")> {
|
|
88
|
+
if (!fsPromisesPromise) {
|
|
89
|
+
fsPromisesPromise = import("node:fs/promises");
|
|
90
|
+
}
|
|
91
|
+
return fsPromisesPromise;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function pathExists(value: string): Promise<boolean> {
|
|
95
|
+
try {
|
|
96
|
+
const fs = await getNodeFsPromises();
|
|
97
|
+
await fs.access(value);
|
|
98
|
+
return true;
|
|
99
|
+
} catch {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function resolveWorkspaceRoot(cwd: string): Promise<string> {
|
|
105
|
+
const normalizedCwd = normalizePath(cwd);
|
|
106
|
+
const cached = workspaceRootCache.get(normalizedCwd);
|
|
107
|
+
if (cached) return cached;
|
|
108
|
+
|
|
109
|
+
const promise = (async () => {
|
|
110
|
+
let current = normalizedCwd;
|
|
111
|
+
while (true) {
|
|
112
|
+
if (await pathExists(joinPaths(current, ".git"))) {
|
|
113
|
+
return current;
|
|
114
|
+
}
|
|
115
|
+
const parent = dirnamePath(current);
|
|
116
|
+
if (parent === current || parent === ".") {
|
|
117
|
+
return normalizedCwd;
|
|
118
|
+
}
|
|
119
|
+
current = parent;
|
|
120
|
+
}
|
|
121
|
+
})();
|
|
122
|
+
|
|
123
|
+
workspaceRootCache.set(normalizedCwd, promise);
|
|
124
|
+
return promise;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function getOpenAICodexImageDirectory(cwd: string): string {
|
|
128
|
+
return joinPaths(cwd, OPENAI_CODEX_IMAGE_DIR);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function getOpenAICodexImagePath(cwd: string, responseId: string | undefined, callId: string, outputFormat?: string): string {
|
|
132
|
+
const ext = normalizeImageOutputFormat(outputFormat);
|
|
133
|
+
const safeCallId = shortenFilePart(callId, "image");
|
|
134
|
+
const safeResponseId = shortenFilePart(responseId, "response");
|
|
135
|
+
return joinPaths(getOpenAICodexImageDirectory(cwd), `${safeCallId}-${safeResponseId}.${ext}`);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function getOpenAICodexLatestImagePath(cwd: string): string {
|
|
139
|
+
return joinPaths(getOpenAICodexImageDirectory(cwd), OPENAI_CODEX_LATEST_IMAGE_NAME);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function buildGeneratedImageDisplayText(savedImage: SavedGeneratedImage, options?: { expanded?: boolean }): string {
|
|
143
|
+
const lines: string[] = [];
|
|
144
|
+
if (options?.expanded && savedImage.revisedPrompt) {
|
|
145
|
+
lines.push(`Prompt: ${savedImage.revisedPrompt}`);
|
|
146
|
+
}
|
|
147
|
+
lines.push(`File: ${savedImage.relativePath}`);
|
|
148
|
+
return lines.join("\n");
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function saveOpenAICodexGeneratedImage(
|
|
152
|
+
cwd: string,
|
|
153
|
+
image: { responseId?: string; callId: string; result: string; outputFormat?: string; imageModel?: string; revisedPrompt?: string },
|
|
154
|
+
): Promise<SavedGeneratedImage> {
|
|
155
|
+
const workspaceRoot = await resolveWorkspaceRoot(cwd);
|
|
156
|
+
const outputFormat = normalizeImageOutputFormat(image.outputFormat);
|
|
157
|
+
const saved = await saveBase64Image({
|
|
158
|
+
base64: image.result,
|
|
159
|
+
callId: image.callId,
|
|
160
|
+
cwd,
|
|
161
|
+
format: outputFormat,
|
|
162
|
+
responseId: image.responseId,
|
|
163
|
+
settings: loadSettings(cwd),
|
|
164
|
+
});
|
|
165
|
+
const absolutePath = saved.path;
|
|
166
|
+
const latestAbsolutePath = saved.latestPath ?? getOpenAICodexLatestImagePath(workspaceRoot);
|
|
167
|
+
|
|
168
|
+
const relativeFilePath = relativePath(workspaceRoot, absolutePath);
|
|
169
|
+
const latestRelativeFilePath = relativePath(workspaceRoot, latestAbsolutePath);
|
|
170
|
+
const relativePathValue = relativeFilePath && !relativeFilePath.startsWith("..") ? relativeFilePath : absolutePath;
|
|
171
|
+
const latestRelativePathValue =
|
|
172
|
+
latestRelativeFilePath && !latestRelativeFilePath.startsWith("..") ? latestRelativeFilePath : latestAbsolutePath;
|
|
173
|
+
|
|
174
|
+
return {
|
|
175
|
+
absolutePath,
|
|
176
|
+
relativePath: relativePathValue,
|
|
177
|
+
latestAbsolutePath,
|
|
178
|
+
latestRelativePath: latestRelativePathValue,
|
|
179
|
+
responseId: image.responseId,
|
|
180
|
+
callId: image.callId,
|
|
181
|
+
outputFormat,
|
|
182
|
+
imageModel: image.imageModel,
|
|
183
|
+
revisedPrompt: image.revisedPrompt,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export const IMAGE_SAVE_DISPLAY_MESSAGE_TYPE = "codex-image-generation-display";
|
|
2
|
+
|
|
3
|
+
export interface SavedGeneratedImage {
|
|
4
|
+
absolutePath: string;
|
|
5
|
+
relativePath: string;
|
|
6
|
+
latestAbsolutePath: string;
|
|
7
|
+
latestRelativePath: string;
|
|
8
|
+
responseId: string | undefined;
|
|
9
|
+
callId: string;
|
|
10
|
+
outputFormat: string;
|
|
11
|
+
imageModel?: string;
|
|
12
|
+
revisedPrompt?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface ImageDisplayMessageDetails {
|
|
16
|
+
savedImages: SavedGeneratedImage[];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface PendingImageDisplay {
|
|
20
|
+
savedImage: SavedGeneratedImage;
|
|
21
|
+
imageData: { data: string; mimeType: string };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface QueuedImageActivity extends PendingImageDisplay {
|
|
25
|
+
kind: "image";
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type PendingActivity = QueuedImageActivity;
|
|
29
|
+
|
|
30
|
+
export interface CachedImagePreview {
|
|
31
|
+
data: string;
|
|
32
|
+
mimeType: string;
|
|
33
|
+
bytes: number;
|
|
34
|
+
widthPx?: number;
|
|
35
|
+
heightPx?: number;
|
|
36
|
+
}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { extname, isAbsolute, resolve } from "node:path";
|
|
4
|
+
import { buildSessionContext, type SessionEntry } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import { saveBase64Image } from "../utils/images.js";
|
|
6
|
+
import type { CodexMinimalToolsSettings } from "@oai404iao/pi-codex-runtime/internal/settings";
|
|
7
|
+
import type { Api, Model, ProviderHeaders } from "@earendil-works/pi-ai";
|
|
8
|
+
import {
|
|
9
|
+
buildCodexJsonHeaders,
|
|
10
|
+
hasCodexRequestAuth,
|
|
11
|
+
resolveCodexApiEndpoint,
|
|
12
|
+
} from "@oai404iao/pi-codex-runtime/internal/codex-http";
|
|
13
|
+
import { loadModelSettings, type ResolvedCodexModelSettings } from "@oai404iao/pi-codex-runtime/internal/model-catalog/runtime";
|
|
14
|
+
|
|
15
|
+
export interface ImageGenerationInput {
|
|
16
|
+
prompt?: string;
|
|
17
|
+
referenced_image_paths?: string[];
|
|
18
|
+
num_last_images_to_include?: number;
|
|
19
|
+
size?: "1024x1024" | "1024x1536" | "1536x1024" | "auto";
|
|
20
|
+
quality?: "low" | "medium" | "high" | "auto";
|
|
21
|
+
background?: "transparent" | "opaque" | "auto";
|
|
22
|
+
output_format?: "png" | "webp" | "jpeg";
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const imageGenerationToolSchema = {
|
|
26
|
+
type: "object",
|
|
27
|
+
additionalProperties: false,
|
|
28
|
+
properties: {
|
|
29
|
+
prompt: { type: "string", description: "Image generation or editing prompt." },
|
|
30
|
+
referenced_image_paths: {
|
|
31
|
+
type: "array",
|
|
32
|
+
maxItems: 5,
|
|
33
|
+
items: { type: "string", minLength: 1 },
|
|
34
|
+
description: "Local PNG, JPEG, or WebP paths to edit.",
|
|
35
|
+
},
|
|
36
|
+
num_last_images_to_include: {
|
|
37
|
+
type: "integer",
|
|
38
|
+
minimum: 1,
|
|
39
|
+
maximum: 5,
|
|
40
|
+
description: "Use the newest conversation images when one or more targets have no local path.",
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
required: ["prompt"],
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
async function urlToBase64(url: string, signal?: AbortSignal): Promise<string> {
|
|
47
|
+
const response = await fetch(url, { signal });
|
|
48
|
+
if (!response.ok) throw new Error(`Failed to download generated image: ${response.status} ${await response.text()}`);
|
|
49
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
50
|
+
return buffer.toString("base64");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function directImageGeneration(input: ImageGenerationInput, cwd: string, settings: CodexMinimalToolsSettings, signal?: AbortSignal) {
|
|
54
|
+
if (!settings.directImageApiFallback) throw new Error("Direct Images API fallback is disabled. Use native openai or openai-codex handling, or enable directImageApiFallback.");
|
|
55
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
56
|
+
if (!apiKey) throw new Error("OPENAI_API_KEY is required for direct image_generation fallback.");
|
|
57
|
+
if (!input.prompt?.trim()) throw new Error("A prompt is required for direct image_generation fallback.");
|
|
58
|
+
const body: Record<string, unknown> = {
|
|
59
|
+
model: settings.imageModel,
|
|
60
|
+
prompt: input.prompt,
|
|
61
|
+
};
|
|
62
|
+
if (input.size && input.size !== "auto") body.size = input.size;
|
|
63
|
+
if (input.quality && input.quality !== "auto") body.quality = input.quality;
|
|
64
|
+
if (input.background && input.background !== "auto") body.background = input.background;
|
|
65
|
+
if (input.output_format) body.output_format = input.output_format;
|
|
66
|
+
const response = await fetch("https://api.openai.com/v1/images/generations", {
|
|
67
|
+
method: "POST",
|
|
68
|
+
headers: { Authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
|
|
69
|
+
body: JSON.stringify(body),
|
|
70
|
+
signal,
|
|
71
|
+
});
|
|
72
|
+
if (!response.ok) throw new Error(`OpenAI Images API failed: ${response.status} ${await response.text()}`);
|
|
73
|
+
const json = await response.json() as { data?: Array<{ b64_json?: string; url?: string; revised_prompt?: string }> };
|
|
74
|
+
const first = json.data?.[0];
|
|
75
|
+
const base64 = first?.b64_json ?? (first?.url ? await urlToBase64(first.url, signal) : undefined);
|
|
76
|
+
if (!base64) throw new Error("OpenAI Images API returned no image data.");
|
|
77
|
+
const saved = await saveBase64Image({ base64, callId: "direct", cwd, format: input.output_format, responseId: settings.imageModel, settings });
|
|
78
|
+
return {
|
|
79
|
+
content: [{ type: "text", text: `Generated image with ${settings.imageModel}; saved to ${saved.path}${saved.latestPath ? ` (latest: ${saved.latestPath})` : ""}.` }],
|
|
80
|
+
details: { saved, revisedPrompt: first?.revised_prompt, mode: "direct-images-api" },
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
interface ImageGenerationToolContext {
|
|
85
|
+
cwd: string;
|
|
86
|
+
model?: Model<Api>;
|
|
87
|
+
modelRegistry?: {
|
|
88
|
+
getApiKeyAndHeaders(model: Model<Api>): Promise<
|
|
89
|
+
| { ok: true; apiKey?: string; headers?: ProviderHeaders }
|
|
90
|
+
| { ok: false; error: string }
|
|
91
|
+
>;
|
|
92
|
+
};
|
|
93
|
+
sessionManager?: {
|
|
94
|
+
getSessionId?(): string;
|
|
95
|
+
getBranch(): SessionEntry[];
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface StandaloneImageGenerationInvocation {
|
|
100
|
+
callId?: string;
|
|
101
|
+
turnId?: string;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function referencedImageUrls(cwd: string, paths: readonly string[]): Promise<Array<{ image_url: string }>> {
|
|
105
|
+
if (paths.length > 5) throw new Error("referenced_image_paths must contain at most 5 paths.");
|
|
106
|
+
return Promise.all(paths.map(async (rawPath) => {
|
|
107
|
+
const normalized = rawPath.replace(/^@/, "");
|
|
108
|
+
const path = isAbsolute(normalized) ? normalized : resolve(cwd, normalized);
|
|
109
|
+
const extension = extname(path).toLowerCase();
|
|
110
|
+
const mimeType = extension === ".png"
|
|
111
|
+
? "image/png"
|
|
112
|
+
: extension === ".jpg" || extension === ".jpeg"
|
|
113
|
+
? "image/jpeg"
|
|
114
|
+
: extension === ".webp"
|
|
115
|
+
? "image/webp"
|
|
116
|
+
: undefined;
|
|
117
|
+
if (!mimeType) throw new Error(`Unsupported reference image type: ${rawPath}. Use PNG, JPEG, or WebP.`);
|
|
118
|
+
const data = await readFile(path);
|
|
119
|
+
return { image_url: `data:${mimeType};base64,${data.toString("base64")}` };
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function recentConversationImageUrls(
|
|
124
|
+
ctx: ImageGenerationToolContext,
|
|
125
|
+
count: number,
|
|
126
|
+
): Array<{ image_url: string }> {
|
|
127
|
+
if (!ctx.sessionManager) {
|
|
128
|
+
throw new Error("Conversation images are unavailable in this tool context; use referenced_image_paths.");
|
|
129
|
+
}
|
|
130
|
+
const messages = buildSessionContext(ctx.sessionManager.getBranch()).messages;
|
|
131
|
+
const images: string[] = [];
|
|
132
|
+
for (const message of messages) {
|
|
133
|
+
if (message.role === "user" || message.role === "toolResult") {
|
|
134
|
+
if (!Array.isArray(message.content)) continue;
|
|
135
|
+
for (const item of message.content) {
|
|
136
|
+
if (item.type === "image") {
|
|
137
|
+
images.push(`data:${item.mimeType};base64,${item.data}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (message.role !== "assistant") continue;
|
|
143
|
+
for (const block of message.content as unknown[]) {
|
|
144
|
+
if (!block || typeof block !== "object") continue;
|
|
145
|
+
const candidate = block as {
|
|
146
|
+
type?: unknown;
|
|
147
|
+
item?: { result?: unknown };
|
|
148
|
+
};
|
|
149
|
+
if (
|
|
150
|
+
candidate.type === "image_generation_call"
|
|
151
|
+
&& typeof candidate.item?.result === "string"
|
|
152
|
+
&& candidate.item.result
|
|
153
|
+
) {
|
|
154
|
+
images.push(`data:image/png;base64,${candidate.item.result}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (images.length < count) {
|
|
159
|
+
throw new Error(`Requested the last ${count} conversation images, but only ${images.length} were available.`);
|
|
160
|
+
}
|
|
161
|
+
return images.slice(-count).map((image_url) => ({ image_url }));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export async function standaloneImageGeneration(
|
|
165
|
+
input: ImageGenerationInput,
|
|
166
|
+
ctx: ImageGenerationToolContext,
|
|
167
|
+
settings: ResolvedCodexModelSettings,
|
|
168
|
+
signal?: AbortSignal,
|
|
169
|
+
invocation: StandaloneImageGenerationInvocation = {},
|
|
170
|
+
) {
|
|
171
|
+
signal?.throwIfAborted();
|
|
172
|
+
const callId = invocation.callId ?? "standalone";
|
|
173
|
+
const turnId = invocation.turnId ?? randomUUID();
|
|
174
|
+
const model = ctx.model;
|
|
175
|
+
if (!model || !ctx.modelRegistry) throw new Error("No active model is available for standalone image generation.");
|
|
176
|
+
if (!settings.enabled) throw new Error("pi-codex-minimal-tools is disabled.");
|
|
177
|
+
if (!input.prompt?.trim()) throw new Error("A prompt is required for standalone image generation.");
|
|
178
|
+
if (input.num_last_images_to_include !== undefined && input.referenced_image_paths?.length) {
|
|
179
|
+
throw new Error("Provide only one of referenced_image_paths or num_last_images_to_include.");
|
|
180
|
+
}
|
|
181
|
+
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
|
|
182
|
+
signal?.throwIfAborted();
|
|
183
|
+
if (!auth.ok) throw new Error(auth.error);
|
|
184
|
+
if (!hasCodexRequestAuth({
|
|
185
|
+
modelHeaders: model.headers,
|
|
186
|
+
auth: { apiKey: auth.apiKey, headers: auth.headers },
|
|
187
|
+
})) {
|
|
188
|
+
throw new Error(`No request authentication for provider: ${model.provider}`);
|
|
189
|
+
}
|
|
190
|
+
const images = input.num_last_images_to_include !== undefined
|
|
191
|
+
? recentConversationImageUrls(ctx, input.num_last_images_to_include)
|
|
192
|
+
: await referencedImageUrls(ctx.cwd, input.referenced_image_paths ?? []);
|
|
193
|
+
const edit = images.length > 0;
|
|
194
|
+
signal?.throwIfAborted();
|
|
195
|
+
const response = await fetch(
|
|
196
|
+
resolveCodexApiEndpoint(
|
|
197
|
+
model.baseUrl,
|
|
198
|
+
settings.apiKeyMode,
|
|
199
|
+
edit ? "images/edits" : "images/generations",
|
|
200
|
+
),
|
|
201
|
+
{
|
|
202
|
+
method: "POST",
|
|
203
|
+
headers: buildCodexJsonHeaders({
|
|
204
|
+
modelHeaders: model.headers,
|
|
205
|
+
auth: { apiKey: auth.apiKey, headers: auth.headers },
|
|
206
|
+
apiKeyMode: settings.apiKeyMode,
|
|
207
|
+
extraHeaders: {
|
|
208
|
+
"x-codex-image-turn-id": turnId,
|
|
209
|
+
},
|
|
210
|
+
}),
|
|
211
|
+
body: JSON.stringify({
|
|
212
|
+
model: settings.imageModel,
|
|
213
|
+
prompt: input.prompt.trim(),
|
|
214
|
+
...(edit ? { images } : {}),
|
|
215
|
+
background: "auto",
|
|
216
|
+
quality: "auto",
|
|
217
|
+
size: "auto",
|
|
218
|
+
}),
|
|
219
|
+
signal,
|
|
220
|
+
},
|
|
221
|
+
);
|
|
222
|
+
signal?.throwIfAborted();
|
|
223
|
+
if (!response.ok) {
|
|
224
|
+
throw new Error(`Standalone image generation failed: HTTP ${response.status}: ${await response.text()}`);
|
|
225
|
+
}
|
|
226
|
+
const result = await response.json() as {
|
|
227
|
+
data?: Array<{ b64_json?: string }>;
|
|
228
|
+
};
|
|
229
|
+
signal?.throwIfAborted();
|
|
230
|
+
const base64 = result.data?.[0]?.b64_json;
|
|
231
|
+
if (!base64) throw new Error("Standalone image generation returned no image data.");
|
|
232
|
+
const saved = await saveBase64Image({
|
|
233
|
+
base64,
|
|
234
|
+
callId,
|
|
235
|
+
cwd: ctx.cwd,
|
|
236
|
+
format: "png",
|
|
237
|
+
responseId: settings.imageModel,
|
|
238
|
+
settings,
|
|
239
|
+
});
|
|
240
|
+
signal?.throwIfAborted();
|
|
241
|
+
return {
|
|
242
|
+
content: [
|
|
243
|
+
{ type: "image", data: base64, mimeType: "image/png" },
|
|
244
|
+
{ type: "text", text: `Generated image with ${settings.imageModel}; saved to ${saved.path}${saved.latestPath ? ` (latest: ${saved.latestPath})` : ""}.` },
|
|
245
|
+
],
|
|
246
|
+
details: { saved, mode: "standalone-images-api" },
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export function createImageGenerationToolDefinition(options: {
|
|
251
|
+
loadSettings?: (cwd: string, model?: Model<Api>) => CodexMinimalToolsSettings | ResolvedCodexModelSettings;
|
|
252
|
+
getCurrentTurnId?: (sessionId: string | undefined) => string | undefined;
|
|
253
|
+
hasProviderRuntime?: () => boolean;
|
|
254
|
+
} = {}) {
|
|
255
|
+
return {
|
|
256
|
+
name: "image_generation",
|
|
257
|
+
label: "Image Generation",
|
|
258
|
+
description: "Generate or edit images using the hosted or standalone implementation selected by the current model profile. Results are saved under imageOutputDir and mirrored to latest.<ext>.",
|
|
259
|
+
promptSnippet: "Generate or edit images with the implementation selected by the current model profile.",
|
|
260
|
+
parameters: imageGenerationToolSchema,
|
|
261
|
+
async execute(toolCallId: string, params: ImageGenerationInput, signal: AbortSignal | undefined, _onUpdate: unknown, ctx: ImageGenerationToolContext) {
|
|
262
|
+
const cwd = ctx?.cwd ?? process.cwd();
|
|
263
|
+
const settings = options.loadSettings?.(cwd, ctx.model) ?? loadModelSettings(ctx.model, cwd);
|
|
264
|
+
const resolvedSettings = "modelProfile" in settings
|
|
265
|
+
? settings as ResolvedCodexModelSettings
|
|
266
|
+
: loadModelSettings(ctx.model, cwd, settings);
|
|
267
|
+
if (resolvedSettings.imageGenerationImplementation === "standalone") {
|
|
268
|
+
const sessionId = ctx.sessionManager?.getSessionId?.();
|
|
269
|
+
return standaloneImageGeneration(params, ctx, resolvedSettings, signal, {
|
|
270
|
+
callId: toolCallId,
|
|
271
|
+
turnId: options.getCurrentTurnId?.(sessionId),
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
if (settings?.directImageApiFallback) return directImageGeneration(params, cwd, settings, signal);
|
|
275
|
+
if (options.hasProviderRuntime?.() === false) {
|
|
276
|
+
throw new Error("Hosted image_generation requires pi-codex-core. Use a catalog-supported standalone profile or explicitly enable directImageApiFallback.");
|
|
277
|
+
}
|
|
278
|
+
return {
|
|
279
|
+
content: [{ type: "text", text: "image_generation should be handled by the current model profile. If no hosted or standalone implementation is configured, enable directImageApiFallback with OPENAI_API_KEY." }],
|
|
280
|
+
details: { phase: "native-provider", nativeTool: "image_generation" },
|
|
281
|
+
};
|
|
282
|
+
},
|
|
283
|
+
};
|
|
284
|
+
}
|