@shanepadgett/tau-agent 0.19.0 → 0.20.0

package/extensions/image-gen/README.md

@@ -1,11 +1,13 @@
 # Image Generation
 
-`image_gen` generates raster images and edits up to three local raster images with Grok Imagine. It uses `grok-imagine-image-quality` and saves results under `~/.local/share/tau-agent/images/` by default. Pass an explicit path with the expected image extension when the image should be saved in the current repository or another chosen location.
+`image_gen` generates raster images and edits up to three local raster images with OpenAI GPT Image or xAI Grok Imagine. It uses `gpt-image-2` for OpenAI and `grok-imagine-image-quality` for xAI.
 
-Run `/login xai` and choose either a subscription or API-key login before invoking the tool.
+By default, the tool follows the parent model: GPT models prefer OpenAI and Grok models prefer xAI. If that provider has no configured authentication, it tries the other provider. The model can pass `provider: "openai"` or `provider: "xai"` to override automatic selection.
+
+Run `/login openai-codex` for GPT Image or `/login xai` for Grok Imagine before invoking the tool. OpenAI image generation uses the existing Codex subscription login; xAI supports its configured login methods.
 
 Run `/reload` after installing or changing the extension.
 
-The model invokes `image_gen` with a prompt. For edits, it also supplies one to three local PNG, JPEG, or WebP paths. Successful images up to 12 MiB are returned inline for inspection; larger results remain available at the saved path.
+Results are saved under `~/.local/share/tau-agent/images/` by default. Pass an explicit path with the expected image extension when the image should be saved in the current repository or another chosen location. For edits, supply one to three local PNG, JPEG, or WebP paths. Successful images up to 12 MiB are returned inline for inspection; larger results remain available at the saved path.
 
-xAI controls model availability and subscription entitlements. A successful login does not guarantee that every account can use Grok Imagine.
+The OpenAI path calls a private Codex backend, which may change without notice. xAI controls Grok Imagine availability and subscription entitlements. A successful login does not guarantee image access.

package/extensions/image-gen/client.ts

@@ -1,4 +1,10 @@
-import { XAI_API_BASE_URL, XAI_IMAGE_MODEL } from "./constants.ts";
+import {
+	OPENAI_IMAGE_API_BASE_URL,
+	OPENAI_IMAGE_MODEL,
+	XAI_API_BASE_URL,
+	XAI_IMAGE_MODEL,
+	type ImageProvider,
+} from "./constants.ts";
 
 const MAX_ERROR_BODY_BYTES = 8192;
 const MAX_ERROR_MESSAGE_LENGTH = 2000;
@@ -6,6 +12,11 @@ const REQUEST_TIMEOUT_MS = 60_000;
 const MAX_ATTEMPTS = 3;
 const RETRY_BASE_DELAY_MS = 500;
 
+export interface CodexAuth {
+	token: string;
+	accountId: string;
+}
+
 export interface EditImage {
 	mimeType: "image/png" | "image/jpeg" | "image/webp";
 	data: string;
@@ -28,6 +39,29 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+export function resolveCodexAuth(token: string): CodexAuth {
+	const invalidCredential = () =>
+		new Error("The OpenAI Codex credential does not contain a usable ChatGPT account ID. Run /login again.");
+	const segments = token.split(".");
+	if (segments.length !== 3 || !segments[1]) throw invalidCredential();
+	if (!/^[A-Za-z0-9_-]+$/.test(segments[1]) || segments[1].length % 4 === 1) throw invalidCredential();
+
+	let payload: unknown;
+	try {
+		const decoded = Buffer.from(segments[1], "base64url");
+		if (decoded.toString("base64url") !== segments[1]) throw invalidCredential();
+		payload = JSON.parse(decoded.toString("utf8"));
+	} catch {
+		throw invalidCredential();
+	}
+	if (!isRecord(payload)) throw invalidCredential();
+	const authClaim = payload["https://api.openai.com/auth"];
+	if (!isRecord(authClaim)) throw invalidCredential();
+	const accountId = authClaim.chatgpt_account_id;
+	if (typeof accountId !== "string" || !accountId.trim()) throw invalidCredential();
+	return { token, accountId: accountId.trim() };
+}
+
 async function boundedError(response: HttpResponse): Promise<string> {
 	if (!response.body) return "";
 	const reader = response.body.getReader();
@@ -85,25 +119,61 @@ export function detectImageMimeType(bytes: Buffer): GeneratedImage["mimeType"] |
 	return undefined;
 }
 
-function decodeImageResponse(value: unknown): GeneratedImage {
+function decodeImageResponse(value: unknown, service: "OpenAI Codex" | "xAI", pngOnly: boolean): GeneratedImage {
 	if (!isRecord(value) || !Array.isArray(value.data) || !isRecord(value.data[0])) {
-		throw new Error("xAI returned an invalid image response");
+		throw new Error(`${service} returned an invalid image response`);
 	}
 	const encoded = value.data[0].b64_json;
-	if (typeof encoded !== "string" || !encoded.trim()) throw new Error("xAI returned an invalid image response");
+	if (typeof encoded !== "string" || !encoded.trim()) throw new Error(`${service} returned an invalid image response`);
 	const base64 = encoded.trim();
 	if (!/^[A-Za-z0-9+/]*={0,2}$/.test(base64) || base64.length % 4 === 1) {
-		throw new Error("xAI returned invalid base64 image data");
+		throw new Error(`${service} returned invalid base64 image data`);
 	}
 	const bytes = Buffer.from(base64, "base64");
 	if (bytes.toString("base64").replace(/=+$/, "") !== base64.replace(/=+$/, "")) {
-		throw new Error("xAI returned invalid base64 image data");
+		throw new Error(`${service} returned invalid base64 image data`);
 	}
 	const mimeType = detectImageMimeType(bytes);
-	if (!mimeType) throw new Error("xAI returned unsupported image data");
+	if (!mimeType || (pngOnly && mimeType !== "image/png")) {
+		throw new Error(`${service} returned unsupported image data`);
+	}
 	return { bytes, base64: bytes.toString("base64"), mimeType };
 }
 
+async function requestOpenAIImage(
+	operation: "generation" | "edit",
+	body: Record<string, unknown>,
+	auth: CodexAuth,
+	signal?: AbortSignal,
+): Promise<GeneratedImage> {
+	const route = operation === "generation" ? "generations" : "edits";
+	const response = (await fetch(`${OPENAI_IMAGE_API_BASE_URL}/${route}`, {
+		method: "POST",
+		headers: {
+			Accept: "application/json",
+			Authorization: `Bearer ${auth.token}`,
+			"chatgpt-account-id": auth.accountId,
+			"Content-Type": "application/json",
+			originator: "pi",
+		},
+		body: JSON.stringify(body),
+		signal,
+	})) as HttpResponse;
+	if (!response.ok) {
+		const message = serverErrorMessage(await boundedError(response), auth.token);
+		throw new Error(
+			`OpenAI Codex image ${operation} failed with status ${response.status}${message ? `: ${message}` : ""}`,
+		);
+	}
+	let value: unknown;
+	try {
+		value = await response.json();
+	} catch {
+		throw new Error("OpenAI Codex returned a non-JSON image response");
+	}
+	return decodeImageResponse(value, "OpenAI Codex", true);
+}
+
 function retryable(status: number): boolean {
 	return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
 }
@@ -124,7 +194,7 @@ async function delay(milliseconds: number, signal?: AbortSignal): Promise<void>
 	});
 }
 
-async function requestImage(
+async function requestXaiImage(
 	operation: "generation" | "edit",
 	body: Record<string, unknown>,
 	token: string,
@@ -160,7 +230,7 @@ async function requestImage(
 			} catch {
 				throw new Error("xAI returned a non-JSON image response");
 			}
-			return decodeImageResponse(value);
+			return decodeImageResponse(value, "xAI", false);
 		}
 		const message = serverErrorMessage(await boundedError(response), token);
 		if (!retryable(response.status) || attempt === MAX_ATTEMPTS) {
@@ -173,8 +243,21 @@ async function requestImage(
 	throw new Error(`xAI image ${operation} failed`);
 }
 
-export function generateImage(prompt: string, token: string, signal?: AbortSignal): Promise<GeneratedImage> {
-	return requestImage(
+export function generateImage(
+	provider: ImageProvider,
+	prompt: string,
+	token: string,
+	signal?: AbortSignal,
+): Promise<GeneratedImage> {
+	if (provider === "openai") {
+		return requestOpenAIImage(
+			"generation",
+			{ prompt, model: OPENAI_IMAGE_MODEL, background: "auto", quality: "auto", size: "auto" },
+			resolveCodexAuth(token),
+			signal,
+		);
+	}
+	return requestXaiImage(
 		"generation",
 		{ model: XAI_IMAGE_MODEL, prompt, n: 1, resolution: "1k", response_format: "b64_json" },
 		token,
@@ -183,13 +266,29 @@ export function generateImage(prompt: string, token: string, signal?: AbortSigna
 }
 
 export function editImage(
+	provider: ImageProvider,
 	prompt: string,
 	images: readonly EditImage[],
 	token: string,
 	signal?: AbortSignal,
 ): Promise<GeneratedImage> {
+	if (provider === "openai") {
+		return requestOpenAIImage(
+			"edit",
+			{
+				images: images.map((image) => ({ image_url: `data:${image.mimeType};base64,${image.data}` })),
+				prompt,
+				model: OPENAI_IMAGE_MODEL,
+				background: "auto",
+				quality: "auto",
+				size: "auto",
+			},
+			resolveCodexAuth(token),
+			signal,
+		);
+	}
 	const references = images.map((image) => ({ url: `data:${image.mimeType};base64,${image.data}` }));
-	return requestImage(
+	return requestXaiImage(
 		"edit",
 		{
 			model: XAI_IMAGE_MODEL,

package/extensions/image-gen/constants.ts

@@ -1,3 +1,8 @@
+export const OPENAI_PROVIDER = "openai-codex";
+export const OPENAI_IMAGE_MODEL = "gpt-image-2";
+export const OPENAI_IMAGE_API_BASE_URL = "https://chatgpt.com/backend-api/codex/images";
 export const XAI_PROVIDER = "xai";
 export const XAI_IMAGE_MODEL = "grok-imagine-image-quality";
 export const XAI_API_BASE_URL = "https://api.x.ai/v1";
+
+export type ImageProvider = "openai" | "xai";

package/extensions/image-gen/index.ts

@@ -5,7 +5,13 @@ import { homedir } from "node:os";
 import { basename, dirname, extname, isAbsolute, join, resolve } from "node:path";
 import { type Static, Type } from "typebox";
 import { detectImageMimeType, editImage, generateImage, type EditImage, type GeneratedImage } from "./client.ts";
-import { XAI_IMAGE_MODEL, XAI_PROVIDER } from "./constants.ts";
+import {
+	OPENAI_IMAGE_MODEL,
+	OPENAI_PROVIDER,
+	XAI_IMAGE_MODEL,
+	XAI_PROVIDER,
+	type ImageProvider,
+} from "./constants.ts";
 
 const MAX_INPUT_BYTES = 50 * 1024 * 1024;
 const MAX_INLINE_BYTES = 12 * 1024 * 1024;
@@ -13,6 +19,12 @@ const MAX_INLINE_BYTES = 12 * 1024 * 1024;
 const imageGenSchema = Type.Object(
 	{
 		prompt: Type.String({ minLength: 1 }),
+		provider: Type.Optional(
+			Type.Union([Type.Literal("openai"), Type.Literal("xai")], {
+				description:
+					"Image provider override. Omit to follow the parent model, preferring OpenAI for GPT and xAI for Grok.",
+			}),
+		),
 		path: Type.Optional(
 			Type.String({ description: "Explicit image destination path; defaults to Tau's external image store" }),
 		),
@@ -25,7 +37,8 @@ type ImageGenParams = Static<typeof imageGenSchema>;
 
 interface ImageGenDetails {
 	path: string;
-	model: typeof XAI_IMAGE_MODEL;
+	provider: ImageProvider;
+	model: typeof OPENAI_IMAGE_MODEL | typeof XAI_IMAGE_MODEL;
 	operation: "generate" | "edit";
 }
 
@@ -41,7 +54,7 @@ export default function imageGenExtension(pi: ExtensionAPI): void {
 			name: "image_gen",
 			label: "Image Generation",
 			description:
-				"Generate a requested raster image or AI-edit existing images with configured xAI authentication. Omit referenced_image_paths to generate; pass one to three local paths to edit or compose. Omit path to use Tau's external image store; pass path only when the user explicitly requests a repository file or other destination. Returns the image for inspection.",
+				"Generate a requested raster image or AI-edit existing images with OpenAI GPT Image or xAI Grok Imagine. Omit provider to follow the parent model; set it to openai or xai to override. Omit referenced_image_paths to generate; pass one to three local paths to edit or compose. Omit path to use Tau's external image store; pass path only when the user explicitly requests a repository file or other destination. Returns the image for inspection.",
 			parameters: imageGenSchema,
 			async execute(_toolCallId, params: ImageGenParams, signal, onUpdate, ctx) {
 				signal?.throwIfAborted();
@@ -61,10 +74,38 @@ export default function imageGenExtension(pi: ExtensionAPI): void {
 					throw new Error("Image path must end in .jpg, .jpeg, .png, or .webp");
 				}
 
-				const token = await ctx.modelRegistry.getApiKeyForProvider(XAI_PROVIDER);
-				if (!token) {
-					throw new Error("xAI authentication is unavailable. Run /login xai and choose a login method.");
+				const parentUsesXai =
+					ctx.model?.provider.toLowerCase() === XAI_PROVIDER || ctx.model?.id.toLowerCase().includes("grok") === true;
+				const preferredProvider: ImageProvider = params.provider ?? (parentUsesXai ? "xai" : "openai");
+				const providers: readonly ImageProvider[] = params.provider
+					? [params.provider]
+					: preferredProvider === "xai"
+						? ["xai", "openai"]
+						: ["openai", "xai"];
+				let provider: ImageProvider | undefined;
+				let token: string | undefined;
+				for (const candidate of providers) {
+					const candidateToken = await ctx.modelRegistry.getApiKeyForProvider(
+						candidate === "openai" ? OPENAI_PROVIDER : XAI_PROVIDER,
+					);
+					if (candidateToken) {
+						provider = candidate;
+						token = candidateToken;
+						break;
+					}
+				}
+				if (!provider || !token) {
+					if (params.provider === "openai") {
+						throw new Error("OpenAI Codex authentication is unavailable. Run /login openai-codex.");
+					}
+					if (params.provider === "xai") {
+						throw new Error("xAI authentication is unavailable. Run /login xai and choose a login method.");
+					}
+					throw new Error(
+						"Image generation authentication is unavailable. Run /login for OpenAI Codex or xAI.",
+					);
 				}
+				const model = provider === "openai" ? OPENAI_IMAGE_MODEL : XAI_IMAGE_MODEL;
 				const images: EditImage[] = [];
 				for (const path of params.referenced_image_paths ?? []) {
 					const rawPath = path.startsWith("@") ? path.slice(1) : path;
@@ -88,16 +129,16 @@ export default function imageGenExtension(pi: ExtensionAPI): void {
 							type: "text",
 							text:
 								operation === "generate"
-									? `Generating image with ${XAI_IMAGE_MODEL}...`
-									: `Editing image with ${XAI_IMAGE_MODEL}...`,
+									? `Generating image with ${model}...`
+									: `Editing image with ${model}...`,
 						},
 					],
 					details: undefined,
 				});
 				const generated =
 					operation === "generate"
-						? await generateImage(prompt, token, signal)
-						: await editImage(prompt, images, token, signal);
+						? await generateImage(provider, prompt, token, signal)
+						: await editImage(provider, prompt, images, token, signal);
 				signal?.throwIfAborted();
 				const generatedExtension = outputExtension(generated);
 				if (requestedAbsolutePath) {
@@ -106,7 +147,9 @@ export default function imageGenExtension(pi: ExtensionAPI): void {
 						requestedExtension === generatedExtension ||
 						(generatedExtension === ".jpg" && requestedExtension === ".jpeg");
 					if (!matches)
-						throw new Error(`xAI returned ${generated.mimeType}; destination must end in ${generatedExtension}`);
+						throw new Error(
+							`${provider === "openai" ? "OpenAI Codex" : "xAI"} returned ${generated.mimeType}; destination must end in ${generatedExtension}`,
+						);
 				}
 				const absolutePath =
 					requestedAbsolutePath ??
@@ -129,13 +172,13 @@ export default function imageGenExtension(pi: ExtensionAPI): void {
 				});
 
 				const verb = operation === "generate" ? "Generated" : "Edited";
-				const details: ImageGenDetails = { path: absolutePath, model: XAI_IMAGE_MODEL, operation };
+				const details: ImageGenDetails = { path: absolutePath, provider, model, operation };
 				if (generated.bytes.length > MAX_INLINE_BYTES) {
 					return {
 						content: [
 							{
 								type: "text",
-								text: `${verb} image saved to ${absolutePath}. The PNG exceeds the 12 MiB attachment limit, so it was not added to model context.`,
+								text: `${verb} image saved to ${absolutePath}. The image exceeds the 12 MiB attachment limit, so it was not added to model context.`,
 							},
 						],
 						details,

package/extensions/tau-help/help.md

@@ -52,7 +52,7 @@ Adds `/ideas` to log rough ideas or open the ideas browser.
 
 ## image-gen
 
-Gives the agent a Grok Imagine generation and editing tool. Run `/login xai` and choose a subscription or API-key login before use. Generated images are saved for inspection.
+Gives the agent an OpenAI GPT Image and xAI Grok Imagine generation and editing tool. It follows the parent model by default and can override the provider per request. Run `/login openai-codex` or `/login xai` before use. Generated images are saved for inspection.
 
 ## manage-sessions
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@shanepadgett/tau-agent",
-	"version": "0.19.0",
+	"version": "0.20.0",
 	"description": "Tau is a custom agentic harness built with pi extensions",
 	"type": "module",
 	"license": "MIT",
@@ -28,7 +28,7 @@
 		"README.md"
 	],
 	"dependencies": {
-		"@shanepadgett/tau-tui": "0.19.0",
+		"@shanepadgett/tau-tui": "0.20.0",
 		"@toon-format/toon": "2.3.0",
 		"smol-toml": "1.7.0"
 	},