@pi-unipi/unipi 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/package.json +21 -21
- package/packages/ask-user/package.json +2 -2
- package/packages/autocomplete/package.json +1 -1
- package/packages/btw/package.json +2 -2
- package/packages/cocoindex/package.json +2 -2
- package/packages/compactor/package.json +3 -3
- package/packages/core/package.json +1 -1
- package/packages/footer/package.json +2 -2
- package/packages/image/package.json +2 -2
- package/packages/image/src/generate.ts +34 -15
- package/packages/image/src/index.ts +8 -0
- package/packages/image/src/models.ts +28 -0
- package/packages/image/src/openai-images-api.ts +282 -0
- package/packages/image/src/register-providers.ts +220 -0
- package/packages/image/src/tools.ts +37 -6
- package/packages/image/src/tui/settings-dialog.ts +6 -1
- package/packages/info-screen/package.json +2 -2
- package/packages/input-shortcuts/package.json +2 -2
- package/packages/kanboard/package.json +2 -2
- package/packages/mcp/package.json +2 -2
- package/packages/memory/package.json +3 -3
- package/packages/milestone/package.json +2 -2
- package/packages/notify/package.json +2 -2
- package/packages/ralph/package.json +3 -3
- package/packages/subagents/package.json +4 -4
- package/packages/unipi/bundled.js +291 -17
- package/packages/updater/package.json +2 -2
- package/packages/utility/package.json +2 -2
- package/packages/web-api/package.json +2 -2
- package/packages/workflow/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [2.4.0] — 2026-08-09
|
|
10
|
+
|
|
11
|
+
Image generation no longer requires an OpenRouter account, and `image_generate` can now edit an existing image.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- **`image`: image editing.** `image_generate` takes an optional `image` argument — a file path, `data:` URL, or raw base64 — and edits that image instead of generating from scratch. The result is saved and returned like any other generation.
|
|
16
|
+
|
|
17
|
+
Editing regenerates the whole image rather than masking a region, so details you did not mention can still shift. Which model you pick matters a lot here: in testing, `flux.2-pro` preserved unmentioned elements faithfully, while the `gemini-*-image` family tended to reinterpret the whole scene. Note also that image models respond to what you *describe*, not to what you negate — "no text" is as likely to add text as remove it.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- **`image`: generation works with any OpenAI-compatible provider configured in pi.** Previously pi-ai shipped exactly one image provider (`openrouter`), so generating an image meant holding an OpenRouter key even when you had several other providers signed in. Every provider in pi's model registry is now bridged into pi-ai's images collection at session start, backed by a single generic adapter that posts to `{baseUrl}/images/generations`.
|
|
22
|
+
|
|
23
|
+
Credentials come from pi's existing auth — there is no separate image login. Gateways disagree on the response shape, so three known forms are normalized (`b64_json` + `media_type`, `b64_json` + `revised_prompt`, and a `data:` URL under `url`); a remote `url` is surfaced as text rather than silently dropped.
|
|
24
|
+
|
|
25
|
+
Model *discovery* remains heuristic. pi's `ProviderModelConfig` has no `output` field and the provider composer rebuilds each model from a fixed field list, so an extension cannot record "this model emits images" — hence the name-based matching, with an explicit `provider/model-id` always accepted as the escape hatch.
|
|
26
|
+
|
|
27
|
+
- **`image`: errors name the provider you are actually using.** A missing key now points at that provider's environment variable and `/login` instead of unconditionally linking OpenRouter, and an unusable provider reports "no image-generation route" rather than the misleading "cannot generate images".
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- **`image`: a valid API key could be reported as missing.** pi-ai resolves credentials to an `AuthResult` — `{ auth: { apiKey } }` — but the key was read from the top level, so generation failed with "No API key for provider" while a perfectly good credential sat one level down. Both shapes are now accepted.
|
|
32
|
+
|
|
33
|
+
- **`image`: models discovered from the registry lost their endpoint.** Provider registration rebuilt each model from a field list that omitted `baseUrl`, so setup appeared to succeed and only the first real request failed with "No baseUrl for image model". The endpoint is now carried through discovery, and filled in from the registry at resolve time for hand-typed `provider/model-id` references, which never carry one.
|
|
34
|
+
|
|
9
35
|
## [2.3.0] — 2026-08-07
|
|
10
36
|
|
|
11
37
|
Startup went from **23.1s to 0.75s** — 31× faster, and within ~0.7s of bare `pi` with no extensions at all. A cold start (empty cache) is now the same speed as a warm one.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/unipi",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "All-in-one extension suite for Pi coding agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -71,26 +71,26 @@
|
|
|
71
71
|
"typebox": "^1.1.38"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@pi-unipi/ask-user": "2.
|
|
75
|
-
"@pi-unipi/btw": "2.
|
|
76
|
-
"@pi-unipi/compactor": "2.
|
|
77
|
-
"@pi-unipi/notify": "2.
|
|
78
|
-
"@pi-unipi/command-enchantment": "2.
|
|
79
|
-
"@pi-unipi/core": "2.
|
|
80
|
-
"@pi-unipi/info-screen": "2.
|
|
81
|
-
"@pi-unipi/mcp": "2.
|
|
82
|
-
"@pi-unipi/memory": "2.
|
|
83
|
-
"@pi-unipi/ralph": "2.
|
|
84
|
-
"@pi-unipi/subagents": "2.
|
|
85
|
-
"@pi-unipi/utility": "2.
|
|
86
|
-
"@pi-unipi/milestone": "2.
|
|
87
|
-
"@pi-unipi/kanboard": "2.
|
|
88
|
-
"@pi-unipi/web-api": "2.
|
|
89
|
-
"@pi-unipi/workflow": "2.
|
|
90
|
-
"@pi-unipi/footer": "2.
|
|
91
|
-
"@pi-unipi/updater": "2.
|
|
92
|
-
"@pi-unipi/input-shortcuts": "2.
|
|
93
|
-
"@pi-unipi/cocoindex": "2.
|
|
74
|
+
"@pi-unipi/ask-user": "2.4.0",
|
|
75
|
+
"@pi-unipi/btw": "2.4.0",
|
|
76
|
+
"@pi-unipi/compactor": "2.4.0",
|
|
77
|
+
"@pi-unipi/notify": "2.4.0",
|
|
78
|
+
"@pi-unipi/command-enchantment": "2.4.0",
|
|
79
|
+
"@pi-unipi/core": "2.4.0",
|
|
80
|
+
"@pi-unipi/info-screen": "2.4.0",
|
|
81
|
+
"@pi-unipi/mcp": "2.4.0",
|
|
82
|
+
"@pi-unipi/memory": "2.4.0",
|
|
83
|
+
"@pi-unipi/ralph": "2.4.0",
|
|
84
|
+
"@pi-unipi/subagents": "2.4.0",
|
|
85
|
+
"@pi-unipi/utility": "2.4.0",
|
|
86
|
+
"@pi-unipi/milestone": "2.4.0",
|
|
87
|
+
"@pi-unipi/kanboard": "2.4.0",
|
|
88
|
+
"@pi-unipi/web-api": "2.4.0",
|
|
89
|
+
"@pi-unipi/workflow": "2.4.0",
|
|
90
|
+
"@pi-unipi/footer": "2.4.0",
|
|
91
|
+
"@pi-unipi/updater": "2.4.0",
|
|
92
|
+
"@pi-unipi/input-shortcuts": "2.4.0",
|
|
93
|
+
"@pi-unipi/cocoindex": "2.4.0"
|
|
94
94
|
},
|
|
95
95
|
"devDependencies": {
|
|
96
96
|
"@earendil-works/pi-agent-core": "^0.80.0",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/ask-user",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Structured user input tool for Pi coding agent — single-select, multi-select, freeform",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"access": "public"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@pi-unipi/core": "2.
|
|
43
|
+
"@pi-unipi/core": "2.4.0"
|
|
44
44
|
},
|
|
45
45
|
"peerDependencies": {
|
|
46
46
|
"@earendil-works/pi-coding-agent": "^0.80.0",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/btw",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "A pi extension for parallel side conversations with /unipi:btw — part of the Unipi suite",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "extensions/btw.ts",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@earendil-works/pi-tui": "^0.80.0"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@pi-unipi/core": "2.
|
|
40
|
+
"@pi-unipi/core": "2.4.0"
|
|
41
41
|
},
|
|
42
42
|
"pi": {
|
|
43
43
|
"extensions": [],
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/cocoindex",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "CocoIndex integration for Pi — AST-aware content indexing, semantic vector search, and incremental pipeline management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.ts",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"typebox": "^1.1.38"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@pi-unipi/core": "2.
|
|
28
|
+
"@pi-unipi/core": "2.4.0"
|
|
29
29
|
},
|
|
30
30
|
"optionalDependencies": {
|
|
31
31
|
"@lancedb/lancedb": "^0.21.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/compactor",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Context engine for Pi — zero-LLM compaction, session continuity, sandbox execution, and tool display optimization",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
"access": "public"
|
|
35
35
|
},
|
|
36
36
|
"dependencies": {
|
|
37
|
-
"@pi-unipi/core": "2.
|
|
38
|
-
"@pi-unipi/info-screen": "2.
|
|
37
|
+
"@pi-unipi/core": "2.4.0",
|
|
38
|
+
"@pi-unipi/info-screen": "2.4.0",
|
|
39
39
|
"@earendil-works/pi-agent-core": "^0.80.0"
|
|
40
40
|
},
|
|
41
41
|
"peerDependencies": {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/footer",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Persistent status bar for Unipi — subscribes to UNIPI_EVENTS and renders key stats from all unipi packages",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"access": "public"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@pi-unipi/core": "2.
|
|
35
|
+
"@pi-unipi/core": "2.4.0"
|
|
36
36
|
},
|
|
37
37
|
"peerDependencies": {
|
|
38
38
|
"@earendil-works/pi-coding-agent": "^0.80.0",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-unipi/image",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Image generation and image recognition tools for the Pi coding agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"access": "public"
|
|
35
35
|
},
|
|
36
36
|
"dependencies": {
|
|
37
|
-
"@pi-unipi/core": "2.
|
|
37
|
+
"@pi-unipi/core": "2.4.0"
|
|
38
38
|
},
|
|
39
39
|
"peerDependencies": {
|
|
40
40
|
"@earendil-works/pi-ai": "^0.80.0",
|
|
@@ -131,6 +131,8 @@ export interface GenerateOptions {
|
|
|
131
131
|
signal?: AbortSignal;
|
|
132
132
|
/** Absolute directory for saved images; omit to skip saving. */
|
|
133
133
|
outputDir?: string;
|
|
134
|
+
/** Source image; when set the request is an edit rather than a generation. */
|
|
135
|
+
inputImage?: { data: string; mimeType: string };
|
|
134
136
|
now?: Date;
|
|
135
137
|
/** Injected images collection, for tests. */
|
|
136
138
|
images?: ImagesModelsLike;
|
|
@@ -141,7 +143,7 @@ export interface GenerateOptions {
|
|
|
141
143
|
* @throws {Error} with an actionable message when generation fails.
|
|
142
144
|
*/
|
|
143
145
|
export async function generateImage(options: GenerateOptions): Promise<GenerateResult> {
|
|
144
|
-
const { prompt, model, signal, outputDir, now } = options;
|
|
146
|
+
const { prompt, model, signal, outputDir, now, inputImage } = options;
|
|
145
147
|
|
|
146
148
|
if (!prompt.trim()) {
|
|
147
149
|
throw new Error("A non-empty prompt is required.");
|
|
@@ -154,19 +156,19 @@ export async function generateImage(options: GenerateOptions): Promise<GenerateR
|
|
|
154
156
|
);
|
|
155
157
|
}
|
|
156
158
|
|
|
157
|
-
// pi-ai's images collection
|
|
158
|
-
// `
|
|
159
|
-
//
|
|
160
|
-
//
|
|
161
|
-
//
|
|
159
|
+
// pi-ai's images collection has its own provider set, separate from pi's
|
|
160
|
+
// chat registry. `registerRegistryImageProviders()` bridges pi's providers
|
|
161
|
+
// in, but a model may still name a provider with no image route at all —
|
|
162
|
+
// pi-ai would answer with a bare "Unknown provider: x", so say something
|
|
163
|
+
// useful instead.
|
|
162
164
|
if (!providerCanGenerate(imagesApi, model.provider)) {
|
|
163
165
|
const supported = supportedProviders(imagesApi);
|
|
164
166
|
throw new Error(
|
|
165
|
-
`Provider "${model.provider}"
|
|
166
|
-
`→
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
"→ Pick
|
|
167
|
+
`Provider "${model.provider}" has no image-generation route.\n` +
|
|
168
|
+
`→ Available: ${supported.join(", ") || "openrouter"}.\n` +
|
|
169
|
+
"→ Providers are bridged from pi automatically; one without a baseUrl " +
|
|
170
|
+
"or an API key cannot be used.\n" +
|
|
171
|
+
"→ Pick another with /unipi:image-settings.",
|
|
170
172
|
);
|
|
171
173
|
}
|
|
172
174
|
|
|
@@ -174,7 +176,13 @@ export async function generateImage(options: GenerateOptions): Promise<GenerateR
|
|
|
174
176
|
// a bare OPENROUTER_API_KEY still works.
|
|
175
177
|
let apiKey: string | undefined;
|
|
176
178
|
try {
|
|
177
|
-
|
|
179
|
+
// pi-ai resolves to an `AuthResult`, i.e. `{ auth: { apiKey } }`. Older
|
|
180
|
+
// shapes put the key at the top level, so accept both — reading only one
|
|
181
|
+
// fails silently and looks like a missing credential.
|
|
182
|
+
const resolvedAuth = (await imagesApi.getAuth(model)) as
|
|
183
|
+
| { apiKey?: string; auth?: { apiKey?: string } }
|
|
184
|
+
| undefined;
|
|
185
|
+
apiKey = resolvedAuth?.auth?.apiKey ?? resolvedAuth?.apiKey;
|
|
178
186
|
} catch {
|
|
179
187
|
// Reported as a missing key below.
|
|
180
188
|
}
|
|
@@ -183,14 +191,25 @@ export async function generateImage(options: GenerateOptions): Promise<GenerateR
|
|
|
183
191
|
if (!apiKey) {
|
|
184
192
|
throw new Error(
|
|
185
193
|
`No API key for provider "${model.provider}".\n` +
|
|
186
|
-
|
|
187
|
-
`→
|
|
194
|
+
"→ Sign in with /login, or set the provider's API key environment variable.\n" +
|
|
195
|
+
`→ Expected environment variable: ` +
|
|
196
|
+
`${model.provider.toUpperCase().replace(/[^A-Z0-9]/g, "_")}_API_KEY`,
|
|
188
197
|
);
|
|
189
198
|
}
|
|
190
199
|
|
|
200
|
+
const input: Array<{ type: string; text?: string; data?: string; mimeType?: string }> =
|
|
201
|
+
[{ type: "text", text: prompt }];
|
|
202
|
+
if (inputImage) {
|
|
203
|
+
input.push({
|
|
204
|
+
type: "image",
|
|
205
|
+
data: inputImage.data,
|
|
206
|
+
mimeType: inputImage.mimeType,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
|
|
191
210
|
const result = (await imagesApi.generateImages(
|
|
192
211
|
model,
|
|
193
|
-
{ input:
|
|
212
|
+
{ input } as { input: Array<{ type: string; text?: string }> },
|
|
194
213
|
{ apiKey, ...(signal ? { signal } : {}) },
|
|
195
214
|
)) as AssistantImagesLike;
|
|
196
215
|
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
import { registerImageCommands } from "./commands.js";
|
|
22
22
|
import { registerImageTools } from "./tools.js";
|
|
23
23
|
import { listImageGenModels, listVisionModels, type ChatModelRegistry } from "./models.js";
|
|
24
|
+
import { registerRegistryImageProviders } from "./register-providers.js";
|
|
24
25
|
import { loadConfig } from "./settings.js";
|
|
25
26
|
|
|
26
27
|
const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
|
|
@@ -43,6 +44,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
43
44
|
pi.on("session_start", async (_event, ctx) => {
|
|
44
45
|
const config = loadConfig();
|
|
45
46
|
|
|
47
|
+
// Bridge pi's configured providers into pi-ai's images collection up front,
|
|
48
|
+
// so the settings picker and the info screen see them without a prior
|
|
49
|
+
// image_generate call. Best-effort: never block session start.
|
|
50
|
+
void registerRegistryImageProviders(
|
|
51
|
+
(ctx as unknown as { modelRegistry?: ChatModelRegistry }).modelRegistry,
|
|
52
|
+
).catch(() => undefined);
|
|
53
|
+
|
|
46
54
|
const tools: string[] = [];
|
|
47
55
|
if (config.generate.enabled) tools.push(IMAGE_TOOLS.GENERATE);
|
|
48
56
|
if (config.recognize.enabled) tools.push(IMAGE_TOOLS.RECOGNIZE);
|
|
@@ -211,12 +211,40 @@ export function listRegistryImageGenModels(
|
|
|
211
211
|
provider: candidate.provider,
|
|
212
212
|
name: candidate.name,
|
|
213
213
|
api: candidate.api ?? "",
|
|
214
|
+
// Carry the endpoint through. The generic images adapter POSTs to
|
|
215
|
+
// `{baseUrl}/images/generations`, and this is the only place the
|
|
216
|
+
// registry's baseUrl is available — dropping it here surfaces later as
|
|
217
|
+
// "No baseUrl for image model ..." once generation is attempted.
|
|
218
|
+
...(candidate.baseUrl ? { baseUrl: candidate.baseUrl } : {}),
|
|
214
219
|
...(candidate.output ? { output: candidate.output } : {}),
|
|
215
220
|
});
|
|
216
221
|
}
|
|
217
222
|
return out;
|
|
218
223
|
}
|
|
219
224
|
|
|
225
|
+
/**
|
|
226
|
+
* Find a provider's API endpoint in pi's registry.
|
|
227
|
+
*
|
|
228
|
+
* Needed because a model can reach generation without one: a user-typed
|
|
229
|
+
* "provider/model-id" is accepted at face value by `asExplicitModelRef`, and
|
|
230
|
+
* carries no baseUrl of its own.
|
|
231
|
+
*/
|
|
232
|
+
export function findProviderBaseUrl(
|
|
233
|
+
registry: ChatModelRegistry | undefined,
|
|
234
|
+
provider: string,
|
|
235
|
+
): string | undefined {
|
|
236
|
+
if (!registry) return undefined;
|
|
237
|
+
try {
|
|
238
|
+
const models = (registry.getAvailable?.() ?? registry.getAll()) as Array<{
|
|
239
|
+
provider?: string;
|
|
240
|
+
baseUrl?: string;
|
|
241
|
+
}>;
|
|
242
|
+
return models.find((m) => m?.provider === provider && m.baseUrl)?.baseUrl;
|
|
243
|
+
} catch {
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
220
248
|
/**
|
|
221
249
|
* Every selectable generation model: pi-ai's built-in catalog plus anything
|
|
222
250
|
* contributed by registered providers, de-duplicated by "provider/id".
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @pi-unipi/image — Generic OpenAI-compatible images adapter
|
|
3
|
+
*
|
|
4
|
+
* ONE adapter for every provider, rather than per-provider code. It speaks the
|
|
5
|
+
* OpenAI `POST {baseUrl}/images/generations` shape, which every gateway we have
|
|
6
|
+
* tested implements (OpenAI itself, OpenRouter, and OmniRoute's fan-out to
|
|
7
|
+
* openrouter/antigravity/codex/fal-ai backends).
|
|
8
|
+
*
|
|
9
|
+
* Why not pi-ai's built-in `api/openrouter-images`?
|
|
10
|
+
* Despite the name it drives `chat.completions` with `modalities:["image"]`.
|
|
11
|
+
* Gateways that do not implement that extension answer HTTP 200 with the model
|
|
12
|
+
* *narrating* the image ("Here's the image with the circle changed…") while
|
|
13
|
+
* silently dropping `message.images`. That is invisible data loss, so we use
|
|
14
|
+
* the dedicated images endpoint instead.
|
|
15
|
+
*
|
|
16
|
+
* Editing rides the same endpoint: `POST /images/generations` with an `image`
|
|
17
|
+
* array. `/images/edits` (multipart) is NOT used — gateways reject it for most
|
|
18
|
+
* providers ("Image edit is not supported for built-in provider ...").
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { ImageGenModel } from "./models.js";
|
|
22
|
+
|
|
23
|
+
/** pi-ai's `ImagesContext` input parts. */
|
|
24
|
+
export interface ImagesInputPart {
|
|
25
|
+
type: string;
|
|
26
|
+
text?: string;
|
|
27
|
+
data?: string;
|
|
28
|
+
mimeType?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface ImagesContextLike {
|
|
32
|
+
input: ImagesInputPart[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface ImagesOptionsLike {
|
|
36
|
+
apiKey?: string;
|
|
37
|
+
signal?: AbortSignal;
|
|
38
|
+
headers?: Record<string, string | null>;
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
/** Injectable fetch, for tests. */
|
|
41
|
+
fetchImpl?: typeof fetch;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** pi-ai's `AssistantImages`. */
|
|
45
|
+
export interface AssistantImagesLike {
|
|
46
|
+
api: string;
|
|
47
|
+
provider: string;
|
|
48
|
+
model: string;
|
|
49
|
+
output: Array<{ type: string; text?: string; data?: string; mimeType?: string }>;
|
|
50
|
+
stopReason: "stop" | "error" | "aborted";
|
|
51
|
+
errorMessage?: string;
|
|
52
|
+
timestamp: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Images are slow — a minute is not unusual for a large model. */
|
|
56
|
+
const DEFAULT_TIMEOUT_MS = 240_000;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* One returned image, normalized.
|
|
60
|
+
*
|
|
61
|
+
* Gateways disagree on the item shape; all three observed forms are accepted:
|
|
62
|
+
* - `{ b64_json, media_type }` — openrouter/* (note `media_type`, not `mimeType`)
|
|
63
|
+
* - `{ b64_json, revised_prompt }` — antigravity/*
|
|
64
|
+
* - `{ url: "data:image/png;base64,…" }` — codex/*
|
|
65
|
+
* A plain http(s) `url` is also tolerated and reported as text, since we cannot
|
|
66
|
+
* inline bytes we did not fetch.
|
|
67
|
+
*/
|
|
68
|
+
interface RawImageItem {
|
|
69
|
+
b64_json?: unknown;
|
|
70
|
+
url?: unknown;
|
|
71
|
+
media_type?: unknown;
|
|
72
|
+
mime_type?: unknown;
|
|
73
|
+
mimeType?: unknown;
|
|
74
|
+
revised_prompt?: unknown;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function asString(value: unknown): string | undefined {
|
|
78
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Pull `{ data, mimeType }` out of one response item, whatever its shape. */
|
|
82
|
+
export function normalizeImageItem(
|
|
83
|
+
item: RawImageItem,
|
|
84
|
+
): { data: string; mimeType: string } | { text: string } | null {
|
|
85
|
+
const declared =
|
|
86
|
+
asString(item.media_type) ?? asString(item.mime_type) ?? asString(item.mimeType);
|
|
87
|
+
|
|
88
|
+
const b64 = asString(item.b64_json);
|
|
89
|
+
if (b64) return { data: b64, mimeType: declared ?? "image/png" };
|
|
90
|
+
|
|
91
|
+
const url = asString(item.url);
|
|
92
|
+
if (!url) return null;
|
|
93
|
+
|
|
94
|
+
// codex/* returns the bytes as a data: URL rather than b64_json.
|
|
95
|
+
const dataUrl = /^data:([^;,]+)(?:;[^,]*)*,(.*)$/s.exec(url);
|
|
96
|
+
if (dataUrl) {
|
|
97
|
+
const [, mime, payload] = dataUrl;
|
|
98
|
+
if (payload) return { data: payload, mimeType: declared ?? mime ?? "image/png" };
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// A remote URL: surface it rather than silently dropping the result.
|
|
103
|
+
return { text: `Image available at: ${url}` };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Strip a trailing slash so `${base}/images/generations` is well-formed. */
|
|
107
|
+
function joinUrl(baseUrl: string, suffix: string): string {
|
|
108
|
+
return `${baseUrl.replace(/\/+$/, "")}/${suffix.replace(/^\/+/, "")}`;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Model ids are sent to the gateway verbatim.
|
|
113
|
+
*
|
|
114
|
+
* Do NOT try to "repair" a doubled-looking segment. OmniRoute genuinely serves
|
|
115
|
+
* `fal-ai/fal-ai/nano-banana-pro` (provider `fal-ai` + model `fal-ai/nano-...`),
|
|
116
|
+
* and rewriting it to `fal-ai/nano-banana-pro` yields a 404. Confusingly the
|
|
117
|
+
* gateway *also* advertises a `fal/...` alias in `/v1/models` that the images
|
|
118
|
+
* endpoint then rejects with "Invalid image model" — an upstream inconsistency
|
|
119
|
+
* we surface rather than guess around, because a wrong guess turns a clear
|
|
120
|
+
* error into a silently different model.
|
|
121
|
+
*/
|
|
122
|
+
export function normalizeModelId(id: string): string {
|
|
123
|
+
return id;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Merge caller headers, dropping keys explicitly suppressed with null. */
|
|
127
|
+
function buildHeaders(
|
|
128
|
+
apiKey: string,
|
|
129
|
+
extra?: Record<string, string | null>,
|
|
130
|
+
): Record<string, string> {
|
|
131
|
+
const headers: Record<string, string> = {
|
|
132
|
+
Authorization: `Bearer ${apiKey}`,
|
|
133
|
+
"Content-Type": "application/json",
|
|
134
|
+
};
|
|
135
|
+
for (const [key, value] of Object.entries(extra ?? {})) {
|
|
136
|
+
if (value === null) delete headers[key];
|
|
137
|
+
else headers[key] = value;
|
|
138
|
+
}
|
|
139
|
+
return headers;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Best-effort extraction of a provider error message. */
|
|
143
|
+
function describeError(status: number, statusText: string, body: string): string {
|
|
144
|
+
let detail = body.slice(0, 300);
|
|
145
|
+
try {
|
|
146
|
+
const parsed = JSON.parse(body) as { error?: { message?: string } | string };
|
|
147
|
+
if (typeof parsed.error === "string") detail = parsed.error;
|
|
148
|
+
else if (parsed.error?.message) detail = parsed.error.message;
|
|
149
|
+
} catch {
|
|
150
|
+
// Non-JSON body — the truncated text is the best we have.
|
|
151
|
+
}
|
|
152
|
+
return `${status} ${statusText}${detail ? `: ${detail}` : ""}`;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Add guidance for the gateway's confusing model-id errors. */
|
|
156
|
+
function annotateModelError(message: string, model: ImageGenModel): string {
|
|
157
|
+
if (/invalid image model|not found|unknown model/i.test(message)) {
|
|
158
|
+
return (
|
|
159
|
+
`${message}\n` +
|
|
160
|
+
`→ Model id sent: "${model.id}" (provider "${model.provider}").\n` +
|
|
161
|
+
"→ Some gateways list aliases they cannot serve. Try the id exactly as it " +
|
|
162
|
+
"appears in the provider's own catalog, or pick another with /unipi:image-settings."
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return message;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Generate (or edit) images against an OpenAI-compatible endpoint.
|
|
170
|
+
*
|
|
171
|
+
* Satisfies pi-ai's `ProviderImages` interface, so the result is returned —
|
|
172
|
+
* never thrown — with `stopReason: "error"` on failure.
|
|
173
|
+
*/
|
|
174
|
+
export async function generateImages(
|
|
175
|
+
model: ImageGenModel,
|
|
176
|
+
context: ImagesContextLike,
|
|
177
|
+
options?: ImagesOptionsLike,
|
|
178
|
+
): Promise<AssistantImagesLike> {
|
|
179
|
+
const result: AssistantImagesLike = {
|
|
180
|
+
api: model.api || "openai-images",
|
|
181
|
+
provider: model.provider,
|
|
182
|
+
model: model.id,
|
|
183
|
+
output: [],
|
|
184
|
+
stopReason: "stop",
|
|
185
|
+
timestamp: Date.now(),
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
const fetchImpl = options?.fetchImpl ?? fetch;
|
|
189
|
+
const controller = new AbortController();
|
|
190
|
+
const timer = setTimeout(
|
|
191
|
+
() => controller.abort(),
|
|
192
|
+
options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
193
|
+
);
|
|
194
|
+
const onAbort = () => controller.abort();
|
|
195
|
+
options?.signal?.addEventListener("abort", onAbort, { once: true });
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
const apiKey = options?.apiKey;
|
|
199
|
+
if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);
|
|
200
|
+
if (!model.baseUrl) {
|
|
201
|
+
throw new Error(`No baseUrl for image model ${model.provider}/${model.id}`);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Text parts form the prompt; image parts switch the request into edit mode.
|
|
205
|
+
const prompt = context.input
|
|
206
|
+
.filter((part) => part.type === "text" && part.text)
|
|
207
|
+
.map((part) => part.text as string)
|
|
208
|
+
.join("\n")
|
|
209
|
+
.trim();
|
|
210
|
+
|
|
211
|
+
const images = context.input
|
|
212
|
+
.filter((part) => part.type === "image" && part.data)
|
|
213
|
+
.map((part) => `data:${part.mimeType || "image/png"};base64,${part.data}`);
|
|
214
|
+
|
|
215
|
+
if (!prompt) throw new Error("A non-empty prompt is required.");
|
|
216
|
+
|
|
217
|
+
const body: Record<string, unknown> = {
|
|
218
|
+
model: normalizeModelId(model.id),
|
|
219
|
+
prompt,
|
|
220
|
+
};
|
|
221
|
+
// Only send `image` for edits; some backends reject an empty array.
|
|
222
|
+
if (images.length > 0) body.image = images;
|
|
223
|
+
|
|
224
|
+
const response = await fetchImpl(joinUrl(model.baseUrl, "images/generations"), {
|
|
225
|
+
method: "POST",
|
|
226
|
+
headers: buildHeaders(apiKey, options?.headers),
|
|
227
|
+
body: JSON.stringify(body),
|
|
228
|
+
signal: controller.signal,
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
if (!response.ok) {
|
|
232
|
+
throw new Error(
|
|
233
|
+
describeError(response.status, response.statusText, await response.text()),
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const payload = (await response.json()) as {
|
|
238
|
+
data?: RawImageItem[];
|
|
239
|
+
error?: { message?: string };
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
if (payload.error?.message) throw new Error(payload.error.message);
|
|
243
|
+
|
|
244
|
+
for (const item of payload.data ?? []) {
|
|
245
|
+
const normalized = normalizeImageItem(item);
|
|
246
|
+
if (!normalized) continue;
|
|
247
|
+
if ("text" in normalized) {
|
|
248
|
+
result.output.push({ type: "text", text: normalized.text });
|
|
249
|
+
} else {
|
|
250
|
+
result.output.push({
|
|
251
|
+
type: "image",
|
|
252
|
+
data: normalized.data,
|
|
253
|
+
mimeType: normalized.mimeType,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
const revised = asString(item.revised_prompt);
|
|
257
|
+
if (revised && revised !== prompt) {
|
|
258
|
+
result.output.push({ type: "text", text: `Revised prompt: ${revised}` });
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
if (result.output.every((part) => part.type !== "image")) {
|
|
263
|
+
throw new Error("The provider returned no image data.");
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return result;
|
|
267
|
+
} catch (error) {
|
|
268
|
+
const aborted = options?.signal?.aborted || controller.signal.aborted;
|
|
269
|
+
result.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
|
270
|
+
result.errorMessage =
|
|
271
|
+
aborted && !options?.signal?.aborted
|
|
272
|
+
? "Image request timed out."
|
|
273
|
+
: annotateModelError(
|
|
274
|
+
error instanceof Error ? error.message : String(error),
|
|
275
|
+
model,
|
|
276
|
+
);
|
|
277
|
+
return result;
|
|
278
|
+
} finally {
|
|
279
|
+
clearTimeout(timer);
|
|
280
|
+
options?.signal?.removeEventListener("abort", onAbort);
|
|
281
|
+
}
|
|
282
|
+
}
|