ima2-gen 1.1.13 → 1.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/bin/commands/doctor.js +195 -0
- package/bin/commands/doctor.ts +202 -0
- package/bin/ima2.js +3 -105
- package/bin/ima2.ts +3 -109
- package/config.js +1 -0
- package/config.ts +5 -0
- package/docs/CLI.md +36 -0
- package/docs/FAQ.ko.md +82 -2
- package/docs/FAQ.md +85 -2
- package/docs/PROMPT_STUDIO.ko.md +111 -0
- package/docs/PROMPT_STUDIO.md +115 -0
- package/docs/README.ko.md +8 -1
- package/docs/migration/runtime-test-inventory.md +6 -1
- package/lib/agentRuntime.js +9 -2
- package/lib/agentRuntime.ts +8 -2
- package/lib/errorClassify.js +1 -1
- package/lib/errorClassify.ts +1 -1
- package/lib/generationErrors.js +121 -23
- package/lib/generationErrors.ts +100 -13
- package/lib/responsesDoctor.js +386 -0
- package/lib/responsesDoctor.ts +456 -0
- package/lib/responsesErrors.js +57 -0
- package/lib/responsesErrors.ts +83 -0
- package/lib/responsesFallback.js +72 -0
- package/lib/responsesFallback.ts +114 -0
- package/lib/responsesImageAdapter.js +121 -174
- package/lib/responsesImageAdapter.ts +136 -211
- package/lib/responsesParse.js +324 -0
- package/lib/responsesParse.ts +452 -0
- package/lib/responsesTools.js +15 -0
- package/lib/responsesTools.ts +28 -0
- package/package.json +1 -1
- package/routes/edit.js +26 -1
- package/routes/edit.ts +26 -1
- package/routes/generate.js +40 -0
- package/routes/generate.ts +47 -0
- package/ui/dist/.vite/manifest.json +12 -12
- package/ui/dist/assets/{AgentWorkspace-BJe9yxPA.js → AgentWorkspace-B6YNOZHi.js} +1 -1
- package/ui/dist/assets/{CardNewsWorkspace-BBLdwzYU.js → CardNewsWorkspace-EFVeg4l_.js} +1 -1
- package/ui/dist/assets/{NodeCanvas-BSZ527J4.js → NodeCanvas-iM6yjHvO.js} +1 -1
- package/ui/dist/assets/{PromptBuilderPanel-Y2VygFc0.js → PromptBuilderPanel-C3GdLDCl.js} +1 -1
- package/ui/dist/assets/{PromptImportDialog-C6lFV-LL.js → PromptImportDialog-DS9vrc_w.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-D8YJFhND.js → PromptImportDiscoverySection-DHFEt_FA.js} +1 -1
- package/ui/dist/assets/{PromptImportFolderSection-ywfcQolW.js → PromptImportFolderSection-BQxb1zs5.js} +1 -1
- package/ui/dist/assets/{PromptLibraryPanel-fk4KmrGy.js → PromptLibraryPanel-NhMKVGfU.js} +2 -2
- package/ui/dist/assets/{SettingsWorkspace-DL5vhAHQ.js → SettingsWorkspace-FjKjaDqj.js} +1 -1
- package/ui/dist/assets/index-BAN6lKgf.js +28 -0
- package/ui/dist/assets/{index-BLx55BOg.js → index-BbFZyM92.js} +1 -1
- package/ui/dist/assets/index-DK1faG9Z.css +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/index-ByViUJfx.css +0 -1
- package/ui/dist/assets/index-Ci36vcFD.js +0 -28
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
import { config as defaultConfig } from "../config.js";
|
|
2
|
+
import { errInfo } from "./errInfo.js";
|
|
3
|
+
import { parseJson, parseStream, safeDiagnosticLabel, type ResponseDiagnostics } from "./responsesParse.js";
|
|
4
|
+
import type { RouteRuntimeContext } from "./runtimeContext.js";
|
|
5
|
+
import {
|
|
6
|
+
GENERATE_DEVELOPER_PROMPT,
|
|
7
|
+
GENERATE_NO_SEARCH_DEVELOPER_PROMPT,
|
|
8
|
+
buildUserTextPrompt,
|
|
9
|
+
waitForOAuthReady,
|
|
10
|
+
} from "./oauthProxy.js";
|
|
11
|
+
|
|
12
|
+
type ToolChoiceSummary = "none" | "required" | "image_generation";
|
|
13
|
+
type ProbeExpectation = "text" | "image";
|
|
14
|
+
|
|
15
|
+
interface ProbeSpec {
|
|
16
|
+
id: string;
|
|
17
|
+
expectation: ProbeExpectation;
|
|
18
|
+
promptId: string;
|
|
19
|
+
promptChars: number;
|
|
20
|
+
stream: boolean;
|
|
21
|
+
toolTypes: string[];
|
|
22
|
+
toolChoiceKind: ToolChoiceSummary;
|
|
23
|
+
payload: Record<string, unknown>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ImageDoctorProbeOptions {
|
|
27
|
+
provider?: string;
|
|
28
|
+
apiKey?: string;
|
|
29
|
+
oauthUrl?: string;
|
|
30
|
+
model?: string;
|
|
31
|
+
size?: string;
|
|
32
|
+
quality?: string;
|
|
33
|
+
moderation?: string;
|
|
34
|
+
prompt?: string;
|
|
35
|
+
matrix?: boolean;
|
|
36
|
+
timeoutMs?: number;
|
|
37
|
+
ctx?: RouteRuntimeContext;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ImageDoctorProbeResult {
|
|
41
|
+
id: string;
|
|
42
|
+
ok: boolean;
|
|
43
|
+
expectation: ProbeExpectation;
|
|
44
|
+
diagnosticReason: string | null;
|
|
45
|
+
request: {
|
|
46
|
+
stream: boolean;
|
|
47
|
+
toolTypes: string[];
|
|
48
|
+
toolChoiceKind: ToolChoiceSummary;
|
|
49
|
+
promptId: string;
|
|
50
|
+
promptChars: number;
|
|
51
|
+
};
|
|
52
|
+
response: {
|
|
53
|
+
httpStatus: number | null;
|
|
54
|
+
contentType: string | null;
|
|
55
|
+
upstreamRequestId: string | null;
|
|
56
|
+
durationMs: number;
|
|
57
|
+
eventCount: number;
|
|
58
|
+
eventTypes: Record<string, number>;
|
|
59
|
+
webSearchCalls: number;
|
|
60
|
+
textOutputChars: number;
|
|
61
|
+
imageResultCount: number;
|
|
62
|
+
firstImageChars: number;
|
|
63
|
+
diagnostics: ResponseDiagnostics | null;
|
|
64
|
+
};
|
|
65
|
+
error: {
|
|
66
|
+
code: string | null;
|
|
67
|
+
type: string | null;
|
|
68
|
+
param: string | null;
|
|
69
|
+
message: string;
|
|
70
|
+
upstreamBodyChars?: number;
|
|
71
|
+
} | null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const BUILTIN_PROMPT_ID = "builtin_cat";
|
|
75
|
+
const BUILTIN_CAT_PROMPT = "고양이";
|
|
76
|
+
const TEXT_OK_PROMPT = "Reply with exactly OK.";
|
|
77
|
+
|
|
78
|
+
function imageTool(size: string, quality: string, moderation: string) {
|
|
79
|
+
return { type: "image_generation", size, quality, moderation, action: "generate" };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function createProbeSpecs({
|
|
83
|
+
model,
|
|
84
|
+
size,
|
|
85
|
+
quality,
|
|
86
|
+
moderation,
|
|
87
|
+
prompt,
|
|
88
|
+
matrix,
|
|
89
|
+
}: Required<Pick<ImageDoctorProbeOptions, "model" | "size" | "quality" | "moderation" | "prompt">> & { matrix: boolean }): ProbeSpec[] {
|
|
90
|
+
const imageOnlyTools = [imageTool(size, quality, moderation)];
|
|
91
|
+
const webSearchImageTools = [{ type: "web_search" }, imageTool(size, quality, moderation)];
|
|
92
|
+
const specs: ProbeSpec[] = [
|
|
93
|
+
{
|
|
94
|
+
id: "text_sanity",
|
|
95
|
+
expectation: "text",
|
|
96
|
+
promptId: "builtin_text_ok",
|
|
97
|
+
promptChars: TEXT_OK_PROMPT.length,
|
|
98
|
+
stream: false,
|
|
99
|
+
toolTypes: [],
|
|
100
|
+
toolChoiceKind: "none",
|
|
101
|
+
payload: {
|
|
102
|
+
model,
|
|
103
|
+
input: [{ role: "user", content: TEXT_OK_PROMPT }],
|
|
104
|
+
stream: false,
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
id: "minimal_image_non_stream",
|
|
109
|
+
expectation: "image",
|
|
110
|
+
promptId: BUILTIN_PROMPT_ID,
|
|
111
|
+
promptChars: prompt.length,
|
|
112
|
+
stream: false,
|
|
113
|
+
toolTypes: ["image_generation"],
|
|
114
|
+
toolChoiceKind: "image_generation",
|
|
115
|
+
payload: {
|
|
116
|
+
model,
|
|
117
|
+
input: [{ role: "user", content: prompt }],
|
|
118
|
+
tools: imageOnlyTools,
|
|
119
|
+
tool_choice: { type: "image_generation" },
|
|
120
|
+
stream: false,
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
id: "minimal_image_stream",
|
|
125
|
+
expectation: "image",
|
|
126
|
+
promptId: BUILTIN_PROMPT_ID,
|
|
127
|
+
promptChars: prompt.length,
|
|
128
|
+
stream: true,
|
|
129
|
+
toolTypes: ["image_generation"],
|
|
130
|
+
toolChoiceKind: "image_generation",
|
|
131
|
+
payload: {
|
|
132
|
+
model,
|
|
133
|
+
input: [{ role: "user", content: prompt }],
|
|
134
|
+
tools: imageOnlyTools,
|
|
135
|
+
tool_choice: { type: "image_generation" },
|
|
136
|
+
stream: true,
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
];
|
|
140
|
+
if (!matrix) return specs;
|
|
141
|
+
return [
|
|
142
|
+
...specs,
|
|
143
|
+
{
|
|
144
|
+
id: "current_payload_no_search_required",
|
|
145
|
+
expectation: "image",
|
|
146
|
+
promptId: BUILTIN_PROMPT_ID,
|
|
147
|
+
promptChars: prompt.length,
|
|
148
|
+
stream: true,
|
|
149
|
+
toolTypes: ["image_generation"],
|
|
150
|
+
toolChoiceKind: "required",
|
|
151
|
+
payload: {
|
|
152
|
+
model,
|
|
153
|
+
input: [
|
|
154
|
+
{ role: "developer", content: GENERATE_NO_SEARCH_DEVELOPER_PROMPT },
|
|
155
|
+
{ role: "user", content: buildUserTextPrompt(prompt, "auto", { webSearchEnabled: false }) },
|
|
156
|
+
],
|
|
157
|
+
tools: imageOnlyTools,
|
|
158
|
+
tool_choice: "required",
|
|
159
|
+
reasoning: { effort: "low" },
|
|
160
|
+
stream: true,
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
id: "current_payload_web_search_required",
|
|
165
|
+
expectation: "image",
|
|
166
|
+
promptId: BUILTIN_PROMPT_ID,
|
|
167
|
+
promptChars: prompt.length,
|
|
168
|
+
stream: true,
|
|
169
|
+
toolTypes: ["web_search", "image_generation"],
|
|
170
|
+
toolChoiceKind: "required",
|
|
171
|
+
payload: {
|
|
172
|
+
model,
|
|
173
|
+
input: [
|
|
174
|
+
{ role: "developer", content: GENERATE_DEVELOPER_PROMPT },
|
|
175
|
+
{ role: "user", content: buildUserTextPrompt(prompt, "auto", { webSearchEnabled: true }) },
|
|
176
|
+
],
|
|
177
|
+
tools: webSearchImageTools,
|
|
178
|
+
tool_choice: "required",
|
|
179
|
+
reasoning: { effort: "low" },
|
|
180
|
+
stream: true,
|
|
181
|
+
},
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
id: "current_payload_web_search_forced_image",
|
|
185
|
+
expectation: "image",
|
|
186
|
+
promptId: BUILTIN_PROMPT_ID,
|
|
187
|
+
promptChars: prompt.length,
|
|
188
|
+
stream: true,
|
|
189
|
+
toolTypes: ["web_search", "image_generation"],
|
|
190
|
+
toolChoiceKind: "image_generation",
|
|
191
|
+
payload: {
|
|
192
|
+
model,
|
|
193
|
+
input: [
|
|
194
|
+
{ role: "developer", content: GENERATE_DEVELOPER_PROMPT },
|
|
195
|
+
{ role: "user", content: buildUserTextPrompt(prompt, "auto", { webSearchEnabled: true }) },
|
|
196
|
+
],
|
|
197
|
+
tools: webSearchImageTools,
|
|
198
|
+
tool_choice: { type: "image_generation" },
|
|
199
|
+
reasoning: { effort: "low" },
|
|
200
|
+
stream: true,
|
|
201
|
+
},
|
|
202
|
+
},
|
|
203
|
+
];
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function endpointFor(provider: string, options: ImageDoctorProbeOptions) {
|
|
207
|
+
const ctx = options.ctx || { config: defaultConfig };
|
|
208
|
+
if (provider === "api") {
|
|
209
|
+
const headers: Record<string, string> = {
|
|
210
|
+
"Content-Type": "application/json",
|
|
211
|
+
Accept: "text/event-stream, application/json",
|
|
212
|
+
Authorization: apiAuthorizationHeader(options.apiKey),
|
|
213
|
+
};
|
|
214
|
+
return {
|
|
215
|
+
url: "https://api.openai.com/v1/responses",
|
|
216
|
+
headers,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
await waitForOAuthReady(ctx);
|
|
220
|
+
const port = ctx.config?.oauth?.proxyPort || defaultConfig.oauth.proxyPort;
|
|
221
|
+
const baseUrl = safeOAuthBaseUrl(options.oauthUrl || `http://127.0.0.1:${port}`);
|
|
222
|
+
const headers: Record<string, string> = {
|
|
223
|
+
"Content-Type": "application/json",
|
|
224
|
+
Accept: "text/event-stream, application/json",
|
|
225
|
+
};
|
|
226
|
+
return {
|
|
227
|
+
url: `${baseUrl}/v1/responses`,
|
|
228
|
+
headers,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function apiAuthorizationHeader(apiKey: string | undefined) {
|
|
233
|
+
const key = typeof apiKey === "string" ? apiKey.trim() : "";
|
|
234
|
+
if (!key) {
|
|
235
|
+
throw Object.assign(new Error("API key is required for API provider image probe"), {
|
|
236
|
+
code: "API_KEY_REQUIRED",
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
if (/[\u0000-\u001f\u007f]/.test(key)) {
|
|
240
|
+
throw Object.assign(new Error("API key contains invalid characters."), {
|
|
241
|
+
code: "AUTH_API_KEY_INVALID",
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
return `Bearer ${key}`;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function safeOAuthBaseUrl(value: string) {
|
|
248
|
+
try {
|
|
249
|
+
const parsed = new URL(value);
|
|
250
|
+
parsed.username = "";
|
|
251
|
+
parsed.password = "";
|
|
252
|
+
return parsed.toString().replace(/\/$/, "");
|
|
253
|
+
} catch {
|
|
254
|
+
return value.replace(/\/$/, "");
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function sanitizeProbeErrorMessage(value: string) {
|
|
259
|
+
return value
|
|
260
|
+
.replace(/([a-z][a-z0-9+.-]*:\/\/)([^/\s@]+)@/gi, "$1[redacted]@")
|
|
261
|
+
.replace(/([?&](?:access_token|api_key|key|secret|token)=)[^&\s]+/gi, "$1[redacted]")
|
|
262
|
+
.replace(/Bearer\s+[^"'\s]+/gi, "Bearer [redacted]")
|
|
263
|
+
.replace(/sk-[A-Za-z0-9_-]+/g, "sk-[redacted]");
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function safeUpstreamError(text: string) {
|
|
267
|
+
try {
|
|
268
|
+
const parsed = JSON.parse(text);
|
|
269
|
+
const error = parsed?.error || {};
|
|
270
|
+
return {
|
|
271
|
+
code: safeDiagnosticLabel(error.code),
|
|
272
|
+
type: safeDiagnosticLabel(error.type),
|
|
273
|
+
param: safeDiagnosticLabel(error.param),
|
|
274
|
+
};
|
|
275
|
+
} catch {
|
|
276
|
+
return { code: null, type: null, param: null };
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function reasonFrom(result: Omit<ImageDoctorProbeResult, "diagnosticReason">): string | null {
|
|
281
|
+
const diagnostics = result.response.diagnostics;
|
|
282
|
+
if (!result.error && result.ok) return null;
|
|
283
|
+
if (result.error?.code) return result.error.code;
|
|
284
|
+
if (!diagnostics) return "probe_failed";
|
|
285
|
+
if (diagnostics.streamStats.bytesRead > 0 && result.response.eventCount === 0) return "stream_parse_failed";
|
|
286
|
+
if (diagnostics.imageCallFailed) return "image_tool_failed";
|
|
287
|
+
if (diagnostics.imageCallCompleted && diagnostics.imageResultCount === 0) return "image_tool_completed_without_result";
|
|
288
|
+
if (!diagnostics.imageCallSeen && result.response.webSearchCalls > 0) return "web_search_only_response";
|
|
289
|
+
if (!diagnostics.imageCallSeen && diagnostics.messageOutputSeen) return "image_tool_not_called";
|
|
290
|
+
if (result.expectation === "text" && result.response.textOutputChars === 0) return "text_output_missing";
|
|
291
|
+
if (result.expectation === "image" && result.response.imageResultCount === 0) return "image_output_missing";
|
|
292
|
+
return "probe_failed";
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
async function runSingleProbe(provider: string, options: ImageDoctorProbeOptions, spec: ProbeSpec): Promise<ImageDoctorProbeResult> {
|
|
296
|
+
const start = Date.now();
|
|
297
|
+
let httpStatus: number | null = null;
|
|
298
|
+
let contentType: string | null = null;
|
|
299
|
+
let upstreamRequestId: string | null = null;
|
|
300
|
+
try {
|
|
301
|
+
const endpoint = await endpointFor(provider, options);
|
|
302
|
+
const controller = new AbortController();
|
|
303
|
+
const timer = setTimeout(() => controller.abort(), options.timeoutMs || defaultConfig.oauth.generationTimeoutMs);
|
|
304
|
+
try {
|
|
305
|
+
const res = await fetch(endpoint.url, {
|
|
306
|
+
method: "POST",
|
|
307
|
+
headers: endpoint.headers,
|
|
308
|
+
signal: controller.signal,
|
|
309
|
+
body: JSON.stringify(spec.payload),
|
|
310
|
+
});
|
|
311
|
+
httpStatus = res.status;
|
|
312
|
+
contentType = res.headers.get("content-type");
|
|
313
|
+
upstreamRequestId = res.headers.get("x-request-id") || res.headers.get("openai-request-id");
|
|
314
|
+
if (!res.ok) {
|
|
315
|
+
const body = await res.text();
|
|
316
|
+
const upstream = safeUpstreamError(body);
|
|
317
|
+
const partial = {
|
|
318
|
+
id: spec.id,
|
|
319
|
+
ok: false,
|
|
320
|
+
expectation: spec.expectation,
|
|
321
|
+
request: {
|
|
322
|
+
stream: spec.stream,
|
|
323
|
+
toolTypes: spec.toolTypes,
|
|
324
|
+
toolChoiceKind: spec.toolChoiceKind,
|
|
325
|
+
promptId: spec.promptId,
|
|
326
|
+
promptChars: spec.promptChars,
|
|
327
|
+
},
|
|
328
|
+
response: {
|
|
329
|
+
httpStatus,
|
|
330
|
+
contentType,
|
|
331
|
+
upstreamRequestId,
|
|
332
|
+
durationMs: Date.now() - start,
|
|
333
|
+
eventCount: 0,
|
|
334
|
+
eventTypes: {},
|
|
335
|
+
webSearchCalls: 0,
|
|
336
|
+
textOutputChars: 0,
|
|
337
|
+
imageResultCount: 0,
|
|
338
|
+
firstImageChars: 0,
|
|
339
|
+
diagnostics: null,
|
|
340
|
+
},
|
|
341
|
+
error: {
|
|
342
|
+
...upstream,
|
|
343
|
+
message: "Upstream returned a non-2xx response",
|
|
344
|
+
upstreamBodyChars: body.length,
|
|
345
|
+
},
|
|
346
|
+
};
|
|
347
|
+
return { ...partial, diagnosticReason: reasonFrom(partial) };
|
|
348
|
+
}
|
|
349
|
+
const parsed = contentType?.includes("text/event-stream")
|
|
350
|
+
? await parseStream(res, { scope: "doctor-image-probe", maxImages: 1 })
|
|
351
|
+
: await parseJson(res, 1);
|
|
352
|
+
const ok = spec.expectation === "image"
|
|
353
|
+
? Boolean(parsed.images[0]?.b64)
|
|
354
|
+
: Boolean(parsed.text && parsed.text.trim());
|
|
355
|
+
const partial = {
|
|
356
|
+
id: spec.id,
|
|
357
|
+
ok,
|
|
358
|
+
expectation: spec.expectation,
|
|
359
|
+
request: {
|
|
360
|
+
stream: spec.stream,
|
|
361
|
+
toolTypes: spec.toolTypes,
|
|
362
|
+
toolChoiceKind: spec.toolChoiceKind,
|
|
363
|
+
promptId: spec.promptId,
|
|
364
|
+
promptChars: spec.promptChars,
|
|
365
|
+
},
|
|
366
|
+
response: {
|
|
367
|
+
httpStatus,
|
|
368
|
+
contentType,
|
|
369
|
+
upstreamRequestId,
|
|
370
|
+
durationMs: Date.now() - start,
|
|
371
|
+
eventCount: parsed.eventCount,
|
|
372
|
+
eventTypes: parsed.eventTypes,
|
|
373
|
+
webSearchCalls: parsed.webSearchCalls,
|
|
374
|
+
textOutputChars: parsed.text?.length || 0,
|
|
375
|
+
imageResultCount: parsed.images.length,
|
|
376
|
+
firstImageChars: parsed.images[0]?.b64.length || 0,
|
|
377
|
+
diagnostics: parsed.diagnostics,
|
|
378
|
+
},
|
|
379
|
+
error: null,
|
|
380
|
+
};
|
|
381
|
+
return { ...partial, diagnosticReason: reasonFrom(partial) };
|
|
382
|
+
} finally {
|
|
383
|
+
clearTimeout(timer);
|
|
384
|
+
}
|
|
385
|
+
} catch (e) {
|
|
386
|
+
const err = errInfo(e);
|
|
387
|
+
const partial = {
|
|
388
|
+
id: spec.id,
|
|
389
|
+
ok: false,
|
|
390
|
+
expectation: spec.expectation,
|
|
391
|
+
request: {
|
|
392
|
+
stream: spec.stream,
|
|
393
|
+
toolTypes: spec.toolTypes,
|
|
394
|
+
toolChoiceKind: spec.toolChoiceKind,
|
|
395
|
+
promptId: spec.promptId,
|
|
396
|
+
promptChars: spec.promptChars,
|
|
397
|
+
},
|
|
398
|
+
response: {
|
|
399
|
+
httpStatus,
|
|
400
|
+
contentType,
|
|
401
|
+
upstreamRequestId,
|
|
402
|
+
durationMs: Date.now() - start,
|
|
403
|
+
eventCount: Number((err.raw as { eventCount?: unknown })?.eventCount) || 0,
|
|
404
|
+
eventTypes: {},
|
|
405
|
+
webSearchCalls: 0,
|
|
406
|
+
textOutputChars: 0,
|
|
407
|
+
imageResultCount: 0,
|
|
408
|
+
firstImageChars: 0,
|
|
409
|
+
diagnostics: null,
|
|
410
|
+
},
|
|
411
|
+
error: {
|
|
412
|
+
code: safeDiagnosticLabel(err.code),
|
|
413
|
+
type: null,
|
|
414
|
+
param: null,
|
|
415
|
+
message: err.name === "AbortError" ? "Probe timed out" : sanitizeProbeErrorMessage(err.message),
|
|
416
|
+
},
|
|
417
|
+
};
|
|
418
|
+
return { ...partial, diagnosticReason: reasonFrom(partial) };
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
export async function runImageDoctorProbe(options: ImageDoctorProbeOptions = {}) {
|
|
423
|
+
const provider = options.provider || "oauth";
|
|
424
|
+
const model = options.model || defaultConfig.imageModels?.default || "gpt-5.4-mini";
|
|
425
|
+
const size = options.size || "1024x1024";
|
|
426
|
+
const quality = options.quality || "low";
|
|
427
|
+
const moderation = options.moderation || "low";
|
|
428
|
+
const prompt = options.prompt || BUILTIN_CAT_PROMPT;
|
|
429
|
+
const specs = createProbeSpecs({
|
|
430
|
+
model,
|
|
431
|
+
size,
|
|
432
|
+
quality,
|
|
433
|
+
moderation,
|
|
434
|
+
prompt,
|
|
435
|
+
matrix: options.matrix === true,
|
|
436
|
+
});
|
|
437
|
+
const probes: ImageDoctorProbeResult[] = [];
|
|
438
|
+
for (const spec of specs) probes.push(await runSingleProbe(provider, options, spec));
|
|
439
|
+
return {
|
|
440
|
+
provider,
|
|
441
|
+
endpointKind: provider === "api" ? "api" : "oauth",
|
|
442
|
+
model,
|
|
443
|
+
size,
|
|
444
|
+
quality,
|
|
445
|
+
moderation,
|
|
446
|
+
promptId: options.prompt ? "custom" : BUILTIN_PROMPT_ID,
|
|
447
|
+
promptChars: prompt.length,
|
|
448
|
+
matrix: options.matrix === true,
|
|
449
|
+
probes,
|
|
450
|
+
summary: {
|
|
451
|
+
ok: probes.every((probe) => probe.ok),
|
|
452
|
+
passed: probes.filter((probe) => probe.ok).length,
|
|
453
|
+
failed: probes.filter((probe) => !probe.ok).length,
|
|
454
|
+
},
|
|
455
|
+
};
|
|
456
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
const RESPONSES_ERROR_MARKER = "ima2ResponsesError";
|
|
2
|
+
export const RESPONSE_DIAGNOSTIC_CODES = new Set([
|
|
3
|
+
"STREAM_PARSE_FAILED",
|
|
4
|
+
"IMAGE_TOOL_NOT_CALLED",
|
|
5
|
+
"WEB_SEARCH_ONLY_RESPONSE",
|
|
6
|
+
"IMAGE_TOOL_FAILED",
|
|
7
|
+
"IMAGE_TOOL_COMPLETED_WITHOUT_RESULT",
|
|
8
|
+
"OAUTH_IMAGE_CAPABILITY_UNAVAILABLE",
|
|
9
|
+
"RESPONSES_STREAM_ERROR",
|
|
10
|
+
]);
|
|
11
|
+
function diagnosticReason(code) {
|
|
12
|
+
return code === "EMPTY_RESPONSE" ? null : code.toLowerCase();
|
|
13
|
+
}
|
|
14
|
+
function messageForCode(code, fallback) {
|
|
15
|
+
if (code === "STREAM_PARSE_FAILED")
|
|
16
|
+
return "Responses image stream could not be parsed.";
|
|
17
|
+
if (code === "WEB_SEARCH_ONLY_RESPONSE")
|
|
18
|
+
return "Responses called web search but not the image tool.";
|
|
19
|
+
if (code === "IMAGE_TOOL_NOT_CALLED")
|
|
20
|
+
return "Responses completed without calling the image tool.";
|
|
21
|
+
if (code === "IMAGE_TOOL_FAILED")
|
|
22
|
+
return "Responses image tool call failed.";
|
|
23
|
+
if (code === "IMAGE_TOOL_COMPLETED_WITHOUT_RESULT")
|
|
24
|
+
return "Responses image tool completed without image data.";
|
|
25
|
+
return fallback;
|
|
26
|
+
}
|
|
27
|
+
export function classifyNoImageResponse(result) {
|
|
28
|
+
const diagnostics = result.diagnostics;
|
|
29
|
+
const bytesRead = Number(diagnostics.streamStats.bytesRead);
|
|
30
|
+
if (Number.isFinite(bytesRead) && bytesRead > 0 && result.eventCount === 0)
|
|
31
|
+
return "STREAM_PARSE_FAILED";
|
|
32
|
+
if (diagnostics.imageCallFailed)
|
|
33
|
+
return "IMAGE_TOOL_FAILED";
|
|
34
|
+
if (diagnostics.imageCallCompleted && diagnostics.imageResultCount === 0)
|
|
35
|
+
return "IMAGE_TOOL_COMPLETED_WITHOUT_RESULT";
|
|
36
|
+
if (!diagnostics.imageCallSeen && (result.webSearchCalls > 0 || diagnostics.webSearchCallSeen))
|
|
37
|
+
return "WEB_SEARCH_ONLY_RESPONSE";
|
|
38
|
+
if (!diagnostics.imageCallSeen && diagnostics.messageOutputSeen)
|
|
39
|
+
return "IMAGE_TOOL_NOT_CALLED";
|
|
40
|
+
return "EMPTY_RESPONSE";
|
|
41
|
+
}
|
|
42
|
+
export function emptyResponseError(message, result, meta) {
|
|
43
|
+
const code = classifyNoImageResponse(result);
|
|
44
|
+
const err = new Error(messageForCode(code, message));
|
|
45
|
+
err.status = 422;
|
|
46
|
+
err.code = code;
|
|
47
|
+
err.eventCount = result.eventCount;
|
|
48
|
+
err.eventTypes = result.eventTypes;
|
|
49
|
+
err.webSearchCalls = result.webSearchCalls;
|
|
50
|
+
err.responseDiagnostics = result.diagnostics;
|
|
51
|
+
Object.assign(err, meta);
|
|
52
|
+
const reason = diagnosticReason(code);
|
|
53
|
+
if (reason)
|
|
54
|
+
err.diagnosticReason = reason;
|
|
55
|
+
Object.defineProperty(err, RESPONSES_ERROR_MARKER, { value: true });
|
|
56
|
+
return err;
|
|
57
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { ParsedResponsesResult } from "./responsesParse.js";
|
|
2
|
+
|
|
3
|
+
const RESPONSES_ERROR_MARKER = "ima2ResponsesError";
|
|
4
|
+
|
|
5
|
+
export const RESPONSE_DIAGNOSTIC_CODES = new Set([
|
|
6
|
+
"STREAM_PARSE_FAILED",
|
|
7
|
+
"IMAGE_TOOL_NOT_CALLED",
|
|
8
|
+
"WEB_SEARCH_ONLY_RESPONSE",
|
|
9
|
+
"IMAGE_TOOL_FAILED",
|
|
10
|
+
"IMAGE_TOOL_COMPLETED_WITHOUT_RESULT",
|
|
11
|
+
"OAUTH_IMAGE_CAPABILITY_UNAVAILABLE",
|
|
12
|
+
"RESPONSES_STREAM_ERROR",
|
|
13
|
+
]);
|
|
14
|
+
|
|
15
|
+
export interface EmptyResponseMeta {
|
|
16
|
+
provider?: string;
|
|
17
|
+
model?: string;
|
|
18
|
+
toolTypes?: string[];
|
|
19
|
+
toolChoiceKind?: string;
|
|
20
|
+
quality?: string;
|
|
21
|
+
size?: string;
|
|
22
|
+
moderation?: string;
|
|
23
|
+
webSearchEnabled?: boolean;
|
|
24
|
+
refsCount?: number;
|
|
25
|
+
inputImageCount?: number;
|
|
26
|
+
promptChars?: number;
|
|
27
|
+
retryKind?: string;
|
|
28
|
+
initialEventCount?: number;
|
|
29
|
+
initialEventTypes?: Record<string, number>;
|
|
30
|
+
referencesDroppedOnRetry?: boolean;
|
|
31
|
+
developerPromptDroppedOnRetry?: boolean;
|
|
32
|
+
webSearchDroppedOnRetry?: boolean;
|
|
33
|
+
fallbackEventCount?: number;
|
|
34
|
+
fallbackEventTypes?: Record<string, number>;
|
|
35
|
+
fallbackImageCallSeen?: boolean;
|
|
36
|
+
fallbackImageResultCount?: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface ResponsesError extends Error {
|
|
40
|
+
status: number;
|
|
41
|
+
code: string;
|
|
42
|
+
[key: string]: unknown;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function diagnosticReason(code: string): string | null {
|
|
46
|
+
return code === "EMPTY_RESPONSE" ? null : code.toLowerCase();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function messageForCode(code: string, fallback: string) {
|
|
50
|
+
if (code === "STREAM_PARSE_FAILED") return "Responses image stream could not be parsed.";
|
|
51
|
+
if (code === "WEB_SEARCH_ONLY_RESPONSE") return "Responses called web search but not the image tool.";
|
|
52
|
+
if (code === "IMAGE_TOOL_NOT_CALLED") return "Responses completed without calling the image tool.";
|
|
53
|
+
if (code === "IMAGE_TOOL_FAILED") return "Responses image tool call failed.";
|
|
54
|
+
if (code === "IMAGE_TOOL_COMPLETED_WITHOUT_RESULT") return "Responses image tool completed without image data.";
|
|
55
|
+
return fallback;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function classifyNoImageResponse(result: ParsedResponsesResult): string {
|
|
59
|
+
const diagnostics = result.diagnostics;
|
|
60
|
+
const bytesRead = Number(diagnostics.streamStats.bytesRead);
|
|
61
|
+
if (Number.isFinite(bytesRead) && bytesRead > 0 && result.eventCount === 0) return "STREAM_PARSE_FAILED";
|
|
62
|
+
if (diagnostics.imageCallFailed) return "IMAGE_TOOL_FAILED";
|
|
63
|
+
if (diagnostics.imageCallCompleted && diagnostics.imageResultCount === 0) return "IMAGE_TOOL_COMPLETED_WITHOUT_RESULT";
|
|
64
|
+
if (!diagnostics.imageCallSeen && (result.webSearchCalls > 0 || diagnostics.webSearchCallSeen)) return "WEB_SEARCH_ONLY_RESPONSE";
|
|
65
|
+
if (!diagnostics.imageCallSeen && diagnostics.messageOutputSeen) return "IMAGE_TOOL_NOT_CALLED";
|
|
66
|
+
return "EMPTY_RESPONSE";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function emptyResponseError(message: string, result: ParsedResponsesResult, meta: EmptyResponseMeta): ResponsesError {
|
|
70
|
+
const code = classifyNoImageResponse(result);
|
|
71
|
+
const err = new Error(messageForCode(code, message)) as ResponsesError;
|
|
72
|
+
err.status = 422;
|
|
73
|
+
err.code = code;
|
|
74
|
+
err.eventCount = result.eventCount;
|
|
75
|
+
err.eventTypes = result.eventTypes;
|
|
76
|
+
err.webSearchCalls = result.webSearchCalls;
|
|
77
|
+
err.responseDiagnostics = result.diagnostics;
|
|
78
|
+
Object.assign(err, meta);
|
|
79
|
+
const reason = diagnosticReason(code);
|
|
80
|
+
if (reason) err.diagnosticReason = reason;
|
|
81
|
+
Object.defineProperty(err, RESPONSES_ERROR_MARKER, { value: true });
|
|
82
|
+
return err;
|
|
83
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { logEvent } from "./logger.js";
|
|
2
|
+
import { imageToolChoice, tools } from "./responsesTools.js";
|
|
3
|
+
import { emptyResponseError } from "./responsesErrors.js";
|
|
4
|
+
import { buildUserTextPrompt } from "./oauthProxy.js";
|
|
5
|
+
export async function retryPromptOnlyJsonImage({ postResponses, ctx, provider, prompt, mode, model, quality, size, moderation, requestId, signal, initial, referencesDroppedOnRetry, webSearchDroppedOnRetry, reasoningEffort, }) {
|
|
6
|
+
if (provider === "api")
|
|
7
|
+
return null;
|
|
8
|
+
const retryKind = "prompt_only_json_image_tool";
|
|
9
|
+
const retryMeta = {
|
|
10
|
+
retryKind,
|
|
11
|
+
initialEventCount: initial.eventCount,
|
|
12
|
+
initialEventTypes: initial.eventTypes,
|
|
13
|
+
referencesDroppedOnRetry,
|
|
14
|
+
developerPromptDroppedOnRetry: true,
|
|
15
|
+
webSearchDroppedOnRetry,
|
|
16
|
+
};
|
|
17
|
+
logEvent("oauth", "retry_json", { requestId, ...retryMeta });
|
|
18
|
+
let retry;
|
|
19
|
+
try {
|
|
20
|
+
retry = await postResponses({
|
|
21
|
+
ctx,
|
|
22
|
+
provider,
|
|
23
|
+
scope: "oauth-fallback",
|
|
24
|
+
requestId,
|
|
25
|
+
maxImages: 1,
|
|
26
|
+
signal,
|
|
27
|
+
payload: {
|
|
28
|
+
model,
|
|
29
|
+
input: [{ role: "user", content: buildUserTextPrompt(prompt, mode, { webSearchEnabled: false }) }],
|
|
30
|
+
tools: tools(false, { quality, size, moderation }),
|
|
31
|
+
tool_choice: imageToolChoice(true),
|
|
32
|
+
reasoning: { effort: reasoningEffort || "low" },
|
|
33
|
+
stream: false,
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
catch (e) {
|
|
38
|
+
if (e && typeof e === "object")
|
|
39
|
+
Object.assign(e, retryMeta);
|
|
40
|
+
throw e;
|
|
41
|
+
}
|
|
42
|
+
const image = retry.images[0];
|
|
43
|
+
if (image?.b64) {
|
|
44
|
+
logEvent("oauth", "retry_image", { requestId, retryKind, imageChars: image.b64.length });
|
|
45
|
+
return { b64: image.b64, usage: retry.usage, webSearchCalls: initial.webSearchCalls, revisedPrompt: image.revisedPrompt, text: retry.text, ...retryMeta };
|
|
46
|
+
}
|
|
47
|
+
logEvent("oauth", "retry_no_image", {
|
|
48
|
+
requestId,
|
|
49
|
+
retryKind,
|
|
50
|
+
fallbackEventCount: retry.eventCount,
|
|
51
|
+
fallbackImageCallSeen: retry.diagnostics.imageCallSeen,
|
|
52
|
+
fallbackImageResultCount: retry.diagnostics.imageResultCount,
|
|
53
|
+
});
|
|
54
|
+
throw emptyResponseError("No image data received from Responses API fallback", retry, {
|
|
55
|
+
provider,
|
|
56
|
+
model,
|
|
57
|
+
quality,
|
|
58
|
+
size,
|
|
59
|
+
moderation,
|
|
60
|
+
webSearchEnabled: false,
|
|
61
|
+
refsCount: 0,
|
|
62
|
+
inputImageCount: 0,
|
|
63
|
+
promptChars: typeof prompt === "string" ? prompt.length : 0,
|
|
64
|
+
toolTypes: ["image_generation"],
|
|
65
|
+
toolChoiceKind: "image_generation",
|
|
66
|
+
...retryMeta,
|
|
67
|
+
fallbackEventCount: retry.eventCount,
|
|
68
|
+
fallbackEventTypes: retry.eventTypes,
|
|
69
|
+
fallbackImageCallSeen: retry.diagnostics.imageCallSeen,
|
|
70
|
+
fallbackImageResultCount: retry.diagnostics.imageResultCount,
|
|
71
|
+
});
|
|
72
|
+
}
|