neoctl 0.2.41 → 0.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/context/prompts.js +2 -1
- package/dist/context/prompts.js.map +1 -1
- package/dist/core/image-registry.js +1 -1
- package/dist/core/image-registry.js.map +1 -1
- package/dist/core/message-pipeline.js +2 -1
- package/dist/core/message-pipeline.js.map +1 -1
- package/dist/core/query-engine.d.ts +6 -0
- package/dist/core/query-engine.js +22 -0
- package/dist/core/query-engine.js.map +1 -1
- package/dist/core/query-timing.d.ts +43 -0
- package/dist/core/query-timing.js +124 -0
- package/dist/core/query-timing.js.map +1 -0
- package/dist/core/query.d.ts +5 -0
- package/dist/core/query.js +60 -10
- package/dist/core/query.js.map +1 -1
- package/dist/core/run-agent.d.ts +2 -0
- package/dist/core/run-agent.js +4 -0
- package/dist/core/run-agent.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/model/model-gateway.d.ts +6 -0
- package/dist/model/model-gateway.js.map +1 -1
- package/dist/model/openai-chat-mapper.js +6 -0
- package/dist/model/openai-chat-mapper.js.map +1 -1
- package/dist/model/openai-mappers.js +2 -2
- package/dist/model/openai-mappers.js.map +1 -1
- package/dist/model/openai-responses-mapper.js +6 -0
- package/dist/model/openai-responses-mapper.js.map +1 -1
- package/dist/repl/index.js +13 -1
- package/dist/repl/index.js.map +1 -1
- package/dist/session/session-store.d.ts +11 -0
- package/dist/session/session-store.js +24 -0
- package/dist/session/session-store.js.map +1 -1
- package/dist/tools/builtins/image-capabilities.d.ts +22 -0
- package/dist/tools/builtins/image-capabilities.js +98 -0
- package/dist/tools/builtins/image-capabilities.js.map +1 -0
- package/dist/tools/builtins/image-generation-tool.d.ts +14 -7
- package/dist/tools/builtins/image-generation-tool.js +183 -158
- package/dist/tools/builtins/image-generation-tool.js.map +1 -1
- package/dist/tools/builtins/image-output-verification.d.ts +14 -0
- package/dist/tools/builtins/image-output-verification.js +183 -0
- package/dist/tools/builtins/image-output-verification.js.map +1 -0
- package/dist/tools/registry.d.ts +2 -0
- package/dist/tools/registry.js +6 -0
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/run-tool-use.js +5 -0
- package/dist/tools/run-tool-use.js.map +1 -1
- package/dist/tools/tool-orchestration.d.ts +5 -0
- package/dist/tools/tool-orchestration.js +1 -0
- package/dist/tools/tool-orchestration.js.map +1 -1
- package/dist/types/events.d.ts +10 -0
- package/dist/web/image-result-metadata.d.ts +20 -0
- package/dist/web/image-result-metadata.js +20 -0
- package/dist/web/image-result-metadata.js.map +1 -0
- package/dist/web/index.d.ts +18 -3
- package/dist/web/index.js +68 -17
- package/dist/web/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -5,14 +5,15 @@ import { existsSync } from "node:fs";
|
|
|
5
5
|
import fs from "node:fs/promises";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { resolveImageBlockDataSync } from "../../core/image-storage.js";
|
|
8
|
+
import { getImageRegistryFromMessages } from "../../core/message-pipeline.js";
|
|
9
|
+
import { loadImageData, resolveImageRefResult } from "../../core/image-registry.js";
|
|
8
10
|
import { getNeoctlHome } from "../../paths.js";
|
|
11
|
+
import { DEFAULT_OPENAI_IMAGE_MODEL, IMAGE_EDIT_REFERENCE_GUIDE, IMAGE_INPUT_SCHEMA, IMAGE_SELECTION_GUIDE, normalizeImageInput, imageInputWarnings, imageValidationError } from "./image-capabilities.js";
|
|
12
|
+
import { decodeImageBase64, inspectImageBytes, outputWarnings } from "./image-output-verification.js";
|
|
13
|
+
export { DEFAULT_OPENAI_IMAGE_MODEL } from "./image-capabilities.js";
|
|
9
14
|
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com";
|
|
10
|
-
export const DEFAULT_OPENAI_IMAGE_MODEL = "gpt-image-2";
|
|
11
15
|
const DEFAULT_IMAGE_MODEL = DEFAULT_OPENAI_IMAGE_MODEL;
|
|
12
16
|
export const DEFAULT_IMAGE_TIMEOUT_MS = 360_000;
|
|
13
|
-
const MAX_IMAGES = 4;
|
|
14
|
-
const SUPPORTED_IMAGE_MODELS = [DEFAULT_OPENAI_IMAGE_MODEL];
|
|
15
|
-
const SUPPORTED_MODEL_LIST = SUPPORTED_IMAGE_MODELS.join(", ");
|
|
16
17
|
/**
|
|
17
18
|
* OpenAI-only image generation tool backed by the Images API.
|
|
18
19
|
*
|
|
@@ -21,32 +22,25 @@ const SUPPORTED_MODEL_LIST = SUPPORTED_IMAGE_MODELS.join(", ");
|
|
|
21
22
|
* model that image generation is unavailable in that configuration.
|
|
22
23
|
*/
|
|
23
24
|
export function createOpenAIImageGenerationTool(options = {}) {
|
|
25
|
+
const normalize = (value) => {
|
|
26
|
+
const input = normalizeImageInput(value, options.model?.trim() || process.env.OPENAI_IMAGE_MODEL?.trim());
|
|
27
|
+
const slug = semanticNameSlug(input.semanticName);
|
|
28
|
+
if (!slug)
|
|
29
|
+
throw new Error(imageValidationError(input.model, "semanticName", "must be a meaningful non-generic Chinese or English name"));
|
|
30
|
+
for (const [index, source] of [input.image, ...(input.images ?? [])].filter((v) => v !== undefined).entries()) {
|
|
31
|
+
try {
|
|
32
|
+
resolveInputImage(source, index);
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
throw new Error(imageValidationError(input.model, `images[${index}]`, error instanceof Error ? error.message : String(error)));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return { ...input, semanticName: slug };
|
|
39
|
+
};
|
|
24
40
|
return {
|
|
25
41
|
name: "image_create",
|
|
26
|
-
description: `Generate or edit images with OpenAI's Images API. Stable tool name: image_create.
|
|
27
|
-
inputSchema: {
|
|
28
|
-
type: "object",
|
|
29
|
-
properties: {
|
|
30
|
-
mode: { type: "string", enum: ["generate", "edit"], description: "Operation mode. generate creates a new image; edit modifies existing image(s). Defaults to generate." },
|
|
31
|
-
semanticName: { type: "string", description: "Required short semantic image name used as the conversation label and default output filename. Chinese and English names are supported, e.g. 下班后30分钟重启仪式, realistic-blue-kitten, hero-banner. Must describe the image meaning; do not use generic names like image, 图片, output, result, gen, or img." },
|
|
32
|
-
prompt: { type: "string", description: "Detailed image prompt/instruction. For edit mode, describe exactly how to modify the source image(s)." },
|
|
33
|
-
model: { type: "string", enum: [...SUPPORTED_IMAGE_MODELS], description: `Optional OpenAI Images API model. Defaults to OPENAI_IMAGE_MODEL or ${DEFAULT_IMAGE_MODEL}. Supported by this tool: ${SUPPORTED_MODEL_LIST}.` },
|
|
34
|
-
size: { type: "string", enum: ["auto", "1024x1024", "1536x1024", "1024x1536"], description: `${DEFAULT_IMAGE_MODEL} output image size. Supported values: auto, 1024x1024, 1536x1024, 1024x1536. Defaults to auto.` },
|
|
35
|
-
quality: { type: "string", enum: ["auto", "low", "medium", "high"], description: `${DEFAULT_IMAGE_MODEL} rendering quality. Supported values: auto, low, medium, high. Defaults to auto.` },
|
|
36
|
-
outputFormat: { type: "string", enum: ["png", "jpeg", "webp"], description: `${DEFAULT_IMAGE_MODEL} returned image format. Supported values: png, jpeg, webp. Defaults to png.` },
|
|
37
|
-
background: { type: "string", enum: ["auto", "transparent", "opaque"], description: `${DEFAULT_IMAGE_MODEL} background handling. Supported values: auto, transparent, opaque. Defaults to auto.` },
|
|
38
|
-
moderation: { type: "string", enum: ["auto", "low"], description: `${DEFAULT_IMAGE_MODEL} moderation setting. Supported values: auto, low. Defaults to auto.` },
|
|
39
|
-
n: { type: "integer", description: `${DEFAULT_IMAGE_MODEL} number of output images, 1-${MAX_IMAGES}. Defaults to 1.` },
|
|
40
|
-
image: { type: "object", description: "Single source image for mode=edit. Provide base64/data/dataUrl plus mimeType when not using a dataUrl.", additionalProperties: true },
|
|
41
|
-
images: { type: "array", description: "Multiple source images for mode=edit.", items: { type: "object", additionalProperties: true } },
|
|
42
|
-
imageRefs: { type: "array", description: "Labels of prior conversation image blocks to edit, e.g. gen#1, Generated image 1, or [img#1].", items: { type: "string" } },
|
|
43
|
-
useLatestImage: { type: "boolean", description: "In edit mode, use the latest prior conversation image when no explicit source image or imageRefs are provided. Defaults to true." },
|
|
44
|
-
outputDir: { type: "string", description: "Optional directory where generated image files should be saved. Defaults to the current session/agent image directory." },
|
|
45
|
-
outputPath: { type: "string", description: "Optional target directory/file hint for a single generated image. The final filename is derived from semanticName and adjusted to avoid collisions. For n > 1, use outputDir instead." },
|
|
46
|
-
},
|
|
47
|
-
required: ["semanticName", "prompt"],
|
|
48
|
-
additionalProperties: false,
|
|
49
|
-
},
|
|
42
|
+
description: `Generate or edit images with OpenAI's Images API. Stable tool name: image_create. ${IMAGE_SELECTION_GUIDE} Use mode=generate for new images; mode=edit for changes to attached/prior images. ${IMAGE_EDIT_REFERENCE_GUIDE} Valid but imperfect results succeed with structured warnings; invalid inputs fail with model, field, and correction guidance. Available only when MODEL_PROVIDER=openai.`,
|
|
43
|
+
inputSchema: { ...IMAGE_INPUT_SCHEMA, properties: { ...IMAGE_INPUT_SCHEMA.properties, model: { ...IMAGE_INPUT_SCHEMA.properties.model, default: options.model?.trim() || process.env.OPENAI_IMAGE_MODEL?.trim() || DEFAULT_IMAGE_MODEL } } },
|
|
50
44
|
metadata: {
|
|
51
45
|
readOnly: false,
|
|
52
46
|
concurrent: true,
|
|
@@ -54,66 +48,31 @@ export function createOpenAIImageGenerationTool(options = {}) {
|
|
|
54
48
|
maxResultSizeChars: 24000,
|
|
55
49
|
},
|
|
56
50
|
validate(input) {
|
|
57
|
-
|
|
58
|
-
return
|
|
59
|
-
mode: record.mode ?? "generate",
|
|
60
|
-
semanticName: record.semanticName ?? "",
|
|
61
|
-
prompt: record.prompt ?? "",
|
|
62
|
-
model: record.model,
|
|
63
|
-
size: record.size ?? "auto",
|
|
64
|
-
quality: record.quality ?? "auto",
|
|
65
|
-
outputFormat: record.outputFormat ?? "png",
|
|
66
|
-
background: record.background ?? "auto",
|
|
67
|
-
moderation: record.moderation ?? "auto",
|
|
68
|
-
n: record.n ?? 1,
|
|
69
|
-
image: record.image,
|
|
70
|
-
images: record.images,
|
|
71
|
-
imageRefs: record.imageRefs,
|
|
72
|
-
useLatestImage: record.useLatestImage ?? true,
|
|
73
|
-
outputDir: record.outputDir,
|
|
74
|
-
outputPath: record.outputPath,
|
|
75
|
-
};
|
|
51
|
+
// Full model-aware validation belongs in validateInput so callers get a structured failure.
|
|
52
|
+
return input;
|
|
76
53
|
},
|
|
77
|
-
validateInput(input) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
return { ok: false, message:
|
|
86
|
-
|
|
87
|
-
return { ok: false, message: imageCreateValidationError(model, "model", `is not supported by image_create. Supported OpenAI Images API models: ${SUPPORTED_MODEL_LIST}`) };
|
|
88
|
-
if (!isImageSize(input.size))
|
|
89
|
-
return { ok: false, message: imageCreateValidationError(model, "size", "must be auto, 1024x1024, 1536x1024, or 1024x1536") };
|
|
90
|
-
if (!isImageQuality(input.quality))
|
|
91
|
-
return { ok: false, message: imageCreateValidationError(model, "quality", "must be auto, low, medium, or high") };
|
|
92
|
-
if (!isImageFormat(input.outputFormat))
|
|
93
|
-
return { ok: false, message: imageCreateValidationError(model, "outputFormat", "must be png, jpeg, or webp") };
|
|
94
|
-
if (!isImageBackground(input.background))
|
|
95
|
-
return { ok: false, message: imageCreateValidationError(model, "background", "must be auto, transparent, or opaque") };
|
|
96
|
-
if (!isImageModeration(input.moderation))
|
|
97
|
-
return { ok: false, message: imageCreateValidationError(model, "moderation", "must be auto or low") };
|
|
98
|
-
const count = input.n ?? 1;
|
|
99
|
-
if (!Number.isInteger(count) || count < 1 || count > MAX_IMAGES)
|
|
100
|
-
return { ok: false, message: imageCreateValidationError(model, "n", `must be between 1 and ${MAX_IMAGES}`) };
|
|
101
|
-
if (input.images !== undefined && (!Array.isArray(input.images) || input.images.length === 0))
|
|
102
|
-
return { ok: false, message: imageCreateValidationError(model, "images", "must be a non-empty array when provided") };
|
|
103
|
-
if (input.imageRefs !== undefined && (!Array.isArray(input.imageRefs) || input.imageRefs.some((ref) => !ref.trim())))
|
|
104
|
-
return { ok: false, message: imageCreateValidationError(model, "imageRefs", "must contain non-empty strings") };
|
|
105
|
-
if (input.outputDir !== undefined && !input.outputDir.trim())
|
|
106
|
-
return { ok: false, message: imageCreateValidationError(model, "outputDir", "must be a non-empty string when provided") };
|
|
107
|
-
if (input.outputPath !== undefined && !input.outputPath.trim())
|
|
108
|
-
return { ok: false, message: imageCreateValidationError(model, "outputPath", "must be a non-empty string when provided") };
|
|
109
|
-
if (input.outputPath !== undefined && count > 1)
|
|
110
|
-
return { ok: false, message: imageCreateValidationError(model, "outputPath", "can only be used when n is 1; use outputDir for multiple images") };
|
|
111
|
-
return { ok: true, value: { ...input, model, n: count, semanticName: semanticSlug } };
|
|
54
|
+
validateInput(input, context) {
|
|
55
|
+
try {
|
|
56
|
+
const normalized = normalize(input);
|
|
57
|
+
if (normalized.mode === "edit")
|
|
58
|
+
validateEditSources(normalized, context.messages);
|
|
59
|
+
return { ok: true, value: normalized };
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
return { ok: false, message: error instanceof Error ? error.message : String(error) };
|
|
63
|
+
}
|
|
112
64
|
},
|
|
113
65
|
isConcurrencySafe() {
|
|
114
66
|
return true;
|
|
115
67
|
},
|
|
116
68
|
async call(input, context, callOptions) {
|
|
69
|
+
try {
|
|
70
|
+
input = normalize(input);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
return { ok: false, output: { provider: "openai", error: error instanceof Error ? error.message : String(error) } };
|
|
74
|
+
}
|
|
75
|
+
const warnings = imageInputWarnings(input);
|
|
117
76
|
const apiKey = resolveApiKey(options.apiKey);
|
|
118
77
|
if (!apiKey) {
|
|
119
78
|
return {
|
|
@@ -124,14 +83,18 @@ export function createOpenAIImageGenerationTool(options = {}) {
|
|
|
124
83
|
},
|
|
125
84
|
};
|
|
126
85
|
}
|
|
127
|
-
const baseUrl = stripTrailingSlash(options.baseUrl?.trim() || process.env.OPENAI_IMAGE_BASE_URL?.trim() || process.env.OPENAI_BASE_URL?.trim() || DEFAULT_OPENAI_BASE_URL);
|
|
128
86
|
const model = input.model?.trim() || DEFAULT_IMAGE_MODEL;
|
|
129
87
|
const timeoutMs = options.timeoutMs ?? parsePositiveNumber(process.env.OPENAI_IMAGE_TIMEOUT_MS) ?? parsePositiveNumber(process.env.MODEL_TIMEOUT_MS) ?? DEFAULT_IMAGE_TIMEOUT_MS;
|
|
130
88
|
const mode = input.mode ?? "generate";
|
|
131
89
|
callOptions.onProgress?.({ toolName: "image_create", message: `${mode === "edit" ? "Editing" : "Generating"} image with OpenAI ${model}` });
|
|
132
90
|
const startedAt = Date.now();
|
|
133
91
|
try {
|
|
134
|
-
const
|
|
92
|
+
const baseUrl = normalizeImageBaseUrl(options.baseUrl?.trim() || process.env.OPENAI_IMAGE_BASE_URL?.trim() || process.env.OPENAI_BASE_URL?.trim() || DEFAULT_OPENAI_BASE_URL);
|
|
93
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647)
|
|
94
|
+
throw new Error("Image timeoutMs must be positive and <=2147483647");
|
|
95
|
+
if (context.abortSignal?.aborted)
|
|
96
|
+
throw new Error("Image request aborted before execution");
|
|
97
|
+
const editSources = mode === "edit" ? validateEditSources(input, context.messages) : [];
|
|
135
98
|
if (mode === "edit" && editSources.length === 0) {
|
|
136
99
|
throw new Error("image_create mode=edit requires a source image. Provide image/images/imageRefs, attach an image, or keep useLatestImage enabled with a prior image in the conversation.");
|
|
137
100
|
}
|
|
@@ -151,7 +114,14 @@ export function createOpenAIImageGenerationTool(options = {}) {
|
|
|
151
114
|
signal: context.abortSignal,
|
|
152
115
|
input: { ...input, model },
|
|
153
116
|
});
|
|
154
|
-
const
|
|
117
|
+
const extracted = extractGeneratedImages(response, warnings);
|
|
118
|
+
if (!extracted.length)
|
|
119
|
+
throw new Error(`OpenAI returned no usable image data. ${openAIErrorMessage(response) ?? "Expected data[].b64_json containing PNG/JPEG/WebP."}`);
|
|
120
|
+
warnings.push(...outputWarnings(input, response, extracted));
|
|
121
|
+
if (input.outputPath && path.extname(input.outputPath).toLowerCase() && path.extname(input.outputPath).toLowerCase() !== `.${extensionForMimeType(extracted[0].mimeType)}`) {
|
|
122
|
+
warnings.push({ code: "OUTPUT_EXTENSION_CORRECTED", field: "outputPath", message: "The filename extension follows actual image bytes, not the filename hint." });
|
|
123
|
+
}
|
|
124
|
+
const images = await persistGeneratedImages(extracted, input, context);
|
|
155
125
|
const timing = imageGenerationTiming(startedAt);
|
|
156
126
|
const output = {
|
|
157
127
|
...timing,
|
|
@@ -168,6 +138,9 @@ export function createOpenAIImageGenerationTool(options = {}) {
|
|
|
168
138
|
sourceImages: mode === "edit" ? editSources.length : undefined,
|
|
169
139
|
imageRefs: mode === "edit" ? editSources.map((source) => formatSourceImageRef(source)).filter((label) => Boolean(label)) : undefined,
|
|
170
140
|
images,
|
|
141
|
+
requested: buildOpenAIImageRequestBody({ ...input, model }),
|
|
142
|
+
actual: { model: stringFrom(response.model) ?? null, quality: stringFrom(response.quality) ?? null, background: stringFrom(response.background) ?? null, returnedImages: images.length, images: images.map(({ width, height, mimeType, hasAlphaChannel, hasTransparentPixels }) => ({ width, height, mimeType, hasAlphaChannel, hasTransparentPixels })) },
|
|
143
|
+
warnings,
|
|
171
144
|
raw: compactRawResponse(response),
|
|
172
145
|
};
|
|
173
146
|
return {
|
|
@@ -183,7 +156,9 @@ export function createOpenAIImageGenerationTool(options = {}) {
|
|
|
183
156
|
mode,
|
|
184
157
|
model,
|
|
185
158
|
prompt: input.prompt,
|
|
186
|
-
|
|
159
|
+
warnings,
|
|
160
|
+
requested: buildOpenAIImageRequestBody({ ...input, model }),
|
|
161
|
+
error: (error instanceof Error ? error.message : String(error)).split(apiKey).join("[REDACTED]"),
|
|
187
162
|
};
|
|
188
163
|
return {
|
|
189
164
|
ok: false,
|
|
@@ -204,9 +179,12 @@ async function callOpenAIImageGeneration(options) {
|
|
|
204
179
|
const controller = new AbortController();
|
|
205
180
|
const abort = () => controller.abort(options.signal?.reason);
|
|
206
181
|
const timeout = setTimeout(() => controller.abort(new Error(`Image generation request timed out after ${options.timeoutMs}ms`)), options.timeoutMs);
|
|
207
|
-
options.signal?.
|
|
182
|
+
if (options.signal?.aborted)
|
|
183
|
+
abort();
|
|
184
|
+
else
|
|
185
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
208
186
|
try {
|
|
209
|
-
const response = await fetch(`${options.baseUrl}/
|
|
187
|
+
const response = await fetch(`${options.baseUrl}/images/generations`, {
|
|
210
188
|
method: "POST",
|
|
211
189
|
headers: {
|
|
212
190
|
Accept: "application/json",
|
|
@@ -218,7 +196,7 @@ async function callOpenAIImageGeneration(options) {
|
|
|
218
196
|
});
|
|
219
197
|
const text = await response.text();
|
|
220
198
|
const body = text ? parseJsonObject(text) : {};
|
|
221
|
-
if (!response.ok) {
|
|
199
|
+
if (!response.ok || body.error) {
|
|
222
200
|
throw new Error(`OpenAI image generation HTTP ${response.status}: ${openAIErrorMessage(body) ?? text.slice(0, 1000)}`);
|
|
223
201
|
}
|
|
224
202
|
return body;
|
|
@@ -232,7 +210,10 @@ async function callOpenAIImageEdit(options) {
|
|
|
232
210
|
const controller = new AbortController();
|
|
233
211
|
const abort = () => controller.abort(options.signal?.reason);
|
|
234
212
|
const timeout = setTimeout(() => controller.abort(new Error(`Image edit request timed out after ${options.timeoutMs}ms`)), options.timeoutMs);
|
|
235
|
-
options.signal?.
|
|
213
|
+
if (options.signal?.aborted)
|
|
214
|
+
abort();
|
|
215
|
+
else
|
|
216
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
236
217
|
try {
|
|
237
218
|
const form = new FormData();
|
|
238
219
|
for (const image of options.images) {
|
|
@@ -242,7 +223,7 @@ async function callOpenAIImageEdit(options) {
|
|
|
242
223
|
if (value !== undefined)
|
|
243
224
|
form.append(key, String(value));
|
|
244
225
|
}
|
|
245
|
-
const response = await fetch(`${options.baseUrl}/
|
|
226
|
+
const response = await fetch(`${options.baseUrl}/images/edits`, {
|
|
246
227
|
method: "POST",
|
|
247
228
|
headers: {
|
|
248
229
|
Accept: "application/json",
|
|
@@ -253,7 +234,7 @@ async function callOpenAIImageEdit(options) {
|
|
|
253
234
|
});
|
|
254
235
|
const text = await response.text();
|
|
255
236
|
const body = text ? parseJsonObject(text) : {};
|
|
256
|
-
if (!response.ok) {
|
|
237
|
+
if (!response.ok || body.error) {
|
|
257
238
|
throw new Error(`OpenAI image edit HTTP ${response.status}: ${openAIErrorMessage(body) ?? text.slice(0, 1000)}`);
|
|
258
239
|
}
|
|
259
240
|
return body;
|
|
@@ -271,8 +252,8 @@ function buildOpenAIImageRequestBody(input) {
|
|
|
271
252
|
size: input.size ?? "auto",
|
|
272
253
|
quality: input.quality ?? "auto",
|
|
273
254
|
output_format: input.outputFormat ?? "png",
|
|
274
|
-
background: input.background
|
|
275
|
-
moderation: input.moderation
|
|
255
|
+
background: input.background ?? "auto",
|
|
256
|
+
moderation: input.moderation ?? "auto",
|
|
276
257
|
});
|
|
277
258
|
}
|
|
278
259
|
function imageGenerationTiming(startedAt, finishedAt = Date.now()) {
|
|
@@ -288,6 +269,24 @@ function imageGenerationTiming(startedAt, finishedAt = Date.now()) {
|
|
|
288
269
|
elapsedMs: duration,
|
|
289
270
|
};
|
|
290
271
|
}
|
|
272
|
+
function validateEditSources(input, messages) {
|
|
273
|
+
try {
|
|
274
|
+
const sources = resolveImageEditSources(input, messages);
|
|
275
|
+
if (!sources.length)
|
|
276
|
+
throw new Error("requires a source image; provide image/images/imageRefs or a prior conversation image with useLatestImage=true");
|
|
277
|
+
if (sources.length > 16)
|
|
278
|
+
throw new Error("maximum 16 edit sources");
|
|
279
|
+
for (const [index, source] of sources.entries()) {
|
|
280
|
+
const meta = inspectImageBytes(decodeImageBase64(source.base64));
|
|
281
|
+
if (meta.mimeType !== source.mimeType)
|
|
282
|
+
throw new Error(`source ${index + 1} MIME does not match image bytes (${meta.mimeType})`);
|
|
283
|
+
}
|
|
284
|
+
return sources;
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
throw new Error(imageValidationError(input.model, "edit sources", error instanceof Error ? error.message : String(error)));
|
|
288
|
+
}
|
|
289
|
+
}
|
|
291
290
|
function resolveImageEditSources(input, messages) {
|
|
292
291
|
const explicit = [input.image, ...(input.images ?? [])].filter((image) => !!image);
|
|
293
292
|
const fromInput = explicit.map((image, index) => resolveInputImage(image, index));
|
|
@@ -297,32 +296,47 @@ function resolveImageEditSources(input, messages) {
|
|
|
297
296
|
return [...fromInput, ...fromRefs, ...(latest ? [latest] : [])];
|
|
298
297
|
}
|
|
299
298
|
function resolveInputImage(image, index) {
|
|
299
|
+
if ([image.dataUrl, image.base64, image.data].filter(v => v !== undefined).length !== 1)
|
|
300
|
+
throw new Error("provide exactly one of base64, data, or dataUrl");
|
|
301
|
+
if (image.dataUrl !== undefined && !/^data:image\/(?:png|jpeg|webp);base64,/u.test(image.dataUrl))
|
|
302
|
+
throw new Error("dataUrl must be a PNG/JPEG/WebP base64 data URL");
|
|
300
303
|
const parsed = parseImageData(image.dataUrl ?? image.base64 ?? image.data);
|
|
301
304
|
const mimeType = image.mimeType?.trim() || parsed.mimeType;
|
|
302
305
|
if (!parsed.base64)
|
|
303
306
|
throw new Error(`image_create edit source image ${index + 1} is missing base64/data/dataUrl`);
|
|
304
307
|
if (!mimeType)
|
|
305
308
|
throw new Error(`image_create edit source image ${index + 1} is missing mimeType`);
|
|
309
|
+
const bytes = decodeImageBase64(parsed.base64);
|
|
310
|
+
const actual = inspectImageBytes(bytes);
|
|
311
|
+
if (mimeType !== actual.mimeType || (parsed.mimeType && parsed.mimeType !== actual.mimeType))
|
|
312
|
+
throw new Error(`mimeType must match actual bytes (${actual.mimeType})`);
|
|
306
313
|
return {
|
|
307
314
|
index,
|
|
308
|
-
base64:
|
|
315
|
+
base64: bytes.toString("base64"),
|
|
309
316
|
mimeType,
|
|
310
317
|
filename: image.name?.trim() || imageFilename(mimeType, index),
|
|
311
318
|
label: image.label,
|
|
312
319
|
};
|
|
313
320
|
}
|
|
314
321
|
function resolveReferencedImages(messages, refs) {
|
|
315
|
-
if (
|
|
322
|
+
if (refs.length === 0)
|
|
316
323
|
return [];
|
|
317
324
|
const imageBlocks = collectConversationImages(messages);
|
|
325
|
+
const registry = getImageRegistryFromMessages(messages ?? []);
|
|
318
326
|
return refs.map((ref, index) => {
|
|
319
|
-
const found = findReferencedImage(imageBlocks, ref);
|
|
327
|
+
const found = findReferencedImage(imageBlocks, registry, ref);
|
|
320
328
|
if (!found)
|
|
321
|
-
throw new Error(`image_create could not find referenced image: ${ref}. Available imageRefs: ${formatAvailableImageRefs(
|
|
329
|
+
throw new Error(`image_create could not find referenced image: ${ref}. Available imageRefs: ${formatAvailableImageRefs(registry)}`);
|
|
322
330
|
return { ...found, filename: found.filename || imageFilename(found.mimeType, index) };
|
|
323
331
|
});
|
|
324
332
|
}
|
|
325
|
-
function findReferencedImage(images, ref) {
|
|
333
|
+
function findReferencedImage(images, registry, ref) {
|
|
334
|
+
const registryResolution = resolveImageRefResult(registry, ref);
|
|
335
|
+
if (registryResolution.status === "ambiguous")
|
|
336
|
+
throw new Error(`image_create image reference is ambiguous: ${ref}. Use a registry ID such as img_1.`);
|
|
337
|
+
if (registryResolution.status === "resolved") {
|
|
338
|
+
return findCollectedImage(images, registryResolution.entry) ?? resolvedRegistryImage(registryResolution.entry);
|
|
339
|
+
}
|
|
326
340
|
const rawRef = ref.trim().toLowerCase();
|
|
327
341
|
const exactIdentity = images.filter((image) => image.imageId?.toLowerCase() === rawRef);
|
|
328
342
|
if (exactIdentity.length === 1)
|
|
@@ -336,24 +350,40 @@ function findReferencedImage(images, ref) {
|
|
|
336
350
|
if (labelMatches.length === 1)
|
|
337
351
|
return labelMatches[0];
|
|
338
352
|
if (labelMatches.length > 1)
|
|
339
|
-
throw new Error(`image_create image reference is ambiguous: ${ref}. Use
|
|
353
|
+
throw new Error(`image_create image reference is ambiguous: ${ref}. Use a registry ID such as img_1.`);
|
|
340
354
|
const numericRef = parseImageRefNumber(normalizedRef);
|
|
341
355
|
if (numericRef !== undefined)
|
|
342
356
|
return images[numericRef - 1];
|
|
343
357
|
return undefined;
|
|
344
358
|
}
|
|
359
|
+
function findCollectedImage(images, entry) {
|
|
360
|
+
if (entry.imageId)
|
|
361
|
+
return images.find((image) => image.imageId === entry.imageId);
|
|
362
|
+
return images.find((image) => image.sourceMessageId === entry.sourceMessageId && image.sourceBlockIndex === entry.sourceBlockIndex);
|
|
363
|
+
}
|
|
364
|
+
function resolvedRegistryImage(entry) {
|
|
365
|
+
const storedData = loadImageData(entry);
|
|
366
|
+
const parsed = parseImageData(storedData);
|
|
367
|
+
if (!parsed.base64)
|
|
368
|
+
return undefined;
|
|
369
|
+
return {
|
|
370
|
+
index: entry.sourceBlockIndex ?? 0,
|
|
371
|
+
imageId: entry.imageId,
|
|
372
|
+
base64: normalizeBase64ImageData(parsed.base64),
|
|
373
|
+
mimeType: entry.mimeType || parsed.mimeType || "image/png",
|
|
374
|
+
filename: imageFilename(entry.mimeType || parsed.mimeType || "image/png", entry.sourceBlockIndex ?? 0),
|
|
375
|
+
label: entry.label,
|
|
376
|
+
storagePath: entry.storagePath,
|
|
377
|
+
sourceMessageId: entry.sourceMessageId,
|
|
378
|
+
sourceBlockIndex: entry.sourceBlockIndex,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
345
381
|
function formatSourceImageRef(image) {
|
|
346
|
-
|
|
347
|
-
return image.storagePath ? `${ref} (${image.storagePath})` : ref;
|
|
382
|
+
return image.imageId || image.label?.trim() || image.filename || String(image.index + 1);
|
|
348
383
|
}
|
|
349
|
-
function formatAvailableImageRefs(
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
return images
|
|
353
|
-
.map(formatSourceImageRef)
|
|
354
|
-
.filter((ref) => Boolean(ref))
|
|
355
|
-
.slice(-10)
|
|
356
|
-
.join(", ");
|
|
384
|
+
function formatAvailableImageRefs(registry) {
|
|
385
|
+
const refs = registry.images.map((entry) => entry.id).slice(-10);
|
|
386
|
+
return refs.length > 0 ? refs.join(", ") : "none";
|
|
357
387
|
}
|
|
358
388
|
function latestConversationImage(messages) {
|
|
359
389
|
const images = collectConversationImages(messages);
|
|
@@ -364,7 +394,7 @@ function collectConversationImages(messages) {
|
|
|
364
394
|
return [];
|
|
365
395
|
const images = [];
|
|
366
396
|
for (const message of messages) {
|
|
367
|
-
for (const block of message.blocks) {
|
|
397
|
+
for (const [sourceBlockIndex, block] of message.blocks.entries()) {
|
|
368
398
|
if (block.type !== "image")
|
|
369
399
|
continue;
|
|
370
400
|
const resolvedData = resolveImageBlockDataSync(block);
|
|
@@ -381,6 +411,8 @@ function collectConversationImages(messages) {
|
|
|
381
411
|
filename: imageFilename(mimeType, images.length),
|
|
382
412
|
label: block.label,
|
|
383
413
|
storagePath: block.storage?.path,
|
|
414
|
+
sourceMessageId: message.id,
|
|
415
|
+
sourceBlockIndex,
|
|
384
416
|
});
|
|
385
417
|
}
|
|
386
418
|
}
|
|
@@ -440,11 +472,22 @@ async function persistGeneratedImages(images, input, context) {
|
|
|
440
472
|
const extension = extensionForMimeType(image.mimeType);
|
|
441
473
|
const requestedBase = images.length === 1 ? input.semanticName : `${input.semanticName}-${offset + 1}`;
|
|
442
474
|
const label = uniqueGeneratedImageLabel(requestedBase, context.messages, allocatedLabels);
|
|
443
|
-
|
|
475
|
+
let binaryPath = path.resolve(input.outputPath && images.length === 1
|
|
444
476
|
? uniqueOutputPath(dockerEnabled() ? path.posix.resolve(executionCwd(context.appState.snapshot().cwd), input.outputPath) : input.outputPath, label, extension)
|
|
445
|
-
: path.join(outputDir, `${sanitizeFilename(label)}.${extension}`));
|
|
477
|
+
: uniqueOutputPath(path.join(outputDir, `${sanitizeFilename(label)}.${extension}`), label, extension));
|
|
446
478
|
await outputFs.mkdir(path.dirname(binaryPath), { recursive: true });
|
|
447
|
-
|
|
479
|
+
// Exclusive creation also prevents collisions between concurrently running tool calls.
|
|
480
|
+
for (;;) {
|
|
481
|
+
try {
|
|
482
|
+
await outputFs.writeFile(binaryPath, Buffer.from(normalizeBase64ImageData(image.base64), "base64"), { flag: "wx" });
|
|
483
|
+
break;
|
|
484
|
+
}
|
|
485
|
+
catch (error) {
|
|
486
|
+
if (!isRecord(error) || error.code !== "EEXIST")
|
|
487
|
+
throw error;
|
|
488
|
+
binaryPath = uniqueOutputPath(binaryPath, label, extension);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
448
491
|
const storagePath = dockerEnabled()
|
|
449
492
|
? path.join(context.session?.sessionDir || getNeoctlHome(), "generated", "images", `${randomUUID()}.base64.txt`)
|
|
450
493
|
: `${binaryPath}.base64.txt`;
|
|
@@ -462,12 +505,12 @@ async function persistGeneratedImages(images, input, context) {
|
|
|
462
505
|
function resolveGeneratedImageOutputDir(input, context) {
|
|
463
506
|
if (dockerEnabled()) {
|
|
464
507
|
const cwd = executionCwd(context.appState.snapshot().cwd);
|
|
465
|
-
return input.
|
|
508
|
+
return input.outputPath?.trim() ? path.posix.dirname(path.posix.resolve(cwd, input.outputPath)) : input.outputDir?.trim() ? path.posix.resolve(cwd, input.outputDir) : path.posix.join(cwd, "generated", "images");
|
|
466
509
|
}
|
|
467
|
-
if (input.outputDir?.trim())
|
|
468
|
-
return path.resolve(input.outputDir.trim());
|
|
469
510
|
if (input.outputPath?.trim())
|
|
470
511
|
return path.dirname(path.resolve(input.outputPath.trim()));
|
|
512
|
+
if (input.outputDir?.trim())
|
|
513
|
+
return path.resolve(input.outputDir.trim());
|
|
471
514
|
if (context.session?.sessionDir)
|
|
472
515
|
return path.join(context.session.sessionDir, "generated", "images");
|
|
473
516
|
return path.join(getNeoctlHome(), "generated", context.agentId || "main", "images");
|
|
@@ -499,7 +542,7 @@ function collectExistingImageLabels(messages) {
|
|
|
499
542
|
function uniqueOutputPath(requestedPath, label, extension) {
|
|
500
543
|
const resolved = path.resolve(requestedPath.trim());
|
|
501
544
|
const directory = path.dirname(resolved);
|
|
502
|
-
const ext =
|
|
545
|
+
const ext = `.${extension}`;
|
|
503
546
|
const base = sanitizeFilename(label);
|
|
504
547
|
let candidate = path.join(directory, `${base}${ext}`);
|
|
505
548
|
let suffix = 2;
|
|
@@ -534,7 +577,7 @@ function semanticNameSlug(value) {
|
|
|
534
577
|
.replace(/^-+|-+$/gu, "")
|
|
535
578
|
.replace(/-{2,}/gu, "-");
|
|
536
579
|
const slug = [...normalized].slice(0, 80).join("");
|
|
537
|
-
if (!slug || isGenericImageName(slug))
|
|
580
|
+
if (!slug || isGenericImageName(slug) || /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/iu.test(slug))
|
|
538
581
|
return "";
|
|
539
582
|
return slug;
|
|
540
583
|
}
|
|
@@ -584,29 +627,35 @@ function compactImageGenerationOutput(output) {
|
|
|
584
627
|
function isImageGenerationToolOutput(value) {
|
|
585
628
|
return isRecord(value) && value.provider === "openai" && Array.isArray(value.images);
|
|
586
629
|
}
|
|
587
|
-
function extractGeneratedImages(response,
|
|
630
|
+
function extractGeneratedImages(response, warnings) {
|
|
588
631
|
const data = Array.isArray(response.data) ? response.data : [];
|
|
589
632
|
return data.flatMap((item, index) => {
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
633
|
+
try {
|
|
634
|
+
if (!isRecord(item))
|
|
635
|
+
throw new Error("Expected an image object");
|
|
636
|
+
const encoded = stringFrom(item.b64_json ?? item.image_base64 ?? item.base64_json ?? item.base64);
|
|
637
|
+
if (!encoded)
|
|
638
|
+
throw new Error("Missing base64 payload; URL-only responses are not supported");
|
|
639
|
+
const bytes = decodeImageBase64(encoded);
|
|
640
|
+
const metadata = inspectImageBytes(bytes);
|
|
641
|
+
const base64 = bytes.toString("base64");
|
|
642
|
+
return [{ index, ...metadata, base64, dataUrl: `data:${metadata.mimeType};base64,${base64}`, revisedPrompt: stringFrom(item.revised_prompt) }];
|
|
643
|
+
}
|
|
644
|
+
catch (error) {
|
|
645
|
+
warnings.push({ code: "INVALID_OUTPUT_IMAGE", field: `data[${index}]`, message: error instanceof Error ? error.message : String(error) });
|
|
594
646
|
return [];
|
|
595
|
-
|
|
596
|
-
return [{
|
|
597
|
-
index,
|
|
598
|
-
mimeType,
|
|
599
|
-
base64,
|
|
600
|
-
dataUrl: `data:${mimeType};base64,${base64}`,
|
|
601
|
-
revisedPrompt: stringFrom(item.revised_prompt),
|
|
602
|
-
}];
|
|
647
|
+
}
|
|
603
648
|
});
|
|
604
649
|
}
|
|
605
650
|
function resolveApiKey(configured) {
|
|
606
651
|
return configured?.trim() || process.env.OPENAI_IMAGE_API_KEY?.trim() || process.env.OPENAI_API_KEY?.trim() || undefined;
|
|
607
652
|
}
|
|
608
|
-
function
|
|
609
|
-
|
|
653
|
+
export function normalizeImageBaseUrl(value) {
|
|
654
|
+
const url = new URL(value);
|
|
655
|
+
if (!["https:", "http:"].includes(url.protocol) || url.username || url.password || url.search || url.hash)
|
|
656
|
+
throw new Error("Image base URL must be an HTTP(S) base URL without credentials, query, or fragment");
|
|
657
|
+
const base = value.replace(/\/+$/, "");
|
|
658
|
+
return base.endsWith("/v1") ? base : `${base}/v1`;
|
|
610
659
|
}
|
|
611
660
|
function parsePositiveNumber(value) {
|
|
612
661
|
if (!value)
|
|
@@ -653,28 +702,4 @@ function stringFrom(value) {
|
|
|
653
702
|
function isRecord(value) {
|
|
654
703
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
655
704
|
}
|
|
656
|
-
function isImageToolMode(value) {
|
|
657
|
-
return value === "generate" || value === "edit";
|
|
658
|
-
}
|
|
659
|
-
function isSupportedImageModel(value) {
|
|
660
|
-
return SUPPORTED_IMAGE_MODELS.includes(value);
|
|
661
|
-
}
|
|
662
|
-
function imageCreateValidationError(model, field, reason) {
|
|
663
|
-
return `image_create validation failed for model ${model}: ${field} ${reason}.`;
|
|
664
|
-
}
|
|
665
|
-
function isImageSize(value) {
|
|
666
|
-
return value === "auto" || value === "1024x1024" || value === "1536x1024" || value === "1024x1536";
|
|
667
|
-
}
|
|
668
|
-
function isImageQuality(value) {
|
|
669
|
-
return value === "auto" || value === "low" || value === "medium" || value === "high";
|
|
670
|
-
}
|
|
671
|
-
function isImageFormat(value) {
|
|
672
|
-
return value === "png" || value === "jpeg" || value === "webp";
|
|
673
|
-
}
|
|
674
|
-
function isImageBackground(value) {
|
|
675
|
-
return value === "auto" || value === "transparent" || value === "opaque";
|
|
676
|
-
}
|
|
677
|
-
function isImageModeration(value) {
|
|
678
|
-
return value === "auto" || value === "low";
|
|
679
|
-
}
|
|
680
705
|
//# sourceMappingURL=image-generation-tool.js.map
|