ima2-gen 3.13.1 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/CHANGELOG.md +14 -0
- package/README.md +15 -2
- package/bin/commands/capabilities.js +5 -3
- package/bin/commands/config.js +15 -4
- package/bin/commands/defaults.js +7 -5
- package/bin/commands/doctor.js +88 -39
- package/bin/commands/gen.js +21 -4
- package/bin/commands/models.js +2 -2
- package/bin/commands/ping.js +1 -1
- package/bin/commands/prompt.js +5 -3
- package/bin/commands/service.js +32 -4
- package/bin/commands/tools.js +19 -3
- package/bin/commands/upscale.js +18 -4
- package/bin/commands/video.js +37 -50
- package/bin/ima2.js +28 -9
- package/bin/lib/characterResolve.js +2 -1
- package/bin/lib/client.js +155 -20
- package/bin/lib/config-store.js +40 -7
- package/bin/lib/doctor-bundle.js +6 -2
- package/bin/lib/doctor-checks.js +13 -29
- package/bin/lib/doctor-media.js +3 -3
- package/bin/lib/doctor-providers.js +48 -35
- package/bin/lib/doctor-report.js +78 -0
- package/bin/lib/doctor-runtime.js +173 -0
- package/bin/lib/mcpJob.js +19 -7
- package/bin/lib/output.js +3 -0
- package/bin/lib/platform.js +12 -9
- package/bin/lib/recover-output.js +9 -2
- package/bin/lib/sse.js +9 -1
- package/bin/lib/videoMcp.js +16 -8
- package/config.js +41 -5
- package/docs/API.md +154 -16
- package/docs/CLI.md +65 -1
- package/docs/DOCKER.md +19 -4
- package/docs/DOCKER.zh-CN.md +11 -4
- package/docs/DOCKER.zh-TW.md +11 -4
- package/docs/FAQ.md +13 -0
- package/docs/NPX_QUICKSTART.md +6 -0
- package/docs/README.ja.md +15 -2
- package/docs/README.ko.md +15 -2
- package/docs/README.zh-CN.md +15 -2
- package/docs/README.zh-TW.md +15 -2
- package/docs/migration/runtime-test-inventory.md +79 -8
- package/lib/agyArtifact.js +82 -0
- package/lib/agyArtifactRead.js +249 -0
- package/lib/agyImageAdapter.js +2 -329
- package/lib/agyProcess.js +122 -0
- package/lib/apiRequestBudget.js +48 -0
- package/lib/assetLifecycle.js +42 -9
- package/lib/canvasVersionStore.js +46 -14
- package/lib/capabilities.js +7 -1
- package/lib/configKeys.js +2 -0
- package/lib/errors/providerMap.js +26 -0
- package/lib/eventBus.js +10 -6
- package/lib/eventsPolicy.js +9 -0
- package/lib/geminiApiImageAdapter.js +1 -214
- package/lib/generatePipeline.js +39 -165
- package/lib/generatedMediaAccess.js +74 -0
- package/lib/generationErrors.js +3 -22
- package/lib/grokImageAdapter.js +2 -353
- package/lib/grokImageCore.js +1 -57
- package/lib/grokImageDownload.js +159 -0
- package/lib/grokImageDownloadPolicy.js +124 -0
- package/lib/grokImagePlanner.js +301 -0
- package/lib/grokMultimodeAdapter.js +1 -75
- package/lib/grokVideoDownload.js +137 -34
- package/lib/inflight.js +112 -112
- package/lib/jobs/terminalStore.js +45 -0
- package/lib/lanSessionStore.js +131 -0
- package/lib/localAccessPolicy.js +180 -0
- package/lib/localLanAccess.js +281 -0
- package/lib/logger.js +22 -13
- package/lib/mcp/downloadMediaResult.js +72 -78
- package/lib/multimodePipeline.js +39 -126
- package/lib/nodeGeneration.js +44 -101
- package/lib/nodeStore.js +25 -11
- package/lib/nodeValidation.js +5 -1
- package/lib/pinnedHttpGet.js +174 -0
- package/lib/promptImport/githubFolder.js +32 -18
- package/lib/promptImport/githubSource.js +14 -10
- package/lib/promptImport/parsePromptCandidates.js +32 -2
- package/lib/providers/adapters/agyOperations.js +179 -0
- package/lib/providers/adapters/comfy.js +1 -0
- package/lib/providers/adapters/geminiOperations.js +214 -0
- package/lib/providers/adapters/googleExecution.js +97 -0
- package/lib/providers/adapters/grokExecution.js +137 -0
- package/lib/providers/adapters/grokMultimodeOperations.js +83 -0
- package/lib/providers/adapters/grokOperations.js +63 -0
- package/lib/providers/adapters/openaiExecution.js +118 -0
- package/lib/providers/adapters/openaiOperations.js +201 -0
- package/lib/providers/adapters/openaiTypes.js +1 -0
- package/lib/providers/derive.js +4 -0
- package/lib/providers/deriveCore.js +2 -7
- package/lib/providers/execution/admission.js +25 -0
- package/lib/providers/execution/index.js +34 -0
- package/lib/providers/execution/legacy.js +17 -0
- package/lib/providers/execution/legacyClassic.js +66 -0
- package/lib/providers/execution/legacyEdit.js +30 -0
- package/lib/providers/execution/legacyMultimode.js +34 -0
- package/lib/providers/execution/legacyNode.js +41 -0
- package/lib/providers/execution/types.js +1 -0
- package/lib/providers/registry.js +19 -14
- package/lib/providers/surfaceSupport.js +22 -0
- package/lib/responsesImageAdapter.js +1 -389
- package/lib/responsesTransport.js +190 -0
- package/lib/spriteJobEvents.js +20 -2
- package/lib/ssePublish.js +5 -4
- package/lib/videoExtendI2vOperation.js +76 -0
- package/package.json +5 -2
- package/routes/edit.js +31 -123
- package/routes/events.js +140 -63
- package/routes/mcpMedia.js +10 -2
- package/routes/models.js +28 -25
- package/routes/videoExtended.js +9 -68
- package/server.js +24 -46
- package/ui/dist/.vite/manifest.json +90 -63
- package/ui/dist/assets/{AgentWorkspace-PbBytTeV.js → AgentWorkspace-CL3MLv3l.js} +2 -2
- package/ui/dist/assets/App-B9cEM6dJ.js +9 -0
- package/ui/dist/assets/AssetGenWorkspace-swWPvXrx.js +2 -0
- package/ui/dist/assets/AssetsWorkspace-CRVeYU2F.js +1 -0
- package/ui/dist/assets/{CardNewsWorkspace-BS3jKOja.js → CardNewsWorkspace-BAOxWLKf.js} +2 -2
- package/ui/dist/assets/GenerationRequestLogPanel-DZtYZm-D.js +1 -0
- package/ui/dist/assets/HomeWorkspace-BeZ1s0Jm.js +1 -0
- package/ui/dist/assets/KeyingPanel-ZyBSXioh.js +1 -0
- package/ui/dist/assets/NodeCanvas-BvvcCErf.js +7 -0
- package/ui/dist/assets/{PromptBuilderPanel-D97A4pwB.js → PromptBuilderPanel-DBycmShA.js} +2 -2
- package/ui/dist/assets/PromptImportDialog-ChvMv7ek.js +2 -0
- package/ui/dist/assets/PromptImportDiscoverySection-60py7whH.js +1 -0
- package/ui/dist/assets/PromptImportFolderSection-CCRh_-_7.js +1 -0
- package/ui/dist/assets/PromptLibraryPanel-BFWQO8OT.js +2 -0
- package/ui/dist/assets/SettingsWorkspace-B5Ghsgdj.js +1 -0
- package/ui/dist/assets/SpriteRecipeWorkspace-DRWdkmwU.js +1 -0
- package/ui/dist/assets/index-7ngEOVxt.js +5 -0
- package/ui/dist/assets/index-B5jy7ePg.css +1 -0
- package/ui/dist/assets/index-gxh_02DQ.js +22 -0
- package/ui/dist/assets/{pptxgen.es-D4pedbwM.js → pptxgen.es-OUNhQLEV.js} +1 -1
- package/ui/dist/assets/promptBuilderStore-BTUUzbBT.js +1 -0
- package/ui/dist/assets/useAgentDialogFocus-DSc2qY8p.js +1 -0
- package/ui/dist/index.html +3 -2
- package/ui/dist/assets/AssetGenWorkspace-B-TGlvVb.js +0 -2
- package/ui/dist/assets/AssetsWorkspace-LnGaW151.js +0 -1
- package/ui/dist/assets/GenerationRequestLogPanel-DuDWD_ic.js +0 -1
- package/ui/dist/assets/HomeWorkspace-DVP7UG6O.js +0 -1
- package/ui/dist/assets/KeyingPanel-5bRUFQaq.js +0 -1
- package/ui/dist/assets/NodeCanvas-BBEpERTb.js +0 -7
- package/ui/dist/assets/PromptImportDialog-C89MN5JY.js +0 -2
- package/ui/dist/assets/PromptImportDiscoverySection-BgRfDOs2.js +0 -1
- package/ui/dist/assets/PromptImportFolderSection-zTMyriT1.js +0 -1
- package/ui/dist/assets/PromptLibraryPanel-BD96qZu_.js +0 -2
- package/ui/dist/assets/SettingsWorkspace-BAOk3X_8.js +0 -1
- package/ui/dist/assets/SpriteRecipeWorkspace-ClIM5QRo.js +0 -1
- package/ui/dist/assets/index-0fpfY8vu.css +0 -1
- package/ui/dist/assets/index-BFciPGTn.js +0 -5
- package/ui/dist/assets/index-BbykeXR9.js +0 -30
- package/ui/dist/assets/promptBuilderStore-Do7M8g7J.js +0 -1
- package/ui/dist/assets/useAgentDialogFocus-B6-QgEVI.js +0 -1
|
@@ -1,214 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { detectImageMimeFromB64 } from "./refs.js";
|
|
3
|
-
import { getVertexAccessToken, getVertexProjectId, isVertexInitialized } from "./vertexAuth.js";
|
|
4
|
-
import { deriveReferenceLimit } from "./providers/derive.js";
|
|
5
|
-
const MODEL_ID_MAP = {
|
|
6
|
-
"nano-banana-2": "gemini-3.1-flash-image",
|
|
7
|
-
"nano-banana-pro": "gemini-3-pro-image",
|
|
8
|
-
};
|
|
9
|
-
const GEMINI_TIMEOUT_MS = 120_000;
|
|
10
|
-
// Public v1beta ImageResponseFormat uses enums (ASPECT_RATIO_*/IMAGE_SIZE_*),
|
|
11
|
-
// while Vertex imageConfig takes plain strings ("1:1"/"1K") — mixing them got
|
|
12
|
-
// every public-API request rejected with invalid aspect_ratio (070 QA).
|
|
13
|
-
const V1BETA_ASPECT_ENUM = {
|
|
14
|
-
"1:1": "ASPECT_RATIO_ONE_BY_ONE", "2:3": "ASPECT_RATIO_TWO_BY_THREE",
|
|
15
|
-
"3:2": "ASPECT_RATIO_THREE_BY_TWO", "3:4": "ASPECT_RATIO_THREE_BY_FOUR",
|
|
16
|
-
"4:3": "ASPECT_RATIO_FOUR_BY_THREE", "4:5": "ASPECT_RATIO_FOUR_BY_FIVE",
|
|
17
|
-
"5:4": "ASPECT_RATIO_FIVE_BY_FOUR", "9:16": "ASPECT_RATIO_NINE_BY_SIXTEEN",
|
|
18
|
-
"16:9": "ASPECT_RATIO_SIXTEEN_BY_NINE", "21:9": "ASPECT_RATIO_TWENTY_ONE_BY_NINE",
|
|
19
|
-
"1:8": "ASPECT_RATIO_ONE_BY_EIGHT", "8:1": "ASPECT_RATIO_EIGHT_BY_ONE",
|
|
20
|
-
"1:4": "ASPECT_RATIO_ONE_BY_FOUR", "4:1": "ASPECT_RATIO_FOUR_BY_ONE",
|
|
21
|
-
};
|
|
22
|
-
const V1BETA_SIZE_ENUM = {
|
|
23
|
-
"512": "IMAGE_SIZE_FIVE_TWELVE",
|
|
24
|
-
"1K": "IMAGE_SIZE_ONE_K",
|
|
25
|
-
"2K": "IMAGE_SIZE_TWO_K",
|
|
26
|
-
"4K": "IMAGE_SIZE_FOUR_K",
|
|
27
|
-
};
|
|
28
|
-
function toV1BetaImageFormat(params) {
|
|
29
|
-
const aspect = V1BETA_ASPECT_ENUM[params.aspectRatio];
|
|
30
|
-
const size = V1BETA_SIZE_ENUM[params.imageSize];
|
|
31
|
-
if (!aspect)
|
|
32
|
-
throw new Error(`gemini-api: no v1beta aspect enum for ${params.aspectRatio}`);
|
|
33
|
-
if (!size)
|
|
34
|
-
throw new Error(`gemini-api: no v1beta image-size enum for ${params.imageSize}`);
|
|
35
|
-
return { aspect_ratio: aspect, image_size: size };
|
|
36
|
-
}
|
|
37
|
-
function parseGeminiImageParams(size) {
|
|
38
|
-
if (!size || size === "auto" || size === "1024x1024")
|
|
39
|
-
return { aspectRatio: "1:1", imageSize: "1K" };
|
|
40
|
-
const match = size.match(/^(\d+)x(\d+)$/);
|
|
41
|
-
if (!match)
|
|
42
|
-
return { aspectRatio: "1:1", imageSize: "1K" };
|
|
43
|
-
const w = Number(match[1]);
|
|
44
|
-
const h = Number(match[2]);
|
|
45
|
-
const ratio = w / h;
|
|
46
|
-
const ratioMap = [
|
|
47
|
-
["1:1", 1], ["2:3", 2 / 3], ["3:2", 3 / 2], ["3:4", 3 / 4], ["4:3", 4 / 3],
|
|
48
|
-
["4:5", 4 / 5], ["5:4", 5 / 4], ["9:16", 9 / 16], ["16:9", 16 / 9], ["21:9", 21 / 9],
|
|
49
|
-
["1:8", 1 / 8], ["8:1", 8], ["1:4", 1 / 4], ["4:1", 4],
|
|
50
|
-
];
|
|
51
|
-
let bestLabel = "1:1";
|
|
52
|
-
let bestDist = Infinity;
|
|
53
|
-
for (const [label, val] of ratioMap) {
|
|
54
|
-
const dist = Math.abs(ratio - val);
|
|
55
|
-
if (dist < bestDist) {
|
|
56
|
-
bestDist = dist;
|
|
57
|
-
bestLabel = label;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
const maxDim = Math.max(w, h);
|
|
61
|
-
const imageSize = maxDim <= 512 ? "512" : maxDim <= 1024 ? "1K" : maxDim <= 2048 ? "2K" : "4K";
|
|
62
|
-
return { aspectRatio: bestLabel, imageSize };
|
|
63
|
-
}
|
|
64
|
-
function geminiApiError(message, status, code) {
|
|
65
|
-
const err = new Error(message);
|
|
66
|
-
err.status = status;
|
|
67
|
-
err.code = code;
|
|
68
|
-
return err;
|
|
69
|
-
}
|
|
70
|
-
function resolveGeminiModelId(model) {
|
|
71
|
-
return MODEL_ID_MAP[model] || model;
|
|
72
|
-
}
|
|
73
|
-
function buildContents(prompt, references) {
|
|
74
|
-
const parts = [];
|
|
75
|
-
// Add reference images first (if any)
|
|
76
|
-
for (const ref of references.slice(0, deriveReferenceLimit("gemini-api", "edit"))) {
|
|
77
|
-
const mime = ref.declaredMime || ref.detectedMime || detectImageMimeFromB64(ref.b64) || "image/png";
|
|
78
|
-
parts.push({
|
|
79
|
-
inlineData: {
|
|
80
|
-
mimeType: mime,
|
|
81
|
-
data: ref.b64,
|
|
82
|
-
},
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
// Add text prompt
|
|
86
|
-
parts.push({ text: prompt });
|
|
87
|
-
return [{ role: "user", parts }];
|
|
88
|
-
}
|
|
89
|
-
export async function generateViaGeminiApi(prompt, ctx, options = {}) {
|
|
90
|
-
const apiKey = ctx.geminiApiKey;
|
|
91
|
-
const vertexReady = ctx.hasVertexKey && isVertexInitialized();
|
|
92
|
-
const authMode = ctx.geminiAuthMode;
|
|
93
|
-
const useVertex = authMode === "vertex" ? vertexReady : (!apiKey && vertexReady);
|
|
94
|
-
if (!apiKey && !useVertex) {
|
|
95
|
-
throw geminiApiError("Gemini API key or Vertex AI credentials not configured", 401, "GEMINI_API_KEY_MISSING");
|
|
96
|
-
}
|
|
97
|
-
const model = options.model || "nano-banana-2";
|
|
98
|
-
const apiModelId = resolveGeminiModelId(model);
|
|
99
|
-
const references = (options.references || []).slice(0, deriveReferenceLimit("gemini-api", "edit"));
|
|
100
|
-
let url;
|
|
101
|
-
let authHeaders;
|
|
102
|
-
if (useVertex) {
|
|
103
|
-
const token = await getVertexAccessToken();
|
|
104
|
-
const projectId = getVertexProjectId();
|
|
105
|
-
url = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/global/publishers/google/models/${apiModelId}:generateContent`;
|
|
106
|
-
authHeaders = { "Content-Type": "application/json", "Authorization": `Bearer ${token}` };
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
url = `https://generativelanguage.googleapis.com/v1beta/models/${apiModelId}:generateContent`;
|
|
110
|
-
authHeaders = { "Content-Type": "application/json", "x-goog-api-key": apiKey };
|
|
111
|
-
}
|
|
112
|
-
// size "auto" → omit image config entirely so the model decides ratio/size
|
|
113
|
-
const isAutoSize = !options.size || options.size === "auto";
|
|
114
|
-
const imageParams = isAutoSize ? null : parseGeminiImageParams(options.size);
|
|
115
|
-
// Vertex AI rejects responseFormat; it expects imageConfig directly under
|
|
116
|
-
// generationConfig (camelCase). Public API uses response_format.image (snake_case).
|
|
117
|
-
const generationConfig = useVertex
|
|
118
|
-
? {
|
|
119
|
-
responseModalities: ["TEXT", "IMAGE"],
|
|
120
|
-
...(imageParams
|
|
121
|
-
? { imageConfig: { aspectRatio: imageParams.aspectRatio, imageSize: imageParams.imageSize } }
|
|
122
|
-
: {}),
|
|
123
|
-
}
|
|
124
|
-
: {
|
|
125
|
-
response_modalities: ["TEXT", "IMAGE"],
|
|
126
|
-
...(imageParams
|
|
127
|
-
? { response_format: { image: toV1BetaImageFormat(imageParams) } }
|
|
128
|
-
: {}),
|
|
129
|
-
};
|
|
130
|
-
const configKey = useVertex ? "generationConfig" : "generation_config";
|
|
131
|
-
const body = { contents: buildContents(prompt, references), [configKey]: generationConfig };
|
|
132
|
-
logEvent("gemini-api", "generate:start", {
|
|
133
|
-
requestId: options.requestId,
|
|
134
|
-
model,
|
|
135
|
-
apiModelId,
|
|
136
|
-
promptChars: prompt.length,
|
|
137
|
-
refs: references.length,
|
|
138
|
-
});
|
|
139
|
-
const timeoutSignal = AbortSignal.timeout(GEMINI_TIMEOUT_MS);
|
|
140
|
-
const combinedSignal = options.signal
|
|
141
|
-
? AbortSignal.any([options.signal, timeoutSignal])
|
|
142
|
-
: timeoutSignal;
|
|
143
|
-
try {
|
|
144
|
-
const res = await fetch(url, {
|
|
145
|
-
method: "POST",
|
|
146
|
-
headers: authHeaders,
|
|
147
|
-
body: JSON.stringify(body),
|
|
148
|
-
signal: combinedSignal,
|
|
149
|
-
});
|
|
150
|
-
if (!res.ok) {
|
|
151
|
-
const text = await res.text().catch(() => "");
|
|
152
|
-
if (res.status === 429) {
|
|
153
|
-
throw geminiApiError(`Gemini API rate limited: ${text.slice(0, 200)}`, 429, "GEMINI_API_RATE_LIMITED");
|
|
154
|
-
}
|
|
155
|
-
if (res.status === 400 || res.status === 403) {
|
|
156
|
-
throw geminiApiError(`Gemini API error: ${text.slice(0, 200)}`, res.status, "GEMINI_API_BAD_REQUEST");
|
|
157
|
-
}
|
|
158
|
-
throw geminiApiError(`Gemini API error (${res.status}): ${text.slice(0, 200)}`, 502, "GEMINI_API_UPSTREAM_ERROR");
|
|
159
|
-
}
|
|
160
|
-
const json = await res.json();
|
|
161
|
-
// Extract image from candidates[0].content.parts[]
|
|
162
|
-
const parts = json?.candidates?.[0]?.content?.parts || [];
|
|
163
|
-
let b64 = null;
|
|
164
|
-
let textResponse = "";
|
|
165
|
-
let mime = "image/png";
|
|
166
|
-
for (const part of parts) {
|
|
167
|
-
if (part.inlineData?.data) {
|
|
168
|
-
b64 = part.inlineData.data;
|
|
169
|
-
mime = part.inlineData.mimeType || "image/png";
|
|
170
|
-
}
|
|
171
|
-
if (part.text) {
|
|
172
|
-
textResponse += part.text;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
if (!b64) {
|
|
176
|
-
// Check for safety block
|
|
177
|
-
const finishReason = json?.candidates?.[0]?.finishReason;
|
|
178
|
-
if (finishReason === "SAFETY") {
|
|
179
|
-
throw geminiApiError("Gemini API: generation blocked by safety filter", 400, "GEMINI_API_SAFETY_BLOCKED");
|
|
180
|
-
}
|
|
181
|
-
throw geminiApiError(`Gemini API: no image in response (finishReason: ${finishReason || "unknown"})`, 502, "GEMINI_API_NO_IMAGE");
|
|
182
|
-
}
|
|
183
|
-
const usageMetadata = json?.usageMetadata || {};
|
|
184
|
-
logEvent("gemini-api", "generate:done", {
|
|
185
|
-
requestId: options.requestId,
|
|
186
|
-
model,
|
|
187
|
-
b64Len: b64.length,
|
|
188
|
-
mime,
|
|
189
|
-
textResponseLen: textResponse.length,
|
|
190
|
-
});
|
|
191
|
-
return {
|
|
192
|
-
b64,
|
|
193
|
-
revisedPrompt: textResponse || prompt,
|
|
194
|
-
usage: {
|
|
195
|
-
promptTokens: usageMetadata.promptTokenCount || 0,
|
|
196
|
-
candidatesTokens: usageMetadata.candidatesTokenCount || 0,
|
|
197
|
-
totalTokens: usageMetadata.totalTokenCount || 0,
|
|
198
|
-
},
|
|
199
|
-
webSearchCalls: 0,
|
|
200
|
-
mime,
|
|
201
|
-
};
|
|
202
|
-
}
|
|
203
|
-
catch (e) {
|
|
204
|
-
if (e.name === "AbortError") {
|
|
205
|
-
if (options.signal?.aborted) {
|
|
206
|
-
throw geminiApiError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
207
|
-
}
|
|
208
|
-
throw geminiApiError("Gemini API generation timed out", 504, "GENERATION_TIMEOUT");
|
|
209
|
-
}
|
|
210
|
-
if (e.code && e.status)
|
|
211
|
-
throw e;
|
|
212
|
-
throw geminiApiError(`Gemini API request failed: ${e.message}`, 502, "GEMINI_API_NETWORK_FAILED");
|
|
213
|
-
}
|
|
214
|
-
}
|
|
1
|
+
export { generateViaGeminiApi } from "./providers/adapters/geminiOperations.js";
|
package/lib/generatePipeline.js
CHANGED
|
@@ -9,17 +9,10 @@ import { classifyUpstreamError } from "./errorClassify.js";
|
|
|
9
9
|
import { appendGenerationRequestLog } from "./generationRequestLog.js";
|
|
10
10
|
import { normalizeOAuthParams } from "./oauthNormalize.js";
|
|
11
11
|
import { resolveProviderOptions } from "./providerOptions.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { prepareImageExecution } from "./providers/execution/index.js";
|
|
13
|
+
import { checkImageExecutionAdmission } from "./providers/execution/admission.js";
|
|
14
14
|
import { resolveGrokQualityModel } from "./imageModels.js";
|
|
15
|
-
import { generateViaAgy } from "./agyImageAdapter.js";
|
|
16
|
-
import { generateViaGeminiApi } from "./geminiApiImageAdapter.js";
|
|
17
|
-
import { generateViaAtlasCloud } from "./atlasCloudImageAdapter.js";
|
|
18
|
-
import { generateViaMinimax } from "./minimaxImageAdapter.js";
|
|
19
|
-
import { generateViaNai } from "./naiImageAdapter.js";
|
|
20
15
|
import { composerNegativePromptMeta, readNaiOptions } from "./naiOptions.js";
|
|
21
|
-
import { generateViaComfy } from "./comfyImageAdapter.js";
|
|
22
|
-
import { isNonRetryableGenerationError, normalizeGenerationFailure } from "./generationErrors.js";
|
|
23
16
|
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, setJobPhase, INFLIGHT_RETRY_AFTER_SECONDS, } from "./inflight.js";
|
|
24
17
|
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "./generationCancel.js";
|
|
25
18
|
import { logEvent, logError } from "./logger.js";
|
|
@@ -27,7 +20,6 @@ import { embedImageMetadataBestEffort } from "./imageMetadataStore.js";
|
|
|
27
20
|
import { invalidateHistoryIndex } from "./historyIndex.js";
|
|
28
21
|
import { normalizeComposerInsertedPrompts, normalizeComposerPrompt, } from "./composerSnapshot.js";
|
|
29
22
|
import { errInfo } from "./errInfo.js";
|
|
30
|
-
import { requireRuntimeContext } from "./runtimeContext.js";
|
|
31
23
|
import { STORYBOARD_PREFIX } from "./storyboardPrefix.js";
|
|
32
24
|
import { parseBackgroundPreset, backgroundPromptSuffix, backgroundPlannerConstraint } from "./backgroundPresets.js";
|
|
33
25
|
import { sizeNudgeSuffix } from "./sizeNudge.js";
|
|
@@ -39,7 +31,7 @@ import { publishJobEvent } from "./ssePublish.js";
|
|
|
39
31
|
import { normalizeBodyRequestId, validateBoundedCount, validateGenerationPrompt } from "./generationInputValidation.js";
|
|
40
32
|
import { getElementById } from "./assetsStore.js";
|
|
41
33
|
import { compileElements, ELEMENT_CAPACITY_DEFAULTS } from "./elementCompiler.js";
|
|
42
|
-
import { deriveReferenceLimit } from "./providers/derive.js";
|
|
34
|
+
import { deriveReferenceLimit, getProviderSurfaceSupport } from "./providers/derive.js";
|
|
43
35
|
import { claimIdempotencyKey, completeIdempotencyKey, fingerprintRequest, IdempotencyFingerprintConflict, IdempotencyKeyInvalid, readIdempotencyKey, } from "./jobs/idempotency.js";
|
|
44
36
|
export async function runGeneratePipeline(req, res, ctx) {
|
|
45
37
|
const requestId = normalizeBodyRequestId(req.body?.requestId, req.id);
|
|
@@ -160,7 +152,7 @@ export async function runGeneratePipeline(req, res, ctx) {
|
|
|
160
152
|
rawSize: size,
|
|
161
153
|
rawWebSearchEnabled,
|
|
162
154
|
});
|
|
163
|
-
if (providerOptions.error) {
|
|
155
|
+
if (providerOptions.error !== undefined) {
|
|
164
156
|
return fail(providerOptions.status, { error: providerOptions.error, code: providerOptions.code });
|
|
165
157
|
}
|
|
166
158
|
const imageModel = providerOptions.model;
|
|
@@ -273,11 +265,8 @@ export async function runGeneratePipeline(req, res, ctx) {
|
|
|
273
265
|
const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http")
|
|
274
266
|
? req.body.providerUrl
|
|
275
267
|
: null;
|
|
276
|
-
const grokRefs = incomingProviderUrl
|
|
277
|
-
? [{ b64: "", url: incomingProviderUrl, declaredMime: "image/png", detectedMime: "image/png" }, ...refCheck.refDetails]
|
|
278
|
-
: refCheck.refDetails;
|
|
279
268
|
const providerRefCount = activeProvider === "grok" || activeProvider === "grok-api"
|
|
280
|
-
?
|
|
269
|
+
? refCheck.refs.length + (incomingProviderUrl ? 1 : 0)
|
|
281
270
|
: refCheck.refs.length;
|
|
282
271
|
const providerReferenceLimit = deriveReferenceLimit(activeProvider, "edit");
|
|
283
272
|
if ((activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api") && providerRefCount > providerReferenceLimit) {
|
|
@@ -304,13 +293,19 @@ export async function runGeneratePipeline(req, res, ctx) {
|
|
|
304
293
|
// Refuse loudly rather than dropping the input: lib/naiImageAdapter.ts is
|
|
305
294
|
// text-to-image only, so a reference passed here would be ignored and the
|
|
306
295
|
// user would get an unrelated image back believing they had edited one.
|
|
307
|
-
if (activeProvider
|
|
296
|
+
if (getProviderSurfaceSupport(activeProvider ?? "", "generate")?.references === false && providerRefCount > 0) {
|
|
308
297
|
return fail(400, {
|
|
309
298
|
error: "NovelAI image generation does not accept reference images yet",
|
|
310
299
|
code: "NAI_REF_UNSUPPORTED",
|
|
311
300
|
requestId,
|
|
312
301
|
});
|
|
313
302
|
}
|
|
303
|
+
const admission = checkImageExecutionAdmission(ctx, {
|
|
304
|
+
provider: activeProvider, surface: "classic", referenceCount: providerRefCount,
|
|
305
|
+
});
|
|
306
|
+
if (admission) {
|
|
307
|
+
return fail(admission.status, { error: admission.message, code: admission.code, requestId });
|
|
308
|
+
}
|
|
314
309
|
const started = startJob({
|
|
315
310
|
requestId,
|
|
316
311
|
kind: "classic",
|
|
@@ -386,155 +381,34 @@ export async function runGeneratePipeline(req, res, ctx) {
|
|
|
386
381
|
: (providerForcesJpeg ? "jpeg" : String(format));
|
|
387
382
|
const mime = mimeMap[effectiveFormat] || "image/png";
|
|
388
383
|
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
389
|
-
const
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
requestId,
|
|
410
|
-
references: refCheck.refDetails,
|
|
411
|
-
});
|
|
412
|
-
throwIfJobCanceled(requestId);
|
|
413
|
-
return r;
|
|
414
|
-
}
|
|
415
|
-
if (activeProvider === "agy") {
|
|
416
|
-
const r = await generateViaAgy(generationPrompt, {
|
|
417
|
-
references: refCheck.refDetails,
|
|
418
|
-
signal: cancelController.signal,
|
|
419
|
-
requestId,
|
|
420
|
-
});
|
|
421
|
-
throwIfJobCanceled(requestId);
|
|
422
|
-
return r;
|
|
423
|
-
}
|
|
424
|
-
if (activeProvider === "atlascloud") {
|
|
425
|
-
const r = await generateViaAtlasCloud(generationPrompt, requireRuntimeContext(ctx), {
|
|
426
|
-
model: imageModel,
|
|
427
|
-
size: effectiveSize,
|
|
428
|
-
quality,
|
|
429
|
-
signal: cancelController.signal,
|
|
430
|
-
requestId,
|
|
431
|
-
references: refCheck.refDetails,
|
|
432
|
-
...(backgroundParams ? { background: backgroundParams.background } : {}),
|
|
433
|
-
...(backgroundParams?.outputFormat ? { outputFormat: backgroundParams.outputFormat } : {}),
|
|
434
|
-
});
|
|
435
|
-
throwIfJobCanceled(requestId);
|
|
436
|
-
return r;
|
|
437
|
-
}
|
|
438
|
-
if (activeProvider === "minimax") {
|
|
439
|
-
const r = await generateViaMinimax(generationPrompt, requireRuntimeContext(ctx), {
|
|
440
|
-
model: imageModel,
|
|
441
|
-
size: effectiveSize,
|
|
442
|
-
signal: cancelController.signal,
|
|
443
|
-
requestId,
|
|
444
|
-
references: refCheck.refDetails,
|
|
445
|
-
});
|
|
446
|
-
throwIfJobCanceled(requestId);
|
|
447
|
-
return r;
|
|
448
|
-
}
|
|
449
|
-
if (activeProvider === "nai") {
|
|
450
|
-
// No references argument: the adapter is text-to-image only, and the
|
|
451
|
-
// guard above already refused any reference input rather than letting
|
|
452
|
-
// it be silently discarded here.
|
|
453
|
-
const r = await generateViaNai(generationPrompt, requireRuntimeContext(ctx), {
|
|
454
|
-
model: imageModel,
|
|
455
|
-
size: effectiveSize,
|
|
456
|
-
signal: cancelController.signal,
|
|
457
|
-
requestId,
|
|
458
|
-
// One normalizer for every request-driven lane: the multimode and
|
|
459
|
-
// node branches spread the same call, so the three cannot drift.
|
|
460
|
-
...readNaiOptions(req.body),
|
|
461
|
-
});
|
|
462
|
-
throwIfJobCanceled(requestId);
|
|
463
|
-
return r;
|
|
464
|
-
}
|
|
465
|
-
if (activeProvider === "comfy") {
|
|
466
|
-
const r = await generateViaComfy(generationPrompt, requireRuntimeContext(ctx), {
|
|
467
|
-
model: imageModel,
|
|
468
|
-
size: effectiveSize,
|
|
469
|
-
signal: cancelController.signal,
|
|
470
|
-
requestId,
|
|
471
|
-
references: refCheck.refDetails,
|
|
472
|
-
...(typeof req.body?.seed === "number" ? { seed: req.body.seed } : {}),
|
|
473
|
-
...(req.body?.comfyParams && typeof req.body.comfyParams === "object"
|
|
474
|
-
? { params: req.body.comfyParams }
|
|
475
|
-
: {}),
|
|
476
|
-
// A local GPU queue is real user-visible waiting. Without this the
|
|
477
|
-
// UI would show "streaming" while the job sits behind three other
|
|
478
|
-
// prompts on someone's workstation.
|
|
479
|
-
onQueue: (info) => {
|
|
480
|
-
const phase = info.running ? "streaming" : "queued";
|
|
481
|
-
setJobPhase(requestId, phase);
|
|
482
|
-
if (asyncMode) {
|
|
483
|
-
publish(requestId, "phase", { requestId, phase, queuePosition: info.position });
|
|
484
|
-
}
|
|
485
|
-
},
|
|
486
|
-
});
|
|
487
|
-
throwIfJobCanceled(requestId);
|
|
488
|
-
return r;
|
|
489
|
-
}
|
|
490
|
-
if (activeProvider === "grok" || activeProvider === "grok-api") {
|
|
491
|
-
const grokModel = resolveGrokQualityModel(imageModel, quality);
|
|
492
|
-
const r = await generateViaGrok(generationPrompt, ctx, {
|
|
493
|
-
model: grokModel,
|
|
494
|
-
size: effectiveSize,
|
|
495
|
-
signal: cancelController.signal,
|
|
496
|
-
requestId,
|
|
497
|
-
plannedPrompt: sharedGrokPlan?.prompt,
|
|
498
|
-
webSearchCalls: sharedGrokPlan?.webSearchCalls,
|
|
499
|
-
references: grokRefs,
|
|
500
|
-
directApiKey: grokDirectApiKey,
|
|
501
|
-
});
|
|
502
|
-
throwIfJobCanceled(requestId);
|
|
503
|
-
return r;
|
|
504
|
-
}
|
|
505
|
-
const MAX_RETRIES = 1;
|
|
506
|
-
let lastErr;
|
|
507
|
-
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
508
|
-
try {
|
|
509
|
-
const r = await generateViaResponses(activeProvider, generationPrompt, quality, effectiveSize, moderation, refCheck.refDetails || refCheck.refs, requestId, normalizedPromptMode, ctx, {
|
|
510
|
-
model: imageModel,
|
|
511
|
-
reasoningEffort,
|
|
512
|
-
webSearchEnabled,
|
|
513
|
-
signal: cancelController.signal,
|
|
514
|
-
allowPromptOnlyOAuthFallback: activeProvider !== "api",
|
|
515
|
-
...(backgroundParams ? { background: backgroundParams.background } : {}),
|
|
516
|
-
...(backgroundParams?.outputFormat ? { outputFormat: backgroundParams.outputFormat } : {}),
|
|
517
|
-
});
|
|
518
|
-
throwIfJobCanceled(requestId);
|
|
519
|
-
if (r.b64)
|
|
520
|
-
return r;
|
|
521
|
-
lastErr = new Error("Empty response (safety refusal)");
|
|
522
|
-
}
|
|
523
|
-
catch (e) {
|
|
524
|
-
lastErr = e;
|
|
525
|
-
if (isNonRetryableGenerationError(e))
|
|
526
|
-
break;
|
|
527
|
-
}
|
|
528
|
-
if (attempt < MAX_RETRIES) {
|
|
529
|
-
const errCode = (lastErr && typeof lastErr === "object" && "code" in lastErr)
|
|
530
|
-
? lastErr.code
|
|
531
|
-
: undefined;
|
|
532
|
-
logEvent("generate", "retry", { requestId, attempt: attempt + 1, errorCode: errCode });
|
|
384
|
+
const execution = await prepareImageExecution(ctx, {
|
|
385
|
+
surface: "classic", provider: activeProvider, requestId,
|
|
386
|
+
signal: cancelController.signal, prompt: generationPrompt, rawPrompt: prompt,
|
|
387
|
+
references: refCheck.refDetails, providerUrl: incomingProviderUrl,
|
|
388
|
+
options: { model: imageModel, quality, size: effectiveSize, moderation,
|
|
389
|
+
mode: normalizedPromptMode, reasoningEffort, webSearchEnabled },
|
|
390
|
+
background: backgroundParams,
|
|
391
|
+
backgroundConstraint: backgroundPreset ? backgroundPlannerConstraint(backgroundPreset) : undefined,
|
|
392
|
+
nai: activeProvider === "nai" ? readNaiOptions(req.body) : {},
|
|
393
|
+
comfy: {
|
|
394
|
+
...(typeof req.body?.seed === "number" ? { seed: req.body.seed } : {}),
|
|
395
|
+
...(req.body?.comfyParams && typeof req.body.comfyParams === "object"
|
|
396
|
+
? { params: req.body.comfyParams }
|
|
397
|
+
: {}),
|
|
398
|
+
},
|
|
399
|
+
}, {
|
|
400
|
+
onQueue: (info) => {
|
|
401
|
+
const phase = info.running ? "streaming" : "queued";
|
|
402
|
+
setJobPhase(requestId, phase);
|
|
403
|
+
if (asyncMode) {
|
|
404
|
+
publish(requestId, "phase", { requestId, phase, queuePosition: info.position });
|
|
533
405
|
}
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
});
|
|
406
|
+
},
|
|
407
|
+
});
|
|
408
|
+
const generateOne = async () => {
|
|
409
|
+
const { value } = await execution.execute();
|
|
410
|
+
throwIfJobCanceled(requestId);
|
|
411
|
+
return value;
|
|
538
412
|
};
|
|
539
413
|
const results = await Promise.allSettled(Array.from({ length: count }, generateOne));
|
|
540
414
|
throwIfJobCanceled(requestId);
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import express from "express";
|
|
2
|
+
import { realpath } from "node:fs/promises";
|
|
3
|
+
import { resolve, sep } from "node:path";
|
|
4
|
+
import { isLocalBind } from "./localAccessPolicy.js";
|
|
5
|
+
function missing(res, sidecar = false) {
|
|
6
|
+
res.status(404).type("text/plain").end(sidecar ? "Generated metadata is not public" : "Generated media not found");
|
|
7
|
+
}
|
|
8
|
+
function contained(root, path) {
|
|
9
|
+
return path !== root && path.startsWith(`${root}${sep}`);
|
|
10
|
+
}
|
|
11
|
+
/** Mount at /generated; access.mediaHeaders must ALSO precede the global guard. */
|
|
12
|
+
export function createGeneratedMediaAccess(ctx, access) {
|
|
13
|
+
const lan = !isLocalBind(ctx.config.server.host);
|
|
14
|
+
const root = resolve(ctx.config.storage.generatedDir);
|
|
15
|
+
const inspect = async (req, res, next) => {
|
|
16
|
+
try {
|
|
17
|
+
const raw = req.url.split("?")[0] || "/";
|
|
18
|
+
if (/%(?:2f|5c)/i.test(raw))
|
|
19
|
+
return missing(res);
|
|
20
|
+
const decoded = decodeURIComponent(raw);
|
|
21
|
+
if (/[\0\\]/.test(decoded))
|
|
22
|
+
return missing(res);
|
|
23
|
+
const lower = decoded.toLowerCase();
|
|
24
|
+
if (lower.endsWith(".json"))
|
|
25
|
+
return missing(res, true);
|
|
26
|
+
const target = resolve(root, `.${decoded}`);
|
|
27
|
+
if (!contained(root, target))
|
|
28
|
+
return missing(res);
|
|
29
|
+
const [realRoot, realTarget] = await Promise.all([realpath(root), realpath(target)]);
|
|
30
|
+
if (!contained(realRoot, realTarget))
|
|
31
|
+
return missing(res);
|
|
32
|
+
if (realTarget.toLowerCase().endsWith(".json"))
|
|
33
|
+
return missing(res, true);
|
|
34
|
+
if (lower.endsWith(".svg") || realTarget.toLowerCase().endsWith(".svg")) {
|
|
35
|
+
res.setHeader("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'");
|
|
36
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
37
|
+
}
|
|
38
|
+
if (!res.writableEnded && !res.destroyed)
|
|
39
|
+
next();
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
if (!res.writableEnded && !res.destroyed)
|
|
43
|
+
missing(res);
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
const serve = express.static(root, lan ? {
|
|
47
|
+
etag: false, lastModified: false, immutable: false, cacheControl: false, redirect: false, index: false,
|
|
48
|
+
} : { maxAge: ctx.config.storage.staticMaxAge, immutable: true, redirect: false, index: false });
|
|
49
|
+
// Containment is a realpath check, not protection against a malicious local FS writer's race.
|
|
50
|
+
return [access.mediaHeaders, access.guard, inspect, (req, res) => {
|
|
51
|
+
if (lan) {
|
|
52
|
+
delete req.headers["if-none-match"];
|
|
53
|
+
delete req.headers["if-modified-since"];
|
|
54
|
+
}
|
|
55
|
+
serve(req, res, error => {
|
|
56
|
+
if (res.writableEnded || res.destroyed)
|
|
57
|
+
return;
|
|
58
|
+
if (error) {
|
|
59
|
+
if (res.headersSent) {
|
|
60
|
+
res.end();
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const status = error.statusCode;
|
|
64
|
+
if (status === 416 || status === 412) {
|
|
65
|
+
res.status(status).type("text/plain").end("Generated media request cannot be satisfied");
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
missing(res);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
missing(res);
|
|
72
|
+
});
|
|
73
|
+
}];
|
|
74
|
+
}
|
package/lib/generationErrors.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { classifyUpstreamError, classifyUpstreamErrorCode, classifyModerationStage } from "./errorClassify.js";
|
|
2
|
-
import { providerErrorClass } from "./errors/providerMap.js";
|
|
2
|
+
import { providerErrorClass, statusForErrorCode } from "./errors/providerMap.js";
|
|
3
|
+
export { statusForErrorCode } from "./errors/providerMap.js";
|
|
3
4
|
import { safeDiagnosticLabel } from "./responsesParse.js";
|
|
4
5
|
import { RESPONSE_DIAGNOSTIC_CODES } from "./responsesErrors.js";
|
|
5
6
|
const PASSTHROUGH_CODES = new Set([
|
|
7
|
+
"GROK_API_KEY_MISSING",
|
|
6
8
|
"OAUTH_UNAVAILABLE",
|
|
7
9
|
"NETWORK_FAILED",
|
|
8
10
|
"AUTH_CHATGPT_EXPIRED",
|
|
@@ -104,27 +106,6 @@ export function isNonRetryableGenerationError(err) {
|
|
|
104
106
|
const status = Number(err?.status);
|
|
105
107
|
return code === "INVALID_REQUEST" || code === "OAUTH_IMAGE_TIMEOUT" || (Number.isFinite(status) && status >= 400 && status < 500);
|
|
106
108
|
}
|
|
107
|
-
export function statusForErrorCode(code, fallback = 500) {
|
|
108
|
-
if (code === "OAUTH_UNAVAILABLE" || code === "NETWORK_FAILED")
|
|
109
|
-
return 503;
|
|
110
|
-
if (code === "AUTH_CHATGPT_EXPIRED" || code === "AUTH_API_KEY_INVALID")
|
|
111
|
-
return 401;
|
|
112
|
-
if (code === "API_KEY_REQUIRED")
|
|
113
|
-
return 401;
|
|
114
|
-
if (code === "UPSTREAM_5XX")
|
|
115
|
-
return 502;
|
|
116
|
-
if (code === "RESPONSES_STREAM_ERROR")
|
|
117
|
-
return 502;
|
|
118
|
-
if (code === "OAUTH_IMAGE_TIMEOUT")
|
|
119
|
-
return 504;
|
|
120
|
-
if (code === "INVALID_REQUEST")
|
|
121
|
-
return 400;
|
|
122
|
-
if (RESPONSE_DIAGNOSTIC_CODES.has(code))
|
|
123
|
-
return 422;
|
|
124
|
-
if (code === "SAFETY_REFUSAL" || code === "MODERATION_REFUSED" || code === "moderation_blocked")
|
|
125
|
-
return 422;
|
|
126
|
-
return fallback;
|
|
127
|
-
}
|
|
128
109
|
function copyEmptyResponseMetadata(target, source) {
|
|
129
110
|
if (!source)
|
|
130
111
|
return;
|