ima2-gen 3.13.1 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/CHANGELOG.md +14 -0
- package/README.md +15 -2
- package/bin/commands/capabilities.js +5 -3
- package/bin/commands/config.js +15 -4
- package/bin/commands/defaults.js +7 -5
- package/bin/commands/doctor.js +88 -39
- package/bin/commands/gen.js +21 -4
- package/bin/commands/models.js +2 -2
- package/bin/commands/ping.js +1 -1
- package/bin/commands/prompt.js +5 -3
- package/bin/commands/service.js +32 -4
- package/bin/commands/tools.js +19 -3
- package/bin/commands/upscale.js +18 -4
- package/bin/commands/video.js +37 -50
- package/bin/ima2.js +28 -9
- package/bin/lib/characterResolve.js +2 -1
- package/bin/lib/client.js +155 -20
- package/bin/lib/config-store.js +40 -7
- package/bin/lib/doctor-bundle.js +6 -2
- package/bin/lib/doctor-checks.js +13 -29
- package/bin/lib/doctor-media.js +3 -3
- package/bin/lib/doctor-providers.js +48 -35
- package/bin/lib/doctor-report.js +78 -0
- package/bin/lib/doctor-runtime.js +173 -0
- package/bin/lib/mcpJob.js +19 -7
- package/bin/lib/output.js +3 -0
- package/bin/lib/platform.js +12 -9
- package/bin/lib/recover-output.js +9 -2
- package/bin/lib/sse.js +9 -1
- package/bin/lib/videoMcp.js +16 -8
- package/config.js +41 -5
- package/docs/API.md +154 -16
- package/docs/CLI.md +65 -1
- package/docs/DOCKER.md +19 -4
- package/docs/DOCKER.zh-CN.md +11 -4
- package/docs/DOCKER.zh-TW.md +11 -4
- package/docs/FAQ.md +13 -0
- package/docs/NPX_QUICKSTART.md +6 -0
- package/docs/README.ja.md +15 -2
- package/docs/README.ko.md +15 -2
- package/docs/README.zh-CN.md +15 -2
- package/docs/README.zh-TW.md +15 -2
- package/docs/migration/runtime-test-inventory.md +79 -8
- package/lib/agyArtifact.js +82 -0
- package/lib/agyArtifactRead.js +249 -0
- package/lib/agyImageAdapter.js +2 -329
- package/lib/agyProcess.js +122 -0
- package/lib/apiRequestBudget.js +48 -0
- package/lib/assetLifecycle.js +42 -9
- package/lib/canvasVersionStore.js +46 -14
- package/lib/capabilities.js +7 -1
- package/lib/configKeys.js +2 -0
- package/lib/errors/providerMap.js +26 -0
- package/lib/eventBus.js +10 -6
- package/lib/eventsPolicy.js +9 -0
- package/lib/geminiApiImageAdapter.js +1 -214
- package/lib/generatePipeline.js +39 -165
- package/lib/generatedMediaAccess.js +74 -0
- package/lib/generationErrors.js +3 -22
- package/lib/grokImageAdapter.js +2 -353
- package/lib/grokImageCore.js +1 -57
- package/lib/grokImageDownload.js +159 -0
- package/lib/grokImageDownloadPolicy.js +124 -0
- package/lib/grokImagePlanner.js +301 -0
- package/lib/grokMultimodeAdapter.js +1 -75
- package/lib/grokVideoDownload.js +137 -34
- package/lib/inflight.js +112 -112
- package/lib/jobs/terminalStore.js +45 -0
- package/lib/lanSessionStore.js +131 -0
- package/lib/localAccessPolicy.js +180 -0
- package/lib/localLanAccess.js +281 -0
- package/lib/logger.js +22 -13
- package/lib/mcp/downloadMediaResult.js +72 -78
- package/lib/multimodePipeline.js +39 -126
- package/lib/nodeGeneration.js +44 -101
- package/lib/nodeStore.js +25 -11
- package/lib/nodeValidation.js +5 -1
- package/lib/pinnedHttpGet.js +174 -0
- package/lib/promptImport/githubFolder.js +32 -18
- package/lib/promptImport/githubSource.js +14 -10
- package/lib/promptImport/parsePromptCandidates.js +32 -2
- package/lib/providers/adapters/agyOperations.js +179 -0
- package/lib/providers/adapters/comfy.js +1 -0
- package/lib/providers/adapters/geminiOperations.js +214 -0
- package/lib/providers/adapters/googleExecution.js +97 -0
- package/lib/providers/adapters/grokExecution.js +137 -0
- package/lib/providers/adapters/grokMultimodeOperations.js +83 -0
- package/lib/providers/adapters/grokOperations.js +63 -0
- package/lib/providers/adapters/openaiExecution.js +118 -0
- package/lib/providers/adapters/openaiOperations.js +201 -0
- package/lib/providers/adapters/openaiTypes.js +1 -0
- package/lib/providers/derive.js +4 -0
- package/lib/providers/deriveCore.js +2 -7
- package/lib/providers/execution/admission.js +25 -0
- package/lib/providers/execution/index.js +34 -0
- package/lib/providers/execution/legacy.js +17 -0
- package/lib/providers/execution/legacyClassic.js +66 -0
- package/lib/providers/execution/legacyEdit.js +30 -0
- package/lib/providers/execution/legacyMultimode.js +34 -0
- package/lib/providers/execution/legacyNode.js +41 -0
- package/lib/providers/execution/types.js +1 -0
- package/lib/providers/registry.js +19 -14
- package/lib/providers/surfaceSupport.js +22 -0
- package/lib/responsesImageAdapter.js +1 -389
- package/lib/responsesTransport.js +190 -0
- package/lib/spriteJobEvents.js +20 -2
- package/lib/ssePublish.js +5 -4
- package/lib/videoExtendI2vOperation.js +76 -0
- package/package.json +5 -2
- package/routes/edit.js +31 -123
- package/routes/events.js +140 -63
- package/routes/mcpMedia.js +10 -2
- package/routes/models.js +28 -25
- package/routes/videoExtended.js +9 -68
- package/server.js +24 -46
- package/ui/dist/.vite/manifest.json +90 -63
- package/ui/dist/assets/{AgentWorkspace-PbBytTeV.js → AgentWorkspace-CL3MLv3l.js} +2 -2
- package/ui/dist/assets/App-B9cEM6dJ.js +9 -0
- package/ui/dist/assets/AssetGenWorkspace-swWPvXrx.js +2 -0
- package/ui/dist/assets/AssetsWorkspace-CRVeYU2F.js +1 -0
- package/ui/dist/assets/{CardNewsWorkspace-BS3jKOja.js → CardNewsWorkspace-BAOxWLKf.js} +2 -2
- package/ui/dist/assets/GenerationRequestLogPanel-DZtYZm-D.js +1 -0
- package/ui/dist/assets/HomeWorkspace-BeZ1s0Jm.js +1 -0
- package/ui/dist/assets/KeyingPanel-ZyBSXioh.js +1 -0
- package/ui/dist/assets/NodeCanvas-BvvcCErf.js +7 -0
- package/ui/dist/assets/{PromptBuilderPanel-D97A4pwB.js → PromptBuilderPanel-DBycmShA.js} +2 -2
- package/ui/dist/assets/PromptImportDialog-ChvMv7ek.js +2 -0
- package/ui/dist/assets/PromptImportDiscoverySection-60py7whH.js +1 -0
- package/ui/dist/assets/PromptImportFolderSection-CCRh_-_7.js +1 -0
- package/ui/dist/assets/PromptLibraryPanel-BFWQO8OT.js +2 -0
- package/ui/dist/assets/SettingsWorkspace-B5Ghsgdj.js +1 -0
- package/ui/dist/assets/SpriteRecipeWorkspace-DRWdkmwU.js +1 -0
- package/ui/dist/assets/index-7ngEOVxt.js +5 -0
- package/ui/dist/assets/index-B5jy7ePg.css +1 -0
- package/ui/dist/assets/index-gxh_02DQ.js +22 -0
- package/ui/dist/assets/{pptxgen.es-D4pedbwM.js → pptxgen.es-OUNhQLEV.js} +1 -1
- package/ui/dist/assets/promptBuilderStore-BTUUzbBT.js +1 -0
- package/ui/dist/assets/useAgentDialogFocus-DSc2qY8p.js +1 -0
- package/ui/dist/index.html +3 -2
- package/ui/dist/assets/AssetGenWorkspace-B-TGlvVb.js +0 -2
- package/ui/dist/assets/AssetsWorkspace-LnGaW151.js +0 -1
- package/ui/dist/assets/GenerationRequestLogPanel-DuDWD_ic.js +0 -1
- package/ui/dist/assets/HomeWorkspace-DVP7UG6O.js +0 -1
- package/ui/dist/assets/KeyingPanel-5bRUFQaq.js +0 -1
- package/ui/dist/assets/NodeCanvas-BBEpERTb.js +0 -7
- package/ui/dist/assets/PromptImportDialog-C89MN5JY.js +0 -2
- package/ui/dist/assets/PromptImportDiscoverySection-BgRfDOs2.js +0 -1
- package/ui/dist/assets/PromptImportFolderSection-zTMyriT1.js +0 -1
- package/ui/dist/assets/PromptLibraryPanel-BD96qZu_.js +0 -2
- package/ui/dist/assets/SettingsWorkspace-BAOk3X_8.js +0 -1
- package/ui/dist/assets/SpriteRecipeWorkspace-ClIM5QRo.js +0 -1
- package/ui/dist/assets/index-0fpfY8vu.css +0 -1
- package/ui/dist/assets/index-BFciPGTn.js +0 -5
- package/ui/dist/assets/index-BbykeXR9.js +0 -30
- package/ui/dist/assets/promptBuilderStore-Do7M8g7J.js +0 -1
- package/ui/dist/assets/useAgentDialogFocus-B6-QgEVI.js +0 -1
|
@@ -3,39 +3,32 @@
|
|
|
3
3
|
// (routes/mcpMedia.ts) owns the atomic commit. Signed URLs are never persisted.
|
|
4
4
|
import { createWriteStream } from "node:fs";
|
|
5
5
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
6
|
-
import { lookup } from "node:dns/promises";
|
|
7
|
-
import { setDefaultResultOrder } from "node:dns";
|
|
8
|
-
import https from "node:https";
|
|
9
|
-
import { isIP } from "node:net";
|
|
10
6
|
import { tmpdir } from "node:os";
|
|
11
7
|
import { join } from "node:path";
|
|
12
8
|
import { Readable } from "node:stream";
|
|
13
9
|
import { pipeline } from "node:stream/promises";
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
// the actual "fetch failed" that dropped completed Runway results. Prefer
|
|
17
|
-
// IPv4 for outbound fetches in this process.
|
|
18
|
-
try {
|
|
19
|
-
setDefaultResultOrder("ipv4first");
|
|
20
|
-
}
|
|
21
|
-
catch { /* older runtimes */ }
|
|
10
|
+
import { pinnedHttpGet } from "../pinnedHttpGet.js";
|
|
11
|
+
import { resolvePublicDownloadTarget } from "../grokImageDownloadPolicy.js";
|
|
22
12
|
const MAX_REDIRECTS = 5;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return
|
|
13
|
+
async function resolvePublicHttps(url, signal) {
|
|
14
|
+
if (url.protocol !== "https:")
|
|
15
|
+
throw new Error(`MCP_DOWNLOAD_INSECURE:${url.protocol}`);
|
|
16
|
+
try {
|
|
17
|
+
return await resolvePublicDownloadTarget(url, signal, "ipv4first");
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
signal.throwIfAborted();
|
|
21
|
+
if (!(error instanceof Error) || Reflect.get(error, "code") !== "GROK_IMAGE_DOWNLOAD_FAILED")
|
|
22
|
+
throw error;
|
|
23
|
+
throw new Error(`MCP_DOWNLOAD_PRIVATE_IP:${url.hostname}`);
|
|
28
24
|
}
|
|
29
|
-
return PRIVATE_V4.some((pattern) => pattern.test(address));
|
|
30
25
|
}
|
|
31
26
|
export async function assertPublicHttps(url) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
if (isPrivateAddress(address))
|
|
38
|
-
throw new Error(`MCP_DOWNLOAD_PRIVATE_IP:${host}`);
|
|
27
|
+
try {
|
|
28
|
+
await resolvePublicHttps(url, new AbortController().signal);
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
throw error;
|
|
39
32
|
}
|
|
40
33
|
}
|
|
41
34
|
/** Transient right after task completion: network-level failures and
|
|
@@ -75,42 +68,49 @@ export async function downloadMediaResult(rawUrl, options) {
|
|
|
75
68
|
async function downloadMediaResultOnce(rawUrl, options) {
|
|
76
69
|
const maxBytes = options.maxBytes ?? (options.kind === "video" ? 800 * 1024 * 1024 : 40 * 1024 * 1024);
|
|
77
70
|
let url = new URL(rawUrl);
|
|
78
|
-
let response
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
71
|
+
let response;
|
|
72
|
+
try {
|
|
73
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
|
|
74
|
+
response = await openGet(url, options.timeoutMs ?? 120_000, options.v4Fallback !== false);
|
|
75
|
+
if (response.status >= 300 && response.status < 400) {
|
|
76
|
+
const location = response.headers.get("location");
|
|
77
|
+
response.cancel();
|
|
78
|
+
if (!location?.trim())
|
|
79
|
+
throw new Error("MCP_DOWNLOAD_REDIRECT_INVALID");
|
|
80
|
+
url = new URL(location, url);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
if (!response || response.status < 200 || response.status >= 300 || !response.body) {
|
|
86
|
+
throw new Error(`MCP_DOWNLOAD_FAILED:${response?.status ?? "no-response"}`);
|
|
89
87
|
}
|
|
90
|
-
|
|
88
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
89
|
+
const expected = options.kind === "video" ? /^(video\/|application\/octet-stream)/ : /^image\//;
|
|
90
|
+
if (!expected.test(contentType))
|
|
91
|
+
throw new Error(`MCP_RESULT_TYPE_MISMATCH:${contentType}`);
|
|
92
|
+
if (Number(response.headers.get("content-length")) > maxBytes)
|
|
93
|
+
throw new Error("MCP_DOWNLOAD_TOO_LARGE");
|
|
94
|
+
return await writeMediaResult(response, url, contentType, maxBytes);
|
|
91
95
|
}
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
finally {
|
|
97
|
+
response?.cancel();
|
|
94
98
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (!expected.test(contentType))
|
|
98
|
-
throw new Error(`MCP_RESULT_TYPE_MISMATCH:${contentType}`);
|
|
99
|
+
}
|
|
100
|
+
async function writeMediaResult(response, url, contentType, maxBytes) {
|
|
99
101
|
const dir = await mkdtemp(join(tmpdir(), "ima2-mcp-dl-"));
|
|
100
102
|
const tempPath = join(dir, "result");
|
|
101
103
|
let bytes = 0;
|
|
102
104
|
try {
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
transform(chunk, controller) {
|
|
105
|
+
const capped = async function* () {
|
|
106
|
+
for await (const chunk of response.body) {
|
|
106
107
|
bytes += chunk.byteLength;
|
|
107
108
|
if (bytes > maxBytes)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
await pipeline(Readable.fromWeb(capped), createWriteStream(tempPath));
|
|
109
|
+
throw new Error("MCP_DOWNLOAD_TOO_LARGE");
|
|
110
|
+
yield chunk;
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
await pipeline(Readable.from(capped()), createWriteStream(tempPath));
|
|
114
114
|
}
|
|
115
115
|
catch (error) {
|
|
116
116
|
await rm(dir, { recursive: true, force: true });
|
|
@@ -124,37 +124,31 @@ async function downloadMediaResultOnce(rawUrl, options) {
|
|
|
124
124
|
cleanup: () => rm(dir, { recursive: true, force: true }),
|
|
125
125
|
};
|
|
126
126
|
}
|
|
127
|
-
/**
|
|
128
|
-
* undici/https happy-eyeballs), fall back to https.request with family 4 —
|
|
129
|
-
* this was the actual ETIMEDOUT behind the lost Runway results (260718). */
|
|
127
|
+
/** Retain IPv4 preference and one optional network fallback without fresh DNS. */
|
|
130
128
|
async function openGet(url, timeoutMs, v4Fallback) {
|
|
129
|
+
let target;
|
|
131
130
|
try {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
body: response.body ? Readable.fromWeb(response.body) : null,
|
|
137
|
-
};
|
|
131
|
+
return await timedGet(async (signal) => {
|
|
132
|
+
target = await resolvePublicHttps(url, signal);
|
|
133
|
+
return target;
|
|
134
|
+
}, timeoutMs);
|
|
138
135
|
}
|
|
139
136
|
catch (error) {
|
|
140
|
-
if (!v4Fallback)
|
|
137
|
+
if (!v4Fallback || !target?.addresses.some(({ family }) => family === 4) || !isRetryableDownloadError(error))
|
|
141
138
|
throw error;
|
|
142
|
-
|
|
143
|
-
throw error;
|
|
144
|
-
return httpsGetV4(url, timeoutMs);
|
|
139
|
+
return timedGet(async () => target, timeoutMs, 4);
|
|
145
140
|
}
|
|
146
141
|
}
|
|
147
|
-
function
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
});
|
|
142
|
+
async function timedGet(resolve, timeoutMs, family) {
|
|
143
|
+
const controller = new AbortController();
|
|
144
|
+
const timer = setTimeout(() => controller.abort(new Error("MCP_DOWNLOAD_TIMEOUT")), timeoutMs);
|
|
145
|
+
try {
|
|
146
|
+
const target = await resolve(controller.signal);
|
|
147
|
+
const response = await pinnedHttpGet(target, controller.signal, family ? { family } : {});
|
|
148
|
+
return { ...response, cancel: (reason) => { clearTimeout(timer); response.cancel(reason); } };
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
clearTimeout(timer);
|
|
152
|
+
throw error;
|
|
153
|
+
}
|
|
160
154
|
}
|
package/lib/multimodePipeline.js
CHANGED
|
@@ -8,14 +8,9 @@ import { generateImageThumbnailFromBuffer } from "./imageThumb.js";
|
|
|
8
8
|
import { classifyUpstreamError } from "./errorClassify.js";
|
|
9
9
|
import { normalizeOAuthParams } from "./oauthNormalize.js";
|
|
10
10
|
import { resolveProviderOptions } from "./providerOptions.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import { prepareImageExecution } from "./providers/execution/index.js";
|
|
12
|
+
import { checkImageExecutionAdmission } from "./providers/execution/admission.js";
|
|
13
13
|
import { resolveGrokQualityModel } from "./imageModels.js";
|
|
14
|
-
import { generateViaAgy } from "./agyImageAdapter.js";
|
|
15
|
-
import { generateViaGeminiApi } from "./geminiApiImageAdapter.js";
|
|
16
|
-
import { generateViaAtlasCloud } from "./atlasCloudImageAdapter.js";
|
|
17
|
-
import { generateViaMinimax } from "./minimaxImageAdapter.js";
|
|
18
|
-
import { generateViaNai } from "./naiImageAdapter.js";
|
|
19
14
|
import { composerNegativePromptMeta, readNaiOptions } from "./naiOptions.js";
|
|
20
15
|
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, INFLIGHT_RETRY_AFTER_SECONDS } from "./inflight.js";
|
|
21
16
|
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "./generationCancel.js";
|
|
@@ -24,7 +19,6 @@ import { embedImageMetadataBestEffort } from "./imageMetadataStore.js";
|
|
|
24
19
|
import { invalidateHistoryIndex } from "./historyIndex.js";
|
|
25
20
|
import { normalizeComposerInsertedPrompts, normalizeComposerPrompt, } from "./composerSnapshot.js";
|
|
26
21
|
import { errInfo } from "./errInfo.js";
|
|
27
|
-
import { requireRuntimeContext } from "./runtimeContext.js";
|
|
28
22
|
import { validateModeration, imageFormatFromMime, writeSse } from "./routeHelpers.js";
|
|
29
23
|
import { publish } from "./eventBus.js";
|
|
30
24
|
import { publishJobEvent } from "./ssePublish.js";
|
|
@@ -33,6 +27,7 @@ import { normalizeBodyRequestId, validateBoundedCount, validateGenerationPrompt
|
|
|
33
27
|
import { getElementById } from "./assetsStore.js";
|
|
34
28
|
import { compileElements, ELEMENT_CAPACITY_DEFAULTS } from "./elementCompiler.js";
|
|
35
29
|
import { errorEnvelopeFields } from "./errors/envelope.js";
|
|
30
|
+
import { getProviderSurfaceSupport } from "./providers/derive.js";
|
|
36
31
|
async function resolveMultimodeElements(elementIds, references, activeProvider, requestId) {
|
|
37
32
|
let notesFragment = "";
|
|
38
33
|
const resolvedRefs = [];
|
|
@@ -142,7 +137,7 @@ export async function runMultimodePipeline(req, res, ctx) {
|
|
|
142
137
|
// request would reach generateViaResponses and bill OAuth for an image
|
|
143
138
|
// the user asked ComfyUI to make — silently, with no error to trace.
|
|
144
139
|
// Removed in wp7 when this surface gains a real comfy branch.
|
|
145
|
-
if (provider
|
|
140
|
+
if (getProviderSurfaceSupport(provider, "multimode")?.supported === false) {
|
|
146
141
|
finishStatus = "error";
|
|
147
142
|
finishHttpStatus = 400;
|
|
148
143
|
finishErrorCode = "COMFY_SURFACE_UNSUPPORTED";
|
|
@@ -160,7 +155,7 @@ export async function runMultimodePipeline(req, res, ctx) {
|
|
|
160
155
|
rawSize: size,
|
|
161
156
|
rawWebSearchEnabled,
|
|
162
157
|
});
|
|
163
|
-
if (providerOptions.error) {
|
|
158
|
+
if (providerOptions.error !== undefined) {
|
|
164
159
|
finishStatus = "error";
|
|
165
160
|
finishHttpStatus = providerOptions.status;
|
|
166
161
|
finishErrorCode = providerOptions.code;
|
|
@@ -211,6 +206,17 @@ export async function runMultimodePipeline(req, res, ctx) {
|
|
|
211
206
|
});
|
|
212
207
|
}
|
|
213
208
|
const refCheck = refCheckResult;
|
|
209
|
+
const admission = checkImageExecutionAdmission(ctx, {
|
|
210
|
+
provider: activeProvider, surface: "multimode", referenceCount: refCheck.refs.length,
|
|
211
|
+
});
|
|
212
|
+
if (admission) {
|
|
213
|
+
finishStatus = "error";
|
|
214
|
+
finishHttpStatus = admission.status;
|
|
215
|
+
finishErrorCode = admission.code;
|
|
216
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, admission.status, {
|
|
217
|
+
error: admission.message, code: admission.code, status: admission.status, requestId,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
214
220
|
const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http") ? req.body.providerUrl : null;
|
|
215
221
|
const referencePayload = summarizeReferencePayload(mergedReferences);
|
|
216
222
|
const started = startJob({
|
|
@@ -347,126 +353,33 @@ export async function runMultimodePipeline(req, res, ctx) {
|
|
|
347
353
|
dualEmitMultimode(res, requestId, "image", item);
|
|
348
354
|
};
|
|
349
355
|
dualEmitMultimode(res, requestId, "phase", { phase: "streaming", requestId, sequenceId, maxImages });
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
generated = {
|
|
372
|
-
images: [{ b64: r.b64, ...(r.revisedPrompt !== undefined ? { revisedPrompt: r.revisedPrompt } : {}) }],
|
|
373
|
-
usage: r.usage,
|
|
374
|
-
webSearchCalls: r.webSearchCalls,
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
else if (activeProvider === "atlascloud") {
|
|
378
|
-
const r = await generateViaAtlasCloud(prompt, requireRuntimeContext(ctx), {
|
|
379
|
-
model: imageModel,
|
|
380
|
-
size: effectiveSize,
|
|
381
|
-
quality: routeQuality,
|
|
382
|
-
signal: cancelController.signal,
|
|
383
|
-
requestId,
|
|
384
|
-
references: refCheck.refDetails,
|
|
385
|
-
});
|
|
386
|
-
generated = {
|
|
387
|
-
images: [{ b64: r.b64, ...(r.revisedPrompt !== undefined ? { revisedPrompt: r.revisedPrompt } : {}) }],
|
|
388
|
-
usage: r.usage,
|
|
389
|
-
webSearchCalls: r.webSearchCalls,
|
|
390
|
-
};
|
|
391
|
-
}
|
|
392
|
-
else if (activeProvider === "minimax") {
|
|
393
|
-
const r = await generateViaMinimax(prompt, requireRuntimeContext(ctx), {
|
|
394
|
-
model: imageModel,
|
|
395
|
-
size: effectiveSize,
|
|
396
|
-
signal: cancelController.signal,
|
|
397
|
-
requestId,
|
|
398
|
-
references: refCheck.refDetails,
|
|
399
|
-
});
|
|
400
|
-
generated = {
|
|
401
|
-
images: [{ b64: r.b64, ...(r.revisedPrompt !== undefined ? { revisedPrompt: r.revisedPrompt } : {}) }],
|
|
402
|
-
usage: r.usage,
|
|
403
|
-
webSearchCalls: r.webSearchCalls,
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
else if (activeProvider === "nai") {
|
|
407
|
-
// No references: the adapter is text-to-image only (see the refusal in
|
|
408
|
-
// lib/generatePipeline.ts), so none are forwarded here either.
|
|
409
|
-
const r = await generateViaNai(prompt, requireRuntimeContext(ctx), {
|
|
410
|
-
model: imageModel,
|
|
411
|
-
size: effectiveSize,
|
|
412
|
-
signal: cancelController.signal,
|
|
413
|
-
requestId,
|
|
414
|
-
...readNaiOptions(req.body),
|
|
415
|
-
});
|
|
416
|
-
generated = {
|
|
417
|
-
// No mime field on this shape; the persist step below detects PNG
|
|
418
|
-
// from the magic bytes, which is authoritative anyway.
|
|
419
|
-
images: [{ b64: r.b64, ...(r.revisedPrompt !== undefined ? { revisedPrompt: r.revisedPrompt } : {}) }],
|
|
420
|
-
usage: r.usage,
|
|
421
|
-
webSearchCalls: r.webSearchCalls,
|
|
422
|
-
};
|
|
423
|
-
}
|
|
424
|
-
else if (activeProvider === "grok" || activeProvider === "grok-api") {
|
|
425
|
-
const directApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
|
|
426
|
-
const grokModel = resolveGrokQualityModel(imageModel, quality);
|
|
427
|
-
const grokRefs = incomingProviderUrl
|
|
428
|
-
? [{ b64: "", url: incomingProviderUrl }, ...refCheck.refDetails]
|
|
429
|
-
: refCheck.refDetails;
|
|
430
|
-
generated = await generateMultimodeViaGrok(generationPrompt, ctx, {
|
|
431
|
-
model: grokModel,
|
|
432
|
-
maxImages,
|
|
433
|
-
size: effectiveSize,
|
|
434
|
-
signal: cancelController.signal,
|
|
435
|
-
requestId,
|
|
436
|
-
references: grokRefs,
|
|
437
|
-
directApiKey,
|
|
438
|
-
onFinalImage: async (image, index) => {
|
|
439
|
-
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
440
|
-
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
441
|
-
},
|
|
442
|
-
});
|
|
443
|
-
}
|
|
444
|
-
else {
|
|
445
|
-
generated = await generateMultimodeViaResponses(activeProvider, generationPrompt, quality, effectiveSize, moderation, refCheck.refDetails || refCheck.refs, requestId, normalizedPromptMode, ctx, {
|
|
446
|
-
model: imageModel,
|
|
447
|
-
maxImages,
|
|
448
|
-
reasoningEffort,
|
|
449
|
-
webSearchEnabled,
|
|
450
|
-
onPartialImage: (partial) => {
|
|
451
|
-
if (isJobCanceled(requestId))
|
|
452
|
-
return;
|
|
453
|
-
const pd = { image: `data:${mime};base64,${partial.b64}`, requestId, sequenceId, index: partial.index };
|
|
454
|
-
if (!res.writableEnded && !res.destroyed)
|
|
455
|
-
writeSse(res, "partial", pd);
|
|
456
|
-
publish(requestId, "partial", pd);
|
|
457
|
-
},
|
|
458
|
-
onFinalImage: async (image, index) => {
|
|
459
|
-
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
460
|
-
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
461
|
-
},
|
|
462
|
-
signal: cancelController.signal,
|
|
463
|
-
});
|
|
464
|
-
}
|
|
356
|
+
const { value: generated } = await (await prepareImageExecution(ctx, {
|
|
357
|
+
surface: "multimode", provider: activeProvider, requestId,
|
|
358
|
+
signal: cancelController.signal, prompt: generationPrompt, rawPrompt: prompt,
|
|
359
|
+
references: refCheck.refDetails, providerUrl: incomingProviderUrl, maxImages,
|
|
360
|
+
options: { model: imageModel, quality, size: effectiveSize, moderation,
|
|
361
|
+
mode: normalizedPromptMode, reasoningEffort, webSearchEnabled },
|
|
362
|
+
nai: activeProvider === "nai" ? readNaiOptions(req.body) : {},
|
|
363
|
+
}, {
|
|
364
|
+
onPartialImage: (partial) => {
|
|
365
|
+
if (isJobCanceled(requestId))
|
|
366
|
+
return;
|
|
367
|
+
const pd = { image: `data:${mime};base64,${partial.b64}`, requestId, sequenceId, index: partial.index };
|
|
368
|
+
if (!res.writableEnded && !res.destroyed)
|
|
369
|
+
writeSse(res, "partial", pd);
|
|
370
|
+
publish(requestId, "partial", pd);
|
|
371
|
+
},
|
|
372
|
+
onFinalImage: async (image, index) => {
|
|
373
|
+
const totalReturned = images.length + 1;
|
|
374
|
+
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
375
|
+
},
|
|
376
|
+
})).execute();
|
|
465
377
|
throwIfJobCanceled(requestId);
|
|
466
378
|
latestUsage = generated.usage || null;
|
|
467
379
|
latestWebSearchCalls = generated.webSearchCalls || 0;
|
|
468
380
|
latestExtraIgnored = generated.extraIgnored || 0;
|
|
469
|
-
for (const [
|
|
381
|
+
for (const [position, image] of generated.images.entries()) {
|
|
382
|
+
const index = generated.originalIndexes?.[position] ?? position;
|
|
470
383
|
await persistAndSendImage(image, index, generated.images.length, sequenceStatus(generated.images.length, maxImages));
|
|
471
384
|
}
|
|
472
385
|
const returned = images.length;
|
package/lib/nodeGeneration.js
CHANGED
|
@@ -6,26 +6,20 @@ import { detectImageMimeFromB64, summarizeReferencePayload } from "./refs.js";
|
|
|
6
6
|
import { classifyUpstreamError } from "./errorClassify.js";
|
|
7
7
|
import { normalizeOAuthParams } from "./oauthNormalize.js";
|
|
8
8
|
import { resolveProviderOptions } from "./providerOptions.js";
|
|
9
|
-
import { generateViaResponses, editViaResponses } from "./responsesImageAdapter.js";
|
|
10
|
-
import { generateViaGrok } from "./grokImageAdapter.js";
|
|
11
9
|
import { resolveGrokQualityModel } from "./imageModels.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { generateViaAtlasCloud } from "./atlasCloudImageAdapter.js";
|
|
15
|
-
import { generateViaMinimax } from "./minimaxImageAdapter.js";
|
|
16
|
-
import { generateViaNai } from "./naiImageAdapter.js";
|
|
10
|
+
import { prepareImageExecution } from "./providers/execution/index.js";
|
|
11
|
+
import { checkImageExecutionAdmission } from "./providers/execution/admission.js";
|
|
17
12
|
import { readNaiOptions } from "./naiOptions.js";
|
|
18
13
|
import { isNonRetryableGenerationError, normalizeGenerationFailure } from "./generationErrors.js";
|
|
19
14
|
import { logEvent, logError } from "./logger.js";
|
|
20
15
|
import { errInfo } from "./errInfo.js";
|
|
21
|
-
import { requireRuntimeContext } from "./runtimeContext.js";
|
|
22
16
|
import { imageFormatFromMime, writeSse, dataUrlFromB64 } from "./routeHelpers.js";
|
|
23
17
|
import { validateNodeInputs } from "./nodeValidation.js";
|
|
24
18
|
import { publish } from "./eventBus.js";
|
|
25
19
|
import { publishJobEvent } from "./ssePublish.js";
|
|
26
|
-
import { asUpstream, wantsSse, writeNodeError, loadParentNodeB64,
|
|
20
|
+
import { asUpstream, wantsSse, writeNodeError, loadParentNodeB64, nodeErrorDetails, } from "./nodeHelpers.js";
|
|
27
21
|
import { normalizeBodyRequestId, validateGenerationPrompt } from "./generationInputValidation.js";
|
|
28
|
-
import { deriveReferenceLimit } from "./providers/derive.js";
|
|
22
|
+
import { deriveReferenceLimit, getProviderSurfaceSupport } from "./providers/derive.js";
|
|
29
23
|
import { errorEnvelopeFields } from "./errors/envelope.js";
|
|
30
24
|
export async function runNodeGeneration(req, res, ctx) {
|
|
31
25
|
const body = (req.body ?? {});
|
|
@@ -51,15 +45,15 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
51
45
|
const { provider = "oauth" } = body;
|
|
52
46
|
const { quality, warnings: qualityWarnings } = normalizeOAuthParams({ provider, quality: rawQuality });
|
|
53
47
|
const normalizedPromptMode = promptMode === "direct" ? "direct" : "auto";
|
|
54
|
-
const contextMode =
|
|
55
|
-
?
|
|
56
|
-
|
|
57
|
-
|
|
48
|
+
const contextMode = rawContextMode === "parent-only" ? "parent-only"
|
|
49
|
+
: rawContextMode === "ancestry" ? "ancestry" : "parent-plus-refs";
|
|
50
|
+
const searchMode = rawSearchMode === "off" || rawSearchMode === "auto"
|
|
51
|
+
? rawSearchMode : "on";
|
|
58
52
|
// Node mode has no comfy dispatch in this unit; without this the request
|
|
59
53
|
// falls through to generateViaResponses and bills OAuth. The envelope is
|
|
60
54
|
// this file's nested shape, not the flat one generate uses.
|
|
61
55
|
// Removed in wp7.
|
|
62
|
-
if (provider
|
|
56
|
+
if (getProviderSurfaceSupport(provider, "node")?.supported === false) {
|
|
63
57
|
finishStatus = "error";
|
|
64
58
|
finishHttpStatus = 400;
|
|
65
59
|
finishErrorCode = "COMFY_SURFACE_UNSUPPORTED";
|
|
@@ -76,7 +70,7 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
76
70
|
rawWebSearchEnabled: body.webSearchEnabled,
|
|
77
71
|
searchMode,
|
|
78
72
|
});
|
|
79
|
-
if (providerOptions.error) {
|
|
73
|
+
if (providerOptions.error !== undefined) {
|
|
80
74
|
finishStatus = "error";
|
|
81
75
|
finishHttpStatus = providerOptions.status;
|
|
82
76
|
finishErrorCode = providerOptions.code;
|
|
@@ -102,7 +96,8 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
102
96
|
parentNodeId,
|
|
103
97
|
});
|
|
104
98
|
}
|
|
105
|
-
|
|
99
|
+
// Only API/OAuth persist the requested format; other providers select their output format.
|
|
100
|
+
const validation = validateNodeInputs(ctx, rawPrompt, references, moderation, activeProvider === "api" || activeProvider === "oauth" ? format : undefined);
|
|
106
101
|
if (validation.error) {
|
|
107
102
|
finishStatus = "error";
|
|
108
103
|
finishHttpStatus = 400;
|
|
@@ -177,7 +172,7 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
177
172
|
}
|
|
178
173
|
// Node mode chains images, so a parent node is exactly the reference the
|
|
179
174
|
// adapter cannot use. Refuse instead of generating something unrelated.
|
|
180
|
-
if (activeProvider
|
|
175
|
+
if (getProviderSurfaceSupport(activeProvider ?? "", "node")?.references === false && inputImageCount > 0) {
|
|
181
176
|
finishStatus = "error";
|
|
182
177
|
finishHttpStatus = 400;
|
|
183
178
|
return res.status(400).json({
|
|
@@ -189,6 +184,18 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
189
184
|
parentNodeId,
|
|
190
185
|
});
|
|
191
186
|
}
|
|
187
|
+
const admission = checkImageExecutionAdmission(ctx, {
|
|
188
|
+
provider: activeProvider, surface: "node", referenceCount: inputImageCount,
|
|
189
|
+
});
|
|
190
|
+
if (admission) {
|
|
191
|
+
finishStatus = "error";
|
|
192
|
+
finishHttpStatus = admission.status;
|
|
193
|
+
finishErrorCode = admission.code;
|
|
194
|
+
return res.status(admission.status).json({
|
|
195
|
+
error: { code: admission.code, message: admission.message },
|
|
196
|
+
code: admission.code, parentNodeId, requestId,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
192
199
|
const started = startJob({
|
|
193
200
|
requestId,
|
|
194
201
|
kind: "node",
|
|
@@ -255,7 +262,24 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
255
262
|
publish(requestId, "phase", { requestId, phase: "streaming" });
|
|
256
263
|
}
|
|
257
264
|
let b64, usage, webSearchCalls = 0, revisedPrompt = null;
|
|
258
|
-
const
|
|
265
|
+
const execution = await prepareImageExecution(ctx, {
|
|
266
|
+
surface: "node", provider: activeProvider, requestId,
|
|
267
|
+
signal: cancelController.signal, prompt: generationPrompt, rawPrompt: prompt,
|
|
268
|
+
references: refCheck.refDetails, sourceImage: parentB64, contextMode, searchMode,
|
|
269
|
+
partialImages: emitProgress ? 2 : 0,
|
|
270
|
+
options: { model: effectiveImageModel, quality, size: effectiveSize, moderation,
|
|
271
|
+
mode: normalizedPromptMode, reasoningEffort, webSearchEnabled },
|
|
272
|
+
nai: activeProvider === "nai" ? readNaiOptions(req.body) : {},
|
|
273
|
+
}, emitProgress ? {
|
|
274
|
+
onPartialImage: (partial) => {
|
|
275
|
+
if (isJobCanceled(requestId))
|
|
276
|
+
return;
|
|
277
|
+
const pd = { requestId, image: dataUrlFromB64(format, partial.b64 ?? ""), index: partial.index };
|
|
278
|
+
if (streamResponse)
|
|
279
|
+
writeSse(res, "partial", pd);
|
|
280
|
+
publish(requestId, "partial", pd);
|
|
281
|
+
},
|
|
282
|
+
} : undefined);
|
|
259
283
|
let resultFormat = activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api" || activeProvider === "atlascloud" || activeProvider === "minimax" ? "jpeg" : format;
|
|
260
284
|
const maxAttempts = inputImageCount > 0 ? 1 : 2;
|
|
261
285
|
let lastErr = null;
|
|
@@ -279,88 +303,7 @@ export async function runNodeGeneration(req, res, ctx) {
|
|
|
279
303
|
searchMode,
|
|
280
304
|
webSearchEnabled,
|
|
281
305
|
});
|
|
282
|
-
const r =
|
|
283
|
-
? await generateViaGeminiApi(parentB64 ? `Edit this image: ${generationPrompt}` : generationPrompt, requireRuntimeContext(ctx), {
|
|
284
|
-
model: effectiveImageModel,
|
|
285
|
-
size: effectiveSize,
|
|
286
|
-
signal: cancelController.signal,
|
|
287
|
-
requestId,
|
|
288
|
-
references: parentB64
|
|
289
|
-
? [{ b64: parentB64, declaredMime: null, detectedMime: null }, ...(refCheck.refDetails || [])]
|
|
290
|
-
: refCheck.refDetails,
|
|
291
|
-
})
|
|
292
|
-
: activeProvider === "agy"
|
|
293
|
-
? await generateViaAgy(parentB64 ? `Edit this image: ${generationPrompt}` : generationPrompt, {
|
|
294
|
-
...(parentB64
|
|
295
|
-
? { references: [{ b64: parentB64, declaredMime: null, detectedMime: null }] }
|
|
296
|
-
: {}),
|
|
297
|
-
signal: cancelController.signal,
|
|
298
|
-
requestId,
|
|
299
|
-
})
|
|
300
|
-
: activeProvider === "atlascloud"
|
|
301
|
-
? await generateViaAtlasCloud(parentB64 ? `Edit this image: ${prompt}` : prompt, requireRuntimeContext(ctx), {
|
|
302
|
-
model: effectiveImageModel,
|
|
303
|
-
size: effectiveSize,
|
|
304
|
-
quality,
|
|
305
|
-
signal: cancelController.signal,
|
|
306
|
-
requestId,
|
|
307
|
-
references: parentB64
|
|
308
|
-
? [{ b64: parentB64, declaredMime: null, detectedMime: null }, ...(refCheck.refDetails || [])]
|
|
309
|
-
: refCheck.refDetails,
|
|
310
|
-
})
|
|
311
|
-
: activeProvider === "minimax"
|
|
312
|
-
? await generateViaMinimax(parentB64 ? `Edit this image: ${prompt}` : prompt, requireRuntimeContext(ctx), {
|
|
313
|
-
model: effectiveImageModel,
|
|
314
|
-
size: effectiveSize,
|
|
315
|
-
signal: cancelController.signal,
|
|
316
|
-
requestId,
|
|
317
|
-
references: parentB64
|
|
318
|
-
? [{ b64: parentB64, declaredMime: null, detectedMime: null }, ...(refCheck.refDetails || [])]
|
|
319
|
-
: refCheck.refDetails,
|
|
320
|
-
})
|
|
321
|
-
: activeProvider === "nai"
|
|
322
|
-
? await generateViaNai(prompt, requireRuntimeContext(ctx), {
|
|
323
|
-
model: effectiveImageModel,
|
|
324
|
-
size: effectiveSize,
|
|
325
|
-
signal: cancelController.signal,
|
|
326
|
-
requestId,
|
|
327
|
-
...readNaiOptions(req.body),
|
|
328
|
-
})
|
|
329
|
-
: activeProvider === "grok" || activeProvider === "grok-api"
|
|
330
|
-
? await generateViaGrok(generationPrompt, ctx, {
|
|
331
|
-
model: effectiveImageModel,
|
|
332
|
-
size: effectiveSize,
|
|
333
|
-
requestId,
|
|
334
|
-
signal: cancelController.signal,
|
|
335
|
-
references: toGrokReferences(parentB64, refsForRequest),
|
|
336
|
-
directApiKey: grokDirectApiKey,
|
|
337
|
-
})
|
|
338
|
-
: parentB64
|
|
339
|
-
? await editViaResponses(activeProvider, generationPrompt, parentB64, quality, effectiveSize, moderation, normalizedPromptMode, ctx, requestId, {
|
|
340
|
-
model: effectiveImageModel,
|
|
341
|
-
references: refsForRequest,
|
|
342
|
-
searchMode,
|
|
343
|
-
reasoningEffort,
|
|
344
|
-
webSearchEnabled,
|
|
345
|
-
signal: cancelController.signal,
|
|
346
|
-
})
|
|
347
|
-
: await generateViaResponses(activeProvider, generationPrompt, quality, effectiveSize, moderation, refsForRequest, requestId, normalizedPromptMode, ctx, {
|
|
348
|
-
model: effectiveImageModel,
|
|
349
|
-
reasoningEffort,
|
|
350
|
-
webSearchEnabled,
|
|
351
|
-
signal: cancelController.signal,
|
|
352
|
-
partialImages: emitProgress ? 2 : 0,
|
|
353
|
-
onPartialImage: emitProgress
|
|
354
|
-
? (partial) => {
|
|
355
|
-
if (isJobCanceled(requestId))
|
|
356
|
-
return;
|
|
357
|
-
const pd = { requestId, image: dataUrlFromB64(format, partial.b64 ?? ""), index: partial.index };
|
|
358
|
-
if (streamResponse)
|
|
359
|
-
writeSse(res, "partial", pd);
|
|
360
|
-
publish(requestId, "partial", pd);
|
|
361
|
-
}
|
|
362
|
-
: null,
|
|
363
|
-
});
|
|
306
|
+
const { value: r } = await execution.execute();
|
|
364
307
|
throwIfJobCanceled(requestId);
|
|
365
308
|
if (r.b64) {
|
|
366
309
|
b64 = r.b64;
|