ima2-gen 2.0.15 → 2.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/bin/commands/skill.js +336 -17
- package/bin/ima2.js +22 -2
- package/bin/lib/doctor-checks.js +6 -4
- package/config.js +1 -0
- package/docs/CLI.md +25 -3
- package/docs/PROMPT_STUDIO.ko.md +29 -0
- package/docs/PROMPT_STUDIO.md +31 -0
- package/docs/migration/runtime-test-inventory.md +7 -1
- package/lib/agentImageVideoGen.js +8 -0
- package/lib/agentQueueStore.js +36 -10
- package/lib/agentQueueWorker.js +79 -7
- package/lib/agentRuntime.js +2 -0
- package/lib/assetLifecycle.js +40 -8
- package/lib/canvasVersionStore.js +86 -0
- package/lib/cardNewsGenerator.js +37 -3
- package/lib/cardNewsJobStore.js +10 -0
- package/lib/cardNewsManifestStore.js +6 -12
- package/lib/cardNewsPath.js +25 -0
- package/lib/db.js +2 -0
- package/lib/generatePipeline.js +474 -0
- package/lib/generationInputValidation.js +32 -0
- package/lib/grokImageCore.js +18 -1
- package/lib/historyIndex.js +11 -4
- package/lib/historyList.js +3 -0
- package/lib/imageMetadata.js +3 -0
- package/lib/inflight.js +30 -3
- package/lib/multimodePipeline.js +462 -0
- package/lib/nodeGeneration.js +440 -0
- package/lib/nodeValidation.js +23 -0
- package/lib/oauthProxy/generators.js +3 -246
- package/lib/oauthProxy/multimodeGenerators.js +251 -0
- package/lib/routeHelpers.js +9 -2
- package/package.json +2 -2
- package/routes/agent.js +4 -4
- package/routes/canvasVersions.js +21 -1
- package/routes/cardNews.js +2 -1
- package/routes/edit.js +25 -9
- package/routes/events.js +5 -1
- package/routes/generate.js +2 -481
- package/routes/keys.js +31 -39
- package/routes/multimode.js +2 -502
- package/routes/nodes.js +3 -448
- package/routes/video.js +12 -3
- package/routes/videoExtended.js +19 -4
- package/server.js +53 -1
- package/skills/ima2/SKILL.md +204 -0
- package/skills/ima2-front/SKILL.md +611 -0
- package/skills/ima2-front/references/a11y-patterns.md +134 -0
- package/skills/ima2-front/references/aesthetics.md +327 -0
- package/skills/ima2-front/references/anti-slop.md +241 -0
- package/skills/ima2-front/references/asset-requirements.md +431 -0
- package/skills/ima2-front/references/brand-asset-sourcing.md +122 -0
- package/skills/ima2-front/references/color-system.md +132 -0
- package/skills/ima2-front/references/consistency-locks.md +20 -0
- package/skills/ima2-front/references/crud-ui.md +53 -0
- package/skills/ima2-front/references/dropdown-layer.md +110 -0
- package/skills/ima2-front/references/i18n-global.md +104 -0
- package/skills/ima2-front/references/iterative-design.md +90 -0
- package/skills/ima2-front/references/korea-2026.md +174 -0
- package/skills/ima2-front/references/layout-discipline.md +161 -0
- package/skills/ima2-front/references/liquid-glass.md +181 -0
- package/skills/ima2-front/references/logo-sections.md +194 -0
- package/skills/ima2-front/references/mobile-ux.md +144 -0
- package/skills/ima2-front/references/performance-budget.md +106 -0
- package/skills/ima2-front/references/preflight-full.md +58 -0
- package/skills/ima2-front/references/product-density.md +53 -0
- package/skills/ima2-front/references/prototype-variants.md +54 -0
- package/skills/ima2-front/references/reference-capture.md +120 -0
- package/skills/ima2-front/references/responsive-viewport.md +139 -0
- package/skills/ima2-front/references/seo-baseline.md +123 -0
- package/skills/ima2-front/references/soft-3d-asset-gates.md +70 -0
- package/skills/ima2-front/references/stacks/astro.md +204 -0
- package/skills/ima2-front/references/stacks/mobile-native.md +281 -0
- package/skills/ima2-front/references/stacks/nextjs.md +219 -0
- package/skills/ima2-front/references/stacks/react.md +264 -0
- package/skills/ima2-front/references/stacks/svelte.md +235 -0
- package/skills/ima2-front/references/stacks/vanilla.md +226 -0
- package/skills/ima2-front/references/theme-switching.md +82 -0
- package/skills/ima2-front/references/top-bar.md +126 -0
- package/skills/ima2-front/references/typography-wrapping.md +270 -0
- package/skills/ima2-front/references/ux-writing-ko.md +62 -0
- package/skills/ima2-front/references/visual-verification.md +102 -0
- package/skills/ima2-uiux/SKILL.md +477 -0
- package/skills/ima2-uiux/references/anti-slop-judgment.md +39 -0
- package/skills/ima2-uiux/references/color-system.md +80 -0
- package/skills/ima2-uiux/references/design-isms.md +194 -0
- package/skills/ima2-uiux/references/design-read-example.md +33 -0
- package/skills/ima2-uiux/references/design-system-bootstrap.md +133 -0
- package/skills/ima2-uiux/references/favicon-logo.md +355 -0
- package/skills/ima2-uiux/references/form-patterns.md +118 -0
- package/skills/ima2-uiux/references/intent-discovery-ladder.md +86 -0
- package/skills/ima2-uiux/references/korean-design-vocabulary.md +52 -0
- package/skills/ima2-uiux/references/layout-macrostructures.md +54 -0
- package/skills/ima2-uiux/references/logo-trust-sections.md +96 -0
- package/skills/ima2-uiux/references/mobile-native-ux.md +207 -0
- package/skills/ima2-uiux/references/product-personalities.md +118 -0
- package/skills/ima2-uiux/references/responsive-nav.md +66 -0
- package/skills/ima2-uiux/references/typography-line-breaks.md +193 -0
- package/skills/ima2-uiux/references/ux-preflight.md +65 -0
- package/skills/ima2-uiux/references/ux-states.md +197 -0
- package/skills/ima2-uiux/references/visual-hierarchy.md +115 -0
- package/ui/dist/.vite/manifest.json +13 -13
- package/ui/dist/assets/AgentWorkspace-CtYt4SF5.js +3 -0
- package/ui/dist/assets/{CardNewsWorkspace-Cdxn1ohU.js → CardNewsWorkspace-DN5cVqG5.js} +1 -1
- package/ui/dist/assets/GenerationRequestLogPanel-DnmUQdKR.js +1 -0
- package/ui/dist/assets/{NodeCanvas-C67sLboA.js → NodeCanvas-CxpfIBI0.js} +1 -1
- package/ui/dist/assets/PromptBuilderPanel-C2R3ObtR.js +2 -0
- package/ui/dist/assets/{PromptImportDialog-DeQXsEEv.js → PromptImportDialog-BIxz_yEx.js} +2 -2
- package/ui/dist/assets/PromptImportDiscoverySection-BQTlPMME.js +1 -0
- package/ui/dist/assets/PromptImportFolderSection-Ds_Th9O3.js +1 -0
- package/ui/dist/assets/{PromptLibraryPanel-Db3xEPM_.js → PromptLibraryPanel-Rs4l45pV.js} +2 -2
- package/ui/dist/assets/{SettingsWorkspace-BDk_1W-V.js → SettingsWorkspace-p1AZ6uT9.js} +1 -1
- package/ui/dist/assets/index-0-_vgFGs.js +4 -0
- package/ui/dist/assets/index-Dm3pFxV4.js +23 -0
- package/ui/dist/assets/index-J8yDF3Ch.css +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/AgentWorkspace-DAg_7zpY.js +0 -3
- package/ui/dist/assets/GenerationRequestLogPanel-C_qvEUi2.js +0 -1
- package/ui/dist/assets/PromptBuilderPanel-DjvVdAie.js +0 -2
- package/ui/dist/assets/PromptImportDiscoverySection-CrwyUKYI.js +0 -1
- package/ui/dist/assets/PromptImportFolderSection-CToGMqYE.js +0 -1
- package/ui/dist/assets/index-2oG6in1i.css +0 -1
- package/ui/dist/assets/index-CoxMOXAU.js +0 -23
- package/ui/dist/assets/index-p8egnGHk.js +0 -4
package/routes/keys.js
CHANGED
|
@@ -1,13 +1,31 @@
|
|
|
1
1
|
import { readFile, writeFile, rename } from "node:fs/promises";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
2
3
|
import { initVertexAuth, clearVertexAuth } from "../lib/vertexAuth.js";
|
|
3
4
|
// Atomic + 0600 config write: temp file then rename, so a crash or concurrent
|
|
4
5
|
// save can't corrupt config.json (which may hold API keys). Rename also forces
|
|
5
6
|
// 0600 perms even if a looser-perm config pre-existed.
|
|
6
7
|
async function writeConfigAtomic(cfgPath, data) {
|
|
7
|
-
const tmp = `${cfgPath}.${process.pid}.tmp`;
|
|
8
|
+
const tmp = `${cfgPath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
|
|
8
9
|
await writeFile(tmp, JSON.stringify(data, null, 2), { mode: 0o600 });
|
|
9
10
|
await rename(tmp, cfgPath);
|
|
10
11
|
}
|
|
12
|
+
let configMutationQueue = Promise.resolve();
|
|
13
|
+
function serializeConfigMutation(mutation) {
|
|
14
|
+
const result = configMutationQueue.then(mutation, mutation);
|
|
15
|
+
configMutationQueue = result.then(() => undefined, () => undefined);
|
|
16
|
+
return result;
|
|
17
|
+
}
|
|
18
|
+
async function updateConfigFile(cfgPath, mutate) {
|
|
19
|
+
await serializeConfigMutation(async () => {
|
|
20
|
+
let existing = {};
|
|
21
|
+
try {
|
|
22
|
+
existing = JSON.parse(await readFile(cfgPath, "utf-8"));
|
|
23
|
+
}
|
|
24
|
+
catch { /* new file */ }
|
|
25
|
+
mutate(existing);
|
|
26
|
+
await writeConfigAtomic(cfgPath, existing);
|
|
27
|
+
});
|
|
28
|
+
}
|
|
11
29
|
const KEY_PREFIX_MAP = {
|
|
12
30
|
openai: ["sk-"],
|
|
13
31
|
xai: ["xai-"],
|
|
@@ -74,13 +92,7 @@ export function mountKeyRoutes(app, ctx) {
|
|
|
74
92
|
return res.status(400).json({ ok: false, error: "mode must be apikey|vertex", code: "INVALID_MODE" });
|
|
75
93
|
}
|
|
76
94
|
const cfgPath = ctx.config.storage.configFile;
|
|
77
|
-
|
|
78
|
-
try {
|
|
79
|
-
existing = JSON.parse(await readFile(cfgPath, "utf-8"));
|
|
80
|
-
}
|
|
81
|
-
catch { /* new file */ }
|
|
82
|
-
existing.geminiAuthMode = mode;
|
|
83
|
-
await writeConfigAtomic(cfgPath, existing);
|
|
95
|
+
await updateConfigFile(cfgPath, (existing) => { existing.geminiAuthMode = mode; });
|
|
84
96
|
ctx.geminiAuthMode = mode;
|
|
85
97
|
return res.json({ ok: true, geminiAuthMode: mode });
|
|
86
98
|
});
|
|
@@ -117,14 +129,10 @@ export function mountKeyRoutes(app, ctx) {
|
|
|
117
129
|
}
|
|
118
130
|
// Save to config.json
|
|
119
131
|
const cfgPath = ctx.config.storage.configFile;
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
existing =
|
|
123
|
-
}
|
|
124
|
-
catch { /* new file */ }
|
|
125
|
-
existing.vertexServiceAccountJson = trimmed;
|
|
126
|
-
existing.geminiAuthMode = "vertex";
|
|
127
|
-
await writeConfigAtomic(cfgPath, existing);
|
|
132
|
+
await updateConfigFile(cfgPath, (existing) => {
|
|
133
|
+
existing.vertexServiceAccountJson = trimmed;
|
|
134
|
+
existing.geminiAuthMode = "vertex";
|
|
135
|
+
});
|
|
128
136
|
// Hot-update runtime
|
|
129
137
|
ctx.vertexServiceAccountJson = trimmed;
|
|
130
138
|
ctx.vertexProjectId = parsed.project_id;
|
|
@@ -140,13 +148,7 @@ export function mountKeyRoutes(app, ctx) {
|
|
|
140
148
|
return res.status(400).json({ ok: false, error: "Cannot remove env-sourced key", code: "ENV_KEY_IMMUTABLE" });
|
|
141
149
|
}
|
|
142
150
|
const cfgPath = ctx.config.storage.configFile;
|
|
143
|
-
|
|
144
|
-
try {
|
|
145
|
-
existing = JSON.parse(await readFile(cfgPath, "utf-8"));
|
|
146
|
-
}
|
|
147
|
-
catch { /* ignore */ }
|
|
148
|
-
delete existing.vertexServiceAccountJson;
|
|
149
|
-
await writeConfigAtomic(cfgPath, existing);
|
|
151
|
+
await updateConfigFile(cfgPath, (existing) => { delete existing.vertexServiceAccountJson; });
|
|
150
152
|
clearVertexAuth();
|
|
151
153
|
ctx.vertexServiceAccountJson = undefined;
|
|
152
154
|
ctx.vertexProjectId = undefined;
|
|
@@ -201,15 +203,11 @@ export function mountKeyRoutes(app, ctx) {
|
|
|
201
203
|
}
|
|
202
204
|
// Save to config.json
|
|
203
205
|
const cfgPath = ctx.config.storage.configFile;
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
existing[CONFIG_KEY_MAP[provider]] = trimmed;
|
|
210
|
-
if (provider === "gemini")
|
|
211
|
-
existing.geminiAuthMode = "apikey";
|
|
212
|
-
await writeConfigAtomic(cfgPath, existing);
|
|
206
|
+
await updateConfigFile(cfgPath, (existing) => {
|
|
207
|
+
existing[CONFIG_KEY_MAP[provider]] = trimmed;
|
|
208
|
+
if (provider === "gemini")
|
|
209
|
+
existing.geminiAuthMode = "apikey";
|
|
210
|
+
});
|
|
213
211
|
// Hot-update runtime context
|
|
214
212
|
if (provider === "openai") {
|
|
215
213
|
ctx.apiKey = trimmed;
|
|
@@ -245,13 +243,7 @@ export function mountKeyRoutes(app, ctx) {
|
|
|
245
243
|
}
|
|
246
244
|
// Remove from config.json
|
|
247
245
|
const cfgPath = ctx.config.storage.configFile;
|
|
248
|
-
|
|
249
|
-
try {
|
|
250
|
-
existing = JSON.parse(await readFile(cfgPath, "utf-8"));
|
|
251
|
-
}
|
|
252
|
-
catch { /* ignore */ }
|
|
253
|
-
delete existing[CONFIG_KEY_MAP[provider]];
|
|
254
|
-
await writeConfigAtomic(cfgPath, existing);
|
|
246
|
+
await updateConfigFile(cfgPath, (existing) => { delete existing[CONFIG_KEY_MAP[provider]]; });
|
|
255
247
|
// Clear runtime
|
|
256
248
|
if (provider === "openai") {
|
|
257
249
|
ctx.apiKey = undefined;
|
package/routes/multimode.js
CHANGED
|
@@ -1,506 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { safeWriteSidecar } from "../lib/atomicWrite.js";
|
|
3
|
-
import { join } from "path";
|
|
4
|
-
import { randomBytes } from "crypto";
|
|
5
|
-
import { detectImageMimeFromB64, summarizeReferencePayload, validateAndNormalizeRefs } from "../lib/refs.js";
|
|
6
|
-
import { generateImageThumbnailFromBuffer } from "../lib/imageThumb.js";
|
|
7
|
-
import { classifyUpstreamError } from "../lib/errorClassify.js";
|
|
8
|
-
import { normalizeOAuthParams } from "../lib/oauthNormalize.js";
|
|
9
|
-
import { resolveProviderOptions } from "../lib/providerOptions.js";
|
|
10
|
-
import { generateMultimodeViaResponses } from "../lib/responsesImageAdapter.js";
|
|
11
|
-
import { generateMultimodeViaGrok } from "../lib/grokMultimodeAdapter.js";
|
|
12
|
-
import { generateViaAgy } from "../lib/agyImageAdapter.js";
|
|
13
|
-
import { generateViaGeminiApi } from "../lib/geminiApiImageAdapter.js";
|
|
14
|
-
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, INFLIGHT_RETRY_AFTER_SECONDS } from "../lib/inflight.js";
|
|
15
|
-
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "../lib/generationCancel.js";
|
|
16
|
-
import { logEvent, logError } from "../lib/logger.js";
|
|
17
|
-
import { embedImageMetadataBestEffort } from "../lib/imageMetadataStore.js";
|
|
18
|
-
import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
19
|
-
import { normalizeComposerInsertedPrompts, normalizeComposerPrompt, } from "../lib/composerSnapshot.js";
|
|
20
|
-
import { errInfo } from "../lib/errInfo.js";
|
|
1
|
+
import { runMultimodePipeline } from "../lib/multimodePipeline.js";
|
|
21
2
|
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
22
|
-
import { validateModeration, imageFormatFromMime, writeSse } from "../lib/routeHelpers.js";
|
|
23
|
-
import { publish } from "../lib/eventBus.js";
|
|
24
|
-
import { publishJobEvent } from "../lib/ssePublish.js";
|
|
25
|
-
import { normalizeMaxImages, sequenceStatus, } from "../lib/multimodeHelpers.js";
|
|
26
|
-
function dualEmitMultimode(res, requestId, event, data) {
|
|
27
|
-
if (!res.writableEnded)
|
|
28
|
-
writeSse(res, event, data);
|
|
29
|
-
if (event === "done") {
|
|
30
|
-
publishJobEvent(requestId, event, data);
|
|
31
|
-
}
|
|
32
|
-
else {
|
|
33
|
-
publish(requestId, event, data);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
function respondMultimodeValidationError(res, requestId, asyncMode, status, payload) {
|
|
37
|
-
publish(requestId, "error", payload);
|
|
38
|
-
if (asyncMode && !res.headersSent) {
|
|
39
|
-
return res.status(status).json(payload);
|
|
40
|
-
}
|
|
41
|
-
if (!res.writableEnded) {
|
|
42
|
-
writeSse(res, "error", payload);
|
|
43
|
-
res.end();
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
3
|
export function registerMultimodeRoutes(app, ctxRaw) {
|
|
47
4
|
const ctx = requireRuntimeContext(ctxRaw);
|
|
48
|
-
app.post("/api/generate/multimode",
|
|
49
|
-
const requestId = typeof req.body?.requestId === "string" ? req.body.requestId : req.id;
|
|
50
|
-
const asyncMode = req.body?.async === true;
|
|
51
|
-
let finishStatus = "completed";
|
|
52
|
-
let finishHttpStatus = 200;
|
|
53
|
-
let finishErrorCode;
|
|
54
|
-
let finishMeta = {};
|
|
55
|
-
let finishCanceled = false;
|
|
56
|
-
const cancelController = new AbortController();
|
|
57
|
-
const images = [];
|
|
58
|
-
const persistedIndexes = new Set();
|
|
59
|
-
let routeMaxImages = 0;
|
|
60
|
-
let routeSequenceId = "";
|
|
61
|
-
let routeStartTime = Date.now();
|
|
62
|
-
let routeActiveProvider = "auto";
|
|
63
|
-
let routeQuality = "medium";
|
|
64
|
-
let routeEffectiveSize = "1024x1024";
|
|
65
|
-
let routeModeration = "low";
|
|
66
|
-
let routeImageModel = null;
|
|
67
|
-
let routeWebSearchEnabled = true;
|
|
68
|
-
let routePromptMode = "auto";
|
|
69
|
-
let routeQualityWarnings = [];
|
|
70
|
-
let routeComposerPrompt = null;
|
|
71
|
-
let routeComposerInsertedPrompts = [];
|
|
72
|
-
let latestUsage = null;
|
|
73
|
-
let latestWebSearchCalls = 0;
|
|
74
|
-
let latestExtraIgnored = 0;
|
|
75
|
-
if (!asyncMode) {
|
|
76
|
-
res.setHeader("Content-Type", "text/event-stream; charset=utf-8");
|
|
77
|
-
res.setHeader("Cache-Control", "no-cache, no-transform");
|
|
78
|
-
res.setHeader("Connection", "keep-alive");
|
|
79
|
-
res.flushHeaders?.();
|
|
80
|
-
}
|
|
81
|
-
try {
|
|
82
|
-
const { prompt, quality: rawQuality = "medium", size = "1024x1024", format = "png", moderation = "low", provider = "auto", references = [], mode: promptMode = "auto", model: rawModel, reasoningEffort: rawReasoningEffort, webSearchEnabled: rawWebSearchEnabled = true, } = req.body;
|
|
83
|
-
const composerPrompt = normalizeComposerPrompt(req.body?.composerPrompt);
|
|
84
|
-
const composerInsertedPrompts = normalizeComposerInsertedPrompts(req.body?.composerInsertedPrompts);
|
|
85
|
-
const maxImages = normalizeMaxImages(req.body?.maxImages, ctx.config.limits.maxGeneratedImages);
|
|
86
|
-
const normalizedPromptMode = promptMode === "direct" ? "direct" : "auto";
|
|
87
|
-
const { quality, warnings: qualityWarnings } = normalizeOAuthParams({ provider, quality: rawQuality });
|
|
88
|
-
const providerOptions = resolveProviderOptions(ctx, {
|
|
89
|
-
provider,
|
|
90
|
-
rawModel,
|
|
91
|
-
rawReasoningEffort,
|
|
92
|
-
rawSize: size,
|
|
93
|
-
rawWebSearchEnabled,
|
|
94
|
-
});
|
|
95
|
-
if (providerOptions.error) {
|
|
96
|
-
finishStatus = "error";
|
|
97
|
-
finishHttpStatus = providerOptions.status;
|
|
98
|
-
finishErrorCode = providerOptions.code;
|
|
99
|
-
return respondMultimodeValidationError(res, requestId, asyncMode, providerOptions.status, {
|
|
100
|
-
error: providerOptions.error,
|
|
101
|
-
code: providerOptions.code,
|
|
102
|
-
status: providerOptions.status,
|
|
103
|
-
requestId,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
const imageModel = providerOptions.model;
|
|
107
|
-
const reasoningEffort = providerOptions.reasoningEffort;
|
|
108
|
-
const effectiveSize = providerOptions.size;
|
|
109
|
-
const webSearchEnabled = providerOptions.webSearchEnabled;
|
|
110
|
-
const activeProvider = providerOptions.provider;
|
|
111
|
-
if (!prompt) {
|
|
112
|
-
finishStatus = "error";
|
|
113
|
-
finishHttpStatus = 400;
|
|
114
|
-
finishErrorCode = "PROMPT_REQUIRED";
|
|
115
|
-
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
116
|
-
error: "Prompt is required",
|
|
117
|
-
code: finishErrorCode,
|
|
118
|
-
status: 400,
|
|
119
|
-
requestId,
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
const moderationCheck = validateModeration(ctx, moderation);
|
|
123
|
-
if (moderationCheck.error) {
|
|
124
|
-
finishStatus = "error";
|
|
125
|
-
finishHttpStatus = 400;
|
|
126
|
-
finishErrorCode = "INVALID_MODERATION";
|
|
127
|
-
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
128
|
-
error: moderationCheck.error,
|
|
129
|
-
code: finishErrorCode,
|
|
130
|
-
status: 400,
|
|
131
|
-
requestId,
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
const refCheckResult = validateAndNormalizeRefs(references);
|
|
135
|
-
if (refCheckResult.error) {
|
|
136
|
-
finishStatus = "error";
|
|
137
|
-
finishHttpStatus = 400;
|
|
138
|
-
finishErrorCode = refCheckResult.code;
|
|
139
|
-
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
140
|
-
error: refCheckResult.error,
|
|
141
|
-
code: refCheckResult.code,
|
|
142
|
-
status: 400,
|
|
143
|
-
requestId,
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
const refCheck = refCheckResult;
|
|
147
|
-
const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http") ? req.body.providerUrl : null;
|
|
148
|
-
const referencePayload = summarizeReferencePayload(references);
|
|
149
|
-
const started = startJob({
|
|
150
|
-
requestId,
|
|
151
|
-
kind: "multimode",
|
|
152
|
-
prompt,
|
|
153
|
-
meta: {
|
|
154
|
-
kind: "multimode",
|
|
155
|
-
quality,
|
|
156
|
-
model: imageModel,
|
|
157
|
-
size: effectiveSize,
|
|
158
|
-
maxImages,
|
|
159
|
-
refsCount: referencePayload.refsCount,
|
|
160
|
-
referenceBytes: referencePayload.referenceBytes,
|
|
161
|
-
referenceB64Chars: referencePayload.referenceB64Chars,
|
|
162
|
-
composerPrompt,
|
|
163
|
-
composerInsertedPrompts,
|
|
164
|
-
},
|
|
165
|
-
});
|
|
166
|
-
if (started && isStartJobFailure(started)) {
|
|
167
|
-
finishStatus = "error";
|
|
168
|
-
finishHttpStatus = started.code === "TOO_MANY_JOBS" ? 429 : 409;
|
|
169
|
-
finishErrorCode = started.code;
|
|
170
|
-
if (started.code === "TOO_MANY_JOBS") {
|
|
171
|
-
res.setHeader("Retry-After", String(INFLIGHT_RETRY_AFTER_SECONDS));
|
|
172
|
-
}
|
|
173
|
-
return respondMultimodeValidationError(res, requestId, asyncMode, finishHttpStatus, {
|
|
174
|
-
error: started.code === "TOO_MANY_JOBS"
|
|
175
|
-
? "Too many concurrent generation jobs"
|
|
176
|
-
: "Request ID already in use",
|
|
177
|
-
code: started.code,
|
|
178
|
-
status: finishHttpStatus,
|
|
179
|
-
requestId,
|
|
180
|
-
});
|
|
181
|
-
}
|
|
182
|
-
registerJobAbortController(requestId, cancelController);
|
|
183
|
-
if (asyncMode)
|
|
184
|
-
res.status(202).json({ requestId });
|
|
185
|
-
logEvent("multimode", "request", {
|
|
186
|
-
requestId,
|
|
187
|
-
quality,
|
|
188
|
-
model: imageModel,
|
|
189
|
-
size: effectiveSize,
|
|
190
|
-
moderation,
|
|
191
|
-
maxImages,
|
|
192
|
-
refs: refCheck.refs.length,
|
|
193
|
-
referenceBytes: referencePayload.referenceBytes,
|
|
194
|
-
promptChars: typeof prompt === "string" ? prompt.length : 0,
|
|
195
|
-
webSearchEnabled,
|
|
196
|
-
});
|
|
197
|
-
const startTime = Date.now();
|
|
198
|
-
const mimeMap = { png: "image/png", jpeg: "image/jpeg", webp: "image/webp" };
|
|
199
|
-
const mmFormat = activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api" ? "jpeg" : String(format);
|
|
200
|
-
const mime = mimeMap[mmFormat] || "image/png";
|
|
201
|
-
const sequenceId = `seq_${Date.now().toString(36)}_${randomBytes(4).toString("hex")}`;
|
|
202
|
-
routeMaxImages = maxImages;
|
|
203
|
-
routeSequenceId = sequenceId;
|
|
204
|
-
routeStartTime = startTime;
|
|
205
|
-
routeActiveProvider = activeProvider ?? "auto";
|
|
206
|
-
routeQuality = quality;
|
|
207
|
-
routeEffectiveSize = effectiveSize;
|
|
208
|
-
routeModeration = moderation;
|
|
209
|
-
routeImageModel = imageModel ?? null;
|
|
210
|
-
routeWebSearchEnabled = webSearchEnabled ?? false;
|
|
211
|
-
routePromptMode = normalizedPromptMode;
|
|
212
|
-
routeQualityWarnings = qualityWarnings;
|
|
213
|
-
routeComposerPrompt = composerPrompt;
|
|
214
|
-
routeComposerInsertedPrompts = composerInsertedPrompts;
|
|
215
|
-
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
216
|
-
const persistAndSendImage = async (image, index, totalReturned, status) => {
|
|
217
|
-
if (persistedIndexes.has(index))
|
|
218
|
-
return;
|
|
219
|
-
throwIfJobCanceled(requestId);
|
|
220
|
-
const resultMime = activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api"
|
|
221
|
-
? (image.mime || detectImageMimeFromB64(image.b64) || mime)
|
|
222
|
-
: mime;
|
|
223
|
-
const resultFormat = activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api" ? imageFormatFromMime(resultMime) : mmFormat;
|
|
224
|
-
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
225
|
-
const filename = `${Date.now()}_${rand}_multimode_${index}.${resultFormat}`;
|
|
226
|
-
const createdAt = Date.now();
|
|
227
|
-
const meta = {
|
|
228
|
-
kind: "multimode-image",
|
|
229
|
-
generationStrategy: "one-call-text-sequence",
|
|
230
|
-
sequenceId,
|
|
231
|
-
sequenceIndex: index + 1,
|
|
232
|
-
sequenceTotalRequested: maxImages,
|
|
233
|
-
sequenceTotalReturned: totalReturned,
|
|
234
|
-
sequenceStatus: status,
|
|
235
|
-
stageLabel: String.fromCharCode(65 + index),
|
|
236
|
-
requestId,
|
|
237
|
-
prompt,
|
|
238
|
-
userPrompt: prompt,
|
|
239
|
-
revisedPrompt: image.revisedPrompt || null,
|
|
240
|
-
promptMode: normalizedPromptMode,
|
|
241
|
-
composerPrompt,
|
|
242
|
-
composerInsertedPrompts,
|
|
243
|
-
quality,
|
|
244
|
-
size: effectiveSize,
|
|
245
|
-
format: resultFormat,
|
|
246
|
-
moderation,
|
|
247
|
-
model: activeProvider === "grok" ? (quality === "high" ? "grok-imagine-image-quality" : imageModel) : imageModel,
|
|
248
|
-
provider: activeProvider,
|
|
249
|
-
createdAt,
|
|
250
|
-
usage: latestUsage,
|
|
251
|
-
webSearchCalls: latestWebSearchCalls,
|
|
252
|
-
webSearchEnabled,
|
|
253
|
-
refsCount: refCheck.refs.length,
|
|
254
|
-
...(image.providerUrl ? { providerUrl: image.providerUrl } : {}),
|
|
255
|
-
};
|
|
256
|
-
const rawBuffer = Buffer.from(image.b64, "base64");
|
|
257
|
-
const embedded = await embedImageMetadataBestEffort(rawBuffer, resultFormat, meta, {
|
|
258
|
-
version: ctx.packageVersion,
|
|
259
|
-
});
|
|
260
|
-
const mmFilePath = join(ctx.config.storage.generatedDir, filename);
|
|
261
|
-
await writeFile(mmFilePath, embedded.buffer);
|
|
262
|
-
await safeWriteSidecar(mmFilePath + ".json", meta);
|
|
263
|
-
generateImageThumbnailFromBuffer(embedded.buffer, mmFilePath).catch(() => { });
|
|
264
|
-
invalidateHistoryIndex();
|
|
265
|
-
const item = {
|
|
266
|
-
image: `data:${resultMime};base64,${image.b64}`,
|
|
267
|
-
filename,
|
|
268
|
-
createdAt,
|
|
269
|
-
...(image.providerUrl ? { providerUrl: image.providerUrl } : {}),
|
|
270
|
-
revisedPrompt: image.revisedPrompt || null,
|
|
271
|
-
sequenceId,
|
|
272
|
-
sequenceIndex: index + 1,
|
|
273
|
-
sequenceTotalRequested: maxImages,
|
|
274
|
-
sequenceTotalReturned: totalReturned,
|
|
275
|
-
sequenceStatus: status,
|
|
276
|
-
};
|
|
277
|
-
persistedIndexes.add(index);
|
|
278
|
-
images.push(item);
|
|
279
|
-
dualEmitMultimode(res, requestId, "image", item);
|
|
280
|
-
};
|
|
281
|
-
dualEmitMultimode(res, requestId, "phase", { phase: "streaming", requestId, sequenceId, maxImages });
|
|
282
|
-
let generated;
|
|
283
|
-
if (activeProvider === "gemini-api") {
|
|
284
|
-
const r = await generateViaGeminiApi(prompt, requireRuntimeContext(ctx), {
|
|
285
|
-
model: imageModel,
|
|
286
|
-
size: effectiveSize,
|
|
287
|
-
signal: cancelController.signal,
|
|
288
|
-
requestId,
|
|
289
|
-
references: refCheck.refDetails,
|
|
290
|
-
});
|
|
291
|
-
generated = {
|
|
292
|
-
images: [{ b64: r.b64, revisedPrompt: r.revisedPrompt }],
|
|
293
|
-
usage: r.usage,
|
|
294
|
-
webSearchCalls: r.webSearchCalls,
|
|
295
|
-
};
|
|
296
|
-
}
|
|
297
|
-
else if (activeProvider === "agy") {
|
|
298
|
-
const r = await generateViaAgy(prompt, {
|
|
299
|
-
references: refCheck.refDetails,
|
|
300
|
-
signal: cancelController.signal,
|
|
301
|
-
requestId,
|
|
302
|
-
});
|
|
303
|
-
generated = {
|
|
304
|
-
images: [{ b64: r.b64, revisedPrompt: r.revisedPrompt }],
|
|
305
|
-
usage: r.usage,
|
|
306
|
-
webSearchCalls: r.webSearchCalls,
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
else if (activeProvider === "grok" || activeProvider === "grok-api") {
|
|
310
|
-
const directApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
|
|
311
|
-
const grokModel = quality === "high" ? "grok-imagine-image-quality" : imageModel;
|
|
312
|
-
const grokRefs = incomingProviderUrl
|
|
313
|
-
? [{ b64: "", url: incomingProviderUrl }, ...refCheck.refDetails]
|
|
314
|
-
: refCheck.refDetails;
|
|
315
|
-
generated = await generateMultimodeViaGrok(prompt, ctx, {
|
|
316
|
-
model: grokModel,
|
|
317
|
-
maxImages,
|
|
318
|
-
size: effectiveSize,
|
|
319
|
-
signal: cancelController.signal,
|
|
320
|
-
requestId,
|
|
321
|
-
references: grokRefs,
|
|
322
|
-
directApiKey,
|
|
323
|
-
onFinalImage: async (image, index) => {
|
|
324
|
-
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
325
|
-
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
326
|
-
},
|
|
327
|
-
});
|
|
328
|
-
}
|
|
329
|
-
else {
|
|
330
|
-
generated = await generateMultimodeViaResponses(activeProvider, prompt, quality, effectiveSize, moderation, refCheck.refDetails || refCheck.refs, requestId, normalizedPromptMode, ctx, {
|
|
331
|
-
model: imageModel,
|
|
332
|
-
maxImages,
|
|
333
|
-
reasoningEffort,
|
|
334
|
-
webSearchEnabled,
|
|
335
|
-
onPartialImage: (partial) => {
|
|
336
|
-
if (isJobCanceled(requestId))
|
|
337
|
-
return;
|
|
338
|
-
const pd = { image: `data:${mime};base64,${partial.b64}`, requestId, sequenceId, index: partial.index };
|
|
339
|
-
if (!res.writableEnded && !res.destroyed)
|
|
340
|
-
writeSse(res, "partial", pd);
|
|
341
|
-
publish(requestId, "partial", pd);
|
|
342
|
-
},
|
|
343
|
-
onFinalImage: async (image, index) => {
|
|
344
|
-
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
345
|
-
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
346
|
-
},
|
|
347
|
-
signal: cancelController.signal,
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
throwIfJobCanceled(requestId);
|
|
351
|
-
latestUsage = generated.usage || null;
|
|
352
|
-
latestWebSearchCalls = generated.webSearchCalls || 0;
|
|
353
|
-
latestExtraIgnored = generated.extraIgnored || 0;
|
|
354
|
-
for (const [index, image] of generated.images.entries()) {
|
|
355
|
-
await persistAndSendImage(image, index, generated.images.length, sequenceStatus(generated.images.length, maxImages));
|
|
356
|
-
}
|
|
357
|
-
const returned = images.length;
|
|
358
|
-
const status = sequenceStatus(returned, maxImages);
|
|
359
|
-
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
360
|
-
if (returned === 0) {
|
|
361
|
-
finishStatus = "error";
|
|
362
|
-
finishHttpStatus = 422;
|
|
363
|
-
finishErrorCode = "EMPTY_RESPONSE";
|
|
364
|
-
finishMeta = { sequenceId, filenames: [], imageCount: 0, maxImages, status, composerPrompt: routeComposerPrompt, composerInsertedPrompts: routeComposerInsertedPrompts };
|
|
365
|
-
dualEmitMultimode(res, requestId, "error", {
|
|
366
|
-
error: "No image data returned from the multimode stream",
|
|
367
|
-
code: finishErrorCode,
|
|
368
|
-
status: finishHttpStatus,
|
|
369
|
-
requestId, sequenceId, requested: maxImages, returned,
|
|
370
|
-
});
|
|
371
|
-
logEvent("multimode", "empty_response", { requestId, sequenceId, maxImages, elapsedMs: Date.now() - startTime });
|
|
372
|
-
return;
|
|
373
|
-
}
|
|
374
|
-
finishMeta = {
|
|
375
|
-
sequenceId,
|
|
376
|
-
filenames: images.map((image) => image.filename),
|
|
377
|
-
imageCount: returned,
|
|
378
|
-
maxImages,
|
|
379
|
-
status,
|
|
380
|
-
composerPrompt: routeComposerPrompt,
|
|
381
|
-
composerInsertedPrompts: routeComposerInsertedPrompts,
|
|
382
|
-
};
|
|
383
|
-
finishHttpStatus = 200;
|
|
384
|
-
dualEmitMultimode(res, requestId, "done", {
|
|
385
|
-
ok: true,
|
|
386
|
-
requestId,
|
|
387
|
-
sequenceId,
|
|
388
|
-
requested: maxImages,
|
|
389
|
-
returned,
|
|
390
|
-
status,
|
|
391
|
-
elapsed,
|
|
392
|
-
images,
|
|
393
|
-
provider: activeProvider,
|
|
394
|
-
quality,
|
|
395
|
-
size: effectiveSize,
|
|
396
|
-
moderation,
|
|
397
|
-
model: imageModel,
|
|
398
|
-
usage: latestUsage,
|
|
399
|
-
webSearchCalls: latestWebSearchCalls,
|
|
400
|
-
webSearchEnabled,
|
|
401
|
-
warnings: qualityWarnings,
|
|
402
|
-
extraIgnored: latestExtraIgnored,
|
|
403
|
-
promptMode: normalizedPromptMode,
|
|
404
|
-
});
|
|
405
|
-
logEvent("multimode", "saved", {
|
|
406
|
-
requestId,
|
|
407
|
-
sequenceId,
|
|
408
|
-
imageCount: returned,
|
|
409
|
-
maxImages,
|
|
410
|
-
status,
|
|
411
|
-
elapsedMs: Date.now() - startTime,
|
|
412
|
-
});
|
|
413
|
-
}
|
|
414
|
-
catch (e) {
|
|
415
|
-
const err = errInfo(e);
|
|
416
|
-
const ext = (err.raw && typeof err.raw === "object" ? err.raw : {});
|
|
417
|
-
const fallbackCode = err.code || classifyUpstreamError(err.message);
|
|
418
|
-
if (isGenerationCanceledError(err.raw) || isJobCanceled(requestId)) {
|
|
419
|
-
const canceled = makeGenerationCanceledError();
|
|
420
|
-
finishCanceled = true;
|
|
421
|
-
finishHttpStatus = canceled.status;
|
|
422
|
-
finishErrorCode = canceled.code;
|
|
423
|
-
dualEmitMultimode(res, requestId, "error", {
|
|
424
|
-
error: canceled.message,
|
|
425
|
-
code: canceled.code,
|
|
426
|
-
status: canceled.status,
|
|
427
|
-
requestId,
|
|
428
|
-
});
|
|
429
|
-
return;
|
|
430
|
-
}
|
|
431
|
-
if ((fallbackCode === "RESPONSES_IMAGE_TIMEOUT" || err.status === 504) && images.length > 0) {
|
|
432
|
-
const status = sequenceStatus(images.length, routeMaxImages);
|
|
433
|
-
const elapsed = ((Date.now() - routeStartTime) / 1000).toFixed(1);
|
|
434
|
-
finishStatus = "completed";
|
|
435
|
-
finishHttpStatus = 206;
|
|
436
|
-
finishMeta = {
|
|
437
|
-
sequenceId: routeSequenceId,
|
|
438
|
-
filenames: images.map((image) => image.filename),
|
|
439
|
-
imageCount: images.length,
|
|
440
|
-
maxImages: routeMaxImages,
|
|
441
|
-
status,
|
|
442
|
-
partialErrorCode: "RESPONSES_IMAGE_TIMEOUT",
|
|
443
|
-
composerPrompt: routeComposerPrompt,
|
|
444
|
-
composerInsertedPrompts: routeComposerInsertedPrompts,
|
|
445
|
-
};
|
|
446
|
-
dualEmitMultimode(res, requestId, "done", {
|
|
447
|
-
ok: true,
|
|
448
|
-
partial: true,
|
|
449
|
-
requestId,
|
|
450
|
-
sequenceId: routeSequenceId,
|
|
451
|
-
requested: routeMaxImages,
|
|
452
|
-
returned: images.length,
|
|
453
|
-
status,
|
|
454
|
-
elapsed,
|
|
455
|
-
images,
|
|
456
|
-
provider: routeActiveProvider,
|
|
457
|
-
quality: routeQuality,
|
|
458
|
-
size: routeEffectiveSize,
|
|
459
|
-
moderation: routeModeration,
|
|
460
|
-
model: routeImageModel,
|
|
461
|
-
usage: latestUsage,
|
|
462
|
-
webSearchCalls: latestWebSearchCalls,
|
|
463
|
-
webSearchEnabled: routeWebSearchEnabled,
|
|
464
|
-
warnings: routeQualityWarnings,
|
|
465
|
-
extraIgnored: latestExtraIgnored,
|
|
466
|
-
promptMode: routePromptMode,
|
|
467
|
-
warning: {
|
|
468
|
-
code: "RESPONSES_IMAGE_TIMEOUT",
|
|
469
|
-
message: "The provider timed out after returning partial multimode results.",
|
|
470
|
-
},
|
|
471
|
-
});
|
|
472
|
-
logEvent("multimode", "partial_timeout", {
|
|
473
|
-
requestId,
|
|
474
|
-
sequenceId: routeSequenceId,
|
|
475
|
-
imageCount: images.length,
|
|
476
|
-
maxImages: routeMaxImages,
|
|
477
|
-
});
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
finishStatus = "error";
|
|
481
|
-
finishHttpStatus = err.status || 500;
|
|
482
|
-
finishErrorCode = fallbackCode || "MULTIMODE_GENERATE_FAILED";
|
|
483
|
-
logError("multimode", "error", err.raw, { requestId, code: finishErrorCode });
|
|
484
|
-
dualEmitMultimode(res, requestId, "error", {
|
|
485
|
-
error: err.message,
|
|
486
|
-
code: finishErrorCode,
|
|
487
|
-
status: finishHttpStatus,
|
|
488
|
-
requestId,
|
|
489
|
-
upstreamCode: ext.upstreamCode || null,
|
|
490
|
-
upstreamType: ext.upstreamType || null,
|
|
491
|
-
upstreamParam: ext.upstreamParam || null,
|
|
492
|
-
});
|
|
493
|
-
}
|
|
494
|
-
finally {
|
|
495
|
-
finishJob(requestId, {
|
|
496
|
-
canceled: finishCanceled,
|
|
497
|
-
status: finishStatus,
|
|
498
|
-
httpStatus: finishHttpStatus,
|
|
499
|
-
errorCode: finishErrorCode,
|
|
500
|
-
meta: finishMeta,
|
|
501
|
-
});
|
|
502
|
-
if (!res.writableEnded)
|
|
503
|
-
res.end();
|
|
504
|
-
}
|
|
505
|
-
});
|
|
5
|
+
app.post("/api/generate/multimode", (req, res) => runMultimodePipeline(req, res, ctx));
|
|
506
6
|
}
|