ima2-gen 2.0.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +10 -1
- package/bin/commands/backfillThumbs.js +6 -0
- package/bin/commands/gen.js +6 -0
- package/bin/ima2.js +14 -10
- package/docs/API.md +131 -8
- package/docs/CLI.md +2 -1
- package/docs/FAQ.ko.md +16 -0
- package/docs/FAQ.md +30 -0
- package/docs/README.ko.md +7 -3
- package/docs/migration/runtime-test-inventory.md +15 -1
- package/lib/agentImageVideoGen.js +261 -0
- package/lib/agentRuntime.js +7 -262
- package/lib/agyImageAdapter.js +35 -8
- package/lib/errorClassify.js +8 -7
- package/lib/eventBus.js +71 -0
- package/lib/geminiApiImageAdapter.js +16 -20
- package/lib/generationErrors.js +3 -1
- package/lib/grokImageAdapter.js +68 -129
- package/lib/grokImageCore.js +153 -0
- package/lib/grokMultimodeAdapter.js +5 -3
- package/lib/grokVideoCanvas.js +13 -0
- package/lib/grokVideoPlannerPrompt.js +53 -6
- package/lib/historyList.js +1 -0
- package/lib/inflight.js +54 -17
- package/lib/multimodeHelpers.js +10 -0
- package/lib/nodeHelpers.js +59 -0
- package/lib/oauthProxy/prompts.js +30 -36
- package/lib/promptBuilder/systemPrompt.js +2 -5
- package/lib/promptSafetyPolicy.js +1 -5
- package/lib/responsesFallback.js +2 -1
- package/lib/routeHelpers.js +44 -0
- package/lib/ssePublish.js +12 -0
- package/lib/storyboardPrefix.js +28 -0
- package/lib/thumbBackfill.js +16 -5
- package/package.json +4 -1
- package/routes/agy.js +44 -0
- package/routes/auth.js +6 -2
- package/routes/edit.js +7 -1
- package/routes/events.js +78 -0
- package/routes/generate.js +99 -127
- package/routes/index.js +4 -0
- package/routes/multimode.js +99 -56
- package/routes/nodes.js +59 -103
- package/routes/video.js +100 -17
- package/skills/ima2/SKILL.md +98 -21
- package/ui/dist/.vite/manifest.json +12 -12
- package/ui/dist/assets/{AgentWorkspace-CYv84Rus.js → AgentWorkspace-Dth6YijN.js} +1 -1
- package/ui/dist/assets/{CardNewsWorkspace-Dqyc1WZ1.js → CardNewsWorkspace-Dav3K5CT.js} +1 -1
- package/ui/dist/assets/{NodeCanvas-ChEXzQbb.js → NodeCanvas-C4ifFzB1.js} +1 -1
- package/ui/dist/assets/{PromptBuilderPanel-B95ZufnR.js → PromptBuilderPanel-CEcyU9PL.js} +1 -1
- package/ui/dist/assets/{PromptImportDialog-DGOwFQET.js → PromptImportDialog-CgQ94Gth.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-CgvdnR49.js → PromptImportDiscoverySection-CuzyzbNI.js} +1 -1
- package/ui/dist/assets/{PromptImportFolderSection-CfUye9J8.js → PromptImportFolderSection-DHLGlO6l.js} +1 -1
- package/ui/dist/assets/{PromptLibraryPanel-B9kndPw1.js → PromptLibraryPanel-BOe18we8.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-Cdgnm4Wa.js +1 -0
- package/ui/dist/assets/{index-BhcvL0g-.js → index-C5PSahkr.js} +1 -1
- package/ui/dist/assets/index-Dn2AhL6d.css +1 -0
- package/ui/dist/assets/index-Tjqx6wUV.js +23 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/SettingsWorkspace-B3tgLrmF.js +0 -1
- package/ui/dist/assets/index-BtK3YhJc.js +0 -39
- package/ui/dist/assets/index-ClOLOjnA.css +0 -1
package/routes/multimode.js
CHANGED
|
@@ -11,7 +11,7 @@ import { generateMultimodeViaResponses } from "../lib/responsesImageAdapter.js";
|
|
|
11
11
|
import { generateMultimodeViaGrok } from "../lib/grokMultimodeAdapter.js";
|
|
12
12
|
import { generateViaAgy } from "../lib/agyImageAdapter.js";
|
|
13
13
|
import { generateViaGeminiApi } from "../lib/geminiApiImageAdapter.js";
|
|
14
|
-
import { startJob, finishJob, registerJobAbortController, isJobCanceled } from "../lib/inflight.js";
|
|
14
|
+
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, INFLIGHT_RETRY_AFTER_SECONDS } from "../lib/inflight.js";
|
|
15
15
|
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "../lib/generationCancel.js";
|
|
16
16
|
import { logEvent, logError } from "../lib/logger.js";
|
|
17
17
|
import { embedImageMetadataBestEffort } from "../lib/imageMetadataStore.js";
|
|
@@ -19,37 +19,35 @@ import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
|
19
19
|
import { normalizeComposerInsertedPrompts, normalizeComposerPrompt, } from "../lib/composerSnapshot.js";
|
|
20
20
|
import { errInfo } from "../lib/errInfo.js";
|
|
21
21
|
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
function
|
|
27
|
-
if (
|
|
28
|
-
|
|
22
|
+
import { validateModeration, imageFormatFromMime, writeSse } from "../lib/routeHelpers.js";
|
|
23
|
+
import { publish } from "../lib/eventBus.js";
|
|
24
|
+
import { publishJobEvent } from "../lib/ssePublish.js";
|
|
25
|
+
import { normalizeMaxImages, sequenceStatus, } from "../lib/multimodeHelpers.js";
|
|
26
|
+
function dualEmitMultimode(res, requestId, event, data) {
|
|
27
|
+
if (!res.writableEnded)
|
|
28
|
+
writeSse(res, event, data);
|
|
29
|
+
if (event === "done") {
|
|
30
|
+
publishJobEvent(requestId, event, data);
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
publish(requestId, event, data);
|
|
29
34
|
}
|
|
30
|
-
return { moderation };
|
|
31
|
-
}
|
|
32
|
-
function normalizeMaxImages(value) {
|
|
33
|
-
return Math.min(8, Math.max(1, Math.trunc(Number(value) || 1)));
|
|
34
|
-
}
|
|
35
|
-
function sequenceStatus(returned, requested) {
|
|
36
|
-
if (returned <= 0)
|
|
37
|
-
return "empty";
|
|
38
|
-
if (returned < requested)
|
|
39
|
-
return "partial";
|
|
40
|
-
return "complete";
|
|
41
35
|
}
|
|
42
|
-
function
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
36
|
+
function respondMultimodeValidationError(res, requestId, asyncMode, status, payload) {
|
|
37
|
+
publish(requestId, "error", payload);
|
|
38
|
+
if (asyncMode && !res.headersSent) {
|
|
39
|
+
return res.status(status).json(payload);
|
|
40
|
+
}
|
|
41
|
+
if (!res.writableEnded) {
|
|
42
|
+
writeSse(res, "error", payload);
|
|
43
|
+
res.end();
|
|
44
|
+
}
|
|
48
45
|
}
|
|
49
46
|
export function registerMultimodeRoutes(app, ctxRaw) {
|
|
50
47
|
const ctx = requireRuntimeContext(ctxRaw);
|
|
51
48
|
app.post("/api/generate/multimode", async (req, res) => {
|
|
52
49
|
const requestId = typeof req.body?.requestId === "string" ? req.body.requestId : req.id;
|
|
50
|
+
const asyncMode = req.body?.async === true;
|
|
53
51
|
let finishStatus = "completed";
|
|
54
52
|
let finishHttpStatus = 200;
|
|
55
53
|
let finishErrorCode;
|
|
@@ -74,10 +72,12 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
74
72
|
let latestUsage = null;
|
|
75
73
|
let latestWebSearchCalls = 0;
|
|
76
74
|
let latestExtraIgnored = 0;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
if (!asyncMode) {
|
|
76
|
+
res.setHeader("Content-Type", "text/event-stream; charset=utf-8");
|
|
77
|
+
res.setHeader("Cache-Control", "no-cache, no-transform");
|
|
78
|
+
res.setHeader("Connection", "keep-alive");
|
|
79
|
+
res.flushHeaders?.();
|
|
80
|
+
}
|
|
81
81
|
try {
|
|
82
82
|
const { prompt, quality: rawQuality = "medium", size = "1024x1024", format = "png", moderation = "low", provider = "auto", references = [], mode: promptMode = "auto", model: rawModel, reasoningEffort: rawReasoningEffort, webSearchEnabled: rawWebSearchEnabled = true, } = req.body;
|
|
83
83
|
const composerPrompt = normalizeComposerPrompt(req.body?.composerPrompt);
|
|
@@ -96,8 +96,12 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
96
96
|
finishStatus = "error";
|
|
97
97
|
finishHttpStatus = providerOptions.status;
|
|
98
98
|
finishErrorCode = providerOptions.code;
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, providerOptions.status, {
|
|
100
|
+
error: providerOptions.error,
|
|
101
|
+
code: providerOptions.code,
|
|
102
|
+
status: providerOptions.status,
|
|
103
|
+
requestId,
|
|
104
|
+
});
|
|
101
105
|
}
|
|
102
106
|
const imageModel = providerOptions.model;
|
|
103
107
|
const reasoningEffort = providerOptions.reasoningEffort;
|
|
@@ -108,28 +112,41 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
108
112
|
finishStatus = "error";
|
|
109
113
|
finishHttpStatus = 400;
|
|
110
114
|
finishErrorCode = "PROMPT_REQUIRED";
|
|
111
|
-
|
|
112
|
-
|
|
115
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
116
|
+
error: "Prompt is required",
|
|
117
|
+
code: finishErrorCode,
|
|
118
|
+
status: 400,
|
|
119
|
+
requestId,
|
|
120
|
+
});
|
|
113
121
|
}
|
|
114
122
|
const moderationCheck = validateModeration(ctx, moderation);
|
|
115
123
|
if (moderationCheck.error) {
|
|
116
124
|
finishStatus = "error";
|
|
117
125
|
finishHttpStatus = 400;
|
|
118
126
|
finishErrorCode = "INVALID_MODERATION";
|
|
119
|
-
|
|
120
|
-
|
|
127
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
128
|
+
error: moderationCheck.error,
|
|
129
|
+
code: finishErrorCode,
|
|
130
|
+
status: 400,
|
|
131
|
+
requestId,
|
|
132
|
+
});
|
|
121
133
|
}
|
|
122
134
|
const refCheckResult = validateAndNormalizeRefs(references);
|
|
123
135
|
if (refCheckResult.error) {
|
|
124
136
|
finishStatus = "error";
|
|
125
137
|
finishHttpStatus = 400;
|
|
126
138
|
finishErrorCode = refCheckResult.code;
|
|
127
|
-
|
|
128
|
-
|
|
139
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, 400, {
|
|
140
|
+
error: refCheckResult.error,
|
|
141
|
+
code: refCheckResult.code,
|
|
142
|
+
status: 400,
|
|
143
|
+
requestId,
|
|
144
|
+
});
|
|
129
145
|
}
|
|
130
146
|
const refCheck = refCheckResult;
|
|
147
|
+
const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http") ? req.body.providerUrl : null;
|
|
131
148
|
const referencePayload = summarizeReferencePayload(references);
|
|
132
|
-
startJob({
|
|
149
|
+
const started = startJob({
|
|
133
150
|
requestId,
|
|
134
151
|
kind: "multimode",
|
|
135
152
|
prompt,
|
|
@@ -146,7 +163,25 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
146
163
|
composerInsertedPrompts,
|
|
147
164
|
},
|
|
148
165
|
});
|
|
166
|
+
if (started && isStartJobFailure(started)) {
|
|
167
|
+
finishStatus = "error";
|
|
168
|
+
finishHttpStatus = started.code === "TOO_MANY_JOBS" ? 429 : 409;
|
|
169
|
+
finishErrorCode = started.code;
|
|
170
|
+
if (started.code === "TOO_MANY_JOBS") {
|
|
171
|
+
res.setHeader("Retry-After", String(INFLIGHT_RETRY_AFTER_SECONDS));
|
|
172
|
+
}
|
|
173
|
+
return respondMultimodeValidationError(res, requestId, asyncMode, finishHttpStatus, {
|
|
174
|
+
error: started.code === "TOO_MANY_JOBS"
|
|
175
|
+
? "Too many concurrent generation jobs"
|
|
176
|
+
: "Request ID already in use",
|
|
177
|
+
code: started.code,
|
|
178
|
+
status: finishHttpStatus,
|
|
179
|
+
requestId,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
149
182
|
registerJobAbortController(requestId, cancelController);
|
|
183
|
+
if (asyncMode)
|
|
184
|
+
res.status(202).json({ requestId });
|
|
150
185
|
logEvent("multimode", "request", {
|
|
151
186
|
requestId,
|
|
152
187
|
quality,
|
|
@@ -188,6 +223,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
188
223
|
const resultFormat = activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api" ? imageFormatFromMime(resultMime) : mmFormat;
|
|
189
224
|
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
190
225
|
const filename = `${Date.now()}_${rand}_multimode_${index}.${resultFormat}`;
|
|
226
|
+
const createdAt = Date.now();
|
|
191
227
|
const meta = {
|
|
192
228
|
kind: "multimode-image",
|
|
193
229
|
generationStrategy: "one-call-text-sequence",
|
|
@@ -210,11 +246,12 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
210
246
|
moderation,
|
|
211
247
|
model: activeProvider === "grok" ? (quality === "high" ? "grok-imagine-image-quality" : imageModel) : imageModel,
|
|
212
248
|
provider: activeProvider,
|
|
213
|
-
createdAt
|
|
249
|
+
createdAt,
|
|
214
250
|
usage: latestUsage,
|
|
215
251
|
webSearchCalls: latestWebSearchCalls,
|
|
216
252
|
webSearchEnabled,
|
|
217
253
|
refsCount: refCheck.refs.length,
|
|
254
|
+
...(image.providerUrl ? { providerUrl: image.providerUrl } : {}),
|
|
218
255
|
};
|
|
219
256
|
const rawBuffer = Buffer.from(image.b64, "base64");
|
|
220
257
|
const embedded = await embedImageMetadataBestEffort(rawBuffer, resultFormat, meta, {
|
|
@@ -228,6 +265,8 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
228
265
|
const item = {
|
|
229
266
|
image: `data:${resultMime};base64,${image.b64}`,
|
|
230
267
|
filename,
|
|
268
|
+
createdAt,
|
|
269
|
+
...(image.providerUrl ? { providerUrl: image.providerUrl } : {}),
|
|
231
270
|
revisedPrompt: image.revisedPrompt || null,
|
|
232
271
|
sequenceId,
|
|
233
272
|
sequenceIndex: index + 1,
|
|
@@ -237,9 +276,9 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
237
276
|
};
|
|
238
277
|
persistedIndexes.add(index);
|
|
239
278
|
images.push(item);
|
|
240
|
-
|
|
279
|
+
dualEmitMultimode(res, requestId, "image", item);
|
|
241
280
|
};
|
|
242
|
-
|
|
281
|
+
dualEmitMultimode(res, requestId, "phase", { phase: "streaming", requestId, sequenceId, maxImages });
|
|
243
282
|
let generated;
|
|
244
283
|
if (activeProvider === "gemini-api") {
|
|
245
284
|
const r = await generateViaGeminiApi(prompt, requireRuntimeContext(ctx), {
|
|
@@ -270,13 +309,16 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
270
309
|
else if (activeProvider === "grok" || activeProvider === "grok-api") {
|
|
271
310
|
const directApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
|
|
272
311
|
const grokModel = quality === "high" ? "grok-imagine-image-quality" : imageModel;
|
|
312
|
+
const grokRefs = incomingProviderUrl
|
|
313
|
+
? [{ b64: "", url: incomingProviderUrl }, ...refCheck.refDetails]
|
|
314
|
+
: refCheck.refDetails;
|
|
273
315
|
generated = await generateMultimodeViaGrok(prompt, ctx, {
|
|
274
316
|
model: grokModel,
|
|
275
317
|
maxImages,
|
|
276
318
|
size: effectiveSize,
|
|
277
319
|
signal: cancelController.signal,
|
|
278
320
|
requestId,
|
|
279
|
-
references:
|
|
321
|
+
references: grokRefs,
|
|
280
322
|
directApiKey,
|
|
281
323
|
onFinalImage: async (image, index) => {
|
|
282
324
|
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
@@ -290,14 +332,14 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
290
332
|
maxImages,
|
|
291
333
|
reasoningEffort,
|
|
292
334
|
webSearchEnabled,
|
|
293
|
-
onPartialImage: (partial) =>
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
335
|
+
onPartialImage: (partial) => {
|
|
336
|
+
if (isJobCanceled(requestId))
|
|
337
|
+
return;
|
|
338
|
+
const pd = { image: `data:${mime};base64,${partial.b64}`, requestId, sequenceId, index: partial.index };
|
|
339
|
+
if (!res.writableEnded && !res.destroyed)
|
|
340
|
+
writeSse(res, "partial", pd);
|
|
341
|
+
publish(requestId, "partial", pd);
|
|
342
|
+
},
|
|
301
343
|
onFinalImage: async (image, index) => {
|
|
302
344
|
const totalReturned = Math.max(index + 1, images.length + 1);
|
|
303
345
|
await persistAndSendImage(image, index, totalReturned, sequenceStatus(totalReturned, maxImages));
|
|
@@ -320,7 +362,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
320
362
|
finishHttpStatus = 422;
|
|
321
363
|
finishErrorCode = "EMPTY_RESPONSE";
|
|
322
364
|
finishMeta = { sequenceId, filenames: [], imageCount: 0, maxImages, status, composerPrompt: routeComposerPrompt, composerInsertedPrompts: routeComposerInsertedPrompts };
|
|
323
|
-
|
|
365
|
+
dualEmitMultimode(res, requestId, "error", {
|
|
324
366
|
error: "No image data returned from the multimode stream",
|
|
325
367
|
code: finishErrorCode,
|
|
326
368
|
status: finishHttpStatus,
|
|
@@ -339,7 +381,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
339
381
|
composerInsertedPrompts: routeComposerInsertedPrompts,
|
|
340
382
|
};
|
|
341
383
|
finishHttpStatus = 200;
|
|
342
|
-
|
|
384
|
+
dualEmitMultimode(res, requestId, "done", {
|
|
343
385
|
ok: true,
|
|
344
386
|
requestId,
|
|
345
387
|
sequenceId,
|
|
@@ -378,7 +420,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
378
420
|
finishCanceled = true;
|
|
379
421
|
finishHttpStatus = canceled.status;
|
|
380
422
|
finishErrorCode = canceled.code;
|
|
381
|
-
|
|
423
|
+
dualEmitMultimode(res, requestId, "error", {
|
|
382
424
|
error: canceled.message,
|
|
383
425
|
code: canceled.code,
|
|
384
426
|
status: canceled.status,
|
|
@@ -401,7 +443,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
401
443
|
composerPrompt: routeComposerPrompt,
|
|
402
444
|
composerInsertedPrompts: routeComposerInsertedPrompts,
|
|
403
445
|
};
|
|
404
|
-
|
|
446
|
+
dualEmitMultimode(res, requestId, "done", {
|
|
405
447
|
ok: true,
|
|
406
448
|
partial: true,
|
|
407
449
|
requestId,
|
|
@@ -439,7 +481,7 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
439
481
|
finishHttpStatus = err.status || 500;
|
|
440
482
|
finishErrorCode = fallbackCode || "MULTIMODE_GENERATE_FAILED";
|
|
441
483
|
logError("multimode", "error", err.raw, { requestId, code: finishErrorCode });
|
|
442
|
-
|
|
484
|
+
dualEmitMultimode(res, requestId, "error", {
|
|
443
485
|
error: err.message,
|
|
444
486
|
code: finishErrorCode,
|
|
445
487
|
status: finishHttpStatus,
|
|
@@ -457,7 +499,8 @@ export function registerMultimodeRoutes(app, ctxRaw) {
|
|
|
457
499
|
errorCode: finishErrorCode,
|
|
458
500
|
meta: finishMeta,
|
|
459
501
|
});
|
|
460
|
-
res.
|
|
502
|
+
if (!res.writableEnded)
|
|
503
|
+
res.end();
|
|
461
504
|
}
|
|
462
505
|
});
|
|
463
506
|
}
|
package/routes/nodes.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mkdir } from "fs/promises";
|
|
2
|
-
import { newNodeId, saveNode,
|
|
3
|
-
import { startJob, finishJob, registerJobAbortController, isJobCanceled } from "../lib/inflight.js";
|
|
2
|
+
import { newNodeId, saveNode, loadNodeMeta, loadAssetB64, } from "../lib/nodeStore.js";
|
|
3
|
+
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, INFLIGHT_RETRY_AFTER_SECONDS } from "../lib/inflight.js";
|
|
4
4
|
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "../lib/generationCancel.js";
|
|
5
5
|
import { detectImageMimeFromB64, summarizeReferencePayload, validateAndNormalizeRefs } from "../lib/refs.js";
|
|
6
6
|
import { classifyUpstreamError } from "../lib/errorClassify.js";
|
|
@@ -14,72 +14,16 @@ import { isNonRetryableGenerationError, normalizeGenerationFailure } from "../li
|
|
|
14
14
|
import { logEvent, logError } from "../lib/logger.js";
|
|
15
15
|
import { errInfo } from "../lib/errInfo.js";
|
|
16
16
|
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
if (typeof moderation !== "string" || !ctx.config.oauth.validModeration.has(moderation)) {
|
|
22
|
-
return { error: "moderation must be one of: auto, low" };
|
|
23
|
-
}
|
|
24
|
-
return { moderation };
|
|
25
|
-
}
|
|
26
|
-
function wantsSse(req) {
|
|
27
|
-
const accept = typeof req.headers.accept === "string" ? req.headers.accept : "";
|
|
28
|
-
return accept.includes("text/event-stream");
|
|
29
|
-
}
|
|
30
|
-
function writeSse(res, event, data) {
|
|
31
|
-
res.write(`event: ${event}\n`);
|
|
32
|
-
res.write(`data: ${JSON.stringify(data)}\n\n`);
|
|
33
|
-
}
|
|
34
|
-
function writeNodeError(res, status, code, message, parentNodeId, details = {}) {
|
|
35
|
-
if (res.headersSent) {
|
|
36
|
-
writeSse(res, "error", {
|
|
37
|
-
error: { code, message },
|
|
38
|
-
parentNodeId,
|
|
39
|
-
status,
|
|
40
|
-
...details,
|
|
41
|
-
});
|
|
42
|
-
res.end();
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
|
-
res.status(status).json({
|
|
46
|
-
error: { code, message },
|
|
47
|
-
parentNodeId,
|
|
48
|
-
status,
|
|
49
|
-
...details,
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
function dataUrlFromB64(format, b64) {
|
|
53
|
-
return `data:image/${format === "jpeg" ? "jpeg" : format};base64,${b64}`;
|
|
54
|
-
}
|
|
55
|
-
function imageFormatFromMime(mime) {
|
|
56
|
-
if (mime === "image/jpeg")
|
|
57
|
-
return "jpeg";
|
|
58
|
-
if (mime === "image/webp")
|
|
59
|
-
return "webp";
|
|
60
|
-
return "png";
|
|
61
|
-
}
|
|
62
|
-
async function loadParentNodeB64(ctx, nodeId) {
|
|
63
|
-
for (const ext of ["png", "jpeg", "webp"]) {
|
|
64
|
-
const meta = await loadNodeMeta(ctx.rootDir, nodeId, ext, ctx.config.storage.generatedDir);
|
|
65
|
-
if (meta)
|
|
66
|
-
return loadNodeB64(ctx.rootDir, `${nodeId}.${ext}`, ctx.config.storage.generatedDir);
|
|
67
|
-
}
|
|
68
|
-
return loadNodeB64(ctx.rootDir, `${nodeId}.png`, ctx.config.storage.generatedDir);
|
|
69
|
-
}
|
|
70
|
-
function toGrokReferences(parentB64, refs) {
|
|
71
|
-
const parentMime = parentB64 ? detectImageMimeFromB64(parentB64) : null;
|
|
72
|
-
const parentRefs = parentB64
|
|
73
|
-
? [{ b64: parentB64, declaredMime: parentMime, detectedMime: parentMime }]
|
|
74
|
-
: [];
|
|
75
|
-
const normalizedRefs = refs.map((ref) => typeof ref === "string" ? { b64: ref } : ref);
|
|
76
|
-
return [...parentRefs, ...normalizedRefs];
|
|
77
|
-
}
|
|
17
|
+
import { validateModeration, imageFormatFromMime, writeSse, dataUrlFromB64 } from "../lib/routeHelpers.js";
|
|
18
|
+
import { publish } from "../lib/eventBus.js";
|
|
19
|
+
import { publishJobEvent } from "../lib/ssePublish.js";
|
|
20
|
+
import { asUpstream, wantsSse, writeNodeError, loadParentNodeB64, toGrokReferences, nodeErrorDetails, } from "../lib/nodeHelpers.js";
|
|
78
21
|
export function registerNodeRoutes(app, ctxRaw) {
|
|
79
22
|
const ctx = requireRuntimeContext(ctxRaw);
|
|
80
23
|
app.post("/api/node/generate", async (req, res) => {
|
|
81
24
|
const body = (req.body ?? {});
|
|
82
|
-
const
|
|
25
|
+
const asyncMode = body.async === true;
|
|
26
|
+
const streamResponse = !asyncMode && wantsSse(req);
|
|
83
27
|
const parentNodeId = (typeof body.parentNodeId === "string" ? body.parentNodeId : null);
|
|
84
28
|
const requestId = typeof body.requestId === "string" ? body.requestId : (req.id ?? "");
|
|
85
29
|
const sessionId = typeof body.sessionId === "string" ? body.sessionId : null;
|
|
@@ -91,21 +35,6 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
91
35
|
let finishCanceled = false;
|
|
92
36
|
const cancelController = new AbortController();
|
|
93
37
|
const referencePayload = summarizeReferencePayload(body.references);
|
|
94
|
-
startJob({
|
|
95
|
-
requestId,
|
|
96
|
-
kind: "node",
|
|
97
|
-
prompt: body.prompt,
|
|
98
|
-
meta: {
|
|
99
|
-
kind: "node",
|
|
100
|
-
sessionId,
|
|
101
|
-
parentNodeId,
|
|
102
|
-
clientNodeId,
|
|
103
|
-
refsCount: referencePayload.refsCount,
|
|
104
|
-
referenceBytes: referencePayload.referenceBytes,
|
|
105
|
-
referenceB64Chars: referencePayload.referenceB64Chars,
|
|
106
|
-
},
|
|
107
|
-
});
|
|
108
|
-
registerJobAbortController(requestId, cancelController);
|
|
109
38
|
try {
|
|
110
39
|
const { prompt, quality: rawQuality = "medium", size = "1024x1024", format = "png", moderation = "low", references = [], externalSrc = null, mode: promptMode = "auto", contextMode: rawContextMode = "parent-plus-refs", searchMode: rawSearchMode = "on", model: rawModel, reasoningEffort: rawReasoningEffort, } = body;
|
|
111
40
|
const { provider = "oauth" } = body;
|
|
@@ -208,6 +137,34 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
208
137
|
parentNodeId,
|
|
209
138
|
});
|
|
210
139
|
}
|
|
140
|
+
const started = startJob({
|
|
141
|
+
requestId,
|
|
142
|
+
kind: "node",
|
|
143
|
+
prompt: body.prompt,
|
|
144
|
+
meta: {
|
|
145
|
+
kind: "node",
|
|
146
|
+
sessionId,
|
|
147
|
+
parentNodeId,
|
|
148
|
+
clientNodeId,
|
|
149
|
+
refsCount: referencePayload.refsCount,
|
|
150
|
+
referenceBytes: referencePayload.referenceBytes,
|
|
151
|
+
referenceB64Chars: referencePayload.referenceB64Chars,
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
if (started && isStartJobFailure(started)) {
|
|
155
|
+
finishStatus = "error";
|
|
156
|
+
finishHttpStatus = started.code === "TOO_MANY_JOBS" ? 429 : 409;
|
|
157
|
+
finishErrorCode = started.code;
|
|
158
|
+
if (started.code === "TOO_MANY_JOBS") {
|
|
159
|
+
res.setHeader("Retry-After", String(INFLIGHT_RETRY_AFTER_SECONDS));
|
|
160
|
+
}
|
|
161
|
+
return writeNodeError(res, finishHttpStatus, started.code, started.code === "TOO_MANY_JOBS"
|
|
162
|
+
? "Too many concurrent generation jobs"
|
|
163
|
+
: "Request ID already in use", parentNodeId, {}, requestId);
|
|
164
|
+
}
|
|
165
|
+
registerJobAbortController(requestId, cancelController);
|
|
166
|
+
if (asyncMode)
|
|
167
|
+
res.status(202).json({ requestId });
|
|
211
168
|
logEvent("node", "request", {
|
|
212
169
|
requestId,
|
|
213
170
|
operation,
|
|
@@ -231,6 +188,7 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
231
188
|
promptChars: prompt.length,
|
|
232
189
|
promptMode: normalizedPromptMode,
|
|
233
190
|
});
|
|
191
|
+
const emitProgress = streamResponse || asyncMode;
|
|
234
192
|
if (streamResponse) {
|
|
235
193
|
res.writeHead(200, {
|
|
236
194
|
"Content-Type": "text/event-stream; charset=utf-8",
|
|
@@ -238,6 +196,10 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
238
196
|
Connection: "keep-alive",
|
|
239
197
|
});
|
|
240
198
|
writeSse(res, "phase", { requestId, phase: "streaming" });
|
|
199
|
+
publish(requestId, "phase", { requestId, phase: "streaming" });
|
|
200
|
+
}
|
|
201
|
+
else if (asyncMode) {
|
|
202
|
+
publish(requestId, "phase", { requestId, phase: "streaming" });
|
|
241
203
|
}
|
|
242
204
|
let b64, usage, webSearchCalls = 0, revisedPrompt = null;
|
|
243
205
|
const grokDirectApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
|
|
@@ -305,15 +267,16 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
305
267
|
reasoningEffort,
|
|
306
268
|
webSearchEnabled,
|
|
307
269
|
signal: cancelController.signal,
|
|
308
|
-
partialImages:
|
|
309
|
-
onPartialImage:
|
|
310
|
-
? (partial) =>
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
270
|
+
partialImages: emitProgress ? 2 : 0,
|
|
271
|
+
onPartialImage: emitProgress
|
|
272
|
+
? (partial) => {
|
|
273
|
+
if (isJobCanceled(requestId))
|
|
274
|
+
return;
|
|
275
|
+
const pd = { requestId, image: dataUrlFromB64(format, partial.b64), index: partial.index };
|
|
276
|
+
if (streamResponse)
|
|
277
|
+
writeSse(res, "partial", pd);
|
|
278
|
+
publish(requestId, "partial", pd);
|
|
279
|
+
}
|
|
317
280
|
: null,
|
|
318
281
|
});
|
|
319
282
|
throwIfJobCanceled(requestId);
|
|
@@ -368,18 +331,7 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
368
331
|
outerHttpAlreadyCommitted: res.headersSent,
|
|
369
332
|
sseErrorSent: streamResponse,
|
|
370
333
|
});
|
|
371
|
-
return writeNodeError(res, finishHttpStatus ?? 500, finishErrorCode ?? "NODE_GEN_FAILED", finalErr.message, parentNodeId,
|
|
372
|
-
upstreamCode: lastErr?.upstreamCode || lastErr?.code || null,
|
|
373
|
-
upstreamType: lastErr?.upstreamType || null,
|
|
374
|
-
upstreamParam: lastErr?.upstreamParam || null,
|
|
375
|
-
errorEventType: lastErr?.eventType || null,
|
|
376
|
-
errorEventCount: lastErr?.eventCount ?? null,
|
|
377
|
-
diagnosticReason: finalErr.diagnosticReason || lastErr?.diagnosticReason || null,
|
|
378
|
-
retryKind: finalErr.retryKind || lastErr?.retryKind || null,
|
|
379
|
-
referencesDroppedOnRetry: finalErr.referencesDroppedOnRetry ?? lastErr?.referencesDroppedOnRetry ?? null,
|
|
380
|
-
refsCount: finalErr.refsCount ?? lastErr?.refsCount ?? null,
|
|
381
|
-
inputImageCount: finalErr.inputImageCount ?? lastErr?.inputImageCount ?? null,
|
|
382
|
-
});
|
|
334
|
+
return writeNodeError(res, finishHttpStatus ?? 500, finishErrorCode ?? "NODE_GEN_FAILED", finalErr.message, parentNodeId, nodeErrorDetails(finalErr, lastErr), requestId);
|
|
383
335
|
}
|
|
384
336
|
const nodeId = newNodeId();
|
|
385
337
|
throwIfJobCanceled(requestId);
|
|
@@ -455,7 +407,11 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
455
407
|
revisedPrompt,
|
|
456
408
|
promptMode: normalizedPromptMode,
|
|
457
409
|
};
|
|
458
|
-
|
|
410
|
+
publishJobEvent(requestId, "done", payload);
|
|
411
|
+
if (res.writableEnded) {
|
|
412
|
+
// async mode — response already sent
|
|
413
|
+
}
|
|
414
|
+
else if (streamResponse) {
|
|
459
415
|
writeSse(res, "done", payload);
|
|
460
416
|
res.end();
|
|
461
417
|
}
|
|
@@ -472,7 +428,7 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
472
428
|
finishCanceled = true;
|
|
473
429
|
finishHttpStatus = canceled.status;
|
|
474
430
|
finishErrorCode = canceled.code;
|
|
475
|
-
return writeNodeError(res, canceled.status, canceled.code, canceled.message, parentNodeId);
|
|
431
|
+
return writeNodeError(res, canceled.status, canceled.code, canceled.message, parentNodeId, {}, requestId);
|
|
476
432
|
}
|
|
477
433
|
finishStatus = "error";
|
|
478
434
|
finishHttpStatus = err.status || 500;
|
|
@@ -482,7 +438,7 @@ export function registerNodeRoutes(app, ctxRaw) {
|
|
|
482
438
|
upstreamCode: ext.upstreamCode || null,
|
|
483
439
|
upstreamType: ext.upstreamType || null,
|
|
484
440
|
upstreamParam: ext.upstreamParam || null,
|
|
485
|
-
});
|
|
441
|
+
}, requestId);
|
|
486
442
|
}
|
|
487
443
|
finally {
|
|
488
444
|
finishJob(requestId, {
|