ima2-gen 2.0.1 → 2.0.2

package/routes/generate.js

@@ -12,7 +12,7 @@ import { generateViaGrok, planGrokImage } from "../lib/grokImageAdapter.js";
 import { generateViaAgy } from "../lib/agyImageAdapter.js";
 import { generateViaGeminiApi } from "../lib/geminiApiImageAdapter.js";
 import { isNonRetryableGenerationError, normalizeGenerationFailure } from "../lib/generationErrors.js";
-import { startJob, finishJob, registerJobAbortController, isJobCanceled } from "../lib/inflight.js";
+import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, setJobPhase, INFLIGHT_RETRY_AFTER_SECONDS, } from "../lib/inflight.js";
 import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "../lib/generationCancel.js";
 import { logEvent, logError } from "../lib/logger.js";
 import { embedImageMetadataBestEffort } from "../lib/imageMetadataStore.js";
@@ -20,57 +20,44 @@ import { invalidateHistoryIndex } from "../lib/historyIndex.js";
 import { normalizeComposerInsertedPrompts, normalizeComposerPrompt, } from "../lib/composerSnapshot.js";
 import { errInfo } from "../lib/errInfo.js";
 import { requireRuntimeContext } from "../lib/runtimeContext.js";
-function validateModeration(ctx, moderation) {
-    if (typeof moderation !== "string" || !ctx.config.oauth.validModeration.has(moderation)) {
-        return { error: "moderation must be one of: auto, low" };
-    }
-    return { moderation };
-}
-function imageFormatFromMime(mime) {
-    if (mime === "image/jpeg")
-        return "jpeg";
-    if (mime === "image/webp")
-        return "webp";
-    return "png";
-}
+import { STORYBOARD_PREFIX } from "../lib/storyboardPrefix.js";
+import { validateModeration, imageFormatFromMime, upstreamErrorFields } from "../lib/routeHelpers.js";
+import { publish } from "../lib/eventBus.js";
+import { publishJobEvent } from "../lib/ssePublish.js";
 export function registerGenerateRoutes(app, ctxRaw) {
     const ctx = requireRuntimeContext(ctxRaw);
     app.post("/api/generate", async (req, res) => {
         const requestId = typeof req.body?.requestId === "string" ? req.body.requestId : req.id;
+        const asyncMode = req.body?.async === true;
         let finishStatus = "completed";
         let finishHttpStatus;
         let finishErrorCode;
         let finishMeta = {};
         let finishCanceled = false;
         const cancelController = new AbortController();
+        const fail = (status, payload) => {
+            finishStatus = "error";
+            finishHttpStatus = status;
+            finishErrorCode = typeof payload.code === "string" ? payload.code : finishErrorCode;
+            if (asyncMode && res.headersSent) {
+                publish(requestId, "error", { ...payload, status, requestId });
+                return;
+            }
+            return res.status(status).json(payload);
+        };
+        const succeed = (payload) => {
+            if (asyncMode) {
+                publishJobEvent(requestId, "done", payload);
+                return;
+            }
+            res.json(payload);
+        };
         try {
             const sessionId = typeof req.body?.sessionId === "string" ? req.body.sessionId : null;
             const clientNodeId = typeof req.body?.clientNodeId === "string" ? req.body.clientNodeId : null;
             const { prompt, quality: rawQuality = "medium", size = "1024x1024", format = "png", moderation = "low", provider = "auto", n = 1, references = [], mode: promptMode = "auto", model: rawModel, reasoningEffort: rawReasoningEffort, webSearchEnabled: rawWebSearchEnabled = true, } = req.body;
             const storyboardActive = req.body?.storyboard === true;
-            const storyboardPrefix = storyboardActive
-                ? [
-                    "[STORYBOARD MODE — Video Production Keyframe]",
-                    "This image is a keyframe for a multi-shot VIDEO storyboard. It will be animated via image-to-video.",
-                    "The prompt and all injected instructions MUST be in English.",
-                    "",
-                    "CHARACTER LOCK:",
-                    "- Identify each character by 2-3 VISUAL identifiers (clothing color + physique + position/props). Never by name alone.",
-                    "- Copy character descriptions VERBATIM from the reference/prior frame. Do NOT rephrase or drift.",
-                    "",
-                    "SCENE CONTINUITY:",
-                    "- Lock lighting direction, color palette, environment, and art style to prior frames.",
-                    "- Change ONLY: action, shot scale, camera angle, or expression.",
-                    "- Reference image = canonical anchor. Preserve it faithfully.",
-                    "",
-                    "VIDEO-READY COMPOSITION:",
-                    "- Frame for animation: leave space for motion, avoid static-only poses.",
-                    "- Use descriptive caption format: shot type + subject action + environment + technical (lens, lighting) + mood.",
-                    "- Specify intended camera movement for the video phase (e.g. 'slow dolly-in', 'static wide').",
-                    "- End pose must be stable and suitable for video continuation.",
-                    "",
-                ].join("\n") + "\n"
-                : "";
+            const storyboardPrefix = storyboardActive ? STORYBOARD_PREFIX : "";
             const composerPrompt = normalizeComposerPrompt(req.body?.composerPrompt);
             const composerInsertedPrompts = normalizeComposerInsertedPrompts(req.body?.composerInsertedPrompts);
             const { quality, warnings: qualityWarnings } = normalizeOAuthParams({ provider, quality: rawQuality });
@@ -82,10 +69,7 @@ export function registerGenerateRoutes(app, ctxRaw) {
                 rawWebSearchEnabled,
             });
             if (providerOptions.error) {
-                finishStatus = "error";
-                finishHttpStatus = providerOptions.status;
-                finishErrorCode = providerOptions.code;
-                return res.status(providerOptions.status).json({ error: providerOptions.error, code: providerOptions.code });
+                return fail(providerOptions.status, { error: providerOptions.error, code: providerOptions.code });
             }
             const imageModel = providerOptions.model;
             const reasoningEffort = providerOptions.reasoningEffort;
@@ -95,13 +79,34 @@ export function registerGenerateRoutes(app, ctxRaw) {
             const normalizedPromptMode = promptMode === "direct" ? "direct" : "auto";
             const generationPrompt = storyboardPrefix + prompt;
             if (!prompt)
-                return res.status(400).json({ error: "Prompt is required" });
+                return fail(400, { error: "Prompt is required" });
             const moderationCheck = validateModeration(ctx, moderation);
             if (moderationCheck.error)
-                return res.status(400).json({ error: moderationCheck.error });
+                return fail(400, { error: moderationCheck.error });
             const count = Math.min(Math.max(parseInt(n) || 1, 1), 8);
             const referencePayload = summarizeReferencePayload(references);
-            startJob({
+            const refCheckResult = validateAndNormalizeRefs(references);
+            if (refCheckResult.error) {
+                return fail(400, { error: refCheckResult.error, code: refCheckResult.code });
+            }
+            const refCheck = refCheckResult;
+            const incomingProviderUrl = typeof req.body?.providerUrl === "string" && req.body.providerUrl.startsWith("http")
+                ? req.body.providerUrl
+                : null;
+            const grokRefs = incomingProviderUrl
+                ? [{ b64: "", url: incomingProviderUrl, declaredMime: "image/png", detectedMime: "image/png" }, ...refCheck.refDetails]
+                : refCheck.refDetails;
+            const providerRefCount = activeProvider === "grok" || activeProvider === "grok-api"
+                ? grokRefs.length
+                : refCheck.refs.length;
+            if ((activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api") && providerRefCount > 3) {
+                return fail(400, {
+                    error: `${activeProvider === "agy" ? "Agy" : "Grok"} image editing supports up to 3 reference images`,
+                    code: activeProvider === "agy" ? "AGY_REF_TOO_MANY" : "GROK_REF_TOO_MANY",
+                    requestId,
+                });
+            }
+            const started = startJob({
                 requestId,
                 kind: "classic",
                 prompt,
@@ -114,32 +119,33 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     model: imageModel,
                     size: effectiveSize,
                     n: count,
-                    refsCount: referencePayload.refsCount,
+                    refsCount: providerRefCount,
                     referenceBytes: referencePayload.referenceBytes,
                     referenceB64Chars: referencePayload.referenceB64Chars,
                     composerPrompt,
                     composerInsertedPrompts,
                 },
             });
-            registerJobAbortController(requestId, cancelController);
-            const refCheckResult = validateAndNormalizeRefs(references);
-            if (refCheckResult.error) {
-                finishStatus = "error";
-                finishHttpStatus = 400;
-                finishErrorCode = refCheckResult.code;
-                return res.status(400).json({ error: refCheckResult.error, code: refCheckResult.code });
-            }
-            const refCheck = refCheckResult;
-            if ((activeProvider === "grok" || activeProvider === "agy" || activeProvider === "grok-api" || activeProvider === "gemini-api") && refCheck.refs.length > 3) {
-                finishStatus = "error";
-                finishHttpStatus = 400;
-                finishErrorCode = activeProvider === "agy" ? "AGY_REF_TOO_MANY" : "GROK_REF_TOO_MANY";
-                return res.status(400).json({
-                    error: `${activeProvider === "agy" ? "Agy" : "Grok"} image editing supports up to 3 reference images`,
-                    code: activeProvider === "agy" ? "AGY_REF_TOO_MANY" : "GROK_REF_TOO_MANY",
+            if (started && isStartJobFailure(started)) {
+                const status = started.code === "TOO_MANY_JOBS" ? 429 : 409;
+                if (started.code === "TOO_MANY_JOBS") {
+                    res.setHeader("Retry-After", String(INFLIGHT_RETRY_AFTER_SECONDS));
+                }
+                return fail(status, {
+                    error: started.code === "TOO_MANY_JOBS"
+                        ? "Too many concurrent generation jobs"
+                        : "Request ID already in use",
+                    code: started.code,
                     requestId,
                 });
             }
+            registerJobAbortController(requestId, cancelController);
+            if (asyncMode) {
+                res.status(202).json({ requestId, async: true });
+            }
+            setJobPhase(requestId, "streaming");
+            if (asyncMode)
+                publish(requestId, "phase", { requestId, phase: "streaming" });
             const client = req.get("x-ima2-client") || "ui";
             const referenceDiagnostics = refCheck.referenceDiagnostics || [];
             const referenceMismatchCount = referenceDiagnostics.filter((ref) => ref.warnings?.includes("mime_mismatch")).length;
@@ -152,7 +158,7 @@ export function registerGenerateRoutes(app, ctxRaw) {
                 size: effectiveSize,
                 moderation,
                 n: count,
-                refs: refCheck.refs.length,
+                refs: providerRefCount,
                 referenceBytes: referencePayload.referenceBytes,
                 referenceMismatchCount,
                 refDetectedMimes: [...new Set(referenceDiagnostics.map((ref) => ref.detectedMime).filter(Boolean))].join(","),
@@ -175,8 +181,8 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     size: effectiveSize,
                     signal: cancelController.signal,
                     requestId,
-                    referenceCount: refCheck.refs.length,
-                    references: refCheck.refDetails,
+                    referenceCount: grokRefs.length,
+                    references: grokRefs,
                     directApiKey: grokDirectApiKey,
                 })
                 : null;
@@ -210,7 +216,7 @@ export function registerGenerateRoutes(app, ctxRaw) {
                         requestId,
                         plannedPrompt: sharedGrokPlan?.prompt,
                         webSearchCalls: sharedGrokPlan?.webSearchCalls,
-                        references: refCheck.refDetails,
+                        references: grokRefs,
                         directApiKey: grokDirectApiKey,
                     });
                     throwIfJobCanceled(requestId);
@@ -275,6 +281,11 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     }
                     const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
                     const filename = `${Date.now()}_${rand}_${images.length}.${resultFormat}`;
+                    const createdAt = Date.now();
+                    const valueWithProviderUrl = r.value;
+                    const providerUrl = typeof valueWithProviderUrl.providerUrl === "string"
+                        ? valueWithProviderUrl.providerUrl
+                        : undefined;
                     const meta = {
                         kind: "classic",
                         requestId,
@@ -293,11 +304,12 @@ export function registerGenerateRoutes(app, ctxRaw) {
                         model: activeProvider === "grok" ? (quality === "high" ? "grok-imagine-image-quality" : imageModel) : imageModel,
                         reasoningEffort,
                         provider: activeProvider,
-                        createdAt: Date.now(),
+                        createdAt,
+                        ...(providerUrl ? { providerUrl } : {}),
                         usage: r.value.usage || null,
                         webSearchCalls: r.value.webSearchCalls || 0,
                         webSearchEnabled,
-                        refsCount: refCheck.refs.length,
+                        refsCount: providerRefCount,
                     };
                     const rawBuffer = Buffer.from(r.value.b64, "base64");
                     const embedded = await embedImageMetadataBestEffort(rawBuffer, resultFormat, meta, {
@@ -320,6 +332,8 @@ export function registerGenerateRoutes(app, ctxRaw) {
                         image: `data:${resultMime};base64,${r.value.b64}`,
                         filename,
                         revisedPrompt: r.value.revisedPrompt || null,
+                        ...(providerUrl ? { providerUrl } : {}),
+                        createdAt,
                     });
                     if (r.value.usage) {
                         const usageValue = r.value.usage;
@@ -349,47 +363,20 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     const status = firstErr.status || 500;
                     if (isGenerationCanceledError(firstErr)) {
                         finishCanceled = true;
-                        finishHttpStatus = firstErr.status;
-                        finishErrorCode = firstErr.code;
-                        return res.status(firstErr.status).json({
+                        return fail(firstErr.status, {
                             error: firstErr.message,
                             code: firstErr.code,
                             requestId,
                         });
                     }
-                    finishStatus = "error";
-                    finishHttpStatus = status;
-                    finishErrorCode = firstErr.code;
-                    return res.status(status).json({
+                    return fail(status, {
                         error: firstErr.message,
                         code: firstErr.code,
-                        upstreamCode: firstErr.upstreamCode || null,
-                        upstreamType: firstErr.upstreamType || null,
-                        upstreamParam: firstErr.upstreamParam || null,
-                        diagnosticReason: firstErr.diagnosticReason || null,
-                        retryKind: firstErr.retryKind || null,
-                        initialEventCount: firstErr.initialEventCount ?? null,
-                        initialEventTypes: firstErr.initialEventTypes || null,
-                        referencesDroppedOnRetry: firstErr.referencesDroppedOnRetry ?? null,
-                        developerPromptDroppedOnRetry: firstErr.developerPromptDroppedOnRetry ?? null,
-                        webSearchDroppedOnRetry: firstErr.webSearchDroppedOnRetry ?? null,
-                        fallbackEventCount: firstErr.fallbackEventCount ?? null,
-                        fallbackEventTypes: firstErr.fallbackEventTypes || null,
-                        fallbackImageCallSeen: firstErr.fallbackImageCallSeen ?? null,
-                        fallbackImageResultCount: firstErr.fallbackImageResultCount ?? null,
-                        errorEventCount: firstErr.eventCount ?? null,
-                        eventTypes: firstErr.eventTypes || null,
-                        webSearchCalls: firstErr.webSearchCalls ?? null,
-                        responseDiagnostics: firstErr.responseDiagnostics || null,
-                        toolTypes: firstErr.toolTypes || null,
-                        toolChoiceKind: firstErr.toolChoiceKind || null,
+                        ...upstreamErrorFields(firstErr),
                         requestId,
                     });
                 }
-                finishStatus = "error";
-                finishHttpStatus = 500;
-                finishErrorCode = "GENERATE_ALL_FAILED";
-                return res.status(500).json({ error: "All generation attempts failed" });
+                return fail(500, { error: "All generation attempts failed", code: "GENERATE_ALL_FAILED", requestId });
             }
             const elapsed = +((Date.now() - startTime) / 1000).toFixed(1);
             // Persist elapsed (computed after the generation loop) into each image's sidecar.
@@ -431,7 +418,15 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     elapsedMs: Date.now() - startTime,
                     filename: images[0].filename,
                 });
-                res.json({ image: images[0].image, elapsed, filename: images[0].filename, requestId, ...extra });
+                succeed({
+                    image: images[0].image,
+                    elapsed,
+                    filename: images[0].filename,
+                    requestId,
+                    providerUrl: images[0].providerUrl ?? null,
+                    createdAt: images[0].createdAt,
+                    ...extra,
+                });
             }
             else {
                 finishHttpStatus = 200;
@@ -441,7 +436,7 @@ export function registerGenerateRoutes(app, ctxRaw) {
                     imageCount: images.length,
                     elapsedMs: Date.now() - startTime,
                 });
-                res.json({ images, elapsed, count: images.length, requestId, ...extra });
+                succeed({ images, elapsed, count: images.length, requestId, ...extra });
             }
         }
         catch (e) {
@@ -451,41 +446,18 @@ export function registerGenerateRoutes(app, ctxRaw) {
             if (isGenerationCanceledError(err.raw) || isJobCanceled(requestId)) {
                 const canceled = makeGenerationCanceledError();
                 finishCanceled = true;
-                finishHttpStatus = canceled.status;
-                finishErrorCode = canceled.code;
-                return res.status(canceled.status).json({
+                return fail(canceled.status, {
                     error: canceled.message,
                     code: canceled.code,
                     requestId,
                 });
             }
-            finishStatus = "error";
-            finishHttpStatus = err.status || 500;
             finishErrorCode = fallbackCode || "GENERATE_FAILED";
             logError("generate", "error", err.raw, { requestId, code: finishErrorCode });
-            res.status(err.status || 500).json({
+            fail(err.status || 500, {
                 error: err.message,
-                code: fallbackCode,
-                upstreamCode: ext.upstreamCode || null,
-                upstreamType: ext.upstreamType || null,
-                upstreamParam: ext.upstreamParam || null,
-                diagnosticReason: ext.diagnosticReason || null,
-                retryKind: ext.retryKind || null,
-                initialEventCount: ext.initialEventCount ?? null,
-                initialEventTypes: ext.initialEventTypes || null,
-                referencesDroppedOnRetry: ext.referencesDroppedOnRetry ?? null,
-                developerPromptDroppedOnRetry: ext.developerPromptDroppedOnRetry ?? null,
-                webSearchDroppedOnRetry: ext.webSearchDroppedOnRetry ?? null,
-                fallbackEventCount: ext.fallbackEventCount ?? null,
-                fallbackEventTypes: ext.fallbackEventTypes || null,
-                fallbackImageCallSeen: ext.fallbackImageCallSeen ?? null,
-                fallbackImageResultCount: ext.fallbackImageResultCount ?? null,
-                errorEventCount: ext.eventCount ?? null,
-                eventTypes: ext.eventTypes || null,
-                webSearchCalls: ext.webSearchCalls ?? null,
-                responseDiagnostics: ext.responseDiagnostics || null,
-                toolTypes: ext.toolTypes || null,
-                toolChoiceKind: ext.toolChoiceKind || null,
+                code: finishErrorCode,
+                ...upstreamErrorFields(ext),
                 requestId,
             });
         }

package/routes/index.js

@@ -1,4 +1,5 @@
 import { registerCapabilitiesRoutes } from "./capabilities.js";
+import { registerEventsRoute } from "./events.js";
 import { registerHealthRoutes } from "./health.js";
 import { registerHistoryRoutes } from "./history.js";
 import { registerSessionRoutes } from "./sessions.js";
@@ -18,6 +19,7 @@ import { registerImageImportRoutes } from "./imageImport.js";
 import { registerPromptBuilderRoutes } from "./promptBuilder.js";
 import { registerAgentRoutes } from "./agent.js";
 import { registerGrokRoutes } from "./grok.js";
+import { registerAgyRoutes } from "./agy.js";
 import { registerVideoRoutes } from "./video.js";
 import { registerVideoExtendedRoutes } from "./videoExtended.js";
 import { registerQuotaRoutes } from "./quota.js";
@@ -26,6 +28,7 @@ import { mountKeyRoutes } from "./keys.js";
 import { requireRuntimeContext } from "../lib/runtimeContext.js";
 export function configureRoutes(app, ctxRaw) {
     const ctx = requireRuntimeContext(ctxRaw);
+    registerEventsRoute(app, ctx);
     registerHealthRoutes(app, ctx);
     registerCapabilitiesRoutes(app, ctx);
     registerStorageRoutes(app, ctx);
@@ -47,6 +50,7 @@ export function configureRoutes(app, ctxRaw) {
     registerPromptRoutes(app, ctx);
     registerPromptImportRoutes(app, ctx);
     registerGrokRoutes(app, ctx);
+    registerAgyRoutes(app);
     registerVideoRoutes(app, ctx);
     registerVideoExtendedRoutes(app, ctx);
     registerQuotaRoutes(app, ctx);