ima2-gen 3.13.1 → 3.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/.env.example +10 -0
  2. package/CHANGELOG.md +14 -0
  3. package/README.md +15 -2
  4. package/bin/commands/capabilities.js +5 -3
  5. package/bin/commands/config.js +15 -4
  6. package/bin/commands/defaults.js +7 -5
  7. package/bin/commands/doctor.js +88 -39
  8. package/bin/commands/gen.js +21 -4
  9. package/bin/commands/models.js +2 -2
  10. package/bin/commands/ping.js +1 -1
  11. package/bin/commands/prompt.js +5 -3
  12. package/bin/commands/service.js +32 -4
  13. package/bin/commands/tools.js +19 -3
  14. package/bin/commands/upscale.js +18 -4
  15. package/bin/commands/video.js +37 -50
  16. package/bin/ima2.js +28 -9
  17. package/bin/lib/characterResolve.js +2 -1
  18. package/bin/lib/client.js +155 -20
  19. package/bin/lib/config-store.js +40 -7
  20. package/bin/lib/doctor-bundle.js +6 -2
  21. package/bin/lib/doctor-checks.js +13 -29
  22. package/bin/lib/doctor-media.js +3 -3
  23. package/bin/lib/doctor-providers.js +48 -35
  24. package/bin/lib/doctor-report.js +78 -0
  25. package/bin/lib/doctor-runtime.js +173 -0
  26. package/bin/lib/mcpJob.js +19 -7
  27. package/bin/lib/output.js +3 -0
  28. package/bin/lib/platform.js +12 -9
  29. package/bin/lib/recover-output.js +9 -2
  30. package/bin/lib/sse.js +9 -1
  31. package/bin/lib/videoMcp.js +16 -8
  32. package/config.js +41 -5
  33. package/docs/API.md +154 -16
  34. package/docs/CLI.md +65 -1
  35. package/docs/DOCKER.md +19 -4
  36. package/docs/DOCKER.zh-CN.md +11 -4
  37. package/docs/DOCKER.zh-TW.md +11 -4
  38. package/docs/FAQ.md +13 -0
  39. package/docs/NPX_QUICKSTART.md +6 -0
  40. package/docs/README.ja.md +15 -2
  41. package/docs/README.ko.md +15 -2
  42. package/docs/README.zh-CN.md +15 -2
  43. package/docs/README.zh-TW.md +15 -2
  44. package/docs/migration/runtime-test-inventory.md +79 -8
  45. package/lib/agyArtifact.js +82 -0
  46. package/lib/agyArtifactRead.js +249 -0
  47. package/lib/agyImageAdapter.js +2 -329
  48. package/lib/agyProcess.js +122 -0
  49. package/lib/apiRequestBudget.js +48 -0
  50. package/lib/assetLifecycle.js +42 -9
  51. package/lib/canvasVersionStore.js +46 -14
  52. package/lib/capabilities.js +7 -1
  53. package/lib/configKeys.js +2 -0
  54. package/lib/errors/providerMap.js +26 -0
  55. package/lib/eventBus.js +10 -6
  56. package/lib/eventsPolicy.js +9 -0
  57. package/lib/geminiApiImageAdapter.js +1 -214
  58. package/lib/generatePipeline.js +39 -165
  59. package/lib/generatedMediaAccess.js +74 -0
  60. package/lib/generationErrors.js +3 -22
  61. package/lib/grokImageAdapter.js +2 -353
  62. package/lib/grokImageCore.js +1 -57
  63. package/lib/grokImageDownload.js +159 -0
  64. package/lib/grokImageDownloadPolicy.js +124 -0
  65. package/lib/grokImagePlanner.js +301 -0
  66. package/lib/grokMultimodeAdapter.js +1 -75
  67. package/lib/grokVideoDownload.js +137 -34
  68. package/lib/inflight.js +112 -112
  69. package/lib/jobs/terminalStore.js +45 -0
  70. package/lib/lanSessionStore.js +131 -0
  71. package/lib/localAccessPolicy.js +180 -0
  72. package/lib/localLanAccess.js +281 -0
  73. package/lib/logger.js +22 -13
  74. package/lib/mcp/downloadMediaResult.js +72 -78
  75. package/lib/multimodePipeline.js +39 -126
  76. package/lib/nodeGeneration.js +44 -101
  77. package/lib/nodeStore.js +25 -11
  78. package/lib/nodeValidation.js +5 -1
  79. package/lib/pinnedHttpGet.js +174 -0
  80. package/lib/promptImport/githubFolder.js +32 -18
  81. package/lib/promptImport/githubSource.js +14 -10
  82. package/lib/promptImport/parsePromptCandidates.js +32 -2
  83. package/lib/providers/adapters/agyOperations.js +179 -0
  84. package/lib/providers/adapters/comfy.js +1 -0
  85. package/lib/providers/adapters/geminiOperations.js +214 -0
  86. package/lib/providers/adapters/googleExecution.js +97 -0
  87. package/lib/providers/adapters/grokExecution.js +137 -0
  88. package/lib/providers/adapters/grokMultimodeOperations.js +83 -0
  89. package/lib/providers/adapters/grokOperations.js +63 -0
  90. package/lib/providers/adapters/openaiExecution.js +118 -0
  91. package/lib/providers/adapters/openaiOperations.js +201 -0
  92. package/lib/providers/adapters/openaiTypes.js +1 -0
  93. package/lib/providers/derive.js +4 -0
  94. package/lib/providers/deriveCore.js +2 -7
  95. package/lib/providers/execution/admission.js +25 -0
  96. package/lib/providers/execution/index.js +34 -0
  97. package/lib/providers/execution/legacy.js +17 -0
  98. package/lib/providers/execution/legacyClassic.js +66 -0
  99. package/lib/providers/execution/legacyEdit.js +30 -0
  100. package/lib/providers/execution/legacyMultimode.js +34 -0
  101. package/lib/providers/execution/legacyNode.js +41 -0
  102. package/lib/providers/execution/types.js +1 -0
  103. package/lib/providers/registry.js +19 -14
  104. package/lib/providers/surfaceSupport.js +22 -0
  105. package/lib/responsesImageAdapter.js +1 -389
  106. package/lib/responsesTransport.js +190 -0
  107. package/lib/spriteJobEvents.js +20 -2
  108. package/lib/ssePublish.js +5 -4
  109. package/lib/videoExtendI2vOperation.js +76 -0
  110. package/package.json +5 -2
  111. package/routes/edit.js +31 -123
  112. package/routes/events.js +140 -63
  113. package/routes/mcpMedia.js +10 -2
  114. package/routes/models.js +28 -25
  115. package/routes/videoExtended.js +9 -68
  116. package/server.js +24 -46
  117. package/ui/dist/.vite/manifest.json +90 -63
  118. package/ui/dist/assets/{AgentWorkspace-PbBytTeV.js → AgentWorkspace-CL3MLv3l.js} +2 -2
  119. package/ui/dist/assets/App-B9cEM6dJ.js +9 -0
  120. package/ui/dist/assets/AssetGenWorkspace-swWPvXrx.js +2 -0
  121. package/ui/dist/assets/AssetsWorkspace-CRVeYU2F.js +1 -0
  122. package/ui/dist/assets/{CardNewsWorkspace-BS3jKOja.js → CardNewsWorkspace-BAOxWLKf.js} +2 -2
  123. package/ui/dist/assets/GenerationRequestLogPanel-DZtYZm-D.js +1 -0
  124. package/ui/dist/assets/HomeWorkspace-BeZ1s0Jm.js +1 -0
  125. package/ui/dist/assets/KeyingPanel-ZyBSXioh.js +1 -0
  126. package/ui/dist/assets/NodeCanvas-BvvcCErf.js +7 -0
  127. package/ui/dist/assets/{PromptBuilderPanel-D97A4pwB.js → PromptBuilderPanel-DBycmShA.js} +2 -2
  128. package/ui/dist/assets/PromptImportDialog-ChvMv7ek.js +2 -0
  129. package/ui/dist/assets/PromptImportDiscoverySection-60py7whH.js +1 -0
  130. package/ui/dist/assets/PromptImportFolderSection-CCRh_-_7.js +1 -0
  131. package/ui/dist/assets/PromptLibraryPanel-BFWQO8OT.js +2 -0
  132. package/ui/dist/assets/SettingsWorkspace-B5Ghsgdj.js +1 -0
  133. package/ui/dist/assets/SpriteRecipeWorkspace-DRWdkmwU.js +1 -0
  134. package/ui/dist/assets/index-7ngEOVxt.js +5 -0
  135. package/ui/dist/assets/index-B5jy7ePg.css +1 -0
  136. package/ui/dist/assets/index-gxh_02DQ.js +22 -0
  137. package/ui/dist/assets/{pptxgen.es-D4pedbwM.js → pptxgen.es-OUNhQLEV.js} +1 -1
  138. package/ui/dist/assets/promptBuilderStore-BTUUzbBT.js +1 -0
  139. package/ui/dist/assets/useAgentDialogFocus-DSc2qY8p.js +1 -0
  140. package/ui/dist/index.html +3 -2
  141. package/ui/dist/assets/AssetGenWorkspace-B-TGlvVb.js +0 -2
  142. package/ui/dist/assets/AssetsWorkspace-LnGaW151.js +0 -1
  143. package/ui/dist/assets/GenerationRequestLogPanel-DuDWD_ic.js +0 -1
  144. package/ui/dist/assets/HomeWorkspace-DVP7UG6O.js +0 -1
  145. package/ui/dist/assets/KeyingPanel-5bRUFQaq.js +0 -1
  146. package/ui/dist/assets/NodeCanvas-BBEpERTb.js +0 -7
  147. package/ui/dist/assets/PromptImportDialog-C89MN5JY.js +0 -2
  148. package/ui/dist/assets/PromptImportDiscoverySection-BgRfDOs2.js +0 -1
  149. package/ui/dist/assets/PromptImportFolderSection-zTMyriT1.js +0 -1
  150. package/ui/dist/assets/PromptLibraryPanel-BD96qZu_.js +0 -2
  151. package/ui/dist/assets/SettingsWorkspace-BAOk3X_8.js +0 -1
  152. package/ui/dist/assets/SpriteRecipeWorkspace-ClIM5QRo.js +0 -1
  153. package/ui/dist/assets/index-0fpfY8vu.css +0 -1
  154. package/ui/dist/assets/index-BFciPGTn.js +0 -5
  155. package/ui/dist/assets/index-BbykeXR9.js +0 -30
  156. package/ui/dist/assets/promptBuilderStore-Do7M8g7J.js +0 -1
  157. package/ui/dist/assets/useAgentDialogFocus-B6-QgEVI.js +0 -1
@@ -0,0 +1,179 @@
1
+ import { rm, writeFile, mkdir } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import { tmpdir } from "node:os";
4
+ import { randomBytes } from "node:crypto";
5
+ import { spawnAgy, agyError } from "../../agyProcess.js";
6
+ import { parseAgyOutput, findRecentAgyArtifact } from "../../agyArtifact.js";
7
+ import { readAgyArtifact, cleanupAgyArtifact } from "../../agyArtifactRead.js";
8
+ import { logEvent } from "../../logger.js";
9
+ import { SAFETY_INTENT_POLICY } from "../../promptSafetyPolicy.js";
10
+ import { detectImageMimeFromB64 } from "../../refs.js";
11
+ import { deriveReferenceLimit } from "../derive.js";
12
+ const AGY_OUTPUT_RESOLUTION = "1024x1024";
13
+ function buildAgyPrompt(userPrompt, referencePaths) {
14
+ const imagePathsJson = referencePaths.length > 0
15
+ ? JSON.stringify(referencePaths)
16
+ : "[]";
17
+ return [
18
+ "You are an image generation assistant inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
19
+ SAFETY_INTENT_POLICY,
20
+ "",
21
+ "Generate one image by calling the tool default_api:generate_image once.",
22
+ "After the tool finishes, print one machine-readable result line so ima2-gen can copy the artifact.",
23
+ "",
24
+ "Tool parameters:",
25
+ ` Prompt: ${JSON.stringify(userPrompt)}`,
26
+ ' ImageName: "ima2_generated"',
27
+ ` ImagePaths: ${imagePathsJson}`,
28
+ ' toolSummary: "ima2 pipeline generation"',
29
+ ' toolAction: "Generating ima2 image"',
30
+ "",
31
+ `Reference count: ${referencePaths.length}. The output resolution is fixed at ${AGY_OUTPUT_RESOLUTION}.`,
32
+ "",
33
+ "PROMPT HANDLING:",
34
+ "- Preserve the user's prompt as-is for compliant requests. Do not translate, restyle, or soften benign adult clothing requests such as swimwear.",
35
+ "- If the user wrote in Korean, Japanese, or any non-English language, keep visible text and dialogue in that language.",
36
+ "- Do not sexualize, intensify exposure, or add sexualized/fetish framing to benign swimwear, sportswear, or catalog fashion prompts.",
37
+ "- Do not copy safety labels, refusal reasons, or negative safety constraints into the final image prompt.",
38
+ "- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
39
+ "- Content moderation is still handled downstream by the provider.",
40
+ "",
41
+ "OUTPUT:",
42
+ "If generation succeeds, print: RESULT|<absolute_artifact_path>|<file_extension>",
43
+ "If generation fails, print: ERROR|<concise error message>",
44
+ ].join("\n");
45
+ }
46
+ const MIME_TO_EXT = {
47
+ "image/png": "png",
48
+ "image/jpeg": "jpg",
49
+ "image/webp": "webp",
50
+ };
51
+ async function writeRefsToTempFiles(refs) {
52
+ if (refs.length === 0)
53
+ return { paths: [], cleanup: async () => { } };
54
+ const dir = join(tmpdir(), `ima2-agy-refs-${randomBytes(6).toString("hex")}`);
55
+ const cleanup = async () => {
56
+ try {
57
+ await rm(dir, { recursive: true, force: true });
58
+ }
59
+ catch { /* best-effort: preserve the primary staging/operation error */ }
60
+ };
61
+ try {
62
+ await mkdir(dir, { recursive: true });
63
+ const paths = [];
64
+ for (let i = 0; i < refs.length; i++) {
65
+ const ref = refs[i];
66
+ if (!ref)
67
+ continue;
68
+ const mime = ref.detectedMime || ref.declaredMime || detectImageMimeFromB64(ref.b64) || "image/png";
69
+ const ext = MIME_TO_EXT[mime] || "png";
70
+ const p = join(dir, `ref_${i}.${ext}`);
71
+ await writeFile(p, Buffer.from(ref.b64, "base64"));
72
+ paths.push(p);
73
+ }
74
+ return { paths, cleanup };
75
+ }
76
+ catch (err) {
77
+ await cleanup();
78
+ throw err;
79
+ }
80
+ }
81
+ function throwIfAgyAborted(signal) {
82
+ if (signal?.aborted) {
83
+ throw agyError("Generation canceled", 499, "GENERATION_CANCELED");
84
+ }
85
+ }
86
+ async function resolveAgyArtifactPath(stdout, stderr, generationStartedAtMs, requestId) {
87
+ if (stderr && stderr.trim().length > 0) {
88
+ logEvent("agy", "generate:stderr", {
89
+ requestId,
90
+ stderrChars: stderr.length,
91
+ stderrPreview: stderr.slice(0, 200),
92
+ });
93
+ }
94
+ const agyCombinedOutput = [stdout, stderr].filter(Boolean).join("\n");
95
+ try {
96
+ return parseAgyOutput(agyCombinedOutput).artifactPath;
97
+ }
98
+ catch (err) {
99
+ if (err?.code !== "AGY_PARSE_FAILED") {
100
+ throw err;
101
+ }
102
+ const fallbackPath = await findRecentAgyArtifact(generationStartedAtMs);
103
+ if (!fallbackPath) {
104
+ throw err;
105
+ }
106
+ logEvent("agy", "generate:fallback_artifact_found", {
107
+ requestId,
108
+ artifactPath: fallbackPath,
109
+ stdoutChars: stdout.length,
110
+ stderrChars: stderr.length,
111
+ });
112
+ return fallbackPath;
113
+ }
114
+ }
115
+ async function readAgyArtifactResult(artifactPath, prompt, signal, requestId) {
116
+ const receipt = await readAgyArtifact(artifactPath, signal);
117
+ const { buffer } = receipt;
118
+ let result;
119
+ try {
120
+ throwIfAgyAborted(signal);
121
+ const b64 = buffer.toString("base64");
122
+ const mime = detectImageMimeFromB64(b64) || "image/png";
123
+ logEvent("agy", "generate:done", {
124
+ requestId,
125
+ artifactPath,
126
+ b64Len: b64.length,
127
+ mime,
128
+ fileBytes: buffer.length,
129
+ });
130
+ result = {
131
+ b64,
132
+ revisedPrompt: prompt,
133
+ usage: { agy_artifact_bytes: buffer.length },
134
+ webSearchCalls: 0,
135
+ mime,
136
+ };
137
+ }
138
+ finally {
139
+ // Only the trusted reader's issued receipt authorizes artifact cleanup.
140
+ await cleanupAgyArtifact(receipt);
141
+ }
142
+ throwIfAgyAborted(signal);
143
+ return result;
144
+ }
145
+ export async function generateViaAgy(prompt, options = {}) {
146
+ throwIfAgyAborted(options.signal);
147
+ const refDetails = (options.references || []).slice(0, deriveReferenceLimit("agy", "edit"));
148
+ const { paths: refPaths, cleanup } = await writeRefsToTempFiles(refDetails);
149
+ let result;
150
+ try {
151
+ throwIfAgyAborted(options.signal);
152
+ const agyPrompt = buildAgyPrompt(prompt, refPaths);
153
+ logEvent("agy", "generate:start", {
154
+ requestId: options.requestId,
155
+ promptChars: prompt.length,
156
+ agyPromptChars: agyPrompt.length,
157
+ refs: refPaths.length,
158
+ });
159
+ const generationStartedAtMs = Date.now();
160
+ const { stdout, stderr } = await spawnAgy(agyPrompt, options.signal);
161
+ throwIfAgyAborted(options.signal);
162
+ const artifactPath = await resolveAgyArtifactPath(stdout, stderr, generationStartedAtMs, options.requestId);
163
+ throwIfAgyAborted(options.signal);
164
+ result = await readAgyArtifactResult(artifactPath, prompt, options.signal, options.requestId);
165
+ }
166
+ catch (err) {
167
+ try {
168
+ logEvent("agy", "generate:failed_cleanup", { requestId: options.requestId });
169
+ }
170
+ catch { /* Logging must not replace the primary operation error. */ }
171
+ throw err;
172
+ }
173
+ finally {
174
+ await cleanup();
175
+ }
176
+ // No await after this barrier: cancellation during reference cleanup must win.
177
+ throwIfAgyAborted(options.signal);
178
+ return result;
179
+ }
@@ -35,6 +35,7 @@ export function createComfyAdapter(ctx) {
35
35
  id: workflow.id,
36
36
  kind: "image",
37
37
  supports: {
38
+ generate: true,
38
39
  edit: Boolean(workflow.bind.refImage),
39
40
  mask: false,
40
41
  streaming: false,
@@ -0,0 +1,214 @@
1
+ import { logEvent } from "../../logger.js";
2
+ import { detectImageMimeFromB64 } from "../../refs.js";
3
+ import { getVertexAccessToken, getVertexProjectId, isVertexInitialized } from "../../vertexAuth.js";
4
+ import { deriveReferenceLimit } from "../../providers/derive.js";
5
+ const MODEL_ID_MAP = {
6
+ "nano-banana-2": "gemini-3.1-flash-image",
7
+ "nano-banana-pro": "gemini-3-pro-image",
8
+ };
9
+ const GEMINI_TIMEOUT_MS = 120_000;
10
+ // Public v1beta ImageResponseFormat uses enums (ASPECT_RATIO_*/IMAGE_SIZE_*),
11
+ // while Vertex imageConfig takes plain strings ("1:1"/"1K") — mixing them got
12
+ // every public-API request rejected with invalid aspect_ratio (070 QA).
13
+ const V1BETA_ASPECT_ENUM = {
14
+ "1:1": "ASPECT_RATIO_ONE_BY_ONE", "2:3": "ASPECT_RATIO_TWO_BY_THREE",
15
+ "3:2": "ASPECT_RATIO_THREE_BY_TWO", "3:4": "ASPECT_RATIO_THREE_BY_FOUR",
16
+ "4:3": "ASPECT_RATIO_FOUR_BY_THREE", "4:5": "ASPECT_RATIO_FOUR_BY_FIVE",
17
+ "5:4": "ASPECT_RATIO_FIVE_BY_FOUR", "9:16": "ASPECT_RATIO_NINE_BY_SIXTEEN",
18
+ "16:9": "ASPECT_RATIO_SIXTEEN_BY_NINE", "21:9": "ASPECT_RATIO_TWENTY_ONE_BY_NINE",
19
+ "1:8": "ASPECT_RATIO_ONE_BY_EIGHT", "8:1": "ASPECT_RATIO_EIGHT_BY_ONE",
20
+ "1:4": "ASPECT_RATIO_ONE_BY_FOUR", "4:1": "ASPECT_RATIO_FOUR_BY_ONE",
21
+ };
22
+ const V1BETA_SIZE_ENUM = {
23
+ "512": "IMAGE_SIZE_FIVE_TWELVE",
24
+ "1K": "IMAGE_SIZE_ONE_K",
25
+ "2K": "IMAGE_SIZE_TWO_K",
26
+ "4K": "IMAGE_SIZE_FOUR_K",
27
+ };
28
+ function toV1BetaImageFormat(params) {
29
+ const aspect = V1BETA_ASPECT_ENUM[params.aspectRatio];
30
+ const size = V1BETA_SIZE_ENUM[params.imageSize];
31
+ if (!aspect)
32
+ throw new Error(`gemini-api: no v1beta aspect enum for ${params.aspectRatio}`);
33
+ if (!size)
34
+ throw new Error(`gemini-api: no v1beta image-size enum for ${params.imageSize}`);
35
+ return { aspect_ratio: aspect, image_size: size };
36
+ }
37
+ function parseGeminiImageParams(size) {
38
+ if (!size || size === "auto" || size === "1024x1024")
39
+ return { aspectRatio: "1:1", imageSize: "1K" };
40
+ const match = size.match(/^(\d+)x(\d+)$/);
41
+ if (!match)
42
+ return { aspectRatio: "1:1", imageSize: "1K" };
43
+ const w = Number(match[1]);
44
+ const h = Number(match[2]);
45
+ const ratio = w / h;
46
+ const ratioMap = [
47
+ ["1:1", 1], ["2:3", 2 / 3], ["3:2", 3 / 2], ["3:4", 3 / 4], ["4:3", 4 / 3],
48
+ ["4:5", 4 / 5], ["5:4", 5 / 4], ["9:16", 9 / 16], ["16:9", 16 / 9], ["21:9", 21 / 9],
49
+ ["1:8", 1 / 8], ["8:1", 8], ["1:4", 1 / 4], ["4:1", 4],
50
+ ];
51
+ let bestLabel = "1:1";
52
+ let bestDist = Infinity;
53
+ for (const [label, val] of ratioMap) {
54
+ const dist = Math.abs(ratio - val);
55
+ if (dist < bestDist) {
56
+ bestDist = dist;
57
+ bestLabel = label;
58
+ }
59
+ }
60
+ const maxDim = Math.max(w, h);
61
+ const imageSize = maxDim <= 512 ? "512" : maxDim <= 1024 ? "1K" : maxDim <= 2048 ? "2K" : "4K";
62
+ return { aspectRatio: bestLabel, imageSize };
63
+ }
64
+ function geminiApiError(message, status, code) {
65
+ const err = new Error(message);
66
+ err.status = status;
67
+ err.code = code;
68
+ return err;
69
+ }
70
+ function resolveGeminiModelId(model) {
71
+ return MODEL_ID_MAP[model] || model;
72
+ }
73
+ function buildContents(prompt, references) {
74
+ const parts = [];
75
+ // Add reference images first (if any)
76
+ for (const ref of references.slice(0, deriveReferenceLimit("gemini-api", "edit"))) {
77
+ const mime = ref.declaredMime || ref.detectedMime || detectImageMimeFromB64(ref.b64) || "image/png";
78
+ parts.push({
79
+ inlineData: {
80
+ mimeType: mime,
81
+ data: ref.b64,
82
+ },
83
+ });
84
+ }
85
+ // Add text prompt
86
+ parts.push({ text: prompt });
87
+ return [{ role: "user", parts }];
88
+ }
89
+ export async function generateViaGeminiApi(prompt, ctx, options = {}) {
90
+ const apiKey = ctx.geminiApiKey;
91
+ const vertexReady = ctx.hasVertexKey && isVertexInitialized();
92
+ const authMode = ctx.geminiAuthMode;
93
+ const useVertex = authMode === "vertex" ? vertexReady : (!apiKey && vertexReady);
94
+ if (!apiKey && !useVertex) {
95
+ throw geminiApiError("Gemini API key or Vertex AI credentials not configured", 401, "GEMINI_API_KEY_MISSING");
96
+ }
97
+ const model = options.model || "nano-banana-2";
98
+ const apiModelId = resolveGeminiModelId(model);
99
+ const references = (options.references || []).slice(0, deriveReferenceLimit("gemini-api", "edit"));
100
+ let url;
101
+ let authHeaders;
102
+ if (useVertex) {
103
+ const token = await getVertexAccessToken();
104
+ const projectId = getVertexProjectId();
105
+ url = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/global/publishers/google/models/${apiModelId}:generateContent`;
106
+ authHeaders = { "Content-Type": "application/json", "Authorization": `Bearer ${token}` };
107
+ }
108
+ else {
109
+ url = `https://generativelanguage.googleapis.com/v1beta/models/${apiModelId}:generateContent`;
110
+ authHeaders = { "Content-Type": "application/json", "x-goog-api-key": apiKey };
111
+ }
112
+ // size "auto" → omit image config entirely so the model decides ratio/size
113
+ const isAutoSize = !options.size || options.size === "auto";
114
+ const imageParams = isAutoSize ? null : parseGeminiImageParams(options.size);
115
+ // Vertex AI rejects responseFormat; it expects imageConfig directly under
116
+ // generationConfig (camelCase). Public API uses response_format.image (snake_case).
117
+ const generationConfig = useVertex
118
+ ? {
119
+ responseModalities: ["TEXT", "IMAGE"],
120
+ ...(imageParams
121
+ ? { imageConfig: { aspectRatio: imageParams.aspectRatio, imageSize: imageParams.imageSize } }
122
+ : {}),
123
+ }
124
+ : {
125
+ response_modalities: ["TEXT", "IMAGE"],
126
+ ...(imageParams
127
+ ? { response_format: { image: toV1BetaImageFormat(imageParams) } }
128
+ : {}),
129
+ };
130
+ const configKey = useVertex ? "generationConfig" : "generation_config";
131
+ const body = { contents: buildContents(prompt, references), [configKey]: generationConfig };
132
+ logEvent("gemini-api", "generate:start", {
133
+ requestId: options.requestId,
134
+ model,
135
+ apiModelId,
136
+ promptChars: prompt.length,
137
+ refs: references.length,
138
+ });
139
+ const timeoutSignal = AbortSignal.timeout(GEMINI_TIMEOUT_MS);
140
+ const combinedSignal = options.signal
141
+ ? AbortSignal.any([options.signal, timeoutSignal])
142
+ : timeoutSignal;
143
+ try {
144
+ const res = await fetch(url, {
145
+ method: "POST",
146
+ headers: authHeaders,
147
+ body: JSON.stringify(body),
148
+ signal: combinedSignal,
149
+ });
150
+ if (!res.ok) {
151
+ const text = await res.text().catch(() => "");
152
+ if (res.status === 429) {
153
+ throw geminiApiError(`Gemini API rate limited: ${text.slice(0, 200)}`, 429, "GEMINI_API_RATE_LIMITED");
154
+ }
155
+ if (res.status === 400 || res.status === 403) {
156
+ throw geminiApiError(`Gemini API error: ${text.slice(0, 200)}`, res.status, "GEMINI_API_BAD_REQUEST");
157
+ }
158
+ throw geminiApiError(`Gemini API error (${res.status}): ${text.slice(0, 200)}`, 502, "GEMINI_API_UPSTREAM_ERROR");
159
+ }
160
+ const json = await res.json();
161
+ // Extract image from candidates[0].content.parts[]
162
+ const parts = json?.candidates?.[0]?.content?.parts || [];
163
+ let b64 = null;
164
+ let textResponse = "";
165
+ let mime = "image/png";
166
+ for (const part of parts) {
167
+ if (part.inlineData?.data) {
168
+ b64 = part.inlineData.data;
169
+ mime = part.inlineData.mimeType || "image/png";
170
+ }
171
+ if (part.text) {
172
+ textResponse += part.text;
173
+ }
174
+ }
175
+ if (!b64) {
176
+ // Check for safety block
177
+ const finishReason = json?.candidates?.[0]?.finishReason;
178
+ if (finishReason === "SAFETY") {
179
+ throw geminiApiError("Gemini API: generation blocked by safety filter", 400, "GEMINI_API_SAFETY_BLOCKED");
180
+ }
181
+ throw geminiApiError(`Gemini API: no image in response (finishReason: ${finishReason || "unknown"})`, 502, "GEMINI_API_NO_IMAGE");
182
+ }
183
+ const usageMetadata = json?.usageMetadata || {};
184
+ logEvent("gemini-api", "generate:done", {
185
+ requestId: options.requestId,
186
+ model,
187
+ b64Len: b64.length,
188
+ mime,
189
+ textResponseLen: textResponse.length,
190
+ });
191
+ return {
192
+ b64,
193
+ revisedPrompt: textResponse || prompt,
194
+ usage: {
195
+ promptTokens: usageMetadata.promptTokenCount || 0,
196
+ candidatesTokens: usageMetadata.candidatesTokenCount || 0,
197
+ totalTokens: usageMetadata.totalTokenCount || 0,
198
+ },
199
+ webSearchCalls: 0,
200
+ mime,
201
+ };
202
+ }
203
+ catch (e) {
204
+ if (e.name === "AbortError") {
205
+ if (options.signal?.aborted) {
206
+ throw geminiApiError("Generation canceled", 499, "GENERATION_CANCELED");
207
+ }
208
+ throw geminiApiError("Gemini API generation timed out", 504, "GENERATION_TIMEOUT");
209
+ }
210
+ if (e.code && e.status)
211
+ throw e;
212
+ throw geminiApiError(`Gemini API request failed: ${e.message}`, 502, "GEMINI_API_NETWORK_FAILED");
213
+ }
214
+ }
@@ -0,0 +1,97 @@
1
+ import { requireRuntimeContext } from "../../runtimeContext.js";
2
+ import { detectImageMimeFromB64 } from "../../refs.js";
3
+ import { generateViaAgy } from "./agyOperations.js";
4
+ import { generateViaGeminiApi } from "./geminiOperations.js";
5
+ export function isGoogleRequest(request) {
6
+ return request.provider === "agy" || request.provider === "gemini-api";
7
+ }
8
+ function googleInput(request) {
9
+ if (request.surface === "edit") {
10
+ return {
11
+ prompt: `Edit this image: ${request.prompt}`,
12
+ references: [{ b64: request.sourceImage, declaredMime: null,
13
+ detectedMime: detectImageMimeFromB64(request.sourceImage) || null }],
14
+ };
15
+ }
16
+ if (request.surface === "node") {
17
+ const selectedRefs = request.contextMode === "parent-only" ? [] : request.references;
18
+ return {
19
+ prompt: request.sourceImage ? `Edit this image: ${request.prompt}` : request.prompt,
20
+ references: request.sourceImage
21
+ ? [{ b64: request.sourceImage, declaredMime: null, detectedMime: null }, ...selectedRefs]
22
+ : selectedRefs,
23
+ };
24
+ }
25
+ return { prompt: request.prompt, references: request.references };
26
+ }
27
+ function prepareGoogleClassic(ctx, request) {
28
+ const { provider, prompt, requestId } = request;
29
+ const { model, size } = request.options;
30
+ // Classic captures scalars once; each attempt reads live refs, signal and credentials.
31
+ return { execute: async () => {
32
+ try {
33
+ const value = provider === "agy"
34
+ ? await generateViaAgy(prompt, { references: request.references, signal: request.signal, requestId })
35
+ : await generateViaGeminiApi(prompt, requireRuntimeContext(ctx), {
36
+ model, size, signal: request.signal,
37
+ ...(requestId !== undefined ? { requestId } : {}),
38
+ references: request.references,
39
+ });
40
+ return { kind: "single", value };
41
+ }
42
+ catch (error) {
43
+ throw error;
44
+ } // Preserve native identity; caller owns normalization.
45
+ } };
46
+ }
47
+ async function runGoogleImage(ctx, request, input) {
48
+ try {
49
+ const { provider, signal, requestId, options } = request;
50
+ if (provider === "agy") {
51
+ return await generateViaAgy(input.prompt, { references: input.references, signal, requestId });
52
+ }
53
+ return await generateViaGeminiApi(input.prompt, request.surface === "node" ? requireRuntimeContext(ctx) : ctx, {
54
+ model: options.model, size: options.size, signal,
55
+ ...(requestId !== undefined ? { requestId } : {}),
56
+ references: input.references,
57
+ });
58
+ }
59
+ catch (error) {
60
+ throw error;
61
+ }
62
+ }
63
+ function googleSequence(result) {
64
+ // Preserve the dense one-image projection; the caller owns the final persistence sweep.
65
+ return {
66
+ images: [{ b64: result.b64, ...(result.revisedPrompt !== undefined ? { revisedPrompt: result.revisedPrompt } : {}) }],
67
+ usage: result.usage, webSearchCalls: result.webSearchCalls,
68
+ };
69
+ }
70
+ export async function prepareGoogleExecution(ctx, request, _progress) {
71
+ try {
72
+ switch (request.surface) {
73
+ case "classic": return prepareGoogleClassic(ctx, request);
74
+ case "node":
75
+ case "edit": return { execute: async () => {
76
+ try {
77
+ return { kind: "single", value: await runGoogleImage(ctx, request, googleInput(request)) };
78
+ }
79
+ catch (error) {
80
+ throw error;
81
+ } // Caller owns retries, cancellation and cleanup.
82
+ } };
83
+ case "multimode": return { execute: async () => {
84
+ try {
85
+ const result = await runGoogleImage(ctx, request, googleInput(request));
86
+ return { kind: "sequence", value: googleSequence(result) };
87
+ }
88
+ catch (error) {
89
+ throw error;
90
+ }
91
+ } };
92
+ }
93
+ }
94
+ catch (error) {
95
+ throw error;
96
+ }
97
+ }
@@ -0,0 +1,137 @@
1
+ import { planGrokImage } from "../../grokImagePlanner.js";
2
+ import { resolveGrokQualityModel } from "../../imageModels.js";
3
+ import { toGrokReferences } from "../../nodeHelpers.js";
4
+ import { generateViaGrok, editViaGrok } from "./grokOperations.js";
5
+ import { generateMultimodeViaGrok } from "./grokMultimodeOperations.js";
6
+ export function isGrokRequest(request) {
7
+ return request.provider === "grok" || request.provider === "grok-api";
8
+ }
9
+ async function prepareGrokClassic(ctx, request) {
10
+ try {
11
+ const { provider: activeProvider, prompt: generationPrompt, requestId } = request;
12
+ const { model: imageModel, quality, size: effectiveSize, webSearchEnabled } = request.options;
13
+ const grokRefs = request.providerUrl
14
+ ? [{ b64: "", url: request.providerUrl, declaredMime: "image/png", detectedMime: "image/png" }, ...request.references]
15
+ : request.references;
16
+ // Keep classic's once-per-batch key and shared plan capture before any image attempt.
17
+ const grokDirectApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
18
+ const sharedGrokPlan = await planGrokImage(generationPrompt, ctx, {
19
+ model: resolveGrokQualityModel(imageModel, quality),
20
+ size: effectiveSize,
21
+ signal: request.signal,
22
+ requestId,
23
+ referenceCount: grokRefs.length,
24
+ references: grokRefs,
25
+ directApiKey: grokDirectApiKey,
26
+ backgroundConstraint: request.backgroundConstraint,
27
+ webSearchEnabled,
28
+ });
29
+ return { execute: async () => {
30
+ try {
31
+ const grokModel = resolveGrokQualityModel(imageModel, quality);
32
+ const value = await generateViaGrok(generationPrompt, ctx, {
33
+ model: grokModel,
34
+ size: effectiveSize,
35
+ signal: request.signal,
36
+ requestId,
37
+ plannedPrompt: sharedGrokPlan?.prompt,
38
+ webSearchCalls: sharedGrokPlan?.webSearchCalls,
39
+ references: grokRefs,
40
+ directApiKey: grokDirectApiKey,
41
+ });
42
+ return { kind: "single", value };
43
+ }
44
+ catch (error) {
45
+ throw error;
46
+ } // Caller owns lifecycle and the existing error envelope.
47
+ } };
48
+ }
49
+ catch (error) {
50
+ throw error;
51
+ }
52
+ }
53
+ function prepareGrokNode(ctx, request) {
54
+ // The outer execution boundary checks current presence; retries retain this key.
55
+ const grokDirectApiKey = request.provider === "grok-api" ? ctx.xaiApiKey : undefined;
56
+ return { execute: async () => {
57
+ try {
58
+ return { kind: "single", value: await executeGrokNode(ctx, request, grokDirectApiKey) };
59
+ }
60
+ catch (error) {
61
+ throw error;
62
+ } // The caller owns retries and normalization.
63
+ } };
64
+ }
65
+ async function executeGrokNode(ctx, request, grokDirectApiKey) {
66
+ try {
67
+ const { sourceImage: parentB64, prompt: generationPrompt, references, requestId, signal, options } = request;
68
+ const { model, size, webSearchEnabled } = options;
69
+ const refsForRequest = request.contextMode === "parent-only" ? [] : references;
70
+ return await generateViaGrok(generationPrompt, ctx, {
71
+ model, size, requestId, signal,
72
+ references: toGrokReferences(parentB64, refsForRequest),
73
+ directApiKey: grokDirectApiKey,
74
+ webSearchEnabled,
75
+ });
76
+ }
77
+ catch (error) {
78
+ throw error;
79
+ }
80
+ }
81
+ async function executeGrokEdit(ctx, request) {
82
+ try {
83
+ const { provider, rawPrompt, sourceImage, signal, requestId, options } = request;
84
+ const directApiKey = provider === "grok-api" ? ctx.xaiApiKey : undefined;
85
+ return await editViaGrok(rawPrompt, sourceImage, ctx, {
86
+ model: resolveGrokQualityModel(options.model, options.quality),
87
+ size: options.size, signal, requestId, directApiKey,
88
+ });
89
+ }
90
+ catch (error) {
91
+ throw error;
92
+ }
93
+ }
94
+ async function executeGrokMultimode(ctx, request, progress) {
95
+ try {
96
+ const { provider, prompt, references, signal, requestId, options, maxImages } = request;
97
+ const directApiKey = provider === "grok-api" ? ctx.xaiApiKey : undefined;
98
+ const grokRefs = request.providerUrl
99
+ ? [{ b64: "", url: request.providerUrl }, ...references] : references;
100
+ return await generateMultimodeViaGrok(prompt, ctx, {
101
+ model: resolveGrokQualityModel(options.model, options.quality), maxImages,
102
+ size: options.size, signal, requestId, references: grokRefs, directApiKey,
103
+ onFinalImage: progress.onFinalImage,
104
+ webSearchEnabled: options.webSearchEnabled,
105
+ });
106
+ }
107
+ catch (error) {
108
+ throw error;
109
+ }
110
+ }
111
+ export async function prepareGrokExecution(ctx, request, progress = {}) {
112
+ try {
113
+ switch (request.surface) {
114
+ case "classic": return await prepareGrokClassic(ctx, request);
115
+ case "node": return prepareGrokNode(ctx, request);
116
+ case "edit": return { execute: async () => {
117
+ try {
118
+ return { kind: "single", value: await executeGrokEdit(ctx, request) };
119
+ }
120
+ catch (error) {
121
+ throw error;
122
+ } // Route owns failure normalization and job cleanup.
123
+ } };
124
+ case "multimode": return { execute: async () => {
125
+ try {
126
+ return { kind: "sequence", value: await executeGrokMultimode(ctx, request, progress) };
127
+ }
128
+ catch (error) {
129
+ throw error;
130
+ } // Caller owns partial-timeout recovery.
131
+ } };
132
+ }
133
+ }
134
+ catch (error) {
135
+ throw error;
136
+ }
137
+ }