ima2-gen 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +151 -0
- package/README.md +10 -1
- package/bin/commands/backfillThumbs.js +6 -0
- package/bin/commands/gen.js +13 -5
- package/bin/commands/multimode.js +7 -5
- package/bin/commands/node.js +2 -1
- package/bin/ima2.js +14 -10
- package/config.js +8 -1
- package/docs/API.md +133 -9
- package/docs/CLI.md +6 -5
- package/docs/FAQ.ko.md +16 -0
- package/docs/FAQ.md +30 -0
- package/docs/README.ko.md +7 -3
- package/docs/migration/runtime-test-inventory.md +25 -1
- package/lib/agentCommandParser.js +10 -6
- package/lib/agentGenerationPlanner.js +96 -19
- package/lib/agentImageVideoGen.js +281 -0
- package/lib/agentPlannerModel.js +172 -0
- package/lib/agentQuestionResponder.js +9 -3
- package/lib/agentQueueStore.js +42 -0
- package/lib/agentQueueWorker.js +57 -7
- package/lib/agentRuntime.js +103 -274
- package/lib/agentSettings.js +9 -6
- package/lib/agentToolManifest.js +90 -0
- package/lib/agentTypes.js +1 -0
- package/lib/agyCli.js +31 -0
- package/lib/agyImageAdapter.js +104 -12
- package/lib/capabilities.js +7 -5
- package/lib/configKeys.js +6 -0
- package/lib/errorClassify.js +8 -7
- package/lib/eventBus.js +71 -0
- package/lib/geminiApiImageAdapter.js +24 -21
- package/lib/generationErrors.js +3 -1
- package/lib/generationRequestLog.js +25 -0
- package/lib/grokImageAdapter.js +76 -144
- package/lib/grokImageCore.js +153 -0
- package/lib/grokMultimodeAdapter.js +7 -4
- package/lib/grokVideoAdapter.js +3 -2
- package/lib/grokVideoCanvas.js +13 -0
- package/lib/grokVideoPlannerPrompt.js +52 -12
- package/lib/historyList.js +1 -0
- package/lib/inflight.js +54 -17
- package/lib/multimodeHelpers.js +11 -0
- package/lib/nodeHelpers.js +59 -0
- package/lib/oauthProxy/generators.js +6 -5
- package/lib/oauthProxy/prompts.js +60 -41
- package/lib/oauthProxy/streams.js +1 -1
- package/lib/promptBuilder/systemPrompt.js +2 -5
- package/lib/promptSafetyPolicy.js +1 -5
- package/lib/responsesDoctor.js +3 -3
- package/lib/responsesFallback.js +29 -15
- package/lib/responsesImageAdapter.js +11 -7
- package/lib/routeHelpers.js +44 -0
- package/lib/ssePublish.js +12 -0
- package/lib/storyboardPrefix.js +28 -0
- package/lib/thumbBackfill.js +16 -5
- package/node_modules/qs/CHANGELOG.md +10 -0
- package/node_modules/qs/README.md +1 -1
- package/node_modules/qs/dist/qs.js +15 -15
- package/node_modules/qs/eslint.config.mjs +1 -0
- package/node_modules/qs/lib/parse.js +52 -22
- package/node_modules/qs/lib/stringify.js +11 -4
- package/node_modules/qs/package.json +2 -2
- package/node_modules/qs/test/parse.js +49 -0
- package/node_modules/qs/test/stringify.js +129 -0
- package/package.json +4 -1
- package/routes/agent.js +19 -2
- package/routes/agy.js +48 -0
- package/routes/auth.js +6 -2
- package/routes/edit.js +7 -1
- package/routes/events.js +78 -0
- package/routes/generate.js +111 -128
- package/routes/generationRequestLog.js +16 -0
- package/routes/index.js +6 -0
- package/routes/keys.js +22 -2
- package/routes/multimode.js +100 -57
- package/routes/nodes.js +64 -108
- package/routes/video.js +100 -17
- package/skills/ima2/SKILL.md +98 -21
- package/ui/dist/.vite/manifest.json +22 -12
- package/ui/dist/assets/AgentWorkspace-qEMrATBP.js +3 -0
- package/ui/dist/assets/{CardNewsWorkspace-Dqyc1WZ1.js → CardNewsWorkspace-DoJnVHmn.js} +1 -1
- package/ui/dist/assets/GenerationRequestLogPanel-BDWRRLNV.js +1 -0
- package/ui/dist/assets/{NodeCanvas-ChEXzQbb.js → NodeCanvas-DIMo45sp.js} +3 -3
- package/ui/dist/assets/{PromptBuilderPanel-B95ZufnR.js → PromptBuilderPanel-BF3lcSJD.js} +2 -2
- package/ui/dist/assets/{PromptImportDialog-DGOwFQET.js → PromptImportDialog-Cy6ZymKG.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-CgvdnR49.js → PromptImportDiscoverySection-CzuW-8P2.js} +1 -1
- package/ui/dist/assets/PromptImportFolderSection-DP5ywnsv.js +1 -0
- package/ui/dist/assets/{PromptLibraryPanel-B9kndPw1.js → PromptLibraryPanel-CAsKr7CV.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-_PdPVsxi.js +1 -0
- package/ui/dist/assets/index-BUinlX2n.js +4 -0
- package/ui/dist/assets/index-CX3fge8X.css +1 -0
- package/ui/dist/assets/index-ygo6nfqx.js +23 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/AgentWorkspace-CYv84Rus.js +0 -3
- package/ui/dist/assets/PromptImportFolderSection-CfUye9J8.js +0 -1
- package/ui/dist/assets/SettingsWorkspace-B3tgLrmF.js +0 -1
- package/ui/dist/assets/index-BhcvL0g-.js +0 -1
- package/ui/dist/assets/index-BtK3YhJc.js +0 -39
- package/ui/dist/assets/index-ClOLOjnA.css +0 -1
package/lib/grokImageAdapter.js
CHANGED
|
@@ -1,117 +1,9 @@
|
|
|
1
1
|
import { logEvent } from "./logger.js";
|
|
2
|
+
import { SAFETY_INTENT_POLICY } from "./promptSafetyPolicy.js";
|
|
2
3
|
import { mapSizeToGrokImageParams } from "./grokSizeMapper.js";
|
|
3
4
|
import { detectImageMimeFromB64 } from "./refs.js";
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
if (directApiKey) {
|
|
7
|
-
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
8
|
-
return {
|
|
9
|
-
url: `https://api.x.ai${normalizedPath}`,
|
|
10
|
-
headers: { "Content-Type": "application/json", Authorization: `Bearer ${directApiKey}` },
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
|
-
return {
|
|
14
|
-
url: getGrokProxyUrl(ctx, path),
|
|
15
|
-
headers: { "Content-Type": "application/json", Authorization: "Bearer dummy" },
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
function getGrokTimeout(ctx) {
|
|
19
|
-
return ctx.config.grokProvider?.generationTimeoutMs || 120_000;
|
|
20
|
-
}
|
|
21
|
-
export function grokError(message, status, code) {
|
|
22
|
-
const err = new Error(message);
|
|
23
|
-
err.status = status;
|
|
24
|
-
err.code = code;
|
|
25
|
-
return err;
|
|
26
|
-
}
|
|
27
|
-
function grokStageError(stage, message, status) {
|
|
28
|
-
const prefix = stage === "search" ? "GROK_SEARCH" : "GROK_PLANNER";
|
|
29
|
-
if (status === 429)
|
|
30
|
-
return grokError(`${stage} rate limited: ${message}`, 429, "GROK_RATE_LIMITED");
|
|
31
|
-
if (status === 401 || status === 403)
|
|
32
|
-
return grokError(`${stage} auth failed: ${message}`, 502, "GROK_AUTH_FAILED");
|
|
33
|
-
if (status >= 500)
|
|
34
|
-
return grokError(`${stage} upstream error: ${message}`, 502, "GROK_UPSTREAM_ERROR");
|
|
35
|
-
return grokError(`Grok ${stage} bad request: ${message}`, status, `${prefix}_BAD_REQUEST`);
|
|
36
|
-
}
|
|
37
|
-
function getPlannerConfig(ctx) {
|
|
38
|
-
const grokCfg = ctx.config.grokProvider || {};
|
|
39
|
-
return {
|
|
40
|
-
model: grokCfg.plannerModel || "grok-4.3",
|
|
41
|
-
timeoutMs: grokCfg.plannerTimeoutMs || 60_000,
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
function withTimeoutSignal(signal, timeoutMs) {
|
|
45
|
-
const timeoutController = new AbortController();
|
|
46
|
-
const timer = setTimeout(() => timeoutController.abort(), timeoutMs);
|
|
47
|
-
const combinedSignal = signal ? AbortSignal.any([signal, timeoutController.signal]) : timeoutController.signal;
|
|
48
|
-
return { combinedSignal, timer };
|
|
49
|
-
}
|
|
50
|
-
export function imagePayload(model, prompt, size) {
|
|
51
|
-
return { model, prompt, n: 1, response_format: "b64_json", ...mapSizeToGrokImageParams(size) };
|
|
52
|
-
}
|
|
53
|
-
function referenceImageUrl(ref) {
|
|
54
|
-
const inputMime = ref.declaredMime || ref.detectedMime || detectImageMimeFromB64(ref.b64) || "image/png";
|
|
55
|
-
return ref.b64.startsWith("data:") ? ref.b64 : `data:${inputMime};base64,${ref.b64}`;
|
|
56
|
-
}
|
|
57
|
-
export function imageEditPayload(model, prompt, references, size) {
|
|
58
|
-
const sourceImages = references.map((ref) => ({ type: "image_url", url: referenceImageUrl(ref) }));
|
|
59
|
-
return { model, prompt, n: 1, response_format: "b64_json", ...(sourceImages.length === 1 ? { image: sourceImages[0] } : { images: sourceImages }), ...mapSizeToGrokImageParams(size) };
|
|
60
|
-
}
|
|
61
|
-
function extractResponsesText(response) {
|
|
62
|
-
const chunks = [];
|
|
63
|
-
for (const item of response.output || []) {
|
|
64
|
-
if (item.type !== "message")
|
|
65
|
-
continue;
|
|
66
|
-
for (const content of item.content || []) {
|
|
67
|
-
if (typeof content.text === "string" && content.text.trim())
|
|
68
|
-
chunks.push(content.text.trim());
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return chunks.join("\n\n").trim();
|
|
72
|
-
}
|
|
73
|
-
export async function postGrokImages(ctx, payload, signal, path = "/v1/images/generations", directApiKey) {
|
|
74
|
-
const { url, headers } = getGrokEndpoint(ctx, path, directApiKey);
|
|
75
|
-
const timeoutMs = getGrokTimeout(ctx);
|
|
76
|
-
const { combinedSignal, timer } = withTimeoutSignal(signal, timeoutMs);
|
|
77
|
-
try {
|
|
78
|
-
const res = await fetch(url, {
|
|
79
|
-
method: "POST",
|
|
80
|
-
headers,
|
|
81
|
-
body: JSON.stringify(payload),
|
|
82
|
-
signal: combinedSignal,
|
|
83
|
-
});
|
|
84
|
-
clearTimeout(timer);
|
|
85
|
-
if (!res.ok) {
|
|
86
|
-
const text = await res.text().catch(() => "");
|
|
87
|
-
let parsed;
|
|
88
|
-
try {
|
|
89
|
-
parsed = JSON.parse(text);
|
|
90
|
-
}
|
|
91
|
-
catch { /* ignore */ }
|
|
92
|
-
const msg = parsed?.error || text || `HTTP ${res.status}`;
|
|
93
|
-
if (res.status === 429)
|
|
94
|
-
throw grokError(`Grok rate limited: ${msg}`, 429, "GROK_RATE_LIMITED");
|
|
95
|
-
if (res.status === 401 || res.status === 403)
|
|
96
|
-
throw grokError(`Grok auth failed: ${msg}`, 502, "GROK_AUTH_FAILED");
|
|
97
|
-
if (res.status >= 500)
|
|
98
|
-
throw grokError(`Grok upstream error: ${msg}`, 502, "GROK_UPSTREAM_ERROR");
|
|
99
|
-
throw grokError(`Grok bad request: ${msg}`, res.status, "GROK_BAD_REQUEST");
|
|
100
|
-
}
|
|
101
|
-
return await res.json();
|
|
102
|
-
}
|
|
103
|
-
catch (e) {
|
|
104
|
-
clearTimeout(timer);
|
|
105
|
-
if (e.name === "AbortError") {
|
|
106
|
-
if (signal?.aborted)
|
|
107
|
-
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
108
|
-
throw grokError("Grok image generation timed out", 504, "GENERATION_TIMEOUT");
|
|
109
|
-
}
|
|
110
|
-
if (e.code && e.status)
|
|
111
|
-
throw e;
|
|
112
|
-
throw grokError(`Grok request failed: ${e.message}`, 502, "GROK_NETWORK_FAILED");
|
|
113
|
-
}
|
|
114
|
-
}
|
|
5
|
+
import { grokError, grokStageError, getPlannerConfig, getGrokEndpoint, withTimeoutSignal, imagePayload, imageEditPayload, referenceImageUrl, extractResponsesText, postGrokImages, downloadGrokImageUrl, } from "./grokImageCore.js";
|
|
6
|
+
export { grokError, imagePayload, imageEditPayload, postGrokImages, downloadGrokImageUrl, } from "./grokImageCore.js";
|
|
115
7
|
export function buildGrokPlannerPayload(prompt, model, size, sizeParams, plannerModel = "grok-4.3", searchSummary = "", references = 0) {
|
|
116
8
|
const referenceImages = Array.isArray(references) ? references : [];
|
|
117
9
|
const referenceCount = Array.isArray(references) ? references.length : references;
|
|
@@ -129,17 +21,19 @@ export function buildGrokPlannerPayload(prompt, model, size, sizeParams, planner
|
|
|
129
21
|
{
|
|
130
22
|
role: "system",
|
|
131
23
|
content: [
|
|
132
|
-
"You are ima2's image generation planner for xAI Grok Imagine (Aurora model).",
|
|
24
|
+
"You are ima2's image generation planner for xAI Grok Imagine (Aurora model), inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
|
|
25
|
+
SAFETY_INTENT_POLICY,
|
|
133
26
|
"",
|
|
134
27
|
"TASK: Rewrite the user's casual request into ONE optimal, production-ready image prompt.",
|
|
135
28
|
"",
|
|
136
29
|
"OUTPUT FORMAT: A single natural-language paragraph (NOT tags, NOT keyword lists, NOT weighted tokens like (word:1.2)).",
|
|
137
|
-
"Structure the paragraph in this
|
|
138
|
-
"1. Core subject/scene — who or what, with specific physical details (face shape, hair, clothing, pose)",
|
|
139
|
-
"2. Environment/setting — where, with concrete spatial details",
|
|
140
|
-
"3. Lighting + mood/emotion — use
|
|
141
|
-
"4. Camera/
|
|
30
|
+
"Structure the paragraph in this MANDATORY order — each element flows into the next:",
|
|
31
|
+
"1. Core subject/scene — who or what, with specific physical details (face shape, hair, clothing, pose, expression)",
|
|
32
|
+
"2. Environment/setting — where, with concrete spatial details and materials (wet concrete, polished marble, weathered wood)",
|
|
33
|
+
"3. Lighting + mood/emotion — use specific terms (golden hour backlight, overcast diffused, hard rim light, soft diffuse, nostalgic, melancholic, electric)",
|
|
34
|
+
"4. Camera/lens — shot type + specific focal length + depth of field (close-up on 85mm f/1.4 with shallow DoF, wide establishing on 24mm with deep focus)",
|
|
142
35
|
"5. Style/technical references — art style, medium, or mood reference (cinematic realism, Studio Ghibli feel, oil painting texture)",
|
|
36
|
+
"6. Quality finish — resolution/detail boosters appropriate to style (8K, highly detailed, sharp focus, film grain, bokeh)",
|
|
143
37
|
"",
|
|
144
38
|
"RULES:",
|
|
145
39
|
"- Write like a director briefing a cinematographer, not like a search query.",
|
|
@@ -148,28 +42,60 @@ export function buildGrokPlannerPayload(prompt, model, size, sizeParams, planner
|
|
|
148
42
|
"- If reference images are attached, inspect them and describe the visual details (subject appearance, style, color palette) that must be preserved in the output.",
|
|
149
43
|
"- Do NOT use Stable Diffusion tags, booru tags, or comma-separated keyword lists.",
|
|
150
44
|
"- Do NOT use weighting syntax like (word:1.2) or [word].",
|
|
151
|
-
"-
|
|
45
|
+
"- Quality boosters like '8K', 'highly detailed', 'sharp focus', 'cinematic lighting' ARE effective for this model — use them when appropriate to the requested style.",
|
|
152
46
|
"- The prompt should be 2-5 sentences (40-120 words). Extremely short prompts produce generic results.",
|
|
153
47
|
"",
|
|
48
|
+
"CAMERA/LENS ENRICHMENT:",
|
|
49
|
+
"- If the user's prompt does not specify camera/lens details, add appropriate ones based on the subject:",
|
|
50
|
+
" - Portraits/people: 85mm f/1.4, shallow depth of field, bokeh background",
|
|
51
|
+
" - Environmental/landscape: 24-35mm wide-angle, deep depth of field",
|
|
52
|
+
" - Product/macro: 100mm macro lens, controlled studio lighting",
|
|
53
|
+
" - Action/dynamic: 35mm, fast shutter, slight motion blur on extremities",
|
|
54
|
+
" - Architecture/interior: 24mm tilt-shift or 14mm ultra-wide",
|
|
55
|
+
"- Always include a specific focal length — this model responds strongly to lens specifications.",
|
|
56
|
+
"",
|
|
154
57
|
"MULTI-CHARACTER IDENTIFICATION:",
|
|
155
58
|
"- Identify each character by VISUAL APPEARANCE, not by name alone.",
|
|
156
59
|
" The image model cannot recognize names — it only sees visual features.",
|
|
157
60
|
"- When multiple characters appear, describe each by clothing, physique, position, or props.",
|
|
158
61
|
"- Characters must be distinguishable by at least two visual attributes.",
|
|
159
62
|
"",
|
|
63
|
+
"BENIGN CLOTHING AND CONTEXT HANDLING:",
|
|
64
|
+
"Before rewriting, distinguish ordinary clothing/swimwear descriptions from sexualized requests.",
|
|
65
|
+
"Preserve benign adult clothing intent without adding erotic details:",
|
|
66
|
+
"",
|
|
67
|
+
"bikini/비키니 → adult two-piece swimwear in a neutral beach, pool, swimwear catalog, or fashion editorial context",
|
|
68
|
+
"swimsuit/수영복 → adult swimwear suitable for swimming, beach, pool, sport, or catalog photography",
|
|
69
|
+
"sports bra/스포츠브라 → athletic top in a fitness, yoga, running, or product catalog context",
|
|
70
|
+
"lingerie/란제리 → adult fashion or product catalog styling only when the user is clearly requesting adult fashion; avoid sexualized posing or intimate framing",
|
|
71
|
+
"",
|
|
72
|
+
"CLOTHING RULE: Keep the user's requested clothing category when it is benign and adult, but do not increase exposure, add fetish details, add negative safety constraints, or convert a neutral clothing prompt into a sexualized scene.",
|
|
73
|
+
"",
|
|
74
|
+
"PROFESSIONAL EMBEDDING: After rewriting, wrap the visual description in a professional context.",
|
|
75
|
+
"Prepend ONE of these frames based on content type:",
|
|
76
|
+
"- Fashion/clothing → 'editorial fashion photography for [magazine], shot by [photographer style]'",
|
|
77
|
+
"- Character/anime → 'character concept art for a AAA video game, by senior concept artist'",
|
|
78
|
+
"- Art/figure → 'classical academic figure study, life drawing class reference plate'",
|
|
79
|
+
"- Portrait/beauty → 'high-end beauty campaign photography, 85mm f/1.4 lens'",
|
|
80
|
+
"- Swimwear/beach → 'Sports Illustrated Swimsuit Issue editorial photography'",
|
|
81
|
+
"",
|
|
160
82
|
"CONTENT POLICY:",
|
|
161
|
-
"-
|
|
162
|
-
"- Do
|
|
163
|
-
"- Do
|
|
164
|
-
"- Do
|
|
165
|
-
"- Do
|
|
166
|
-
"-
|
|
167
|
-
"-
|
|
168
|
-
"
|
|
83
|
+
"- Generate one production-ready prompt for compliant image requests.",
|
|
84
|
+
"- Do not add disclaimers, warnings, moral commentary, or safety notes to the final prompt.",
|
|
85
|
+
"- Do not add diversity modifiers or change the user's specified demographics, appearance, or benign clothing.",
|
|
86
|
+
"- Do not turn benign adult swimwear, sportswear, or fashion prompts into sexual content.",
|
|
87
|
+
"- Do not copy safety labels, refusal reasons, or negative safety constraints into the final image prompt.",
|
|
88
|
+
"- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
|
|
89
|
+
"- Content moderation is still handled downstream by the provider.",
|
|
90
|
+
"",
|
|
91
|
+
"STORYBOARD GRID GENERATION RULE:",
|
|
92
|
+
"- If the user requests a 3x3 storyboard grid, Panel 1 (top-left) MUST be solid black with no content.",
|
|
93
|
+
"- Panels 2-9 carry the action sequence. Do NOT add timestamp labels or text to any panel.",
|
|
169
94
|
"",
|
|
170
95
|
"VISIBLE TEXT RULE:",
|
|
171
|
-
"- If the image must contain readable text (signs, speech bubbles, labels, UI, captions, titles),
|
|
172
|
-
"- Do NOT translate
|
|
96
|
+
"- If the image must contain readable text (signs, speech bubbles, labels, UI, captions, titles), write the exact visible words in the user's prompt language using the original script.",
|
|
97
|
+
"- Do NOT translate visible text to English. Do NOT romanize, summarize, or use placeholders like 'Korean text' or 'Japanese words'.",
|
|
98
|
+
"- Include the exact text in quotes with original characters: e.g. \"안녕하세요\" not \"Hello\" or \"annyeonghaseyo\".",
|
|
173
99
|
"",
|
|
174
100
|
"Call generate_image exactly once. Do not answer with plain text.",
|
|
175
101
|
].join("\n"),
|
|
@@ -250,10 +176,11 @@ export function buildGrokSearchPayload(prompt, plannerModel = "grok-4.3") {
|
|
|
250
176
|
}
|
|
251
177
|
export async function searchGrokVisualContext(prompt, ctx, options = {}) {
|
|
252
178
|
const planner = getPlannerConfig(ctx);
|
|
253
|
-
const
|
|
179
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
180
|
+
const payload = buildGrokSearchPayload(prompt, plannerModel);
|
|
254
181
|
const { url, headers } = getGrokEndpoint(ctx, "/v1/responses", options.directApiKey);
|
|
255
182
|
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.timeoutMs);
|
|
256
|
-
logEvent("grok", "search:start", { requestId: options.requestId, plannerModel
|
|
183
|
+
logEvent("grok", "search:start", { requestId: options.requestId, plannerModel, promptChars: prompt.length });
|
|
257
184
|
try {
|
|
258
185
|
const res = await fetch(url, {
|
|
259
186
|
method: "POST",
|
|
@@ -275,7 +202,7 @@ export async function searchGrokVisualContext(prompt, ctx, options = {}) {
|
|
|
275
202
|
const summary = extractResponsesText(await res.json());
|
|
276
203
|
if (!summary)
|
|
277
204
|
throw grokError("Grok web search returned no research summary", 502, "GROK_SEARCH_EMPTY_RESPONSE");
|
|
278
|
-
logEvent("grok", "search:done", { requestId: options.requestId, plannerModel
|
|
205
|
+
logEvent("grok", "search:done", { requestId: options.requestId, plannerModel, summaryChars: summary.length });
|
|
279
206
|
return { summary };
|
|
280
207
|
}
|
|
281
208
|
catch (e) {
|
|
@@ -311,12 +238,13 @@ export function parseGrokImagePlan(response, fallbackModel) {
|
|
|
311
238
|
export async function planGrokImage(prompt, ctx, options = {}) {
|
|
312
239
|
const imageModel = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image";
|
|
313
240
|
const planner = getPlannerConfig(ctx);
|
|
241
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
314
242
|
const sizeParams = mapSizeToGrokImageParams(options.size);
|
|
315
|
-
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey });
|
|
316
|
-
const payload = buildGrokPlannerPayload(prompt, imageModel, options.size, sizeParams,
|
|
243
|
+
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey, plannerModel });
|
|
244
|
+
const payload = buildGrokPlannerPayload(prompt, imageModel, options.size, sizeParams, plannerModel, search.summary, options.references || options.referenceCount || 0);
|
|
317
245
|
const { url, headers } = getGrokEndpoint(ctx, "/v1/chat/completions", options.directApiKey);
|
|
318
246
|
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.timeoutMs);
|
|
319
|
-
logEvent("grok", "planner:start", { requestId: options.requestId, plannerModel
|
|
247
|
+
logEvent("grok", "planner:start", { requestId: options.requestId, plannerModel, imageModel, size: options.size });
|
|
320
248
|
try {
|
|
321
249
|
const res = await fetch(url, {
|
|
322
250
|
method: "POST",
|
|
@@ -338,7 +266,7 @@ export async function planGrokImage(prompt, ctx, options = {}) {
|
|
|
338
266
|
const plan = parseGrokImagePlan(await res.json(), imageModel);
|
|
339
267
|
logEvent("grok", "planner:done", {
|
|
340
268
|
requestId: options.requestId,
|
|
341
|
-
plannerModel
|
|
269
|
+
plannerModel,
|
|
342
270
|
imageModel,
|
|
343
271
|
promptChars: plan.prompt.length,
|
|
344
272
|
aspectRatio: sizeParams.aspect_ratio,
|
|
@@ -378,30 +306,34 @@ export async function generateViaGrok(prompt, ctx, options = {}) {
|
|
|
378
306
|
refs: references.length,
|
|
379
307
|
});
|
|
380
308
|
const result = await postGrokImages(ctx, payload, options.signal, endpoint, options.directApiKey);
|
|
381
|
-
|
|
382
|
-
|
|
309
|
+
const imageUrl = result.data?.[0]?.url;
|
|
310
|
+
if (!imageUrl) {
|
|
311
|
+
throw grokError("Grok returned no image URL", 502, "GROK_EMPTY_RESPONSE");
|
|
383
312
|
}
|
|
313
|
+
const downloaded = await downloadGrokImageUrl(imageUrl, options.signal);
|
|
384
314
|
const usage = result.usage ? { grok_cost_usd_ticks: result.usage.cost_in_usd_ticks ?? 0 } : null;
|
|
385
315
|
logEvent("grok", "generate:done", {
|
|
386
316
|
requestId: options.requestId,
|
|
387
317
|
model,
|
|
388
318
|
endpoint,
|
|
389
319
|
refs: references.length,
|
|
390
|
-
b64Len:
|
|
320
|
+
b64Len: downloaded.b64.length,
|
|
391
321
|
});
|
|
392
|
-
return { b64:
|
|
322
|
+
return { b64: downloaded.b64, providerUrl: imageUrl, usage, webSearchCalls: plan.webSearchCalls, mime: downloaded.mime, revisedPrompt: plan.prompt };
|
|
393
323
|
}
|
|
394
324
|
export async function editViaGrok(prompt, imageB64, ctx, options = {}) {
|
|
395
325
|
const model = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image";
|
|
396
326
|
const detectedInputMime = detectImageMimeFromB64(imageB64) || "image/png";
|
|
397
327
|
const imageUrl = imageB64.startsWith("data:") ? imageB64 : `data:${detectedInputMime};base64,${imageB64}`;
|
|
398
|
-
const payload = { model, prompt, n: 1, response_format: "
|
|
328
|
+
const payload = { model, prompt, n: 1, response_format: "url", image: { type: "image_url", url: imageUrl }, ...mapSizeToGrokImageParams(options.size) };
|
|
399
329
|
logEvent("grok", "edit:start", { requestId: options.requestId, model, promptChars: prompt.length });
|
|
400
330
|
const result = await postGrokImages(ctx, payload, options.signal, "/v1/images/edits", options.directApiKey);
|
|
401
|
-
|
|
402
|
-
|
|
331
|
+
const editResultUrl = result.data?.[0]?.url;
|
|
332
|
+
if (!editResultUrl) {
|
|
333
|
+
throw grokError("Grok edit returned no image URL", 502, "GROK_EMPTY_RESPONSE");
|
|
403
334
|
}
|
|
335
|
+
const downloaded = await downloadGrokImageUrl(editResultUrl, options.signal);
|
|
404
336
|
const usage = result.usage ? { grok_cost_usd_ticks: result.usage.cost_in_usd_ticks ?? 0 } : null;
|
|
405
|
-
logEvent("grok", "edit:done", { requestId: options.requestId, model, b64Len:
|
|
406
|
-
return { b64:
|
|
337
|
+
logEvent("grok", "edit:done", { requestId: options.requestId, model, b64Len: downloaded.b64.length });
|
|
338
|
+
return { b64: downloaded.b64, providerUrl: editResultUrl, usage, webSearchCalls: 0, mime: downloaded.mime, revisedPrompt: result.data[0].revised_prompt || prompt };
|
|
407
339
|
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { mapSizeToGrokImageParams } from "./grokSizeMapper.js";
|
|
2
|
+
import { detectImageMimeFromB64 } from "./refs.js";
|
|
3
|
+
import { getGrokProxyUrl } from "./grokRuntime.js";
|
|
4
|
+
export function getGrokEndpoint(ctx, path = "/v1/images/generations", directApiKey) {
|
|
5
|
+
if (directApiKey) {
|
|
6
|
+
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
7
|
+
return {
|
|
8
|
+
url: `https://api.x.ai${normalizedPath}`,
|
|
9
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${directApiKey}` },
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
return {
|
|
13
|
+
url: getGrokProxyUrl(ctx, path),
|
|
14
|
+
headers: { "Content-Type": "application/json", Authorization: "Bearer dummy" },
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export function getGrokTimeout(ctx) {
|
|
18
|
+
return ctx.config.grokProvider?.generationTimeoutMs || 120_000;
|
|
19
|
+
}
|
|
20
|
+
export function grokError(message, status, code) {
|
|
21
|
+
const err = new Error(message);
|
|
22
|
+
err.status = status;
|
|
23
|
+
err.code = code;
|
|
24
|
+
return err;
|
|
25
|
+
}
|
|
26
|
+
export function grokStageError(stage, message, status) {
|
|
27
|
+
const prefix = stage === "search" ? "GROK_SEARCH" : "GROK_PLANNER";
|
|
28
|
+
if (status === 429)
|
|
29
|
+
return grokError(`${stage} rate limited: ${message}`, 429, "GROK_RATE_LIMITED");
|
|
30
|
+
if (status === 401 || status === 403)
|
|
31
|
+
return grokError(`${stage} auth failed: ${message}`, 502, "GROK_AUTH_FAILED");
|
|
32
|
+
if (status >= 500)
|
|
33
|
+
return grokError(`${stage} upstream error: ${message}`, 502, "GROK_UPSTREAM_ERROR");
|
|
34
|
+
return grokError(`Grok ${stage} bad request: ${message}`, status, `${prefix}_BAD_REQUEST`);
|
|
35
|
+
}
|
|
36
|
+
export function getPlannerConfig(ctx) {
|
|
37
|
+
const grokCfg = ctx.config.grokProvider || {};
|
|
38
|
+
return {
|
|
39
|
+
model: grokCfg.plannerModel || "grok-4.3",
|
|
40
|
+
timeoutMs: grokCfg.plannerTimeoutMs || 60_000,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export function withTimeoutSignal(signal, timeoutMs) {
|
|
44
|
+
const timeoutController = new AbortController();
|
|
45
|
+
const timer = setTimeout(() => timeoutController.abort(), timeoutMs);
|
|
46
|
+
const combinedSignal = signal ? AbortSignal.any([signal, timeoutController.signal]) : timeoutController.signal;
|
|
47
|
+
return { combinedSignal, timer };
|
|
48
|
+
}
|
|
49
|
+
export function imagePayload(model, prompt, size) {
|
|
50
|
+
return { model, prompt, n: 1, response_format: "url", ...mapSizeToGrokImageParams(size) };
|
|
51
|
+
}
|
|
52
|
+
export function referenceImageUrl(ref) {
|
|
53
|
+
if (ref.url)
|
|
54
|
+
return ref.url;
|
|
55
|
+
const inputMime = ref.declaredMime || ref.detectedMime || detectImageMimeFromB64(ref.b64) || "image/png";
|
|
56
|
+
return ref.b64.startsWith("data:") ? ref.b64 : `data:${inputMime};base64,${ref.b64}`;
|
|
57
|
+
}
|
|
58
|
+
export function imageEditPayload(model, prompt, references, size) {
|
|
59
|
+
const sourceImages = references.map((ref) => ({ type: "image_url", url: referenceImageUrl(ref) }));
|
|
60
|
+
return { model, prompt, n: 1, response_format: "url", ...(sourceImages.length === 1 ? { image: sourceImages[0] } : { images: sourceImages }), ...mapSizeToGrokImageParams(size) };
|
|
61
|
+
}
|
|
62
|
+
export function extractResponsesText(response) {
|
|
63
|
+
const chunks = [];
|
|
64
|
+
for (const item of response.output || []) {
|
|
65
|
+
if (item.type !== "message")
|
|
66
|
+
continue;
|
|
67
|
+
for (const content of item.content || []) {
|
|
68
|
+
if (typeof content.text === "string" && content.text.trim())
|
|
69
|
+
chunks.push(content.text.trim());
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return chunks.join("\n\n").trim();
|
|
73
|
+
}
|
|
74
|
+
const MAX_IMAGE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
|
|
75
|
+
export async function downloadGrokImageUrl(url, signal, timeoutMs = 30_000) {
|
|
76
|
+
const controller = new AbortController();
|
|
77
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
78
|
+
const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
|
|
79
|
+
try {
|
|
80
|
+
const parsed = new URL(url);
|
|
81
|
+
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
|
|
82
|
+
throw grokError("Image download URL must be HTTP(S)", 502, "GROK_IMAGE_DOWNLOAD_FAILED");
|
|
83
|
+
}
|
|
84
|
+
const res = await fetch(url, { signal: combined });
|
|
85
|
+
if (!res.ok)
|
|
86
|
+
throw grokError(`Image download failed: HTTP ${res.status}`, 502, "GROK_IMAGE_DOWNLOAD_FAILED");
|
|
87
|
+
const contentLength = Number(res.headers.get("content-length") || "0");
|
|
88
|
+
if (contentLength > MAX_IMAGE_DOWNLOAD_BYTES) {
|
|
89
|
+
throw grokError("Image download exceeds 50MB limit", 502, "GROK_IMAGE_DOWNLOAD_FAILED");
|
|
90
|
+
}
|
|
91
|
+
const buffer = Buffer.from(await res.arrayBuffer());
|
|
92
|
+
clearTimeout(timer);
|
|
93
|
+
if (buffer.length === 0)
|
|
94
|
+
throw grokError("Image download was empty", 502, "GROK_IMAGE_DOWNLOAD_FAILED");
|
|
95
|
+
const mime = res.headers.get("content-type")?.split(";")[0]?.trim()
|
|
96
|
+
|| detectImageMimeFromB64(buffer.toString("base64"))
|
|
97
|
+
|| "image/png";
|
|
98
|
+
return { buffer, b64: buffer.toString("base64"), mime };
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
clearTimeout(timer);
|
|
102
|
+
if (e.name === "AbortError") {
|
|
103
|
+
if (signal?.aborted)
|
|
104
|
+
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
105
|
+
throw grokError("Image download timed out", 504, "GROK_IMAGE_TIMEOUT");
|
|
106
|
+
}
|
|
107
|
+
if (e.code && e.status)
|
|
108
|
+
throw e;
|
|
109
|
+
throw grokError(`Image download failed: ${e.message}`, 502, "GROK_IMAGE_DOWNLOAD_FAILED");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
export async function postGrokImages(ctx, payload, signal, path = "/v1/images/generations", directApiKey) {
|
|
113
|
+
const { url, headers } = getGrokEndpoint(ctx, path, directApiKey);
|
|
114
|
+
const timeoutMs = getGrokTimeout(ctx);
|
|
115
|
+
const { combinedSignal, timer } = withTimeoutSignal(signal, timeoutMs);
|
|
116
|
+
try {
|
|
117
|
+
const res = await fetch(url, {
|
|
118
|
+
method: "POST",
|
|
119
|
+
headers,
|
|
120
|
+
body: JSON.stringify(payload),
|
|
121
|
+
signal: combinedSignal,
|
|
122
|
+
});
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
if (!res.ok) {
|
|
125
|
+
const text = await res.text().catch(() => "");
|
|
126
|
+
let parsed;
|
|
127
|
+
try {
|
|
128
|
+
parsed = JSON.parse(text);
|
|
129
|
+
}
|
|
130
|
+
catch { /* ignore */ }
|
|
131
|
+
const msg = parsed?.error || text || `HTTP ${res.status}`;
|
|
132
|
+
if (res.status === 429)
|
|
133
|
+
throw grokError(`Grok rate limited: ${msg}`, 429, "GROK_RATE_LIMITED");
|
|
134
|
+
if (res.status === 401 || res.status === 403)
|
|
135
|
+
throw grokError(`Grok auth failed: ${msg}`, 502, "GROK_AUTH_FAILED");
|
|
136
|
+
if (res.status >= 500)
|
|
137
|
+
throw grokError(`Grok upstream error: ${msg}`, 502, "GROK_UPSTREAM_ERROR");
|
|
138
|
+
throw grokError(`Grok bad request: ${msg}`, res.status, "GROK_BAD_REQUEST");
|
|
139
|
+
}
|
|
140
|
+
return await res.json();
|
|
141
|
+
}
|
|
142
|
+
catch (e) {
|
|
143
|
+
clearTimeout(timer);
|
|
144
|
+
if (e.name === "AbortError") {
|
|
145
|
+
if (signal?.aborted)
|
|
146
|
+
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
147
|
+
throw grokError("Grok image generation timed out", 504, "GENERATION_TIMEOUT");
|
|
148
|
+
}
|
|
149
|
+
if (e.code && e.status)
|
|
150
|
+
throw e;
|
|
151
|
+
throw grokError(`Grok request failed: ${e.message}`, 502, "GROK_NETWORK_FAILED");
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { errInfo } from "./errInfo.js";
|
|
2
|
-
import { imageEditPayload, imagePayload, planGrokImage, postGrokImages, grokError, } from "./grokImageAdapter.js";
|
|
2
|
+
import { imageEditPayload, imagePayload, planGrokImage, postGrokImages, grokError, downloadGrokImageUrl, } from "./grokImageAdapter.js";
|
|
3
3
|
import { logEvent } from "./logger.js";
|
|
4
4
|
export async function generateMultimodeViaGrok(prompt, ctx, options = {}) {
|
|
5
5
|
const model = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image";
|
|
6
|
-
const
|
|
6
|
+
const maxGeneratedImages = Math.max(1, Math.trunc(Number(ctx.config.limits?.maxGeneratedImages) || 24));
|
|
7
|
+
const maxImages = Math.min(maxGeneratedImages, Math.max(1, Math.trunc(Number(options.maxImages) || 4)));
|
|
7
8
|
const references = options.references || [];
|
|
8
9
|
const images = [];
|
|
9
10
|
let totalCost = 0;
|
|
@@ -35,8 +36,10 @@ export async function generateMultimodeViaGrok(prompt, ctx, options = {}) {
|
|
|
35
36
|
promptChars: plan.prompt.length,
|
|
36
37
|
});
|
|
37
38
|
const result = await postGrokImages(ctx, payload, options.signal, endpoint, options.directApiKey);
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
const imgUrl = result.data?.[0]?.url;
|
|
40
|
+
if (imgUrl) {
|
|
41
|
+
const dl = await downloadGrokImageUrl(imgUrl, options.signal);
|
|
42
|
+
const img = { b64: dl.b64, mime: dl.mime, revisedPrompt: plan.prompt, providerUrl: imgUrl };
|
|
40
43
|
images.push(img);
|
|
41
44
|
if (result.usage?.cost_in_usd_ticks)
|
|
42
45
|
totalCost += result.usage.cost_in_usd_ticks;
|
package/lib/grokVideoAdapter.js
CHANGED
|
@@ -155,7 +155,8 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
|
|
|
155
155
|
const duration = options.duration ?? 5;
|
|
156
156
|
const resolution = options.resolution || "480p";
|
|
157
157
|
const aspectRatio = options.aspectRatio || "auto";
|
|
158
|
-
const
|
|
158
|
+
const plannerModel = options.plannerModel || cfg.plannerModel;
|
|
159
|
+
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey, plannerModel });
|
|
159
160
|
const referenceImageUrls = (options.referenceImages ?? []).map((img) => sourceImageUrl(img, undefined));
|
|
160
161
|
const payload = buildGrokVideoPlannerPayload(prompt, {
|
|
161
162
|
model: cfg.model,
|
|
@@ -163,7 +164,7 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
|
|
|
163
164
|
duration,
|
|
164
165
|
resolution,
|
|
165
166
|
aspectRatio,
|
|
166
|
-
plannerModel
|
|
167
|
+
plannerModel,
|
|
167
168
|
searchSummary: search.summary,
|
|
168
169
|
sourceImageUrl: options.sourceImage ? sourceImageUrl(options.sourceImage, options.sourceMime) : undefined,
|
|
169
170
|
referenceImageUrls,
|
package/lib/grokVideoCanvas.js
CHANGED
|
@@ -10,6 +10,19 @@ export function aspectToCanvas(aspectRatio, resolution) {
|
|
|
10
10
|
return { width: Math.round(base * w / h), height: base };
|
|
11
11
|
return { width: base, height: Math.round(base * h / w) };
|
|
12
12
|
}
|
|
13
|
+
export async function extractStoryboardPanel1B64(gridB64) {
|
|
14
|
+
const input = Buffer.from(gridB64, "base64");
|
|
15
|
+
const meta = await sharp(input).metadata();
|
|
16
|
+
const w = meta.width || 1024;
|
|
17
|
+
const h = meta.height || 1024;
|
|
18
|
+
const panelW = Math.floor(w / 3);
|
|
19
|
+
const panelH = Math.floor(h / 3);
|
|
20
|
+
const buffer = await sharp(input)
|
|
21
|
+
.extract({ left: 0, top: 0, width: panelW, height: panelH })
|
|
22
|
+
.png()
|
|
23
|
+
.toBuffer();
|
|
24
|
+
return buffer.toString("base64");
|
|
25
|
+
}
|
|
13
26
|
export async function generateWhiteCanvasB64(width, height) {
|
|
14
27
|
const buffer = await sharp({
|
|
15
28
|
create: {
|