ima2-gen 3.13.1 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/CHANGELOG.md +14 -0
- package/README.md +15 -2
- package/bin/commands/capabilities.js +5 -3
- package/bin/commands/config.js +15 -4
- package/bin/commands/defaults.js +7 -5
- package/bin/commands/doctor.js +88 -39
- package/bin/commands/gen.js +21 -4
- package/bin/commands/models.js +2 -2
- package/bin/commands/ping.js +1 -1
- package/bin/commands/prompt.js +5 -3
- package/bin/commands/service.js +32 -4
- package/bin/commands/tools.js +19 -3
- package/bin/commands/upscale.js +18 -4
- package/bin/commands/video.js +37 -50
- package/bin/ima2.js +28 -9
- package/bin/lib/characterResolve.js +2 -1
- package/bin/lib/client.js +155 -20
- package/bin/lib/config-store.js +40 -7
- package/bin/lib/doctor-bundle.js +6 -2
- package/bin/lib/doctor-checks.js +13 -29
- package/bin/lib/doctor-media.js +3 -3
- package/bin/lib/doctor-providers.js +48 -35
- package/bin/lib/doctor-report.js +78 -0
- package/bin/lib/doctor-runtime.js +173 -0
- package/bin/lib/mcpJob.js +19 -7
- package/bin/lib/output.js +3 -0
- package/bin/lib/platform.js +12 -9
- package/bin/lib/recover-output.js +9 -2
- package/bin/lib/sse.js +9 -1
- package/bin/lib/videoMcp.js +16 -8
- package/config.js +41 -5
- package/docs/API.md +154 -16
- package/docs/CLI.md +65 -1
- package/docs/DOCKER.md +19 -4
- package/docs/DOCKER.zh-CN.md +11 -4
- package/docs/DOCKER.zh-TW.md +11 -4
- package/docs/FAQ.md +13 -0
- package/docs/NPX_QUICKSTART.md +6 -0
- package/docs/README.ja.md +15 -2
- package/docs/README.ko.md +15 -2
- package/docs/README.zh-CN.md +15 -2
- package/docs/README.zh-TW.md +15 -2
- package/docs/migration/runtime-test-inventory.md +79 -8
- package/lib/agyArtifact.js +82 -0
- package/lib/agyArtifactRead.js +249 -0
- package/lib/agyImageAdapter.js +2 -329
- package/lib/agyProcess.js +122 -0
- package/lib/apiRequestBudget.js +48 -0
- package/lib/assetLifecycle.js +42 -9
- package/lib/canvasVersionStore.js +46 -14
- package/lib/capabilities.js +7 -1
- package/lib/configKeys.js +2 -0
- package/lib/errors/providerMap.js +26 -0
- package/lib/eventBus.js +10 -6
- package/lib/eventsPolicy.js +9 -0
- package/lib/geminiApiImageAdapter.js +1 -214
- package/lib/generatePipeline.js +39 -165
- package/lib/generatedMediaAccess.js +74 -0
- package/lib/generationErrors.js +3 -22
- package/lib/grokImageAdapter.js +2 -353
- package/lib/grokImageCore.js +1 -57
- package/lib/grokImageDownload.js +159 -0
- package/lib/grokImageDownloadPolicy.js +124 -0
- package/lib/grokImagePlanner.js +301 -0
- package/lib/grokMultimodeAdapter.js +1 -75
- package/lib/grokVideoDownload.js +137 -34
- package/lib/inflight.js +112 -112
- package/lib/jobs/terminalStore.js +45 -0
- package/lib/lanSessionStore.js +131 -0
- package/lib/localAccessPolicy.js +180 -0
- package/lib/localLanAccess.js +281 -0
- package/lib/logger.js +22 -13
- package/lib/mcp/downloadMediaResult.js +72 -78
- package/lib/multimodePipeline.js +39 -126
- package/lib/nodeGeneration.js +44 -101
- package/lib/nodeStore.js +25 -11
- package/lib/nodeValidation.js +5 -1
- package/lib/pinnedHttpGet.js +174 -0
- package/lib/promptImport/githubFolder.js +32 -18
- package/lib/promptImport/githubSource.js +14 -10
- package/lib/promptImport/parsePromptCandidates.js +32 -2
- package/lib/providers/adapters/agyOperations.js +179 -0
- package/lib/providers/adapters/comfy.js +1 -0
- package/lib/providers/adapters/geminiOperations.js +214 -0
- package/lib/providers/adapters/googleExecution.js +97 -0
- package/lib/providers/adapters/grokExecution.js +137 -0
- package/lib/providers/adapters/grokMultimodeOperations.js +83 -0
- package/lib/providers/adapters/grokOperations.js +63 -0
- package/lib/providers/adapters/openaiExecution.js +118 -0
- package/lib/providers/adapters/openaiOperations.js +201 -0
- package/lib/providers/adapters/openaiTypes.js +1 -0
- package/lib/providers/derive.js +4 -0
- package/lib/providers/deriveCore.js +2 -7
- package/lib/providers/execution/admission.js +25 -0
- package/lib/providers/execution/index.js +34 -0
- package/lib/providers/execution/legacy.js +17 -0
- package/lib/providers/execution/legacyClassic.js +66 -0
- package/lib/providers/execution/legacyEdit.js +30 -0
- package/lib/providers/execution/legacyMultimode.js +34 -0
- package/lib/providers/execution/legacyNode.js +41 -0
- package/lib/providers/execution/types.js +1 -0
- package/lib/providers/registry.js +19 -14
- package/lib/providers/surfaceSupport.js +22 -0
- package/lib/responsesImageAdapter.js +1 -389
- package/lib/responsesTransport.js +190 -0
- package/lib/spriteJobEvents.js +20 -2
- package/lib/ssePublish.js +5 -4
- package/lib/videoExtendI2vOperation.js +76 -0
- package/package.json +5 -2
- package/routes/edit.js +31 -123
- package/routes/events.js +140 -63
- package/routes/mcpMedia.js +10 -2
- package/routes/models.js +28 -25
- package/routes/videoExtended.js +9 -68
- package/server.js +24 -46
- package/ui/dist/.vite/manifest.json +90 -63
- package/ui/dist/assets/{AgentWorkspace-PbBytTeV.js → AgentWorkspace-CL3MLv3l.js} +2 -2
- package/ui/dist/assets/App-B9cEM6dJ.js +9 -0
- package/ui/dist/assets/AssetGenWorkspace-swWPvXrx.js +2 -0
- package/ui/dist/assets/AssetsWorkspace-CRVeYU2F.js +1 -0
- package/ui/dist/assets/{CardNewsWorkspace-BS3jKOja.js → CardNewsWorkspace-BAOxWLKf.js} +2 -2
- package/ui/dist/assets/GenerationRequestLogPanel-DZtYZm-D.js +1 -0
- package/ui/dist/assets/HomeWorkspace-BeZ1s0Jm.js +1 -0
- package/ui/dist/assets/KeyingPanel-ZyBSXioh.js +1 -0
- package/ui/dist/assets/NodeCanvas-BvvcCErf.js +7 -0
- package/ui/dist/assets/{PromptBuilderPanel-D97A4pwB.js → PromptBuilderPanel-DBycmShA.js} +2 -2
- package/ui/dist/assets/PromptImportDialog-ChvMv7ek.js +2 -0
- package/ui/dist/assets/PromptImportDiscoverySection-60py7whH.js +1 -0
- package/ui/dist/assets/PromptImportFolderSection-CCRh_-_7.js +1 -0
- package/ui/dist/assets/PromptLibraryPanel-BFWQO8OT.js +2 -0
- package/ui/dist/assets/SettingsWorkspace-B5Ghsgdj.js +1 -0
- package/ui/dist/assets/SpriteRecipeWorkspace-DRWdkmwU.js +1 -0
- package/ui/dist/assets/index-7ngEOVxt.js +5 -0
- package/ui/dist/assets/index-B5jy7ePg.css +1 -0
- package/ui/dist/assets/index-gxh_02DQ.js +22 -0
- package/ui/dist/assets/{pptxgen.es-D4pedbwM.js → pptxgen.es-OUNhQLEV.js} +1 -1
- package/ui/dist/assets/promptBuilderStore-BTUUzbBT.js +1 -0
- package/ui/dist/assets/useAgentDialogFocus-DSc2qY8p.js +1 -0
- package/ui/dist/index.html +3 -2
- package/ui/dist/assets/AssetGenWorkspace-B-TGlvVb.js +0 -2
- package/ui/dist/assets/AssetsWorkspace-LnGaW151.js +0 -1
- package/ui/dist/assets/GenerationRequestLogPanel-DuDWD_ic.js +0 -1
- package/ui/dist/assets/HomeWorkspace-DVP7UG6O.js +0 -1
- package/ui/dist/assets/KeyingPanel-5bRUFQaq.js +0 -1
- package/ui/dist/assets/NodeCanvas-BBEpERTb.js +0 -7
- package/ui/dist/assets/PromptImportDialog-C89MN5JY.js +0 -2
- package/ui/dist/assets/PromptImportDiscoverySection-BgRfDOs2.js +0 -1
- package/ui/dist/assets/PromptImportFolderSection-zTMyriT1.js +0 -1
- package/ui/dist/assets/PromptLibraryPanel-BD96qZu_.js +0 -2
- package/ui/dist/assets/SettingsWorkspace-BAOk3X_8.js +0 -1
- package/ui/dist/assets/SpriteRecipeWorkspace-ClIM5QRo.js +0 -1
- package/ui/dist/assets/index-0fpfY8vu.css +0 -1
- package/ui/dist/assets/index-BFciPGTn.js +0 -5
- package/ui/dist/assets/index-BbykeXR9.js +0 -30
- package/ui/dist/assets/promptBuilderStore-Do7M8g7J.js +0 -1
- package/ui/dist/assets/useAgentDialogFocus-B6-QgEVI.js +0 -1
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { lookup } from "node:dns/promises";
|
|
2
|
+
import { BlockList, isIP } from "node:net";
|
|
3
|
+
// Dated conservative WP05 policy (050/051), not an exhaustive IANA mirror.
|
|
4
|
+
const deniedIPv4 = new BlockList();
|
|
5
|
+
for (const [network, prefix] of [
|
|
6
|
+
["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10],
|
|
7
|
+
["127.0.0.0", 8], ["169.254.0.0", 16], ["172.16.0.0", 12],
|
|
8
|
+
["192.0.0.0", 24], ["192.0.2.0", 24], ["192.88.99.0", 24],
|
|
9
|
+
["192.168.0.0", 16], ["198.18.0.0", 15], ["198.51.100.0", 24],
|
|
10
|
+
["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4],
|
|
11
|
+
])
|
|
12
|
+
deniedIPv4.addSubnet(network, prefix, "ipv4");
|
|
13
|
+
const mappedIPv6 = new BlockList();
|
|
14
|
+
mappedIPv6.addSubnet("::ffff:0:0", 96, "ipv6");
|
|
15
|
+
const globalIPv6 = new BlockList();
|
|
16
|
+
globalIPv6.addSubnet("2000::", 3, "ipv6");
|
|
17
|
+
const deniedIPv6 = new BlockList();
|
|
18
|
+
for (const [network, prefix] of [
|
|
19
|
+
["2001::", 23], ["2001:db8::", 32], ["2002::", 16], ["3fff::", 20],
|
|
20
|
+
])
|
|
21
|
+
deniedIPv6.addSubnet(network, prefix, "ipv6");
|
|
22
|
+
function policyError() {
|
|
23
|
+
return Object.assign(new Error("Image download destination is unavailable or disallowed"), {
|
|
24
|
+
status: 502, code: "GROK_IMAGE_DOWNLOAD_FAILED",
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function isAllowedAddress(address, family) {
|
|
28
|
+
if (family === 4)
|
|
29
|
+
return !deniedIPv4.check(address, "ipv4");
|
|
30
|
+
// BlockList applies IPv4 CIDRs to mapped IPv6 numerically, in dotted or hex form.
|
|
31
|
+
// Do this BEFORE the native IPv6 global-prefix check; public mapped IPv4 is valid.
|
|
32
|
+
if (mappedIPv6.check(address, "ipv6"))
|
|
33
|
+
return !deniedIPv4.check(address, "ipv6");
|
|
34
|
+
return globalIPv6.check(address, "ipv6") && !deniedIPv6.check(address, "ipv6");
|
|
35
|
+
}
|
|
36
|
+
function isTrustedOrigin(url, policy) {
|
|
37
|
+
if (!policy.trustedProxyOrigin)
|
|
38
|
+
return false;
|
|
39
|
+
const trusted = new URL(policy.trustedProxyOrigin);
|
|
40
|
+
return url.origin === trusted.origin;
|
|
41
|
+
}
|
|
42
|
+
function validateAddresses(answers, trusted) {
|
|
43
|
+
if (answers.length === 0)
|
|
44
|
+
throw policyError();
|
|
45
|
+
const addresses = [];
|
|
46
|
+
for (const { address, family } of answers) {
|
|
47
|
+
if ((family !== 4 && family !== 6) || address.includes("%") || isIP(address) !== family) {
|
|
48
|
+
throw policyError();
|
|
49
|
+
}
|
|
50
|
+
if (!trusted && !isAllowedAddress(address, family))
|
|
51
|
+
throw policyError();
|
|
52
|
+
addresses.push({ address, family });
|
|
53
|
+
}
|
|
54
|
+
return addresses;
|
|
55
|
+
}
|
|
56
|
+
async function lookupWithSignal(hostname, signal, order) {
|
|
57
|
+
signal.throwIfAborted();
|
|
58
|
+
let onAbort = () => { };
|
|
59
|
+
try {
|
|
60
|
+
return await new Promise((resolve, reject) => {
|
|
61
|
+
onAbort = () => reject(signal.reason);
|
|
62
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
63
|
+
if (signal.aborted) {
|
|
64
|
+
onAbort();
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
Promise.resolve().then(() => {
|
|
68
|
+
signal.throwIfAborted();
|
|
69
|
+
return lookup(hostname, { all: true, ...(order ? { order } : {}) });
|
|
70
|
+
}).then(resolve, reject); // Consume late fulfillment AND rejection after abort.
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
signal.removeEventListener("abort", onAbort);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
export async function resolveImageDownloadTarget(url, policy, signal) {
|
|
81
|
+
try {
|
|
82
|
+
return await resolveDownloadTarget(url, policy, signal);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
// The wrapper owns public 499/504 mapping; preserve any caller abort reason.
|
|
86
|
+
signal.throwIfAborted();
|
|
87
|
+
throw policyError();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/** Same public-address policy, with no trusted-proxy argument. DNS errors remain retryable. */
|
|
91
|
+
export async function resolvePublicDownloadTarget(url, signal, order) {
|
|
92
|
+
try {
|
|
93
|
+
return await resolveDownloadTarget(url, {}, signal, order);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
signal.throwIfAborted();
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async function resolveDownloadTarget(url, policy, signal, order) {
|
|
101
|
+
try {
|
|
102
|
+
signal.throwIfAborted();
|
|
103
|
+
// Snapshot the destination so caller mutation cannot change it during DNS wait.
|
|
104
|
+
const targetUrl = new URL(url.href);
|
|
105
|
+
const hostname = targetUrl.hostname.replace(/^\[|\]$/g, "");
|
|
106
|
+
if (!hostname || hostname.includes("%") || targetUrl.username || targetUrl.password
|
|
107
|
+
|| (targetUrl.protocol !== "https:" && targetUrl.protocol !== "http:"))
|
|
108
|
+
throw policyError();
|
|
109
|
+
const trusted = isTrustedOrigin(targetUrl, policy);
|
|
110
|
+
if (!trusted && targetUrl.protocol !== "https:")
|
|
111
|
+
throw policyError();
|
|
112
|
+
const family = isIP(hostname);
|
|
113
|
+
const answers = family === 0
|
|
114
|
+
? await lookupWithSignal(hostname, signal, order) : [{ address: hostname, family }];
|
|
115
|
+
signal.throwIfAborted();
|
|
116
|
+
const addresses = validateAddresses(answers, trusted);
|
|
117
|
+
signal.throwIfAborted();
|
|
118
|
+
return { url: targetUrl, addresses };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
signal.throwIfAborted();
|
|
122
|
+
throw error;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import { logEvent } from "./logger.js";
|
|
2
|
+
import { SAFETY_INTENT_POLICY } from "./promptSafetyPolicy.js";
|
|
3
|
+
import { mapSizeToGrokImageParams } from "./grokSizeMapper.js";
|
|
4
|
+
import { DEFAULT_GROK_PLANNER_MODEL } from "../config.js";
|
|
5
|
+
import { grokError, grokStageError, getPlannerConfig, getGrokEndpoint, withTimeoutSignal, referenceImageUrl, extractResponsesText, } from "./grokImageCore.js";
|
|
6
|
+
import { grokFetchWithRetry } from "./grokUpstreamRetry.js";
|
|
7
|
+
export function buildGrokPlannerPayload(prompt, model, size, sizeParams, plannerModel = DEFAULT_GROK_PLANNER_MODEL, searchSummary = "", references = 0, backgroundConstraint = "") {
|
|
8
|
+
const referenceImages = Array.isArray(references) ? references : [];
|
|
9
|
+
const referenceCount = Array.isArray(references) ? references.length : references;
|
|
10
|
+
const sizeLine = size
|
|
11
|
+
? `Requested ima2 size: ${size}; xAI parameters: ${JSON.stringify(sizeParams)}.`
|
|
12
|
+
: `Requested ima2 size: auto; xAI parameters: ${JSON.stringify(sizeParams)}.`;
|
|
13
|
+
const referenceLine = referenceCount > 0
|
|
14
|
+
? `Reference images attached: ${referenceCount}. The final image call will use xAI image editing with these source images; preserve relevant subject, composition, style, and product details from them.`
|
|
15
|
+
: "Reference images attached: 0. The final image call will use text-to-image generation.";
|
|
16
|
+
return {
|
|
17
|
+
model: plannerModel,
|
|
18
|
+
stream: false,
|
|
19
|
+
parallel_tool_calls: false,
|
|
20
|
+
messages: [
|
|
21
|
+
{
|
|
22
|
+
role: "system",
|
|
23
|
+
content: [
|
|
24
|
+
"You are ima2's image generation planner for xAI Grok Imagine (Aurora model), inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
|
|
25
|
+
SAFETY_INTENT_POLICY,
|
|
26
|
+
"",
|
|
27
|
+
"TASK: Rewrite the user's casual request into ONE optimal, production-ready image prompt.",
|
|
28
|
+
"",
|
|
29
|
+
"OUTPUT FORMAT: A single natural-language paragraph (NOT tags, NOT keyword lists, NOT weighted tokens like (word:1.2)).",
|
|
30
|
+
"Structure the paragraph in this MANDATORY order — each element flows into the next:",
|
|
31
|
+
"1. Core subject/scene — who or what, with specific physical details (face shape, hair, clothing, pose, expression)",
|
|
32
|
+
"2. Environment/setting — where, with concrete spatial details and materials (wet concrete, polished marble, weathered wood)",
|
|
33
|
+
"3. Lighting + mood/emotion — use specific terms (golden hour backlight, overcast diffused, hard rim light, soft diffuse, nostalgic, melancholic, electric)",
|
|
34
|
+
"4. Camera/lens — shot type + specific focal length + depth of field (close-up on 85mm f/1.4 with shallow DoF, wide establishing on 24mm with deep focus)",
|
|
35
|
+
"5. Style/technical references — art style, medium, or mood reference (cinematic realism, Studio Ghibli feel, oil painting texture)",
|
|
36
|
+
"6. Quality finish — resolution/detail boosters appropriate to style (8K, highly detailed, sharp focus, film grain, bokeh)",
|
|
37
|
+
"",
|
|
38
|
+
"RULES:",
|
|
39
|
+
"- Write like a director briefing a cinematographer, not like a search query.",
|
|
40
|
+
"- Be hyper-specific: name real locations, brands, materials, textures. Vague = generic output.",
|
|
41
|
+
"- The prompt MUST be in English. Exception: any visible text the user wants rendered in the image (signs, speech bubbles, titles) must be kept in the ORIGINAL language characters verbatim (e.g. '한국어', '日本語').",
|
|
42
|
+
"- If reference images are attached, inspect them and describe the visual details (subject appearance, style, color palette) that must be preserved in the output.",
|
|
43
|
+
"- Do NOT use Stable Diffusion tags, booru tags, or comma-separated keyword lists.",
|
|
44
|
+
"- Do NOT use weighting syntax like (word:1.2) or [word].",
|
|
45
|
+
"- Quality boosters like '8K', 'highly detailed', 'sharp focus', 'cinematic lighting' ARE effective for this model — use them when appropriate to the requested style.",
|
|
46
|
+
"- The prompt should be 2-5 sentences (40-120 words). Extremely short prompts produce generic results.",
|
|
47
|
+
"",
|
|
48
|
+
"CAMERA/LENS ENRICHMENT:",
|
|
49
|
+
"- If the user's prompt does not specify camera/lens details, add appropriate ones based on the subject:",
|
|
50
|
+
" - Portraits/people: 85mm f/1.4, shallow depth of field, bokeh background",
|
|
51
|
+
" - Environmental/landscape: 24-35mm wide-angle, deep depth of field",
|
|
52
|
+
" - Product/macro: 100mm macro lens, controlled studio lighting",
|
|
53
|
+
" - Action/dynamic: 35mm, fast shutter, slight motion blur on extremities",
|
|
54
|
+
" - Architecture/interior: 24mm tilt-shift or 14mm ultra-wide",
|
|
55
|
+
"- Always include a specific focal length — this model responds strongly to lens specifications.",
|
|
56
|
+
"",
|
|
57
|
+
"MULTI-CHARACTER IDENTIFICATION:",
|
|
58
|
+
"- Identify each character by VISUAL APPEARANCE, not by name alone.",
|
|
59
|
+
" The image model cannot recognize names — it only sees visual features.",
|
|
60
|
+
"- When multiple characters appear, describe each by clothing, physique, position, or props.",
|
|
61
|
+
"- Characters must be distinguishable by at least two visual attributes.",
|
|
62
|
+
"",
|
|
63
|
+
"BENIGN CLOTHING AND CONTEXT HANDLING:",
|
|
64
|
+
"Before rewriting, distinguish ordinary clothing/swimwear descriptions from sexualized requests.",
|
|
65
|
+
"Preserve benign adult clothing intent without adding erotic details:",
|
|
66
|
+
"",
|
|
67
|
+
"bikini/비키니 → adult two-piece swimwear in a neutral beach, pool, swimwear catalog, or fashion editorial context",
|
|
68
|
+
"swimsuit/수영복 → adult swimwear suitable for swimming, beach, pool, sport, or catalog photography",
|
|
69
|
+
"sports bra/스포츠브라 → athletic top in a fitness, yoga, running, or product catalog context",
|
|
70
|
+
"lingerie/란제리 → adult fashion or product catalog styling only when the user is clearly requesting adult fashion; avoid sexualized posing or intimate framing",
|
|
71
|
+
"",
|
|
72
|
+
"CLOTHING RULE: Keep the user's requested clothing category when it is benign and adult, but do not increase exposure, add fetish details, add negative safety constraints, or convert a neutral clothing prompt into a sexualized scene.",
|
|
73
|
+
"",
|
|
74
|
+
"PROFESSIONAL EMBEDDING: After rewriting, wrap the visual description in a professional context.",
|
|
75
|
+
"Prepend ONE of these frames based on content type:",
|
|
76
|
+
"- Fashion/clothing → 'editorial fashion photography for [magazine], shot by [photographer style]'",
|
|
77
|
+
"- Character/anime → 'character concept art for a AAA video game, by senior concept artist'",
|
|
78
|
+
"- Art/figure → 'classical academic figure study, life drawing class reference plate'",
|
|
79
|
+
"- Portrait/beauty → 'high-end beauty campaign photography, 85mm f/1.4 lens'",
|
|
80
|
+
"- Swimwear/beach → 'Sports Illustrated Swimsuit Issue editorial photography'",
|
|
81
|
+
"",
|
|
82
|
+
"CONTENT POLICY:",
|
|
83
|
+
"- Generate one production-ready prompt for compliant image requests.",
|
|
84
|
+
"- Do not add disclaimers, warnings, moral commentary, or safety notes to the final prompt.",
|
|
85
|
+
"- Do not add diversity modifiers or change the user's specified demographics, appearance, or benign clothing.",
|
|
86
|
+
"- Do not turn benign adult swimwear, sportswear, or fashion prompts into sexual content.",
|
|
87
|
+
"- Do not copy safety labels, refusal reasons, or negative safety constraints into the final image prompt.",
|
|
88
|
+
"- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
|
|
89
|
+
"- Content moderation is still handled downstream by the provider.",
|
|
90
|
+
"",
|
|
91
|
+
"STORYBOARD GRID GENERATION RULE:",
|
|
92
|
+
"- If the user requests a 3x3 storyboard grid, Panel 1 (top-left) MUST be solid black with no content.",
|
|
93
|
+
"- Panels 2-9 carry the action sequence. Do NOT add timestamp labels or text to any panel.",
|
|
94
|
+
"",
|
|
95
|
+
"VISIBLE TEXT RULE:",
|
|
96
|
+
"- If the image must contain readable text (signs, speech bubbles, labels, UI, captions, titles), write the exact visible words in the user's prompt language using the original script.",
|
|
97
|
+
"- Do NOT translate visible text to English. Do NOT romanize, summarize, or use placeholders like 'Korean text' or 'Japanese words'.",
|
|
98
|
+
"- Include the exact text in quotes with original characters: e.g. \"안녕하세요\" not \"Hello\" or \"annyeonghaseyo\".",
|
|
99
|
+
"",
|
|
100
|
+
"Call generate_image exactly once. Do not answer with plain text.",
|
|
101
|
+
].join("\n"),
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
role: "user",
|
|
105
|
+
content: [
|
|
106
|
+
{
|
|
107
|
+
type: "text",
|
|
108
|
+
text: [
|
|
109
|
+
`Selected image model: ${model}.`,
|
|
110
|
+
sizeLine,
|
|
111
|
+
referenceLine,
|
|
112
|
+
...(backgroundConstraint ? [backgroundConstraint] : []),
|
|
113
|
+
searchSummary ? `Mandatory web-search brief:\n${searchSummary}` : "Mandatory web-search brief: unavailable.",
|
|
114
|
+
"Create the best final prompt for the image generator.",
|
|
115
|
+
"Return the generate_image.prompt argument in English only, except for exact visible text that the user explicitly requested.",
|
|
116
|
+
"",
|
|
117
|
+
"User prompt:",
|
|
118
|
+
prompt,
|
|
119
|
+
].join("\n"),
|
|
120
|
+
},
|
|
121
|
+
...referenceImages.map((ref) => ({
|
|
122
|
+
type: "image_url",
|
|
123
|
+
image_url: { url: referenceImageUrl(ref), detail: "high" },
|
|
124
|
+
})),
|
|
125
|
+
],
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
tools: [
|
|
129
|
+
{
|
|
130
|
+
type: "function",
|
|
131
|
+
function: {
|
|
132
|
+
name: "generate_image",
|
|
133
|
+
description: "Generate a single image through xAI Images API.",
|
|
134
|
+
parameters: {
|
|
135
|
+
type: "object",
|
|
136
|
+
properties: {
|
|
137
|
+
prompt: {
|
|
138
|
+
type: "string",
|
|
139
|
+
description: "Final image-generation prompt to send to xAI Images API.",
|
|
140
|
+
},
|
|
141
|
+
model: {
|
|
142
|
+
type: "string",
|
|
143
|
+
enum: ["grok-imagine-image-2.0", "grok-imagine-image", "grok-imagine-image-quality"],
|
|
144
|
+
description: "The xAI image model. The server may override this with the user's selected model.",
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
required: ["prompt", "model"],
|
|
148
|
+
},
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
],
|
|
152
|
+
tool_choice: { type: "function", function: { name: "generate_image" } },
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
export function buildGrokSearchPayload(prompt, plannerModel = DEFAULT_GROK_PLANNER_MODEL) {
|
|
156
|
+
return {
|
|
157
|
+
model: plannerModel,
|
|
158
|
+
stream: false,
|
|
159
|
+
input: [
|
|
160
|
+
{
|
|
161
|
+
role: "system",
|
|
162
|
+
content: [
|
|
163
|
+
"You are ima2's visual research assistant.",
|
|
164
|
+
"You must use web_search before producing the brief.",
|
|
165
|
+
"Return a concise image-generation research brief: visual facts, current references, style cues, and text-rendering constraints.",
|
|
166
|
+
"Do not generate an image prompt yet.",
|
|
167
|
+
].join(" "),
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
role: "user",
|
|
171
|
+
content: prompt,
|
|
172
|
+
},
|
|
173
|
+
],
|
|
174
|
+
tools: [{ type: "web_search" }],
|
|
175
|
+
tool_choice: "required",
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
export async function searchGrokVisualContext(prompt, ctx, options = {}) {
|
|
179
|
+
const planner = getPlannerConfig(ctx);
|
|
180
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
181
|
+
const payload = buildGrokSearchPayload(prompt, plannerModel);
|
|
182
|
+
const { url, headers } = getGrokEndpoint(ctx, "/v1/responses", options.directApiKey);
|
|
183
|
+
// The search brief has its own, smaller budget: it runs BEFORE the planner call inside
|
|
184
|
+
// the same user request, so charging it the full planner budget lets one slow stage eat
|
|
185
|
+
// the other's time. See devlog/_plan/260817_grok_video_planner_timeout/010.
|
|
186
|
+
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.searchTimeoutMs);
|
|
187
|
+
logEvent("grok", "search:start", { requestId: options.requestId, plannerModel, promptChars: prompt.length });
|
|
188
|
+
try {
|
|
189
|
+
// Safe to replay: search produces no billable artifact, only a research summary.
|
|
190
|
+
const res = await grokFetchWithRetry(() => fetch(url, {
|
|
191
|
+
method: "POST",
|
|
192
|
+
headers,
|
|
193
|
+
body: JSON.stringify(payload),
|
|
194
|
+
signal: combinedSignal,
|
|
195
|
+
}), { signal: combinedSignal, label: "search" });
|
|
196
|
+
clearTimeout(timer);
|
|
197
|
+
if (!res.ok) {
|
|
198
|
+
const text = await res.text().catch(() => "");
|
|
199
|
+
let parsed;
|
|
200
|
+
try {
|
|
201
|
+
parsed = JSON.parse(text);
|
|
202
|
+
}
|
|
203
|
+
catch { /* ignore */ }
|
|
204
|
+
const msg = parsed?.error || text || `HTTP ${res.status}`;
|
|
205
|
+
throw grokStageError("search", msg, res.status);
|
|
206
|
+
}
|
|
207
|
+
const summary = extractResponsesText(await res.json());
|
|
208
|
+
if (!summary)
|
|
209
|
+
throw grokError("Grok web search returned no research summary", 502, "GROK_SEARCH_EMPTY_RESPONSE");
|
|
210
|
+
logEvent("grok", "search:done", { requestId: options.requestId, plannerModel, summaryChars: summary.length });
|
|
211
|
+
return { summary };
|
|
212
|
+
}
|
|
213
|
+
catch (e) {
|
|
214
|
+
clearTimeout(timer);
|
|
215
|
+
if (e.name === "AbortError") {
|
|
216
|
+
if (options.signal?.aborted)
|
|
217
|
+
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
218
|
+
throw grokError("Grok web search timed out", 504, "GROK_SEARCH_TIMEOUT");
|
|
219
|
+
}
|
|
220
|
+
if (e.code && e.status)
|
|
221
|
+
throw e;
|
|
222
|
+
throw grokError(`Grok web search request failed: ${e.message}`, 502, "GROK_SEARCH_NETWORK_FAILED");
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
export function parseGrokImagePlan(response, fallbackModel) {
|
|
226
|
+
const toolCalls = response.choices?.[0]?.message?.tool_calls || [];
|
|
227
|
+
const call = toolCalls.find((item) => item.type === "function" && item.function?.name === "generate_image");
|
|
228
|
+
if (!call?.function?.arguments) {
|
|
229
|
+
throw grokError("Grok planner did not call generate_image", 502, "GROK_PLANNER_EMPTY_TOOL_CALL");
|
|
230
|
+
}
|
|
231
|
+
let args;
|
|
232
|
+
try {
|
|
233
|
+
args = JSON.parse(call.function.arguments);
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
throw grokError("Grok planner returned invalid tool arguments", 502, "GROK_PLANNER_INVALID_TOOL_ARGS");
|
|
237
|
+
}
|
|
238
|
+
if (typeof args?.prompt !== "string" || !args.prompt.trim()) {
|
|
239
|
+
throw grokError("Grok planner returned an empty image prompt", 502, "GROK_PLANNER_INVALID_TOOL_ARGS");
|
|
240
|
+
}
|
|
241
|
+
return { prompt: args.prompt.trim(), model: fallbackModel, webSearchCalls: 1 };
|
|
242
|
+
}
|
|
243
|
+
export async function planGrokImage(prompt, ctx, options = {}) {
|
|
244
|
+
const imageModel = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image-quality";
|
|
245
|
+
const planner = getPlannerConfig(ctx);
|
|
246
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
247
|
+
const sizeParams = mapSizeToGrokImageParams(options.size);
|
|
248
|
+
// Honor the web-search toggle end to end: skip the forced search call and
|
|
249
|
+
// record calls actually made (070 QA: search ran even with
|
|
250
|
+
// webSearchEnabled:false, and webSearchCalls was hardcoded to 1).
|
|
251
|
+
const searchEnabled = options.webSearchEnabled !== false;
|
|
252
|
+
const search = searchEnabled
|
|
253
|
+
? await searchGrokVisualContext(prompt, ctx, { ...(options.signal ? { signal: options.signal } : {}), ...(options.requestId ? { requestId: options.requestId } : {}), ...(options.directApiKey ? { directApiKey: options.directApiKey } : {}), plannerModel })
|
|
254
|
+
: { summary: "", skipped: true };
|
|
255
|
+
const payload = buildGrokPlannerPayload(prompt, imageModel, options.size, sizeParams, plannerModel, search.summary, options.references || options.referenceCount || 0, options.backgroundConstraint || "");
|
|
256
|
+
const { url, headers } = getGrokEndpoint(ctx, "/v1/chat/completions", options.directApiKey);
|
|
257
|
+
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.timeoutMs);
|
|
258
|
+
logEvent("grok", "planner:start", { requestId: options.requestId, plannerModel, imageModel, size: options.size });
|
|
259
|
+
try {
|
|
260
|
+
// Safe to replay: the planner produces no billable artifact, only a prompt.
|
|
261
|
+
const res = await grokFetchWithRetry(() => fetch(url, {
|
|
262
|
+
method: "POST",
|
|
263
|
+
headers,
|
|
264
|
+
body: JSON.stringify(payload),
|
|
265
|
+
signal: combinedSignal,
|
|
266
|
+
}), { signal: combinedSignal, label: "planner" });
|
|
267
|
+
clearTimeout(timer);
|
|
268
|
+
if (!res.ok) {
|
|
269
|
+
const text = await res.text().catch(() => "");
|
|
270
|
+
let parsed;
|
|
271
|
+
try {
|
|
272
|
+
parsed = JSON.parse(text);
|
|
273
|
+
}
|
|
274
|
+
catch { /* ignore */ }
|
|
275
|
+
const msg = parsed?.error || text || `HTTP ${res.status}`;
|
|
276
|
+
throw grokStageError("planner", msg, res.status);
|
|
277
|
+
}
|
|
278
|
+
const parsed = parseGrokImagePlan(await res.json(), imageModel);
|
|
279
|
+
const plan = { ...parsed, webSearchCalls: searchEnabled ? 1 : 0 };
|
|
280
|
+
logEvent("grok", "planner:done", {
|
|
281
|
+
requestId: options.requestId,
|
|
282
|
+
plannerModel,
|
|
283
|
+
imageModel,
|
|
284
|
+
promptChars: plan.prompt.length,
|
|
285
|
+
aspectRatio: sizeParams.aspect_ratio,
|
|
286
|
+
resolution: sizeParams.resolution,
|
|
287
|
+
});
|
|
288
|
+
return plan;
|
|
289
|
+
}
|
|
290
|
+
catch (e) {
|
|
291
|
+
clearTimeout(timer);
|
|
292
|
+
if (e.name === "AbortError") {
|
|
293
|
+
if (options.signal?.aborted)
|
|
294
|
+
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
295
|
+
throw grokError("Grok planner timed out", 504, "GROK_PLANNER_TIMEOUT");
|
|
296
|
+
}
|
|
297
|
+
if (e.code && e.status)
|
|
298
|
+
throw e;
|
|
299
|
+
throw grokError(`Grok planner request failed: ${e.message}`, 502, "GROK_PLANNER_NETWORK_FAILED");
|
|
300
|
+
}
|
|
301
|
+
}
|
|
@@ -1,75 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { imageEditPayload, imagePayload, planGrokImage, postGrokImages, grokError, downloadGrokImageUrl, } from "./grokImageAdapter.js";
|
|
3
|
-
import { logEvent } from "./logger.js";
|
|
4
|
-
/**
|
|
5
|
-
* Representative-error policy for a multimode run.
|
|
6
|
-
*
|
|
7
|
-
* Items are attempted in order and a mixed-cause run has no principled winner,
|
|
8
|
-
* so the last failure is chosen by construction. The result is only meaningful
|
|
9
|
-
* when every item failed: with any image returned the run is a success and
|
|
10
|
-
* callers must not report an error (lib/multimodePipeline.ts consults it solely
|
|
11
|
-
* under `returned === 0`).
|
|
12
|
-
*/
|
|
13
|
-
export function representativeItemError(errors, returnedImages) {
|
|
14
|
-
if (returnedImages > 0)
|
|
15
|
-
return undefined;
|
|
16
|
-
return errors.length > 0 ? errors[errors.length - 1] : undefined;
|
|
17
|
-
}
|
|
18
|
-
export async function generateMultimodeViaGrok(prompt, ctx, options = {}) {
|
|
19
|
-
const model = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image-quality";
|
|
20
|
-
const maxGeneratedImages = Math.max(1, Math.trunc(Number(ctx.config.limits?.maxGeneratedImages) || 24));
|
|
21
|
-
const maxImages = Math.min(maxGeneratedImages, Math.max(1, Math.trunc(Number(options.maxImages) || 4)));
|
|
22
|
-
const references = options.references || [];
|
|
23
|
-
const images = [];
|
|
24
|
-
let totalCost = 0;
|
|
25
|
-
let totalWebSearchCalls = 0;
|
|
26
|
-
const itemErrors = [];
|
|
27
|
-
logEvent("grok", "multimode:start", { requestId: options.requestId, model, maxImages, refs: references.length });
|
|
28
|
-
for (let i = 0; i < maxImages; i++) {
|
|
29
|
-
if (options.signal?.aborted)
|
|
30
|
-
throw grokError("Generation canceled", 499, "GENERATION_CANCELED");
|
|
31
|
-
const indexedPrompt = maxImages > 1 ? `[Image ${i + 1} of ${maxImages}] ${prompt}` : prompt;
|
|
32
|
-
const plan = await planGrokImage(indexedPrompt, ctx, {
|
|
33
|
-
model,
|
|
34
|
-
size: options.size,
|
|
35
|
-
signal: options.signal,
|
|
36
|
-
requestId: options.requestId,
|
|
37
|
-
references,
|
|
38
|
-
directApiKey: options.directApiKey,
|
|
39
|
-
});
|
|
40
|
-
totalWebSearchCalls += plan.webSearchCalls;
|
|
41
|
-
const endpoint = references.length > 0 ? "/v1/images/edits" : "/v1/images/generations";
|
|
42
|
-
const payload = references.length > 0
|
|
43
|
-
? imageEditPayload(model, plan.prompt, references, options.size)
|
|
44
|
-
: imagePayload(model, plan.prompt, options.size);
|
|
45
|
-
try {
|
|
46
|
-
logEvent("grok", "multimode:item-start", {
|
|
47
|
-
requestId: options.requestId,
|
|
48
|
-
index: i,
|
|
49
|
-
endpoint,
|
|
50
|
-
refs: references.length,
|
|
51
|
-
promptChars: plan.prompt.length,
|
|
52
|
-
});
|
|
53
|
-
const result = await postGrokImages(ctx, payload, options.signal, endpoint, options.directApiKey);
|
|
54
|
-
const imgUrl = result.data?.[0]?.url;
|
|
55
|
-
if (imgUrl) {
|
|
56
|
-
const dl = await downloadGrokImageUrl(imgUrl, options.signal);
|
|
57
|
-
const img = { b64: dl.b64, mime: dl.mime, revisedPrompt: plan.prompt, providerUrl: imgUrl };
|
|
58
|
-
images.push(img);
|
|
59
|
-
if (result.usage?.cost_in_usd_ticks)
|
|
60
|
-
totalCost += result.usage.cost_in_usd_ticks;
|
|
61
|
-
await options.onFinalImage?.(img, i);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
catch (e) {
|
|
65
|
-
if (e.code === "GENERATION_CANCELED")
|
|
66
|
-
throw e;
|
|
67
|
-
logEvent("grok", "multimode:item-error", { requestId: options.requestId, index: i, error: errInfo(e) });
|
|
68
|
-
itemErrors.push(e);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
logEvent("grok", "multimode:done", { requestId: options.requestId, model, returned: images.length, requested: maxImages, refs: references.length });
|
|
72
|
-
const usage = totalCost > 0 ? { grok_cost_usd_ticks: totalCost } : null;
|
|
73
|
-
const representative = representativeItemError(itemErrors, images.length);
|
|
74
|
-
return { images, usage, webSearchCalls: totalWebSearchCalls, extraIgnored: 0, ...(representative !== undefined ? { error: representative } : {}) };
|
|
75
|
-
}
|
|
1
|
+
export { generateMultimodeViaGrok, representativeItemError, } from "./providers/adapters/grokMultimodeOperations.js";
|