ima2-gen 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/bin/commands/gen.js +7 -5
- package/bin/commands/multimode.js +7 -5
- package/bin/commands/node.js +2 -1
- package/config.js +8 -1
- package/docs/API.md +5 -4
- package/docs/CLI.md +4 -4
- package/docs/FAQ.ko.md +1 -1
- package/docs/FAQ.md +1 -1
- package/docs/migration/runtime-test-inventory.md +11 -1
- package/lib/agentCommandParser.js +10 -6
- package/lib/agentGenerationPlanner.js +96 -19
- package/lib/agentImageVideoGen.js +26 -6
- package/lib/agentPlannerModel.js +172 -0
- package/lib/agentQuestionResponder.js +9 -3
- package/lib/agentQueueStore.js +42 -0
- package/lib/agentQueueWorker.js +57 -7
- package/lib/agentRuntime.js +96 -12
- package/lib/agentSettings.js +9 -6
- package/lib/agentToolManifest.js +90 -0
- package/lib/agentTypes.js +1 -0
- package/lib/agyCli.js +31 -0
- package/lib/agyImageAdapter.js +73 -8
- package/lib/capabilities.js +7 -5
- package/lib/configKeys.js +6 -0
- package/lib/geminiApiImageAdapter.js +11 -4
- package/lib/generationRequestLog.js +25 -0
- package/lib/grokImageAdapter.js +27 -34
- package/lib/grokMultimodeAdapter.js +2 -1
- package/lib/grokVideoAdapter.js +3 -2
- package/lib/grokVideoPlannerPrompt.js +18 -25
- package/lib/inflight.js +1 -1
- package/lib/multimodeHelpers.js +3 -2
- package/lib/oauthProxy/generators.js +6 -5
- package/lib/oauthProxy/prompts.js +41 -16
- package/lib/oauthProxy/streams.js +1 -1
- package/lib/promptSafetyPolicy.js +1 -1
- package/lib/responsesDoctor.js +3 -3
- package/lib/responsesFallback.js +27 -14
- package/lib/responsesImageAdapter.js +11 -7
- package/node_modules/qs/CHANGELOG.md +10 -0
- package/node_modules/qs/README.md +1 -1
- package/node_modules/qs/dist/qs.js +15 -15
- package/node_modules/qs/eslint.config.mjs +1 -0
- package/node_modules/qs/lib/parse.js +52 -22
- package/node_modules/qs/lib/stringify.js +11 -4
- package/node_modules/qs/package.json +2 -2
- package/node_modules/qs/test/parse.js +49 -0
- package/node_modules/qs/test/stringify.js +129 -0
- package/package.json +1 -1
- package/routes/agent.js +19 -2
- package/routes/agy.js +5 -1
- package/routes/generate.js +12 -1
- package/routes/generationRequestLog.js +16 -0
- package/routes/index.js +2 -0
- package/routes/keys.js +22 -2
- package/routes/multimode.js +1 -1
- package/routes/nodes.js +5 -5
- package/ui/dist/.vite/manifest.json +22 -12
- package/ui/dist/assets/AgentWorkspace-qEMrATBP.js +3 -0
- package/ui/dist/assets/{CardNewsWorkspace-Dav3K5CT.js → CardNewsWorkspace-DoJnVHmn.js} +1 -1
- package/ui/dist/assets/GenerationRequestLogPanel-BDWRRLNV.js +1 -0
- package/ui/dist/assets/{NodeCanvas-C4ifFzB1.js → NodeCanvas-DIMo45sp.js} +3 -3
- package/ui/dist/assets/{PromptBuilderPanel-CEcyU9PL.js → PromptBuilderPanel-BF3lcSJD.js} +2 -2
- package/ui/dist/assets/{PromptImportDialog-CgQ94Gth.js → PromptImportDialog-Cy6ZymKG.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-CuzyzbNI.js → PromptImportDiscoverySection-CzuW-8P2.js} +1 -1
- package/ui/dist/assets/PromptImportFolderSection-DP5ywnsv.js +1 -0
- package/ui/dist/assets/{PromptLibraryPanel-BOe18we8.js → PromptLibraryPanel-CAsKr7CV.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-_PdPVsxi.js +1 -0
- package/ui/dist/assets/index-BUinlX2n.js +4 -0
- package/ui/dist/assets/index-CX3fge8X.css +1 -0
- package/ui/dist/assets/index-ygo6nfqx.js +23 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/AgentWorkspace-Dth6YijN.js +0 -3
- package/ui/dist/assets/PromptImportFolderSection-DHLGlO6l.js +0 -1
- package/ui/dist/assets/SettingsWorkspace-Cdgnm4Wa.js +0 -1
- package/ui/dist/assets/index-C5PSahkr.js +0 -1
- package/ui/dist/assets/index-Dn2AhL6d.css +0 -1
- package/ui/dist/assets/index-Tjqx6wUV.js +0 -23
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { AGENT_ALLOWED_TOOLS } from "./agentTypes.js";
|
|
2
|
+
// Single source of truth for the agent tool surface. Consumed by the LLM
|
|
3
|
+
// planner developer prompt, /api/agent/tools, and capabilities.agentMode.
|
|
4
|
+
export const AGENT_TOOL_MANIFEST = [
|
|
5
|
+
{
|
|
6
|
+
name: "ima2.get_image_context",
|
|
7
|
+
description: "Load the session image context manifest (previous images, current image, locks). Runs automatically before image generation.",
|
|
8
|
+
parameters: {
|
|
9
|
+
type: "object",
|
|
10
|
+
properties: {},
|
|
11
|
+
additionalProperties: false,
|
|
12
|
+
},
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
name: "ima2.web_search",
|
|
16
|
+
description: "Search the web for factual visual references before generating. Only available when web search is enabled for the session.",
|
|
17
|
+
parameters: {
|
|
18
|
+
type: "object",
|
|
19
|
+
properties: {
|
|
20
|
+
query: { type: "string", description: "Search query for factual visual accuracy." },
|
|
21
|
+
},
|
|
22
|
+
required: ["query"],
|
|
23
|
+
additionalProperties: false,
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
name: "ima2.generate_image",
|
|
28
|
+
description: "Generate one or more images. Supports fanout: provide one prompt per variant.",
|
|
29
|
+
parameters: {
|
|
30
|
+
type: "object",
|
|
31
|
+
properties: {
|
|
32
|
+
prompts: {
|
|
33
|
+
type: "array",
|
|
34
|
+
items: { type: "string" },
|
|
35
|
+
description: "One generation prompt per planned variant (1 to the configured image limit).",
|
|
36
|
+
},
|
|
37
|
+
plannedVariants: { type: "integer", minimum: 1, description: "Number of images to generate." },
|
|
38
|
+
plannedParallelism: { type: "integer", minimum: 1, description: "Concurrent generation calls." },
|
|
39
|
+
sourceImagePolicy: {
|
|
40
|
+
type: "string",
|
|
41
|
+
enum: ["auto", "none", "current"],
|
|
42
|
+
description: "none creates a fresh image and ignores the current session image; current uses the current session image as edit/reference input; auto lets the runtime choose.",
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
required: ["prompts"],
|
|
46
|
+
additionalProperties: false,
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
name: "ima2.generate_video",
|
|
51
|
+
description: "Generate a single video with Grok Imagine. If the session has a last image, it is used as the image-to-video source automatically.",
|
|
52
|
+
parameters: {
|
|
53
|
+
type: "object",
|
|
54
|
+
properties: {
|
|
55
|
+
prompt: { type: "string", description: "Video prompt describing visual flow and motion." },
|
|
56
|
+
duration: { type: "integer", minimum: 1, maximum: 15, description: "Video duration in seconds. Default 5." },
|
|
57
|
+
resolution: { type: "string", enum: ["480p", "720p"], description: "Output resolution. Default 480p." },
|
|
58
|
+
aspectRatio: {
|
|
59
|
+
type: "string",
|
|
60
|
+
enum: ["auto", "1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"],
|
|
61
|
+
description: "Output aspect ratio. Default auto.",
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
required: ["prompt"],
|
|
65
|
+
additionalProperties: false,
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
name: "ima2.get_generation_errors",
|
|
70
|
+
description: "Read-only lookup of the session's recent generation failures (failed queue jobs and error turns). Use when the user asks why a generation failed.",
|
|
71
|
+
parameters: {
|
|
72
|
+
type: "object",
|
|
73
|
+
properties: {
|
|
74
|
+
limit: { type: "integer", minimum: 1, maximum: 20, description: "Maximum error records to return. Default 10." },
|
|
75
|
+
},
|
|
76
|
+
additionalProperties: false,
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
];
|
|
80
|
+
const MANIFEST_NAMES = new Set(AGENT_TOOL_MANIFEST.map((entry) => entry.name));
|
|
81
|
+
for (const tool of AGENT_ALLOWED_TOOLS) {
|
|
82
|
+
if (!MANIFEST_NAMES.has(tool)) {
|
|
83
|
+
throw new Error(`Agent tool manifest is missing an entry for: ${tool}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export function formatToolManifestForPrompt() {
|
|
87
|
+
return AGENT_TOOL_MANIFEST
|
|
88
|
+
.map((entry) => `- ${entry.name}: ${entry.description}\n parameters: ${JSON.stringify(entry.parameters)}`)
|
|
89
|
+
.join("\n");
|
|
90
|
+
}
|
package/lib/agentTypes.js
CHANGED
package/lib/agyCli.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { delimiter, dirname, join } from "node:path";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
export function agyCommandName(platform = process.platform) {
|
|
5
|
+
return platform === "win32" ? "agy.cmd" : "agy";
|
|
6
|
+
}
|
|
7
|
+
export function agyLocalBinCandidates(home = homedir(), platform = process.platform) {
|
|
8
|
+
const command = agyCommandName(platform);
|
|
9
|
+
return [
|
|
10
|
+
join(home, ".local", "bin", command),
|
|
11
|
+
join(home, ".npm-global", "bin", command),
|
|
12
|
+
];
|
|
13
|
+
}
|
|
14
|
+
export function resolveAgyBin(env = process.env, home = homedir(), platform = process.platform) {
|
|
15
|
+
if (env.IMA2_AGY_BIN) {
|
|
16
|
+
if (!existsSync(env.IMA2_AGY_BIN)) {
|
|
17
|
+
console.warn(`[ima2-agy] IMA2_AGY_BIN is set to "${env.IMA2_AGY_BIN}" but the file does not exist. Falling back to PATH lookup.`);
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
return env.IMA2_AGY_BIN;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return agyLocalBinCandidates(home, platform).find((candidate) => existsSync(candidate))
|
|
24
|
+
?? agyCommandName(platform);
|
|
25
|
+
}
|
|
26
|
+
export function buildAgyPathEnv(env = process.env, agyBin = resolveAgyBin(env)) {
|
|
27
|
+
const basePath = env.PATH || "";
|
|
28
|
+
if (agyBin === agyCommandName() || agyBin === "agy")
|
|
29
|
+
return basePath;
|
|
30
|
+
return `${dirname(agyBin)}${delimiter}${basePath}`;
|
|
31
|
+
}
|
package/lib/agyImageAdapter.js
CHANGED
|
@@ -3,7 +3,9 @@ import { readFile, readdir, rm, stat, writeFile, mkdir } from "node:fs/promises"
|
|
|
3
3
|
import { extname, dirname, join, resolve } from "node:path";
|
|
4
4
|
import { homedir, tmpdir } from "node:os";
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { buildAgyPathEnv, resolveAgyBin } from "./agyCli.js";
|
|
6
7
|
import { logEvent } from "./logger.js";
|
|
8
|
+
import { SAFETY_INTENT_POLICY } from "./promptSafetyPolicy.js";
|
|
7
9
|
import { detectImageMimeFromB64 } from "./refs.js";
|
|
8
10
|
const AGY_TIMEOUT_MS = 360_000;
|
|
9
11
|
const AGY_OUTPUT_RESOLUTION = "1024x1024";
|
|
@@ -19,7 +21,8 @@ function buildAgyPrompt(userPrompt, referencePaths) {
|
|
|
19
21
|
? JSON.stringify(referencePaths)
|
|
20
22
|
: "[]";
|
|
21
23
|
return [
|
|
22
|
-
"You are an image generation assistant inside a professional creative tool used by designers and artists.
|
|
24
|
+
"You are an image generation assistant inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
|
|
25
|
+
SAFETY_INTENT_POLICY,
|
|
23
26
|
"",
|
|
24
27
|
"Generate one image by calling the tool default_api:generate_image once.",
|
|
25
28
|
"After the tool finishes, print one machine-readable result line so ima2-gen can copy the artifact.",
|
|
@@ -34,10 +37,12 @@ function buildAgyPrompt(userPrompt, referencePaths) {
|
|
|
34
37
|
`Reference count: ${referencePaths.length}. The output resolution is fixed at ${AGY_OUTPUT_RESOLUTION}.`,
|
|
35
38
|
"",
|
|
36
39
|
"PROMPT HANDLING:",
|
|
37
|
-
"- Preserve the user's prompt as-is. Do not translate,
|
|
40
|
+
"- Preserve the user's prompt as-is for compliant requests. Do not translate, restyle, or soften benign adult clothing requests such as swimwear.",
|
|
38
41
|
"- If the user wrote in Korean, Japanese, or any non-English language, keep visible text and dialogue in that language.",
|
|
39
|
-
"- Do
|
|
40
|
-
"-
|
|
42
|
+
"- Do not sexualize, intensify exposure, or add sexualized/fetish framing to benign swimwear, sportswear, or catalog fashion prompts.",
|
|
43
|
+
"- Do not copy safety labels, refusal reasons, or negative safety constraints into the final image prompt.",
|
|
44
|
+
"- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
|
|
45
|
+
"- Content moderation is still handled downstream by the provider.",
|
|
41
46
|
"",
|
|
42
47
|
"OUTPUT:",
|
|
43
48
|
"If generation succeeds, print: RESULT|<absolute_artifact_path>|<file_extension>",
|
|
@@ -83,12 +88,47 @@ function parseAgyOutput(stdout) {
|
|
|
83
88
|
}
|
|
84
89
|
throw agyError(`Could not parse artifact path from agy output (${stdout.length} chars): ${stdout.slice(0, 200)}`, 502, "AGY_PARSE_FAILED");
|
|
85
90
|
}
|
|
91
|
+
export async function findRecentAgyArtifact(sinceMs, rootOverrides) {
|
|
92
|
+
const roots = rootOverrides ?? [
|
|
93
|
+
join(homedir(), ".gemini", "antigravity-cli", "brain"),
|
|
94
|
+
join(homedir(), ".gemini"),
|
|
95
|
+
];
|
|
96
|
+
const candidates = [];
|
|
97
|
+
const seen = new Set();
|
|
98
|
+
async function walk(dir, depth) {
|
|
99
|
+
if (depth > 5 || seen.has(dir))
|
|
100
|
+
return;
|
|
101
|
+
seen.add(dir);
|
|
102
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
103
|
+
for (const entry of entries) {
|
|
104
|
+
const p = join(dir, entry.name);
|
|
105
|
+
if (!entry.isSymbolicLink() && entry.isDirectory()) {
|
|
106
|
+
await walk(p, depth + 1);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
if (!/^ima2_generated.*\.(png|jpg|jpeg|webp)$/i.test(entry.name)) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
const s = await stat(p).catch(() => null);
|
|
113
|
+
if (!s)
|
|
114
|
+
continue;
|
|
115
|
+
if (s.mtimeMs >= sinceMs - 5_000) {
|
|
116
|
+
candidates.push({ path: p, mtimeMs: s.mtimeMs });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
for (const root of roots) {
|
|
121
|
+
await walk(root, 0);
|
|
122
|
+
}
|
|
123
|
+
candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
124
|
+
return candidates[0]?.path ?? null;
|
|
125
|
+
}
|
|
86
126
|
function spawnAgy(prompt, signal) {
|
|
87
127
|
return new Promise((resolve, reject) => {
|
|
88
|
-
const child = spawn(
|
|
128
|
+
const child = spawn(resolveAgyBin(), ["-p", "-"], {
|
|
89
129
|
stdio: ["pipe", "pipe", "pipe"],
|
|
90
130
|
env: {
|
|
91
|
-
PATH:
|
|
131
|
+
PATH: buildAgyPathEnv(),
|
|
92
132
|
HOME: process.env.HOME,
|
|
93
133
|
USERPROFILE: process.env.USERPROFILE,
|
|
94
134
|
TMPDIR: process.env.TMPDIR,
|
|
@@ -116,7 +156,11 @@ function spawnAgy(prompt, signal) {
|
|
|
116
156
|
return;
|
|
117
157
|
settled = true;
|
|
118
158
|
clearTimeout(timer);
|
|
119
|
-
|
|
159
|
+
const bin = resolveAgyBin();
|
|
160
|
+
const hint = err.code === "ENOENT"
|
|
161
|
+
? `. "${bin}" was not found — install agy or set IMA2_AGY_BIN=/absolute/path/to/agy`
|
|
162
|
+
: "";
|
|
163
|
+
reject(agyError(`Agy process error: ${err.message}${hint}`, 502, "AGY_PROCESS_ERROR"));
|
|
120
164
|
});
|
|
121
165
|
child.on("close", (code) => {
|
|
122
166
|
if (settled)
|
|
@@ -200,6 +244,7 @@ export async function generateViaAgy(prompt, options = {}) {
|
|
|
200
244
|
refs: refPaths.length,
|
|
201
245
|
});
|
|
202
246
|
try {
|
|
247
|
+
const generationStartedAtMs = Date.now();
|
|
203
248
|
const { stdout, stderr } = await spawnAgy(agyPrompt, options.signal);
|
|
204
249
|
if (stderr && stderr.trim().length > 0) {
|
|
205
250
|
logEvent("agy", "generate:stderr", {
|
|
@@ -208,7 +253,27 @@ export async function generateViaAgy(prompt, options = {}) {
|
|
|
208
253
|
stderrPreview: stderr.slice(0, 200),
|
|
209
254
|
});
|
|
210
255
|
}
|
|
211
|
-
const
|
|
256
|
+
const agyCombinedOutput = [stdout, stderr].filter(Boolean).join("\n");
|
|
257
|
+
let artifactPath;
|
|
258
|
+
try {
|
|
259
|
+
artifactPath = parseAgyOutput(agyCombinedOutput).artifactPath;
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
if (err?.code !== "AGY_PARSE_FAILED") {
|
|
263
|
+
throw err;
|
|
264
|
+
}
|
|
265
|
+
const fallbackPath = await findRecentAgyArtifact(generationStartedAtMs);
|
|
266
|
+
if (!fallbackPath) {
|
|
267
|
+
throw err;
|
|
268
|
+
}
|
|
269
|
+
logEvent("agy", "generate:fallback_artifact_found", {
|
|
270
|
+
requestId: options.requestId,
|
|
271
|
+
artifactPath: fallbackPath,
|
|
272
|
+
stdoutChars: stdout.length,
|
|
273
|
+
stderrChars: stderr.length,
|
|
274
|
+
});
|
|
275
|
+
artifactPath = fallbackPath;
|
|
276
|
+
}
|
|
212
277
|
// Validate artifact path is within allowed directories
|
|
213
278
|
const resolvedPath = resolve(artifactPath);
|
|
214
279
|
const allowedPrefixes = [
|
package/lib/capabilities.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { config as runtimeConfigDefault } from "../config.js";
|
|
2
|
+
import { AGENT_ALLOWED_TOOLS } from "./agentTypes.js";
|
|
3
|
+
import { AGENT_TOOL_MANIFEST } from "./agentToolManifest.js";
|
|
2
4
|
import { KEY_TO_ENV, WRITABLE_CONFIG_KEYS } from "./configKeys.js";
|
|
3
5
|
import { DEFAULT_IMAGE_QUALITY, VALID_IMAGE_QUALITIES } from "./oauthNormalize.js";
|
|
4
|
-
const MAX_GENERATED_IMAGES = 8;
|
|
5
6
|
const VALID_MODES = ["auto", "direct"];
|
|
6
7
|
const VALID_PROVIDERS = ["auto", "oauth", "api", "grok", "grok-api", "agy", "gemini-api"];
|
|
7
8
|
const AGENT_COMMANDS = [
|
|
@@ -79,11 +80,11 @@ export function buildIma2Capabilities({ appConfig = runtimeConfigDefault, packag
|
|
|
79
80
|
},
|
|
80
81
|
limits: {
|
|
81
82
|
maxRefCount: appConfig.limits.maxRefCount,
|
|
82
|
-
maxGeneratedImages:
|
|
83
|
+
maxGeneratedImages: appConfig.limits.maxGeneratedImages,
|
|
83
84
|
maxParallel: {
|
|
84
85
|
value: appConfig.limits.maxParallel,
|
|
85
|
-
enforced:
|
|
86
|
-
note: "
|
|
86
|
+
enforced: true,
|
|
87
|
+
note: "server-side inflight capacity guard uses this runtime limit",
|
|
87
88
|
},
|
|
88
89
|
},
|
|
89
90
|
promptBuilder: {
|
|
@@ -96,7 +97,8 @@ export function buildIma2Capabilities({ appConfig = runtimeConfigDefault, packag
|
|
|
96
97
|
agentMode: {
|
|
97
98
|
available: true,
|
|
98
99
|
route: "/api/agent/sessions",
|
|
99
|
-
allowedTools: [
|
|
100
|
+
allowedTools: [...AGENT_ALLOWED_TOOLS],
|
|
101
|
+
toolManifest: [...AGENT_TOOL_MANIFEST],
|
|
100
102
|
finalArtifact: "image",
|
|
101
103
|
uiOnly: true,
|
|
102
104
|
cliCommand: null,
|
package/lib/configKeys.js
CHANGED
|
@@ -10,6 +10,8 @@ export const WRITABLE_CONFIG_KEYS = new Set([
|
|
|
10
10
|
"cardNewsPlanner.model",
|
|
11
11
|
"cardNewsPlanner.timeoutMs",
|
|
12
12
|
"cardNewsPlanner.deterministicFallback",
|
|
13
|
+
"agentPlanner.enabled",
|
|
14
|
+
"agentPlanner.timeoutMs",
|
|
13
15
|
"grokProvider.plannerModel",
|
|
14
16
|
"grokProvider.plannerTimeoutMs",
|
|
15
17
|
"grokProvider.defaultImageModel",
|
|
@@ -25,6 +27,7 @@ export const WRITABLE_CONFIG_KEYS = new Set([
|
|
|
25
27
|
"oauth.statusTimeoutMs",
|
|
26
28
|
"oauth.restartDelayMs",
|
|
27
29
|
"limits.maxRefCount",
|
|
30
|
+
"limits.maxGeneratedImages",
|
|
28
31
|
"limits.maxParallel",
|
|
29
32
|
"history.defaultPageSize",
|
|
30
33
|
"history.maxPageCap",
|
|
@@ -44,9 +47,12 @@ export const KEY_TO_ENV = {
|
|
|
44
47
|
"cardNewsPlanner.enabled": "IMA2_CARD_NEWS_PLANNER",
|
|
45
48
|
"cardNewsPlanner.model": "IMA2_CARD_NEWS_PLANNER_MODEL",
|
|
46
49
|
"cardNewsPlanner.timeoutMs": "IMA2_CARD_NEWS_PLANNER_TIMEOUT_MS",
|
|
50
|
+
"agentPlanner.enabled": "IMA2_AGENT_PLANNER_ENABLED",
|
|
51
|
+
"agentPlanner.timeoutMs": "IMA2_AGENT_PLANNER_TIMEOUT_MS",
|
|
47
52
|
"grokProvider.plannerModel": "IMA2_GROK_PLANNER_MODEL",
|
|
48
53
|
"grokProvider.plannerTimeoutMs": "IMA2_GROK_PLANNER_TIMEOUT_MS",
|
|
49
54
|
"grokProvider.defaultImageModel": "IMA2_GROK_IMAGE_MODEL_DEFAULT",
|
|
55
|
+
"limits.maxGeneratedImages": "IMA2_MAX_GENERATED_IMAGES",
|
|
50
56
|
"limits.maxParallel": "IMA2_MAX_PARALLEL",
|
|
51
57
|
"limits.maxRefCount": "IMA2_MAX_REF_COUNT",
|
|
52
58
|
"history.defaultPageSize": "IMA2_HISTORY_PAGE_SIZE",
|
|
@@ -81,16 +81,23 @@ export async function generateViaGeminiApi(prompt, ctx, options = {}) {
|
|
|
81
81
|
url = `https://generativelanguage.googleapis.com/v1beta/models/${apiModelId}:generateContent`;
|
|
82
82
|
authHeaders = { "Content-Type": "application/json", "x-goog-api-key": apiKey };
|
|
83
83
|
}
|
|
84
|
-
|
|
85
|
-
const
|
|
84
|
+
// size "auto" → omit image config entirely so the model decides ratio/size
|
|
85
|
+
const isAutoSize = !options.size || options.size === "auto";
|
|
86
|
+
const imageParams = isAutoSize ? null : parseGeminiImageParams(options.size);
|
|
87
|
+
// Vertex AI rejects responseFormat; it expects imageConfig directly under
|
|
88
|
+
// generationConfig (camelCase). Public API uses response_format.image (snake_case).
|
|
86
89
|
const generationConfig = useVertex
|
|
87
90
|
? {
|
|
88
91
|
responseModalities: ["TEXT", "IMAGE"],
|
|
89
|
-
|
|
92
|
+
...(imageParams
|
|
93
|
+
? { imageConfig: { aspectRatio: imageParams.aspectRatio, imageSize: imageParams.imageSize } }
|
|
94
|
+
: {}),
|
|
90
95
|
}
|
|
91
96
|
: {
|
|
92
97
|
response_modalities: ["TEXT", "IMAGE"],
|
|
93
|
-
|
|
98
|
+
...(imageParams
|
|
99
|
+
? { response_format: { image: { aspect_ratio: imageParams.aspectRatio, image_size: imageParams.imageSize } } }
|
|
100
|
+
: {}),
|
|
94
101
|
};
|
|
95
102
|
const configKey = useVertex ? "generationConfig" : "generation_config";
|
|
96
103
|
const body = { contents: buildContents(prompt, references), [configKey]: generationConfig };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { atomicWriteJson } from "./atomicWrite.js";
|
|
4
|
+
const MAX_ENTRIES = 200;
|
|
5
|
+
let writeQueue = Promise.resolve();
|
|
6
|
+
async function readEntries(path) {
|
|
7
|
+
try {
|
|
8
|
+
const parsed = JSON.parse(await readFile(path, "utf8"));
|
|
9
|
+
return Array.isArray(parsed) ? parsed.slice(0, MAX_ENTRIES) : [];
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export async function listGenerationRequestLog(path) {
|
|
16
|
+
return readEntries(path);
|
|
17
|
+
}
|
|
18
|
+
export function appendGenerationRequestLog(path, entry) {
|
|
19
|
+
writeQueue = writeQueue.catch(() => undefined).then(async () => {
|
|
20
|
+
await mkdir(dirname(path), { recursive: true });
|
|
21
|
+
const entries = await readEntries(path);
|
|
22
|
+
await atomicWriteJson(path, [entry, ...entries].slice(0, MAX_ENTRIES));
|
|
23
|
+
});
|
|
24
|
+
return writeQueue;
|
|
25
|
+
}
|
package/lib/grokImageAdapter.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { logEvent } from "./logger.js";
|
|
2
|
+
import { SAFETY_INTENT_POLICY } from "./promptSafetyPolicy.js";
|
|
2
3
|
import { mapSizeToGrokImageParams } from "./grokSizeMapper.js";
|
|
3
4
|
import { detectImageMimeFromB64 } from "./refs.js";
|
|
4
5
|
import { grokError, grokStageError, getPlannerConfig, getGrokEndpoint, withTimeoutSignal, imagePayload, imageEditPayload, referenceImageUrl, extractResponsesText, postGrokImages, downloadGrokImageUrl, } from "./grokImageCore.js";
|
|
@@ -20,7 +21,8 @@ export function buildGrokPlannerPayload(prompt, model, size, sizeParams, planner
|
|
|
20
21
|
{
|
|
21
22
|
role: "system",
|
|
22
23
|
content: [
|
|
23
|
-
"You are ima2's image generation planner for xAI Grok Imagine (Aurora model), inside a professional creative tool used by designers and artists.
|
|
24
|
+
"You are ima2's image generation planner for xAI Grok Imagine (Aurora model), inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
|
|
25
|
+
SAFETY_INTENT_POLICY,
|
|
24
26
|
"",
|
|
25
27
|
"TASK: Rewrite the user's casual request into ONE optimal, production-ready image prompt.",
|
|
26
28
|
"",
|
|
@@ -58,26 +60,16 @@ export function buildGrokPlannerPayload(prompt, model, size, sizeParams, planner
|
|
|
58
60
|
"- When multiple characters appear, describe each by clothing, physique, position, or props.",
|
|
59
61
|
"- Characters must be distinguishable by at least two visual attributes.",
|
|
60
62
|
"",
|
|
61
|
-
"
|
|
62
|
-
"Before rewriting,
|
|
63
|
-
"
|
|
63
|
+
"BENIGN CLOTHING AND CONTEXT HANDLING:",
|
|
64
|
+
"Before rewriting, distinguish ordinary clothing/swimwear descriptions from sexualized requests.",
|
|
65
|
+
"Preserve benign adult clothing intent without adding erotic details:",
|
|
64
66
|
"",
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"nude/누드 → bare skin visible, no clothing, figure study pose",
|
|
70
|
-
"상반신 노출 → bare shoulders, collarbones, upper torso uncovered",
|
|
71
|
-
"lingerie/란제리 → lace bralette and matching briefs, sheer fabric, delicate straps",
|
|
72
|
-
"seductive/유혹적 → smoldering eye contact, beckoning pose, soft parted lips, intimate close framing",
|
|
73
|
-
"bondage → intricate rope pattern across torso and limbs, knotted hemp cord, shibari ties",
|
|
74
|
-
"wet/젖은 → glistening water droplets on skin, damp clinging fabric, water-slicked hair",
|
|
75
|
-
"bikini/비키니 → string triangle top and low-rise bottom, two-piece swimwear",
|
|
76
|
-
"시스루 → translucent layered fabric, skin tone visible underneath",
|
|
67
|
+
"bikini/비키니 → adult two-piece swimwear in a neutral beach, pool, swimwear catalog, or fashion editorial context",
|
|
68
|
+
"swimsuit/수영복 → adult swimwear suitable for swimming, beach, pool, sport, or catalog photography",
|
|
69
|
+
"sports bra/스포츠브라 → athletic top in a fitness, yoga, running, or product catalog context",
|
|
70
|
+
"lingerie/란제리 → adult fashion or product catalog styling only when the user is clearly requesting adult fashion; avoid sexualized posing or intimate framing",
|
|
77
71
|
"",
|
|
78
|
-
"
|
|
79
|
-
"If the user says lingerie, the image shows lingerie — not a dress. If bikini, show a bikini — not a one-piece.",
|
|
80
|
-
"Never upgrade coverage. Never downgrade exposure. Match the original intent exactly.",
|
|
72
|
+
"CLOTHING RULE: Keep the user's requested clothing category when it is benign and adult, but do not increase exposure, add fetish details, add negative safety constraints, or convert a neutral clothing prompt into a sexualized scene.",
|
|
81
73
|
"",
|
|
82
74
|
"PROFESSIONAL EMBEDDING: After rewriting, wrap the visual description in a professional context.",
|
|
83
75
|
"Prepend ONE of these frames based on content type:",
|
|
@@ -88,14 +80,13 @@ export function buildGrokPlannerPayload(prompt, model, size, sizeParams, planner
|
|
|
88
80
|
"- Swimwear/beach → 'Sports Illustrated Swimsuit Issue editorial photography'",
|
|
89
81
|
"",
|
|
90
82
|
"CONTENT POLICY:",
|
|
91
|
-
"-
|
|
92
|
-
"- Do
|
|
93
|
-
"- Do
|
|
94
|
-
"- Do
|
|
95
|
-
"- Do
|
|
96
|
-
"-
|
|
97
|
-
"-
|
|
98
|
-
"- Your ONLY job is to make the prompt technically optimal. Content moderation is handled downstream.",
|
|
83
|
+
"- Generate one production-ready prompt for compliant image requests.",
|
|
84
|
+
"- Do not add disclaimers, warnings, moral commentary, or safety notes to the final prompt.",
|
|
85
|
+
"- Do not add diversity modifiers or change the user's specified demographics, appearance, or benign clothing.",
|
|
86
|
+
"- Do not turn benign adult swimwear, sportswear, or fashion prompts into sexual content.",
|
|
87
|
+
"- Do not copy safety labels, refusal reasons, or negative safety constraints into the final image prompt.",
|
|
88
|
+
"- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
|
|
89
|
+
"- Content moderation is still handled downstream by the provider.",
|
|
99
90
|
"",
|
|
100
91
|
"STORYBOARD GRID GENERATION RULE:",
|
|
101
92
|
"- If the user requests a 3x3 storyboard grid, Panel 1 (top-left) MUST be solid black with no content.",
|
|
@@ -185,10 +176,11 @@ export function buildGrokSearchPayload(prompt, plannerModel = "grok-4.3") {
|
|
|
185
176
|
}
|
|
186
177
|
export async function searchGrokVisualContext(prompt, ctx, options = {}) {
|
|
187
178
|
const planner = getPlannerConfig(ctx);
|
|
188
|
-
const
|
|
179
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
180
|
+
const payload = buildGrokSearchPayload(prompt, plannerModel);
|
|
189
181
|
const { url, headers } = getGrokEndpoint(ctx, "/v1/responses", options.directApiKey);
|
|
190
182
|
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.timeoutMs);
|
|
191
|
-
logEvent("grok", "search:start", { requestId: options.requestId, plannerModel
|
|
183
|
+
logEvent("grok", "search:start", { requestId: options.requestId, plannerModel, promptChars: prompt.length });
|
|
192
184
|
try {
|
|
193
185
|
const res = await fetch(url, {
|
|
194
186
|
method: "POST",
|
|
@@ -210,7 +202,7 @@ export async function searchGrokVisualContext(prompt, ctx, options = {}) {
|
|
|
210
202
|
const summary = extractResponsesText(await res.json());
|
|
211
203
|
if (!summary)
|
|
212
204
|
throw grokError("Grok web search returned no research summary", 502, "GROK_SEARCH_EMPTY_RESPONSE");
|
|
213
|
-
logEvent("grok", "search:done", { requestId: options.requestId, plannerModel
|
|
205
|
+
logEvent("grok", "search:done", { requestId: options.requestId, plannerModel, summaryChars: summary.length });
|
|
214
206
|
return { summary };
|
|
215
207
|
}
|
|
216
208
|
catch (e) {
|
|
@@ -246,12 +238,13 @@ export function parseGrokImagePlan(response, fallbackModel) {
|
|
|
246
238
|
export async function planGrokImage(prompt, ctx, options = {}) {
|
|
247
239
|
const imageModel = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image";
|
|
248
240
|
const planner = getPlannerConfig(ctx);
|
|
241
|
+
const plannerModel = options.plannerModel || planner.model;
|
|
249
242
|
const sizeParams = mapSizeToGrokImageParams(options.size);
|
|
250
|
-
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey });
|
|
251
|
-
const payload = buildGrokPlannerPayload(prompt, imageModel, options.size, sizeParams,
|
|
243
|
+
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey, plannerModel });
|
|
244
|
+
const payload = buildGrokPlannerPayload(prompt, imageModel, options.size, sizeParams, plannerModel, search.summary, options.references || options.referenceCount || 0);
|
|
252
245
|
const { url, headers } = getGrokEndpoint(ctx, "/v1/chat/completions", options.directApiKey);
|
|
253
246
|
const { combinedSignal, timer } = withTimeoutSignal(options.signal, planner.timeoutMs);
|
|
254
|
-
logEvent("grok", "planner:start", { requestId: options.requestId, plannerModel
|
|
247
|
+
logEvent("grok", "planner:start", { requestId: options.requestId, plannerModel, imageModel, size: options.size });
|
|
255
248
|
try {
|
|
256
249
|
const res = await fetch(url, {
|
|
257
250
|
method: "POST",
|
|
@@ -273,7 +266,7 @@ export async function planGrokImage(prompt, ctx, options = {}) {
|
|
|
273
266
|
const plan = parseGrokImagePlan(await res.json(), imageModel);
|
|
274
267
|
logEvent("grok", "planner:done", {
|
|
275
268
|
requestId: options.requestId,
|
|
276
|
-
plannerModel
|
|
269
|
+
plannerModel,
|
|
277
270
|
imageModel,
|
|
278
271
|
promptChars: plan.prompt.length,
|
|
279
272
|
aspectRatio: sizeParams.aspect_ratio,
|
|
@@ -3,7 +3,8 @@ import { imageEditPayload, imagePayload, planGrokImage, postGrokImages, grokErro
|
|
|
3
3
|
import { logEvent } from "./logger.js";
|
|
4
4
|
export async function generateMultimodeViaGrok(prompt, ctx, options = {}) {
|
|
5
5
|
const model = options.model || ctx.config.grokProvider?.defaultImageModel || "grok-imagine-image";
|
|
6
|
-
const
|
|
6
|
+
const maxGeneratedImages = Math.max(1, Math.trunc(Number(ctx.config.limits?.maxGeneratedImages) || 24));
|
|
7
|
+
const maxImages = Math.min(maxGeneratedImages, Math.max(1, Math.trunc(Number(options.maxImages) || 4)));
|
|
7
8
|
const references = options.references || [];
|
|
8
9
|
const images = [];
|
|
9
10
|
let totalCost = 0;
|
package/lib/grokVideoAdapter.js
CHANGED
|
@@ -155,7 +155,8 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
|
|
|
155
155
|
const duration = options.duration ?? 5;
|
|
156
156
|
const resolution = options.resolution || "480p";
|
|
157
157
|
const aspectRatio = options.aspectRatio || "auto";
|
|
158
|
-
const
|
|
158
|
+
const plannerModel = options.plannerModel || cfg.plannerModel;
|
|
159
|
+
const search = await searchGrokVisualContext(prompt, ctx, { signal: options.signal, requestId: options.requestId, directApiKey: options.directApiKey, plannerModel });
|
|
159
160
|
const referenceImageUrls = (options.referenceImages ?? []).map((img) => sourceImageUrl(img, undefined));
|
|
160
161
|
const payload = buildGrokVideoPlannerPayload(prompt, {
|
|
161
162
|
model: cfg.model,
|
|
@@ -163,7 +164,7 @@ export async function planGrokVideo(prompt, ctx, options = {}) {
|
|
|
163
164
|
duration,
|
|
164
165
|
resolution,
|
|
165
166
|
aspectRatio,
|
|
166
|
-
plannerModel
|
|
167
|
+
plannerModel,
|
|
167
168
|
searchSummary: search.summary,
|
|
168
169
|
sourceImageUrl: options.sourceImage ? sourceImageUrl(options.sourceImage, options.sourceMime) : undefined,
|
|
169
170
|
referenceImageUrls,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { SAFETY_INTENT_POLICY } from "./promptSafetyPolicy.js";
|
|
1
2
|
export function formatDurationPacingGuidance(duration, mode) {
|
|
2
3
|
const roundedDuration = Number.isFinite(duration) && duration > 0 ? Math.round(duration) : 5;
|
|
3
4
|
const modeGuidance = mode === "image-to-video"
|
|
@@ -16,7 +17,8 @@ export function formatDurationPacingGuidance(duration, mode) {
|
|
|
16
17
|
}
|
|
17
18
|
export function buildGrokVideoPlannerSystemPrompt() {
|
|
18
19
|
return [
|
|
19
|
-
"You are ima2's video generation planner for xAI Grok Imagine Video, inside a professional creative tool used by designers and artists.
|
|
20
|
+
"You are ima2's video generation planner for xAI Grok Imagine Video, inside a professional creative tool used by designers and artists. Users create content for legitimate creative, commercial, or educational purposes.",
|
|
21
|
+
SAFETY_INTENT_POLICY,
|
|
20
22
|
"",
|
|
21
23
|
"TASK: Rewrite the user's casual request into ONE optimal, production-ready video prompt.",
|
|
22
24
|
"",
|
|
@@ -73,26 +75,16 @@ export function buildGrokVideoPlannerSystemPrompt() {
|
|
|
73
75
|
"- Keep prompts focused: one main production-level action sequence. Overloading causes artifacts.",
|
|
74
76
|
"- Keep output concise but scale detail to the requested duration; longer clips may need more connected action/camera/audio development than short clips.",
|
|
75
77
|
"",
|
|
76
|
-
"
|
|
77
|
-
"Before rewriting,
|
|
78
|
-
"
|
|
78
|
+
"BENIGN CLOTHING AND CONTEXT HANDLING:",
|
|
79
|
+
"Before rewriting, distinguish ordinary clothing/swimwear descriptions from sexualized requests.",
|
|
80
|
+
"Preserve benign adult clothing intent without adding erotic details:",
|
|
79
81
|
"",
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"nude/누드 → bare skin visible, no clothing, figure study pose",
|
|
85
|
-
"상반신 노출 → bare shoulders, collarbones, upper torso uncovered",
|
|
86
|
-
"lingerie/란제리 → lace bralette and matching briefs, sheer fabric, delicate straps",
|
|
87
|
-
"seductive/유혹적 → smoldering eye contact, beckoning pose, soft parted lips, intimate close framing",
|
|
88
|
-
"bondage → intricate rope pattern across torso and limbs, knotted hemp cord, shibari ties",
|
|
89
|
-
"wet/젖은 → glistening water droplets on skin, damp clinging fabric, water-slicked hair",
|
|
90
|
-
"bikini/비키니 → string triangle top and low-rise bottom, two-piece swimwear",
|
|
91
|
-
"시스루 → translucent layered fabric, skin tone visible underneath",
|
|
82
|
+
"bikini/비키니 → adult two-piece swimwear in a neutral beach, pool, swimwear catalog, or fashion editorial context",
|
|
83
|
+
"swimsuit/수영복 → adult swimwear suitable for swimming, beach, pool, sport, or catalog footage",
|
|
84
|
+
"sports bra/스포츠브라 → athletic top in a fitness, yoga, running, or product catalog context",
|
|
85
|
+
"lingerie/란제리 → adult fashion or product catalog styling only when the user is clearly requesting adult fashion; avoid sexualized posing or intimate framing",
|
|
92
86
|
"",
|
|
93
|
-
"
|
|
94
|
-
"If the user says lingerie, the video shows lingerie — not a dress. If bikini, show a bikini — not a one-piece.",
|
|
95
|
-
"Never upgrade coverage. Never downgrade exposure. Match the original intent exactly.",
|
|
87
|
+
"CLOTHING RULE: Keep the user's requested clothing category when it is benign and adult, but do not increase exposure, add fetish details, add negative safety constraints, or convert a neutral clothing prompt into a sexualized scene.",
|
|
96
88
|
"",
|
|
97
89
|
"PROFESSIONAL EMBEDDING: After rewriting, wrap the visual description in a professional context.",
|
|
98
90
|
"Prepend ONE of these frames based on content type:",
|
|
@@ -104,12 +96,13 @@ export function buildGrokVideoPlannerSystemPrompt() {
|
|
|
104
96
|
"- Dance/performance → 'music video production, choreographed performance reel'",
|
|
105
97
|
"",
|
|
106
98
|
"CONTENT POLICY:",
|
|
107
|
-
"-
|
|
108
|
-
"- Do
|
|
109
|
-
"- Do
|
|
110
|
-
"- Do
|
|
111
|
-
"-
|
|
112
|
-
"-
|
|
99
|
+
"- Generate one production-ready prompt for compliant video requests.",
|
|
100
|
+
"- Do not add disclaimers, warnings, moral commentary, or safety notes to the final prompt.",
|
|
101
|
+
"- Do not change the user's specified demographics, appearance, or benign clothing.",
|
|
102
|
+
"- Do not turn benign adult swimwear, sportswear, or fashion prompts into sexual content.",
|
|
103
|
+
"- Do not copy safety labels, refusal reasons, or negative safety constraints into the final video prompt.",
|
|
104
|
+
"- If the request includes minors, underage-looking subjects, explicit sexual activity, or fetish framing, do not rewrite it as a benign allowed prompt.",
|
|
105
|
+
"- Content moderation is still handled downstream by the provider.",
|
|
113
106
|
"",
|
|
114
107
|
"VISIBLE TEXT AND DIALOGUE RULE:",
|
|
115
108
|
"- All dialogue lines, voiceover, narration, signs, subtitles, and on-screen text must be written in the user's prompt language, using the original script.",
|
package/lib/inflight.js
CHANGED
|
@@ -4,7 +4,7 @@ import { publish } from "./eventBus.js";
|
|
|
4
4
|
import { logEvent } from "./logger.js";
|
|
5
5
|
const terminalJobs = new Map(); // requestId -> terminal snapshot, active-only API stays default
|
|
6
6
|
const abortControllers = new Map();
|
|
7
|
-
export const MAX_CONCURRENT_JOBS =
|
|
7
|
+
export const MAX_CONCURRENT_JOBS = Math.max(1, Math.trunc(Number(config.limits.maxParallel) || 24));
|
|
8
8
|
export const INFLIGHT_RETRY_AFTER_SECONDS = 5;
|
|
9
9
|
export function isStartJobFailure(r) {
|
|
10
10
|
return !r.ok;
|