ima2-gen 2.0.2 → 2.0.4-preview-260626

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/CHANGELOG.md +1 -0
  2. package/bin/commands/gen.js +7 -5
  3. package/bin/commands/multimode.js +7 -5
  4. package/bin/commands/node.js +2 -1
  5. package/config.js +8 -1
  6. package/docs/API.md +5 -4
  7. package/docs/CLI.md +4 -4
  8. package/docs/FAQ.ko.md +1 -1
  9. package/docs/FAQ.md +1 -1
  10. package/docs/IMAGE_RESOLUTION.md +29 -0
  11. package/docs/migration/runtime-test-inventory.md +11 -1
  12. package/lib/agentCommandParser.js +10 -6
  13. package/lib/agentGenerationPlanner.js +96 -19
  14. package/lib/agentImageVideoGen.js +26 -6
  15. package/lib/agentPlannerModel.js +172 -0
  16. package/lib/agentQuestionResponder.js +9 -3
  17. package/lib/agentQueueStore.js +42 -0
  18. package/lib/agentQueueWorker.js +57 -7
  19. package/lib/agentRuntime.js +96 -12
  20. package/lib/agentSettings.js +9 -6
  21. package/lib/agentToolManifest.js +90 -0
  22. package/lib/agentTypes.js +1 -0
  23. package/lib/agyCli.js +31 -0
  24. package/lib/agyImageAdapter.js +73 -8
  25. package/lib/capabilities.js +7 -5
  26. package/lib/configKeys.js +6 -0
  27. package/lib/geminiApiImageAdapter.js +11 -4
  28. package/lib/generationRequestLog.js +25 -0
  29. package/lib/grokImageAdapter.js +27 -34
  30. package/lib/grokMultimodeAdapter.js +2 -1
  31. package/lib/grokVideoAdapter.js +3 -2
  32. package/lib/grokVideoPlannerPrompt.js +18 -25
  33. package/lib/inflight.js +1 -1
  34. package/lib/multimodeHelpers.js +3 -2
  35. package/lib/oauthProxy/generators.js +6 -5
  36. package/lib/oauthProxy/prompts.js +55 -16
  37. package/lib/oauthProxy/streams.js +1 -1
  38. package/lib/promptSafetyPolicy.js +1 -1
  39. package/lib/responsesDoctor.js +3 -3
  40. package/lib/responsesFallback.js +27 -14
  41. package/lib/responsesImageAdapter.js +11 -7
  42. package/node_modules/qs/CHANGELOG.md +10 -0
  43. package/node_modules/qs/README.md +1 -1
  44. package/node_modules/qs/dist/qs.js +15 -15
  45. package/node_modules/qs/eslint.config.mjs +1 -0
  46. package/node_modules/qs/lib/parse.js +52 -22
  47. package/node_modules/qs/lib/stringify.js +11 -4
  48. package/node_modules/qs/package.json +2 -2
  49. package/node_modules/qs/test/parse.js +49 -0
  50. package/node_modules/qs/test/stringify.js +129 -0
  51. package/package.json +1 -1
  52. package/routes/agent.js +19 -2
  53. package/routes/agy.js +5 -1
  54. package/routes/generate.js +12 -1
  55. package/routes/generationRequestLog.js +16 -0
  56. package/routes/index.js +2 -0
  57. package/routes/keys.js +22 -2
  58. package/routes/multimode.js +1 -1
  59. package/routes/nodes.js +5 -5
  60. package/ui/dist/.vite/manifest.json +22 -12
  61. package/ui/dist/assets/AgentWorkspace-CMP9XZrL.js +3 -0
  62. package/ui/dist/assets/{CardNewsWorkspace-Dav3K5CT.js → CardNewsWorkspace-DtOwgYzf.js} +1 -1
  63. package/ui/dist/assets/GenerationRequestLogPanel-CB1i5dWd.js +1 -0
  64. package/ui/dist/assets/{NodeCanvas-C4ifFzB1.js → NodeCanvas-DwX2jRHf.js} +3 -3
  65. package/ui/dist/assets/{PromptBuilderPanel-CEcyU9PL.js → PromptBuilderPanel-B3VhUc5u.js} +2 -2
  66. package/ui/dist/assets/{PromptImportDialog-CgQ94Gth.js → PromptImportDialog-4FQd2YVS.js} +2 -2
  67. package/ui/dist/assets/{PromptImportDiscoverySection-CuzyzbNI.js → PromptImportDiscoverySection-CTNr6ft3.js} +1 -1
  68. package/ui/dist/assets/PromptImportFolderSection-zCJ1lBH8.js +1 -0
  69. package/ui/dist/assets/{PromptLibraryPanel-BOe18we8.js → PromptLibraryPanel-B784T1KR.js} +2 -2
  70. package/ui/dist/assets/SettingsWorkspace-BpM6lWH-.js +1 -0
  71. package/ui/dist/assets/index-2oG6in1i.css +1 -0
  72. package/ui/dist/assets/index-BzjrmmIG.js +23 -0
  73. package/ui/dist/assets/index-CwAzer33.js +4 -0
  74. package/ui/dist/index.html +2 -2
  75. package/ui/dist/assets/AgentWorkspace-Dth6YijN.js +0 -3
  76. package/ui/dist/assets/PromptImportFolderSection-DHLGlO6l.js +0 -1
  77. package/ui/dist/assets/SettingsWorkspace-Cdgnm4Wa.js +0 -1
  78. package/ui/dist/assets/index-C5PSahkr.js +0 -1
  79. package/ui/dist/assets/index-Dn2AhL6d.css +0 -1
  80. package/ui/dist/assets/index-Tjqx6wUV.js +0 -23
@@ -16,6 +16,7 @@ import { parseVideoParams } from "./agentGenerationPlanner.js";
16
16
  import { appendAgentTurn, getAgentImages, getAgentSession, importAgentImage, } from "./agentStore.js";
17
17
  import { errInfo } from "./errInfo.js";
18
18
  import { forceImagePrompt, isTextOnlyResult, textOnlyError, notFound } from "./agentRuntime.js";
19
+ const AGENT_GROK_PLANNER_MODEL = "grok-4.3";
19
20
  export async function generateAgentImageWithRetry(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
20
21
  let lastError = null;
21
22
  for (let attempt = 0; attempt < 2; attempt++) {
@@ -43,9 +44,10 @@ export async function generateAgentImageWithRetry(ctx, sessionId, prompt, manife
43
44
  }
44
45
  async function generateAgentImage(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
45
46
  const requestId = options.requestId ?? `agent_${ulid()}`;
47
+ const grokPlannerModel = isAgentGrokPlannerModel(options.model) ? options.model : undefined;
46
48
  const providerOptions = resolveProviderOptions(ctx, {
47
49
  provider: options.provider ?? "oauth",
48
- rawModel: options.model,
50
+ rawModel: grokPlannerModel ? undefined : options.model,
49
51
  rawReasoningEffort: options.reasoningEffort,
50
52
  rawSize: options.size ?? "1024x1024",
51
53
  rawWebSearchEnabled: webSearchEnabled,
@@ -72,7 +74,8 @@ async function generateAgentImage(ctx, sessionId, prompt, manifest, webSearchEna
72
74
  size: providerOptions.size,
73
75
  requestId,
74
76
  signal: options.signal ?? undefined,
75
- references: await loadAgentCurrentImageReferences(ctx, sessionId),
77
+ references: await loadAgentCurrentImageReferences(ctx, sessionId, options.sourceImagePolicy ?? "none"),
78
+ plannerModel: grokPlannerModel,
76
79
  })
77
80
  : await generateViaResponses(activeProvider, `${manifest}\n\nUser request:\n${prompt}`, options.quality ?? "medium", providerOptions.size, options.moderation ?? "low", [], requestId, "auto", ctx, {
78
81
  model: providerOptions.model,
@@ -90,16 +93,23 @@ async function generateAgentImage(ctx, sessionId, prompt, manifest, webSearchEna
90
93
  const responseText = "text" in response && typeof response.text === "string" ? response.text : null;
91
94
  return { image, webSearchCalls: response.webSearchCalls || 0, text: responseText, provider: activeProvider };
92
95
  }
93
- async function loadAgentCurrentImageReferences(ctx, sessionId) {
96
+ async function loadAgentCurrentImageReferences(ctx, sessionId, policy) {
97
+ if (policy === "none") {
98
+ logEvent("agent", "grok_ref_policy", { sessionId, policy, attached: false });
99
+ return [];
100
+ }
94
101
  const session = getAgentSession(sessionId);
95
102
  const currentImage = session?.lastImageId
96
103
  ? getAgentImages(sessionId).find((image) => image.id === session.lastImageId)
97
104
  : null;
98
- if (!currentImage?.filename)
105
+ if (!currentImage?.filename) {
106
+ logEvent("agent", "grok_ref_policy", { sessionId, policy, attached: false });
99
107
  return [];
108
+ }
100
109
  try {
101
110
  const b64 = (await readFile(join(ctx.config.storage.generatedDir, currentImage.filename))).toString("base64");
102
111
  const mime = detectImageMimeFromB64(b64);
112
+ logEvent("agent", "grok_ref_policy", { sessionId, policy, attached: true, filename: currentImage.filename });
103
113
  return [{ b64, declaredMime: mime, detectedMime: mime }];
104
114
  }
105
115
  catch (error) {
@@ -179,7 +189,13 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
179
189
  catch { /* fallback to T2V */ }
180
190
  }
181
191
  }
182
- const videoParams = parseVideoParams(prompt);
192
+ // LLM-planned params win; the prompt regex remains the fallback extractor.
193
+ const parsedParams = parseVideoParams(prompt);
194
+ const videoParams = {
195
+ duration: options.videoParams?.duration ?? parsedParams.duration,
196
+ resolution: options.videoParams?.resolution ?? parsedParams.resolution,
197
+ aspectRatio: options.videoParams?.aspectRatio ?? parsedParams.aspectRatio,
198
+ };
183
199
  const result = await generateVideoViaGrok(prompt, ctx, {
184
200
  model: "grok-imagine-video",
185
201
  mode,
@@ -189,6 +205,7 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
189
205
  aspectRatio: (videoParams.aspectRatio ?? "auto"),
190
206
  requestId,
191
207
  signal: options.signal ?? undefined,
208
+ plannerModel: isAgentGrokPlannerModel(options.model) ? options.model : undefined,
192
209
  });
193
210
  const video = await persistAgentVideo(ctx, sessionId, prompt, requestId, result);
194
211
  const finishedAt = Date.now();
@@ -215,12 +232,15 @@ export async function runAgentVideoGeneration(ctx, sessionId, prompt, options =
215
232
  const assistantTurn = appendAgentTurn({
216
233
  sessionId,
217
234
  role: "assistant",
218
- text: `Generated 1 video artifact. ${result.revisedPrompt}`,
235
+ text: options.assistantText?.trim() || `Generated 1 video artifact. ${result.revisedPrompt}`,
219
236
  imageIds: [video.id],
220
237
  status: "complete",
221
238
  });
222
239
  return { assistantTurn, imageIds: [video.id], webFindingIds: [] };
223
240
  }
241
+ function isAgentGrokPlannerModel(model) {
242
+ return model === AGENT_GROK_PLANNER_MODEL;
243
+ }
224
244
  async function persistAgentVideo(ctx, sessionId, prompt, requestId, result) {
225
245
  await mkdir(ctx.config.storage.generatedDir, { recursive: true });
226
246
  const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
@@ -0,0 +1,172 @@
1
+ import { normalizeAgentGenerationPlan } from "./agentGenerationPlanner.js";
2
+ import { readResponsesTextPayload } from "./agentQuestionResponder.js";
3
+ import { formatToolManifestForPrompt } from "./agentToolManifest.js";
4
+ import { getAgentSession } from "./agentStore.js";
5
+ import { errInfo } from "./errInfo.js";
6
+ import { logEvent } from "./logger.js";
7
+ import { getGrokEndpoint, getPlannerConfig } from "./grokImageCore.js";
8
+ import { waitForOAuthReady } from "./oauthProxy/runtime.js";
9
+ import { requireRuntimeContext } from "./runtimeContext.js";
10
+ function buildPlannerDeveloperPrompt(hasSourceImage, imageCount) {
11
+ return [
12
+ "You are the generation planner for the ima2 Agent. Decide how to fulfill the user's request using the available tools.",
13
+ "",
14
+ "Available tools (name, purpose, parameter schema):",
15
+ formatToolManifestForPrompt(),
16
+ "",
17
+ "Tool execution contract:",
18
+ "- You do not call provider image/video APIs directly. You choose a plan; the ima2 runtime executes the corresponding ima2.* tools.",
19
+ "- The session model is the planner/LLM model, not an image or video model. For example, grok-4.3 means Grok planner/provider routing; image generation still uses ima2.generate_image with the configured Grok image backend.",
20
+ "- For image creation/edit requests choose mode single or fanout, which maps to ima2.get_image_context followed by ima2.generate_image.",
21
+ "- For image creation/edit requests, also choose sourceImagePolicy: none for a fresh image, current to use the session's current image as an edit/reference input, or auto only when genuinely ambiguous.",
22
+ "- For video creation requests choose mode video, which maps to ima2.generate_video. Never put video model names in prompts.",
23
+ "- For failure questions choose mode errors, which maps to ima2.get_generation_errors.",
24
+ "",
25
+ "Session context:",
26
+ `- Images in session: ${imageCount}`,
27
+ `- Last image available as image-to-video source: ${hasSourceImage ? "yes" : "no"}`,
28
+ "",
29
+ "Decide ONE plan and respond with ONLY a JSON object (no prose, no code fences):",
30
+ '{"mode":"single|fanout|video|question|errors","prompts":["..."],"plannedVariants":1,"plannedParallelism":1,"sourceImagePolicy":"none|current|auto","videoParams":{"duration":5,"resolution":"480p","aspectRatio":"auto"},"assistantText":"...","reason":"short reason"}',
31
+ "",
32
+ "Rules:",
33
+ "- You are a conversational assistant first. Generate media ONLY when the user clearly asks you to create or edit an image/video. Everything else (questions, chat, greetings, feedback, follow-ups) is mode question.",
34
+ "- mode single: one image. prompts has exactly 1 entry (the generation prompt, user language preserved).",
35
+ "- mode fanout: multiple image variants. prompts has one entry per variant; respect any count the user asked for.",
36
+ "- sourceImagePolicy for single/fanout: use none for new/fresh/separate/from-scratch requests, including '새로', '별도', 'i2i 말고', '새로운 방식', 'new image', 'from scratch', 'without reference'.",
37
+ "- sourceImagePolicy for single/fanout: use current only when the user explicitly asks to use/edit/modify/transform/reference the current image, including '이 이미지', '현재 이미지', '방금 그거', '참조', 'reference', 'i2i', 'image-to-image', '유지해서'.",
38
+ "- sourceImagePolicy for plain image requests with no explicit reference wording is none.",
39
+ "- mode video: one video via ima2.generate_video. Choose it only when the user asks to CREATE a video. prompts has exactly 1 entry. Extract duration (1-15 s), resolution (480p|720p), aspectRatio (auto|1:1|16:9|9:16|4:3|3:4|3:2|2:3) from the request into videoParams; omit fields the user did not specify.",
40
+ "- mode question: the user is NOT requesting generation — a question (capabilities, how-to, status), small talk, a greeting, or feedback — e.g. '영상 생성가능하니?', 'can you make videos?', '고마워'. prompts must be []. Write the full answer in assistantText. Mentioning a media word like 'video' or '영상' inside a question does NOT make it a generation request.",
41
+ "- mode errors: the user is asking why a previous generation failed or about recent errors. prompts must be [].",
42
+ "- assistantText: REQUIRED for every mode, written in the user's language. For question/errors it is the full reply. For single/fanout/video it is a short natural chat reply telling the user what you are creating (1-2 sentences, no markdown headings).",
43
+ "- Preserve the user's prompt content; do not censor, embellish, or translate it.",
44
+ "- reason: one short sentence explaining the decision.",
45
+ ].join("\n");
46
+ }
47
+ export async function requestAgentPlanFromModel(ctxRaw, input) {
48
+ const ctx = requireRuntimeContext(ctxRaw);
49
+ const plannerCfg = ctx.config.agentPlanner;
50
+ if (!plannerCfg?.enabled)
51
+ return null;
52
+ if (input.settings.provider === "agy")
53
+ return null;
54
+ const timeoutMs = plannerCfg.timeoutMs ?? 30_000;
55
+ const controller = new AbortController();
56
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
57
+ const signal = input.signal ? AbortSignal.any([controller.signal, input.signal]) : controller.signal;
58
+ try {
59
+ const session = getAgentSession(input.sessionId);
60
+ const developerPrompt = buildPlannerDeveloperPrompt(Boolean(session?.lastImageId), session?.imageCount ?? 0);
61
+ const rawText = input.settings.provider === "grok"
62
+ ? await requestGrokPlan(ctx, developerPrompt, input.prompt, signal)
63
+ : await requestResponsesPlan(ctx, developerPrompt, input.prompt, input.settings, signal);
64
+ const parsed = extractJsonObject(rawText);
65
+ if (!parsed) {
66
+ logEvent("agent_planner", "parse_failed", { requestId: input.requestId, provider: input.settings.provider, chars: rawText.length });
67
+ return null;
68
+ }
69
+ const plan = normalizeAgentGenerationPlan(input.prompt, { ...parsed, source: "llm-planner" }, input.settings);
70
+ logEvent("agent_planner", "planned", {
71
+ requestId: input.requestId,
72
+ provider: input.settings.provider,
73
+ mode: plan.mode,
74
+ plannedVariants: plan.plannedVariants,
75
+ source: plan.source,
76
+ });
77
+ return plan;
78
+ }
79
+ catch (error) {
80
+ const err = errInfo(error);
81
+ logEvent("agent_planner", "fallback", {
82
+ requestId: input.requestId,
83
+ provider: input.settings.provider,
84
+ code: err.name === "AbortError" ? "AGENT_PLANNER_TIMEOUT" : err.code,
85
+ message: err.message,
86
+ });
87
+ return null;
88
+ }
89
+ finally {
90
+ clearTimeout(timer);
91
+ }
92
+ }
93
+ async function requestGrokPlan(ctx, developerPrompt, userPrompt, signal) {
94
+ const { url, headers } = getGrokEndpoint(ctx, "/v1/chat/completions");
95
+ const planner = getPlannerConfig(ctx);
96
+ const res = await fetch(url, {
97
+ method: "POST",
98
+ headers,
99
+ signal,
100
+ body: JSON.stringify({
101
+ model: planner.model,
102
+ stream: false,
103
+ messages: [
104
+ { role: "system", content: developerPrompt },
105
+ { role: "user", content: userPrompt },
106
+ ],
107
+ }),
108
+ });
109
+ if (!res.ok)
110
+ throw plannerHttpError("grok", res.status);
111
+ const body = await res.json();
112
+ return typeof body.choices?.[0]?.message?.content === "string" ? body.choices[0].message.content : "";
113
+ }
114
+ async function requestResponsesPlan(ctx, developerPrompt, userPrompt, settings, signal) {
115
+ let url;
116
+ let headers;
117
+ if (settings.provider === "api") {
118
+ if (!ctx.apiKey)
119
+ throw plannerError("API key is required for Agent planner", "API_KEY_REQUIRED", 401);
120
+ url = "https://api.openai.com/v1/responses";
121
+ headers = { "Content-Type": "application/json", Accept: "text/event-stream", Authorization: `Bearer ${ctx.apiKey}` };
122
+ }
123
+ else {
124
+ await waitForOAuthReady(ctx);
125
+ url = `${ctx.oauthUrl}/v1/responses`;
126
+ headers = { "Content-Type": "application/json", Accept: "text/event-stream" };
127
+ }
128
+ // stream:true is required — the bundled OAuth proxy returns an empty
129
+ // `output` array for non-streaming Responses calls, which used to make the
130
+ // planner silently fall back to the regex-derived image plan.
131
+ const res = await fetch(url, {
132
+ method: "POST",
133
+ headers,
134
+ signal,
135
+ body: JSON.stringify({
136
+ model: settings.model,
137
+ input: [
138
+ { role: "developer", content: developerPrompt },
139
+ { role: "user", content: userPrompt },
140
+ ],
141
+ reasoning: { effort: "low" },
142
+ stream: true,
143
+ }),
144
+ });
145
+ if (!res.ok)
146
+ throw plannerHttpError(settings.provider, res.status);
147
+ const payload = await readResponsesTextPayload(res);
148
+ return payload.text;
149
+ }
150
+ export function extractJsonObject(raw) {
151
+ const text = raw.replace(/```(?:json)?/gi, "").trim();
152
+ const start = text.indexOf("{");
153
+ const end = text.lastIndexOf("}");
154
+ if (start === -1 || end === -1 || end <= start)
155
+ return null;
156
+ try {
157
+ const parsed = JSON.parse(text.slice(start, end + 1));
158
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
159
+ }
160
+ catch {
161
+ return null;
162
+ }
163
+ }
164
+ function plannerHttpError(provider, status) {
165
+ return plannerError(`Agent planner upstream rejected the request (${provider})`, "AGENT_PLANNER_UPSTREAM_FAILED", status >= 400 && status < 600 ? status : 502);
166
+ }
167
+ function plannerError(message, code, status) {
168
+ const err = new Error(message);
169
+ err.code = code;
170
+ err.status = status;
171
+ return err;
172
+ }
@@ -40,9 +40,7 @@ export async function requestAgentQuestionAnswer(ctxRaw, question, options = {})
40
40
  });
41
41
  if (!res.ok)
42
42
  throw await questionHttpError(res, options.provider);
43
- const parsed = res.headers.get("content-type")?.includes("text/event-stream")
44
- ? await readTextStream(res)
45
- : readTextBody(await res.json());
43
+ const parsed = await readResponsesTextPayload(res);
46
44
  if (!parsed.text.trim())
47
45
  throw questionError("Agent question returned an empty response", "AGENT_QUESTION_EMPTY", 502);
48
46
  return { text: parsed.text.trim().slice(0, 4_000), usage: parsed.usage };
@@ -94,6 +92,14 @@ async function questionHttpError(res, provider) {
94
92
  throw errInfo(error).raw;
95
93
  }
96
94
  }
95
+ // Shared with the agent planner: the bundled OAuth proxy returns an empty
96
+ // `output` array for stream:false Responses calls, so any text consumer must
97
+ // be able to read the SSE stream instead of relying on the JSON body.
98
+ export async function readResponsesTextPayload(res) {
99
+ return res.headers.get("content-type")?.includes("text/event-stream")
100
+ ? readTextStream(res)
101
+ : readTextBody(await res.json());
102
+ }
97
103
  async function readTextStream(res) {
98
104
  try {
99
105
  const reader = res.body?.getReader();
@@ -143,6 +143,48 @@ export function cancelAgentQueueItem(id) {
143
143
  `).run(Date.now(), id);
144
144
  return res.changes > 0;
145
145
  }
146
+ export function updateAgentQueueItemPlan(id, plan) {
147
+ const res = getDb().prepare(`
148
+ UPDATE agent_queue_items
149
+ SET tool_plan = ?
150
+ WHERE id = ?
151
+ `).run(JSON.stringify(plan), id);
152
+ return res.changes > 0;
153
+ }
154
+ export function getAgentGenerationErrors(sessionId, limit = 10) {
155
+ const cap = Math.max(1, Math.min(20, Math.round(limit)));
156
+ const queueRows = getDb().prepare(`
157
+ SELECT error_code AS code, error_message AS message, prompt, finished_at AS at
158
+ FROM agent_queue_items
159
+ WHERE session_id = ? AND status = 'failed'
160
+ ORDER BY finished_at DESC
161
+ LIMIT ?
162
+ `).all(sessionId, cap);
163
+ const turnRows = getDb().prepare(`
164
+ SELECT text, created_at AS at
165
+ FROM agent_turns
166
+ WHERE session_id = ? AND status = 'error'
167
+ ORDER BY created_at DESC
168
+ LIMIT ?
169
+ `).all(sessionId, cap);
170
+ const records = [
171
+ ...queueRows.map((row) => ({
172
+ scope: "queue",
173
+ code: row.code,
174
+ message: row.message ?? "Generation failed without a recorded message.",
175
+ prompt: row.prompt,
176
+ at: row.at ?? 0,
177
+ })),
178
+ ...turnRows.map((row) => ({
179
+ scope: "turn",
180
+ code: null,
181
+ message: row.text,
182
+ prompt: null,
183
+ at: row.at,
184
+ })),
185
+ ];
186
+ return records.sort((a, b) => b.at - a.at).slice(0, cap);
187
+ }
146
188
  export function retryAgentQueueItem(id) {
147
189
  const res = getDb().prepare(`
148
190
  UPDATE agent_queue_items
@@ -1,5 +1,8 @@
1
- import { claimNextAgentQueueItem, completeAgentQueueItem, failAgentQueueItem, getAgentQueueItem, } from "./agentQueueStore.js";
2
- import { runAgentGenerationPlan } from "./agentRuntime.js";
1
+ import { config } from "../config.js";
2
+ import { claimNextAgentQueueItem, completeAgentQueueItem, failAgentQueueItem, getAgentQueueItem, updateAgentQueueItemPlan, } from "./agentQueueStore.js";
3
+ import { requestAgentPlanFromModel } from "./agentPlannerModel.js";
4
+ import { hasAgentErrorTurnRecorded, runAgentGenerationPlan } from "./agentRuntime.js";
5
+ import { appendAgentTurn } from "./agentStore.js";
3
6
  import { errInfo } from "./errInfo.js";
4
7
  import { finishJob, setJobPhase, startJob } from "./inflight.js";
5
8
  import { logEvent } from "./logger.js";
@@ -42,10 +45,42 @@ export async function tickAgentQueueWorker(ctx) {
42
45
  ticking = false;
43
46
  }
44
47
  }
48
+ // LLM planning runs only for plans the regex deriver produced automatically.
49
+ // Slash commands, manual settings, agy sessions, and already-LLM-planned
50
+ // retries keep their stored plan (audit decision F4/F5).
51
+ function isLlmPlanningEligible(item) {
52
+ return Boolean(config.agentPlanner.enabled &&
53
+ (item.plan.source === "auto-default" || item.plan.source === "auto-request") &&
54
+ item.plan.command == null &&
55
+ item.options.generationStrategy === "auto" &&
56
+ item.options.provider !== "agy");
57
+ }
58
+ async function resolveRuntimePlan(ctx, item) {
59
+ if (!isLlmPlanningEligible(item))
60
+ return item.plan;
61
+ try {
62
+ const plan = await requestAgentPlanFromModel(ctx, {
63
+ sessionId: item.sessionId,
64
+ prompt: item.prompt,
65
+ settings: item.options,
66
+ requestId: item.requestId,
67
+ });
68
+ if (!plan)
69
+ return item.plan;
70
+ updateAgentQueueItemPlan(item.id, plan);
71
+ return plan;
72
+ }
73
+ catch (error) {
74
+ const err = errInfo(error);
75
+ logEvent("agent_queue", "planner_fallback", { itemId: item.id, code: err.code, message: err.message });
76
+ return item.plan;
77
+ }
78
+ }
45
79
  async function runClaimedQueueItem(ctx, itemId) {
46
80
  const item = getAgentQueueItem(itemId);
47
81
  if (!item)
48
82
  return;
83
+ const plan = await resolveRuntimePlan(ctx, item);
49
84
  startJob({
50
85
  requestId: item.requestId,
51
86
  kind: "agent_queue",
@@ -53,19 +88,19 @@ async function runClaimedQueueItem(ctx, itemId) {
53
88
  meta: {
54
89
  sessionId: item.sessionId,
55
90
  queueItemId: item.id,
56
- variants: item.plan.plannedVariants,
57
- parallelism: item.plan.plannedParallelism,
58
- requestedVariants: item.plan.requestedVariants,
91
+ variants: plan.plannedVariants,
92
+ parallelism: plan.plannedParallelism,
93
+ requestedVariants: plan.requestedVariants,
59
94
  },
60
95
  });
61
96
  try {
62
97
  logEvent("agent_queue", "start", { itemId: item.id, sessionId: item.sessionId });
63
98
  setJobPhase(item.requestId, "streaming");
64
- const result = await runAgentGenerationPlan(ctx, item.sessionId, item.prompt, item.plan, {
99
+ const result = await runAgentGenerationPlan(ctx, item.sessionId, item.prompt, plan, {
65
100
  ...item.options,
66
101
  requestId: item.requestId,
67
102
  webSearchEnabled: item.options.webSearchEnabled,
68
- parallelism: item.plan.plannedParallelism,
103
+ parallelism: plan.plannedParallelism,
69
104
  }, {
70
105
  appendUserTurn: false,
71
106
  });
@@ -79,6 +114,21 @@ async function runClaimedQueueItem(ctx, itemId) {
79
114
  catch (error) {
80
115
  const err = errInfo(error);
81
116
  failAgentQueueItem(item.id, { code: err.code, message: err.message });
117
+ // The chat pane must never fail silently — surface the failure as an
118
+ // assistant error turn unless the runtime already recorded one.
119
+ if (!hasAgentErrorTurnRecorded(error)) {
120
+ try {
121
+ appendAgentTurn({
122
+ sessionId: item.sessionId,
123
+ role: "assistant",
124
+ text: err.code ? `${err.message} [${err.code}]` : err.message,
125
+ status: "error",
126
+ });
127
+ }
128
+ catch (turnError) {
129
+ logEvent("agent_queue", "error_turn_failed", { itemId: item.id, message: errInfo(turnError).message });
130
+ }
131
+ }
82
132
  finishJob(item.requestId, {
83
133
  status: "failed",
84
134
  errorCode: err.code,
@@ -1,8 +1,11 @@
1
1
  import { ulid } from "ulid";
2
+ import { config } from "../config.js";
2
3
  import { generateAgentImageWithRetry } from "./agentImageVideoGen.js";
3
4
  import { runAgentVideoGeneration } from "./agentImageVideoGen.js";
4
5
  import { appendAgentTurn, buildImageContextManifest, getAgentSession, recordAgentWebFinding, restartAgentRuntimeSession, } from "./agentStore.js";
5
6
  import { AGENT_ALLOWED_TOOLS, } from "./agentTypes.js";
7
+ import { getAgentGenerationErrors } from "./agentQueueStore.js";
8
+ import { AGENT_TOOL_MANIFEST } from "./agentToolManifest.js";
6
9
  import { errInfo } from "./errInfo.js";
7
10
  export function assertAgentAllowedTools(tools) {
8
11
  const allowed = new Set(AGENT_ALLOWED_TOOLS);
@@ -16,7 +19,7 @@ export function assertAgentAllowedTools(tools) {
16
19
  }
17
20
  }
18
21
  export function agentAllowedToolPayload() {
19
- return { tools: [...AGENT_ALLOWED_TOOLS] };
22
+ return { tools: [...AGENT_ALLOWED_TOOLS], manifest: [...AGENT_TOOL_MANIFEST] };
20
23
  }
21
24
  export async function runAgentTurn(ctx, sessionId, prompt, options = {}) {
22
25
  return runAgentGenerationPlan(ctx, sessionId, prompt, {
@@ -29,6 +32,7 @@ export async function runAgentTurn(ctx, sessionId, prompt, options = {}) {
29
32
  reason: "Direct turn endpoint defaults to one image.",
30
33
  command: null,
31
34
  assistantText: null,
35
+ sourceImagePolicy: "none",
32
36
  }, options, { appendUserTurn: true });
33
37
  }
34
38
  export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, options = {}, behavior = {}) {
@@ -38,7 +42,7 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
38
42
  const webSearchEnabled = options.provider === "agy" ? false : options.provider === "grok" ? true : options.webSearchEnabled ?? session.webSearchEnabled;
39
43
  const enabledTools = webSearchEnabled
40
44
  ? [...AGENT_ALLOWED_TOOLS]
41
- : ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video"];
45
+ : ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video", "ima2.get_generation_errors"];
42
46
  assertAgentAllowedTools(enabledTools);
43
47
  if (behavior.appendUserTurn !== false) {
44
48
  appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
@@ -54,9 +58,15 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
54
58
  });
55
59
  return { assistantTurn, imageIds: [], webFindingIds: [] };
56
60
  }
61
+ if (plan.mode === "errors") {
62
+ return runAgentErrorLookup(sessionId, plan);
63
+ }
64
+ const preludeSent = appendPlannerPreludeTurn(sessionId, plan);
57
65
  if (plan.mode === "video") {
58
- return runAgentVideoGeneration(ctx, sessionId, prompt, {
66
+ return runAgentVideoGeneration(ctx, sessionId, plan.prompts[0] ?? prompt, {
59
67
  ...options,
68
+ videoParams: plan.videoParams ?? options.videoParams ?? null,
69
+ assistantText: preludeSent ? null : plan.assistantText,
60
70
  requestId: options.requestId ?? `agent_video_${ulid()}`,
61
71
  skipUserTurn: true,
62
72
  });
@@ -88,6 +98,7 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
88
98
  const result = await runGeneratorWithRuntimeRecovery(ctx, sessionId, generationPrompt, manifest, webSearchEnabled, {
89
99
  ...options,
90
100
  requestId,
101
+ sourceImagePolicy: plan.sourceImagePolicy ?? "none",
91
102
  });
92
103
  const findingIds = recordSearchFindings(sessionId, generationPrompt, result.webSearchCalls, result.provider ?? "oauth");
93
104
  const finishedAt = Date.now();
@@ -142,20 +153,84 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
142
153
  const assistantTurn = appendAgentTurn({
143
154
  sessionId,
144
155
  role: "assistant",
145
- text: formatAgentAssistantText(plan, imageIds.length, responseTexts),
156
+ text: formatAgentAssistantText(plan, prompt, imageIds.length, responseTexts, preludeSent),
146
157
  imageIds,
147
158
  webFindingIds: findingIds,
148
159
  status: "complete",
149
160
  });
150
161
  return { assistantTurn, imageIds, webFindingIds: findingIds };
151
162
  }
152
- function formatAgentAssistantText(plan, imageCount, responseTexts) {
153
- const countText = imageCount === 1 ? "Generated 1 image artifact." : `Generated ${imageCount} image artifacts.`;
154
- const modeText = plan.mode === "fanout"
155
- ? `Fanout used ${plan.plannedParallelism} concurrent tool call${plan.plannedParallelism === 1 ? "" : "s"}.`
156
- : "Single-image plan completed.";
157
- const modelText = responseTexts.length > 0 ? `${responseTexts.join("\n\n")}\n\n` : "";
158
- return `${modelText}${countText} ${modeText} ${plan.reason}`.trim();
163
+ function appendPlannerPreludeTurn(sessionId, plan) {
164
+ const text = plan.assistantText?.trim();
165
+ if (!text)
166
+ return false;
167
+ appendAgentTurn({
168
+ sessionId,
169
+ role: "assistant",
170
+ text,
171
+ imageIds: [],
172
+ webFindingIds: [],
173
+ status: "complete",
174
+ });
175
+ return true;
176
+ }
177
+ function runAgentErrorLookup(sessionId, plan) {
178
+ const startedAt = Date.now();
179
+ const errors = getAgentGenerationErrors(sessionId, 10);
180
+ appendAgentTurn({
181
+ sessionId,
182
+ role: "tool",
183
+ text: "ima2.get_generation_errors",
184
+ status: "complete",
185
+ raw: {
186
+ toolCalls: [{
187
+ id: `tc_errors_${ulid()}`,
188
+ name: "ima2.get_generation_errors",
189
+ status: "complete",
190
+ startedAt,
191
+ finishedAt: Date.now(),
192
+ durationMs: Date.now() - startedAt,
193
+ outputSummary: errors.length > 0
194
+ ? `Found ${errors.length} recent generation error${errors.length === 1 ? "" : "s"}.`
195
+ : "No recent generation errors recorded for this session.",
196
+ }],
197
+ },
198
+ });
199
+ const assistantTurn = appendAgentTurn({
200
+ sessionId,
201
+ role: "assistant",
202
+ text: plan.assistantText?.trim() || formatGenerationErrors(errors),
203
+ imageIds: [],
204
+ webFindingIds: [],
205
+ status: "complete",
206
+ });
207
+ return { assistantTurn, imageIds: [], webFindingIds: [] };
208
+ }
209
+ function formatGenerationErrors(errors) {
210
+ if (errors.length === 0)
211
+ return "No generation errors are recorded for this session.";
212
+ const lines = errors.map((error, index) => {
213
+ const when = new Date(error.at).toISOString();
214
+ const code = error.code ? ` [${error.code}]` : "";
215
+ const promptPart = error.prompt ? ` (prompt: ${error.prompt.slice(0, 80)})` : "";
216
+ return `${index + 1}. ${when}${code} ${error.message}${promptPart}`;
217
+ });
218
+ return `Recent generation errors (most recent first):\n${lines.join("\n")}`;
219
+ }
220
+ function formatAgentAssistantText(plan, prompt, imageCount, responseTexts, omitPlannerText = false) {
221
+ // Behave like a normal chat agent: prefer the planner's natural-language
222
+ // reply, then any text the image model returned. The mechanical summary is
223
+ // only the fallback when neither produced prose.
224
+ const plannerText = omitPlannerText ? "" : (plan.assistantText?.trim() ?? "");
225
+ const modelText = responseTexts.join("\n\n").trim();
226
+ const prose = [plannerText, modelText].filter(Boolean).join("\n\n");
227
+ if (prose)
228
+ return prose;
229
+ const languageProbe = [prompt, ...plan.prompts].find((item) => item.trim().length > 0) ?? "";
230
+ const isKorean = /[\uAC00-\uD7AF]/u.test(languageProbe);
231
+ if (isKorean)
232
+ return imageCount === 1 ? "이미지 생성이 완료됐어요." : `이미지 ${imageCount}장을 생성했어요.`;
233
+ return imageCount === 1 ? "Done - I generated the image." : `Done - I generated ${imageCount} images.`;
159
234
  }
160
235
  async function runGeneratorWithRuntimeRecovery(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
161
236
  try {
@@ -167,9 +242,18 @@ async function runGeneratorWithRuntimeRecovery(ctx, sessionId, prompt, manifest,
167
242
  restartAgentRuntimeSession(sessionId, err.code || err.message);
168
243
  }
169
244
  appendAgentTurn({ sessionId, role: "assistant", text: err.message, status: "error" });
245
+ markAgentErrorTurnRecorded(error);
170
246
  throw error;
171
247
  }
172
248
  }
249
+ export function markAgentErrorTurnRecorded(error) {
250
+ if (error && typeof error === "object") {
251
+ error.agentErrorTurnRecorded = true;
252
+ }
253
+ }
254
+ export function hasAgentErrorTurnRecorded(error) {
255
+ return Boolean(error && typeof error === "object" && error.agentErrorTurnRecorded);
256
+ }
173
257
  export function isRuntimeRestartableError(error) {
174
258
  const err = errInfo(error);
175
259
  const code = err.code || "";
@@ -231,7 +315,7 @@ function cleanParallelism(value) {
231
315
  const numeric = typeof value === "number" ? value : Number(value);
232
316
  if (!Number.isFinite(numeric))
233
317
  return 2;
234
- return Math.max(1, Math.min(8, Math.round(numeric)));
318
+ return Math.max(1, Math.min(config.limits.maxParallel, Math.round(numeric)));
235
319
  }
236
320
  export function notFound(sessionId) {
237
321
  const err = new Error(`Agent session not found: ${sessionId}`);
@@ -1,9 +1,12 @@
1
+ import { config } from "../config.js";
1
2
  const PROVIDERS = new Set(["oauth", "api", "grok", "grok-api", "agy", "gemini-api"]);
2
3
  const QUALITIES = new Set(["low", "medium", "high"]);
3
4
  const FORMATS = new Set(["png", "jpeg", "webp"]);
4
5
  const MODERATIONS = new Set(["auto", "low"]);
5
- const REASONING_EFFORTS = new Set(["low", "medium", "high", "xhigh"]);
6
+ const REASONING_EFFORTS = new Set(["none", "low", "medium", "high", "xhigh"]);
6
7
  const GENERATION_STRATEGIES = new Set(["auto", "manual"]);
8
+ const MAX_AGENT_VARIANTS = Math.max(1, Math.trunc(config.limits.maxGeneratedImages));
9
+ const MAX_AGENT_PARALLELISM = Math.max(1, Math.trunc(config.limits.maxParallel));
7
10
  export const DEFAULT_AGENT_GENERATION_SETTINGS = {
8
11
  provider: "oauth",
9
12
  model: "gpt-5.4-mini",
@@ -11,11 +14,11 @@ export const DEFAULT_AGENT_GENERATION_SETTINGS = {
11
14
  size: "1024x1024",
12
15
  format: "png",
13
16
  moderation: "low",
14
- reasoningEffort: "medium",
17
+ reasoningEffort: "none",
15
18
  webSearchEnabled: true,
16
19
  generationStrategy: "auto",
17
20
  variants: 1,
18
- maxAutoVariants: 8,
21
+ maxAutoVariants: MAX_AGENT_VARIANTS,
19
22
  parallelism: 2,
20
23
  };
21
24
  export function normalizeAgentGenerationSettings(value, fallback = DEFAULT_AGENT_GENERATION_SETTINGS) {
@@ -30,9 +33,9 @@ export function normalizeAgentGenerationSettings(value, fallback = DEFAULT_AGENT
30
33
  reasoningEffort: cleanEnum(input.reasoningEffort, REASONING_EFFORTS, fallback.reasoningEffort),
31
34
  webSearchEnabled: typeof input.webSearchEnabled === "boolean" ? input.webSearchEnabled : fallback.webSearchEnabled,
32
35
  generationStrategy: cleanEnum(input.generationStrategy, GENERATION_STRATEGIES, fallback.generationStrategy),
33
- variants: cleanPositiveInt(input.variants, fallback.variants, 1, 8),
34
- maxAutoVariants: cleanPositiveInt(input.maxAutoVariants, fallback.maxAutoVariants, 1, 8),
35
- parallelism: cleanPositiveInt(input.parallelism, fallback.parallelism, 1, 8),
36
+ variants: cleanPositiveInt(input.variants, fallback.variants, 1, MAX_AGENT_VARIANTS),
37
+ maxAutoVariants: cleanPositiveInt(input.maxAutoVariants, fallback.maxAutoVariants, 1, MAX_AGENT_VARIANTS),
38
+ parallelism: cleanPositiveInt(input.parallelism, fallback.parallelism, 1, MAX_AGENT_PARALLELISM),
36
39
  };
37
40
  }
38
41
  export function mergeAgentGenerationSettings(current, patch) {