ima2-gen 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +151 -0
- package/README.md +10 -1
- package/bin/commands/backfillThumbs.js +6 -0
- package/bin/commands/gen.js +13 -5
- package/bin/commands/multimode.js +7 -5
- package/bin/commands/node.js +2 -1
- package/bin/ima2.js +14 -10
- package/config.js +8 -1
- package/docs/API.md +133 -9
- package/docs/CLI.md +6 -5
- package/docs/FAQ.ko.md +16 -0
- package/docs/FAQ.md +30 -0
- package/docs/README.ko.md +7 -3
- package/docs/migration/runtime-test-inventory.md +25 -1
- package/lib/agentCommandParser.js +10 -6
- package/lib/agentGenerationPlanner.js +96 -19
- package/lib/agentImageVideoGen.js +281 -0
- package/lib/agentPlannerModel.js +172 -0
- package/lib/agentQuestionResponder.js +9 -3
- package/lib/agentQueueStore.js +42 -0
- package/lib/agentQueueWorker.js +57 -7
- package/lib/agentRuntime.js +103 -274
- package/lib/agentSettings.js +9 -6
- package/lib/agentToolManifest.js +90 -0
- package/lib/agentTypes.js +1 -0
- package/lib/agyCli.js +31 -0
- package/lib/agyImageAdapter.js +104 -12
- package/lib/capabilities.js +7 -5
- package/lib/configKeys.js +6 -0
- package/lib/errorClassify.js +8 -7
- package/lib/eventBus.js +71 -0
- package/lib/geminiApiImageAdapter.js +24 -21
- package/lib/generationErrors.js +3 -1
- package/lib/generationRequestLog.js +25 -0
- package/lib/grokImageAdapter.js +76 -144
- package/lib/grokImageCore.js +153 -0
- package/lib/grokMultimodeAdapter.js +7 -4
- package/lib/grokVideoAdapter.js +3 -2
- package/lib/grokVideoCanvas.js +13 -0
- package/lib/grokVideoPlannerPrompt.js +52 -12
- package/lib/historyList.js +1 -0
- package/lib/inflight.js +54 -17
- package/lib/multimodeHelpers.js +11 -0
- package/lib/nodeHelpers.js +59 -0
- package/lib/oauthProxy/generators.js +6 -5
- package/lib/oauthProxy/prompts.js +60 -41
- package/lib/oauthProxy/streams.js +1 -1
- package/lib/promptBuilder/systemPrompt.js +2 -5
- package/lib/promptSafetyPolicy.js +1 -5
- package/lib/responsesDoctor.js +3 -3
- package/lib/responsesFallback.js +29 -15
- package/lib/responsesImageAdapter.js +11 -7
- package/lib/routeHelpers.js +44 -0
- package/lib/ssePublish.js +12 -0
- package/lib/storyboardPrefix.js +28 -0
- package/lib/thumbBackfill.js +16 -5
- package/node_modules/qs/CHANGELOG.md +10 -0
- package/node_modules/qs/README.md +1 -1
- package/node_modules/qs/dist/qs.js +15 -15
- package/node_modules/qs/eslint.config.mjs +1 -0
- package/node_modules/qs/lib/parse.js +52 -22
- package/node_modules/qs/lib/stringify.js +11 -4
- package/node_modules/qs/package.json +2 -2
- package/node_modules/qs/test/parse.js +49 -0
- package/node_modules/qs/test/stringify.js +129 -0
- package/package.json +4 -1
- package/routes/agent.js +19 -2
- package/routes/agy.js +48 -0
- package/routes/auth.js +6 -2
- package/routes/edit.js +7 -1
- package/routes/events.js +78 -0
- package/routes/generate.js +111 -128
- package/routes/generationRequestLog.js +16 -0
- package/routes/index.js +6 -0
- package/routes/keys.js +22 -2
- package/routes/multimode.js +100 -57
- package/routes/nodes.js +64 -108
- package/routes/video.js +100 -17
- package/skills/ima2/SKILL.md +98 -21
- package/ui/dist/.vite/manifest.json +22 -12
- package/ui/dist/assets/AgentWorkspace-qEMrATBP.js +3 -0
- package/ui/dist/assets/{CardNewsWorkspace-Dqyc1WZ1.js → CardNewsWorkspace-DoJnVHmn.js} +1 -1
- package/ui/dist/assets/GenerationRequestLogPanel-BDWRRLNV.js +1 -0
- package/ui/dist/assets/{NodeCanvas-ChEXzQbb.js → NodeCanvas-DIMo45sp.js} +3 -3
- package/ui/dist/assets/{PromptBuilderPanel-B95ZufnR.js → PromptBuilderPanel-BF3lcSJD.js} +2 -2
- package/ui/dist/assets/{PromptImportDialog-DGOwFQET.js → PromptImportDialog-Cy6ZymKG.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-CgvdnR49.js → PromptImportDiscoverySection-CzuW-8P2.js} +1 -1
- package/ui/dist/assets/PromptImportFolderSection-DP5ywnsv.js +1 -0
- package/ui/dist/assets/{PromptLibraryPanel-B9kndPw1.js → PromptLibraryPanel-CAsKr7CV.js} +2 -2
- package/ui/dist/assets/SettingsWorkspace-_PdPVsxi.js +1 -0
- package/ui/dist/assets/index-BUinlX2n.js +4 -0
- package/ui/dist/assets/index-CX3fge8X.css +1 -0
- package/ui/dist/assets/index-ygo6nfqx.js +23 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/AgentWorkspace-CYv84Rus.js +0 -3
- package/ui/dist/assets/PromptImportFolderSection-CfUye9J8.js +0 -1
- package/ui/dist/assets/SettingsWorkspace-B3tgLrmF.js +0 -1
- package/ui/dist/assets/index-BhcvL0g-.js +0 -1
- package/ui/dist/assets/index-BtK3YhJc.js +0 -39
- package/ui/dist/assets/index-ClOLOjnA.css +0 -1
package/lib/agentQueueStore.js
CHANGED
|
@@ -143,6 +143,48 @@ export function cancelAgentQueueItem(id) {
|
|
|
143
143
|
`).run(Date.now(), id);
|
|
144
144
|
return res.changes > 0;
|
|
145
145
|
}
|
|
146
|
+
export function updateAgentQueueItemPlan(id, plan) {
|
|
147
|
+
const res = getDb().prepare(`
|
|
148
|
+
UPDATE agent_queue_items
|
|
149
|
+
SET tool_plan = ?
|
|
150
|
+
WHERE id = ?
|
|
151
|
+
`).run(JSON.stringify(plan), id);
|
|
152
|
+
return res.changes > 0;
|
|
153
|
+
}
|
|
154
|
+
export function getAgentGenerationErrors(sessionId, limit = 10) {
|
|
155
|
+
const cap = Math.max(1, Math.min(20, Math.round(limit)));
|
|
156
|
+
const queueRows = getDb().prepare(`
|
|
157
|
+
SELECT error_code AS code, error_message AS message, prompt, finished_at AS at
|
|
158
|
+
FROM agent_queue_items
|
|
159
|
+
WHERE session_id = ? AND status = 'failed'
|
|
160
|
+
ORDER BY finished_at DESC
|
|
161
|
+
LIMIT ?
|
|
162
|
+
`).all(sessionId, cap);
|
|
163
|
+
const turnRows = getDb().prepare(`
|
|
164
|
+
SELECT text, created_at AS at
|
|
165
|
+
FROM agent_turns
|
|
166
|
+
WHERE session_id = ? AND status = 'error'
|
|
167
|
+
ORDER BY created_at DESC
|
|
168
|
+
LIMIT ?
|
|
169
|
+
`).all(sessionId, cap);
|
|
170
|
+
const records = [
|
|
171
|
+
...queueRows.map((row) => ({
|
|
172
|
+
scope: "queue",
|
|
173
|
+
code: row.code,
|
|
174
|
+
message: row.message ?? "Generation failed without a recorded message.",
|
|
175
|
+
prompt: row.prompt,
|
|
176
|
+
at: row.at ?? 0,
|
|
177
|
+
})),
|
|
178
|
+
...turnRows.map((row) => ({
|
|
179
|
+
scope: "turn",
|
|
180
|
+
code: null,
|
|
181
|
+
message: row.text,
|
|
182
|
+
prompt: null,
|
|
183
|
+
at: row.at,
|
|
184
|
+
})),
|
|
185
|
+
];
|
|
186
|
+
return records.sort((a, b) => b.at - a.at).slice(0, cap);
|
|
187
|
+
}
|
|
146
188
|
export function retryAgentQueueItem(id) {
|
|
147
189
|
const res = getDb().prepare(`
|
|
148
190
|
UPDATE agent_queue_items
|
package/lib/agentQueueWorker.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { config } from "../config.js";
|
|
2
|
+
import { claimNextAgentQueueItem, completeAgentQueueItem, failAgentQueueItem, getAgentQueueItem, updateAgentQueueItemPlan, } from "./agentQueueStore.js";
|
|
3
|
+
import { requestAgentPlanFromModel } from "./agentPlannerModel.js";
|
|
4
|
+
import { hasAgentErrorTurnRecorded, runAgentGenerationPlan } from "./agentRuntime.js";
|
|
5
|
+
import { appendAgentTurn } from "./agentStore.js";
|
|
3
6
|
import { errInfo } from "./errInfo.js";
|
|
4
7
|
import { finishJob, setJobPhase, startJob } from "./inflight.js";
|
|
5
8
|
import { logEvent } from "./logger.js";
|
|
@@ -42,10 +45,42 @@ export async function tickAgentQueueWorker(ctx) {
|
|
|
42
45
|
ticking = false;
|
|
43
46
|
}
|
|
44
47
|
}
|
|
48
|
+
// LLM planning runs only for plans the regex deriver produced automatically.
|
|
49
|
+
// Slash commands, manual settings, agy sessions, and already-LLM-planned
|
|
50
|
+
// retries keep their stored plan (audit decision F4/F5).
|
|
51
|
+
function isLlmPlanningEligible(item) {
|
|
52
|
+
return Boolean(config.agentPlanner.enabled &&
|
|
53
|
+
(item.plan.source === "auto-default" || item.plan.source === "auto-request") &&
|
|
54
|
+
item.plan.command == null &&
|
|
55
|
+
item.options.generationStrategy === "auto" &&
|
|
56
|
+
item.options.provider !== "agy");
|
|
57
|
+
}
|
|
58
|
+
async function resolveRuntimePlan(ctx, item) {
|
|
59
|
+
if (!isLlmPlanningEligible(item))
|
|
60
|
+
return item.plan;
|
|
61
|
+
try {
|
|
62
|
+
const plan = await requestAgentPlanFromModel(ctx, {
|
|
63
|
+
sessionId: item.sessionId,
|
|
64
|
+
prompt: item.prompt,
|
|
65
|
+
settings: item.options,
|
|
66
|
+
requestId: item.requestId,
|
|
67
|
+
});
|
|
68
|
+
if (!plan)
|
|
69
|
+
return item.plan;
|
|
70
|
+
updateAgentQueueItemPlan(item.id, plan);
|
|
71
|
+
return plan;
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const err = errInfo(error);
|
|
75
|
+
logEvent("agent_queue", "planner_fallback", { itemId: item.id, code: err.code, message: err.message });
|
|
76
|
+
return item.plan;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
45
79
|
async function runClaimedQueueItem(ctx, itemId) {
|
|
46
80
|
const item = getAgentQueueItem(itemId);
|
|
47
81
|
if (!item)
|
|
48
82
|
return;
|
|
83
|
+
const plan = await resolveRuntimePlan(ctx, item);
|
|
49
84
|
startJob({
|
|
50
85
|
requestId: item.requestId,
|
|
51
86
|
kind: "agent_queue",
|
|
@@ -53,19 +88,19 @@ async function runClaimedQueueItem(ctx, itemId) {
|
|
|
53
88
|
meta: {
|
|
54
89
|
sessionId: item.sessionId,
|
|
55
90
|
queueItemId: item.id,
|
|
56
|
-
variants:
|
|
57
|
-
parallelism:
|
|
58
|
-
requestedVariants:
|
|
91
|
+
variants: plan.plannedVariants,
|
|
92
|
+
parallelism: plan.plannedParallelism,
|
|
93
|
+
requestedVariants: plan.requestedVariants,
|
|
59
94
|
},
|
|
60
95
|
});
|
|
61
96
|
try {
|
|
62
97
|
logEvent("agent_queue", "start", { itemId: item.id, sessionId: item.sessionId });
|
|
63
98
|
setJobPhase(item.requestId, "streaming");
|
|
64
|
-
const result = await runAgentGenerationPlan(ctx, item.sessionId, item.prompt,
|
|
99
|
+
const result = await runAgentGenerationPlan(ctx, item.sessionId, item.prompt, plan, {
|
|
65
100
|
...item.options,
|
|
66
101
|
requestId: item.requestId,
|
|
67
102
|
webSearchEnabled: item.options.webSearchEnabled,
|
|
68
|
-
parallelism:
|
|
103
|
+
parallelism: plan.plannedParallelism,
|
|
69
104
|
}, {
|
|
70
105
|
appendUserTurn: false,
|
|
71
106
|
});
|
|
@@ -79,6 +114,21 @@ async function runClaimedQueueItem(ctx, itemId) {
|
|
|
79
114
|
catch (error) {
|
|
80
115
|
const err = errInfo(error);
|
|
81
116
|
failAgentQueueItem(item.id, { code: err.code, message: err.message });
|
|
117
|
+
// The chat pane must never fail silently — surface the failure as an
|
|
118
|
+
// assistant error turn unless the runtime already recorded one.
|
|
119
|
+
if (!hasAgentErrorTurnRecorded(error)) {
|
|
120
|
+
try {
|
|
121
|
+
appendAgentTurn({
|
|
122
|
+
sessionId: item.sessionId,
|
|
123
|
+
role: "assistant",
|
|
124
|
+
text: err.code ? `${err.message} [${err.code}]` : err.message,
|
|
125
|
+
status: "error",
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
catch (turnError) {
|
|
129
|
+
logEvent("agent_queue", "error_turn_failed", { itemId: item.id, message: errInfo(turnError).message });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
82
132
|
finishJob(item.requestId, {
|
|
83
133
|
status: "failed",
|
|
84
134
|
errorCode: err.code,
|
package/lib/agentRuntime.js
CHANGED
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
import { randomBytes } from "node:crypto";
|
|
2
|
-
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
3
|
-
import { atomicWriteJson } from "./atomicWrite.js";
|
|
4
|
-
import { join } from "node:path";
|
|
5
1
|
import { ulid } from "ulid";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { resolveProviderOptions } from "./providerOptions.js";
|
|
11
|
-
import { generateViaResponses } from "./responsesImageAdapter.js";
|
|
12
|
-
import { generateViaGrok } from "./grokImageAdapter.js";
|
|
13
|
-
import { generateViaAgy } from "./agyImageAdapter.js";
|
|
14
|
-
import { generateVideoViaGrok } from "./grokVideoAdapter.js";
|
|
15
|
-
import { parseVideoParams } from "./agentGenerationPlanner.js";
|
|
16
|
-
import { appendAgentTurn, buildImageContextManifest, getAgentImages, getAgentSession, importAgentImage, recordAgentWebFinding, restartAgentRuntimeSession, } from "./agentStore.js";
|
|
2
|
+
import { config } from "../config.js";
|
|
3
|
+
import { generateAgentImageWithRetry } from "./agentImageVideoGen.js";
|
|
4
|
+
import { runAgentVideoGeneration } from "./agentImageVideoGen.js";
|
|
5
|
+
import { appendAgentTurn, buildImageContextManifest, getAgentSession, recordAgentWebFinding, restartAgentRuntimeSession, } from "./agentStore.js";
|
|
17
6
|
import { AGENT_ALLOWED_TOOLS, } from "./agentTypes.js";
|
|
7
|
+
import { getAgentGenerationErrors } from "./agentQueueStore.js";
|
|
8
|
+
import { AGENT_TOOL_MANIFEST } from "./agentToolManifest.js";
|
|
18
9
|
import { errInfo } from "./errInfo.js";
|
|
19
10
|
export function assertAgentAllowedTools(tools) {
|
|
20
11
|
const allowed = new Set(AGENT_ALLOWED_TOOLS);
|
|
@@ -28,7 +19,7 @@ export function assertAgentAllowedTools(tools) {
|
|
|
28
19
|
}
|
|
29
20
|
}
|
|
30
21
|
export function agentAllowedToolPayload() {
|
|
31
|
-
return { tools: [...AGENT_ALLOWED_TOOLS] };
|
|
22
|
+
return { tools: [...AGENT_ALLOWED_TOOLS], manifest: [...AGENT_TOOL_MANIFEST] };
|
|
32
23
|
}
|
|
33
24
|
export async function runAgentTurn(ctx, sessionId, prompt, options = {}) {
|
|
34
25
|
return runAgentGenerationPlan(ctx, sessionId, prompt, {
|
|
@@ -41,6 +32,7 @@ export async function runAgentTurn(ctx, sessionId, prompt, options = {}) {
|
|
|
41
32
|
reason: "Direct turn endpoint defaults to one image.",
|
|
42
33
|
command: null,
|
|
43
34
|
assistantText: null,
|
|
35
|
+
sourceImagePolicy: "none",
|
|
44
36
|
}, options, { appendUserTurn: true });
|
|
45
37
|
}
|
|
46
38
|
export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, options = {}, behavior = {}) {
|
|
@@ -50,7 +42,7 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
50
42
|
const webSearchEnabled = options.provider === "agy" ? false : options.provider === "grok" ? true : options.webSearchEnabled ?? session.webSearchEnabled;
|
|
51
43
|
const enabledTools = webSearchEnabled
|
|
52
44
|
? [...AGENT_ALLOWED_TOOLS]
|
|
53
|
-
: ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video"];
|
|
45
|
+
: ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video", "ima2.get_generation_errors"];
|
|
54
46
|
assertAgentAllowedTools(enabledTools);
|
|
55
47
|
if (behavior.appendUserTurn !== false) {
|
|
56
48
|
appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
|
|
@@ -66,9 +58,15 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
66
58
|
});
|
|
67
59
|
return { assistantTurn, imageIds: [], webFindingIds: [] };
|
|
68
60
|
}
|
|
61
|
+
if (plan.mode === "errors") {
|
|
62
|
+
return runAgentErrorLookup(sessionId, plan);
|
|
63
|
+
}
|
|
64
|
+
const preludeSent = appendPlannerPreludeTurn(sessionId, plan);
|
|
69
65
|
if (plan.mode === "video") {
|
|
70
|
-
return runAgentVideoGeneration(ctx, sessionId, prompt, {
|
|
66
|
+
return runAgentVideoGeneration(ctx, sessionId, plan.prompts[0] ?? prompt, {
|
|
71
67
|
...options,
|
|
68
|
+
videoParams: plan.videoParams ?? options.videoParams ?? null,
|
|
69
|
+
assistantText: preludeSent ? null : plan.assistantText,
|
|
72
70
|
requestId: options.requestId ?? `agent_video_${ulid()}`,
|
|
73
71
|
skipUserTurn: true,
|
|
74
72
|
});
|
|
@@ -100,6 +98,7 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
100
98
|
const result = await runGeneratorWithRuntimeRecovery(ctx, sessionId, generationPrompt, manifest, webSearchEnabled, {
|
|
101
99
|
...options,
|
|
102
100
|
requestId,
|
|
101
|
+
sourceImagePolicy: plan.sourceImagePolicy ?? "none",
|
|
103
102
|
});
|
|
104
103
|
const findingIds = recordSearchFindings(sessionId, generationPrompt, result.webSearchCalls, result.provider ?? "oauth");
|
|
105
104
|
const finishedAt = Date.now();
|
|
@@ -154,20 +153,84 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
154
153
|
const assistantTurn = appendAgentTurn({
|
|
155
154
|
sessionId,
|
|
156
155
|
role: "assistant",
|
|
157
|
-
text: formatAgentAssistantText(plan, imageIds.length, responseTexts),
|
|
156
|
+
text: formatAgentAssistantText(plan, prompt, imageIds.length, responseTexts, preludeSent),
|
|
158
157
|
imageIds,
|
|
159
158
|
webFindingIds: findingIds,
|
|
160
159
|
status: "complete",
|
|
161
160
|
});
|
|
162
161
|
return { assistantTurn, imageIds, webFindingIds: findingIds };
|
|
163
162
|
}
|
|
164
|
-
function
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
163
|
+
function appendPlannerPreludeTurn(sessionId, plan) {
|
|
164
|
+
const text = plan.assistantText?.trim();
|
|
165
|
+
if (!text)
|
|
166
|
+
return false;
|
|
167
|
+
appendAgentTurn({
|
|
168
|
+
sessionId,
|
|
169
|
+
role: "assistant",
|
|
170
|
+
text,
|
|
171
|
+
imageIds: [],
|
|
172
|
+
webFindingIds: [],
|
|
173
|
+
status: "complete",
|
|
174
|
+
});
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
function runAgentErrorLookup(sessionId, plan) {
|
|
178
|
+
const startedAt = Date.now();
|
|
179
|
+
const errors = getAgentGenerationErrors(sessionId, 10);
|
|
180
|
+
appendAgentTurn({
|
|
181
|
+
sessionId,
|
|
182
|
+
role: "tool",
|
|
183
|
+
text: "ima2.get_generation_errors",
|
|
184
|
+
status: "complete",
|
|
185
|
+
raw: {
|
|
186
|
+
toolCalls: [{
|
|
187
|
+
id: `tc_errors_${ulid()}`,
|
|
188
|
+
name: "ima2.get_generation_errors",
|
|
189
|
+
status: "complete",
|
|
190
|
+
startedAt,
|
|
191
|
+
finishedAt: Date.now(),
|
|
192
|
+
durationMs: Date.now() - startedAt,
|
|
193
|
+
outputSummary: errors.length > 0
|
|
194
|
+
? `Found ${errors.length} recent generation error${errors.length === 1 ? "" : "s"}.`
|
|
195
|
+
: "No recent generation errors recorded for this session.",
|
|
196
|
+
}],
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
const assistantTurn = appendAgentTurn({
|
|
200
|
+
sessionId,
|
|
201
|
+
role: "assistant",
|
|
202
|
+
text: plan.assistantText?.trim() || formatGenerationErrors(errors),
|
|
203
|
+
imageIds: [],
|
|
204
|
+
webFindingIds: [],
|
|
205
|
+
status: "complete",
|
|
206
|
+
});
|
|
207
|
+
return { assistantTurn, imageIds: [], webFindingIds: [] };
|
|
208
|
+
}
|
|
209
|
+
function formatGenerationErrors(errors) {
|
|
210
|
+
if (errors.length === 0)
|
|
211
|
+
return "No generation errors are recorded for this session.";
|
|
212
|
+
const lines = errors.map((error, index) => {
|
|
213
|
+
const when = new Date(error.at).toISOString();
|
|
214
|
+
const code = error.code ? ` [${error.code}]` : "";
|
|
215
|
+
const promptPart = error.prompt ? ` (prompt: ${error.prompt.slice(0, 80)})` : "";
|
|
216
|
+
return `${index + 1}. ${when}${code} ${error.message}${promptPart}`;
|
|
217
|
+
});
|
|
218
|
+
return `Recent generation errors (most recent first):\n${lines.join("\n")}`;
|
|
219
|
+
}
|
|
220
|
+
function formatAgentAssistantText(plan, prompt, imageCount, responseTexts, omitPlannerText = false) {
|
|
221
|
+
// Behave like a normal chat agent: prefer the planner's natural-language
|
|
222
|
+
// reply, then any text the image model returned. The mechanical summary is
|
|
223
|
+
// only the fallback when neither produced prose.
|
|
224
|
+
const plannerText = omitPlannerText ? "" : (plan.assistantText?.trim() ?? "");
|
|
225
|
+
const modelText = responseTexts.join("\n\n").trim();
|
|
226
|
+
const prose = [plannerText, modelText].filter(Boolean).join("\n\n");
|
|
227
|
+
if (prose)
|
|
228
|
+
return prose;
|
|
229
|
+
const languageProbe = [prompt, ...plan.prompts].find((item) => item.trim().length > 0) ?? "";
|
|
230
|
+
const isKorean = /[\uAC00-\uD7AF]/u.test(languageProbe);
|
|
231
|
+
if (isKorean)
|
|
232
|
+
return imageCount === 1 ? "이미지 생성이 완료됐어요." : `이미지 ${imageCount}장을 생성했어요.`;
|
|
233
|
+
return imageCount === 1 ? "Done - I generated the image." : `Done - I generated ${imageCount} images.`;
|
|
171
234
|
}
|
|
172
235
|
async function runGeneratorWithRuntimeRecovery(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
|
|
173
236
|
try {
|
|
@@ -179,9 +242,18 @@ async function runGeneratorWithRuntimeRecovery(ctx, sessionId, prompt, manifest,
|
|
|
179
242
|
restartAgentRuntimeSession(sessionId, err.code || err.message);
|
|
180
243
|
}
|
|
181
244
|
appendAgentTurn({ sessionId, role: "assistant", text: err.message, status: "error" });
|
|
245
|
+
markAgentErrorTurnRecorded(error);
|
|
182
246
|
throw error;
|
|
183
247
|
}
|
|
184
248
|
}
|
|
249
|
+
export function markAgentErrorTurnRecorded(error) {
|
|
250
|
+
if (error && typeof error === "object") {
|
|
251
|
+
error.agentErrorTurnRecorded = true;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
export function hasAgentErrorTurnRecorded(error) {
|
|
255
|
+
return Boolean(error && typeof error === "object" && error.agentErrorTurnRecorded);
|
|
256
|
+
}
|
|
185
257
|
export function isRuntimeRestartableError(error) {
|
|
186
258
|
const err = errInfo(error);
|
|
187
259
|
const code = err.code || "";
|
|
@@ -190,249 +262,6 @@ export function isRuntimeRestartableError(error) {
|
|
|
190
262
|
code.includes("PROTOCOL") ||
|
|
191
263
|
err.message.toLowerCase().includes("protocol wedge"));
|
|
192
264
|
}
|
|
193
|
-
async function generateAgentImageWithRetry(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
|
|
194
|
-
let lastError = null;
|
|
195
|
-
for (let attempt = 0; attempt < 2; attempt++) {
|
|
196
|
-
try {
|
|
197
|
-
const forcedPrompt = attempt === 0 ? prompt : forceImagePrompt(prompt);
|
|
198
|
-
const result = await generateAgentImage(ctx, sessionId, forcedPrompt, manifest, webSearchEnabled, options);
|
|
199
|
-
if (result.image)
|
|
200
|
-
return result;
|
|
201
|
-
}
|
|
202
|
-
catch (error) {
|
|
203
|
-
lastError = error;
|
|
204
|
-
if (!isTextOnlyResult(error))
|
|
205
|
-
throw error;
|
|
206
|
-
if (attempt === 1)
|
|
207
|
-
break;
|
|
208
|
-
appendAgentTurn({
|
|
209
|
-
sessionId,
|
|
210
|
-
role: "tool",
|
|
211
|
-
text: "ima2.generate_image retry: text-only result rejected",
|
|
212
|
-
status: "error",
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
throw textOnlyError(lastError);
|
|
217
|
-
}
|
|
218
|
-
async function generateAgentImage(ctx, sessionId, prompt, manifest, webSearchEnabled, options) {
|
|
219
|
-
const requestId = options.requestId ?? `agent_${ulid()}`;
|
|
220
|
-
const providerOptions = resolveProviderOptions(ctx, {
|
|
221
|
-
provider: options.provider ?? "oauth",
|
|
222
|
-
rawModel: options.model,
|
|
223
|
-
rawReasoningEffort: options.reasoningEffort,
|
|
224
|
-
rawSize: options.size ?? "1024x1024",
|
|
225
|
-
rawWebSearchEnabled: webSearchEnabled,
|
|
226
|
-
searchMode: webSearchEnabled ? "on" : "off",
|
|
227
|
-
});
|
|
228
|
-
if (providerOptions.error) {
|
|
229
|
-
const err = new Error(providerOptions.error);
|
|
230
|
-
err.code = providerOptions.code;
|
|
231
|
-
err.status = providerOptions.status;
|
|
232
|
-
throw err;
|
|
233
|
-
}
|
|
234
|
-
const activeProvider = providerOptions.provider;
|
|
235
|
-
const effectiveModel = activeProvider === "grok" && options.quality === "high"
|
|
236
|
-
? "grok-imagine-image-quality"
|
|
237
|
-
: providerOptions.model;
|
|
238
|
-
const response = activeProvider === "agy"
|
|
239
|
-
? await generateViaAgy(`${manifest}\n\nUser request:\n${prompt}`, {
|
|
240
|
-
requestId,
|
|
241
|
-
signal: options.signal ?? undefined,
|
|
242
|
-
})
|
|
243
|
-
: activeProvider === "grok"
|
|
244
|
-
? await generateViaGrok(`${manifest}\n\nUser request:\n${prompt}`, ctx, {
|
|
245
|
-
model: effectiveModel,
|
|
246
|
-
size: providerOptions.size,
|
|
247
|
-
requestId,
|
|
248
|
-
signal: options.signal ?? undefined,
|
|
249
|
-
references: await loadAgentCurrentImageReferences(ctx, sessionId),
|
|
250
|
-
})
|
|
251
|
-
: await generateViaResponses(activeProvider, `${manifest}\n\nUser request:\n${prompt}`, options.quality ?? "medium", providerOptions.size, options.moderation ?? "low", [], requestId, "auto", ctx, {
|
|
252
|
-
model: providerOptions.model,
|
|
253
|
-
reasoningEffort: providerOptions.reasoningEffort,
|
|
254
|
-
webSearchEnabled,
|
|
255
|
-
signal: options.signal,
|
|
256
|
-
});
|
|
257
|
-
const format = activeProvider === "grok" || activeProvider === "agy"
|
|
258
|
-
? imageFormatFromMime(("mime" in response ? response.mime : undefined) || detectImageMimeFromB64(response.b64) || "image/jpeg")
|
|
259
|
-
: options.format ?? "png";
|
|
260
|
-
const image = await persistAgentImage(ctx, sessionId, prompt, format, requestId, response, {
|
|
261
|
-
provider: String(activeProvider),
|
|
262
|
-
model: String(effectiveModel),
|
|
263
|
-
});
|
|
264
|
-
const responseText = "text" in response && typeof response.text === "string" ? response.text : null;
|
|
265
|
-
return { image, webSearchCalls: response.webSearchCalls || 0, text: responseText, provider: activeProvider };
|
|
266
|
-
}
|
|
267
|
-
async function loadAgentCurrentImageReferences(ctx, sessionId) {
|
|
268
|
-
const session = getAgentSession(sessionId);
|
|
269
|
-
const currentImage = session?.lastImageId
|
|
270
|
-
? getAgentImages(sessionId).find((image) => image.id === session.lastImageId)
|
|
271
|
-
: null;
|
|
272
|
-
if (!currentImage?.filename)
|
|
273
|
-
return [];
|
|
274
|
-
try {
|
|
275
|
-
const b64 = (await readFile(join(ctx.config.storage.generatedDir, currentImage.filename))).toString("base64");
|
|
276
|
-
const mime = detectImageMimeFromB64(b64);
|
|
277
|
-
return [{ b64, declaredMime: mime, detectedMime: mime }];
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
const err = errInfo(error);
|
|
281
|
-
logEvent("agent", "grok_ref_missing", { sessionId, filename: currentImage.filename, code: err.code, message: err.message });
|
|
282
|
-
return [];
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
function imageFormatFromMime(mime) {
|
|
286
|
-
if (mime === "image/jpeg")
|
|
287
|
-
return "jpeg";
|
|
288
|
-
if (mime === "image/webp")
|
|
289
|
-
return "webp";
|
|
290
|
-
return "png";
|
|
291
|
-
}
|
|
292
|
-
async function persistAgentImage(ctx, sessionId, prompt, format, requestId, response, generation) {
|
|
293
|
-
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
294
|
-
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
295
|
-
const filename = `${Date.now()}_${rand}_agent.${format}`;
|
|
296
|
-
const meta = {
|
|
297
|
-
kind: "agent",
|
|
298
|
-
requestId,
|
|
299
|
-
sessionId,
|
|
300
|
-
prompt,
|
|
301
|
-
userPrompt: prompt,
|
|
302
|
-
revisedPrompt: response.revisedPrompt ?? null,
|
|
303
|
-
provider: generation.provider,
|
|
304
|
-
model: generation.model,
|
|
305
|
-
createdAt: Date.now(),
|
|
306
|
-
usage: response.usage ?? null,
|
|
307
|
-
webSearchCalls: response.webSearchCalls ?? 0,
|
|
308
|
-
};
|
|
309
|
-
const embedded = await embedImageMetadataBestEffort(Buffer.from(response.b64, "base64"), format, meta, {
|
|
310
|
-
version: ctx.packageVersion,
|
|
311
|
-
});
|
|
312
|
-
const filePath = join(ctx.config.storage.generatedDir, filename);
|
|
313
|
-
await writeFile(filePath, embedded.buffer);
|
|
314
|
-
try {
|
|
315
|
-
await atomicWriteJson(`${filePath}.json`, meta);
|
|
316
|
-
}
|
|
317
|
-
catch (err) {
|
|
318
|
-
await unlink(filePath).catch(() => { });
|
|
319
|
-
throw err;
|
|
320
|
-
}
|
|
321
|
-
invalidateHistoryIndex();
|
|
322
|
-
logEvent("agent", "saved", { requestId, sessionId, filename });
|
|
323
|
-
return importAgentImage(sessionId, {
|
|
324
|
-
id: `ai_${ulid()}`,
|
|
325
|
-
filename,
|
|
326
|
-
url: `/generated/${filename}`,
|
|
327
|
-
prompt,
|
|
328
|
-
revisedPrompt: response.revisedPrompt ?? null,
|
|
329
|
-
createdAt: Date.now(),
|
|
330
|
-
});
|
|
331
|
-
}
|
|
332
|
-
export async function runAgentVideoGeneration(ctx, sessionId, prompt, options = {}) {
|
|
333
|
-
const session = getAgentSession(sessionId);
|
|
334
|
-
if (!session)
|
|
335
|
-
throw notFound(sessionId);
|
|
336
|
-
if (!options.skipUserTurn) {
|
|
337
|
-
appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
|
|
338
|
-
}
|
|
339
|
-
const requestId = options.requestId ?? `agent_video_${ulid()}`;
|
|
340
|
-
const startedAt = Date.now();
|
|
341
|
-
// Auto I2V: if session has a last image, use it as source
|
|
342
|
-
let sourceImage;
|
|
343
|
-
let mode = "text-to-video";
|
|
344
|
-
if (session.lastImageId) {
|
|
345
|
-
const images = getAgentImages(sessionId);
|
|
346
|
-
const lastImage = images.find((img) => img.id === session.lastImageId);
|
|
347
|
-
if (lastImage?.filename && !lastImage.filename.endsWith(".mp4")) {
|
|
348
|
-
try {
|
|
349
|
-
const { loadAssetB64 } = await import("./nodeStore.js");
|
|
350
|
-
sourceImage = await loadAssetB64(ctx.rootDir, lastImage.filename, ctx.config.storage.generatedDir);
|
|
351
|
-
mode = "image-to-video";
|
|
352
|
-
}
|
|
353
|
-
catch { /* fallback to T2V */ }
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
const videoParams = parseVideoParams(prompt);
|
|
357
|
-
const result = await generateVideoViaGrok(prompt, ctx, {
|
|
358
|
-
model: "grok-imagine-video",
|
|
359
|
-
mode,
|
|
360
|
-
sourceImage,
|
|
361
|
-
duration: videoParams.duration ?? 5,
|
|
362
|
-
resolution: videoParams.resolution ?? "480p",
|
|
363
|
-
aspectRatio: (videoParams.aspectRatio ?? "auto"),
|
|
364
|
-
requestId,
|
|
365
|
-
signal: options.signal ?? undefined,
|
|
366
|
-
});
|
|
367
|
-
const video = await persistAgentVideo(ctx, sessionId, prompt, requestId, result);
|
|
368
|
-
const finishedAt = Date.now();
|
|
369
|
-
const toolCall = {
|
|
370
|
-
id: `tc_video_${ulid()}`,
|
|
371
|
-
name: "ima2.generate_video",
|
|
372
|
-
status: "complete",
|
|
373
|
-
startedAt,
|
|
374
|
-
finishedAt,
|
|
375
|
-
durationMs: finishedAt - startedAt,
|
|
376
|
-
requestId,
|
|
377
|
-
inputSummary: prompt,
|
|
378
|
-
outputSummary: `Generated video ${video.filename}.`,
|
|
379
|
-
imageIds: [video.id],
|
|
380
|
-
};
|
|
381
|
-
appendAgentTurn({
|
|
382
|
-
sessionId,
|
|
383
|
-
role: "tool",
|
|
384
|
-
text: "ima2.generate_video",
|
|
385
|
-
imageIds: [video.id],
|
|
386
|
-
status: "complete",
|
|
387
|
-
raw: { toolCalls: [toolCall] },
|
|
388
|
-
});
|
|
389
|
-
const assistantTurn = appendAgentTurn({
|
|
390
|
-
sessionId,
|
|
391
|
-
role: "assistant",
|
|
392
|
-
text: `Generated 1 video artifact. ${result.revisedPrompt}`,
|
|
393
|
-
imageIds: [video.id],
|
|
394
|
-
status: "complete",
|
|
395
|
-
});
|
|
396
|
-
return { assistantTurn, imageIds: [video.id], webFindingIds: [] };
|
|
397
|
-
}
|
|
398
|
-
async function persistAgentVideo(ctx, sessionId, prompt, requestId, result) {
|
|
399
|
-
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
400
|
-
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
401
|
-
const filename = `${Date.now()}_${rand}_agent.mp4`;
|
|
402
|
-
const meta = {
|
|
403
|
-
kind: "agent",
|
|
404
|
-
mediaType: "video",
|
|
405
|
-
requestId,
|
|
406
|
-
sessionId,
|
|
407
|
-
prompt,
|
|
408
|
-
userPrompt: prompt,
|
|
409
|
-
revisedPrompt: result.revisedPrompt,
|
|
410
|
-
provider: "grok",
|
|
411
|
-
model: "grok-imagine-video",
|
|
412
|
-
createdAt: Date.now(),
|
|
413
|
-
usage: result.usage,
|
|
414
|
-
webSearchCalls: result.webSearchCalls,
|
|
415
|
-
};
|
|
416
|
-
const filePath = join(ctx.config.storage.generatedDir, filename);
|
|
417
|
-
await writeFile(filePath, result.videoBuffer);
|
|
418
|
-
try {
|
|
419
|
-
await atomicWriteJson(`${filePath}.json`, meta);
|
|
420
|
-
}
|
|
421
|
-
catch (err) {
|
|
422
|
-
await unlink(filePath).catch(() => { });
|
|
423
|
-
throw err;
|
|
424
|
-
}
|
|
425
|
-
invalidateHistoryIndex();
|
|
426
|
-
logEvent("agent", "video_saved", { requestId, sessionId, filename });
|
|
427
|
-
return importAgentImage(sessionId, {
|
|
428
|
-
id: `ai_${ulid()}`,
|
|
429
|
-
filename,
|
|
430
|
-
url: `/generated/${filename}`,
|
|
431
|
-
prompt,
|
|
432
|
-
revisedPrompt: result.revisedPrompt,
|
|
433
|
-
createdAt: Date.now(),
|
|
434
|
-
});
|
|
435
|
-
}
|
|
436
265
|
function recordSearchFindings(sessionId, prompt, count, provider) {
|
|
437
266
|
if (!count)
|
|
438
267
|
return [];
|
|
@@ -446,14 +275,14 @@ function recordSearchFindings(sessionId, prompt, count, provider) {
|
|
|
446
275
|
}),
|
|
447
276
|
];
|
|
448
277
|
}
|
|
449
|
-
function forceImagePrompt(prompt) {
|
|
278
|
+
export function forceImagePrompt(prompt) {
|
|
450
279
|
return [
|
|
451
280
|
"The previous turn did not return an image artifact.",
|
|
452
281
|
"Return a final image using ima2.generate_image/image_generation now.",
|
|
453
282
|
`User request: ${prompt}`,
|
|
454
283
|
].join("\n");
|
|
455
284
|
}
|
|
456
|
-
function isTextOnlyResult(error) {
|
|
285
|
+
export function isTextOnlyResult(error) {
|
|
457
286
|
const err = errInfo(error);
|
|
458
287
|
return [
|
|
459
288
|
"EMPTY_RESPONSE",
|
|
@@ -462,7 +291,7 @@ function isTextOnlyResult(error) {
|
|
|
462
291
|
"IMAGE_TOOL_COMPLETED_WITHOUT_RESULT",
|
|
463
292
|
].includes(err.code || "") || err.message.includes("No image data");
|
|
464
293
|
}
|
|
465
|
-
function textOnlyError(cause) {
|
|
294
|
+
export function textOnlyError(cause) {
|
|
466
295
|
const err = new Error("Agent result did not include an image artifact.");
|
|
467
296
|
err.code = "AGENT_TEXT_ONLY_RESULT";
|
|
468
297
|
err.status = 422;
|
|
@@ -486,9 +315,9 @@ function cleanParallelism(value) {
|
|
|
486
315
|
const numeric = typeof value === "number" ? value : Number(value);
|
|
487
316
|
if (!Number.isFinite(numeric))
|
|
488
317
|
return 2;
|
|
489
|
-
return Math.max(1, Math.min(
|
|
318
|
+
return Math.max(1, Math.min(config.limits.maxParallel, Math.round(numeric)));
|
|
490
319
|
}
|
|
491
|
-
function notFound(sessionId) {
|
|
320
|
+
export function notFound(sessionId) {
|
|
492
321
|
const err = new Error(`Agent session not found: ${sessionId}`);
|
|
493
322
|
err.code = "AGENT_SESSION_NOT_FOUND";
|
|
494
323
|
err.status = 404;
|
package/lib/agentSettings.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { config } from "../config.js";
|
|
1
2
|
const PROVIDERS = new Set(["oauth", "api", "grok", "grok-api", "agy", "gemini-api"]);
|
|
2
3
|
const QUALITIES = new Set(["low", "medium", "high"]);
|
|
3
4
|
const FORMATS = new Set(["png", "jpeg", "webp"]);
|
|
4
5
|
const MODERATIONS = new Set(["auto", "low"]);
|
|
5
|
-
const REASONING_EFFORTS = new Set(["low", "medium", "high", "xhigh"]);
|
|
6
|
+
const REASONING_EFFORTS = new Set(["none", "low", "medium", "high", "xhigh"]);
|
|
6
7
|
const GENERATION_STRATEGIES = new Set(["auto", "manual"]);
|
|
8
|
+
const MAX_AGENT_VARIANTS = Math.max(1, Math.trunc(config.limits.maxGeneratedImages));
|
|
9
|
+
const MAX_AGENT_PARALLELISM = Math.max(1, Math.trunc(config.limits.maxParallel));
|
|
7
10
|
export const DEFAULT_AGENT_GENERATION_SETTINGS = {
|
|
8
11
|
provider: "oauth",
|
|
9
12
|
model: "gpt-5.4-mini",
|
|
@@ -11,11 +14,11 @@ export const DEFAULT_AGENT_GENERATION_SETTINGS = {
|
|
|
11
14
|
size: "1024x1024",
|
|
12
15
|
format: "png",
|
|
13
16
|
moderation: "low",
|
|
14
|
-
reasoningEffort: "
|
|
17
|
+
reasoningEffort: "none",
|
|
15
18
|
webSearchEnabled: true,
|
|
16
19
|
generationStrategy: "auto",
|
|
17
20
|
variants: 1,
|
|
18
|
-
maxAutoVariants:
|
|
21
|
+
maxAutoVariants: MAX_AGENT_VARIANTS,
|
|
19
22
|
parallelism: 2,
|
|
20
23
|
};
|
|
21
24
|
export function normalizeAgentGenerationSettings(value, fallback = DEFAULT_AGENT_GENERATION_SETTINGS) {
|
|
@@ -30,9 +33,9 @@ export function normalizeAgentGenerationSettings(value, fallback = DEFAULT_AGENT
|
|
|
30
33
|
reasoningEffort: cleanEnum(input.reasoningEffort, REASONING_EFFORTS, fallback.reasoningEffort),
|
|
31
34
|
webSearchEnabled: typeof input.webSearchEnabled === "boolean" ? input.webSearchEnabled : fallback.webSearchEnabled,
|
|
32
35
|
generationStrategy: cleanEnum(input.generationStrategy, GENERATION_STRATEGIES, fallback.generationStrategy),
|
|
33
|
-
variants: cleanPositiveInt(input.variants, fallback.variants, 1,
|
|
34
|
-
maxAutoVariants: cleanPositiveInt(input.maxAutoVariants, fallback.maxAutoVariants, 1,
|
|
35
|
-
parallelism: cleanPositiveInt(input.parallelism, fallback.parallelism, 1,
|
|
36
|
+
variants: cleanPositiveInt(input.variants, fallback.variants, 1, MAX_AGENT_VARIANTS),
|
|
37
|
+
maxAutoVariants: cleanPositiveInt(input.maxAutoVariants, fallback.maxAutoVariants, 1, MAX_AGENT_VARIANTS),
|
|
38
|
+
parallelism: cleanPositiveInt(input.parallelism, fallback.parallelism, 1, MAX_AGENT_PARALLELISM),
|
|
36
39
|
};
|
|
37
40
|
}
|
|
38
41
|
export function mergeAgentGenerationSettings(current, patch) {
|