ima2-gen 3.13.1 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/CHANGELOG.md +14 -0
- package/README.md +15 -2
- package/bin/commands/capabilities.js +5 -3
- package/bin/commands/config.js +15 -4
- package/bin/commands/defaults.js +7 -5
- package/bin/commands/doctor.js +88 -39
- package/bin/commands/gen.js +21 -4
- package/bin/commands/models.js +2 -2
- package/bin/commands/ping.js +1 -1
- package/bin/commands/prompt.js +5 -3
- package/bin/commands/service.js +32 -4
- package/bin/commands/tools.js +19 -3
- package/bin/commands/upscale.js +18 -4
- package/bin/commands/video.js +37 -50
- package/bin/ima2.js +28 -9
- package/bin/lib/characterResolve.js +2 -1
- package/bin/lib/client.js +155 -20
- package/bin/lib/config-store.js +40 -7
- package/bin/lib/doctor-bundle.js +6 -2
- package/bin/lib/doctor-checks.js +13 -29
- package/bin/lib/doctor-media.js +3 -3
- package/bin/lib/doctor-providers.js +48 -35
- package/bin/lib/doctor-report.js +78 -0
- package/bin/lib/doctor-runtime.js +173 -0
- package/bin/lib/mcpJob.js +19 -7
- package/bin/lib/output.js +3 -0
- package/bin/lib/platform.js +12 -9
- package/bin/lib/recover-output.js +9 -2
- package/bin/lib/sse.js +9 -1
- package/bin/lib/videoMcp.js +16 -8
- package/config.js +41 -5
- package/docs/API.md +154 -16
- package/docs/CLI.md +65 -1
- package/docs/DOCKER.md +19 -4
- package/docs/DOCKER.zh-CN.md +11 -4
- package/docs/DOCKER.zh-TW.md +11 -4
- package/docs/FAQ.md +13 -0
- package/docs/NPX_QUICKSTART.md +6 -0
- package/docs/README.ja.md +15 -2
- package/docs/README.ko.md +15 -2
- package/docs/README.zh-CN.md +15 -2
- package/docs/README.zh-TW.md +15 -2
- package/docs/migration/runtime-test-inventory.md +79 -8
- package/lib/agyArtifact.js +82 -0
- package/lib/agyArtifactRead.js +249 -0
- package/lib/agyImageAdapter.js +2 -329
- package/lib/agyProcess.js +122 -0
- package/lib/apiRequestBudget.js +48 -0
- package/lib/assetLifecycle.js +42 -9
- package/lib/canvasVersionStore.js +46 -14
- package/lib/capabilities.js +7 -1
- package/lib/configKeys.js +2 -0
- package/lib/errors/providerMap.js +26 -0
- package/lib/eventBus.js +10 -6
- package/lib/eventsPolicy.js +9 -0
- package/lib/geminiApiImageAdapter.js +1 -214
- package/lib/generatePipeline.js +39 -165
- package/lib/generatedMediaAccess.js +74 -0
- package/lib/generationErrors.js +3 -22
- package/lib/grokImageAdapter.js +2 -353
- package/lib/grokImageCore.js +1 -57
- package/lib/grokImageDownload.js +159 -0
- package/lib/grokImageDownloadPolicy.js +124 -0
- package/lib/grokImagePlanner.js +301 -0
- package/lib/grokMultimodeAdapter.js +1 -75
- package/lib/grokVideoDownload.js +137 -34
- package/lib/inflight.js +112 -112
- package/lib/jobs/terminalStore.js +45 -0
- package/lib/lanSessionStore.js +131 -0
- package/lib/localAccessPolicy.js +180 -0
- package/lib/localLanAccess.js +281 -0
- package/lib/logger.js +22 -13
- package/lib/mcp/downloadMediaResult.js +72 -78
- package/lib/multimodePipeline.js +39 -126
- package/lib/nodeGeneration.js +44 -101
- package/lib/nodeStore.js +25 -11
- package/lib/nodeValidation.js +5 -1
- package/lib/pinnedHttpGet.js +174 -0
- package/lib/promptImport/githubFolder.js +32 -18
- package/lib/promptImport/githubSource.js +14 -10
- package/lib/promptImport/parsePromptCandidates.js +32 -2
- package/lib/providers/adapters/agyOperations.js +179 -0
- package/lib/providers/adapters/comfy.js +1 -0
- package/lib/providers/adapters/geminiOperations.js +214 -0
- package/lib/providers/adapters/googleExecution.js +97 -0
- package/lib/providers/adapters/grokExecution.js +137 -0
- package/lib/providers/adapters/grokMultimodeOperations.js +83 -0
- package/lib/providers/adapters/grokOperations.js +63 -0
- package/lib/providers/adapters/openaiExecution.js +118 -0
- package/lib/providers/adapters/openaiOperations.js +201 -0
- package/lib/providers/adapters/openaiTypes.js +1 -0
- package/lib/providers/derive.js +4 -0
- package/lib/providers/deriveCore.js +2 -7
- package/lib/providers/execution/admission.js +25 -0
- package/lib/providers/execution/index.js +34 -0
- package/lib/providers/execution/legacy.js +17 -0
- package/lib/providers/execution/legacyClassic.js +66 -0
- package/lib/providers/execution/legacyEdit.js +30 -0
- package/lib/providers/execution/legacyMultimode.js +34 -0
- package/lib/providers/execution/legacyNode.js +41 -0
- package/lib/providers/execution/types.js +1 -0
- package/lib/providers/registry.js +19 -14
- package/lib/providers/surfaceSupport.js +22 -0
- package/lib/responsesImageAdapter.js +1 -389
- package/lib/responsesTransport.js +190 -0
- package/lib/spriteJobEvents.js +20 -2
- package/lib/ssePublish.js +5 -4
- package/lib/videoExtendI2vOperation.js +76 -0
- package/package.json +5 -2
- package/routes/edit.js +31 -123
- package/routes/events.js +140 -63
- package/routes/mcpMedia.js +10 -2
- package/routes/models.js +28 -25
- package/routes/videoExtended.js +9 -68
- package/server.js +24 -46
- package/ui/dist/.vite/manifest.json +90 -63
- package/ui/dist/assets/{AgentWorkspace-PbBytTeV.js → AgentWorkspace-CL3MLv3l.js} +2 -2
- package/ui/dist/assets/App-B9cEM6dJ.js +9 -0
- package/ui/dist/assets/AssetGenWorkspace-swWPvXrx.js +2 -0
- package/ui/dist/assets/AssetsWorkspace-CRVeYU2F.js +1 -0
- package/ui/dist/assets/{CardNewsWorkspace-BS3jKOja.js → CardNewsWorkspace-BAOxWLKf.js} +2 -2
- package/ui/dist/assets/GenerationRequestLogPanel-DZtYZm-D.js +1 -0
- package/ui/dist/assets/HomeWorkspace-BeZ1s0Jm.js +1 -0
- package/ui/dist/assets/KeyingPanel-ZyBSXioh.js +1 -0
- package/ui/dist/assets/NodeCanvas-BvvcCErf.js +7 -0
- package/ui/dist/assets/{PromptBuilderPanel-D97A4pwB.js → PromptBuilderPanel-DBycmShA.js} +2 -2
- package/ui/dist/assets/PromptImportDialog-ChvMv7ek.js +2 -0
- package/ui/dist/assets/PromptImportDiscoverySection-60py7whH.js +1 -0
- package/ui/dist/assets/PromptImportFolderSection-CCRh_-_7.js +1 -0
- package/ui/dist/assets/PromptLibraryPanel-BFWQO8OT.js +2 -0
- package/ui/dist/assets/SettingsWorkspace-B5Ghsgdj.js +1 -0
- package/ui/dist/assets/SpriteRecipeWorkspace-DRWdkmwU.js +1 -0
- package/ui/dist/assets/index-7ngEOVxt.js +5 -0
- package/ui/dist/assets/index-B5jy7ePg.css +1 -0
- package/ui/dist/assets/index-gxh_02DQ.js +22 -0
- package/ui/dist/assets/{pptxgen.es-D4pedbwM.js → pptxgen.es-OUNhQLEV.js} +1 -1
- package/ui/dist/assets/promptBuilderStore-BTUUzbBT.js +1 -0
- package/ui/dist/assets/useAgentDialogFocus-DSc2qY8p.js +1 -0
- package/ui/dist/index.html +3 -2
- package/ui/dist/assets/AssetGenWorkspace-B-TGlvVb.js +0 -2
- package/ui/dist/assets/AssetsWorkspace-LnGaW151.js +0 -1
- package/ui/dist/assets/GenerationRequestLogPanel-DuDWD_ic.js +0 -1
- package/ui/dist/assets/HomeWorkspace-DVP7UG6O.js +0 -1
- package/ui/dist/assets/KeyingPanel-5bRUFQaq.js +0 -1
- package/ui/dist/assets/NodeCanvas-BBEpERTb.js +0 -7
- package/ui/dist/assets/PromptImportDialog-C89MN5JY.js +0 -2
- package/ui/dist/assets/PromptImportDiscoverySection-BgRfDOs2.js +0 -1
- package/ui/dist/assets/PromptImportFolderSection-zTMyriT1.js +0 -1
- package/ui/dist/assets/PromptLibraryPanel-BD96qZu_.js +0 -2
- package/ui/dist/assets/SettingsWorkspace-BAOk3X_8.js +0 -1
- package/ui/dist/assets/SpriteRecipeWorkspace-ClIM5QRo.js +0 -1
- package/ui/dist/assets/index-0fpfY8vu.css +0 -1
- package/ui/dist/assets/index-BFciPGTn.js +0 -5
- package/ui/dist/assets/index-BbykeXR9.js +0 -30
- package/ui/dist/assets/promptBuilderStore-Do7M8g7J.js +0 -1
- package/ui/dist/assets/useAgentDialogFocus-B6-QgEVI.js +0 -1
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { logEvent } from "./logger.js";
|
|
2
|
+
import { classifyUpstreamError, classifyUpstreamErrorCode } from "./errorClassify.js";
|
|
3
|
+
import { errInfo } from "./errInfo.js";
|
|
4
|
+
import { setJobPhase } from "./inflight.js";
|
|
5
|
+
import { parseJson, parseStream, safeDiagnosticLabel, } from "./responsesParse.js";
|
|
6
|
+
import { waitForOAuthReady } from "./oauthProxy.js";
|
|
7
|
+
const RESPONSES_ERROR_MARKER = "ima2ResponsesError";
|
|
8
|
+
function makeError(message, { status = 500, code = "RESPONSES_IMAGE_ERROR", cause, ...rest } = {}) {
|
|
9
|
+
const err = new Error(message);
|
|
10
|
+
err.status = status;
|
|
11
|
+
err.code = code;
|
|
12
|
+
if (cause)
|
|
13
|
+
err.cause = cause;
|
|
14
|
+
Object.assign(err, rest);
|
|
15
|
+
Object.defineProperty(err, RESPONSES_ERROR_MARKER, { value: true });
|
|
16
|
+
return err;
|
|
17
|
+
}
|
|
18
|
+
function parseOpenAIErrorBody(text) {
|
|
19
|
+
try {
|
|
20
|
+
const parsed = JSON.parse(text);
|
|
21
|
+
const error = parsed?.error || {};
|
|
22
|
+
return {
|
|
23
|
+
message: typeof error.message === "string" && error.message ? error.message : "OpenAI request failed",
|
|
24
|
+
code: safeDiagnosticLabel(error.code),
|
|
25
|
+
type: safeDiagnosticLabel(error.type),
|
|
26
|
+
param: safeDiagnosticLabel(error.param),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function normalizedCode(upstream) {
|
|
34
|
+
const byCode = classifyUpstreamErrorCode(upstream?.code);
|
|
35
|
+
if (byCode !== "UNKNOWN")
|
|
36
|
+
return byCode;
|
|
37
|
+
const byType = classifyUpstreamErrorCode(upstream?.type);
|
|
38
|
+
if (byType !== "UNKNOWN")
|
|
39
|
+
return byType;
|
|
40
|
+
const byMessage = classifyUpstreamError(upstream?.message);
|
|
41
|
+
return byMessage !== "UNKNOWN" ? byMessage : "RESPONSES_IMAGE_ERROR";
|
|
42
|
+
}
|
|
43
|
+
function safeUpstreamClientMessage(upstream, status) {
|
|
44
|
+
const code = normalizedCode(upstream);
|
|
45
|
+
if (code === "AUTH_API_KEY_INVALID")
|
|
46
|
+
return "API key is invalid or unavailable.";
|
|
47
|
+
if (code === "MODERATION_REFUSED")
|
|
48
|
+
return "OpenAI refused the image request for safety reasons.";
|
|
49
|
+
if (code === "INVALID_REQUEST") {
|
|
50
|
+
return upstream?.param
|
|
51
|
+
? "OpenAI rejected the image request parameters."
|
|
52
|
+
: "OpenAI rejected the image request.";
|
|
53
|
+
}
|
|
54
|
+
if (status === 401 || status === 403)
|
|
55
|
+
return "OpenAI authentication failed.";
|
|
56
|
+
if (status === 429)
|
|
57
|
+
return "OpenAI rate limited the image request.";
|
|
58
|
+
return "OpenAI rejected the image request.";
|
|
59
|
+
}
|
|
60
|
+
function safeBaseUrl(value) {
|
|
61
|
+
try {
|
|
62
|
+
const parsed = new URL(value);
|
|
63
|
+
parsed.username = "";
|
|
64
|
+
parsed.password = "";
|
|
65
|
+
return parsed.toString().replace(/\/$/, "");
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return value.replace(/\/$/, "");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function apiAuthorizationHeader(apiKey) {
|
|
72
|
+
const key = typeof apiKey === "string" ? apiKey.trim() : "";
|
|
73
|
+
if (!key) {
|
|
74
|
+
throw makeError("API key is required for API provider image generation", {
|
|
75
|
+
status: 401,
|
|
76
|
+
code: "API_KEY_REQUIRED",
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
if (/[\u0000-\u001f\u007f]/.test(key)) {
|
|
80
|
+
throw makeError("API key contains invalid characters.", {
|
|
81
|
+
status: 401,
|
|
82
|
+
code: "AUTH_API_KEY_INVALID",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return `Bearer ${key}`;
|
|
86
|
+
}
|
|
87
|
+
function isKnownResponsesError(value) {
|
|
88
|
+
return Boolean(value &&
|
|
89
|
+
typeof value === "object" &&
|
|
90
|
+
value.ima2ResponsesError === true);
|
|
91
|
+
}
|
|
92
|
+
async function getEndpoint(ctx, provider, _scope) {
|
|
93
|
+
if (provider === "api") {
|
|
94
|
+
return {
|
|
95
|
+
url: "https://api.openai.com/v1/responses",
|
|
96
|
+
headers: {
|
|
97
|
+
"Content-Type": "application/json",
|
|
98
|
+
Accept: "text/event-stream",
|
|
99
|
+
Authorization: apiAuthorizationHeader(ctx.apiKey),
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
await waitForOAuthReady(ctx);
|
|
104
|
+
const port = ctx?.config?.oauth?.proxyPort || 10531;
|
|
105
|
+
return {
|
|
106
|
+
url: `${safeBaseUrl(ctx?.oauthUrl || `http://127.0.0.1:${port}`)}/v1/responses`,
|
|
107
|
+
headers: { "Content-Type": "application/json", Accept: "text/event-stream" },
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
function combineAbortSignals(signals) {
|
|
111
|
+
const only = signals[0];
|
|
112
|
+
if (signals.length === 1 && only)
|
|
113
|
+
return only;
|
|
114
|
+
const controller = new AbortController();
|
|
115
|
+
for (const signal of signals) {
|
|
116
|
+
if (signal.aborted) {
|
|
117
|
+
controller.abort();
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
signal.addEventListener("abort", () => controller.abort(), { once: true });
|
|
121
|
+
}
|
|
122
|
+
return controller.signal;
|
|
123
|
+
}
|
|
124
|
+
export async function postResponses({ ctx, provider, scope, payload, requestId, maxImages = 1, signal = null, onPartialImage = null, onFinalImage = null, }) {
|
|
125
|
+
const { url, headers } = await getEndpoint(ctx, provider, scope);
|
|
126
|
+
const timeoutMs = ctx?.config?.oauth?.generationTimeoutMs || 400 * 1000;
|
|
127
|
+
const controller = new AbortController();
|
|
128
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
129
|
+
const fetchSignal = signal
|
|
130
|
+
? combineAbortSignals([controller.signal, signal])
|
|
131
|
+
: controller.signal;
|
|
132
|
+
try {
|
|
133
|
+
const res = await fetch(url, {
|
|
134
|
+
method: "POST",
|
|
135
|
+
headers: headers,
|
|
136
|
+
signal: fetchSignal,
|
|
137
|
+
body: JSON.stringify(payload),
|
|
138
|
+
});
|
|
139
|
+
logEvent(scope, "response", { requestId, provider, status: res.status, contentType: res.headers.get("content-type") });
|
|
140
|
+
if (!res.ok) {
|
|
141
|
+
const text = await res.text();
|
|
142
|
+
const upstream = parseOpenAIErrorBody(text);
|
|
143
|
+
if (res.status >= 400 && res.status < 500 && upstream?.message) {
|
|
144
|
+
throw makeError(safeUpstreamClientMessage(upstream, res.status), {
|
|
145
|
+
status: res.status,
|
|
146
|
+
code: normalizedCode(upstream),
|
|
147
|
+
upstreamBodyChars: text.length,
|
|
148
|
+
upstreamCode: upstream.code,
|
|
149
|
+
upstreamType: upstream.type,
|
|
150
|
+
upstreamParam: upstream.param,
|
|
151
|
+
upstreamMessageRedacted: true,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
throw makeError(`${provider === "api" ? "OpenAI API" : "OAuth proxy"} returned ${res.status}`, {
|
|
155
|
+
status: res.status,
|
|
156
|
+
upstreamBodyChars: text.length,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
if (requestId)
|
|
160
|
+
setJobPhase(requestId, "streaming");
|
|
161
|
+
const contentType = res.headers.get("content-type") || "";
|
|
162
|
+
return contentType.includes("text/event-stream")
|
|
163
|
+
? await parseStream(res, { requestId, scope, maxImages, onPartialImage, onFinalImage })
|
|
164
|
+
: await parseJson(res, maxImages);
|
|
165
|
+
}
|
|
166
|
+
catch (e) {
|
|
167
|
+
const err = errInfo(e);
|
|
168
|
+
if (err.name === "AbortError") {
|
|
169
|
+
if (signal?.aborted) {
|
|
170
|
+
throw makeError("Generation canceled", {
|
|
171
|
+
status: 499,
|
|
172
|
+
code: "GENERATION_CANCELED",
|
|
173
|
+
cause: err.raw,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
throw makeError("Responses image generation timed out", { status: 504, code: "RESPONSES_IMAGE_TIMEOUT", cause: err.raw });
|
|
177
|
+
}
|
|
178
|
+
if (isKnownResponsesError(err.raw))
|
|
179
|
+
throw err.raw;
|
|
180
|
+
throw makeError("Responses request failed before receiving a response", {
|
|
181
|
+
status: 502,
|
|
182
|
+
code: "NETWORK_FAILED",
|
|
183
|
+
errorName: err.name,
|
|
184
|
+
upstreamMessageRedacted: true,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
finally {
|
|
188
|
+
clearTimeout(timer);
|
|
189
|
+
}
|
|
190
|
+
}
|
package/lib/spriteJobEvents.js
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
import { publish } from "./eventBus.js";
|
|
2
2
|
import { writeSse } from "./routeHelpers.js";
|
|
3
3
|
import { publishJobEvent } from "./ssePublish.js";
|
|
4
|
-
export function createSpriteJobEmitter(res, requestId) {
|
|
5
|
-
|
|
4
|
+
export function createSpriteJobEmitter(res, requestId) {
|
|
5
|
+
return {
|
|
6
|
+
emit(event, data) {
|
|
7
|
+
const wrote = !res.writableEnded ? writeSse(res, event, data) : false;
|
|
8
|
+
if (event !== "done" && event !== "error") {
|
|
9
|
+
publish(requestId, event, data);
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
// Direct legacy SSE keeps its nested error. The shared envelope reads flat fields.
|
|
13
|
+
const nested = data.error;
|
|
14
|
+
const error = nested && typeof nested === "object" && !Array.isArray(nested)
|
|
15
|
+
? nested : null;
|
|
16
|
+
const payload = event === "error" && error ? { ...data,
|
|
17
|
+
code: error.code, error: error.message } : data;
|
|
18
|
+
return publishJobEvent(requestId, event, payload) || wrote;
|
|
19
|
+
},
|
|
20
|
+
end() { if (!res.writableEnded)
|
|
21
|
+
res.end(); },
|
|
22
|
+
};
|
|
23
|
+
}
|
package/lib/ssePublish.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import { publish } from "./eventBus.js";
|
|
2
|
-
import { getJobPhase, isJobCanceled } from "./inflight.js";
|
|
2
|
+
import { getJobPhase, isJobCanceled, isJobTrackingExpired } from "./inflight.js";
|
|
3
3
|
import { buildEnvelope } from "./jobs/envelope.js";
|
|
4
4
|
/**
|
|
5
|
-
* Publish a multiplexed job event.
|
|
6
|
-
*
|
|
5
|
+
* Publish a multiplexed job event. Cancellation/expiry dominate later terminal
|
|
6
|
+
* errors and success, so abort listeners cannot replace the established outcome.
|
|
7
7
|
*
|
|
8
8
|
* Also attaches the canonical envelope (#151). The snapshot is taken here,
|
|
9
9
|
* before the event is queued, so replay reproduces what was true at publish
|
|
10
10
|
* time rather than re-deriving state that has since moved on.
|
|
11
11
|
*/
|
|
12
12
|
export function publishJobEvent(requestId, event, data) {
|
|
13
|
-
if (event === "done"
|
|
13
|
+
if ((event === "done" || event === "error") &&
|
|
14
|
+
(isJobCanceled(requestId) || isJobTrackingExpired(requestId)))
|
|
14
15
|
return false;
|
|
15
16
|
const inflightPhase = getJobPhase(requestId);
|
|
16
17
|
publish(requestId, event, data, {
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { appendVideoContinuityEntry, lineageFromVideoMetadata } from "./videoContinuity.js";
|
|
2
|
+
import { deriveChildVideoLineage } from "./videoLineage.js";
|
|
3
|
+
import { invalidateHistoryIndex } from "./historyIndex.js";
|
|
4
|
+
import { finishJob, isJobCanceled, setJobPhase } from "./inflight.js";
|
|
5
|
+
import { makeGenerationCanceledError } from "./generationCancel.js";
|
|
6
|
+
import { publish } from "./eventBus.js";
|
|
7
|
+
import { publishJobEvent } from "./ssePublish.js";
|
|
8
|
+
import { errInfo } from "./errInfo.js";
|
|
9
|
+
import { codedVideoError as codedError, emitPhase, extractError, retryableData } from "./videoExtendedHelpers.js";
|
|
10
|
+
import { errorEnvelopeFields } from "./errors/envelope.js";
|
|
11
|
+
// Approved relocation: keep the original operation body and ordering unchanged.
|
|
12
|
+
export async function runLastFrameI2v(task) {
|
|
13
|
+
const { ctx, requestId, sourceVideoId, prompt, model, provider, duration, resolution, aspectRatio, parent, motion, cancelController, extractFrame, generateVideo, persistArtifact, createFilename } = task;
|
|
14
|
+
const startedAt = Date.now();
|
|
15
|
+
let stage = "extracting-frame";
|
|
16
|
+
try {
|
|
17
|
+
emitPhase(requestId, stage);
|
|
18
|
+
let sourceImage;
|
|
19
|
+
try {
|
|
20
|
+
sourceImage = await extractFrame(ctx.config.storage.generatedDir, sourceVideoId, "last", { signal: cancelController.signal });
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
throw extractError(error, cancelController.signal);
|
|
24
|
+
}
|
|
25
|
+
if (cancelController.signal.aborted)
|
|
26
|
+
throw makeGenerationCanceledError();
|
|
27
|
+
const parentContinuity = lineageFromVideoMetadata(sourceVideoId, parent);
|
|
28
|
+
const onEvent = (event) => {
|
|
29
|
+
setJobPhase(requestId, event.phase === "submitted" ? "streaming" : event.phase);
|
|
30
|
+
publish(requestId, event.phase, {
|
|
31
|
+
requestId,
|
|
32
|
+
...(event.phase === "submitted" ? { xaiVideoRequestId: event.xaiVideoRequestId, requestedModel: event.requestedModel, effectiveModel: event.effectiveModel, modelFallback: event.modelFallback ?? null } : {}),
|
|
33
|
+
...(event.phase === "progress" ? { progress: typeof event.progress === "number" ? event.progress / 100 : null, stalled: Boolean(event.stalled) } : {}),
|
|
34
|
+
});
|
|
35
|
+
};
|
|
36
|
+
const compiledPrompt = motion.fragment ? `${prompt}\n\nCamera motion: ${motion.fragment}.` : prompt;
|
|
37
|
+
const result = await generateVideo(compiledPrompt, ctx, {
|
|
38
|
+
model, mode: "image-to-video", duration, resolution, aspectRatio,
|
|
39
|
+
sourceImage, sourceMime: "image/png", signal: cancelController.signal,
|
|
40
|
+
requestId, continuityLineage: parentContinuity,
|
|
41
|
+
directApiKey: provider === "grok-api" ? ctx.xaiApiKey ?? undefined : undefined,
|
|
42
|
+
onEvent,
|
|
43
|
+
});
|
|
44
|
+
if (cancelController.signal.aborted)
|
|
45
|
+
throw makeGenerationCanceledError();
|
|
46
|
+
stage = "persisting";
|
|
47
|
+
emitPhase(requestId, stage);
|
|
48
|
+
const filename = createFilename(ctx);
|
|
49
|
+
const createdAt = Date.now();
|
|
50
|
+
const videoLineage = deriveChildVideoLineage(filename, sourceVideoId, parent);
|
|
51
|
+
const videoContinuity = appendVideoContinuityEntry(parentContinuity, { filename, userPrompt: prompt, revisedPrompt: result.revisedPrompt, createdAt });
|
|
52
|
+
const elapsed = +((createdAt - startedAt) / 1000).toFixed(1);
|
|
53
|
+
const video = { operation: "extend", mode: "image-to-video", sourceVideoId, sourceFrame: "last", duration: result.duration, resolution: result.resolution, aspectRatio: result.aspectRatio, xaiVideoRequestId: result.xaiVideoRequestId };
|
|
54
|
+
const metadata = { kind: "video", mediaType: "video", providerUrl: result.url, requestId, prompt, userPrompt: prompt, revisedPrompt: result.revisedPrompt, motionPresetIds: motion.ids, provider, model: result.effectiveModel, createdAt, elapsed, usage: result.usage, webSearchCalls: result.webSearchCalls, video, videoLineage, videoContinuity };
|
|
55
|
+
try {
|
|
56
|
+
await persistArtifact(ctx.config.storage.generatedDir, filename, result.videoBuffer, metadata);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
throw codedError(errInfo(error).message, 500, "VIDEO_PERSIST_FAILED");
|
|
60
|
+
}
|
|
61
|
+
invalidateHistoryIndex();
|
|
62
|
+
const done = { requestId, filename, url: `/generated/${encodeURIComponent(filename)}`, providerUrl: result.url, mediaType: "video", provider, model: result.effectiveModel, prompt, userPrompt: prompt, revisedPrompt: result.revisedPrompt, createdAt, elapsed, usage: result.usage, webSearchCalls: result.webSearchCalls, video, videoLineage, videoContinuity };
|
|
63
|
+
// finishJob BEFORE done (audit blocker B4): a done event must never be
|
|
64
|
+
// followed by an error from a failing inflight-completion write.
|
|
65
|
+
finishJob(requestId, { status: "completed", meta: { filename, xaiVideoRequestId: result.xaiVideoRequestId } });
|
|
66
|
+
publishJobEvent(requestId, "done", done);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (!isJobCanceled(requestId)) {
|
|
70
|
+
const info = errInfo(error);
|
|
71
|
+
// #151 stage 2: terminal failure carries the canonical envelope.
|
|
72
|
+
publishJobEvent(requestId, "error", { requestId, error: info.message, code: info.code ?? "VIDEO_EXTEND_FAILED", status: info.status ?? 500, ...retryableData(error), ...errorEnvelopeFields(error) });
|
|
73
|
+
finishJob(requestId, { status: "error", httpStatus: info.status ?? 500, errorCode: info.code ?? "VIDEO_EXTEND_FAILED", meta: { stage } });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ima2-gen",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.14.0",
|
|
4
4
|
"packageManager": "npm@11.18.0",
|
|
5
5
|
"description": "Local-first visual generation runtime and studio for people and coding agents, with reproducible image and video workflows across multiple providers.",
|
|
6
6
|
"type": "module",
|
|
@@ -31,6 +31,8 @@
|
|
|
31
31
|
"prepublishOnly": "node scripts/release-contract.mjs assert-publish-context && npm run verify:release",
|
|
32
32
|
"lint:pkg": "node -e \"const p=require('./package.json'); const req=['name','version','bin']; for(const k of req){if(!p[k])throw new Error('missing '+k)} const mustInclude=['bin/**/*.js','lib/**/*.js','routes/**/*.js','skills/','integrations/comfyui/ima2_gen_bridge/__init__.py','integrations/comfyui/ima2_gen_bridge/nodes.py','integrations/comfyui/ima2_gen_bridge/README.md','assets/card-news/templates/','assets/mcp-snapshots/','vendor/','server.js','LICENSE']; for(const f of mustInclude){if(!p.files.includes(f))throw new Error('files[] must include '+f)}\"",
|
|
33
33
|
"docs:refresh-line-counts": "node scripts/refresh-structure-line-counts.mjs",
|
|
34
|
+
"docs:runtime": "node --import tsx scripts/generate-runtime-install-contract.mjs",
|
|
35
|
+
"docs:runtime:check": "node --import tsx scripts/generate-runtime-install-contract.mjs --check",
|
|
34
36
|
"publish:dry-run": "node scripts/publish-dry-run.mjs",
|
|
35
37
|
"release:patch": "gh workflow run release.yml -f bump=patch -f dry_run=false",
|
|
36
38
|
"release:minor": "gh workflow run release.yml -f bump=minor -f dry_run=false",
|
|
@@ -120,9 +122,10 @@
|
|
|
120
122
|
"@types/better-sqlite3": "^9.6.0",
|
|
121
123
|
"@types/express": "^5.0.6",
|
|
122
124
|
"@types/node": "^22.20.1",
|
|
125
|
+
"node-gyp": "13.0.1",
|
|
123
126
|
"tsx": "^4.23.12",
|
|
124
127
|
"typescript": "^5.9.3",
|
|
125
128
|
"yaml": "2.9.0"
|
|
126
129
|
},
|
|
127
|
-
"gitHead": "
|
|
130
|
+
"gitHead": "36aa6fcea62f753d858d20c4823e04922951c954"
|
|
128
131
|
}
|
package/routes/edit.js
CHANGED
|
@@ -7,14 +7,9 @@ import { generateImageThumbnailFromBuffer } from "../lib/imageThumb.js";
|
|
|
7
7
|
import { classifyUpstreamError } from "../lib/errorClassify.js";
|
|
8
8
|
import { normalizeOAuthParams } from "../lib/oauthNormalize.js";
|
|
9
9
|
import { resolveProviderOptions } from "../lib/providerOptions.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
10
|
+
import { prepareImageExecution } from "../lib/providers/execution/index.js";
|
|
11
|
+
import { checkImageExecutionAdmission } from "../lib/providers/execution/admission.js";
|
|
12
12
|
import { resolveGrokQualityModel } from "../lib/imageModels.js";
|
|
13
|
-
import { generateViaAgy } from "../lib/agyImageAdapter.js";
|
|
14
|
-
import { generateViaGeminiApi } from "../lib/geminiApiImageAdapter.js";
|
|
15
|
-
import { generateViaAtlasCloud } from "../lib/atlasCloudImageAdapter.js";
|
|
16
|
-
import { generateViaMinimax } from "../lib/minimaxImageAdapter.js";
|
|
17
|
-
import { generateViaComfy } from "../lib/comfyImageAdapter.js";
|
|
18
13
|
import { startJob, finishJob, registerJobAbortController, isJobCanceled, isStartJobFailure, INFLIGHT_RETRY_AFTER_SECONDS } from "../lib/inflight.js";
|
|
19
14
|
import { isGenerationCanceledError, makeGenerationCanceledError, throwIfJobCanceled, } from "../lib/generationCancel.js";
|
|
20
15
|
import { logEvent, logError } from "../lib/logger.js";
|
|
@@ -25,6 +20,7 @@ import { invalidateHistoryIndex } from "../lib/historyIndex.js";
|
|
|
25
20
|
import { errInfo } from "../lib/errInfo.js";
|
|
26
21
|
import { requireRuntimeContext } from "../lib/runtimeContext.js";
|
|
27
22
|
import { errorEnvelopeFields } from "../lib/errors/envelope.js";
|
|
23
|
+
import { getProviderSurfaceSupport } from "../lib/providers/derive.js";
|
|
28
24
|
function validateModeration(ctx, moderation) {
|
|
29
25
|
if (typeof moderation !== "string" || !ctx.config.oauth.validModeration.has(moderation)) {
|
|
30
26
|
return { error: "moderation must be one of: auto, low" };
|
|
@@ -103,7 +99,7 @@ export function registerEditRoutes(app, ctxRaw) {
|
|
|
103
99
|
rawSize: size,
|
|
104
100
|
rawWebSearchEnabled,
|
|
105
101
|
});
|
|
106
|
-
if (providerOptions.error) {
|
|
102
|
+
if (providerOptions.error !== undefined) {
|
|
107
103
|
finishStatus = "error";
|
|
108
104
|
finishHttpStatus = providerOptions.status;
|
|
109
105
|
finishErrorCode = providerOptions.code;
|
|
@@ -116,6 +112,18 @@ export function registerEditRoutes(app, ctxRaw) {
|
|
|
116
112
|
const activeProvider = providerOptions.provider;
|
|
117
113
|
const sessionId = typeof req.body?.sessionId === "string" ? req.body.sessionId : null;
|
|
118
114
|
const normalizedPromptMode = promptMode === "direct" ? "direct" : "auto";
|
|
115
|
+
const admission = checkImageExecutionAdmission(ctx, {
|
|
116
|
+
provider: activeProvider, surface: "edit", referenceCount: imageB64 ? 1 : 0,
|
|
117
|
+
});
|
|
118
|
+
if (admission) {
|
|
119
|
+
finishStatus = "error";
|
|
120
|
+
finishHttpStatus = admission.status;
|
|
121
|
+
finishErrorCode = admission.code;
|
|
122
|
+
return res.status(admission.status).json({
|
|
123
|
+
error: admission.message, code: admission.code, requestId,
|
|
124
|
+
...errorEnvelopeFields({ code: admission.code, status: admission.status }),
|
|
125
|
+
});
|
|
126
|
+
}
|
|
119
127
|
const started = startJob({
|
|
120
128
|
requestId,
|
|
121
129
|
kind: "classic",
|
|
@@ -154,7 +162,7 @@ export function registerEditRoutes(app, ctxRaw) {
|
|
|
154
162
|
// NovelAI joins this list because its infill is a distinct action with its
|
|
155
163
|
// own model ids, not a mask flag on generate; accepting a mask here would
|
|
156
164
|
// silently drop it.
|
|
157
|
-
if ((activeProvider
|
|
165
|
+
if (getProviderSurfaceSupport(activeProvider ?? "", "edit")?.mask === false && rawMask) {
|
|
158
166
|
finishStatus = "error";
|
|
159
167
|
finishHttpStatus = 400;
|
|
160
168
|
const code = activeProvider === "agy" ? "AGY_MASK_UNSUPPORTED" : activeProvider === "gemini-api" ? "GEMINI_API_MASK_UNSUPPORTED" : activeProvider === "atlascloud" ? "ATLASCLOUD_MASK_UNSUPPORTED" : activeProvider === "minimax" ? "MINIMAX_MASK_UNSUPPORTED" : activeProvider === "nai" ? "NAI_MASK_UNSUPPORTED" : activeProvider === "comfy" ? "COMFY_MASK_UNSUPPORTED" : "GROK_MASK_UNSUPPORTED";
|
|
@@ -165,7 +173,7 @@ export function registerEditRoutes(app, ctxRaw) {
|
|
|
165
173
|
// is text-to-image only. Dispatching here would ignore that image and
|
|
166
174
|
// return an unrelated generation, so the lane refuses instead. NovelAI's
|
|
167
175
|
// img2img and infill are separate actions and a follow-on unit.
|
|
168
|
-
if (activeProvider
|
|
176
|
+
if (getProviderSurfaceSupport(activeProvider ?? "", "edit")?.supported === false) {
|
|
169
177
|
finishStatus = "error";
|
|
170
178
|
finishHttpStatus = 400;
|
|
171
179
|
finishErrorCode = "NAI_EDIT_UNSUPPORTED";
|
|
@@ -206,119 +214,19 @@ export function registerEditRoutes(app, ctxRaw) {
|
|
|
206
214
|
maskBytes: maskCheck.maskBytes ?? 0,
|
|
207
215
|
});
|
|
208
216
|
const startTime = Date.now();
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
});
|
|
223
|
-
resultB64 = r.b64;
|
|
224
|
-
usage = r.usage;
|
|
225
|
-
revisedPrompt = r.revisedPrompt;
|
|
226
|
-
webSearchCalls = r.webSearchCalls;
|
|
227
|
-
resultMimeFromProvider = r.mime;
|
|
228
|
-
}
|
|
229
|
-
else if (activeProvider === "agy") {
|
|
230
|
-
const r = await generateViaAgy(`Edit this image: ${prompt}`, {
|
|
231
|
-
references: [{ b64: imageB64, declaredMime: null, detectedMime: detectImageMimeFromB64(imageB64) || null }],
|
|
232
|
-
signal: cancelController.signal,
|
|
233
|
-
requestId,
|
|
234
|
-
});
|
|
235
|
-
resultB64 = r.b64;
|
|
236
|
-
usage = r.usage;
|
|
237
|
-
revisedPrompt = r.revisedPrompt;
|
|
238
|
-
webSearchCalls = r.webSearchCalls;
|
|
239
|
-
resultMimeFromProvider = r.mime;
|
|
240
|
-
}
|
|
241
|
-
else if (activeProvider === "atlascloud") {
|
|
242
|
-
const r = await generateViaAtlasCloud(`Edit this image: ${prompt}`, requireRuntimeContext(ctx), {
|
|
243
|
-
model: imageModel,
|
|
244
|
-
size: effectiveSize,
|
|
245
|
-
quality,
|
|
246
|
-
signal: cancelController.signal,
|
|
247
|
-
requestId,
|
|
248
|
-
references: [{ b64: imageB64, declaredMime: null, detectedMime: detectImageMimeFromB64(imageB64) || null }],
|
|
249
|
-
});
|
|
250
|
-
resultB64 = r.b64;
|
|
251
|
-
providerUrl = r.providerUrl ?? null;
|
|
252
|
-
usage = r.usage;
|
|
253
|
-
revisedPrompt = r.revisedPrompt ?? undefined;
|
|
254
|
-
webSearchCalls = r.webSearchCalls;
|
|
255
|
-
resultMimeFromProvider = r.mime;
|
|
256
|
-
}
|
|
257
|
-
else if (activeProvider === "minimax") {
|
|
258
|
-
const r = await generateViaMinimax(`Edit this image: ${prompt}`, requireRuntimeContext(ctx), {
|
|
259
|
-
model: imageModel,
|
|
260
|
-
size: effectiveSize,
|
|
261
|
-
signal: cancelController.signal,
|
|
262
|
-
requestId,
|
|
263
|
-
references: [{ b64: imageB64, declaredMime: null, detectedMime: detectImageMimeFromB64(imageB64) || null }],
|
|
264
|
-
});
|
|
265
|
-
resultB64 = r.b64;
|
|
266
|
-
providerUrl = r.providerUrl ?? null;
|
|
267
|
-
usage = r.usage;
|
|
268
|
-
revisedPrompt = r.revisedPrompt ?? undefined;
|
|
269
|
-
webSearchCalls = r.webSearchCalls;
|
|
270
|
-
resultMimeFromProvider = r.mime;
|
|
271
|
-
}
|
|
272
|
-
else if (activeProvider === "comfy") {
|
|
273
|
-
// i2i goes through the same adapter as generate: the reference is
|
|
274
|
-
// uploaded to the instance and its filename injected into the
|
|
275
|
-
// workflow's LoadImage binding. A graph without that binding refuses
|
|
276
|
-
// with COMFY_WORKFLOW_BIND_INVALID rather than silently ignoring the
|
|
277
|
-
// input image.
|
|
278
|
-
const r = await generateViaComfy(`Edit this image: ${prompt}`, requireRuntimeContext(ctx), {
|
|
279
|
-
model: imageModel,
|
|
280
|
-
size: effectiveSize,
|
|
281
|
-
signal: cancelController.signal,
|
|
282
|
-
requestId,
|
|
283
|
-
references: [{ b64: imageB64, declaredMime: null, detectedMime: detectImageMimeFromB64(imageB64) || null }],
|
|
284
|
-
});
|
|
285
|
-
resultB64 = r.b64;
|
|
286
|
-
providerUrl = r.providerUrl ?? null;
|
|
287
|
-
usage = r.usage;
|
|
288
|
-
revisedPrompt = r.revisedPrompt ?? undefined;
|
|
289
|
-
webSearchCalls = r.webSearchCalls;
|
|
290
|
-
resultMimeFromProvider = r.mime;
|
|
291
|
-
}
|
|
292
|
-
else if (activeProvider === "grok" || activeProvider === "grok-api") {
|
|
293
|
-
const directApiKey = activeProvider === "grok-api" ? ctx.xaiApiKey : undefined;
|
|
294
|
-
const grokModel = resolveGrokQualityModel(imageModel, quality);
|
|
295
|
-
const r = await editViaGrok(prompt, imageB64, ctx, {
|
|
296
|
-
model: grokModel,
|
|
297
|
-
size: effectiveSize,
|
|
298
|
-
signal: cancelController.signal,
|
|
299
|
-
requestId,
|
|
300
|
-
directApiKey,
|
|
301
|
-
});
|
|
302
|
-
resultB64 = r.b64;
|
|
303
|
-
providerUrl = r.providerUrl ?? null;
|
|
304
|
-
usage = r.usage;
|
|
305
|
-
revisedPrompt = r.revisedPrompt;
|
|
306
|
-
webSearchCalls = r.webSearchCalls;
|
|
307
|
-
resultMimeFromProvider = r.mime;
|
|
308
|
-
}
|
|
309
|
-
else {
|
|
310
|
-
const r = await editViaResponses(activeProvider, prompt, imageB64, quality, effectiveSize, moderation, normalizedPromptMode, ctx, requestId, {
|
|
311
|
-
model: imageModel,
|
|
312
|
-
reasoningEffort,
|
|
313
|
-
webSearchEnabled,
|
|
314
|
-
mask: maskCheck.mask,
|
|
315
|
-
signal: cancelController.signal,
|
|
316
|
-
});
|
|
317
|
-
resultB64 = r.b64;
|
|
318
|
-
usage = r.usage ?? null;
|
|
319
|
-
revisedPrompt = r.revisedPrompt ?? undefined;
|
|
320
|
-
webSearchCalls = r.webSearchCalls ?? 0;
|
|
321
|
-
}
|
|
217
|
+
const { value: r } = await (await prepareImageExecution(ctx, {
|
|
218
|
+
surface: "edit", provider: activeProvider, requestId,
|
|
219
|
+
signal: cancelController.signal, prompt, rawPrompt: prompt,
|
|
220
|
+
sourceImage: imageB64, mask: maskCheck.mask ?? null, references: [],
|
|
221
|
+
options: { model: imageModel, quality, size: effectiveSize, moderation,
|
|
222
|
+
mode: normalizedPromptMode, reasoningEffort, webSearchEnabled },
|
|
223
|
+
})).execute();
|
|
224
|
+
const resultB64 = r.b64;
|
|
225
|
+
const usage = r.usage;
|
|
226
|
+
const revisedPrompt = r.revisedPrompt ?? undefined;
|
|
227
|
+
const webSearchCalls = r.webSearchCalls;
|
|
228
|
+
const resultMimeFromProvider = r.mime;
|
|
229
|
+
const providerUrl = r.providerUrl ?? null;
|
|
322
230
|
throwIfJobCanceled(requestId);
|
|
323
231
|
const elapsed = +((Date.now() - startTime) / 1000).toFixed(1);
|
|
324
232
|
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|