ima2-gen 1.1.17 → 1.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -4
- package/bin/commands/capabilities.js +6 -0
- package/bin/commands/capabilities.ts +6 -0
- package/bin/commands/video.js +215 -0
- package/bin/commands/video.ts +205 -0
- package/bin/ima2.js +61 -6
- package/bin/ima2.ts +54 -6
- package/docs/API.md +73 -4
- package/docs/CLI.md +38 -0
- package/docs/README.ja.md +2 -2
- package/docs/README.ko.md +15 -3
- package/docs/README.zh-CN.md +2 -2
- package/lib/agentGenerationPlanner.js +18 -1
- package/lib/agentGenerationPlanner.ts +21 -1
- package/lib/agentRuntime.js +105 -1
- package/lib/agentRuntime.ts +118 -1
- package/lib/agentTypes.js +1 -0
- package/lib/agentTypes.ts +2 -1
- package/lib/assetLifecycle.js +12 -8
- package/lib/assetLifecycle.ts +12 -8
- package/lib/capabilities.js +9 -0
- package/lib/capabilities.ts +9 -0
- package/lib/grokVideoAdapter.js +45 -1
- package/lib/grokVideoAdapter.ts +49 -1
- package/lib/historyList.js +1 -0
- package/lib/historyList.ts +1 -0
- package/lib/imageModels.js +1 -1
- package/lib/imageModels.ts +2 -2
- package/lib/oauthLauncher.js +5 -2
- package/lib/oauthLauncher.ts +5 -3
- package/lib/videoSeriesChain.js +24 -0
- package/lib/videoSeriesChain.ts +29 -0
- package/node_modules/progrok/README.md +300 -22
- package/node_modules/progrok/dist/index.js +558 -173
- package/node_modules/progrok/dist/index.js.map +1 -1
- package/node_modules/progrok/package.json +3 -3
- package/node_modules/progrok/skills/progrok/SKILL.md +145 -109
- package/package.json +2 -2
- package/routes/video.js +10 -1
- package/routes/video.ts +11 -1
- package/ui/dist/.vite/manifest.json +12 -12
- package/ui/dist/assets/AgentWorkspace-DE_wg90f.js +3 -0
- package/ui/dist/assets/{CardNewsWorkspace-6y_HNp3I.js → CardNewsWorkspace--Myc5pAp.js} +1 -1
- package/ui/dist/assets/NodeCanvas-4U5oOT2y.js +7 -0
- package/ui/dist/assets/{PromptBuilderPanel-BQlPtGGm.js → PromptBuilderPanel-DNW1U8zI.js} +2 -2
- package/ui/dist/assets/{PromptImportDialog-aNk40wLt.js → PromptImportDialog-o-4Sqki1.js} +2 -2
- package/ui/dist/assets/{PromptImportDiscoverySection-B6NKkVBz.js → PromptImportDiscoverySection-BAbrRP8B.js} +1 -1
- package/ui/dist/assets/{PromptImportFolderSection-9-xbe-FM.js → PromptImportFolderSection-L-XI2noz.js} +1 -1
- package/ui/dist/assets/{PromptLibraryPanel-CbEY0AM6.js → PromptLibraryPanel-CrW9LYGD.js} +2 -2
- package/ui/dist/assets/{SettingsWorkspace-ao9ymIWt.js → SettingsWorkspace-Dn4SYTyZ.js} +1 -1
- package/ui/dist/assets/index-B6tcw_UF.css +1 -0
- package/ui/dist/assets/{index-DP88bEQf.js → index-BONbNNIi.js} +1 -1
- package/ui/dist/assets/index-CeSZ2L3-.js +32 -0
- package/ui/dist/index.html +2 -2
- package/vendor/progrok-0.1.1.tgz +0 -0
- package/ui/dist/assets/AgentWorkspace-CLHwx6u4.js +0 -3
- package/ui/dist/assets/NodeCanvas-DR2N5Dib.js +0 -7
- package/ui/dist/assets/index-B0re600T.js +0 -32
- package/ui/dist/assets/index-CXJEgTOQ.css +0 -1
- package/vendor/progrok-0.1.0.tgz +0 -0
package/bin/ima2.ts
CHANGED
|
@@ -64,13 +64,15 @@ async function setup() {
|
|
|
64
64
|
|
|
65
65
|
console.log("\n ima2-gen — GPT Image 2 Generator\n");
|
|
66
66
|
console.log(" Choose authentication method:\n");
|
|
67
|
-
console.log(" 1)
|
|
68
|
-
console.log(" 2) OAuth
|
|
67
|
+
console.log(" 1) GPT OAuth — login with ChatGPT account (free, images only)");
|
|
68
|
+
console.log(" 2) Grok OAuth — login with xAI/Grok account (images + video)");
|
|
69
|
+
console.log(" 3) Both — GPT OAuth + Grok OAuth");
|
|
70
|
+
console.log(" 4) API Key — paste your OpenAI API key (paid)\n");
|
|
69
71
|
|
|
70
|
-
const choice = await rl.question(" Enter 1
|
|
72
|
+
const choice = await rl.question(" Enter 1-4: ");
|
|
71
73
|
const config = loadConfig();
|
|
72
74
|
|
|
73
|
-
if (choice.trim() === "
|
|
75
|
+
if (choice.trim() === "4") {
|
|
74
76
|
const key = await rl.question(" OpenAI API Key: ");
|
|
75
77
|
if (!key.startsWith("sk-")) {
|
|
76
78
|
console.log(" Invalid API key format. Expected sk-...");
|
|
@@ -81,13 +83,56 @@ async function setup() {
|
|
|
81
83
|
config.apiKey = key.trim();
|
|
82
84
|
saveConfig(config);
|
|
83
85
|
console.log("\n API key saved. Starting server...\n");
|
|
86
|
+
} else if (choice.trim() === "2") {
|
|
87
|
+
config.provider = "grok";
|
|
88
|
+
config.oauth = config.oauth || {};
|
|
89
|
+
config.oauth.disableAutoStart = true;
|
|
90
|
+
delete config.apiKey;
|
|
91
|
+
saveConfig(config);
|
|
92
|
+
console.log("\n Starting Grok OAuth login...\n");
|
|
93
|
+
try {
|
|
94
|
+
execSync(`node ${JSON.stringify(join(ROOT, "bin", "ima2.js"))} grok login`, { stdio: "inherit" });
|
|
95
|
+
} catch {
|
|
96
|
+
console.log("\n Grok login failed or cancelled. You can retry with 'ima2 grok login'.\n");
|
|
97
|
+
rl.close();
|
|
98
|
+
process.exit(1);
|
|
99
|
+
}
|
|
100
|
+
console.log(" Grok configured. Run 'ima2 serve' to start.\n");
|
|
101
|
+
} else if (choice.trim() === "3") {
|
|
102
|
+
config.provider = "oauth";
|
|
103
|
+
delete config.apiKey;
|
|
104
|
+
if (config.oauth) delete config.oauth.disableAutoStart;
|
|
105
|
+
saveConfig(config);
|
|
106
|
+
console.log("\n Setting up both GPT OAuth + Grok OAuth...\n");
|
|
107
|
+
// GPT OAuth
|
|
108
|
+
const auth = detectCodexAuth();
|
|
109
|
+
if (!auth.authed) {
|
|
110
|
+
console.log(" Running GPT OAuth login...\n");
|
|
111
|
+
try {
|
|
112
|
+
execSync(`${resolveBin("npx")} @openai/codex login`, { stdio: "inherit" });
|
|
113
|
+
} catch {
|
|
114
|
+
console.log("\n GPT login failed. Continuing with Grok...\n");
|
|
115
|
+
}
|
|
116
|
+
} else {
|
|
117
|
+
console.log(` GPT OAuth session found.\n`);
|
|
118
|
+
}
|
|
119
|
+
// Grok OAuth
|
|
120
|
+
console.log(" Running Grok OAuth login...\n");
|
|
121
|
+
try {
|
|
122
|
+
execSync(`node ${JSON.stringify(join(ROOT, "bin", "ima2.js"))} grok login`, { stdio: "inherit" });
|
|
123
|
+
} catch {
|
|
124
|
+
console.log("\n Grok login failed. You can retry with 'ima2 grok login'.\n");
|
|
125
|
+
}
|
|
126
|
+
console.log(" Both providers configured.\n");
|
|
84
127
|
} else {
|
|
128
|
+
// Default: GPT OAuth (choice 1 or anything else)
|
|
85
129
|
config.provider = "oauth";
|
|
130
|
+
config.oauth = config.oauth || {};
|
|
131
|
+
config.oauth.disableAutoStart = false;
|
|
86
132
|
delete config.apiKey;
|
|
87
133
|
saveConfig(config);
|
|
88
134
|
console.log("\n Starting OAuth login...\n");
|
|
89
135
|
|
|
90
|
-
// Check if codex auth exists (file OR keyring via `codex login status`)
|
|
91
136
|
const auth = detectCodexAuth();
|
|
92
137
|
const hasAuth = auth.authed;
|
|
93
138
|
|
|
@@ -232,6 +277,7 @@ function showHelp() {
|
|
|
232
277
|
|
|
233
278
|
Client commands (require a running 'ima2 serve'):
|
|
234
279
|
gen <prompt> Generate image(s) from prompt (ima2 gen --help)
|
|
280
|
+
video <prompt> Generate video via Grok (ima2 video --help)
|
|
235
281
|
edit <file> Edit an existing image (ima2 edit --help)
|
|
236
282
|
ls List recent history (ima2 ls --help)
|
|
237
283
|
show <name> Show one history item (ima2 show --help)
|
|
@@ -277,6 +323,7 @@ function showHelp() {
|
|
|
277
323
|
ima2 serve --dev Start with verbose server diagnostics
|
|
278
324
|
ima2 gen "a shiba in space" Generate from CLI
|
|
279
325
|
ima2 gen "merge" --ref a.png --ref b.png -q high -o out.png
|
|
326
|
+
ima2 video "a cat playing piano" --duration 10
|
|
280
327
|
ima2 ls -n 10 Last 10 generations
|
|
281
328
|
ima2 skill Print agent usage skill
|
|
282
329
|
ima2 capabilities --json Inspect supported models/options
|
|
@@ -295,7 +342,7 @@ if (args.includes("-v") || args.includes("--version")) {
|
|
|
295
342
|
}
|
|
296
343
|
|
|
297
344
|
if ((!command || args.includes("-h") || args.includes("--help"))
|
|
298
|
-
&& !["doctor", "gen", "edit", "ls", "show", "ps", "cancel", "session", "history", "prompt", "multimode", "node", "annotate", "canvas-versions", "metadata", "comfy", "cardnews", "inflight", "storage", "billing", "providers", "oauth", "grok", "config", "defaults", "capabilities", "skill", "ping"].includes(command)) {
|
|
345
|
+
&& !["doctor", "gen", "video", "edit", "ls", "show", "ps", "cancel", "session", "history", "prompt", "multimode", "node", "annotate", "canvas-versions", "metadata", "comfy", "cardnews", "inflight", "storage", "billing", "providers", "oauth", "grok", "config", "defaults", "capabilities", "skill", "ping"].includes(command)) {
|
|
299
346
|
showHelp();
|
|
300
347
|
process.exit(command ? 0 : 1);
|
|
301
348
|
}
|
|
@@ -337,6 +384,7 @@ switch (command) {
|
|
|
337
384
|
}
|
|
338
385
|
break;
|
|
339
386
|
case "gen":
|
|
387
|
+
case "video":
|
|
340
388
|
case "edit":
|
|
341
389
|
case "ls":
|
|
342
390
|
case "show":
|
package/docs/API.md
CHANGED
|
@@ -20,10 +20,8 @@ Image generation supports OAuth, API-key, and Grok providers.
|
|
|
20
20
|
- Grok generation maps `size` to xAI `aspect_ratio` and `resolution`; it does not send an OpenAI-style `size` field upstream. Grok edit uses xAI `/v1/images/edits`; Grok mask edit remains unsupported and returns `GROK_MASK_UNSUPPORTED`.
|
|
21
21
|
- Mask edits are mask/selection guided edits, not pixel-perfect inpaint guarantees.
|
|
22
22
|
|
|
23
|
-
Grok video generation
|
|
24
|
-
|
|
25
|
-
planning and research notes only; no `/api/video` route or Grok video endpoint
|
|
26
|
-
wrapper is shipped in this release.
|
|
23
|
+
Grok video generation uses `POST /api/video/generate` (SSE). See the Video
|
|
24
|
+
Generation section below for the full endpoint specification.
|
|
27
25
|
|
|
28
26
|
## Health And Status
|
|
29
27
|
|
|
@@ -219,6 +217,76 @@ Server-side validation may return these reference codes:
|
|
|
219
217
|
| `GROK_REF_TOO_MANY` | Grok classic generation received more than three reference images |
|
|
220
218
|
| `GROK_MASK_UNSUPPORTED` | Grok edit was requested with a mask; xAI mask edit is not wired in this release |
|
|
221
219
|
|
|
220
|
+
## Video Generation
|
|
221
|
+
|
|
222
|
+
### `POST /api/video/generate` (SSE)
|
|
223
|
+
|
|
224
|
+
Generate a video via the Grok video provider. Returns Server-Sent Events.
|
|
225
|
+
|
|
226
|
+
```json
|
|
227
|
+
{
|
|
228
|
+
"prompt": "a cat playing piano",
|
|
229
|
+
"provider": "grok",
|
|
230
|
+
"model": "grok-imagine-video",
|
|
231
|
+
"duration": 5,
|
|
232
|
+
"resolution": "480p",
|
|
233
|
+
"aspectRatio": "auto",
|
|
234
|
+
"sourceImage": "<base64>",
|
|
235
|
+
"referenceImages": ["<base64>", "<base64>"],
|
|
236
|
+
"referenceFilenames": ["existing-file.png"],
|
|
237
|
+
"sessionId": "optional",
|
|
238
|
+
"requestId": "optional-client-id"
|
|
239
|
+
}
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
**Models**: `grok-imagine-video` (default), `grok-imagine-video-1.5-preview`.
|
|
243
|
+
|
|
244
|
+
**Mode** is auto-detected from reference inputs:
|
|
245
|
+
|
|
246
|
+
| Inputs | Mode | Duration cap |
|
|
247
|
+
|---|---|---|
|
|
248
|
+
| No images | text-to-video | 1–15s |
|
|
249
|
+
| 1 image (`sourceImage` or `sourceFilename`) | image-to-video | 1–15s |
|
|
250
|
+
| 2–7 images (`referenceImages` / `referenceFilenames`) | reference-to-video | 1–10s |
|
|
251
|
+
|
|
252
|
+
**Parameters**:
|
|
253
|
+
|
|
254
|
+
| Field | Type | Default | Notes |
|
|
255
|
+
|---|---|---|---|
|
|
256
|
+
| `prompt` | string | — | Required |
|
|
257
|
+
| `provider` | string | `"grok"` | Must be `"grok"` |
|
|
258
|
+
| `model` | string | `grok-imagine-video` | Video model |
|
|
259
|
+
| `duration` | integer | `5` | 1–15 seconds (clamped to 10 for reference-to-video) |
|
|
260
|
+
| `resolution` | string | `"480p"` | `480p` or `720p` |
|
|
261
|
+
| `aspectRatio` | string | `"auto"` | 1:1, 16:9, 9:16, 4:3, 3:4, 3:2, 2:3, auto |
|
|
262
|
+
| `sourceImage` | string | — | Base64 image for image-to-video |
|
|
263
|
+
| `sourceFilename` | string | — | Existing generated file for image-to-video |
|
|
264
|
+
| `referenceImages` | string[] | — | Base64 images for reference-to-video |
|
|
265
|
+
| `referenceFilenames` | string[] | — | Existing generated files for reference-to-video |
|
|
266
|
+
|
|
267
|
+
**SSE events**:
|
|
268
|
+
|
|
269
|
+
| Event | Data | Description |
|
|
270
|
+
|---|---|---|
|
|
271
|
+
| `planning` | `{ requestId }` | Preparing video generation |
|
|
272
|
+
| `submitted` | `{ requestId, xaiVideoRequestId }` | Submitted to xAI |
|
|
273
|
+
| `progress` | `{ requestId, progress, stalled }` | Progress 0.0–1.0 |
|
|
274
|
+
| `done` | `{ requestId, filename, url, mediaType, revisedPrompt, elapsed, usage, video }` | Video ready |
|
|
275
|
+
| `error` | `{ error, code, status, requestId }` | Generation failed |
|
|
276
|
+
|
|
277
|
+
**Video error codes**:
|
|
278
|
+
|
|
279
|
+
| Code | Meaning |
|
|
280
|
+
|---|---|
|
|
281
|
+
| `VIDEO_PROVIDER_UNSUPPORTED` | Provider is not `"grok"` |
|
|
282
|
+
| `PROMPT_REQUIRED` | Empty or missing prompt |
|
|
283
|
+
| `INVALID_GROK_VIDEO_MODEL` | Model not in valid set |
|
|
284
|
+
| `INVALID_VIDEO_RESOLUTION` | Resolution not 480p or 720p |
|
|
285
|
+
| `INVALID_VIDEO_ASPECT_RATIO` | Aspect ratio not in valid set |
|
|
286
|
+
| `INVALID_VIDEO_DURATION` | Duration not 1–15 integer |
|
|
287
|
+
| `GROK_VIDEO_REF_TOO_MANY` | More than 7 reference images |
|
|
288
|
+
| `GROK_VIDEO_FAILED` | Upstream xAI video generation failed |
|
|
289
|
+
|
|
222
290
|
## History
|
|
223
291
|
|
|
224
292
|
| Method | Path | Notes |
|
|
@@ -299,6 +367,7 @@ Most server routes under `/api/*` have a CLI wrapper. The exception is **Agent M
|
|
|
299
367
|
| `POST /api/generate` | `ima2 gen` |
|
|
300
368
|
| `POST /api/edit` | `ima2 edit` |
|
|
301
369
|
| `POST /api/generate/multimode` (SSE) | `ima2 multimode` |
|
|
370
|
+
| `POST /api/video/generate` (SSE) | `ima2 video` |
|
|
302
371
|
| `POST /api/node/generate` (SSE) / `GET /api/node/:id` | `ima2 node generate` / `ima2 node show` |
|
|
303
372
|
| `GET /api/history` | `ima2 ls` |
|
|
304
373
|
| `DELETE /api/history/:name` / `…/permanent` | `ima2 history rm [--permanent]` |
|
package/docs/CLI.md
CHANGED
|
@@ -49,6 +49,7 @@ Agents should start from the packaged skill and capability commands instead of g
|
|
|
49
49
|
| `ima2 gen <prompt>` | Generate from the CLI |
|
|
50
50
|
| `ima2 edit <file> --prompt <text>` | Edit an existing image |
|
|
51
51
|
| `ima2 multimode <prompt>` | Multi-image SSE generation (streams `phase` / `partial` / `image` events) |
|
|
52
|
+
| `ima2 video <prompt>` | Video generation via Grok (SSE streaming with progress) |
|
|
52
53
|
| `ima2 node generate` | Node-mode generate (SSE; supports `--no-stream`) |
|
|
53
54
|
| `ima2 node show <nodeId>` | Read node metadata |
|
|
54
55
|
|
|
@@ -105,6 +106,43 @@ small text, and pixel-perfect typography can still need iteration or post-editin
|
|
|
105
106
|
|
|
106
107
|
Multimode-specific flags include `--max-images <1..8>`, `--ref <file>` (repeatable, max 5), `--mode <auto|direct>`, `--provider <auto|oauth|api|grok>`, and `--show-partial`. `ima2 edit --mask` remains intentionally deferred to #31 because current mask plumbing is guided edit rather than guaranteed true masked/inpaint semantics.
|
|
107
108
|
|
|
109
|
+
## Video
|
|
110
|
+
|
|
111
|
+
| Command | Description |
|
|
112
|
+
|---|---|
|
|
113
|
+
| `ima2 video <prompt>` | Generate a video via Grok (SSE streaming with progress) |
|
|
114
|
+
|
|
115
|
+
Video flags:
|
|
116
|
+
|
|
117
|
+
| Flag | Meaning |
|
|
118
|
+
|---|---|
|
|
119
|
+
| `--duration <1..15>` | Duration in seconds (default: 5) |
|
|
120
|
+
| `--resolution <480p\|720p>` | Video resolution (default: 480p) |
|
|
121
|
+
| `--aspect-ratio <ratio\|auto>` | 1:1, 16:9, 9:16, 4:3, 3:4, 3:2, 2:3, auto (default: auto) |
|
|
122
|
+
| `--model <name>` | `grok-imagine-video` or `grok-imagine-video-1.5-preview` |
|
|
123
|
+
| `--ref <file>` | Attach source/reference image (repeatable, max 7) |
|
|
124
|
+
| `-o, --out <file>` | Output file path |
|
|
125
|
+
| `-d, --out-dir <dir>` | Output directory |
|
|
126
|
+
| `--timeout <sec>` | Timeout in seconds (default: 600) |
|
|
127
|
+
| `--session <id>` | Session ID |
|
|
128
|
+
|
|
129
|
+
Video mode is auto-detected from `--ref` count:
|
|
130
|
+
|
|
131
|
+
| Refs | Mode |
|
|
132
|
+
|---|---|
|
|
133
|
+
| 0 | text-to-video |
|
|
134
|
+
| 1 | image-to-video |
|
|
135
|
+
| 2–7 | reference-to-video (max 10s duration) |
|
|
136
|
+
|
|
137
|
+
SSE events: `planning` → `submitted` → `progress` (0–100%) → `done` or `error`.
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
ima2 video "a cat playing piano"
|
|
141
|
+
ima2 video "animate this" --ref photo.png --duration 10
|
|
142
|
+
ima2 video "cinematic" --resolution 720p --aspect-ratio 16:9 -o out.mp4
|
|
143
|
+
ima2 video "style transfer" --ref a.png --ref b.png --ref c.png --model grok-imagine-video-1.5-preview
|
|
144
|
+
```
|
|
145
|
+
|
|
108
146
|
## Diagnostics
|
|
109
147
|
|
|
110
148
|
`ima2 doctor image-probe` runs live Responses probes that help classify image
|
package/docs/README.ja.md
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
>
|
|
9
9
|
> **他の言語で読む**: [English](../README.md) · [한국어](README.ko.md) · [简体中文](README.zh-CN.md)
|
|
10
10
|
|
|
11
|
-
`ima2-gen`
|
|
11
|
+
`ima2-gen` は、無料の ChatGPT と SuperGrok だけで画像と動画を作れるローカル AI スタジオです。
|
|
12
12
|
|
|
13
|
-
`npx` で起動し、
|
|
13
|
+
`npx` で起動し、ChatGPT または Grok OAuth でログインすれば、すぐに画像・動画生成を始められます。API キー不要で、ノード分岐、multimode batch、Grok Video、Canvas Mode まで全機能が使えます。
|
|
14
14
|
|
|
15
15
|

|
|
16
16
|
|
package/docs/README.ko.md
CHANGED
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
>
|
|
11
11
|
> **다른 언어로 읽기**: [English](../README.md) · [日本語](README.ja.md) · [简体中文](README.zh-CN.md)
|
|
12
12
|
|
|
13
|
-
`ima2-gen`은 ChatGPT
|
|
13
|
+
`ima2-gen`은 무료 ChatGPT와 SuperGrok만으로 이미지와 영상을 만드는 로컬 AI 스튜디오입니다.
|
|
14
14
|
|
|
15
|
-
`npx
|
|
15
|
+
`npx` 한 줄로 실행하고, ChatGPT 또는 Grok OAuth로 로그인하면 바로 시작됩니다. API 키 없이 이미지 생성, 비디오 생성, 노드 분기, 멀티모드 배치, Canvas 정리까지 전부 가능합니다.
|
|
16
16
|
|
|
17
17
|

|
|
18
18
|
|
|
@@ -40,12 +40,24 @@ npm install -g ima2-gen
|
|
|
40
40
|
ima2 serve
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
### 설정
|
|
44
|
+
|
|
45
|
+
`ima2 setup`으로 인증 방식을 선택합니다:
|
|
46
|
+
|
|
47
|
+
1. **GPT OAuth** — ChatGPT 계정으로 로그인 (무료, 이미지만)
|
|
48
|
+
2. **Grok OAuth** — xAI/Grok 계정으로 로그인 (이미지 + 영상)
|
|
49
|
+
3. **Both** — GPT + Grok 둘 다 (전체 기능)
|
|
50
|
+
4. **API Key** — OpenAI API 키 입력 (유료)
|
|
51
|
+
|
|
52
|
+
영상 생성은 Grok OAuth(2번 또는 3번)가 필요합니다.
|
|
53
|
+
|
|
43
54
|
## 무엇을 할 수 있나요?
|
|
44
55
|
|
|
45
56
|
- **Classic mode**: 빠르게 이미지를 만들고, 수정하고, 현재 결과를 다시 레퍼런스로 사용합니다.
|
|
46
57
|
- **Node mode**: 마음에 드는 이미지를 여러 방향으로 분기해 실험합니다.
|
|
47
58
|
- **Multimode batches**: 하나의 프롬프트에서 여러 후보 슬롯을 동시에 만들고, 가장 좋은 결과에서 이어갑니다.
|
|
48
59
|
- **Canvas Mode**: 확대/이동, 주석, 지우개, 배경 정리, 투명 체크보드 미리보기, alpha/matte export를 지원합니다.
|
|
60
|
+
- **Video 생성**: 텍스트, 이미지, 또는 여러 레퍼런스에서 짧은 영상을 만듭니다. 기획→제출→진행률→완료까지 실시간으로 보여줍니다.
|
|
49
61
|
- **Local gallery**: 생성물을 내 컴퓨터에 저장하고 세션별 히스토리로 봅니다.
|
|
50
62
|
- **Reference images**: 레퍼런스를 드래그, 붙여넣기, 파일 선택으로 추가합니다. 큰 이미지는 업로드 전에 자동 압축됩니다.
|
|
51
63
|
- **Prompt library imports**: 로컬 prompt pack, GitHub folder, curated GPT-image hint를 내장 prompt library로 가져옵니다.
|
|
@@ -62,7 +74,7 @@ ima2 serve
|
|
|
62
74
|
|
|
63
75
|
Grok은 Classic, Node, Agent 흐름을 지원합니다. Classic 레퍼런스, Node 부모 이미지, Agent 현재 이미지가 있으면 최종 Grok 호출은 xAI image edit 경로로 전환되어 image-to-image 맥락을 유지합니다. 기본 모델은 `grok-imagine-image`이고, `quality: "high"`에서는 `grok-imagine-image-quality`를 사용합니다.
|
|
64
76
|
|
|
65
|
-
Grok video 생성(T2V/I2V)은
|
|
77
|
+
Grok video 생성(T2V/I2V/ref2v)은 v1.1.16부터 사용 가능합니다. 텍스트 프롬프트, 단일 이미지, 또는 최대 7장의 레퍼런스에서 짧은 영상을 만들 수 있으며, 실시간 진행률 스트리밍을 지원합니다.
|
|
66
78
|
|
|
67
79
|
설정 화면에 **API key provider available**이나 **Grok provider available**이 보이면 해당 공급자가 감지됐고 생성 요청에 사용할 수 있다는 뜻입니다.
|
|
68
80
|
|
package/docs/README.zh-CN.md
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
>
|
|
9
9
|
> **其他语言**: [English](../README.md) · [한국어](README.ko.md) · [日本語](README.ja.md)
|
|
10
10
|
|
|
11
|
-
`ima2-gen`
|
|
11
|
+
`ima2-gen` 是一个本地 AI 工作室,只需免费 ChatGPT 和 SuperGrok 即可生成图像和视频。
|
|
12
12
|
|
|
13
|
-
用 `npx`
|
|
13
|
+
用 `npx` 启动,通过 ChatGPT 或 Grok OAuth 登录即可开始生成图像和视频。无需 API 密钥,节点分支、multimode 批量、Grok Video、Canvas Mode 全部可用。
|
|
14
14
|
|
|
15
15
|

|
|
16
16
|
|
|
@@ -34,6 +34,19 @@ export function deriveAgentGenerationPlan({ prompt, settings, command = null })
|
|
|
34
34
|
assistantText: null,
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
|
+
if (isVideoIntent(prompt)) {
|
|
38
|
+
return {
|
|
39
|
+
mode: "video",
|
|
40
|
+
prompts: [prompt],
|
|
41
|
+
requestedVariants: 1,
|
|
42
|
+
plannedVariants: 1,
|
|
43
|
+
plannedParallelism: 1,
|
|
44
|
+
source: "auto-request",
|
|
45
|
+
reason: "Video generation detected from prompt keywords.",
|
|
46
|
+
command: command?.name ?? null,
|
|
47
|
+
assistantText: null,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
37
50
|
const variantDecision = decideVariantCount(prompt, settings, command);
|
|
38
51
|
const plannedParallelism = resolvePlannedParallelism(settings, variantDecision.count, command);
|
|
39
52
|
const prompts = buildGenerationPrompts(prompt, variantDecision.count);
|
|
@@ -58,7 +71,7 @@ export function normalizeAgentGenerationPlan(prompt, value, settings) {
|
|
|
58
71
|
const requestedParallelism = cleanCount(input.plannedParallelism, settings.parallelism, 1, HARD_MAX_VARIANTS);
|
|
59
72
|
const plannedParallelism = resolvePlannedParallelism({ ...settings, parallelism: requestedParallelism }, plannedVariants, null);
|
|
60
73
|
return {
|
|
61
|
-
mode: input.mode === "question" ? "question" : prompts.length > 1 ? "fanout" : "single",
|
|
74
|
+
mode: input.mode === "question" ? "question" : input.mode === "video" ? "video" : prompts.length > 1 ? "fanout" : "single",
|
|
62
75
|
prompts,
|
|
63
76
|
requestedVariants: cleanCount(input.requestedVariants, plannedVariants, 0, HARD_MAX_VARIANTS),
|
|
64
77
|
plannedVariants,
|
|
@@ -183,3 +196,7 @@ function cleanCount(value, fallback, min, max) {
|
|
|
183
196
|
function clampCount(value, max) {
|
|
184
197
|
return Math.max(1, Math.min(max, Math.round(value)));
|
|
185
198
|
}
|
|
199
|
+
const VIDEO_INTENT_PATTERN = /\b(?:video|animate|animation|동영상|비디오|영상|애니메이트|움직이|클립)\b/iu;
|
|
200
|
+
function isVideoIntent(prompt) {
|
|
201
|
+
return VIDEO_INTENT_PATTERN.test(prompt);
|
|
202
|
+
}
|
|
@@ -56,6 +56,20 @@ export function deriveAgentGenerationPlan({ prompt, settings, command = null }:
|
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
if (isVideoIntent(prompt)) {
|
|
60
|
+
return {
|
|
61
|
+
mode: "video",
|
|
62
|
+
prompts: [prompt],
|
|
63
|
+
requestedVariants: 1,
|
|
64
|
+
plannedVariants: 1,
|
|
65
|
+
plannedParallelism: 1,
|
|
66
|
+
source: "auto-request",
|
|
67
|
+
reason: "Video generation detected from prompt keywords.",
|
|
68
|
+
command: command?.name ?? null,
|
|
69
|
+
assistantText: null,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
59
73
|
const variantDecision = decideVariantCount(prompt, settings, command);
|
|
60
74
|
const plannedParallelism = resolvePlannedParallelism(settings, variantDecision.count, command);
|
|
61
75
|
const prompts = buildGenerationPrompts(prompt, variantDecision.count);
|
|
@@ -85,7 +99,7 @@ export function normalizeAgentGenerationPlan(
|
|
|
85
99
|
const requestedParallelism = cleanCount(input.plannedParallelism, settings.parallelism, 1, HARD_MAX_VARIANTS);
|
|
86
100
|
const plannedParallelism = resolvePlannedParallelism({ ...settings, parallelism: requestedParallelism }, plannedVariants, null);
|
|
87
101
|
return {
|
|
88
|
-
mode: input.mode === "question" ? "question" : prompts.length > 1 ? "fanout" : "single",
|
|
102
|
+
mode: input.mode === "question" ? "question" : input.mode === "video" ? "video" : prompts.length > 1 ? "fanout" : "single",
|
|
89
103
|
prompts,
|
|
90
104
|
requestedVariants: cleanCount(input.requestedVariants, plannedVariants, 0, HARD_MAX_VARIANTS),
|
|
91
105
|
plannedVariants,
|
|
@@ -227,3 +241,9 @@ function cleanCount(value: unknown, fallback: number, min: number, max: number):
|
|
|
227
241
|
function clampCount(value: number, max: number): number {
|
|
228
242
|
return Math.max(1, Math.min(max, Math.round(value)));
|
|
229
243
|
}
|
|
244
|
+
|
|
245
|
+
const VIDEO_INTENT_PATTERN = /\b(?:video|animate|animation|동영상|비디오|영상|애니메이트|움직이|클립)\b/iu;
|
|
246
|
+
|
|
247
|
+
function isVideoIntent(prompt: string): boolean {
|
|
248
|
+
return VIDEO_INTENT_PATTERN.test(prompt);
|
|
249
|
+
}
|
package/lib/agentRuntime.js
CHANGED
|
@@ -9,6 +9,7 @@ import { detectImageMimeFromB64 } from "./refs.js";
|
|
|
9
9
|
import { resolveProviderOptions } from "./providerOptions.js";
|
|
10
10
|
import { generateViaResponses } from "./responsesImageAdapter.js";
|
|
11
11
|
import { generateViaGrok } from "./grokImageAdapter.js";
|
|
12
|
+
import { generateVideoViaGrok } from "./grokVideoAdapter.js";
|
|
12
13
|
import { appendAgentTurn, buildImageContextManifest, getAgentImages, getAgentSession, importAgentImage, recordAgentWebFinding, restartAgentRuntimeSession, } from "./agentStore.js";
|
|
13
14
|
import { AGENT_ALLOWED_TOOLS, } from "./agentTypes.js";
|
|
14
15
|
import { errInfo } from "./errInfo.js";
|
|
@@ -46,7 +47,7 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
46
47
|
const webSearchEnabled = options.provider === "grok" ? true : options.webSearchEnabled ?? session.webSearchEnabled;
|
|
47
48
|
const enabledTools = webSearchEnabled
|
|
48
49
|
? [...AGENT_ALLOWED_TOOLS]
|
|
49
|
-
: ["ima2.get_image_context", "ima2.generate_image"];
|
|
50
|
+
: ["ima2.get_image_context", "ima2.generate_image", "ima2.generate_video"];
|
|
50
51
|
assertAgentAllowedTools(enabledTools);
|
|
51
52
|
if (behavior.appendUserTurn !== false) {
|
|
52
53
|
appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
|
|
@@ -62,6 +63,13 @@ export async function runAgentGenerationPlan(ctx, sessionId, prompt, plan, optio
|
|
|
62
63
|
});
|
|
63
64
|
return { assistantTurn, imageIds: [], webFindingIds: [] };
|
|
64
65
|
}
|
|
66
|
+
if (plan.mode === "video") {
|
|
67
|
+
return runAgentVideoGeneration(ctx, sessionId, prompt, {
|
|
68
|
+
...options,
|
|
69
|
+
requestId: options.requestId ?? `agent_video_${ulid()}`,
|
|
70
|
+
skipUserTurn: true,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
65
73
|
const manifest = buildImageContextManifest(sessionId);
|
|
66
74
|
const contextStartedAt = Date.now();
|
|
67
75
|
appendAgentTurn({
|
|
@@ -306,6 +314,102 @@ async function persistAgentImage(ctx, sessionId, prompt, format, requestId, resp
|
|
|
306
314
|
createdAt: Date.now(),
|
|
307
315
|
});
|
|
308
316
|
}
|
|
317
|
+
export async function runAgentVideoGeneration(ctx, sessionId, prompt, options = {}) {
|
|
318
|
+
const session = getAgentSession(sessionId);
|
|
319
|
+
if (!session)
|
|
320
|
+
throw notFound(sessionId);
|
|
321
|
+
if (!options.skipUserTurn) {
|
|
322
|
+
appendAgentTurn({ sessionId, role: "user", text: prompt, status: "complete" });
|
|
323
|
+
}
|
|
324
|
+
const requestId = options.requestId ?? `agent_video_${ulid()}`;
|
|
325
|
+
const startedAt = Date.now();
|
|
326
|
+
// Auto I2V: if session has a last image, use it as source
|
|
327
|
+
let sourceImage;
|
|
328
|
+
let mode = "text-to-video";
|
|
329
|
+
if (session.lastImageId) {
|
|
330
|
+
const images = getAgentImages(sessionId);
|
|
331
|
+
const lastImage = images.find((img) => img.id === session.lastImageId);
|
|
332
|
+
if (lastImage?.filename && !lastImage.filename.endsWith(".mp4")) {
|
|
333
|
+
try {
|
|
334
|
+
const { loadAssetB64 } = await import("./nodeStore.js");
|
|
335
|
+
sourceImage = await loadAssetB64(ctx.rootDir, lastImage.filename, ctx.config.storage.generatedDir);
|
|
336
|
+
mode = "image-to-video";
|
|
337
|
+
}
|
|
338
|
+
catch { /* fallback to T2V */ }
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
const result = await generateVideoViaGrok(prompt, ctx, {
|
|
342
|
+
model: "grok-imagine-video",
|
|
343
|
+
mode,
|
|
344
|
+
sourceImage,
|
|
345
|
+
duration: 5,
|
|
346
|
+
resolution: "480p",
|
|
347
|
+
aspectRatio: "auto",
|
|
348
|
+
requestId,
|
|
349
|
+
signal: options.signal ?? undefined,
|
|
350
|
+
});
|
|
351
|
+
const video = await persistAgentVideo(ctx, sessionId, prompt, requestId, result);
|
|
352
|
+
const finishedAt = Date.now();
|
|
353
|
+
const toolCall = {
|
|
354
|
+
id: `tc_video_${ulid()}`,
|
|
355
|
+
name: "ima2.generate_video",
|
|
356
|
+
status: "complete",
|
|
357
|
+
startedAt,
|
|
358
|
+
finishedAt,
|
|
359
|
+
durationMs: finishedAt - startedAt,
|
|
360
|
+
requestId,
|
|
361
|
+
inputSummary: prompt,
|
|
362
|
+
outputSummary: `Generated video ${video.filename}.`,
|
|
363
|
+
imageIds: [video.id],
|
|
364
|
+
};
|
|
365
|
+
appendAgentTurn({
|
|
366
|
+
sessionId,
|
|
367
|
+
role: "tool",
|
|
368
|
+
text: "ima2.generate_video",
|
|
369
|
+
imageIds: [video.id],
|
|
370
|
+
status: "complete",
|
|
371
|
+
raw: { toolCalls: [toolCall] },
|
|
372
|
+
});
|
|
373
|
+
const assistantTurn = appendAgentTurn({
|
|
374
|
+
sessionId,
|
|
375
|
+
role: "assistant",
|
|
376
|
+
text: `Generated 1 video artifact. ${result.revisedPrompt}`,
|
|
377
|
+
imageIds: [video.id],
|
|
378
|
+
status: "complete",
|
|
379
|
+
});
|
|
380
|
+
return { assistantTurn, imageIds: [video.id], webFindingIds: [] };
|
|
381
|
+
}
|
|
382
|
+
async function persistAgentVideo(ctx, sessionId, prompt, requestId, result) {
|
|
383
|
+
await mkdir(ctx.config.storage.generatedDir, { recursive: true });
|
|
384
|
+
const rand = randomBytes(ctx.config.ids.generatedHexBytes).toString("hex");
|
|
385
|
+
const filename = `${Date.now()}_${rand}_agent.mp4`;
|
|
386
|
+
const meta = {
|
|
387
|
+
kind: "agent",
|
|
388
|
+
mediaType: "video",
|
|
389
|
+
requestId,
|
|
390
|
+
sessionId,
|
|
391
|
+
prompt,
|
|
392
|
+
userPrompt: prompt,
|
|
393
|
+
revisedPrompt: result.revisedPrompt,
|
|
394
|
+
provider: "grok",
|
|
395
|
+
model: "grok-imagine-video",
|
|
396
|
+
createdAt: Date.now(),
|
|
397
|
+
usage: result.usage,
|
|
398
|
+
webSearchCalls: result.webSearchCalls,
|
|
399
|
+
};
|
|
400
|
+
await writeFile(join(ctx.config.storage.generatedDir, filename), result.videoBuffer);
|
|
401
|
+
await writeFile(join(ctx.config.storage.generatedDir, `${filename}.json`), JSON.stringify(meta)).catch(() => { });
|
|
402
|
+
invalidateHistoryIndex();
|
|
403
|
+
logEvent("agent", "video_saved", { requestId, sessionId, filename });
|
|
404
|
+
return importAgentImage(sessionId, {
|
|
405
|
+
id: `ai_${ulid()}`,
|
|
406
|
+
filename,
|
|
407
|
+
url: `/generated/${filename}`,
|
|
408
|
+
prompt,
|
|
409
|
+
revisedPrompt: result.revisedPrompt,
|
|
410
|
+
createdAt: Date.now(),
|
|
411
|
+
});
|
|
412
|
+
}
|
|
309
413
|
function recordSearchFindings(sessionId, prompt, count, provider) {
|
|
310
414
|
if (!count)
|
|
311
415
|
return [];
|