agens-studio 0.1.0

package/src/routes/prompt.js

@@ -0,0 +1,279 @@
+import { Router } from 'express';
+import { optimizePrompt, optimizeNegativePrompt, rewritePrompt, generateCandidates } from '../services/mimoClient.js';
+import { describeImage, descriptionToText, reviewImage } from '../services/mimoVision.js';
+import * as mem from '../services/promptMemory.js';
+
+const router = Router();
+
+/**
+ * POST /api/prompt/optimize
+ * body: { text, mode, extraHints?, imageUrl? }
+ *   - imageUrl: 图生图模式必填（本地上传图路径），用于识图
+ * 返回: { optimized, sampleId, dimensions?, imageDescription? }
+ *   - dimensions: 文生图/图生图的结构化维度（前端分维度展示）
+ *   - imageDescription: 图生图/图生视频的识图结果（前端展示原图识别）
+ *   - imageDescriptions: 多图生视频/关键帧的多张识图结果
+ *
+ * 识图流程（两阶段）：
+ *   1. mimo-v2.5 识图（单张逐个，带缓存）→ 结构化描述
+ *   2. mimo-v2.5-pro 基于识图结果 + 用户要求 → 优化
+ * 需要识图的模式：img2img（1张）、image2video（1张）、multi2video（N张）、keyframe（N张）
+ */
+router.post('/optimize', async (req, res) => {
+  const { text, mode, extraHints, imageUrl, imageUrls } = req.body || {};
+  if (!text || !text.trim()) {
+    return res.status(400).json({ error: '请输入要优化的提示词' });
+  }
+  try {
+    // 判断该模式是否需要识图
+    const needVision = ['img2img', 'image2video', 'multi2video', 'keyframe'].includes(mode);
+    // 统一成数组处理：图生图/图生视频单图 → [imageUrl]；多图/关键帧 → imageUrls
+    const visionUrls = needVision
+      ? (mode === 'img2img' || mode === 'image2video'
+          ? (imageUrl ? [imageUrl] : [])
+          : (Array.isArray(imageUrls) ? imageUrls : []))
+      : [];
+
+    if (needVision && visionUrls.length === 0) {
+      return res.status(400).json({ error: '该模式需要上传参考图才能优化' });
+    }
+    if (mode === 'keyframe' && visionUrls.length < 2) {
+      return res.status(400).json({ error: '关键帧动画至少需要首帧和尾帧' });
+    }
+
+    // 识图（单张逐个，带缓存）
+    let imageDescriptionObj = null;      // 单图模式：识图对象
+    let imageDescription = '';           // 单图：拼成文本
+    let multiImageDescriptions = [];     // 多图：[{ index, label, description, cached }]
+    let imageCached = false;
+
+    if (visionUrls.length === 1) {
+      const vision = await describeImage(visionUrls[0]);
+      imageDescriptionObj = vision.description;
+      imageDescription = descriptionToText(imageDescriptionObj);
+      imageCached = vision.cached;
+    } else if (visionUrls.length > 1) {
+      // 多图：逐个识，拼成「图1：...；图2：...」
+      const labels = mode === 'keyframe'
+        ? ['首帧', '尾帧', '中间帧1', '中间帧2', '中间帧3']
+        : ['图1', '图2', '图3', '图4', '图5', '图6'];
+      const parts = [];
+      for (let i = 0; i < visionUrls.length; i++) {
+        const vision = await describeImage(visionUrls[i]);
+        const label = labels[i] || `图${i + 1}`;
+        multiImageDescriptions.push({
+          index: i,
+          label,
+          description: vision.description,
+          cached: vision.cached,
+        });
+        parts.push(`${label}：${descriptionToText(vision.description)}`);
+      }
+      imageDescription = parts.join('\n');
+      imageCached = multiImageDescriptions.every((d) => d.cached);
+    }
+
+    // 优化（带上识图上下文）
+    const result = await optimizePrompt({
+      text: text.trim(),
+      mode,
+      extraHints,
+      imageDescription,
+    });
+
+    // 存入记忆库（带识图结果）
+    const sample = await mem.addSample({
+      kind: 'positive',
+      original: text.trim(),
+      optimized: result.optimized,
+      mode,
+      imageDescription,
+    });
+
+    res.json({
+      optimized: result.optimized,
+      dimensions: result.dimensions || null,
+      sampleId: sample.id,
+      imageDescription: imageDescriptionObj,
+      multiImageDescriptions,
+      imageCached,
+    });
+  } catch (e) {
+    res.status(500).json({ error: '优化失败：' + e.message });
+  }
+});
+
+/**
+ * POST /api/prompt/optimize-negative
+ * body: { text, mode, positive? }
+ * 返回: { optimized, sampleId }
+ *
+ * 反向提示词优化，独立记忆库（kind=negative）
+ */
+router.post('/optimize-negative', async (req, res) => {
+  const { text, mode, positive } = req.body || {};
+  if (!text || !text.trim()) {
+    return res.status(400).json({ error: '请输入要优化的反向提示词' });
+  }
+  try {
+    const { optimized } = await optimizeNegativePrompt({
+      text: text.trim(),
+      mode,
+      positive: positive || '',
+    });
+    const sample = await mem.addSample({ kind: 'negative', original: text.trim(), optimized, mode });
+    res.json({ optimized, sampleId: sample.id });
+  } catch (e) {
+    res.status(500).json({ error: '反向优化失败：' + e.message });
+  }
+});
+
+/**
+ * POST /api/prompt/feedback
+ * body: { sampleId, rating, correction? }   rating: 1=赞, -1=踩
+ *   - correction: 点踩原因（层级1：让模型知道具体哪里不行）
+ */
+router.post('/feedback', async (req, res) => {
+  const { sampleId, rating, correction } = req.body || {};
+  if (!sampleId) return res.status(400).json({ error: '缺少 sampleId' });
+  if (![1, -1].includes(rating)) return res.status(400).json({ error: 'rating 必须是 1 或 -1' });
+  const patch = { rating };
+  // 点踩时附带原因；点赞时清空原因（用户改主意了）
+  if (rating === -1 && typeof correction === 'string') patch.correction = correction;
+  if (rating === 1) patch.correction = '';
+  const s = await mem.updateSample(sampleId, patch);
+  if (!s) return res.status(404).json({ error: '样本不存在' });
+  res.json({ ok: true });
+});
+
+/**
+ * POST /api/prompt/rewrite
+ * 重写：用户带具体修改指令重新优化（层级2）
+ * body: { sampleId, instruction }   sampleId 指明要重写哪条，instruction 是修改要求
+ * 返回: { optimized }  并把重写历史写回该样本
+ */
+router.post('/rewrite', async (req, res) => {
+  const { sampleId, instruction } = req.body || {};
+  if (!sampleId) return res.status(400).json({ error: '缺少 sampleId' });
+  if (!instruction || !instruction.trim()) {
+    return res.status(400).json({ error: '请输入修改要求' });
+  }
+  // 取出该样本，拿到原始输入/模式/上次结果/识图描述
+  const sample = await mem.getSample(sampleId);
+  if (!sample) return res.status(404).json({ error: '样本不存在' });
+
+  try {
+    const result = await rewritePrompt({
+      text: sample.original,
+      mode: sample.mode,
+      lastResult: sample.optimized,
+      instruction: instruction.trim(),
+      isNegative: sample.kind === 'negative',
+      // 图生图重写时把识图结果带上，让 [保留] 维度有依据
+      imageDescription: sample.imageDescription || '',
+    });
+    // 写回重写历史（注意：updateSample 会把 optimized 更新为最新重写结果）
+    await mem.updateSample(sampleId, {
+      rewrite: { instruction: instruction.trim(), result: result.optimized, accepted: false },
+    });
+    // 文生图/图生图：返回 dimensions 让前端分维度渲染
+    res.json({
+      optimized: result.optimized,
+      dimensions: result.dimensions || null,
+    });
+  } catch (e) {
+    res.status(500).json({ error: '重写失败：' + e.message });
+  }
+});
+
+/**
+ * POST /api/prompt/adopt
+ * 标记某个样本被「采用」（用户点了采用按钮）
+ * body: { sampleId }
+ */
+router.post('/adopt', async (req, res) => {
+  const { sampleId } = req.body || {};
+  if (!sampleId) return res.status(400).json({ error: '缺少 sampleId' });
+  const s = await mem.updateSample(sampleId, { adoptedDelta: 1 });
+  if (!s) return res.status(404).json({ error: '样本不存在' });
+  res.json({ ok: true });
+});
+
+/**
+ * POST /api/prompt/candidates
+ * 生成 3 个不同方向的候选提示词
+ * body: { text, mode, imageUrl?, imageUrls? }
+ * 返回: { candidates: [{ label, optimized, dimensions }] }
+ */
+router.post('/candidates', async (req, res) => {
+  const { text, mode, imageUrl, imageUrls } = req.body || {};
+  if (!text || !text.trim()) {
+    return res.status(400).json({ error: '请输入提示词' });
+  }
+  try {
+    // 识图（需要识图的模式）
+    let imageDescription = '';
+    const needVision = ['img2img', 'image2video', 'multi2video', 'keyframe'].includes(mode);
+    if (needVision) {
+      const urls = (mode === 'img2img' || mode === 'image2video')
+        ? (Array.isArray(imageUrls) && imageUrls.length ? imageUrls : (imageUrl ? [imageUrl] : []))
+        : (Array.isArray(imageUrls) ? imageUrls : []);
+      if (urls.length === 0) {
+        return res.status(400).json({ error: '该模式需要上传参考图' });
+      }
+      const parts = [];
+      const labels = mode === 'keyframe' ? ['首帧', '尾帧'] : ['图1', '图2', '图3'];
+      for (let i = 0; i < urls.length; i++) {
+        const v = await describeImage(urls[i]);
+        parts.push(`${labels[i] || '图' + (i+1)}：${descriptionToText(v.description)}`);
+      }
+      imageDescription = parts.join('\n');
+    }
+
+    const { candidates } = await generateCandidates({
+      text: text.trim(),
+      mode,
+      imageDescription,
+    });
+    const optimized = candidates
+      .map((c) => c.optimized || '')
+      .filter(Boolean)
+      .join('\n\n');
+    const sample = await mem.addSample({
+      kind: 'positive',
+      original: text.trim(),
+      optimized,
+      mode,
+      imageDescription,
+    });
+    res.json({ candidates, sampleId: sample.id, imageDescription: imageDescription || null });
+  } catch (e) {
+    res.status(500).json({ error: '候选生成失败：' + e.message });
+  }
+});
+
+/**
+ * POST /api/prompt/review
+ * AI 复看：看 agnes 生成的图，评价 + 改进提示词
+ * body: { assetPath, prompt, mode }
+ * 返回: { rating, comment, problems, suggestion, improvedPrompt }
+ */
+router.post('/review', async (req, res) => {
+  const { assetPath, prompt, mode } = req.body || {};
+  if (!assetPath) return res.status(400).json({ error: '缺少 assetPath' });
+  try {
+    const review = await reviewImage(assetPath, prompt || '', mode || 'image');
+    res.json(review);
+  } catch (e) {
+    res.status(500).json({ error: '复看失败：' + e.message });
+  }
+});
+
+/**
+ * GET /api/prompt/stats —— 记忆库统计（调试/管理用）
+ */
+router.get('/stats', async (req, res) => {
+  res.json(await mem.stats());
+});
+
+export default router;

package/src/routes/task.js

@@ -0,0 +1,19 @@
+import { Router } from 'express';
+import { getTask, recentTasks } from '../services/taskManager.js';
+
+const router = Router();
+
+/** GET /api/task/:id 查单个任务状态 */
+router.get('/:id', (req, res) => {
+  const t = getTask(req.params.id);
+  if (!t) return res.status(404).json({ error: '任务不存在' });
+  res.json(t);
+});
+
+/** GET /api/task 最近任务列表（供前端任务队列展示） */
+router.get('/', (req, res) => {
+  const limit = Math.min(parseInt(req.query.limit, 10) || 20, 100);
+  res.json({ items: recentTasks(limit) });
+});
+
+export default router;

package/src/services/agensClient.js

@@ -0,0 +1,257 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { config } from '../../config.js';
+
+function authHeaders(apiKey) {
+  return {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${apiKey}`,
+  };
+}
+
+function ratioToImageSize(ratio) {
+  const map = {
+    '1:1': '2048x2048',
+    '3:4': '1536x2048',
+    '4:3': '2048x1536',
+    '16:9': '2560x1440',
+    '9:16': '1440x2560',
+  };
+  return map[ratio] || '2048x2048';
+}
+
+function ratioToVideoSize(ratio, resolution) {
+  const r = resolution || '1080p';
+  const short = r === '2k' ? 1440 : r === '1080p' ? 1080 : 720;
+  const long = Math.round(short * 16 / 9);
+  const map = {
+    '16:9': { width: long, height: short },
+    '9:16': { width: short, height: long },
+    '1:1': { width: short, height: short },
+    '4:3': { width: Math.round(short * 4 / 3), height: short },
+    '3:4': { width: short, height: Math.round(short * 4 / 3) },
+  };
+  return map[ratio] || { width: 1920, height: 1080 };
+}
+
+function styleToPromptSuffix(style) {
+  const map = {
+    photographic: ', photographic, realistic, high detail',
+    anime: ', anime style, illustration',
+    '3d': ', 3D render, cgi',
+    art: ', artistic, digital art',
+  };
+  return map[style] || '';
+}
+
+function durationToFrames(seconds) {
+  const fps = 30;
+  const candidates = [81, 121, 161, 201, 241, 281, 321, 361, 401, 441];
+  const target = seconds * fps;
+  let best = candidates[0];
+  let bestDiff = Math.abs(candidates[0] - target);
+  for (const c of candidates) {
+    const d = Math.abs(c - target);
+    if (d < bestDiff) { bestDiff = d; best = c; }
+  }
+  return { num_frames: best, frame_rate: fps };
+}
+
+async function localImageToString(localPath, asDataUri) {
+  let absPath;
+  if (localPath.startsWith('/uploads/')) {
+    absPath = path.join(config.dirs.uploads, path.basename(localPath));
+  } else if (localPath.startsWith('/assets/images/')) {
+    absPath = path.join(config.dirs.images, path.basename(localPath));
+  } else {
+    return localPath;
+  }
+  const buf = await fs.readFile(absPath);
+  const b64 = buf.toString('base64');
+  if (!asDataUri) return b64;
+  const ext = path.extname(absPath).slice(1).toLowerCase();
+  const mime = 'image/' + (ext === 'jpg' ? 'jpeg' : (ext || 'png'));
+  return 'data:' + mime + ';base64,' + b64;
+}
+
+async function agnesFetch(url, opts, options) {
+  const opts2 = options || {};
+  const retries = opts2.retries != null ? opts2.retries : 3;
+  const timeoutMs = opts2.timeoutMs != null ? opts2.timeoutMs : 120000;
+  let lastErr;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    const controller = new AbortController();
+    const timer = setTimeout(function () { controller.abort(); }, timeoutMs);
+    try {
+      const res = await fetch(url, Object.assign({}, opts, { signal: controller.signal }));
+      clearTimeout(timer);
+      if ((res.status === 429 || res.status === 503) && attempt < retries) {
+        const t = await res.text().catch(function () { return ''; });
+        console.log('[agnes] ' + res.status + ' busy, retry ' + (attempt + 1) + ' in 5s: ' + t.slice(0, 100));
+        await new Promise(function (r) { setTimeout(r, 5000); });
+        continue;
+      }
+      return res;
+    } catch (e) {
+      clearTimeout(timer);
+      lastErr = e;
+      if (attempt < retries) {
+        console.log('[agnes] network error, retry ' + (attempt + 1) + ': ' + e.message);
+        await new Promise(function (r) { setTimeout(r, 3000); });
+        continue;
+      }
+    }
+  }
+  throw lastErr || new Error('agnes request failed after retries');
+}
+
+function agensFetch(url, opts, options) {
+  return agnesFetch(url, opts, options);
+}
+
+export function buildImageRequestBody({ prompt, params = {} }) {
+  const cfg = config.agens.image;
+  const size = ratioToImageSize(params.ratio);
+  const fullPrompt = prompt + styleToPromptSuffix(params.style);
+  const body = {
+    model: cfg.model,
+    prompt: fullPrompt,
+    size: size,
+    extra_body: { response_format: 'url' },
+  };
+  if (Array.isArray(params.imageUrls) && params.imageUrls.length > 0) {
+    body.image = params.imageUrls;
+  }
+  return body;
+}
+
+export async function generateImage(input) {
+  const params = input.params || {};
+  const cfg = config.agens.image;
+  const imageUrls = Array.isArray(params.imageUrls) ? params.imageUrls : [];
+  let requestParams = params;
+  if (imageUrls.length > 0) {
+    const images = [];
+    for (let i = 0; i < imageUrls.length; i++) {
+      images.push(await localImageToString(imageUrls[i], true));
+    }
+    requestParams = { ...params, imageUrls: images };
+  }
+  const body = buildImageRequestBody({ prompt: input.prompt, params: requestParams });
+  const res = await agnesFetch(cfg.baseUrl + cfg.endpoint, {
+    method: 'POST',
+    headers: authHeaders(cfg.apiKey),
+    body: JSON.stringify(body),
+  });
+  const text = await res.text();
+  if (!res.ok) {
+    let detail = text;
+    try { detail = JSON.stringify(JSON.parse(text)); } catch (e) {}
+    throw new Error('agnes image failed (' + res.status + '): ' + detail.slice(0, 500));
+  }
+  let data;
+  try { data = JSON.parse(text); } catch (e) { throw new Error('agnes image response invalid JSON'); }
+  var item = data.data && data.data[0];
+  var url = item && item.url;
+  if (!url) {
+    if (item && item.b64_json) {
+      return { url: 'data:image/png;base64,' + item.b64_json, mime: 'image/png', meta: data };
+    }
+    throw new Error('agnes image response missing url');
+  }
+  return { url: url, mime: 'image/png', meta: data };
+}
+
+export async function generateVideo(input) {
+  const mode = input.mode;
+  const prompt = input.prompt;
+  const imageUrls = input.imageUrls || [];
+  const params = input.params || {};
+  const cfg = config.agens.video;
+  const wh = ratioToVideoSize(params.ratio, params.resolution);
+  const fr = durationToFrames(parseInt(params.duration, 10) || 5);
+  var images = [];
+  if (imageUrls.length) {
+    for (let i = 0; i < imageUrls.length; i++) {
+      images.push(await localImageToString(imageUrls[i], false));
+    }
+  }
+  const body = {
+    model: cfg.model,
+    prompt: prompt,
+    width: wh.width,
+    height: wh.height,
+    num_frames: fr.num_frames,
+    frame_rate: fr.frame_rate,
+  };
+  if (params.negativePrompt && params.negativePrompt.trim()) {
+    body.negative_prompt = params.negativePrompt.trim();
+  }
+  if (mode === 'image2video') {
+    body.image = images[0];
+  } else if (mode === 'multi2video') {
+    body.extra_body = { image: images };
+  } else if (mode === 'keyframe') {
+    body.extra_body = { image: images, mode: 'keyframes' };
+  }
+  const res = await agnesFetch(cfg.baseUrl + cfg.createEndpoint, {
+    method: 'POST',
+    headers: authHeaders(cfg.apiKey),
+    body: JSON.stringify(body),
+  });
+  const text = await res.text();
+  if (!res.ok) {
+    let detail = text;
+    try { detail = JSON.stringify(JSON.parse(text)); } catch (e) {}
+    throw new Error('agnes video create failed (' + res.status + '): ' + detail.slice(0, 500));
+  }
+  let data;
+  try { data = JSON.parse(text); } catch (e) { throw new Error('agnes video response invalid JSON'); }
+  const videoId = data.video_id || data.id || data.task_id;
+  if (!videoId) throw new Error('agnes video response missing video_id');
+  return { videoId: videoId, raw: data };
+}
+
+export async function getTaskStatus(videoId) {
+  const cfg = config.agens.video;
+  const url = cfg.queryBaseUrl + cfg.queryPath + '?video_id=' + encodeURIComponent(videoId) + '&model_name=' + encodeURIComponent(cfg.model);
+  const res = await agnesFetch(url, { method: 'GET', headers: authHeaders(cfg.apiKey) });
+  const text = await res.text();
+  if (!res.ok) {
+    throw new Error('agnes video query failed (' + res.status + '): ' + text.slice(0, 200));
+  }
+  let data;
+  try { data = JSON.parse(text); } catch (e) { throw new Error('agnes video query invalid JSON'); }
+  const status = normalizeStatus(data.status);
+  const result = {};
+  if (typeof data.progress === 'number') result.progress = data.progress;
+  if (status === 'success') {
+    result.videoUrl = data.remixed_from_video_id
+      || data.video_url
+      || data.url
+      || data.output
+      || (data.result && data.result.video_url)
+      || (data.result && data.result.url)
+      || (Array.isArray(data.output) ? data.output[0] : null);
+    result.duration = data.seconds ? parseFloat(data.seconds) : undefined;
+    if (!result.videoUrl) {
+      console.error('[agnes] video success but no URL found, response:', JSON.stringify(data).slice(0, 500));
+    }
+  }
+  if (status === 'failed') {
+    result.error = (data.error && (data.error.message || data.error)) || data.message || 'video generation failed';
+  }
+  return { status: status, progress: result.progress, result: result, raw: data };
+}
+
+function normalizeStatus(s) {
+  const v = String(s || '').toLowerCase();
+  if (['completed', 'complete', 'succeed', 'success', 'done'].indexOf(v) >= 0) return 'success';
+  if (['fail', 'failed', 'error'].indexOf(v) >= 0) return 'failed';
+  if (['in_progress', 'processing', 'running', 'generating'].indexOf(v) >= 0) return 'processing';
+  return 'pending';
+}
+
+export function isMockMode() {
+  return false;
+}