@mokuyoaxis/dsh-iris 0.2.0 → 0.2.1

package/lib/actions.js

@@ -1,4 +1,5 @@
 'use strict';
+import { visionInputLimits } from './vision-image-input.js';
 /**
  * Iris 动作执行层（阶段 5 GUI 操作面板）。
  *
@@ -11,6 +12,7 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import crypto from 'node:crypto';
 import * as adapters from './adapters.js';
 import * as store from './config.js';
 import * as cap from './capability.js';
@@ -18,28 +20,35 @@ import * as models from './models.js';
 import * as tasks from './tasks.js';
 import * as artifacts from './artifacts.js';
 import { registerMedia, mediaLinksOf, webBase } from './media.js';
-import { buildVisionBackendsFromHost, askWithBackends, testVisionCapability, RED_TEST_IMAGE } from './vision.js';
+import { RED_TEST_IMAGE } from './vision.js';
+import { runVisionRequest, readVisionImageFile, probeVisionModel, visionImageFromDataUrl } from './vision-model-routing.js';
 import { HOST_CONTRACT_VERSION, HOST_PORTS, defineHostAdapter, hasHostPort, requireHostPort } from './host-contract.js';
-import { coreSnapshotForDsh, cropForDsh, diffForDsh, framesForDsh, inspectProviderTaskForDsh, submitProviderTaskForDsh, watchProviderTaskForDsh } from './dsh-core-adapter.js';
-import { imageDimensions } from './pixels.js';
-import { locateObject } from './locate.js';
-import { longOcr } from './ocr.js';
+import { coreSnapshotForDsh, cropForDsh, diffForDsh, framesForDsh, summaryFramesForDsh, visionImageForDsh, htmlForDsh, inspectProviderTaskForDsh, submitProviderTaskForDsh, watchProviderTaskForDsh } from './dsh-core-adapter.js';
+import { runLocateRequest, runSummaryRequest } from './composite-vision-routing.js';
+import { normalizeSummarySource, summaryFrameRecords, summaryTimelineLabel } from './summary-input.js';
+import { normalizeVisionImageSource } from './vision-input.js';
+import { formatOcrResult } from './ocr.js';
+import { runOcrRequest } from './ocr-model-routing.js';
 import { ffmpegAvailable, extractFrames, probeVideo } from './media-probe.js';
-import { buildContactSheet, summarizeMedia } from './summarize.js';
 import { ensurePrivateDir, writePrivateFile } from './private-storage.js';
-import { selectMediaProtocol } from './provider-protocol.js';
+import { selectMediaProtocol, selectImageProtocol, providerForImageModel } from './provider-protocol.js';
 import { providerTaskBinding } from './provider-catalog.js';
-import { ProviderContractError, submitWithAcceptanceBoundary } from './provider-contract.js';
 import { invokeProviderOperation, providerPollTaskView, prepareProviderInput } from './provider-adapter.js';
-import { createConfiguredProviderAdapter } from './provider-adapters.js';
+import { createConfiguredProviderAdapter as createProviderAdapter } from './provider-adapters.js';
 import { deriveUserState } from './task-semantics.js';
 import { normalizeGenerationInput } from './generation-input.js';
+import { videoTaskCandidates } from './video-input.js';
 
 const MEDIA_TYPES = {
   '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
   '.webp': 'image/webp', '.gif': 'image/gif'
 };
 
+function createConfiguredProviderAdapter(provider, options = {}) {
+  return createProviderAdapter(provider, { ...options,
+    onRateLimit: options.allowRateLimited ? undefined : store.recordRateLimit });
+}
+
 const EMPTY_ACTION_HOST = defineHostAdapter({
   id: 'iris-hostless',
   ports: {},
@@ -59,22 +68,6 @@ function actionHost(value) {
 // 工作台清空可选文本框会发送空字符串；在 Host 边界解释为未提供，保持默认值。
 function optionalActionText(value) { return value === '' ? undefined : value; }
 
-/* ---------------- 视觉效果自述（辅助 iris_draw_image） ---------------- */
-async function describeFirstImage(host, ref, absPath, originalPrompt) {
-  try {
-    const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
-    const buf = fs.readFileSync(absPath);
-    const dataUrl = `data:${mt};base64,${buf.toString('base64')}`;
-    const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION) });
-    const { answer } = await askWithHealth(backends, {
-      question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是：「${String(originalPrompt).slice(0, 200)}」`,
-      imageDataUrl: dataUrl,
-      signal: undefined
-    });
-    return answer.slice(0, 300);
-  } catch (_) { return ''; }
-}
-
 /* ---------------- IO 目录 ---------------- */
 function outputDir() {
   const dir = path.join(store.irisHome(), 'outputs');
@@ -99,6 +92,11 @@ function selectProviders(capability, label, requestedModelRef) {
   if (requested) {
     const pool = models.modelPool(store.providers());
     const selected = models.resolvePoolModel(pool, requested, capability);
+    if (selected?.rateLimited) throw Object.assign(new Error(store.modelHealth(selected.providerId, selected.id, capability).retryAt
+      ? 'iris: 指定模型正在限流冷却，请稍后使用或手动实测'
+      : 'iris: 指定模型额度或预算耗尽，请手动实测成功后再使用'), {
+      code: 'IRIS_PROVIDER_MODEL_RATE_LIMITED'
+    });
     const field = models.CAP_FIELD[capability];
     providers = selected ? providers.filter((p) => p.id === selected.providerId && p[field] === selected.id) : [];
     if (!providers.length) throw new Error('iris: 指定的模型复合引用不可用或不具备' + label + '能力');
@@ -127,8 +125,9 @@ function routeSelectionReason(provider, capability, requested) {
 }
 
 function imageRoute(provider, args) {
-  const protocol = selectMediaProtocol(provider).mediaProtocol;
   const model = configuredModel(provider, cap.CAPABILITIES.IMAGE, args.model);
+  provider = providerForImageModel(provider, model);
+  const protocol = selectMediaProtocol(provider).mediaProtocol;
   return {
     provider, protocol, model, modelRef: models.modelRef(provider.id, model),
     selectionReason: routeSelectionReason(provider, cap.CAPABILITIES.IMAGE, args.model),
@@ -193,20 +192,15 @@ function recordVisionHealth(errors) {
   }
 }
 
-async function askWithHealth(backends, request) {
+async function askSingleVisionWithHealth(host, request) {
   try {
-    const result = await askWithBackends(backends, request);
+    const result = await runVisionRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, ...request });
     recordVisionHealth(result.errors);
-    if (result.via === 'selfstack') {
-      recordProviderHealthSafe(result.backendId, result.model, cap.CAPABILITIES.VISION, {
-        ok: true, category: 'success', note: '视觉请求成功'
-      });
-    }
+    if (result.via === 'selfstack') recordProviderHealthSafe(result.backendId, result.model, cap.CAPABILITIES.VISION, {
+      ok: true, category: 'success', note: '视觉请求成功'
+    });
     return result;
-  } catch (error) {
-    recordVisionHealth(error && error.errors);
-    throw error;
-  }
+  } catch (error) { recordVisionHealth(error?.errors); throw error; }
 }
 
 function recordCoreTaskAttemptHealth(task, capability) {
@@ -229,7 +223,7 @@ function recordCoreTaskAttemptHealth(task, capability) {
 }
 
 /** DSH 图片入口：同步和异步候选共用一份 Core Task/Attempt/Artifact 事实。 */
-async function submitCoreImage(routes, providerInput, signal) {
+async function submitCoreImage(routes, providerInput, signal, sourceArtifactId) {
   const submitted = await submitProviderTaskForDsh({
     capability: 'image',
     candidates: routes.map((route) => ({
@@ -239,6 +233,7 @@ async function submitCoreImage(routes, providerInput, signal) {
       providerBinding: providerTaskBinding(route.provider)
     })),
     providerInput,
+    sourceArtifactId,
     signal
   });
   const task = submitted.task;
@@ -263,24 +258,25 @@ async function submitCoreImage(routes, providerInput, signal) {
 /**
  * DSH 视频入口（E 阶段）：t2v/i2v 共用 Core Task/Attempt/Artifact 事实轴，
  * 不共用图片状态机——视频走自己的交付 Profile（video/mp4 / generated-video）。
- * s2v 数字人（需先上传首帧与音频到临时存储）仍走 legacy 链路，见 register('video')。
+ * s2v 在写前 Attempt 之后按账号上传首帧和音频，交付复用视频 Profile。
  */
 async function submitCoreVideo(routes, frame, prompt, args, signal) {
-  const { providerInput } = normalizeGenerationInput('video', {
-    prompt,
-    img_data_url: frame ? fileDataUrl(frame) : undefined,
+  const normalized = normalizeGenerationInput('video', {
+    ...(prompt ? { prompt } : {}),
+    ...(frame ? { first_frame_path: frame.absPath } : {}),
+    ...(args.audio_path ? { audio_path: args.audio_path, resolution: args.resolution } : {}),
     size: optionalActionText(args.size),
     duration: args.duration
-  });
+  }, { allowVideoPaths: true });
   const submitted = await submitProviderTaskForDsh({
     capability: 'video',
-    candidates: routes.map((route) => ({
+    candidates: videoTaskCandidates(routes.map((route) => ({
       adapter: route.adapter,
       model: route.modelRef,
       selectionReason: route.selectionReason,
       providerBinding: providerTaskBinding(route.provider)
-    })),
-    providerInput,
+    })), normalized),
+    providerInput: normalized.providerInput,
     signal
   });
   const task = submitted.task;
@@ -313,12 +309,26 @@ function findGeneratedAttachment(attachmentId) {
   return null;
 }
 
-function resolveFirstFrameFile(args) {
+async function resolveFirstFrameFile(host, args, signal) {
   if (args.first_frame_attachment_id) {
     const hit = findGeneratedAttachment(String(args.first_frame_attachment_id).trim());
-    if (!hit) throw new Error('iris: 找不到该 first_frame_attachment_id');
-    if (!fs.existsSync(hit.absPath)) throw new Error('iris: 首帧图片的本地缓存已被清理');
-    return hit;
+    if (hit) {
+      if (!fs.existsSync(hit.absPath)) throw new Error('iris: 首帧图片的本地缓存已被清理');
+      return hit;
+    }
+    const sessions = requireHostPort(host, 'sessions', '读取首帧附件');
+    const attachments = requireHostPort(host, 'attachments', '读取首帧附件');
+    const ref = await sessions.findImageAttachment(args.session_id, String(args.first_frame_attachment_id).trim());
+    if (!ref) throw new Error('iris: 找不到本会话中的 first_frame_attachment_id');
+    const stored = await attachments.readImage(ref, signal);
+    signal?.throwIfAborted();
+    const mediaType = stored.mediaType || ref.mediaType;
+    const extension = Object.keys(MEDIA_TYPES).find(key => MEDIA_TYPES[key] === mediaType);
+    if (!extension) throw new Error('iris: 不支持的首帧图片格式');
+    const bytes = Buffer.from(stored.data);
+    const file = path.join(store.irisHome(), 'uploads', 'frame-' + crypto.createHash('sha256').update(bytes).digest('hex') + extension);
+    if (!fs.existsSync(file)) writePrivateFile(file, bytes, { flag: 'wx' });
+    return { absPath: file, mediaType };
   }
   if (args.first_frame_path) {
     const p = String(args.first_frame_path).trim();
@@ -331,10 +341,6 @@ function resolveFirstFrameFile(args) {
   return null;
 }
 
-function fileDataUrl(file) {
-  return 'data:' + file.mediaType + ';base64,' + fs.readFileSync(file.absPath).toString('base64');
-}
-
 function videoRoute(provider, args, frame) {
   const model = configuredModel(provider, cap.CAPABILITIES.VIDEO, args.model);
   const protocol = selectMediaProtocol(provider).mediaProtocol;
@@ -352,126 +358,6 @@ function videoRoute(provider, args, frame) {
   };
 }
 
-function beforeGenerationError(error, stage) {
-  const category = error && (error.category === 'network' || error instanceof TypeError)
-    ? 'network'
-    : (stage === 'validate' ? 'invalid_request' : 'local_io');
-  return new ProviderContractError(String((error && error.message) || error || '提交前准备失败'), {
-    stage,
-    category,
-    acceptance: 'not_accepted',
-    retryable: category === 'network'
-  });
-}
-
-async function submitVideoV2(routes, frame, prompt, args, signal) {
-  const task = tasks.createV2({
-    cap: 'video', prompt: prompt.slice(0, 4000),
-    ...(args.model ? { requestedModelRef: String(args.model) } : {}),
-    ...(args._retryOf ? { retryOf: String(args._retryOf) } : {})
-  });
-  const candidates = routes.map((route) => ({
-    id: route.provider.id,
-    model: route.modelRef,
-    route,
-    submit: async (_input, context) => {
-      const attemptId = context.attempt.id;
-      let submitArgs;
-      if (route.isS2V) {
-        if (!frame) throw beforeGenerationError(new Error('iris: s2v 数字人需要首帧'), 'validate');
-        const audioPath = String(args.audio_path || '').trim();
-        if (!audioPath || !path.isAbsolute(audioPath)) {
-          throw beforeGenerationError(new Error('iris: s2v 的 audio_path 必须是绝对路径'), 'validate');
-        }
-        if (!fs.existsSync(audioPath)) throw beforeGenerationError(new Error('iris: 音频文件不存在'), 'validate');
-        tasks.markAttemptStage(task.id, attemptId, 'upload');
-        const { url: imageUrl } = await prepareProviderInput(route.adapter, {
-          model: route.model, filePath: frame.absPath, signal
-        });
-        const { url: audioUrl } = await prepareProviderInput(route.adapter, {
-          model: route.model, filePath: audioPath, signal
-        });
-        submitArgs = { imgDataUrl: imageUrl, audioUrl, resolution: args.resolution || '480P' };
-      } else {
-        if (!prompt) throw beforeGenerationError(new Error('iris: 文生/图生视频需要 prompt'), 'validate');
-        tasks.markAttemptStage(task.id, attemptId, 'prepare');
-        try {
-          submitArgs = {
-            prompt,
-            imgDataUrl: frame ? fileDataUrl(frame) : undefined,
-            size: args.size || '1280*720',
-            duration: args.duration ? Number(args.duration) : undefined
-          };
-        } catch (error) {
-          throw beforeGenerationError(error, 'prepare');
-        }
-      }
-      tasks.markAttemptStage(task.id, attemptId, 'submit');
-      return observeProviderCapability(route, cap.CAPABILITIES.VIDEO, () =>
-        invokeProviderOperation(route.adapter, 'submit', {
-          capability: 'video', model: route.model, input: submitArgs, signal
-        }));
-    }
-  }));
-
-  const submitted = await submitWithAcceptanceBoundary(candidates, { capability: 'video' }, {
-    beforeAttempt: ({ providerId, model }) => {
-      const route = routes.find((item) => item.provider.id === providerId && item.modelRef === model);
-      if (!route) throw new Error('候选路由与 Attempt 身份不一致');
-      const attempt = tasks.beginAttempt(task.id, {
-        providerId, providerName: route.provider.name, model, protocol: route.protocol
-      });
-      tasks.update(task.id, {
-        mode: route.mode,
-        ...(route.isS2V ? { resolution: args.resolution || '480P' } : {
-          size: args.size || '1280*720', ...(args.duration ? { duration: Number(args.duration) } : {})
-        })
-      });
-      return attempt;
-    },
-    afterResult: (snapshot) => tasks.recordAttemptResult(task.id, snapshot)
-  });
-
-  if (submitted.localError) {
-    throw taskActionError(task.id, '视频提交结果未能可靠落盘，已停止后续候选', submitted.localError);
-  }
-  if (!submitted.candidate) {
-    const lastError = submitted.result && submitted.result.error;
-    tasks.finalizeNoAcceptance(task.id, lastError || '所有视频候选均明确未受理');
-    throw taskActionError(task.id, '所有视频候选均明确未受理', submitted.attempts);
-  }
-  if (submitted.result.kind !== 'accepted') {
-    const safe = submitted.result.error && submitted.result.error.safeMessage;
-    throw taskActionError(task.id, '视频受理状态未知，禁止自动重提' + (safe ? '：' + safe : ''), submitted.result.error);
-  }
-
-  const route = submitted.candidate.route;
-  tasks.activateWatch(task.id);
-  tasks.watch(tasks.get(task.id), {
-    poll: ({ remoteTaskId: id, signal: pollSignal }) => observeProviderCapability(
-      route, cap.CAPABILITIES.VIDEO,
-      () => invokeProviderOperation(route.adapter, 'poll', {
-        capability: 'video', remoteTaskId: id, signal: pollSignal
-      }),
-      { recordSuccess: false }
-    ),
-    intervalMs: 6000,
-    onSuccess: async (record, result) => {
-      const names = [];
-      for (const url of result.urls || []) {
-        const name = record.id + '-' + names.length + '.' + adapters.extFromUrl(url, 'mp4');
-        await invokeProviderOperation(route.adapter, 'download', {
-          artifact: { kind: 'remote-url', url }, targetPath: path.join(outputDir(), name)
-        });
-        names.push(name);
-      }
-      for (const name of names) registerMedia(record.id, path.join(outputDir(), name));
-      return names;
-    }
-  });
-  return { task: tasks.get(task.id), provider: route.provider, model: route.model, remoteTaskId: submitted.result.remoteTaskId };
-}
-
 function simpleModelRoute(provider, args, capability) {
   const protocol = selectMediaProtocol(provider).mediaProtocol;
   const model = configuredModel(provider, capability, args.model);
@@ -493,7 +379,8 @@ export function taskPollDeps(provider, task) {
     : 'dashscope');
   let providerAdapter;
   try {
-    providerAdapter = createConfiguredProviderAdapter(provider, { protocol });
+    providerAdapter = createConfiguredProviderAdapter(provider, { protocol,
+      modelId: models.parseModelRef(String(task.modelRef || task.model || ''))?.modelId || String(task.model || '') });
   } catch (_) {
     return null;
   }
@@ -575,14 +462,16 @@ export async function runAction(host, name, args, { signal } = {}) {
 /* ============ 动作实现 ============ */
 
 /* ---------- 🖼️ 画图 ---------- */
-register('image', async (host, args, { signal }) => {
-  const { providerInput } = normalizeGenerationInput('image', {
-    prompt: args.prompt, size: optionalActionText(args.size), n: args.n
+async function imageAction(host, args, { signal }, editing = false) {
+  if (editing && !args.source_artifact_id) throw new Error('iris: 改图必须提供 source_artifact_id');
+  const { providerInput, sourceArtifactId } = normalizeGenerationInput('image', {
+    prompt: args.prompt, size: optionalActionText(args.size), n: args.n,
+    ...(editing ? { source_artifact_id: args.source_artifact_id } : {})
   });
 
   const imageProviders = selectProviders(cap.CAPABILITIES.IMAGE, '图像', args.model);
   const routes = imageProviders.map((provider) => imageRoute(provider, args));
-  const submitted = await submitCoreImage(routes, providerInput, signal);
+  const submitted = await submitCoreImage(routes, providerInput, signal, sourceArtifactId);
   const firstArtifactId = submitted.task.artifactIds[0];
   const ready = submitted.task.outcome === 'succeeded'
     && submitted.task.deliveryState === 'ready' && Boolean(firstArtifactId);
@@ -590,44 +479,36 @@ register('image', async (host, args, { signal }) => {
     ok: true, storage: 'core', taskId: submitted.task.id, model: submitted.model,
     providerId: submitted.provider.id, remoteTaskId: submitted.task.remoteTaskId || null,
     artifactIds: submitted.task.artifactIds,
+    ...(sourceArtifactId ? { sourceArtifactId } : {}),
     ...(ready ? { imageUrl: '/iris/api/core/artifact/' + encodeURIComponent(firstArtifactId) + '/media' } : {}),
     text: ready
       ? '[iris] 图像任务已完成（' + submitted.model + '，Core task: ' + submitted.task.id + '）'
       : '[iris] 图像任务已提交并由 Core 后台观察（' + submitted.model + '，Core task: ' + submitted.task.id + '）'
   };
-});
+}
+register('image', imageAction);
+register('image_edit', (host, args, context) => imageAction(host, args, context, true));
 
 /* ---------- 🎬 视频生成 ---------- */
 register('video', async (host, args, { signal }) => {
   const prompt = String(args.prompt || '').trim();
-  const frame = resolveFirstFrameFile(args);
+  const frame = await resolveFirstFrameFile(host, args, signal);
   const providers = selectProviders(cap.CAPABILITIES.VIDEO, '视频', args.model);
   const routes = providers.map((provider) => videoRoute(provider, args, frame));
-  // s2v 数字人需要在受理边界前按候选上传首帧+音频，暂留 legacy 链路（见 DSH_CORE_MIGRATION）。
-  // 判定看显式意图（audio_path 或 model 指定 s2v），不看候选池——池里登记 s2v 模型
-  // 不应把 t2v/i2v 请求挡回 legacy。
+  // 请求子类型决定候选过滤；池中存在 s2v 不影响普通视频。
   const hasAudioPath = Boolean(String(args.audio_path || '').trim());
   const s2v = hasAudioPath || (args.model && /s2v/i.test(String(args.model)));
   // B-7：audio_path 与选中模型必须语义一致。若用户提供了音频却选中非 s2v 模型，
   // 旧实现会落入 else 分支静默丢弃 audio_path 并生成普通视频（可能产生非预期计费）。
   // 这里在受理边界与任何网络调用之前明确拒绝，不猜测用户意图。
-  if (hasAudioPath && routes.some((route) => !route.s2vModel)) {
+  if (hasAudioPath && args.model && routes.some((route) => !route.s2vModel)) {
     const mismatched = routes.filter((route) => !route.s2vModel).map((route) => route.model).join(', ');
     throw new Error('iris: audio_path 只能配合支持 s2v 数字人的模型；当前候选 ' + mismatched
-      + ' 不是 s2v 模型。请改用 s2v 模型（如 wan2.2-s2v-flash），或移除 audio_path 走文生/图生视频。');
-  }
-  if (s2v) {
-    if (!prompt && !args.audio_path) throw new Error('iris: prompt 不能为空');
-    const submitted = await submitVideoV2(routes, frame, prompt, args, signal);
-    return {
-      ok: true, storage: 'legacy', taskId: submitted.task.id, model: submitted.model, mode: submitted.task.mode,
-      providerId: submitted.provider.id, remoteTaskId: submitted.remoteTaskId,
-      text: '[iris] 视频任务已提交（' + submitted.model + '，task: ' + submitted.task.id + '）'
-    };
+      + ' 不是 s2v 模型。请改用 s2v 模型（如 wan2.2-s2v），或移除 audio_path 走文生/图生视频。');
   }
-  const coreRoutes = routes.filter((route) => !route.isS2V);
+  const coreRoutes = routes.filter((route) => route.isS2V === Boolean(s2v));
   if (!coreRoutes.length) {
-    throw new Error('iris: 当前视频候选只有 s2v 数字人模型；请使用 audio_path 的数字人入口');
+    throw new Error(s2v ? 'iris: 未配置可用 s2v 数字人模型' : 'iris: 当前视频候选只有 s2v 数字人模型；请使用 audio_path 的数字人入口');
   }
   const submitted = await submitCoreVideo(coreRoutes, frame, prompt, args, signal);
   const firstArtifactId = submitted.task.artifactIds[0];
@@ -745,16 +626,21 @@ async function submitCoreTranscribe(routes, normalized, signal) {
 
 /* ---------- 👁 看图 ---------- */
 register('look', async (host, args, { signal }) => {
+  const source = normalizeVisionImageSource(args);
   const p = String(args.image_path || '').trim();
-  if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
   const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
-  if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式');
-  if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
-  const dataUrl = `data:${mt};base64,${fs.readFileSync(p).toString('base64')}`;
-  const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model });
-  const { answer, via, model } = await askWithHealth(backends, { question: (args.question || '详细描述这张图片。'), imageDataUrl: dataUrl, signal });
+  if (source === 'image_path') {
+    if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
+    if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式');
+    if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
+  }
+  const { answer, via, model } = await askSingleVisionWithHealth(host, { model: args.model,
+    question: String(args.question || '').trim() || '详细描述这张图片。',
+    prepareImage: operationSignal => source === 'artifact_id'
+      ? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mt, operationSignal), signal });
   const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
-  return { ok: true, text: `[iris] 看图回答（${model} · ${viaLabel}）：\n${answer}` };
+  return { ok: true, ...(args.artifact_id ? { artifactId: args.artifact_id } : {}),
+    text: `[iris] 看图回答（${model} · ${viaLabel}）：\n${answer}` };
 });
 
 /* ---------- ✂️ 裁剪 ---------- */
@@ -789,39 +675,47 @@ register('diff', async (host, args, { signal }) => {
 
 /* ---------- 📍 定位 ---------- */
 register('locate', async (host, args, { signal }) => {
+  const source = normalizeVisionImageSource(args);
   const p = String(args.image_path || '').trim();
-  if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
-  if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
-  const buf = fs.readFileSync(p);
+  if (source === 'image_path' && (!p || !path.isAbsolute(p))) throw new Error('iris: image_path 必须是绝对路径');
   const mt = MEDIA_TYPES[path.extname(p).toLowerCase()] || 'image/png';
-  const dataUrl = `data:${mt};base64,${buf.toString('base64')}`;
-  const { width, height } = await imageDimensions(buf);
-  const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model });
-  const r = await locateObject(backends, { target: args.target, imageDataUrl: dataUrl, width, height, signal });
-  if (!r.found) return { ok: true, text: `[iris] 在图片中未找到「${args.target}」` };
-  return { ok: true, text: `[iris] 定位「${args.target}」：bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}（${r.via} · ${r.model}）` };
+  const r = await runLocateRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model,
+    target: args.target, prepareImage: operationSignal => source === 'artifact_id'
+      ? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mt, operationSignal), signal });
+  const { width, height } = r;
+  const provenance = args.artifact_id ? { artifactId: args.artifact_id } : {};
+  if (!r.found) return { ok: true, ...provenance, text: `[iris] 在图片中未找到「${args.target}」` };
+  return { ok: true, ...provenance, text: `[iris] 定位「${args.target}」：bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}（${r.via} · ${r.model}）` };
 });
 /* ---------- 🖼️ HTML 截图 ---------- */
 register('html', async (host, args, { signal }) => {
   const html = String(args.html || '').trim();
   if (!html) throw new Error('iris: html 不能为空');
   const browser = requireHostPort(host, 'browser', 'HTML 截图');
-  const rendered = await browser.renderHtml({ html, width: args.width, height: args.height,
+  const rendered = await htmlForDsh({ browser, html, width: args.width, height: args.height,
     fullPage: args.fullPage !== false, signal });
   const png = Buffer.from(rendered.bytes);
-  return { ok: true, text: `[iris] HTML 截图完成（fullPage: ${args.fullPage !== false}）`, imageDataUrl: `data:${rendered.mediaType || 'image/png'};base64,${png.toString('base64')}` };
+  return { ok: true, storage: 'core', artifactIds: [rendered.artifact.id], text: `[iris] HTML 截图完成（fullPage: ${args.fullPage !== false}）`, imageDataUrl: `data:image/png;base64,${png.toString('base64')}` };
 });
 
 /* ---------- 📄 长截图 OCR ---------- */
 register('ocr', async (host, args, { signal }) => {
+  const source = normalizeVisionImageSource(args);
   const p = String(args.image_path || '').trim();
-  if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
-  if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
-  const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION) });
-  const result = await longOcr({ input: p, backends, chunkHeight: args.chunk_height || 1200, overlap: args.overlap || 120, signal });
-  const errCount = result.chunks.filter((c) => c.error).length;
-  const head = `[iris] 长截图 OCR 完成：${result.width}x${result.height}px，${result.totalChunks} 块${errCount ? '（' + errCount + ' 块失败）' : ''}\n`;
-  return { ok: true, text: result.fullText ? head + result.fullText : head + '（未识别到文字）' };
+  const mediaType = MEDIA_TYPES[path.extname(p).toLowerCase()];
+  if (source === 'image_path') {
+    if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
+    if (!mediaType || mediaType === 'image/gif') throw new Error('iris: 不支持的图片格式（png/jpg/jpeg/webp）');
+  }
+  const result = await runOcrRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION),
+    onRateLimit: store.recordRateLimit,
+    prepareImage: operationSignal => source === 'artifact_id'
+      ? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mediaType, operationSignal),
+    chunkHeight: args.chunk_height, overlap: args.overlap, signal });
+  return { ok: result.status !== 'failed', status: result.status,
+    totalChunks: result.totalChunks, successfulChunks: result.successfulChunks,
+    failedChunks: result.failedChunks, skippedChunks: result.skippedChunks,
+    ...(args.artifact_id ? { artifactId: args.artifact_id } : {}), text: formatOcrResult(result) };
 });
 
 /* ---------- 📋 任务查询 ---------- */
@@ -1070,22 +964,23 @@ register('relook', async (host, args, { signal }) => {
   if (!id) throw new Error('iris: attachment_id 不能为空');
   const question = String(args.question || '').trim();
   if (!question) throw new Error('iris: question 不能为空');
-  // GUI 无会话事件扫描能力：只在 iris 自己的任务产物里找（findOwnAttachment 逻辑）
-  let hit = null;
-  for (const t of tasks.all()) {
-    for (const a of t.attachments || []) {
-      if (a.attachmentId === id) {
-        const absPath = path.join(store.irisHome(), 'outputs', a.file);
-        if (fs.existsSync(absPath)) { hit = { absPath, mediaType: a.mediaType }; break; }
+  const { answer, via, model } = await askSingleVisionWithHealth(host, { question, model: args.model,
+    async prepareImage(operationSignal) {
+      // GUI 没有会话上下文；记录扫描也在预取消检查及整体预算之后执行。
+      let hit;
+      for (const task of tasks.all()) {
+        for (const attachment of task.attachments || []) {
+          if (attachment.attachmentId === id) {
+            const absPath = path.join(store.irisHome(), 'outputs', attachment.file);
+            if (fs.existsSync(absPath)) { hit = { absPath, mediaType: attachment.mediaType }; break; }
+          }
+        }
+        if (hit) break;
       }
-    }
-    if (hit) break;
-  }
-  if (!hit) throw new Error('iris: 该 attachment 不在 iris 生成的任务产物中（GUI 重看仅支持 iris 画图产物）');
-  const mt = hit.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png';
-  const dataUrl = `data:${mt};base64,${fs.readFileSync(hit.absPath).toString('base64')}`;
-  const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model });
-  const { answer, via, model } = await askWithHealth(backends, { question, imageDataUrl: dataUrl, signal });
+      if (!hit) throw new Error('iris: 图片引用不可用');
+      const mt = hit.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png';
+      return readVisionImageFile(hit.absPath, mt, operationSignal);
+    }, signal });
   const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
   return { ok: true, text: `[iris] 重看回答（${model} · ${viaLabel}）：\n${answer}` };
 });
@@ -1106,10 +1001,15 @@ register('providers_list', async (host, args) => {
       enabled: p.enabled !== false,
       auth: p.auth === 'none' ? 'none' : 'bearer',
       ...selectMediaProtocol(p),
+      visionInput: p.visionInput || null,
       apiKeyHint: String(p.apiKey || '').length <= 8 ? '****' : String(p.apiKey).slice(0, 3) + '****' + String(p.apiKey).slice(-4),
       models: models.providerModels(p).map((m) => ({
         id: m.id,
         capabilities: m.capabilities,
+        ...(m.imageProtocol !== undefined ? { imageProtocol: m.imageProtocol } : {}),
+        ...(m.visionInput !== undefined ? { visionInput: m.visionInput } : {}),
+        ...(m.capabilities.includes(cap.CAPABILITIES.VISION) ? { visionInputEffective: visionInputLimits(p, m.id) } : {}),
+        ...(m.capabilities.includes(cap.CAPABILITIES.IMAGE) ? { imageRouting: selectImageProtocol(p, m.id) } : {}),
         verified: m.verified || null,
         health: Object.fromEntries(m.capabilities.map((capability) => [
           capability, store.modelHealth(p.id, m.id, capability)
@@ -1126,7 +1026,7 @@ register('providers_upsert', async (host, args) => {
   const id = args.id || undefined;
   const patch = { id };
   const allowed = ['auth', 'name', 'type', 'baseUrl', 'mediaBaseUrl', 'apiKey', 'enabled', 'mediaProtocol',
-    'imageModel', 'videoModel', 'ttsModel', 'transcribeModel', 'visionModel', 'models'];
+    'imageModel', 'videoModel', 'ttsModel', 'transcribeModel', 'visionModel', 'visionInput', 'models'];
   for (const key of allowed) {
     if (Object.prototype.hasOwnProperty.call(args, key)) patch[key] = args[key];
   }
@@ -1179,14 +1079,12 @@ register('providers_discover', async (host, args, { signal }) => {
 register('providers_test_vision', async (host, args, { signal }) => {
   const p = store.providerById(String(args.id || ''));
   if (!p) throw new Error('iris: 供应商不存在');
-  const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION).filter((x) => x.id === p.id);
+  const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION, { includeRateLimited: true }).filter((x) => x.id === p.id);
   if (!visionProviders.length) return { ok: true, tested: false, text: '该供应商未配置 vision 模型，跳过' };
-  const backends = buildVisionBackendsFromHost(host, { providers: visionProviders });
-  const backend = backends[0];
-  if (!backend) return { ok: true, tested: false, text: '该供应商的 vision 配置不可用（类型非 openai）' };
+  const model = configuredModel(visionProviders[0], cap.CAPABILITIES.VISION);
   if (args.confirm_paid !== true) throw new Error('iris: 视觉实测会发起真实供应商请求，请先明确确认');
-  const r = await testVisionCapability(backend, { timeoutMs: 15000, signal });
-  store.setModelVerified(p.id, backend.model, cap.CAPABILITIES.VISION, {
+  const r = await probeVisionModel(visionProviders[0], model, visionImageFromDataUrl(RED_TEST_IMAGE), { timeoutMs: 15000, signal });
+  store.setModelVerified(p.id, model, cap.CAPABILITIES.VISION, {
     ok: r.ok,
     note: r.ok ? '认出红色测试图' : (r.error || '未识别出红色'),
     category: r.category,
@@ -1197,7 +1095,7 @@ register('providers_test_vision', async (host, args, { signal }) => {
     tested: true,
     passed: r.ok,
     text: r.ok
-      ? `[iris] 视觉能力实测通过：模型认出红色测试图（${backend.model}）`
+      ? `[iris] 视觉能力实测通过：模型认出红色测试图（${model}）`
       : `[iris] 视觉能力实测未通过：${r.error || '未识别出红色'}`
   };
 });
@@ -1207,9 +1105,7 @@ register('providers_test_vision', async (host, args, { signal }) => {
 /** 按能力实测单个模型；返回 {ok, note}。真实调用仅在 UI 明确确认后执行。 */
 async function probeModel(host, p, modelId, capability, signal) {
   if (capability === cap.CAPABILITIES.VISION) {
-    const backend = buildVisionBackendsFromHost(host, { providers: [p], model: modelId })[0];
-    if (!backend) return { ok: false, note: '非 openai 类型，无法直测' };
-    const r = await testVisionCapability(backend, { timeoutMs: 15000, signal });
+    const r = await probeVisionModel(p, modelId, visionImageFromDataUrl(RED_TEST_IMAGE), { timeoutMs: 15000, signal });
     return {
       ok: r.ok,
       note: r.ok ? '认出红色测试图' : (r.error || '未识别出红色'),
@@ -1218,24 +1114,28 @@ async function probeModel(host, p, modelId, capability, signal) {
     };
   }
   if (capability === cap.CAPABILITIES.TRANSCRIBE) {
-    return { skipped: true, ok: false, note: '需真实音频，请通过“音频转写”动作验证' };
+    return { skipped: true, ok: false, note: '需真实音频；可用 CLI models test --capability transcribe --input 提供素材进行实测恢复' };
   }
   if (capability === cap.CAPABILITIES.TTS) {
-    const providerAdapter = createConfiguredProviderAdapter(p);
+    const providerAdapter = createConfiguredProviderAdapter(p, { allowRateLimited: true });
     const submitted = await invokeProviderOperation(providerAdapter, 'submit', {
       capability: 'tts', model: modelId, input: { text: '你好' }, signal
     });
     const value = submitted.kind === 'completed' ? submitted.value : null;
     const ok = Boolean(value && (value.audioUrl || value.audioB64));
-    return { ok, note: ok ? '合成返回音频' : (submitted.error?.safeMessage || '无音频返回') };
+    return { ok, note: ok ? '合成返回音频' : (submitted.error?.safeMessage || '无音频返回'),
+      category: submitted.error?.category, httpStatus: submitted.error?.httpStatus };
   }
   if (capability === cap.CAPABILITIES.IMAGE) {
-    const providerAdapter = createConfiguredProviderAdapter(p);
+    p = providerForImageModel(p, modelId);
+    const providerAdapter = createConfiguredProviderAdapter(p, { allowRateLimited: true, modelId });
     const submitted = await invokeProviderOperation(providerAdapter, 'submit', {
       capability: 'image', model: modelId,
       input: {
         prompt: providerAdapter.protocol === 'openai-images' ? 'red circle' : '红色圆形',
-        size: providerAdapter.protocol === 'openai-images' ? '256x256' : '512*512',
+        ...(['openai-chat-images', 'openai-responses-images'].includes(providerAdapter.protocol) ? {} : {
+          size: providerAdapter.protocol === 'openai-images' ? '256x256' : '512*512'
+        }),
         n: 1
       },
       signal
@@ -1272,7 +1172,7 @@ async function probeModel(host, p, modelId, capability, signal) {
     return { ok: false, note: '生成超时（>60s）' };
   }
   if (capability === cap.CAPABILITIES.VIDEO) {
-    return { skipped: true, ok: false, note: '视频提交即可能计费，请通过“视频生成”动作验证' };
+    return { skipped: true, ok: false, note: '视频提交即可能计费；可用 CLI models test --capability video-gen --input 提供素材进行实测恢复' };
   }
   return { ok: false, note: '未知能力' };
 }
@@ -1305,6 +1205,19 @@ register('providers_set_model_caps', async (host, args) => {
   return { ok: true, text: `[iris] ${args.model_id} 能力已设为：${caps.join(',') || '（空）'}` };
 });
 
+register('providers_set_model_image_protocol', async (host, args) => {
+  const ok = store.setModelImageProtocol(String(args.id || ''), String(args.model_id || ''), args.imageProtocol);
+  if (!ok) throw new Error('iris: 模型不在池中');
+  return { ok: true, text: '[iris] 模型图片协议已更新' };
+});
+
+register('providers_set_model_vision_input', async (host, args) => {
+  if (!Object.hasOwn(args, 'visionInput')) throw new Error('iris: 需要 visionInput 对象或 null');
+  const ok = store.setModelVisionInput(String(args.id || ''), String(args.model_id || ''), args.visionInput);
+  if (!ok) throw new Error('iris: 模型不在池中');
+  return { ok: true, text: '[iris] 模型看图输入预算已更新' };
+});
+
 /** 逐模型实测某能力 → 存 verified（"test it, don't guess it"；用户手动触发，发现不自动跑） */
 register('providers_test_model', async (host, args, { signal }) => {
   const p = store.providerById(String(args.id || ''));
@@ -1491,28 +1404,32 @@ register('video_frames', async (host, args, { signal }) => {
 /* ============ 多模态视频摘要（阶段 7.3，GUI 同步快速版） ============ */
 
 register('media_summarize', async (host, args, { signal }) => {
+  const source = normalizeSummarySource(args);
   const p = String(args.video_path || '').trim();
-  if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
-  if (!fs.existsSync(p)) throw new Error('iris: 视频文件不存在');
-  if (!ffmpegAvailable()) throw new Error('iris: ffmpeg/ffprobe 不可用，无法分析视频（可选安装 ffmpeg 后启用）');
-  const meta = probeVideo(p);
-  const frames = await extractFrames({
-    inputPath: p, maxFrames: Math.min(Number(args.max_frames) || 8, 12), targetWidth: args.target_width,
-    format: 'jpeg', quality: 70, signal
-  });
-  const sheet = await buildContactSheet({ frames, cols: 3 });
-  const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model });
-  const { answer, via, model } = await summarizeMedia({
-    backends, question: args.question,
-    transcript: args.transcribe_text, // GUI 同步版：不自动转写，可粘贴已有转写文本
-    imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
-    signal
+  if (source === 'video') {
+    if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
+    if (!fs.existsSync(p)) throw new Error('iris: 视频文件不存在');
+    if (!ffmpegAvailable()) throw new Error('iris: ffmpeg/ffprobe 不可用，无法分析视频（可选安装 ffmpeg 后启用）');
+  }
+  const { answer, via, model, sheet, meta, frames } = await runSummaryRequest(host, {
+    onRateLimit: store.recordRateLimit,
+    providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model, question: args.question, signal,
+    async prepareMedia(operationSignal) {
+      if (source === 'core-artifacts') {
+        return { ...await summaryFramesForDsh({ frameArtifactIds: args.frame_artifact_ids, signal: operationSignal }), transcript: args.transcribe_text };
+      }
+      const meta = probeVideo(p);
+      const frames = await extractFrames({ inputPath: p, maxFrames: Math.min(Number(args.max_frames) || 8, 12),
+        targetWidth: args.target_width, format: 'jpeg', quality: 70, signal: operationSignal });
+      return { meta, frames, transcript: args.transcribe_text };
+    }
   });
   const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
   const hasTrans = args.transcribe_text && String(args.transcribe_text).trim();
   return {
     ok: true,
-    text: `[iris] 视频摘要完成（${model} · ${viaLabel}，${meta.durationSec.toFixed(1)}s / ${frames.length} 帧联系表${hasTrans ? ' + 转写文本' : ''}）：\n${answer}`,
+    frames: summaryFrameRecords(frames),
+    text: `[iris] 视频摘要完成（${model} · ${viaLabel}，${summaryTimelineLabel(meta)} / ${frames.length} 帧联系表${hasTrans ? ' + 转写文本' : ''}）：\n${answer}`,
     imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`
   };
 });