@mokuyoaxis/dsh-iris 0.1.4 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.dsh/skills/iris-compose-media/SKILL.md +10 -5
  2. package/.dsh/skills/iris-compose-media/references/workflows.md +7 -1
  3. package/.dsh/skills/iris-verify-ui/SKILL.md +2 -1
  4. package/CHANGELOG.md +99 -1
  5. package/README.en.md +124 -31
  6. package/README.md +124 -31
  7. package/bin/dsh-iris.js +593 -13
  8. package/docs/AGENT_QUICKSTART.md +39 -0
  9. package/docs/ARCHITECTURE.md +87 -64
  10. package/docs/ARTIFACT_LIBRARY.md +5 -5
  11. package/docs/ARTIFACT_MANIFEST.md +60 -0
  12. package/docs/CLI_MANAGEMENT.md +181 -0
  13. package/docs/COMPOSITE_VISION.md +35 -0
  14. package/docs/CORE_RUNTIME_CONTRACT.md +84 -0
  15. package/docs/DSH_CORE_MIGRATION.md +135 -0
  16. package/docs/DSH_RC2_ADAPTATION.md +38 -0
  17. package/docs/FAULT_INJECTION.md +12 -2
  18. package/docs/HEADLESS_CLI.md +223 -0
  19. package/docs/HOST_ADAPTER_CONTRACT.md +36 -17
  20. package/docs/HOST_DOCTOR.md +4 -2
  21. package/docs/MODEL_PORT_CONTRACT.md +305 -0
  22. package/docs/OCR_MODEL.md +39 -0
  23. package/docs/PROMPT_OPTIMIZER.md +47 -0
  24. package/docs/PROVIDER_ADAPTER_CONTRACT.md +53 -10
  25. package/docs/PROVIDER_HEALTH.md +22 -5
  26. package/docs/PROVIDER_RUNTIME_HARNESS.md +33 -0
  27. package/docs/PROVIDER_SUBMISSION_CONTRACT.md +2 -2
  28. package/docs/ROADMAP.md +49 -83
  29. package/docs/SECURITY.md +8 -7
  30. package/docs/TASK_SEMANTICS.md +2 -2
  31. package/docs/VISION_CLI.md +132 -0
  32. package/docs/VISION_MODEL.md +51 -0
  33. package/docs/WORKBENCH_ARTIFACTS.md +67 -0
  34. package/docs/assets/logo/README.md +55 -0
  35. package/docs/assets/logo/iris-bubble-concept.svg +67 -0
  36. package/docs/assets/logo/iris-bubble.svg +14 -0
  37. package/docs/assets/logo/iris-interim-flower.png +0 -0
  38. package/docs/assets/logo/iris-logo-mono-inverse.svg +78 -0
  39. package/docs/assets/logo/iris-logo-mono.svg +78 -0
  40. package/docs/assets/logo/iris-logo.svg +78 -0
  41. package/docs/assets/logo/iris-mark-mono-inverse.svg +42 -0
  42. package/docs/assets/logo/iris-mark-mono.svg +42 -0
  43. package/docs/assets/logo/iris-mark.svg +42 -0
  44. package/docs/file-access-across-environments.md +8 -5
  45. package/docs/releases/0.2.0.md +48 -0
  46. package/docs/releases/0.2.1.md +56 -0
  47. package/lib/actions.js +520 -591
  48. package/lib/adapters.js +184 -36
  49. package/lib/api.js +200 -11
  50. package/lib/chromium-browser.js +117 -0
  51. package/lib/client.js +1390 -105
  52. package/lib/command-service.js +534 -0
  53. package/lib/composite-vision-routing.js +42 -0
  54. package/lib/config.js +143 -51
  55. package/lib/core-artifact-store.js +630 -0
  56. package/lib/core-artifacts.js +4 -0
  57. package/lib/core-attention.js +103 -0
  58. package/lib/core-contract.js +105 -0
  59. package/lib/core-doctor.js +41 -0
  60. package/lib/core-lease-recovery.js +170 -0
  61. package/lib/core-maintenance.js +213 -0
  62. package/lib/core-runtime.js +232 -0
  63. package/lib/core-tasks.js +567 -0
  64. package/lib/core-user-projection.js +193 -0
  65. package/lib/doctor.js +92 -33
  66. package/lib/dsh-core-adapter.js +653 -0
  67. package/lib/dsh-host-adapter.js +58 -7
  68. package/lib/dsh-text-model-adapter.js +222 -0
  69. package/lib/dsh-vision-model-adapter.js +75 -0
  70. package/lib/generation-input.js +126 -0
  71. package/lib/guard.js +3 -3
  72. package/lib/headless-vision.js +181 -0
  73. package/lib/host-runtime.js +1 -0
  74. package/lib/http-vision-model-adapter.js +187 -0
  75. package/lib/image-edit-input.js +34 -0
  76. package/lib/index.js +323 -188
  77. package/lib/locate.js +86 -28
  78. package/lib/media-probe.js +2 -1
  79. package/lib/model-call-runtime.js +45 -0
  80. package/lib/model-invoker.js +220 -0
  81. package/lib/model-port-contract.js +341 -0
  82. package/lib/models.js +105 -14
  83. package/lib/ocr-model-routing.js +26 -0
  84. package/lib/ocr.js +110 -64
  85. package/lib/prompt-optimizer-config.js +2 -1
  86. package/lib/prompt-optimizer-core.js +121 -0
  87. package/lib/prompt-optimizer.js +37 -193
  88. package/lib/provider-adapter.js +49 -23
  89. package/lib/provider-adapters.js +165 -26
  90. package/lib/provider-catalog.js +183 -0
  91. package/lib/provider-config-service.js +236 -0
  92. package/lib/provider-contract.js +74 -4
  93. package/lib/provider-health.js +80 -1
  94. package/lib/provider-protocol.js +74 -2
  95. package/lib/provider-task-runner.js +370 -0
  96. package/lib/summarize.js +31 -20
  97. package/lib/summary-input.js +52 -0
  98. package/lib/task-wait.js +40 -0
  99. package/lib/tasks.js +16 -4
  100. package/lib/video-input.js +30 -0
  101. package/lib/vision-core.js +42 -0
  102. package/lib/vision-image-input.js +83 -0
  103. package/lib/vision-input.js +31 -0
  104. package/lib/vision-model-routing.js +180 -0
  105. package/lib/vision.js +31 -9
  106. package/lib/workbench-artifacts.js +52 -0
  107. package/lib/workbench-management.js +151 -0
  108. package/lib/workbench-zip.js +39 -0
  109. package/package.json +8 -8
  110. package/user_guide.md +71 -26
package/lib/index.js CHANGED
@@ -14,18 +14,22 @@ import * as cap from './capability.js';
14
14
  import * as tasks from './tasks.js';
15
15
  import { registerMedia, mediaLinksOf, authorizeMedia, serveMedia } from './media.js';
16
16
  import { serveApi, closeAllSse, purgeStaleUploads } from './api.js';
17
- import { buildVisionBackendsFromHost, askWithBackends } from './vision.js';
17
+ import { runVisionRequest, readVisionImageFile } from './vision-model-routing.js';
18
+ import { waitForModelWork } from './model-call-runtime.js';
18
19
  import { hasHostPort, requireHostPort } from './host-contract.js';
19
20
  import { createDshHostAdapter } from './dsh-host-adapter.js';
20
- import { cropImage, pixelDiff, imageDimensions } from './pixels.js';
21
- import { locateObject } from './locate.js';
21
+ import { cropForDsh, diffForDsh, framesForDsh, summaryFramesForDsh, visionImageForDsh, htmlForDsh, inspectProviderTaskForDsh, projectCoreTaskForDsh, readCoreArtifactMediaForDsh, resumeProviderTaskWatchesForDsh, stopProviderTaskWatchesForDsh } from './dsh-core-adapter.js';
22
+ import { imageDimensions } from './pixels.js';
23
+ import { runLocateRequest, runSummaryRequest, describeGeneratedImage } from './composite-vision-routing.js';
24
+ import { normalizeSummarySource, summaryFrameRecords, summaryTimelineLabel } from './summary-input.js';
25
+ import { normalizeVisionImageSource } from './vision-input.js';
22
26
  import { serveRender } from './render.js';
23
27
  import { ensurePrivateDir, hardenPrivateTree } from './private-storage.js';
24
28
  import { guarded } from './guard.js';
25
- import { longOcr } from './ocr.js';
29
+ import { formatOcrResult } from './ocr.js';
30
+ import { runOcrRequest } from './ocr-model-routing.js';
26
31
  import { runAction, listActions, taskPollDeps } from './actions.js';
27
32
  import { ffmpegAvailable, extractFrames, probeVideo, extractAudioTrack } from './media-probe.js';
28
- import { buildContactSheet, summarizeMedia } from './summarize.js';
29
33
  import { registerBundledSkills } from './bundled-skills.js';
30
34
  import {
31
35
  EXPECTED_IRIS_ROUTES, IRIS_PLUGIN_ID, beginHostRuntime, recordHostRoutes, recordHostSkills, recordHostTool
@@ -51,7 +55,7 @@ function dshHost(ctx) {
51
55
  }
52
56
 
53
57
  /** 工具内同步等待上限;超过即转后台盯守,用 iris_task_status 查询 */
54
- const AWAIT_MS = { image: 180000, video: 480000 };
58
+ const AWAIT_MS = { image: 180000, video: 480000, transcribe: 120000 };
55
59
 
56
60
  function outputDir() {
57
61
  const dir = tasks.outputsDir();
@@ -153,22 +157,37 @@ async function awaitTerminal(taskId, { timeoutMs, signal }) {
153
157
  }
154
158
  }
155
159
 
156
- /* ---------------- 图片 data URL(视觉工具共用) ---------------- */
160
+ function coreTaskWaitFinished(task) {
161
+ return Boolean(task && (task.phase === 'terminal'
162
+ || (task.outcome === 'succeeded' && ['ready', 'failed'].includes(task.deliveryState))));
163
+ }
157
164
 
158
- function toDataUrl(absPath, mediaType) {
159
- const buf = fs.readFileSync(absPath);
160
- return `data:${mediaType};base64,${buf.toString('base64')}`;
165
+ function coreTaskTerminalProblem(task, label = '任务') {
166
+ if (!task) return label + '记录不存在';
167
+ if (task.outcome === 'succeeded' && task.deliveryState === 'ready') return '';
168
+ if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
169
+ return label + '已成功,但 Core Artifact 交付失败:'
170
+ + (task.lastError?.safeMessage || '可稍后重新交付;禁止重新生成');
171
+ }
172
+ if (task.acceptance === 'unknown' || task.outcome === 'unknown') {
173
+ return label + '的远端受理或结果状态未知:'
174
+ + (task.lastError?.safeMessage || '禁止自动重提,请先人工确认');
175
+ }
176
+ if (task.outcome === 'canceled') return label + '已取消';
177
+ if (task.outcome === 'failed') return label + '失败:'
178
+ + (task.lastError?.safeMessage || '供应商明确返回失败');
179
+ return label + '需要人工处理';
161
180
  }
162
181
 
163
- function fmtTask(t) {
164
- const lines = [`[${t.cap}] ${t.id} · ${t.status}${t.progress ? ' · ' + t.progress : ''} · ${t.model}`];
165
- lines.push('prompt: ' + String(t.prompt || '').slice(0, 120));
166
- if (t.error) lines.push('error: ' + t.error);
167
- if ((t.files || []).length) lines.push('files:\n' + t.files.map((f) => path.join(tasks.outputsDir(), f)).join('\n'));
168
- const links = mediaLinksOf(t);
169
- if (links.length) lines.push('links:\n' + links.join('\n'));
170
- if (t.remoteTaskId) lines.push('remoteTask: ' + t.remoteTaskId);
171
- return lines.join('\n');
182
+ async function awaitCoreTerminal(taskId, { timeoutMs, signal }) {
183
+ const deadline = Date.now() + timeoutMs;
184
+ for (;;) {
185
+ if (signal?.aborted) throw new Error('已取消;Core 任务仍保留远端受理事实,可稍后查询');
186
+ const task = await inspectProviderTaskForDsh(taskId, { signal });
187
+ if (coreTaskWaitFinished(task)) return task;
188
+ if (Date.now() > deadline) return null;
189
+ await new Promise((resolve) => setTimeout(resolve, 1000));
190
+ }
172
191
  }
173
192
 
174
193
  export const name = 'mokuyoaxis-dsh-iris';
@@ -223,9 +242,19 @@ export async function apply(ctx) {
223
242
  if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个后台任务`);
224
243
  });
225
244
 
226
- // Fiber 清理:插件停用/更新时停掉全部盯守句柄
245
+ try {
246
+ const resumed = await resumeProviderTaskWatchesForDsh();
247
+ if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个 Core 图片任务`);
248
+ } catch (error) {
249
+ console.error('[iris] 恢复 Core 图片后台任务失败(已隔离,不影响宿主):', error?.message || error);
250
+ }
251
+
252
+ // Fiber 清理:插件停用/更新时停掉 legacy 与 Core 的全部 Host 盯守句柄
227
253
  guard('盯守句柄注册', () => {
228
- ctx.effect(() => () => tasks.stopWatchAll());
254
+ ctx.effect(() => () => {
255
+ tasks.stopWatchAll();
256
+ stopProviderTaskWatchesForDsh();
257
+ });
229
258
  });
230
259
 
231
260
  // 媒体路由(视频/音频在对话流里的可播通路;图片仍走原生附件)
@@ -250,11 +279,41 @@ export async function apply(ctx) {
250
279
  return;
251
280
  }
252
281
 
282
+ async function coreImageToolResult(submitted, prompt, signal) {
283
+ let final = await inspectProviderTaskForDsh(submitted.taskId, { signal });
284
+ if (!coreTaskWaitFinished(final)) final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal });
285
+ if (!final) return { blocks: [{ type: 'text', text: `[iris] Core 图片任务仍在后台观察(task: ${submitted.taskId})。稍后可用 iris_task_status 查询进度与 Artifact。` }] };
286
+ const terminalProblem = coreTaskTerminalProblem(final, '图像生成');
287
+ if (terminalProblem) throw new Error(terminalProblem);
288
+ const projected = await projectCoreTaskForDsh(dshHost(ctx), submitted.taskId, { signal });
289
+ const refs = projected.artifacts.map((item) => item.attachment);
290
+ const blocks = projected.artifacts.map((item) => ({ type: 'image', attachment: item.attachment }));
291
+ let note = '';
292
+ try {
293
+ const described = await describeGeneratedImage(dshHost(ctx), {
294
+ providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal,
295
+ async prepareImage(inputSignal) {
296
+ const first = await readCoreArtifactMediaForDsh(projected.artifacts[0].artifact.id, { signal: inputSignal });
297
+ return { bytes: new Uint8Array(first.bytes), mediaType: first.artifact.mediaType };
298
+ }
299
+ });
300
+ if (described) note = `画面内容:${described}`;
301
+ } catch (_) { /* 自述失败不影响 Core 交付 */ }
302
+ blocks.unshift({ type: 'text', text:
303
+ `[iris] 图像已生成(${submitted.model},Core task: ${submitted.taskId})。` +
304
+ (submitted.sourceArtifactId ? `\n原图 artifact: ${submitted.sourceArtifactId}` : '') +
305
+ (note ? `\n${note}` : '') +
306
+ `\nartifact: ${projected.artifacts.map((item) => item.artifact.id).join(', ')}` +
307
+ `\nattachment: ${refs.map((ref) => ref.attachmentId).join(', ')}`
308
+ });
309
+ return { blocks };
310
+ }
311
+
253
312
  /* ---------- 🖼️ 画图 ---------- */
254
313
  safeRegister(host, {
255
314
  name: 'iris_draw_image',
256
315
  description:
257
- 'Generate an image from a detailed prompt using the user-configured image model (DashScope wan* or OpenAI-images-compatible). ' +
316
+ 'Generate an image from a detailed prompt using the user-configured image model and its configured image protocol. ' +
258
317
  'Returns a durable DSH attachment rendered in the conversation plus a vision-model description so you know what was drawn.',
259
318
  parameters: {
260
319
  type: 'object',
@@ -273,8 +332,10 @@ export async function apply(ctx) {
273
332
  isConcurrencySafe: () => false,
274
333
  async execute(args, exec) {
275
334
  const prompt = String(args.prompt || '').trim();
276
- if (!prompt) throw new Error('iris: prompt 不能为空');
277
335
  const submitted = await runAction(dshHost(ctx), 'image', args, { signal: exec.signal });
336
+ if (submitted.storage === 'core') {
337
+ return coreImageToolResult(submitted, prompt, exec.signal);
338
+ }
278
339
  let final = tasks.get(submitted.taskId);
279
340
  if (final && !taskWaitFinished(final)) {
280
341
  final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
@@ -304,7 +365,10 @@ export async function apply(ctx) {
304
365
 
305
366
  let note = '';
306
367
  try {
307
- const described = await describeFirstImage(dshHost(ctx), refs[0], files[0], prompt);
368
+ const described = await describeGeneratedImage(dshHost(ctx), {
369
+ providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal: exec.signal,
370
+ prepareImage: signal => readVisionImageFile(files[0], MEDIA_TYPES[path.extname(files[0]).toLowerCase()] || 'image/png', signal)
371
+ });
308
372
  if (described) note = `画面内容:${described}`;
309
373
  } catch (_) { /* 增强失败不影响生成结果 */ }
310
374
  blocks.unshift({
@@ -317,6 +381,25 @@ export async function apply(ctx) {
317
381
  }
318
382
  });
319
383
 
384
+ safeRegister(host, {
385
+ name: 'iris_edit_image',
386
+ description: 'Edit an existing Core image by Artifact ID and an instruction using a configured openai-chat-images model. Returns new Core Artifacts and DSH image attachments; preserves the source image and records its provenance.',
387
+ parameters: {
388
+ type: 'object', properties: {
389
+ source_artifact_id: { type: 'string', description: 'Core Artifact ID of the source PNG, JPEG or WebP image (up to 20 MiB)' },
390
+ prompt: { type: 'string', description: 'Describe the changes and what should be preserved in the source image' },
391
+ model: { type: 'string', description: 'Optional providerId::modelId; image protocol must be openai-chat-images' },
392
+ n: { type: 'string', description: "Number of images, default '1'" }
393
+ }, required: ['source_artifact_id', 'prompt']
394
+ },
395
+ output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
396
+ isConcurrencySafe: () => false,
397
+ async execute(args, exec) {
398
+ const submitted = await runAction(dshHost(ctx), 'image_edit', args, { signal: exec.signal });
399
+ return coreImageToolResult(submitted, String(args.prompt || '').trim(), exec.signal);
400
+ }
401
+ });
402
+
320
403
  /* ---------- 🎬 视频生成 ---------- */
321
404
  safeRegister(host, {
322
405
  name: 'iris_generate_video',
@@ -329,7 +412,7 @@ export async function apply(ctx) {
329
412
  parameters: {
330
413
  type: 'object',
331
414
  properties: {
332
- prompt: { type: 'string', description: 'Detailed description of the motion/scene for the video' },
415
+ prompt: { type: 'string', description: 'Motion/scene description, required for t2v/i2v; optional for s2v' },
333
416
  first_frame_attachment_id: { type: 'string', description: 'Attachment id of an image generated by iris_draw_image, used as the first frame' },
334
417
  first_frame_path: { type: 'string', description: 'Absolute path to a local image used as the first frame (alternative to attachment id)' },
335
418
  audio_path: { type: 'string', description: 'Absolute path to a wav/mp3 file (<15MB, <20s, clear human voice). Required for s2v models like wan2.2-s2v' },
@@ -338,12 +421,31 @@ export async function apply(ctx) {
338
421
  duration: { type: 'number', description: 't2v/i2v only: duration in seconds, if the model supports it' },
339
422
  model: { type: 'string', description: 'Override with a providerId::modelId reference (legacy bare model id accepted)' }
340
423
  },
341
- required: ['prompt']
424
+ required: []
342
425
  },
343
426
  output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
344
427
  isConcurrencySafe: () => false,
345
428
  async execute(args, exec) {
346
- const submitted = await runAction(dshHost(ctx), 'video', args, { signal: exec.signal });
429
+ const submitted = await runAction(dshHost(ctx), 'video', { ...args, session_id: exec.agent?.session?.id }, { signal: exec.signal });
430
+ if (submitted.storage === 'core') {
431
+ let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
432
+ if (!coreTaskWaitFinished(final)) {
433
+ final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
434
+ }
435
+ if (!final) {
436
+ return `[iris] Core 视频任务已提交并转入后台观察(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
437
+ '渲染通常需要数分钟;完成后用 iris_task_status 查询进度与 Artifact,或在工作台「重新观察」。';
438
+ }
439
+ const terminalProblem = coreTaskTerminalProblem(final, '视频生成');
440
+ if (terminalProblem) throw new Error(terminalProblem);
441
+ const secs = Math.round((Date.now() - new Date(final.createdAt).getTime()) / 1000);
442
+ const links = final.artifactIds.map((artifactId) =>
443
+ '/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
444
+ return `[iris] 视频已生成(${submitted.model || final.modelRef},${submitted.mode || 't2v'},约 ${secs}s,Core task: ${final.id}):\n` +
445
+ `artifact: ${final.artifactIds.join(', ')}\n` +
446
+ links.join('\n') +
447
+ '\n(链接受 DSH Host 与浏览器同源守卫保护;Artifact ID 持有者可访问,请勿公开)';
448
+ }
347
449
  const final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
348
450
  if (!final) {
349
451
  return `[iris] 视频任务已提交并转入后台盯守(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
@@ -378,6 +480,16 @@ export async function apply(ctx) {
378
480
  isConcurrencySafe: () => true,
379
481
  async execute(args, exec) {
380
482
  const submitted = await runAction(dshHost(ctx), 'tts', args, { signal: exec.signal });
483
+ if (submitted.storage === 'core') {
484
+ const final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
485
+ const terminalProblem = coreTaskTerminalProblem(final, '语音合成');
486
+ if (terminalProblem) throw new Error(terminalProblem);
487
+ const links = final.artifactIds.map((artifactId) =>
488
+ '/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
489
+ return `[iris] 语音已合成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
490
+ `artifact: ${final.artifactIds.join(', ')}\n` +
491
+ links.map((link) => `[♪ 音频播放](${link})`).join('\n');
492
+ }
381
493
  const final = tasks.get(submitted.taskId);
382
494
  const terminalProblem = taskTerminalProblem(final, '语音合成');
383
495
  if (terminalProblem) throw new Error(terminalProblem);
@@ -405,6 +517,25 @@ export async function apply(ctx) {
405
517
  isConcurrencySafe: () => false,
406
518
  async execute(args, exec) {
407
519
  const submitted = await runAction(dshHost(ctx), 'transcribe', args, { signal: exec.signal });
520
+ if (submitted.storage === 'core') {
521
+ let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
522
+ if (!coreTaskWaitFinished(final)) {
523
+ final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.transcribe, signal: exec.signal });
524
+ }
525
+ if (!final) {
526
+ return `[iris] Core 转写任务已提交并转入后台观察(task: ${submitted.taskId})。完成后用 iris_task_status 查询进度与 Artifact。`;
527
+ }
528
+ const terminalProblem = coreTaskTerminalProblem(final, '音频转写');
529
+ if (terminalProblem) throw new Error(terminalProblem);
530
+ const texts = [];
531
+ for (const artifactId of final.artifactIds) {
532
+ const media = await readCoreArtifactMediaForDsh(artifactId, { signal: exec.signal });
533
+ texts.push(Buffer.from(media.bytes).toString('utf8'));
534
+ }
535
+ return `[iris] 音频转写完成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
536
+ texts.join('\n') +
537
+ `\nartifact: ${final.artifactIds.join(', ')}`;
538
+ }
408
539
  const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
409
540
  if (!final) return `[iris] 音频转写任务已提交并转入后台(task: ${submitted.taskId})。完成后用 iris_task_status 查询。`;
410
541
  const terminalProblem = taskTerminalProblem(final, '音频转写');
@@ -435,26 +566,22 @@ export async function apply(ctx) {
435
566
  isConcurrencySafe: () => false,
436
567
  async execute(args, exec) {
437
568
  if (exec.signal && exec.signal.aborted) throw new Error('已取消');
438
- const p = String(args.video_path || '').trim();
439
- if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
440
- if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法抽帧(可选安装 ffmpeg/ffprobe 后启用;不影响其他 iris 工具)');
441
- const meta = probeVideo(p);
442
- const frames = await extractFrames({
443
- inputPath: p,
569
+ const { media: meta, frames } = await framesForDsh({
570
+ videoPath: args.video_path,
444
571
  maxFrames: args.max_frames,
445
572
  targetWidth: args.target_width,
446
573
  format: args.format,
447
574
  quality: args.quality,
448
575
  signal: exec.signal
449
576
  });
450
- const mediaType = args.format === 'png' ? 'image/png' : 'image/jpeg';
451
- const ext = args.format === 'png' ? 'png' : 'jpg';
577
+ const mediaType = frames[0].mediaType;
578
+ const ext = mediaType === 'image/png' ? 'png' : 'jpg';
452
579
  const blocks = [];
453
580
  const refs = [];
454
581
  for (let i = 0; i < frames.length; i++) {
455
582
  const f = frames[i];
456
583
  const ref = await attachmentService(host).saveImage({
457
- data: new Uint8Array(f.buffer),
584
+ data: new Uint8Array(f.bytes),
458
585
  mediaType,
459
586
  name: `iris-frame-${Date.now()}-${i + 1}.${ext}`
460
587
  });
@@ -468,9 +595,10 @@ export async function apply(ctx) {
468
595
  `[iris] 视频抽帧完成:${frames.length} 帧(${meta.durationSec.toFixed(1)}s 视频,` +
469
596
  `${frames[0].width}x${frames[0].height},${mediaType === 'image/png' ? 'PNG' : 'JPEG'})。\n` +
470
597
  `时间戳:${ats}\n` +
598
+ `frame_artifact_ids: ${JSON.stringify(frames.map(f => f.artifact.id))}\n` +
471
599
  `attachments: ${refs.map((r) => r.attachmentId).join(', ')}`
472
600
  });
473
- return { blocks };
601
+ return { blocks, artifactIds: frames.map((frame) => frame.artifact.id) };
474
602
  }
475
603
  });
476
604
 
@@ -478,83 +606,91 @@ export async function apply(ctx) {
478
606
  safeRegister(host, {
479
607
  name: 'iris_media_summarize',
480
608
  description:
481
- 'Summarize a local video by compressing it into a contact sheet of sampled frames (with timestamps) plus an optional ' +
609
+ 'Summarize a local video or existing Core video-frame Artifacts as one contact sheet (with timestamps) plus an optional ' +
482
610
  'auto-transcribed audio track, then asking the vision model to describe content. Returns the summary text plus the contact sheet as a DSH image attachment. ' +
483
- 'Requires ffmpeg + ffprobe on PATH. Transcription reuses the configured provider (qwen-audio-3.0-asr-flash-filetrans) when the video has an audio track.',
611
+ 'Provide exactly one of video_path or frame_artifact_ids. Existing frames are read in timestamp order without ffmpeg or the original video. ' +
612
+ 'The video_path mode requires ffmpeg + ffprobe and can auto-transcribe the audio track with the configured provider.',
484
613
  parameters: {
485
614
  type: 'object',
486
615
  properties: {
487
616
  video_path: { type: 'string', description: 'Absolute path to the local video file (mp4/webm/mov etc.)' },
617
+ frame_artifact_ids: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: 20, uniqueItems: true,
618
+ description: 'Existing Core video-frame IDs from iris_video_frames; sorted by timestamp. Use instead of video_path, without sampling options or auto-transcription.' },
488
619
  question: { type: 'string', description: 'Custom question about the video content (default: summarize scenes/theme)' },
489
620
  max_frames: { type: 'integer', description: 'Frames to sample for the contact sheet, clamped to 1..12 (default 8)' },
490
621
  target_width: { type: 'integer', description: 'Frame width for sampling (default 640)' },
491
- transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track if present (default true)' },
622
+ transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track for video_path (default true unless transcribe_text is provided). Existing frames only consume explicit text.' },
623
+ transcribe_text: { type: 'string', description: 'Optional existing transcript text; does not create a transcription task' },
492
624
  model: { type: 'string', description: 'Override the configured vision model id' }
493
625
  },
494
- required: ['video_path']
626
+ required: []
495
627
  },
496
628
  output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
497
629
  isConcurrencySafe: () => false,
498
630
  async execute(args, exec) {
499
631
  if (exec.signal && exec.signal.aborted) throw new Error('已取消');
632
+ const source = normalizeSummarySource(args);
500
633
  const p = String(args.video_path || '').trim();
501
- if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
502
- if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法分析视频(可选安装 ffmpeg/ffprobe 后启用)');
503
- const meta = probeVideo(p);
504
- const frames = await extractFrames({
505
- inputPath: p,
506
- maxFrames: Math.min(Number(args.max_frames) || 8, 12),
507
- targetWidth: args.target_width,
508
- format: 'jpeg',
509
- quality: 70,
510
- signal: exec.signal
511
- });
512
-
513
- // 音轨转写(可选;与独立 transcribe capability 共用同一动作与 failover)
514
- let transcript = '';
515
- let transcriptNote = '';
516
- const wantTranscribe = args.transcribe !== false && meta.hasAudio;
517
- if (wantTranscribe) {
518
- if (store.pickAllFor(cap.CAPABILITIES.TRANSCRIBE).length) {
519
- try {
520
- const audio = await extractAudioTrack({ inputPath: p, signal: exec.signal });
521
- try {
522
- const submitted = await runAction(dshHost(ctx), 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
523
- const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
524
- if (final && final.status === 'succeeded') transcript = final.transcribeText || '';
525
- else if (!final) transcriptNote = `\n(音轨转写仍在后台:task ${submitted.taskId},本次摘要未含语音内容)`;
526
- } finally {
527
- fs.rmSync(audio.outDir, { recursive: true, force: true });
528
- }
529
- } catch (err) {
530
- transcriptNote = '\n(音轨转写失败,本次摘要仅画面:' + String(err.message || err).slice(0, 160) + ')';
634
+ if (source === 'video') {
635
+ if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
636
+ if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法分析视频(可选安装 ffmpeg/ffprobe 后启用)');
637
+ }
638
+ const { answer, via, model, sheet, meta, frames, transcriptNote } = await runSummaryRequest(dshHost(ctx), {
639
+ providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, question: args.question, signal: exec.signal,
640
+ async prepareMedia(signal) {
641
+ if (source === 'core-artifacts') {
642
+ return { ...await summaryFramesForDsh({ frameArtifactIds: args.frame_artifact_ids, signal }), transcript: args.transcribe_text };
643
+ }
644
+ const meta = probeVideo(p);
645
+ const frames = await extractFrames({
646
+ inputPath: p, maxFrames: Math.min(Number(args.max_frames) || 8, 12),
647
+ targetWidth: args.target_width, format: 'jpeg', quality: 70, signal
648
+ });
649
+ // 显式选项保留自动转写;消费已完成的 Core Artifact,不重提远端任务。
650
+ let transcript = args.transcribe_text || '', transcriptNote = '';
651
+ if (args.transcribe !== false && !transcript.trim() && meta.hasAudio) {
652
+ if (store.pickAllFor(cap.CAPABILITIES.TRANSCRIBE).length) {
653
+ try {
654
+ const audio = await extractAudioTrack({ inputPath: p, signal });
655
+ try {
656
+ const submitted = await runAction(dshHost(ctx), 'transcribe', { audio_path: audio.filePath }, { signal });
657
+ const final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: 120000, signal });
658
+ if (final?.outcome === 'succeeded' && final.deliveryState === 'ready') {
659
+ const texts = [];
660
+ for (const artifactId of final.artifactIds) {
661
+ const media = await readCoreArtifactMediaForDsh(artifactId, { signal });
662
+ texts.push(Buffer.from(media.bytes).toString('utf8'));
663
+ }
664
+ transcript = texts.join('\n');
665
+ } else transcriptNote = '\n(音轨转写未完成,本次摘要仅画面;可用 iris_task_status 查询 Core task)';
666
+ } finally {
667
+ fs.rmSync(audio.outDir, { recursive: true, force: true });
668
+ }
669
+ } catch (err) {
670
+ if (signal.aborted) throw signal.reason;
671
+ transcriptNote = '\n(音轨转写失败,本次摘要仅画面;可用 iris_task_status 查询转写任务)';
672
+ }
673
+ } else transcriptNote = '\n(视频含音轨但未配置转写供应商,本次摘要仅画面)';
531
674
  }
532
- } else {
533
- transcriptNote = '\n(视频含音轨但未配置转写供应商,本次摘要仅画面)';
675
+ return { meta, frames, transcript, transcriptNote };
534
676
  }
535
- }
536
-
537
- // contact sheet → attachment → 视觉摘要
538
- const sheet = await buildContactSheet({ frames, cols: 3 });
677
+ });
678
+ // 成功摘要的同一张拼图作为显式工具产物发布。
679
+ if (exec.signal?.aborted) throw exec.signal.reason;
539
680
  const ref = await attachmentService(host).saveImage({
540
681
  data: new Uint8Array(sheet.buffer),
541
682
  mediaType: 'image/png',
542
683
  name: `iris-sheet-${Date.now()}.png`
543
684
  });
544
- const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
545
- const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
546
- const { answer, via, model } = await summarizeMedia({
547
- backends, question: args.question, transcript,
548
- imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
549
- ref, signal: exec.signal
550
- });
685
+ if (exec.signal?.aborted) throw exec.signal.reason;
551
686
  const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
552
687
  return {
688
+ frames: summaryFrameRecords(frames),
553
689
  blocks: [
554
690
  {
555
691
  type: 'text',
556
692
  text:
557
- `[iris] 视频摘要完成(${model} · ${viaLabel},${meta.durationSec.toFixed(1)}s / ${frames.length} 帧联系表):\n` +
693
+ `[iris] 视频摘要完成(${model} · ${viaLabel},${summaryTimelineLabel(meta)} / ${frames.length} 帧联系表):\n` +
558
694
  `${answer}${transcriptNote || ''}\n` +
559
695
  `contact sheet attachment: ${ref.attachmentId}`
560
696
  },
@@ -569,30 +705,39 @@ export async function apply(ctx) {
569
705
  safeRegister(host, {
570
706
  name: 'iris_look_at_image',
571
707
  description:
572
- 'Look at a local image file and answer one question about it with the configured vision model (qwen-vl via the self-hosted provider stack; falls back to the global DSH vision model). The image is saved as a durable attachment so follow-ups can reference it.',
708
+ 'Look at a local image file or an existing Core image Artifact and answer one question with the configured vision model (self-hosted provider stack, then DSH default vision). Provide exactly one of image_path or artifact_id. Local files are saved as durable DSH attachments; Core images are read directly without export or Core writes.',
573
709
  parameters: {
574
710
  type: 'object',
575
711
  properties: {
576
712
  image_path: { type: 'string', description: 'Absolute path to the image file (png/jpg/jpeg/webp)' },
713
+ artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path)' },
577
714
  question: { type: 'string', description: 'Question or extraction request about the image; defaults to a detailed description' },
578
715
  model: { type: 'string', description: 'Override the configured vision model id (default qwen-vl-plus)' }
579
716
  },
580
- required: ['image_path']
717
+ required: []
581
718
  },
582
719
  output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
583
720
  isConcurrencySafe: () => true,
584
721
  async execute(args, exec) {
722
+ const source = normalizeVisionImageSource(args);
585
723
  const p = String(args.image_path || '').trim();
586
- if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
587
724
  const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
588
- if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp):' + p);
589
- if (!fs.existsSync(p)) throw new Error('iris: 图片不存在:' + p);
725
+ if (source === 'image_path') {
726
+ if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
727
+ if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp):' + p);
728
+ if (!fs.existsSync(p)) throw new Error('iris: 图片不存在:' + p);
729
+ }
590
730
  const question = String(args.question || '').trim() || '详细描述这张图片。';
591
731
  if (exec.signal && exec.signal.aborted) throw new Error('已取消');
592
732
 
593
- const dataUrl = toDataUrl(p, mt);
594
- const ref = await attachmentService(host).saveImage({ data: new Uint8Array(fs.readFileSync(p)), mediaType: mt, name: path.basename(p) });
595
- return runVisionTool(dshHost(ctx), exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
733
+ return runVisionTool(dshHost(ctx), exec, { origin: 'tool', model: args.model, question, artifactId: args.artifact_id,
734
+ async prepareImage(signal) {
735
+ if (source === 'artifact_id') return visionImageForDsh({ artifactId: args.artifact_id, signal });
736
+ const image = await readVisionImageFile(p, mt, signal);
737
+ await waitForModelWork(() => attachmentService(host).saveImage({ data: image.bytes, mediaType: mt, name: path.basename(p) }), signal);
738
+ return image;
739
+ }
740
+ });
596
741
  }
597
742
  });
598
743
 
@@ -618,18 +763,16 @@ export async function apply(ctx) {
618
763
  if (!question) throw new Error('iris: question 不能为空');
619
764
  if (exec.signal && exec.signal.aborted) throw new Error('已取消');
620
765
 
621
- // 定位:本会话出现过的图片优先,其次本插件产物记录
622
- const hit = (await sessionAttachmentRef(host, exec, id)) || findOwnAttachment(id);
623
- if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
624
-
625
- let dataUrl;
626
- if (hit.absPath) {
627
- dataUrl = toDataUrl(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png');
628
- } else {
629
- const stored = await attachmentService(host).readImage(hit.ref, exec.signal);
630
- dataUrl = `data:${stored.mediaType || hit.ref.mediaType};base64,${Buffer.from(stored.data).toString('base64')}`;
631
- }
632
- return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
766
+ return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question,
767
+ async prepareImage(signal) {
768
+ // 会话扫描、解引用与读图都属于这次操作的总预算。
769
+ const hit = (await waitForModelWork(() => sessionAttachmentRef(host, exec, id), signal)) || findOwnAttachment(id);
770
+ if (!hit) throw new Error('iris: 图片引用不可用');
771
+ if (hit.absPath) return readVisionImageFile(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png', signal);
772
+ const stored = await waitForModelWork(() => attachmentService(host).readImage(hit.ref, signal), signal);
773
+ return { bytes: new Uint8Array(stored.data), mediaType: stored.mediaType || hit.ref.mediaType };
774
+ }
775
+ });
633
776
  }
634
777
  });
635
778
 
@@ -648,14 +791,8 @@ export async function apply(ctx) {
648
791
  output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
649
792
  isConcurrencySafe: () => true,
650
793
  async execute(args) {
651
- if (args.task_id) {
652
- const t = tasks.get(String(args.task_id).trim());
653
- if (!t) throw new Error('iris: 没有这个任务:' + args.task_id);
654
- return fmtTask(t);
655
- }
656
- const recent = tasks.list().slice(0, 5);
657
- if (!recent.length) return '[iris] 还没有生成任务记录。';
658
- return recent.map(fmtTask).join('\n---\n');
794
+ const result = await runAction(dshHost(ctx), 'status', args || {});
795
+ return result.text;
659
796
  }
660
797
  });
661
798
 
@@ -685,21 +822,25 @@ export async function apply(ctx) {
685
822
  image_path: args.image_path,
686
823
  attachment_id: args.attachment_id
687
824
  });
688
- const { buffer, width, height, mime } = await cropImage({
689
- input: input.buffer,
825
+ const result = await cropForDsh({
826
+ ...(args.image_path
827
+ ? { imagePath: String(args.image_path).trim() }
828
+ : { bytes: input.buffer, mediaType: input.mediaType }),
690
829
  left: args.left,
691
830
  top: args.top,
692
831
  width: args.width,
693
- height: args.height
832
+ height: args.height,
833
+ signal: exec.signal
694
834
  });
835
+ if (exec.signal && exec.signal.aborted) throw new Error('已取消');
695
836
  const ref = await attachmentService(host).saveImage({
696
- data: new Uint8Array(buffer),
697
- mediaType: mime,
837
+ data: new Uint8Array(result.bytes),
838
+ mediaType: result.mediaType,
698
839
  name: `iris-crop-${Date.now()}.png`
699
840
  });
700
841
  return {
701
842
  blocks: [
702
- { type: 'text', text: `[iris] 裁剪完成:${width}x${height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
843
+ { type: 'text', text: `[iris] 裁剪完成:${result.width}x${result.height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
703
844
  { type: 'image', attachment: ref }
704
845
  ]
705
846
  };
@@ -732,20 +873,21 @@ export async function apply(ctx) {
732
873
  if (!(args.image_b_path || args.attachment_b_id)) throw new Error('iris: 需要 image_b_path 或 attachment_b_id');
733
874
  const a = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
734
875
  const b = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
735
- const { ratio, diffPixels, totalPixels, width, height, worstRegions, heatmap, mime } = await pixelDiff({
736
- inputA: a.buffer,
737
- inputB: b.buffer,
738
- grid: args.grid || 8,
739
- topRegions: args.top_regions || 3
876
+ const result = await diffForDsh({
877
+ imageA: { bytes: a.buffer, mediaType: a.mediaType },
878
+ imageB: { bytes: b.buffer, mediaType: b.mediaType },
879
+ grid: args.grid, topRegions: args.top_regions, signal: exec.signal
740
880
  });
881
+ const { ratio, diffPixels, totalPixels, width, height, worstRegions } = result.metrics;
741
882
  const ref = await attachmentService(host).saveImage({
742
- data: new Uint8Array(heatmap),
743
- mediaType: mime,
883
+ data: new Uint8Array(result.bytes),
884
+ mediaType: result.mediaType,
744
885
  name: `iris-diff-${Date.now()}.png`
745
886
  });
746
887
  const pct = (ratio * 100).toFixed(2);
747
888
  const regions = worstRegions.map((r) => `(col ${r.col},row ${r.row}) ${(r.score * 100).toFixed(1)}%`).join(' ');
748
889
  return {
890
+ artifactIds: [result.artifact.id],
749
891
  blocks: [
750
892
  {
751
893
  type: 'text',
@@ -763,13 +905,14 @@ export async function apply(ctx) {
763
905
  safeRegister(host, {
764
906
  name: 'iris_locate',
765
907
  description:
766
- 'Locate an object/region described by `target` in an image (absolute path or session attachment_id), ' +
908
+ 'Locate an object/region described by `target` in an image (exactly one of absolute image_path, Core artifact_id, or session attachment_id), ' +
767
909
  'returning the pixel bounding box (x1,y1,x2,y2) in original image coordinates. ' +
768
- 'Use the returned bbox with iris_crop to extract the region.',
910
+ 'Use the bbox with iris_crop for path/session images or CLI run crop with artifact_id for Core images; do not treat a Core ID as a file path.',
769
911
  parameters: {
770
912
  type: 'object',
771
913
  properties: {
772
914
  image_path: { type: 'string', description: 'Absolute path to the source image (png/jpg/jpeg/webp)' },
915
+ artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path or attachment_id)' },
773
916
  attachment_id: { type: 'string', description: 'Session attachment id of an image (alternative to image_path)' },
774
917
  target: { type: 'string', description: 'The object or UI element to locate, e.g. "send button"' },
775
918
  model: { type: 'string', description: 'Override the vision model for this locate call (e.g. qwen3-vl-235b-a22b-thinking for higher grounding precision; default is the provider visionModel, typically qwen-vl-plus)' }
@@ -779,23 +922,15 @@ export async function apply(ctx) {
779
922
  output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
780
923
  isConcurrencySafe: () => true,
781
924
  async execute(args, exec) {
782
- if (exec.signal && exec.signal.aborted) throw new Error('已取消');
783
- const input = await resolveImageInput(dshHost(ctx), exec, {
784
- image_path: args.image_path,
785
- attachment_id: args.attachment_id
786
- });
787
- const { width, height } = await imageDimensions(input.buffer);
788
- const dataUrl = `data:${input.mediaType};base64,${input.buffer.toString('base64')}`;
789
- const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
790
- const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
791
- const r = await locateObject(backends, {
792
- target: args.target,
793
- imageDataUrl: dataUrl,
794
- width,
795
- height,
796
- signal: exec.signal
925
+ const r = await runLocateRequest(dshHost(ctx), {
926
+ providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, target: args.target,
927
+ prepareImage: signal => resolveVisionImageInput(dshHost(ctx), exec, args, signal), signal: exec.signal
797
928
  });
798
- if (!r.found) return `[iris] 在图片中未找到「${args.target}」(${r.via} · ${r.model})`;
929
+ const { width, height } = r;
930
+ const sourceNote = args.artifact_id ? `\nCore Artifact: ${args.artifact_id}` : '';
931
+ if (!r.found) return `[iris] 在图片中未找到「${args.target}」(${r.via} · ${r.model})${sourceNote}`;
932
+ if (args.artifact_id) return `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})${sourceNote}\n` +
933
+ `裁剪区域:left=${r.x1}, top=${r.y1}, width=${r.x2 - r.x1}, height=${r.y2 - r.y1}`;
799
934
  return `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})\n` +
800
935
  `裁剪指令:iris_crop(image_path="${args.image_path || args.attachment_id}", left=${r.x1}, top=${r.y1}, width=${r.x2 - r.x1}, height=${r.y2 - r.y1})`;
801
936
  }
@@ -825,7 +960,8 @@ export async function apply(ctx) {
825
960
  async execute(args, exec) {
826
961
  if (exec.signal && exec.signal.aborted) throw new Error('已取消');
827
962
  const browser = requireHostPort(dshHost(ctx), 'browser', 'HTML 截图');
828
- const rendered = await browser.renderHtml({
963
+ const rendered = await htmlForDsh({
964
+ browser,
829
965
  html: args.html,
830
966
  width: args.width,
831
967
  height: args.height,
@@ -844,7 +980,7 @@ export async function apply(ctx) {
844
980
  {
845
981
  type: 'text',
846
982
  text: `[iris] HTML 截图完成(${outW}x${outH},fullPage: ${args.fullPage !== false})\n` +
847
- `attachment: ${ref.attachmentId}` +
983
+ `artifact: ${rendered.artifact.id}\nattachment: ${ref.attachmentId}` +
848
984
  `\n注意:渲染在共享浏览器临时标签页中完成,页面短暂可见属正常行为。`
849
985
  },
850
986
  { type: 'image', attachment: ref }
@@ -857,13 +993,14 @@ export async function apply(ctx) {
857
993
  safeRegister(host, {
858
994
  name: 'iris_long_ocr',
859
995
  description:
860
- 'OCR a (possibly long) image: slice it into chunks (1200px tall with 120px overlap by default) ' +
996
+ 'OCR a (possibly long) image from exactly one of image_path, Core artifact_id, or session attachment_id: slice it into chunks (1200px tall with 120px overlap by default) ' +
861
997
  'and read the text chunk by chunk with the vision model, then join the results in top-to-bottom order. ' +
862
- 'Returns the full recognized text plus per-chunk details. Good for long screenshots / documents.',
998
+ 'Marks partial results when chunks fail. Cancellation or timeout stops the whole operation. Good for long screenshots / documents.',
863
999
  parameters: {
864
1000
  type: 'object',
865
1001
  properties: {
866
1002
  image_path: { type: 'string', description: 'Absolute path to the image (png/jpg/jpeg/webp)' },
1003
+ artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path or attachment_id)' },
867
1004
  attachment_id: { type: 'string', description: 'Session attachment id of the image (alternative to image_path)' },
868
1005
  chunk_height: { type: 'integer', description: 'Chunk height in pixels, default 1200' },
869
1006
  overlap: { type: 'integer', description: 'Overlap between chunks in pixels to avoid cutting text lines, default 120' }
@@ -873,28 +1010,19 @@ export async function apply(ctx) {
873
1010
  output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
874
1011
  isConcurrencySafe: () => true,
875
1012
  async execute(args, exec) {
876
- if (exec.signal && exec.signal.aborted) throw new Error('已取消');
877
- const input = await resolveImageInput(dshHost(ctx), exec, {
878
- image_path: args.image_path,
879
- attachment_id: args.attachment_id
880
- });
881
- const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
882
- const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders });
883
- const result = await longOcr({
884
- input: input.buffer,
885
- backends,
886
- chunkHeight: args.chunk_height || 1200,
887
- overlap: args.overlap || 120,
1013
+ const host = dshHost(ctx);
1014
+ const result = await runOcrRequest(host, {
1015
+ providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit,
1016
+ prepareImage: operationSignal => resolveVisionImageInput(host, exec, args, operationSignal),
1017
+ chunkHeight: args.chunk_height,
1018
+ overlap: args.overlap,
888
1019
  signal: exec.signal
889
1020
  });
890
- const errCount = result.chunks.filter((c) => c.error).length;
891
- const head = `[iris] 长截图 OCR 完成:${result.width}x${result.height}px,${result.totalChunks} 块${errCount ? '(' + errCount + ' 块失败)' : ''}\n`;
892
- if (!result.fullText) return head + '(未识别到文字)';
893
- return head + result.fullText;
1021
+ return formatOcrResult(result) + (args.artifact_id ? `\nCore Artifact: ${args.artifact_id}` : '');
894
1022
  }
895
1023
  });
896
1024
 
897
- console.log('[iris] tools registered: iris_draw_image, iris_generate_video, iris_speak_text, iris_transcribe_audio, iris_task_status, iris_look_at_image, iris_relook_attachment, iris_crop, iris_pixel_diff, iris_locate, iris_html_screenshot, iris_long_ocr, iris_video_frames, iris_media_summarize');
1025
+ console.log('[iris] tools registered: iris_draw_image, iris_edit_image, iris_generate_video, iris_speak_text, iris_transcribe_audio, iris_task_status, iris_look_at_image, iris_relook_attachment, iris_crop, iris_pixel_diff, iris_locate, iris_html_screenshot, iris_long_ocr, iris_video_frames, iris_media_summarize');
898
1026
  };
899
1027
 
900
1028
  /* ---------------- 视觉路由(M3 补全):look/relook 共用执行器 + 降级链 ---------------- */
@@ -943,12 +1071,35 @@ async function resolveImageInput(host, exec, { image_path, attachment_id }) {
943
1071
  throw new Error('iris: 需要 image_path 或 attachment_id');
944
1072
  }
945
1073
 
1074
+ /** 视觉解引用在整体预算内;路径限额读,Host 会话/附件等待可取消。 */
1075
+ async function resolveVisionImageInput(host, exec, input, signal) {
1076
+ const source = normalizeVisionImageSource(input, { allowAttachment: true });
1077
+ const { image_path, artifact_id, attachment_id } = input;
1078
+ if (source === 'artifact_id') return visionImageForDsh({ artifactId: artifact_id, signal });
1079
+ if (image_path) {
1080
+ const file = String(image_path).trim();
1081
+ const mediaType = MEDIA_TYPES[path.extname(file).toLowerCase()];
1082
+ if (!path.isAbsolute(file) || !mediaType || mediaType === 'image/gif') throw new Error('invalid vision image');
1083
+ return readVisionImageFile(file, mediaType, signal);
1084
+ }
1085
+ if (attachment_id) {
1086
+ const id = String(attachment_id).trim();
1087
+ const hit = await waitForModelWork(() => sessionAttachmentRef(host, exec, id), signal) || findOwnAttachment(id);
1088
+ if (!hit) throw new Error('unavailable vision attachment');
1089
+ if (hit.absPath) return readVisionImageFile(hit.absPath,
1090
+ hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png', signal);
1091
+ const stored = await waitForModelWork(() => attachmentService(host).readImage(hit.ref, signal), signal);
1092
+ return { bytes: stored.data, mediaType: stored.mediaType || hit.ref.mediaType || 'image/png' };
1093
+ }
1094
+ throw new Error('missing vision image');
1095
+ }
1096
+
946
1097
  /** 在本插件的任务产物里按 attachmentId 找 iris 自生成的图片(relook 的兜底) */
947
1098
  export function findOwnAttachment(attachmentId) {
948
1099
  for (const t of tasks.all()) {
949
1100
  for (const a of t.attachments || []) {
950
1101
  if (a.attachmentId === attachmentId) {
951
- const absPath = path.join(outputDir(), a.file);
1102
+ const absPath = path.join(tasks.outputsDir(), a.file);
952
1103
  if (fs.existsSync(absPath)) {
953
1104
  return { ref: { attachmentId: a.attachmentId, mediaType: a.mediaType }, absPath };
954
1105
  }
@@ -984,42 +1135,26 @@ export async function sessionAttachmentRef(host, exec, attachmentId) {
984
1135
  * 全局视觉模型兜底;每个后端的失败现场保留在 errors 里。
985
1136
  * @returns {{answer:string,via:'selfstack'|'global',model?:string,backendId:string}}
986
1137
  */
987
- export async function askVision(host, { question, ref, dataUrl, signal, model }) {
1138
+ export async function askVision(host, { question, dataUrl, image, prepareImage, signal, model, budget }) {
988
1139
  const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
989
- const backends = buildVisionBackendsFromHost(host, { providers: visionProviders, model });
990
- return askWithBackends(backends, { question, ref, imageDataUrl: dataUrl, signal });
1140
+ return runVisionRequest(host, { providers: visionProviders, model, question, dataUrl, image, prepareImage, signal, budget, onRateLimit: store.recordRateLimit });
991
1141
  }
992
1142
 
993
1143
  /**
994
1144
  * look / relook 共用执行器:看图问答 → 人话结果(标注模型归属,方便排查走哪条链)。
995
1145
  * @returns {Promise<string>} 直接作为工具输出
996
1146
  */
997
- export async function runVisionTool(host, exec, { origin, model, question, ref, dataUrl }) {
1147
+ export async function runVisionTool(host, exec, { origin, model, question, dataUrl, image, prepareImage, artifactId }) {
998
1148
  if (exec && exec.signal && exec.signal.aborted) throw new Error('已取消');
999
1149
  const { answer, via, model: usedModel } = await askVision(host, {
1000
1150
  question,
1001
- ref,
1002
1151
  dataUrl,
1152
+ image,
1153
+ prepareImage,
1003
1154
  signal: exec && exec.signal,
1004
1155
  model
1005
1156
  });
1006
1157
  const label = origin === 'relook' ? '重看回答' : '看图回答';
1007
1158
  const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
1008
- return `[iris] ${label}(${usedModel || '默认'} · ${viaLabel}):\n${answer}`;
1009
- }
1010
-
1011
- /* ---------------- 视觉自述(画图后增强项):走 askVision 降级链,失败静默 ---------------- */
1012
- async function describeFirstImage(host, ref, absPath, originalPrompt) {
1013
- try {
1014
- const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
1015
- const { answer } = await askVision(host, {
1016
- question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
1017
- ref,
1018
- dataUrl: toDataUrl(absPath, mt),
1019
- signal: undefined
1020
- });
1021
- return answer.slice(0, 300);
1022
- } catch (_) {
1023
- return ''; // 自述是增强项,失败不影响交付
1024
- }
1159
+ return `[iris] ${label}(${usedModel || '默认'} · ${viaLabel}):\n${answer}` + (artifactId ? `\nCore Artifact: ${artifactId}` : '');
1025
1160
  }