@mokuyoaxis/dsh-iris 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-compose-media/SKILL.md +10 -5
- package/.dsh/skills/iris-compose-media/references/workflows.md +7 -1
- package/.dsh/skills/iris-verify-ui/SKILL.md +2 -1
- package/CHANGELOG.md +99 -1
- package/README.en.md +124 -31
- package/README.md +124 -31
- package/bin/dsh-iris.js +593 -13
- package/docs/AGENT_QUICKSTART.md +39 -0
- package/docs/ARCHITECTURE.md +87 -64
- package/docs/ARTIFACT_LIBRARY.md +5 -5
- package/docs/ARTIFACT_MANIFEST.md +60 -0
- package/docs/CLI_MANAGEMENT.md +181 -0
- package/docs/COMPOSITE_VISION.md +35 -0
- package/docs/CORE_RUNTIME_CONTRACT.md +84 -0
- package/docs/DSH_CORE_MIGRATION.md +135 -0
- package/docs/DSH_RC2_ADAPTATION.md +38 -0
- package/docs/FAULT_INJECTION.md +12 -2
- package/docs/HEADLESS_CLI.md +223 -0
- package/docs/HOST_ADAPTER_CONTRACT.md +36 -17
- package/docs/HOST_DOCTOR.md +4 -2
- package/docs/MODEL_PORT_CONTRACT.md +305 -0
- package/docs/OCR_MODEL.md +39 -0
- package/docs/PROMPT_OPTIMIZER.md +47 -0
- package/docs/PROVIDER_ADAPTER_CONTRACT.md +53 -10
- package/docs/PROVIDER_HEALTH.md +22 -5
- package/docs/PROVIDER_RUNTIME_HARNESS.md +33 -0
- package/docs/PROVIDER_SUBMISSION_CONTRACT.md +2 -2
- package/docs/ROADMAP.md +49 -83
- package/docs/SECURITY.md +8 -7
- package/docs/TASK_SEMANTICS.md +2 -2
- package/docs/VISION_CLI.md +132 -0
- package/docs/VISION_MODEL.md +51 -0
- package/docs/WORKBENCH_ARTIFACTS.md +67 -0
- package/docs/assets/logo/README.md +55 -0
- package/docs/assets/logo/iris-bubble-concept.svg +67 -0
- package/docs/assets/logo/iris-bubble.svg +14 -0
- package/docs/assets/logo/iris-interim-flower.png +0 -0
- package/docs/assets/logo/iris-logo-mono-inverse.svg +78 -0
- package/docs/assets/logo/iris-logo-mono.svg +78 -0
- package/docs/assets/logo/iris-logo.svg +78 -0
- package/docs/assets/logo/iris-mark-mono-inverse.svg +42 -0
- package/docs/assets/logo/iris-mark-mono.svg +42 -0
- package/docs/assets/logo/iris-mark.svg +42 -0
- package/docs/file-access-across-environments.md +8 -5
- package/docs/releases/0.2.0.md +48 -0
- package/docs/releases/0.2.1.md +56 -0
- package/lib/actions.js +520 -591
- package/lib/adapters.js +184 -36
- package/lib/api.js +200 -11
- package/lib/chromium-browser.js +117 -0
- package/lib/client.js +1390 -105
- package/lib/command-service.js +534 -0
- package/lib/composite-vision-routing.js +42 -0
- package/lib/config.js +143 -51
- package/lib/core-artifact-store.js +630 -0
- package/lib/core-artifacts.js +4 -0
- package/lib/core-attention.js +103 -0
- package/lib/core-contract.js +105 -0
- package/lib/core-doctor.js +41 -0
- package/lib/core-lease-recovery.js +170 -0
- package/lib/core-maintenance.js +213 -0
- package/lib/core-runtime.js +232 -0
- package/lib/core-tasks.js +567 -0
- package/lib/core-user-projection.js +193 -0
- package/lib/doctor.js +92 -33
- package/lib/dsh-core-adapter.js +653 -0
- package/lib/dsh-host-adapter.js +58 -7
- package/lib/dsh-text-model-adapter.js +222 -0
- package/lib/dsh-vision-model-adapter.js +75 -0
- package/lib/generation-input.js +126 -0
- package/lib/guard.js +3 -3
- package/lib/headless-vision.js +181 -0
- package/lib/host-runtime.js +1 -0
- package/lib/http-vision-model-adapter.js +187 -0
- package/lib/image-edit-input.js +34 -0
- package/lib/index.js +323 -188
- package/lib/locate.js +86 -28
- package/lib/media-probe.js +2 -1
- package/lib/model-call-runtime.js +45 -0
- package/lib/model-invoker.js +220 -0
- package/lib/model-port-contract.js +341 -0
- package/lib/models.js +105 -14
- package/lib/ocr-model-routing.js +26 -0
- package/lib/ocr.js +110 -64
- package/lib/prompt-optimizer-config.js +2 -1
- package/lib/prompt-optimizer-core.js +121 -0
- package/lib/prompt-optimizer.js +37 -193
- package/lib/provider-adapter.js +49 -23
- package/lib/provider-adapters.js +165 -26
- package/lib/provider-catalog.js +183 -0
- package/lib/provider-config-service.js +236 -0
- package/lib/provider-contract.js +74 -4
- package/lib/provider-health.js +80 -1
- package/lib/provider-protocol.js +74 -2
- package/lib/provider-task-runner.js +370 -0
- package/lib/summarize.js +31 -20
- package/lib/summary-input.js +52 -0
- package/lib/task-wait.js +40 -0
- package/lib/tasks.js +16 -4
- package/lib/video-input.js +30 -0
- package/lib/vision-core.js +42 -0
- package/lib/vision-image-input.js +83 -0
- package/lib/vision-input.js +31 -0
- package/lib/vision-model-routing.js +180 -0
- package/lib/vision.js +31 -9
- package/lib/workbench-artifacts.js +52 -0
- package/lib/workbench-management.js +151 -0
- package/lib/workbench-zip.js +39 -0
- package/package.json +8 -8
- package/user_guide.md +71 -26
package/lib/index.js
CHANGED
|
@@ -14,18 +14,22 @@ import * as cap from './capability.js';
|
|
|
14
14
|
import * as tasks from './tasks.js';
|
|
15
15
|
import { registerMedia, mediaLinksOf, authorizeMedia, serveMedia } from './media.js';
|
|
16
16
|
import { serveApi, closeAllSse, purgeStaleUploads } from './api.js';
|
|
17
|
-
import {
|
|
17
|
+
import { runVisionRequest, readVisionImageFile } from './vision-model-routing.js';
|
|
18
|
+
import { waitForModelWork } from './model-call-runtime.js';
|
|
18
19
|
import { hasHostPort, requireHostPort } from './host-contract.js';
|
|
19
20
|
import { createDshHostAdapter } from './dsh-host-adapter.js';
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
21
|
+
import { cropForDsh, diffForDsh, framesForDsh, summaryFramesForDsh, visionImageForDsh, htmlForDsh, inspectProviderTaskForDsh, projectCoreTaskForDsh, readCoreArtifactMediaForDsh, resumeProviderTaskWatchesForDsh, stopProviderTaskWatchesForDsh } from './dsh-core-adapter.js';
|
|
22
|
+
import { imageDimensions } from './pixels.js';
|
|
23
|
+
import { runLocateRequest, runSummaryRequest, describeGeneratedImage } from './composite-vision-routing.js';
|
|
24
|
+
import { normalizeSummarySource, summaryFrameRecords, summaryTimelineLabel } from './summary-input.js';
|
|
25
|
+
import { normalizeVisionImageSource } from './vision-input.js';
|
|
22
26
|
import { serveRender } from './render.js';
|
|
23
27
|
import { ensurePrivateDir, hardenPrivateTree } from './private-storage.js';
|
|
24
28
|
import { guarded } from './guard.js';
|
|
25
|
-
import {
|
|
29
|
+
import { formatOcrResult } from './ocr.js';
|
|
30
|
+
import { runOcrRequest } from './ocr-model-routing.js';
|
|
26
31
|
import { runAction, listActions, taskPollDeps } from './actions.js';
|
|
27
32
|
import { ffmpegAvailable, extractFrames, probeVideo, extractAudioTrack } from './media-probe.js';
|
|
28
|
-
import { buildContactSheet, summarizeMedia } from './summarize.js';
|
|
29
33
|
import { registerBundledSkills } from './bundled-skills.js';
|
|
30
34
|
import {
|
|
31
35
|
EXPECTED_IRIS_ROUTES, IRIS_PLUGIN_ID, beginHostRuntime, recordHostRoutes, recordHostSkills, recordHostTool
|
|
@@ -51,7 +55,7 @@ function dshHost(ctx) {
|
|
|
51
55
|
}
|
|
52
56
|
|
|
53
57
|
/** 工具内同步等待上限;超过即转后台盯守,用 iris_task_status 查询 */
|
|
54
|
-
const AWAIT_MS = { image: 180000, video: 480000 };
|
|
58
|
+
const AWAIT_MS = { image: 180000, video: 480000, transcribe: 120000 };
|
|
55
59
|
|
|
56
60
|
function outputDir() {
|
|
57
61
|
const dir = tasks.outputsDir();
|
|
@@ -153,22 +157,37 @@ async function awaitTerminal(taskId, { timeoutMs, signal }) {
|
|
|
153
157
|
}
|
|
154
158
|
}
|
|
155
159
|
|
|
156
|
-
|
|
160
|
+
function coreTaskWaitFinished(task) {
|
|
161
|
+
return Boolean(task && (task.phase === 'terminal'
|
|
162
|
+
|| (task.outcome === 'succeeded' && ['ready', 'failed'].includes(task.deliveryState))));
|
|
163
|
+
}
|
|
157
164
|
|
|
158
|
-
function
|
|
159
|
-
|
|
160
|
-
|
|
165
|
+
function coreTaskTerminalProblem(task, label = '任务') {
|
|
166
|
+
if (!task) return label + '记录不存在';
|
|
167
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'ready') return '';
|
|
168
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
169
|
+
return label + '已成功,但 Core Artifact 交付失败:'
|
|
170
|
+
+ (task.lastError?.safeMessage || '可稍后重新交付;禁止重新生成');
|
|
171
|
+
}
|
|
172
|
+
if (task.acceptance === 'unknown' || task.outcome === 'unknown') {
|
|
173
|
+
return label + '的远端受理或结果状态未知:'
|
|
174
|
+
+ (task.lastError?.safeMessage || '禁止自动重提,请先人工确认');
|
|
175
|
+
}
|
|
176
|
+
if (task.outcome === 'canceled') return label + '已取消';
|
|
177
|
+
if (task.outcome === 'failed') return label + '失败:'
|
|
178
|
+
+ (task.lastError?.safeMessage || '供应商明确返回失败');
|
|
179
|
+
return label + '需要人工处理';
|
|
161
180
|
}
|
|
162
181
|
|
|
163
|
-
function
|
|
164
|
-
const
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
182
|
+
async function awaitCoreTerminal(taskId, { timeoutMs, signal }) {
|
|
183
|
+
const deadline = Date.now() + timeoutMs;
|
|
184
|
+
for (;;) {
|
|
185
|
+
if (signal?.aborted) throw new Error('已取消;Core 任务仍保留远端受理事实,可稍后查询');
|
|
186
|
+
const task = await inspectProviderTaskForDsh(taskId, { signal });
|
|
187
|
+
if (coreTaskWaitFinished(task)) return task;
|
|
188
|
+
if (Date.now() > deadline) return null;
|
|
189
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
190
|
+
}
|
|
172
191
|
}
|
|
173
192
|
|
|
174
193
|
export const name = 'mokuyoaxis-dsh-iris';
|
|
@@ -223,9 +242,19 @@ export async function apply(ctx) {
|
|
|
223
242
|
if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个后台任务`);
|
|
224
243
|
});
|
|
225
244
|
|
|
226
|
-
|
|
245
|
+
try {
|
|
246
|
+
const resumed = await resumeProviderTaskWatchesForDsh();
|
|
247
|
+
if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个 Core 图片任务`);
|
|
248
|
+
} catch (error) {
|
|
249
|
+
console.error('[iris] 恢复 Core 图片后台任务失败(已隔离,不影响宿主):', error?.message || error);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Fiber 清理:插件停用/更新时停掉 legacy 与 Core 的全部 Host 盯守句柄
|
|
227
253
|
guard('盯守句柄注册', () => {
|
|
228
|
-
ctx.effect(() => () =>
|
|
254
|
+
ctx.effect(() => () => {
|
|
255
|
+
tasks.stopWatchAll();
|
|
256
|
+
stopProviderTaskWatchesForDsh();
|
|
257
|
+
});
|
|
229
258
|
});
|
|
230
259
|
|
|
231
260
|
// 媒体路由(视频/音频在对话流里的可播通路;图片仍走原生附件)
|
|
@@ -250,11 +279,41 @@ export async function apply(ctx) {
|
|
|
250
279
|
return;
|
|
251
280
|
}
|
|
252
281
|
|
|
282
|
+
async function coreImageToolResult(submitted, prompt, signal) {
|
|
283
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal });
|
|
284
|
+
if (!coreTaskWaitFinished(final)) final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal });
|
|
285
|
+
if (!final) return { blocks: [{ type: 'text', text: `[iris] Core 图片任务仍在后台观察(task: ${submitted.taskId})。稍后可用 iris_task_status 查询进度与 Artifact。` }] };
|
|
286
|
+
const terminalProblem = coreTaskTerminalProblem(final, '图像生成');
|
|
287
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
288
|
+
const projected = await projectCoreTaskForDsh(dshHost(ctx), submitted.taskId, { signal });
|
|
289
|
+
const refs = projected.artifacts.map((item) => item.attachment);
|
|
290
|
+
const blocks = projected.artifacts.map((item) => ({ type: 'image', attachment: item.attachment }));
|
|
291
|
+
let note = '';
|
|
292
|
+
try {
|
|
293
|
+
const described = await describeGeneratedImage(dshHost(ctx), {
|
|
294
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal,
|
|
295
|
+
async prepareImage(inputSignal) {
|
|
296
|
+
const first = await readCoreArtifactMediaForDsh(projected.artifacts[0].artifact.id, { signal: inputSignal });
|
|
297
|
+
return { bytes: new Uint8Array(first.bytes), mediaType: first.artifact.mediaType };
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
if (described) note = `画面内容:${described}`;
|
|
301
|
+
} catch (_) { /* 自述失败不影响 Core 交付 */ }
|
|
302
|
+
blocks.unshift({ type: 'text', text:
|
|
303
|
+
`[iris] 图像已生成(${submitted.model},Core task: ${submitted.taskId})。` +
|
|
304
|
+
(submitted.sourceArtifactId ? `\n原图 artifact: ${submitted.sourceArtifactId}` : '') +
|
|
305
|
+
(note ? `\n${note}` : '') +
|
|
306
|
+
`\nartifact: ${projected.artifacts.map((item) => item.artifact.id).join(', ')}` +
|
|
307
|
+
`\nattachment: ${refs.map((ref) => ref.attachmentId).join(', ')}`
|
|
308
|
+
});
|
|
309
|
+
return { blocks };
|
|
310
|
+
}
|
|
311
|
+
|
|
253
312
|
/* ---------- 🖼️ 画图 ---------- */
|
|
254
313
|
safeRegister(host, {
|
|
255
314
|
name: 'iris_draw_image',
|
|
256
315
|
description:
|
|
257
|
-
'Generate an image from a detailed prompt using the user-configured image model
|
|
316
|
+
'Generate an image from a detailed prompt using the user-configured image model and its configured image protocol. ' +
|
|
258
317
|
'Returns a durable DSH attachment rendered in the conversation plus a vision-model description so you know what was drawn.',
|
|
259
318
|
parameters: {
|
|
260
319
|
type: 'object',
|
|
@@ -273,8 +332,10 @@ export async function apply(ctx) {
|
|
|
273
332
|
isConcurrencySafe: () => false,
|
|
274
333
|
async execute(args, exec) {
|
|
275
334
|
const prompt = String(args.prompt || '').trim();
|
|
276
|
-
if (!prompt) throw new Error('iris: prompt 不能为空');
|
|
277
335
|
const submitted = await runAction(dshHost(ctx), 'image', args, { signal: exec.signal });
|
|
336
|
+
if (submitted.storage === 'core') {
|
|
337
|
+
return coreImageToolResult(submitted, prompt, exec.signal);
|
|
338
|
+
}
|
|
278
339
|
let final = tasks.get(submitted.taskId);
|
|
279
340
|
if (final && !taskWaitFinished(final)) {
|
|
280
341
|
final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
|
|
@@ -304,7 +365,10 @@ export async function apply(ctx) {
|
|
|
304
365
|
|
|
305
366
|
let note = '';
|
|
306
367
|
try {
|
|
307
|
-
const described = await
|
|
368
|
+
const described = await describeGeneratedImage(dshHost(ctx), {
|
|
369
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal: exec.signal,
|
|
370
|
+
prepareImage: signal => readVisionImageFile(files[0], MEDIA_TYPES[path.extname(files[0]).toLowerCase()] || 'image/png', signal)
|
|
371
|
+
});
|
|
308
372
|
if (described) note = `画面内容:${described}`;
|
|
309
373
|
} catch (_) { /* 增强失败不影响生成结果 */ }
|
|
310
374
|
blocks.unshift({
|
|
@@ -317,6 +381,25 @@ export async function apply(ctx) {
|
|
|
317
381
|
}
|
|
318
382
|
});
|
|
319
383
|
|
|
384
|
+
safeRegister(host, {
|
|
385
|
+
name: 'iris_edit_image',
|
|
386
|
+
description: 'Edit an existing Core image by Artifact ID and an instruction using a configured openai-chat-images model. Returns new Core Artifacts and DSH image attachments; preserves the source image and records its provenance.',
|
|
387
|
+
parameters: {
|
|
388
|
+
type: 'object', properties: {
|
|
389
|
+
source_artifact_id: { type: 'string', description: 'Core Artifact ID of the source PNG, JPEG or WebP image (up to 20 MiB)' },
|
|
390
|
+
prompt: { type: 'string', description: 'Describe the changes and what should be preserved in the source image' },
|
|
391
|
+
model: { type: 'string', description: 'Optional providerId::modelId; image protocol must be openai-chat-images' },
|
|
392
|
+
n: { type: 'string', description: "Number of images, default '1'" }
|
|
393
|
+
}, required: ['source_artifact_id', 'prompt']
|
|
394
|
+
},
|
|
395
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
396
|
+
isConcurrencySafe: () => false,
|
|
397
|
+
async execute(args, exec) {
|
|
398
|
+
const submitted = await runAction(dshHost(ctx), 'image_edit', args, { signal: exec.signal });
|
|
399
|
+
return coreImageToolResult(submitted, String(args.prompt || '').trim(), exec.signal);
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
|
|
320
403
|
/* ---------- 🎬 视频生成 ---------- */
|
|
321
404
|
safeRegister(host, {
|
|
322
405
|
name: 'iris_generate_video',
|
|
@@ -329,7 +412,7 @@ export async function apply(ctx) {
|
|
|
329
412
|
parameters: {
|
|
330
413
|
type: 'object',
|
|
331
414
|
properties: {
|
|
332
|
-
prompt: { type: 'string', description: '
|
|
415
|
+
prompt: { type: 'string', description: 'Motion/scene description, required for t2v/i2v; optional for s2v' },
|
|
333
416
|
first_frame_attachment_id: { type: 'string', description: 'Attachment id of an image generated by iris_draw_image, used as the first frame' },
|
|
334
417
|
first_frame_path: { type: 'string', description: 'Absolute path to a local image used as the first frame (alternative to attachment id)' },
|
|
335
418
|
audio_path: { type: 'string', description: 'Absolute path to a wav/mp3 file (<15MB, <20s, clear human voice). Required for s2v models like wan2.2-s2v' },
|
|
@@ -338,12 +421,31 @@ export async function apply(ctx) {
|
|
|
338
421
|
duration: { type: 'number', description: 't2v/i2v only: duration in seconds, if the model supports it' },
|
|
339
422
|
model: { type: 'string', description: 'Override with a providerId::modelId reference (legacy bare model id accepted)' }
|
|
340
423
|
},
|
|
341
|
-
required: [
|
|
424
|
+
required: []
|
|
342
425
|
},
|
|
343
426
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
344
427
|
isConcurrencySafe: () => false,
|
|
345
428
|
async execute(args, exec) {
|
|
346
|
-
const submitted = await runAction(dshHost(ctx), 'video', args, { signal: exec.signal });
|
|
429
|
+
const submitted = await runAction(dshHost(ctx), 'video', { ...args, session_id: exec.agent?.session?.id }, { signal: exec.signal });
|
|
430
|
+
if (submitted.storage === 'core') {
|
|
431
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
432
|
+
if (!coreTaskWaitFinished(final)) {
|
|
433
|
+
final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
434
|
+
}
|
|
435
|
+
if (!final) {
|
|
436
|
+
return `[iris] Core 视频任务已提交并转入后台观察(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
437
|
+
'渲染通常需要数分钟;完成后用 iris_task_status 查询进度与 Artifact,或在工作台「重新观察」。';
|
|
438
|
+
}
|
|
439
|
+
const terminalProblem = coreTaskTerminalProblem(final, '视频生成');
|
|
440
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
441
|
+
const secs = Math.round((Date.now() - new Date(final.createdAt).getTime()) / 1000);
|
|
442
|
+
const links = final.artifactIds.map((artifactId) =>
|
|
443
|
+
'/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
|
|
444
|
+
return `[iris] 视频已生成(${submitted.model || final.modelRef},${submitted.mode || 't2v'},约 ${secs}s,Core task: ${final.id}):\n` +
|
|
445
|
+
`artifact: ${final.artifactIds.join(', ')}\n` +
|
|
446
|
+
links.join('\n') +
|
|
447
|
+
'\n(链接受 DSH Host 与浏览器同源守卫保护;Artifact ID 持有者可访问,请勿公开)';
|
|
448
|
+
}
|
|
347
449
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
348
450
|
if (!final) {
|
|
349
451
|
return `[iris] 视频任务已提交并转入后台盯守(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
@@ -378,6 +480,16 @@ export async function apply(ctx) {
|
|
|
378
480
|
isConcurrencySafe: () => true,
|
|
379
481
|
async execute(args, exec) {
|
|
380
482
|
const submitted = await runAction(dshHost(ctx), 'tts', args, { signal: exec.signal });
|
|
483
|
+
if (submitted.storage === 'core') {
|
|
484
|
+
const final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
485
|
+
const terminalProblem = coreTaskTerminalProblem(final, '语音合成');
|
|
486
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
487
|
+
const links = final.artifactIds.map((artifactId) =>
|
|
488
|
+
'/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
|
|
489
|
+
return `[iris] 语音已合成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
|
|
490
|
+
`artifact: ${final.artifactIds.join(', ')}\n` +
|
|
491
|
+
links.map((link) => `[♪ 音频播放](${link})`).join('\n');
|
|
492
|
+
}
|
|
381
493
|
const final = tasks.get(submitted.taskId);
|
|
382
494
|
const terminalProblem = taskTerminalProblem(final, '语音合成');
|
|
383
495
|
if (terminalProblem) throw new Error(terminalProblem);
|
|
@@ -405,6 +517,25 @@ export async function apply(ctx) {
|
|
|
405
517
|
isConcurrencySafe: () => false,
|
|
406
518
|
async execute(args, exec) {
|
|
407
519
|
const submitted = await runAction(dshHost(ctx), 'transcribe', args, { signal: exec.signal });
|
|
520
|
+
if (submitted.storage === 'core') {
|
|
521
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
522
|
+
if (!coreTaskWaitFinished(final)) {
|
|
523
|
+
final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.transcribe, signal: exec.signal });
|
|
524
|
+
}
|
|
525
|
+
if (!final) {
|
|
526
|
+
return `[iris] Core 转写任务已提交并转入后台观察(task: ${submitted.taskId})。完成后用 iris_task_status 查询进度与 Artifact。`;
|
|
527
|
+
}
|
|
528
|
+
const terminalProblem = coreTaskTerminalProblem(final, '音频转写');
|
|
529
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
530
|
+
const texts = [];
|
|
531
|
+
for (const artifactId of final.artifactIds) {
|
|
532
|
+
const media = await readCoreArtifactMediaForDsh(artifactId, { signal: exec.signal });
|
|
533
|
+
texts.push(Buffer.from(media.bytes).toString('utf8'));
|
|
534
|
+
}
|
|
535
|
+
return `[iris] 音频转写完成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
|
|
536
|
+
texts.join('\n') +
|
|
537
|
+
`\nartifact: ${final.artifactIds.join(', ')}`;
|
|
538
|
+
}
|
|
408
539
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
409
540
|
if (!final) return `[iris] 音频转写任务已提交并转入后台(task: ${submitted.taskId})。完成后用 iris_task_status 查询。`;
|
|
410
541
|
const terminalProblem = taskTerminalProblem(final, '音频转写');
|
|
@@ -435,26 +566,22 @@ export async function apply(ctx) {
|
|
|
435
566
|
isConcurrencySafe: () => false,
|
|
436
567
|
async execute(args, exec) {
|
|
437
568
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
438
|
-
const
|
|
439
|
-
|
|
440
|
-
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法抽帧(可选安装 ffmpeg/ffprobe 后启用;不影响其他 iris 工具)');
|
|
441
|
-
const meta = probeVideo(p);
|
|
442
|
-
const frames = await extractFrames({
|
|
443
|
-
inputPath: p,
|
|
569
|
+
const { media: meta, frames } = await framesForDsh({
|
|
570
|
+
videoPath: args.video_path,
|
|
444
571
|
maxFrames: args.max_frames,
|
|
445
572
|
targetWidth: args.target_width,
|
|
446
573
|
format: args.format,
|
|
447
574
|
quality: args.quality,
|
|
448
575
|
signal: exec.signal
|
|
449
576
|
});
|
|
450
|
-
const mediaType =
|
|
451
|
-
const ext =
|
|
577
|
+
const mediaType = frames[0].mediaType;
|
|
578
|
+
const ext = mediaType === 'image/png' ? 'png' : 'jpg';
|
|
452
579
|
const blocks = [];
|
|
453
580
|
const refs = [];
|
|
454
581
|
for (let i = 0; i < frames.length; i++) {
|
|
455
582
|
const f = frames[i];
|
|
456
583
|
const ref = await attachmentService(host).saveImage({
|
|
457
|
-
data: new Uint8Array(f.
|
|
584
|
+
data: new Uint8Array(f.bytes),
|
|
458
585
|
mediaType,
|
|
459
586
|
name: `iris-frame-${Date.now()}-${i + 1}.${ext}`
|
|
460
587
|
});
|
|
@@ -468,9 +595,10 @@ export async function apply(ctx) {
|
|
|
468
595
|
`[iris] 视频抽帧完成:${frames.length} 帧(${meta.durationSec.toFixed(1)}s 视频,` +
|
|
469
596
|
`${frames[0].width}x${frames[0].height},${mediaType === 'image/png' ? 'PNG' : 'JPEG'})。\n` +
|
|
470
597
|
`时间戳:${ats}\n` +
|
|
598
|
+
`frame_artifact_ids: ${JSON.stringify(frames.map(f => f.artifact.id))}\n` +
|
|
471
599
|
`attachments: ${refs.map((r) => r.attachmentId).join(', ')}`
|
|
472
600
|
});
|
|
473
|
-
return { blocks };
|
|
601
|
+
return { blocks, artifactIds: frames.map((frame) => frame.artifact.id) };
|
|
474
602
|
}
|
|
475
603
|
});
|
|
476
604
|
|
|
@@ -478,83 +606,91 @@ export async function apply(ctx) {
|
|
|
478
606
|
safeRegister(host, {
|
|
479
607
|
name: 'iris_media_summarize',
|
|
480
608
|
description:
|
|
481
|
-
'Summarize a local video
|
|
609
|
+
'Summarize a local video or existing Core video-frame Artifacts as one contact sheet (with timestamps) plus an optional ' +
|
|
482
610
|
'auto-transcribed audio track, then asking the vision model to describe content. Returns the summary text plus the contact sheet as a DSH image attachment. ' +
|
|
483
|
-
'
|
|
611
|
+
'Provide exactly one of video_path or frame_artifact_ids. Existing frames are read in timestamp order without ffmpeg or the original video. ' +
|
|
612
|
+
'The video_path mode requires ffmpeg + ffprobe and can auto-transcribe the audio track with the configured provider.',
|
|
484
613
|
parameters: {
|
|
485
614
|
type: 'object',
|
|
486
615
|
properties: {
|
|
487
616
|
video_path: { type: 'string', description: 'Absolute path to the local video file (mp4/webm/mov etc.)' },
|
|
617
|
+
frame_artifact_ids: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: 20, uniqueItems: true,
|
|
618
|
+
description: 'Existing Core video-frame IDs from iris_video_frames; sorted by timestamp. Use instead of video_path, without sampling options or auto-transcription.' },
|
|
488
619
|
question: { type: 'string', description: 'Custom question about the video content (default: summarize scenes/theme)' },
|
|
489
620
|
max_frames: { type: 'integer', description: 'Frames to sample for the contact sheet, clamped to 1..12 (default 8)' },
|
|
490
621
|
target_width: { type: 'integer', description: 'Frame width for sampling (default 640)' },
|
|
491
|
-
transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track
|
|
622
|
+
transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track for video_path (default true unless transcribe_text is provided). Existing frames only consume explicit text.' },
|
|
623
|
+
transcribe_text: { type: 'string', description: 'Optional existing transcript text; does not create a transcription task' },
|
|
492
624
|
model: { type: 'string', description: 'Override the configured vision model id' }
|
|
493
625
|
},
|
|
494
|
-
required: [
|
|
626
|
+
required: []
|
|
495
627
|
},
|
|
496
628
|
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
497
629
|
isConcurrencySafe: () => false,
|
|
498
630
|
async execute(args, exec) {
|
|
499
631
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
632
|
+
const source = normalizeSummarySource(args);
|
|
500
633
|
const p = String(args.video_path || '').trim();
|
|
501
|
-
if (
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
634
|
+
if (source === 'video') {
|
|
635
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
|
|
636
|
+
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法分析视频(可选安装 ffmpeg/ffprobe 后启用)');
|
|
637
|
+
}
|
|
638
|
+
const { answer, via, model, sheet, meta, frames, transcriptNote } = await runSummaryRequest(dshHost(ctx), {
|
|
639
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, question: args.question, signal: exec.signal,
|
|
640
|
+
async prepareMedia(signal) {
|
|
641
|
+
if (source === 'core-artifacts') {
|
|
642
|
+
return { ...await summaryFramesForDsh({ frameArtifactIds: args.frame_artifact_ids, signal }), transcript: args.transcribe_text };
|
|
643
|
+
}
|
|
644
|
+
const meta = probeVideo(p);
|
|
645
|
+
const frames = await extractFrames({
|
|
646
|
+
inputPath: p, maxFrames: Math.min(Number(args.max_frames) || 8, 12),
|
|
647
|
+
targetWidth: args.target_width, format: 'jpeg', quality: 70, signal
|
|
648
|
+
});
|
|
649
|
+
// 显式选项保留自动转写;消费已完成的 Core Artifact,不重提远端任务。
|
|
650
|
+
let transcript = args.transcribe_text || '', transcriptNote = '';
|
|
651
|
+
if (args.transcribe !== false && !transcript.trim() && meta.hasAudio) {
|
|
652
|
+
if (store.pickAllFor(cap.CAPABILITIES.TRANSCRIBE).length) {
|
|
653
|
+
try {
|
|
654
|
+
const audio = await extractAudioTrack({ inputPath: p, signal });
|
|
655
|
+
try {
|
|
656
|
+
const submitted = await runAction(dshHost(ctx), 'transcribe', { audio_path: audio.filePath }, { signal });
|
|
657
|
+
const final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: 120000, signal });
|
|
658
|
+
if (final?.outcome === 'succeeded' && final.deliveryState === 'ready') {
|
|
659
|
+
const texts = [];
|
|
660
|
+
for (const artifactId of final.artifactIds) {
|
|
661
|
+
const media = await readCoreArtifactMediaForDsh(artifactId, { signal });
|
|
662
|
+
texts.push(Buffer.from(media.bytes).toString('utf8'));
|
|
663
|
+
}
|
|
664
|
+
transcript = texts.join('\n');
|
|
665
|
+
} else transcriptNote = '\n(音轨转写未完成,本次摘要仅画面;可用 iris_task_status 查询 Core task)';
|
|
666
|
+
} finally {
|
|
667
|
+
fs.rmSync(audio.outDir, { recursive: true, force: true });
|
|
668
|
+
}
|
|
669
|
+
} catch (err) {
|
|
670
|
+
if (signal.aborted) throw signal.reason;
|
|
671
|
+
transcriptNote = '\n(音轨转写失败,本次摘要仅画面;可用 iris_task_status 查询转写任务)';
|
|
672
|
+
}
|
|
673
|
+
} else transcriptNote = '\n(视频含音轨但未配置转写供应商,本次摘要仅画面)';
|
|
531
674
|
}
|
|
532
|
-
|
|
533
|
-
transcriptNote = '\n(视频含音轨但未配置转写供应商,本次摘要仅画面)';
|
|
675
|
+
return { meta, frames, transcript, transcriptNote };
|
|
534
676
|
}
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
const sheet = await buildContactSheet({ frames, cols: 3 });
|
|
677
|
+
});
|
|
678
|
+
// 成功摘要的同一张拼图作为显式工具产物发布。
|
|
679
|
+
if (exec.signal?.aborted) throw exec.signal.reason;
|
|
539
680
|
const ref = await attachmentService(host).saveImage({
|
|
540
681
|
data: new Uint8Array(sheet.buffer),
|
|
541
682
|
mediaType: 'image/png',
|
|
542
683
|
name: `iris-sheet-${Date.now()}.png`
|
|
543
684
|
});
|
|
544
|
-
|
|
545
|
-
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
546
|
-
const { answer, via, model } = await summarizeMedia({
|
|
547
|
-
backends, question: args.question, transcript,
|
|
548
|
-
imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
|
|
549
|
-
ref, signal: exec.signal
|
|
550
|
-
});
|
|
685
|
+
if (exec.signal?.aborted) throw exec.signal.reason;
|
|
551
686
|
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
552
687
|
return {
|
|
688
|
+
frames: summaryFrameRecords(frames),
|
|
553
689
|
blocks: [
|
|
554
690
|
{
|
|
555
691
|
type: 'text',
|
|
556
692
|
text:
|
|
557
|
-
`[iris] 视频摘要完成(${model} · ${viaLabel},${meta
|
|
693
|
+
`[iris] 视频摘要完成(${model} · ${viaLabel},${summaryTimelineLabel(meta)} / ${frames.length} 帧联系表):\n` +
|
|
558
694
|
`${answer}${transcriptNote || ''}\n` +
|
|
559
695
|
`contact sheet attachment: ${ref.attachmentId}`
|
|
560
696
|
},
|
|
@@ -569,30 +705,39 @@ export async function apply(ctx) {
|
|
|
569
705
|
safeRegister(host, {
|
|
570
706
|
name: 'iris_look_at_image',
|
|
571
707
|
description:
|
|
572
|
-
'Look at a local image file and answer one question
|
|
708
|
+
'Look at a local image file or an existing Core image Artifact and answer one question with the configured vision model (self-hosted provider stack, then DSH default vision). Provide exactly one of image_path or artifact_id. Local files are saved as durable DSH attachments; Core images are read directly without export or Core writes.',
|
|
573
709
|
parameters: {
|
|
574
710
|
type: 'object',
|
|
575
711
|
properties: {
|
|
576
712
|
image_path: { type: 'string', description: 'Absolute path to the image file (png/jpg/jpeg/webp)' },
|
|
713
|
+
artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path)' },
|
|
577
714
|
question: { type: 'string', description: 'Question or extraction request about the image; defaults to a detailed description' },
|
|
578
715
|
model: { type: 'string', description: 'Override the configured vision model id (default qwen-vl-plus)' }
|
|
579
716
|
},
|
|
580
|
-
required: [
|
|
717
|
+
required: []
|
|
581
718
|
},
|
|
582
719
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
583
720
|
isConcurrencySafe: () => true,
|
|
584
721
|
async execute(args, exec) {
|
|
722
|
+
const source = normalizeVisionImageSource(args);
|
|
585
723
|
const p = String(args.image_path || '').trim();
|
|
586
|
-
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
587
724
|
const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
|
|
588
|
-
if (
|
|
589
|
-
|
|
725
|
+
if (source === 'image_path') {
|
|
726
|
+
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
727
|
+
if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp):' + p);
|
|
728
|
+
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在:' + p);
|
|
729
|
+
}
|
|
590
730
|
const question = String(args.question || '').trim() || '详细描述这张图片。';
|
|
591
731
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
592
732
|
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
733
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'tool', model: args.model, question, artifactId: args.artifact_id,
|
|
734
|
+
async prepareImage(signal) {
|
|
735
|
+
if (source === 'artifact_id') return visionImageForDsh({ artifactId: args.artifact_id, signal });
|
|
736
|
+
const image = await readVisionImageFile(p, mt, signal);
|
|
737
|
+
await waitForModelWork(() => attachmentService(host).saveImage({ data: image.bytes, mediaType: mt, name: path.basename(p) }), signal);
|
|
738
|
+
return image;
|
|
739
|
+
}
|
|
740
|
+
});
|
|
596
741
|
}
|
|
597
742
|
});
|
|
598
743
|
|
|
@@ -618,18 +763,16 @@ export async function apply(ctx) {
|
|
|
618
763
|
if (!question) throw new Error('iris: question 不能为空');
|
|
619
764
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
620
765
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
}
|
|
632
|
-
return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
766
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question,
|
|
767
|
+
async prepareImage(signal) {
|
|
768
|
+
// 会话扫描、解引用与读图都属于这次操作的总预算。
|
|
769
|
+
const hit = (await waitForModelWork(() => sessionAttachmentRef(host, exec, id), signal)) || findOwnAttachment(id);
|
|
770
|
+
if (!hit) throw new Error('iris: 图片引用不可用');
|
|
771
|
+
if (hit.absPath) return readVisionImageFile(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png', signal);
|
|
772
|
+
const stored = await waitForModelWork(() => attachmentService(host).readImage(hit.ref, signal), signal);
|
|
773
|
+
return { bytes: new Uint8Array(stored.data), mediaType: stored.mediaType || hit.ref.mediaType };
|
|
774
|
+
}
|
|
775
|
+
});
|
|
633
776
|
}
|
|
634
777
|
});
|
|
635
778
|
|
|
@@ -648,14 +791,8 @@ export async function apply(ctx) {
|
|
|
648
791
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
649
792
|
isConcurrencySafe: () => true,
|
|
650
793
|
async execute(args) {
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
if (!t) throw new Error('iris: 没有这个任务:' + args.task_id);
|
|
654
|
-
return fmtTask(t);
|
|
655
|
-
}
|
|
656
|
-
const recent = tasks.list().slice(0, 5);
|
|
657
|
-
if (!recent.length) return '[iris] 还没有生成任务记录。';
|
|
658
|
-
return recent.map(fmtTask).join('\n---\n');
|
|
794
|
+
const result = await runAction(dshHost(ctx), 'status', args || {});
|
|
795
|
+
return result.text;
|
|
659
796
|
}
|
|
660
797
|
});
|
|
661
798
|
|
|
@@ -685,21 +822,25 @@ export async function apply(ctx) {
|
|
|
685
822
|
image_path: args.image_path,
|
|
686
823
|
attachment_id: args.attachment_id
|
|
687
824
|
});
|
|
688
|
-
const
|
|
689
|
-
|
|
825
|
+
const result = await cropForDsh({
|
|
826
|
+
...(args.image_path
|
|
827
|
+
? { imagePath: String(args.image_path).trim() }
|
|
828
|
+
: { bytes: input.buffer, mediaType: input.mediaType }),
|
|
690
829
|
left: args.left,
|
|
691
830
|
top: args.top,
|
|
692
831
|
width: args.width,
|
|
693
|
-
height: args.height
|
|
832
|
+
height: args.height,
|
|
833
|
+
signal: exec.signal
|
|
694
834
|
});
|
|
835
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
695
836
|
const ref = await attachmentService(host).saveImage({
|
|
696
|
-
data: new Uint8Array(
|
|
697
|
-
mediaType:
|
|
837
|
+
data: new Uint8Array(result.bytes),
|
|
838
|
+
mediaType: result.mediaType,
|
|
698
839
|
name: `iris-crop-${Date.now()}.png`
|
|
699
840
|
});
|
|
700
841
|
return {
|
|
701
842
|
blocks: [
|
|
702
|
-
{ type: 'text', text: `[iris] 裁剪完成:${width}x${height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
|
|
843
|
+
{ type: 'text', text: `[iris] 裁剪完成:${result.width}x${result.height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
|
|
703
844
|
{ type: 'image', attachment: ref }
|
|
704
845
|
]
|
|
705
846
|
};
|
|
@@ -732,20 +873,21 @@ export async function apply(ctx) {
|
|
|
732
873
|
if (!(args.image_b_path || args.attachment_b_id)) throw new Error('iris: 需要 image_b_path 或 attachment_b_id');
|
|
733
874
|
const a = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
734
875
|
const b = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
735
|
-
const
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
grid: args.grid
|
|
739
|
-
topRegions: args.top_regions || 3
|
|
876
|
+
const result = await diffForDsh({
|
|
877
|
+
imageA: { bytes: a.buffer, mediaType: a.mediaType },
|
|
878
|
+
imageB: { bytes: b.buffer, mediaType: b.mediaType },
|
|
879
|
+
grid: args.grid, topRegions: args.top_regions, signal: exec.signal
|
|
740
880
|
});
|
|
881
|
+
const { ratio, diffPixels, totalPixels, width, height, worstRegions } = result.metrics;
|
|
741
882
|
const ref = await attachmentService(host).saveImage({
|
|
742
|
-
data: new Uint8Array(
|
|
743
|
-
mediaType:
|
|
883
|
+
data: new Uint8Array(result.bytes),
|
|
884
|
+
mediaType: result.mediaType,
|
|
744
885
|
name: `iris-diff-${Date.now()}.png`
|
|
745
886
|
});
|
|
746
887
|
const pct = (ratio * 100).toFixed(2);
|
|
747
888
|
const regions = worstRegions.map((r) => `(col ${r.col},row ${r.row}) ${(r.score * 100).toFixed(1)}%`).join(' ');
|
|
748
889
|
return {
|
|
890
|
+
artifactIds: [result.artifact.id],
|
|
749
891
|
blocks: [
|
|
750
892
|
{
|
|
751
893
|
type: 'text',
|
|
@@ -763,13 +905,14 @@ export async function apply(ctx) {
|
|
|
763
905
|
safeRegister(host, {
|
|
764
906
|
name: 'iris_locate',
|
|
765
907
|
description:
|
|
766
|
-
'Locate an object/region described by `target` in an image (absolute
|
|
908
|
+
'Locate an object/region described by `target` in an image (exactly one of absolute image_path, Core artifact_id, or session attachment_id), ' +
|
|
767
909
|
'returning the pixel bounding box (x1,y1,x2,y2) in original image coordinates. ' +
|
|
768
|
-
'Use the
|
|
910
|
+
'Use the bbox with iris_crop for path/session images or CLI run crop with artifact_id for Core images; do not treat a Core ID as a file path.',
|
|
769
911
|
parameters: {
|
|
770
912
|
type: 'object',
|
|
771
913
|
properties: {
|
|
772
914
|
image_path: { type: 'string', description: 'Absolute path to the source image (png/jpg/jpeg/webp)' },
|
|
915
|
+
artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path or attachment_id)' },
|
|
773
916
|
attachment_id: { type: 'string', description: 'Session attachment id of an image (alternative to image_path)' },
|
|
774
917
|
target: { type: 'string', description: 'The object or UI element to locate, e.g. "send button"' },
|
|
775
918
|
model: { type: 'string', description: 'Override the vision model for this locate call (e.g. qwen3-vl-235b-a22b-thinking for higher grounding precision; default is the provider visionModel, typically qwen-vl-plus)' }
|
|
@@ -779,23 +922,15 @@ export async function apply(ctx) {
|
|
|
779
922
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
780
923
|
isConcurrencySafe: () => true,
|
|
781
924
|
async execute(args, exec) {
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
attachment_id: args.attachment_id
|
|
786
|
-
});
|
|
787
|
-
const { width, height } = await imageDimensions(input.buffer);
|
|
788
|
-
const dataUrl = `data:${input.mediaType};base64,${input.buffer.toString('base64')}`;
|
|
789
|
-
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
790
|
-
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
791
|
-
const r = await locateObject(backends, {
|
|
792
|
-
target: args.target,
|
|
793
|
-
imageDataUrl: dataUrl,
|
|
794
|
-
width,
|
|
795
|
-
height,
|
|
796
|
-
signal: exec.signal
|
|
925
|
+
const r = await runLocateRequest(dshHost(ctx), {
|
|
926
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, target: args.target,
|
|
927
|
+
prepareImage: signal => resolveVisionImageInput(dshHost(ctx), exec, args, signal), signal: exec.signal
|
|
797
928
|
});
|
|
798
|
-
|
|
929
|
+
const { width, height } = r;
|
|
930
|
+
const sourceNote = args.artifact_id ? `\nCore Artifact: ${args.artifact_id}` : '';
|
|
931
|
+
if (!r.found) return `[iris] 在图片中未找到「${args.target}」(${r.via} · ${r.model})${sourceNote}`;
|
|
932
|
+
if (args.artifact_id) return `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})${sourceNote}\n` +
|
|
933
|
+
`裁剪区域:left=${r.x1}, top=${r.y1}, width=${r.x2 - r.x1}, height=${r.y2 - r.y1}`;
|
|
799
934
|
return `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})\n` +
|
|
800
935
|
`裁剪指令:iris_crop(image_path="${args.image_path || args.attachment_id}", left=${r.x1}, top=${r.y1}, width=${r.x2 - r.x1}, height=${r.y2 - r.y1})`;
|
|
801
936
|
}
|
|
@@ -825,7 +960,8 @@ export async function apply(ctx) {
|
|
|
825
960
|
async execute(args, exec) {
|
|
826
961
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
827
962
|
const browser = requireHostPort(dshHost(ctx), 'browser', 'HTML 截图');
|
|
828
|
-
const rendered = await
|
|
963
|
+
const rendered = await htmlForDsh({
|
|
964
|
+
browser,
|
|
829
965
|
html: args.html,
|
|
830
966
|
width: args.width,
|
|
831
967
|
height: args.height,
|
|
@@ -844,7 +980,7 @@ export async function apply(ctx) {
|
|
|
844
980
|
{
|
|
845
981
|
type: 'text',
|
|
846
982
|
text: `[iris] HTML 截图完成(${outW}x${outH},fullPage: ${args.fullPage !== false})\n` +
|
|
847
|
-
`
|
|
983
|
+
`artifact: ${rendered.artifact.id}\nattachment: ${ref.attachmentId}` +
|
|
848
984
|
`\n注意:渲染在共享浏览器临时标签页中完成,页面短暂可见属正常行为。`
|
|
849
985
|
},
|
|
850
986
|
{ type: 'image', attachment: ref }
|
|
@@ -857,13 +993,14 @@ export async function apply(ctx) {
|
|
|
857
993
|
safeRegister(host, {
|
|
858
994
|
name: 'iris_long_ocr',
|
|
859
995
|
description:
|
|
860
|
-
'OCR a (possibly long) image: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
996
|
+
'OCR a (possibly long) image from exactly one of image_path, Core artifact_id, or session attachment_id: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
861
997
|
'and read the text chunk by chunk with the vision model, then join the results in top-to-bottom order. ' +
|
|
862
|
-
'
|
|
998
|
+
'Marks partial results when chunks fail. Cancellation or timeout stops the whole operation. Good for long screenshots / documents.',
|
|
863
999
|
parameters: {
|
|
864
1000
|
type: 'object',
|
|
865
1001
|
properties: {
|
|
866
1002
|
image_path: { type: 'string', description: 'Absolute path to the image (png/jpg/jpeg/webp)' },
|
|
1003
|
+
artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path or attachment_id)' },
|
|
867
1004
|
attachment_id: { type: 'string', description: 'Session attachment id of the image (alternative to image_path)' },
|
|
868
1005
|
chunk_height: { type: 'integer', description: 'Chunk height in pixels, default 1200' },
|
|
869
1006
|
overlap: { type: 'integer', description: 'Overlap between chunks in pixels to avoid cutting text lines, default 120' }
|
|
@@ -873,28 +1010,19 @@ export async function apply(ctx) {
|
|
|
873
1010
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
874
1011
|
isConcurrencySafe: () => true,
|
|
875
1012
|
async execute(args, exec) {
|
|
876
|
-
|
|
877
|
-
const
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders });
|
|
883
|
-
const result = await longOcr({
|
|
884
|
-
input: input.buffer,
|
|
885
|
-
backends,
|
|
886
|
-
chunkHeight: args.chunk_height || 1200,
|
|
887
|
-
overlap: args.overlap || 120,
|
|
1013
|
+
const host = dshHost(ctx);
|
|
1014
|
+
const result = await runOcrRequest(host, {
|
|
1015
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit,
|
|
1016
|
+
prepareImage: operationSignal => resolveVisionImageInput(host, exec, args, operationSignal),
|
|
1017
|
+
chunkHeight: args.chunk_height,
|
|
1018
|
+
overlap: args.overlap,
|
|
888
1019
|
signal: exec.signal
|
|
889
1020
|
});
|
|
890
|
-
|
|
891
|
-
const head = `[iris] 长截图 OCR 完成:${result.width}x${result.height}px,${result.totalChunks} 块${errCount ? '(' + errCount + ' 块失败)' : ''}\n`;
|
|
892
|
-
if (!result.fullText) return head + '(未识别到文字)';
|
|
893
|
-
return head + result.fullText;
|
|
1021
|
+
return formatOcrResult(result) + (args.artifact_id ? `\nCore Artifact: ${args.artifact_id}` : '');
|
|
894
1022
|
}
|
|
895
1023
|
});
|
|
896
1024
|
|
|
897
|
-
console.log('[iris] tools registered: iris_draw_image, iris_generate_video, iris_speak_text, iris_transcribe_audio, iris_task_status, iris_look_at_image, iris_relook_attachment, iris_crop, iris_pixel_diff, iris_locate, iris_html_screenshot, iris_long_ocr, iris_video_frames, iris_media_summarize');
|
|
1025
|
+
console.log('[iris] tools registered: iris_draw_image, iris_edit_image, iris_generate_video, iris_speak_text, iris_transcribe_audio, iris_task_status, iris_look_at_image, iris_relook_attachment, iris_crop, iris_pixel_diff, iris_locate, iris_html_screenshot, iris_long_ocr, iris_video_frames, iris_media_summarize');
|
|
898
1026
|
};
|
|
899
1027
|
|
|
900
1028
|
/* ---------------- 视觉路由(M3 补全):look/relook 共用执行器 + 降级链 ---------------- */
|
|
@@ -943,12 +1071,35 @@ async function resolveImageInput(host, exec, { image_path, attachment_id }) {
|
|
|
943
1071
|
throw new Error('iris: 需要 image_path 或 attachment_id');
|
|
944
1072
|
}
|
|
945
1073
|
|
|
1074
|
+
/** 视觉解引用在整体预算内;路径限额读,Host 会话/附件等待可取消。 */
|
|
1075
|
+
async function resolveVisionImageInput(host, exec, input, signal) {
|
|
1076
|
+
const source = normalizeVisionImageSource(input, { allowAttachment: true });
|
|
1077
|
+
const { image_path, artifact_id, attachment_id } = input;
|
|
1078
|
+
if (source === 'artifact_id') return visionImageForDsh({ artifactId: artifact_id, signal });
|
|
1079
|
+
if (image_path) {
|
|
1080
|
+
const file = String(image_path).trim();
|
|
1081
|
+
const mediaType = MEDIA_TYPES[path.extname(file).toLowerCase()];
|
|
1082
|
+
if (!path.isAbsolute(file) || !mediaType || mediaType === 'image/gif') throw new Error('invalid vision image');
|
|
1083
|
+
return readVisionImageFile(file, mediaType, signal);
|
|
1084
|
+
}
|
|
1085
|
+
if (attachment_id) {
|
|
1086
|
+
const id = String(attachment_id).trim();
|
|
1087
|
+
const hit = await waitForModelWork(() => sessionAttachmentRef(host, exec, id), signal) || findOwnAttachment(id);
|
|
1088
|
+
if (!hit) throw new Error('unavailable vision attachment');
|
|
1089
|
+
if (hit.absPath) return readVisionImageFile(hit.absPath,
|
|
1090
|
+
hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png', signal);
|
|
1091
|
+
const stored = await waitForModelWork(() => attachmentService(host).readImage(hit.ref, signal), signal);
|
|
1092
|
+
return { bytes: stored.data, mediaType: stored.mediaType || hit.ref.mediaType || 'image/png' };
|
|
1093
|
+
}
|
|
1094
|
+
throw new Error('missing vision image');
|
|
1095
|
+
}
|
|
1096
|
+
|
|
946
1097
|
/** 在本插件的任务产物里按 attachmentId 找 iris 自生成的图片(relook 的兜底) */
|
|
947
1098
|
export function findOwnAttachment(attachmentId) {
|
|
948
1099
|
for (const t of tasks.all()) {
|
|
949
1100
|
for (const a of t.attachments || []) {
|
|
950
1101
|
if (a.attachmentId === attachmentId) {
|
|
951
|
-
const absPath = path.join(
|
|
1102
|
+
const absPath = path.join(tasks.outputsDir(), a.file);
|
|
952
1103
|
if (fs.existsSync(absPath)) {
|
|
953
1104
|
return { ref: { attachmentId: a.attachmentId, mediaType: a.mediaType }, absPath };
|
|
954
1105
|
}
|
|
@@ -984,42 +1135,26 @@ export async function sessionAttachmentRef(host, exec, attachmentId) {
|
|
|
984
1135
|
* 全局视觉模型兜底;每个后端的失败现场保留在 errors 里。
|
|
985
1136
|
* @returns {{answer:string,via:'selfstack'|'global',model?:string,backendId:string}}
|
|
986
1137
|
*/
|
|
987
|
-
export async function askVision(host, { question,
|
|
1138
|
+
export async function askVision(host, { question, dataUrl, image, prepareImage, signal, model, budget }) {
|
|
988
1139
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
989
|
-
|
|
990
|
-
return askWithBackends(backends, { question, ref, imageDataUrl: dataUrl, signal });
|
|
1140
|
+
return runVisionRequest(host, { providers: visionProviders, model, question, dataUrl, image, prepareImage, signal, budget, onRateLimit: store.recordRateLimit });
|
|
991
1141
|
}
|
|
992
1142
|
|
|
993
1143
|
/**
|
|
994
1144
|
* look / relook 共用执行器:看图问答 → 人话结果(标注模型归属,方便排查走哪条链)。
|
|
995
1145
|
* @returns {Promise<string>} 直接作为工具输出
|
|
996
1146
|
*/
|
|
997
|
-
export async function runVisionTool(host, exec, { origin, model, question,
|
|
1147
|
+
export async function runVisionTool(host, exec, { origin, model, question, dataUrl, image, prepareImage, artifactId }) {
|
|
998
1148
|
if (exec && exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
999
1149
|
const { answer, via, model: usedModel } = await askVision(host, {
|
|
1000
1150
|
question,
|
|
1001
|
-
ref,
|
|
1002
1151
|
dataUrl,
|
|
1152
|
+
image,
|
|
1153
|
+
prepareImage,
|
|
1003
1154
|
signal: exec && exec.signal,
|
|
1004
1155
|
model
|
|
1005
1156
|
});
|
|
1006
1157
|
const label = origin === 'relook' ? '重看回答' : '看图回答';
|
|
1007
1158
|
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
1008
|
-
return `[iris] ${label}(${usedModel || '默认'} · ${viaLabel}):\n${answer}
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
/* ---------------- 视觉自述(画图后增强项):走 askVision 降级链,失败静默 ---------------- */
|
|
1012
|
-
async function describeFirstImage(host, ref, absPath, originalPrompt) {
|
|
1013
|
-
try {
|
|
1014
|
-
const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
|
|
1015
|
-
const { answer } = await askVision(host, {
|
|
1016
|
-
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
1017
|
-
ref,
|
|
1018
|
-
dataUrl: toDataUrl(absPath, mt),
|
|
1019
|
-
signal: undefined
|
|
1020
|
-
});
|
|
1021
|
-
return answer.slice(0, 300);
|
|
1022
|
-
} catch (_) {
|
|
1023
|
-
return ''; // 自述是增强项,失败不影响交付
|
|
1024
|
-
}
|
|
1159
|
+
return `[iris] ${label}(${usedModel || '默认'} · ${viaLabel}):\n${answer}` + (artifactId ? `\nCore Artifact: ${artifactId}` : '');
|
|
1025
1160
|
}
|