@mokuyoaxis/dsh-iris 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-compose-media/SKILL.md +23 -49
- package/.dsh/skills/iris-compose-media/references/task-v2-recovery.md +30 -0
- package/.dsh/skills/iris-compose-media/references/workflows.md +38 -0
- package/.dsh/skills/iris-verify-ui/SKILL.md +3 -25
- package/.dsh/skills/iris-verify-ui/agents/openai.yaml +4 -0
- package/.dsh/skills/iris-verify-ui/references/verdict-and-degradation.md +31 -0
- package/CHANGELOG.md +77 -2
- package/README.en.md +98 -19
- package/README.md +98 -19
- package/bin/dsh-iris.js +392 -13
- package/docs/AGENT_QUICKSTART.md +35 -0
- package/docs/ARCHITECTURE.md +82 -52
- package/docs/ARTIFACT_LIBRARY.md +24 -0
- package/docs/ARTIFACT_MANIFEST.md +60 -0
- package/docs/CORE_RUNTIME_CONTRACT.md +83 -0
- package/docs/DSH_CORE_MIGRATION.md +106 -0
- package/docs/DSH_RC2_ADAPTATION.md +36 -0
- package/docs/HEADLESS_CLI.md +217 -0
- package/docs/HOST_ADAPTER_CONTRACT.md +105 -0
- package/docs/HOST_DOCTOR.md +49 -0
- package/docs/MODEL_PORT_CONTRACT.md +300 -0
- package/docs/PROVIDER_ADAPTER_CONTRACT.md +138 -0
- package/docs/PROVIDER_HEALTH.md +39 -0
- package/docs/PROVIDER_RUNTIME_HARNESS.md +33 -0
- package/docs/PROVIDER_SUBMISSION_CONTRACT.md +2 -2
- package/docs/ROADMAP.md +42 -76
- package/docs/SECURITY.md +10 -5
- package/docs/TASK_SEMANTICS.md +3 -2
- package/docs/assets/diagrams/iris-architecture.png +0 -0
- package/docs/assets/diagrams/iris-architecture.svg +5 -5
- package/docs/assets/logo/README.md +55 -0
- package/docs/assets/logo/iris-bubble-concept.svg +67 -0
- package/docs/assets/logo/iris-bubble.svg +14 -0
- package/docs/assets/logo/iris-interim-flower.png +0 -0
- package/docs/assets/logo/iris-logo-mono-inverse.svg +78 -0
- package/docs/assets/logo/iris-logo-mono.svg +78 -0
- package/docs/assets/logo/iris-logo.svg +78 -0
- package/docs/assets/logo/iris-mark-mono-inverse.svg +42 -0
- package/docs/assets/logo/iris-mark-mono.svg +42 -0
- package/docs/assets/logo/iris-mark.svg +42 -0
- package/docs/releases/0.1.4.md +59 -0
- package/docs/releases/0.2.0.md +48 -0
- package/lib/actions.js +673 -525
- package/lib/adapters.js +58 -21
- package/lib/api.js +182 -16
- package/lib/artifacts.js +239 -0
- package/lib/bundled-skills.js +5 -5
- package/lib/client.js +1116 -86
- package/lib/command-service.js +420 -0
- package/lib/config.js +245 -49
- package/lib/core-artifact-store.js +598 -0
- package/lib/core-artifacts.js +4 -0
- package/lib/core-attention.js +103 -0
- package/lib/core-contract.js +105 -0
- package/lib/core-doctor.js +41 -0
- package/lib/core-lease-recovery.js +170 -0
- package/lib/core-runtime.js +232 -0
- package/lib/core-tasks.js +560 -0
- package/lib/core-user-projection.js +192 -0
- package/lib/doctor.js +260 -30
- package/lib/dsh-core-adapter.js +599 -0
- package/lib/dsh-host-adapter.js +240 -0
- package/lib/generation-input.js +95 -0
- package/lib/guard.js +3 -3
- package/lib/host-contract.js +131 -0
- package/lib/host-runtime.js +156 -0
- package/lib/index.js +255 -166
- package/lib/locate.js +1 -1
- package/lib/media.js +16 -6
- package/lib/model-invoker.js +218 -0
- package/lib/model-port-contract.js +333 -0
- package/lib/models.js +97 -11
- package/lib/prompt-optimizer.js +10 -20
- package/lib/provider-adapter.js +397 -0
- package/lib/provider-adapters.js +342 -0
- package/lib/provider-catalog.js +171 -0
- package/lib/provider-contract.js +72 -3
- package/lib/provider-health.js +59 -0
- package/lib/provider-protocol.js +51 -2
- package/lib/provider-task-runner.js +328 -0
- package/lib/render.js +43 -12
- package/lib/tasks.js +45 -9
- package/lib/vision.js +51 -56
- package/package.json +20 -14
- package/user_guide.md +67 -12
- package/docs/assets/diagrams/iris-architecture.drawio +0 -1
- package/docs/assets/diagrams/iris-task-lifecycle.drawio +0 -1
- package/docs/assets/diagrams/iris-workflow-compose.drawio +0 -1
- package/docs/releases/0.1.1.md +0 -51
- package/docs/releases/0.1.2.md +0 -26
- package/docs/releases/0.1.3.md +0 -30
package/lib/index.js
CHANGED
|
@@ -14,10 +14,13 @@ import * as cap from './capability.js';
|
|
|
14
14
|
import * as tasks from './tasks.js';
|
|
15
15
|
import { registerMedia, mediaLinksOf, authorizeMedia, serveMedia } from './media.js';
|
|
16
16
|
import { serveApi, closeAllSse, purgeStaleUploads } from './api.js';
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
17
|
+
import { buildVisionBackendsFromHost, askWithBackends } from './vision.js';
|
|
18
|
+
import { hasHostPort, requireHostPort } from './host-contract.js';
|
|
19
|
+
import { createDshHostAdapter } from './dsh-host-adapter.js';
|
|
20
|
+
import { cropForDsh, diffForDsh, framesForDsh, inspectProviderTaskForDsh, projectCoreTaskForDsh, readCoreArtifactMediaForDsh, resumeProviderTaskWatchesForDsh, stopProviderTaskWatchesForDsh } from './dsh-core-adapter.js';
|
|
21
|
+
import { imageDimensions } from './pixels.js';
|
|
19
22
|
import { locateObject } from './locate.js';
|
|
20
|
-
import { serveRender
|
|
23
|
+
import { serveRender } from './render.js';
|
|
21
24
|
import { ensurePrivateDir, hardenPrivateTree } from './private-storage.js';
|
|
22
25
|
import { guarded } from './guard.js';
|
|
23
26
|
import { longOcr } from './ocr.js';
|
|
@@ -25,7 +28,15 @@ import { runAction, listActions, taskPollDeps } from './actions.js';
|
|
|
25
28
|
import { ffmpegAvailable, extractFrames, probeVideo, extractAudioTrack } from './media-probe.js';
|
|
26
29
|
import { buildContactSheet, summarizeMedia } from './summarize.js';
|
|
27
30
|
import { registerBundledSkills } from './bundled-skills.js';
|
|
28
|
-
|
|
31
|
+
import {
|
|
32
|
+
EXPECTED_IRIS_ROUTES, IRIS_PLUGIN_ID, beginHostRuntime, recordHostRoutes, recordHostSkills, recordHostTool
|
|
33
|
+
} from './host-runtime.js';
|
|
34
|
+
export { doctor, formatDoctorReport, hostDoctor } from './doctor.js';
|
|
35
|
+
|
|
36
|
+
const PACKAGE_VERSION = (() => {
|
|
37
|
+
try { return JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version || 'unknown'; }
|
|
38
|
+
catch (_) { return 'unknown'; }
|
|
39
|
+
})();
|
|
29
40
|
|
|
30
41
|
const MEDIA_TYPES = {
|
|
31
42
|
'.png': 'image/png',
|
|
@@ -35,8 +46,13 @@ const MEDIA_TYPES = {
|
|
|
35
46
|
'.gif': 'image/gif'
|
|
36
47
|
};
|
|
37
48
|
|
|
49
|
+
/** DSH 私有对象只在宿主边界转换;可选服务可能晚到,因此按操作刷新快照。 */
|
|
50
|
+
function dshHost(ctx) {
|
|
51
|
+
return createDshHostAdapter(ctx);
|
|
52
|
+
}
|
|
53
|
+
|
|
38
54
|
/** 工具内同步等待上限;超过即转后台盯守,用 iris_task_status 查询 */
|
|
39
|
-
const AWAIT_MS = { image: 180000, video: 480000 };
|
|
55
|
+
const AWAIT_MS = { image: 180000, video: 480000, transcribe: 120000 };
|
|
40
56
|
|
|
41
57
|
function outputDir() {
|
|
42
58
|
const dir = tasks.outputsDir();
|
|
@@ -47,10 +63,9 @@ function outputDir() {
|
|
|
47
63
|
|
|
48
64
|
let mediaRouteMounted = false;
|
|
49
65
|
function mountIrisRoutes(routeCtx) {
|
|
50
|
-
// Cordis
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
: undefined;
|
|
66
|
+
// Cordis 服务只在 DSH 宿主边界探测;Command 与通用工具辅助函数不接收原始 ctx
|
|
67
|
+
const host = dshHost(routeCtx);
|
|
68
|
+
const reg = hasHostPort(host, 'routes') ? host.ports.routes : undefined;
|
|
54
69
|
if (!reg || typeof reg.register !== 'function' || mediaRouteMounted) return;
|
|
55
70
|
try {
|
|
56
71
|
const disposeMedia = reg.register({
|
|
@@ -61,7 +76,7 @@ function mountIrisRoutes(routeCtx) {
|
|
|
61
76
|
const disposeApi = reg.register({
|
|
62
77
|
kind: 'prefix',
|
|
63
78
|
path: '/iris/api',
|
|
64
|
-
handler: guarded((req, res) => serveApi(req, res, routeCtx))
|
|
79
|
+
handler: guarded((req, res) => serveApi(req, res, dshHost(routeCtx)))
|
|
65
80
|
});
|
|
66
81
|
const disposeRender = reg.register({
|
|
67
82
|
kind: 'prefix',
|
|
@@ -69,6 +84,7 @@ function mountIrisRoutes(routeCtx) {
|
|
|
69
84
|
handler: guarded(serveRender)
|
|
70
85
|
});
|
|
71
86
|
mediaRouteMounted = true;
|
|
87
|
+
recordHostRoutes(EXPECTED_IRIS_ROUTES);
|
|
72
88
|
routeCtx.effect(() => () => {
|
|
73
89
|
if (typeof disposeMedia === 'function') disposeMedia();
|
|
74
90
|
if (typeof disposeApi === 'function') disposeApi();
|
|
@@ -138,6 +154,39 @@ async function awaitTerminal(taskId, { timeoutMs, signal }) {
|
|
|
138
154
|
}
|
|
139
155
|
}
|
|
140
156
|
|
|
157
|
+
function coreTaskWaitFinished(task) {
|
|
158
|
+
return Boolean(task && (task.phase === 'terminal'
|
|
159
|
+
|| (task.outcome === 'succeeded' && ['ready', 'failed'].includes(task.deliveryState))));
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function coreTaskTerminalProblem(task, label = '任务') {
|
|
163
|
+
if (!task) return label + '记录不存在';
|
|
164
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'ready') return '';
|
|
165
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
166
|
+
return label + '已成功,但 Core Artifact 交付失败:'
|
|
167
|
+
+ (task.lastError?.safeMessage || '可稍后重新交付;禁止重新生成');
|
|
168
|
+
}
|
|
169
|
+
if (task.acceptance === 'unknown' || task.outcome === 'unknown') {
|
|
170
|
+
return label + '的远端受理或结果状态未知:'
|
|
171
|
+
+ (task.lastError?.safeMessage || '禁止自动重提,请先人工确认');
|
|
172
|
+
}
|
|
173
|
+
if (task.outcome === 'canceled') return label + '已取消';
|
|
174
|
+
if (task.outcome === 'failed') return label + '失败:'
|
|
175
|
+
+ (task.lastError?.safeMessage || '供应商明确返回失败');
|
|
176
|
+
return label + '需要人工处理';
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function awaitCoreTerminal(taskId, { timeoutMs, signal }) {
|
|
180
|
+
const deadline = Date.now() + timeoutMs;
|
|
181
|
+
for (;;) {
|
|
182
|
+
if (signal?.aborted) throw new Error('已取消;Core 任务仍保留远端受理事实,可稍后查询');
|
|
183
|
+
const task = await inspectProviderTaskForDsh(taskId, { signal });
|
|
184
|
+
if (coreTaskWaitFinished(task)) return task;
|
|
185
|
+
if (Date.now() > deadline) return null;
|
|
186
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
141
190
|
/* ---------------- 图片 data URL(视觉工具共用) ---------------- */
|
|
142
191
|
|
|
143
192
|
function toDataUrl(absPath, mediaType) {
|
|
@@ -145,17 +194,6 @@ function toDataUrl(absPath, mediaType) {
|
|
|
145
194
|
return `data:${mediaType};base64,${buf.toString('base64')}`;
|
|
146
195
|
}
|
|
147
196
|
|
|
148
|
-
function fmtTask(t) {
|
|
149
|
-
const lines = [`[${t.cap}] ${t.id} · ${t.status}${t.progress ? ' · ' + t.progress : ''} · ${t.model}`];
|
|
150
|
-
lines.push('prompt: ' + String(t.prompt || '').slice(0, 120));
|
|
151
|
-
if (t.error) lines.push('error: ' + t.error);
|
|
152
|
-
if ((t.files || []).length) lines.push('files:\n' + t.files.map((f) => path.join(tasks.outputsDir(), f)).join('\n'));
|
|
153
|
-
const links = mediaLinksOf(t);
|
|
154
|
-
if (links.length) lines.push('links:\n' + links.join('\n'));
|
|
155
|
-
if (t.remoteTaskId) lines.push('remoteTask: ' + t.remoteTaskId);
|
|
156
|
-
return lines.join('\n');
|
|
157
|
-
}
|
|
158
|
-
|
|
159
197
|
export const name = 'mokuyoaxis-dsh-iris';
|
|
160
198
|
export const inject = ['tools'];
|
|
161
199
|
|
|
@@ -174,13 +212,17 @@ function guard(label, fn) {
|
|
|
174
212
|
}
|
|
175
213
|
|
|
176
214
|
/** 单工具注册隔离:注册失败的收益是诊断日志,而不是宿主陪葬 */
|
|
177
|
-
function safeRegister(
|
|
215
|
+
function safeRegister(host, definition) {
|
|
216
|
+
let registered = false;
|
|
178
217
|
guard(`工具注册 ${definition && definition.name}`, () => {
|
|
179
|
-
|
|
218
|
+
requireHostPort(host, 'tools', '注册 Agent 工具').register(definition);
|
|
219
|
+
registered = true;
|
|
180
220
|
});
|
|
221
|
+
if (registered) recordHostTool(definition.name);
|
|
181
222
|
}
|
|
182
223
|
|
|
183
224
|
export async function apply(ctx) {
|
|
225
|
+
beginHostRuntime({ pluginId: IRIS_PLUGIN_ID, version: PACKAGE_VERSION });
|
|
184
226
|
guard('Iris 私有存储权限收紧', () => hardenPrivateTree(store.irisHome()));
|
|
185
227
|
|
|
186
228
|
guard('上传临时文件清理', () => purgeStaleUploads());
|
|
@@ -204,9 +246,19 @@ export async function apply(ctx) {
|
|
|
204
246
|
if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个后台任务`);
|
|
205
247
|
});
|
|
206
248
|
|
|
207
|
-
|
|
249
|
+
try {
|
|
250
|
+
const resumed = await resumeProviderTaskWatchesForDsh();
|
|
251
|
+
if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个 Core 图片任务`);
|
|
252
|
+
} catch (error) {
|
|
253
|
+
console.error('[iris] 恢复 Core 图片后台任务失败(已隔离,不影响宿主):', error?.message || error);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Fiber 清理:插件停用/更新时停掉 legacy 与 Core 的全部 Host 盯守句柄
|
|
208
257
|
guard('盯守句柄注册', () => {
|
|
209
|
-
ctx.effect(() => () =>
|
|
258
|
+
ctx.effect(() => () => {
|
|
259
|
+
tasks.stopWatchAll();
|
|
260
|
+
stopProviderTaskWatchesForDsh();
|
|
261
|
+
});
|
|
210
262
|
});
|
|
211
263
|
|
|
212
264
|
// 媒体路由(视频/音频在对话流里的可播通路;图片仍走原生附件)
|
|
@@ -219,19 +271,20 @@ export async function apply(ctx) {
|
|
|
219
271
|
// 全局层;项目目录中的同名 Skill 仍由 DSH 原生优先级覆盖。
|
|
220
272
|
ctx.inject(['skills'], (skillCtx) => {
|
|
221
273
|
guard('随包 Skill 注册', () => {
|
|
222
|
-
const names = registerBundledSkills(skillCtx);
|
|
274
|
+
const names = registerBundledSkills(dshHost(skillCtx));
|
|
275
|
+
recordHostSkills(names);
|
|
223
276
|
console.log('[iris] 随包 Skill 已注册:' + names.join(' · '));
|
|
224
277
|
});
|
|
225
278
|
});
|
|
226
279
|
|
|
227
|
-
const
|
|
228
|
-
if (!
|
|
280
|
+
const host = dshHost(ctx);
|
|
281
|
+
if (!hasHostPort(host, 'tools')) {
|
|
229
282
|
console.error('[iris] tools 服务不可用,跳过全部工具注册(宿主不受影响)');
|
|
230
283
|
return;
|
|
231
284
|
}
|
|
232
285
|
|
|
233
286
|
/* ---------- 🖼️ 画图 ---------- */
|
|
234
|
-
safeRegister(
|
|
287
|
+
safeRegister(host, {
|
|
235
288
|
name: 'iris_draw_image',
|
|
236
289
|
description:
|
|
237
290
|
'Generate an image from a detailed prompt using the user-configured image model (DashScope wan* or OpenAI-images-compatible). ' +
|
|
@@ -253,8 +306,35 @@ export async function apply(ctx) {
|
|
|
253
306
|
isConcurrencySafe: () => false,
|
|
254
307
|
async execute(args, exec) {
|
|
255
308
|
const prompt = String(args.prompt || '').trim();
|
|
256
|
-
|
|
257
|
-
|
|
309
|
+
const submitted = await runAction(dshHost(ctx), 'image', args, { signal: exec.signal });
|
|
310
|
+
if (submitted.storage === 'core') {
|
|
311
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
312
|
+
if (!coreTaskWaitFinished(final)) {
|
|
313
|
+
final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
|
|
314
|
+
}
|
|
315
|
+
if (!final) {
|
|
316
|
+
return { blocks: [{ type: 'text', text: `[iris] Core 图片任务仍在后台观察(task: ${submitted.taskId})。稍后可用 iris_task_status 查询进度与 Artifact。` }] };
|
|
317
|
+
}
|
|
318
|
+
const terminalProblem = coreTaskTerminalProblem(final, '图像生成');
|
|
319
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
320
|
+
const projected = await projectCoreTaskForDsh(dshHost(ctx), submitted.taskId, { signal: exec.signal });
|
|
321
|
+
const refs = projected.artifacts.map((item) => item.attachment);
|
|
322
|
+
const blocks = projected.artifacts.map((item) => ({ type: 'image', attachment: item.attachment }));
|
|
323
|
+
let note = '';
|
|
324
|
+
try {
|
|
325
|
+
const first = await readCoreArtifactMediaForDsh(projected.artifacts[0].artifact.id, { signal: exec.signal });
|
|
326
|
+
const described = await describeImageData(dshHost(ctx), refs[0],
|
|
327
|
+
`data:${first.artifact.mediaType};base64,${first.bytes.toString('base64')}`, prompt);
|
|
328
|
+
if (described) note = `画面内容:${described}`;
|
|
329
|
+
} catch (_) { /* 自述是增强项,失败不影响 Core 交付 */ }
|
|
330
|
+
blocks.unshift({ type: 'text', text:
|
|
331
|
+
`[iris] 图像已生成(${submitted.model},Core task: ${submitted.taskId})。` +
|
|
332
|
+
(note ? `\n${note}` : '') +
|
|
333
|
+
`\nartifact: ${projected.artifacts.map((item) => item.artifact.id).join(', ')}` +
|
|
334
|
+
`\nattachment: ${refs.map((ref) => ref.attachmentId).join(', ')}`
|
|
335
|
+
});
|
|
336
|
+
return { blocks };
|
|
337
|
+
}
|
|
258
338
|
let final = tasks.get(submitted.taskId);
|
|
259
339
|
if (final && !taskWaitFinished(final)) {
|
|
260
340
|
final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
|
|
@@ -270,7 +350,7 @@ export async function apply(ctx) {
|
|
|
270
350
|
const refs = [];
|
|
271
351
|
for (const f of files) {
|
|
272
352
|
const mediaType = MEDIA_TYPES[path.extname(f).toLowerCase()] || 'image/png';
|
|
273
|
-
const ref = await attachmentService(
|
|
353
|
+
const ref = await attachmentService(host).saveImage({ data: new Uint8Array(fs.readFileSync(f)), mediaType, name: path.basename(f) });
|
|
274
354
|
refs.push(ref);
|
|
275
355
|
blocks.push({ type: 'image', attachment: ref });
|
|
276
356
|
}
|
|
@@ -284,7 +364,7 @@ export async function apply(ctx) {
|
|
|
284
364
|
|
|
285
365
|
let note = '';
|
|
286
366
|
try {
|
|
287
|
-
const described = await describeFirstImage(ctx, refs[0], files[0], prompt);
|
|
367
|
+
const described = await describeFirstImage(dshHost(ctx), refs[0], files[0], prompt);
|
|
288
368
|
if (described) note = `画面内容:${described}`;
|
|
289
369
|
} catch (_) { /* 增强失败不影响生成结果 */ }
|
|
290
370
|
blocks.unshift({
|
|
@@ -298,7 +378,7 @@ export async function apply(ctx) {
|
|
|
298
378
|
});
|
|
299
379
|
|
|
300
380
|
/* ---------- 🎬 视频生成 ---------- */
|
|
301
|
-
safeRegister(
|
|
381
|
+
safeRegister(host, {
|
|
302
382
|
name: 'iris_generate_video',
|
|
303
383
|
description:
|
|
304
384
|
'Generate a video with the configured DashScope video model. Three modes: ' +
|
|
@@ -323,7 +403,26 @@ export async function apply(ctx) {
|
|
|
323
403
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
324
404
|
isConcurrencySafe: () => false,
|
|
325
405
|
async execute(args, exec) {
|
|
326
|
-
const submitted = await runAction(ctx, 'video', args, { signal: exec.signal });
|
|
406
|
+
const submitted = await runAction(dshHost(ctx), 'video', args, { signal: exec.signal });
|
|
407
|
+
if (submitted.storage === 'core') {
|
|
408
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
409
|
+
if (!coreTaskWaitFinished(final)) {
|
|
410
|
+
final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
411
|
+
}
|
|
412
|
+
if (!final) {
|
|
413
|
+
return `[iris] Core 视频任务已提交并转入后台观察(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
414
|
+
'渲染通常需要数分钟;完成后用 iris_task_status 查询进度与 Artifact,或在工作台「重新观察」。';
|
|
415
|
+
}
|
|
416
|
+
const terminalProblem = coreTaskTerminalProblem(final, '视频生成');
|
|
417
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
418
|
+
const secs = Math.round((Date.now() - new Date(final.createdAt).getTime()) / 1000);
|
|
419
|
+
const links = final.artifactIds.map((artifactId) =>
|
|
420
|
+
'/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
|
|
421
|
+
return `[iris] 视频已生成(${submitted.model || final.modelRef},${submitted.mode || 't2v'},约 ${secs}s,Core task: ${final.id}):\n` +
|
|
422
|
+
`artifact: ${final.artifactIds.join(', ')}\n` +
|
|
423
|
+
links.join('\n') +
|
|
424
|
+
'\n(链接受 DSH Host 与浏览器同源守卫保护;Artifact ID 持有者可访问,请勿公开)';
|
|
425
|
+
}
|
|
327
426
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
328
427
|
if (!final) {
|
|
329
428
|
return `[iris] 视频任务已提交并转入后台盯守(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
@@ -342,7 +441,7 @@ export async function apply(ctx) {
|
|
|
342
441
|
});
|
|
343
442
|
|
|
344
443
|
/* ---------- 🔊 语音合成 ---------- */
|
|
345
|
-
safeRegister(
|
|
444
|
+
safeRegister(host, {
|
|
346
445
|
name: 'iris_speak_text',
|
|
347
446
|
description: 'Synthesize speech audio from text using the configured TTS model (qwen-tts / compatible). Returns the saved audio file path.',
|
|
348
447
|
parameters: {
|
|
@@ -357,7 +456,17 @@ export async function apply(ctx) {
|
|
|
357
456
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
358
457
|
isConcurrencySafe: () => true,
|
|
359
458
|
async execute(args, exec) {
|
|
360
|
-
const submitted = await runAction(ctx, 'tts', args, { signal: exec.signal });
|
|
459
|
+
const submitted = await runAction(dshHost(ctx), 'tts', args, { signal: exec.signal });
|
|
460
|
+
if (submitted.storage === 'core') {
|
|
461
|
+
const final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
462
|
+
const terminalProblem = coreTaskTerminalProblem(final, '语音合成');
|
|
463
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
464
|
+
const links = final.artifactIds.map((artifactId) =>
|
|
465
|
+
'/iris/api/core/artifact/' + encodeURIComponent(artifactId) + '/media');
|
|
466
|
+
return `[iris] 语音已合成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
|
|
467
|
+
`artifact: ${final.artifactIds.join(', ')}\n` +
|
|
468
|
+
links.map((link) => `[♪ 音频播放](${link})`).join('\n');
|
|
469
|
+
}
|
|
361
470
|
const final = tasks.get(submitted.taskId);
|
|
362
471
|
const terminalProblem = taskTerminalProblem(final, '语音合成');
|
|
363
472
|
if (terminalProblem) throw new Error(terminalProblem);
|
|
@@ -370,7 +479,7 @@ export async function apply(ctx) {
|
|
|
370
479
|
});
|
|
371
480
|
|
|
372
481
|
/* ---------- 🎙️ 音频转写(阶段 7.2) ---------- */
|
|
373
|
-
safeRegister(
|
|
482
|
+
safeRegister(host, {
|
|
374
483
|
name: 'iris_transcribe_audio',
|
|
375
484
|
description: 'Transcribe an audio file (wav/mp3) to text using the configured DashScope provider (qwen-audio-3.0-asr-flash-filetrans). The audio is uploaded to Bailian temp storage, then an async task is submitted. Returns the full recognized text.',
|
|
376
485
|
parameters: {
|
|
@@ -384,7 +493,26 @@ export async function apply(ctx) {
|
|
|
384
493
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
385
494
|
isConcurrencySafe: () => false,
|
|
386
495
|
async execute(args, exec) {
|
|
387
|
-
const submitted = await runAction(ctx, 'transcribe', args, { signal: exec.signal });
|
|
496
|
+
const submitted = await runAction(dshHost(ctx), 'transcribe', args, { signal: exec.signal });
|
|
497
|
+
if (submitted.storage === 'core') {
|
|
498
|
+
let final = await inspectProviderTaskForDsh(submitted.taskId, { signal: exec.signal });
|
|
499
|
+
if (!coreTaskWaitFinished(final)) {
|
|
500
|
+
final = await awaitCoreTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.transcribe, signal: exec.signal });
|
|
501
|
+
}
|
|
502
|
+
if (!final) {
|
|
503
|
+
return `[iris] Core 转写任务已提交并转入后台观察(task: ${submitted.taskId})。完成后用 iris_task_status 查询进度与 Artifact。`;
|
|
504
|
+
}
|
|
505
|
+
const terminalProblem = coreTaskTerminalProblem(final, '音频转写');
|
|
506
|
+
if (terminalProblem) throw new Error(terminalProblem);
|
|
507
|
+
const texts = [];
|
|
508
|
+
for (const artifactId of final.artifactIds) {
|
|
509
|
+
const media = await readCoreArtifactMediaForDsh(artifactId, { signal: exec.signal });
|
|
510
|
+
texts.push(Buffer.from(media.bytes).toString('utf8'));
|
|
511
|
+
}
|
|
512
|
+
return `[iris] 音频转写完成(${submitted.model || final.modelRef},Core task: ${final.id}):\n` +
|
|
513
|
+
texts.join('\n') +
|
|
514
|
+
`\nartifact: ${final.artifactIds.join(', ')}`;
|
|
515
|
+
}
|
|
388
516
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
389
517
|
if (!final) return `[iris] 音频转写任务已提交并转入后台(task: ${submitted.taskId})。完成后用 iris_task_status 查询。`;
|
|
390
518
|
const terminalProblem = taskTerminalProblem(final, '音频转写');
|
|
@@ -394,7 +522,7 @@ export async function apply(ctx) {
|
|
|
394
522
|
});
|
|
395
523
|
|
|
396
524
|
/* ---------- 🎞️ 视频抽帧(阶段 7.1,ffmpeg 可选) ---------- */
|
|
397
|
-
safeRegister(
|
|
525
|
+
safeRegister(host, {
|
|
398
526
|
name: 'iris_video_frames',
|
|
399
527
|
description:
|
|
400
528
|
'Extract N frames (uniformly sampled across the video) from a local video file using the system ffmpeg. ' +
|
|
@@ -415,26 +543,22 @@ export async function apply(ctx) {
|
|
|
415
543
|
isConcurrencySafe: () => false,
|
|
416
544
|
async execute(args, exec) {
|
|
417
545
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
418
|
-
const
|
|
419
|
-
|
|
420
|
-
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法抽帧(可选安装 ffmpeg/ffprobe 后启用;不影响其他 iris 工具)');
|
|
421
|
-
const meta = probeVideo(p);
|
|
422
|
-
const frames = await extractFrames({
|
|
423
|
-
inputPath: p,
|
|
546
|
+
const { media: meta, frames } = await framesForDsh({
|
|
547
|
+
videoPath: args.video_path,
|
|
424
548
|
maxFrames: args.max_frames,
|
|
425
549
|
targetWidth: args.target_width,
|
|
426
550
|
format: args.format,
|
|
427
551
|
quality: args.quality,
|
|
428
552
|
signal: exec.signal
|
|
429
553
|
});
|
|
430
|
-
const mediaType =
|
|
431
|
-
const ext =
|
|
554
|
+
const mediaType = frames[0].mediaType;
|
|
555
|
+
const ext = mediaType === 'image/png' ? 'png' : 'jpg';
|
|
432
556
|
const blocks = [];
|
|
433
557
|
const refs = [];
|
|
434
558
|
for (let i = 0; i < frames.length; i++) {
|
|
435
559
|
const f = frames[i];
|
|
436
|
-
const ref = await attachmentService(
|
|
437
|
-
data: new Uint8Array(f.
|
|
560
|
+
const ref = await attachmentService(host).saveImage({
|
|
561
|
+
data: new Uint8Array(f.bytes),
|
|
438
562
|
mediaType,
|
|
439
563
|
name: `iris-frame-${Date.now()}-${i + 1}.${ext}`
|
|
440
564
|
});
|
|
@@ -450,12 +574,12 @@ export async function apply(ctx) {
|
|
|
450
574
|
`时间戳:${ats}\n` +
|
|
451
575
|
`attachments: ${refs.map((r) => r.attachmentId).join(', ')}`
|
|
452
576
|
});
|
|
453
|
-
return { blocks };
|
|
577
|
+
return { blocks, artifactIds: frames.map((frame) => frame.artifact.id) };
|
|
454
578
|
}
|
|
455
579
|
});
|
|
456
580
|
|
|
457
581
|
/* ---------- 📝 多模态视频摘要(阶段 7.3,ffmpeg 可选) ---------- */
|
|
458
|
-
safeRegister(
|
|
582
|
+
safeRegister(host, {
|
|
459
583
|
name: 'iris_media_summarize',
|
|
460
584
|
description:
|
|
461
585
|
'Summarize a local video by compressing it into a contact sheet of sampled frames (with timestamps) plus an optional ' +
|
|
@@ -499,7 +623,7 @@ export async function apply(ctx) {
|
|
|
499
623
|
try {
|
|
500
624
|
const audio = await extractAudioTrack({ inputPath: p, signal: exec.signal });
|
|
501
625
|
try {
|
|
502
|
-
const submitted = await runAction(ctx, 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
|
|
626
|
+
const submitted = await runAction(dshHost(ctx), 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
|
|
503
627
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
504
628
|
if (final && final.status === 'succeeded') transcript = final.transcribeText || '';
|
|
505
629
|
else if (!final) transcriptNote = `\n(音轨转写仍在后台:task ${submitted.taskId},本次摘要未含语音内容)`;
|
|
@@ -516,13 +640,13 @@ export async function apply(ctx) {
|
|
|
516
640
|
|
|
517
641
|
// contact sheet → attachment → 视觉摘要
|
|
518
642
|
const sheet = await buildContactSheet({ frames, cols: 3 });
|
|
519
|
-
const ref = await attachmentService(
|
|
643
|
+
const ref = await attachmentService(host).saveImage({
|
|
520
644
|
data: new Uint8Array(sheet.buffer),
|
|
521
645
|
mediaType: 'image/png',
|
|
522
646
|
name: `iris-sheet-${Date.now()}.png`
|
|
523
647
|
});
|
|
524
648
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
525
|
-
const backends =
|
|
649
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
526
650
|
const { answer, via, model } = await summarizeMedia({
|
|
527
651
|
backends, question: args.question, transcript,
|
|
528
652
|
imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
|
|
@@ -546,7 +670,7 @@ export async function apply(ctx) {
|
|
|
546
670
|
|
|
547
671
|
/* ---------- 👁 视觉路由(M3):显式工具,自持栈为主 ---------- */
|
|
548
672
|
|
|
549
|
-
safeRegister(
|
|
673
|
+
safeRegister(host, {
|
|
550
674
|
name: 'iris_look_at_image',
|
|
551
675
|
description:
|
|
552
676
|
'Look at a local image file and answer one question about it with the configured vision model (qwen-vl via the self-hosted provider stack; falls back to the global DSH vision model). The image is saved as a durable attachment so follow-ups can reference it.',
|
|
@@ -571,12 +695,12 @@ export async function apply(ctx) {
|
|
|
571
695
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
572
696
|
|
|
573
697
|
const dataUrl = toDataUrl(p, mt);
|
|
574
|
-
const ref = await attachmentService(
|
|
575
|
-
return runVisionTool(ctx, exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
|
|
698
|
+
const ref = await attachmentService(host).saveImage({ data: new Uint8Array(fs.readFileSync(p)), mediaType: mt, name: path.basename(p) });
|
|
699
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
|
|
576
700
|
}
|
|
577
701
|
});
|
|
578
702
|
|
|
579
|
-
safeRegister(
|
|
703
|
+
safeRegister(host, {
|
|
580
704
|
name: 'iris_relook_attachment',
|
|
581
705
|
description:
|
|
582
706
|
'Ask a NEW question about an image that was already seen in this session: pass its attachment_id (from user uploads, tool results, or any iris-generated image). Pixels are re-read via the vision model.',
|
|
@@ -599,22 +723,22 @@ export async function apply(ctx) {
|
|
|
599
723
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
600
724
|
|
|
601
725
|
// 定位:本会话出现过的图片优先,其次本插件产物记录
|
|
602
|
-
const hit = (await sessionAttachmentRef(
|
|
726
|
+
const hit = (await sessionAttachmentRef(host, exec, id)) || findOwnAttachment(id);
|
|
603
727
|
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
604
728
|
|
|
605
729
|
let dataUrl;
|
|
606
730
|
if (hit.absPath) {
|
|
607
731
|
dataUrl = toDataUrl(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png');
|
|
608
732
|
} else {
|
|
609
|
-
const stored = await attachmentService(
|
|
733
|
+
const stored = await attachmentService(host).readImage(hit.ref, exec.signal);
|
|
610
734
|
dataUrl = `data:${stored.mediaType || hit.ref.mediaType};base64,${Buffer.from(stored.data).toString('base64')}`;
|
|
611
735
|
}
|
|
612
|
-
return runVisionTool(ctx, exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
736
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
613
737
|
}
|
|
614
738
|
});
|
|
615
739
|
|
|
616
740
|
/* ---------- 📋 任务查询 ---------- */
|
|
617
|
-
safeRegister(
|
|
741
|
+
safeRegister(host, {
|
|
618
742
|
name: 'iris_task_status',
|
|
619
743
|
description:
|
|
620
744
|
'Query iris generation tasks (image/video/tts). Pass a task_id for one task, or omit it to list the latest tasks with status, progress, errors and output file paths.',
|
|
@@ -628,19 +752,13 @@ export async function apply(ctx) {
|
|
|
628
752
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
629
753
|
isConcurrencySafe: () => true,
|
|
630
754
|
async execute(args) {
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
if (!t) throw new Error('iris: 没有这个任务:' + args.task_id);
|
|
634
|
-
return fmtTask(t);
|
|
635
|
-
}
|
|
636
|
-
const recent = tasks.list().slice(0, 5);
|
|
637
|
-
if (!recent.length) return '[iris] 还没有生成任务记录。';
|
|
638
|
-
return recent.map(fmtTask).join('\n---\n');
|
|
755
|
+
const result = await runAction(dshHost(ctx), 'status', args || {});
|
|
756
|
+
return result.text;
|
|
639
757
|
}
|
|
640
758
|
});
|
|
641
759
|
|
|
642
760
|
/* ---------- ✂️ 确定性像素工具(阶段 2) ---------- */
|
|
643
|
-
safeRegister(
|
|
761
|
+
safeRegister(host, {
|
|
644
762
|
name: 'iris_crop',
|
|
645
763
|
description:
|
|
646
764
|
'Crop a rectangular region from an image (absolute local path or a session attachment_id) and save the result as a durable DSH attachment. ' +
|
|
@@ -661,32 +779,36 @@ export async function apply(ctx) {
|
|
|
661
779
|
isConcurrencySafe: () => true,
|
|
662
780
|
async execute(args, exec) {
|
|
663
781
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
664
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
782
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
665
783
|
image_path: args.image_path,
|
|
666
784
|
attachment_id: args.attachment_id
|
|
667
785
|
});
|
|
668
|
-
const
|
|
669
|
-
|
|
786
|
+
const result = await cropForDsh({
|
|
787
|
+
...(args.image_path
|
|
788
|
+
? { imagePath: String(args.image_path).trim() }
|
|
789
|
+
: { bytes: input.buffer, mediaType: input.mediaType }),
|
|
670
790
|
left: args.left,
|
|
671
791
|
top: args.top,
|
|
672
792
|
width: args.width,
|
|
673
|
-
height: args.height
|
|
793
|
+
height: args.height,
|
|
794
|
+
signal: exec.signal
|
|
674
795
|
});
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
796
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
797
|
+
const ref = await attachmentService(host).saveImage({
|
|
798
|
+
data: new Uint8Array(result.bytes),
|
|
799
|
+
mediaType: result.mediaType,
|
|
678
800
|
name: `iris-crop-${Date.now()}.png`
|
|
679
801
|
});
|
|
680
802
|
return {
|
|
681
803
|
blocks: [
|
|
682
|
-
{ type: 'text', text: `[iris] 裁剪完成:${width}x${height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
|
|
804
|
+
{ type: 'text', text: `[iris] 裁剪完成:${result.width}x${result.height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
|
|
683
805
|
{ type: 'image', attachment: ref }
|
|
684
806
|
]
|
|
685
807
|
};
|
|
686
808
|
}
|
|
687
809
|
});
|
|
688
810
|
|
|
689
|
-
safeRegister(
|
|
811
|
+
safeRegister(host, {
|
|
690
812
|
name: 'iris_pixel_diff',
|
|
691
813
|
description:
|
|
692
814
|
'Compute a pixel-level difference between two images (absolute local paths or session attachment_ids). ' +
|
|
@@ -710,22 +832,23 @@ export async function apply(ctx) {
|
|
|
710
832
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
711
833
|
if (!(args.image_a_path || args.attachment_a_id)) throw new Error('iris: 需要 image_a_path 或 attachment_a_id');
|
|
712
834
|
if (!(args.image_b_path || args.attachment_b_id)) throw new Error('iris: 需要 image_b_path 或 attachment_b_id');
|
|
713
|
-
const a = await resolveImageInput(ctx, exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
714
|
-
const b = await resolveImageInput(ctx, exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
715
|
-
const
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
grid: args.grid
|
|
719
|
-
topRegions: args.top_regions || 3
|
|
835
|
+
const a = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
836
|
+
const b = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
837
|
+
const result = await diffForDsh({
|
|
838
|
+
imageA: { bytes: a.buffer, mediaType: a.mediaType },
|
|
839
|
+
imageB: { bytes: b.buffer, mediaType: b.mediaType },
|
|
840
|
+
grid: args.grid, topRegions: args.top_regions, signal: exec.signal
|
|
720
841
|
});
|
|
721
|
-
const
|
|
722
|
-
|
|
723
|
-
|
|
842
|
+
const { ratio, diffPixels, totalPixels, width, height, worstRegions } = result.metrics;
|
|
843
|
+
const ref = await attachmentService(host).saveImage({
|
|
844
|
+
data: new Uint8Array(result.bytes),
|
|
845
|
+
mediaType: result.mediaType,
|
|
724
846
|
name: `iris-diff-${Date.now()}.png`
|
|
725
847
|
});
|
|
726
848
|
const pct = (ratio * 100).toFixed(2);
|
|
727
849
|
const regions = worstRegions.map((r) => `(col ${r.col},row ${r.row}) ${(r.score * 100).toFixed(1)}%`).join(' ');
|
|
728
850
|
return {
|
|
851
|
+
artifactIds: [result.artifact.id],
|
|
729
852
|
blocks: [
|
|
730
853
|
{
|
|
731
854
|
type: 'text',
|
|
@@ -740,7 +863,7 @@ export async function apply(ctx) {
|
|
|
740
863
|
});
|
|
741
864
|
|
|
742
865
|
/* ---------- 📍 模型驱动定位(阶段 3A) ---------- */
|
|
743
|
-
safeRegister(
|
|
866
|
+
safeRegister(host, {
|
|
744
867
|
name: 'iris_locate',
|
|
745
868
|
description:
|
|
746
869
|
'Locate an object/region described by `target` in an image (absolute path or session attachment_id), ' +
|
|
@@ -760,14 +883,14 @@ export async function apply(ctx) {
|
|
|
760
883
|
isConcurrencySafe: () => true,
|
|
761
884
|
async execute(args, exec) {
|
|
762
885
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
763
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
886
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
764
887
|
image_path: args.image_path,
|
|
765
888
|
attachment_id: args.attachment_id
|
|
766
889
|
});
|
|
767
890
|
const { width, height } = await imageDimensions(input.buffer);
|
|
768
891
|
const dataUrl = `data:${input.mediaType};base64,${input.buffer.toString('base64')}`;
|
|
769
892
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
770
|
-
const backends =
|
|
893
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
771
894
|
const r = await locateObject(backends, {
|
|
772
895
|
target: args.target,
|
|
773
896
|
imageDataUrl: dataUrl,
|
|
@@ -782,7 +905,7 @@ export async function apply(ctx) {
|
|
|
782
905
|
});
|
|
783
906
|
|
|
784
907
|
/* ---------- 🖼️ HTML 截图(阶段 3C,基于 dsh-builtin-browser) ---------- */
|
|
785
|
-
safeRegister(
|
|
908
|
+
safeRegister(host, {
|
|
786
909
|
name: 'iris_html_screenshot',
|
|
787
910
|
description:
|
|
788
911
|
'Render an HTML string to a page screenshot using the shared browser. ' +
|
|
@@ -804,19 +927,17 @@ export async function apply(ctx) {
|
|
|
804
927
|
isConcurrencySafe: () => false,
|
|
805
928
|
async execute(args, exec) {
|
|
806
929
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
807
|
-
const browser = ctx
|
|
808
|
-
|
|
809
|
-
throw new Error('iris: 浏览器服务不可用(ctx.get("browser") 为空)——需要启用 dsh-builtin-browser 插件');
|
|
810
|
-
}
|
|
811
|
-
const renderer = new BrowserHtmlRenderer({ browser });
|
|
812
|
-
const { png } = await renderer.render({
|
|
930
|
+
const browser = requireHostPort(dshHost(ctx), 'browser', 'HTML 截图');
|
|
931
|
+
const rendered = await browser.renderHtml({
|
|
813
932
|
html: args.html,
|
|
814
933
|
width: args.width,
|
|
815
934
|
height: args.height,
|
|
816
|
-
fullPage: args.fullPage !== false
|
|
935
|
+
fullPage: args.fullPage !== false,
|
|
936
|
+
signal: exec.signal
|
|
817
937
|
});
|
|
938
|
+
const png = Buffer.from(rendered.bytes);
|
|
818
939
|
const { width: outW, height: outH } = await imageDimensions(png);
|
|
819
|
-
const ref = await attachmentService(
|
|
940
|
+
const ref = await attachmentService(host).saveImage({
|
|
820
941
|
data: new Uint8Array(png),
|
|
821
942
|
mediaType: 'image/png',
|
|
822
943
|
name: `iris-html-${Date.now()}.png`
|
|
@@ -836,7 +957,7 @@ export async function apply(ctx) {
|
|
|
836
957
|
});
|
|
837
958
|
|
|
838
959
|
/* ---------- 📄 长截图 OCR(阶段 3B,视觉模型分块) ---------- */
|
|
839
|
-
safeRegister(
|
|
960
|
+
safeRegister(host, {
|
|
840
961
|
name: 'iris_long_ocr',
|
|
841
962
|
description:
|
|
842
963
|
'OCR a (possibly long) image: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
@@ -856,12 +977,12 @@ export async function apply(ctx) {
|
|
|
856
977
|
isConcurrencySafe: () => true,
|
|
857
978
|
async execute(args, exec) {
|
|
858
979
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
859
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
980
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
860
981
|
image_path: args.image_path,
|
|
861
982
|
attachment_id: args.attachment_id
|
|
862
983
|
});
|
|
863
984
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
864
|
-
const backends =
|
|
985
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders });
|
|
865
986
|
const result = await longOcr({
|
|
866
987
|
input: input.buffer,
|
|
867
988
|
backends,
|
|
@@ -881,13 +1002,13 @@ export async function apply(ctx) {
|
|
|
881
1002
|
|
|
882
1003
|
/* ---------------- 视觉路由(M3 补全):look/relook 共用执行器 + 降级链 ---------------- */
|
|
883
1004
|
|
|
884
|
-
/**
|
|
885
|
-
function attachmentService(
|
|
886
|
-
const
|
|
887
|
-
if (
|
|
888
|
-
throw new Error('iris:
|
|
1005
|
+
/** 通用工具只消费 Host attachments 端口。 */
|
|
1006
|
+
function attachmentService(host) {
|
|
1007
|
+
const attachments = requireHostPort(host, 'attachments', '处理 DSH 附件');
|
|
1008
|
+
if (typeof attachments.saveImage !== 'function' || typeof attachments.readImage !== 'function') {
|
|
1009
|
+
throw new Error('iris: Host attachments 端口缺少 saveImage/readImage');
|
|
889
1010
|
}
|
|
890
|
-
return
|
|
1011
|
+
return attachments;
|
|
891
1012
|
}
|
|
892
1013
|
|
|
893
1014
|
/**
|
|
@@ -895,7 +1016,7 @@ function attachmentService(ctx) {
|
|
|
895
1016
|
* crop / pixel_diff 共用(阶段 2)。
|
|
896
1017
|
* @returns {Promise<{buffer:Buffer, mediaType:string, name:string}>}
|
|
897
1018
|
*/
|
|
898
|
-
async function resolveImageInput(
|
|
1019
|
+
async function resolveImageInput(host, exec, { image_path, attachment_id }) {
|
|
899
1020
|
if (image_path) {
|
|
900
1021
|
const p = String(image_path).trim();
|
|
901
1022
|
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
@@ -906,7 +1027,7 @@ async function resolveImageInput(ctx, exec, { image_path, attachment_id }) {
|
|
|
906
1027
|
}
|
|
907
1028
|
if (attachment_id) {
|
|
908
1029
|
const id = String(attachment_id).trim();
|
|
909
|
-
const hit = (await sessionAttachmentRef(
|
|
1030
|
+
const hit = (await sessionAttachmentRef(host, exec, id)) || findOwnAttachment(id);
|
|
910
1031
|
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
911
1032
|
if (hit.absPath) {
|
|
912
1033
|
return {
|
|
@@ -915,7 +1036,7 @@ async function resolveImageInput(ctx, exec, { image_path, attachment_id }) {
|
|
|
915
1036
|
name: path.basename(hit.absPath)
|
|
916
1037
|
};
|
|
917
1038
|
}
|
|
918
|
-
const stored = await attachmentService(
|
|
1039
|
+
const stored = await attachmentService(host).readImage(hit.ref, exec && exec.signal);
|
|
919
1040
|
return {
|
|
920
1041
|
buffer: Buffer.from(stored.data),
|
|
921
1042
|
mediaType: stored.mediaType || hit.ref.mediaType || 'image/png',
|
|
@@ -947,51 +1068,17 @@ export function findOwnAttachment(attachmentId) {
|
|
|
947
1068
|
* 不依赖精确事件类型:递归扫任何载荷中的 attachment 块 / attachmentId 字段。
|
|
948
1069
|
* @returns {Promise<{ref:object}|null>} 找到则带可直接 readImage 的 ref
|
|
949
1070
|
*/
|
|
950
|
-
export async function sessionAttachmentRef(
|
|
1071
|
+
export async function sessionAttachmentRef(host, exec, attachmentId) {
|
|
951
1072
|
const agent = exec && exec.agent;
|
|
952
1073
|
const session = agent && agent.session;
|
|
953
|
-
const
|
|
954
|
-
if (!
|
|
955
|
-
let events;
|
|
1074
|
+
const wanted = String(attachmentId || '');
|
|
1075
|
+
if (!session || !wanted || !host?.ports?.sessions) return null;
|
|
956
1076
|
try {
|
|
957
|
-
|
|
1077
|
+
const ref = await host.ports.sessions.findImageAttachment(session.id, wanted);
|
|
1078
|
+
return ref ? { ref } : null;
|
|
958
1079
|
} catch (_) {
|
|
959
1080
|
return null; // 读不到会话记录 → 交给 findOwnAttachment 兜底
|
|
960
1081
|
}
|
|
961
|
-
if (!Array.isArray(events)) return null;
|
|
962
|
-
const wanted = String(attachmentId || '');
|
|
963
|
-
if (!wanted) return null;
|
|
964
|
-
const ref = findAttachmentRefIn(events, wanted);
|
|
965
|
-
return ref ? { ref } : null;
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
/** 递归扫描对象/数组,返回第一个匹配 wanted 的图片附件引用(防循环、限扫描量) */
|
|
969
|
-
function findAttachmentRefIn(nodes, wanted) {
|
|
970
|
-
const seen = new Set();
|
|
971
|
-
const stack = Array.isArray(nodes) ? [...nodes] : [nodes];
|
|
972
|
-
let scanned = 0;
|
|
973
|
-
while (stack.length && scanned < 4000) {
|
|
974
|
-
const node = stack.pop();
|
|
975
|
-
scanned++;
|
|
976
|
-
if (!node || typeof node !== 'object') continue;
|
|
977
|
-
if (seen.has(node)) continue;
|
|
978
|
-
seen.add(node);
|
|
979
|
-
// 形态 1:附件块 { type:'image', attachment:{ attachmentId,... } } —— 最优先
|
|
980
|
-
if (
|
|
981
|
-
node.attachment && typeof node.attachment === 'object' &&
|
|
982
|
-
node.attachment.attachmentId === wanted && typeof node.attachment.mediaType === 'string'
|
|
983
|
-
) {
|
|
984
|
-
return node.attachment;
|
|
985
|
-
}
|
|
986
|
-
// 形态 2:引用对象本身 { attachmentId, mediaType }(兜底)
|
|
987
|
-
if (node.attachmentId === wanted && typeof node.mediaType === 'string') {
|
|
988
|
-
return node;
|
|
989
|
-
}
|
|
990
|
-
for (const v of Object.values(node)) {
|
|
991
|
-
if (v && typeof v === 'object') stack.push(v);
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
return null;
|
|
995
1082
|
}
|
|
996
1083
|
|
|
997
1084
|
/**
|
|
@@ -1000,9 +1087,9 @@ function findAttachmentRefIn(nodes, wanted) {
|
|
|
1000
1087
|
* 全局视觉模型兜底;每个后端的失败现场保留在 errors 里。
|
|
1001
1088
|
* @returns {{answer:string,via:'selfstack'|'global',model?:string,backendId:string}}
|
|
1002
1089
|
*/
|
|
1003
|
-
export async function askVision(
|
|
1090
|
+
export async function askVision(host, { question, ref, dataUrl, signal, model }) {
|
|
1004
1091
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
1005
|
-
const backends =
|
|
1092
|
+
const backends = buildVisionBackendsFromHost(host, { providers: visionProviders, model });
|
|
1006
1093
|
return askWithBackends(backends, { question, ref, imageDataUrl: dataUrl, signal });
|
|
1007
1094
|
}
|
|
1008
1095
|
|
|
@@ -1010,9 +1097,9 @@ export async function askVision(ctx, { question, ref, dataUrl, signal, model })
|
|
|
1010
1097
|
* look / relook 共用执行器:看图问答 → 人话结果(标注模型归属,方便排查走哪条链)。
|
|
1011
1098
|
* @returns {Promise<string>} 直接作为工具输出
|
|
1012
1099
|
*/
|
|
1013
|
-
export async function runVisionTool(
|
|
1100
|
+
export async function runVisionTool(host, exec, { origin, model, question, ref, dataUrl }) {
|
|
1014
1101
|
if (exec && exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
1015
|
-
const { answer, via, model: usedModel } = await askVision(
|
|
1102
|
+
const { answer, via, model: usedModel } = await askVision(host, {
|
|
1016
1103
|
question,
|
|
1017
1104
|
ref,
|
|
1018
1105
|
dataUrl,
|
|
@@ -1025,17 +1112,19 @@ export async function runVisionTool(ctx, exec, { origin, model, question, ref, d
|
|
|
1025
1112
|
}
|
|
1026
1113
|
|
|
1027
1114
|
/* ---------------- 视觉自述(画图后增强项):走 askVision 降级链,失败静默 ---------------- */
|
|
1028
|
-
async function describeFirstImage(
|
|
1115
|
+
async function describeFirstImage(host, ref, absPath, originalPrompt) {
|
|
1029
1116
|
try {
|
|
1030
1117
|
const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
|
|
1031
|
-
|
|
1032
|
-
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
1033
|
-
ref,
|
|
1034
|
-
dataUrl: toDataUrl(absPath, mt),
|
|
1035
|
-
signal: undefined
|
|
1036
|
-
});
|
|
1037
|
-
return answer.slice(0, 300);
|
|
1118
|
+
return await describeImageData(host, ref, toDataUrl(absPath, mt), originalPrompt);
|
|
1038
1119
|
} catch (_) {
|
|
1039
1120
|
return ''; // 自述是增强项,失败不影响交付
|
|
1040
1121
|
}
|
|
1041
1122
|
}
|
|
1123
|
+
|
|
1124
|
+
async function describeImageData(host, ref, dataUrl, originalPrompt) {
|
|
1125
|
+
const { answer } = await askVision(host, {
|
|
1126
|
+
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
1127
|
+
ref, dataUrl, signal: undefined
|
|
1128
|
+
});
|
|
1129
|
+
return answer.slice(0, 300);
|
|
1130
|
+
}
|