@mokuyoaxis/dsh-iris 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-compose-media/SKILL.md +23 -49
- package/.dsh/skills/iris-compose-media/references/task-v2-recovery.md +30 -0
- package/.dsh/skills/iris-compose-media/references/workflows.md +38 -0
- package/.dsh/skills/iris-verify-ui/SKILL.md +3 -25
- package/.dsh/skills/iris-verify-ui/agents/openai.yaml +4 -0
- package/.dsh/skills/iris-verify-ui/references/verdict-and-degradation.md +31 -0
- package/CHANGELOG.md +24 -2
- package/README.en.md +29 -7
- package/README.md +29 -7
- package/docs/ARCHITECTURE.md +27 -13
- package/docs/ARTIFACT_LIBRARY.md +24 -0
- package/docs/HOST_ADAPTER_CONTRACT.md +86 -0
- package/docs/HOST_DOCTOR.md +47 -0
- package/docs/PROVIDER_ADAPTER_CONTRACT.md +97 -0
- package/docs/PROVIDER_HEALTH.md +39 -0
- package/docs/PROVIDER_SUBMISSION_CONTRACT.md +2 -2
- package/docs/ROADMAP.md +18 -8
- package/docs/SECURITY.md +6 -2
- package/docs/TASK_SEMANTICS.md +3 -2
- package/docs/assets/diagrams/iris-architecture.png +0 -0
- package/docs/assets/diagrams/iris-architecture.svg +5 -5
- package/docs/releases/0.1.4.md +59 -0
- package/lib/actions.js +357 -221
- package/lib/adapters.js +7 -7
- package/lib/api.js +30 -11
- package/lib/artifacts.js +239 -0
- package/lib/bundled-skills.js +5 -5
- package/lib/client.js +370 -55
- package/lib/config.js +192 -9
- package/lib/doctor.js +175 -4
- package/lib/dsh-host-adapter.js +189 -0
- package/lib/host-contract.js +131 -0
- package/lib/host-runtime.js +156 -0
- package/lib/index.js +97 -113
- package/lib/locate.js +1 -1
- package/lib/media.js +16 -6
- package/lib/prompt-optimizer.js +10 -20
- package/lib/provider-adapter.js +371 -0
- package/lib/provider-adapters.js +279 -0
- package/lib/provider-health.js +59 -0
- package/lib/render.js +43 -12
- package/lib/tasks.js +29 -5
- package/lib/vision.js +23 -49
- package/package.json +14 -8
- package/user_guide.md +42 -2
- package/docs/assets/diagrams/iris-architecture.drawio +0 -1
- package/docs/assets/diagrams/iris-task-lifecycle.drawio +0 -1
- package/docs/assets/diagrams/iris-workflow-compose.drawio +0 -1
- package/docs/releases/0.1.1.md +0 -51
- package/docs/releases/0.1.2.md +0 -26
- package/docs/releases/0.1.3.md +0 -30
package/lib/index.js
CHANGED
|
@@ -14,10 +14,12 @@ import * as cap from './capability.js';
|
|
|
14
14
|
import * as tasks from './tasks.js';
|
|
15
15
|
import { registerMedia, mediaLinksOf, authorizeMedia, serveMedia } from './media.js';
|
|
16
16
|
import { serveApi, closeAllSse, purgeStaleUploads } from './api.js';
|
|
17
|
-
import {
|
|
17
|
+
import { buildVisionBackendsFromHost, askWithBackends } from './vision.js';
|
|
18
|
+
import { hasHostPort, requireHostPort } from './host-contract.js';
|
|
19
|
+
import { createDshHostAdapter } from './dsh-host-adapter.js';
|
|
18
20
|
import { cropImage, pixelDiff, imageDimensions } from './pixels.js';
|
|
19
21
|
import { locateObject } from './locate.js';
|
|
20
|
-
import { serveRender
|
|
22
|
+
import { serveRender } from './render.js';
|
|
21
23
|
import { ensurePrivateDir, hardenPrivateTree } from './private-storage.js';
|
|
22
24
|
import { guarded } from './guard.js';
|
|
23
25
|
import { longOcr } from './ocr.js';
|
|
@@ -25,7 +27,15 @@ import { runAction, listActions, taskPollDeps } from './actions.js';
|
|
|
25
27
|
import { ffmpegAvailable, extractFrames, probeVideo, extractAudioTrack } from './media-probe.js';
|
|
26
28
|
import { buildContactSheet, summarizeMedia } from './summarize.js';
|
|
27
29
|
import { registerBundledSkills } from './bundled-skills.js';
|
|
28
|
-
|
|
30
|
+
import {
|
|
31
|
+
EXPECTED_IRIS_ROUTES, IRIS_PLUGIN_ID, beginHostRuntime, recordHostRoutes, recordHostSkills, recordHostTool
|
|
32
|
+
} from './host-runtime.js';
|
|
33
|
+
export { doctor, formatDoctorReport, hostDoctor } from './doctor.js';
|
|
34
|
+
|
|
35
|
+
const PACKAGE_VERSION = (() => {
|
|
36
|
+
try { return JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version || 'unknown'; }
|
|
37
|
+
catch (_) { return 'unknown'; }
|
|
38
|
+
})();
|
|
29
39
|
|
|
30
40
|
const MEDIA_TYPES = {
|
|
31
41
|
'.png': 'image/png',
|
|
@@ -35,6 +45,11 @@ const MEDIA_TYPES = {
|
|
|
35
45
|
'.gif': 'image/gif'
|
|
36
46
|
};
|
|
37
47
|
|
|
48
|
+
/** DSH 私有对象只在宿主边界转换;可选服务可能晚到,因此按操作刷新快照。 */
|
|
49
|
+
function dshHost(ctx) {
|
|
50
|
+
return createDshHostAdapter(ctx);
|
|
51
|
+
}
|
|
52
|
+
|
|
38
53
|
/** 工具内同步等待上限;超过即转后台盯守,用 iris_task_status 查询 */
|
|
39
54
|
const AWAIT_MS = { image: 180000, video: 480000 };
|
|
40
55
|
|
|
@@ -47,10 +62,9 @@ function outputDir() {
|
|
|
47
62
|
|
|
48
63
|
let mediaRouteMounted = false;
|
|
49
64
|
function mountIrisRoutes(routeCtx) {
|
|
50
|
-
// Cordis
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
: undefined;
|
|
65
|
+
// Cordis 服务只在 DSH 宿主边界探测;Command 与通用工具辅助函数不接收原始 ctx
|
|
66
|
+
const host = dshHost(routeCtx);
|
|
67
|
+
const reg = hasHostPort(host, 'routes') ? host.ports.routes : undefined;
|
|
54
68
|
if (!reg || typeof reg.register !== 'function' || mediaRouteMounted) return;
|
|
55
69
|
try {
|
|
56
70
|
const disposeMedia = reg.register({
|
|
@@ -61,7 +75,7 @@ function mountIrisRoutes(routeCtx) {
|
|
|
61
75
|
const disposeApi = reg.register({
|
|
62
76
|
kind: 'prefix',
|
|
63
77
|
path: '/iris/api',
|
|
64
|
-
handler: guarded((req, res) => serveApi(req, res, routeCtx))
|
|
78
|
+
handler: guarded((req, res) => serveApi(req, res, dshHost(routeCtx)))
|
|
65
79
|
});
|
|
66
80
|
const disposeRender = reg.register({
|
|
67
81
|
kind: 'prefix',
|
|
@@ -69,6 +83,7 @@ function mountIrisRoutes(routeCtx) {
|
|
|
69
83
|
handler: guarded(serveRender)
|
|
70
84
|
});
|
|
71
85
|
mediaRouteMounted = true;
|
|
86
|
+
recordHostRoutes(EXPECTED_IRIS_ROUTES);
|
|
72
87
|
routeCtx.effect(() => () => {
|
|
73
88
|
if (typeof disposeMedia === 'function') disposeMedia();
|
|
74
89
|
if (typeof disposeApi === 'function') disposeApi();
|
|
@@ -174,13 +189,17 @@ function guard(label, fn) {
|
|
|
174
189
|
}
|
|
175
190
|
|
|
176
191
|
/** 单工具注册隔离:注册失败的收益是诊断日志,而不是宿主陪葬 */
|
|
177
|
-
function safeRegister(
|
|
192
|
+
function safeRegister(host, definition) {
|
|
193
|
+
let registered = false;
|
|
178
194
|
guard(`工具注册 ${definition && definition.name}`, () => {
|
|
179
|
-
|
|
195
|
+
requireHostPort(host, 'tools', '注册 Agent 工具').register(definition);
|
|
196
|
+
registered = true;
|
|
180
197
|
});
|
|
198
|
+
if (registered) recordHostTool(definition.name);
|
|
181
199
|
}
|
|
182
200
|
|
|
183
201
|
export async function apply(ctx) {
|
|
202
|
+
beginHostRuntime({ pluginId: IRIS_PLUGIN_ID, version: PACKAGE_VERSION });
|
|
184
203
|
guard('Iris 私有存储权限收紧', () => hardenPrivateTree(store.irisHome()));
|
|
185
204
|
|
|
186
205
|
guard('上传临时文件清理', () => purgeStaleUploads());
|
|
@@ -219,19 +238,20 @@ export async function apply(ctx) {
|
|
|
219
238
|
// 全局层;项目目录中的同名 Skill 仍由 DSH 原生优先级覆盖。
|
|
220
239
|
ctx.inject(['skills'], (skillCtx) => {
|
|
221
240
|
guard('随包 Skill 注册', () => {
|
|
222
|
-
const names = registerBundledSkills(skillCtx);
|
|
241
|
+
const names = registerBundledSkills(dshHost(skillCtx));
|
|
242
|
+
recordHostSkills(names);
|
|
223
243
|
console.log('[iris] 随包 Skill 已注册:' + names.join(' · '));
|
|
224
244
|
});
|
|
225
245
|
});
|
|
226
246
|
|
|
227
|
-
const
|
|
228
|
-
if (!
|
|
247
|
+
const host = dshHost(ctx);
|
|
248
|
+
if (!hasHostPort(host, 'tools')) {
|
|
229
249
|
console.error('[iris] tools 服务不可用,跳过全部工具注册(宿主不受影响)');
|
|
230
250
|
return;
|
|
231
251
|
}
|
|
232
252
|
|
|
233
253
|
/* ---------- 🖼️ 画图 ---------- */
|
|
234
|
-
safeRegister(
|
|
254
|
+
safeRegister(host, {
|
|
235
255
|
name: 'iris_draw_image',
|
|
236
256
|
description:
|
|
237
257
|
'Generate an image from a detailed prompt using the user-configured image model (DashScope wan* or OpenAI-images-compatible). ' +
|
|
@@ -254,7 +274,7 @@ export async function apply(ctx) {
|
|
|
254
274
|
async execute(args, exec) {
|
|
255
275
|
const prompt = String(args.prompt || '').trim();
|
|
256
276
|
if (!prompt) throw new Error('iris: prompt 不能为空');
|
|
257
|
-
const submitted = await runAction(ctx, 'image', args, { signal: exec.signal });
|
|
277
|
+
const submitted = await runAction(dshHost(ctx), 'image', args, { signal: exec.signal });
|
|
258
278
|
let final = tasks.get(submitted.taskId);
|
|
259
279
|
if (final && !taskWaitFinished(final)) {
|
|
260
280
|
final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
|
|
@@ -270,7 +290,7 @@ export async function apply(ctx) {
|
|
|
270
290
|
const refs = [];
|
|
271
291
|
for (const f of files) {
|
|
272
292
|
const mediaType = MEDIA_TYPES[path.extname(f).toLowerCase()] || 'image/png';
|
|
273
|
-
const ref = await attachmentService(
|
|
293
|
+
const ref = await attachmentService(host).saveImage({ data: new Uint8Array(fs.readFileSync(f)), mediaType, name: path.basename(f) });
|
|
274
294
|
refs.push(ref);
|
|
275
295
|
blocks.push({ type: 'image', attachment: ref });
|
|
276
296
|
}
|
|
@@ -284,7 +304,7 @@ export async function apply(ctx) {
|
|
|
284
304
|
|
|
285
305
|
let note = '';
|
|
286
306
|
try {
|
|
287
|
-
const described = await describeFirstImage(ctx, refs[0], files[0], prompt);
|
|
307
|
+
const described = await describeFirstImage(dshHost(ctx), refs[0], files[0], prompt);
|
|
288
308
|
if (described) note = `画面内容:${described}`;
|
|
289
309
|
} catch (_) { /* 增强失败不影响生成结果 */ }
|
|
290
310
|
blocks.unshift({
|
|
@@ -298,7 +318,7 @@ export async function apply(ctx) {
|
|
|
298
318
|
});
|
|
299
319
|
|
|
300
320
|
/* ---------- 🎬 视频生成 ---------- */
|
|
301
|
-
safeRegister(
|
|
321
|
+
safeRegister(host, {
|
|
302
322
|
name: 'iris_generate_video',
|
|
303
323
|
description:
|
|
304
324
|
'Generate a video with the configured DashScope video model. Three modes: ' +
|
|
@@ -323,7 +343,7 @@ export async function apply(ctx) {
|
|
|
323
343
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
324
344
|
isConcurrencySafe: () => false,
|
|
325
345
|
async execute(args, exec) {
|
|
326
|
-
const submitted = await runAction(ctx, 'video', args, { signal: exec.signal });
|
|
346
|
+
const submitted = await runAction(dshHost(ctx), 'video', args, { signal: exec.signal });
|
|
327
347
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
328
348
|
if (!final) {
|
|
329
349
|
return `[iris] 视频任务已提交并转入后台盯守(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
@@ -342,7 +362,7 @@ export async function apply(ctx) {
|
|
|
342
362
|
});
|
|
343
363
|
|
|
344
364
|
/* ---------- 🔊 语音合成 ---------- */
|
|
345
|
-
safeRegister(
|
|
365
|
+
safeRegister(host, {
|
|
346
366
|
name: 'iris_speak_text',
|
|
347
367
|
description: 'Synthesize speech audio from text using the configured TTS model (qwen-tts / compatible). Returns the saved audio file path.',
|
|
348
368
|
parameters: {
|
|
@@ -357,7 +377,7 @@ export async function apply(ctx) {
|
|
|
357
377
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
358
378
|
isConcurrencySafe: () => true,
|
|
359
379
|
async execute(args, exec) {
|
|
360
|
-
const submitted = await runAction(ctx, 'tts', args, { signal: exec.signal });
|
|
380
|
+
const submitted = await runAction(dshHost(ctx), 'tts', args, { signal: exec.signal });
|
|
361
381
|
const final = tasks.get(submitted.taskId);
|
|
362
382
|
const terminalProblem = taskTerminalProblem(final, '语音合成');
|
|
363
383
|
if (terminalProblem) throw new Error(terminalProblem);
|
|
@@ -370,7 +390,7 @@ export async function apply(ctx) {
|
|
|
370
390
|
});
|
|
371
391
|
|
|
372
392
|
/* ---------- 🎙️ 音频转写(阶段 7.2) ---------- */
|
|
373
|
-
safeRegister(
|
|
393
|
+
safeRegister(host, {
|
|
374
394
|
name: 'iris_transcribe_audio',
|
|
375
395
|
description: 'Transcribe an audio file (wav/mp3) to text using the configured DashScope provider (qwen-audio-3.0-asr-flash-filetrans). The audio is uploaded to Bailian temp storage, then an async task is submitted. Returns the full recognized text.',
|
|
376
396
|
parameters: {
|
|
@@ -384,7 +404,7 @@ export async function apply(ctx) {
|
|
|
384
404
|
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
385
405
|
isConcurrencySafe: () => false,
|
|
386
406
|
async execute(args, exec) {
|
|
387
|
-
const submitted = await runAction(ctx, 'transcribe', args, { signal: exec.signal });
|
|
407
|
+
const submitted = await runAction(dshHost(ctx), 'transcribe', args, { signal: exec.signal });
|
|
388
408
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
389
409
|
if (!final) return `[iris] 音频转写任务已提交并转入后台(task: ${submitted.taskId})。完成后用 iris_task_status 查询。`;
|
|
390
410
|
const terminalProblem = taskTerminalProblem(final, '音频转写');
|
|
@@ -394,7 +414,7 @@ export async function apply(ctx) {
|
|
|
394
414
|
});
|
|
395
415
|
|
|
396
416
|
/* ---------- 🎞️ 视频抽帧(阶段 7.1,ffmpeg 可选) ---------- */
|
|
397
|
-
safeRegister(
|
|
417
|
+
safeRegister(host, {
|
|
398
418
|
name: 'iris_video_frames',
|
|
399
419
|
description:
|
|
400
420
|
'Extract N frames (uniformly sampled across the video) from a local video file using the system ffmpeg. ' +
|
|
@@ -433,7 +453,7 @@ export async function apply(ctx) {
|
|
|
433
453
|
const refs = [];
|
|
434
454
|
for (let i = 0; i < frames.length; i++) {
|
|
435
455
|
const f = frames[i];
|
|
436
|
-
const ref = await attachmentService(
|
|
456
|
+
const ref = await attachmentService(host).saveImage({
|
|
437
457
|
data: new Uint8Array(f.buffer),
|
|
438
458
|
mediaType,
|
|
439
459
|
name: `iris-frame-${Date.now()}-${i + 1}.${ext}`
|
|
@@ -455,7 +475,7 @@ export async function apply(ctx) {
|
|
|
455
475
|
});
|
|
456
476
|
|
|
457
477
|
/* ---------- 📝 多模态视频摘要(阶段 7.3,ffmpeg 可选) ---------- */
|
|
458
|
-
safeRegister(
|
|
478
|
+
safeRegister(host, {
|
|
459
479
|
name: 'iris_media_summarize',
|
|
460
480
|
description:
|
|
461
481
|
'Summarize a local video by compressing it into a contact sheet of sampled frames (with timestamps) plus an optional ' +
|
|
@@ -499,7 +519,7 @@ export async function apply(ctx) {
|
|
|
499
519
|
try {
|
|
500
520
|
const audio = await extractAudioTrack({ inputPath: p, signal: exec.signal });
|
|
501
521
|
try {
|
|
502
|
-
const submitted = await runAction(ctx, 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
|
|
522
|
+
const submitted = await runAction(dshHost(ctx), 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
|
|
503
523
|
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
504
524
|
if (final && final.status === 'succeeded') transcript = final.transcribeText || '';
|
|
505
525
|
else if (!final) transcriptNote = `\n(音轨转写仍在后台:task ${submitted.taskId},本次摘要未含语音内容)`;
|
|
@@ -516,13 +536,13 @@ export async function apply(ctx) {
|
|
|
516
536
|
|
|
517
537
|
// contact sheet → attachment → 视觉摘要
|
|
518
538
|
const sheet = await buildContactSheet({ frames, cols: 3 });
|
|
519
|
-
const ref = await attachmentService(
|
|
539
|
+
const ref = await attachmentService(host).saveImage({
|
|
520
540
|
data: new Uint8Array(sheet.buffer),
|
|
521
541
|
mediaType: 'image/png',
|
|
522
542
|
name: `iris-sheet-${Date.now()}.png`
|
|
523
543
|
});
|
|
524
544
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
525
|
-
const backends =
|
|
545
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
526
546
|
const { answer, via, model } = await summarizeMedia({
|
|
527
547
|
backends, question: args.question, transcript,
|
|
528
548
|
imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
|
|
@@ -546,7 +566,7 @@ export async function apply(ctx) {
|
|
|
546
566
|
|
|
547
567
|
/* ---------- 👁 视觉路由(M3):显式工具,自持栈为主 ---------- */
|
|
548
568
|
|
|
549
|
-
safeRegister(
|
|
569
|
+
safeRegister(host, {
|
|
550
570
|
name: 'iris_look_at_image',
|
|
551
571
|
description:
|
|
552
572
|
'Look at a local image file and answer one question about it with the configured vision model (qwen-vl via the self-hosted provider stack; falls back to the global DSH vision model). The image is saved as a durable attachment so follow-ups can reference it.',
|
|
@@ -571,12 +591,12 @@ export async function apply(ctx) {
|
|
|
571
591
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
572
592
|
|
|
573
593
|
const dataUrl = toDataUrl(p, mt);
|
|
574
|
-
const ref = await attachmentService(
|
|
575
|
-
return runVisionTool(ctx, exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
|
|
594
|
+
const ref = await attachmentService(host).saveImage({ data: new Uint8Array(fs.readFileSync(p)), mediaType: mt, name: path.basename(p) });
|
|
595
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
|
|
576
596
|
}
|
|
577
597
|
});
|
|
578
598
|
|
|
579
|
-
safeRegister(
|
|
599
|
+
safeRegister(host, {
|
|
580
600
|
name: 'iris_relook_attachment',
|
|
581
601
|
description:
|
|
582
602
|
'Ask a NEW question about an image that was already seen in this session: pass its attachment_id (from user uploads, tool results, or any iris-generated image). Pixels are re-read via the vision model.',
|
|
@@ -599,22 +619,22 @@ export async function apply(ctx) {
|
|
|
599
619
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
600
620
|
|
|
601
621
|
// 定位:本会话出现过的图片优先,其次本插件产物记录
|
|
602
|
-
const hit = (await sessionAttachmentRef(
|
|
622
|
+
const hit = (await sessionAttachmentRef(host, exec, id)) || findOwnAttachment(id);
|
|
603
623
|
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
604
624
|
|
|
605
625
|
let dataUrl;
|
|
606
626
|
if (hit.absPath) {
|
|
607
627
|
dataUrl = toDataUrl(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png');
|
|
608
628
|
} else {
|
|
609
|
-
const stored = await attachmentService(
|
|
629
|
+
const stored = await attachmentService(host).readImage(hit.ref, exec.signal);
|
|
610
630
|
dataUrl = `data:${stored.mediaType || hit.ref.mediaType};base64,${Buffer.from(stored.data).toString('base64')}`;
|
|
611
631
|
}
|
|
612
|
-
return runVisionTool(ctx, exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
632
|
+
return runVisionTool(dshHost(ctx), exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
613
633
|
}
|
|
614
634
|
});
|
|
615
635
|
|
|
616
636
|
/* ---------- 📋 任务查询 ---------- */
|
|
617
|
-
safeRegister(
|
|
637
|
+
safeRegister(host, {
|
|
618
638
|
name: 'iris_task_status',
|
|
619
639
|
description:
|
|
620
640
|
'Query iris generation tasks (image/video/tts). Pass a task_id for one task, or omit it to list the latest tasks with status, progress, errors and output file paths.',
|
|
@@ -640,7 +660,7 @@ export async function apply(ctx) {
|
|
|
640
660
|
});
|
|
641
661
|
|
|
642
662
|
/* ---------- ✂️ 确定性像素工具(阶段 2) ---------- */
|
|
643
|
-
safeRegister(
|
|
663
|
+
safeRegister(host, {
|
|
644
664
|
name: 'iris_crop',
|
|
645
665
|
description:
|
|
646
666
|
'Crop a rectangular region from an image (absolute local path or a session attachment_id) and save the result as a durable DSH attachment. ' +
|
|
@@ -661,7 +681,7 @@ export async function apply(ctx) {
|
|
|
661
681
|
isConcurrencySafe: () => true,
|
|
662
682
|
async execute(args, exec) {
|
|
663
683
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
664
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
684
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
665
685
|
image_path: args.image_path,
|
|
666
686
|
attachment_id: args.attachment_id
|
|
667
687
|
});
|
|
@@ -672,7 +692,7 @@ export async function apply(ctx) {
|
|
|
672
692
|
width: args.width,
|
|
673
693
|
height: args.height
|
|
674
694
|
});
|
|
675
|
-
const ref = await attachmentService(
|
|
695
|
+
const ref = await attachmentService(host).saveImage({
|
|
676
696
|
data: new Uint8Array(buffer),
|
|
677
697
|
mediaType: mime,
|
|
678
698
|
name: `iris-crop-${Date.now()}.png`
|
|
@@ -686,7 +706,7 @@ export async function apply(ctx) {
|
|
|
686
706
|
}
|
|
687
707
|
});
|
|
688
708
|
|
|
689
|
-
safeRegister(
|
|
709
|
+
safeRegister(host, {
|
|
690
710
|
name: 'iris_pixel_diff',
|
|
691
711
|
description:
|
|
692
712
|
'Compute a pixel-level difference between two images (absolute local paths or session attachment_ids). ' +
|
|
@@ -710,15 +730,15 @@ export async function apply(ctx) {
|
|
|
710
730
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
711
731
|
if (!(args.image_a_path || args.attachment_a_id)) throw new Error('iris: 需要 image_a_path 或 attachment_a_id');
|
|
712
732
|
if (!(args.image_b_path || args.attachment_b_id)) throw new Error('iris: 需要 image_b_path 或 attachment_b_id');
|
|
713
|
-
const a = await resolveImageInput(ctx, exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
714
|
-
const b = await resolveImageInput(ctx, exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
733
|
+
const a = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
734
|
+
const b = await resolveImageInput(dshHost(ctx), exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
715
735
|
const { ratio, diffPixels, totalPixels, width, height, worstRegions, heatmap, mime } = await pixelDiff({
|
|
716
736
|
inputA: a.buffer,
|
|
717
737
|
inputB: b.buffer,
|
|
718
738
|
grid: args.grid || 8,
|
|
719
739
|
topRegions: args.top_regions || 3
|
|
720
740
|
});
|
|
721
|
-
const ref = await attachmentService(
|
|
741
|
+
const ref = await attachmentService(host).saveImage({
|
|
722
742
|
data: new Uint8Array(heatmap),
|
|
723
743
|
mediaType: mime,
|
|
724
744
|
name: `iris-diff-${Date.now()}.png`
|
|
@@ -740,7 +760,7 @@ export async function apply(ctx) {
|
|
|
740
760
|
});
|
|
741
761
|
|
|
742
762
|
/* ---------- 📍 模型驱动定位(阶段 3A) ---------- */
|
|
743
|
-
safeRegister(
|
|
763
|
+
safeRegister(host, {
|
|
744
764
|
name: 'iris_locate',
|
|
745
765
|
description:
|
|
746
766
|
'Locate an object/region described by `target` in an image (absolute path or session attachment_id), ' +
|
|
@@ -760,14 +780,14 @@ export async function apply(ctx) {
|
|
|
760
780
|
isConcurrencySafe: () => true,
|
|
761
781
|
async execute(args, exec) {
|
|
762
782
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
763
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
783
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
764
784
|
image_path: args.image_path,
|
|
765
785
|
attachment_id: args.attachment_id
|
|
766
786
|
});
|
|
767
787
|
const { width, height } = await imageDimensions(input.buffer);
|
|
768
788
|
const dataUrl = `data:${input.mediaType};base64,${input.buffer.toString('base64')}`;
|
|
769
789
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
770
|
-
const backends =
|
|
790
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders, model: args.model });
|
|
771
791
|
const r = await locateObject(backends, {
|
|
772
792
|
target: args.target,
|
|
773
793
|
imageDataUrl: dataUrl,
|
|
@@ -782,7 +802,7 @@ export async function apply(ctx) {
|
|
|
782
802
|
});
|
|
783
803
|
|
|
784
804
|
/* ---------- 🖼️ HTML 截图(阶段 3C,基于 dsh-builtin-browser) ---------- */
|
|
785
|
-
safeRegister(
|
|
805
|
+
safeRegister(host, {
|
|
786
806
|
name: 'iris_html_screenshot',
|
|
787
807
|
description:
|
|
788
808
|
'Render an HTML string to a page screenshot using the shared browser. ' +
|
|
@@ -804,19 +824,17 @@ export async function apply(ctx) {
|
|
|
804
824
|
isConcurrencySafe: () => false,
|
|
805
825
|
async execute(args, exec) {
|
|
806
826
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
807
|
-
const browser = ctx
|
|
808
|
-
|
|
809
|
-
throw new Error('iris: 浏览器服务不可用(ctx.get("browser") 为空)——需要启用 dsh-builtin-browser 插件');
|
|
810
|
-
}
|
|
811
|
-
const renderer = new BrowserHtmlRenderer({ browser });
|
|
812
|
-
const { png } = await renderer.render({
|
|
827
|
+
const browser = requireHostPort(dshHost(ctx), 'browser', 'HTML 截图');
|
|
828
|
+
const rendered = await browser.renderHtml({
|
|
813
829
|
html: args.html,
|
|
814
830
|
width: args.width,
|
|
815
831
|
height: args.height,
|
|
816
|
-
fullPage: args.fullPage !== false
|
|
832
|
+
fullPage: args.fullPage !== false,
|
|
833
|
+
signal: exec.signal
|
|
817
834
|
});
|
|
835
|
+
const png = Buffer.from(rendered.bytes);
|
|
818
836
|
const { width: outW, height: outH } = await imageDimensions(png);
|
|
819
|
-
const ref = await attachmentService(
|
|
837
|
+
const ref = await attachmentService(host).saveImage({
|
|
820
838
|
data: new Uint8Array(png),
|
|
821
839
|
mediaType: 'image/png',
|
|
822
840
|
name: `iris-html-${Date.now()}.png`
|
|
@@ -836,7 +854,7 @@ export async function apply(ctx) {
|
|
|
836
854
|
});
|
|
837
855
|
|
|
838
856
|
/* ---------- 📄 长截图 OCR(阶段 3B,视觉模型分块) ---------- */
|
|
839
|
-
safeRegister(
|
|
857
|
+
safeRegister(host, {
|
|
840
858
|
name: 'iris_long_ocr',
|
|
841
859
|
description:
|
|
842
860
|
'OCR a (possibly long) image: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
@@ -856,12 +874,12 @@ export async function apply(ctx) {
|
|
|
856
874
|
isConcurrencySafe: () => true,
|
|
857
875
|
async execute(args, exec) {
|
|
858
876
|
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
859
|
-
const input = await resolveImageInput(ctx, exec, {
|
|
877
|
+
const input = await resolveImageInput(dshHost(ctx), exec, {
|
|
860
878
|
image_path: args.image_path,
|
|
861
879
|
attachment_id: args.attachment_id
|
|
862
880
|
});
|
|
863
881
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
864
|
-
const backends =
|
|
882
|
+
const backends = buildVisionBackendsFromHost(dshHost(ctx), { providers: visionProviders });
|
|
865
883
|
const result = await longOcr({
|
|
866
884
|
input: input.buffer,
|
|
867
885
|
backends,
|
|
@@ -881,13 +899,13 @@ export async function apply(ctx) {
|
|
|
881
899
|
|
|
882
900
|
/* ---------------- 视觉路由(M3 补全):look/relook 共用执行器 + 降级链 ---------------- */
|
|
883
901
|
|
|
884
|
-
/**
|
|
885
|
-
function attachmentService(
|
|
886
|
-
const
|
|
887
|
-
if (
|
|
888
|
-
throw new Error('iris:
|
|
902
|
+
/** 通用工具只消费 Host attachments 端口。 */
|
|
903
|
+
function attachmentService(host) {
|
|
904
|
+
const attachments = requireHostPort(host, 'attachments', '处理 DSH 附件');
|
|
905
|
+
if (typeof attachments.saveImage !== 'function' || typeof attachments.readImage !== 'function') {
|
|
906
|
+
throw new Error('iris: Host attachments 端口缺少 saveImage/readImage');
|
|
889
907
|
}
|
|
890
|
-
return
|
|
908
|
+
return attachments;
|
|
891
909
|
}
|
|
892
910
|
|
|
893
911
|
/**
|
|
@@ -895,7 +913,7 @@ function attachmentService(ctx) {
|
|
|
895
913
|
* crop / pixel_diff 共用(阶段 2)。
|
|
896
914
|
* @returns {Promise<{buffer:Buffer, mediaType:string, name:string}>}
|
|
897
915
|
*/
|
|
898
|
-
async function resolveImageInput(
|
|
916
|
+
async function resolveImageInput(host, exec, { image_path, attachment_id }) {
|
|
899
917
|
if (image_path) {
|
|
900
918
|
const p = String(image_path).trim();
|
|
901
919
|
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
@@ -906,7 +924,7 @@ async function resolveImageInput(ctx, exec, { image_path, attachment_id }) {
|
|
|
906
924
|
}
|
|
907
925
|
if (attachment_id) {
|
|
908
926
|
const id = String(attachment_id).trim();
|
|
909
|
-
const hit = (await sessionAttachmentRef(
|
|
927
|
+
const hit = (await sessionAttachmentRef(host, exec, id)) || findOwnAttachment(id);
|
|
910
928
|
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
911
929
|
if (hit.absPath) {
|
|
912
930
|
return {
|
|
@@ -915,7 +933,7 @@ async function resolveImageInput(ctx, exec, { image_path, attachment_id }) {
|
|
|
915
933
|
name: path.basename(hit.absPath)
|
|
916
934
|
};
|
|
917
935
|
}
|
|
918
|
-
const stored = await attachmentService(
|
|
936
|
+
const stored = await attachmentService(host).readImage(hit.ref, exec && exec.signal);
|
|
919
937
|
return {
|
|
920
938
|
buffer: Buffer.from(stored.data),
|
|
921
939
|
mediaType: stored.mediaType || hit.ref.mediaType || 'image/png',
|
|
@@ -947,51 +965,17 @@ export function findOwnAttachment(attachmentId) {
|
|
|
947
965
|
* 不依赖精确事件类型:递归扫任何载荷中的 attachment 块 / attachmentId 字段。
|
|
948
966
|
* @returns {Promise<{ref:object}|null>} 找到则带可直接 readImage 的 ref
|
|
949
967
|
*/
|
|
950
|
-
export async function sessionAttachmentRef(
|
|
968
|
+
export async function sessionAttachmentRef(host, exec, attachmentId) {
|
|
951
969
|
const agent = exec && exec.agent;
|
|
952
970
|
const session = agent && agent.session;
|
|
953
|
-
const
|
|
954
|
-
if (!
|
|
955
|
-
let events;
|
|
971
|
+
const wanted = String(attachmentId || '');
|
|
972
|
+
if (!session || !wanted || !host?.ports?.sessions) return null;
|
|
956
973
|
try {
|
|
957
|
-
|
|
974
|
+
const ref = await host.ports.sessions.findImageAttachment(session.id, wanted);
|
|
975
|
+
return ref ? { ref } : null;
|
|
958
976
|
} catch (_) {
|
|
959
977
|
return null; // 读不到会话记录 → 交给 findOwnAttachment 兜底
|
|
960
978
|
}
|
|
961
|
-
if (!Array.isArray(events)) return null;
|
|
962
|
-
const wanted = String(attachmentId || '');
|
|
963
|
-
if (!wanted) return null;
|
|
964
|
-
const ref = findAttachmentRefIn(events, wanted);
|
|
965
|
-
return ref ? { ref } : null;
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
/** 递归扫描对象/数组,返回第一个匹配 wanted 的图片附件引用(防循环、限扫描量) */
|
|
969
|
-
function findAttachmentRefIn(nodes, wanted) {
|
|
970
|
-
const seen = new Set();
|
|
971
|
-
const stack = Array.isArray(nodes) ? [...nodes] : [nodes];
|
|
972
|
-
let scanned = 0;
|
|
973
|
-
while (stack.length && scanned < 4000) {
|
|
974
|
-
const node = stack.pop();
|
|
975
|
-
scanned++;
|
|
976
|
-
if (!node || typeof node !== 'object') continue;
|
|
977
|
-
if (seen.has(node)) continue;
|
|
978
|
-
seen.add(node);
|
|
979
|
-
// 形态 1:附件块 { type:'image', attachment:{ attachmentId,... } } —— 最优先
|
|
980
|
-
if (
|
|
981
|
-
node.attachment && typeof node.attachment === 'object' &&
|
|
982
|
-
node.attachment.attachmentId === wanted && typeof node.attachment.mediaType === 'string'
|
|
983
|
-
) {
|
|
984
|
-
return node.attachment;
|
|
985
|
-
}
|
|
986
|
-
// 形态 2:引用对象本身 { attachmentId, mediaType }(兜底)
|
|
987
|
-
if (node.attachmentId === wanted && typeof node.mediaType === 'string') {
|
|
988
|
-
return node;
|
|
989
|
-
}
|
|
990
|
-
for (const v of Object.values(node)) {
|
|
991
|
-
if (v && typeof v === 'object') stack.push(v);
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
return null;
|
|
995
979
|
}
|
|
996
980
|
|
|
997
981
|
/**
|
|
@@ -1000,9 +984,9 @@ function findAttachmentRefIn(nodes, wanted) {
|
|
|
1000
984
|
* 全局视觉模型兜底;每个后端的失败现场保留在 errors 里。
|
|
1001
985
|
* @returns {{answer:string,via:'selfstack'|'global',model?:string,backendId:string}}
|
|
1002
986
|
*/
|
|
1003
|
-
export async function askVision(
|
|
987
|
+
export async function askVision(host, { question, ref, dataUrl, signal, model }) {
|
|
1004
988
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
1005
|
-
const backends =
|
|
989
|
+
const backends = buildVisionBackendsFromHost(host, { providers: visionProviders, model });
|
|
1006
990
|
return askWithBackends(backends, { question, ref, imageDataUrl: dataUrl, signal });
|
|
1007
991
|
}
|
|
1008
992
|
|
|
@@ -1010,9 +994,9 @@ export async function askVision(ctx, { question, ref, dataUrl, signal, model })
|
|
|
1010
994
|
* look / relook 共用执行器:看图问答 → 人话结果(标注模型归属,方便排查走哪条链)。
|
|
1011
995
|
* @returns {Promise<string>} 直接作为工具输出
|
|
1012
996
|
*/
|
|
1013
|
-
export async function runVisionTool(
|
|
997
|
+
export async function runVisionTool(host, exec, { origin, model, question, ref, dataUrl }) {
|
|
1014
998
|
if (exec && exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
1015
|
-
const { answer, via, model: usedModel } = await askVision(
|
|
999
|
+
const { answer, via, model: usedModel } = await askVision(host, {
|
|
1016
1000
|
question,
|
|
1017
1001
|
ref,
|
|
1018
1002
|
dataUrl,
|
|
@@ -1025,10 +1009,10 @@ export async function runVisionTool(ctx, exec, { origin, model, question, ref, d
|
|
|
1025
1009
|
}
|
|
1026
1010
|
|
|
1027
1011
|
/* ---------------- 视觉自述(画图后增强项):走 askVision 降级链,失败静默 ---------------- */
|
|
1028
|
-
async function describeFirstImage(
|
|
1012
|
+
async function describeFirstImage(host, ref, absPath, originalPrompt) {
|
|
1029
1013
|
try {
|
|
1030
1014
|
const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
|
|
1031
|
-
const { answer } = await askVision(
|
|
1015
|
+
const { answer } = await askVision(host, {
|
|
1032
1016
|
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
1033
1017
|
ref,
|
|
1034
1018
|
dataUrl: toDataUrl(absPath, mt),
|
package/lib/locate.js
CHANGED
|
@@ -38,7 +38,7 @@ export function extractBboxJson(answer) {
|
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
40
|
* 在原图上定位目标,返回原像素 bbox。
|
|
41
|
-
* @param {Array} backends 有序视觉后端(
|
|
41
|
+
* @param {Array} backends 有序视觉后端(buildVisionBackendsFromHost 产物)
|
|
42
42
|
* @param {{target:string, imageDataUrl:string, width:number, height:number, signal?:AbortSignal}}
|
|
43
43
|
* @returns {Promise<{found:boolean, x1?:number, y1?:number, x2?:number, y2?:number, via?:string, model?:string}>}
|
|
44
44
|
* @throws LocateError 模型未返回有效 bbox / bbox 完全越界 / 字段非法
|
package/lib/media.js
CHANGED
|
@@ -15,13 +15,16 @@ import crypto from 'node:crypto';
|
|
|
15
15
|
import fs from 'node:fs';
|
|
16
16
|
import path from 'node:path';
|
|
17
17
|
import * as tasks from './tasks.js';
|
|
18
|
+
import * as artifacts from './artifacts.js';
|
|
18
19
|
|
|
19
20
|
const MIME = {
|
|
20
21
|
'.mp4': 'video/mp4',
|
|
21
22
|
'.mov': 'video/quicktime',
|
|
22
23
|
'.webm': 'video/webm',
|
|
23
|
-
'.wav': 'audio/wav',
|
|
24
|
-
'.
|
|
24
|
+
'.wav': 'audio/wav', '.mp3': 'audio/mpeg', '.m4a': 'audio/mp4',
|
|
25
|
+
'.aac': 'audio/aac', '.ogg': 'audio/ogg', '.opus': 'audio/ogg', '.flac': 'audio/flac',
|
|
26
|
+
'.mkv': 'video/x-matroska', '.avi': 'video/x-msvideo', '.mpeg': 'video/mpeg', '.mpg': 'video/mpeg',
|
|
27
|
+
'.avif': 'image/avif', '.bmp': 'image/bmp',
|
|
25
28
|
'.png': 'image/png',
|
|
26
29
|
'.jpg': 'image/jpeg',
|
|
27
30
|
'.jpeg': 'image/jpeg',
|
|
@@ -61,9 +64,11 @@ export function registerMedia(taskId, absPath) {
|
|
|
61
64
|
createdAt: new Date().toISOString()
|
|
62
65
|
};
|
|
63
66
|
tasks.update(taskId, { media: [...(t.media || []), entry] });
|
|
67
|
+
artifacts.register(absPath);
|
|
64
68
|
return { ...entry, url: mediaUrl(taskId, entry) };
|
|
65
69
|
}
|
|
66
70
|
const entry = t.media.find((e) => e.file === file);
|
|
71
|
+
artifacts.register(absPath);
|
|
67
72
|
return { ...entry, url: mediaUrl(taskId, entry) };
|
|
68
73
|
}
|
|
69
74
|
|
|
@@ -127,15 +132,20 @@ export function serveMedia(req, res) {
|
|
|
127
132
|
const method = String(req.method || 'GET').toUpperCase();
|
|
128
133
|
if (method !== 'GET' && method !== 'HEAD') return sendText(res, 405, 'method not allowed');
|
|
129
134
|
const parts = String(req.url || '').split('?')[0].split('/').filter(Boolean);
|
|
130
|
-
// /iris/media/:taskId/:token/:name
|
|
131
|
-
|
|
135
|
+
// Task media: /iris/media/:taskId/:token/:name
|
|
136
|
+
// Independent work: /iris/media/artifact/:artifactId/:token/:name
|
|
137
|
+
const artifactRoute = parts.length === 6 && parts[0] === 'iris' && parts[1] === 'media' && parts[2] === 'artifact';
|
|
138
|
+
const taskRoute = parts.length === 5 && parts[0] === 'iris' && parts[1] === 'media';
|
|
139
|
+
if (!artifactRoute && !taskRoute) return sendText(res, 404, 'not found');
|
|
132
140
|
let name;
|
|
133
141
|
try {
|
|
134
|
-
name = decodeURIComponent(parts[4]);
|
|
142
|
+
name = decodeURIComponent(parts[artifactRoute ? 5 : 4]);
|
|
135
143
|
} catch (_) {
|
|
136
144
|
return sendText(res, 404, 'not found');
|
|
137
145
|
}
|
|
138
|
-
const hit =
|
|
146
|
+
const hit = artifactRoute
|
|
147
|
+
? artifacts.authorize(parts[3], parts[4], name)
|
|
148
|
+
: authorizeMedia(parts[2], parts[3], name);
|
|
139
149
|
if (!hit) return sendText(res, 404, 'not found'); // 未命中一律 404,不给枚举线索
|
|
140
150
|
const base = {
|
|
141
151
|
'Content-Type': hit.entry.mime,
|