@mokuyoaxis/dsh-iris 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-compose-media/SKILL.md +114 -0
- package/.dsh/skills/iris-compose-media/agents/openai.yaml +4 -0
- package/.dsh/skills/iris-verify-ui/SKILL.md +127 -0
- package/CHANGELOG.md +53 -0
- package/LICENSE +21 -0
- package/README.md +177 -0
- package/cordis.patch.yml +3 -0
- package/docs/ROADMAP.md +72 -0
- package/docs/assets/iris-wordmark.svg +25 -0
- package/docs/file-access-across-environments.md +78 -0
- package/lib/actions.js +930 -0
- package/lib/adapters.js +504 -0
- package/lib/api.js +467 -0
- package/lib/capability.js +103 -0
- package/lib/client.d.ts +5 -0
- package/lib/client.js +1187 -0
- package/lib/config.js +384 -0
- package/lib/guard.js +73 -0
- package/lib/index.js +1028 -0
- package/lib/locate.js +94 -0
- package/lib/media-probe.js +169 -0
- package/lib/media.js +172 -0
- package/lib/models.js +128 -0
- package/lib/ocr.js +80 -0
- package/lib/pixels.js +140 -0
- package/lib/render.js +138 -0
- package/lib/summarize.js +111 -0
- package/lib/tasks.js +315 -0
- package/lib/vision.js +171 -0
- package/package.json +95 -0
- package/user_guide.md +212 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,1028 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* dsh-iris —— Host 入口。
|
|
4
|
+
* 给 DSH 装上眼睛和双手:多供应商媒体生成 + 视觉路由 + Iris 工作台。
|
|
5
|
+
* M2:新增视频生成(文生/图生);全部异步生成统一走任务盯守框架
|
|
6
|
+
* (提交即盯守、工具内等待超时自动转后台、DSH 重启后恢复接管)。
|
|
7
|
+
* 架构纪律:无独立后台,与 DSH 俱荣俱损——定时器归 Fiber,产物落 $DSH_HOME。
|
|
8
|
+
*/
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import os from 'node:os';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import * as adapters from './adapters.js';
|
|
13
|
+
import * as store from './config.js';
|
|
14
|
+
import * as cap from './capability.js';
|
|
15
|
+
import * as tasks from './tasks.js';
|
|
16
|
+
import { registerMedia, mediaLinksOf, authorizeMedia, serveMedia } from './media.js';
|
|
17
|
+
import { serveApi, closeAllSse, purgeStaleUploads } from './api.js';
|
|
18
|
+
import { buildVisionBackends, askWithBackends } from './vision.js';
|
|
19
|
+
import { cropImage, pixelDiff, imageDimensions } from './pixels.js';
|
|
20
|
+
import { locateObject } from './locate.js';
|
|
21
|
+
import { serveRender, BrowserHtmlRenderer } from './render.js';
|
|
22
|
+
import { guarded } from './guard.js';
|
|
23
|
+
import { longOcr } from './ocr.js';
|
|
24
|
+
import { runAction, listActions } from './actions.js';
|
|
25
|
+
import { ffmpegAvailable, extractFrames, probeVideo, extractAudioTrack } from './media-probe.js';
|
|
26
|
+
import { buildContactSheet, summarizeMedia } from './summarize.js';
|
|
27
|
+
|
|
28
|
+
const MEDIA_TYPES = {
|
|
29
|
+
'.png': 'image/png',
|
|
30
|
+
'.jpg': 'image/jpeg',
|
|
31
|
+
'.jpeg': 'image/jpeg',
|
|
32
|
+
'.webp': 'image/webp',
|
|
33
|
+
'.gif': 'image/gif'
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/** 工具内同步等待上限;超过即转后台盯守,用 iris_task_status 查询 */
|
|
37
|
+
const AWAIT_MS = { image: 180000, video: 480000 };
|
|
38
|
+
|
|
39
|
+
function outputDir() {
|
|
40
|
+
const dir = tasks.outputsDir();
|
|
41
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
42
|
+
return dir;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/* ---------------- 媒体路由:/iris/media/:taskId/:token/:name ---------------- */
|
|
46
|
+
|
|
47
|
+
let mediaRouteMounted = false;
|
|
48
|
+
function mountIrisRoutes(routeCtx) {
|
|
49
|
+
// Cordis 铁律:服务一律走 ctx.get() 可选访问;属性直读会在未 inject 时抛错炸掉整个插件
|
|
50
|
+
const reg = routeCtx && typeof routeCtx.get === 'function'
|
|
51
|
+
? (routeCtx.get('webServer') || routeCtx.get('httpServer'))
|
|
52
|
+
: undefined;
|
|
53
|
+
if (!reg || typeof reg.register !== 'function' || mediaRouteMounted) return;
|
|
54
|
+
try {
|
|
55
|
+
const disposeMedia = reg.register({
|
|
56
|
+
kind: 'prefix',
|
|
57
|
+
path: '/iris/media',
|
|
58
|
+
handler: guarded(serveMedia)
|
|
59
|
+
});
|
|
60
|
+
const disposeApi = reg.register({
|
|
61
|
+
kind: 'prefix',
|
|
62
|
+
path: '/iris/api',
|
|
63
|
+
handler: guarded((req, res) => serveApi(req, res, routeCtx))
|
|
64
|
+
});
|
|
65
|
+
const disposeRender = reg.register({
|
|
66
|
+
kind: 'prefix',
|
|
67
|
+
path: '/iris/render',
|
|
68
|
+
handler: guarded(serveRender)
|
|
69
|
+
});
|
|
70
|
+
mediaRouteMounted = true;
|
|
71
|
+
routeCtx.effect(() => () => {
|
|
72
|
+
if (typeof disposeMedia === 'function') disposeMedia();
|
|
73
|
+
if (typeof disposeApi === 'function') disposeApi();
|
|
74
|
+
if (typeof disposeRender === 'function') disposeRender();
|
|
75
|
+
closeAllSse(); // SSE 长连接随插件停用一起关闭,不留悬挂连接
|
|
76
|
+
mediaRouteMounted = false;
|
|
77
|
+
}, 'iris: media+api+actions+render routes');
|
|
78
|
+
console.log('[iris] 媒体+工作台+操作路由已挂载:/iris/media · /iris/api · /iris/api/actions · /iris/render');
|
|
79
|
+
} catch (err) {
|
|
80
|
+
console.error('[iris] 路由挂载失败:', err && err.message);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** 盯守依赖:按能力给轮询间隔与结果转存扩展名 */
|
|
85
|
+
export function pollDeps(provider, cap) {
|
|
86
|
+
if (cap === 'transcribe') {
|
|
87
|
+
// 转写任务的结果在 transcription_text 而非 urls——重启接管必须走专用轮询器,
|
|
88
|
+
// 否则 pollTask 拿到空 urls,盯守会「成功」但转写文本静默丢失。
|
|
89
|
+
// (与 iris_transcribe_audio 工具内的盯守依赖保持同构。)
|
|
90
|
+
return {
|
|
91
|
+
key: () => provider.apiKey,
|
|
92
|
+
poll: async ({ key, remoteTaskId }) => {
|
|
93
|
+
const r = await adapters.pollTranscriptionTask({ key, remoteTaskId });
|
|
94
|
+
return { done: r.done, ok: r.ok, urls: r.text ? [r.text] : [], message: r.message };
|
|
95
|
+
},
|
|
96
|
+
intervalMs: 2500,
|
|
97
|
+
onSuccess: async (t, r) => {
|
|
98
|
+
tasks.update(t.id, { transcribeText: (r.urls && r.urls.join('')) || '' });
|
|
99
|
+
return [];
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
key: () => provider.apiKey,
|
|
105
|
+
poll: ({ key, remoteTaskId }) => adapters.pollTask({ key, remoteTaskId }),
|
|
106
|
+
intervalMs: cap === 'video' ? 6000 : 2500,
|
|
107
|
+
onSuccess: async (t, r) => {
|
|
108
|
+
fs.mkdirSync(outputDir(), { recursive: true });
|
|
109
|
+
const names = [];
|
|
110
|
+
for (const url of r.urls || []) {
|
|
111
|
+
const name = `${t.id}-${names.length}.${adapters.extFromUrl(url, cap === 'video' ? 'mp4' : 'png')}`;
|
|
112
|
+
await adapters.downloadTo(url, path.join(outputDir(), name));
|
|
113
|
+
names.push(name);
|
|
114
|
+
}
|
|
115
|
+
// 登记媒体链接(token 授权),对话流里可点播
|
|
116
|
+
for (const n of names) registerMedia(t.id, path.join(outputDir(), n));
|
|
117
|
+
return names;
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* 在工具调用内同步等待任务到终态。
|
|
124
|
+
* @returns 终态任务记录;null = 等待超时(任务已转后台继续盯守)
|
|
125
|
+
* @throws 取消时抛错并标记 canceled
|
|
126
|
+
*/
|
|
127
|
+
async function awaitTerminal(taskId, { timeoutMs, signal }) {
|
|
128
|
+
const deadline = Date.now() + timeoutMs;
|
|
129
|
+
for (;;) {
|
|
130
|
+
if (signal && signal.aborted) {
|
|
131
|
+
tasks.cancel(taskId, '用户取消');
|
|
132
|
+
throw new Error('已取消');
|
|
133
|
+
}
|
|
134
|
+
const t = tasks.get(taskId);
|
|
135
|
+
if (t && t.status !== 'running') return t;
|
|
136
|
+
if (Date.now() > deadline) return null;
|
|
137
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/* ---------------- 图片 data URL(视觉工具共用) ---------------- */
|
|
142
|
+
|
|
143
|
+
function toDataUrl(absPath, mediaType) {
|
|
144
|
+
const buf = fs.readFileSync(absPath);
|
|
145
|
+
return `data:${mediaType};base64,${buf.toString('base64')}`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function fmtTask(t) {
|
|
149
|
+
const lines = [`[${t.cap}] ${t.id} · ${t.status}${t.progress ? ' · ' + t.progress : ''} · ${t.model}`];
|
|
150
|
+
lines.push('prompt: ' + String(t.prompt || '').slice(0, 120));
|
|
151
|
+
if (t.error) lines.push('error: ' + t.error);
|
|
152
|
+
if ((t.files || []).length) lines.push('files:\n' + t.files.map((f) => path.join(tasks.outputsDir(), f)).join('\n'));
|
|
153
|
+
const links = mediaLinksOf(t);
|
|
154
|
+
if (links.length) lines.push('links:\n' + links.join('\n'));
|
|
155
|
+
if (t.remoteTaskId) lines.push('remoteTask: ' + t.remoteTaskId);
|
|
156
|
+
return lines.join('\n');
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export const name = 'dsh-iris';
|
|
160
|
+
export const inject = ['tools'];
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* 隔离单点失败:任何非核心环节(导入/恢复/路由/单个工具注册)抛错
|
|
164
|
+
* 都只记日志,绝不让插件条目 reject —— 历史教训:iris 一次直读未 inject
|
|
165
|
+
* 的服务就把整个宿主炸掉(见 dsh-web.log 的 boot 崩溃)。这里一律不许再犯。
|
|
166
|
+
*/
|
|
167
|
+
function guard(label, fn) {
|
|
168
|
+
try {
|
|
169
|
+
return fn();
|
|
170
|
+
} catch (err) {
|
|
171
|
+
console.error(`[iris] ${label} 失败(已隔离,不影响宿主):`, err && err.message || err);
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** 单工具注册隔离:注册失败的收益是诊断日志,而不是宿主陪葬 */
|
|
177
|
+
function safeRegister(ctx, definition) {
|
|
178
|
+
guard(`工具注册 ${definition && definition.name}`, () => {
|
|
179
|
+
ctx.effect(() => ctx.tools.register(definition));
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export async function apply(ctx) {
|
|
184
|
+
|
|
185
|
+
guard('上传临时文件清理', () => purgeStaleUploads());
|
|
186
|
+
|
|
187
|
+
// 首次运行:从工作台导入(幂等)
|
|
188
|
+
guard('工作台导入', () => {
|
|
189
|
+
const wb = path.join(process.env.HOME || os.homedir(), 'projects', 'ai-paint', 'data', 'config.json');
|
|
190
|
+
const imported = store.importFromWorkbench(wb);
|
|
191
|
+
if (imported.imported) console.log(`[iris] 已从工作台导入 ${imported.imported} 个服务商`);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
// 接管上次进程未跑完的异步任务(恢复循环本身在 tasks.js 里逐任务防御)
|
|
195
|
+
guard('恢复后台任务', () => {
|
|
196
|
+
const resumed = tasks.resumePending((t) => {
|
|
197
|
+
const p = store.providerById(t.providerId);
|
|
198
|
+
return p ? pollDeps(p, t.cap) : null;
|
|
199
|
+
});
|
|
200
|
+
if (resumed.length) console.log(`[iris] 已恢复接管 ${resumed.length} 个后台任务`);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
// Fiber 清理:插件停用/更新时停掉全部盯守句柄
|
|
204
|
+
guard('盯守句柄注册', () => {
|
|
205
|
+
ctx.effect(() => () => tasks.stopWatchAll());
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
// 媒体路由(视频/音频在对话流里的可播通路;图片仍走原生附件)
|
|
209
|
+
ctx.inject(['webServer'], mountIrisRoutes);
|
|
210
|
+
ctx.inject(['httpServer'], mountIrisRoutes);
|
|
211
|
+
// 若 webServer 已在运行,inject 回调也会触发;这里兜底立即挂载一次
|
|
212
|
+
guard('媒体路由兜底挂载', () => mountIrisRoutes(ctx));
|
|
213
|
+
|
|
214
|
+
const tools = ctx.tools;
|
|
215
|
+
if (!tools || typeof tools.register !== 'function') {
|
|
216
|
+
console.error('[iris] tools 服务不可用,跳过全部工具注册(宿主不受影响)');
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/* ---------- 🖼️ 画图 ---------- */
|
|
221
|
+
safeRegister(ctx, {
|
|
222
|
+
name: 'iris_draw_image',
|
|
223
|
+
description:
|
|
224
|
+
'Generate an image from a detailed prompt using the user-configured image model (DashScope wan* or OpenAI-images-compatible). ' +
|
|
225
|
+
'Returns a durable DSH attachment rendered in the conversation plus a vision-model description so you know what was drawn.',
|
|
226
|
+
parameters: {
|
|
227
|
+
type: 'object',
|
|
228
|
+
properties: {
|
|
229
|
+
prompt: { type: 'string', description: 'Detailed visual description of the image to create' },
|
|
230
|
+
size: { type: 'string', description: "Output size; DashScope uses 'W*H' (e.g. '1024*1024'), OpenAI uses 'WxH'. Default provider default." },
|
|
231
|
+
n: { type: 'string', description: "Number of images, default '1'" },
|
|
232
|
+
model: { type: 'string', description: 'Override with a providerId::modelId reference (legacy bare model id also accepted)' }
|
|
233
|
+
},
|
|
234
|
+
required: ['prompt']
|
|
235
|
+
},
|
|
236
|
+
output: {
|
|
237
|
+
schema: { type: 'object' },
|
|
238
|
+
render: (_args, value) => value.blocks
|
|
239
|
+
},
|
|
240
|
+
isConcurrencySafe: () => false,
|
|
241
|
+
async execute(args, exec) {
|
|
242
|
+
const prompt = String(args.prompt || '').trim();
|
|
243
|
+
if (!prompt) throw new Error('iris: prompt 不能为空');
|
|
244
|
+
const submitted = await runAction(ctx, 'image', args, { signal: exec.signal });
|
|
245
|
+
let final = tasks.get(submitted.taskId);
|
|
246
|
+
if (final && final.status === 'running') {
|
|
247
|
+
final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.image, signal: exec.signal });
|
|
248
|
+
}
|
|
249
|
+
if (!final) {
|
|
250
|
+
return { blocks: [{ type: 'text', text: `[iris] 图像任务仍在后台盯守(task: ${submitted.taskId})。稍后可用 iris_task_status 查询进度与文件路径。` }] };
|
|
251
|
+
}
|
|
252
|
+
if (final.status === 'canceled') throw new Error('已取消');
|
|
253
|
+
if (final.status !== 'succeeded') throw new Error('生成失败:' + (final.error || '未知原因'));
|
|
254
|
+
|
|
255
|
+
const files = (final.files || []).map((f) => path.join(outputDir(), f));
|
|
256
|
+
const blocks = [];
|
|
257
|
+
const refs = [];
|
|
258
|
+
for (const f of files) {
|
|
259
|
+
const mediaType = MEDIA_TYPES[path.extname(f).toLowerCase()] || 'image/png';
|
|
260
|
+
const ref = await attachmentService(ctx).saveImage({ data: new Uint8Array(fs.readFileSync(f)), mediaType, name: path.basename(f) });
|
|
261
|
+
refs.push(ref);
|
|
262
|
+
blocks.push({ type: 'image', attachment: ref });
|
|
263
|
+
}
|
|
264
|
+
tasks.update(final.id, {
|
|
265
|
+
attachments: refs.map((r, i) => ({
|
|
266
|
+
attachmentId: r.attachmentId,
|
|
267
|
+
file: path.basename(files[i]),
|
|
268
|
+
mediaType: MEDIA_TYPES[path.extname(files[i]).toLowerCase()] || 'image/png'
|
|
269
|
+
}))
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
let note = '';
|
|
273
|
+
try {
|
|
274
|
+
const described = await describeFirstImage(ctx, refs[0], files[0], prompt);
|
|
275
|
+
if (described) note = `画面内容:${described}`;
|
|
276
|
+
} catch (_) { /* 增强失败不影响生成结果 */ }
|
|
277
|
+
blocks.unshift({
|
|
278
|
+
type: 'text',
|
|
279
|
+
text: `[iris] 图像已生成(${submitted.model || final.model},task: ${final.id})。` +
|
|
280
|
+
(note ? `\n${note}` : '') +
|
|
281
|
+
`\nattachment: ${refs.map((r) => r.attachmentId).join(', ')}`
|
|
282
|
+
});
|
|
283
|
+
return { blocks };
|
|
284
|
+
}
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
/* ---------- 🎬 视频生成 ---------- */
|
|
288
|
+
safeRegister(ctx, {
|
|
289
|
+
name: 'iris_generate_video',
|
|
290
|
+
description:
|
|
291
|
+
'Generate a video with the configured DashScope video model. Three modes: ' +
|
|
292
|
+
'(1) text-to-video from a prompt (wan* t2v); ' +
|
|
293
|
+
'(2) image-to-video with a first frame — an attachment id returned by iris_draw_image or an absolute local path; ' +
|
|
294
|
+
'(3) s2v digital-human talking video (model wan2.2-s2v): first frame + audio_path (wav/mp3, <20s), local files are auto-uploaded to Bailian temp storage. ' +
|
|
295
|
+
'Submission returns fast; the render is watched in the background — if it exceeds the inline wait you get a task id for iris_task_status.',
|
|
296
|
+
parameters: {
|
|
297
|
+
type: 'object',
|
|
298
|
+
properties: {
|
|
299
|
+
prompt: { type: 'string', description: 'Detailed description of the motion/scene for the video' },
|
|
300
|
+
first_frame_attachment_id: { type: 'string', description: 'Attachment id of an image generated by iris_draw_image, used as the first frame' },
|
|
301
|
+
first_frame_path: { type: 'string', description: 'Absolute path to a local image used as the first frame (alternative to attachment id)' },
|
|
302
|
+
audio_path: { type: 'string', description: 'Absolute path to a wav/mp3 file (<15MB, <20s, clear human voice). Required for s2v models like wan2.2-s2v' },
|
|
303
|
+
resolution: { type: 'string', description: "s2v only: output tier '480P' (default) or '720P'" },
|
|
304
|
+
size: { type: 'string', description: "t2v/i2v only: output size 'W*H', e.g. '1280*720' (default)" },
|
|
305
|
+
duration: { type: 'number', description: 't2v/i2v only: duration in seconds, if the model supports it' },
|
|
306
|
+
model: { type: 'string', description: 'Override with a providerId::modelId reference (legacy bare model id accepted)' }
|
|
307
|
+
},
|
|
308
|
+
required: ['prompt']
|
|
309
|
+
},
|
|
310
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
311
|
+
isConcurrencySafe: () => false,
|
|
312
|
+
async execute(args, exec) {
|
|
313
|
+
const submitted = await runAction(ctx, 'video', args, { signal: exec.signal });
|
|
314
|
+
const final = await awaitTerminal(submitted.taskId, { timeoutMs: AWAIT_MS.video, signal: exec.signal });
|
|
315
|
+
if (!final) {
|
|
316
|
+
return `[iris] 视频任务已提交并转入后台盯守(task: ${submitted.taskId},remoteTask: ${submitted.remoteTaskId})。\n` +
|
|
317
|
+
'渲染通常需要数分钟;完成后用 iris_task_status 查询产物路径。';
|
|
318
|
+
}
|
|
319
|
+
if (final.status === 'canceled') throw new Error('已取消');
|
|
320
|
+
if (final.status !== 'succeeded') throw new Error('视频生成失败:' + (final.error || '未知原因'));
|
|
321
|
+
const paths = (final.files || []).map((f) => path.join(outputDir(), f));
|
|
322
|
+
const links = mediaLinksOf(final);
|
|
323
|
+
const secs = Math.round((Date.now() - new Date(final.createdAt).getTime()) / 1000);
|
|
324
|
+
return `[iris] 视频已生成(${submitted.model || final.model},${final.mode || 't2v'},约 ${secs}s):\n` +
|
|
325
|
+
paths.join('\n') +
|
|
326
|
+
(links.length ? `\n\n${links.join('\n')}\n(点击即在本机浏览器播放)` : '') +
|
|
327
|
+
`\ntask: ${final.id}`;
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
/* ---------- 🔊 语音合成 ---------- */
|
|
332
|
+
safeRegister(ctx, {
|
|
333
|
+
name: 'iris_speak_text',
|
|
334
|
+
description: 'Synthesize speech audio from text using the configured TTS model (qwen-tts / compatible). Returns the saved audio file path.',
|
|
335
|
+
parameters: {
|
|
336
|
+
type: 'object',
|
|
337
|
+
properties: {
|
|
338
|
+
text: { type: 'string', description: 'Text to speak' },
|
|
339
|
+
voice: { type: 'string', description: "Voice name, e.g. 'Cherry' (DashScope qwen-tts voices)" },
|
|
340
|
+
model: { type: 'string', description: 'Override with a providerId::modelId reference (legacy bare model id accepted)' }
|
|
341
|
+
},
|
|
342
|
+
required: ['text']
|
|
343
|
+
},
|
|
344
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
345
|
+
isConcurrencySafe: () => true,
|
|
346
|
+
async execute(args, exec) {
|
|
347
|
+
const submitted = await runAction(ctx, 'tts', args, { signal: exec.signal });
|
|
348
|
+
const final = tasks.get(submitted.taskId);
|
|
349
|
+
if (!final || final.status !== 'succeeded') throw new Error('语音合成失败:' + ((final && final.error) || '未知原因'));
|
|
350
|
+
const fileName = (final.files || [])[0];
|
|
351
|
+
const p = path.join(outputDir(), fileName);
|
|
352
|
+
const media = (final.media || [])[0];
|
|
353
|
+
return `[iris] 语音已合成(${submitted.model || final.model}):${p}\ntask: ${final.id}` +
|
|
354
|
+
(media ? `\n[♪ 音频播放](${mediaLinksOf(final)[0]})` : '');
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
/* ---------- 🎙️ 音频转写(阶段 7.2) ---------- */
|
|
359
|
+
safeRegister(ctx, {
|
|
360
|
+
name: 'iris_transcribe_audio',
|
|
361
|
+
description: 'Transcribe an audio file (wav/mp3) to text using the configured DashScope provider (qwen-audio-3.0-asr-flash-filetrans). The audio is uploaded to Bailian temp storage, then an async task is submitted. Returns the full recognized text.',
|
|
362
|
+
parameters: {
|
|
363
|
+
type: 'object',
|
|
364
|
+
properties: {
|
|
365
|
+
audio_path: { type: 'string', description: 'Absolute path to the audio file (wav/mp3)' },
|
|
366
|
+
model: { type: 'string', description: 'Override with a providerId::modelId transcription reference (legacy bare model id accepted)' }
|
|
367
|
+
},
|
|
368
|
+
required: ['audio_path']
|
|
369
|
+
},
|
|
370
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
371
|
+
isConcurrencySafe: () => false,
|
|
372
|
+
async execute(args, exec) {
|
|
373
|
+
const submitted = await runAction(ctx, 'transcribe', args, { signal: exec.signal });
|
|
374
|
+
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
375
|
+
if (!final) return `[iris] 音频转写任务已提交并转入后台(task: ${submitted.taskId})。完成后用 iris_task_status 查询。`;
|
|
376
|
+
if (final.status === 'canceled') throw new Error('已取消');
|
|
377
|
+
if (final.status !== 'succeeded') throw new Error('转写失败:' + (final.error || '未知原因'));
|
|
378
|
+
return `[iris] 音频转写完成(${submitted.model || final.model}):\n${final.transcribeText || ''}`;
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
/* ---------- 🎞️ 视频抽帧(阶段 7.1,ffmpeg 可选) ---------- */
|
|
383
|
+
safeRegister(ctx, {
|
|
384
|
+
name: 'iris_video_frames',
|
|
385
|
+
description:
|
|
386
|
+
'Extract N frames (uniformly sampled across the video) from a local video file using the system ffmpeg. ' +
|
|
387
|
+
'Frames are scaled to a target width, saved as durable DSH attachments (jpeg/png), and returned as image blocks ' +
|
|
388
|
+
'so you can inspect the video content. Requires ffmpeg + ffprobe on PATH (optional system dependency; other iris tools are unaffected).',
|
|
389
|
+
parameters: {
|
|
390
|
+
type: 'object',
|
|
391
|
+
properties: {
|
|
392
|
+
video_path: { type: 'string', description: 'Absolute path to the local video file (mp4/webm/mov etc.)' },
|
|
393
|
+
max_frames: { type: 'integer', description: 'Max frames to extract, clamped to 1..20 (default 8)' },
|
|
394
|
+
target_width: { type: 'integer', description: 'Scale frames to this width, keeping aspect ratio (default 640, max 4096)' },
|
|
395
|
+
format: { type: 'string', enum: ['jpeg', 'png'], description: 'Output frame format (default jpeg)' },
|
|
396
|
+
quality: { type: 'integer', description: 'JPEG quality 1-100 (default 85)' }
|
|
397
|
+
},
|
|
398
|
+
required: ['video_path']
|
|
399
|
+
},
|
|
400
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
401
|
+
isConcurrencySafe: () => false,
|
|
402
|
+
async execute(args, exec) {
|
|
403
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
404
|
+
const p = String(args.video_path || '').trim();
|
|
405
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
|
|
406
|
+
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法抽帧(可选安装 ffmpeg/ffprobe 后启用;不影响其他 iris 工具)');
|
|
407
|
+
const meta = probeVideo(p);
|
|
408
|
+
const frames = await extractFrames({
|
|
409
|
+
inputPath: p,
|
|
410
|
+
maxFrames: args.max_frames,
|
|
411
|
+
targetWidth: args.target_width,
|
|
412
|
+
format: args.format,
|
|
413
|
+
quality: args.quality,
|
|
414
|
+
signal: exec.signal
|
|
415
|
+
});
|
|
416
|
+
const mediaType = args.format === 'png' ? 'image/png' : 'image/jpeg';
|
|
417
|
+
const ext = args.format === 'png' ? 'png' : 'jpg';
|
|
418
|
+
const blocks = [];
|
|
419
|
+
const refs = [];
|
|
420
|
+
for (let i = 0; i < frames.length; i++) {
|
|
421
|
+
const f = frames[i];
|
|
422
|
+
const ref = await attachmentService(ctx).saveImage({
|
|
423
|
+
data: new Uint8Array(f.buffer),
|
|
424
|
+
mediaType,
|
|
425
|
+
name: `iris-frame-${Date.now()}-${i + 1}.${ext}`
|
|
426
|
+
});
|
|
427
|
+
refs.push(ref);
|
|
428
|
+
blocks.push({ type: 'image', attachment: ref });
|
|
429
|
+
}
|
|
430
|
+
const ats = frames.map((f) => f.atSec.toFixed(1) + 's').join(', ');
|
|
431
|
+
blocks.unshift({
|
|
432
|
+
type: 'text',
|
|
433
|
+
text:
|
|
434
|
+
`[iris] 视频抽帧完成:${frames.length} 帧(${meta.durationSec.toFixed(1)}s 视频,` +
|
|
435
|
+
`${frames[0].width}x${frames[0].height},${mediaType === 'image/png' ? 'PNG' : 'JPEG'})。\n` +
|
|
436
|
+
`时间戳:${ats}\n` +
|
|
437
|
+
`attachments: ${refs.map((r) => r.attachmentId).join(', ')}`
|
|
438
|
+
});
|
|
439
|
+
return { blocks };
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
/* ---------- 📝 多模态视频摘要(阶段 7.3,ffmpeg 可选) ---------- */
|
|
444
|
+
safeRegister(ctx, {
|
|
445
|
+
name: 'iris_media_summarize',
|
|
446
|
+
description:
|
|
447
|
+
'Summarize a local video by compressing it into a contact sheet of sampled frames (with timestamps) plus an optional ' +
|
|
448
|
+
'auto-transcribed audio track, then asking the vision model to describe content. Returns the summary text plus the contact sheet as a DSH image attachment. ' +
|
|
449
|
+
'Requires ffmpeg + ffprobe on PATH. Transcription reuses the configured provider (qwen-audio-3.0-asr-flash-filetrans) when the video has an audio track.',
|
|
450
|
+
parameters: {
|
|
451
|
+
type: 'object',
|
|
452
|
+
properties: {
|
|
453
|
+
video_path: { type: 'string', description: 'Absolute path to the local video file (mp4/webm/mov etc.)' },
|
|
454
|
+
question: { type: 'string', description: 'Custom question about the video content (default: summarize scenes/theme)' },
|
|
455
|
+
max_frames: { type: 'integer', description: 'Frames to sample for the contact sheet, clamped to 1..12 (default 8)' },
|
|
456
|
+
target_width: { type: 'integer', description: 'Frame width for sampling (default 640)' },
|
|
457
|
+
transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track if present (default true)' },
|
|
458
|
+
model: { type: 'string', description: 'Override the configured vision model id' }
|
|
459
|
+
},
|
|
460
|
+
required: ['video_path']
|
|
461
|
+
},
|
|
462
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
463
|
+
isConcurrencySafe: () => false,
|
|
464
|
+
async execute(args, exec) {
|
|
465
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
466
|
+
const p = String(args.video_path || '').trim();
|
|
467
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
|
|
468
|
+
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法分析视频(可选安装 ffmpeg/ffprobe 后启用)');
|
|
469
|
+
const meta = probeVideo(p);
|
|
470
|
+
const frames = await extractFrames({
|
|
471
|
+
inputPath: p,
|
|
472
|
+
maxFrames: Math.min(Number(args.max_frames) || 8, 12),
|
|
473
|
+
targetWidth: args.target_width,
|
|
474
|
+
format: 'jpeg',
|
|
475
|
+
quality: 70,
|
|
476
|
+
signal: exec.signal
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
// 音轨转写(可选;与独立 transcribe capability 共用同一动作与 failover)
|
|
480
|
+
let transcript = '';
|
|
481
|
+
let transcriptNote = '';
|
|
482
|
+
const wantTranscribe = args.transcribe !== false && meta.hasAudio;
|
|
483
|
+
if (wantTranscribe) {
|
|
484
|
+
if (store.pickAllFor(cap.CAPABILITIES.TRANSCRIBE).length) {
|
|
485
|
+
try {
|
|
486
|
+
const audio = await extractAudioTrack({ inputPath: p, signal: exec.signal });
|
|
487
|
+
try {
|
|
488
|
+
const submitted = await runAction(ctx, 'transcribe', { audio_path: audio.filePath }, { signal: exec.signal });
|
|
489
|
+
const final = await awaitTerminal(submitted.taskId, { timeoutMs: 120000, signal: exec.signal });
|
|
490
|
+
if (final && final.status === 'succeeded') transcript = final.transcribeText || '';
|
|
491
|
+
else if (!final) transcriptNote = `\n(音轨转写仍在后台:task ${submitted.taskId},本次摘要未含语音内容)`;
|
|
492
|
+
} finally {
|
|
493
|
+
fs.rmSync(audio.outDir, { recursive: true, force: true });
|
|
494
|
+
}
|
|
495
|
+
} catch (err) {
|
|
496
|
+
transcriptNote = '\n(音轨转写失败,本次摘要仅画面:' + String(err.message || err).slice(0, 160) + ')';
|
|
497
|
+
}
|
|
498
|
+
} else {
|
|
499
|
+
transcriptNote = '\n(视频含音轨但未配置转写供应商,本次摘要仅画面)';
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// contact sheet → attachment → 视觉摘要
|
|
504
|
+
const sheet = await buildContactSheet({ frames, cols: 3 });
|
|
505
|
+
const ref = await attachmentService(ctx).saveImage({
|
|
506
|
+
data: new Uint8Array(sheet.buffer),
|
|
507
|
+
mediaType: 'image/png',
|
|
508
|
+
name: `iris-sheet-${Date.now()}.png`
|
|
509
|
+
});
|
|
510
|
+
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
511
|
+
const backends = buildVisionBackends(ctx, { providers: visionProviders, model: args.model });
|
|
512
|
+
const { answer, via, model } = await summarizeMedia({
|
|
513
|
+
backends, question: args.question, transcript,
|
|
514
|
+
imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`,
|
|
515
|
+
ref, signal: exec.signal
|
|
516
|
+
});
|
|
517
|
+
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
518
|
+
return {
|
|
519
|
+
blocks: [
|
|
520
|
+
{
|
|
521
|
+
type: 'text',
|
|
522
|
+
text:
|
|
523
|
+
`[iris] 视频摘要完成(${model} · ${viaLabel},${meta.durationSec.toFixed(1)}s / ${frames.length} 帧联系表):\n` +
|
|
524
|
+
`${answer}${transcriptNote || ''}\n` +
|
|
525
|
+
`contact sheet attachment: ${ref.attachmentId}`
|
|
526
|
+
},
|
|
527
|
+
{ type: 'image', attachment: ref }
|
|
528
|
+
]
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
/* ---------- 👁 视觉路由(M3):显式工具,自持栈为主 ---------- */
|
|
534
|
+
|
|
535
|
+
safeRegister(ctx, {
|
|
536
|
+
name: 'iris_look_at_image',
|
|
537
|
+
description:
|
|
538
|
+
'Look at a local image file and answer one question about it with the configured vision model (qwen-vl via the self-hosted provider stack; falls back to the global DSH vision model). The image is saved as a durable attachment so follow-ups can reference it.',
|
|
539
|
+
parameters: {
|
|
540
|
+
type: 'object',
|
|
541
|
+
properties: {
|
|
542
|
+
image_path: { type: 'string', description: 'Absolute path to the image file (png/jpg/jpeg/webp)' },
|
|
543
|
+
question: { type: 'string', description: 'Question or extraction request about the image; defaults to a detailed description' },
|
|
544
|
+
model: { type: 'string', description: 'Override the configured vision model id (default qwen-vl-plus)' }
|
|
545
|
+
},
|
|
546
|
+
required: ['image_path']
|
|
547
|
+
},
|
|
548
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
549
|
+
isConcurrencySafe: () => true,
|
|
550
|
+
async execute(args, exec) {
|
|
551
|
+
const p = String(args.image_path || '').trim();
|
|
552
|
+
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
553
|
+
const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
|
|
554
|
+
if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp):' + p);
|
|
555
|
+
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在:' + p);
|
|
556
|
+
const question = String(args.question || '').trim() || '详细描述这张图片。';
|
|
557
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
558
|
+
|
|
559
|
+
const dataUrl = toDataUrl(p, mt);
|
|
560
|
+
const ref = await attachmentService(ctx).saveImage({ data: new Uint8Array(fs.readFileSync(p)), mediaType: mt, name: path.basename(p) });
|
|
561
|
+
return runVisionTool(ctx, exec, { origin: 'tool', model: args.model, question, ref, dataUrl });
|
|
562
|
+
}
|
|
563
|
+
});
|
|
564
|
+
|
|
565
|
+
safeRegister(ctx, {
|
|
566
|
+
name: 'iris_relook_attachment',
|
|
567
|
+
description:
|
|
568
|
+
'Ask a NEW question about an image that was already seen in this session: pass its attachment_id (from user uploads, tool results, or any iris-generated image). Pixels are re-read via the vision model.',
|
|
569
|
+
parameters: {
|
|
570
|
+
type: 'object',
|
|
571
|
+
properties: {
|
|
572
|
+
attachment_id: { type: 'string', description: 'The attachment_id of an image that appeared in this session or was generated by iris' },
|
|
573
|
+
question: { type: 'string', description: 'New question about the image pixels' },
|
|
574
|
+
model: { type: 'string', description: 'Override the configured vision model id' }
|
|
575
|
+
},
|
|
576
|
+
required: ['attachment_id', 'question']
|
|
577
|
+
},
|
|
578
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
579
|
+
isConcurrencySafe: () => true,
|
|
580
|
+
async execute(args, exec) {
|
|
581
|
+
const id = String(args.attachment_id || '').trim();
|
|
582
|
+
if (!id) throw new Error('iris: attachment_id 不能为空');
|
|
583
|
+
const question = String(args.question || '').trim();
|
|
584
|
+
if (!question) throw new Error('iris: question 不能为空');
|
|
585
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
586
|
+
|
|
587
|
+
// 定位:本会话出现过的图片优先,其次本插件产物记录
|
|
588
|
+
const hit = (await sessionAttachmentRef(ctx, exec, id)) || findOwnAttachment(id);
|
|
589
|
+
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
590
|
+
|
|
591
|
+
let dataUrl;
|
|
592
|
+
if (hit.absPath) {
|
|
593
|
+
dataUrl = toDataUrl(hit.absPath, hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png');
|
|
594
|
+
} else {
|
|
595
|
+
const stored = await attachmentService(ctx).readImage(hit.ref, exec.signal);
|
|
596
|
+
dataUrl = `data:${stored.mediaType || hit.ref.mediaType};base64,${Buffer.from(stored.data).toString('base64')}`;
|
|
597
|
+
}
|
|
598
|
+
return runVisionTool(ctx, exec, { origin: 'relook', model: args.model, question, ref: hit.ref, dataUrl });
|
|
599
|
+
}
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
/* ---------- 📋 任务查询 ---------- */
|
|
603
|
+
safeRegister(ctx, {
|
|
604
|
+
name: 'iris_task_status',
|
|
605
|
+
description:
|
|
606
|
+
'Query iris generation tasks (image/video/tts). Pass a task_id for one task, or omit it to list the latest tasks with status, progress, errors and output file paths.',
|
|
607
|
+
parameters: {
|
|
608
|
+
type: 'object',
|
|
609
|
+
properties: {
|
|
610
|
+
task_id: { type: 'string', description: 'Task id, e.g. from a background-handoff notice; omit to list recent tasks' }
|
|
611
|
+
},
|
|
612
|
+
required: []
|
|
613
|
+
},
|
|
614
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
615
|
+
isConcurrencySafe: () => true,
|
|
616
|
+
async execute(args) {
|
|
617
|
+
if (args.task_id) {
|
|
618
|
+
const t = tasks.get(String(args.task_id).trim());
|
|
619
|
+
if (!t) throw new Error('iris: 没有这个任务:' + args.task_id);
|
|
620
|
+
return fmtTask(t);
|
|
621
|
+
}
|
|
622
|
+
const recent = tasks.list().slice(0, 5);
|
|
623
|
+
if (!recent.length) return '[iris] 还没有生成任务记录。';
|
|
624
|
+
return recent.map(fmtTask).join('\n---\n');
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
/* ---------- ✂️ 确定性像素工具(阶段 2) ---------- */
|
|
629
|
+
safeRegister(ctx, {
|
|
630
|
+
name: 'iris_crop',
|
|
631
|
+
description:
|
|
632
|
+
'Crop a rectangular region from an image (absolute local path or a session attachment_id) and save the result as a durable DSH attachment. ' +
|
|
633
|
+
'Returns the new attachment id and the cropped dimensions.',
|
|
634
|
+
parameters: {
|
|
635
|
+
type: 'object',
|
|
636
|
+
properties: {
|
|
637
|
+
image_path: { type: 'string', description: 'Absolute path to the source image (png/jpg/jpeg/webp)' },
|
|
638
|
+
attachment_id: { type: 'string', description: 'Attachment id of an image seen in this session or generated by iris (alternative to image_path)' },
|
|
639
|
+
left: { type: 'integer', description: 'Left edge x, 0-based, in pixels' },
|
|
640
|
+
top: { type: 'integer', description: 'Top edge y, 0-based, in pixels' },
|
|
641
|
+
width: { type: 'integer', description: 'Crop width in pixels (must be positive)' },
|
|
642
|
+
height: { type: 'integer', description: 'Crop height in pixels (must be positive)' }
|
|
643
|
+
},
|
|
644
|
+
required: ['left', 'top', 'width', 'height']
|
|
645
|
+
},
|
|
646
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
647
|
+
isConcurrencySafe: () => true,
|
|
648
|
+
async execute(args, exec) {
|
|
649
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
650
|
+
const input = await resolveImageInput(ctx, exec, {
|
|
651
|
+
image_path: args.image_path,
|
|
652
|
+
attachment_id: args.attachment_id
|
|
653
|
+
});
|
|
654
|
+
const { buffer, width, height, mime } = await cropImage({
|
|
655
|
+
input: input.buffer,
|
|
656
|
+
left: args.left,
|
|
657
|
+
top: args.top,
|
|
658
|
+
width: args.width,
|
|
659
|
+
height: args.height
|
|
660
|
+
});
|
|
661
|
+
const ref = await attachmentService(ctx).saveImage({
|
|
662
|
+
data: new Uint8Array(buffer),
|
|
663
|
+
mediaType: mime,
|
|
664
|
+
name: `iris-crop-${Date.now()}.png`
|
|
665
|
+
});
|
|
666
|
+
return {
|
|
667
|
+
blocks: [
|
|
668
|
+
{ type: 'text', text: `[iris] 裁剪完成:${width}x${height}(原区域 ${args.left},${args.top},${args.width},${args.height})\nattachment: ${ref.attachmentId}` },
|
|
669
|
+
{ type: 'image', attachment: ref }
|
|
670
|
+
]
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
safeRegister(ctx, {
|
|
676
|
+
name: 'iris_pixel_diff',
|
|
677
|
+
description:
|
|
678
|
+
'Compute a pixel-level difference between two images (absolute local paths or session attachment_ids). ' +
|
|
679
|
+
'Images of different sizes are normalized to the smaller one. ' +
|
|
680
|
+
'Returns the diff ratio (0-1), the worst regions on an 8x8 grid, and saves a heatmap PNG as a durable DSH attachment.',
|
|
681
|
+
parameters: {
|
|
682
|
+
type: 'object',
|
|
683
|
+
properties: {
|
|
684
|
+
image_a_path: { type: 'string', description: 'Absolute path to the first image' },
|
|
685
|
+
image_b_path: { type: 'string', description: 'Absolute path to the second image' },
|
|
686
|
+
attachment_a_id: { type: 'string', description: 'Session attachment id of the first image (alternative to image_a_path)' },
|
|
687
|
+
attachment_b_id: { type: 'string', description: 'Session attachment id of the second image (alternative to image_b_path)' },
|
|
688
|
+
grid: { type: 'integer', description: 'Grid size for worst-region analysis, default 8' },
|
|
689
|
+
top_regions: { type: 'integer', description: 'How many worst regions to report, default 3' }
|
|
690
|
+
},
|
|
691
|
+
required: []
|
|
692
|
+
},
|
|
693
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
694
|
+
isConcurrencySafe: () => true,
|
|
695
|
+
async execute(args, exec) {
|
|
696
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
697
|
+
if (!(args.image_a_path || args.attachment_a_id)) throw new Error('iris: 需要 image_a_path 或 attachment_a_id');
|
|
698
|
+
if (!(args.image_b_path || args.attachment_b_id)) throw new Error('iris: 需要 image_b_path 或 attachment_b_id');
|
|
699
|
+
const a = await resolveImageInput(ctx, exec, { image_path: args.image_a_path, attachment_id: args.attachment_a_id });
|
|
700
|
+
const b = await resolveImageInput(ctx, exec, { image_path: args.image_b_path, attachment_id: args.attachment_b_id });
|
|
701
|
+
const { ratio, diffPixels, totalPixels, width, height, worstRegions, heatmap, mime } = await pixelDiff({
|
|
702
|
+
inputA: a.buffer,
|
|
703
|
+
inputB: b.buffer,
|
|
704
|
+
grid: args.grid || 8,
|
|
705
|
+
topRegions: args.top_regions || 3
|
|
706
|
+
});
|
|
707
|
+
const ref = await attachmentService(ctx).saveImage({
|
|
708
|
+
data: new Uint8Array(heatmap),
|
|
709
|
+
mediaType: mime,
|
|
710
|
+
name: `iris-diff-${Date.now()}.png`
|
|
711
|
+
});
|
|
712
|
+
const pct = (ratio * 100).toFixed(2);
|
|
713
|
+
const regions = worstRegions.map((r) => `(col ${r.col},row ${r.row}) ${(r.score * 100).toFixed(1)}%`).join(' ');
|
|
714
|
+
return {
|
|
715
|
+
blocks: [
|
|
716
|
+
{
|
|
717
|
+
type: 'text',
|
|
718
|
+
text: `[iris] 像素差异 ${pct}%(${diffPixels}/${totalPixels}px,归一化 ${width}x${height})\n` +
|
|
719
|
+
`最差区域(${worstRegions.length} 格): ${regions}\n` +
|
|
720
|
+
`热力图 attachment: ${ref.attachmentId}`
|
|
721
|
+
},
|
|
722
|
+
{ type: 'image', attachment: ref }
|
|
723
|
+
]
|
|
724
|
+
};
|
|
725
|
+
}
|
|
726
|
+
});
|
|
727
|
+
|
|
728
|
+
/* ---------- 📍 模型驱动定位(阶段 3A) ---------- */
|
|
729
|
+
safeRegister(ctx, {
|
|
730
|
+
name: 'iris_locate',
|
|
731
|
+
description:
|
|
732
|
+
'Locate an object/region described by `target` in an image (absolute path or session attachment_id), ' +
|
|
733
|
+
'returning the pixel bounding box (x1,y1,x2,y2) in original image coordinates. ' +
|
|
734
|
+
'Use the returned bbox with iris_crop to extract the region.',
|
|
735
|
+
parameters: {
|
|
736
|
+
type: 'object',
|
|
737
|
+
properties: {
|
|
738
|
+
image_path: { type: 'string', description: 'Absolute path to the source image (png/jpg/jpeg/webp)' },
|
|
739
|
+
attachment_id: { type: 'string', description: 'Session attachment id of an image (alternative to image_path)' },
|
|
740
|
+
target: { type: 'string', description: 'The object or UI element to locate, e.g. "send button"' },
|
|
741
|
+
model: { type: 'string', description: 'Override the vision model for this locate call (e.g. qwen3-vl-235b-a22b-thinking for higher grounding precision; default is the provider visionModel, typically qwen-vl-plus)' }
|
|
742
|
+
},
|
|
743
|
+
required: ['target']
|
|
744
|
+
},
|
|
745
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
746
|
+
isConcurrencySafe: () => true,
|
|
747
|
+
async execute(args, exec) {
|
|
748
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
749
|
+
const input = await resolveImageInput(ctx, exec, {
|
|
750
|
+
image_path: args.image_path,
|
|
751
|
+
attachment_id: args.attachment_id
|
|
752
|
+
});
|
|
753
|
+
const { width, height } = await imageDimensions(input.buffer);
|
|
754
|
+
const dataUrl = `data:${input.mediaType};base64,${input.buffer.toString('base64')}`;
|
|
755
|
+
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
756
|
+
const backends = buildVisionBackends(ctx, { providers: visionProviders, model: args.model });
|
|
757
|
+
const r = await locateObject(backends, {
|
|
758
|
+
target: args.target,
|
|
759
|
+
imageDataUrl: dataUrl,
|
|
760
|
+
width,
|
|
761
|
+
height,
|
|
762
|
+
signal: exec.signal
|
|
763
|
+
});
|
|
764
|
+
if (!r.found) return `[iris] 在图片中未找到「${args.target}」(${r.via} · ${r.model})`;
|
|
765
|
+
return `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})\n` +
|
|
766
|
+
`裁剪指令:iris_crop(image_path="${args.image_path || args.attachment_id}", left=${r.x1}, top=${r.y1}, width=${r.x2 - r.x1}, height=${r.y2 - r.y1})`;
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
/* ---------- 🖼️ HTML 截图(阶段 3C,基于 dsh-builtin-browser) ---------- */
|
|
771
|
+
safeRegister(ctx, {
|
|
772
|
+
name: 'iris_html_screenshot',
|
|
773
|
+
description:
|
|
774
|
+
'Render an HTML string to a page screenshot using the shared browser. ' +
|
|
775
|
+
'Creates a temporary HTML file, opens it in the browser via the host web server, ' +
|
|
776
|
+
'captures a full-page screenshot, saves the PNG as a durable DSH attachment, ' +
|
|
777
|
+
'and cleans up the temporary files. ' +
|
|
778
|
+
'Requires the dsh-builtin-browser plugin to be enabled.',
|
|
779
|
+
parameters: {
|
|
780
|
+
type: 'object',
|
|
781
|
+
properties: {
|
|
782
|
+
html: { type: 'string', description: 'Raw HTML content to render' },
|
|
783
|
+
width: { type: 'integer', description: 'Minimum container width in pixels (optional, advisory — no viewport control)' },
|
|
784
|
+
height: { type: 'integer', description: 'Minimum container height in pixels (optional, advisory)' },
|
|
785
|
+
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport. Default true.' }
|
|
786
|
+
},
|
|
787
|
+
required: ['html']
|
|
788
|
+
},
|
|
789
|
+
output: { schema: { type: 'object' }, render: (_args, value) => value.blocks },
|
|
790
|
+
isConcurrencySafe: () => false,
|
|
791
|
+
async execute(args, exec) {
|
|
792
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
793
|
+
const browser = ctx.get('browser');
|
|
794
|
+
if (!browser) {
|
|
795
|
+
throw new Error('iris: 浏览器服务不可用(ctx.get("browser") 为空)——需要启用 dsh-builtin-browser 插件');
|
|
796
|
+
}
|
|
797
|
+
const renderer = new BrowserHtmlRenderer({ browser });
|
|
798
|
+
const { png } = await renderer.render({
|
|
799
|
+
html: args.html,
|
|
800
|
+
width: args.width,
|
|
801
|
+
height: args.height,
|
|
802
|
+
fullPage: args.fullPage !== false
|
|
803
|
+
});
|
|
804
|
+
const { width: outW, height: outH } = await imageDimensions(png);
|
|
805
|
+
const ref = await attachmentService(ctx).saveImage({
|
|
806
|
+
data: new Uint8Array(png),
|
|
807
|
+
mediaType: 'image/png',
|
|
808
|
+
name: `iris-html-${Date.now()}.png`
|
|
809
|
+
});
|
|
810
|
+
return {
|
|
811
|
+
blocks: [
|
|
812
|
+
{
|
|
813
|
+
type: 'text',
|
|
814
|
+
text: `[iris] HTML 截图完成(${outW}x${outH},fullPage: ${args.fullPage !== false})\n` +
|
|
815
|
+
`attachment: ${ref.attachmentId}` +
|
|
816
|
+
`\n注意:渲染在共享浏览器临时标签页中完成,页面短暂可见属正常行为。`
|
|
817
|
+
},
|
|
818
|
+
{ type: 'image', attachment: ref }
|
|
819
|
+
]
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
});
|
|
823
|
+
|
|
824
|
+
/* ---------- 📄 长截图 OCR(阶段 3B,视觉模型分块) ---------- */
|
|
825
|
+
safeRegister(ctx, {
|
|
826
|
+
name: 'iris_long_ocr',
|
|
827
|
+
description:
|
|
828
|
+
'OCR a (possibly long) image: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
829
|
+
'and read the text chunk by chunk with the vision model, then join the results in top-to-bottom order. ' +
|
|
830
|
+
'Returns the full recognized text plus per-chunk details. Good for long screenshots / documents.',
|
|
831
|
+
parameters: {
|
|
832
|
+
type: 'object',
|
|
833
|
+
properties: {
|
|
834
|
+
image_path: { type: 'string', description: 'Absolute path to the image (png/jpg/jpeg/webp)' },
|
|
835
|
+
attachment_id: { type: 'string', description: 'Session attachment id of the image (alternative to image_path)' },
|
|
836
|
+
chunk_height: { type: 'integer', description: 'Chunk height in pixels, default 1200' },
|
|
837
|
+
overlap: { type: 'integer', description: 'Overlap between chunks in pixels to avoid cutting text lines, default 120' }
|
|
838
|
+
},
|
|
839
|
+
required: []
|
|
840
|
+
},
|
|
841
|
+
output: { schema: { type: 'string' }, render: (_args, v) => [{ type: 'text', text: v }] },
|
|
842
|
+
isConcurrencySafe: () => true,
|
|
843
|
+
async execute(args, exec) {
|
|
844
|
+
if (exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
845
|
+
const input = await resolveImageInput(ctx, exec, {
|
|
846
|
+
image_path: args.image_path,
|
|
847
|
+
attachment_id: args.attachment_id
|
|
848
|
+
});
|
|
849
|
+
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
850
|
+
const backends = buildVisionBackends(ctx, { providers: visionProviders });
|
|
851
|
+
const result = await longOcr({
|
|
852
|
+
input: input.buffer,
|
|
853
|
+
backends,
|
|
854
|
+
chunkHeight: args.chunk_height || 1200,
|
|
855
|
+
overlap: args.overlap || 120,
|
|
856
|
+
signal: exec.signal
|
|
857
|
+
});
|
|
858
|
+
const errCount = result.chunks.filter((c) => c.error).length;
|
|
859
|
+
const head = `[iris] 长截图 OCR 完成:${result.width}x${result.height}px,${result.totalChunks} 块${errCount ? '(' + errCount + ' 块失败)' : ''}\n`;
|
|
860
|
+
if (!result.fullText) return head + '(未识别到文字)';
|
|
861
|
+
return head + result.fullText;
|
|
862
|
+
}
|
|
863
|
+
});
|
|
864
|
+
|
|
865
|
+
console.log('[iris] tools registered: iris_draw_image, iris_generate_video, iris_speak_text, iris_transcribe_audio, iris_task_status, iris_look_at_image, iris_relook_attachment, iris_crop, iris_pixel_diff, iris_locate, iris_html_screenshot, iris_long_ocr, iris_video_frames, iris_media_summarize');
|
|
866
|
+
};
|
|
867
|
+
|
|
868
|
+
/* ---------------- 视觉路由(M3 补全):look/relook 共用执行器 + 降级链 ---------------- */
|
|
869
|
+
|
|
870
|
+
/** 附件服务必须走 ctx.get()(不在 inject 里,直接属性访问会炸掉插件) */
|
|
871
|
+
function attachmentService(ctx) {
|
|
872
|
+
const s = ctx && typeof ctx.get === 'function' ? ctx.get('attachments') : undefined;
|
|
873
|
+
if (!s || typeof s.saveImage !== 'function' || typeof s.readImage !== 'function') {
|
|
874
|
+
throw new Error('iris: 附件服务不可用(ctx.get("attachments") 为空)');
|
|
875
|
+
}
|
|
876
|
+
return s;
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
/**
|
|
880
|
+
* 把「本地绝对路径 或 本会话 attachment_id」解析为图片 buffer。
|
|
881
|
+
* crop / pixel_diff 共用(阶段 2)。
|
|
882
|
+
* @returns {Promise<{buffer:Buffer, mediaType:string, name:string}>}
|
|
883
|
+
*/
|
|
884
|
+
async function resolveImageInput(ctx, exec, { image_path, attachment_id }) {
|
|
885
|
+
if (image_path) {
|
|
886
|
+
const p = String(image_path).trim();
|
|
887
|
+
if (!path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
888
|
+
const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
|
|
889
|
+
if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp):' + p);
|
|
890
|
+
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在:' + p);
|
|
891
|
+
return { buffer: fs.readFileSync(p), mediaType: mt, name: path.basename(p) };
|
|
892
|
+
}
|
|
893
|
+
if (attachment_id) {
|
|
894
|
+
const id = String(attachment_id).trim();
|
|
895
|
+
const hit = (await sessionAttachmentRef(ctx, exec, id)) || findOwnAttachment(id);
|
|
896
|
+
if (!hit) throw new Error('iris: 该 attachment 不在本会话中、也不是 iris 生成的图片:' + id);
|
|
897
|
+
if (hit.absPath) {
|
|
898
|
+
return {
|
|
899
|
+
buffer: fs.readFileSync(hit.absPath),
|
|
900
|
+
mediaType: hit.ref.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png',
|
|
901
|
+
name: path.basename(hit.absPath)
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
const stored = await attachmentService(ctx).readImage(hit.ref, exec && exec.signal);
|
|
905
|
+
return {
|
|
906
|
+
buffer: Buffer.from(stored.data),
|
|
907
|
+
mediaType: stored.mediaType || hit.ref.mediaType || 'image/png',
|
|
908
|
+
name: hit.ref.name || 'image.png'
|
|
909
|
+
};
|
|
910
|
+
}
|
|
911
|
+
throw new Error('iris: 需要 image_path 或 attachment_id');
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/** 在本插件的任务产物里按 attachmentId 找 iris 自生成的图片(relook 的兜底) */
|
|
915
|
+
export function findOwnAttachment(attachmentId) {
|
|
916
|
+
for (const t of tasks.all()) {
|
|
917
|
+
for (const a of t.attachments || []) {
|
|
918
|
+
if (a.attachmentId === attachmentId) {
|
|
919
|
+
const absPath = path.join(outputDir(), a.file);
|
|
920
|
+
if (fs.existsSync(absPath)) {
|
|
921
|
+
return { ref: { attachmentId: a.attachmentId, mediaType: a.mediaType }, absPath };
|
|
922
|
+
}
|
|
923
|
+
return null; // 本地缓存已被清理策略删除,需重新生成
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
return null;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* 防御式扫描当前会话的完整事件日志,找「本会话出现过的图片」附件引用
|
|
932
|
+
* (用户上传、其他工具产物、iris 生成的图都会出现在会话事件里)。
|
|
933
|
+
* 不依赖精确事件类型:递归扫任何载荷中的 attachment 块 / attachmentId 字段。
|
|
934
|
+
* @returns {Promise<{ref:object}|null>} 找到则带可直接 readImage 的 ref
|
|
935
|
+
*/
|
|
936
|
+
export async function sessionAttachmentRef(ctx, exec, attachmentId) {
|
|
937
|
+
const agent = exec && exec.agent;
|
|
938
|
+
const session = agent && agent.session;
|
|
939
|
+
const sq = ctx && typeof ctx.get === 'function' ? ctx.get('sessionQuery') : undefined;
|
|
940
|
+
if (!sq || !session || typeof sq.readSession !== 'function') return null;
|
|
941
|
+
let events;
|
|
942
|
+
try {
|
|
943
|
+
events = (await sq.readSession(session.id)).events;
|
|
944
|
+
} catch (_) {
|
|
945
|
+
return null; // 读不到会话记录 → 交给 findOwnAttachment 兜底
|
|
946
|
+
}
|
|
947
|
+
if (!Array.isArray(events)) return null;
|
|
948
|
+
const wanted = String(attachmentId || '');
|
|
949
|
+
if (!wanted) return null;
|
|
950
|
+
const ref = findAttachmentRefIn(events, wanted);
|
|
951
|
+
return ref ? { ref } : null;
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
/** 递归扫描对象/数组,返回第一个匹配 wanted 的图片附件引用(防循环、限扫描量) */
|
|
955
|
+
function findAttachmentRefIn(nodes, wanted) {
|
|
956
|
+
const seen = new Set();
|
|
957
|
+
const stack = Array.isArray(nodes) ? [...nodes] : [nodes];
|
|
958
|
+
let scanned = 0;
|
|
959
|
+
while (stack.length && scanned < 4000) {
|
|
960
|
+
const node = stack.pop();
|
|
961
|
+
scanned++;
|
|
962
|
+
if (!node || typeof node !== 'object') continue;
|
|
963
|
+
if (seen.has(node)) continue;
|
|
964
|
+
seen.add(node);
|
|
965
|
+
// 形态 1:附件块 { type:'image', attachment:{ attachmentId,... } } —— 最优先
|
|
966
|
+
if (
|
|
967
|
+
node.attachment && typeof node.attachment === 'object' &&
|
|
968
|
+
node.attachment.attachmentId === wanted && typeof node.attachment.mediaType === 'string'
|
|
969
|
+
) {
|
|
970
|
+
return node.attachment;
|
|
971
|
+
}
|
|
972
|
+
// 形态 2:引用对象本身 { attachmentId, mediaType }(兜底)
|
|
973
|
+
if (node.attachmentId === wanted && typeof node.mediaType === 'string') {
|
|
974
|
+
return node;
|
|
975
|
+
}
|
|
976
|
+
for (const v of Object.values(node)) {
|
|
977
|
+
if (v && typeof v === 'object') stack.push(v);
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
return null;
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/**
|
|
984
|
+
* 看图问答(阶段 1:VisionBackend 降级链)。
|
|
985
|
+
* 只取声明/推断出 vision 能力的自持栈 provider(严格选择),按序组装后端链,
|
|
986
|
+
* 全局视觉模型兜底;每个后端的失败现场保留在 errors 里。
|
|
987
|
+
* @returns {{answer:string,via:'selfstack'|'global',model?:string,backendId:string}}
|
|
988
|
+
*/
|
|
989
|
+
export async function askVision(ctx, { question, ref, dataUrl, signal, model }) {
|
|
990
|
+
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
991
|
+
const backends = buildVisionBackends(ctx, { providers: visionProviders, model });
|
|
992
|
+
return askWithBackends(backends, { question, ref, imageDataUrl: dataUrl, signal });
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
/**
|
|
996
|
+
* look / relook 共用执行器:看图问答 → 人话结果(标注模型归属,方便排查走哪条链)。
|
|
997
|
+
* @returns {Promise<string>} 直接作为工具输出
|
|
998
|
+
*/
|
|
999
|
+
export async function runVisionTool(ctx, exec, { origin, model, question, ref, dataUrl }) {
|
|
1000
|
+
if (exec && exec.signal && exec.signal.aborted) throw new Error('已取消');
|
|
1001
|
+
const { answer, via, model: usedModel } = await askVision(ctx, {
|
|
1002
|
+
question,
|
|
1003
|
+
ref,
|
|
1004
|
+
dataUrl,
|
|
1005
|
+
signal: exec && exec.signal,
|
|
1006
|
+
model
|
|
1007
|
+
});
|
|
1008
|
+
const label = origin === 'relook' ? '重看回答' : '看图回答';
|
|
1009
|
+
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
1010
|
+
return `[iris] ${label}(${usedModel || '默认'} · ${viaLabel}):\n${answer}`;
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/* ---------------- 视觉自述(画图后增强项):走 askVision 降级链,失败静默 ---------------- */
|
|
1014
|
+
async function describeFirstImage(ctx, ref, absPath, originalPrompt) {
|
|
1015
|
+
try {
|
|
1016
|
+
const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
|
|
1017
|
+
const { answer } = await askVision(ctx, {
|
|
1018
|
+
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
1019
|
+
ref,
|
|
1020
|
+
dataUrl: toDataUrl(absPath, mt),
|
|
1021
|
+
signal: undefined
|
|
1022
|
+
});
|
|
1023
|
+
return answer.slice(0, 300);
|
|
1024
|
+
} catch (_) {
|
|
1025
|
+
return ''; // 自述是增强项,失败不影响交付
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|