@mokuyoaxis/dsh-iris 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-compose-media/SKILL.md +10 -5
- package/.dsh/skills/iris-compose-media/references/workflows.md +7 -1
- package/.dsh/skills/iris-verify-ui/SKILL.md +2 -1
- package/CHANGELOG.md +99 -1
- package/README.en.md +124 -31
- package/README.md +124 -31
- package/bin/dsh-iris.js +593 -13
- package/docs/AGENT_QUICKSTART.md +39 -0
- package/docs/ARCHITECTURE.md +87 -64
- package/docs/ARTIFACT_LIBRARY.md +5 -5
- package/docs/ARTIFACT_MANIFEST.md +60 -0
- package/docs/CLI_MANAGEMENT.md +181 -0
- package/docs/COMPOSITE_VISION.md +35 -0
- package/docs/CORE_RUNTIME_CONTRACT.md +84 -0
- package/docs/DSH_CORE_MIGRATION.md +135 -0
- package/docs/DSH_RC2_ADAPTATION.md +38 -0
- package/docs/FAULT_INJECTION.md +12 -2
- package/docs/HEADLESS_CLI.md +223 -0
- package/docs/HOST_ADAPTER_CONTRACT.md +36 -17
- package/docs/HOST_DOCTOR.md +4 -2
- package/docs/MODEL_PORT_CONTRACT.md +305 -0
- package/docs/OCR_MODEL.md +39 -0
- package/docs/PROMPT_OPTIMIZER.md +47 -0
- package/docs/PROVIDER_ADAPTER_CONTRACT.md +53 -10
- package/docs/PROVIDER_HEALTH.md +22 -5
- package/docs/PROVIDER_RUNTIME_HARNESS.md +33 -0
- package/docs/PROVIDER_SUBMISSION_CONTRACT.md +2 -2
- package/docs/ROADMAP.md +49 -83
- package/docs/SECURITY.md +8 -7
- package/docs/TASK_SEMANTICS.md +2 -2
- package/docs/VISION_CLI.md +132 -0
- package/docs/VISION_MODEL.md +51 -0
- package/docs/WORKBENCH_ARTIFACTS.md +67 -0
- package/docs/assets/logo/README.md +55 -0
- package/docs/assets/logo/iris-bubble-concept.svg +67 -0
- package/docs/assets/logo/iris-bubble.svg +14 -0
- package/docs/assets/logo/iris-interim-flower.png +0 -0
- package/docs/assets/logo/iris-logo-mono-inverse.svg +78 -0
- package/docs/assets/logo/iris-logo-mono.svg +78 -0
- package/docs/assets/logo/iris-logo.svg +78 -0
- package/docs/assets/logo/iris-mark-mono-inverse.svg +42 -0
- package/docs/assets/logo/iris-mark-mono.svg +42 -0
- package/docs/assets/logo/iris-mark.svg +42 -0
- package/docs/file-access-across-environments.md +8 -5
- package/docs/releases/0.2.0.md +48 -0
- package/docs/releases/0.2.1.md +56 -0
- package/lib/actions.js +520 -591
- package/lib/adapters.js +184 -36
- package/lib/api.js +200 -11
- package/lib/chromium-browser.js +117 -0
- package/lib/client.js +1390 -105
- package/lib/command-service.js +534 -0
- package/lib/composite-vision-routing.js +42 -0
- package/lib/config.js +143 -51
- package/lib/core-artifact-store.js +630 -0
- package/lib/core-artifacts.js +4 -0
- package/lib/core-attention.js +103 -0
- package/lib/core-contract.js +105 -0
- package/lib/core-doctor.js +41 -0
- package/lib/core-lease-recovery.js +170 -0
- package/lib/core-maintenance.js +213 -0
- package/lib/core-runtime.js +232 -0
- package/lib/core-tasks.js +567 -0
- package/lib/core-user-projection.js +193 -0
- package/lib/doctor.js +92 -33
- package/lib/dsh-core-adapter.js +653 -0
- package/lib/dsh-host-adapter.js +58 -7
- package/lib/dsh-text-model-adapter.js +222 -0
- package/lib/dsh-vision-model-adapter.js +75 -0
- package/lib/generation-input.js +126 -0
- package/lib/guard.js +3 -3
- package/lib/headless-vision.js +181 -0
- package/lib/host-runtime.js +1 -0
- package/lib/http-vision-model-adapter.js +187 -0
- package/lib/image-edit-input.js +34 -0
- package/lib/index.js +323 -188
- package/lib/locate.js +86 -28
- package/lib/media-probe.js +2 -1
- package/lib/model-call-runtime.js +45 -0
- package/lib/model-invoker.js +220 -0
- package/lib/model-port-contract.js +341 -0
- package/lib/models.js +105 -14
- package/lib/ocr-model-routing.js +26 -0
- package/lib/ocr.js +110 -64
- package/lib/prompt-optimizer-config.js +2 -1
- package/lib/prompt-optimizer-core.js +121 -0
- package/lib/prompt-optimizer.js +37 -193
- package/lib/provider-adapter.js +49 -23
- package/lib/provider-adapters.js +165 -26
- package/lib/provider-catalog.js +183 -0
- package/lib/provider-config-service.js +236 -0
- package/lib/provider-contract.js +74 -4
- package/lib/provider-health.js +80 -1
- package/lib/provider-protocol.js +74 -2
- package/lib/provider-task-runner.js +370 -0
- package/lib/summarize.js +31 -20
- package/lib/summary-input.js +52 -0
- package/lib/task-wait.js +40 -0
- package/lib/tasks.js +16 -4
- package/lib/video-input.js +30 -0
- package/lib/vision-core.js +42 -0
- package/lib/vision-image-input.js +83 -0
- package/lib/vision-input.js +31 -0
- package/lib/vision-model-routing.js +180 -0
- package/lib/vision.js +31 -9
- package/lib/workbench-artifacts.js +52 -0
- package/lib/workbench-management.js +151 -0
- package/lib/workbench-zip.js +39 -0
- package/package.json +8 -8
- package/user_guide.md +71 -26
package/lib/actions.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
+
import { visionInputLimits } from './vision-image-input.js';
|
|
2
3
|
/**
|
|
3
4
|
* Iris 动作执行层(阶段 5 GUI 操作面板)。
|
|
4
5
|
*
|
|
@@ -11,6 +12,7 @@
|
|
|
11
12
|
*/
|
|
12
13
|
import fs from 'node:fs';
|
|
13
14
|
import path from 'node:path';
|
|
15
|
+
import crypto from 'node:crypto';
|
|
14
16
|
import * as adapters from './adapters.js';
|
|
15
17
|
import * as store from './config.js';
|
|
16
18
|
import * as cap from './capability.js';
|
|
@@ -18,25 +20,35 @@ import * as models from './models.js';
|
|
|
18
20
|
import * as tasks from './tasks.js';
|
|
19
21
|
import * as artifacts from './artifacts.js';
|
|
20
22
|
import { registerMedia, mediaLinksOf, webBase } from './media.js';
|
|
21
|
-
import {
|
|
23
|
+
import { RED_TEST_IMAGE } from './vision.js';
|
|
24
|
+
import { runVisionRequest, readVisionImageFile, probeVisionModel, visionImageFromDataUrl } from './vision-model-routing.js';
|
|
22
25
|
import { HOST_CONTRACT_VERSION, HOST_PORTS, defineHostAdapter, hasHostPort, requireHostPort } from './host-contract.js';
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
+
import { coreSnapshotForDsh, cropForDsh, diffForDsh, framesForDsh, summaryFramesForDsh, visionImageForDsh, htmlForDsh, inspectProviderTaskForDsh, submitProviderTaskForDsh, watchProviderTaskForDsh } from './dsh-core-adapter.js';
|
|
27
|
+
import { runLocateRequest, runSummaryRequest } from './composite-vision-routing.js';
|
|
28
|
+
import { normalizeSummarySource, summaryFrameRecords, summaryTimelineLabel } from './summary-input.js';
|
|
29
|
+
import { normalizeVisionImageSource } from './vision-input.js';
|
|
30
|
+
import { formatOcrResult } from './ocr.js';
|
|
31
|
+
import { runOcrRequest } from './ocr-model-routing.js';
|
|
26
32
|
import { ffmpegAvailable, extractFrames, probeVideo } from './media-probe.js';
|
|
27
|
-
import { buildContactSheet, summarizeMedia } from './summarize.js';
|
|
28
33
|
import { ensurePrivateDir, writePrivateFile } from './private-storage.js';
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import { invokeProviderOperation, providerPollTaskView } from './provider-adapter.js';
|
|
32
|
-
import { createConfiguredProviderAdapter } from './provider-adapters.js';
|
|
34
|
+
import { selectMediaProtocol, selectImageProtocol, providerForImageModel } from './provider-protocol.js';
|
|
35
|
+
import { providerTaskBinding } from './provider-catalog.js';
|
|
36
|
+
import { invokeProviderOperation, providerPollTaskView, prepareProviderInput } from './provider-adapter.js';
|
|
37
|
+
import { createConfiguredProviderAdapter as createProviderAdapter } from './provider-adapters.js';
|
|
33
38
|
import { deriveUserState } from './task-semantics.js';
|
|
39
|
+
import { normalizeGenerationInput } from './generation-input.js';
|
|
40
|
+
import { videoTaskCandidates } from './video-input.js';
|
|
34
41
|
|
|
35
42
|
const MEDIA_TYPES = {
|
|
36
43
|
'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
|
37
44
|
'.webp': 'image/webp', '.gif': 'image/gif'
|
|
38
45
|
};
|
|
39
46
|
|
|
47
|
+
function createConfiguredProviderAdapter(provider, options = {}) {
|
|
48
|
+
return createProviderAdapter(provider, { ...options,
|
|
49
|
+
onRateLimit: options.allowRateLimited ? undefined : store.recordRateLimit });
|
|
50
|
+
}
|
|
51
|
+
|
|
40
52
|
const EMPTY_ACTION_HOST = defineHostAdapter({
|
|
41
53
|
id: 'iris-hostless',
|
|
42
54
|
ports: {},
|
|
@@ -53,21 +65,8 @@ function actionHost(value) {
|
|
|
53
65
|
throw new TypeError('Iris Action 只接受有效的 Host Adapter,不接受原始 ctx');
|
|
54
66
|
}
|
|
55
67
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
try {
|
|
59
|
-
const mt = MEDIA_TYPES[path.extname(absPath).toLowerCase()] || 'image/png';
|
|
60
|
-
const buf = fs.readFileSync(absPath);
|
|
61
|
-
const dataUrl = `data:${mt};base64,${buf.toString('base64')}`;
|
|
62
|
-
const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION) });
|
|
63
|
-
const { answer } = await askWithHealth(backends, {
|
|
64
|
-
question: `用不超过两句话描述这张图片的主题与构图。生成该图的提示词是:「${String(originalPrompt).slice(0, 200)}」`,
|
|
65
|
-
imageDataUrl: dataUrl,
|
|
66
|
-
signal: undefined
|
|
67
|
-
});
|
|
68
|
-
return answer.slice(0, 300);
|
|
69
|
-
} catch (_) { return ''; }
|
|
70
|
-
}
|
|
68
|
+
// 工作台清空可选文本框会发送空字符串;在 Host 边界解释为未提供,保持默认值。
|
|
69
|
+
function optionalActionText(value) { return value === '' ? undefined : value; }
|
|
71
70
|
|
|
72
71
|
/* ---------------- IO 目录 ---------------- */
|
|
73
72
|
function outputDir() {
|
|
@@ -89,24 +88,49 @@ function throwIfAborted(signal) {
|
|
|
89
88
|
*/
|
|
90
89
|
function selectProviders(capability, label, requestedModelRef) {
|
|
91
90
|
let providers = store.pickAllFor(capability);
|
|
92
|
-
const
|
|
93
|
-
if (
|
|
91
|
+
const requested = String(requestedModelRef || '').trim();
|
|
92
|
+
if (requested) {
|
|
93
|
+
const pool = models.modelPool(store.providers());
|
|
94
|
+
const selected = models.resolvePoolModel(pool, requested, capability);
|
|
95
|
+
if (selected?.rateLimited) throw Object.assign(new Error(store.modelHealth(selected.providerId, selected.id, capability).retryAt
|
|
96
|
+
? 'iris: 指定模型正在限流冷却,请稍后使用或手动实测'
|
|
97
|
+
: 'iris: 指定模型额度或预算耗尽,请手动实测成功后再使用'), {
|
|
98
|
+
code: 'IRIS_PROVIDER_MODEL_RATE_LIMITED'
|
|
99
|
+
});
|
|
94
100
|
const field = models.CAP_FIELD[capability];
|
|
95
|
-
providers = providers.filter((p) => p.id ===
|
|
101
|
+
providers = selected ? providers.filter((p) => p.id === selected.providerId && p[field] === selected.id) : [];
|
|
96
102
|
if (!providers.length) throw new Error('iris: 指定的模型复合引用不可用或不具备' + label + '能力');
|
|
97
103
|
}
|
|
98
104
|
if (!providers.length) throw new Error('iris: 没有可用' + label + '供应商——先在 Iris 设置里添加、标注并分配模型');
|
|
99
105
|
return providers;
|
|
100
106
|
}
|
|
101
107
|
|
|
108
|
+
function configuredModel(provider, capability, requested) {
|
|
109
|
+
const ref = models.resolveModelRef(provider, capability, requested);
|
|
110
|
+
if (!ref) throw new Error('iris: 未配置可用模型,请发现或手工添加并标注对应能力');
|
|
111
|
+
return models.parseModelRef(ref).modelId;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function routeSelectionReason(provider, capability, requested) {
|
|
115
|
+
if (String(requested || '').trim()) return 'explicit';
|
|
116
|
+
const field = models.CAP_FIELD[capability];
|
|
117
|
+
const selectedRef = field && provider?.[field]
|
|
118
|
+
? models.modelRef(provider.id, provider[field]) : '';
|
|
119
|
+
return models.selectionReasonForModel(
|
|
120
|
+
models.modelPool(store.providers()),
|
|
121
|
+
capability,
|
|
122
|
+
store.assignments()[capability],
|
|
123
|
+
selectedRef
|
|
124
|
+
) || 'pool';
|
|
125
|
+
}
|
|
126
|
+
|
|
102
127
|
function imageRoute(provider, args) {
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
const
|
|
106
|
-
? (parsed?.modelId || String(args.model))
|
|
107
|
-
: (provider.imageModel || (protocol === 'openai-images' ? 'gpt-image-1' : 'wan2.2-t2i-flash'));
|
|
128
|
+
const model = configuredModel(provider, cap.CAPABILITIES.IMAGE, args.model);
|
|
129
|
+
provider = providerForImageModel(provider, model);
|
|
130
|
+
const protocol = selectMediaProtocol(provider).mediaProtocol;
|
|
108
131
|
return {
|
|
109
132
|
provider, protocol, model, modelRef: models.modelRef(provider.id, model),
|
|
133
|
+
selectionReason: routeSelectionReason(provider, cap.CAPABILITIES.IMAGE, args.model),
|
|
110
134
|
adapter: createConfiguredProviderAdapter(provider, { protocol })
|
|
111
135
|
};
|
|
112
136
|
}
|
|
@@ -168,164 +192,110 @@ function recordVisionHealth(errors) {
|
|
|
168
192
|
}
|
|
169
193
|
}
|
|
170
194
|
|
|
171
|
-
async function
|
|
195
|
+
async function askSingleVisionWithHealth(host, request) {
|
|
172
196
|
try {
|
|
173
|
-
const result = await
|
|
197
|
+
const result = await runVisionRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, ...request });
|
|
174
198
|
recordVisionHealth(result.errors);
|
|
175
|
-
if (result.via === 'selfstack') {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
});
|
|
179
|
-
}
|
|
199
|
+
if (result.via === 'selfstack') recordProviderHealthSafe(result.backendId, result.model, cap.CAPABILITIES.VISION, {
|
|
200
|
+
ok: true, category: 'success', note: '视觉请求成功'
|
|
201
|
+
});
|
|
180
202
|
return result;
|
|
181
|
-
} catch (error) {
|
|
182
|
-
recordVisionHealth(error && error.errors);
|
|
183
|
-
throw error;
|
|
184
|
-
}
|
|
203
|
+
} catch (error) { recordVisionHealth(error?.errors); throw error; }
|
|
185
204
|
}
|
|
186
205
|
|
|
187
|
-
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
if (
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const name = taskId + '-' + names.length + '.png';
|
|
202
|
-
const target = path.join(outputDir(), name);
|
|
203
|
-
if (item && item.b64) writePrivateFile(target, Buffer.from(item.b64, 'base64'));
|
|
204
|
-
else if (item && item.url) await invokeProviderOperation(providerAdapter, 'download', {
|
|
205
|
-
artifact: { kind: 'remote-url', url: item.url }, targetPath: target, signal
|
|
206
|
-
});
|
|
207
|
-
else throw new Error('同步图片结果缺少 b64 或 URL');
|
|
208
|
-
names.push(name);
|
|
209
|
-
}
|
|
210
|
-
} else {
|
|
211
|
-
throw new Error('未知同步图片结果结构');
|
|
212
|
-
}
|
|
213
|
-
if (!names.length) throw new Error('生成已成功,但没有可交付的图片');
|
|
214
|
-
for (const name of names) registerMedia(taskId, path.join(outputDir(), name));
|
|
215
|
-
tasks.completeDelivery(taskId, names);
|
|
216
|
-
return tasks.get(taskId);
|
|
217
|
-
} catch (error) {
|
|
218
|
-
let recorded;
|
|
219
|
-
try {
|
|
220
|
-
recorded = tasks.failDelivery(taskId, error, names);
|
|
221
|
-
} catch (persistError) {
|
|
222
|
-
throw taskActionError(taskId, '图像已生成,但交付状态无法可靠落盘;禁止重新生成', {
|
|
223
|
-
message: String((persistError && persistError.message) || persistError)
|
|
206
|
+
function recordCoreTaskAttemptHealth(task, capability) {
|
|
207
|
+
const label = capability === cap.CAPABILITIES.VIDEO ? '视频' : '图片';
|
|
208
|
+
for (const attempt of task?.attempts || []) {
|
|
209
|
+
const parsed = models.parseModelRef(attempt.model);
|
|
210
|
+
if (!parsed) continue;
|
|
211
|
+
if (attempt.acceptance === 'accepted') {
|
|
212
|
+
recordProviderHealthSafe(attempt.providerId, parsed.modelId, capability, {
|
|
213
|
+
ok: true, category: 'success',
|
|
214
|
+
note: attempt.resultKind === 'completed' ? `同步${label}生成成功` : `${label}任务已受理`
|
|
215
|
+
});
|
|
216
|
+
} else if (attempt.error) {
|
|
217
|
+
recordProviderHealthSafe(attempt.providerId, parsed.modelId, capability, {
|
|
218
|
+
ok: false, category: attempt.error.category, httpStatus: attempt.error.httpStatus,
|
|
219
|
+
note: attempt.error.safeMessage
|
|
224
220
|
});
|
|
225
221
|
}
|
|
226
|
-
throw taskActionError(taskId, '图像已生成,但产物交付失败:' + recorded.error, recorded.lastError);
|
|
227
222
|
}
|
|
228
223
|
}
|
|
229
224
|
|
|
230
|
-
/**
|
|
231
|
-
async function
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
route,
|
|
244
|
-
submit: async () => {
|
|
245
|
-
if (signal && signal.aborted) {
|
|
246
|
-
throw new ProviderContractError('提交前收到取消信号', {
|
|
247
|
-
stage: 'submit', category: 'aborted', acceptance: 'unknown'
|
|
248
|
-
});
|
|
249
|
-
}
|
|
250
|
-
return observeProviderCapability(route, cap.CAPABILITIES.IMAGE, () =>
|
|
251
|
-
invokeProviderOperation(route.adapter, 'submit', {
|
|
252
|
-
capability: 'image',
|
|
253
|
-
model: route.model,
|
|
254
|
-
input: { prompt, size: args.size, n: args.n },
|
|
255
|
-
signal
|
|
256
|
-
}));
|
|
257
|
-
}
|
|
258
|
-
}));
|
|
259
|
-
|
|
260
|
-
const submitted = await submitWithAcceptanceBoundary(candidates, { capability: 'image' }, {
|
|
261
|
-
beforeAttempt: ({ providerId, model }) => {
|
|
262
|
-
const route = routes.find((item) => item.provider.id === providerId && item.modelRef === model);
|
|
263
|
-
if (!route) throw new Error('候选路由与 Attempt 身份不一致');
|
|
264
|
-
return tasks.beginAttempt(task.id, {
|
|
265
|
-
providerId,
|
|
266
|
-
providerName: route.provider.name,
|
|
267
|
-
model,
|
|
268
|
-
protocol: route.protocol
|
|
269
|
-
});
|
|
270
|
-
},
|
|
271
|
-
afterResult: (snapshot) => tasks.recordAttemptResult(task.id, snapshot)
|
|
225
|
+
/** DSH 图片入口:同步和异步候选共用一份 Core Task/Attempt/Artifact 事实。 */
|
|
226
|
+
async function submitCoreImage(routes, providerInput, signal, sourceArtifactId) {
|
|
227
|
+
const submitted = await submitProviderTaskForDsh({
|
|
228
|
+
capability: 'image',
|
|
229
|
+
candidates: routes.map((route) => ({
|
|
230
|
+
adapter: route.adapter,
|
|
231
|
+
model: route.modelRef,
|
|
232
|
+
selectionReason: route.selectionReason,
|
|
233
|
+
providerBinding: providerTaskBinding(route.provider)
|
|
234
|
+
})),
|
|
235
|
+
providerInput,
|
|
236
|
+
sourceArtifactId,
|
|
237
|
+
signal
|
|
272
238
|
});
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
239
|
+
const task = submitted.task;
|
|
240
|
+
recordCoreTaskAttemptHealth(task, cap.CAPABILITIES.IMAGE);
|
|
241
|
+
if (submitted.localError) throw taskActionError(task.id, 'Core 提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
242
|
+
if (task.acceptance === 'unknown') throw taskActionError(task.id, '供应商受理状态未知,禁止自动重提', task.lastError);
|
|
243
|
+
if (task.acceptance === 'not_accepted') throw taskActionError(task.id, '所有候选均明确未受理', task.attempts);
|
|
244
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
245
|
+
throw taskActionError(task.id, '图像已生成,但 Core Artifact 交付失败;禁止重新生成', task.lastError);
|
|
276
246
|
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
if (
|
|
283
|
-
|
|
284
|
-
throw taskActionError(task.id, '供应商受理状态未知,禁止自动重提' + (safe ? ':' + safe : ''), submitted.result.error);
|
|
247
|
+
const selected = routes.find((route) => route.provider.id === task.providerId && route.modelRef === task.modelRef);
|
|
248
|
+
if (!selected) throw taskActionError(task.id, 'Core Task 的供应商身份无法映射回当前路由', task);
|
|
249
|
+
if (task.acceptance === 'accepted' && task.remoteTaskId
|
|
250
|
+
&& ['none', 'unknown'].includes(task.outcome) && task.phase !== 'terminal') {
|
|
251
|
+
await watchProviderTaskForDsh({ taskId: task.id, signal });
|
|
252
|
+
} else if (task.outcome !== 'succeeded' || task.deliveryState !== 'ready' || !task.artifactIds.length) {
|
|
253
|
+
throw taskActionError(task.id, '图片路由没有得到可观察任务或可交付产物;事实已保留在 Core,未回退重提', task);
|
|
285
254
|
}
|
|
255
|
+
return { task, provider: selected.provider, model: selected.model };
|
|
256
|
+
}
|
|
286
257
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
258
|
+
/**
|
|
259
|
+
* DSH 视频入口(E 阶段):t2v/i2v 共用 Core Task/Attempt/Artifact 事实轴,
|
|
260
|
+
* 不共用图片状态机——视频走自己的交付 Profile(video/mp4 / generated-video)。
|
|
261
|
+
* s2v 在写前 Attempt 之后按账号上传首帧和音频,交付复用视频 Profile。
|
|
262
|
+
*/
|
|
263
|
+
async function submitCoreVideo(routes, frame, prompt, args, signal) {
|
|
264
|
+
const normalized = normalizeGenerationInput('video', {
|
|
265
|
+
...(prompt ? { prompt } : {}),
|
|
266
|
+
...(frame ? { first_frame_path: frame.absPath } : {}),
|
|
267
|
+
...(args.audio_path ? { audio_path: args.audio_path, resolution: args.resolution } : {}),
|
|
268
|
+
size: optionalActionText(args.size),
|
|
269
|
+
duration: args.duration
|
|
270
|
+
}, { allowVideoPaths: true });
|
|
271
|
+
const submitted = await submitProviderTaskForDsh({
|
|
272
|
+
capability: 'video',
|
|
273
|
+
candidates: videoTaskCandidates(routes.map((route) => ({
|
|
274
|
+
adapter: route.adapter,
|
|
275
|
+
model: route.modelRef,
|
|
276
|
+
selectionReason: route.selectionReason,
|
|
277
|
+
providerBinding: providerTaskBinding(route.provider)
|
|
278
|
+
})), normalized),
|
|
279
|
+
providerInput: normalized.providerInput,
|
|
280
|
+
signal
|
|
281
|
+
});
|
|
282
|
+
const task = submitted.task;
|
|
283
|
+
recordCoreTaskAttemptHealth(task, cap.CAPABILITIES.VIDEO);
|
|
284
|
+
if (submitted.localError) throw taskActionError(task.id, 'Core 提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
285
|
+
if (task.acceptance === 'unknown') throw taskActionError(task.id, '供应商受理状态未知,禁止自动重提', task.lastError);
|
|
286
|
+
if (task.acceptance === 'not_accepted') throw taskActionError(task.id, '所有候选均明确未受理', task.attempts);
|
|
287
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
288
|
+
throw taskActionError(task.id, '视频已生成,但 Core Artifact 交付失败;禁止重新生成', task.lastError);
|
|
294
289
|
}
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
290
|
+
const selected = routes.find((route) => route.provider.id === task.providerId && route.modelRef === task.modelRef);
|
|
291
|
+
if (!selected) throw taskActionError(task.id, 'Core Task 的供应商身份无法映射回当前路由', task);
|
|
292
|
+
if (task.acceptance === 'accepted' && task.remoteTaskId
|
|
293
|
+
&& ['none', 'unknown'].includes(task.outcome) && task.phase !== 'terminal') {
|
|
294
|
+
await watchProviderTaskForDsh({ taskId: task.id, signal });
|
|
295
|
+
} else if (task.outcome !== 'succeeded' || task.deliveryState !== 'ready' || !task.artifactIds.length) {
|
|
296
|
+
throw taskActionError(task.id, '视频路由没有得到可观察任务或可交付产物;事实已保留在 Core,未回退重提', task);
|
|
300
297
|
}
|
|
301
|
-
|
|
302
|
-
poll: ({ remoteTaskId, signal: pollSignal }) => observeProviderCapability(
|
|
303
|
-
route, cap.CAPABILITIES.IMAGE,
|
|
304
|
-
() => invokeProviderOperation(route.adapter, 'poll', {
|
|
305
|
-
capability: 'image', remoteTaskId, signal: pollSignal
|
|
306
|
-
}),
|
|
307
|
-
{ recordSuccess: false }
|
|
308
|
-
),
|
|
309
|
-
intervalMs: 2500,
|
|
310
|
-
onSuccess: async (record, result) => {
|
|
311
|
-
const names = [];
|
|
312
|
-
for (const url of result.urls || []) {
|
|
313
|
-
const name = record.id + '-' + names.length + '.' + adapters.extFromUrl(url, 'png');
|
|
314
|
-
await invokeProviderOperation(route.adapter, 'download', {
|
|
315
|
-
artifact: { kind: 'remote-url', url }, targetPath: path.join(outputDir(), name)
|
|
316
|
-
});
|
|
317
|
-
names.push(name);
|
|
318
|
-
}
|
|
319
|
-
for (const name of names) registerMedia(record.id, path.join(outputDir(), name));
|
|
320
|
-
return names;
|
|
321
|
-
}
|
|
322
|
-
});
|
|
323
|
-
return {
|
|
324
|
-
task: tasks.get(task.id),
|
|
325
|
-
provider: route.provider,
|
|
326
|
-
model: route.model,
|
|
327
|
-
remoteTaskId: submitted.result.remoteTaskId
|
|
328
|
-
};
|
|
298
|
+
return { task, provider: selected.provider, model: selected.model, mode: selected.mode };
|
|
329
299
|
}
|
|
330
300
|
|
|
331
301
|
function findGeneratedAttachment(attachmentId) {
|
|
@@ -339,12 +309,26 @@ function findGeneratedAttachment(attachmentId) {
|
|
|
339
309
|
return null;
|
|
340
310
|
}
|
|
341
311
|
|
|
342
|
-
function resolveFirstFrameFile(args) {
|
|
312
|
+
async function resolveFirstFrameFile(host, args, signal) {
|
|
343
313
|
if (args.first_frame_attachment_id) {
|
|
344
314
|
const hit = findGeneratedAttachment(String(args.first_frame_attachment_id).trim());
|
|
345
|
-
if (
|
|
346
|
-
|
|
347
|
-
|
|
315
|
+
if (hit) {
|
|
316
|
+
if (!fs.existsSync(hit.absPath)) throw new Error('iris: 首帧图片的本地缓存已被清理');
|
|
317
|
+
return hit;
|
|
318
|
+
}
|
|
319
|
+
const sessions = requireHostPort(host, 'sessions', '读取首帧附件');
|
|
320
|
+
const attachments = requireHostPort(host, 'attachments', '读取首帧附件');
|
|
321
|
+
const ref = await sessions.findImageAttachment(args.session_id, String(args.first_frame_attachment_id).trim());
|
|
322
|
+
if (!ref) throw new Error('iris: 找不到本会话中的 first_frame_attachment_id');
|
|
323
|
+
const stored = await attachments.readImage(ref, signal);
|
|
324
|
+
signal?.throwIfAborted();
|
|
325
|
+
const mediaType = stored.mediaType || ref.mediaType;
|
|
326
|
+
const extension = Object.keys(MEDIA_TYPES).find(key => MEDIA_TYPES[key] === mediaType);
|
|
327
|
+
if (!extension) throw new Error('iris: 不支持的首帧图片格式');
|
|
328
|
+
const bytes = Buffer.from(stored.data);
|
|
329
|
+
const file = path.join(store.irisHome(), 'uploads', 'frame-' + crypto.createHash('sha256').update(bytes).digest('hex') + extension);
|
|
330
|
+
if (!fs.existsSync(file)) writePrivateFile(file, bytes, { flag: 'wx' });
|
|
331
|
+
return { absPath: file, mediaType };
|
|
348
332
|
}
|
|
349
333
|
if (args.first_frame_path) {
|
|
350
334
|
const p = String(args.first_frame_path).trim();
|
|
@@ -357,292 +341,46 @@ function resolveFirstFrameFile(args) {
|
|
|
357
341
|
return null;
|
|
358
342
|
}
|
|
359
343
|
|
|
360
|
-
function fileDataUrl(file) {
|
|
361
|
-
return 'data:' + file.mediaType + ';base64,' + fs.readFileSync(file.absPath).toString('base64');
|
|
362
|
-
}
|
|
363
|
-
|
|
364
344
|
function videoRoute(provider, args, frame) {
|
|
365
|
-
const
|
|
366
|
-
const
|
|
367
|
-
|
|
345
|
+
const model = configuredModel(provider, cap.CAPABILITIES.VIDEO, args.model);
|
|
346
|
+
const protocol = selectMediaProtocol(provider).mediaProtocol;
|
|
347
|
+
// B-7:必须区分两件事——
|
|
348
|
+
// s2vModel : 该模型【本身】是否是数字人模型(决定它能否接受 audio_path)
|
|
349
|
+
// isS2V : 本次【请求】是否应走 s2v 链路(显式 audio_path 意图 或 模型本身是 s2v)
|
|
350
|
+
// 旧实现只有 isS2V 且只看模型名,导致 audio_path + 非 s2v 模型被静默降级为 t2v。
|
|
351
|
+
const s2vModel = /s2v/i.test(model);
|
|
352
|
+
const isS2V = Boolean(String(args.audio_path || '').trim()) || s2vModel;
|
|
368
353
|
const mode = isS2V ? 's2v' : (frame ? 'i2v' : 't2v');
|
|
369
354
|
return {
|
|
370
|
-
provider, model, modelRef: models.modelRef(provider.id, model), isS2V, mode, protocol
|
|
371
|
-
adapter: createConfiguredProviderAdapter(provider, { protocol
|
|
355
|
+
provider, model, modelRef: models.modelRef(provider.id, model), isS2V, s2vModel, mode, protocol,
|
|
356
|
+
adapter: createConfiguredProviderAdapter(provider, { protocol }),
|
|
357
|
+
selectionReason: routeSelectionReason(provider, cap.CAPABILITIES.VIDEO, args.model)
|
|
372
358
|
};
|
|
373
359
|
}
|
|
374
360
|
|
|
375
|
-
function
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
: (stage === 'validate' ? 'invalid_request' : 'local_io');
|
|
379
|
-
return new ProviderContractError(String((error && error.message) || error || '提交前准备失败'), {
|
|
380
|
-
stage,
|
|
381
|
-
category,
|
|
382
|
-
acceptance: 'not_accepted',
|
|
383
|
-
retryable: category === 'network'
|
|
384
|
-
});
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
async function submitVideoV2(routes, frame, prompt, args, signal) {
|
|
388
|
-
const task = tasks.createV2({
|
|
389
|
-
cap: 'video', prompt: prompt.slice(0, 4000),
|
|
390
|
-
...(args.model ? { requestedModelRef: String(args.model) } : {}),
|
|
391
|
-
...(args._retryOf ? { retryOf: String(args._retryOf) } : {})
|
|
392
|
-
});
|
|
393
|
-
const candidates = routes.map((route) => ({
|
|
394
|
-
id: route.provider.id,
|
|
395
|
-
model: route.modelRef,
|
|
396
|
-
route,
|
|
397
|
-
submit: async (_input, context) => {
|
|
398
|
-
const attemptId = context.attempt.id;
|
|
399
|
-
let submitArgs;
|
|
400
|
-
if (route.isS2V) {
|
|
401
|
-
if (!frame) throw beforeGenerationError(new Error('iris: s2v 数字人需要首帧'), 'validate');
|
|
402
|
-
const audioPath = String(args.audio_path || '').trim();
|
|
403
|
-
if (!audioPath || !path.isAbsolute(audioPath)) {
|
|
404
|
-
throw beforeGenerationError(new Error('iris: s2v 的 audio_path 必须是绝对路径'), 'validate');
|
|
405
|
-
}
|
|
406
|
-
if (!fs.existsSync(audioPath)) throw beforeGenerationError(new Error('iris: 音频文件不存在'), 'validate');
|
|
407
|
-
tasks.markAttemptStage(task.id, attemptId, 'upload');
|
|
408
|
-
try {
|
|
409
|
-
const imageUrl = await adapters.uploadTempFile({
|
|
410
|
-
key: route.provider.apiKey, baseUrl: route.provider.baseUrl,
|
|
411
|
-
model: route.model, filePath: frame.absPath, signal
|
|
412
|
-
});
|
|
413
|
-
const audioUrl = await adapters.uploadTempFile({
|
|
414
|
-
key: route.provider.apiKey, baseUrl: route.provider.baseUrl,
|
|
415
|
-
model: route.model, filePath: audioPath, signal
|
|
416
|
-
});
|
|
417
|
-
submitArgs = { imgDataUrl: imageUrl, audioUrl, resolution: args.resolution || '480P' };
|
|
418
|
-
} catch (error) {
|
|
419
|
-
throw beforeGenerationError(error, 'upload');
|
|
420
|
-
}
|
|
421
|
-
} else {
|
|
422
|
-
if (!prompt) throw beforeGenerationError(new Error('iris: 文生/图生视频需要 prompt'), 'validate');
|
|
423
|
-
tasks.markAttemptStage(task.id, attemptId, 'prepare');
|
|
424
|
-
try {
|
|
425
|
-
submitArgs = {
|
|
426
|
-
prompt,
|
|
427
|
-
imgDataUrl: frame ? fileDataUrl(frame) : undefined,
|
|
428
|
-
size: args.size || '1280*720',
|
|
429
|
-
duration: args.duration ? Number(args.duration) : undefined
|
|
430
|
-
};
|
|
431
|
-
} catch (error) {
|
|
432
|
-
throw beforeGenerationError(error, 'prepare');
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
tasks.markAttemptStage(task.id, attemptId, 'submit');
|
|
436
|
-
return observeProviderCapability(route, cap.CAPABILITIES.VIDEO, () =>
|
|
437
|
-
invokeProviderOperation(route.adapter, 'submit', {
|
|
438
|
-
capability: 'video', model: route.model, input: submitArgs, signal
|
|
439
|
-
}));
|
|
440
|
-
}
|
|
441
|
-
}));
|
|
442
|
-
|
|
443
|
-
const submitted = await submitWithAcceptanceBoundary(candidates, { capability: 'video' }, {
|
|
444
|
-
beforeAttempt: ({ providerId, model }) => {
|
|
445
|
-
const route = routes.find((item) => item.provider.id === providerId && item.modelRef === model);
|
|
446
|
-
if (!route) throw new Error('候选路由与 Attempt 身份不一致');
|
|
447
|
-
const attempt = tasks.beginAttempt(task.id, {
|
|
448
|
-
providerId, providerName: route.provider.name, model, protocol: route.protocol
|
|
449
|
-
});
|
|
450
|
-
tasks.update(task.id, {
|
|
451
|
-
mode: route.mode,
|
|
452
|
-
...(route.isS2V ? { resolution: args.resolution || '480P' } : {
|
|
453
|
-
size: args.size || '1280*720', ...(args.duration ? { duration: Number(args.duration) } : {})
|
|
454
|
-
})
|
|
455
|
-
});
|
|
456
|
-
return attempt;
|
|
457
|
-
},
|
|
458
|
-
afterResult: (snapshot) => tasks.recordAttemptResult(task.id, snapshot)
|
|
459
|
-
});
|
|
460
|
-
|
|
461
|
-
if (submitted.localError) {
|
|
462
|
-
throw taskActionError(task.id, '视频提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
463
|
-
}
|
|
464
|
-
if (!submitted.candidate) {
|
|
465
|
-
const lastError = submitted.result && submitted.result.error;
|
|
466
|
-
tasks.finalizeNoAcceptance(task.id, lastError || '所有视频候选均明确未受理');
|
|
467
|
-
throw taskActionError(task.id, '所有视频候选均明确未受理', submitted.attempts);
|
|
468
|
-
}
|
|
469
|
-
if (submitted.result.kind !== 'accepted') {
|
|
470
|
-
const safe = submitted.result.error && submitted.result.error.safeMessage;
|
|
471
|
-
throw taskActionError(task.id, '视频受理状态未知,禁止自动重提' + (safe ? ':' + safe : ''), submitted.result.error);
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
const route = submitted.candidate.route;
|
|
475
|
-
tasks.activateWatch(task.id);
|
|
476
|
-
tasks.watch(tasks.get(task.id), {
|
|
477
|
-
poll: ({ remoteTaskId: id, signal: pollSignal }) => observeProviderCapability(
|
|
478
|
-
route, cap.CAPABILITIES.VIDEO,
|
|
479
|
-
() => invokeProviderOperation(route.adapter, 'poll', {
|
|
480
|
-
capability: 'video', remoteTaskId: id, signal: pollSignal
|
|
481
|
-
}),
|
|
482
|
-
{ recordSuccess: false }
|
|
483
|
-
),
|
|
484
|
-
intervalMs: 6000,
|
|
485
|
-
onSuccess: async (record, result) => {
|
|
486
|
-
const names = [];
|
|
487
|
-
for (const url of result.urls || []) {
|
|
488
|
-
const name = record.id + '-' + names.length + '.' + adapters.extFromUrl(url, 'mp4');
|
|
489
|
-
await invokeProviderOperation(route.adapter, 'download', {
|
|
490
|
-
artifact: { kind: 'remote-url', url }, targetPath: path.join(outputDir(), name)
|
|
491
|
-
});
|
|
492
|
-
names.push(name);
|
|
493
|
-
}
|
|
494
|
-
for (const name of names) registerMedia(record.id, path.join(outputDir(), name));
|
|
495
|
-
return names;
|
|
496
|
-
}
|
|
497
|
-
});
|
|
498
|
-
return { task: tasks.get(task.id), provider: route.provider, model: route.model, remoteTaskId: submitted.result.remoteTaskId };
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
function simpleModelRoute(provider, args, field, fallback, protocol = 'dashscope') {
|
|
502
|
-
const parsed = models.parseModelRef(String(args.model || ''));
|
|
503
|
-
const model = args.model ? (parsed?.modelId || String(args.model)) : (provider[field] || fallback);
|
|
361
|
+
function simpleModelRoute(provider, args, capability) {
|
|
362
|
+
const protocol = selectMediaProtocol(provider).mediaProtocol;
|
|
363
|
+
const model = configuredModel(provider, capability, args.model);
|
|
504
364
|
return {
|
|
505
365
|
provider, model, modelRef: models.modelRef(provider.id, model), protocol,
|
|
506
|
-
adapter: createConfiguredProviderAdapter(provider, { protocol })
|
|
366
|
+
adapter: createConfiguredProviderAdapter(provider, { protocol }),
|
|
367
|
+
selectionReason: routeSelectionReason(provider, capability, args.model)
|
|
507
368
|
};
|
|
508
369
|
}
|
|
509
370
|
|
|
510
|
-
async function submitTranscriptionV2(routes, audioPath, args, signal) {
|
|
511
|
-
const task = tasks.createV2({
|
|
512
|
-
cap: 'transcribe', prompt: 'audio transcribe: ' + path.basename(audioPath),
|
|
513
|
-
...(args.model ? { requestedModelRef: String(args.model) } : {})
|
|
514
|
-
});
|
|
515
|
-
const candidates = routes.map((route) => ({
|
|
516
|
-
id: route.provider.id,
|
|
517
|
-
model: route.modelRef,
|
|
518
|
-
route,
|
|
519
|
-
submit: async (_input, context) => {
|
|
520
|
-
tasks.markAttemptStage(task.id, context.attempt.id, 'upload');
|
|
521
|
-
let audioUrl;
|
|
522
|
-
try {
|
|
523
|
-
audioUrl = await adapters.uploadTempFile({
|
|
524
|
-
key: route.provider.apiKey, baseUrl: route.provider.baseUrl,
|
|
525
|
-
model: route.model, filePath: audioPath, signal
|
|
526
|
-
});
|
|
527
|
-
} catch (error) {
|
|
528
|
-
throw beforeGenerationError(error, 'upload');
|
|
529
|
-
}
|
|
530
|
-
tasks.markAttemptStage(task.id, context.attempt.id, 'submit');
|
|
531
|
-
return observeProviderCapability(route, cap.CAPABILITIES.TRANSCRIBE, () =>
|
|
532
|
-
invokeProviderOperation(route.adapter, 'submit', {
|
|
533
|
-
capability: 'transcribe', model: route.model, input: { audioUrl }, signal
|
|
534
|
-
}));
|
|
535
|
-
}
|
|
536
|
-
}));
|
|
537
|
-
const submitted = await submitWithAcceptanceBoundary(candidates, { capability: 'transcribe' }, {
|
|
538
|
-
beforeAttempt: ({ providerId, model }) => {
|
|
539
|
-
const route = routes.find((item) => item.provider.id === providerId && item.modelRef === model);
|
|
540
|
-
if (!route) throw new Error('候选路由与 Attempt 身份不一致');
|
|
541
|
-
return tasks.beginAttempt(task.id, {
|
|
542
|
-
providerId, providerName: route.provider.name, model, protocol: route.protocol
|
|
543
|
-
});
|
|
544
|
-
},
|
|
545
|
-
afterResult: (snapshot) => tasks.recordAttemptResult(task.id, snapshot)
|
|
546
|
-
});
|
|
547
|
-
if (submitted.localError) throw taskActionError(task.id, '转写提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
548
|
-
if (!submitted.candidate) {
|
|
549
|
-
tasks.finalizeNoAcceptance(task.id, submitted.result?.error || '所有转写候选均明确未受理');
|
|
550
|
-
throw taskActionError(task.id, '所有转写候选均明确未受理', submitted.attempts);
|
|
551
|
-
}
|
|
552
|
-
if (submitted.result.kind !== 'accepted') {
|
|
553
|
-
const safe = submitted.result.error && submitted.result.error.safeMessage;
|
|
554
|
-
throw taskActionError(task.id, '转写受理状态未知,禁止自动重提' + (safe ? ':' + safe : ''), submitted.result.error);
|
|
555
|
-
}
|
|
556
|
-
const route = submitted.candidate.route;
|
|
557
|
-
tasks.activateWatch(task.id);
|
|
558
|
-
tasks.watch(tasks.get(task.id), {
|
|
559
|
-
poll: ({ remoteTaskId: id, signal: pollSignal }) => observeProviderCapability(
|
|
560
|
-
route, cap.CAPABILITIES.TRANSCRIBE,
|
|
561
|
-
() => invokeProviderOperation(route.adapter, 'poll', {
|
|
562
|
-
capability: 'transcribe', remoteTaskId: id, signal: pollSignal
|
|
563
|
-
}),
|
|
564
|
-
{ recordSuccess: false }
|
|
565
|
-
),
|
|
566
|
-
intervalMs: 2500,
|
|
567
|
-
allowEmptyResult: true,
|
|
568
|
-
onSuccess: async (record, result) => {
|
|
569
|
-
tasks.update(record.id, { transcribeText: (result.urls && result.urls.join('')) || '' });
|
|
570
|
-
return [];
|
|
571
|
-
}
|
|
572
|
-
});
|
|
573
|
-
return { task: tasks.get(task.id), provider: route.provider, model: route.model, remoteTaskId: submitted.result.remoteTaskId };
|
|
574
|
-
}
|
|
575
371
|
|
|
576
|
-
async function submitTtsV2(routes, text, args, signal) {
|
|
577
|
-
const task = tasks.createV2({
|
|
578
|
-
cap: 'tts', prompt: text.slice(0, 500),
|
|
579
|
-
...(args.model ? { requestedModelRef: String(args.model) } : {})
|
|
580
|
-
});
|
|
581
|
-
const candidates = routes.map((route) => ({
|
|
582
|
-
id: route.provider.id,
|
|
583
|
-
model: route.modelRef,
|
|
584
|
-
route,
|
|
585
|
-
submit: async (_input, context) => {
|
|
586
|
-
tasks.markAttemptStage(task.id, context.attempt.id, 'submit');
|
|
587
|
-
return observeProviderCapability(route, cap.CAPABILITIES.TTS, () =>
|
|
588
|
-
invokeProviderOperation(route.adapter, 'submit', {
|
|
589
|
-
capability: 'tts', model: route.model, input: { text, voice: route.voice }, signal
|
|
590
|
-
}));
|
|
591
|
-
}
|
|
592
|
-
}));
|
|
593
|
-
const submitted = await submitWithAcceptanceBoundary(candidates, { capability: 'tts' }, {
|
|
594
|
-
beforeAttempt: ({ providerId, model }) => {
|
|
595
|
-
const route = routes.find((item) => item.provider.id === providerId && item.modelRef === model);
|
|
596
|
-
if (!route) throw new Error('候选路由与 Attempt 身份不一致');
|
|
597
|
-
const attempt = tasks.beginAttempt(task.id, {
|
|
598
|
-
providerId, providerName: route.provider.name, model, protocol: route.protocol
|
|
599
|
-
});
|
|
600
|
-
tasks.update(task.id, { voice: route.voice });
|
|
601
|
-
return attempt;
|
|
602
|
-
},
|
|
603
|
-
afterResult: (snapshot) => tasks.recordAttemptResult(task.id, snapshot)
|
|
604
|
-
});
|
|
605
|
-
if (submitted.localError) throw taskActionError(task.id, '语音合成结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
606
|
-
if (!submitted.candidate) {
|
|
607
|
-
tasks.finalizeNoAcceptance(task.id, submitted.result?.error || '所有语音候选均明确未受理');
|
|
608
|
-
throw taskActionError(task.id, '所有语音候选均明确未受理', submitted.attempts);
|
|
609
|
-
}
|
|
610
|
-
if (submitted.result.kind !== 'completed') {
|
|
611
|
-
const safe = submitted.result.error && submitted.result.error.safeMessage;
|
|
612
|
-
throw taskActionError(task.id, '语音合成受理状态未知,禁止自动重提' + (safe ? ':' + safe : ''), submitted.result.error);
|
|
613
|
-
}
|
|
614
|
-
const route = submitted.candidate.route;
|
|
615
|
-
const audio = submitted.result.value || {};
|
|
616
|
-
const names = [];
|
|
617
|
-
try {
|
|
618
|
-
tasks.beginDelivery(task.id);
|
|
619
|
-
const name = task.id + '.' + (audio.audioUrl ? adapters.extFromUrl(audio.audioUrl, 'wav') : 'wav');
|
|
620
|
-
const target = path.join(outputDir(), name);
|
|
621
|
-
if (audio.audioUrl) await invokeProviderOperation(route.adapter, 'download', {
|
|
622
|
-
artifact: { kind: 'remote-url', url: audio.audioUrl }, targetPath: target, signal
|
|
623
|
-
});
|
|
624
|
-
else if (audio.audioB64) writePrivateFile(target, Buffer.from(audio.audioB64, 'base64'));
|
|
625
|
-
else throw new Error('语音合成成功响应缺少音频数据');
|
|
626
|
-
names.push(name);
|
|
627
|
-
registerMedia(task.id, target);
|
|
628
|
-
tasks.completeDelivery(task.id, names);
|
|
629
|
-
} catch (error) {
|
|
630
|
-
const recorded = tasks.failDelivery(task.id, error, names);
|
|
631
|
-
throw taskActionError(task.id, '语音已合成,但产物交付失败:' + recorded.error, recorded.lastError);
|
|
632
|
-
}
|
|
633
|
-
return { task: tasks.get(task.id), provider: route.provider, model: route.model };
|
|
634
|
-
}
|
|
635
372
|
|
|
636
373
|
/** 重新观察/交付与启动恢复共享的远端查询和本地落盘依赖。 */
|
|
637
374
|
export function taskPollDeps(provider, task) {
|
|
638
375
|
if (!provider || !task) return null;
|
|
639
376
|
if (!['image', 'video', 'transcribe'].includes(task.cap)) return null;
|
|
640
377
|
const protocol = task.protocol || (task.cap === 'image'
|
|
641
|
-
? (provider.mediaProtocol
|
|
378
|
+
? selectMediaProtocol(provider).mediaProtocol
|
|
642
379
|
: 'dashscope');
|
|
643
380
|
let providerAdapter;
|
|
644
381
|
try {
|
|
645
|
-
providerAdapter = createConfiguredProviderAdapter(provider, { protocol
|
|
382
|
+
providerAdapter = createConfiguredProviderAdapter(provider, { protocol,
|
|
383
|
+
modelId: models.parseModelRef(String(task.modelRef || task.model || ''))?.modelId || String(task.model || '') });
|
|
646
384
|
} catch (_) {
|
|
647
385
|
return null;
|
|
648
386
|
}
|
|
@@ -724,65 +462,185 @@ export async function runAction(host, name, args, { signal } = {}) {
|
|
|
724
462
|
/* ============ 动作实现 ============ */
|
|
725
463
|
|
|
726
464
|
/* ---------- 🖼️ 画图 ---------- */
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
465
|
+
async function imageAction(host, args, { signal }, editing = false) {
|
|
466
|
+
if (editing && !args.source_artifact_id) throw new Error('iris: 改图必须提供 source_artifact_id');
|
|
467
|
+
const { providerInput, sourceArtifactId } = normalizeGenerationInput('image', {
|
|
468
|
+
prompt: args.prompt, size: optionalActionText(args.size), n: args.n,
|
|
469
|
+
...(editing ? { source_artifact_id: args.source_artifact_id } : {})
|
|
470
|
+
});
|
|
730
471
|
|
|
731
472
|
const imageProviders = selectProviders(cap.CAPABILITIES.IMAGE, '图像', args.model);
|
|
732
473
|
const routes = imageProviders.map((provider) => imageRoute(provider, args));
|
|
733
|
-
const submitted = await
|
|
474
|
+
const submitted = await submitCoreImage(routes, providerInput, signal, sourceArtifactId);
|
|
475
|
+
const firstArtifactId = submitted.task.artifactIds[0];
|
|
476
|
+
const ready = submitted.task.outcome === 'succeeded'
|
|
477
|
+
&& submitted.task.deliveryState === 'ready' && Boolean(firstArtifactId);
|
|
734
478
|
return {
|
|
735
|
-
ok: true, taskId: submitted.task.id, model: submitted.model,
|
|
736
|
-
remoteTaskId: submitted.remoteTaskId,
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
479
|
+
ok: true, storage: 'core', taskId: submitted.task.id, model: submitted.model,
|
|
480
|
+
providerId: submitted.provider.id, remoteTaskId: submitted.task.remoteTaskId || null,
|
|
481
|
+
artifactIds: submitted.task.artifactIds,
|
|
482
|
+
...(sourceArtifactId ? { sourceArtifactId } : {}),
|
|
483
|
+
...(ready ? { imageUrl: '/iris/api/core/artifact/' + encodeURIComponent(firstArtifactId) + '/media' } : {}),
|
|
484
|
+
text: ready
|
|
485
|
+
? '[iris] 图像任务已完成(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
486
|
+
: '[iris] 图像任务已提交并由 Core 后台观察(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
740
487
|
};
|
|
741
|
-
}
|
|
488
|
+
}
|
|
489
|
+
register('image', imageAction);
|
|
490
|
+
register('image_edit', (host, args, context) => imageAction(host, args, context, true));
|
|
742
491
|
|
|
743
492
|
/* ---------- 🎬 视频生成 ---------- */
|
|
744
493
|
register('video', async (host, args, { signal }) => {
|
|
745
494
|
const prompt = String(args.prompt || '').trim();
|
|
746
|
-
|
|
747
|
-
const frame = resolveFirstFrameFile(args);
|
|
495
|
+
const frame = await resolveFirstFrameFile(host, args, signal);
|
|
748
496
|
const providers = selectProviders(cap.CAPABILITIES.VIDEO, '视频', args.model);
|
|
749
497
|
const routes = providers.map((provider) => videoRoute(provider, args, frame));
|
|
750
|
-
|
|
498
|
+
// 请求子类型决定候选过滤;池中存在 s2v 不影响普通视频。
|
|
499
|
+
const hasAudioPath = Boolean(String(args.audio_path || '').trim());
|
|
500
|
+
const s2v = hasAudioPath || (args.model && /s2v/i.test(String(args.model)));
|
|
501
|
+
// B-7:audio_path 与选中模型必须语义一致。若用户提供了音频却选中非 s2v 模型,
|
|
502
|
+
// 旧实现会落入 else 分支静默丢弃 audio_path 并生成普通视频(可能产生非预期计费)。
|
|
503
|
+
// 这里在受理边界与任何网络调用之前明确拒绝,不猜测用户意图。
|
|
504
|
+
if (hasAudioPath && args.model && routes.some((route) => !route.s2vModel)) {
|
|
505
|
+
const mismatched = routes.filter((route) => !route.s2vModel).map((route) => route.model).join(', ');
|
|
506
|
+
throw new Error('iris: audio_path 只能配合支持 s2v 数字人的模型;当前候选 ' + mismatched
|
|
507
|
+
+ ' 不是 s2v 模型。请改用 s2v 模型(如 wan2.2-s2v),或移除 audio_path 走文生/图生视频。');
|
|
508
|
+
}
|
|
509
|
+
const coreRoutes = routes.filter((route) => route.isS2V === Boolean(s2v));
|
|
510
|
+
if (!coreRoutes.length) {
|
|
511
|
+
throw new Error(s2v ? 'iris: 未配置可用 s2v 数字人模型' : 'iris: 当前视频候选只有 s2v 数字人模型;请使用 audio_path 的数字人入口');
|
|
512
|
+
}
|
|
513
|
+
const submitted = await submitCoreVideo(coreRoutes, frame, prompt, args, signal);
|
|
514
|
+
const firstArtifactId = submitted.task.artifactIds[0];
|
|
515
|
+
const ready = submitted.task.outcome === 'succeeded'
|
|
516
|
+
&& submitted.task.deliveryState === 'ready' && Boolean(firstArtifactId);
|
|
751
517
|
return {
|
|
752
|
-
ok: true, taskId: submitted.task.id, model: submitted.model, mode: submitted.
|
|
753
|
-
providerId: submitted.provider.id, remoteTaskId: submitted.remoteTaskId,
|
|
754
|
-
|
|
518
|
+
ok: true, storage: 'core', taskId: submitted.task.id, model: submitted.model, mode: submitted.mode,
|
|
519
|
+
providerId: submitted.provider.id, remoteTaskId: submitted.task.remoteTaskId || null,
|
|
520
|
+
artifactIds: submitted.task.artifactIds,
|
|
521
|
+
...(ready ? { videoUrl: '/iris/api/core/artifact/' + encodeURIComponent(firstArtifactId) + '/media' } : {}),
|
|
522
|
+
text: ready
|
|
523
|
+
? '[iris] 视频任务已完成(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
524
|
+
: '[iris] 视频任务已提交并由 Core 后台观察(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
755
525
|
};
|
|
756
526
|
});
|
|
757
527
|
|
|
528
|
+
/**
|
|
529
|
+
* DSH 语音合成入口(E2):TTS 是同步完成型——submit 内 completed,同一次调用
|
|
530
|
+
* 直接交付 generated-audio Artifact(无 remote 长轮询)。整条候选链共用一个
|
|
531
|
+
* voice(Core providerInput 静态;legacy 曾按候选各自取 provider.ttsVoice)。
|
|
532
|
+
*/
|
|
533
|
+
async function submitCoreTts(routes, normalizedInput, signal) {
|
|
534
|
+
const voice = normalizedInput.voice || String(routes[0].provider.ttsVoice || '').trim();
|
|
535
|
+
const { providerInput } = normalizeGenerationInput('tts', {
|
|
536
|
+
text: normalizedInput.text, ...(voice ? { voice } : {})
|
|
537
|
+
});
|
|
538
|
+
const submitted = await submitProviderTaskForDsh({
|
|
539
|
+
capability: 'tts',
|
|
540
|
+
candidates: routes.map((route) => ({
|
|
541
|
+
adapter: route.adapter,
|
|
542
|
+
model: route.modelRef,
|
|
543
|
+
selectionReason: route.selectionReason,
|
|
544
|
+
providerBinding: providerTaskBinding(route.provider)
|
|
545
|
+
})),
|
|
546
|
+
providerInput,
|
|
547
|
+
signal
|
|
548
|
+
});
|
|
549
|
+
const task = submitted.task;
|
|
550
|
+
recordCoreTaskAttemptHealth(task, cap.CAPABILITIES.TTS);
|
|
551
|
+
if (submitted.localError) throw taskActionError(task.id, 'Core 提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
552
|
+
if (task.acceptance === 'unknown') throw taskActionError(task.id, '语音合成受理状态未知,禁止自动重提', task.lastError);
|
|
553
|
+
if (task.acceptance === 'not_accepted') throw taskActionError(task.id, '所有候选均明确未受理', task.attempts);
|
|
554
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
555
|
+
throw taskActionError(task.id, '语音已合成,但 Core Artifact 交付失败;禁止重新生成', task.lastError);
|
|
556
|
+
}
|
|
557
|
+
if (task.outcome !== 'succeeded' || task.deliveryState !== 'ready' || !task.artifactIds.length) {
|
|
558
|
+
throw taskActionError(task.id, '语音合成没有得到可交付产物;事实已保留在 Core,未回退重提', task);
|
|
559
|
+
}
|
|
560
|
+
const selected = routes.find((route) => route.provider.id === task.providerId && route.modelRef === task.modelRef);
|
|
561
|
+
if (!selected) throw taskActionError(task.id, 'Core Task 的供应商身份无法映射回当前路由', task);
|
|
562
|
+
return { task, provider: selected.provider, model: selected.model };
|
|
563
|
+
}
|
|
564
|
+
|
|
758
565
|
/* ---------- 🔊 语音合成 ---------- */
|
|
759
566
|
register('tts', async (host, args, { signal }) => {
|
|
760
|
-
const
|
|
761
|
-
|
|
567
|
+
const { providerInput } = normalizeGenerationInput('tts', {
|
|
568
|
+
text: args.text, voice: optionalActionText(args.voice)
|
|
569
|
+
});
|
|
762
570
|
const providers = selectProviders(cap.CAPABILITIES.TTS, '语音', args.model);
|
|
763
|
-
const routes = providers.map((provider) => (
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
}));
|
|
767
|
-
const submitted = await submitTtsV2(routes, text, args, signal);
|
|
571
|
+
const routes = providers.map((provider) => simpleModelRoute(provider, args, cap.CAPABILITIES.TTS));
|
|
572
|
+
const submitted = await submitCoreTts(routes, providerInput, signal);
|
|
573
|
+
const firstArtifactId = submitted.task.artifactIds[0];
|
|
768
574
|
return {
|
|
769
|
-
ok: true, taskId: submitted.task.id, model: submitted.model,
|
|
770
|
-
|
|
575
|
+
ok: true, storage: 'core', taskId: submitted.task.id, model: submitted.model,
|
|
576
|
+
providerId: submitted.provider.id,
|
|
577
|
+
artifactIds: submitted.task.artifactIds,
|
|
578
|
+
audioUrl: '/iris/api/core/artifact/' + encodeURIComponent(firstArtifactId) + '/media',
|
|
579
|
+
text: '[iris] 语音已合成(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
771
580
|
};
|
|
772
581
|
});
|
|
773
582
|
|
|
583
|
+
/**
|
|
584
|
+
* DSH 转写入口(E3):上传型异步——音频先在 Host 边界上传到首选候选的临时存储
|
|
585
|
+
* (签名 URL 只进 providerInput,绝不落 Core 记录),再经受理边界提交;观察/接管
|
|
586
|
+
* 与人工控制面复用同一 Task/Attempt 轴;正文物化为 text/plain transcript Artifact。
|
|
587
|
+
* 已知边界:临时音频 URL 属于首选候选的账号,failover 到不同账号时如实记录失败。
|
|
588
|
+
*/
|
|
589
|
+
async function submitCoreTranscribe(routes, normalized, signal) {
|
|
590
|
+
let providerInput = normalized.providerInput;
|
|
591
|
+
if (normalized.audioPath) {
|
|
592
|
+
const prepared = await prepareProviderInput(routes[0].adapter, {
|
|
593
|
+
model: routes[0].model, filePath: normalized.audioPath, signal
|
|
594
|
+
});
|
|
595
|
+
providerInput = normalizeGenerationInput('transcribe', { audio_url: prepared.url }).providerInput;
|
|
596
|
+
}
|
|
597
|
+
const submitted = await submitProviderTaskForDsh({
|
|
598
|
+
capability: 'transcribe',
|
|
599
|
+
candidates: routes.map((route) => ({
|
|
600
|
+
adapter: route.adapter,
|
|
601
|
+
model: route.modelRef,
|
|
602
|
+
selectionReason: route.selectionReason,
|
|
603
|
+
providerBinding: providerTaskBinding(route.provider)
|
|
604
|
+
})),
|
|
605
|
+
providerInput,
|
|
606
|
+
signal
|
|
607
|
+
});
|
|
608
|
+
const task = submitted.task;
|
|
609
|
+
recordCoreTaskAttemptHealth(task, cap.CAPABILITIES.TRANSCRIBE);
|
|
610
|
+
if (submitted.localError) throw taskActionError(task.id, 'Core 提交结果未能可靠落盘,已停止后续候选', submitted.localError);
|
|
611
|
+
if (task.acceptance === 'unknown') throw taskActionError(task.id, '转写受理状态未知,禁止自动重提', task.lastError);
|
|
612
|
+
if (task.acceptance === 'not_accepted') throw taskActionError(task.id, '所有候选均明确未受理', task.attempts);
|
|
613
|
+
if (task.outcome === 'succeeded' && task.deliveryState === 'failed') {
|
|
614
|
+
throw taskActionError(task.id, '转写已完成,但 Core Artifact 交付失败;禁止重新生成', task.lastError);
|
|
615
|
+
}
|
|
616
|
+
const selected = routes.find((route) => route.provider.id === task.providerId && route.modelRef === task.modelRef);
|
|
617
|
+
if (!selected) throw taskActionError(task.id, 'Core Task 的供应商身份无法映射回当前路由', task);
|
|
618
|
+
if (task.acceptance === 'accepted' && task.remoteTaskId
|
|
619
|
+
&& ['none', 'unknown'].includes(task.outcome) && task.phase !== 'terminal') {
|
|
620
|
+
await watchProviderTaskForDsh({ taskId: task.id, signal });
|
|
621
|
+
} else if (task.outcome !== 'succeeded' || task.deliveryState !== 'ready' || !task.artifactIds.length) {
|
|
622
|
+
throw taskActionError(task.id, '转写路由没有得到可观察任务或可交付产物;事实已保留在 Core,未回退重提', task);
|
|
623
|
+
}
|
|
624
|
+
return { task, provider: selected.provider, model: selected.model };
|
|
625
|
+
}
|
|
626
|
+
|
|
774
627
|
/* ---------- 👁 看图 ---------- */
|
|
775
628
|
register('look', async (host, args, { signal }) => {
|
|
629
|
+
const source = normalizeVisionImageSource(args);
|
|
776
630
|
const p = String(args.image_path || '').trim();
|
|
777
|
-
if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
778
631
|
const mt = MEDIA_TYPES[path.extname(p).toLowerCase()];
|
|
779
|
-
if (
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
632
|
+
if (source === 'image_path') {
|
|
633
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
634
|
+
if (!mt || mt === 'image/gif') throw new Error('iris: 不支持的图片格式');
|
|
635
|
+
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
|
|
636
|
+
}
|
|
637
|
+
const { answer, via, model } = await askSingleVisionWithHealth(host, { model: args.model,
|
|
638
|
+
question: String(args.question || '').trim() || '详细描述这张图片。',
|
|
639
|
+
prepareImage: operationSignal => source === 'artifact_id'
|
|
640
|
+
? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mt, operationSignal), signal });
|
|
784
641
|
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
785
|
-
return { ok: true,
|
|
642
|
+
return { ok: true, ...(args.artifact_id ? { artifactId: args.artifact_id } : {}),
|
|
643
|
+
text: `[iris] 看图回答(${model} · ${viaLabel}):\n${answer}` };
|
|
786
644
|
});
|
|
787
645
|
|
|
788
646
|
/* ---------- ✂️ 裁剪 ---------- */
|
|
@@ -790,79 +648,121 @@ register('crop', async (host, args, { signal }) => {
|
|
|
790
648
|
const p = String(args.image_path || '').trim();
|
|
791
649
|
if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
792
650
|
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
|
|
793
|
-
const
|
|
794
|
-
|
|
795
|
-
|
|
651
|
+
const result = await cropForDsh({
|
|
652
|
+
imagePath: p, left: args.left, top: args.top, width: args.width, height: args.height, signal
|
|
653
|
+
});
|
|
654
|
+
const b64 = result.bytes.toString('base64');
|
|
655
|
+
return { ok: true, text: `[iris] 裁剪完成:${result.width}x${result.height}(原区域 ${args.left},${args.top},${args.width},${args.height})`, imageDataUrl: `data:${result.mediaType};base64,${b64}` };
|
|
796
656
|
});
|
|
797
657
|
|
|
798
658
|
/* ---------- 📷 像素 diff ---------- */
|
|
799
659
|
register('diff', async (host, args, { signal }) => {
|
|
800
660
|
if (!args.image_a_path || !args.image_b_path) throw new Error('iris: 需要 image_a_path 和 image_b_path');
|
|
801
|
-
const
|
|
802
|
-
|
|
661
|
+
const result = await diffForDsh({
|
|
662
|
+
imageA: { imagePath: args.image_a_path }, imageB: { imagePath: args.image_b_path },
|
|
663
|
+
grid: args.grid, topRegions: args.top_regions, signal
|
|
803
664
|
});
|
|
665
|
+
const { ratio, diffPixels, totalPixels, width, height, worstRegions } = result.metrics;
|
|
804
666
|
const pct = (ratio * 100).toFixed(2);
|
|
805
667
|
const regions = worstRegions.map((r) => `(col ${r.col},row ${r.row}) ${(r.score * 100).toFixed(1)}%`).join(' ');
|
|
806
668
|
return {
|
|
807
669
|
ok: true,
|
|
670
|
+
artifactIds: [result.artifact.id],
|
|
808
671
|
text: `[iris] 像素差异 ${pct}%(${diffPixels}/${totalPixels}px,归一化 ${width}x${height})\n最差区域(${worstRegions.length} 格): ${regions}`,
|
|
809
|
-
imageDataUrl: `data:${
|
|
672
|
+
imageDataUrl: `data:${result.mediaType};base64,${result.bytes.toString('base64')}`
|
|
810
673
|
};
|
|
811
674
|
});
|
|
812
675
|
|
|
813
676
|
/* ---------- 📍 定位 ---------- */
|
|
814
677
|
register('locate', async (host, args, { signal }) => {
|
|
678
|
+
const source = normalizeVisionImageSource(args);
|
|
815
679
|
const p = String(args.image_path || '').trim();
|
|
816
|
-
if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
817
|
-
if (!fs.existsSync(p)) throw new Error('iris: 图片不存在');
|
|
818
|
-
const buf = fs.readFileSync(p);
|
|
680
|
+
if (source === 'image_path' && (!p || !path.isAbsolute(p))) throw new Error('iris: image_path 必须是绝对路径');
|
|
819
681
|
const mt = MEDIA_TYPES[path.extname(p).toLowerCase()] || 'image/png';
|
|
820
|
-
const
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
const
|
|
824
|
-
|
|
825
|
-
return { ok: true, text: `[iris]
|
|
682
|
+
const r = await runLocateRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model,
|
|
683
|
+
target: args.target, prepareImage: operationSignal => source === 'artifact_id'
|
|
684
|
+
? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mt, operationSignal), signal });
|
|
685
|
+
const { width, height } = r;
|
|
686
|
+
const provenance = args.artifact_id ? { artifactId: args.artifact_id } : {};
|
|
687
|
+
if (!r.found) return { ok: true, ...provenance, text: `[iris] 在图片中未找到「${args.target}」` };
|
|
688
|
+
return { ok: true, ...provenance, text: `[iris] 定位「${args.target}」:bbox (${r.x1},${r.y1},${r.x2},${r.y2}) / ${width}x${height}(${r.via} · ${r.model})` };
|
|
826
689
|
});
|
|
827
690
|
/* ---------- 🖼️ HTML 截图 ---------- */
|
|
828
691
|
register('html', async (host, args, { signal }) => {
|
|
829
692
|
const html = String(args.html || '').trim();
|
|
830
693
|
if (!html) throw new Error('iris: html 不能为空');
|
|
831
694
|
const browser = requireHostPort(host, 'browser', 'HTML 截图');
|
|
832
|
-
const rendered = await
|
|
695
|
+
const rendered = await htmlForDsh({ browser, html, width: args.width, height: args.height,
|
|
833
696
|
fullPage: args.fullPage !== false, signal });
|
|
834
697
|
const png = Buffer.from(rendered.bytes);
|
|
835
|
-
return { ok: true, text: `[iris] HTML 截图完成(fullPage: ${args.fullPage !== false})`, imageDataUrl: `data
|
|
698
|
+
return { ok: true, storage: 'core', artifactIds: [rendered.artifact.id], text: `[iris] HTML 截图完成(fullPage: ${args.fullPage !== false})`, imageDataUrl: `data:image/png;base64,${png.toString('base64')}` };
|
|
836
699
|
});
|
|
837
700
|
|
|
838
701
|
/* ---------- 📄 长截图 OCR ---------- */
|
|
839
702
|
register('ocr', async (host, args, { signal }) => {
|
|
703
|
+
const source = normalizeVisionImageSource(args);
|
|
840
704
|
const p = String(args.image_path || '').trim();
|
|
841
|
-
|
|
842
|
-
if (
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
const
|
|
847
|
-
|
|
705
|
+
const mediaType = MEDIA_TYPES[path.extname(p).toLowerCase()];
|
|
706
|
+
if (source === 'image_path') {
|
|
707
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: image_path 必须是绝对路径');
|
|
708
|
+
if (!mediaType || mediaType === 'image/gif') throw new Error('iris: 不支持的图片格式(png/jpg/jpeg/webp)');
|
|
709
|
+
}
|
|
710
|
+
const result = await runOcrRequest(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION),
|
|
711
|
+
onRateLimit: store.recordRateLimit,
|
|
712
|
+
prepareImage: operationSignal => source === 'artifact_id'
|
|
713
|
+
? visionImageForDsh({ artifactId: args.artifact_id, signal: operationSignal }) : readVisionImageFile(p, mediaType, operationSignal),
|
|
714
|
+
chunkHeight: args.chunk_height, overlap: args.overlap, signal });
|
|
715
|
+
return { ok: result.status !== 'failed', status: result.status,
|
|
716
|
+
totalChunks: result.totalChunks, successfulChunks: result.successfulChunks,
|
|
717
|
+
failedChunks: result.failedChunks, skippedChunks: result.skippedChunks,
|
|
718
|
+
...(args.artifact_id ? { artifactId: args.artifact_id } : {}), text: formatOcrResult(result) };
|
|
848
719
|
});
|
|
849
720
|
|
|
850
721
|
/* ---------- 📋 任务查询 ---------- */
|
|
851
722
|
register('status', async (host, args, { signal }) => {
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
723
|
+
const requestedId = String(args.task_id || '').trim();
|
|
724
|
+
if (requestedId) {
|
|
725
|
+
const legacy = tasks.get(requestedId);
|
|
726
|
+
if (legacy) {
|
|
727
|
+
const lines = [`[${legacy.cap}] ${legacy.id} · ${legacy.status}${legacy.progress ? ' · ' + legacy.progress : ''} · ${legacy.model}`];
|
|
728
|
+
if (legacy.prompt) lines.push('prompt: ' + String(legacy.prompt).slice(0, 120));
|
|
729
|
+
if (legacy.error) lines.push('error: ' + legacy.error);
|
|
730
|
+
if ((legacy.files || []).length) lines.push('files:\n' + legacy.files.map((f) => path.join(outputDir(), f)).join('\n'));
|
|
731
|
+
const links = mediaLinksOf(legacy);
|
|
732
|
+
if (links.length) lines.push('links:\n' + links.join('\n'));
|
|
733
|
+
return { ok: true, storage: 'legacy', taskId: legacy.id, text: lines.join('\n') };
|
|
734
|
+
}
|
|
735
|
+
if (/^task_[a-f0-9]{24}$/.test(requestedId)) {
|
|
736
|
+
try {
|
|
737
|
+
const core = await inspectProviderTaskForDsh(requestedId, { signal });
|
|
738
|
+
const lines = [`[${core.capability}] ${core.id} · Core ${core.phase} · ${core.outcome} / ${core.deliveryState} · ${core.modelRef || '未选择模型'}`];
|
|
739
|
+
if (core.lastError) lines.push('error: ' + core.lastError.safeMessage);
|
|
740
|
+
if (core.remoteTaskId) lines.push('remoteTask: ' + core.remoteTaskId);
|
|
741
|
+
if (core.artifactIds.length) {
|
|
742
|
+
lines.push('artifacts:\n' + core.artifactIds.map((id) => id + ' · /iris/api/core/artifact/' + encodeURIComponent(id) + '/media').join('\n'));
|
|
743
|
+
}
|
|
744
|
+
return { ok: true, storage: 'core', taskId: core.id, artifactIds: core.artifactIds, text: lines.join('\n') };
|
|
745
|
+
} catch (_) { /* 对外统一为不存在,不暴露 Core 数据根 */ }
|
|
746
|
+
}
|
|
747
|
+
throw new Error('iris: 没有这个任务:' + args.task_id);
|
|
862
748
|
}
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
749
|
+
|
|
750
|
+
const legacyRows = tasks.list().slice(0, 5).map((task) => ({
|
|
751
|
+
at: task.createdAt,
|
|
752
|
+
text: `[${task.cap}] ${task.id} · ${task.status} · ${task.model}`
|
|
753
|
+
}));
|
|
754
|
+
let coreRows = [];
|
|
755
|
+
try {
|
|
756
|
+
const snapshot = await coreSnapshotForDsh({ limit: 5, signal });
|
|
757
|
+
coreRows = (snapshot.tasks.recent || []).map((task) => ({
|
|
758
|
+
at: task.updatedAt,
|
|
759
|
+
text: `[${task.capability}] ${task.id} · Core ${task.phase} · ${task.outcome} / ${task.deliveryState} · ${task.modelRef || '未选择模型'}`
|
|
760
|
+
}));
|
|
761
|
+
} catch (_) { /* legacy 查询仍然可用 */ }
|
|
762
|
+
const rows = legacyRows.concat(coreRows)
|
|
763
|
+
.sort((a, b) => String(b.at || '').localeCompare(String(a.at || '')))
|
|
764
|
+
.slice(0, 5);
|
|
765
|
+
return { ok: true, text: rows.length ? rows.map((item) => item.text).join('\n') : '[iris] 还没有生成任务记录。' };
|
|
866
766
|
});
|
|
867
767
|
|
|
868
768
|
/* ---------- Task v2 人工接管 ---------- */
|
|
@@ -1064,22 +964,23 @@ register('relook', async (host, args, { signal }) => {
|
|
|
1064
964
|
if (!id) throw new Error('iris: attachment_id 不能为空');
|
|
1065
965
|
const question = String(args.question || '').trim();
|
|
1066
966
|
if (!question) throw new Error('iris: question 不能为空');
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
const
|
|
1073
|
-
|
|
967
|
+
const { answer, via, model } = await askSingleVisionWithHealth(host, { question, model: args.model,
|
|
968
|
+
async prepareImage(operationSignal) {
|
|
969
|
+
// GUI 没有会话上下文;记录扫描也在预取消检查及整体预算之后执行。
|
|
970
|
+
let hit;
|
|
971
|
+
for (const task of tasks.all()) {
|
|
972
|
+
for (const attachment of task.attachments || []) {
|
|
973
|
+
if (attachment.attachmentId === id) {
|
|
974
|
+
const absPath = path.join(store.irisHome(), 'outputs', attachment.file);
|
|
975
|
+
if (fs.existsSync(absPath)) { hit = { absPath, mediaType: attachment.mediaType }; break; }
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
if (hit) break;
|
|
1074
979
|
}
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
const mt = hit.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png';
|
|
1080
|
-
const dataUrl = `data:${mt};base64,${fs.readFileSync(hit.absPath).toString('base64')}`;
|
|
1081
|
-
const backends = buildVisionBackendsFromHost(host, { providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model });
|
|
1082
|
-
const { answer, via, model } = await askWithHealth(backends, { question, imageDataUrl: dataUrl, signal });
|
|
980
|
+
if (!hit) throw new Error('iris: 图片引用不可用');
|
|
981
|
+
const mt = hit.mediaType || MEDIA_TYPES[path.extname(hit.absPath).toLowerCase()] || 'image/png';
|
|
982
|
+
return readVisionImageFile(hit.absPath, mt, operationSignal);
|
|
983
|
+
}, signal });
|
|
1083
984
|
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
1084
985
|
return { ok: true, text: `[iris] 重看回答(${model} · ${viaLabel}):\n${answer}` };
|
|
1085
986
|
});
|
|
@@ -1096,12 +997,19 @@ register('providers_list', async (host, args) => {
|
|
|
1096
997
|
name: p.name || p.id,
|
|
1097
998
|
type: p.type || 'openai',
|
|
1098
999
|
baseUrl: p.baseUrl,
|
|
1000
|
+
mediaBaseUrl: p.mediaBaseUrl || '',
|
|
1099
1001
|
enabled: p.enabled !== false,
|
|
1100
|
-
|
|
1002
|
+
auth: p.auth === 'none' ? 'none' : 'bearer',
|
|
1003
|
+
...selectMediaProtocol(p),
|
|
1004
|
+
visionInput: p.visionInput || null,
|
|
1101
1005
|
apiKeyHint: String(p.apiKey || '').length <= 8 ? '****' : String(p.apiKey).slice(0, 3) + '****' + String(p.apiKey).slice(-4),
|
|
1102
1006
|
models: models.providerModels(p).map((m) => ({
|
|
1103
1007
|
id: m.id,
|
|
1104
1008
|
capabilities: m.capabilities,
|
|
1009
|
+
...(m.imageProtocol !== undefined ? { imageProtocol: m.imageProtocol } : {}),
|
|
1010
|
+
...(m.visionInput !== undefined ? { visionInput: m.visionInput } : {}),
|
|
1011
|
+
...(m.capabilities.includes(cap.CAPABILITIES.VISION) ? { visionInputEffective: visionInputLimits(p, m.id) } : {}),
|
|
1012
|
+
...(m.capabilities.includes(cap.CAPABILITIES.IMAGE) ? { imageRouting: selectImageProtocol(p, m.id) } : {}),
|
|
1105
1013
|
verified: m.verified || null,
|
|
1106
1014
|
health: Object.fromEntries(m.capabilities.map((capability) => [
|
|
1107
1015
|
capability, store.modelHealth(p.id, m.id, capability)
|
|
@@ -1117,8 +1025,8 @@ register('providers_list', async (host, args) => {
|
|
|
1117
1025
|
register('providers_upsert', async (host, args) => {
|
|
1118
1026
|
const id = args.id || undefined;
|
|
1119
1027
|
const patch = { id };
|
|
1120
|
-
const allowed = ['name', 'type', 'baseUrl', 'apiKey', 'enabled', 'mediaProtocol',
|
|
1121
|
-
'imageModel', 'videoModel', 'ttsModel', 'transcribeModel', 'visionModel', 'models'];
|
|
1028
|
+
const allowed = ['auth', 'name', 'type', 'baseUrl', 'mediaBaseUrl', 'apiKey', 'enabled', 'mediaProtocol',
|
|
1029
|
+
'imageModel', 'videoModel', 'ttsModel', 'transcribeModel', 'visionModel', 'visionInput', 'models'];
|
|
1122
1030
|
for (const key of allowed) {
|
|
1123
1031
|
if (Object.prototype.hasOwnProperty.call(args, key)) patch[key] = args[key];
|
|
1124
1032
|
}
|
|
@@ -1141,44 +1049,42 @@ register('providers_set_models', async (host, args) => {
|
|
|
1141
1049
|
});
|
|
1142
1050
|
|
|
1143
1051
|
/**
|
|
1144
|
-
*
|
|
1052
|
+
* 模型发现:读取供应商目录 → 官方能力元数据优先、名称规则兜底 → 写入显式池。
|
|
1145
1053
|
* 免费只读;拉不到/不支持则原样保留现有池(优雅降级),报人话错误。
|
|
1146
1054
|
*/
|
|
1147
1055
|
register('providers_discover', async (host, args, { signal }) => {
|
|
1148
1056
|
const p = store.providerById(String(args.id || ''));
|
|
1149
1057
|
if (!p) throw new Error('iris: 供应商不存在或未启用');
|
|
1150
|
-
let
|
|
1058
|
+
let discoveredModels;
|
|
1151
1059
|
try {
|
|
1152
1060
|
const providerAdapter = createConfiguredProviderAdapter(p);
|
|
1153
1061
|
const discovered = await invokeProviderOperation(providerAdapter, 'discover', { signal });
|
|
1154
|
-
|
|
1062
|
+
discoveredModels = discovered.models;
|
|
1155
1063
|
} catch (err) {
|
|
1156
1064
|
throw new Error('iris: 模型发现失败(保留现有池):' + String(err.message || err));
|
|
1157
1065
|
}
|
|
1158
1066
|
const media = [];
|
|
1159
|
-
for (const
|
|
1160
|
-
const caps = models.
|
|
1161
|
-
if (caps.length) media.push({ id, capabilities: caps });
|
|
1067
|
+
for (const model of discoveredModels) {
|
|
1068
|
+
const caps = models.capabilitiesOfDiscoveredModel(model);
|
|
1069
|
+
if (caps.length) media.push({ id: model.id, capabilities: caps });
|
|
1162
1070
|
}
|
|
1163
1071
|
if (!media.length) {
|
|
1164
|
-
return { ok: true, total:
|
|
1072
|
+
return { ok: true, total: discoveredModels.length, media: 0, text: `[iris] ${p.name}:拉到 ${discoveredModels.length} 个模型,无媒体能力匹配——可在模型池区手动添加并标能力` };
|
|
1165
1073
|
}
|
|
1166
|
-
store.
|
|
1167
|
-
return { ok: true, total:
|
|
1074
|
+
store.mergeDiscoveredModels(p.id, media);
|
|
1075
|
+
return { ok: true, total: discoveredModels.length, media: media.length, text: `[iris] ${p.name}:发现 ${discoveredModels.length} 个模型,${media.length} 个媒体能力模型已入池(官方元数据优先)` };
|
|
1168
1076
|
});
|
|
1169
1077
|
|
|
1170
1078
|
/** 能力实测(红色测试图):对某供应商的 vision 模型跑 testVisionCapability */
|
|
1171
1079
|
register('providers_test_vision', async (host, args, { signal }) => {
|
|
1172
1080
|
const p = store.providerById(String(args.id || ''));
|
|
1173
1081
|
if (!p) throw new Error('iris: 供应商不存在');
|
|
1174
|
-
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION).filter((x) => x.id === p.id);
|
|
1082
|
+
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION, { includeRateLimited: true }).filter((x) => x.id === p.id);
|
|
1175
1083
|
if (!visionProviders.length) return { ok: true, tested: false, text: '该供应商未配置 vision 模型,跳过' };
|
|
1176
|
-
const
|
|
1177
|
-
const backend = backends[0];
|
|
1178
|
-
if (!backend) return { ok: true, tested: false, text: '该供应商的 vision 配置不可用(类型非 openai)' };
|
|
1084
|
+
const model = configuredModel(visionProviders[0], cap.CAPABILITIES.VISION);
|
|
1179
1085
|
if (args.confirm_paid !== true) throw new Error('iris: 视觉实测会发起真实供应商请求,请先明确确认');
|
|
1180
|
-
const r = await
|
|
1181
|
-
store.setModelVerified(p.id,
|
|
1086
|
+
const r = await probeVisionModel(visionProviders[0], model, visionImageFromDataUrl(RED_TEST_IMAGE), { timeoutMs: 15000, signal });
|
|
1087
|
+
store.setModelVerified(p.id, model, cap.CAPABILITIES.VISION, {
|
|
1182
1088
|
ok: r.ok,
|
|
1183
1089
|
note: r.ok ? '认出红色测试图' : (r.error || '未识别出红色'),
|
|
1184
1090
|
category: r.category,
|
|
@@ -1189,7 +1095,7 @@ register('providers_test_vision', async (host, args, { signal }) => {
|
|
|
1189
1095
|
tested: true,
|
|
1190
1096
|
passed: r.ok,
|
|
1191
1097
|
text: r.ok
|
|
1192
|
-
? `[iris] 视觉能力实测通过:模型认出红色测试图(${
|
|
1098
|
+
? `[iris] 视觉能力实测通过:模型认出红色测试图(${model})`
|
|
1193
1099
|
: `[iris] 视觉能力实测未通过:${r.error || '未识别出红色'}`
|
|
1194
1100
|
};
|
|
1195
1101
|
});
|
|
@@ -1199,9 +1105,7 @@ register('providers_test_vision', async (host, args, { signal }) => {
|
|
|
1199
1105
|
/** 按能力实测单个模型;返回 {ok, note}。真实调用仅在 UI 明确确认后执行。 */
|
|
1200
1106
|
async function probeModel(host, p, modelId, capability, signal) {
|
|
1201
1107
|
if (capability === cap.CAPABILITIES.VISION) {
|
|
1202
|
-
const
|
|
1203
|
-
if (!backend) return { ok: false, note: '非 openai 类型,无法直测' };
|
|
1204
|
-
const r = await testVisionCapability(backend, { timeoutMs: 15000, signal });
|
|
1108
|
+
const r = await probeVisionModel(p, modelId, visionImageFromDataUrl(RED_TEST_IMAGE), { timeoutMs: 15000, signal });
|
|
1205
1109
|
return {
|
|
1206
1110
|
ok: r.ok,
|
|
1207
1111
|
note: r.ok ? '认出红色测试图' : (r.error || '未识别出红色'),
|
|
@@ -1210,24 +1114,28 @@ async function probeModel(host, p, modelId, capability, signal) {
|
|
|
1210
1114
|
};
|
|
1211
1115
|
}
|
|
1212
1116
|
if (capability === cap.CAPABILITIES.TRANSCRIBE) {
|
|
1213
|
-
return { skipped: true, ok: false, note: '
|
|
1117
|
+
return { skipped: true, ok: false, note: '需真实音频;可用 CLI models test --capability transcribe --input 提供素材进行实测恢复' };
|
|
1214
1118
|
}
|
|
1215
1119
|
if (capability === cap.CAPABILITIES.TTS) {
|
|
1216
|
-
const providerAdapter = createConfiguredProviderAdapter(p, {
|
|
1120
|
+
const providerAdapter = createConfiguredProviderAdapter(p, { allowRateLimited: true });
|
|
1217
1121
|
const submitted = await invokeProviderOperation(providerAdapter, 'submit', {
|
|
1218
|
-
capability: 'tts', model: modelId, input: { text: '你好'
|
|
1122
|
+
capability: 'tts', model: modelId, input: { text: '你好' }, signal
|
|
1219
1123
|
});
|
|
1220
1124
|
const value = submitted.kind === 'completed' ? submitted.value : null;
|
|
1221
1125
|
const ok = Boolean(value && (value.audioUrl || value.audioB64));
|
|
1222
|
-
return { ok, note: ok ? '合成返回音频' : (submitted.error?.safeMessage || '无音频返回')
|
|
1126
|
+
return { ok, note: ok ? '合成返回音频' : (submitted.error?.safeMessage || '无音频返回'),
|
|
1127
|
+
category: submitted.error?.category, httpStatus: submitted.error?.httpStatus };
|
|
1223
1128
|
}
|
|
1224
1129
|
if (capability === cap.CAPABILITIES.IMAGE) {
|
|
1225
|
-
|
|
1130
|
+
p = providerForImageModel(p, modelId);
|
|
1131
|
+
const providerAdapter = createConfiguredProviderAdapter(p, { allowRateLimited: true, modelId });
|
|
1226
1132
|
const submitted = await invokeProviderOperation(providerAdapter, 'submit', {
|
|
1227
1133
|
capability: 'image', model: modelId,
|
|
1228
1134
|
input: {
|
|
1229
1135
|
prompt: providerAdapter.protocol === 'openai-images' ? 'red circle' : '红色圆形',
|
|
1230
|
-
|
|
1136
|
+
...(['openai-chat-images', 'openai-responses-images'].includes(providerAdapter.protocol) ? {} : {
|
|
1137
|
+
size: providerAdapter.protocol === 'openai-images' ? '256x256' : '512*512'
|
|
1138
|
+
}),
|
|
1231
1139
|
n: 1
|
|
1232
1140
|
},
|
|
1233
1141
|
signal
|
|
@@ -1264,7 +1172,7 @@ async function probeModel(host, p, modelId, capability, signal) {
|
|
|
1264
1172
|
return { ok: false, note: '生成超时(>60s)' };
|
|
1265
1173
|
}
|
|
1266
1174
|
if (capability === cap.CAPABILITIES.VIDEO) {
|
|
1267
|
-
return { skipped: true, ok: false, note: '
|
|
1175
|
+
return { skipped: true, ok: false, note: '视频提交即可能计费;可用 CLI models test --capability video-gen --input 提供素材进行实测恢复' };
|
|
1268
1176
|
}
|
|
1269
1177
|
return { ok: false, note: '未知能力' };
|
|
1270
1178
|
}
|
|
@@ -1297,6 +1205,19 @@ register('providers_set_model_caps', async (host, args) => {
|
|
|
1297
1205
|
return { ok: true, text: `[iris] ${args.model_id} 能力已设为:${caps.join(',') || '(空)'}` };
|
|
1298
1206
|
});
|
|
1299
1207
|
|
|
1208
|
+
register('providers_set_model_image_protocol', async (host, args) => {
|
|
1209
|
+
const ok = store.setModelImageProtocol(String(args.id || ''), String(args.model_id || ''), args.imageProtocol);
|
|
1210
|
+
if (!ok) throw new Error('iris: 模型不在池中');
|
|
1211
|
+
return { ok: true, text: '[iris] 模型图片协议已更新' };
|
|
1212
|
+
});
|
|
1213
|
+
|
|
1214
|
+
register('providers_set_model_vision_input', async (host, args) => {
|
|
1215
|
+
if (!Object.hasOwn(args, 'visionInput')) throw new Error('iris: 需要 visionInput 对象或 null');
|
|
1216
|
+
const ok = store.setModelVisionInput(String(args.id || ''), String(args.model_id || ''), args.visionInput);
|
|
1217
|
+
if (!ok) throw new Error('iris: 模型不在池中');
|
|
1218
|
+
return { ok: true, text: '[iris] 模型看图输入预算已更新' };
|
|
1219
|
+
});
|
|
1220
|
+
|
|
1300
1221
|
/** 逐模型实测某能力 → 存 verified("test it, don't guess it";用户手动触发,发现不自动跑) */
|
|
1301
1222
|
register('providers_test_model', async (host, args, { signal }) => {
|
|
1302
1223
|
const p = store.providerById(String(args.id || ''));
|
|
@@ -1436,41 +1357,45 @@ register('attachment_export', async (host, args, { signal }) => {
|
|
|
1436
1357
|
/* ============ 音频转写(阶段 7.2) ============ */
|
|
1437
1358
|
|
|
1438
1359
|
register('transcribe', async (host, args, { signal }) => {
|
|
1439
|
-
const
|
|
1440
|
-
|
|
1441
|
-
|
|
1360
|
+
const normalized = normalizeGenerationInput('transcribe', {
|
|
1361
|
+
audio_url: args.audio_url, audio_path: args.audio_path
|
|
1362
|
+
}, { allowAudioPath: true });
|
|
1363
|
+
if (normalized.audioPath && !fs.existsSync(normalized.audioPath)) throw new Error('iris: 音频文件不存在');
|
|
1442
1364
|
const providers = selectProviders(cap.CAPABILITIES.TRANSCRIBE, '转写', args.model);
|
|
1443
1365
|
const routes = providers.map((provider) =>
|
|
1444
|
-
simpleModelRoute(provider, args,
|
|
1366
|
+
simpleModelRoute(provider, args, cap.CAPABILITIES.TRANSCRIBE)
|
|
1445
1367
|
);
|
|
1446
|
-
const submitted = await
|
|
1368
|
+
const submitted = await submitCoreTranscribe(routes, normalized, signal);
|
|
1369
|
+
const firstArtifactId = submitted.task.artifactIds[0];
|
|
1370
|
+
const ready = submitted.task.outcome === 'succeeded'
|
|
1371
|
+
&& submitted.task.deliveryState === 'ready' && Boolean(firstArtifactId);
|
|
1447
1372
|
return {
|
|
1448
|
-
ok: true, taskId: submitted.task.id, model: submitted.model,
|
|
1449
|
-
remoteTaskId: submitted.remoteTaskId,
|
|
1450
|
-
|
|
1373
|
+
ok: true, storage: 'core', taskId: submitted.task.id, model: submitted.model,
|
|
1374
|
+
providerId: submitted.provider.id, remoteTaskId: submitted.task.remoteTaskId || null,
|
|
1375
|
+
artifactIds: submitted.task.artifactIds,
|
|
1376
|
+
...(ready ? { transcriptUrl: '/iris/api/core/artifact/' + encodeURIComponent(firstArtifactId) + '/media' } : {}),
|
|
1377
|
+
text: ready
|
|
1378
|
+
? '[iris] 音频转写已完成(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
1379
|
+
: '[iris] 音频转写任务已提交并由 Core 后台观察(' + submitted.model + ',Core task: ' + submitted.task.id + ')'
|
|
1451
1380
|
};
|
|
1452
1381
|
});
|
|
1453
1382
|
|
|
1454
1383
|
/* ============ 视频抽帧(阶段 7.1) ============ */
|
|
1455
1384
|
|
|
1456
1385
|
register('video_frames', async (host, args, { signal }) => {
|
|
1457
|
-
const
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg/ffprobe 不可用,无法抽帧(可选安装 ffmpeg 后启用)');
|
|
1461
|
-
const meta = probeVideo(p);
|
|
1462
|
-
const frames = await extractFrames({
|
|
1463
|
-
inputPath: p, maxFrames: args.max_frames, targetWidth: args.target_width,
|
|
1464
|
-
format: args.format || 'jpeg', quality: args.quality, signal
|
|
1386
|
+
const { media: meta, frames } = await framesForDsh({
|
|
1387
|
+
videoPath: args.video_path, maxFrames: args.max_frames, targetWidth: args.target_width,
|
|
1388
|
+
format: optionalActionText(args.format), quality: args.quality, signal
|
|
1465
1389
|
});
|
|
1466
|
-
const mime =
|
|
1390
|
+
const mime = frames[0].mediaType;
|
|
1467
1391
|
const summary = frames.map((f, i) =>
|
|
1468
|
-
`帧${i + 1} @${f.atSec.toFixed(1)}s ${f.width}x${f.height} ${(f.
|
|
1392
|
+
`帧${i + 1} @${f.atSec.toFixed(1)}s ${f.width}x${f.height} ${(f.bytes.length / 1024).toFixed(0)}KB`
|
|
1469
1393
|
).join('\n');
|
|
1470
1394
|
// 首帧预览 base64
|
|
1471
|
-
const firstFrame = `data:${mime};base64,${frames[0].
|
|
1395
|
+
const firstFrame = `data:${mime};base64,${frames[0].bytes.toString('base64')}`;
|
|
1472
1396
|
return {
|
|
1473
1397
|
ok: true,
|
|
1398
|
+
artifactIds: frames.map((frame) => frame.artifact.id),
|
|
1474
1399
|
text: `[iris] 视频抽帧完成:${frames.length} 帧(${meta.durationSec.toFixed(1)}s ${meta.width}x${meta.height},${mime === 'image/png' ? 'PNG' : 'JPEG'})\n${summary}\n\n(预览为首帧;全部帧请用对话工具 iris_video_frames 获取 attachment)`,
|
|
1475
1400
|
imageDataUrl: firstFrame
|
|
1476
1401
|
};
|
|
@@ -1479,28 +1404,32 @@ register('video_frames', async (host, args, { signal }) => {
|
|
|
1479
1404
|
/* ============ 多模态视频摘要(阶段 7.3,GUI 同步快速版) ============ */
|
|
1480
1405
|
|
|
1481
1406
|
register('media_summarize', async (host, args, { signal }) => {
|
|
1407
|
+
const source = normalizeSummarySource(args);
|
|
1482
1408
|
const p = String(args.video_path || '').trim();
|
|
1483
|
-
if (
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1409
|
+
if (source === 'video') {
|
|
1410
|
+
if (!p || !path.isAbsolute(p)) throw new Error('iris: video_path 必须是绝对路径');
|
|
1411
|
+
if (!fs.existsSync(p)) throw new Error('iris: 视频文件不存在');
|
|
1412
|
+
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg/ffprobe 不可用,无法分析视频(可选安装 ffmpeg 后启用)');
|
|
1413
|
+
}
|
|
1414
|
+
const { answer, via, model, sheet, meta, frames } = await runSummaryRequest(host, {
|
|
1415
|
+
onRateLimit: store.recordRateLimit,
|
|
1416
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), model: args.model, question: args.question, signal,
|
|
1417
|
+
async prepareMedia(operationSignal) {
|
|
1418
|
+
if (source === 'core-artifacts') {
|
|
1419
|
+
return { ...await summaryFramesForDsh({ frameArtifactIds: args.frame_artifact_ids, signal: operationSignal }), transcript: args.transcribe_text };
|
|
1420
|
+
}
|
|
1421
|
+
const meta = probeVideo(p);
|
|
1422
|
+
const frames = await extractFrames({ inputPath: p, maxFrames: Math.min(Number(args.max_frames) || 8, 12),
|
|
1423
|
+
targetWidth: args.target_width, format: 'jpeg', quality: 70, signal: operationSignal });
|
|
1424
|
+
return { meta, frames, transcript: args.transcribe_text };
|
|
1425
|
+
}
|
|
1498
1426
|
});
|
|
1499
1427
|
const viaLabel = via === 'selfstack' ? 'iris 自持栈' : 'DSH 全局视觉模型';
|
|
1500
1428
|
const hasTrans = args.transcribe_text && String(args.transcribe_text).trim();
|
|
1501
1429
|
return {
|
|
1502
1430
|
ok: true,
|
|
1503
|
-
|
|
1431
|
+
frames: summaryFrameRecords(frames),
|
|
1432
|
+
text: `[iris] 视频摘要完成(${model} · ${viaLabel},${summaryTimelineLabel(meta)} / ${frames.length} 帧联系表${hasTrans ? ' + 转写文本' : ''}):\n${answer}`,
|
|
1504
1433
|
imageDataUrl: `data:image/png;base64,${sheet.buffer.toString('base64')}`
|
|
1505
1434
|
};
|
|
1506
1435
|
});
|