@mokuyoaxis/dsh-iris 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh/skills/iris-use/SKILL.md +2 -2
- package/.dsh/skills/iris-use/references/media-contract.md +2 -2
- package/CHANGELOG.md +27 -1
- package/README.en.md +27 -19
- package/README.md +28 -18
- package/bin/dsh-iris.js +29 -12
- package/docs/AGENT_QUICKSTART.md +7 -7
- package/docs/CLI_MANAGEMENT.md +5 -1
- package/docs/DSH_COMPATIBILITY.md +8 -4
- package/docs/DSH_PROVIDER_CONNECTIONS.md +34 -0
- package/docs/FUNCTION_MODEL_ASSIGNMENTS.md +2 -0
- package/docs/FUNCTION_MODEL_POOLS.md +84 -0
- package/docs/HEADLESS_CLI.md +7 -7
- package/docs/HOST_ADAPTER_CONTRACT.md +7 -2
- package/docs/HOST_DOCTOR.md +2 -2
- package/docs/MUSIC.md +2 -2
- package/docs/PROVIDER_HEALTH.md +26 -2
- package/docs/ROADMAP.md +5 -3
- package/docs/SECURITY.md +12 -2
- package/docs/VISION_CLI.md +2 -2
- package/docs/releases/0.2.6.md +50 -0
- package/lib/actions.js +137 -30
- package/lib/adapters.js +36 -9
- package/lib/api.js +2 -1
- package/lib/client.js +717 -180
- package/lib/composite-vision-routing.js +6 -6
- package/lib/config.js +140 -29
- package/lib/doctor.js +8 -1
- package/lib/dsh-core-adapter.js +22 -11
- package/lib/dsh-host-adapter.js +10 -0
- package/lib/dsh-provider-config.js +35 -0
- package/lib/dsh-provider-source.js +102 -0
- package/lib/headless-vision.js +5 -5
- package/lib/host-contract.js +1 -0
- package/lib/http-vision-model-adapter.js +5 -2
- package/lib/index.js +54 -35
- package/lib/model-directory-service.js +53 -0
- package/lib/model-directory.js +103 -0
- package/lib/models.js +71 -6
- package/lib/ocr-model-routing.js +2 -2
- package/lib/provider-adapters.js +42 -9
- package/lib/provider-catalog.js +14 -9
- package/lib/provider-config-scope.js +16 -0
- package/lib/provider-config-service.js +82 -13
- package/lib/provider-health.js +33 -0
- package/lib/provider-protocol.js +30 -0
- package/lib/vision-model-routing.js +25 -12
- package/package.json +2 -2
- package/user_guide.md +25 -19
package/lib/headless-vision.js
CHANGED
|
@@ -70,9 +70,9 @@ function completionResult(completion, routes) {
|
|
|
70
70
|
return { ...result, ...selectedModel(completion, routes) };
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
async function transcribeVideo({ catalog, input, runtime, operation, onRateLimit }) {
|
|
73
|
+
async function transcribeVideo({ catalog, input, runtime, operation, onRateLimit, onModelAvailability }) {
|
|
74
74
|
const routes = transcribeCandidatesFromCatalog(catalog, input.transcribe_model_ref);
|
|
75
|
-
const candidates = routes.map(route => ({ adapter: createConfiguredProviderAdapter(route.provider, { onRateLimit }),
|
|
75
|
+
const candidates = routes.map(route => ({ adapter: createConfiguredProviderAdapter(route.provider, { onRateLimit, onModelAvailability }),
|
|
76
76
|
model: route.modelRef, selectionReason: route.selectionReason, providerBinding: providerTaskBinding(route.provider) }));
|
|
77
77
|
const audio = await waitForModelWork(() => extractAudioTrack({ inputPath: input.video_path, signal: operation.signal }), operation.signal);
|
|
78
78
|
const abortRuntime = () => { void runtime.dispose().catch(() => {}); };
|
|
@@ -111,7 +111,7 @@ async function transcribeVideo({ catalog, input, runtime, operation, onRateLimit
|
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
-
export async function executeVisionCommand({ command, input, catalog, modelRef = '', runtime, signal, timeoutMs = VISION_BUDGET.timeoutMs, onRateLimit }) {
|
|
114
|
+
export async function executeVisionCommand({ command, input, catalog, modelRef = '', runtime, signal, timeoutMs = VISION_BUDGET.timeoutMs, onRateLimit, onModelAvailability }) {
|
|
115
115
|
normalizeVisionInput(command, input);
|
|
116
116
|
if (modelRef && !/^[^:]+::.+$/.test(modelRef)) invalid('--model-ref 必须是 providerId::modelId');
|
|
117
117
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > VISION_BUDGET.timeoutMs) invalid('--timeout-ms 必须是 1–120000 的整数');
|
|
@@ -121,7 +121,7 @@ export async function executeVisionCommand({ command, input, catalog, modelRef =
|
|
|
121
121
|
const inputs = new Map();
|
|
122
122
|
const prepareImageRequest = command === 'locate' ? createLocateImageRequest(input.target, inputs)
|
|
123
123
|
: command === 'ocr' ? createVisionInputObserver(inputs) : undefined;
|
|
124
|
-
const ports = routes.map(route => createConfiguredVisionModelPort(route.provider, route.model, { onRateLimit,
|
|
124
|
+
const ports = routes.map(route => createConfiguredVisionModelPort(route.provider, route.model, { onRateLimit, onModelAvailability,
|
|
125
125
|
prepareImages: ['look', 'ocr', 'locate'].includes(command), orientImages: command === 'locate', prepareImageRequest }));
|
|
126
126
|
const budget = { ...VISION_BUDGET, timeoutMs };
|
|
127
127
|
const operation = createModelOperation({ signal, budget: { timeoutMs, maxInvocations: command === 'ocr' ? input.max_invocations ?? 64 : ports.length } });
|
|
@@ -140,7 +140,7 @@ export async function executeVisionCommand({ command, input, catalog, modelRef =
|
|
|
140
140
|
}
|
|
141
141
|
let transcript = input.transcribe_text || '', transcription = { status: transcript.trim() ? 'provided' : 'disabled' };
|
|
142
142
|
if (input.transcribe) {
|
|
143
|
-
const asr = meta.hasAudio ? await transcribeVideo({ catalog, input, runtime, operation, onRateLimit }) : { report: { status: 'no_audio' }, text: '' };
|
|
143
|
+
const asr = meta.hasAudio ? await transcribeVideo({ catalog, input, runtime, operation, onRateLimit, onModelAvailability }) : { report: { status: 'no_audio' }, text: '' };
|
|
144
144
|
transcript = asr.text; transcription = asr.report;
|
|
145
145
|
}
|
|
146
146
|
const completion = await summarizeMedia({ ports, frames, question: input.question, transcript, operation, budget });
|
package/lib/host-contract.js
CHANGED
|
@@ -5,6 +5,7 @@ import { ModelPortError, normalizeModelDescriptor, normalizeModelCall, normalize
|
|
|
5
5
|
import { modelCallScope, modelAbortError, waitForModelWork } from './model-call-runtime.js';
|
|
6
6
|
import { createHash } from 'node:crypto';
|
|
7
7
|
import { isFreeQuotaExhausted, retryAfterDelay } from './provider-health.js';
|
|
8
|
+
import { isModelNotFound } from './model-directory.js';
|
|
8
9
|
import { prepareVisionImage } from './vision-image-input.js';
|
|
9
10
|
|
|
10
11
|
const object = value => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
@@ -103,7 +104,7 @@ export function createHttpVisionModelPort({ providerId, modelId, baseUrl, apiKey
|
|
|
103
104
|
});
|
|
104
105
|
}, signal, 'invoke', 'unknown');
|
|
105
106
|
if (!response.ok) {
|
|
106
|
-
const body = [400, 403, 413, 429].includes(response.status) ? await waitForModelWork(() => response.json().catch(() => null), signal, 'read', 'rejected') : null;
|
|
107
|
+
const body = [400, 403, 404, 413, 429].includes(response.status) ? await waitForModelWork(() => response.json().catch(() => null), signal, 'read', 'rejected') : null;
|
|
107
108
|
// 只提取可验证的数值事实,不把供应商原文或凭据带到 UI/CLI。
|
|
108
109
|
const limit = [400, 413].includes(response.status)
|
|
109
110
|
? /\b(\d+) bytes exceeds the (\d+)-byte limit\b/i.exec(String(body?.error?.message || body?.message || '')) : null;
|
|
@@ -114,10 +115,12 @@ export function createHttpVisionModelPort({ providerId, modelId, baseUrl, apiKey
|
|
|
114
115
|
imageBytes: image.bytes.byteLength, ...(maxBytes ? { imageMaxBytes: maxBytes } : {}) });
|
|
115
116
|
}
|
|
116
117
|
const quota = isFreeQuotaExhausted({ status: response.status, ...(body?.error || body) });
|
|
117
|
-
const
|
|
118
|
+
const missing = isModelNotFound({ status: response.status, ...(body?.error || body) });
|
|
119
|
+
const code = missing ? 'IRIS_MODEL_UNAVAILABLE' : quota ? 'IRIS_MODEL_RATE_LIMITED' : [401, 403].includes(response.status) ? 'IRIS_MODEL_AUTH_FAILED'
|
|
118
120
|
: response.status === 429 ? 'IRIS_MODEL_RATE_LIMITED' : response.status === 400 ? 'IRIS_MODEL_INPUT_INVALID' : 'IRIS_MODEL_REQUEST_FAILED';
|
|
119
121
|
const error = new ModelPortError(code, { stage: 'invoke', invocation: code === 'IRIS_MODEL_REQUEST_FAILED' ? 'unknown' : 'rejected', status: response.status });
|
|
120
122
|
error.providerCode = (body?.error || body)?.code;
|
|
123
|
+
if (missing) error.providerCode = 'model_not_found';
|
|
121
124
|
if (response.status === 429) error.retryAfterMs = retryAfterDelay(response.headers.get('retry-after'));
|
|
122
125
|
if (quota) error.quotaExhausted = true;
|
|
123
126
|
throw error;
|
package/lib/index.js
CHANGED
|
@@ -18,6 +18,8 @@ import { runVisionRequest, readVisionImageFile } from './vision-model-routing.js
|
|
|
18
18
|
import { waitForModelWork } from './model-call-runtime.js';
|
|
19
19
|
import { hasHostPort, requireHostPort } from './host-contract.js';
|
|
20
20
|
import { createDshHostAdapter } from './dsh-host-adapter.js';
|
|
21
|
+
import { withDshProviderConfig } from './dsh-provider-config.js';
|
|
22
|
+
import { configureProviderHostForDsh } from './dsh-core-adapter.js';
|
|
21
23
|
import { cropForDsh, diffForDsh, framesForDsh, summaryFramesForDsh, visionImageForDsh, htmlForDsh, inspectProviderTaskForDsh, projectCoreTaskForDsh, readCoreArtifactMediaForDsh, resumeProviderTaskWatchesForDsh, stopProviderTaskWatchesForDsh } from './dsh-core-adapter.js';
|
|
22
24
|
import { imageDimensions } from './pixels.js';
|
|
23
25
|
import { runLocateRequest, runSummaryRequest, describeGeneratedImage } from './composite-vision-routing.js';
|
|
@@ -219,6 +221,7 @@ function safeRegister(host, definition) {
|
|
|
219
221
|
|
|
220
222
|
export async function apply(ctx) {
|
|
221
223
|
beginHostRuntime({ pluginId: IRIS_PLUGIN_ID, version: PACKAGE_VERSION });
|
|
224
|
+
const releaseProviderHost = configureProviderHostForDsh(() => dshHost(ctx));
|
|
222
225
|
guard('Iris 私有存储权限收紧', () => hardenPrivateTree(store.irisHome()));
|
|
223
226
|
|
|
224
227
|
guard('上传临时文件清理', () => purgeStaleUploads());
|
|
@@ -254,6 +257,7 @@ export async function apply(ctx) {
|
|
|
254
257
|
ctx.effect(() => () => {
|
|
255
258
|
tasks.stopWatchAll();
|
|
256
259
|
stopProviderTaskWatchesForDsh();
|
|
260
|
+
releaseProviderHost();
|
|
257
261
|
});
|
|
258
262
|
});
|
|
259
263
|
|
|
@@ -274,6 +278,20 @@ export async function apply(ctx) {
|
|
|
274
278
|
});
|
|
275
279
|
|
|
276
280
|
const host = dshHost(ctx);
|
|
281
|
+
const providerTools = new Set(['iris_draw_image', 'iris_edit_image', 'iris_generate_video', 'iris_generate_music',
|
|
282
|
+
'iris_speak_text', 'iris_transcribe_audio', 'iris_look_at_image', 'iris_relook_attachment',
|
|
283
|
+
'iris_media_summarize', 'iris_locate', 'iris_long_ocr']);
|
|
284
|
+
const visionTools = new Set(['iris_look_at_image', 'iris_relook_attachment', 'iris_media_summarize', 'iris_locate', 'iris_long_ocr']);
|
|
285
|
+
function registerIrisTool(definition) {
|
|
286
|
+
safeRegister(host, { ...definition, execute(args, exec) {
|
|
287
|
+
const execute = async () => {
|
|
288
|
+
if (visionTools.has(definition.name) && !args?.model) await store.refreshAutomaticDirectories({ capability: 'vision', signal: exec.signal });
|
|
289
|
+
return definition.execute(args, exec);
|
|
290
|
+
};
|
|
291
|
+
return providerTools.has(definition.name)
|
|
292
|
+
? withDshProviderConfig(dshHost(ctx), execute, { signal: exec.signal }) : execute();
|
|
293
|
+
} });
|
|
294
|
+
}
|
|
277
295
|
if (!hasHostPort(host, 'tools')) {
|
|
278
296
|
console.error('[iris] tools 服务不可用,跳过全部工具注册(宿主不受影响)');
|
|
279
297
|
return;
|
|
@@ -291,7 +309,7 @@ export async function apply(ctx) {
|
|
|
291
309
|
let note = '';
|
|
292
310
|
try {
|
|
293
311
|
const described = await describeGeneratedImage(dshHost(ctx), {
|
|
294
|
-
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal,
|
|
312
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability, originalPrompt: prompt, signal,
|
|
295
313
|
async prepareImage(inputSignal) {
|
|
296
314
|
const first = await readCoreArtifactMediaForDsh(projected.artifacts[0].artifact.id, { signal: inputSignal });
|
|
297
315
|
return { bytes: new Uint8Array(first.bytes), mediaType: first.artifact.mediaType };
|
|
@@ -309,7 +327,7 @@ export async function apply(ctx) {
|
|
|
309
327
|
return { blocks };
|
|
310
328
|
}
|
|
311
329
|
|
|
312
|
-
|
|
330
|
+
registerIrisTool({
|
|
313
331
|
name: 'iris_import_image',
|
|
314
332
|
description: 'Save a local image or a current-session image attachment as a persistent Iris image Artifact for later viewing, editing and export. Provide exactly one of image_path or attachment_id. Supports static PNG/JPEG/WebP up to 20 MiB, preserves the source, and never calls a model.',
|
|
315
333
|
parameters: {
|
|
@@ -334,7 +352,7 @@ export async function apply(ctx) {
|
|
|
334
352
|
});
|
|
335
353
|
|
|
336
354
|
/* ---------- 🖼️ 画图 ---------- */
|
|
337
|
-
|
|
355
|
+
registerIrisTool({
|
|
338
356
|
name: 'iris_draw_image',
|
|
339
357
|
description:
|
|
340
358
|
'Generate an image from a detailed prompt using the user-configured image model and its configured image protocol. ' +
|
|
@@ -345,7 +363,7 @@ export async function apply(ctx) {
|
|
|
345
363
|
prompt: { type: 'string', description: 'Detailed visual description of the image to create' },
|
|
346
364
|
size: { type: 'string', description: "Output size; DashScope uses 'W*H' (e.g. '1024*1024'), OpenAI uses 'WxH'. Default provider default." },
|
|
347
365
|
n: { type: 'string', description: "Number of images, default '1'" },
|
|
348
|
-
model: { type: 'string', description: 'Optional
|
|
366
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled function pool; selects only this item (legacy bare model id also accepted)' }
|
|
349
367
|
},
|
|
350
368
|
required: ['prompt']
|
|
351
369
|
},
|
|
@@ -390,7 +408,7 @@ export async function apply(ctx) {
|
|
|
390
408
|
let note = '';
|
|
391
409
|
try {
|
|
392
410
|
const described = await describeGeneratedImage(dshHost(ctx), {
|
|
393
|
-
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, originalPrompt: prompt, signal: exec.signal,
|
|
411
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability, originalPrompt: prompt, signal: exec.signal,
|
|
394
412
|
prepareImage: signal => readVisionImageFile(files[0], MEDIA_TYPES[path.extname(files[0]).toLowerCase()] || 'image/png', signal)
|
|
395
413
|
});
|
|
396
414
|
if (described) note = `画面内容:${described}`;
|
|
@@ -405,14 +423,14 @@ export async function apply(ctx) {
|
|
|
405
423
|
}
|
|
406
424
|
});
|
|
407
425
|
|
|
408
|
-
|
|
426
|
+
registerIrisTool({
|
|
409
427
|
name: 'iris_edit_image',
|
|
410
|
-
description: 'Edit an existing Core image by Artifact ID and an instruction using an
|
|
428
|
+
description: 'Edit an existing Core image by Artifact ID and an instruction using an allowed chat-image model or native DashScope Qwen edit model. Returns new Core Artifacts and DSH image attachments; preserves the source image and records its provenance.',
|
|
411
429
|
parameters: {
|
|
412
430
|
type: 'object', properties: {
|
|
413
431
|
source_artifact_id: { type: 'string', description: 'Core Artifact ID of the source PNG, JPEG or WebP image (up to 20 MiB)' },
|
|
414
432
|
prompt: { type: 'string', description: 'Describe the changes and what should be preserved in the source image' },
|
|
415
|
-
model: { type: 'string', description: 'Optional
|
|
433
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled image pool; selects only this item; supports openai-chat-images or DashScope Qwen edit/max/plus and 2.0/pro models' },
|
|
416
434
|
n: { type: 'string', description: "Number of images, default '1'" }
|
|
417
435
|
}, required: ['source_artifact_id', 'prompt']
|
|
418
436
|
},
|
|
@@ -425,7 +443,7 @@ export async function apply(ctx) {
|
|
|
425
443
|
});
|
|
426
444
|
|
|
427
445
|
/* ---------- 🎬 视频生成 ---------- */
|
|
428
|
-
|
|
446
|
+
registerIrisTool({
|
|
429
447
|
name: 'iris_generate_video',
|
|
430
448
|
description:
|
|
431
449
|
'Generate a video with the configured DashScope video model. Three modes: ' +
|
|
@@ -443,7 +461,7 @@ export async function apply(ctx) {
|
|
|
443
461
|
resolution: { type: 'string', description: "s2v only: output tier '480P' (default) or '720P'" },
|
|
444
462
|
size: { type: 'string', description: "t2v/i2v only: output size 'W*H', e.g. '1280*720' (default)" },
|
|
445
463
|
duration: { type: 'number', description: 't2v/i2v only: duration in seconds, if the model supports it' },
|
|
446
|
-
model: { type: 'string', description: 'Optional
|
|
464
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled function pool; selects only this item (legacy bare model id accepted)' }
|
|
447
465
|
},
|
|
448
466
|
required: []
|
|
449
467
|
},
|
|
@@ -487,12 +505,12 @@ export async function apply(ctx) {
|
|
|
487
505
|
}
|
|
488
506
|
});
|
|
489
507
|
|
|
490
|
-
|
|
508
|
+
registerIrisTool({
|
|
491
509
|
name: 'iris_generate_music',
|
|
492
|
-
description: 'Generate music using a model
|
|
510
|
+
description: 'Generate music using a model in the enabled Music pool. Returns audio artifact IDs and playback links. Lyrics, vocals and duration are prompt intentions. Configured 502 retries may generate again.',
|
|
493
511
|
parameters: { type: 'object', properties: {
|
|
494
512
|
prompt: { type: 'string', description: 'Describe the music, including any lyrics, vocals, mood or intended duration' },
|
|
495
|
-
model: { type: 'string', description: 'Optional
|
|
513
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled Music pool; selects only this item' },
|
|
496
514
|
timeout_ms: { type: 'integer', description: 'Per-attempt timeout; default 300000 ms' },
|
|
497
515
|
retry_502: { type: 'integer', description: '502 regeneration retries, 0–5; default 2; each retry consumes another request' },
|
|
498
516
|
retry_delay_ms: { type: 'integer', description: 'Delay between 502 retries; default 5000 ms' }
|
|
@@ -507,7 +525,7 @@ export async function apply(ctx) {
|
|
|
507
525
|
});
|
|
508
526
|
|
|
509
527
|
/* ---------- 🔊 语音合成 ---------- */
|
|
510
|
-
|
|
528
|
+
registerIrisTool({
|
|
511
529
|
name: 'iris_speak_text',
|
|
512
530
|
description: 'Synthesize spoken audio from text using the configured TTS model. Returns Core audio Artifact IDs and playback links; export an Artifact to a host file before passing it as audio_path to another tool. Use iris_generate_music for songs or background music.',
|
|
513
531
|
parameters: {
|
|
@@ -515,7 +533,7 @@ export async function apply(ctx) {
|
|
|
515
533
|
properties: {
|
|
516
534
|
text: { type: 'string', description: 'Text to speak' },
|
|
517
535
|
voice: { type: 'string', description: "Voice name, e.g. 'Cherry' (DashScope qwen-tts voices)" },
|
|
518
|
-
model: { type: 'string', description: 'Optional
|
|
536
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled function pool; selects only this item (legacy bare model id accepted)' }
|
|
519
537
|
},
|
|
520
538
|
required: ['text']
|
|
521
539
|
},
|
|
@@ -545,14 +563,14 @@ export async function apply(ctx) {
|
|
|
545
563
|
});
|
|
546
564
|
|
|
547
565
|
/* ---------- 🎙️ 音频转写(阶段 7.2) ---------- */
|
|
548
|
-
|
|
566
|
+
registerIrisTool({
|
|
549
567
|
name: 'iris_transcribe_audio',
|
|
550
568
|
description: 'Transcribe an audio file (wav/mp3) to text using the configured DashScope provider (qwen-audio-3.0-asr-flash-filetrans). The audio is uploaded to Bailian temp storage, then an async task is submitted. Returns the full recognized text.',
|
|
551
569
|
parameters: {
|
|
552
570
|
type: 'object',
|
|
553
571
|
properties: {
|
|
554
572
|
audio_path: { type: 'string', description: 'Absolute path to the audio file (wav/mp3)' },
|
|
555
|
-
model: { type: 'string', description: 'Optional
|
|
573
|
+
model: { type: 'string', description: 'Optional providerId::modelId in the enabled transcription pool; selects only this item (legacy bare model id accepted)' }
|
|
556
574
|
},
|
|
557
575
|
required: ['audio_path']
|
|
558
576
|
},
|
|
@@ -588,7 +606,7 @@ export async function apply(ctx) {
|
|
|
588
606
|
});
|
|
589
607
|
|
|
590
608
|
/* ---------- 🎞️ 视频抽帧(阶段 7.1,ffmpeg 可选) ---------- */
|
|
591
|
-
|
|
609
|
+
registerIrisTool({
|
|
592
610
|
name: 'iris_video_frames',
|
|
593
611
|
description:
|
|
594
612
|
'Extract N frames (uniformly sampled across the video) from a local video file using the system ffmpeg. ' +
|
|
@@ -646,7 +664,7 @@ export async function apply(ctx) {
|
|
|
646
664
|
});
|
|
647
665
|
|
|
648
666
|
/* ---------- 📝 多模态视频摘要(阶段 7.3,ffmpeg 可选) ---------- */
|
|
649
|
-
|
|
667
|
+
registerIrisTool({
|
|
650
668
|
name: 'iris_media_summarize',
|
|
651
669
|
description:
|
|
652
670
|
'Summarize a local video or existing Core video-frame Artifacts as one contact sheet (with timestamps) plus an optional ' +
|
|
@@ -664,7 +682,7 @@ export async function apply(ctx) {
|
|
|
664
682
|
target_width: { type: 'integer', description: 'Frame width for sampling (default 640)' },
|
|
665
683
|
transcribe: { type: 'boolean', description: 'Auto-transcribe the audio track for video_path (default true unless transcribe_text is provided). Existing frames only consume explicit text.' },
|
|
666
684
|
transcribe_text: { type: 'string', description: 'Optional existing transcript text; does not create a transcription task' },
|
|
667
|
-
model: { type: 'string', description: 'Optional item from the vision
|
|
685
|
+
model: { type: 'string', description: 'Optional item from the enabled vision pool (providerId::modelId or host:current-vision); selects only this item' }
|
|
668
686
|
},
|
|
669
687
|
required: []
|
|
670
688
|
},
|
|
@@ -679,7 +697,7 @@ export async function apply(ctx) {
|
|
|
679
697
|
if (!ffmpegAvailable()) throw new Error('iris: ffmpeg 不可用,无法分析视频(可选安装 ffmpeg/ffprobe 后启用)');
|
|
680
698
|
}
|
|
681
699
|
const { answer, via, model, sheet, meta, frames, transcriptNote } = await runSummaryRequest(dshHost(ctx), {
|
|
682
|
-
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, question: args.question, signal: exec.signal,
|
|
700
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability, model: args.model, question: args.question, signal: exec.signal,
|
|
683
701
|
async prepareMedia(signal) {
|
|
684
702
|
if (source === 'core-artifacts') {
|
|
685
703
|
return { ...await summaryFramesForDsh({ frameArtifactIds: args.frame_artifact_ids, signal }), transcript: args.transcribe_text };
|
|
@@ -745,7 +763,7 @@ export async function apply(ctx) {
|
|
|
745
763
|
|
|
746
764
|
/* ---------- 👁 视觉路由(M3):显式工具,自持栈为主 ---------- */
|
|
747
765
|
|
|
748
|
-
|
|
766
|
+
registerIrisTool({
|
|
749
767
|
name: 'iris_look_at_image',
|
|
750
768
|
description:
|
|
751
769
|
'Look at a local image file or an existing Core image Artifact and answer one question with the configured vision model (self-hosted provider stack, then DSH default vision). Provide exactly one of image_path or artifact_id. Local files are saved as durable DSH attachments; Core images are read directly without export or Core writes.',
|
|
@@ -755,7 +773,7 @@ export async function apply(ctx) {
|
|
|
755
773
|
image_path: { type: 'string', description: 'Absolute path to the image file (png/jpg/jpeg/webp)' },
|
|
756
774
|
artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path)' },
|
|
757
775
|
question: { type: 'string', description: 'Question or extraction request about the image; defaults to a detailed description' },
|
|
758
|
-
model: { type: 'string', description: 'Optional item from the vision
|
|
776
|
+
model: { type: 'string', description: 'Optional item from the enabled vision pool (providerId::modelId or host:current-vision); selects only this item' }
|
|
759
777
|
},
|
|
760
778
|
required: []
|
|
761
779
|
},
|
|
@@ -784,7 +802,7 @@ export async function apply(ctx) {
|
|
|
784
802
|
}
|
|
785
803
|
});
|
|
786
804
|
|
|
787
|
-
|
|
805
|
+
registerIrisTool({
|
|
788
806
|
name: 'iris_relook_attachment',
|
|
789
807
|
description:
|
|
790
808
|
'Ask a NEW question about an image that was already seen in this session: pass its attachment_id (from user uploads, tool results, or any iris-generated image). Pixels are re-read via the vision model.',
|
|
@@ -793,7 +811,7 @@ export async function apply(ctx) {
|
|
|
793
811
|
properties: {
|
|
794
812
|
attachment_id: { type: 'string', description: 'The attachment_id of an image that appeared in this session or was generated by iris' },
|
|
795
813
|
question: { type: 'string', description: 'New question about the image pixels' },
|
|
796
|
-
model: { type: 'string', description: 'Optional item from the vision
|
|
814
|
+
model: { type: 'string', description: 'Optional item from the enabled vision pool (providerId::modelId or host:current-vision); selects only this item' }
|
|
797
815
|
},
|
|
798
816
|
required: ['attachment_id', 'question']
|
|
799
817
|
},
|
|
@@ -820,7 +838,7 @@ export async function apply(ctx) {
|
|
|
820
838
|
});
|
|
821
839
|
|
|
822
840
|
/* ---------- 📋 任务查询 ---------- */
|
|
823
|
-
|
|
841
|
+
registerIrisTool({
|
|
824
842
|
name: 'iris_task_status',
|
|
825
843
|
description:
|
|
826
844
|
'Query Iris tasks (image/video/tts/music/transcription). Pass a task_id for one task, or omit it to list recent tasks. Core results provide Artifact IDs and playback links; legacy tasks may provide file paths. Querying does not re-observe or redeliver a task.',
|
|
@@ -840,7 +858,7 @@ export async function apply(ctx) {
|
|
|
840
858
|
});
|
|
841
859
|
|
|
842
860
|
/* ---------- ✂️ 确定性像素工具(阶段 2) ---------- */
|
|
843
|
-
|
|
861
|
+
registerIrisTool({
|
|
844
862
|
name: 'iris_crop',
|
|
845
863
|
description:
|
|
846
864
|
'Crop a rectangular region from an image (absolute local path or a session attachment_id) and save the result as a durable DSH attachment. ' +
|
|
@@ -890,7 +908,7 @@ export async function apply(ctx) {
|
|
|
890
908
|
}
|
|
891
909
|
});
|
|
892
910
|
|
|
893
|
-
|
|
911
|
+
registerIrisTool({
|
|
894
912
|
name: 'iris_pixel_diff',
|
|
895
913
|
description:
|
|
896
914
|
'Compute a pixel-level difference between two images (absolute local paths or session attachment_ids). ' +
|
|
@@ -945,7 +963,7 @@ export async function apply(ctx) {
|
|
|
945
963
|
});
|
|
946
964
|
|
|
947
965
|
/* ---------- 📍 模型驱动定位(阶段 3A) ---------- */
|
|
948
|
-
|
|
966
|
+
registerIrisTool({
|
|
949
967
|
name: 'iris_locate',
|
|
950
968
|
description:
|
|
951
969
|
'Locate an object/region described by `target` in an image (exactly one of absolute image_path, Core artifact_id, or session attachment_id), ' +
|
|
@@ -958,7 +976,7 @@ export async function apply(ctx) {
|
|
|
958
976
|
artifact_id: { type: 'string', description: 'Existing Core image Artifact ID in the current profile (alternative to image_path or attachment_id)' },
|
|
959
977
|
attachment_id: { type: 'string', description: 'Session attachment id of an image (alternative to image_path)' },
|
|
960
978
|
target: { type: 'string', description: 'The object or UI element to locate, e.g. "send button"' },
|
|
961
|
-
model: { type: 'string', description: 'Optional item from the vision
|
|
979
|
+
model: { type: 'string', description: 'Optional item from the enabled vision pool (providerId::modelId or host:current-vision); selects only this item' }
|
|
962
980
|
},
|
|
963
981
|
required: ['target']
|
|
964
982
|
},
|
|
@@ -966,7 +984,7 @@ export async function apply(ctx) {
|
|
|
966
984
|
isConcurrencySafe: () => true,
|
|
967
985
|
async execute(args, exec) {
|
|
968
986
|
const r = await runLocateRequest(dshHost(ctx), {
|
|
969
|
-
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, model: args.model, target: args.target,
|
|
987
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability, model: args.model, target: args.target,
|
|
970
988
|
prepareImage: signal => resolveVisionImageInput(dshHost(ctx), exec, args, signal), signal: exec.signal
|
|
971
989
|
});
|
|
972
990
|
const { width, height } = r;
|
|
@@ -980,7 +998,7 @@ export async function apply(ctx) {
|
|
|
980
998
|
});
|
|
981
999
|
|
|
982
1000
|
/* ---------- 🖼️ HTML 截图(阶段 3C,基于 dsh-builtin-browser) ---------- */
|
|
983
|
-
|
|
1001
|
+
registerIrisTool({
|
|
984
1002
|
name: 'iris_html_screenshot',
|
|
985
1003
|
description:
|
|
986
1004
|
'Render an HTML string to a page screenshot using the shared browser. ' +
|
|
@@ -1033,7 +1051,7 @@ export async function apply(ctx) {
|
|
|
1033
1051
|
});
|
|
1034
1052
|
|
|
1035
1053
|
/* ---------- 📄 长截图 OCR(阶段 3B,视觉模型分块) ---------- */
|
|
1036
|
-
|
|
1054
|
+
registerIrisTool({
|
|
1037
1055
|
name: 'iris_long_ocr',
|
|
1038
1056
|
description:
|
|
1039
1057
|
'OCR a (possibly long) image from exactly one of image_path, Core artifact_id, or session attachment_id: slice it into chunks (1200px tall with 120px overlap by default) ' +
|
|
@@ -1055,7 +1073,7 @@ export async function apply(ctx) {
|
|
|
1055
1073
|
async execute(args, exec) {
|
|
1056
1074
|
const host = dshHost(ctx);
|
|
1057
1075
|
const result = await runOcrRequest(host, {
|
|
1058
|
-
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit,
|
|
1076
|
+
providers: store.pickAllFor(cap.CAPABILITIES.VISION), onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability,
|
|
1059
1077
|
prepareImage: operationSignal => resolveVisionImageInput(host, exec, args, operationSignal),
|
|
1060
1078
|
chunkHeight: args.chunk_height,
|
|
1061
1079
|
overlap: args.overlap,
|
|
@@ -1180,7 +1198,8 @@ export async function sessionAttachmentRef(host, exec, attachmentId) {
|
|
|
1180
1198
|
*/
|
|
1181
1199
|
export async function askVision(host, { question, dataUrl, image, prepareImage, signal, model, budget }) {
|
|
1182
1200
|
const visionProviders = store.pickAllFor(cap.CAPABILITIES.VISION);
|
|
1183
|
-
return runVisionRequest(host, { providers: visionProviders, model, question, dataUrl, image, prepareImage, signal, budget,
|
|
1201
|
+
return runVisionRequest(host, { providers: visionProviders, model, question, dataUrl, image, prepareImage, signal, budget,
|
|
1202
|
+
onRateLimit: store.recordRateLimit, onModelAvailability: store.recordAvailability });
|
|
1184
1203
|
}
|
|
1185
1204
|
|
|
1186
1205
|
/**
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { listModels } from './adapters.js';
|
|
2
|
+
import { capabilitiesOfDiscoveredModel } from './models.js';
|
|
3
|
+
import { providerApiKey, providerMediaBaseUrl, selectMediaProtocol } from './provider-protocol.js';
|
|
4
|
+
import { redactProviderMessage } from './provider-contract.js';
|
|
5
|
+
import { MODEL_DIRECTORY_TTL_MS, MODEL_DIRECTORY_RETRY_MS, currentModelDirectory, modelDirectoryBinding } from './model-directory.js';
|
|
6
|
+
|
|
7
|
+
const pending = new Map();
|
|
8
|
+
|
|
9
|
+
/** 只有完整发现成功才替换快照;相同连接合并网络读取,各调用方保存自己的配置。 */
|
|
10
|
+
export async function refreshModelDirectory(provider, { force = false, signal, save, discover = listModels, now = Date.now() } = {}) {
|
|
11
|
+
if (!['dashscope', 'openai-images', 'openai-chat-images', 'openai-responses-images', 'gemini-web-music'].includes(selectMediaProtocol(provider).mediaProtocol)) {
|
|
12
|
+
return { refreshed: false, error: '当前协议未提供模型目录发现' };
|
|
13
|
+
}
|
|
14
|
+
const previous = currentModelDirectory(provider);
|
|
15
|
+
if (!force && previous) {
|
|
16
|
+
const stale = Boolean(previous.lastError) || !previous.checkedAt || now - Date.parse(previous.checkedAt) >= MODEL_DIRECTORY_TTL_MS
|
|
17
|
+
|| Date.parse(previous.invalidatedAt || '') >= Date.parse(previous.checkedAt);
|
|
18
|
+
if (!stale || (previous.failedAt && now - Date.parse(previous.failedAt) < MODEL_DIRECTORY_RETRY_MS)) {
|
|
19
|
+
return { refreshed: false, directory: previous, ...(previous.lastError ? { error: previous.lastError } : {}) };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
signal?.throwIfAborted();
|
|
23
|
+
const binding = modelDirectoryBinding(provider);
|
|
24
|
+
let job = pending.get(binding);
|
|
25
|
+
if (!job) {
|
|
26
|
+
const budget = AbortSignal.timeout(15000);
|
|
27
|
+
job = discover({ key: providerApiKey(provider), baseUrl: providerMediaBaseUrl(provider),
|
|
28
|
+
signal: signal ? AbortSignal.any([signal, budget]) : budget, timeoutMs: 15000 });
|
|
29
|
+
pending.set(binding, job);
|
|
30
|
+
job.finally(() => { if (pending.get(binding) === job) pending.delete(binding); }).catch(() => {});
|
|
31
|
+
}
|
|
32
|
+
let discovered;
|
|
33
|
+
try {
|
|
34
|
+
discovered = await job;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
signal?.throwIfAborted();
|
|
37
|
+
const key = providerApiKey(provider);
|
|
38
|
+
const message = String(error?.message || error);
|
|
39
|
+
const directory = { version: 1, binding, ...(previous || {}), failedAt: new Date(now).toISOString(),
|
|
40
|
+
lastError: redactProviderMessage(key ? message.split(key).join('[REDACTED]') : message) };
|
|
41
|
+
if (await save(directory) === false) return { refreshed: false, error: '连接在目录同步期间发生变化,请重新同步' };
|
|
42
|
+
return { refreshed: false, directory, error: directory.lastError };
|
|
43
|
+
}
|
|
44
|
+
signal?.throwIfAborted();
|
|
45
|
+
const models = discovered.map(model => ({ id: typeof model === 'string' ? model : model.id,
|
|
46
|
+
capabilities: capabilitiesOfDiscoveredModel(model) }));
|
|
47
|
+
const known = new Set(previous?.models?.map(model => model.id)
|
|
48
|
+
|| provider.models?.map(model => typeof model === 'string' ? model : model.id) || []);
|
|
49
|
+
const directory = { version: 1, binding, checkedAt: new Date(now).toISOString(), models,
|
|
50
|
+
added: models.filter(model => !known.has(model.id)).map(model => model.id) };
|
|
51
|
+
if (await save(directory) === false) return { refreshed: false, error: '连接在目录同步期间发生变化,请重新同步' };
|
|
52
|
+
return { refreshed: true, directory };
|
|
53
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import { isKnownCapability } from './capability.js';
|
|
3
|
+
import { providerApiKey, providerMediaBaseUrl, selectMediaProtocol, providerForImageModel, providerForMusicModel } from './provider-protocol.js';
|
|
4
|
+
|
|
5
|
+
export const MODEL_DIRECTORY_TTL_MS = 10 * 60 * 1000;
|
|
6
|
+
export const MODEL_DIRECTORY_RETRY_MS = 30 * 1000;
|
|
7
|
+
|
|
8
|
+
/** 配置是外部输入,目录只接受不含连接凭据的固定数据形状。 */
|
|
9
|
+
export function validateModelDirectory(value) {
|
|
10
|
+
if (value === undefined) return;
|
|
11
|
+
if (!value || value.version !== 1 || !/^[a-f0-9]{64}$/.test(value.binding)
|
|
12
|
+
|| (value.checkedAt !== undefined && !Number.isFinite(Date.parse(value.checkedAt)))
|
|
13
|
+
|| (value.checkedAt && !Array.isArray(value.models))
|
|
14
|
+
|| (value.models !== undefined && (!Array.isArray(value.models) || value.models.some(model =>
|
|
15
|
+
!model || typeof model.id !== 'string' || !model.id.trim() || !Array.isArray(model.capabilities)
|
|
16
|
+
|| !model.capabilities.every(isKnownCapability))))
|
|
17
|
+
|| (value.added !== undefined && (!Array.isArray(value.added) || !value.added.every(id => typeof id === 'string')))) {
|
|
18
|
+
throw new TypeError('模型目录快照结构无效');
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** 摘要绑定实际连接,不保存或回传密钥。 */
|
|
23
|
+
export function modelDirectoryBinding(provider) {
|
|
24
|
+
return crypto.createHash('sha256').update(JSON.stringify([
|
|
25
|
+
provider.id, provider.type || '', provider.baseUrl || '', providerMediaBaseUrl(provider), selectMediaProtocol(provider).mediaProtocol,
|
|
26
|
+
provider.auth || 'bearer', providerApiKey(provider)
|
|
27
|
+
])).digest('hex');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function currentModelDirectory(provider, { allowUnresolvedConnections = false } = {}) {
|
|
31
|
+
const directory = provider?.modelDirectory;
|
|
32
|
+
if (!directory || directory.version !== 1) return null;
|
|
33
|
+
if (directory.binding === modelDirectoryBinding(provider)) return directory;
|
|
34
|
+
return allowUnresolvedConnections && provider.dshProvider && !provider.baseUrl ? directory : null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function modelAvailabilityBinding(provider, modelId, capability) {
|
|
38
|
+
if (capability === 'image-gen') return modelDirectoryBinding(providerForImageModel(provider, modelId));
|
|
39
|
+
if (capability === 'music-gen' && provider.models?.some(model => model?.id === modelId && model.musicProtocol)) {
|
|
40
|
+
return modelDirectoryBinding(providerForMusicModel(provider, modelId));
|
|
41
|
+
}
|
|
42
|
+
return modelDirectoryBinding(provider);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function modelDirectoryEntry(provider, modelId, options) {
|
|
46
|
+
return currentModelDirectory(provider, options)?.models?.find(model => model.id === modelId) || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function modelDirectoryStatus(provider, modelId) {
|
|
50
|
+
const directory = currentModelDirectory(provider, { allowUnresolvedConnections: true });
|
|
51
|
+
if (!directory?.checkedAt) return 'unknown';
|
|
52
|
+
return directory.models.some(model => model.id === modelId) ? 'listed' : 'unlisted';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 普通请求路径404或参数400不代表模型不存在。兼容已有精确404记录。 */
|
|
56
|
+
export function isModelNotFound(result) {
|
|
57
|
+
if (result?.stage === 'poll') return false;
|
|
58
|
+
const status = Number(result?.httpStatus ?? result?.status);
|
|
59
|
+
if (status !== 404) return false;
|
|
60
|
+
const code = String(result?.providerCode || result?.code || '');
|
|
61
|
+
const message = String(result?.note || result?.safeMessage || result?.message || '');
|
|
62
|
+
return /^(?:NotFound\.Model|model_not_found|ModelNotFound)$/i.test(code)
|
|
63
|
+
|| /\bModel not exist\./i.test(message);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** 当前可选性与历史成功分开;旧Task成功不能解除新提交已确认的阻断。 */
|
|
67
|
+
export function modelUnavailable(provider, modelId, capability, { allowUnresolvedConnections = false } = {}) {
|
|
68
|
+
const observation = provider?.health?.observations?.find(item => item.modelId === modelId && item.capability === capability);
|
|
69
|
+
const availability = observation?.availability;
|
|
70
|
+
if (availability) {
|
|
71
|
+
if (availability.binding !== modelAvailabilityBinding(provider, modelId, capability)
|
|
72
|
+
&& !(allowUnresolvedConnections && provider.dshProvider && !provider.baseUrl)) return null;
|
|
73
|
+
return availability.state === 'blocked' ? availability : null;
|
|
74
|
+
}
|
|
75
|
+
const failure = [observation?.lastTransient, observation?.lastFailure].filter(isModelNotFound)
|
|
76
|
+
.sort((a, b) => String(b.at).localeCompare(String(a.at)))[0];
|
|
77
|
+
if (!failure) return null;
|
|
78
|
+
if (observation.lastSuccess?.source === 'probe' && observation.lastSuccess.at >= failure.at) return null;
|
|
79
|
+
return { ...failure, state: 'blocked' };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function modelUnavailableReason() {
|
|
83
|
+
return '当前连接已返回模型不存在,已暂停自动选用;请核对目录后单独实测恢复';
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function modelDirectorySummary(provider) {
|
|
87
|
+
const directory = currentModelDirectory(provider, { allowUnresolvedConnections: true });
|
|
88
|
+
if (!directory) return { status: 'unknown', note: '尚未同步当前连接的模型目录' };
|
|
89
|
+
const models = directory.models || [];
|
|
90
|
+
const configuredIds = Array.isArray(provider.models)
|
|
91
|
+
? provider.models.map(model => typeof model === 'string' ? model : model.id)
|
|
92
|
+
: [...new Set(['imageModel', 'videoModel', 'ttsModel', 'musicModel', 'transcribeModel', 'visionModel']
|
|
93
|
+
.map(field => provider[field]).filter(id => typeof id === 'string' && id.trim()).map(id => id.trim()))];
|
|
94
|
+
const missing = directory.checkedAt ? configuredIds.filter(id => !models.some(model => model.id === id)) : [];
|
|
95
|
+
const stale = Boolean(directory.lastError) || Date.now() - Date.parse(directory.checkedAt || '') >= MODEL_DIRECTORY_TTL_MS
|
|
96
|
+
|| Date.parse(directory.invalidatedAt || '') >= Date.parse(directory.checkedAt || '');
|
|
97
|
+
return {
|
|
98
|
+
status: directory.checkedAt ? stale ? 'stale' : 'current' : 'unknown',
|
|
99
|
+
checkedAt: directory.checkedAt || null, modelCount: models.length,
|
|
100
|
+
added: directory.added || [], missing,
|
|
101
|
+
...(directory.lastError ? { error: directory.lastError, failedAt: directory.failedAt } : {})
|
|
102
|
+
};
|
|
103
|
+
}
|