@oadank/dsh-input-tools 0.3.11 → 0.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +44 -7
- package/package.json +1 -1
- package/patches/dsh-voice-rc8.patch +379 -6
package/lib/client.js
CHANGED
|
@@ -32,6 +32,10 @@ window.__ModuleLoader__.load({
|
|
|
32
32
|
|
|
33
33
|
// ── 附件槽桥:官方 onAddImages 由 attachments 槽组件挂载时存入,left 按钮调用 ──
|
|
34
34
|
let sharedOnAddImages = null;
|
|
35
|
+
// [2026-08-21] draft 图片共享:attachments 槽挂载时把当前 draft 图(ComposerAttachment[])
|
|
36
|
+
// 与移除回调存入模块级,语音发送时可一起带上、发完清掉(解决"选了图发语音图被留下")。
|
|
37
|
+
let sharedDraftImages = [];
|
|
38
|
+
let sharedRemoveImage = null;
|
|
35
39
|
|
|
36
40
|
// ── [2026-08-21] 语音气泡(聊天界面 DOM 注入,安装即用,不依赖 dsh 源码支持)────
|
|
37
41
|
// 录音 → 存服务器(/voice/outbox/save)→ ASR 转文本 → 发【用户语音】标记文本;
|
|
@@ -158,13 +162,38 @@ window.__ModuleLoader__.load({
|
|
|
158
162
|
if (voiceErrorTimerRef.current !== null) window.clearTimeout(voiceErrorTimerRef.current);
|
|
159
163
|
voiceErrorTimerRef.current = window.setTimeout(() => setVoiceError(null), 6000);
|
|
160
164
|
};
|
|
161
|
-
|
|
165
|
+
// [2026-08-21] draft 图片转 image content(File→base64),语音可与图片一起发送
|
|
166
|
+
const draftImageContents = async () => {
|
|
167
|
+
const imgs = Array.isArray(sharedDraftImages) ? sharedDraftImages : [];
|
|
168
|
+
const out = [];
|
|
169
|
+
for (const a of imgs) {
|
|
170
|
+
const file = a?.file;
|
|
171
|
+
if (!file) continue;
|
|
172
|
+
const b64 = await new Promise((resolve) => {
|
|
173
|
+
const r = new FileReader();
|
|
174
|
+
r.onload = () => resolve(String(r.result).split(",")[1] ?? "");
|
|
175
|
+
r.onerror = () => resolve("");
|
|
176
|
+
r.readAsDataURL(file);
|
|
177
|
+
});
|
|
178
|
+
if (b64 !== "") out.push({ type: "image", mediaType: file.type || "image/jpeg", data: b64, name: file.name });
|
|
179
|
+
}
|
|
180
|
+
return out;
|
|
181
|
+
};
|
|
182
|
+
// [2026-08-21] 语音发送成功后清掉 draft 图片(否则图还留在输入框上)
|
|
183
|
+
const clearDraftImages = () => {
|
|
184
|
+
const imgs = Array.isArray(sharedDraftImages) ? sharedDraftImages : [];
|
|
185
|
+
if (typeof sharedRemoveImage === "function") {
|
|
186
|
+
for (const a of imgs) { try { sharedRemoveImage(a.id); } catch { /* ignore */ } }
|
|
187
|
+
}
|
|
188
|
+
sharedDraftImages = [];
|
|
189
|
+
};
|
|
190
|
+
const sendAsText = async (text, images) => {
|
|
162
191
|
// [2026-08-21] 降级路径:XDN(npm rc.7) 不支持 voice content。带【用户语音】标记让 AI
|
|
163
192
|
// 知道这是语音转的文本,可以按规则(自动 TTS)回复。
|
|
164
193
|
const marked = "【用户语音】" + text;
|
|
165
194
|
const response = await connection.api.sessions.prompt({
|
|
166
195
|
sessionId, mode: "queue",
|
|
167
|
-
content: [{ type: "text", text: marked }],
|
|
196
|
+
content: [{ type: "text", text: marked }, ...images],
|
|
168
197
|
});
|
|
169
198
|
const result = response?.result;
|
|
170
199
|
if (!result || !result.ok) {
|
|
@@ -172,25 +201,26 @@ window.__ModuleLoader__.load({
|
|
|
172
201
|
? result.error.message : "语音发送失败,请重试");
|
|
173
202
|
}
|
|
174
203
|
};
|
|
175
|
-
const sendAsVoice = async () => {
|
|
204
|
+
const sendAsVoice = async (images) => {
|
|
176
205
|
// 首选:多模态直发(AI 能听原音,消息渲染为语音气泡)——本机 lecoo / dev rc.8 支持
|
|
177
206
|
const response = await connection.api.sessions.prompt({
|
|
178
207
|
sessionId, mode: "queue",
|
|
179
|
-
content: [{ type: "voice", mediaType, data }],
|
|
208
|
+
content: [{ type: "voice", mediaType, data }, ...images],
|
|
180
209
|
});
|
|
181
210
|
return response?.result;
|
|
182
211
|
};
|
|
183
212
|
try {
|
|
213
|
+
const images = await draftImageContents();
|
|
184
214
|
// [2026-08-21 修] 先直发 voice,失败时降级 ASR 转文本(rc.7 兼容)。
|
|
185
215
|
// 这样本机/rc.8 享受多模态(AI 听到原音 + 语音消息气泡),XDN/rc.7 自动降级不报错。
|
|
186
216
|
let result;
|
|
187
217
|
try {
|
|
188
|
-
result = await sendAsVoice();
|
|
218
|
+
result = await sendAsVoice(images);
|
|
189
219
|
} catch (voiceErr) {
|
|
190
220
|
result = null;
|
|
191
221
|
}
|
|
192
222
|
// 直发成功(rc.8/dev):result.ok true
|
|
193
|
-
if (result && result.ok) return;
|
|
223
|
+
if (result && result.ok) { clearDraftImages(); return; }
|
|
194
224
|
// 失败或不支持:尝试降级
|
|
195
225
|
const errMsg = result?.error?.message ?? "";
|
|
196
226
|
// 只有"contract/payload"类错误才降级;其他业务错误直接提示
|
|
@@ -216,7 +246,8 @@ window.__ModuleLoader__.load({
|
|
|
216
246
|
if (!td?.ok) { fail(td?.error ?? "语音识别失败,请检查 ASR 配置"); return; }
|
|
217
247
|
const text = typeof td?.text === "string" ? td.text.trim() : "";
|
|
218
248
|
if (text === "") { fail("没听清,请再说一次"); return; }
|
|
219
|
-
await sendAsText(text);
|
|
249
|
+
await sendAsText(text, images);
|
|
250
|
+
clearDraftImages();
|
|
220
251
|
} catch (e) {
|
|
221
252
|
fail(String(e && typeof e.message === "string" && e.message !== "" ? e.message : e));
|
|
222
253
|
}
|
|
@@ -318,6 +349,12 @@ window.__ModuleLoader__.load({
|
|
|
318
349
|
if (typeof onAddImages === "function") sharedOnAddImages = onAddImages;
|
|
319
350
|
}, [onAddImages]);
|
|
320
351
|
|
|
352
|
+
// [2026-08-21] draft 图同步到模块级(语音发送一起带 + 发完清掉)
|
|
353
|
+
useEffect(() => {
|
|
354
|
+
sharedDraftImages = Array.isArray(attachments) ? attachments : [];
|
|
355
|
+
if (typeof onRemoveImage === "function") sharedRemoveImage = onRemoveImage;
|
|
356
|
+
}, [attachments, onRemoveImage]);
|
|
357
|
+
|
|
321
358
|
useEffect(() => {
|
|
322
359
|
if (zoom !== null && !items.some((a) => a.id === zoom.id)) setZoom(null);
|
|
323
360
|
}, [items, zoom]);
|
package/package.json
CHANGED
|
@@ -10,6 +10,56 @@ index 6a5164f160..f5a57a5e65 100644
|
|
|
10
10
|
WorkspaceId, WorkspaceView,
|
|
11
11
|
} from '@deepseek-ai/dsh-client-connection/client'
|
|
12
12
|
export type {} from '@deepseek-ai/dsh-api-gateway/client'
|
|
13
|
+
diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts
|
|
14
|
+
index 723df98720..0068b2f00b 100644
|
|
15
|
+
--- a/packages/attachment/attachment-local/src/store.ts
|
|
16
|
+
+++ b/packages/attachment/attachment-local/src/store.ts
|
|
17
|
+
@@ -1,7 +1,7 @@
|
|
18
|
+
/** Content-addressed, owner-private local attachment storage. */
|
|
19
|
+
|
|
20
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
21
|
+
-import { constants } from 'node:fs'
|
|
22
|
+
+import { constants, existsSync } from 'node:fs'
|
|
23
|
+
import { chmod, link, mkdir, open, readFile, unlink } from 'node:fs/promises'
|
|
24
|
+
import { dirname, join, parse, resolve } from 'node:path'
|
|
25
|
+
import {
|
|
26
|
+
@@ -19,6 +19,16 @@ import { detectImage, probeImage } from './image.ts'
|
|
27
|
+
const ID_PATTERN = /^sha256:([a-f0-9]{64})$/
|
|
28
|
+
const durableHomes = new Set<string>()
|
|
29
|
+
|
|
30
|
+
+/**
|
|
31
|
+
+ * [本地改造 2026-08-21] 内容寻址对象(无扩展名)的"带扩展名别名"路径。
|
|
32
|
+
+ * zai-vision 等视觉 MCP 按扩展名(.jpg/.jpeg/.png)校验,无扩展名会被拒绝。
|
|
33
|
+
+ * jpeg→.jpg、png→.png、webp→.png(webp 由调用方用 sharp 转码成 png 别名)。
|
|
34
|
+
+ */
|
|
35
|
+
+function extensionAliasPath(target: string, mediaType: ImageAttachmentRef['mediaType']): string | null {
|
|
36
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : mediaType === 'image/png' ? '.png' : mediaType === 'image/webp' ? '.png' : null
|
|
37
|
+
+ return ext === null ? null : `${target}${ext}`
|
|
38
|
+
+}
|
|
39
|
+
+
|
|
40
|
+
function digest(data: Uint8Array): string {
|
|
41
|
+
return createHash('sha256').update(data).digest('hex')
|
|
42
|
+
}
|
|
43
|
+
@@ -162,6 +172,19 @@ export async function saveImageFile(root: string, input: SaveImageAttachment, li
|
|
44
|
+
const existing = new Uint8Array(await readFile(target))
|
|
45
|
+
if (digest(existing) !== sha256) throw new AttachmentError('Stored attachment failed integrity verification.', 'ATTACHMENT_CORRUPT')
|
|
46
|
+
}
|
|
47
|
+
+ // [本地改造 2026-08-21] 确保带扩展名别名存在(首次存储与 dedup 命中都执行):
|
|
48
|
+
+ // jpeg/png 硬链接(零拷贝);webp 用 sharp 转成 png 别名。供 zai-vision 等按扩展名校验的 MCP 使用。
|
|
49
|
+
+ const alias = extensionAliasPath(target, metadata.mediaType)
|
|
50
|
+
+ if (alias !== null && !(await existsSync(alias))) {
|
|
51
|
+
+ try {
|
|
52
|
+
+ if (metadata.mediaType === 'image/webp') {
|
|
53
|
+
+ const { default: sharpMod } = await import('sharp')
|
|
54
|
+
+ await sharpMod(input.data).png().toFile(alias)
|
|
55
|
+
+ } else {
|
|
56
|
+
+ await link(target, alias)
|
|
57
|
+
+ }
|
|
58
|
+
+ } catch { /* 别名失败可忽略(主对象已持久化) */ }
|
|
59
|
+
+ }
|
|
60
|
+
// Persist the target entry and close a concurrent bucket-creation window
|
|
61
|
+
// before the reference can reach a session checkpoint. The dedup path
|
|
62
|
+
// repeats both syncs because it may observe another writer's link before
|
|
13
63
|
diff --git a/packages/attachment/attachment/src/error.ts b/packages/attachment/attachment/src/error.ts
|
|
14
64
|
index 2e2d695dae..c91a4f862f 100644
|
|
15
65
|
--- a/packages/attachment/attachment/src/error.ts
|
|
@@ -3645,7 +3695,7 @@ index 638d555b1e..f93aaa2ed8 100644
|
|
|
3645
3695
|
const apiKey = await this.config.resolveApiKey(connection)
|
|
3646
3696
|
const userId = this.config.resolveUserId()
|
|
3647
3697
|
diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3648
|
-
index 498b3fb2f7..
|
|
3698
|
+
index 498b3fb2f7..1a5c9aa6e4 100644
|
|
3649
3699
|
--- a/packages/llm/llm-deepseek/src/serialize.ts
|
|
3650
3700
|
+++ b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3651
3701
|
@@ -17,6 +17,7 @@ import type {
|
|
@@ -3656,7 +3706,7 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3656
3706
|
|
|
3657
3707
|
/** Adapter-level request defaults (from plugin config). */
|
|
3658
3708
|
export interface RequestDefaults {
|
|
3659
|
-
@@ -79,11 +80,
|
|
3709
|
+
@@ -79,11 +80,60 @@ function flattenText(blocks: ContentBlock[]): string {
|
|
3660
3710
|
.join('')
|
|
3661
3711
|
}
|
|
3662
3712
|
|
|
@@ -3666,7 +3716,8 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3666
3716
|
- throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
|
|
3667
3717
|
+/** [本地改造 2026-08-16] 把 image 块转成含本地附件路径的文本(参考 dsh-vscode-layout 补丁):
|
|
3668
3718
|
+ * 文本模型收到路径后,必须通过视觉 MCP(mcp__visionqa__look / mcp__zai-vision__analyze_image)
|
|
3669
|
-
+ *
|
|
3719
|
+
+ * 识图。路径带扩展名(jpeg→.jpg / png→.png / webp→.png,attachment-local 存储时已生成
|
|
3720
|
+
+ * 带扩展名别名,见 store.ts extensionAliasPath),zai-vision 等按扩展名校验的工具可用。 */
|
|
3670
3721
|
+function imageAsText(block: ContentBlock): ContentBlock {
|
|
3671
3722
|
+ const ref = (block as { attachment?: { attachmentId?: unknown; name?: string; mediaType?: string } }).attachment
|
|
3672
3723
|
+ const rawId = typeof ref?.attachmentId === 'string' ? ref.attachmentId : ''
|
|
@@ -3674,10 +3725,11 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3674
3725
|
+ const name = typeof ref?.name === 'string' && ref.name.length > 0 ? ref.name : 'image'
|
|
3675
3726
|
+ const mediaType = ref?.mediaType ?? 'image/jpeg'
|
|
3676
3727
|
+ const home = process.env.DSH_HOME ?? ''
|
|
3728
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : '.png'
|
|
3677
3729
|
+ const path = hex.length > 0 && home !== ''
|
|
3678
|
-
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3730
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex) + ext
|
|
3679
3731
|
+ : '(unknown)'
|
|
3680
|
-
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path
|
|
3732
|
+
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path):${path}]` }
|
|
3681
3733
|
+}
|
|
3682
3734
|
+
|
|
3683
3735
|
+function imagesAsText(blocks: readonly ContentBlock[]): ContentBlock[] {
|
|
@@ -3719,7 +3771,7 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3719
3771
|
}
|
|
3720
3772
|
|
|
3721
3773
|
/** Reject roles whose DeepSeek history format cannot carry image input. */
|
|
3722
|
-
@@ -203,19 +
|
|
3774
|
+
@@ -203,19 +253,22 @@ function serializeAssistant(message: Message): WireMessage {
|
|
3723
3775
|
export function serializeMessages(messages: Message[]): WireMessage[] {
|
|
3724
3776
|
const wire: WireMessage[] = []
|
|
3725
3777
|
for (const message of messages) {
|
|
@@ -3747,6 +3799,327 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3747
3799
|
if (text.length > 0 || toolResults.length === 0) {
|
|
3748
3800
|
wire.push({ role: 'user', content: text })
|
|
3749
3801
|
}
|
|
3802
|
+
diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts
|
|
3803
|
+
index d66a48115d..9e3127d2e6 100644
|
|
3804
|
+
--- a/packages/llm/llm-pi-ai/src/context.ts
|
|
3805
|
+
+++ b/packages/llm/llm-pi-ai/src/context.ts
|
|
3806
|
+
@@ -4,51 +4,122 @@
|
|
3807
|
+
* @module dsh-llm-pi-ai/context
|
|
3808
|
+
*/
|
|
3809
|
+
|
|
3810
|
+
-import { CallId, contentHasImage, LlmError, offloadRequestImages } from '@deepseek-ai/dsh-llm'
|
|
3811
|
+
-import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
|
|
3812
|
+
+import { CallId, contentHasImage, LlmError } from '@deepseek-ai/dsh-llm'
|
|
3813
|
+
+import type { ContentBlock, GenerateOptions } from '@deepseek-ai/dsh-llm'
|
|
3814
|
+
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
|
|
3815
|
+
import type { Context as PiContext, ImageContent, Message as PiMessage, TextContent, Tool as PiTool } from '@earendil-works/pi-ai'
|
|
3816
|
+
import { toPiAssistant } from './replay.ts'
|
|
3817
|
+
+import { join } from 'node:path'
|
|
3818
|
+
|
|
3819
|
+
-/** Join the text blocks of a harness message. */
|
|
3820
|
+
-function flattenText(message: Message): string {
|
|
3821
|
+
- return message.content
|
|
3822
|
+
- .filter(block => block.type === 'text')
|
|
3823
|
+
- .map(block => block.text)
|
|
3824
|
+
- .join('')
|
|
3825
|
+
+/** [本地改造 2026-08-16] 把已转为文本的内容块扁平化为纯文本(非视觉模型路径)。 */
|
|
3826
|
+
+function flattenBlocks(blocks: readonly ContentBlock[]): string {
|
|
3827
|
+
+ let out = ''
|
|
3828
|
+
+ for (const block of blocks) {
|
|
3829
|
+
+ if (block.type === 'text') {
|
|
3830
|
+
+ out += block.text
|
|
3831
|
+
+ } else if (block.type === 'voice') {
|
|
3832
|
+
+ const asText = voiceAsText(block)
|
|
3833
|
+
+ if (asText.type === 'text') out += asText.text
|
|
3834
|
+
+ } else if (block.type === 'tool-result') {
|
|
3835
|
+
+ out += flattenBlocks(block.content)
|
|
3836
|
+
+ }
|
|
3837
|
+
+ // image 已在 imagesToText 转成 text,此处不再出现。
|
|
3838
|
+
+ }
|
|
3839
|
+
+ return out
|
|
3840
|
+
}
|
|
3841
|
+
|
|
3842
|
+
+/**
|
|
3843
|
+
+ * [本地改造 2026-08-16] 把 voice 块转成文本:attachment.transcript 存在时直接给出
|
|
3844
|
+
+ * 识别文本;否则输出本地语音文件路径——与 llm-deepseek serialize.ts 的 voiceAsText
|
|
3845
|
+
+ * 同一策略,保证切换 provider(deepseek-official ↔ qwen/pi-ai)后语音识别行为一致。
|
|
3846
|
+
+ */
|
|
3847
|
+
+function voiceAsText(block: ContentBlock): ContentBlock {
|
|
3848
|
+
+ if (block.type !== 'voice') return block
|
|
3849
|
+
+ const ref = (block as { attachment?: { voiceId?: unknown; durationMs?: unknown; transcript?: unknown } }).attachment
|
|
3850
|
+
+ const rawId = typeof ref?.voiceId === 'string' ? ref.voiceId : ''
|
|
3851
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3852
|
+
+ const transcript = typeof ref?.transcript === 'string' && ref.transcript.length > 0
|
|
3853
|
+
+ ? ref.transcript
|
|
3854
|
+
+ : null
|
|
3855
|
+
+ const durationMs = typeof ref?.durationMs === 'number' ? ref.durationMs : null
|
|
3856
|
+
+ const duration = durationMs === null ? '' : `(时长 ${Math.round(durationMs / 1000)} 秒)`
|
|
3857
|
+
+ if (transcript !== null) {
|
|
3858
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},识别内容:${transcript}]` }
|
|
3859
|
+
+ }
|
|
3860
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3861
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3862
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3863
|
+
+ : '(unknown)'
|
|
3864
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},本地语音文件路径: ${path}]` }
|
|
3865
|
+
+}
|
|
3866
|
+
|
|
3867
|
+
-/** Flatten text recursively inside one tool result. */
|
|
3868
|
+
-function toolResultText(blocks: readonly ContentBlock[]): string {
|
|
3869
|
+
- return blocks.map(block => block.type === 'text'
|
|
3870
|
+
- ? block.text
|
|
3871
|
+
- : block.type === 'tool-result' ? toolResultText(block.content) : '').join('')
|
|
3872
|
+
+/**
|
|
3873
|
+
+ * [本地改造 2026-08-16] 把 image 块转成含本地附件路径的文本(与 llm-deepseek
|
|
3874
|
+
+ * serialize.ts 的 imageAsText 同一策略):非视觉模型(pi-ai input 不含 image)
|
|
3875
|
+
+ * 收到路径文本后,必须通过视觉 MCP(mcp__visionqa__look / mcp__zai-vision__analyze_image)
|
|
3876
|
+
+ * 识图。路径带扩展名(jpeg→.jpg / png→.png / webp→.png,attachment-local 存储时已生成
|
|
3877
|
+
+ * 带扩展名别名,见 store.ts extensionAliasPath),zai-vision 等按扩展名校验的工具可用。
|
|
3878
|
+
+ */
|
|
3879
|
+
+function imageAsText(block: ContentBlock): ContentBlock {
|
|
3880
|
+
+ if (block.type !== 'image') return block
|
|
3881
|
+
+ const ref = (block as { attachment?: { attachmentId?: unknown; name?: unknown; mediaType?: unknown } }).attachment
|
|
3882
|
+
+ const rawId = typeof ref?.attachmentId === 'string' ? ref.attachmentId : ''
|
|
3883
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3884
|
+
+ const name = typeof ref?.name === 'string' && ref.name.length > 0 ? ref.name : 'image'
|
|
3885
|
+
+ const mediaType = typeof ref?.mediaType === 'string' ? ref.mediaType : 'image/jpeg'
|
|
3886
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3887
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : '.png'
|
|
3888
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3889
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex) + ext
|
|
3890
|
+
+ : '(unknown)'
|
|
3891
|
+
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path):${path}]` }
|
|
3892
|
+
}
|
|
3893
|
+
|
|
3894
|
+
-/** Reject image roles that pi-ai cannot replay before request-size offloading can replace them. */
|
|
3895
|
+
-function assertSupportedImageRoles(messages: readonly Message[]): void {
|
|
3896
|
+
- for (const message of messages) {
|
|
3897
|
+
- if (message.role !== 'user' && contentHasImage(message.content)) {
|
|
3898
|
+
- throw new LlmError(
|
|
3899
|
+
- `pi-ai cannot represent an image in an in-history ${message.role} message`,
|
|
3900
|
+
- 'UNSUPPORTED_CONTENT',
|
|
3901
|
+
- )
|
|
3902
|
+
+/** Convert image blocks to path-text when the route model is not a vision model. */
|
|
3903
|
+
+function imagesToText(blocks: readonly ContentBlock[], vision: boolean): readonly ContentBlock[] {
|
|
3904
|
+
+ if (vision) return blocks
|
|
3905
|
+
+ const out: ContentBlock[] = []
|
|
3906
|
+
+ for (const block of blocks) {
|
|
3907
|
+
+ if (block.type === 'image') {
|
|
3908
|
+
+ out.push(imageAsText(block))
|
|
3909
|
+
+ } else if (block.type === 'tool-result') {
|
|
3910
|
+
+ out.push({ ...block, content: [...imagesToText(block.content, vision)] })
|
|
3911
|
+
+ } else {
|
|
3912
|
+
+ out.push(block)
|
|
3913
|
+
}
|
|
3914
|
+
}
|
|
3915
|
+
+ return out
|
|
3916
|
+
}
|
|
3917
|
+
|
|
3918
|
+
async function userContent(
|
|
3919
|
+
blocks: readonly ContentBlock[],
|
|
3920
|
+
- attachments: AttachmentStore,
|
|
3921
|
+
+ attachments: AttachmentStore | undefined,
|
|
3922
|
+
+ vision: boolean,
|
|
3923
|
+
): Promise<string | (TextContent | ImageContent)[]> {
|
|
3924
|
+
+ // [本地改造 2026-08-16] 非视觉模型:图片块先整体转路径文本(agent 用视觉 MCP 看图),
|
|
3925
|
+
+ // 不再需要 attachments;视觉模型保持原逻辑(读原图送 pi-ai)。
|
|
3926
|
+
+ const converted = imagesToText(blocks, vision)
|
|
3927
|
+
+ if (!vision) {
|
|
3928
|
+
+ return flattenBlocks(converted)
|
|
3929
|
+
+ }
|
|
3930
|
+
const content: (TextContent | ImageContent)[] = []
|
|
3931
|
+
- for (const block of blocks) {
|
|
3932
|
+
+ for (const block of converted) {
|
|
3933
|
+
switch (block.type) {
|
|
3934
|
+
case 'text':
|
|
3935
|
+
if (block.text.length > 0) content.push({ type: 'text', text: block.text })
|
|
3936
|
+
break
|
|
3937
|
+
+ case 'voice': {
|
|
3938
|
+
+ // [本地改造 2026-08-16] 语音块转文本(识别文本/本地路径),与文本块同路进模型。
|
|
3939
|
+
+ const asText = voiceAsText(block)
|
|
3940
|
+
+ if (asText.type === 'text' && asText.text.length > 0) {
|
|
3941
|
+
+ content.push({ type: 'text', text: asText.text })
|
|
3942
|
+
+ }
|
|
3943
|
+
+ break
|
|
3944
|
+
+ }
|
|
3945
|
+
case 'image': {
|
|
3946
|
+
+ if (attachments === undefined) {
|
|
3947
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3948
|
+
+ }
|
|
3949
|
+
const stored = await attachments.readImage(block.attachment)
|
|
3950
|
+
content.push({
|
|
3951
|
+
type: 'image',
|
|
3952
|
+
@@ -59,7 +130,7 @@ async function userContent(
|
|
3953
|
+
}
|
|
3954
|
+
case 'tool-result':
|
|
3955
|
+
{
|
|
3956
|
+
- const nested = await userContent(block.content, attachments)
|
|
3957
|
+
+ const nested = await userContent(block.content, attachments, vision)
|
|
3958
|
+
if (typeof nested === 'string') {
|
|
3959
|
+
if (nested.length > 0) content.push({ type: 'text', text: nested })
|
|
3960
|
+
} else {
|
|
3961
|
+
@@ -96,34 +167,45 @@ function piContext(options: GenerateOptions, messages: PiMessage[]): PiContext {
|
|
3962
|
+
}
|
|
3963
|
+
}
|
|
3964
|
+
|
|
3965
|
+
-function textOnlyContext(options: GenerateOptions, onReplayDegrade?: (reason: string) => void): PiContext {
|
|
3966
|
+
+function textOnlyContext(options: GenerateOptions, vision: boolean): PiContext {
|
|
3967
|
+
const toolNames = new Map<CallId, string>()
|
|
3968
|
+
const messages: PiMessage[] = []
|
|
3969
|
+
for (const message of options.messages) {
|
|
3970
|
+
- if (contentHasImage(message.content)) {
|
|
3971
|
+
- throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3972
|
+
- }
|
|
3973
|
+
if (message.role === 'system') {
|
|
3974
|
+
- messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
|
|
3975
|
+
+ // 视觉模型无法在 pi-ai 单一 systemPrompt 槽内表达图片;非视觉模型
|
|
3976
|
+
+ // (vision=false)走文本路径(imagesToText 转路径文本)。
|
|
3977
|
+
+ if (vision && contentHasImage(message.content)) {
|
|
3978
|
+
+ throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
|
|
3979
|
+
+ }
|
|
3980
|
+
+ messages.push({ role: 'user', content: flattenBlocks(imagesToText(message.content, vision)), timestamp: 0 })
|
|
3981
|
+
continue
|
|
3982
|
+
}
|
|
3983
|
+
if (message.role === 'assistant') {
|
|
3984
|
+
- const assistant = toPiAssistant(message, onReplayDegrade)
|
|
3985
|
+
+ const assistant = toPiAssistant(message)
|
|
3986
|
+
for (const block of assistant.content) if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
|
|
3987
|
+
messages.push(assistant)
|
|
3988
|
+
continue
|
|
3989
|
+
}
|
|
3990
|
+
- const text = flattenText(message)
|
|
3991
|
+
+ const regular = message.content.filter(block => block.type !== 'tool-result')
|
|
3992
|
+
+ // [本地改造 2026-08-16] 非视觉模型(vision=false):图片块转路径文本后扁平化;
|
|
3993
|
+
+ // 视觉模型(vision=true)无 durable attachment 服务时仍拒绝(必须读原图)。
|
|
3994
|
+
+ if (vision && contentHasImage(regular)) {
|
|
3995
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3996
|
+
+ }
|
|
3997
|
+
+ const text = flattenBlocks(imagesToText(regular, vision))
|
|
3998
|
+
const results = message.content.filter(block => block.type === 'tool-result')
|
|
3999
|
+
if (text.length > 0 || results.length === 0) messages.push({ role: 'user', content: text, timestamp: 0 })
|
|
4000
|
+
for (const result of results) {
|
|
4001
|
+
+ if (vision && contentHasImage(result.content)) {
|
|
4002
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
4003
|
+
+ }
|
|
4004
|
+
messages.push({
|
|
4005
|
+
role: 'toolResult',
|
|
4006
|
+
toolCallId: result.toolCallId,
|
|
4007
|
+
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
|
|
4008
|
+
content: [{
|
|
4009
|
+
type: 'text',
|
|
4010
|
+
- text: toolResultText(result.content) || '(no output)',
|
|
4011
|
+
+ text: flattenBlocks(imagesToText(result.content, vision)) || '(no output)',
|
|
4012
|
+
}],
|
|
4013
|
+
isError: result.isError ?? false,
|
|
4014
|
+
timestamp: 0,
|
|
4015
|
+
@@ -133,69 +215,42 @@ function textOnlyContext(options: GenerateOptions, onReplayDegrade?: (reason: st
|
|
4016
|
+
return piContext(options, messages)
|
|
4017
|
+
}
|
|
4018
|
+
|
|
4019
|
+
-/**
|
|
4020
|
+
- * Convert text-only harness history to a synchronous pi-ai Context. Tool
|
|
4021
|
+
- * result names are recovered from preceding assistant tool calls.
|
|
4022
|
+
- * @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
|
|
4023
|
+
- * @param attachments - absent; selects the synchronous conversion.
|
|
4024
|
+
- * @param onReplayDegrade - forwarded to {@link toPiAssistant} for each assistant message.
|
|
4025
|
+
- * @returns the pi-ai context; `tools` is omitted when the request declares none.
|
|
4026
|
+
- */
|
|
4027
|
+
-export function toPiContext(
|
|
4028
|
+
- options: GenerateOptions,
|
|
4029
|
+
- attachments?: undefined,
|
|
4030
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4031
|
+
-): PiContext
|
|
4032
|
+
/**
|
|
4033
|
+
* Convert harness history to a pi-ai Context while resolving durable images.
|
|
4034
|
+
- * Tool result names are recovered from preceding assistant tool calls. When
|
|
4035
|
+
- * the accumulated base64 image payload exceeds `maxRequestImageBytes`, the
|
|
4036
|
+
- * oldest images are replaced by text placeholders until the request fits, so
|
|
4037
|
+
- * an image-heavy session keeps clearing gateway request-size caps.
|
|
4038
|
+
+ * Tool result names are recovered from preceding assistant tool calls.
|
|
4039
|
+
+ * [本地改造 2026-08-16] vision=false(模型 input 不含 image)时,图片块转为
|
|
4040
|
+
+ * 本地路径文本(agent 用视觉 MCP 识图),与 llm-deepseek serialize.ts 一致;
|
|
4041
|
+
+ * attachments 可为 undefined(非视觉路径不需要 durable attachment 服务)。
|
|
4042
|
+
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
|
|
4043
|
+
- * @param attachments - durable byte resolver for image references.
|
|
4044
|
+
- * @param onReplayDegrade - forwarded to {@link toPiAssistant} for each assistant message.
|
|
4045
|
+
- * @param maxRequestImageBytes - request-level bound on base64-encoded image payload; omission leaves every image in place.
|
|
4046
|
+
+ * @param attachments - durable byte resolver for image references (vision models); optional.
|
|
4047
|
+
+ * @param vision - whether the route model accepts image input.
|
|
4048
|
+
* @returns the asynchronously resolved pi-ai context.
|
|
4049
|
+
*/
|
|
4050
|
+
-export function toPiContext(
|
|
4051
|
+
- options: GenerateOptions,
|
|
4052
|
+
- attachments: AttachmentStore,
|
|
4053
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4054
|
+
- maxRequestImageBytes?: number,
|
|
4055
|
+
-): Promise<PiContext>
|
|
4056
|
+
-export function toPiContext(
|
|
4057
|
+
- options: GenerateOptions,
|
|
4058
|
+
- attachments?: AttachmentStore,
|
|
4059
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4060
|
+
- maxRequestImageBytes?: number,
|
|
4061
|
+
-): PiContext | Promise<PiContext> {
|
|
4062
|
+
- return attachments === undefined
|
|
4063
|
+
- ? textOnlyContext(options, onReplayDegrade)
|
|
4064
|
+
- : toPiContextWithImages(options, attachments, onReplayDegrade, maxRequestImageBytes)
|
|
4065
|
+
+export function toPiContext(options: GenerateOptions, attachments?: undefined, vision?: boolean): PiContext
|
|
4066
|
+
+export function toPiContext(options: GenerateOptions, attachments: AttachmentStore, vision?: boolean): Promise<PiContext>
|
|
4067
|
+
+export function toPiContext(options: GenerateOptions, attachments?: AttachmentStore, vision = true): PiContext | Promise<PiContext> {
|
|
4068
|
+
+ return attachments === undefined ? textOnlyContext(options, vision) : toPiContextWithImages(options, attachments, vision)
|
|
4069
|
+
}
|
|
4070
|
+
|
|
4071
|
+
-async function toPiContextWithImages(
|
|
4072
|
+
- options: GenerateOptions,
|
|
4073
|
+
- attachments: AttachmentStore,
|
|
4074
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4075
|
+
- maxRequestImageBytes?: number,
|
|
4076
|
+
-): Promise<PiContext> {
|
|
4077
|
+
- assertSupportedImageRoles(options.messages)
|
|
4078
|
+
- const requestMessages = offloadRequestImages(options.messages, maxRequestImageBytes)
|
|
4079
|
+
+async function toPiContextWithImages(options: GenerateOptions, attachments: AttachmentStore, vision: boolean): Promise<PiContext> {
|
|
4080
|
+
const toolNames = new Map<CallId, string>()
|
|
4081
|
+
const messages: PiMessage[] = []
|
|
4082
|
+
|
|
4083
|
+
- for (const message of requestMessages) {
|
|
4084
|
+
+ for (const message of options.messages) {
|
|
4085
|
+
if (message.role === 'system') {
|
|
4086
|
+
+ // 视觉模型无法在 pi-ai 单一 systemPrompt 槽内表达图片;非视觉模型
|
|
4087
|
+
+ // (vision=false)走文本路径(imagesToText 转路径文本)。
|
|
4088
|
+
+ if (vision && contentHasImage(message.content)) {
|
|
4089
|
+
+ throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
|
|
4090
|
+
+ }
|
|
4091
|
+
// pi-ai has a single systemPrompt slot; in-history system messages are
|
|
4092
|
+
// folded into user messages to preserve order (rare in practice — the
|
|
4093
|
+
// harness sends the system prompt via options.system).
|
|
4094
|
+
- messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
|
|
4095
|
+
+ messages.push({ role: 'user', content: flattenBlocks(imagesToText(message.content, vision)), timestamp: 0 })
|
|
4096
|
+
continue
|
|
4097
|
+
}
|
|
4098
|
+
if (message.role === 'assistant') {
|
|
4099
|
+
- const assistant = toPiAssistant(message, onReplayDegrade)
|
|
4100
|
+
+ const assistant = toPiAssistant(message)
|
|
4101
|
+
for (const block of assistant.content) {
|
|
4102
|
+
if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
|
|
4103
|
+
}
|
|
4104
|
+
@@ -204,15 +259,13 @@ async function toPiContextWithImages(
|
|
4105
|
+
}
|
|
4106
|
+
// user role: text + tool results (each result becomes its own message).
|
|
4107
|
+
const regular = message.content.filter(block => block.type !== 'tool-result')
|
|
4108
|
+
- const content = await userContent(regular, attachments)
|
|
4109
|
+
- const results = message.content.filter((block): block is Extract<ContentBlock, { type: 'tool-result' }> => (
|
|
4110
|
+
- block.type === 'tool-result'
|
|
4111
|
+
- ))
|
|
4112
|
+
+ const content = await userContent(regular, attachments, vision)
|
|
4113
|
+
+ const results = message.content.filter(block => block.type === 'tool-result')
|
|
4114
|
+
if (content.length > 0 || results.length === 0) {
|
|
4115
|
+
messages.push({ role: 'user', content, timestamp: 0 })
|
|
4116
|
+
}
|
|
4117
|
+
for (const result of results) {
|
|
4118
|
+
- const resultContent = await userContent(result.content, attachments)
|
|
4119
|
+
+ const resultContent = await userContent(result.content, attachments, vision)
|
|
4120
|
+
messages.push({
|
|
4121
|
+
role: 'toolResult',
|
|
4122
|
+
toolCallId: result.toolCallId,
|
|
3750
4123
|
diff --git a/packages/llm/llm/src/types.ts b/packages/llm/llm/src/types.ts
|
|
3751
4124
|
index 8c5be187dd..750496f164 100644
|
|
3752
4125
|
--- a/packages/llm/llm/src/types.ts
|