@oadank/dsh-input-tools 0.3.12 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/package.json +1 -1
- package/patches/dsh-voice-rc8.patch +379 -6
package/README.md
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
| 语音气泡 | 用户/AI 语音消息可点击播放,尾部复制转写文本 |
|
|
19
19
|
|
|
20
20
|
### 图片
|
|
21
|
-
- 输入框图片按钮上传 →
|
|
21
|
+
- 输入框图片按钮上传 → **文本模型也能发图**:图片转本地路径文本,AI 自动调视觉 MCP 识图后回答
|
|
22
|
+
- 附件存储自动生成**带扩展名的别名**(jpg/png/webp→png),zai-vision 等按扩展名校验的
|
|
23
|
+
识图工具可直接读取(源码补丁包含,无需额外配置)
|
|
22
24
|
|
|
23
25
|
### 余额
|
|
24
26
|
- 直连模型时输入框右侧实时显示余额(¥xx)
|
|
@@ -81,7 +83,9 @@ setup 脚本自动完成:装插件进 profile → 注册 → 检查 ffmpeg →
|
|
|
81
83
|
### 依赖
|
|
82
84
|
|
|
83
85
|
- **ffmpeg**(语音转码必需):Windows `winget install ffmpeg`;Linux `sudo apt install ffmpeg`
|
|
84
|
-
-
|
|
86
|
+
- **视觉 MCP**(图片识图必需):在 dsh 设置里配置至少一个视觉 MCP 服务,AI 用它的工具识图:
|
|
87
|
+
- `zai-vision`(推荐,通用):`npx -y @z_ai/mcp-server`,配 `Z_AI_BASE_URL=http://localhost:11434/v1/`(本地 ollama 跑 qwen3-vl 等视觉模型),按扩展名校验(已由补丁解决)
|
|
88
|
+
- `visionqa`(本机自建服务)
|
|
85
89
|
|
|
86
90
|
## 配置
|
|
87
91
|
|
package/package.json
CHANGED
|
@@ -10,6 +10,56 @@ index 6a5164f160..f5a57a5e65 100644
|
|
|
10
10
|
WorkspaceId, WorkspaceView,
|
|
11
11
|
} from '@deepseek-ai/dsh-client-connection/client'
|
|
12
12
|
export type {} from '@deepseek-ai/dsh-api-gateway/client'
|
|
13
|
+
diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts
|
|
14
|
+
index 723df98720..0068b2f00b 100644
|
|
15
|
+
--- a/packages/attachment/attachment-local/src/store.ts
|
|
16
|
+
+++ b/packages/attachment/attachment-local/src/store.ts
|
|
17
|
+
@@ -1,7 +1,7 @@
|
|
18
|
+
/** Content-addressed, owner-private local attachment storage. */
|
|
19
|
+
|
|
20
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
21
|
+
-import { constants } from 'node:fs'
|
|
22
|
+
+import { constants, existsSync } from 'node:fs'
|
|
23
|
+
import { chmod, link, mkdir, open, readFile, unlink } from 'node:fs/promises'
|
|
24
|
+
import { dirname, join, parse, resolve } from 'node:path'
|
|
25
|
+
import {
|
|
26
|
+
@@ -19,6 +19,16 @@ import { detectImage, probeImage } from './image.ts'
|
|
27
|
+
const ID_PATTERN = /^sha256:([a-f0-9]{64})$/
|
|
28
|
+
const durableHomes = new Set<string>()
|
|
29
|
+
|
|
30
|
+
+/**
|
|
31
|
+
+ * [本地改造 2026-08-21] 内容寻址对象(无扩展名)的"带扩展名别名"路径。
|
|
32
|
+
+ * zai-vision 等视觉 MCP 按扩展名(.jpg/.jpeg/.png)校验,无扩展名会被拒绝。
|
|
33
|
+
+ * jpeg→.jpg、png→.png、webp→.png(webp 由调用方用 sharp 转码成 png 别名)。
|
|
34
|
+
+ */
|
|
35
|
+
+function extensionAliasPath(target: string, mediaType: ImageAttachmentRef['mediaType']): string | null {
|
|
36
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : mediaType === 'image/png' ? '.png' : mediaType === 'image/webp' ? '.png' : null
|
|
37
|
+
+ return ext === null ? null : `${target}${ext}`
|
|
38
|
+
+}
|
|
39
|
+
+
|
|
40
|
+
function digest(data: Uint8Array): string {
|
|
41
|
+
return createHash('sha256').update(data).digest('hex')
|
|
42
|
+
}
|
|
43
|
+
@@ -162,6 +172,19 @@ export async function saveImageFile(root: string, input: SaveImageAttachment, li
|
|
44
|
+
const existing = new Uint8Array(await readFile(target))
|
|
45
|
+
if (digest(existing) !== sha256) throw new AttachmentError('Stored attachment failed integrity verification.', 'ATTACHMENT_CORRUPT')
|
|
46
|
+
}
|
|
47
|
+
+ // [本地改造 2026-08-21] 确保带扩展名别名存在(首次存储与 dedup 命中都执行):
|
|
48
|
+
+ // jpeg/png 硬链接(零拷贝);webp 用 sharp 转成 png 别名。供 zai-vision 等按扩展名校验的 MCP 使用。
|
|
49
|
+
+ const alias = extensionAliasPath(target, metadata.mediaType)
|
|
50
|
+
+ if (alias !== null && !(await existsSync(alias))) {
|
|
51
|
+
+ try {
|
|
52
|
+
+ if (metadata.mediaType === 'image/webp') {
|
|
53
|
+
+ const { default: sharpMod } = await import('sharp')
|
|
54
|
+
+ await sharpMod(input.data).png().toFile(alias)
|
|
55
|
+
+ } else {
|
|
56
|
+
+ await link(target, alias)
|
|
57
|
+
+ }
|
|
58
|
+
+ } catch { /* 别名失败可忽略(主对象已持久化) */ }
|
|
59
|
+
+ }
|
|
60
|
+
// Persist the target entry and close a concurrent bucket-creation window
|
|
61
|
+
// before the reference can reach a session checkpoint. The dedup path
|
|
62
|
+
// repeats both syncs because it may observe another writer's link before
|
|
13
63
|
diff --git a/packages/attachment/attachment/src/error.ts b/packages/attachment/attachment/src/error.ts
|
|
14
64
|
index 2e2d695dae..c91a4f862f 100644
|
|
15
65
|
--- a/packages/attachment/attachment/src/error.ts
|
|
@@ -3645,7 +3695,7 @@ index 638d555b1e..f93aaa2ed8 100644
|
|
|
3645
3695
|
const apiKey = await this.config.resolveApiKey(connection)
|
|
3646
3696
|
const userId = this.config.resolveUserId()
|
|
3647
3697
|
diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3648
|
-
index 498b3fb2f7..
|
|
3698
|
+
index 498b3fb2f7..1a5c9aa6e4 100644
|
|
3649
3699
|
--- a/packages/llm/llm-deepseek/src/serialize.ts
|
|
3650
3700
|
+++ b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3651
3701
|
@@ -17,6 +17,7 @@ import type {
|
|
@@ -3656,7 +3706,7 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3656
3706
|
|
|
3657
3707
|
/** Adapter-level request defaults (from plugin config). */
|
|
3658
3708
|
export interface RequestDefaults {
|
|
3659
|
-
@@ -79,11 +80,
|
|
3709
|
+
@@ -79,11 +80,60 @@ function flattenText(blocks: ContentBlock[]): string {
|
|
3660
3710
|
.join('')
|
|
3661
3711
|
}
|
|
3662
3712
|
|
|
@@ -3666,7 +3716,8 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3666
3716
|
- throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
|
|
3667
3717
|
+/** [本地改造 2026-08-16] 把 image 块转成含本地附件路径的文本(参考 dsh-vscode-layout 补丁):
|
|
3668
3718
|
+ * 文本模型收到路径后,必须通过视觉 MCP(mcp__visionqa__look / mcp__zai-vision__analyze_image)
|
|
3669
|
-
+ *
|
|
3719
|
+
+ * 识图。路径带扩展名(jpeg→.jpg / png→.png / webp→.png,attachment-local 存储时已生成
|
|
3720
|
+
+ * 带扩展名别名,见 store.ts extensionAliasPath),zai-vision 等按扩展名校验的工具可用。 */
|
|
3670
3721
|
+function imageAsText(block: ContentBlock): ContentBlock {
|
|
3671
3722
|
+ const ref = (block as { attachment?: { attachmentId?: unknown; name?: string; mediaType?: string } }).attachment
|
|
3672
3723
|
+ const rawId = typeof ref?.attachmentId === 'string' ? ref.attachmentId : ''
|
|
@@ -3674,10 +3725,11 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3674
3725
|
+ const name = typeof ref?.name === 'string' && ref.name.length > 0 ? ref.name : 'image'
|
|
3675
3726
|
+ const mediaType = ref?.mediaType ?? 'image/jpeg'
|
|
3676
3727
|
+ const home = process.env.DSH_HOME ?? ''
|
|
3728
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : '.png'
|
|
3677
3729
|
+ const path = hex.length > 0 && home !== ''
|
|
3678
|
-
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3730
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex) + ext
|
|
3679
3731
|
+ : '(unknown)'
|
|
3680
|
-
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path
|
|
3732
|
+
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path):${path}]` }
|
|
3681
3733
|
+}
|
|
3682
3734
|
+
|
|
3683
3735
|
+function imagesAsText(blocks: readonly ContentBlock[]): ContentBlock[] {
|
|
@@ -3719,7 +3771,7 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3719
3771
|
}
|
|
3720
3772
|
|
|
3721
3773
|
/** Reject roles whose DeepSeek history format cannot carry image input. */
|
|
3722
|
-
@@ -203,19 +
|
|
3774
|
+
@@ -203,19 +253,22 @@ function serializeAssistant(message: Message): WireMessage {
|
|
3723
3775
|
export function serializeMessages(messages: Message[]): WireMessage[] {
|
|
3724
3776
|
const wire: WireMessage[] = []
|
|
3725
3777
|
for (const message of messages) {
|
|
@@ -3747,6 +3799,327 @@ index 498b3fb2f7..a5e8458366 100644
|
|
|
3747
3799
|
if (text.length > 0 || toolResults.length === 0) {
|
|
3748
3800
|
wire.push({ role: 'user', content: text })
|
|
3749
3801
|
}
|
|
3802
|
+
diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts
|
|
3803
|
+
index d66a48115d..9e3127d2e6 100644
|
|
3804
|
+
--- a/packages/llm/llm-pi-ai/src/context.ts
|
|
3805
|
+
+++ b/packages/llm/llm-pi-ai/src/context.ts
|
|
3806
|
+
@@ -4,51 +4,122 @@
|
|
3807
|
+
* @module dsh-llm-pi-ai/context
|
|
3808
|
+
*/
|
|
3809
|
+
|
|
3810
|
+
-import { CallId, contentHasImage, LlmError, offloadRequestImages } from '@deepseek-ai/dsh-llm'
|
|
3811
|
+
-import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
|
|
3812
|
+
+import { CallId, contentHasImage, LlmError } from '@deepseek-ai/dsh-llm'
|
|
3813
|
+
+import type { ContentBlock, GenerateOptions } from '@deepseek-ai/dsh-llm'
|
|
3814
|
+
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
|
|
3815
|
+
import type { Context as PiContext, ImageContent, Message as PiMessage, TextContent, Tool as PiTool } from '@earendil-works/pi-ai'
|
|
3816
|
+
import { toPiAssistant } from './replay.ts'
|
|
3817
|
+
+import { join } from 'node:path'
|
|
3818
|
+
|
|
3819
|
+
-/** Join the text blocks of a harness message. */
|
|
3820
|
+
-function flattenText(message: Message): string {
|
|
3821
|
+
- return message.content
|
|
3822
|
+
- .filter(block => block.type === 'text')
|
|
3823
|
+
- .map(block => block.text)
|
|
3824
|
+
- .join('')
|
|
3825
|
+
+/** [本地改造 2026-08-16] 把已转为文本的内容块扁平化为纯文本(非视觉模型路径)。 */
|
|
3826
|
+
+function flattenBlocks(blocks: readonly ContentBlock[]): string {
|
|
3827
|
+
+ let out = ''
|
|
3828
|
+
+ for (const block of blocks) {
|
|
3829
|
+
+ if (block.type === 'text') {
|
|
3830
|
+
+ out += block.text
|
|
3831
|
+
+ } else if (block.type === 'voice') {
|
|
3832
|
+
+ const asText = voiceAsText(block)
|
|
3833
|
+
+ if (asText.type === 'text') out += asText.text
|
|
3834
|
+
+ } else if (block.type === 'tool-result') {
|
|
3835
|
+
+ out += flattenBlocks(block.content)
|
|
3836
|
+
+ }
|
|
3837
|
+
+ // image 已在 imagesToText 转成 text,此处不再出现。
|
|
3838
|
+
+ }
|
|
3839
|
+
+ return out
|
|
3840
|
+
}
|
|
3841
|
+
|
|
3842
|
+
+/**
|
|
3843
|
+
+ * [本地改造 2026-08-16] 把 voice 块转成文本:attachment.transcript 存在时直接给出
|
|
3844
|
+
+ * 识别文本;否则输出本地语音文件路径——与 llm-deepseek serialize.ts 的 voiceAsText
|
|
3845
|
+
+ * 同一策略,保证切换 provider(deepseek-official ↔ qwen/pi-ai)后语音识别行为一致。
|
|
3846
|
+
+ */
|
|
3847
|
+
+function voiceAsText(block: ContentBlock): ContentBlock {
|
|
3848
|
+
+ if (block.type !== 'voice') return block
|
|
3849
|
+
+ const ref = (block as { attachment?: { voiceId?: unknown; durationMs?: unknown; transcript?: unknown } }).attachment
|
|
3850
|
+
+ const rawId = typeof ref?.voiceId === 'string' ? ref.voiceId : ''
|
|
3851
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3852
|
+
+ const transcript = typeof ref?.transcript === 'string' && ref.transcript.length > 0
|
|
3853
|
+
+ ? ref.transcript
|
|
3854
|
+
+ : null
|
|
3855
|
+
+ const durationMs = typeof ref?.durationMs === 'number' ? ref.durationMs : null
|
|
3856
|
+
+ const duration = durationMs === null ? '' : `(时长 ${Math.round(durationMs / 1000)} 秒)`
|
|
3857
|
+
+ if (transcript !== null) {
|
|
3858
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},识别内容:${transcript}]` }
|
|
3859
|
+
+ }
|
|
3860
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3861
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3862
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3863
|
+
+ : '(unknown)'
|
|
3864
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},本地语音文件路径: ${path}]` }
|
|
3865
|
+
+}
|
|
3866
|
+
|
|
3867
|
+
-/** Flatten text recursively inside one tool result. */
|
|
3868
|
+
-function toolResultText(blocks: readonly ContentBlock[]): string {
|
|
3869
|
+
- return blocks.map(block => block.type === 'text'
|
|
3870
|
+
- ? block.text
|
|
3871
|
+
- : block.type === 'tool-result' ? toolResultText(block.content) : '').join('')
|
|
3872
|
+
+/**
|
|
3873
|
+
+ * [本地改造 2026-08-16] 把 image 块转成含本地附件路径的文本(与 llm-deepseek
|
|
3874
|
+
+ * serialize.ts 的 imageAsText 同一策略):非视觉模型(pi-ai input 不含 image)
|
|
3875
|
+
+ * 收到路径文本后,必须通过视觉 MCP(mcp__visionqa__look / mcp__zai-vision__analyze_image)
|
|
3876
|
+
+ * 识图。路径带扩展名(jpeg→.jpg / png→.png / webp→.png,attachment-local 存储时已生成
|
|
3877
|
+
+ * 带扩展名别名,见 store.ts extensionAliasPath),zai-vision 等按扩展名校验的工具可用。
|
|
3878
|
+
+ */
|
|
3879
|
+
+function imageAsText(block: ContentBlock): ContentBlock {
|
|
3880
|
+
+ if (block.type !== 'image') return block
|
|
3881
|
+
+ const ref = (block as { attachment?: { attachmentId?: unknown; name?: unknown; mediaType?: unknown } }).attachment
|
|
3882
|
+
+ const rawId = typeof ref?.attachmentId === 'string' ? ref.attachmentId : ''
|
|
3883
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3884
|
+
+ const name = typeof ref?.name === 'string' && ref.name.length > 0 ? ref.name : 'image'
|
|
3885
|
+
+ const mediaType = typeof ref?.mediaType === 'string' ? ref.mediaType : 'image/jpeg'
|
|
3886
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3887
|
+
+ const ext = mediaType === 'image/jpeg' ? '.jpg' : '.png'
|
|
3888
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3889
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex) + ext
|
|
3890
|
+
+ : '(unknown)'
|
|
3891
|
+
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path):${path}]` }
|
|
3892
|
+
}
|
|
3893
|
+
|
|
3894
|
+
-/** Reject image roles that pi-ai cannot replay before request-size offloading can replace them. */
|
|
3895
|
+
-function assertSupportedImageRoles(messages: readonly Message[]): void {
|
|
3896
|
+
- for (const message of messages) {
|
|
3897
|
+
- if (message.role !== 'user' && contentHasImage(message.content)) {
|
|
3898
|
+
- throw new LlmError(
|
|
3899
|
+
- `pi-ai cannot represent an image in an in-history ${message.role} message`,
|
|
3900
|
+
- 'UNSUPPORTED_CONTENT',
|
|
3901
|
+
- )
|
|
3902
|
+
+/** Convert image blocks to path-text when the route model is not a vision model. */
|
|
3903
|
+
+function imagesToText(blocks: readonly ContentBlock[], vision: boolean): readonly ContentBlock[] {
|
|
3904
|
+
+ if (vision) return blocks
|
|
3905
|
+
+ const out: ContentBlock[] = []
|
|
3906
|
+
+ for (const block of blocks) {
|
|
3907
|
+
+ if (block.type === 'image') {
|
|
3908
|
+
+ out.push(imageAsText(block))
|
|
3909
|
+
+ } else if (block.type === 'tool-result') {
|
|
3910
|
+
+ out.push({ ...block, content: [...imagesToText(block.content, vision)] })
|
|
3911
|
+
+ } else {
|
|
3912
|
+
+ out.push(block)
|
|
3913
|
+
}
|
|
3914
|
+
}
|
|
3915
|
+
+ return out
|
|
3916
|
+
}
|
|
3917
|
+
|
|
3918
|
+
async function userContent(
|
|
3919
|
+
blocks: readonly ContentBlock[],
|
|
3920
|
+
- attachments: AttachmentStore,
|
|
3921
|
+
+ attachments: AttachmentStore | undefined,
|
|
3922
|
+
+ vision: boolean,
|
|
3923
|
+
): Promise<string | (TextContent | ImageContent)[]> {
|
|
3924
|
+
+ // [本地改造 2026-08-16] 非视觉模型:图片块先整体转路径文本(agent 用视觉 MCP 看图),
|
|
3925
|
+
+ // 不再需要 attachments;视觉模型保持原逻辑(读原图送 pi-ai)。
|
|
3926
|
+
+ const converted = imagesToText(blocks, vision)
|
|
3927
|
+
+ if (!vision) {
|
|
3928
|
+
+ return flattenBlocks(converted)
|
|
3929
|
+
+ }
|
|
3930
|
+
const content: (TextContent | ImageContent)[] = []
|
|
3931
|
+
- for (const block of blocks) {
|
|
3932
|
+
+ for (const block of converted) {
|
|
3933
|
+
switch (block.type) {
|
|
3934
|
+
case 'text':
|
|
3935
|
+
if (block.text.length > 0) content.push({ type: 'text', text: block.text })
|
|
3936
|
+
break
|
|
3937
|
+
+ case 'voice': {
|
|
3938
|
+
+ // [本地改造 2026-08-16] 语音块转文本(识别文本/本地路径),与文本块同路进模型。
|
|
3939
|
+
+ const asText = voiceAsText(block)
|
|
3940
|
+
+ if (asText.type === 'text' && asText.text.length > 0) {
|
|
3941
|
+
+ content.push({ type: 'text', text: asText.text })
|
|
3942
|
+
+ }
|
|
3943
|
+
+ break
|
|
3944
|
+
+ }
|
|
3945
|
+
case 'image': {
|
|
3946
|
+
+ if (attachments === undefined) {
|
|
3947
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3948
|
+
+ }
|
|
3949
|
+
const stored = await attachments.readImage(block.attachment)
|
|
3950
|
+
content.push({
|
|
3951
|
+
type: 'image',
|
|
3952
|
+
@@ -59,7 +130,7 @@ async function userContent(
|
|
3953
|
+
}
|
|
3954
|
+
case 'tool-result':
|
|
3955
|
+
{
|
|
3956
|
+
- const nested = await userContent(block.content, attachments)
|
|
3957
|
+
+ const nested = await userContent(block.content, attachments, vision)
|
|
3958
|
+
if (typeof nested === 'string') {
|
|
3959
|
+
if (nested.length > 0) content.push({ type: 'text', text: nested })
|
|
3960
|
+
} else {
|
|
3961
|
+
@@ -96,34 +167,45 @@ function piContext(options: GenerateOptions, messages: PiMessage[]): PiContext {
|
|
3962
|
+
}
|
|
3963
|
+
}
|
|
3964
|
+
|
|
3965
|
+
-function textOnlyContext(options: GenerateOptions, onReplayDegrade?: (reason: string) => void): PiContext {
|
|
3966
|
+
+function textOnlyContext(options: GenerateOptions, vision: boolean): PiContext {
|
|
3967
|
+
const toolNames = new Map<CallId, string>()
|
|
3968
|
+
const messages: PiMessage[] = []
|
|
3969
|
+
for (const message of options.messages) {
|
|
3970
|
+
- if (contentHasImage(message.content)) {
|
|
3971
|
+
- throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3972
|
+
- }
|
|
3973
|
+
if (message.role === 'system') {
|
|
3974
|
+
- messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
|
|
3975
|
+
+ // 视觉模型无法在 pi-ai 单一 systemPrompt 槽内表达图片;非视觉模型
|
|
3976
|
+
+ // (vision=false)走文本路径(imagesToText 转路径文本)。
|
|
3977
|
+
+ if (vision && contentHasImage(message.content)) {
|
|
3978
|
+
+ throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
|
|
3979
|
+
+ }
|
|
3980
|
+
+ messages.push({ role: 'user', content: flattenBlocks(imagesToText(message.content, vision)), timestamp: 0 })
|
|
3981
|
+
continue
|
|
3982
|
+
}
|
|
3983
|
+
if (message.role === 'assistant') {
|
|
3984
|
+
- const assistant = toPiAssistant(message, onReplayDegrade)
|
|
3985
|
+
+ const assistant = toPiAssistant(message)
|
|
3986
|
+
for (const block of assistant.content) if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
|
|
3987
|
+
messages.push(assistant)
|
|
3988
|
+
continue
|
|
3989
|
+
}
|
|
3990
|
+
- const text = flattenText(message)
|
|
3991
|
+
+ const regular = message.content.filter(block => block.type !== 'tool-result')
|
|
3992
|
+
+ // [本地改造 2026-08-16] 非视觉模型(vision=false):图片块转路径文本后扁平化;
|
|
3993
|
+
+ // 视觉模型(vision=true)无 durable attachment 服务时仍拒绝(必须读原图)。
|
|
3994
|
+
+ if (vision && contentHasImage(regular)) {
|
|
3995
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
3996
|
+
+ }
|
|
3997
|
+
+ const text = flattenBlocks(imagesToText(regular, vision))
|
|
3998
|
+
const results = message.content.filter(block => block.type === 'tool-result')
|
|
3999
|
+
if (text.length > 0 || results.length === 0) messages.push({ role: 'user', content: text, timestamp: 0 })
|
|
4000
|
+
for (const result of results) {
|
|
4001
|
+
+ if (vision && contentHasImage(result.content)) {
|
|
4002
|
+
+ throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
|
|
4003
|
+
+ }
|
|
4004
|
+
messages.push({
|
|
4005
|
+
role: 'toolResult',
|
|
4006
|
+
toolCallId: result.toolCallId,
|
|
4007
|
+
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
|
|
4008
|
+
content: [{
|
|
4009
|
+
type: 'text',
|
|
4010
|
+
- text: toolResultText(result.content) || '(no output)',
|
|
4011
|
+
+ text: flattenBlocks(imagesToText(result.content, vision)) || '(no output)',
|
|
4012
|
+
}],
|
|
4013
|
+
isError: result.isError ?? false,
|
|
4014
|
+
timestamp: 0,
|
|
4015
|
+
@@ -133,69 +215,42 @@ function textOnlyContext(options: GenerateOptions, onReplayDegrade?: (reason: st
|
|
4016
|
+
return piContext(options, messages)
|
|
4017
|
+
}
|
|
4018
|
+
|
|
4019
|
+
-/**
|
|
4020
|
+
- * Convert text-only harness history to a synchronous pi-ai Context. Tool
|
|
4021
|
+
- * result names are recovered from preceding assistant tool calls.
|
|
4022
|
+
- * @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
|
|
4023
|
+
- * @param attachments - absent; selects the synchronous conversion.
|
|
4024
|
+
- * @param onReplayDegrade - forwarded to {@link toPiAssistant} for each assistant message.
|
|
4025
|
+
- * @returns the pi-ai context; `tools` is omitted when the request declares none.
|
|
4026
|
+
- */
|
|
4027
|
+
-export function toPiContext(
|
|
4028
|
+
- options: GenerateOptions,
|
|
4029
|
+
- attachments?: undefined,
|
|
4030
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4031
|
+
-): PiContext
|
|
4032
|
+
/**
|
|
4033
|
+
* Convert harness history to a pi-ai Context while resolving durable images.
|
|
4034
|
+
- * Tool result names are recovered from preceding assistant tool calls. When
|
|
4035
|
+
- * the accumulated base64 image payload exceeds `maxRequestImageBytes`, the
|
|
4036
|
+
- * oldest images are replaced by text placeholders until the request fits, so
|
|
4037
|
+
- * an image-heavy session keeps clearing gateway request-size caps.
|
|
4038
|
+
+ * Tool result names are recovered from preceding assistant tool calls.
|
|
4039
|
+
+ * [本地改造 2026-08-16] vision=false(模型 input 不含 image)时,图片块转为
|
|
4040
|
+
+ * 本地路径文本(agent 用视觉 MCP 识图),与 llm-deepseek serialize.ts 一致;
|
|
4041
|
+
+ * attachments 可为 undefined(非视觉路径不需要 durable attachment 服务)。
|
|
4042
|
+
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
|
|
4043
|
+
- * @param attachments - durable byte resolver for image references.
|
|
4044
|
+
- * @param onReplayDegrade - forwarded to {@link toPiAssistant} for each assistant message.
|
|
4045
|
+
- * @param maxRequestImageBytes - request-level bound on base64-encoded image payload; omission leaves every image in place.
|
|
4046
|
+
+ * @param attachments - durable byte resolver for image references (vision models); optional.
|
|
4047
|
+
+ * @param vision - whether the route model accepts image input.
|
|
4048
|
+
* @returns the asynchronously resolved pi-ai context.
|
|
4049
|
+
*/
|
|
4050
|
+
-export function toPiContext(
|
|
4051
|
+
- options: GenerateOptions,
|
|
4052
|
+
- attachments: AttachmentStore,
|
|
4053
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4054
|
+
- maxRequestImageBytes?: number,
|
|
4055
|
+
-): Promise<PiContext>
|
|
4056
|
+
-export function toPiContext(
|
|
4057
|
+
- options: GenerateOptions,
|
|
4058
|
+
- attachments?: AttachmentStore,
|
|
4059
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4060
|
+
- maxRequestImageBytes?: number,
|
|
4061
|
+
-): PiContext | Promise<PiContext> {
|
|
4062
|
+
- return attachments === undefined
|
|
4063
|
+
- ? textOnlyContext(options, onReplayDegrade)
|
|
4064
|
+
- : toPiContextWithImages(options, attachments, onReplayDegrade, maxRequestImageBytes)
|
|
4065
|
+
+export function toPiContext(options: GenerateOptions, attachments?: undefined, vision?: boolean): PiContext
|
|
4066
|
+
+export function toPiContext(options: GenerateOptions, attachments: AttachmentStore, vision?: boolean): Promise<PiContext>
|
|
4067
|
+
+export function toPiContext(options: GenerateOptions, attachments?: AttachmentStore, vision = true): PiContext | Promise<PiContext> {
|
|
4068
|
+
+ return attachments === undefined ? textOnlyContext(options, vision) : toPiContextWithImages(options, attachments, vision)
|
|
4069
|
+
}
|
|
4070
|
+
|
|
4071
|
+
-async function toPiContextWithImages(
|
|
4072
|
+
- options: GenerateOptions,
|
|
4073
|
+
- attachments: AttachmentStore,
|
|
4074
|
+
- onReplayDegrade?: (reason: string) => void,
|
|
4075
|
+
- maxRequestImageBytes?: number,
|
|
4076
|
+
-): Promise<PiContext> {
|
|
4077
|
+
- assertSupportedImageRoles(options.messages)
|
|
4078
|
+
- const requestMessages = offloadRequestImages(options.messages, maxRequestImageBytes)
|
|
4079
|
+
+async function toPiContextWithImages(options: GenerateOptions, attachments: AttachmentStore, vision: boolean): Promise<PiContext> {
|
|
4080
|
+
const toolNames = new Map<CallId, string>()
|
|
4081
|
+
const messages: PiMessage[] = []
|
|
4082
|
+
|
|
4083
|
+
- for (const message of requestMessages) {
|
|
4084
|
+
+ for (const message of options.messages) {
|
|
4085
|
+
if (message.role === 'system') {
|
|
4086
|
+
+ // 视觉模型无法在 pi-ai 单一 systemPrompt 槽内表达图片;非视觉模型
|
|
4087
|
+
+ // (vision=false)走文本路径(imagesToText 转路径文本)。
|
|
4088
|
+
+ if (vision && contentHasImage(message.content)) {
|
|
4089
|
+
+ throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
|
|
4090
|
+
+ }
|
|
4091
|
+
// pi-ai has a single systemPrompt slot; in-history system messages are
|
|
4092
|
+
// folded into user messages to preserve order (rare in practice — the
|
|
4093
|
+
// harness sends the system prompt via options.system).
|
|
4094
|
+
- messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
|
|
4095
|
+
+ messages.push({ role: 'user', content: flattenBlocks(imagesToText(message.content, vision)), timestamp: 0 })
|
|
4096
|
+
continue
|
|
4097
|
+
}
|
|
4098
|
+
if (message.role === 'assistant') {
|
|
4099
|
+
- const assistant = toPiAssistant(message, onReplayDegrade)
|
|
4100
|
+
+ const assistant = toPiAssistant(message)
|
|
4101
|
+
for (const block of assistant.content) {
|
|
4102
|
+
if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
|
|
4103
|
+
}
|
|
4104
|
+
@@ -204,15 +259,13 @@ async function toPiContextWithImages(
|
|
4105
|
+
}
|
|
4106
|
+
// user role: text + tool results (each result becomes its own message).
|
|
4107
|
+
const regular = message.content.filter(block => block.type !== 'tool-result')
|
|
4108
|
+
- const content = await userContent(regular, attachments)
|
|
4109
|
+
- const results = message.content.filter((block): block is Extract<ContentBlock, { type: 'tool-result' }> => (
|
|
4110
|
+
- block.type === 'tool-result'
|
|
4111
|
+
- ))
|
|
4112
|
+
+ const content = await userContent(regular, attachments, vision)
|
|
4113
|
+
+ const results = message.content.filter(block => block.type === 'tool-result')
|
|
4114
|
+
if (content.length > 0 || results.length === 0) {
|
|
4115
|
+
messages.push({ role: 'user', content, timestamp: 0 })
|
|
4116
|
+
}
|
|
4117
|
+
for (const result of results) {
|
|
4118
|
+
- const resultContent = await userContent(result.content, attachments)
|
|
4119
|
+
+ const resultContent = await userContent(result.content, attachments, vision)
|
|
4120
|
+
messages.push({
|
|
4121
|
+
role: 'toolResult',
|
|
4122
|
+
toolCallId: result.toolCallId,
|
|
3750
4123
|
diff --git a/packages/llm/llm/src/types.ts b/packages/llm/llm/src/types.ts
|
|
3751
4124
|
index 8c5be187dd..750496f164 100644
|
|
3752
4125
|
--- a/packages/llm/llm/src/types.ts
|