dsh-agent-sdk-bridge 1.1.5 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEVELOPMENT.md +34 -0
- package/FAQ.md +24 -0
- package/client.js +140 -62
- package/lib/index.js +282 -60
- package/lib/types/adapter.d.ts +59 -12
- package/lib/types/api.d.ts +7 -0
- package/lib/types/codebuddy-adapter.d.ts +33 -0
- package/lib/types/codebuddy.d.ts +39 -0
- package/lib/types/index.d.ts +10 -0
- package/lib/types/minimax-adapter.d.ts +8 -0
- package/lib/types/render.d.ts +17 -4
- package/package.json +1 -1
- package/src/adapter.ts +125 -25
- package/src/api.ts +15 -1
- package/src/codebuddy-adapter.ts +91 -16
- package/src/codebuddy.ts +66 -3
- package/src/index.ts +33 -0
- package/src/minimax-adapter.ts +37 -13
- package/src/render.ts +42 -12
package/src/codebuddy.ts
CHANGED
|
@@ -108,7 +108,16 @@ export interface CodeBuddyModel {
|
|
|
108
108
|
description?: string
|
|
109
109
|
maxInputTokens?: number
|
|
110
110
|
maxOutputTokens?: number
|
|
111
|
+
/**
|
|
112
|
+
* Context budgets the provider actually accepts for this model.
|
|
113
|
+
*
|
|
114
|
+
* Empty means the model exposes no selectable window: CodeBuddy validates
|
|
115
|
+
* `set_config.contextWindow` against `contextWindow.supportedLengths` and
|
|
116
|
+
* rejects everything else with `Allowed: <none>`, so the settings page must
|
|
117
|
+
* not offer a choice the CLI will refuse.
|
|
118
|
+
*/
|
|
111
119
|
contextWindows: number[]
|
|
120
|
+
/** The window the model actually runs with (its physical input limit). */
|
|
112
121
|
defaultContextWindow: number
|
|
113
122
|
vision: boolean
|
|
114
123
|
reasoningEfforts?: string[]
|
|
@@ -256,12 +265,15 @@ function priceFactorOf(model: RawLanguageModel): number | undefined {
|
|
|
256
265
|
|
|
257
266
|
/** Turn one raw catalog entry into the model row this plugin uses. */
|
|
258
267
|
export function toCodeBuddyModel(model: RawLanguageModel): CodeBuddyModel {
|
|
268
|
+
// Only what the provider declares as selectable is a choice. The physical
|
|
269
|
+
// `maxInputTokens` is the window the model runs with, not a menu entry:
|
|
270
|
+
// folding it into `contextWindows` offered 200K/1M switches that the CLI
|
|
271
|
+
// answered with `Invalid context_window value … Allowed: <none>`.
|
|
259
272
|
const windows = new Set<number>(model.contextWindow?.supportedLengths ?? [])
|
|
260
273
|
const declared = model.contextWindow?.defaultLength
|
|
261
274
|
?? model.maxInputTokens
|
|
262
275
|
?? model.maxAllowedSize
|
|
263
276
|
?? CODEBUDDY_DEFAULT_CONTEXT_WINDOW
|
|
264
|
-
windows.add(declared)
|
|
265
277
|
return {
|
|
266
278
|
id: model.id,
|
|
267
279
|
name: model.name ?? model.id,
|
|
@@ -334,6 +346,16 @@ export class CodeBuddyCatalog {
|
|
|
334
346
|
private readonly timeoutMs: Live<number> = CODEBUDDY_FETCH_TIMEOUT_MS,
|
|
335
347
|
) {}
|
|
336
348
|
|
|
349
|
+
/**
|
|
350
|
+
* The last catalog read for a region, without fetching.
|
|
351
|
+
*
|
|
352
|
+
* Callers that must answer synchronously (deciding whether a model exposes a
|
|
353
|
+
* context-budget choice) use this instead of awaiting a refresh.
|
|
354
|
+
*/
|
|
355
|
+
snapshot(requested?: CodeBuddyRegion): readonly CodeBuddyModel[] {
|
|
356
|
+
return this.snapshots.get(requested ?? readLive(this.region, 'global'))?.models ?? []
|
|
357
|
+
}
|
|
358
|
+
|
|
337
359
|
/** Models for one region; a failed refresh keeps the last good list. */
|
|
338
360
|
async models(requested?: CodeBuddyRegion): Promise<readonly CodeBuddyModel[]> {
|
|
339
361
|
const region = requested ?? readLive(this.region, 'global')
|
|
@@ -911,6 +933,10 @@ export class CodeBuddySession {
|
|
|
911
933
|
private effort: string | undefined
|
|
912
934
|
private systemPrompt: string | undefined
|
|
913
935
|
private cwd: string | undefined
|
|
936
|
+
/** Context budget the host shows for this session, when a model offers one. */
|
|
937
|
+
private contextWindow: number | undefined
|
|
938
|
+
/** Budget already pushed to the CLI, keyed by model and value. */
|
|
939
|
+
private appliedWindow: string | undefined
|
|
914
940
|
/** Host messages already fed to the inner session, for continuation planning. */
|
|
915
941
|
fedMessages: readonly import('@deepseek-ai/dsh-llm').RequestMessage[] | undefined
|
|
916
942
|
/** Host system prompt last applied; informational for the adapter. */
|
|
@@ -927,12 +953,47 @@ export class CodeBuddySession {
|
|
|
927
953
|
}
|
|
928
954
|
|
|
929
955
|
/** Point the session at a region, model and optional system prompt. */
|
|
930
|
-
configure(
|
|
956
|
+
configure(
|
|
957
|
+
region: CodeBuddyRegion,
|
|
958
|
+
model: string,
|
|
959
|
+
options: { effort?: string, system?: string, cwd?: string, contextWindow?: number } = {},
|
|
960
|
+
): void {
|
|
931
961
|
this.region = region
|
|
932
962
|
this.model = model
|
|
933
963
|
this.effort = options.effort
|
|
934
964
|
this.systemPrompt = options.system
|
|
935
965
|
if (options.cwd !== undefined) this.cwd = options.cwd
|
|
966
|
+
if (options.contextWindow !== undefined) this.contextWindow = options.contextWindow
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
/**
|
|
970
|
+
* Push the host-visible context window into the inner session.
|
|
971
|
+
*
|
|
972
|
+
* The CLI resolves its own budget (`override > model default > physical
|
|
973
|
+
* limit`) and both its compaction trigger and the upstream gateway enforce
|
|
974
|
+
* that budget. A global-catalog model such as `deepseek-v4.1-flash` declares
|
|
975
|
+
* `supportedLengths [300K, 1M]` with a 300K default, so a session left alone
|
|
976
|
+
* compacts (and the gateway rejects) anything past 276K — while the host,
|
|
977
|
+
* told 1M, keeps showing and feeding a larger history. Replaying that history
|
|
978
|
+
* then collapses to a summary. Applying the same number the host divides by
|
|
979
|
+
* keeps the two in step; models that expose no choice reject it and keep
|
|
980
|
+
* their default.
|
|
981
|
+
*
|
|
982
|
+
* The CLI refuses to clear the override (`contextWindow must be a positive
|
|
983
|
+
* safe integer`), so the adapter sends the model's default instead when the
|
|
984
|
+
* page returns to "default".
|
|
985
|
+
*/
|
|
986
|
+
private async applyContextWindow(session: SessionImpl): Promise<void> {
|
|
987
|
+
const window = this.contextWindow
|
|
988
|
+
if (window === undefined) return
|
|
989
|
+
const key = `${this.model}:${window}`
|
|
990
|
+
if (this.appliedWindow === key) return
|
|
991
|
+
this.appliedWindow = key
|
|
992
|
+
try {
|
|
993
|
+
await session.setConfig({ contextWindow: window })
|
|
994
|
+
} catch {
|
|
995
|
+
// `Allowed: <none>` models have one fixed window; their default stands.
|
|
996
|
+
}
|
|
936
997
|
}
|
|
937
998
|
|
|
938
999
|
/**
|
|
@@ -1009,8 +1070,10 @@ export class CodeBuddySession {
|
|
|
1009
1070
|
signal?.addEventListener('abort', onAbort, { once: true })
|
|
1010
1071
|
try {
|
|
1011
1072
|
// A model switch on a warm session is a control request; waiting for it
|
|
1012
|
-
// keeps the next prompt on the model the host just resolved.
|
|
1073
|
+
// keeps the next prompt on the model the host just resolved. The window
|
|
1074
|
+
// follows it: the CLI validates the value against the current model.
|
|
1013
1075
|
await session.setModel(this.model)
|
|
1076
|
+
await this.applyContextWindow(session)
|
|
1014
1077
|
if (feed !== null && feed.length > 0) {
|
|
1015
1078
|
const content = typeof feed === 'string'
|
|
1016
1079
|
? feed
|
package/src/index.ts
CHANGED
|
@@ -54,10 +54,21 @@ interface WebServerService {
|
|
|
54
54
|
}): () => void
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The mounted filesystem provider, used only to translate a host path into
|
|
59
|
+
* the execution world the host tools run in. Declared minimally so the plugin
|
|
60
|
+
* keeps only the peer dependencies it actually needs.
|
|
61
|
+
*/
|
|
62
|
+
interface FileSystemService {
|
|
63
|
+
processPathFromHostPath?(hostPath: string): string | undefined
|
|
64
|
+
}
|
|
65
|
+
|
|
57
66
|
declare module '@deepseek-ai/cordis' {
|
|
58
67
|
interface Context {
|
|
59
68
|
/** Present only in web compositions; reached through a nested inject. */
|
|
60
69
|
webServer?: WebServerService
|
|
70
|
+
/** Present when a filesystem provider is mounted; reached by inject. */
|
|
71
|
+
fs?: FileSystemService
|
|
61
72
|
}
|
|
62
73
|
|
|
63
74
|
interface Events {
|
|
@@ -208,6 +219,7 @@ export function apply(ctx: Context, config: ParsedConfig): void {
|
|
|
208
219
|
// plugin's answer for the rest of the process lifetime.
|
|
209
220
|
const live = liveConfig(config)
|
|
210
221
|
let attachments: AttachmentStore | undefined
|
|
222
|
+
let filesystem: FileSystemService | undefined
|
|
211
223
|
|
|
212
224
|
/**
|
|
213
225
|
* Everything both editions share.
|
|
@@ -235,6 +247,20 @@ export function apply(ctx: Context, config: ParsedConfig): void {
|
|
|
235
247
|
return store.readImageRequest(ref, { width: ref.width, height: ref.height, maxBytes: REQUEST_IMAGE_MAX_BYTES }, signal)
|
|
236
248
|
.then(image => ({ bytes: image.data, mediaType: image.mediaType }))
|
|
237
249
|
},
|
|
250
|
+
/**
|
|
251
|
+
* Where the model can find the image on disk. Pixels let it describe the
|
|
252
|
+
* picture; this path lets a "modify and save it" turn act on the file
|
|
253
|
+
* instead of searching the workspace for something that was never there.
|
|
254
|
+
*/
|
|
255
|
+
imagePath: (ref: ImageAttachmentRef): string | undefined => {
|
|
256
|
+
const store = attachments
|
|
257
|
+
if (store === undefined) return undefined
|
|
258
|
+
const hostPath = store.imageHostPath(ref)
|
|
259
|
+
if (hostPath === undefined) return undefined
|
|
260
|
+
// Prefer the mounted tools' view of the same file; the local provider
|
|
261
|
+
// maps identically, a sandboxed one may not.
|
|
262
|
+
return filesystem?.processPathFromHostPath?.(hostPath) ?? hostPath
|
|
263
|
+
},
|
|
238
264
|
}
|
|
239
265
|
|
|
240
266
|
/**
|
|
@@ -279,6 +305,7 @@ export function apply(ctx: Context, config: ParsedConfig): void {
|
|
|
279
305
|
: live.codebuddyGlobalModelContextWindows(),
|
|
280
306
|
modelAliases: live.codebuddyModelAliases,
|
|
281
307
|
readImage: shared.readImage,
|
|
308
|
+
imagePath: shared.imagePath,
|
|
282
309
|
}),
|
|
283
310
|
minimax: new MinimaxAdapter({
|
|
284
311
|
maxSessions: live.maxSessions,
|
|
@@ -298,6 +325,7 @@ export function apply(ctx: Context, config: ParsedConfig): void {
|
|
|
298
325
|
: live.minimaxGlobalModelContextWindows(),
|
|
299
326
|
modelAliases: live.minimaxModelAliases,
|
|
300
327
|
readImage: shared.readImage,
|
|
328
|
+
imagePath: shared.imagePath,
|
|
301
329
|
}),
|
|
302
330
|
}
|
|
303
331
|
|
|
@@ -306,6 +334,11 @@ export function apply(ctx: Context, config: ParsedConfig): void {
|
|
|
306
334
|
scope.effect(() => () => { attachments = undefined }, 'llm-qoder-cn.attachments')
|
|
307
335
|
})
|
|
308
336
|
|
|
337
|
+
ctx.inject(['fs'], (scope) => {
|
|
338
|
+
filesystem = scope.fs
|
|
339
|
+
scope.effect(() => () => { filesystem = undefined }, 'llm-qoder-cn.fs')
|
|
340
|
+
})
|
|
341
|
+
|
|
309
342
|
// Routes and the provider directory are the two shapes a live value cannot
|
|
310
343
|
// reach on its own: which provider ids exist, and the labels the selection
|
|
311
344
|
// surfaces render. A disabled provider must own neither, so both are kept in
|
package/src/minimax-adapter.ts
CHANGED
|
@@ -22,15 +22,14 @@ import {
|
|
|
22
22
|
minimaxColdStream, minimaxRegionBlockMessage, minimaxRegionBlocked,
|
|
23
23
|
minimaxRegionBlockedCached, minimaxSignedIn, minimaxUsage,
|
|
24
24
|
} from './minimax.ts'
|
|
25
|
-
import { textOnlyImageText } from '@deepseek-ai/dsh-llm'
|
|
26
25
|
import type { MinimaxModel, MinimaxPromptBlock, MinimaxUsageSnapshot } from './minimax.ts'
|
|
27
26
|
import { resolveMinimaxCli } from './minimax-cli.ts'
|
|
28
27
|
import { reasoningInfo, planContinuation } from './adapter.ts'
|
|
29
28
|
import type { ImageRequestReader } from './adapter.ts'
|
|
30
29
|
import type { Live } from './live.ts'
|
|
31
30
|
import { readLive, readLiveOptional } from './live.ts'
|
|
32
|
-
import type { Feed } from './render.ts'
|
|
33
|
-
import { feedImageRefs, feedToText, renderInitialFeed, resolvedFeedParts } from './render.ts'
|
|
31
|
+
import type { Feed, ImagePathResolver } from './render.ts'
|
|
32
|
+
import { feedImageRefs, feedToText, imageRefText, renderInitialFeed, resolvedFeedParts } from './render.ts'
|
|
34
33
|
|
|
35
34
|
/** The MiniMax Code provider route. */
|
|
36
35
|
export const MINIMAX_PROVIDER = 'minimax-acp'
|
|
@@ -56,6 +55,11 @@ export interface MinimaxAdapterOptions {
|
|
|
56
55
|
* store) → images degrade to their handle text instead of failing the turn.
|
|
57
56
|
*/
|
|
58
57
|
readImage?: ImageRequestReader
|
|
58
|
+
/**
|
|
59
|
+
* The tool-world path of one image, for the handle text that lets the model
|
|
60
|
+
* edit it. Absent → an unreadable image keeps the harness placeholder.
|
|
61
|
+
*/
|
|
62
|
+
imagePath?: ImagePathResolver
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
const NO_MODELS: readonly string[] = []
|
|
@@ -90,6 +94,8 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
90
94
|
private readonly auth = new MinimaxAuth()
|
|
91
95
|
private readonly mcpServer = new MinimaxMcpServer()
|
|
92
96
|
private readonly disabledCache = new Map<MinimaxRegion, { key: string, set: ReadonlySet<string> }>()
|
|
97
|
+
/** Last verified image capability, keyed `${region}:${model}`. */
|
|
98
|
+
private readonly knownVision = new Map<string, boolean>()
|
|
93
99
|
|
|
94
100
|
constructor(options: MinimaxAdapterOptions = {}) {
|
|
95
101
|
super()
|
|
@@ -136,7 +142,9 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
136
142
|
async describeCatalog(region: MinimaxRegion = this.region): Promise<readonly MinimaxModel[]> {
|
|
137
143
|
if (!this.isSignedIn(region)) return []
|
|
138
144
|
if (await minimaxRegionBlocked(region)) return []
|
|
139
|
-
|
|
145
|
+
const models = await this.catalog.models(region)
|
|
146
|
+
for (const model of models) this.knownVision.set(`${region}:${model.id}`, model.vision)
|
|
147
|
+
return models
|
|
140
148
|
}
|
|
141
149
|
|
|
142
150
|
/** Whether the region's endpoints are refusing this machine's network. */
|
|
@@ -246,8 +254,10 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
246
254
|
override async resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
|
247
255
|
const region = this.region
|
|
248
256
|
const target = resolveMinimaxModelId(model, this.aliases)
|
|
257
|
+
const key = `${region}:${target}`
|
|
249
258
|
const live = (await this.catalog.models(region)).find(entry => entry.id === target)
|
|
250
259
|
if (live !== undefined) {
|
|
260
|
+
this.knownVision.set(key, live.vision)
|
|
251
261
|
const reasoning = reasoningInfo(live.reasoningEfforts, live.defaultEffort, live.reasoning)
|
|
252
262
|
return {
|
|
253
263
|
provider,
|
|
@@ -269,7 +279,7 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
269
279
|
provider,
|
|
270
280
|
id: model,
|
|
271
281
|
name: model,
|
|
272
|
-
inputModalities: ['text'],
|
|
282
|
+
inputModalities: this.knownVision.get(key) === true ? ['text', 'image'] : ['text'],
|
|
273
283
|
context: { contextWindow: this.contextWindowFor(target, region) ?? MINIMAX_DEFAULT_CONTEXT_WINDOW },
|
|
274
284
|
defaultMaxTokens: readLive(this.options.defaultMaxTokens, MINIMAX_DEFAULT_MAX_TOKENS),
|
|
275
285
|
...reasoning === undefined ? {} : { reasoning },
|
|
@@ -297,7 +307,7 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
297
307
|
}
|
|
298
308
|
const model = resolveMinimaxModelId(options.model, this.aliases)
|
|
299
309
|
if (options.sessionId === undefined || options.purpose !== undefined) {
|
|
300
|
-
const prompt = feedToText(renderInitialFeed(options.system, options.messages))
|
|
310
|
+
const prompt = feedToText(renderInitialFeed(options.system, options.messages, this.options.imagePath), this.options.imagePath)
|
|
301
311
|
+ '\n(这是一次性旁路请求,直接输出下一条助手回复。)'
|
|
302
312
|
for (const chunk of await minimaxColdStream(region, prompt, model, options)) yield chunk
|
|
303
313
|
return
|
|
@@ -367,7 +377,7 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
367
377
|
session.fedSystem = options.system
|
|
368
378
|
const vision = await this.modelVision(model, region)
|
|
369
379
|
const feed = await this.buildFeed(
|
|
370
|
-
renderInitialFeed(undefined, options.messages),
|
|
380
|
+
renderInitialFeed(undefined, options.messages, this.options.imagePath),
|
|
371
381
|
vision,
|
|
372
382
|
options.signal,
|
|
373
383
|
)
|
|
@@ -377,8 +387,15 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
377
387
|
/** Whether the resolved model accepts images (drives whether pixels are sent). */
|
|
378
388
|
private async modelVision(model: string, region: MinimaxRegion): Promise<boolean> {
|
|
379
389
|
const target = resolveMinimaxModelId(model, this.aliases)
|
|
390
|
+
const key = `${region}:${target}`
|
|
380
391
|
const live = (await this.catalog.models(region).catch(() => [])).find(entry => entry.id === target)
|
|
381
|
-
|
|
392
|
+
if (live !== undefined) {
|
|
393
|
+
this.knownVision.set(key, live.vision)
|
|
394
|
+
return live.vision
|
|
395
|
+
}
|
|
396
|
+
// Keep a verified vision capability across a missed catalog read; otherwise
|
|
397
|
+
// DSH strips the pixels before this adapter ever runs.
|
|
398
|
+
return this.knownVision.get(key) === true
|
|
382
399
|
}
|
|
383
400
|
|
|
384
401
|
/**
|
|
@@ -395,7 +412,7 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
395
412
|
signal?: AbortSignal,
|
|
396
413
|
): Promise<string | MinimaxPromptBlock[]> {
|
|
397
414
|
if (typeof feed === 'string') return feed
|
|
398
|
-
if (!vision) return feedToText(feed)
|
|
415
|
+
if (!vision) return feedToText(feed, this.options.imagePath)
|
|
399
416
|
const images = new Map<string, { data: string, mediaType: string }>()
|
|
400
417
|
const readImage = this.options.readImage
|
|
401
418
|
if (readImage !== undefined) {
|
|
@@ -422,10 +439,17 @@ export class MinimaxAdapter extends LlmAdapter {
|
|
|
422
439
|
content.push({ type: 'text', text: part.text })
|
|
423
440
|
continue
|
|
424
441
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
442
|
+
// The pixels let the model read the image; this handle lets it find and
|
|
443
|
+
// edit the source file. Without it a "modify this image" turn searches
|
|
444
|
+
// the workspace for a file that was never there.
|
|
445
|
+
const path = this.options.imagePath?.(part.attachment)
|
|
446
|
+
content.push({
|
|
447
|
+
type: 'text',
|
|
448
|
+
text: part.type === 'image-unreadable' && path === undefined
|
|
449
|
+
? `[图片附件 ${String(part.attachment.attachmentId)} 本次未能读取,请基于文字内容继续]`
|
|
450
|
+
: imageRefText(part.attachment, path),
|
|
451
|
+
})
|
|
452
|
+
if (part.type === 'image-unreadable') continue
|
|
429
453
|
content.push({ type: 'image', data: part.data, mimeType: part.mediaType })
|
|
430
454
|
}
|
|
431
455
|
return content
|
package/src/render.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* @module dsh-llm-qoder/render
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { fileHandleText, offloadedImageText, textOnlyImageText } from '@deepseek-ai/dsh-llm'
|
|
16
|
+
import { fileHandleText, offloadedImageText, requestImageHandleText, textOnlyImageText } from '@deepseek-ai/dsh-llm'
|
|
17
17
|
import type { ContentBlock, RequestMessage } from '@deepseek-ai/dsh-llm'
|
|
18
18
|
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
19
19
|
import { countWideChars } from './tokens.ts'
|
|
@@ -23,6 +23,25 @@ export type FeedPart =
|
|
|
23
23
|
| { type: 'text', text: string }
|
|
24
24
|
| { type: 'image', attachment: ImageAttachmentRef }
|
|
25
25
|
|
|
26
|
+
/** Resolve the tool-world path of one durable image attachment. */
|
|
27
|
+
export type ImagePathResolver = (ref: ImageAttachmentRef) => string | undefined
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Model-facing handle for one image.
|
|
31
|
+
*
|
|
32
|
+
* The host's native adapters use the same text: it names the attachment, its
|
|
33
|
+
* pixel size and media type, and the read-only normalized copy on disk, and it
|
|
34
|
+
* tells the model to copy that file to a writable path before editing. Pixels
|
|
35
|
+
* alone let a model describe an image but not process it — without this path
|
|
36
|
+
* the model goes looking through the workspace for a file that was never there.
|
|
37
|
+
*/
|
|
38
|
+
export function imageRefText(ref: ImageAttachmentRef, path: string | undefined): string {
|
|
39
|
+
if (path === undefined) {
|
|
40
|
+
return `[图片附件 ${String(ref.attachmentId)}(${ref.width}×${ref.height}px,${ref.mediaType})]`
|
|
41
|
+
}
|
|
42
|
+
return requestImageHandleText(ref, { width: ref.width, height: ref.height }, { readonlyPath: path })
|
|
43
|
+
}
|
|
44
|
+
|
|
26
45
|
/** What one inner turn receives: text (the historical shape) or text + images. */
|
|
27
46
|
export type Feed = string | FeedPart[]
|
|
28
47
|
|
|
@@ -40,12 +59,13 @@ export function imageRefs(blocks: readonly ContentBlock[]): ImageAttachmentRef[]
|
|
|
40
59
|
}
|
|
41
60
|
|
|
42
61
|
/** Render one content-block list as plain text; non-text blocks get handles. */
|
|
43
|
-
export function renderBlocks(blocks: readonly ContentBlock[]): string {
|
|
62
|
+
export function renderBlocks(blocks: readonly ContentBlock[], imagePath?: ImagePathResolver): string {
|
|
63
|
+
const parts: string[] = []
|
|
44
64
|
for (const block of blocks) {
|
|
45
65
|
switch (block.type) {
|
|
46
66
|
case 'text': parts.push(block.text); break
|
|
47
67
|
case 'reasoning': break
|
|
48
|
-
case 'image': parts.push(block.offloaded === true ? offloadedImageText(block.attachment) :
|
|
68
|
+
case 'image': parts.push(block.offloaded === true ? offloadedImageText(block.attachment) : imageRefText(block.attachment, imagePath?.(block.attachment))); break
|
|
49
69
|
case 'file': parts.push(fileHandleText(block.attachment, undefined)); break
|
|
50
70
|
case 'tool-call': parts.push(`[调用了工具 ${block.name}(${block.arguments})]`); break
|
|
51
71
|
case 'tool-addition': parts.push(`[工具启用 ${block.toolName}]`); break
|
|
@@ -71,8 +91,8 @@ function roleTag(message: RequestMessage): string {
|
|
|
71
91
|
* Render one request message with its role tag. A request-only user input has
|
|
72
92
|
* no durable identity or source, so it reads exactly like a user message.
|
|
73
93
|
*/
|
|
74
|
-
export function renderMessage(message: RequestMessage): string {
|
|
75
|
-
return `[${roleTag(message)}] ${renderBlocks(message.content)}`
|
|
94
|
+
export function renderMessage(message: RequestMessage, imagePath?: ImagePathResolver): string {
|
|
95
|
+
return `[${roleTag(message)}] ${renderBlocks(message.content, imagePath)}`
|
|
76
96
|
}
|
|
77
97
|
|
|
78
98
|
/** Preamble establishing the inner model's role as this agent's LLM backend. */
|
|
@@ -158,10 +178,16 @@ export function feedImageRefs(feed: Feed): ImageAttachmentRef[] {
|
|
|
158
178
|
}
|
|
159
179
|
|
|
160
180
|
/** Flatten a feed back to text, replacing images with the harness handle text. */
|
|
161
|
-
export function feedToText(feed: Feed): string {
|
|
181
|
+
export function feedToText(feed: Feed, imagePath?: ImagePathResolver): string {
|
|
162
182
|
if (typeof feed === 'string') return feed
|
|
163
183
|
return feed
|
|
164
|
-
.map(part =>
|
|
184
|
+
.map(part => {
|
|
185
|
+
if (part.type === 'text') return part.text
|
|
186
|
+
const path = imagePath?.(part.attachment)
|
|
187
|
+
// This function is used only where pixels are not sent, so the harness's
|
|
188
|
+
// "text only" wording is accurate when no path is available.
|
|
189
|
+
return path === undefined ? textOnlyImageText(part.attachment) : imageRefText(part.attachment, path)
|
|
190
|
+
})
|
|
165
191
|
.filter(text => text.length > 0)
|
|
166
192
|
.join('\n\n')
|
|
167
193
|
}
|
|
@@ -169,7 +195,7 @@ export function feedToText(feed: Feed): string {
|
|
|
169
195
|
/** One feed element after image bytes have been resolved (base64). */
|
|
170
196
|
export type ResolvedFeedPart =
|
|
171
197
|
| { type: 'text', text: string }
|
|
172
|
-
| { type: 'image', data: string, mediaType: string }
|
|
198
|
+
| { type: 'image', attachment: ImageAttachmentRef, data: string, mediaType: string }
|
|
173
199
|
| { type: 'image-unreadable', attachment: ImageAttachmentRef }
|
|
174
200
|
|
|
175
201
|
/**
|
|
@@ -195,7 +221,7 @@ export function resolvedFeedParts(feed: Feed, images: ReadonlyMap<string, { data
|
|
|
195
221
|
const resolved = images.get(String(part.attachment.attachmentId))
|
|
196
222
|
parts.push(resolved === undefined
|
|
197
223
|
? { type: 'image-unreadable', attachment: part.attachment }
|
|
198
|
-
: { type: 'image', data: resolved.data, mediaType: resolved.mediaType })
|
|
224
|
+
: { type: 'image', attachment: part.attachment, data: resolved.data, mediaType: resolved.mediaType })
|
|
199
225
|
}
|
|
200
226
|
return parts
|
|
201
227
|
}
|
|
@@ -222,14 +248,18 @@ export function feedWideCharCount(feed: Feed): number {
|
|
|
222
248
|
* prompt, and the existing conversation as compact context. Messages of the
|
|
223
249
|
* current turn keep their image references; earlier ones are handles.
|
|
224
250
|
*/
|
|
225
|
-
export function renderInitialFeed(
|
|
251
|
+
export function renderInitialFeed(
|
|
252
|
+
system: string | undefined,
|
|
253
|
+
messages: readonly RequestMessage[],
|
|
254
|
+
imagePath?: ImagePathResolver,
|
|
255
|
+
): Feed {
|
|
226
256
|
const from = currentTurnStart(messages)
|
|
227
257
|
const history: Feed[] = []
|
|
228
258
|
for (const [index, message] of messages.entries()) {
|
|
229
259
|
const entry: Feed = index >= from && imageRefs(message.content).length > 0
|
|
230
260
|
? label(`[${roleTag(message)}] `, blockParts(message.content))
|
|
231
|
-
: renderMessage(message)
|
|
232
|
-
if (feedToText(entry).length > 0) history.push(entry)
|
|
261
|
+
: renderMessage(message, imagePath)
|
|
262
|
+
if (feedToText(entry, imagePath).length > 0) history.push(entry)
|
|
233
263
|
}
|
|
234
264
|
const sections: Feed[] = [BACKEND_ROLE]
|
|
235
265
|
if (system !== undefined && system.length > 0) {
|