@namzu/sdk 31.1.0 → 32.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/dist/agents/ReactiveAgent.d.ts.map +1 -1
- package/dist/agents/ReactiveAgent.js +4 -0
- package/dist/agents/ReactiveAgent.js.map +1 -1
- package/dist/agents/SupervisorAgent.d.ts.map +1 -1
- package/dist/agents/SupervisorAgent.js +4 -0
- package/dist/agents/SupervisorAgent.js.map +1 -1
- package/dist/agents/runAgent.d.ts +4 -0
- package/dist/agents/runAgent.d.ts.map +1 -1
- package/dist/agents/runAgent.js +4 -0
- package/dist/agents/runAgent.js.map +1 -1
- package/dist/directory/derive-supervisor.d.ts.map +1 -1
- package/dist/directory/derive-supervisor.js +5 -0
- package/dist/directory/derive-supervisor.js.map +1 -1
- package/dist/directory/derive.d.ts.map +1 -1
- package/dist/directory/derive.js +5 -0
- package/dist/directory/derive.js.map +1 -1
- package/dist/directory/load.d.ts.map +1 -1
- package/dist/directory/load.js +1 -0
- package/dist/directory/load.js.map +1 -1
- package/dist/directory/types.d.ts +2 -0
- package/dist/directory/types.d.ts.map +1 -1
- package/dist/execution/code-runtime/types.d.ts +11 -1
- package/dist/execution/code-runtime/types.d.ts.map +1 -1
- package/dist/execution/code-runtime/types.js.map +1 -1
- package/dist/execution/code-runtime/worker.d.ts.map +1 -1
- package/dist/execution/code-runtime/worker.js +67 -9
- package/dist/execution/code-runtime/worker.js.map +1 -1
- package/dist/manager/agent/lifecycle.d.ts.map +1 -1
- package/dist/manager/agent/lifecycle.js +1 -0
- package/dist/manager/agent/lifecycle.js.map +1 -1
- package/dist/provider/errors.d.ts +1 -0
- package/dist/provider/errors.d.ts.map +1 -1
- package/dist/provider/errors.js +39 -1
- package/dist/provider/errors.js.map +1 -1
- package/dist/public-runtime.d.ts +5 -3
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +7 -2
- package/dist/public-runtime.js.map +1 -1
- package/dist/public-types.d.ts +2 -1
- package/dist/public-types.d.ts.map +1 -1
- package/dist/run/reporter.d.ts.map +1 -1
- package/dist/run/reporter.js +6 -1
- package/dist/run/reporter.js.map +1 -1
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +214 -29
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/index.d.ts +5 -0
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +1 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts +1 -0
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +34 -14
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/advisory.js +2 -2
- package/dist/runtime/query/iteration/phases/advisory.js.map +1 -1
- package/dist/runtime/query/iteration/provider-rejected-image.d.ts +13 -0
- package/dist/runtime/query/iteration/provider-rejected-image.d.ts.map +1 -0
- package/dist/runtime/query/iteration/provider-rejected-image.js +91 -0
- package/dist/runtime/query/iteration/provider-rejected-image.js.map +1 -0
- package/dist/runtime/query/iteration/stream-turn.d.ts +4 -1
- package/dist/runtime/query/iteration/stream-turn.d.ts.map +1 -1
- package/dist/runtime/query/iteration/stream-turn.js +7 -3
- package/dist/runtime/query/iteration/stream-turn.js.map +1 -1
- package/dist/runtime/query/request-rich-content.d.ts +13 -0
- package/dist/runtime/query/request-rich-content.d.ts.map +1 -1
- package/dist/runtime/query/request-rich-content.js +153 -17
- package/dist/runtime/query/request-rich-content.js.map +1 -1
- package/dist/runtime/query/result.d.ts.map +1 -1
- package/dist/runtime/query/result.js +1 -0
- package/dist/runtime/query/result.js.map +1 -1
- package/dist/store/attachment/index.d.ts +24 -2
- package/dist/store/attachment/index.d.ts.map +1 -1
- package/dist/store/attachment/index.js +52 -5
- package/dist/store/attachment/index.js.map +1 -1
- package/dist/tools/builtins/lsp.d.ts +1 -1
- package/dist/tools/builtins/run-code.d.ts.map +1 -1
- package/dist/tools/builtins/run-code.js +15 -5
- package/dist/tools/builtins/run-code.js.map +1 -1
- package/dist/types/agent/base.d.ts +4 -0
- package/dist/types/agent/base.d.ts.map +1 -1
- package/dist/types/agent/factory.d.ts +1 -0
- package/dist/types/agent/factory.d.ts.map +1 -1
- package/dist/types/message/index.d.ts +31 -1
- package/dist/types/message/index.d.ts.map +1 -1
- package/dist/types/message/index.js +42 -0
- package/dist/types/message/index.js.map +1 -1
- package/dist/types/provider/error.d.ts +4 -1
- package/dist/types/provider/error.d.ts.map +1 -1
- package/dist/types/run/events.d.ts +10 -2
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/types/tool/index.d.ts +52 -6
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/package.json +1 -1
- package/src/agents/ReactiveAgent.ts +4 -0
- package/src/agents/SupervisorAgent.ts +4 -0
- package/src/agents/runAgent.ts +8 -0
- package/src/directory/derive-supervisor.ts +5 -0
- package/src/directory/derive.ts +5 -0
- package/src/directory/load.ts +1 -0
- package/src/directory/types.ts +2 -0
- package/src/execution/code-runtime/types.ts +15 -1
- package/src/execution/code-runtime/worker.ts +72 -9
- package/src/manager/agent/lifecycle.ts +1 -0
- package/src/provider/errors.ts +40 -1
- package/src/public-runtime.ts +16 -1
- package/src/public-types.ts +11 -0
- package/src/run/reporter.ts +7 -1
- package/src/runtime/query/executor.ts +235 -26
- package/src/runtime/query/index.ts +6 -0
- package/src/runtime/query/iteration/index.ts +69 -29
- package/src/runtime/query/iteration/phases/advisory.ts +2 -2
- package/src/runtime/query/iteration/provider-rejected-image.ts +106 -0
- package/src/runtime/query/iteration/stream-turn.ts +12 -2
- package/src/runtime/query/request-rich-content.ts +188 -15
- package/src/runtime/query/result.ts +1 -0
- package/src/store/attachment/index.ts +100 -15
- package/src/tools/builtins/run-code.ts +15 -5
- package/src/types/agent/base.ts +4 -0
- package/src/types/agent/factory.ts +1 -0
- package/src/types/message/index.ts +77 -1
- package/src/types/provider/error.ts +4 -1
- package/src/types/run/events.ts +18 -4
- package/src/types/tool/index.ts +74 -9
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { isProviderRequestError } from '../../../provider/errors.js'
|
|
2
|
+
import type {
|
|
3
|
+
ChatCompletionParams,
|
|
4
|
+
LLMProvider,
|
|
5
|
+
StreamChunk,
|
|
6
|
+
} from '../../../types/provider/index.js'
|
|
7
|
+
import {
|
|
8
|
+
type RequestImageIdentity,
|
|
9
|
+
findSingleRequestImage,
|
|
10
|
+
projectRequestWithoutImage,
|
|
11
|
+
} from '../request-rich-content.js'
|
|
12
|
+
|
|
13
|
+
const IMAGE_REJECTION_PATTERNS: readonly RegExp[] = [
|
|
14
|
+
/\binvalid[_ -]?image\b/i,
|
|
15
|
+
/\bimage\b.{0,100}\b(?:cannot|could not|failed to)\s+(?:be\s+)?(?:decoded|processed|loaded|parsed|read)\b/i,
|
|
16
|
+
/\b(?:cannot|could not|failed to)\s+(?:decode|process|load|parse|read)\b.{0,100}\bimage\b/i,
|
|
17
|
+
/\bimage\b.{0,100}\b(?:malformed|corrupt|unsupported format)\b/i,
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
type ImageRejectionConfidence = 'server-confirmed' | 'heuristic'
|
|
21
|
+
|
|
22
|
+
function classifyProviderRejectedImageError(error: unknown): ImageRejectionConfidence | null {
|
|
23
|
+
if (!isProviderRequestError(error) || error.kind !== 'bad_request') return null
|
|
24
|
+
if (error.status === 400 && error.providerCode === 'invalid_image') return 'server-confirmed'
|
|
25
|
+
const detail = error.detail ?? error.message
|
|
26
|
+
return IMAGE_REJECTION_PATTERNS.some((pattern) => pattern.test(detail)) ? 'heuristic' : null
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function isProviderRejectedImageError(error: unknown): boolean {
|
|
30
|
+
return classifyProviderRejectedImageError(error) !== null
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function isAcceptedChunk(chunk: StreamChunk): boolean {
|
|
34
|
+
if (chunk.retry !== undefined || chunk.fallback !== undefined || chunk.error !== undefined) {
|
|
35
|
+
return false
|
|
36
|
+
}
|
|
37
|
+
return Boolean(
|
|
38
|
+
chunk.delta.content ||
|
|
39
|
+
chunk.delta.toolCalls?.length ||
|
|
40
|
+
chunk.delta.toolCallEnd ||
|
|
41
|
+
chunk.delta.reasoning ||
|
|
42
|
+
chunk.delta.citation ||
|
|
43
|
+
chunk.finishReason !== undefined ||
|
|
44
|
+
chunk.usage !== undefined ||
|
|
45
|
+
chunk.replayState !== undefined,
|
|
46
|
+
)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
50
|
+
if (signal?.aborted) throw signal.reason
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Retry one provider-invalid request without its only distinct image.
|
|
55
|
+
*
|
|
56
|
+
* The callback is the durable commit point for an HTTP 400 carrying the exact
|
|
57
|
+
* provider code `invalid_image`. It runs only after the retry has produced an
|
|
58
|
+
* accepted chunk or reached a clean EOF. A legacy phrase can still recover the
|
|
59
|
+
* current request, but cannot assert durable server proof.
|
|
60
|
+
*/
|
|
61
|
+
export async function* streamWithProviderRejectedImageRecovery(
|
|
62
|
+
provider: LLMProvider,
|
|
63
|
+
params: ChatCompletionParams,
|
|
64
|
+
onAccepted: (identity: RequestImageIdentity) => Promise<void>,
|
|
65
|
+
): AsyncIterable<StreamChunk> {
|
|
66
|
+
const candidate = findSingleRequestImage(params.messages)
|
|
67
|
+
if (candidate === null) {
|
|
68
|
+
yield* provider.chatStream(params)
|
|
69
|
+
return
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let produced = false
|
|
73
|
+
let confidence: ImageRejectionConfidence | null = null
|
|
74
|
+
try {
|
|
75
|
+
for await (const chunk of provider.chatStream(params)) {
|
|
76
|
+
if (isAcceptedChunk(chunk)) produced = true
|
|
77
|
+
yield chunk
|
|
78
|
+
}
|
|
79
|
+
return
|
|
80
|
+
} catch (error) {
|
|
81
|
+
throwIfAborted(params.signal)
|
|
82
|
+
confidence = classifyProviderRejectedImageError(error)
|
|
83
|
+
if (produced || confidence === null) throw error
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
throwIfAborted(params.signal)
|
|
87
|
+
const retryParams: ChatCompletionParams = {
|
|
88
|
+
...params,
|
|
89
|
+
messages: projectRequestWithoutImage(params.messages, candidate),
|
|
90
|
+
}
|
|
91
|
+
let accepted = false
|
|
92
|
+
let retryReportedError = false
|
|
93
|
+
for await (const chunk of provider.chatStream(retryParams)) {
|
|
94
|
+
if (chunk.error !== undefined) retryReportedError = true
|
|
95
|
+
if (!accepted && isAcceptedChunk(chunk)) {
|
|
96
|
+
throwIfAborted(params.signal)
|
|
97
|
+
if (confidence === 'server-confirmed') await onAccepted(candidate)
|
|
98
|
+
accepted = true
|
|
99
|
+
}
|
|
100
|
+
yield chunk
|
|
101
|
+
}
|
|
102
|
+
if (!accepted && !retryReportedError) {
|
|
103
|
+
throwIfAborted(params.signal)
|
|
104
|
+
if (confidence === 'server-confirmed') await onAccepted(candidate)
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -22,6 +22,8 @@ import type { MessageStopReason } from '../../../types/run/stop-reason.js'
|
|
|
22
22
|
import { generateMessageId } from '../../../utils/id.js'
|
|
23
23
|
import type { Logger } from '../../../utils/logger.js'
|
|
24
24
|
import type { EmitEvent } from '../events.js'
|
|
25
|
+
import type { RequestImageIdentity } from '../request-rich-content.js'
|
|
26
|
+
import { streamWithProviderRejectedImageRecovery } from './provider-rejected-image.js'
|
|
25
27
|
|
|
26
28
|
/**
|
|
27
29
|
* Map a provider's coarse `finishReason` plus the orchestrator's
|
|
@@ -147,6 +149,9 @@ export async function* streamProviderTurn(
|
|
|
147
149
|
* Optional, so a caller with no use for the id is unchanged.
|
|
148
150
|
*/
|
|
149
151
|
announceAs?: import('../../../types/ids/index.js').MessageId,
|
|
152
|
+
imageRecovery?: {
|
|
153
|
+
readonly onAccepted: (identity: RequestImageIdentity) => Promise<void>
|
|
154
|
+
},
|
|
150
155
|
): AsyncGenerator<RunEvent, StreamingTurnResult> {
|
|
151
156
|
// The `chat {model}` span the GenAI conventions require. There was none:
|
|
152
157
|
// `chatSpanName` existed with zero call sites, so a trace carried no LLM
|
|
@@ -224,10 +229,15 @@ export async function* streamProviderTurn(
|
|
|
224
229
|
let streamError: string | undefined
|
|
225
230
|
let streamCause: unknown
|
|
226
231
|
|
|
227
|
-
const
|
|
232
|
+
const streamParams = {
|
|
228
233
|
...params,
|
|
229
234
|
stream: true,
|
|
230
|
-
}
|
|
235
|
+
} satisfies import('../../../types/provider/index.js').ChatCompletionParams
|
|
236
|
+
const stream = (
|
|
237
|
+
imageRecovery
|
|
238
|
+
? streamWithProviderRejectedImageRecovery(provider, streamParams, imageRecovery.onAccepted)
|
|
239
|
+
: provider.chatStream(streamParams)
|
|
240
|
+
) as AsyncIterable<StreamChunk>
|
|
231
241
|
|
|
232
242
|
// Drive the stream manually so each `.next()` can be RACED against the run
|
|
233
243
|
// abort: a Stop tears the in-flight model request down (the provider got
|
|
@@ -34,6 +34,12 @@ interface RichOccurrence {
|
|
|
34
34
|
readonly source: 'user' | 'tool'
|
|
35
35
|
readonly kind: RichKind
|
|
36
36
|
readonly bytes: number
|
|
37
|
+
readonly reason: 'budget' | 'provider-rejected'
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface RequestImageIdentity {
|
|
41
|
+
readonly data: string
|
|
42
|
+
readonly mediaType: string
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
const userMarker = (kind: RichKind): string =>
|
|
@@ -42,6 +48,12 @@ const userMarker = (kind: RichKind): string =>
|
|
|
42
48
|
const toolMarker = (kind: RichKind): string =>
|
|
43
49
|
`[${kind} omitted from this model request to keep the accumulated rich-content payload within its configured size limit; call the producing tool again if it is still needed.]`
|
|
44
50
|
|
|
51
|
+
const rejectedUserImageMarker = (): string =>
|
|
52
|
+
'[image omitted from this model request because the provider rejected this image; attach a corrected image in a new message if it is still needed.]'
|
|
53
|
+
|
|
54
|
+
const rejectedToolImageMarker = (): string =>
|
|
55
|
+
'[image omitted from this model request because the provider rejected this image; call the producing tool again after correcting its image source if it is still needed.]'
|
|
56
|
+
|
|
45
57
|
function assertInlineAttachment(
|
|
46
58
|
attachment: MessageAttachment,
|
|
47
59
|
): asserts attachment is Exclude<MessageAttachment, { readonly type: 'stored' }> {
|
|
@@ -65,6 +77,11 @@ function collectRichOccurrences(messages: readonly Message[]): RichOccurrence[]
|
|
|
65
77
|
source: 'user',
|
|
66
78
|
kind: attachment.type === 'document' ? 'document' : 'image',
|
|
67
79
|
bytes: attachment.data.length,
|
|
80
|
+
reason:
|
|
81
|
+
attachment.type !== 'document' &&
|
|
82
|
+
attachment.modelOmission?.reason === 'provider-rejected'
|
|
83
|
+
? 'provider-rejected'
|
|
84
|
+
: 'budget',
|
|
68
85
|
})
|
|
69
86
|
}
|
|
70
87
|
continue
|
|
@@ -79,6 +96,10 @@ function collectRichOccurrences(messages: readonly Message[]): RichOccurrence[]
|
|
|
79
96
|
source: 'tool',
|
|
80
97
|
kind: block.type,
|
|
81
98
|
bytes: block.data.length,
|
|
99
|
+
reason:
|
|
100
|
+
block.type === 'image' && block.modelOmission?.reason === 'provider-rejected'
|
|
101
|
+
? 'provider-rejected'
|
|
102
|
+
: 'budget',
|
|
82
103
|
})
|
|
83
104
|
}
|
|
84
105
|
}
|
|
@@ -86,10 +107,19 @@ function collectRichOccurrences(messages: readonly Message[]): RichOccurrence[]
|
|
|
86
107
|
return occurrences
|
|
87
108
|
}
|
|
88
109
|
|
|
89
|
-
function
|
|
90
|
-
if (
|
|
91
|
-
const
|
|
92
|
-
return content.length > 0 ? `${content}\n\n${
|
|
110
|
+
function appendTextMarkers(content: string, markers: readonly string[]): string {
|
|
111
|
+
if (markers.length === 0) return content
|
|
112
|
+
const joined = markers.join('\n\n')
|
|
113
|
+
return content.length > 0 ? `${content}\n\n${joined}` : joined
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function appendMarkers(content: string, occurrences: readonly RichOccurrence[]): string {
|
|
117
|
+
const markers = occurrences.map((occurrence) =>
|
|
118
|
+
occurrence.reason === 'provider-rejected'
|
|
119
|
+
? rejectedUserImageMarker()
|
|
120
|
+
: userMarker(occurrence.kind),
|
|
121
|
+
)
|
|
122
|
+
return appendTextMarkers(content, markers)
|
|
93
123
|
}
|
|
94
124
|
|
|
95
125
|
/**
|
|
@@ -103,15 +133,10 @@ function appendMarkers(content: string, kinds: readonly RichKind[]): string {
|
|
|
103
133
|
*/
|
|
104
134
|
export function projectRequestRichContent(messages: Message[], maxBytes: number): Message[] {
|
|
105
135
|
const occurrences = collectRichOccurrences(messages)
|
|
106
|
-
if (
|
|
107
|
-
|
|
108
|
-
let total = occurrences.reduce((sum, occurrence) => sum + occurrence.bytes, 0)
|
|
109
|
-
if (total <= maxBytes) return messages
|
|
136
|
+
if (occurrences.length === 0) return messages
|
|
110
137
|
|
|
111
138
|
const omitted = new Map<number, Map<number, RichOccurrence>>()
|
|
112
|
-
|
|
113
|
-
if (total <= maxBytes) break
|
|
114
|
-
total -= occurrence.bytes
|
|
139
|
+
const omit = (occurrence: RichOccurrence): void => {
|
|
115
140
|
let byItem = omitted.get(occurrence.messageIndex)
|
|
116
141
|
if (!byItem) {
|
|
117
142
|
byItem = new Map()
|
|
@@ -120,6 +145,24 @@ export function projectRequestRichContent(messages: Message[], maxBytes: number)
|
|
|
120
145
|
byItem.set(occurrence.itemIndex, occurrence)
|
|
121
146
|
}
|
|
122
147
|
|
|
148
|
+
for (const occurrence of occurrences) {
|
|
149
|
+
if (occurrence.reason === 'provider-rejected') omit(occurrence)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
let total = occurrences.reduce(
|
|
153
|
+
(sum, occurrence) => sum + (occurrence.reason === 'budget' ? occurrence.bytes : 0),
|
|
154
|
+
0,
|
|
155
|
+
)
|
|
156
|
+
if (maxBytes > 0 && total > maxBytes) {
|
|
157
|
+
for (const occurrence of occurrences) {
|
|
158
|
+
if (total <= maxBytes) break
|
|
159
|
+
if (occurrence.reason !== 'budget') continue
|
|
160
|
+
total -= occurrence.bytes
|
|
161
|
+
omit(occurrence)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (omitted.size === 0) return messages
|
|
165
|
+
|
|
123
166
|
return messages.map((message, messageIndex) => {
|
|
124
167
|
const byItem = omitted.get(messageIndex)
|
|
125
168
|
if (!byItem) return message
|
|
@@ -127,13 +170,12 @@ export function projectRequestRichContent(messages: Message[], maxBytes: number)
|
|
|
127
170
|
if (message.role === 'user') {
|
|
128
171
|
const { attachments = [], ...withoutAttachments } = message
|
|
129
172
|
const kept = attachments.filter((_attachment, itemIndex) => !byItem.has(itemIndex))
|
|
130
|
-
const
|
|
173
|
+
const omittedOccurrences = [...byItem.values()]
|
|
131
174
|
.filter((occurrence) => occurrence.source === 'user')
|
|
132
175
|
.sort((a, b) => a.itemIndex - b.itemIndex)
|
|
133
|
-
.map((occurrence) => occurrence.kind)
|
|
134
176
|
return {
|
|
135
177
|
...withoutAttachments,
|
|
136
|
-
content: appendMarkers(message.content,
|
|
178
|
+
content: appendMarkers(message.content, omittedOccurrences),
|
|
137
179
|
...(kept.length > 0 ? { attachments: kept } : {}),
|
|
138
180
|
}
|
|
139
181
|
}
|
|
@@ -142,7 +184,13 @@ export function projectRequestRichContent(messages: Message[], maxBytes: number)
|
|
|
142
184
|
const content: ToolResultBlock[] = message.content.map((block, itemIndex) => {
|
|
143
185
|
const occurrence = byItem.get(itemIndex)
|
|
144
186
|
return occurrence?.source === 'tool'
|
|
145
|
-
? {
|
|
187
|
+
? {
|
|
188
|
+
type: 'text',
|
|
189
|
+
text:
|
|
190
|
+
occurrence.reason === 'provider-rejected'
|
|
191
|
+
? rejectedToolImageMarker()
|
|
192
|
+
: toolMarker(occurrence.kind),
|
|
193
|
+
}
|
|
146
194
|
: block
|
|
147
195
|
})
|
|
148
196
|
return { ...message, content }
|
|
@@ -151,3 +199,128 @@ export function projectRequestRichContent(messages: Message[], maxBytes: number)
|
|
|
151
199
|
return message
|
|
152
200
|
})
|
|
153
201
|
}
|
|
202
|
+
|
|
203
|
+
function sameImage(
|
|
204
|
+
image: { readonly data: string; readonly mediaType: string },
|
|
205
|
+
identity: RequestImageIdentity,
|
|
206
|
+
): boolean {
|
|
207
|
+
return image.data === identity.data && image.mediaType === identity.mediaType
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Find the one distinct image in the exact provider-bound request, if one exists. */
|
|
211
|
+
export function findSingleRequestImage(messages: readonly Message[]): RequestImageIdentity | null {
|
|
212
|
+
let candidate: RequestImageIdentity | null = null
|
|
213
|
+
for (const message of messages) {
|
|
214
|
+
if (message.role === 'user') {
|
|
215
|
+
for (const attachment of message.attachments ?? []) {
|
|
216
|
+
if (attachment.type === 'stored') {
|
|
217
|
+
throw new TypeError(
|
|
218
|
+
'Cannot inspect an unresolved stored attachment at the provider boundary',
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
if (attachment.type === 'document') continue
|
|
222
|
+
if (candidate === null) {
|
|
223
|
+
candidate = { data: attachment.data, mediaType: attachment.mediaType }
|
|
224
|
+
} else if (!sameImage(attachment, candidate)) {
|
|
225
|
+
return null
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
continue
|
|
229
|
+
}
|
|
230
|
+
if (message.role !== 'tool' || !Array.isArray(message.content)) continue
|
|
231
|
+
for (const block of message.content) {
|
|
232
|
+
if (block.type !== 'image') continue
|
|
233
|
+
if (candidate === null) {
|
|
234
|
+
candidate = { data: block.data, mediaType: block.mediaType }
|
|
235
|
+
} else if (!sameImage(block, candidate)) {
|
|
236
|
+
return null
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return candidate
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Build the one-off retry request without changing durable conversation state. */
|
|
244
|
+
export function projectRequestWithoutImage(
|
|
245
|
+
messages: Message[],
|
|
246
|
+
identity: RequestImageIdentity,
|
|
247
|
+
): Message[] {
|
|
248
|
+
return messages.map((message) => {
|
|
249
|
+
if (message.role === 'user') {
|
|
250
|
+
const attachments = message.attachments ?? []
|
|
251
|
+
const rejected = attachments.filter(
|
|
252
|
+
(attachment) =>
|
|
253
|
+
attachment.type !== 'stored' &&
|
|
254
|
+
attachment.type !== 'document' &&
|
|
255
|
+
sameImage(attachment, identity),
|
|
256
|
+
)
|
|
257
|
+
if (rejected.length === 0) return message
|
|
258
|
+
const kept = attachments.filter(
|
|
259
|
+
(attachment) =>
|
|
260
|
+
attachment.type === 'stored' ||
|
|
261
|
+
attachment.type === 'document' ||
|
|
262
|
+
!sameImage(attachment, identity),
|
|
263
|
+
)
|
|
264
|
+
const { attachments: _attachments, ...withoutAttachments } = message
|
|
265
|
+
return {
|
|
266
|
+
...withoutAttachments,
|
|
267
|
+
content: appendTextMarkers(
|
|
268
|
+
message.content,
|
|
269
|
+
rejected.map(() => rejectedUserImageMarker()),
|
|
270
|
+
),
|
|
271
|
+
...(kept.length > 0 ? { attachments: kept } : {}),
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (message.role !== 'tool' || !Array.isArray(message.content)) return message
|
|
275
|
+
let changed = false
|
|
276
|
+
const content: ToolResultBlock[] = message.content.map((block) => {
|
|
277
|
+
if (block.type !== 'image' || !sameImage(block, identity)) return block
|
|
278
|
+
changed = true
|
|
279
|
+
return { type: 'text', text: rejectedToolImageMarker() }
|
|
280
|
+
})
|
|
281
|
+
return changed ? { ...message, content } : message
|
|
282
|
+
})
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Persist a successful recovery while retaining exact image bytes. */
|
|
286
|
+
export function markProviderRejectedImage(
|
|
287
|
+
messages: readonly Message[],
|
|
288
|
+
identity: RequestImageIdentity,
|
|
289
|
+
): { readonly messages: Message[]; readonly count: number } {
|
|
290
|
+
let count = 0
|
|
291
|
+
const marked = messages.map((message) => {
|
|
292
|
+
if (message.role === 'user') {
|
|
293
|
+
let changed = false
|
|
294
|
+
const attachments = message.attachments?.map((attachment) => {
|
|
295
|
+
if (
|
|
296
|
+
attachment.type === 'stored' ||
|
|
297
|
+
attachment.type === 'document' ||
|
|
298
|
+
!sameImage(attachment, identity) ||
|
|
299
|
+
attachment.modelOmission?.reason === 'provider-rejected'
|
|
300
|
+
) {
|
|
301
|
+
return attachment
|
|
302
|
+
}
|
|
303
|
+
changed = true
|
|
304
|
+
count += 1
|
|
305
|
+
return { ...attachment, modelOmission: { reason: 'provider-rejected' as const } }
|
|
306
|
+
})
|
|
307
|
+
return changed ? { ...message, attachments } : message
|
|
308
|
+
}
|
|
309
|
+
if (message.role !== 'tool' || !Array.isArray(message.content)) return message
|
|
310
|
+
let changed = false
|
|
311
|
+
const content: ToolResultBlock[] = message.content.map((block) => {
|
|
312
|
+
if (
|
|
313
|
+
block.type !== 'image' ||
|
|
314
|
+
!sameImage(block, identity) ||
|
|
315
|
+
block.modelOmission?.reason === 'provider-rejected'
|
|
316
|
+
) {
|
|
317
|
+
return block
|
|
318
|
+
}
|
|
319
|
+
changed = true
|
|
320
|
+
count += 1
|
|
321
|
+
return { ...block, modelOmission: { reason: 'provider-rejected' as const } }
|
|
322
|
+
})
|
|
323
|
+
return changed ? { ...message, content } : message
|
|
324
|
+
})
|
|
325
|
+
return { messages: marked, count }
|
|
326
|
+
}
|
|
@@ -183,6 +183,7 @@ export class ResultAssembler {
|
|
|
183
183
|
? {
|
|
184
184
|
kind: err.kind,
|
|
185
185
|
providerId: err.providerId,
|
|
186
|
+
...(err.providerCode !== undefined ? { providerCode: err.providerCode } : {}),
|
|
186
187
|
...(err.status !== undefined ? { status: err.status } : {}),
|
|
187
188
|
...(err.retryAfterMs !== undefined ? { retryAfterMs: err.retryAfterMs } : {}),
|
|
188
189
|
// The provider's own sentence, already truncated and scrubbed
|
|
@@ -60,6 +60,23 @@ export interface AttachmentOperationOptions {
|
|
|
60
60
|
readonly signal?: AbortSignal
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
/** Policy owned by the operation that materializes one or more references. */
|
|
64
|
+
export interface AttachmentResolutionOptions extends AttachmentOperationOptions {
|
|
65
|
+
/**
|
|
66
|
+
* Maximum wall-clock time for the complete materialization phase.
|
|
67
|
+
*
|
|
68
|
+
* Defaults to one minute. `0` retains the prior unbounded wait. The SDK
|
|
69
|
+
* still races the wait itself, so a custom store cannot defeat this bound
|
|
70
|
+
* by ignoring the signal it receives.
|
|
71
|
+
*/
|
|
72
|
+
readonly timeoutMs?: number
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** One minute is long enough for remote stores without letting a run wedge forever. */
|
|
76
|
+
export const DEFAULT_ATTACHMENT_RESOLVE_TIMEOUT_MS = 60_000
|
|
77
|
+
|
|
78
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647
|
|
79
|
+
|
|
63
80
|
export interface AttachmentStore {
|
|
64
81
|
/**
|
|
65
82
|
* Take bytes, return a ref.
|
|
@@ -109,6 +126,17 @@ export class AttachmentMediaTypeMismatchError extends Error {
|
|
|
109
126
|
}
|
|
110
127
|
}
|
|
111
128
|
|
|
129
|
+
/** A store did not settle the materialization phase within its declared bound. */
|
|
130
|
+
export class AttachmentResolutionTimeoutError extends Error {
|
|
131
|
+
readonly details: { timeoutMs: number }
|
|
132
|
+
|
|
133
|
+
constructor(details: { timeoutMs: number }) {
|
|
134
|
+
super(`Stored attachment resolution timed out after ${details.timeoutMs}ms.`)
|
|
135
|
+
this.name = 'AttachmentResolutionTimeoutError'
|
|
136
|
+
this.details = details
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
112
140
|
export const isStoredAttachment = (
|
|
113
141
|
attachment: MessageAttachment,
|
|
114
142
|
): attachment is MessageAttachment & StoredAttachment =>
|
|
@@ -126,11 +154,23 @@ export const isStoredAttachment = (
|
|
|
126
154
|
export async function resolveAttachment(
|
|
127
155
|
attachment: MessageAttachment,
|
|
128
156
|
store: AttachmentStore | undefined,
|
|
129
|
-
options:
|
|
157
|
+
options: AttachmentResolutionOptions = {},
|
|
130
158
|
): Promise<MessageAttachment> {
|
|
131
|
-
|
|
132
|
-
|
|
159
|
+
options.signal?.throwIfAborted()
|
|
160
|
+
const timeoutMs = resolveAttachmentTimeoutMs(options.timeoutMs)
|
|
133
161
|
if (!isStoredAttachment(attachment)) return attachment
|
|
162
|
+
|
|
163
|
+
return await withAttachmentDeadline(options.signal, timeoutMs, async (signal) =>
|
|
164
|
+
resolveAttachmentWithSignal(attachment, store, signal),
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async function resolveAttachmentWithSignal(
|
|
169
|
+
attachment: MessageAttachment & StoredAttachment,
|
|
170
|
+
store: AttachmentStore | undefined,
|
|
171
|
+
signal: AbortSignal | undefined,
|
|
172
|
+
): Promise<MessageAttachment> {
|
|
173
|
+
signal?.throwIfAborted()
|
|
134
174
|
if (!store) throw new NoAttachmentStoreError({ ref: attachment.ref })
|
|
135
175
|
|
|
136
176
|
const bytes = await awaitStoreRead(signal, () =>
|
|
@@ -177,9 +217,10 @@ export async function resolveAttachment(
|
|
|
177
217
|
export async function resolveAttachments(
|
|
178
218
|
messages: readonly Message[],
|
|
179
219
|
store: AttachmentStore | undefined,
|
|
180
|
-
options:
|
|
220
|
+
options: AttachmentResolutionOptions = {},
|
|
181
221
|
): Promise<readonly Message[]> {
|
|
182
222
|
options.signal?.throwIfAborted()
|
|
223
|
+
const timeoutMs = resolveAttachmentTimeoutMs(options.timeoutMs)
|
|
183
224
|
// `Message` rather than a structural constraint: only some members of
|
|
184
225
|
// the union carry `attachments`, and a structural bound over a union
|
|
185
226
|
// like that is not assignable in either direction. Naming the real type
|
|
@@ -189,20 +230,64 @@ export async function resolveAttachments(
|
|
|
189
230
|
|
|
190
231
|
if (!messages.some((m) => has(m)?.some(isStoredAttachment))) return messages
|
|
191
232
|
|
|
192
|
-
return await
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
attachments
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
233
|
+
return await withAttachmentDeadline(options.signal, timeoutMs, async (signal) =>
|
|
234
|
+
Promise.all(
|
|
235
|
+
messages.map(async (message) => {
|
|
236
|
+
const attachments = has(message)
|
|
237
|
+
if (!attachments?.some(isStoredAttachment)) return message
|
|
238
|
+
return {
|
|
239
|
+
...message,
|
|
240
|
+
attachments: await Promise.all(
|
|
241
|
+
attachments.map((attachment) =>
|
|
242
|
+
isStoredAttachment(attachment)
|
|
243
|
+
? resolveAttachmentWithSignal(attachment, store, signal)
|
|
244
|
+
: attachment,
|
|
245
|
+
),
|
|
246
|
+
),
|
|
247
|
+
}
|
|
248
|
+
}),
|
|
249
|
+
),
|
|
203
250
|
)
|
|
204
251
|
}
|
|
205
252
|
|
|
253
|
+
function resolveAttachmentTimeoutMs(value: number | undefined): number {
|
|
254
|
+
const resolved = value ?? DEFAULT_ATTACHMENT_RESOLVE_TIMEOUT_MS
|
|
255
|
+
if (!Number.isInteger(resolved) || resolved < 0 || resolved > MAX_TIMER_DELAY_MS) {
|
|
256
|
+
throw new RangeError(
|
|
257
|
+
`attachmentResolveTimeoutMs must be an integer from 0 to ${MAX_TIMER_DELAY_MS}; received ${String(resolved)}`,
|
|
258
|
+
)
|
|
259
|
+
}
|
|
260
|
+
return resolved
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Own one timer for the whole parallel materialization phase.
|
|
265
|
+
*
|
|
266
|
+
* The caller's signal is an input, never the controller we abort. Whichever
|
|
267
|
+
* cause wins is latched by `AbortSignal.any`, and every store wait races that
|
|
268
|
+
* fused signal independently through `awaitStoreRead` below.
|
|
269
|
+
*/
|
|
270
|
+
async function withAttachmentDeadline<T>(
|
|
271
|
+
upstream: AbortSignal | undefined,
|
|
272
|
+
timeoutMs: number,
|
|
273
|
+
start: (signal: AbortSignal | undefined) => Promise<T>,
|
|
274
|
+
): Promise<T> {
|
|
275
|
+
upstream?.throwIfAborted()
|
|
276
|
+
if (timeoutMs === 0) return await start(upstream)
|
|
277
|
+
|
|
278
|
+
const timeout = new AbortController()
|
|
279
|
+
const signal = upstream ? AbortSignal.any([upstream, timeout.signal]) : timeout.signal
|
|
280
|
+
const timer = setTimeout(
|
|
281
|
+
() => timeout.abort(new AttachmentResolutionTimeoutError({ timeoutMs })),
|
|
282
|
+
timeoutMs,
|
|
283
|
+
)
|
|
284
|
+
try {
|
|
285
|
+
return await start(signal)
|
|
286
|
+
} finally {
|
|
287
|
+
clearTimeout(timer)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
206
291
|
/**
|
|
207
292
|
* Await a host store without handing it ownership of run cancellation.
|
|
208
293
|
*
|
|
@@ -99,19 +99,25 @@ export function buildRunCodeTool(options: RunCodeToolOptions = {}) {
|
|
|
99
99
|
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
100
100
|
maxOutputBytes: options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT,
|
|
101
101
|
...(context.abortSignal ? { signal: context.abortSignal } : {}),
|
|
102
|
-
onHostCall: async (request) => {
|
|
102
|
+
onHostCall: async (request, operation) => {
|
|
103
103
|
// Through the SAME registry, with the SAME context. Not a
|
|
104
104
|
// parallel dispatch path: a second path is a second place
|
|
105
105
|
// for the permission gate to be forgotten, and the one that
|
|
106
106
|
// forgot it would be the one a model reached through a
|
|
107
107
|
// program.
|
|
108
108
|
try {
|
|
109
|
-
const toolResult = await dispatch(request.name, request.input
|
|
109
|
+
const toolResult = await dispatch(request.name, request.input, {
|
|
110
|
+
signal: operation.signal,
|
|
111
|
+
runtimeToolCallId: operation.runtimeToolCallId,
|
|
112
|
+
})
|
|
110
113
|
return toolResult.success
|
|
111
114
|
? { ok: true, value: toolResult.output }
|
|
112
115
|
: { ok: false, error: toolResult.error ?? 'the tool failed' }
|
|
113
116
|
} catch (err) {
|
|
114
|
-
return {
|
|
117
|
+
return {
|
|
118
|
+
ok: false,
|
|
119
|
+
error: err instanceof Error ? err.message : String(err),
|
|
120
|
+
}
|
|
115
121
|
}
|
|
116
122
|
},
|
|
117
123
|
})
|
|
@@ -142,10 +148,14 @@ export function buildRunCodeTool(options: RunCodeToolOptions = {}) {
|
|
|
142
148
|
// printed its progress and then hung has told the model
|
|
143
149
|
// where it got to, and discarding that leaves the model
|
|
144
150
|
// retrying from the start.
|
|
145
|
-
error: `The program ran longer than ${options.timeoutMs ?? DEFAULT_TIMEOUT_MS}ms and was stopped.`,
|
|
151
|
+
error: `The program ran longer than ${options.timeoutMs ?? DEFAULT_TIMEOUT_MS}ms and its worker was stopped. Admitted host calls were asked to cancel; an uncooperative one may still be running.`,
|
|
146
152
|
}
|
|
147
153
|
case 'cancelled':
|
|
148
|
-
return {
|
|
154
|
+
return {
|
|
155
|
+
success: false,
|
|
156
|
+
output: body,
|
|
157
|
+
error: 'The program was cancelled.',
|
|
158
|
+
}
|
|
149
159
|
default:
|
|
150
160
|
return { success: false, output: body, error: result.outcome.error }
|
|
151
161
|
}
|
package/src/types/agent/base.ts
CHANGED
|
@@ -23,6 +23,8 @@ export interface BaseAgentConfig {
|
|
|
23
23
|
streamIdleTimeoutMs?: number
|
|
24
24
|
/** See {@link import('../run/config.js').AgentRunConfig.maxRequestRichContentBytes}. */
|
|
25
25
|
maxRequestRichContentBytes?: number
|
|
26
|
+
/** Maximum stored-attachment materialization time; defaults to one minute. `0` disables. */
|
|
27
|
+
attachmentResolveTimeoutMs?: number
|
|
26
28
|
maxIterations?: number
|
|
27
29
|
temperature?: number
|
|
28
30
|
maxResponseTokens?: number
|
|
@@ -239,6 +241,8 @@ export interface AgentInput {
|
|
|
239
241
|
messages: Message[]
|
|
240
242
|
workingDirectory: string
|
|
241
243
|
signal?: AbortSignal
|
|
244
|
+
/** Store that owns any `stored` attachment refs carried by `messages`. */
|
|
245
|
+
attachmentStore?: import('../../store/attachment/index.js').AttachmentStore
|
|
242
246
|
|
|
243
247
|
taskStore?: TaskStore
|
|
244
248
|
|