dsh-lcx-codex 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/LICENSE +21 -0
- package/README.md +120 -0
- package/cordis.patch.yml +18 -0
- package/lib/checkpoint-store-v3.js +459 -0
- package/lib/client.js +248 -0
- package/lib/compact-v2.js +305 -0
- package/lib/compact.js +653 -0
- package/lib/dsh-pi-responses.js +339 -0
- package/lib/index.js +1757 -0
- package/lib/private-file.js +18 -0
- package/lib/session-lease.js +70 -0
- package/lib/transport.js +396 -0
- package/lib/web-run-output.js +157 -0
- package/lib/web-search-alpha.js +451 -0
- package/lib/web-search-capability.js +78 -0
- package/lib/web-search-hosted.js +383 -0
- package/lib/web-search-ref-store.js +70 -0
- package/lib/web-search-store.js +181 -0
- package/package.json +99 -0
- package/scripts/probe-alpha.mjs +91 -0
package/lib/compact.js
ADDED
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { offloadDshRequestImages, resolveDshImage, serializeDshResponsesInput } from './dsh-pi-responses.js'
|
|
3
|
+
|
|
4
|
+
const UNSUPPORTED_CHECKPOINT_PATTERN = /\[dsh-lcx-codex-checkpoint:[0-9a-f-]{36}\]/iu
|
|
5
|
+
const PORTABLE_CHECKPOINT_PATTERN = /\[dsh-lcx-codex-v3-checkpoint:([0-9a-f-]{36})\]/giu
|
|
6
|
+
const PORTABLE_HISTORY_TOKEN_BUDGET = 20_000
|
|
7
|
+
export const PORTABLE_HISTORY_BYTE_BUDGET = 2 * 1024 * 1024
|
|
8
|
+
const UNSUPPORTED_IMAGE_PLACEHOLDER = '[image omitted because the target model does not support image input]'
|
|
9
|
+
|
|
10
|
+
export function portableCheckpointIds(text) {
|
|
11
|
+
return [...String(text ?? '').matchAll(PORTABLE_CHECKPOINT_PATTERN)].map((match) => match[1].toLowerCase())
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function textOfContent(content) {
|
|
15
|
+
return (Array.isArray(content) ? content : [])
|
|
16
|
+
.filter((block) => block?.type === 'text' && typeof block.text === 'string')
|
|
17
|
+
.map((block) => block.text)
|
|
18
|
+
.join('')
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function textOfMessage(message) {
|
|
22
|
+
return textOfContent(message?.content)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function toolResultText(block) {
|
|
26
|
+
return textOfContent(block?.content) || '(no output)'
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function unsupportedPortableContentError(type) {
|
|
30
|
+
const error = new Error(`LCX Compact cannot safely portable-replay message content type: ${String(type)}`)
|
|
31
|
+
error.code = 'LCX_CHECKPOINT_PORTABLE_UNSUPPORTED_CONTENT'
|
|
32
|
+
return error
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function compactImageError(message, code, cause) {
|
|
36
|
+
const error = new Error(message, cause === undefined ? undefined : { cause })
|
|
37
|
+
error.code = code
|
|
38
|
+
return error
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function containsImage(content) {
|
|
42
|
+
return (Array.isArray(content) ? content : []).some((block) =>
|
|
43
|
+
block?.type === 'image' || (block?.type === 'tool-result' && containsImage(block.content)))
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function inputImageCount(input) {
|
|
47
|
+
return (input ?? []).reduce((count, item) => {
|
|
48
|
+
const content = item?.type === 'message' ? item.content : item?.type === 'function_call_output' ? item.output : undefined
|
|
49
|
+
return count + (Array.isArray(content)
|
|
50
|
+
? content.filter((part) => part?.type === 'input_image' || part?.type === 'dsh_image_attachment').length
|
|
51
|
+
: 0)
|
|
52
|
+
}, 0)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function inputImagePart(block, options = {}) {
|
|
56
|
+
const image = await resolveDshImage(block, options)
|
|
57
|
+
return { type: 'input_image', image_url: `data:${image.mimeType};base64,${image.data}` }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function assertSupportedBlocks(blocks, { allowImage }) {
|
|
61
|
+
const supported = new Set(['text', 'tool-call', 'tool-result', 'reasoning', ...(allowImage ? ['image'] : [])])
|
|
62
|
+
for (const block of Array.isArray(blocks) ? blocks : []) {
|
|
63
|
+
if (!supported.has(block?.type)) throw unsupportedPortableContentError(block?.type)
|
|
64
|
+
if (block.type === 'tool-result') assertSupportedBlocks(block.content, { allowImage })
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function toolResultOutputWithImages(block, options) {
|
|
69
|
+
const parts = []
|
|
70
|
+
let text = ''
|
|
71
|
+
const flushText = () => {
|
|
72
|
+
if (text) parts.push({ type: 'input_text', text })
|
|
73
|
+
text = ''
|
|
74
|
+
}
|
|
75
|
+
for (const part of block?.content ?? []) {
|
|
76
|
+
if (part?.type === 'text') text += part.text
|
|
77
|
+
else if (part?.type === 'image') {
|
|
78
|
+
flushText()
|
|
79
|
+
parts.push(await inputImagePart(part, options))
|
|
80
|
+
} else if (part?.type === 'tool-result') {
|
|
81
|
+
const nested = await toolResultOutputWithImages(part, options)
|
|
82
|
+
if (typeof nested === 'string') text += nested === '(no output)' ? '' : nested
|
|
83
|
+
else {
|
|
84
|
+
flushText()
|
|
85
|
+
parts.push(...nested)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
flushText()
|
|
90
|
+
return parts.some((part) => part.type === 'input_image') ? parts : parts.map((part) => part.text).join('') || '(no output)'
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function messageInputItemsWithImages(message, options = {}) {
|
|
94
|
+
const blocks = Array.isArray(message?.content) ? message.content : []
|
|
95
|
+
if (!containsImage(blocks)) return messageInputItems(message, options)
|
|
96
|
+
if (message?.role !== 'user') {
|
|
97
|
+
throw compactImageError('LCX Compact only supports image attachments in user messages or tool results', 'LCX_COMPACT_IMAGE_UNSUPPORTED')
|
|
98
|
+
}
|
|
99
|
+
if (options.strict) assertSupportedBlocks(blocks, { allowImage: true })
|
|
100
|
+
const content = []
|
|
101
|
+
let pendingText = ''
|
|
102
|
+
const flushText = () => {
|
|
103
|
+
if (pendingText) content.push({ type: 'input_text', text: pendingText })
|
|
104
|
+
pendingText = ''
|
|
105
|
+
}
|
|
106
|
+
for (const block of blocks) {
|
|
107
|
+
if (block?.type === 'text') pendingText += block.text
|
|
108
|
+
else if (block?.type === 'image') {
|
|
109
|
+
flushText()
|
|
110
|
+
content.push(await inputImagePart(block, options))
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
flushText()
|
|
114
|
+
const items = []
|
|
115
|
+
if (content.length > 0) items.push({ type: 'message', role: 'user', content })
|
|
116
|
+
for (const call of blocks.filter((block) => block?.type === 'tool-call')) {
|
|
117
|
+
items.push({
|
|
118
|
+
type: 'function_call',
|
|
119
|
+
call_id: String(call.id),
|
|
120
|
+
name: String(call.name),
|
|
121
|
+
arguments: typeof call.arguments === 'string' ? call.arguments : JSON.stringify(call.arguments ?? {}),
|
|
122
|
+
})
|
|
123
|
+
}
|
|
124
|
+
for (const result of blocks.filter((block) => block?.type === 'tool-result')) {
|
|
125
|
+
items.push({
|
|
126
|
+
type: 'function_call_output',
|
|
127
|
+
call_id: String(result.toolCallId),
|
|
128
|
+
output: containsImage(result.content) ? await toolResultOutputWithImages(result, options) : toolResultText(result),
|
|
129
|
+
})
|
|
130
|
+
}
|
|
131
|
+
return items
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export async function normalInputWithImages(messages, options = {}) {
|
|
135
|
+
const input = []
|
|
136
|
+
for (const message of messages ?? []) {
|
|
137
|
+
if (message?.role === 'system') continue
|
|
138
|
+
input.push(...await messageInputItemsWithImages(message, options))
|
|
139
|
+
}
|
|
140
|
+
return input
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function mapImageContent(value, mapper) {
|
|
144
|
+
if (!Array.isArray(value)) return value
|
|
145
|
+
return value.map((part) => {
|
|
146
|
+
if (part?.type === 'input_image' || part?.type === 'dsh_image_attachment') return mapper(part)
|
|
147
|
+
return part?.type === 'function_call_output' && Array.isArray(part.output)
|
|
148
|
+
? { ...part, output: mapImageContent(part.output, mapper) }
|
|
149
|
+
: structuredClone(part)
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function persistNativeImageReferences(output, references) {
|
|
154
|
+
const referenceMap = references instanceof Map ? references : new Map()
|
|
155
|
+
const persistPart = (part) => {
|
|
156
|
+
if (part.type !== 'input_image' || typeof part.image_url !== 'string') {
|
|
157
|
+
throw compactImageError('LCX Compact response contains an invalid image item', 'LCX_COMPACT_INVALID_RESPONSE')
|
|
158
|
+
}
|
|
159
|
+
const attachment = referenceMap.get(part.image_url)
|
|
160
|
+
if (!attachment) {
|
|
161
|
+
throw compactImageError('LCX Compact response contains an untracked image item', 'LCX_COMPACT_INVALID_RESPONSE')
|
|
162
|
+
}
|
|
163
|
+
return { type: 'dsh_image_attachment', attachment: structuredClone(attachment) }
|
|
164
|
+
}
|
|
165
|
+
return (output ?? []).map((item) => {
|
|
166
|
+
if (item?.type === 'message' && Array.isArray(item.content)) {
|
|
167
|
+
return { ...structuredClone(item), content: mapImageContent(item.content, persistPart) }
|
|
168
|
+
}
|
|
169
|
+
if (item?.type === 'function_call_output' && Array.isArray(item.output)) {
|
|
170
|
+
return { ...structuredClone(item), output: mapImageContent(item.output, persistPart) }
|
|
171
|
+
}
|
|
172
|
+
return structuredClone(item)
|
|
173
|
+
})
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function hydrateImageContent(content, options) {
|
|
177
|
+
const hydrated = []
|
|
178
|
+
for (const part of content ?? []) {
|
|
179
|
+
if (part?.type === 'dsh_image_attachment') {
|
|
180
|
+
hydrated.push(await inputImagePart({ type: 'image', attachment: part.attachment }, options))
|
|
181
|
+
} else {
|
|
182
|
+
hydrated.push(structuredClone(part))
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return hydrated
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export async function hydrateNativeImageReferences(output, options = {}) {
|
|
189
|
+
const hydrated = []
|
|
190
|
+
for (const item of output ?? []) {
|
|
191
|
+
if (item?.type === 'message' && Array.isArray(item.content)) {
|
|
192
|
+
hydrated.push({ ...structuredClone(item), content: await hydrateImageContent(item.content, options) })
|
|
193
|
+
} else if (item?.type === 'function_call_output' && Array.isArray(item.output)) {
|
|
194
|
+
hydrated.push({ ...structuredClone(item), output: await hydrateImageContent(item.output, options) })
|
|
195
|
+
} else {
|
|
196
|
+
hydrated.push(structuredClone(item))
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return hydrated
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function messageInputItems(message, { strict = false } = {}) {
|
|
203
|
+
const blocks = Array.isArray(message?.content) ? message.content : []
|
|
204
|
+
if (strict) assertSupportedBlocks(blocks, { allowImage: false })
|
|
205
|
+
const text = textOfContent(blocks)
|
|
206
|
+
const calls = blocks.filter((block) => block?.type === 'tool-call')
|
|
207
|
+
const results = blocks.filter((block) => block?.type === 'tool-result')
|
|
208
|
+
const items = []
|
|
209
|
+
|
|
210
|
+
if (message?.role === 'assistant') {
|
|
211
|
+
if (text) items.push({ type: 'message', role: 'assistant', content: [{ type: 'output_text', text }] })
|
|
212
|
+
for (const call of calls) {
|
|
213
|
+
items.push({
|
|
214
|
+
type: 'function_call',
|
|
215
|
+
call_id: String(call.id),
|
|
216
|
+
name: String(call.name),
|
|
217
|
+
arguments: typeof call.arguments === 'string' ? call.arguments : JSON.stringify(call.arguments ?? {}),
|
|
218
|
+
})
|
|
219
|
+
}
|
|
220
|
+
return items
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if (text) items.push({ type: 'message', role: 'user', content: [{ type: 'input_text', text }] })
|
|
224
|
+
for (const result of results) {
|
|
225
|
+
items.push({
|
|
226
|
+
type: 'function_call_output',
|
|
227
|
+
call_id: String(result.toolCallId),
|
|
228
|
+
output: toolResultText(result),
|
|
229
|
+
})
|
|
230
|
+
}
|
|
231
|
+
return items
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export function normalInput(messages, options = {}) {
|
|
235
|
+
const input = []
|
|
236
|
+
for (const message of messages ?? []) {
|
|
237
|
+
if (message?.role === 'system') continue
|
|
238
|
+
input.push(...messageInputItems(message, options))
|
|
239
|
+
}
|
|
240
|
+
return input
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export function routeFingerprint(route) {
|
|
244
|
+
// Keep the final empty slot for compatibility with v3 fingerprints already
|
|
245
|
+
// written before the nonexistent GenerateOptions.branchId field was removed.
|
|
246
|
+
const value = [route?.provider ?? '', route?.model ?? '', route?.baseURL ?? '', route?.sessionId ?? '', ''].join('\u001f')
|
|
247
|
+
return createHash('sha256').update(value, 'utf8').digest('hex')
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function checkpointRouteMismatchDimensions(record, route) {
|
|
251
|
+
const dimensions = []
|
|
252
|
+
if (record.provider !== String(route?.provider ?? '')) dimensions.push('provider')
|
|
253
|
+
if (record.model !== String(route?.model ?? '')) dimensions.push('model')
|
|
254
|
+
if ((record.sessionId ?? '') !== String(route?.sessionId ?? '')) dimensions.push('session')
|
|
255
|
+
if (dimensions.length === 0) dimensions.push('base URL or route configuration')
|
|
256
|
+
return dimensions
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export function assertCheckpointRoute(record, route) {
|
|
260
|
+
const expected = routeFingerprint(route)
|
|
261
|
+
const semanticMismatch = record.modelKey !== `${record.provider}:${record.model}` ||
|
|
262
|
+
record.provider !== String(route?.provider ?? '') ||
|
|
263
|
+
record.model !== String(route?.model ?? '')
|
|
264
|
+
if (semanticMismatch || record.routeFingerprint !== expected) {
|
|
265
|
+
const dimensions = checkpointRouteMismatchDimensions(record, route)
|
|
266
|
+
const guidance = dimensions.includes('session')
|
|
267
|
+
? 'Resume the original DSH session or start a new session without this checkpoint marker.'
|
|
268
|
+
: 'Use the original provider/model/base URL or create a new checkpoint.'
|
|
269
|
+
const error = new Error(`LCX checkpoint route mismatch (${dimensions.join(', ')}): checkpoint=${record.routeFingerprint ?? 'missing'} current=${expected}. ${guidance}`)
|
|
270
|
+
error.code = 'LCX_CHECKPOINT_ROUTE_MISMATCH'
|
|
271
|
+
throw error
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function assertNoUnsupportedCheckpointMarker(messages) {
|
|
276
|
+
for (const message of messages ?? []) {
|
|
277
|
+
if (message?.role !== 'user') continue
|
|
278
|
+
if (!UNSUPPORTED_CHECKPOINT_PATTERN.test(textOfMessage(message))) continue
|
|
279
|
+
const error = new Error('LCX checkpoint v2 markers are unsupported; use a v3 checkpoint')
|
|
280
|
+
error.code = 'LCX_CHECKPOINT_V2_UNSUPPORTED'
|
|
281
|
+
throw error
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export function baseURLFingerprint(baseURL) {
|
|
286
|
+
return createHash('sha256').update(String(baseURL ?? '').replace(/\/+$/u, ''), 'utf8').digest('hex')
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
export function latestPortableMarker(messages) {
|
|
290
|
+
assertNoUnsupportedCheckpointMarker(messages)
|
|
291
|
+
let found
|
|
292
|
+
for (let index = 0; index < (messages?.length ?? 0); index += 1) {
|
|
293
|
+
if (messages[index]?.role !== 'user') continue
|
|
294
|
+
const ids = portableCheckpointIds(textOfMessage(messages[index]))
|
|
295
|
+
if (ids.length > 1) {
|
|
296
|
+
const error = new Error('LCX message contains multiple v3 checkpoint markers')
|
|
297
|
+
error.code = 'LCX_CHECKPOINT_V3_CORRUPT'
|
|
298
|
+
throw error
|
|
299
|
+
}
|
|
300
|
+
if (ids.length === 1) found = { id: ids[0], index }
|
|
301
|
+
}
|
|
302
|
+
return found
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export function hasPortableCheckpoint(messages) {
|
|
306
|
+
return latestPortableMarker(messages) !== undefined
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export function buildPortableHistory(input, options = {}) {
|
|
310
|
+
const tokenBudget = options.tokenBudget ?? PORTABLE_HISTORY_TOKEN_BUDGET
|
|
311
|
+
const byteBudget = options.byteBudget ?? PORTABLE_HISTORY_BYTE_BUDGET
|
|
312
|
+
let remainingChars = tokenBudget * 4
|
|
313
|
+
let remainingBytes = byteBudget
|
|
314
|
+
const candidates = []
|
|
315
|
+
for (const item of input ?? []) {
|
|
316
|
+
if (item === null || typeof item !== 'object' || Array.isArray(item)) continue
|
|
317
|
+
if (typeof item.type !== 'string') continue
|
|
318
|
+
if (item.type === 'compaction' || item.type === 'context_compaction' || item.type === 'compaction_trigger') continue
|
|
319
|
+
if (item.type.startsWith('response.')) continue
|
|
320
|
+
if (Object.prototype.hasOwnProperty.call(item, 'encrypted_content')) continue
|
|
321
|
+
let candidate = item
|
|
322
|
+
if (item.type === 'message') {
|
|
323
|
+
const expectedType = item.role === 'assistant' ? 'output_text' : item.role === 'user' ? 'input_text' : undefined
|
|
324
|
+
const parts = Array.isArray(item.content) ? item.content : []
|
|
325
|
+
const imageParts = parts.filter((part) => part?.type === 'input_image')
|
|
326
|
+
if (imageParts.length > 0) {
|
|
327
|
+
if (!options.omitInputImages) throw unsupportedPortableContentError('input_image')
|
|
328
|
+
candidate = { ...item, content: parts.filter((part) => part?.type === expectedType) }
|
|
329
|
+
if (candidate.content.length === 0) continue
|
|
330
|
+
}
|
|
331
|
+
if (!expectedType || !Array.isArray(candidate.content) || candidate.content.some((part) =>
|
|
332
|
+
!isObject(part) || (part.type !== 'dsh_image_attachment' &&
|
|
333
|
+
(part.type !== expectedType || typeof part.text !== 'string')))) {
|
|
334
|
+
throw unsupportedPortableContentError(item.type)
|
|
335
|
+
}
|
|
336
|
+
} else if (item.type === 'function_call_output' && Array.isArray(item.output)) {
|
|
337
|
+
const imageParts = item.output.filter((part) => part?.type === 'input_image')
|
|
338
|
+
if (imageParts.length > 0) {
|
|
339
|
+
if (!options.omitInputImages) throw unsupportedPortableContentError('input_image')
|
|
340
|
+
candidate = { ...item, output: item.output.filter((part) => part?.type !== 'input_image') }
|
|
341
|
+
}
|
|
342
|
+
if (candidate.output.some((part) => !isObject(part) ||
|
|
343
|
+
(part.type !== 'dsh_image_attachment' &&
|
|
344
|
+
(!['input_text', 'output_text'].includes(part.type) || typeof part.text !== 'string')))) {
|
|
345
|
+
throw unsupportedPortableContentError('function_call_output')
|
|
346
|
+
}
|
|
347
|
+
} else if (item.type !== 'function_call' && item.type !== 'function_call_output') {
|
|
348
|
+
throw unsupportedPortableContentError(item.type)
|
|
349
|
+
}
|
|
350
|
+
candidates.push(candidate)
|
|
351
|
+
}
|
|
352
|
+
const callsById = new Map()
|
|
353
|
+
const outputsById = new Map()
|
|
354
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
355
|
+
const item = candidates[index]
|
|
356
|
+
if (item.type === 'function_call' && typeof item.call_id === 'string' && item.call_id.length > 0 &&
|
|
357
|
+
typeof item.name === 'string' && item.name.length > 0 && typeof item.arguments === 'string') {
|
|
358
|
+
const indexes = callsById.get(item.call_id) ?? []
|
|
359
|
+
indexes.push(index)
|
|
360
|
+
callsById.set(item.call_id, indexes)
|
|
361
|
+
}
|
|
362
|
+
if (item.type === 'function_call_output' && typeof item.call_id === 'string' && item.call_id.length > 0 &&
|
|
363
|
+
Object.prototype.hasOwnProperty.call(item, 'output')) {
|
|
364
|
+
const indexes = outputsById.get(item.call_id) ?? []
|
|
365
|
+
indexes.push(index)
|
|
366
|
+
outputsById.set(item.call_id, indexes)
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const pairedIndexes = new Set()
|
|
371
|
+
const units = []
|
|
372
|
+
for (const [callId, callIndexes] of callsById) {
|
|
373
|
+
const outputIndexes = outputsById.get(callId)
|
|
374
|
+
if (callIndexes.length !== 1 || outputIndexes?.length !== 1) continue
|
|
375
|
+
const callIndex = callIndexes[0]
|
|
376
|
+
const outputIndex = outputIndexes[0]
|
|
377
|
+
if (callIndex >= outputIndex) continue
|
|
378
|
+
const indexes = [callIndex, outputIndex]
|
|
379
|
+
for (const index of indexes) pairedIndexes.add(index)
|
|
380
|
+
units.push({ indexes, items: indexes.map((index) => candidates[index]) })
|
|
381
|
+
}
|
|
382
|
+
for (let index = 0; index < candidates.length; index += 1) {
|
|
383
|
+
const item = candidates[index]
|
|
384
|
+
if (pairedIndexes.has(index)) continue
|
|
385
|
+
if (item.type === 'function_call' || item.type === 'function_call_output') continue
|
|
386
|
+
units.push({ indexes: [index], items: [item] })
|
|
387
|
+
}
|
|
388
|
+
units.sort((left, right) => left.indexes[0] - right.indexes[0])
|
|
389
|
+
|
|
390
|
+
const retainedUnits = []
|
|
391
|
+
for (let index = units.length - 1; index >= 0; index -= 1) {
|
|
392
|
+
const unit = units[index]
|
|
393
|
+
let unitBytes = 0
|
|
394
|
+
let unitChars = 0
|
|
395
|
+
for (const item of unit.items) {
|
|
396
|
+
const serialized = JSON.stringify(item)
|
|
397
|
+
unitBytes += Buffer.byteLength(serialized, 'utf8')
|
|
398
|
+
unitChars += serialized.length
|
|
399
|
+
}
|
|
400
|
+
if (unitBytes > remainingBytes || unitChars > remainingChars) continue
|
|
401
|
+
retainedUnits.push(unit)
|
|
402
|
+
remainingBytes -= unitBytes
|
|
403
|
+
remainingChars -= unitChars
|
|
404
|
+
if (remainingBytes <= 128 || remainingChars <= 32) break
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
return retainedUnits
|
|
408
|
+
.flatMap((unit) => unit.indexes.map((index, itemIndex) => ({ index, item: structuredClone(unit.items[itemIndex]) })))
|
|
409
|
+
.sort((left, right) => left.index - right.index)
|
|
410
|
+
.map((entry) => entry.item)
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function portableCheckpointMismatchDimensions(record, route) {
|
|
414
|
+
if (!record || !route) return ['route']
|
|
415
|
+
const dimensions = []
|
|
416
|
+
if (record.provider !== String(route.provider ?? '')) dimensions.push('provider')
|
|
417
|
+
if (record.baseURLFingerprint !== baseURLFingerprint(route.baseURL)) dimensions.push('endpoint')
|
|
418
|
+
const sessionId = String(route.sessionId ?? '')
|
|
419
|
+
const sameSessionLineage = !sessionId || record.lineageId === sessionId ||
|
|
420
|
+
(Array.isArray(route.ancestorSessionIds) && route.ancestorSessionIds.includes(record.lineageId))
|
|
421
|
+
if (!sameSessionLineage) dimensions.push('session')
|
|
422
|
+
return dimensions.length > 0 ? dimensions : ['route']
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export function portableCheckpointState(record, route) {
|
|
426
|
+
if (!record || !route) return 'route-mismatch'
|
|
427
|
+
const currentSessionId = String(route.sessionId ?? '')
|
|
428
|
+
const hasSessionIdentity = currentSessionId.length > 0
|
|
429
|
+
const sameLineage = record.lineageId === currentSessionId ||
|
|
430
|
+
(Array.isArray(route.ancestorSessionIds) && route.ancestorSessionIds.includes(record.lineageId))
|
|
431
|
+
const sameProvider = record.provider === String(route.provider ?? '')
|
|
432
|
+
const sameBaseURL = record.baseURLFingerprint === baseURLFingerprint(route.baseURL)
|
|
433
|
+
if (hasSessionIdentity && sameLineage && record.routeFingerprint === routeFingerprint(route)) return 'native-compatible'
|
|
434
|
+
if (sameProvider && sameBaseURL && (!hasSessionIdentity || sameLineage)) return 'portable-migratable'
|
|
435
|
+
return 'route-mismatch'
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function portableSummaryItem(summary) {
|
|
439
|
+
if (typeof summary !== 'string' || summary.length === 0) return []
|
|
440
|
+
return [{
|
|
441
|
+
type: 'message',
|
|
442
|
+
role: 'assistant',
|
|
443
|
+
content: [{ type: 'output_text', text: summary }],
|
|
444
|
+
}]
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
export function buildPortableResponsesInput(messages, store, route) {
|
|
448
|
+
const marker = latestPortableMarker(messages)
|
|
449
|
+
if (!marker) return normalInput(messages, { strict: true })
|
|
450
|
+
const record = store.get(marker.id)
|
|
451
|
+
if (!record) {
|
|
452
|
+
const error = new Error(`LCX v3 checkpoint ${marker.id} is missing from ${store.file}`)
|
|
453
|
+
error.code = 'LCX_CHECKPOINT_V3_CORRUPT'
|
|
454
|
+
throw error
|
|
455
|
+
}
|
|
456
|
+
const state = portableCheckpointState(record, route)
|
|
457
|
+
if (state === 'route-mismatch') {
|
|
458
|
+
const dimensions = portableCheckpointMismatchDimensions(record, route)
|
|
459
|
+
const error = new Error(`LCX v3 checkpoint cannot migrate; route mismatch dimensions: ${dimensions.join(', ')}`)
|
|
460
|
+
error.code = 'LCX_CHECKPOINT_ROUTE_MISMATCH'
|
|
461
|
+
throw error
|
|
462
|
+
}
|
|
463
|
+
const prefix = state === 'native-compatible'
|
|
464
|
+
? record.nativeOutput
|
|
465
|
+
: [...portableSummaryItem(record.portableSummary), ...buildPortableHistory(record.portableHistory)]
|
|
466
|
+
const tail = normalInput((messages ?? []).slice(marker.index + 1), { strict: true })
|
|
467
|
+
return [...structuredClone(prefix), ...tail]
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
export async function buildPortableResponsesInputWithImages(messages, store, route, options = {}) {
|
|
471
|
+
const marker = latestPortableMarker(messages)
|
|
472
|
+
const imageSupport = options.imageSupport ?? (typeof options.resolveImage === 'function' ? 'supported' : 'unknown')
|
|
473
|
+
const requestMessages = offloadDshRequestImages(messages, options.maxRequestImageBytes)
|
|
474
|
+
if (!marker) return serializeDshResponsesInput(requestMessages, { ...options, imageSupport, route })
|
|
475
|
+
const record = store.get(marker.id)
|
|
476
|
+
if (!record) {
|
|
477
|
+
const error = new Error(`LCX v3 checkpoint ${marker.id} is missing from ${store.file}`)
|
|
478
|
+
error.code = 'LCX_CHECKPOINT_V3_CORRUPT'
|
|
479
|
+
throw error
|
|
480
|
+
}
|
|
481
|
+
const state = portableCheckpointState(record, route)
|
|
482
|
+
if (state === 'route-mismatch') {
|
|
483
|
+
const dimensions = portableCheckpointMismatchDimensions(record, route)
|
|
484
|
+
const error = new Error(`LCX v3 checkpoint cannot migrate; route mismatch dimensions: ${dimensions.join(', ')}`)
|
|
485
|
+
error.code = 'LCX_CHECKPOINT_ROUTE_MISMATCH'
|
|
486
|
+
throw error
|
|
487
|
+
}
|
|
488
|
+
const portableImages = Number(record.portableImageCount ?? inputImageCount(record.portableHistory))
|
|
489
|
+
const durableImages = inputImageCount(record.portableHistory)
|
|
490
|
+
if (state === 'portable-migratable' && portableImages > durableImages) {
|
|
491
|
+
throw compactImageError(`LCX checkpoint portable migration cannot restore ${portableImages} image attachment(s)`, 'LCX_CHECKPOINT_PORTABLE_UNSUPPORTED_CONTENT')
|
|
492
|
+
}
|
|
493
|
+
let portableHistory
|
|
494
|
+
if (state === 'portable-migratable') {
|
|
495
|
+
portableHistory = buildPortableHistory(record.portableHistory)
|
|
496
|
+
if (durableImages > 0) {
|
|
497
|
+
if (imageSupport === 'unknown') {
|
|
498
|
+
throw compactImageError('LCX Compact cannot determine whether the target model accepts checkpoint images', 'LCX_COMPACT_IMAGE_CAPABILITY_UNKNOWN')
|
|
499
|
+
}
|
|
500
|
+
portableHistory = imageSupport === 'supported'
|
|
501
|
+
? await hydrateNativeImageReferences(portableHistory, options)
|
|
502
|
+
: portableHistory.map((item) => {
|
|
503
|
+
const key = item?.type === 'message' ? 'content' : item?.type === 'function_call_output' ? 'output' : undefined
|
|
504
|
+
if (!key || !Array.isArray(item[key])) return structuredClone(item)
|
|
505
|
+
const textType = item.type === 'message' && item.role === 'assistant' ? 'output_text' : 'input_text'
|
|
506
|
+
return {
|
|
507
|
+
...structuredClone(item),
|
|
508
|
+
[key]: item[key].map((part) => part?.type === 'dsh_image_attachment'
|
|
509
|
+
? { type: textType, text: UNSUPPORTED_IMAGE_PLACEHOLDER }
|
|
510
|
+
: structuredClone(part)),
|
|
511
|
+
}
|
|
512
|
+
})
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
const prefix = state === 'native-compatible'
|
|
516
|
+
? await hydrateNativeImageReferences(record.nativeOutput, options)
|
|
517
|
+
: [...portableSummaryItem(record.portableSummary), ...portableHistory]
|
|
518
|
+
const tail = state === 'native-compatible'
|
|
519
|
+
? await serializeDshResponsesInput(requestMessages.slice(marker.index + 1), { ...options, imageSupport, route })
|
|
520
|
+
: await serializeDshResponsesInput(requestMessages.slice(marker.index + 1), { ...options, imageSupport, route })
|
|
521
|
+
return [...prefix, ...tail]
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function isObject(value) {
|
|
525
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
function compactOutputShape(response) {
|
|
529
|
+
const output = Array.isArray(response?.output) ? response.output : []
|
|
530
|
+
const types = output.slice(0, 20).map((item) => {
|
|
531
|
+
const type = item && typeof item === 'object' && typeof item.type === 'string' ? item.type : typeof item
|
|
532
|
+
return String(type).replace(/[^a-zA-Z0-9._:-]/gu, '').slice(0, 48) || 'unknown'
|
|
533
|
+
})
|
|
534
|
+
return `object=${String(response?.object ?? 'missing')} outputLength=${output.length} outputTypes=[${types.join(',')}]`
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
function invalidOutputItem(message) {
|
|
538
|
+
const error = new Error(`LCX compact response contains an invalid output item: ${message}`)
|
|
539
|
+
error.code = 'LCX_COMPACT_INVALID_RESPONSE'
|
|
540
|
+
return error
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function validContent(content) {
|
|
544
|
+
return Array.isArray(content) && content.every((part) => isObject(part) && typeof part.type === 'string' &&
|
|
545
|
+
(part.text === undefined || typeof part.text === 'string'))
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function validateOutputItems(output) {
|
|
549
|
+
const itemIds = new Set()
|
|
550
|
+
const callIds = new Set()
|
|
551
|
+
const outputIds = new Set()
|
|
552
|
+
const calls = new Set()
|
|
553
|
+
const outputs = new Set()
|
|
554
|
+
for (const item of output) {
|
|
555
|
+
if (!isObject(item) || typeof item.type !== 'string' || item.type.length === 0) throw invalidOutputItem('missing type')
|
|
556
|
+
if (item.id !== undefined) {
|
|
557
|
+
if (typeof item.id !== 'string' || item.id.length === 0 || itemIds.has(item.id)) throw invalidOutputItem('duplicate or invalid id')
|
|
558
|
+
itemIds.add(item.id)
|
|
559
|
+
}
|
|
560
|
+
if (item.type === 'compaction_trigger' || item.type === 'context_compaction' || item.type.startsWith('response.')) {
|
|
561
|
+
throw invalidOutputItem(`invalid trigger/type ${item.type}`)
|
|
562
|
+
}
|
|
563
|
+
if (item.type === 'message') {
|
|
564
|
+
if (!['assistant', 'developer', 'system', 'user'].includes(item.role) || !validContent(item.content)) {
|
|
565
|
+
throw invalidOutputItem('message shape')
|
|
566
|
+
}
|
|
567
|
+
continue
|
|
568
|
+
}
|
|
569
|
+
if (item.type === 'function_call') {
|
|
570
|
+
if (typeof item.call_id !== 'string' || item.call_id.length === 0 || callIds.has(item.call_id) ||
|
|
571
|
+
typeof item.name !== 'string' || item.name.length === 0 || typeof item.arguments !== 'string') {
|
|
572
|
+
throw invalidOutputItem('function_call shape')
|
|
573
|
+
}
|
|
574
|
+
callIds.add(item.call_id)
|
|
575
|
+
calls.add(item.call_id)
|
|
576
|
+
continue
|
|
577
|
+
}
|
|
578
|
+
if (item.type === 'function_call_output') {
|
|
579
|
+
if (typeof item.call_id !== 'string' || item.call_id.length === 0 || outputIds.has(item.call_id) ||
|
|
580
|
+
!(typeof item.output === 'string' || Array.isArray(item.output))) {
|
|
581
|
+
throw invalidOutputItem('function_call_output shape')
|
|
582
|
+
}
|
|
583
|
+
outputIds.add(item.call_id)
|
|
584
|
+
outputs.add(item.call_id)
|
|
585
|
+
continue
|
|
586
|
+
}
|
|
587
|
+
if (item.type === 'compaction') {
|
|
588
|
+
if (typeof item.encrypted_content !== 'string' || item.encrypted_content.length === 0) {
|
|
589
|
+
throw invalidOutputItem('compaction encrypted_content')
|
|
590
|
+
}
|
|
591
|
+
continue
|
|
592
|
+
}
|
|
593
|
+
if (item.type === 'reasoning') {
|
|
594
|
+
if (item.encrypted_content !== undefined && typeof item.encrypted_content !== 'string') {
|
|
595
|
+
throw invalidOutputItem('reasoning encrypted_content')
|
|
596
|
+
}
|
|
597
|
+
if (item.content !== undefined && !validContent(item.content)) throw invalidOutputItem('reasoning content')
|
|
598
|
+
if (item.summary !== undefined && !validContent(item.summary)) throw invalidOutputItem('reasoning summary')
|
|
599
|
+
continue
|
|
600
|
+
}
|
|
601
|
+
if (Object.prototype.hasOwnProperty.call(item, 'encrypted_content')) throw invalidOutputItem('encrypted non-compaction item')
|
|
602
|
+
}
|
|
603
|
+
for (const callId of outputs) {
|
|
604
|
+
if (!calls.has(callId)) throw invalidOutputItem(`orphan function_call_output ${callId}`)
|
|
605
|
+
}
|
|
606
|
+
for (const callId of calls) {
|
|
607
|
+
if (!outputs.has(callId)) throw invalidOutputItem(`orphan function_call ${callId}`)
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
export function normalizeCompactionResponse(response) {
|
|
612
|
+
if (response?.error) {
|
|
613
|
+
const error = new Error(response.error.message ?? 'LCX compact request failed')
|
|
614
|
+
error.code = 'LCX_COMPACT_HTTP_ERROR'
|
|
615
|
+
throw error
|
|
616
|
+
}
|
|
617
|
+
if (response?.object !== undefined && !['response.compaction', 'response'].includes(response.object)) {
|
|
618
|
+
const error = new Error(`Unexpected compact response object: ${String(response.object)}`)
|
|
619
|
+
error.code = 'LCX_COMPACT_INVALID_RESPONSE'
|
|
620
|
+
throw error
|
|
621
|
+
}
|
|
622
|
+
if (!Array.isArray(response?.output)) {
|
|
623
|
+
const error = new Error('LCX compact response has no output array')
|
|
624
|
+
error.code = 'LCX_COMPACT_INVALID_RESPONSE'
|
|
625
|
+
throw error
|
|
626
|
+
}
|
|
627
|
+
if (response.output.some((item) => item?.type === 'compaction_trigger')) {
|
|
628
|
+
throw invalidOutputItem('invalid trigger compaction_trigger')
|
|
629
|
+
}
|
|
630
|
+
const compactions = response.output.filter((item) => item?.type === 'compaction')
|
|
631
|
+
if (compactions.length === 0) {
|
|
632
|
+
const error = new Error(`LCX compact response has no compaction item (${compactOutputShape(response)})`)
|
|
633
|
+
error.code = 'LCX_COMPACT_MISSING_ITEM'
|
|
634
|
+
throw error
|
|
635
|
+
}
|
|
636
|
+
if (compactions.length !== 1) {
|
|
637
|
+
const error = new Error(`LCX compact response has ${compactions.length} compaction items`)
|
|
638
|
+
error.code = 'LCX_COMPACT_MULTIPLE_ITEMS'
|
|
639
|
+
throw error
|
|
640
|
+
}
|
|
641
|
+
if (typeof compactions[0].encrypted_content !== 'string' || compactions[0].encrypted_content.length === 0) {
|
|
642
|
+
const error = new Error('LCX compact response has empty encrypted_content')
|
|
643
|
+
error.code = 'LCX_COMPACT_EMPTY_ENCRYPTED_CONTENT'
|
|
644
|
+
throw error
|
|
645
|
+
}
|
|
646
|
+
validateOutputItems(response.output)
|
|
647
|
+
const compaction = compactions[0]
|
|
648
|
+
return {
|
|
649
|
+
output: structuredClone(response.output),
|
|
650
|
+
compaction: structuredClone(compaction),
|
|
651
|
+
responseId: typeof response.id === 'string' ? response.id : undefined,
|
|
652
|
+
}
|
|
653
|
+
}
|