dsh-lcx-codex 0.4.3-pre.3 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -99
- package/THIRD_PARTY_NOTICES.md +64 -0
- package/lib/auxiliary-usage.js +63 -0
- package/lib/client.js +1363 -295
- package/lib/dsh-compat.js +80 -18
- package/lib/dsh-responses.js +75 -8
- package/lib/grok-native-search.js +153 -79
- package/lib/index.js +159 -101
- package/lib/invocation-policy-scope.js +261 -0
- package/lib/json-store.js +11 -4
- package/lib/pi-responses-runtime.js +1571 -0
- package/lib/responses-request.js +1 -1
- package/lib/responses-stream.js +163 -92
- package/lib/route.js +17 -3
- package/lib/search-accounting.js +86 -0
- package/lib/search-usage.js +86 -0
- package/lib/transport.js +38 -10
- package/lib/types/client/index.d.ts +12 -0
- package/lib/types/client/search-media.d.ts +16 -0
- package/lib/web-run-output.js +22 -0
- package/lib/web-search-alpha.js +230 -28
- package/lib/web-search-capability.js +26 -1
- package/lib/web-search-hosted.js +119 -9
- package/lib/web-search-ref-store.js +88 -7
- package/package.json +50 -26
- package/lib/types/compact-v2.d.ts +0 -104
- package/lib/types/dsh-compat.d.ts +0 -78
- package/lib/types/dsh-responses.d.ts +0 -82
- package/lib/types/grok-native-search.d.ts +0 -39
- package/lib/types/json-store.d.ts +0 -10
- package/lib/types/native-checkpoint.d.ts +0 -213
- package/lib/types/responses-request.d.ts +0 -59
- package/lib/types/responses-stream.d.ts +0 -64
- package/lib/types/route.d.ts +0 -153
- package/lib/types/service-mutex.d.ts +0 -14
- package/lib/types/token-budget.d.ts +0 -50
- package/lib/types/transport.d.ts +0 -21
- package/lib/types/web-run-output.d.ts +0 -29
- package/lib/types/web-search-alpha.d.ts +0 -286
- package/lib/types/web-search-capability.d.ts +0 -26
- package/lib/types/web-search-hosted.d.ts +0 -246
- package/lib/types/web-search-ref-store.d.ts +0 -22
package/lib/dsh-compat.js
CHANGED
|
@@ -132,43 +132,105 @@ export function compactionPatchCandidate(value, records) {
|
|
|
132
132
|
typeof compaction.compactIfNeeded !== "function" ||
|
|
133
133
|
records.has(compaction))
|
|
134
134
|
return undefined;
|
|
135
|
+
let originalOwnDescriptor;
|
|
136
|
+
try {
|
|
137
|
+
originalOwnDescriptor = Object.getOwnPropertyDescriptor(compaction, "compactIfNeeded");
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
135
142
|
return {
|
|
136
143
|
compaction: compaction,
|
|
137
144
|
original: compaction.compactIfNeeded,
|
|
145
|
+
originalOwnDescriptor,
|
|
138
146
|
};
|
|
139
147
|
}
|
|
140
|
-
|
|
141
|
-
|
|
148
|
+
function inheritedPropertyDescriptor(target, key) {
|
|
149
|
+
try {
|
|
150
|
+
for (let current = Object.getPrototypeOf(target); current; current = Object.getPrototypeOf(current)) {
|
|
151
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key);
|
|
152
|
+
if (descriptor)
|
|
153
|
+
return descriptor;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
catch { }
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
function sameDataDescriptor(left, right) {
|
|
160
|
+
return (left !== undefined &&
|
|
161
|
+
"value" in left &&
|
|
162
|
+
"value" in right &&
|
|
163
|
+
left.value === right.value &&
|
|
164
|
+
left.writable === right.writable &&
|
|
165
|
+
left.enumerable === right.enumerable &&
|
|
166
|
+
left.configurable === right.configurable);
|
|
167
|
+
}
|
|
168
|
+
export function installCompactionPatch(records, candidate, mutex, policyScope, lifecycle, behavior) {
|
|
169
|
+
const { compaction, original, originalOwnDescriptor } = candidate;
|
|
170
|
+
if (originalOwnDescriptor &&
|
|
171
|
+
(!("value" in originalOwnDescriptor) || originalOwnDescriptor.writable !== true))
|
|
172
|
+
return false;
|
|
173
|
+
if (!originalOwnDescriptor) {
|
|
174
|
+
const inherited = inheritedPropertyDescriptor(compaction, "compactIfNeeded");
|
|
175
|
+
if (!inherited || !("value" in inherited) || inherited.writable !== true)
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
142
178
|
const callOriginal = async (agent, trigger, signal) => {
|
|
143
179
|
const result = Reflect.apply(original, compaction, [agent, trigger, signal]);
|
|
144
180
|
return Promise.resolve(result);
|
|
145
181
|
};
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
original,
|
|
149
|
-
mutex,
|
|
150
|
-
lifecycle,
|
|
151
|
-
wrapper: async function (agent, trigger, signal) {
|
|
152
|
-
return behavior(agent, trigger, signal, (activeSignal) => callOriginal(agent, trigger, activeSignal));
|
|
153
|
-
},
|
|
182
|
+
const wrapper = async function (agent, trigger, signal) {
|
|
183
|
+
return behavior(agent, trigger, signal, (activeSignal) => callOriginal(agent, trigger, activeSignal));
|
|
154
184
|
};
|
|
185
|
+
const installedOwnDescriptor = originalOwnDescriptor
|
|
186
|
+
? { ...originalOwnDescriptor, value: wrapper }
|
|
187
|
+
: {
|
|
188
|
+
configurable: true,
|
|
189
|
+
enumerable: true,
|
|
190
|
+
writable: true,
|
|
191
|
+
value: wrapper,
|
|
192
|
+
};
|
|
155
193
|
try {
|
|
156
|
-
compaction
|
|
194
|
+
Object.defineProperty(compaction, "compactIfNeeded", installedOwnDescriptor);
|
|
157
195
|
}
|
|
158
196
|
catch {
|
|
159
197
|
return false;
|
|
160
198
|
}
|
|
161
|
-
|
|
199
|
+
if (!sameDataDescriptor(Object.getOwnPropertyDescriptor(compaction, "compactIfNeeded"), installedOwnDescriptor)) {
|
|
200
|
+
try {
|
|
201
|
+
if (originalOwnDescriptor)
|
|
202
|
+
Object.defineProperty(compaction, "compactIfNeeded", originalOwnDescriptor);
|
|
203
|
+
else
|
|
204
|
+
Reflect.deleteProperty(compaction, "compactIfNeeded");
|
|
205
|
+
}
|
|
206
|
+
catch { }
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
records.set(compaction, {
|
|
210
|
+
compaction,
|
|
211
|
+
original,
|
|
212
|
+
originalOwnDescriptor,
|
|
213
|
+
installedOwnDescriptor,
|
|
214
|
+
wrapper,
|
|
215
|
+
mutex,
|
|
216
|
+
policyScope,
|
|
217
|
+
lifecycle,
|
|
218
|
+
});
|
|
162
219
|
return true;
|
|
163
220
|
}
|
|
164
221
|
export function restoreCompactionPatches(records, entries = records.values()) {
|
|
165
|
-
for (const record of entries)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
222
|
+
for (const record of entries) {
|
|
223
|
+
const current = Object.getOwnPropertyDescriptor(record.compaction, "compactIfNeeded");
|
|
224
|
+
if (!sameDataDescriptor(current, record.installedOwnDescriptor))
|
|
225
|
+
continue;
|
|
226
|
+
try {
|
|
227
|
+
if (record.originalOwnDescriptor)
|
|
228
|
+
Object.defineProperty(record.compaction, "compactIfNeeded", record.originalOwnDescriptor);
|
|
229
|
+
else
|
|
230
|
+
Reflect.deleteProperty(record.compaction, "compactIfNeeded");
|
|
171
231
|
}
|
|
232
|
+
catch { }
|
|
233
|
+
}
|
|
172
234
|
records.clear();
|
|
173
235
|
}
|
|
174
236
|
export function toolResultPrunerState(value) {
|
package/lib/dsh-responses.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { offloadedImageText, offloadRequestImagesWithPolicy, requestImageHandleText, resolveImageAttachmentAccess, } from "@deepseek-ai/dsh-llm";
|
|
2
|
-
import { convertResponsesMessages, convertResponsesTools, } from "
|
|
3
|
-
import { createGrammarToolInputProperties } from "@earendil-works/pi-ai/api/constrained-sampling";
|
|
4
|
-
import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all";
|
|
2
|
+
import { convertResponsesMessages, convertResponsesTools, createGrammarToolInputProperties, getBuiltinModels, } from "./pi-responses-runtime.js";
|
|
5
3
|
const DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
6
4
|
const DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET = 2048 * 2048;
|
|
7
5
|
const DEFAULT_REQUEST_IMAGE_MAX_BYTES = 1024 * 1024;
|
|
@@ -93,7 +91,7 @@ function attachmentResolver(ctx, options) {
|
|
|
93
91
|
const attachments = attachmentServices(ctx?.attachments);
|
|
94
92
|
return async (block, signal) => {
|
|
95
93
|
if (attachments === undefined)
|
|
96
|
-
throw error("LCX requires the DSH
|
|
94
|
+
throw error("LCX requires the DSH request-image attachment API", "LCX_COMPACT_IMAGE_API_UNAVAILABLE");
|
|
97
95
|
const request = await attachments.store.readImageRequest(block.attachment, { maxPixels: options.requestImagePixelBudget, maxBytes: options.requestImageMaxBytes }, signal);
|
|
98
96
|
const data = request.data;
|
|
99
97
|
const mediaType = request.mediaType ?? block.attachment.mediaType;
|
|
@@ -159,6 +157,57 @@ function unsupportedContent(type) {
|
|
|
159
157
|
function replayBlockType(type) {
|
|
160
158
|
return type === "text" || type === "reasoning" || type === "tool-call" ? type : undefined;
|
|
161
159
|
}
|
|
160
|
+
function replayTextPhase(signature) {
|
|
161
|
+
if (typeof signature !== "string" || !signature.startsWith("{"))
|
|
162
|
+
return undefined;
|
|
163
|
+
try {
|
|
164
|
+
const parsed = JSON.parse(signature);
|
|
165
|
+
if (!isObject(parsed) || parsed.v !== 1 || typeof parsed.id !== "string")
|
|
166
|
+
return undefined;
|
|
167
|
+
return parsed.phase === "commentary" || parsed.phase === "final_answer"
|
|
168
|
+
? parsed.phase
|
|
169
|
+
: undefined;
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function restorePortableAssistantPhases(input, messages, target) {
|
|
176
|
+
const slots = [];
|
|
177
|
+
for (const message of messages) {
|
|
178
|
+
if (message.role !== "assistant" || message.stopReason === "error" || message.stopReason === "aborted")
|
|
179
|
+
continue;
|
|
180
|
+
const sameProviderAndApi = message.provider === target.provider && message.api === target.api;
|
|
181
|
+
const sameModel = sameProviderAndApi && message.model === target.id;
|
|
182
|
+
for (const block of message.content) {
|
|
183
|
+
if (block.type === "text") {
|
|
184
|
+
const phase = sameProviderAndApi && !sameModel
|
|
185
|
+
? replayTextPhase(block.textSignature)
|
|
186
|
+
: undefined;
|
|
187
|
+
slots.push({ text: block.text, ...(phase === undefined ? {} : { phase }) });
|
|
188
|
+
}
|
|
189
|
+
else if (block.type === "thinking" && !sameModel && !block.redacted && block.thinking.trim()) {
|
|
190
|
+
slots.push({ text: block.thinking });
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const output = structuredClone(input);
|
|
195
|
+
let slotIndex = 0;
|
|
196
|
+
for (const [index, candidate] of output.entries()) {
|
|
197
|
+
if (!isObject(candidate) || candidate.type !== "message" || candidate.role !== "assistant" ||
|
|
198
|
+
!Array.isArray(candidate.content))
|
|
199
|
+
continue;
|
|
200
|
+
const slot = slots[slotIndex++];
|
|
201
|
+
if (slot?.phase === undefined)
|
|
202
|
+
continue;
|
|
203
|
+
const text = candidate.content
|
|
204
|
+
.map((part) => isObject(part) && part.type === "output_text" ? String(part.text ?? "") : "")
|
|
205
|
+
.join("");
|
|
206
|
+
if (text === slot.text)
|
|
207
|
+
output[index] = { ...candidate, phase: slot.phase };
|
|
208
|
+
}
|
|
209
|
+
return output;
|
|
210
|
+
}
|
|
162
211
|
export function readDshPiReplayState(value) {
|
|
163
212
|
if (!isObject(value))
|
|
164
213
|
throw invalidReplay("expected a replay envelope");
|
|
@@ -250,6 +299,8 @@ async function piToolContent(blocks, ctx, options, imageMap) {
|
|
|
250
299
|
content.push({ type: "text", text: block.text });
|
|
251
300
|
else if (block.type === "image")
|
|
252
301
|
content.push(...(await piImageParts(block, ctx, options, imageMap)));
|
|
302
|
+
else if (block.type === "reasoning")
|
|
303
|
+
continue;
|
|
253
304
|
else if (block.type === "tool-result")
|
|
254
305
|
content.push(...(await piToolContent(block.content, ctx, options, imageMap)));
|
|
255
306
|
else
|
|
@@ -280,13 +331,25 @@ function projectFiles(messages, ctx) {
|
|
|
280
331
|
content: projectFileContent(message.content, fileRequestText),
|
|
281
332
|
}));
|
|
282
333
|
}
|
|
334
|
+
function flattenMessageText(message) {
|
|
335
|
+
return message.content
|
|
336
|
+
.map((block) => (block.type === "text" ? block.text : ""))
|
|
337
|
+
.join("");
|
|
338
|
+
}
|
|
283
339
|
async function dshToPiMessages(messages, ctx, options, imageMap) {
|
|
284
340
|
const result = [];
|
|
341
|
+
const toolNames = new Map();
|
|
285
342
|
for (const message of messages) {
|
|
286
|
-
if (message.role === "system")
|
|
343
|
+
if (message.role === "system") {
|
|
344
|
+
result.push({ role: "user", content: flattenMessageText(message), timestamp: 0 });
|
|
287
345
|
continue;
|
|
346
|
+
}
|
|
288
347
|
if (message.role === "assistant") {
|
|
289
|
-
|
|
348
|
+
const assistant = toPiAssistant(message, options.onReplayDegrade);
|
|
349
|
+
for (const block of assistant.content)
|
|
350
|
+
if (block.type === "toolCall")
|
|
351
|
+
toolNames.set(String(block.id), block.name);
|
|
352
|
+
result.push(assistant);
|
|
290
353
|
continue;
|
|
291
354
|
}
|
|
292
355
|
const ordinary = [];
|
|
@@ -294,6 +357,8 @@ async function dshToPiMessages(messages, ctx, options, imageMap) {
|
|
|
294
357
|
for (const block of message.content) {
|
|
295
358
|
if (block.type === "text" || block.type === "image")
|
|
296
359
|
ordinary.push(block);
|
|
360
|
+
else if (block.type === "reasoning")
|
|
361
|
+
continue;
|
|
297
362
|
else if (block.type === "tool-result")
|
|
298
363
|
toolResults.push(block);
|
|
299
364
|
else
|
|
@@ -313,7 +378,8 @@ async function dshToPiMessages(messages, ctx, options, imageMap) {
|
|
|
313
378
|
for (const block of toolResults) {
|
|
314
379
|
if (block.type !== "tool-result")
|
|
315
380
|
continue;
|
|
316
|
-
|
|
381
|
+
const toolCallId = String(block.toolCallId);
|
|
382
|
+
result.push({ role: "toolResult", toolCallId, toolName: toolNames.get(toolCallId) ?? "unknown", content: await piToolContent(block.content, ctx, options, imageMap), addedToolNames: [], isError: block.isError === true, timestamp: 0 });
|
|
317
383
|
}
|
|
318
384
|
}
|
|
319
385
|
return result;
|
|
@@ -379,7 +445,8 @@ export async function serializeDshMessages(messages, ctx, options = {}) {
|
|
|
379
445
|
tools: context.tools,
|
|
380
446
|
};
|
|
381
447
|
const piModel = model;
|
|
382
|
-
const
|
|
448
|
+
const convertedInput = convertResponsesMessages(piModel, piContext, new Set(["openai", "openai-codex", "opencode"]), { includeSystemPrompt: normalized.includeSystemPrompt, grammarToolInputProperties, deferredTools: new Map(), deferredToolsMode, toolOptions });
|
|
449
|
+
const input = restorePortableAssistantPhases(convertedInput, context.messages, { provider: model.provider, id: model.id, api: model.api });
|
|
383
450
|
const tools = options.tools === undefined ? undefined : convertResponsesTools([...immediateTools.values()], toolOptions);
|
|
384
451
|
return { input, imageMap, tools, model: piModel, grammarToolInputProperties, deferredToolsMode };
|
|
385
452
|
}
|
|
@@ -8,6 +8,13 @@ const GROK_EXCLUDED_SEARCH_FUNCTIONS = new Set([
|
|
|
8
8
|
"websearch_gpt_advanced",
|
|
9
9
|
"websearch_alpha",
|
|
10
10
|
]);
|
|
11
|
+
const GROK_NATIVE_X_EXTENSION_NAMES = new Set([
|
|
12
|
+
"x_user_search",
|
|
13
|
+
"x_keyword_search",
|
|
14
|
+
"x_semantic_search",
|
|
15
|
+
"x_thread_fetch",
|
|
16
|
+
]);
|
|
17
|
+
const GROK_NATIVE_WEB_EXTENSION_NAMES = new Set(["browse_page"]);
|
|
11
18
|
const GROK_NATIVE_VISIBLE_ITEM_TYPES = new Set([
|
|
12
19
|
"message",
|
|
13
20
|
"reasoning",
|
|
@@ -16,9 +23,104 @@ const GROK_NATIVE_VISIBLE_ITEM_TYPES = new Set([
|
|
|
16
23
|
]);
|
|
17
24
|
const GROK_NATIVE_REPLAY_MAX_ITEMS = 256;
|
|
18
25
|
const GROK_NATIVE_REPLAY_MAX_CHARS = 8 * 1024 * 1024;
|
|
26
|
+
const GROK_RENDER_MARKER = /(?:\bstateless_invoke\s+)?\{?render_inline_citation(?:\s*\(\s*citation_id\s*=\s*\d+\s*\)|\s+with\s+citation_id\s+is\s+\d+)\}?[ \t]*/giu;
|
|
27
|
+
const GROK_CITATION_ID_MARKER = /([ \t]?)\{render_inline_citation:citation_id=(\d+)\}/giu;
|
|
28
|
+
const GROK_UI_CITATION_MARKER = /render_inline_citation[^\r\n]*[ \t]*/giu;
|
|
29
|
+
const GROK_PRIVATE_RENDER_MARKER = /[\uE000-\uF8FF]render_inline_citation[^\uE000-\uF8FF\r\n]*[\uE000-\uF8FF][ \t]*/giu;
|
|
30
|
+
const GROK_EOS_MARKER = /<\|eos\|>/giu;
|
|
19
31
|
function isRecord(value) {
|
|
20
32
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
21
33
|
}
|
|
34
|
+
function citationId(annotation) {
|
|
35
|
+
const value = annotation.citation_id ?? annotation.citationId ?? annotation.id;
|
|
36
|
+
return typeof value === "string" || typeof value === "number"
|
|
37
|
+
? String(value)
|
|
38
|
+
: undefined;
|
|
39
|
+
}
|
|
40
|
+
function citationUrl(annotation) {
|
|
41
|
+
if (annotation.type !== "url_citation" || typeof annotation.url !== "string")
|
|
42
|
+
return undefined;
|
|
43
|
+
try {
|
|
44
|
+
const parsed = new URL(annotation.url);
|
|
45
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
|
|
46
|
+
return undefined;
|
|
47
|
+
return annotation.url.replace(/\)/gu, "%29");
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function stripTrailingSourcesList(value) {
|
|
54
|
+
const newline = value.includes("\r\n") ? "\r\n" : "\n";
|
|
55
|
+
const lines = value.split(/\r?\n/u);
|
|
56
|
+
let fence;
|
|
57
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
58
|
+
const fenceMatch = /^\s*(```|~~~)/u.exec(lines[index] ?? "");
|
|
59
|
+
if (fenceMatch) {
|
|
60
|
+
const marker = fenceMatch[1];
|
|
61
|
+
fence = fence === marker ? undefined : fence ?? marker;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (fence !== undefined || !/^\s*Sources:[ \t]*$/iu.test(lines[index] ?? ""))
|
|
65
|
+
continue;
|
|
66
|
+
let sourceCount = 0;
|
|
67
|
+
let validTail = true;
|
|
68
|
+
for (const line of lines.slice(index + 1)) {
|
|
69
|
+
if (!line.trim())
|
|
70
|
+
continue;
|
|
71
|
+
if (/^\s*[-*][ \t]+.*https?:\/\/\S+/iu.test(line))
|
|
72
|
+
sourceCount += 1;
|
|
73
|
+
else {
|
|
74
|
+
validTail = false;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (validTail && sourceCount > 0)
|
|
79
|
+
return lines.slice(0, index).join(newline);
|
|
80
|
+
}
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
export function grokPendingSourcesStart(value) {
|
|
84
|
+
const lines = value.split(/(?<=\n)/u);
|
|
85
|
+
let offset = 0;
|
|
86
|
+
let fence;
|
|
87
|
+
let candidate;
|
|
88
|
+
for (const lineWithEnding of lines) {
|
|
89
|
+
const line = lineWithEnding.replace(/\r?\n$/u, "");
|
|
90
|
+
const fenceMatch = /^\s*(```|~~~)/u.exec(line);
|
|
91
|
+
if (fenceMatch) {
|
|
92
|
+
const marker = fenceMatch[1];
|
|
93
|
+
fence = fence === marker ? undefined : fence ?? marker;
|
|
94
|
+
}
|
|
95
|
+
else if (fence === undefined && /^\s*Sources:[ \t]*$/iu.test(line)) {
|
|
96
|
+
candidate = offset;
|
|
97
|
+
}
|
|
98
|
+
offset += lineWithEnding.length;
|
|
99
|
+
}
|
|
100
|
+
return candidate;
|
|
101
|
+
}
|
|
102
|
+
export function sanitizeGrokVisibleText(value, annotations = []) {
|
|
103
|
+
const citations = new Map();
|
|
104
|
+
for (const candidate of annotations) {
|
|
105
|
+
if (!isRecord(candidate))
|
|
106
|
+
continue;
|
|
107
|
+
const id = citationId(candidate);
|
|
108
|
+
const url = citationUrl(candidate);
|
|
109
|
+
if (id !== undefined && url !== undefined && !citations.has(id))
|
|
110
|
+
citations.set(id, url);
|
|
111
|
+
}
|
|
112
|
+
let text = value
|
|
113
|
+
.replace(GROK_CITATION_ID_MARKER, (_marker, leading, id) => {
|
|
114
|
+
const url = citations.get(id);
|
|
115
|
+
return url === undefined ? "" : `${leading}[[${id}]](${url})`;
|
|
116
|
+
})
|
|
117
|
+
.replace(GROK_RENDER_MARKER, "")
|
|
118
|
+
.replace(GROK_UI_CITATION_MARKER, "")
|
|
119
|
+
.replace(GROK_PRIVATE_RENDER_MARKER, "")
|
|
120
|
+
.replace(GROK_EOS_MARKER, "");
|
|
121
|
+
text = stripTrailingSourcesList(text);
|
|
122
|
+
return text.trimEnd();
|
|
123
|
+
}
|
|
22
124
|
function canonicalHeaders(headers) {
|
|
23
125
|
return Object.entries(headers ?? {})
|
|
24
126
|
.map(([name, value]) => [name.toLowerCase(), value])
|
|
@@ -43,6 +145,21 @@ export function grokVisibleFunctionTools(tools, state = { web: false, x: false }
|
|
|
43
145
|
return tools?.filter((tool) => !GROK_EXCLUDED_SEARCH_FUNCTIONS.has(tool.name) &&
|
|
44
146
|
!(state.x && tool.name === "x_search"));
|
|
45
147
|
}
|
|
148
|
+
export function isGrokNativeServerToolItem(value, state, declaredToolNames) {
|
|
149
|
+
if (!isRecord(value) || typeof value.type !== "string")
|
|
150
|
+
return false;
|
|
151
|
+
if (GROK_NATIVE_SERVER_TOOL_TYPES.has(value.type))
|
|
152
|
+
return true;
|
|
153
|
+
if (value.type !== "custom_tool_call" || typeof value.name !== "string" ||
|
|
154
|
+
declaredToolNames.has(value.name) || typeof value.id !== "string" ||
|
|
155
|
+
!value.id.startsWith("ctc_") || typeof value.call_id !== "string")
|
|
156
|
+
return false;
|
|
157
|
+
if (state.x && GROK_NATIVE_X_EXTENSION_NAMES.has(value.name) &&
|
|
158
|
+
value.call_id.startsWith("xs_call-"))
|
|
159
|
+
return true;
|
|
160
|
+
return state.web && GROK_NATIVE_WEB_EXTENSION_NAMES.has(value.name) &&
|
|
161
|
+
(value.call_id.startsWith("ws_call-") || value.call_id.startsWith("web_search_call-"));
|
|
162
|
+
}
|
|
46
163
|
export function grokWireTools(tools, state) {
|
|
47
164
|
const nativeTypes = new Set(["web_search", "x_search"]);
|
|
48
165
|
const result = (tools ?? [])
|
|
@@ -75,37 +192,10 @@ function validVisibleItem(item) {
|
|
|
75
192
|
typeof item.name === "string" && typeof item.input === "string";
|
|
76
193
|
return false;
|
|
77
194
|
}
|
|
78
|
-
function textSha256(text) {
|
|
79
|
-
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
80
|
-
}
|
|
81
|
-
function lcxCitationAddition(item) {
|
|
82
|
-
if (item.type !== "message" ||
|
|
83
|
-
typeof item.id !== "string" ||
|
|
84
|
-
!item.id.startsWith("msg_lcx_sources_"))
|
|
85
|
-
return undefined;
|
|
86
|
-
const text = itemText(item);
|
|
87
|
-
return text.startsWith("\n\nSources:\n")
|
|
88
|
-
? { kind: "citation-sources", textSha256: textSha256(text) }
|
|
89
|
-
: undefined;
|
|
90
|
-
}
|
|
91
|
-
function validVisibleAdditions(value) {
|
|
92
|
-
if (!Array.isArray(value) || value.length > GROK_NATIVE_REPLAY_MAX_ITEMS)
|
|
93
|
-
return undefined;
|
|
94
|
-
const additions = [];
|
|
95
|
-
for (const candidate of value) {
|
|
96
|
-
if (!isRecord(candidate) ||
|
|
97
|
-
candidate.kind !== "citation-sources" ||
|
|
98
|
-
typeof candidate.textSha256 !== "string" ||
|
|
99
|
-
!/^[0-9a-f]{64}$/u.test(candidate.textSha256))
|
|
100
|
-
return undefined;
|
|
101
|
-
additions.push({ kind: "citation-sources", textSha256: candidate.textSha256 });
|
|
102
|
-
}
|
|
103
|
-
return additions;
|
|
104
|
-
}
|
|
105
195
|
function readGrokNativeReplayEnvelope(value, route) {
|
|
106
196
|
if (!isRecord(value) ||
|
|
107
197
|
value.kind !== "xai-responses-native-search" ||
|
|
108
|
-
|
|
198
|
+
value.version !== 3)
|
|
109
199
|
return undefined;
|
|
110
200
|
if (value.provider !== route.provider ||
|
|
111
201
|
value.model !== route.model ||
|
|
@@ -127,16 +217,16 @@ function readGrokNativeReplayEnvelope(value, route) {
|
|
|
127
217
|
const output = [];
|
|
128
218
|
const itemKinds = new Map();
|
|
129
219
|
const callIds = new Set();
|
|
130
|
-
|
|
220
|
+
if (value.serverSearchEchoCallIds !== undefined &&
|
|
221
|
+
(!Array.isArray(value.serverSearchEchoCallIds) ||
|
|
222
|
+
!value.serverSearchEchoCallIds.every((id) => typeof id === "string" && id.length > 0) ||
|
|
223
|
+
new Set(value.serverSearchEchoCallIds).size !== value.serverSearchEchoCallIds.length))
|
|
224
|
+
return undefined;
|
|
225
|
+
const serverSearchEchoCallIds = new Set(value.serverSearchEchoCallIds ?? []);
|
|
131
226
|
for (const candidate of value.output) {
|
|
132
227
|
if (!isRecord(candidate) || typeof candidate.type !== "string")
|
|
133
228
|
return undefined;
|
|
134
229
|
const item = candidate;
|
|
135
|
-
const legacyAddition = value.version === 1 ? lcxCitationAddition(item) : undefined;
|
|
136
|
-
if (legacyAddition) {
|
|
137
|
-
migratedAdditions.push(legacyAddition);
|
|
138
|
-
continue;
|
|
139
|
-
}
|
|
140
230
|
if (GROK_NATIVE_SERVER_TOOL_TYPES.has(item.type)) {
|
|
141
231
|
if (typeof item.id !== "string" || item.id.length === 0)
|
|
142
232
|
return undefined;
|
|
@@ -163,45 +253,45 @@ function readGrokNativeReplayEnvelope(value, route) {
|
|
|
163
253
|
}
|
|
164
254
|
if (output.length === 0)
|
|
165
255
|
return undefined;
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
if (!visibleAdditions)
|
|
170
|
-
return undefined;
|
|
256
|
+
for (const id of serverSearchEchoCallIds)
|
|
257
|
+
if (!output.some((item) => item.type === "custom_tool_call" && item.call_id === id))
|
|
258
|
+
return undefined;
|
|
171
259
|
return {
|
|
172
260
|
kind: "xai-responses-native-search",
|
|
173
|
-
version:
|
|
261
|
+
version: 3,
|
|
174
262
|
provider: route.provider,
|
|
175
263
|
model: route.model,
|
|
176
264
|
sourceSessionId: route.sessionId,
|
|
177
265
|
routeAuthorityFingerprint: grokNativeReplayRouteFingerprint(route),
|
|
266
|
+
...(serverSearchEchoCallIds.size > 0 ? { serverSearchEchoCallIds: [...serverSearchEchoCallIds] } : {}),
|
|
178
267
|
output,
|
|
179
|
-
visibleAdditions,
|
|
180
268
|
};
|
|
181
269
|
}
|
|
182
|
-
export function createGrokNativeReplayEnvelope(output, route,
|
|
270
|
+
export function createGrokNativeReplayEnvelope(output, route, serverSearchEchoCallIds = []) {
|
|
183
271
|
if (!route || !Array.isArray(output))
|
|
184
272
|
return undefined;
|
|
185
|
-
const additions = visibleAdditions.map((candidate) => isRecord(candidate) && typeof candidate.type === "string"
|
|
186
|
-
? lcxCitationAddition(candidate)
|
|
187
|
-
: undefined);
|
|
188
|
-
if (additions.some((addition) => addition === undefined))
|
|
189
|
-
return undefined;
|
|
190
273
|
return readGrokNativeReplayEnvelope({
|
|
191
274
|
kind: "xai-responses-native-search",
|
|
192
|
-
version:
|
|
275
|
+
version: 3,
|
|
193
276
|
provider: route.provider,
|
|
194
277
|
model: route.model,
|
|
195
278
|
sourceSessionId: route.sessionId,
|
|
196
279
|
routeAuthorityFingerprint: grokNativeReplayRouteFingerprint(route),
|
|
280
|
+
...(serverSearchEchoCallIds.length > 0 ? { serverSearchEchoCallIds: [...serverSearchEchoCallIds] } : {}),
|
|
197
281
|
output,
|
|
198
|
-
visibleAdditions: additions,
|
|
199
282
|
}, route);
|
|
200
283
|
}
|
|
201
|
-
function itemText(item) {
|
|
284
|
+
function itemText(item, visible = false) {
|
|
202
285
|
if (item.type === "message")
|
|
203
286
|
return (Array.isArray(item.content) ? item.content : [])
|
|
204
|
-
.map((part) =>
|
|
287
|
+
.map((part) => {
|
|
288
|
+
if (!isRecord(part))
|
|
289
|
+
return "";
|
|
290
|
+
const text = String(part.text ?? part.refusal ?? "");
|
|
291
|
+
return visible
|
|
292
|
+
? sanitizeGrokVisibleText(text, Array.isArray(part.annotations) ? part.annotations : [])
|
|
293
|
+
: text;
|
|
294
|
+
})
|
|
205
295
|
.join("");
|
|
206
296
|
if (item.type === "reasoning") {
|
|
207
297
|
const parts = Array.isArray(item.summary) && item.summary.length > 0
|
|
@@ -213,7 +303,7 @@ function itemText(item) {
|
|
|
213
303
|
}
|
|
214
304
|
function visibleItemMatchesBlock(item, block) {
|
|
215
305
|
if (item.type === "message")
|
|
216
|
-
return block.type === "text" && block.text === itemText(item);
|
|
306
|
+
return block.type === "text" && block.text === itemText(item, true);
|
|
217
307
|
if (item.type === "reasoning")
|
|
218
308
|
return block.type === "reasoning" && block.text === itemText(item);
|
|
219
309
|
if (item.type === "function_call")
|
|
@@ -233,19 +323,6 @@ function visibleItemMatchesInput(item, candidate) {
|
|
|
233
323
|
return candidate.call_id === item.call_id;
|
|
234
324
|
return true;
|
|
235
325
|
}
|
|
236
|
-
function additionMatchesBlock(addition, candidate) {
|
|
237
|
-
return isRecord(candidate) && candidate.type === "text" &&
|
|
238
|
-
typeof candidate.text === "string" &&
|
|
239
|
-
candidate.text.startsWith("\n\nSources:\n") &&
|
|
240
|
-
textSha256(candidate.text) === addition.textSha256;
|
|
241
|
-
}
|
|
242
|
-
function additionMatchesInput(addition, candidate) {
|
|
243
|
-
if (!isRecord(candidate) || candidate.type !== "message")
|
|
244
|
-
return false;
|
|
245
|
-
if (typeof candidate.id !== "string" || !candidate.id.startsWith("msg_lcx_sources_"))
|
|
246
|
-
return false;
|
|
247
|
-
return textSha256(itemText(candidate)) === addition.textSha256;
|
|
248
|
-
}
|
|
249
326
|
function replayForMessage(message, route) {
|
|
250
327
|
if (message.role !== "assistant" || message.source.kind !== "model")
|
|
251
328
|
return undefined;
|
|
@@ -257,7 +334,9 @@ function replayForMessage(message, route) {
|
|
|
257
334
|
const envelope = readGrokNativeReplayEnvelope(replayState.grokNative, route);
|
|
258
335
|
if (!envelope)
|
|
259
336
|
return undefined;
|
|
260
|
-
const
|
|
337
|
+
const serverSearchEchoCallIds = new Set(envelope.serverSearchEchoCallIds ?? []);
|
|
338
|
+
const visible = envelope.output.filter((item) => !GROK_NATIVE_SERVER_TOOL_TYPES.has(item.type) &&
|
|
339
|
+
!(item.type === "custom_tool_call" && serverSearchEchoCallIds.has(String(item.call_id))));
|
|
261
340
|
if (visible.length === 0)
|
|
262
341
|
return undefined;
|
|
263
342
|
const reasoningCounts = new Map();
|
|
@@ -272,22 +351,18 @@ function replayForMessage(message, route) {
|
|
|
272
351
|
anchors.push(item);
|
|
273
352
|
blockIndex += 1;
|
|
274
353
|
}
|
|
275
|
-
else if (!(item.type === "reasoning" &&
|
|
354
|
+
else if (!(item.type === "reasoning" &&
|
|
355
|
+
(!itemText(item).trim() || (reasoningCounts.get(String(item.id)) ?? 0) > 1))) {
|
|
276
356
|
return undefined;
|
|
277
357
|
}
|
|
278
358
|
}
|
|
279
|
-
// Pi can coalesce repeated reasoning IDs
|
|
280
|
-
//
|
|
359
|
+
// Empty Grok reasoning has no UI block. Pi can coalesce repeated reasoning IDs
|
|
360
|
+
// in terminal-only gateway responses; every actual DSH-visible block must still match.
|
|
281
361
|
if (anchors.length === 0)
|
|
282
362
|
return undefined;
|
|
283
|
-
for (const addition of envelope.visibleAdditions) {
|
|
284
|
-
if (!additionMatchesBlock(addition, message.content[blockIndex]))
|
|
285
|
-
return undefined;
|
|
286
|
-
blockIndex += 1;
|
|
287
|
-
}
|
|
288
363
|
if (blockIndex !== message.content.length)
|
|
289
364
|
return undefined;
|
|
290
|
-
return { envelope, visible: anchors
|
|
365
|
+
return { envelope, visible: anchors };
|
|
291
366
|
}
|
|
292
367
|
export function restoreGrokNativeReplay(input, messages, route) {
|
|
293
368
|
const restored = structuredClone(input);
|
|
@@ -298,12 +373,11 @@ export function restoreGrokNativeReplay(input, messages, route) {
|
|
|
298
373
|
const replay = replayForMessage(message, route);
|
|
299
374
|
if (!replay)
|
|
300
375
|
continue;
|
|
301
|
-
const { envelope, visible
|
|
302
|
-
const replacedLength = visible.length
|
|
376
|
+
const { envelope, visible } = replay;
|
|
377
|
+
const replacedLength = visible.length;
|
|
303
378
|
let start = -1;
|
|
304
379
|
for (let index = cursor; index <= restored.length - replacedLength; index += 1) {
|
|
305
|
-
if (visible.every((item, offset) => visibleItemMatchesInput(item, restored[index + offset]))
|
|
306
|
-
additions.every((addition, offset) => additionMatchesInput(addition, restored[index + visible.length + offset]))) {
|
|
380
|
+
if (visible.every((item, offset) => visibleItemMatchesInput(item, restored[index + offset]))) {
|
|
307
381
|
start = index;
|
|
308
382
|
break;
|
|
309
383
|
}
|