@bitkyc08/opencodex 2.28.0 → 2.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/gui/dist/assets/index-BNESwCzn.js +102 -0
- package/gui/dist/assets/index-CH7ncHCC.css +1 -0
- package/gui/dist/index.html +2 -2
- package/package.json +1 -1
- package/src/adapters/base.ts +3 -1
- package/src/adapters/cursor/checkpoint-store.ts +303 -0
- package/src/adapters/cursor/discovery.ts +25 -0
- package/src/adapters/cursor/live-transport.ts +19 -2
- package/src/adapters/cursor/native-exec.ts +47 -2
- package/src/adapters/cursor/protobuf-request.ts +80 -23
- package/src/adapters/cursor/request-builder.ts +119 -5
- package/src/adapters/cursor/transport.ts +5 -0
- package/src/adapters/cursor/types.ts +13 -0
- package/src/adapters/cursor.ts +109 -3
- package/src/adapters/google-antigravity-replay.ts +31 -4
- package/src/adapters/google.ts +323 -38
- package/src/adapters/openai-chat.ts +19 -0
- package/src/adapters/openai-responses.ts +218 -37
- package/src/bridge.ts +38 -20
- package/src/claude/desktop-3p.ts +15 -6
- package/src/cli/agent.ts +44 -1
- package/src/cli/claude-agent-startup-sync.ts +73 -0
- package/src/cli/dispatch.ts +24 -1
- package/src/cli/ensure-desired-integrations.ts +152 -0
- package/src/cli/help.ts +1 -1
- package/src/cli/index.ts +40 -38
- package/src/cli/integrations.ts +1 -1
- package/src/cli/registry.ts +2 -2
- package/src/clients/config-export.ts +119 -20
- package/src/codex/affinity-debug.ts +162 -0
- package/src/codex/inject.ts +46 -14
- package/src/codex/journal.ts +22 -8
- package/src/config.ts +1 -0
- package/src/generated/compatibility-version.json +134 -66
- package/src/integrations/mutation-flight.ts +71 -0
- package/src/integrations/owned-refresh.ts +74 -0
- package/src/integrations/registry.ts +25 -2
- package/src/integrations/writer.ts +32 -1
- package/src/lab/public/signature.ts +25 -1
- package/src/lab/subject/behavior-fingerprint.ts +1 -1
- package/src/lib/redact.ts +2 -2
- package/src/oauth/log.ts +3 -1
- package/src/providers/derive.ts +9 -0
- package/src/providers/fastwire.ts +24 -18
- package/src/providers/openai-tiers.ts +60 -1
- package/src/providers/registry.ts +14 -7
- package/src/providers/xai-responses-opt-in.ts +15 -0
- package/src/responses/compaction.ts +18 -0
- package/src/responses/custom-tool-compat.ts +70 -5
- package/src/responses/namespace-tool-compat.ts +356 -0
- package/src/responses/parser.ts +2 -2
- package/src/responses/provider-continuation.ts +98 -0
- package/src/responses/reasoning-replay-cache.ts +125 -7
- package/src/responses/spill-store.ts +6 -1
- package/src/responses/state.ts +11 -0
- package/src/router.ts +14 -0
- package/src/routing/compatibility/behavior.ts +1 -0
- package/src/server/auth-cors.ts +4 -0
- package/src/server/management/agent-settings-routes.ts +57 -9
- package/src/server/management/config-routes.ts +134 -15
- package/src/server/management/integration-routes.ts +8 -55
- package/src/server/management/model-routes.ts +23 -1
- package/src/server/management/provider-routes.ts +22 -0
- package/src/server/management/vision-sidecar-options.ts +18 -11
- package/src/server/management/web-search-sidecar-options.ts +120 -0
- package/src/server/responses/core.ts +588 -78
- package/src/server/responses/responses-field-backfill.ts +7 -11
- package/src/server/responses/terminal-guard.ts +22 -11
- package/src/server/responses-custom-tool-repair.ts +3 -1
- package/src/server/responses-reasoning-summary-rewrite.ts +7 -0
- package/src/server/responses-tool-search-repair.ts +64 -14
- package/src/sidecar/auth.ts +92 -0
- package/src/sidecar/candidates.ts +83 -0
- package/src/types/config.ts +26 -5
- package/src/types/provider.ts +18 -0
- package/src/types/request.ts +26 -0
- package/src/types.ts +1 -0
- package/src/usage/log.ts +2 -0
- package/src/vision/index.ts +8 -9
- package/src/web-search/backends.ts +108 -0
- package/src/web-search/exa-executor.ts +88 -0
- package/src/web-search/gemini-executor.ts +141 -0
- package/src/web-search/index.ts +139 -12
- package/src/web-search/loop.ts +45 -5
- package/src/web-search/parse.ts +34 -23
- package/src/web-search/sources.ts +60 -0
- package/src/web-search/xai-executor.ts +219 -0
- package/gui/dist/assets/index-D2sP-biU.js +0 -102
- package/gui/dist/assets/index-DQsMZzI5.css +0 -1
package/src/web-search/loop.ts
CHANGED
|
@@ -6,6 +6,10 @@ import type { AttemptRecoveryKind } from "../usage/log";
|
|
|
6
6
|
import { bridgeToResponsesSSE } from "../bridge";
|
|
7
7
|
import { runWebSearch, type SidecarOutcome, type SidecarOutcomeRecorder, type SidecarSettings } from "./executor";
|
|
8
8
|
import { runAnthropicWebSearch } from "./anthropic-executor";
|
|
9
|
+
import { runXaiWebSearch, type XaiSearchOptions } from "./xai-executor";
|
|
10
|
+
import { runGeminiWebSearch } from "./gemini-executor";
|
|
11
|
+
import { runExaWebSearch } from "./exa-executor";
|
|
12
|
+
import type { WebSearchBackendId } from "./index";
|
|
9
13
|
import { clearableDeadline } from "../lib/abort";
|
|
10
14
|
import { redactSecretString } from "../lib/redact";
|
|
11
15
|
import { readBoundedResponseBody } from "../lib/bounded-body";
|
|
@@ -250,12 +254,24 @@ export interface WebSearchLoopDeps {
|
|
|
250
254
|
parsed: OcxParsedRequest;
|
|
251
255
|
adapter: ProviderAdapter;
|
|
252
256
|
incomingMeta: IncomingMeta;
|
|
253
|
-
/**
|
|
254
|
-
|
|
257
|
+
/**
|
|
258
|
+
* Which executor runs searches. Defaults to "openai" so existing callers keep the ChatGPT path
|
|
259
|
+
* (audit F4). The widened ids (xai/gemini/exa) cannot reach the loop yet: planWebSearch returns
|
|
260
|
+
* no plan for them (inert 060 arms), and the dispatch below only branches on "anthropic".
|
|
261
|
+
*/
|
|
262
|
+
backend?: WebSearchBackendId;
|
|
255
263
|
/** Required for the openai backend; unused (and typically undefined) for the anthropic backend. */
|
|
256
264
|
forwardProvider?: OcxProviderConfig;
|
|
257
265
|
/** Required for the anthropic backend: the stored-OAuth provider that runs web_search_20250305. */
|
|
258
266
|
anthropicSidecar?: { providerName: string; provider: OcxProviderConfig };
|
|
267
|
+
/** Required for the xai backend: the stored Grok OAuth provider (L7). */
|
|
268
|
+
xaiSidecar?: { providerName: string; provider: OcxProviderConfig };
|
|
269
|
+
/** Required for the gemini backend: the stored Antigravity CCA provider (L8). */
|
|
270
|
+
geminiSidecar?: { providerName: string; provider: OcxProviderConfig };
|
|
271
|
+
/** Required for the exa backend: the operator key, read from config at plan unpack (L9). */
|
|
272
|
+
exaApiKey?: string;
|
|
273
|
+
/** Opt-in x_search options for the xai backend. */
|
|
274
|
+
xaiSearchOptions?: XaiSearchOptions;
|
|
259
275
|
hostedTool: Record<string, unknown>;
|
|
260
276
|
selectedForwardHeaders: Headers;
|
|
261
277
|
settings: SidecarSettings;
|
|
@@ -294,6 +310,8 @@ export interface WebSearchLoopDeps {
|
|
|
294
310
|
on429?: (retryAfterHeader: string | null) => ProviderAdapter | null;
|
|
295
311
|
/** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */
|
|
296
312
|
retryOn429Policy?: Required<RateLimitRetryPolicy> | null;
|
|
313
|
+
/** Called only when the final bridged Responses stream reaches completed or incomplete. */
|
|
314
|
+
onCompletedResponse?: (response: Record<string, unknown>) => void;
|
|
297
315
|
}
|
|
298
316
|
|
|
299
317
|
/**
|
|
@@ -656,9 +674,30 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
|
|
|
656
674
|
// signal.aborted both after the await and in the catch (a fulfilled {error} on an aborted
|
|
657
675
|
// signal would otherwise look like an ordinary degradable failure).
|
|
658
676
|
try {
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
677
|
+
if (backend === "anthropic" && anthropicSidecar) {
|
|
678
|
+
outcome = await runAnthropicWebSearch(query, anthropicSidecar.providerName, anthropicSidecar.provider, settings, signal);
|
|
679
|
+
} else if (backend === "xai") {
|
|
680
|
+
// L7: stored Grok OAuth to the pinned api.x.ai Responses endpoint; same
|
|
681
|
+
// never-throws contract and no Codex/OpenAI pool outcome recording (F5 parity).
|
|
682
|
+
// A missing xaiSidecar is an invariant violation — fail CLOSED with an error
|
|
683
|
+
// outcome rather than falling through to the forward-header OpenAI executor
|
|
684
|
+
// (review High: that fallthrough would be credential-sensitive).
|
|
685
|
+
outcome = deps.xaiSidecar
|
|
686
|
+
? await runXaiWebSearch(query, deps.xaiSidecar.providerName, deps.xaiSidecar.provider, settings, deps.xaiSearchOptions ?? {}, signal)
|
|
687
|
+
: { text: "", sources: [], error: "xai backend selected without a resolved Grok OAuth provider" };
|
|
688
|
+
} else if (backend === "gemini") {
|
|
689
|
+
// L8: Antigravity CCA grounding; same fail-closed invariant stance as xai.
|
|
690
|
+
outcome = deps.geminiSidecar
|
|
691
|
+
? await runGeminiWebSearch(query, deps.geminiSidecar.providerName, deps.geminiSidecar.provider, settings, signal)
|
|
692
|
+
: { text: "", sources: [], error: "gemini backend selected without a resolved Antigravity provider" };
|
|
693
|
+
} else if (backend === "exa") {
|
|
694
|
+
// L9: non-LLM lane; key comes from the loop deps, never the plan. Fail closed.
|
|
695
|
+
outcome = deps.exaApiKey
|
|
696
|
+
? await runExaWebSearch(query, deps.exaApiKey, settings, signal)
|
|
697
|
+
: { text: "", sources: [], error: "exa backend selected without an exaApiKey" };
|
|
698
|
+
} else {
|
|
699
|
+
outcome = await runWebSearch(query, hostedTool, forwardProvider!, selectedForwardHeaders, settings, signal, recordSidecarOutcome);
|
|
700
|
+
}
|
|
662
701
|
if (signal.aborted) throw new LoopError(499, "client closed request during web-search");
|
|
663
702
|
} catch (e) {
|
|
664
703
|
if (e instanceof LoopError) throw e;
|
|
@@ -847,6 +886,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise<Respons
|
|
|
847
886
|
...(deps.stallTimeoutSec !== undefined ? { stallTimeoutSec: deps.stallTimeoutSec } : {}),
|
|
848
887
|
...(deps.onFirstOutput ? { onFirstOutput: deps.onFirstOutput } : {}),
|
|
849
888
|
...(deps.onUsage ? { onUsage: deps.onUsage } : {}),
|
|
889
|
+
...(deps.onCompletedResponse ? { onCompletedResponse: deps.onCompletedResponse } : {}),
|
|
850
890
|
},
|
|
851
891
|
);
|
|
852
892
|
return new Response(sse, { headers: SSE_HEADERS });
|
package/src/web-search/parse.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { sseFieldValue } from "../lib/sse-decoder";
|
|
2
|
+
import {
|
|
3
|
+
appendSafeWebSearchSource,
|
|
4
|
+
safeWebSearchSources,
|
|
5
|
+
type SafeWebSearchSource,
|
|
6
|
+
} from "./sources";
|
|
2
7
|
|
|
3
8
|
/** A single web source backing the sidecar's answer. */
|
|
4
|
-
export
|
|
5
|
-
url: string;
|
|
6
|
-
title?: string;
|
|
7
|
-
}
|
|
9
|
+
export type WebSearchSource = SafeWebSearchSource;
|
|
8
10
|
|
|
9
11
|
/** The sidecar's synthesized answer plus its sources (empty `sources` is fine). */
|
|
10
12
|
export interface WebSearchResult {
|
|
@@ -36,8 +38,10 @@ export const MAX_SIDECAR_RESPONSE_BYTES = 64 * 1024;
|
|
|
36
38
|
/** Push a `url_citation` annotation as a source, de-duplicated by URL. */
|
|
37
39
|
function collectAnnotation(ann: AnnotationLike | undefined, sources: WebSearchSource[], seen: Set<string>): void {
|
|
38
40
|
if (!ann || ann.type !== "url_citation" || typeof ann.url !== "string" || seen.has(ann.url)) return;
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
if (appendSafeWebSearchSource(sources, {
|
|
42
|
+
url: ann.url,
|
|
43
|
+
...(ann.title !== undefined ? { title: ann.title } : {}),
|
|
44
|
+
})) seen.add(ann.url);
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
/**
|
|
@@ -51,7 +55,11 @@ function collectAnnotation(ann: AnnotationLike | undefined, sources: WebSearchSo
|
|
|
51
55
|
* (`### Sources:`, `**Sources**`), a title line whose URL sits on the FOLLOWING line, and trailing
|
|
52
56
|
* URL punctuation (`;`, `,`, `)`, `]`, `.`). Prose that follows the source list is preserved.
|
|
53
57
|
*/
|
|
54
|
-
const URL_RE = /https?:\/\/[^\s<>()\[\]]
|
|
58
|
+
const URL_RE = /https?:\/\/[^\s<>()\[\]]+/i;
|
|
59
|
+
// Recognize URI-like candidates separately from the HTTP(S)-only acceptance boundary. A rejected
|
|
60
|
+
// citation (for example `javascript:`) still belongs to the trailing Sources block and must not be
|
|
61
|
+
// left behind as ordinary assistant text.
|
|
62
|
+
const URI_LIKE_RE = /[a-z][a-z0-9+.-]*:[^\s<>()\[\]]+/i;
|
|
55
63
|
// A "Sources:" / "Source:" header, allowing markdown prefixes (#, *, -, >) and bold/italic wrappers.
|
|
56
64
|
const SOURCES_WORD_RE = /^sources?/i;
|
|
57
65
|
|
|
@@ -113,14 +121,14 @@ function cleanTitle(prefix: string): string {
|
|
|
113
121
|
return title;
|
|
114
122
|
}
|
|
115
123
|
|
|
116
|
-
function extractTrailingSources(text: string): { text: string; sources: WebSearchSource[] } {
|
|
124
|
+
function extractTrailingSources(text: string): { text: string; sources: WebSearchSource[]; stripped: boolean } {
|
|
117
125
|
const lines = text.split("\n");
|
|
118
126
|
// Find the LAST line that is a "Sources:" header (markdown prefixes allowed).
|
|
119
127
|
let headerIdx = -1;
|
|
120
128
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
121
129
|
if (isSourcesHeader(lines[i])) { headerIdx = i; break; }
|
|
122
130
|
}
|
|
123
|
-
if (headerIdx === -1) return { text, sources: [] };
|
|
131
|
+
if (headerIdx === -1) return { text, sources: [], stripped: false };
|
|
124
132
|
const sources: WebSearchSource[] = [];
|
|
125
133
|
const seen = new Set<string>();
|
|
126
134
|
// Track the last line index actually consumed as part of the source list so trailing prose after
|
|
@@ -128,14 +136,16 @@ function extractTrailingSources(text: string): { text: string; sources: WebSearc
|
|
|
128
136
|
let lastConsumed = headerIdx;
|
|
129
137
|
// A title line whose URL is expected on a following line (multiline entry).
|
|
130
138
|
let pendingTitle: string | null = null;
|
|
139
|
+
let consumedSourceLine = false;
|
|
131
140
|
for (let i = headerIdx + 1; i < lines.length; i++) {
|
|
132
141
|
const raw = lines[i].trim();
|
|
133
142
|
if (raw === "") {
|
|
134
143
|
// Blank line between header and first entry is fine; a blank AFTER entries ends the list.
|
|
135
|
-
if (
|
|
144
|
+
if (consumedSourceLine || pendingTitle !== null) break;
|
|
136
145
|
continue;
|
|
137
146
|
}
|
|
138
|
-
const
|
|
147
|
+
const httpMatch = raw.match(URL_RE);
|
|
148
|
+
const m = httpMatch ?? raw.match(URI_LIKE_RE);
|
|
139
149
|
if (!m) {
|
|
140
150
|
// A list-ish line with no URL may be a title whose URL is on the next line. Only treat it as a
|
|
141
151
|
// pending title when it looks like a list item; otherwise it's prose → stop.
|
|
@@ -144,23 +154,27 @@ function extractTrailingSources(text: string): { text: string; sources: WebSearc
|
|
|
144
154
|
}
|
|
145
155
|
break;
|
|
146
156
|
}
|
|
157
|
+
// A non-HTTP URI embedded in prose is not sufficient to classify the line as a citation.
|
|
158
|
+
// Accept it as a consumed source line only when it is a list item or the whole line starts with
|
|
159
|
+
// the URI candidate, matching the existing bare-URL grammar.
|
|
160
|
+
if (!httpMatch && !/^[-*>\d.)]/.test(raw) && m.index !== 0) break;
|
|
147
161
|
const url = cleanUrl(m[0]);
|
|
148
162
|
if (!url) { break; }
|
|
163
|
+
consumedSourceLine = true;
|
|
149
164
|
lastConsumed = i;
|
|
150
165
|
// Title: text before the URL on this line, else a buffered title from a preceding line.
|
|
151
166
|
const inlinePrefix = raw.slice(0, m.index);
|
|
152
167
|
const title = cleanTitle(inlinePrefix) || (pendingTitle ? cleanTitle(pendingTitle) : "");
|
|
153
168
|
pendingTitle = null;
|
|
154
169
|
if (seen.has(url)) continue;
|
|
155
|
-
seen.add(url);
|
|
156
|
-
sources.push(title ? { url, title } : { url });
|
|
170
|
+
if (appendSafeWebSearchSource(sources, title ? { url, title } : { url })) seen.add(url);
|
|
157
171
|
}
|
|
158
|
-
if (
|
|
172
|
+
if (!consumedSourceLine) return { text, sources: [], stripped: false };
|
|
159
173
|
// Keep text before the header AND any prose after the consumed source lines.
|
|
160
174
|
const before = lines.slice(0, headerIdx).join("\n").replace(/\s+$/, "");
|
|
161
175
|
const after = lines.slice(lastConsumed + 1).join("\n").replace(/^\s+/, "");
|
|
162
176
|
const body = after ? (before ? `${before}\n\n${after}` : after) : before;
|
|
163
|
-
return { text: body, sources };
|
|
177
|
+
return { text: body, sources, stripped: true };
|
|
164
178
|
}
|
|
165
179
|
|
|
166
180
|
/** Pull final text + url_citation sources from a completed Responses `output[]` array. */
|
|
@@ -283,22 +297,19 @@ export async function parseSidecarSSE(response: Response): Promise<WebSearchResu
|
|
|
283
297
|
|| acc.doneText.trim() && acc.doneText
|
|
284
298
|
|| acc.deltaText;
|
|
285
299
|
// Merge sources from the final output[] and the streaming annotation events.
|
|
286
|
-
const sources =
|
|
287
|
-
const seenMerge = new Set(sources.map(s => s.url));
|
|
300
|
+
const sources = safeWebSearchSources(acc.final?.sources ?? []);
|
|
288
301
|
for (const s of acc.streamSources) {
|
|
289
|
-
|
|
302
|
+
appendSafeWebSearchSource(sources, s);
|
|
290
303
|
}
|
|
291
304
|
// Hosted web_search usually omits url_citation annotations and lists sources in a trailing
|
|
292
305
|
// `Sources:` markdown block instead. Pull those out (and strip the block from the answer so the
|
|
293
306
|
// tool_result renderer doesn't print sources twice). Annotation titles win; text-block titles
|
|
294
307
|
// only fill a gap. URL-deduped against annotation sources.
|
|
295
|
-
const { text: body, sources: textSources } = extractTrailingSources(typeof text === "string" ? text : "");
|
|
308
|
+
const { text: body, sources: textSources, stripped } = extractTrailingSources(typeof text === "string" ? text : "");
|
|
296
309
|
for (const s of textSources) {
|
|
297
|
-
|
|
298
|
-
seenMerge.add(s.url);
|
|
299
|
-
sources.push(s);
|
|
310
|
+
appendSafeWebSearchSource(sources, s);
|
|
300
311
|
}
|
|
301
|
-
const finalText =
|
|
312
|
+
const finalText = stripped ? body : (typeof text === "string" ? text : "");
|
|
302
313
|
if (!finalText.trim() && acc.error) return { text: "", sources, error: acc.error };
|
|
303
314
|
return { text: finalText, sources };
|
|
304
315
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
export interface SafeWebSearchSource {
|
|
2
|
+
url: string;
|
|
3
|
+
title?: string;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export const MAX_WEB_SEARCH_SOURCES = 20;
|
|
7
|
+
export const MAX_WEB_SEARCH_URL_BYTES = 2_048;
|
|
8
|
+
export const MAX_WEB_SEARCH_TITLE_BYTES = 256;
|
|
9
|
+
export const MAX_WEB_SEARCH_SOURCE_BYTES = 16_384;
|
|
10
|
+
|
|
11
|
+
const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u;
|
|
12
|
+
const encoder = new TextEncoder();
|
|
13
|
+
|
|
14
|
+
function byteLength(value: string): number {
|
|
15
|
+
return encoder.encode(value).byteLength;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function serializedSourceBytes(source: SafeWebSearchSource): number {
|
|
19
|
+
return byteLength(JSON.stringify(source));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function safeTitle(value: unknown): string | undefined {
|
|
23
|
+
if (typeof value !== "string"
|
|
24
|
+
|| value.trim().length === 0
|
|
25
|
+
|| CONTROL_CHARACTERS.test(value)
|
|
26
|
+
|| byteLength(value) > MAX_WEB_SEARCH_TITLE_BYTES) return undefined;
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Add one client-safe citation while enforcing per-message count and byte budgets. */
|
|
31
|
+
export function appendSafeWebSearchSource(target: SafeWebSearchSource[], source: unknown): boolean {
|
|
32
|
+
if (target.length >= MAX_WEB_SEARCH_SOURCES || !source || typeof source !== "object") return false;
|
|
33
|
+
const candidate = source as { url?: unknown; title?: unknown };
|
|
34
|
+
if (typeof candidate.url !== "string"
|
|
35
|
+
|| candidate.url.length === 0
|
|
36
|
+
|| candidate.url.trim() !== candidate.url
|
|
37
|
+
|| CONTROL_CHARACTERS.test(candidate.url)
|
|
38
|
+
|| byteLength(candidate.url) > MAX_WEB_SEARCH_URL_BYTES
|
|
39
|
+
|| target.some(existing => existing.url === candidate.url)) return false;
|
|
40
|
+
|
|
41
|
+
let parsed: URL;
|
|
42
|
+
try { parsed = new URL(candidate.url); } catch { return false; }
|
|
43
|
+
if ((parsed.protocol !== "http:" && parsed.protocol !== "https:")
|
|
44
|
+
|| parsed.username.length > 0
|
|
45
|
+
|| parsed.password.length > 0) return false;
|
|
46
|
+
|
|
47
|
+
const title = safeTitle(candidate.title);
|
|
48
|
+
const next = title === undefined ? { url: candidate.url } : { url: candidate.url, title };
|
|
49
|
+
const usedBytes = target.reduce((sum, item) => sum + serializedSourceBytes(item), 0);
|
|
50
|
+
if (usedBytes + serializedSourceBytes(next) > MAX_WEB_SEARCH_SOURCE_BYTES) return false;
|
|
51
|
+
target.push(next);
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function safeWebSearchSources(sources: unknown): SafeWebSearchSource[] {
|
|
56
|
+
if (!Array.isArray(sources)) return [];
|
|
57
|
+
const safe: SafeWebSearchSource[] = [];
|
|
58
|
+
for (const source of sources) appendSafeWebSearchSource(safe, source);
|
|
59
|
+
return safe;
|
|
60
|
+
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Execute ONE web search via a Grok sidecar through the STORED xai OAuth credential (#2188 L7).
|
|
3
|
+
*
|
|
4
|
+
* POSTs the canonical api.x.ai Responses endpoint with the hosted web_search tool (plus the
|
|
5
|
+
* opt-in x_search tool) and reduces the SSE stream to a SidecarOutcome. Wire shapes are
|
|
6
|
+
* probe-verified (devlog 003 / 070): reasoning.effort is accepted alongside tools; both hosted
|
|
7
|
+
* tools may share one request; annotations arrive as url_citation events on message items;
|
|
8
|
+
* x_search activity surfaces as custom_tool_call items (ctc_) rather than the documented
|
|
9
|
+
* x_search_call — both are tolerated, neither is required. `action` may be absent on
|
|
10
|
+
* output_item.added (skeleton-first) and fills in later. Never throws — returns `{error}` so
|
|
11
|
+
* the caller injects a graceful tool result.
|
|
12
|
+
*/
|
|
13
|
+
import type { OcxProviderConfig } from "../types";
|
|
14
|
+
import { getValidAccessToken, publicOAuthAuthenticationErrorMessage } from "../oauth";
|
|
15
|
+
import { fetchWithResetRetry } from "../lib/upstream-retry";
|
|
16
|
+
import { cancelBodyOnAbort, signalWithTimeout } from "../lib/abort";
|
|
17
|
+
import { sidecarEnter } from "../lib/sidecar-tracker";
|
|
18
|
+
import { redactSecretString } from "../lib/redact";
|
|
19
|
+
import { MAX_SIDECAR_RESPONSE_BYTES, type WebSearchSource } from "./parse";
|
|
20
|
+
import { BASE_INSTRUCTION, IMAGE_INSTRUCTION, type SidecarOutcome, type SidecarSettings } from "./executor";
|
|
21
|
+
|
|
22
|
+
/** The one destination the OAuth credential may be sent to; provider.baseUrl is honored only on the same origin. */
|
|
23
|
+
const XAI_RESPONSES_ORIGIN = "https://api.x.ai";
|
|
24
|
+
|
|
25
|
+
export interface XaiSearchOptions {
|
|
26
|
+
/** Opt-in: add the hosted x_search tool next to web_search. */
|
|
27
|
+
xSearch?: boolean;
|
|
28
|
+
allowedXHandles?: string[];
|
|
29
|
+
excludedXHandles?: string[];
|
|
30
|
+
fromDate?: string;
|
|
31
|
+
toDate?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
35
|
+
|
|
36
|
+
/** Doc-validated limits (docs.x.ai x-search): <=20 handles, allow XOR exclude, ISO dates. */
|
|
37
|
+
export function validateXaiSearchOptions(options: XaiSearchOptions): string | undefined {
|
|
38
|
+
const allowed = options.allowedXHandles ?? [];
|
|
39
|
+
const excluded = options.excludedXHandles ?? [];
|
|
40
|
+
if (allowed.length > 0 && excluded.length > 0) return "allowedXHandles and excludedXHandles are mutually exclusive";
|
|
41
|
+
if (allowed.length > 20) return "allowedXHandles admits at most 20 handles";
|
|
42
|
+
if (excluded.length > 20) return "excludedXHandles admits at most 20 handles";
|
|
43
|
+
for (const [field, value] of [["fromDate", options.fromDate], ["toDate", options.toDate]] as const) {
|
|
44
|
+
if (value !== undefined && !ISO_DATE.test(value)) return `${field} must be an ISO-8601 date (YYYY-MM-DD)`;
|
|
45
|
+
}
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function buildXSearchTool(options: XaiSearchOptions): Record<string, unknown> {
|
|
50
|
+
return {
|
|
51
|
+
type: "x_search",
|
|
52
|
+
...(options.allowedXHandles?.length ? { allowed_x_handles: options.allowedXHandles } : {}),
|
|
53
|
+
...(options.excludedXHandles?.length ? { excluded_x_handles: options.excludedXHandles } : {}),
|
|
54
|
+
...(options.fromDate ? { from_date: options.fromDate } : {}),
|
|
55
|
+
...(options.toDate ? { to_date: options.toDate } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function isRec(v: unknown): v is Record<string, unknown> {
|
|
60
|
+
return !!v && typeof v === "object" && !Array.isArray(v);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function runXaiWebSearch(
|
|
64
|
+
query: string,
|
|
65
|
+
providerName: string,
|
|
66
|
+
provider: OcxProviderConfig,
|
|
67
|
+
settings: SidecarSettings,
|
|
68
|
+
options: XaiSearchOptions = {},
|
|
69
|
+
abortSignal?: AbortSignal,
|
|
70
|
+
): Promise<SidecarOutcome> {
|
|
71
|
+
const invalid = validateXaiSearchOptions(options);
|
|
72
|
+
if (invalid) return { text: "", sources: [], error: `xai sidecar options invalid: ${invalid}` };
|
|
73
|
+
let token: string;
|
|
74
|
+
try {
|
|
75
|
+
token = await getValidAccessToken(providerName);
|
|
76
|
+
} catch (e) {
|
|
77
|
+
return { text: "", sources: [], error: `xai sidecar auth failed: ${publicOAuthAuthenticationErrorMessage(e)}` };
|
|
78
|
+
}
|
|
79
|
+
// Credential pinning: only the EXACT api.x.ai origin may carry the OAuth bearer.
|
|
80
|
+
// A prefix check would admit https://api.x.ai.evil/ (review Critical); parse and
|
|
81
|
+
// compare origins, falling back to the canonical endpoint on any mismatch.
|
|
82
|
+
let base = `${XAI_RESPONSES_ORIGIN}/v1`;
|
|
83
|
+
if (provider.baseUrl) {
|
|
84
|
+
try {
|
|
85
|
+
const parsed = new URL(provider.baseUrl);
|
|
86
|
+
if (parsed.origin === XAI_RESPONSES_ORIGIN) base = provider.baseUrl.replace(/\/+$/, "");
|
|
87
|
+
} catch { /* malformed baseUrl: keep the canonical endpoint */ }
|
|
88
|
+
}
|
|
89
|
+
const url = `${base}/responses`;
|
|
90
|
+
const instruction = settings.describeImages ? BASE_INSTRUCTION + IMAGE_INSTRUCTION : BASE_INSTRUCTION;
|
|
91
|
+
const body = {
|
|
92
|
+
model: settings.model,
|
|
93
|
+
instructions: instruction,
|
|
94
|
+
input: [{ role: "user", content: query }],
|
|
95
|
+
tools: [{ type: "web_search" }, ...(options.xSearch ? [buildXSearchTool(options)] : [])],
|
|
96
|
+
include: ["web_search_call.action.sources"],
|
|
97
|
+
reasoning: { effort: settings.reasoning },
|
|
98
|
+
stream: true,
|
|
99
|
+
};
|
|
100
|
+
const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
|
|
101
|
+
const sidecarExit = sidecarEnter("web-search");
|
|
102
|
+
const t0 = Date.now();
|
|
103
|
+
try {
|
|
104
|
+
const res = await fetchWithResetRetry(
|
|
105
|
+
() => fetch(url, {
|
|
106
|
+
method: "POST",
|
|
107
|
+
headers: { "Content-Type": "application/json", "Authorization": `Bearer ${token}` },
|
|
108
|
+
body: JSON.stringify(body),
|
|
109
|
+
signal: linkedSignal.signal,
|
|
110
|
+
// Credential-bearing: never follow a redirect off the pinned origin.
|
|
111
|
+
redirect: "manual",
|
|
112
|
+
}),
|
|
113
|
+
{ abortSignal: linkedSignal.signal, label: "xai-web-search-sidecar" },
|
|
114
|
+
);
|
|
115
|
+
const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
|
|
116
|
+
if (!res.ok) {
|
|
117
|
+
const t = await res.text().catch(() => "");
|
|
118
|
+
detachBodyGuard();
|
|
119
|
+
const entitlement = res.status === 401 || res.status === 403 ? " (Grok OAuth entitlement — re-run ocx login xai?)" : "";
|
|
120
|
+
return { text: "", sources: [], error: `xai sidecar HTTP ${res.status}${entitlement}: ${redactSecretString(t.slice(0, 200))}` };
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
return await parseXaiResponsesSSE(res);
|
|
124
|
+
} finally {
|
|
125
|
+
detachBodyGuard();
|
|
126
|
+
}
|
|
127
|
+
} catch (e) {
|
|
128
|
+
const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
|
|
129
|
+
console.warn(`[web-search] xai sidecar ${kind} (${Date.now() - t0}ms)`);
|
|
130
|
+
return { text: "", sources: [], error: redactSecretString(e instanceof Error ? e.message : String(e)) };
|
|
131
|
+
} finally {
|
|
132
|
+
sidecarExit();
|
|
133
|
+
linkedSignal.cleanup();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Reduce a Grok Responses SSE stream to a SidecarOutcome (probe-shapes in devlog 003):
|
|
139
|
+
* text from output_text deltas on message items; sources from url_citation annotations
|
|
140
|
+
* unioned with web_search_call action.sources (deduped by url). Tolerates custom_tool_call
|
|
141
|
+
* and x_search_call items, absent `action`, and unknown event names. Bounds raw bytes.
|
|
142
|
+
*/
|
|
143
|
+
export async function parseXaiResponsesSSE(response: Response): Promise<SidecarOutcome> {
|
|
144
|
+
if (!response.body) return { text: "", sources: [] };
|
|
145
|
+
const reader = response.body.getReader();
|
|
146
|
+
const decoder = new TextDecoder();
|
|
147
|
+
let buffer = "";
|
|
148
|
+
let responseBytes = 0;
|
|
149
|
+
let text = "";
|
|
150
|
+
let doneText = "";
|
|
151
|
+
const sourceUrls = new Map<string, WebSearchSource>();
|
|
152
|
+
let error: string | null = null;
|
|
153
|
+
const addSource = (url: unknown, title?: unknown) => {
|
|
154
|
+
if (typeof url !== "string" || url.length === 0) return;
|
|
155
|
+
if (!sourceUrls.has(url)) sourceUrls.set(url, { url, ...(typeof title === "string" && title.length > 0 && title !== url ? { title } : {}) });
|
|
156
|
+
};
|
|
157
|
+
try {
|
|
158
|
+
while (true) {
|
|
159
|
+
const { done, value } = await reader.read();
|
|
160
|
+
if (done) break;
|
|
161
|
+
responseBytes += value.byteLength;
|
|
162
|
+
if (responseBytes > MAX_SIDECAR_RESPONSE_BYTES * 8) {
|
|
163
|
+
error = "xai sidecar stream exceeded byte bound";
|
|
164
|
+
await reader.cancel(error).catch(() => {});
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
buffer += decoder.decode(value, { stream: true });
|
|
168
|
+
let idx;
|
|
169
|
+
while ((idx = buffer.indexOf("\n\n")) >= 0) {
|
|
170
|
+
const frame = buffer.slice(0, idx);
|
|
171
|
+
buffer = buffer.slice(idx + 2);
|
|
172
|
+
const dataLine = frame.match(/^data: (.+)$/m)?.[1];
|
|
173
|
+
if (!dataLine || dataLine === "[DONE]") continue;
|
|
174
|
+
let payload: unknown;
|
|
175
|
+
try { payload = JSON.parse(dataLine); } catch { continue; }
|
|
176
|
+
if (!isRec(payload) || typeof payload.type !== "string") continue;
|
|
177
|
+
switch (payload.type) {
|
|
178
|
+
case "response.output_text.delta": {
|
|
179
|
+
if (typeof payload.delta === "string") text += payload.delta;
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
case "response.output_text.done": {
|
|
183
|
+
if (typeof payload.text === "string") doneText = payload.text;
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
case "response.output_text.annotation.added": {
|
|
187
|
+
const annotation = payload.annotation;
|
|
188
|
+
if (isRec(annotation) && annotation.type === "url_citation") addSource(annotation.url, annotation.title);
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
case "response.output_item.done": {
|
|
192
|
+
const item = payload.item;
|
|
193
|
+
if (isRec(item) && item.type === "web_search_call" && isRec(item.action) && Array.isArray(item.action.sources)) {
|
|
194
|
+
for (const s of item.action.sources) if (isRec(s)) addSource(s.url);
|
|
195
|
+
}
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
case "response.failed":
|
|
199
|
+
case "error": {
|
|
200
|
+
const message = isRec(payload.response) && isRec(payload.response.error) && typeof payload.response.error.message === "string"
|
|
201
|
+
? payload.response.error.message
|
|
202
|
+
: typeof payload.message === "string" ? payload.message : "upstream reported failure";
|
|
203
|
+
error = redactSecretString(String(message).slice(0, 200));
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
default: break;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
} catch (e) {
|
|
211
|
+
error = redactSecretString(e instanceof Error ? e.message : String(e));
|
|
212
|
+
} finally {
|
|
213
|
+
reader.releaseLock();
|
|
214
|
+
}
|
|
215
|
+
const finalText = doneText.length >= text.length ? doneText : text;
|
|
216
|
+
const outcome: SidecarOutcome = { text: finalText, sources: [...sourceUrls.values()] };
|
|
217
|
+
if (error && finalText.length === 0) outcome.error = error;
|
|
218
|
+
return outcome;
|
|
219
|
+
}
|