dsh-lcx-codex 0.4.3-pre.2 → 0.4.3-pre.3

package/lib/index.js

@@ -1,4 +1,5 @@
 import z from "@deepseek-ai/schemastery";
+import { WEB_SEARCH_MAX_QUERIES, applyWebSearchTool, } from "@deepseek-ai/dsh-tool-web";
 import "@deepseek-ai/dsh-agent";
 import "@deepseek-ai/dsh-tools";
 import "@deepseek-ai/dsh-web";
@@ -6,9 +7,10 @@ import { randomUUID } from "node:crypto";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { AsyncLocalStorage } from "node:async_hooks";
+import { getSupportedThinkingLevels } from "@earendil-works/pi-ai";
 import { fetchJsonWithRetry } from "./transport.js";
-import { agentSessionId, agentUsesSession, compactionConfigState, compactionPatchCandidate, installCompactionPatch, patchCompactionConfig, patchToolResultPruner, patchVisibleWebSearchTimeout, readAgentRouteState, readWebSearchProvider, refreshVisibleWebSearchTimeouts, resolveAgentService, resolveContextService, resolveScopedService, restoreCompactionConfig, restoreCompactionPatches, restoreToolResultPruner, restoreVisibleWebSearchTimeouts, sessionFor, sessionFromAgent, scopedToolRuntime, tokenMeterTotal, toolResultPrunerState, writeWebSearchProvider, } from "./dsh-compat.js";
-import { authenticatedHeaders, currentRoute, generationControlsFromSession, promptCacheKey, promptCacheRetention, promptCacheSessionId, resolveResponsesRouteConfig, routeFingerprint, } from "./route.js";
+import { agentSessionId, agentUsesSession, compactionConfigState, compactionPatchCandidate, installCompactionPatch, patchCompactionConfig, patchToolResultPruner, readAgentRouteState, resolveAgentService, resolveContextService, resolveScopedService, restoreCompactionConfig, restoreCompactionPatches, restoreToolResultPruner, sessionFor, sessionFromAgent, scopedToolRuntime, tokenMeterTotal, toolResultPrunerState, } from "./dsh-compat.js";
+import { authenticatedGrokHeaders, authenticatedHeaders, currentRoute, generationControlsFromSession, grokPromptCacheSessionId, promptCacheKey, promptCacheRetention, promptCacheSessionId, resolveGrokResponsesRouteConfig, resolveResponsesRouteConfig, routeFingerprint, } from "./route.js";
 import { DEFAULT_MAX_REQUEST_IMAGE_BYTES, DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET, DEFAULT_REQUEST_IMAGE_MAX_BYTES, hydrateNativeImageReferences, resolveModelImageSupport, responseInputItems, serializeDshMessages, } from "./dsh-responses.js";
 import { mergeFeatureHeader, requestNativeCompaction } from "./compact-v2.js";
 import { buildResponsesBody } from "./responses-request.js";
@@ -19,6 +21,7 @@ import { ALPHA_SCHEMA_FINGERPRINT, alphaRefRequiresStore, isAlphaContinuationUrl
 import { AlphaCapabilityStore, alphaCapabilityFingerprint, alphaCapabilityUsable, } from "./web-search-capability.js";
 import { AlphaRefStore } from "./web-search-ref-store.js";
 import { ServiceMutex } from "./service-mutex.js";
+import { GROK_NATIVE_SERVER_TOOL_TYPES, grokNativeSearchEnabled, grokVisibleFunctionTools, grokWireTools, restoreGrokNativeReplay, } from "./grok-native-search.js";
 export const name = "lcx-codex";
 export const inject = ["llm", "web", "sessions", "tools", "settings", "credentials", "attachments", "fs"];
 const SETTINGS_NS = "lcx-codex";
@@ -53,6 +56,13 @@ function routeWithPolicies(route, config) {
         webMaxResults: config.webMaxResults,
     };
 }
+function grokReplayRoute(route, config) {
+    return {
+        ...route,
+        apiKeyEnv: config.apiKeyEnv,
+        headers: config.headers,
+    };
+}
 function isRecord(value) {
     return value !== null && typeof value === "object";
 }
@@ -106,6 +116,8 @@ const SettingsSchema = z.object({
     webSearch: z.boolean().default(false),
     advancedHostedSearch: z.boolean().default(false),
     alphaSearch: z.boolean().default(false),
+    grokNativeWebSearch: z.boolean().default(false),
+    grokNativeXSearch: z.boolean().default(false),
 });
 function normalizeConfig(input = {}) {
     return {
@@ -156,10 +168,10 @@ function activeAgentRoute(exec) {
 function selectedAgentRoute(agent) {
     const route = readAgentRouteState(agent);
     return {
-        provider: stringValue(routeConfigValue(route.options, "provider") ??
-            routeConfigValue(route.requestConfig, "provider"), ""),
-        model: stringValue(routeConfigValue(route.options, "model") ??
-            routeConfigValue(route.requestConfig, "model"), ""),
+        provider: stringValue(routeConfigValue(route.requestConfig, "provider") ??
+            routeConfigValue(route.options, "provider"), ""),
+        model: stringValue(routeConfigValue(route.requestConfig, "model") ??
+            routeConfigValue(route.options, "model"), ""),
         sessionId: route.sessionId,
     };
 }
@@ -228,6 +240,40 @@ class LcxResponsesSearchProvider {
         };
     }
 }
+function createScopedGptWebSearchTool(ctx, state, getConfig, provider, fetchEnabled) {
+    let definition;
+    const composition = {
+        web: {
+            search: (request, signal) => provider.search(request, signal),
+        },
+        tools: {
+            register(candidate) {
+                definition = candidate;
+                return () => { };
+            },
+        },
+        systemPrompt: {
+            getSectionOrder: () => 0,
+            section: () => () => { },
+        },
+    };
+    applyWebSearchTool(composition, getConfig().webMaxResults, WEB_SEARCH_MAX_QUERIES, WEB_SEARCH_TIMEOUT_MS, fetchEnabled);
+    if (!definition)
+        throw new Error("DSH web_search composition did not register a tool");
+    const execute = definition.execute.bind(definition);
+    return {
+        ...definition,
+        async execute(args, exec) {
+            if (!state.enabled || !state.webSearch)
+                throw webError("web_search GPT Hosted Search is disabled", "LCX_WEB_DISABLED");
+            const active = activeAgentRoute(exec);
+            const route = resolveResponsesRouteConfig(ctx, active, getConfig());
+            if (!route)
+                throw webError("GPT Hosted Search requires the active GPT openai-responses route", "LCX_WEB_ROUTE_UNAVAILABLE");
+            return hostedSearchRouteContext.run(active, () => execute(args, exec));
+        },
+    };
+}
 function createAdvancedHostedTool(ctx, state, getConfig) {
     return {
         name: ADVANCED_HOSTED_TOOL,
@@ -249,6 +295,61 @@ function createAdvancedHostedTool(ctx, state, getConfig) {
         },
     };
 }
+function disposeGptToolsForAgent(agent, registrations) {
+    const registration = registrations.get(agent);
+    if (!registration)
+        return;
+    registration.dispose();
+    registrations.delete(agent);
+}
+function syncGptToolsForAgent(ctx, agent, state, getConfig, provider, registrations) {
+    const config = getConfig();
+    const active = selectedAgentRoute(agent);
+    const route = resolveResponsesRouteConfig(ctx, active, config);
+    const webSearch = state.enabled && state.webSearch && Boolean(route);
+    const advanced = webSearch && state.advancedHostedSearch;
+    if (!webSearch && !advanced) {
+        disposeGptToolsForAgent(agent, registrations);
+        return false;
+    }
+    const fingerprint = JSON.stringify({
+        provider: route?.provider,
+        model: route?.model,
+        baseURL: route?.baseURL,
+        webSearch,
+        advanced,
+        webMaxResults: config.webMaxResults,
+    });
+    if (registrations.get(agent)?.fingerprint === fingerprint)
+        return true;
+    disposeGptToolsForAgent(agent, registrations);
+    const scopedTools = scopedToolRuntime(agent);
+    if (!scopedTools)
+        return false;
+    const disposers = [];
+    try {
+        if (webSearch) {
+            const fetchEnabled = Boolean(scopedTools.get?.("web_fetch", agent));
+            disposers.push(scopedTools.register(createScopedGptWebSearchTool(ctx, state, getConfig, provider, fetchEnabled)));
+        }
+        if (advanced)
+            disposers.push(scopedTools.register(createAdvancedHostedTool(ctx, state, getConfig)));
+        registrations.set(agent, {
+            fingerprint,
+            dispose() {
+                for (const dispose of disposers.reverse())
+                    dispose();
+            },
+        });
+        return true;
+    }
+    catch (error) {
+        for (const dispose of disposers.reverse())
+            dispose();
+        ctx.logger?.warn?.(`[lcx-codex] GPT search tool composition unavailable: ${errorDetails(error).message ?? String(error)}`);
+        return false;
+    }
+}
 function alphaCapabilityFor(config, store) {
     const fingerprint = alphaCapabilityFingerprint({
         baseURL: config.baseURL,
@@ -433,14 +534,15 @@ async function serializeNativeAware(messages, route, routeConfig, ctx, options)
     const flush = async () => {
         if (!normal.length)
             return;
-        const serialized = await serializeDshMessages(normal, ctx, serializeOptions());
+        const batch = normal;
+        normal = [];
+        const serialized = await serializeDshMessages(batch, ctx, serializeOptions());
         nativeTools = serialized.tools ?? nativeTools;
         nativeModel = serialized.model ?? nativeModel;
         grammarToolInputProperties =
             serialized.grammarToolInputProperties ?? grammarToolInputProperties;
-        input.push(...serialized.input);
+        input.push(...restoreGrokNativeReplay(serialized.input, batch, options.grokNativeReplayRoute));
         mergeMap(imageMap, serialized.imageMap);
-        normal = [];
     };
     for (const message of messages ?? []) {
         assertSupportedCheckpointMessage(message);
@@ -679,9 +781,6 @@ async function restoreCompactionPressure(records) {
     await Promise.allSettled(entries.map((record) => record.mutex.close(reason)));
     restoreCompactionPatches(records, entries);
 }
-function visibleWebSearchTimeout(state) {
-    return state.enabled && state.webSearch ? WEB_SEARCH_TIMEOUT_MS : undefined;
-}
 function messagesContainNativeCheckpoint(messages, session) {
     return session !== undefined && messages.some((message) => checkpointStateForMessage(session, message));
 }
@@ -731,6 +830,78 @@ async function* managedResponsesStream(options, routeConfig, ctx) {
         yield managedFailureChunk(error, options.signal);
     }
 }
+async function* managedGrokNativeSearchStream(options, routeConfig, nativeSearch, ctx) {
+    const route = currentRoute(options, routeConfig);
+    const nativeReplayRoute = grokReplayRoute(route, routeConfig);
+    try {
+        if (options.stop !== undefined)
+            throw Object.assign(new Error("LCX Responses does not support GenerateOptions.stop"), { code: "LCX_RESPONSES_UNSUPPORTED_OPTION" });
+        const visibleTools = grokVisibleFunctionTools(options.tools, nativeSearch);
+        const prepared = await serializeNativeAware(options.messages, route, routeConfig, ctx, {
+            signal: options.signal,
+            system: options.system,
+            tools: visibleTools,
+            grokNativeReplayRoute: nativeReplayRoute,
+        });
+        const cacheSessionId = grokPromptCacheSessionId(route, routeConfig);
+        const headers = await authenticatedGrokHeaders(ctx, routeConfig, cacheSessionId);
+        const model = {
+            ...prepared.model,
+            ...routeConfig.modelDefaults,
+            ...routeConfig.modelControls,
+            id: route.model,
+            provider: route.provider,
+            baseUrl: routeConfig.baseURL,
+            api: "openai-responses",
+        };
+        const effectiveReasoning = options.reasoningEffort ?? routeConfig.profileReasoning;
+        if (effectiveReasoning !== undefined &&
+            !getSupportedThinkingLevels(model).some((level) => level === effectiveReasoning))
+            throw Object.assign(new Error(`Grok route does not support reasoning effort "${String(effectiveReasoning)}"`), { code: "LCX_RESPONSES_UNSUPPORTED_OPTION" });
+        const body = buildResponsesBody({
+            model,
+            input: prepared.input,
+            tools: grokWireTools(prepared.tools ?? visibleTools, nativeSearch),
+            sessionId: cacheSessionId,
+            promptCacheRetention: promptCacheRetention(routeConfig),
+            cacheRetention: routeConfig.cacheRetention,
+            reasoningEffort: effectiveReasoning,
+            temperature: options.temperature,
+            maxTokens: options.maxTokens,
+        });
+        if (nativeSearch.web) {
+            const include = new Set(Array.isArray(body.include) ? body.include : []);
+            include.add("web_search_call.action.sources");
+            body.include = [...include];
+        }
+        yield* streamResponsesRequest({
+            baseURL: routeConfig.baseURL,
+            provider: route.provider,
+            model: route.model,
+            piModel: model,
+            body,
+            grammarToolInputProperties: prepared.grammarToolInputProperties,
+            headers,
+            signal: options.signal,
+            timeoutMs: routeConfig.timeoutMs,
+            applyDefaultTimeout: false,
+            streamIdleTimeoutMs: routeConfig.streamIdleTimeoutMs,
+            maxAttempts: 1,
+            maxResponseBytes: routeConfig.maxResponseBytes,
+            serverToolTypes: GROK_NATIVE_SERVER_TOOL_TYPES,
+            nativeReplayRoute,
+            onServerToolUsage(usage) {
+                ctx.logger?.info?.(`[lcx-codex] Grok native search used ${usage.total} server-side tool call(s) (web=${usage.webSearchCalls}, x=${usage.xSearchCalls})`);
+            },
+        });
+    }
+    catch (error) {
+        yield managedFailureChunk(error, options.signal);
+    }
+}
+function isGptLifecycleTarget(options) {
+    return /^gpt-/iu.test(String(options.model ?? ""));
+}
 async function* unavailableManagedRouteStream(options) {
     yield managedFailureChunk(Object.assign(new Error("LCX is enabled but the selected DSH route cannot be resolved as an authenticated OpenAI Responses wire route"), { code: "LCX_RESPONSES_ROUTE_UNAVAILABLE" }), options.signal);
 }
@@ -742,66 +913,45 @@ function installInjected(ctx, configInput = {}) {
         webSearch: false,
         advancedHostedSearch: false,
         alphaSearch: false,
+        grokNativeWebSearch: false,
+        grokNativeXSearch: false,
     };
-    const originalSearchProvider = readWebSearchProvider(ctx);
-    let warnedWebSelection = false;
     const provider = new LcxResponsesSearchProvider(ctx, () => runtimeConfig, () => state.enabled && state.webSearch);
-    ctx.web.registerSearchProvider(provider);
-    const tools = ctx.tools;
-    let disposeAdvanced;
     const capabilityStore = new AlphaCapabilityStore(baseConfig.alphaCapabilityPath);
     const refStore = new AlphaRefStore(baseConfig.alphaRefPath);
-    const alphaAgents = new Set();
+    const managedAgents = new Set();
+    const gptToolRegistrations = new Map();
     const alphaToolRegistrations = new Map();
     const compactionPatchRecords = new Map();
-    const patchedWebSearchDefinitions = new Map();
+    const syncAgentTools = (agent) => {
+        syncGptToolsForAgent(ctx, agent, state, () => runtimeConfig, provider, gptToolRegistrations);
+        syncAlphaToolForAgent(ctx, agent, state, () => runtimeConfig, capabilityStore, refStore, alphaToolRegistrations);
+    };
+    const refreshTools = () => {
+        for (const agent of managedAgents)
+            syncAgentTools(agent);
+    };
     ctx.on("session/disposed", (session) => {
-        for (const agent of alphaAgents)
+        for (const agent of managedAgents)
             if (agentUsesSession(agent, session)) {
+                disposeGptToolsForAgent(agent, gptToolRegistrations);
                 disposeAlphaToolForAgent(agent, alphaToolRegistrations);
-                alphaAgents.delete(agent);
+                managedAgents.delete(agent);
             }
     }, { global: true });
     ctx.on("session/event", (session, event) => {
         if (event.type === "request/header")
-            for (const agent of alphaAgents)
-                if (agentUsesSession(agent, session)) {
-                    syncAlphaToolForAgent(ctx, agent, state, () => runtimeConfig, capabilityStore, refStore, alphaToolRegistrations);
-                }
+            for (const agent of managedAgents)
+                if (agentUsesSession(agent, session))
+                    syncAgentTools(agent);
     }, { global: true });
-    const refreshTools = () => {
-        if (state.enabled &&
-            state.webSearch &&
-            state.advancedHostedSearch &&
-            !disposeAdvanced &&
-            tools?.register)
-            disposeAdvanced = tools.register(createAdvancedHostedTool(ctx, state, () => runtimeConfig));
-        if ((!state.enabled || !state.webSearch || !state.advancedHostedSearch) &&
-            disposeAdvanced) {
-            disposeAdvanced();
-            disposeAdvanced = undefined;
-        }
-        for (const agent of alphaAgents)
-            syncAlphaToolForAgent(ctx, agent, state, () => runtimeConfig, capabilityStore, refStore, alphaToolRegistrations);
-        if (ctx.web) {
-            const target = state.enabled && state.webSearch
-                ? runtimeConfig.webSearchProvider
-                : originalSearchProvider;
-            const selected = writeWebSearchProvider(ctx, target);
-            if (!selected &&
-                state.enabled &&
-                state.webSearch &&
-                !warnedWebSelection) {
-                warnedWebSelection = true;
-                ctx.logger?.warn?.("[lcx-codex] DSH web provider selection could not be changed at runtime; websearch_gpt_advanced still works, but DSH web_search may remain on its configured provider. Pin web.searchProvider=lcx-responses in a profile overlay if this DSH version removes the runtime compatibility field.");
-            }
-        }
-    };
     const settingsEntry = {
         enabled: false,
         webSearch: false,
         advancedHostedSearch: false,
         alphaSearch: false,
+        grokNativeWebSearch: false,
+        grokNativeXSearch: false,
     };
     let source = () => settingsEntry;
     ctx.settings.installSection(ctx, SETTINGS_NS, SettingsSchema, settingsEntry, {
@@ -814,9 +964,10 @@ function installInjected(ctx, configInput = {}) {
             state.webSearch = value.webSearch;
             state.advancedHostedSearch = value.advancedHostedSearch;
             state.alphaSearch = value.alphaSearch;
+            state.grokNativeWebSearch = value.grokNativeWebSearch;
+            state.grokNativeXSearch = value.grokNativeXSearch;
             runtimeConfig = baseConfig;
             refreshTools();
-            refreshVisibleWebSearchTimeouts(patchedWebSearchDefinitions, visibleWebSearchTimeout(state));
         },
     });
     try {
@@ -826,6 +977,8 @@ function installInjected(ctx, configInput = {}) {
             webSearch: Boolean(value.webSearch),
             advancedHostedSearch: Boolean(value.advancedHostedSearch),
             alphaSearch: Boolean(value.alphaSearch),
+            grokNativeWebSearch: Boolean(value.grokNativeWebSearch),
+            grokNativeXSearch: Boolean(value.grokNativeXSearch),
         });
         runtimeConfig = baseConfig;
     }
@@ -834,17 +987,11 @@ function installInjected(ctx, configInput = {}) {
     ctx.inject(["compaction"], (compactionCtx) => {
         patchCompactionPressureService(resolveContextService(compactionCtx, "compaction"), state, () => runtimeConfig, ctx, compactionPatchRecords);
     });
-    ctx.on("tools/execute", async (exec, next) => {
-        if (!state.enabled || !state.webSearch || exec.name !== "web_search")
-            return next();
-        const active = activeAgentRoute(exec);
-        return hostedSearchRouteContext.run(active, () => next());
-    });
     ctx.on("agent/created", ({ agent }) => {
         if (agent === null || typeof agent !== "object")
             return;
-        alphaAgents.add(agent);
-        syncAlphaToolForAgent(ctx, agent, state, () => runtimeConfig, capabilityStore, refStore, alphaToolRegistrations);
+        managedAgents.add(agent);
+        syncAgentTools(agent);
         const installed = patchCompactionPressureForAgent(agent, state, () => runtimeConfig, ctx, compactionPatchRecords);
         if (installed)
             ctx.logger?.info?.("[lcx-codex] pressure coordination installed through AgentPresets service resolver");
@@ -854,15 +1001,25 @@ function installInjected(ctx, configInput = {}) {
             agent === null ||
             typeof agent !== "object")
             return;
-        alphaAgents.add(agent);
-        syncAlphaToolForAgent(ctx, agent, state, () => runtimeConfig, capabilityStore, refStore, alphaToolRegistrations);
+        managedAgents.add(agent);
+        syncAgentTools(agent);
         const installed = patchCompactionPressureForAgent(agent, state, () => runtimeConfig, ctx, compactionPatchRecords);
         if (installed)
             ctx.logger?.info?.("[lcx-codex] pressure coordination installed through AgentPresets service resolver");
-        patchVisibleWebSearchTimeout(agent, () => visibleWebSearchTimeout(state), patchedWebSearchDefinitions);
     }, { global: true });
     ctx.on("llm/stream", (options, next) => {
-        if (!state.enabled || options.purpose === "session-title")
+        if (options.purpose === "session-title")
+            return next();
+        const grokNativeSearch = {
+            web: state.grokNativeWebSearch,
+            x: state.grokNativeXSearch,
+        };
+        if (options.purpose === undefined && grokNativeSearchEnabled(grokNativeSearch)) {
+            const grokRoute = resolveGrokResponsesRouteConfig(ctx, options, runtimeConfig);
+            if (grokRoute)
+                return managedGrokNativeSearchStream(options, routeWithPolicies(grokRoute, runtimeConfig), grokNativeSearch, ctx);
+        }
+        if (!state.enabled || !isGptLifecycleTarget(options))
             return next();
         const routeConfig = resolveResponsesRouteConfig(ctx, options, runtimeConfig);
         if (options.purpose === "compaction") {
@@ -875,13 +1032,12 @@ function installInjected(ctx, configInput = {}) {
         return managedResponsesStream(options, routeWithPolicies(routeConfig, runtimeConfig), ctx);
     });
     ctx.effect?.(() => async () => {
-        disposeAdvanced?.();
-        for (const agent of alphaAgents)
+        for (const agent of managedAgents) {
+            disposeGptToolsForAgent(agent, gptToolRegistrations);
             disposeAlphaToolForAgent(agent, alphaToolRegistrations);
-        alphaAgents.clear();
+        }
+        managedAgents.clear();
         await restoreCompactionPressure(compactionPatchRecords);
-        restoreVisibleWebSearchTimeouts(patchedWebSearchDefinitions);
-        writeWebSearchProvider(ctx, originalSearchProvider);
     }, "lcx-codex cleanup");
 }
 export function apply(ctx, configInput = {}) {

package/lib/responses-request.js

@@ -33,6 +33,9 @@ export function responsesGenerationEnvelope({ model = "unknown", reasoningEffort
     const descriptor = modelRecord(model);
     const result = {};
     if (descriptor.reasoning !== false) {
+        if (descriptor.provider === "xai" ||
+            descriptor.includeEncryptedReasoning === true)
+            result.include = ["reasoning.encrypted_content"];
         if (reasoningEffort !== undefined && reasoningEffort !== "off") {
             const requested = String(reasoningEffort);
             const wire = descriptor.thinkingLevelMap?.[requested] ?? requested;