@fgv/ts-extras 5.1.0-55 → 5.1.0-56

package/lib/packlets/ai-assist/completionClient.js

@@ -21,6 +21,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.callProviderCompletion = callProviderCompletion;
 exports.callProxiedCompletion = callProxiedCompletion;
+/**
+ * Chat completion clients for AI assist. One provider dispatcher over
+ * `IAiProviderDescriptor.apiFormat`, the four adapters it routes to (OpenAI Chat
+ * Completions, OpenAI/xAI Responses, Anthropic, Google Gemini), the response
+ * validators those adapters use, and the proxied variant of the same modality.
+ *
+ * @packageDocumentation
+ */
+const ts_json_base_1 = require("@fgv/ts-json-base");
 const ts_utils_1 = require("@fgv/ts-utils");
 const model_1 = require("./model");
 const thinkingOptionsResolver_1 = require("./thinkingOptionsResolver");
@@ -30,6 +39,8 @@ const http_1 = require("./http");
 const toolFormats_1 = require("./toolFormats");
 const structuredOutput_1 = require("./structuredOutput");
 const registry_1 = require("./registry");
+const streamUsageCapability_1 = require("./streamUsageCapability");
+const usageNormalization_1 = require("./usageNormalization");
 const openAiMessage = ts_utils_1.Validators.object({
     content: ts_utils_1.Validators.string
 });
@@ -55,15 +66,22 @@ const responsesApiResponse = ts_utils_1.Validators.object({
     status: ts_utils_1.Validators.string
 });
 const geminiPart = ts_utils_1.Validators.object({
-    text: ts_utils_1.Validators.string
+    text: ts_utils_1.Validators.string.optional()
 });
 const geminiContent = ts_utils_1.Validators.object({
-    parts: ts_utils_1.Validators.arrayOf(geminiPart).withConstraint((arr) => arr.length > 0)
+    // No non-empty constraint: `parts: []` is a shape Gemini actually returns, and rejecting it
+    // here reports a schema violation for what is really "the model produced no output".
+    parts: ts_utils_1.Validators.arrayOf(geminiPart).optional()
 });
 const geminiCandidate = ts_utils_1.Validators.object({
-    content: geminiContent,
-    finishReason: ts_utils_1.Validators.string
+    content: geminiContent.optional(),
+    finishReason: ts_utils_1.Validators.string.optional(),
+    finishMessage: ts_utils_1.Validators.string.optional()
 });
+// Terminal reasons that mean "the model finished normally", as distinct from a refusal. Kept in
+// step with the identical set in `imageGenerationClient.ts`; both exist so an empty response can
+// be reported as the decline it is rather than as an empty string or a parse error.
+const benignGeminiFinishReasons = new Set(['STOP', 'MAX_TOKENS']);
 const geminiResponse = ts_utils_1.Validators.object({
     candidates: ts_utils_1.Validators.arrayOf(geminiCandidate).withConstraint((arr) => arr.length > 0)
 });
@@ -75,36 +93,52 @@ const geminiResponse = ts_utils_1.Validators.object({
  * Works for xAI Grok, OpenAI, Groq, and Mistral.
  * @internal
  */
-async function callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, useMaxCompletionTokensField = false, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT) {
+async function callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, useMaxCompletionTokensField = false, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT, reportsUsage = false, cache, cacheKeyHeader) {
     var _a;
     const url = `${config.baseUrl}/chat/completions`;
-    const messages = (0, chatRequestBuilders_1.buildMessages)(prompt.system, (0, chatRequestBuilders_1.buildOpenAiChatUserContent)(prompt), {
+    const systemContentResult = (0, chatRequestBuilders_1.buildOpenAiChatSystemContent)(prompt.system, cache);
+    if (systemContentResult.isFailure()) {
+        return (0, ts_utils_1.fail)(systemContentResult.message);
+    }
+    const messages = (0, chatRequestBuilders_1.buildMessages)(systemContentResult.value, (0, chatRequestBuilders_1.buildOpenAiChatUserContent)(prompt), {
         head
     });
     const effort = (_a = resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.openAiEffort) !== null && _a !== void 0 ? _a : resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.xaiEffort;
     const maxTokensField = useMaxCompletionTokensField ? 'max_completion_tokens' : 'max_tokens';
-    const body = Object.assign(Object.assign(Object.assign({ model: config.model, messages }, (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning_effort: effort } : {})), (maxTokens !== undefined ? { [maxTokensField]: maxTokens } : {}));
+    const body = Object.assign(Object.assign(Object.assign(Object.assign({ model: config.model, messages }, (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning_effort: effort } : {})), (maxTokens !== undefined ? { [maxTokensField]: maxTokens } : {})), ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined && cacheKeyHeader === undefined
+        ? // eslint-disable-next-line @typescript-eslint/naming-convention -- wire field name
+            { prompt_cache_key: cache.cacheKey }
+        : {}));
     if ((resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.otherParams) !== undefined) {
         Object.assign(body, resolvedThinking.otherParams);
     }
     Object.assign(body, structured.wire);
     const headers = (0, endpoint_1.bearerAuthHeader)(config.apiKey);
+    // Sticky-routing key, when the provider carries it as a header rather than a body field.
+    // xAI's prompt cache is per-server, so an identical prefix can still miss if the request is
+    // routed to a different box; `x-grok-conv-id` pins it.
+    if ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined && cacheKeyHeader !== undefined) {
+        headers[cacheKeyHeader] = cache.cacheKey;
+    }
     /* c8 ignore next 1 - optional logger */
     logger === null || logger === void 0 ? void 0 : logger.info(`OpenAI completion: model=${config.model}`);
     const jsonResult = await (0, http_1.fetchJson)(url, headers, body, logger, signal);
     if (jsonResult.isFailure()) {
         return (0, ts_utils_1.fail)(jsonResult.message);
     }
+    // Only descriptors with confirmed cache-relevant usage reporting are normalized — the
+    // adapter is shared by Groq/Mistral/Ollama/openai-compat too, and an ordinary usage block
+    // from one of those carries no cache information. See supportsCacheUsageReporting.
+    const rawUsage = jsonResult.value.usage;
+    const usage = reportsUsage
+        ? (0, usageNormalization_1.normalizeOpenAiChatUsage)((0, ts_json_base_1.isJsonObject)(rawUsage) ? rawUsage : undefined)
+        : undefined;
     return openAiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `OpenAI API response: ${msg}`)
         .onSuccess((response) => {
         const choice = response.choices[0];
-        return (0, ts_utils_1.succeed)({
-            content: choice.message.content,
-            truncated: choice.finish_reason === 'length',
-            structuredOutput: structured.enforcement
-        });
+        return (0, ts_utils_1.succeed)(Object.assign({ content: choice.message.content, truncated: choice.finish_reason === 'length', structuredOutput: structured.enforcement }, (usage !== undefined ? { usage } : {})));
     });
 }
 // ============================================================================
@@ -131,14 +165,21 @@ function extractResponsesApiText(output) {
  * Used when tools are configured for an openai-format provider.
  * @internal
  */
-async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, temperature, logger, signal, resolvedThinking, maxTokens, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT) {
+async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, temperature, logger, signal, resolvedThinking, maxTokens, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT, reportsUsage = false, cache) {
     var _a;
     const url = `${config.baseUrl}/responses`;
-    const input = (0, chatRequestBuilders_1.buildMessages)(prompt.system, (0, chatRequestBuilders_1.buildOpenAiResponsesUserContent)(prompt), {
+    const systemContentResult = (0, chatRequestBuilders_1.buildOpenAiResponsesSystemContent)(prompt.system, cache);
+    if (systemContentResult.isFailure()) {
+        return (0, ts_utils_1.fail)(systemContentResult.message);
+    }
+    const input = (0, chatRequestBuilders_1.buildMessages)(systemContentResult.value, (0, chatRequestBuilders_1.buildOpenAiResponsesUserContent)(prompt), {
         head
     });
     const effort = (_a = resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.openAiEffort) !== null && _a !== void 0 ? _a : resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.xaiEffort;
-    const body = Object.assign(Object.assign(Object.assign({ model: config.model, input }, (tools.length > 0 ? { tools: (0, toolFormats_1.toResponsesApiTools)(tools) } : {})), (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning: { effort } } : {}));
+    const body = Object.assign(Object.assign(Object.assign(Object.assign({ model: config.model, input }, (tools.length > 0 ? { tools: (0, toolFormats_1.toResponsesApiTools)(tools) } : {})), (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning: { effort } } : {})), ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined
+        ? // eslint-disable-next-line @typescript-eslint/naming-convention -- wire field name
+            { prompt_cache_key: cache.cacheKey }
+        : {}));
     // Shared by OpenAI and xAI — both route through the Responses API with the same field name.
     if (maxTokens !== undefined) {
         body.max_output_tokens = maxTokens;
@@ -154,15 +195,16 @@ async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, t
     if (jsonResult.isFailure()) {
         return (0, ts_utils_1.fail)(jsonResult.message);
     }
+    // See the identical gate in callOpenAiCompletion above — this route is shared the same way.
+    const rawResponsesUsage = jsonResult.value.usage;
+    const responsesUsage = reportsUsage
+        ? (0, usageNormalization_1.normalizeOpenAiResponsesUsage)((0, ts_json_base_1.isJsonObject)(rawResponsesUsage) ? rawResponsesUsage : undefined)
+        : undefined;
     return responsesApiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `Responses API response: ${msg}`)
         .onSuccess((response) => {
-        return extractResponsesApiText(response.output).onSuccess((text) => (0, ts_utils_1.succeed)({
-            content: text,
-            truncated: response.status === 'incomplete',
-            structuredOutput: structured.enforcement
-        }));
+        return extractResponsesApiText(response.output).onSuccess((text) => (0, ts_utils_1.succeed)(Object.assign({ content: text, truncated: response.status === 'incomplete', structuredOutput: structured.enforcement }, (responsesUsage !== undefined ? { usage: responsesUsage } : {}))));
     });
 }
 // ============================================================================
@@ -228,10 +270,14 @@ function extractAnthropicStructuredOutput(content) {
     return (0, ts_utils_1.fail)(`Anthropic API response: structured output was forced but no '${structuredOutput_1.ANTHROPIC_STRUCTURED_OUTPUT_TOOL_NAME}' tool_use block was returned`);
 }
 /** Calls the Anthropic Messages API with optional tool support. @internal */
-async function callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, useAdaptiveThinking = false, maxTokens, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT) {
+async function callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, useAdaptiveThinking = false, maxTokens, structured = structuredOutput_1.NO_STRUCTURED_OUTPUT, cache) {
     const url = `${config.baseUrl}/messages`;
     const messages = (0, chatRequestBuilders_1.buildAnthropicMessages)(prompt, { head });
-    const body = Object.assign({ model: config.model, system: prompt.system, messages, 
+    const systemResult = (0, chatRequestBuilders_1.buildAnthropicSystem)(prompt.system, cache);
+    if (systemResult.isFailure()) {
+        return (0, ts_utils_1.fail)(systemResult.message);
+    }
+    const body = Object.assign({ model: config.model, system: systemResult.value, messages, 
         // Anthropic's Messages API requires max_tokens on every request — see
         // AiAssist.DEFAULT_ANTHROPIC_MAX_TOKENS for why only this provider defaults it.
         max_tokens: maxTokens !== null && maxTokens !== void 0 ? maxTokens : model_1.DEFAULT_ANTHROPIC_MAX_TOKENS }, (temperature !== undefined ? { temperature } : {}));
@@ -282,6 +328,8 @@ async function callAnthropicCompletion(config, prompt, head, temperature, logger
     }
     const rawContent = jsonResult.value.content;
     const stopReason = jsonResult.value.stop_reason;
+    const rawAnthropicUsage = jsonResult.value.usage;
+    const anthropicUsage = (0, usageNormalization_1.normalizeAnthropicUsage)((0, ts_json_base_1.isJsonObject)(rawAnthropicUsage) ? rawAnthropicUsage : undefined);
     if (!Array.isArray(rawContent)) {
         return (0, ts_utils_1.fail)('Anthropic API response: content is not an array');
     }
@@ -291,11 +339,7 @@ async function callAnthropicCompletion(config, prompt, head, temperature, logger
     const extracted = structured.enforcement === 'tool-forced'
         ? extractAnthropicStructuredOutput(rawContent)
         : extractAnthropicText(rawContent);
-    return extracted.onSuccess((text) => (0, ts_utils_1.succeed)({
-        content: text,
-        truncated: stopReason === 'max_tokens',
-        structuredOutput: structured.enforcement
-    }));
+    return extracted.onSuccess((text) => (0, ts_utils_1.succeed)(Object.assign({ content: text, truncated: stopReason === 'max_tokens', structuredOutput: structured.enforcement }, (anthropicUsage !== undefined ? { usage: anthropicUsage } : {}))));
 }
 // ============================================================================
 // Google Gemini adapter
@@ -343,22 +387,35 @@ async function callGeminiCompletion(config, prompt, head, temperature, logger, t
     if (jsonResult.isFailure()) {
         return (0, ts_utils_1.fail)(jsonResult.message);
     }
+    const rawGeminiUsage = jsonResult.value.usageMetadata;
+    const geminiUsage = (0, usageNormalization_1.normalizeGeminiUsage)((0, ts_json_base_1.isJsonObject)(rawGeminiUsage) ? rawGeminiUsage : undefined);
     return geminiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `Gemini API response: ${msg}`)
         .onSuccess((response) => {
+        var _a, _b, _c;
         const candidate = response.candidates[0];
-        return (0, ts_utils_1.succeed)({
-            // ALL parts, not `parts[0]`. Gemini may split one reply across several
-            // text parts, and reading only the first silently discards the rest —
-            // yielding a truncated document that often still parses, which is the
-            // worst way to be wrong. The streaming adapter has always concatenated
-            // (`fullText += part.text`); this path did not, so the same response gave
-            // different text depending on which one you called.
-            content: candidate.content.parts.map((part) => part.text).join(''),
-            truncated: candidate.finishReason === 'MAX_TOKENS',
-            structuredOutput: structured.enforcement
-        });
+        // ALL parts, not `parts[0]`. Gemini may split one reply across several
+        // text parts, and reading only the first silently discards the rest —
+        // yielding a truncated document that often still parses, which is the
+        // worst way to be wrong. The streaming adapter has always concatenated
+        // (`fullText += part.text`); this path did not, so the same response gave
+        // different text depending on which one you called.
+        //
+        // `part.text ?? ''` rather than `part.text`: a reasoning or tool part contributes no text,
+        // and joining `undefined` would put the literal string "undefined" into the reply.
+        const content = (_c = (_b = (_a = candidate.content) === null || _a === void 0 ? void 0 : _a.parts) === null || _b === void 0 ? void 0 : _b.map((part) => { var _a; return (_a = part.text) !== null && _a !== void 0 ? _a : ''; }).join('')) !== null && _c !== void 0 ? _c : '';
+        if (content.length === 0) {
+            // Nothing came back. If the candidate says why, say so — an empty string handed to a
+            // caller is indistinguishable from a model that legitimately replied with nothing, and
+            // sends them looking in the wrong place.
+            const reason = candidate.finishReason;
+            if (reason !== undefined && !benignGeminiFinishReasons.has(reason)) {
+                const suffix = candidate.finishMessage ? ` — ${candidate.finishMessage}` : '';
+                return (0, ts_utils_1.fail)(`Gemini completion declined: ${reason}${suffix}`);
+            }
+        }
+        return (0, ts_utils_1.succeed)(Object.assign({ content, truncated: candidate.finishReason === 'MAX_TOKENS', structuredOutput: structured.enforcement }, (geminiUsage !== undefined ? { usage: geminiUsage } : {})));
     });
 }
 // ============================================================================
@@ -371,7 +428,7 @@ async function callGeminiCompletion(config, prompt, head, temperature, logger, t
  * @public
  */
 async function callProviderCompletion(params) {
-    const { descriptor, apiKey, system, messages, temperature, modelOverride, tier, logger, tools, signal, endpoint, thinking, maxTokens, structuredOutput } = params;
+    const { descriptor, apiKey, system, messages, temperature, modelOverride, tier, logger, tools, signal, endpoint, thinking, maxTokens, structuredOutput, cache } = params;
     const splitResult = (0, chatRequestBuilders_1.splitChatRequest)(system, messages);
     if (splitResult.isFailure()) {
         return (0, ts_utils_1.fail)(splitResult.message);
@@ -448,15 +505,30 @@ async function callProviderCompletion(params) {
             `tools=${toolTypes}, supported=${supported}`);
     }
     switch (descriptor.apiFormat) {
-        case 'openai':
+        case 'openai': {
             // Responses-API-only models (e.g. gpt-5.5-pro) 400 on /chat/completions, so they route
             // to the Responses path even with no tools requested — same path the tools case uses.
+            // `cache` carries two independently-supported things, and they are gated separately.
+            // `systemBreakpoints` restructures `content` and needs the server to tolerate an
+            // unrecognized field inside it (OpenAI only). `cacheKey` is an opaque routing string that
+            // makes repeated requests with a shared prefix land on the same cache-holding server —
+            // which xAI needs precisely because it does per-server byte-for-byte prefix matching.
+            // Gating them together withheld the routing key from the provider that depends on it most.
+            // Every descriptor supporting neither (Groq, Mistral, Ollama, openai-compat) still gets the
+            // exact request body it would have gotten had the caller passed no `cache` at all.
+            const routing = (0, streamUsageCapability_1.supportsPromptCacheRouting)(descriptor);
+            const breakpoints = (0, streamUsageCapability_1.supportsPromptCacheBreakpoints)(descriptor) ? cache === null || cache === void 0 ? void 0 : cache.systemBreakpoints : undefined;
+            const routedKey = routing !== undefined ? cache === null || cache === void 0 ? void 0 : cache.cacheKey : undefined;
+            const gatedCache = breakpoints !== undefined || routedKey !== undefined
+                ? { systemBreakpoints: breakpoints, cacheKey: routedKey }
+                : undefined;
             if (usesResponsesApi) {
-                return callOpenAiResponsesCompletion(config, prompt, tools, head, temperature, logger, signal, resolvedThinking, maxTokens, resolvedStructured);
+                return callOpenAiResponsesCompletion(config, prompt, tools, head, temperature, logger, signal, resolvedThinking, maxTokens, resolvedStructured, (0, streamUsageCapability_1.supportsCacheUsageReporting)(descriptor), gatedCache);
             }
-            return callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, (0, model_1.usesMaxCompletionTokensField)(descriptor), resolvedStructured);
+            return callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, (0, model_1.usesMaxCompletionTokensField)(descriptor), resolvedStructured, (0, streamUsageCapability_1.supportsCacheUsageReporting)(descriptor), gatedCache, routing === null || routing === void 0 ? void 0 : routing.chatCompletionsHeader);
+        }
         case 'anthropic':
-            return callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, (0, model_1.isAdaptiveThinkingModel)(descriptor, config.model), maxTokens, resolvedStructured);
+            return callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, (0, model_1.isAdaptiveThinkingModel)(descriptor, config.model), maxTokens, resolvedStructured, cache);
         case 'gemini':
             return callGeminiCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, maxTokens, resolvedStructured);
         /* c8 ignore next 4 - defensive coding: exhaustive switch guaranteed by TypeScript */
@@ -480,7 +552,7 @@ async function callProviderCompletion(params) {
  * @public
  */
 async function callProxiedCompletion(proxyUrl, params) {
-    const { descriptor, apiKey, system, messages, temperature, modelOverride, logger, tools, signal, thinking, maxTokens, structuredOutput } = params;
+    const { descriptor, apiKey, system, messages, temperature, modelOverride, logger, tools, signal, thinking, maxTokens, structuredOutput, cache } = params;
     const splitResult = (0, chatRequestBuilders_1.splitChatRequest)(system, messages);
     if (splitResult.isFailure()) {
         return (0, ts_utils_1.fail)(splitResult.message);
@@ -527,6 +599,11 @@ async function callProxiedCompletion(proxyUrl, params) {
                 ? { onUnsupported: structuredOutput.onUnsupported }
                 : {}));
     }
+    // `IAiCacheRequest` is plain numbers and a string — JSON-serializable as-is, unlike
+    // `structuredOutput`'s schema above, so it needs no wire projection before forwarding.
+    if (cache !== undefined) {
+        body.cache = cache;
+    }
     /* c8 ignore next 1 - optional logger */
     logger === null || logger === void 0 ? void 0 : logger.info(`AI proxy request: provider=${descriptor.id}, proxy=${proxyUrl}`);
     const url = `${proxyUrl}/api/ai/completion`;