@fgv/ts-extras 5.1.0-54 → 5.1.0-56

package/dist/packlets/ai-assist/completionClient.js

@@ -17,15 +17,26 @@
 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
+/**
+ * Chat completion clients for AI assist. One provider dispatcher over
+ * `IAiProviderDescriptor.apiFormat`, the four adapters it routes to (OpenAI Chat
+ * Completions, OpenAI/xAI Responses, Anthropic, Google Gemini), the response
+ * validators those adapters use, and the proxied variant of the same modality.
+ *
+ * @packageDocumentation
+ */
+import { isJsonObject } from '@fgv/ts-json-base';
 import { captureResult, fail, succeed, Validators } from '@fgv/ts-utils';
 import { DEFAULT_ANTHROPIC_MAX_TOKENS, isAdaptiveThinkingModel, isResponsesOnlyModel, resolveProviderModel, usesMaxCompletionTokensField } from './model';
 import { anthropicEffortToBudgetTokens, checkTemperatureConflict, mergeThinkingConfig, providerDiscriminatorForId } from './thinkingOptionsResolver';
-import { buildAnthropicMessages, buildGeminiContents, buildMessages, buildOpenAiChatUserContent, buildOpenAiResponsesUserContent, normalizeOutboundMessages, splitChatRequest } from './chatRequestBuilders';
+import { buildAnthropicMessages, buildAnthropicSystem, buildGeminiContents, buildMessages, buildOpenAiChatSystemContent, buildOpenAiChatUserContent, buildOpenAiResponsesSystemContent, buildOpenAiResponsesUserContent, normalizeOutboundMessages, splitChatRequest } from './chatRequestBuilders';
 import { anthropicAuthHeaders, bearerAuthHeader, geminiAuthHeader, resolveEffectiveBaseUrl } from './endpoint';
 import { fetchJson } from './http';
 import { toAnthropicTools, toGeminiTools, toResponsesApiTools } from './toolFormats';
 import { ANTHROPIC_STRUCTURED_OUTPUT_TOOL_NAME, NO_STRUCTURED_OUTPUT, isStructuredOutputEnforcement, resolveStructuredOutput } from './structuredOutput';
 import { resolveStructuredOutputCapability } from './registry';
+import { supportsCacheUsageReporting, supportsPromptCacheBreakpoints, supportsPromptCacheRouting } from './streamUsageCapability';
+import { normalizeAnthropicUsage, normalizeGeminiUsage, normalizeOpenAiChatUsage, normalizeOpenAiResponsesUsage } from './usageNormalization';
 const openAiMessage = Validators.object({
     content: Validators.string
 });
@@ -51,15 +62,22 @@ const responsesApiResponse = Validators.object({
     status: Validators.string
 });
 const geminiPart = Validators.object({
-    text: Validators.string
+    text: Validators.string.optional()
 });
 const geminiContent = Validators.object({
-    parts: Validators.arrayOf(geminiPart).withConstraint((arr) => arr.length > 0)
+    // No non-empty constraint: `parts: []` is a shape Gemini actually returns, and rejecting it
+    // here reports a schema violation for what is really "the model produced no output".
+    parts: Validators.arrayOf(geminiPart).optional()
 });
 const geminiCandidate = Validators.object({
-    content: geminiContent,
-    finishReason: Validators.string
+    content: geminiContent.optional(),
+    finishReason: Validators.string.optional(),
+    finishMessage: Validators.string.optional()
 });
+// Terminal reasons that mean "the model finished normally", as distinct from a refusal. Kept in
+// step with the identical set in `imageGenerationClient.ts`; both exist so an empty response can
+// be reported as the decline it is rather than as an empty string or a parse error.
+const benignGeminiFinishReasons = new Set(['STOP', 'MAX_TOKENS']);
 const geminiResponse = Validators.object({
     candidates: Validators.arrayOf(geminiCandidate).withConstraint((arr) => arr.length > 0)
 });
@@ -71,36 +89,52 @@ const geminiResponse = Validators.object({
  * Works for xAI Grok, OpenAI, Groq, and Mistral.
  * @internal
  */
-async function callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, useMaxCompletionTokensField = false, structured = NO_STRUCTURED_OUTPUT) {
+async function callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, useMaxCompletionTokensField = false, structured = NO_STRUCTURED_OUTPUT, reportsUsage = false, cache, cacheKeyHeader) {
     var _a;
     const url = `${config.baseUrl}/chat/completions`;
-    const messages = buildMessages(prompt.system, buildOpenAiChatUserContent(prompt), {
+    const systemContentResult = buildOpenAiChatSystemContent(prompt.system, cache);
+    if (systemContentResult.isFailure()) {
+        return fail(systemContentResult.message);
+    }
+    const messages = buildMessages(systemContentResult.value, buildOpenAiChatUserContent(prompt), {
         head
     });
     const effort = (_a = resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.openAiEffort) !== null && _a !== void 0 ? _a : resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.xaiEffort;
     const maxTokensField = useMaxCompletionTokensField ? 'max_completion_tokens' : 'max_tokens';
-    const body = Object.assign(Object.assign(Object.assign({ model: config.model, messages }, (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning_effort: effort } : {})), (maxTokens !== undefined ? { [maxTokensField]: maxTokens } : {}));
+    const body = Object.assign(Object.assign(Object.assign(Object.assign({ model: config.model, messages }, (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning_effort: effort } : {})), (maxTokens !== undefined ? { [maxTokensField]: maxTokens } : {})), ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined && cacheKeyHeader === undefined
+        ? // eslint-disable-next-line @typescript-eslint/naming-convention -- wire field name
+            { prompt_cache_key: cache.cacheKey }
+        : {}));
     if ((resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.otherParams) !== undefined) {
         Object.assign(body, resolvedThinking.otherParams);
     }
     Object.assign(body, structured.wire);
     const headers = bearerAuthHeader(config.apiKey);
+    // Sticky-routing key, when the provider carries it as a header rather than a body field.
+    // xAI's prompt cache is per-server, so an identical prefix can still miss if the request is
+    // routed to a different box; `x-grok-conv-id` pins it.
+    if ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined && cacheKeyHeader !== undefined) {
+        headers[cacheKeyHeader] = cache.cacheKey;
+    }
     /* c8 ignore next 1 - optional logger */
     logger === null || logger === void 0 ? void 0 : logger.info(`OpenAI completion: model=${config.model}`);
     const jsonResult = await fetchJson(url, headers, body, logger, signal);
     if (jsonResult.isFailure()) {
         return fail(jsonResult.message);
     }
+    // Only descriptors with confirmed cache-relevant usage reporting are normalized — the
+    // adapter is shared by Groq/Mistral/Ollama/openai-compat too, and an ordinary usage block
+    // from one of those carries no cache information. See supportsCacheUsageReporting.
+    const rawUsage = jsonResult.value.usage;
+    const usage = reportsUsage
+        ? normalizeOpenAiChatUsage(isJsonObject(rawUsage) ? rawUsage : undefined)
+        : undefined;
     return openAiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `OpenAI API response: ${msg}`)
         .onSuccess((response) => {
         const choice = response.choices[0];
-        return succeed({
-            content: choice.message.content,
-            truncated: choice.finish_reason === 'length',
-            structuredOutput: structured.enforcement
-        });
+        return succeed(Object.assign({ content: choice.message.content, truncated: choice.finish_reason === 'length', structuredOutput: structured.enforcement }, (usage !== undefined ? { usage } : {})));
     });
 }
 // ============================================================================
@@ -127,14 +161,21 @@ function extractResponsesApiText(output) {
  * Used when tools are configured for an openai-format provider.
  * @internal
  */
-async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, temperature, logger, signal, resolvedThinking, maxTokens, structured = NO_STRUCTURED_OUTPUT) {
+async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, temperature, logger, signal, resolvedThinking, maxTokens, structured = NO_STRUCTURED_OUTPUT, reportsUsage = false, cache) {
     var _a;
     const url = `${config.baseUrl}/responses`;
-    const input = buildMessages(prompt.system, buildOpenAiResponsesUserContent(prompt), {
+    const systemContentResult = buildOpenAiResponsesSystemContent(prompt.system, cache);
+    if (systemContentResult.isFailure()) {
+        return fail(systemContentResult.message);
+    }
+    const input = buildMessages(systemContentResult.value, buildOpenAiResponsesUserContent(prompt), {
         head
     });
     const effort = (_a = resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.openAiEffort) !== null && _a !== void 0 ? _a : resolvedThinking === null || resolvedThinking === void 0 ? void 0 : resolvedThinking.xaiEffort;
-    const body = Object.assign(Object.assign(Object.assign({ model: config.model, input }, (tools.length > 0 ? { tools: toResponsesApiTools(tools) } : {})), (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning: { effort } } : {}));
+    const body = Object.assign(Object.assign(Object.assign(Object.assign({ model: config.model, input }, (tools.length > 0 ? { tools: toResponsesApiTools(tools) } : {})), (temperature !== undefined ? { temperature } : {})), (effort !== undefined && config.model !== 'grok-4' ? { reasoning: { effort } } : {})), ((cache === null || cache === void 0 ? void 0 : cache.cacheKey) !== undefined
+        ? // eslint-disable-next-line @typescript-eslint/naming-convention -- wire field name
+            { prompt_cache_key: cache.cacheKey }
+        : {}));
     // Shared by OpenAI and xAI — both route through the Responses API with the same field name.
     if (maxTokens !== undefined) {
         body.max_output_tokens = maxTokens;
@@ -150,15 +191,16 @@ async function callOpenAiResponsesCompletion(config, prompt, tools = [], head, t
     if (jsonResult.isFailure()) {
         return fail(jsonResult.message);
     }
+    // See the identical gate in callOpenAiCompletion above — this route is shared the same way.
+    const rawResponsesUsage = jsonResult.value.usage;
+    const responsesUsage = reportsUsage
+        ? normalizeOpenAiResponsesUsage(isJsonObject(rawResponsesUsage) ? rawResponsesUsage : undefined)
+        : undefined;
     return responsesApiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `Responses API response: ${msg}`)
         .onSuccess((response) => {
-        return extractResponsesApiText(response.output).onSuccess((text) => succeed({
-            content: text,
-            truncated: response.status === 'incomplete',
-            structuredOutput: structured.enforcement
-        }));
+        return extractResponsesApiText(response.output).onSuccess((text) => succeed(Object.assign({ content: text, truncated: response.status === 'incomplete', structuredOutput: structured.enforcement }, (responsesUsage !== undefined ? { usage: responsesUsage } : {}))));
     });
 }
 // ============================================================================
@@ -224,10 +266,14 @@ function extractAnthropicStructuredOutput(content) {
     return fail(`Anthropic API response: structured output was forced but no '${ANTHROPIC_STRUCTURED_OUTPUT_TOOL_NAME}' tool_use block was returned`);
 }
 /** Calls the Anthropic Messages API with optional tool support. @internal */
-async function callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, useAdaptiveThinking = false, maxTokens, structured = NO_STRUCTURED_OUTPUT) {
+async function callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, useAdaptiveThinking = false, maxTokens, structured = NO_STRUCTURED_OUTPUT, cache) {
     const url = `${config.baseUrl}/messages`;
     const messages = buildAnthropicMessages(prompt, { head });
-    const body = Object.assign({ model: config.model, system: prompt.system, messages, 
+    const systemResult = buildAnthropicSystem(prompt.system, cache);
+    if (systemResult.isFailure()) {
+        return fail(systemResult.message);
+    }
+    const body = Object.assign({ model: config.model, system: systemResult.value, messages, 
         // Anthropic's Messages API requires max_tokens on every request — see
         // AiAssist.DEFAULT_ANTHROPIC_MAX_TOKENS for why only this provider defaults it.
         max_tokens: maxTokens !== null && maxTokens !== void 0 ? maxTokens : DEFAULT_ANTHROPIC_MAX_TOKENS }, (temperature !== undefined ? { temperature } : {}));
@@ -278,6 +324,8 @@ async function callAnthropicCompletion(config, prompt, head, temperature, logger
     }
     const rawContent = jsonResult.value.content;
     const stopReason = jsonResult.value.stop_reason;
+    const rawAnthropicUsage = jsonResult.value.usage;
+    const anthropicUsage = normalizeAnthropicUsage(isJsonObject(rawAnthropicUsage) ? rawAnthropicUsage : undefined);
     if (!Array.isArray(rawContent)) {
         return fail('Anthropic API response: content is not an array');
     }
@@ -287,11 +335,7 @@ async function callAnthropicCompletion(config, prompt, head, temperature, logger
     const extracted = structured.enforcement === 'tool-forced'
         ? extractAnthropicStructuredOutput(rawContent)
         : extractAnthropicText(rawContent);
-    return extracted.onSuccess((text) => succeed({
-        content: text,
-        truncated: stopReason === 'max_tokens',
-        structuredOutput: structured.enforcement
-    }));
+    return extracted.onSuccess((text) => succeed(Object.assign({ content: text, truncated: stopReason === 'max_tokens', structuredOutput: structured.enforcement }, (anthropicUsage !== undefined ? { usage: anthropicUsage } : {}))));
 }
 // ============================================================================
 // Google Gemini adapter
@@ -339,22 +383,35 @@ async function callGeminiCompletion(config, prompt, head, temperature, logger, t
     if (jsonResult.isFailure()) {
         return fail(jsonResult.message);
     }
+    const rawGeminiUsage = jsonResult.value.usageMetadata;
+    const geminiUsage = normalizeGeminiUsage(isJsonObject(rawGeminiUsage) ? rawGeminiUsage : undefined);
     return geminiResponse
         .validate(jsonResult.value)
         .withErrorFormat((msg) => `Gemini API response: ${msg}`)
         .onSuccess((response) => {
+        var _a, _b, _c;
         const candidate = response.candidates[0];
-        return succeed({
-            // ALL parts, not `parts[0]`. Gemini may split one reply across several
-            // text parts, and reading only the first silently discards the rest —
-            // yielding a truncated document that often still parses, which is the
-            // worst way to be wrong. The streaming adapter has always concatenated
-            // (`fullText += part.text`); this path did not, so the same response gave
-            // different text depending on which one you called.
-            content: candidate.content.parts.map((part) => part.text).join(''),
-            truncated: candidate.finishReason === 'MAX_TOKENS',
-            structuredOutput: structured.enforcement
-        });
+        // ALL parts, not `parts[0]`. Gemini may split one reply across several
+        // text parts, and reading only the first silently discards the rest —
+        // yielding a truncated document that often still parses, which is the
+        // worst way to be wrong. The streaming adapter has always concatenated
+        // (`fullText += part.text`); this path did not, so the same response gave
+        // different text depending on which one you called.
+        //
+        // `part.text ?? ''` rather than `part.text`: a reasoning or tool part contributes no text,
+        // and joining `undefined` would put the literal string "undefined" into the reply.
+        const content = (_c = (_b = (_a = candidate.content) === null || _a === void 0 ? void 0 : _a.parts) === null || _b === void 0 ? void 0 : _b.map((part) => { var _a; return (_a = part.text) !== null && _a !== void 0 ? _a : ''; }).join('')) !== null && _c !== void 0 ? _c : '';
+        if (content.length === 0) {
+            // Nothing came back. If the candidate says why, say so — an empty string handed to a
+            // caller is indistinguishable from a model that legitimately replied with nothing, and
+            // sends them looking in the wrong place.
+            const reason = candidate.finishReason;
+            if (reason !== undefined && !benignGeminiFinishReasons.has(reason)) {
+                const suffix = candidate.finishMessage ? ` — ${candidate.finishMessage}` : '';
+                return fail(`Gemini completion declined: ${reason}${suffix}`);
+            }
+        }
+        return succeed(Object.assign({ content, truncated: candidate.finishReason === 'MAX_TOKENS', structuredOutput: structured.enforcement }, (geminiUsage !== undefined ? { usage: geminiUsage } : {})));
     });
 }
 // ============================================================================
@@ -367,7 +424,7 @@ async function callGeminiCompletion(config, prompt, head, temperature, logger, t
  * @public
  */
 export async function callProviderCompletion(params) {
-    const { descriptor, apiKey, system, messages, temperature, modelOverride, tier, logger, tools, signal, endpoint, thinking, maxTokens, structuredOutput } = params;
+    const { descriptor, apiKey, system, messages, temperature, modelOverride, tier, logger, tools, signal, endpoint, thinking, maxTokens, structuredOutput, cache } = params;
     const splitResult = splitChatRequest(system, messages);
     if (splitResult.isFailure()) {
         return fail(splitResult.message);
@@ -444,15 +501,30 @@ export async function callProviderCompletion(params) {
             `tools=${toolTypes}, supported=${supported}`);
     }
     switch (descriptor.apiFormat) {
-        case 'openai':
+        case 'openai': {
             // Responses-API-only models (e.g. gpt-5.5-pro) 400 on /chat/completions, so they route
             // to the Responses path even with no tools requested — same path the tools case uses.
+            // `cache` carries two independently-supported things, and they are gated separately.
+            // `systemBreakpoints` restructures `content` and needs the server to tolerate an
+            // unrecognized field inside it (OpenAI only). `cacheKey` is an opaque routing string that
+            // makes repeated requests with a shared prefix land on the same cache-holding server —
+            // which xAI needs precisely because it does per-server byte-for-byte prefix matching.
+            // Gating them together withheld the routing key from the provider that depends on it most.
+            // Every descriptor supporting neither (Groq, Mistral, Ollama, openai-compat) still gets the
+            // exact request body it would have gotten had the caller passed no `cache` at all.
+            const routing = supportsPromptCacheRouting(descriptor);
+            const breakpoints = supportsPromptCacheBreakpoints(descriptor) ? cache === null || cache === void 0 ? void 0 : cache.systemBreakpoints : undefined;
+            const routedKey = routing !== undefined ? cache === null || cache === void 0 ? void 0 : cache.cacheKey : undefined;
+            const gatedCache = breakpoints !== undefined || routedKey !== undefined
+                ? { systemBreakpoints: breakpoints, cacheKey: routedKey }
+                : undefined;
             if (usesResponsesApi) {
-                return callOpenAiResponsesCompletion(config, prompt, tools, head, temperature, logger, signal, resolvedThinking, maxTokens, resolvedStructured);
+                return callOpenAiResponsesCompletion(config, prompt, tools, head, temperature, logger, signal, resolvedThinking, maxTokens, resolvedStructured, supportsCacheUsageReporting(descriptor), gatedCache);
             }
-            return callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, usesMaxCompletionTokensField(descriptor), resolvedStructured);
+            return callOpenAiCompletion(config, prompt, head, temperature, logger, signal, resolvedThinking, maxTokens, usesMaxCompletionTokensField(descriptor), resolvedStructured, supportsCacheUsageReporting(descriptor), gatedCache, routing === null || routing === void 0 ? void 0 : routing.chatCompletionsHeader);
+        }
         case 'anthropic':
-            return callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, isAdaptiveThinkingModel(descriptor, config.model), maxTokens, resolvedStructured);
+            return callAnthropicCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, isAdaptiveThinkingModel(descriptor, config.model), maxTokens, resolvedStructured, cache);
         case 'gemini':
             return callGeminiCompletion(config, prompt, head, temperature, logger, tools, signal, resolvedThinking, maxTokens, resolvedStructured);
         /* c8 ignore next 4 - defensive coding: exhaustive switch guaranteed by TypeScript */
@@ -476,7 +548,7 @@ export async function callProviderCompletion(params) {
  * @public
  */
 export async function callProxiedCompletion(proxyUrl, params) {
-    const { descriptor, apiKey, system, messages, temperature, modelOverride, logger, tools, signal, thinking, maxTokens, structuredOutput } = params;
+    const { descriptor, apiKey, system, messages, temperature, modelOverride, logger, tools, signal, thinking, maxTokens, structuredOutput, cache } = params;
     const splitResult = splitChatRequest(system, messages);
     if (splitResult.isFailure()) {
         return fail(splitResult.message);
@@ -523,6 +595,11 @@ export async function callProxiedCompletion(proxyUrl, params) {
                 ? { onUnsupported: structuredOutput.onUnsupported }
                 : {}));
     }
+    // `IAiCacheRequest` is plain numbers and a string — JSON-serializable as-is, unlike
+    // `structuredOutput`'s schema above, so it needs no wire projection before forwarding.
+    if (cache !== undefined) {
+        body.cache = cache;
+    }
     /* c8 ignore next 1 - optional logger */
     logger === null || logger === void 0 ? void 0 : logger.info(`AI proxy request: provider=${descriptor.id}, proxy=${proxyUrl}`);
     const url = `${proxyUrl}/api/ai/completion`;