@librechat/agents 4.0.1 → 4.0.2

package/src/decisions/structuredChat.ts

@@ -0,0 +1,364 @@
+import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type {
+  DecisionModel,
+  DecisionQuestion,
+  DecisionResult,
+  DecisionRequest,
+} from './types';
+import { DecisionError, isDecisionObject, readDecisionUsage } from './types';
+import { failureForStatus, retryAfterMs } from './transport';
+import { validateDecisionQuestions } from './questions';
+import { DECISION_PROMPT_PREFIX } from './traceMarker';
+import { withDecisionDeadline } from './deadline';
+
+const MAX_QUESTIONS = 32;
+const MAX_CHOICE_OPTIONS = 128;
+const DEFAULT_TIMEOUT_MS = 20_000;
+const PROVIDER_ID = 'structured-chat';
+
+export function readChatUsage(raw: unknown): DecisionResult['usage'] {
+  if (!isDecisionObject(raw)) {
+    return null;
+  }
+  const usage = readDecisionUsage(raw.usage_metadata);
+  const response = isDecisionObject(raw.response_metadata)
+    ? raw.response_metadata
+    : null;
+  const nested = isDecisionObject(response?.metadata)
+    ? response.metadata
+    : null;
+  const bedrock = isDecisionObject(nested?.usage)
+    ? nested.usage
+    : response?.usage;
+  if (
+    usage?.inputTokens === undefined ||
+    !isDecisionObject(bedrock) ||
+    usage.inputTokens !== bedrock.inputTokens
+  ) {
+    return usage;
+  }
+  const read = bedrock.cacheReadInputTokens;
+  const write = bedrock.cacheWriteInputTokens;
+  if (
+    (read != null &&
+      (typeof read !== 'number' || !Number.isSafeInteger(read) || read < 0)) ||
+    (write != null &&
+      (typeof write !== 'number' || !Number.isSafeInteger(write) || write < 0))
+  ) {
+    return usage;
+  }
+  const inputTokens =
+    usage.inputTokens +
+    (typeof read === 'number' ? read : 0) +
+    (typeof write === 'number' ? write : 0);
+  return Number.isSafeInteger(inputTokens) ? { ...usage, inputTokens } : usage;
+}
+
+function providerFailure(error: unknown, provider: string): DecisionError {
+  const details = isDecisionObject(error) ? error : null;
+  const metadata = isDecisionObject(details?.$metadata)
+    ? details.$metadata
+    : null;
+  const rawStatus =
+    details?.status ?? details?.statusCode ?? metadata?.httpStatusCode;
+  const status =
+    typeof rawStatus === 'number' &&
+    Number.isInteger(rawStatus) &&
+    rawStatus >= 100 &&
+    rawStatus <= 599
+      ? rawStatus
+      : undefined;
+  const failure = new DecisionError(
+    status == null ? 'network' : failureForStatus(status),
+    'structured decision model request failed',
+    { provider, status }
+  );
+  const headers = details?.headers;
+  if (headers instanceof Headers) {
+    failure.retryAfterMs = retryAfterMs(headers.get('retry-after'));
+  }
+  return failure;
+}
+
+export interface StructuredChatDecisionModelOptions {
+  /** An already configured chat model whose chosen method enforces strict schemas. */
+  model: BaseChatModel;
+  modelId: string;
+  providerId?: string;
+  /** Verified OpenAI strict modes or Anthropic strict tool calling; other adapters fail closed. */
+  method: 'jsonSchema' | 'functionCalling';
+  timeoutMs?: number;
+  maxQuestions?: number;
+  onAnswered?: (label: string, ms: number) => void;
+}
+
+function decisionSchema(
+  entries: Array<[string, DecisionQuestion]>
+): Record<string, unknown> {
+  const properties: Record<string, object> = Object.create(null);
+  for (const [id, question] of entries) {
+    if (question.type === 'boolean') {
+      properties[id] = {
+        type: 'object',
+        properties: { decision: { type: 'boolean' } },
+        required: ['decision'],
+        additionalProperties: false,
+      };
+      continue;
+    }
+    properties[id] = {
+      type: 'object',
+      properties: {
+        choice: { type: 'string', enum: Object.keys(question.criteria) },
+      },
+      required: ['choice'],
+      additionalProperties: false,
+    };
+  }
+  return {
+    type: 'object',
+    properties: {
+      answers: {
+        type: 'object',
+        properties,
+        required: entries.map(([id]) => id),
+        additionalProperties: false,
+      },
+    },
+    required: ['answers'],
+    additionalProperties: false,
+  };
+}
+
+/** Do not trust even a provider-parsed response: check every required id and choice locally. */
+function readDecisions(
+  parsed: unknown,
+  entries: Array<[string, DecisionQuestion]>,
+  provider: string
+): DecisionResult['answers'] {
+  const invalid = (): DecisionError =>
+    new DecisionError(
+      'malformed_response',
+      'invalid structured decision model answer',
+      {
+        provider,
+      }
+    );
+  if (
+    !isDecisionObject(parsed) ||
+    Object.keys(parsed).length !== 1 ||
+    !isDecisionObject(parsed.answers) ||
+    Object.keys(parsed.answers).length !== entries.length
+  ) {
+    throw invalid();
+  }
+  const answers: DecisionResult['answers'] = Object.create(null);
+  for (const [id, question] of entries) {
+    if (!Object.hasOwn(parsed.answers, id)) {
+      throw invalid();
+    }
+    const raw = parsed.answers[id];
+    if (!isDecisionObject(raw) || Object.keys(raw).length !== 1) {
+      throw invalid();
+    }
+    if (question.type === 'boolean') {
+      if (typeof raw.decision !== 'boolean') {
+        throw invalid();
+      }
+      answers[id] = {
+        type: 'boolean',
+        decision: raw.decision,
+        probability: null,
+      };
+      continue;
+    }
+    if (
+      question.type !== 'choice' ||
+      typeof raw.choice !== 'string' ||
+      !Object.hasOwn(question.criteria, raw.choice)
+    ) {
+      throw invalid();
+    }
+    answers[id] = {
+      type: 'choice',
+      choice: raw.choice,
+      confidence: null,
+      probabilities: null,
+    };
+  }
+  return answers;
+}
+
+function questionsForChat(
+  questions: DecisionRequest['questions'],
+  provider: string,
+  maxQuestions: number
+): Array<[string, DecisionQuestion]> {
+  const entries = validateDecisionQuestions(questions, provider);
+  let choices = 0;
+  for (const [, question] of entries) {
+    if (question.type === 'score') {
+      throw new DecisionError(
+        'unsupported_question',
+        'chat decision cannot score expected values',
+        { provider }
+      );
+    }
+    if (question.type === 'choice') {
+      choices += Object.keys(question.criteria).length;
+    }
+  }
+  if (entries.length > maxQuestions || choices > MAX_CHOICE_OPTIONS) {
+    throw new DecisionError(
+      'unsupported_question',
+      'decision model question batch too large',
+      { provider }
+    );
+  }
+  return entries;
+}
+
+/** Strict provider schema or strict tool calling, without an agent loop or fabricated probabilities. */
+export function createStructuredChatDecisionModel(
+  options: StructuredChatDecisionModelOptions
+): DecisionModel {
+  const provider = options.providerId ?? PROVIDER_ID;
+  if (!['jsonSchema', 'functionCalling'].includes(options.method)) {
+    throw new DecisionError(
+      'unsupported_mode',
+      'unsupported structured decision model mode',
+      {
+        provider,
+      }
+    );
+  }
+  if (!options.modelId.trim()) {
+    throw new DecisionError(
+      'bad_request',
+      'decision model requires a model id',
+      {
+        provider,
+      }
+    );
+  }
+  const maxQuestions = options.maxQuestions ?? MAX_QUESTIONS;
+  if (
+    !Number.isSafeInteger(maxQuestions) ||
+    maxQuestions < 1 ||
+    maxQuestions > MAX_QUESTIONS
+  ) {
+    throw new DecisionError(
+      'bad_request',
+      'invalid decision model question limit',
+      {
+        provider,
+      }
+    );
+  }
+  const { model, method } = options;
+  const modelId = options.modelId.trim();
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const onAnswered = options.onAnswered;
+
+  return {
+    id: provider,
+    model: modelId,
+    async decide(request: DecisionRequest): Promise<DecisionResult> {
+      const started = performance.now();
+      return withDecisionDeadline(
+        provider,
+        request.timeoutMs ?? timeoutMs,
+        request.signal,
+        async (signal, waitFor) => {
+          let entries: Array<[string, DecisionQuestion]>;
+          let input: string;
+          try {
+            entries = questionsForChat(
+              request.questions,
+              provider,
+              maxQuestions
+            );
+            input =
+              DECISION_PROMPT_PREFIX +
+              JSON.stringify({
+                state: request.state,
+                questions: Object.fromEntries(entries),
+              });
+          } catch (error) {
+            if (error instanceof DecisionError) {
+              throw error;
+            }
+            throw new DecisionError(
+              'bad_request',
+              'invalid decision model request',
+              {
+                provider,
+              }
+            );
+          }
+          let structured: ReturnType<typeof model.withStructuredOutput>;
+          try {
+            const adapter = model._llmType();
+            if (
+              adapter !== 'openai' &&
+              (adapter !== 'anthropic' || method !== 'functionCalling')
+            ) {
+              throw new Error('unverified strict decision model adapter');
+            }
+            structured = model.withStructuredOutput(decisionSchema(entries), {
+              name: 'DecideQuestions',
+              method,
+              strict: true,
+              includeRaw: true,
+            });
+          } catch {
+            throw new DecisionError(
+              'unsupported_mode',
+              'model cannot enforce the requested strict mode',
+              {
+                provider,
+              }
+            );
+          }
+          await waitFor(Promise.resolve());
+          let output: Awaited<ReturnType<typeof structured.invoke>>;
+          try {
+            output = await waitFor(
+              structured.invoke(
+                [
+                  new SystemMessage(
+                    'Classify the state using the questions. Return only the required decisions. Do not estimate probabilities, confidence, or token usage.'
+                  ),
+                  new HumanMessage(input),
+                ],
+                { signal }
+              )
+            );
+          } catch (error) {
+            if (error instanceof DecisionError) {
+              throw error;
+            }
+            throw providerFailure(error, provider);
+          }
+          const answers = readDecisions(output.parsed, entries, provider);
+          const result: DecisionResult = {
+            model: modelId,
+            answers,
+            usage: readChatUsage(output.raw),
+          };
+          await waitFor(Promise.resolve());
+          try {
+            onAnswered?.(
+              request.label ?? 'decide',
+              performance.now() - started
+            );
+          } catch {
+            // A callback error must not make a valid decision fail.
+          }
+          return result;
+        }
+      );
+    },
+  };
+}

package/src/decisions/traceMarker.ts

@@ -0,0 +1,2 @@
+/** Marks a JSON decision prompt so trace redaction can safely inspect its nested state. */
+export const DECISION_PROMPT_PREFIX = 'librechat-decision-state:';

package/src/decisions/transport.ts

@@ -0,0 +1,335 @@
+import type { AwaitWithinDeadline } from './deadline';
+import type { DecisionCredential } from './types';
+import { withDecisionDeadline } from './deadline';
+import { DecisionError } from './types';
+
+const DEFAULT_TIMEOUT_MS = 4_000;
+const DEFAULT_MAX_RETRIES = 2;
+const MAX_RESPONSE_BYTES = 256 * 1024;
+const BACKOFF_MS = [250, 750, 1_500, 3_000, 6_000];
+const MAX_RETRY_AFTER_MS = 10_000;
+
+export type DecisionFetch = (
+  input: string,
+  init: {
+    method: string;
+    headers: Record<string, string>;
+    body: string;
+    signal: AbortSignal;
+    redirect: 'error';
+  }
+) => Promise<Pick<Response, 'ok' | 'status' | 'headers' | 'body'>>;
+
+export interface TransportOptions {
+  providerId: string;
+  apiKey?: DecisionCredential;
+  requiresAuth?: boolean;
+  /** Full URL, not a base path. */
+  endpoint: string;
+  timeoutMs?: number;
+  maxRetries?: number;
+  fetch?: DecisionFetch;
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  /** Called only after the response parses, with no state, answers, or credentials. */
+  onAnswered?: (label: string, ms: number) => void;
+}
+
+export type Transport = <T>(
+  prepare: () => { payload: string; parse: (body: string) => T },
+  signal: AbortSignal | undefined,
+  label: string,
+  timeoutOverrideMs?: number
+) => Promise<T>;
+
+export function failureForStatus(
+  status: number
+): 'unauthorized' | 'rate_limited' | 'server_error' | 'bad_request' {
+  if (status === 401 || status === 403) {
+    return 'unauthorized';
+  }
+  if (status === 429) {
+    return 'rate_limited';
+  }
+  if (status >= 500) {
+    return 'server_error';
+  }
+  return 'bad_request';
+}
+
+export function retryAfterMs(header: string | null): number | undefined {
+  if (header == null || header === '') {
+    return undefined;
+  }
+  const seconds = Number(header);
+  if (Number.isFinite(seconds) && seconds >= 0) {
+    return Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS);
+  }
+  const at = Date.parse(header);
+  return Number.isFinite(at)
+    ? Math.min(Math.max(at - Date.now(), 0), MAX_RETRY_AFTER_MS)
+    : undefined;
+}
+
+function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted === true) {
+    return Promise.resolve();
+  }
+  return new Promise((resolve) => {
+    const done = (): void => {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', done);
+      resolve();
+    };
+    const timer = setTimeout(done, ms);
+    signal?.addEventListener('abort', done, { once: true });
+  });
+}
+
+/** Stream successful bodies only, so neither errors nor oversized replies are ever buffered. */
+async function readResponse(
+  body: ReadableStream<Uint8Array> | null,
+  waitFor: AwaitWithinDeadline,
+  provider: string
+): Promise<string> {
+  if (!body) {
+    return '';
+  }
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let bytes = 0;
+  let text = '';
+  try {
+    for (;;) {
+      const chunk = await waitFor(reader.read());
+      if (chunk.done) {
+        return text + decoder.decode();
+      }
+      bytes += chunk.value.byteLength;
+      if (bytes > MAX_RESPONSE_BYTES) {
+        throw new DecisionError(
+          'malformed_response',
+          'decision model response too large',
+          {
+            provider,
+          }
+        );
+      }
+      text += decoder.decode(chunk.value, { stream: true });
+    }
+  } catch (error) {
+    void reader.cancel().catch(() => {});
+    throw error;
+  } finally {
+    try {
+      reader.releaseLock();
+    } catch {
+      // A timed-out read still holds the lock until cancellation settles.
+    }
+  }
+}
+
+/** One immutable transport; every call owns its signal, deadline, and credentials. */
+export function createTransport(options: TransportOptions): Transport {
+  const { providerId } = options;
+  const credential = options.apiKey;
+  if (
+    options.requiresAuth !== false &&
+    (credential == null ||
+      (typeof credential === 'string' && credential.trim() === ''))
+  ) {
+    throw new DecisionError(
+      'unauthorized',
+      'decision model requires an API key',
+      {
+        provider: providerId,
+      }
+    );
+  }
+  const endpoint = options.endpoint.trim();
+  if (!endpoint) {
+    throw new DecisionError(
+      'bad_request',
+      'decision model requires an endpoint',
+      {
+        provider: providerId,
+      }
+    );
+  }
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
+  const sleep = options.sleep ?? defaultSleep;
+  const onAnswered = options.onAnswered;
+  if (!Number.isSafeInteger(maxRetries) || maxRetries < 0 || maxRetries > 5) {
+    throw new DecisionError(
+      'bad_request',
+      'invalid decision model retry limit',
+      {
+        provider: providerId,
+      }
+    );
+  }
+  const fetchImpl: DecisionFetch = options.fetch ?? globalThis.fetch;
+  if (typeof fetchImpl !== 'function') {
+    throw new DecisionError('network', 'no fetch implementation available', {
+      provider: providerId,
+    });
+  }
+
+  async function resolveKey(
+    refresh: boolean,
+    signal: AbortSignal
+  ): Promise<string | null> {
+    if (typeof credential === 'string') {
+      return credential.trim() || null;
+    }
+    if (!credential) {
+      return null;
+    }
+    const minted: string = await credential({ refresh, signal });
+    const token = typeof minted === 'string' ? minted.trim() : '';
+    if (!token) {
+      throw new DecisionError(
+        'unauthorized',
+        'credential function returned no token',
+        {
+          provider: providerId,
+        }
+      );
+    }
+    return token;
+  }
+
+  return async function send(prepare, callerSignal, label, timeoutOverrideMs) {
+    const started = performance.now();
+    return withDecisionDeadline(
+      providerId,
+      timeoutOverrideMs ?? timeoutMs,
+      callerSignal,
+      async (signal, waitFor) => {
+        let prepared: ReturnType<typeof prepare>;
+        try {
+          prepared = prepare();
+        } catch (error) {
+          if (error instanceof DecisionError) {
+            throw error;
+          }
+          throw new DecisionError(
+            'bad_request',
+            'invalid decision model request',
+            {
+              provider: providerId,
+            }
+          );
+        }
+        const { payload, parse } = prepared;
+        await waitFor(Promise.resolve());
+        const wasAborted = (): boolean => signal.aborted;
+        const interrupted = (): DecisionError =>
+          new DecisionError(
+            callerSignal?.aborted === true ? 'aborted' : 'timeout',
+            'decision model request interrupted',
+            { provider: providerId }
+          );
+        let refreshed = false;
+        let key: string | null | undefined;
+        let lastError: DecisionError | undefined;
+        for (let attemptNo = 0; attemptNo <= maxRetries; attemptNo++) {
+          if (wasAborted()) {
+            throw interrupted();
+          }
+          try {
+            if (key === undefined) {
+              key = await waitFor(resolveKey(refreshed, signal));
+            }
+            const response = await waitFor(
+              fetchImpl(endpoint, {
+                method: 'POST',
+                headers: {
+                  ...(key !== null ? { Authorization: `Bearer ${key}` } : {}),
+                  'Content-Type': 'application/json',
+                },
+                body: payload,
+                signal,
+                redirect: 'error',
+              })
+            );
+            if (!response.ok) {
+              void response.body?.cancel().catch(() => {});
+              const error = new DecisionError(
+                failureForStatus(response.status),
+                `decision model returned HTTP ${response.status}`,
+                { provider: providerId, status: response.status }
+              );
+              error.retryAfterMs = retryAfterMs(
+                response.headers.get('retry-after')
+              );
+              throw error;
+            }
+            const body = await readResponse(response.body, waitFor, providerId);
+            const result = await waitFor(Promise.resolve(parse(body)));
+            try {
+              onAnswered?.(label, performance.now() - started);
+            } catch {
+              // Observability must not turn a valid answer into a retried request.
+            }
+            return result;
+          } catch (error) {
+            if (wasAborted()) {
+              throw interrupted();
+            }
+            if (error instanceof DecisionError) {
+              lastError = error;
+            } else {
+              lastError = new DecisionError(
+                'network',
+                'decision model request failed',
+                { provider: providerId }
+              );
+            }
+            if (
+              lastError.status === 401 &&
+              typeof credential === 'function' &&
+              !refreshed
+            ) {
+              refreshed = true;
+              key = undefined;
+              attemptNo -= 1;
+              continue;
+            }
+            if (
+              attemptNo === maxRetries ||
+              !['rate_limited', 'server_error', 'network'].includes(
+                lastError.failure
+              )
+            ) {
+              break;
+            }
+            const wait =
+              lastError.retryAfterMs ??
+              BACKOFF_MS[Math.min(attemptNo, BACKOFF_MS.length - 1)];
+            try {
+              await waitFor(sleep(wait, signal));
+            } catch (error) {
+              if (error instanceof DecisionError) {
+                throw error;
+              }
+              throw new DecisionError(
+                'network',
+                'decision model backoff failed',
+                {
+                  provider: providerId,
+                }
+              );
+            }
+          }
+        }
+        throw (
+          lastError ??
+          new DecisionError('network', 'decision model request failed', {
+            provider: providerId,
+          })
+        );
+      }
+    );
+  };
+}