@falai/agent 2.4.3 → 2.6.0

package/src/providers/GenericOpenAICompatibleProvider.ts

@@ -0,0 +1,138 @@
+/**
+ * Generic OpenAI-compatible provider + factory.
+ *
+ * Most "OpenAI-compatible" endpoints (Azure OpenAI, Groq, Together, Fireworks,
+ * vLLM, LM Studio, Ollama, a self-hosted gateway…) differ from OpenAI only in
+ * base URL, headers, and how structured output is requested. `createOpenAICompatibleProvider`
+ * builds a working provider from that config alone — no subclass required — so
+ * adding one ("fake" provider, same wire protocol, different base URL) is a few lines:
+ *
+ *   const ollama = createOpenAICompatibleProvider({
+ *     name: "ollama",
+ *     baseURL: "http://localhost:11434/v1",
+ *     apiKey: "ollama",            // local servers ignore it; pass any non-empty string
+ *     model: "llama3.3",
+ *   });
+ *
+ *   const azure = createOpenAICompatibleProvider({
+ *     name: "azure",
+ *     baseURL: `https://${resource}.openai.azure.com/openai/deployments/${deployment}`,
+ *     apiKey: process.env.AZURE_OPENAI_KEY!,
+ *     model: deployment,
+ *     defaultHeaders: { "api-key": process.env.AZURE_OPENAI_KEY! },
+ *   });
+ *
+ * For OpenAI/OpenRouter (native `responses.parse`) and Anthropic/Gemini, prefer
+ * their dedicated provider classes.
+ */
+
+import OpenAI from "openai";
+
+import type { ProviderCapabilities } from "../types/ai";
+import {
+  OpenAICompatibleProvider,
+  type OpenAICompatibleRequestConfig,
+  type StructuredOutputMode,
+} from "./OpenAICompatibleProvider";
+
+export interface OpenAICompatibleOptions {
+  /** Provider identifier, e.g. "azure", "ollama", "groq". */
+  name: string;
+  /** Base URL of the OpenAI-compatible endpoint. */
+  baseURL: string;
+  /** API key. Local servers often ignore it — pass any non-empty string. */
+  apiKey: string;
+  /** Primary model / deployment name. */
+  model: string;
+  /** Backup models to try if the primary fails. */
+  backupModels?: string[];
+  /** Human-readable name for error messages. Defaults to `name`. */
+  displayName?: string;
+  /** Uppercase tag for log lines. Defaults to `name.toUpperCase()`. */
+  logLabel?: string;
+  /** Capability overrides, merged over the defaults (all true except caching). */
+  capabilities?: Partial<ProviderCapabilities>;
+  /** Extra request headers (e.g. Azure's `api-key`, a gateway's auth header). */
+  defaultHeaders?: Record<string, string>;
+  /**
+   * How structured output is requested. Defaults to `"json_schema"` — the
+   * broadest enforced mode for arbitrary compatible endpoints (`responses.parse`
+   * is OpenAI-only). See {@link StructuredOutputMode}.
+   */
+  structuredOutput?: StructuredOutputMode;
+  /** Default request parameters merged into every call. */
+  config?: OpenAICompatibleRequestConfig;
+  /** Per-call timeout (ms) and retry count. */
+  retryConfig?: { timeout?: number; retries?: number };
+}
+
+/** Sensible defaults for a modern OpenAI-compatible endpoint. */
+const DEFAULT_CAPABILITIES: ProviderCapabilities = {
+  supportsTools: true,
+  supportsNativeJsonSchema: true,
+  supportsStreaming: true,
+  supportsStreamingToolCalls: true,
+  supportsPromptCaching: false,
+};
+
+/**
+ * Concrete OpenAI-compatible provider configured entirely from
+ * {@link OpenAICompatibleOptions}. Kept private — construct via
+ * {@link createOpenAICompatibleProvider}; subclass {@link OpenAICompatibleProvider}
+ * directly if you need behavior beyond these knobs (e.g. provider-specific
+ * error classification or streaming hooks).
+ */
+class GenericOpenAICompatibleProvider extends OpenAICompatibleProvider {
+  public readonly name: string;
+  public readonly capabilities: ProviderCapabilities;
+  protected readonly logLabel: string;
+  protected readonly displayName: string;
+
+  constructor(options: OpenAICompatibleOptions) {
+    if (!options.name) {
+      throw new Error("An OpenAI-compatible provider needs a `name`.");
+    }
+    if (!options.baseURL) {
+      throw new Error(`[${options.name}] A \`baseURL\` is required.`);
+    }
+    if (!options.apiKey) {
+      throw new Error(
+        `[${options.name}] An \`apiKey\` is required — use any non-empty string for servers that ignore it.`
+      );
+    }
+    if (!options.model) {
+      throw new Error(`[${options.name}] A \`model\` is required.`);
+    }
+
+    super({
+      client: new OpenAI({
+        apiKey: options.apiKey,
+        baseURL: options.baseURL,
+        ...(options.defaultHeaders ? { defaultHeaders: options.defaultHeaders } : {}),
+      }),
+      model: options.model,
+      backupModels: options.backupModels,
+      config: options.config,
+      retryConfig: options.retryConfig,
+      // Arbitrary compatible endpoints rarely have responses.parse; default to
+      // the broadly-supported chat json_schema strategy.
+      structuredOutput: options.structuredOutput ?? "json_schema",
+    });
+
+    this.name = options.name;
+    this.displayName = options.displayName ?? options.name;
+    this.logLabel = options.logLabel ?? options.name.toUpperCase();
+    this.capabilities = { ...DEFAULT_CAPABILITIES, ...options.capabilities };
+  }
+}
+
+/**
+ * Build an `AiProvider` for any OpenAI-compatible endpoint from config alone.
+ * See {@link OpenAICompatibleOptions} for the knobs and the file header for
+ * Azure/Ollama examples.
+ */
+export function createOpenAICompatibleProvider(
+  options: OpenAICompatibleOptions
+): OpenAICompatibleProvider {
+  return new GenericOpenAICompatibleProvider(options);
+}

package/src/providers/OpenAICompatibleProvider.ts

@@ -27,7 +27,7 @@ import type {
 } from "../types";
 import type { ProviderCapabilities } from "../types/ai";
 import type { HistoryItem } from "../types/history";
-import { withTimeoutAndRetry, withStreamRetry, logger } from "../utils";
+import { withTimeoutAndRetry, withStreamRetry, resolveRetryConfig, logger, assertUsableCompletion, combineAbortSignals } from "../utils";
 import {
   classifyProviderError,
   getErrorMessage,
@@ -36,11 +36,6 @@ import {
   type ErrorClassificationOptions,
 } from "./errorClassification";
 
-const DEFAULT_RETRY_CONFIG = {
-  timeout: 60000,
-  retries: 3,
-};
-
 /**
  * Default request parameters shared by OpenAI-compatible providers
  */
@@ -48,6 +43,17 @@ export type OpenAICompatibleRequestConfig = Partial<
   Omit<ChatCompletionCreateParamsNonStreaming, "model" | "messages">
 >;
 
+/**
+ * How structured (JSON-schema) output is requested from the endpoint:
+ * - `"responses_parse"` (default) — OpenAI's native `responses.parse` API
+ *   (OpenAI, OpenRouter). Most compatible endpoints do not implement it.
+ * - `"json_schema"` — chat completions with a `json_schema` response_format.
+ *   The broadest enforced mode (DeepSeek, Groq, Together, Fireworks, vLLM, Azure…).
+ * - `"json_object"` — chat completions with a `json_object` response_format,
+ *   for servers that lack json_schema enforcement (parsed at the end).
+ */
+export type StructuredOutputMode = "responses_parse" | "json_schema" | "json_object";
+
 /**
  * Initialization values supplied by subclasses
  */
@@ -65,6 +71,11 @@ export interface OpenAICompatibleProviderInit {
     timeout?: number;
     retries?: number;
   };
+  /**
+   * How structured output is requested (default `"responses_parse"`). Lifts the
+   * strategy that used to be a per-subclass override into shared config.
+   */
+  structuredOutput?: StructuredOutputMode;
 }
 
 /**
@@ -85,6 +96,8 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
   protected readonly backupModels: string[];
   protected readonly config?: OpenAICompatibleRequestConfig;
   protected readonly retryConfig: { timeout: number; retries: number };
+  /** Structured-output strategy; see {@link StructuredOutputMode}. */
+  protected readonly structuredOutput: StructuredOutputMode;
 
   /**
    * Provider-specific error classification signals.
@@ -99,14 +112,8 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
     this.primaryModel = init.model;
     this.backupModels = init.backupModels ?? [];
     this.config = init.config;
-    this.retryConfig = {
-      // `||` is intentional: a 0ms timeout is degenerate (aborts every call
-      // immediately), so fall back to the default.
-      timeout: init.retryConfig?.timeout || DEFAULT_RETRY_CONFIG.timeout,
-      // `??` so an explicit `retries: 0` (disable retries) is honored instead of
-      // being clobbered to the default by a falsy-zero check.
-      retries: init.retryConfig?.retries ?? DEFAULT_RETRY_CONFIG.retries,
-    };
+    this.retryConfig = resolveRetryConfig(init.retryConfig);
+    this.structuredOutput = init.structuredOutput ?? "responses_parse";
   }
 
   // ---------------------------------------------------------------------
@@ -146,22 +153,31 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
     input: GenerateMessageInput<unknown>,
     jsonSchema: StructuredSchema
   ): Promise<GenerateMessageOutput> {
-    const response = await this.client.responses.parse({
-      model,
-      instructions: input.prompt,
-      input: "",
-      reasoning: {
-        effort: input.parameters?.reasoning?.effort || "low",
-      },
-      text: {
-        format: {
-          type: "json_schema",
-          name: input.parameters?.schemaName || "structured_output",
-          // Adapt common schema format to the provider's format
-          schema: this.adaptSchema(jsonSchema),
+    // Non-native modes go through chat completions, where
+    // `structuredResponseFormat` selects the response_format.
+    if (this.structuredOutput !== "responses_parse") {
+      return this.executeChatCompletion(model, input);
+    }
+
+    const response = await this.client.responses.parse(
+      {
+        model,
+        instructions: input.prompt,
+        input: "",
+        reasoning: {
+          effort: input.parameters?.reasoning?.effort || "low",
+        },
+        text: {
+          format: {
+            type: "json_schema",
+            name: input.parameters?.schemaName || "structured_output",
+            // Adapt common schema format to the provider's format
+            schema: this.adaptSchema(jsonSchema),
+          },
         },
       },
-    });
+      { signal: input.signal }
+    );
 
     if (!response.output_parsed) {
       throw new Error(`No parsed output returned from ${this.displayName}`);
@@ -183,15 +199,25 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
   }
 
   /**
-   * response_format applied to chat-completions requests when a JSON
-   * schema is present. Default requests a generic JSON object (parsed at
-   * the end); subclasses with native json_schema support in chat
-   * completions (e.g. DeepSeek) override this.
+   * response_format applied to chat-completions requests when a JSON schema is
+   * present, selected by {@link StructuredOutputMode}: `"json_schema"` for native
+   * enforcement, else a generic JSON object parsed at the end. (`"responses_parse"`
+   * never reaches chat completions for non-streaming, but streaming has no parse
+   * API, so it falls back to `json_object` here.)
    */
   protected structuredResponseFormat(
-    _jsonSchema: StructuredSchema,
-    _schemaName: string | undefined
+    jsonSchema: StructuredSchema,
+    schemaName: string | undefined
   ): ChatCompletionCreateParamsNonStreaming["response_format"] {
+    if (this.structuredOutput === "json_schema") {
+      return {
+        type: "json_schema" as const,
+        json_schema: {
+          name: schemaName || "structured_output",
+          schema: this.adaptSchema(jsonSchema),
+        },
+      };
+    }
     return { type: "json_object" };
   }
 
@@ -413,8 +439,8 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
     // Try primary model first
     try {
       yield* withStreamRetry(
-        () => this.generateStreamWithModel<TContext, TStructured>(this.primaryModel, input),
-        { maxRetries: this.retryConfig.retries, operationName: `${this.logLabel} ${this.primaryModel} stream` }
+        (signal) => this.generateStreamWithModel<TContext, TStructured>(this.primaryModel, input, signal),
+        { maxRetries: this.retryConfig.retries, firstChunkTimeoutMs: this.retryConfig.timeout, operationName: `${this.logLabel} ${this.primaryModel} stream` }
       );
     } catch (primaryError: unknown) {
       const primaryErrMsg = getErrorMessage(primaryError);
@@ -440,8 +466,8 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
 
         try {
           yield* withStreamRetry(
-            () => this.generateStreamWithModel<TContext, TStructured>(backupModel, input),
-            { maxRetries: this.retryConfig.retries, operationName: `${this.logLabel} ${backupModel} stream` }
+            (signal) => this.generateStreamWithModel<TContext, TStructured>(backupModel, input, signal),
+            { maxRetries: this.retryConfig.retries, firstChunkTimeoutMs: this.retryConfig.timeout, operationName: `${this.logLabel} ${backupModel} stream` }
           );
           logger.debug(`[${this.logLabel}] Backup model ${backupModel} succeeded`);
           return;
@@ -483,16 +509,20 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
     model: string,
     input: GenerateMessageInput<TContext>
   ): Promise<GenerateMessageOutput<TStructured>> {
-    const operation = async (): Promise<GenerateMessageOutput> => {
+    const operation = async (signal: AbortSignal): Promise<GenerateMessageOutput> => {
+      // Thread the per-attempt timeout signal through input.signal so whichever
+      // path runs — responses.parse (structured) or chat.completions — cancels
+      // its upstream call when the deadline fires.
+      const reqInput = { ...input, signal: combineAbortSignals(input.signal, signal) };
       // Use structured output path if JSON schema is provided
-      if (input.parameters?.jsonSchema) {
+      if (reqInput.parameters?.jsonSchema) {
         return this.executeStructuredGenerate(
           model,
-          input,
-          input.parameters.jsonSchema
+          reqInput,
+          reqInput.parameters.jsonSchema
         );
       }
-      return this.executeChatCompletion(model, input);
+      return this.executeChatCompletion(model, reqInput);
     };
 
     return withTimeoutAndRetry(
@@ -540,7 +570,9 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
       );
     }
 
-    const response = await this.client.chat.completions.create(params);
+    const response = await this.client.chat.completions.create(params, {
+      signal: input.signal,
+    });
 
     const message = response.choices[0]?.message?.content || "";
 
@@ -584,16 +616,10 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
       } as AgentStructuredResponse;
     }
 
-    // A parsed-but-blank structured message with no tool calls is just as empty
-    // as no text at all — throw so withTimeoutAndRetry retries instead of
+    // A parsed-but-blank message with no tool calls is as empty as no text;
+    // the shared guard throws so withTimeoutAndRetry retries instead of
     // returning {"message":""}.
-    if (
-      toolCalls.length === 0 &&
-      typeof structured?.message === "string" &&
-      !structured.message.trim()
-    ) {
-      throw new Error(`No response from ${this.displayName}`);
-    }
+    assertUsableCompletion(structured, message, toolCalls.length, this.displayName);
 
     return {
       message,
@@ -617,7 +643,8 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
     TStructured = AgentStructuredResponse
   >(
     model: string,
-    input: GenerateMessageInput<TContext>
+    input: GenerateMessageInput<TContext>,
+    attemptSignal?: AbortSignal
   ): AsyncGenerator<GenerateMessageStreamChunk<TStructured>> {
     // Build messages from history and append prompt as final user message
     const historyMessages = this.buildChatMessages(input.history);
@@ -652,7 +679,9 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
       );
     }
 
-    const stream = await this.client.chat.completions.create(params);
+    const stream = await this.client.chat.completions.create(params, {
+      signal: combineAbortSignals(input.signal, attemptSignal),
+    });
 
     let accumulated = "";
     let currentModel = model;
@@ -735,18 +764,9 @@ export abstract class OpenAICompatibleProvider implements AiProvider {
       } as TStructured;
     }
 
-    // Empty-completion guard — mirror of the non-streaming path. A blank
-    // effective message (structured message under a schema, else accumulated
-    // text) with no tool calls means the model produced nothing usable; throw
-    // so withStreamRetry/generateStreamWithBackup retry instead of silently
-    // emitting an empty message.
-    const structuredMessage = (structured as AgentStructuredResponse | undefined)?.message;
-    const messageText = (
-      typeof structuredMessage === "string" ? structuredMessage : accumulated
-    ).trim();
-    if (!messageText && toolCalls.length === 0) {
-      throw new Error(`No response from ${this.displayName}`);
-    }
+    // Empty-completion guard — same definition as the non-streaming path, so the
+    // stream retries / falls back to backup instead of emitting an empty message.
+    assertUsableCompletion(structured, accumulated, toolCalls.length, this.displayName);
 
     // Yield final chunk
     yield {

package/src/providers/index.ts

@@ -22,8 +22,12 @@ export { OpenAICompatibleProvider } from "./OpenAICompatibleProvider";
 export type {
   OpenAICompatibleProviderInit,
   OpenAICompatibleRequestConfig,
+  StructuredOutputMode,
 } from "./OpenAICompatibleProvider";
 
+export { createOpenAICompatibleProvider } from "./GenericOpenAICompatibleProvider";
+export type { OpenAICompatibleOptions } from "./GenericOpenAICompatibleProvider";
+
 export {
   classifyProviderError,
   isBackupEligible,

package/src/utils/completion.ts

@@ -0,0 +1,38 @@
+/**
+ * The user-facing text of a (possibly structured) model completion, trimmed.
+ *
+ * Under a JSON schema the real message lives in `structured.message`; otherwise
+ * it's the raw/accumulated text passed as `fallbackText`. A blank result with no
+ * tool calls means the model produced nothing usable — providers throw on that
+ * so the retry/backup path runs instead of emitting an empty message. Shared by
+ * the streaming and non-streaming empty-completion guards across all providers.
+ */
+export function effectiveMessageText(
+  structured: unknown,
+  fallbackText: string
+): string {
+  const message =
+    typeof structured === "object" && structured !== null && "message" in structured
+      ? structured.message
+      : undefined;
+  return (typeof message === "string" ? message : fallbackText).trim();
+}
+
+/**
+ * The single definition of "the model produced nothing usable": a blank
+ * effective message ({@link effectiveMessageText}) and no tool calls. When that
+ * holds, throw `No response from <provider>` so the caller's retry/backup path
+ * runs instead of surfacing an empty turn. Lifting this above the providers
+ * keeps the streaming and non-streaming guards — six call sites across three
+ * providers — from drifting on what counts as empty.
+ */
+export function assertUsableCompletion(
+  structured: unknown,
+  fallbackText: string,
+  toolCallCount: number,
+  providerLabel: string
+): void {
+  if (toolCallCount === 0 && !effectiveMessageText(structured, fallbackText)) {
+    throw new Error(`No response from ${providerLabel}`);
+  }
+}

package/src/utils/index.ts

@@ -55,8 +55,11 @@ export {
 export { LoggerLevel, logger } from "./logger";
 
 // Retry utilities
-export type { RetryOptions, StreamRetryOptions } from "./retry";
-export { retry, withTimeoutAndRetry, withStreamRetry } from "./retry";
+export type { RetryOptions, StreamRetryOptions, RetryConfig } from "./retry";
+export { retry, withTimeoutAndRetry, withStreamRetry, resolveRetryConfig, combineAbortSignals } from "./retry";
+
+// Completion helpers
+export { effectiveMessageText, assertUsableCompletion } from "./completion";
 
 // Condition utilities
 export {