@falai/agent 0.3.10 → 0.3.11

package/src/providers/OpenAIProvider.ts

@@ -9,6 +9,7 @@ import type {
   AiProvider,
   GenerateMessageInput,
   GenerateMessageOutput,
+  GenerateMessageStreamChunk,
   AgentStructuredResponse,
 } from "../types/ai";
 import { withTimeoutAndRetry } from "../utils/retry";
@@ -164,6 +165,12 @@ export class OpenAIProvider implements AiProvider {
     return this.generateWithBackup(input);
   }
 
+  async *generateMessageStream<TContext = unknown>(
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    yield* this.generateStreamWithBackup(input);
+  }
+
   private async generateWithBackup<TContext = unknown>(
     input: GenerateMessageInput<TContext>
   ): Promise<GenerateMessageOutput> {
@@ -353,4 +360,154 @@ export class OpenAIProvider implements AiProvider {
       `OpenAI ${model}`
     );
   }
+
+  private async *generateStreamWithBackup<TContext = unknown>(
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    // Try primary model first
+    try {
+      yield* this.generateStreamWithModel(this.primaryModel, input);
+    } catch (primaryError: unknown) {
+      const primaryErrMsg = getErrorMessage(primaryError);
+      console.warn(
+        `[OPENAI] Primary model ${this.primaryModel} failed: ${primaryErrMsg}`
+      );
+
+      if (!shouldUseBackupModel(primaryError)) {
+        throw primaryError;
+      }
+
+      console.log(`[OPENAI] Trying backup models for streaming`);
+
+      let lastBackupError: unknown = primaryError;
+
+      for (let i = 0; i < this.backupModels.length; i++) {
+        const backupModel = this.backupModels[i];
+        console.log(
+          `[OPENAI] Trying backup model ${i + 1}/${
+            this.backupModels.length
+          }: ${backupModel}`
+        );
+
+        try {
+          yield* this.generateStreamWithModel(backupModel, input);
+          console.log(`[OPENAI] Backup model ${backupModel} succeeded`);
+          return;
+        } catch (backupError: unknown) {
+          const backupErrMsg = getErrorMessage(backupError);
+          console.warn(
+            `[OPENAI] Backup model ${backupModel} failed: ${backupErrMsg}`
+          );
+          lastBackupError = backupError;
+
+          if (
+            !shouldUseBackupModel(backupError) &&
+            i < this.backupModels.length - 1
+          ) {
+            console.log(
+              `[OPENAI] Backup model error doesn't qualify for further attempts`
+            );
+            break;
+          }
+        }
+      }
+
+      const lastBackupErrMsg = getErrorMessage(lastBackupError);
+      console.error(
+        `[OPENAI] All models failed. Primary: ${primaryErrMsg}, Last backup: ${lastBackupErrMsg}`
+      );
+      throw lastBackupError;
+    }
+  }
+
+  private async *generateStreamWithModel<TContext = unknown>(
+    model: string,
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    const params = {
+      ...this.config,
+      model,
+      messages: [
+        {
+          role: "user" as const,
+          content: input.prompt,
+        },
+      ],
+      stream: true as const,
+    };
+
+    // Override with input parameters if provided
+    if (input.parameters?.maxOutputTokens !== undefined) {
+      params.max_tokens = input.parameters.maxOutputTokens;
+    }
+
+    // Use JSON mode if requested
+    // Note: OpenAI streaming doesn't support the responses.parse API,
+    // so we use response_format with JSON mode instead
+    if (input.parameters?.jsonMode) {
+      params.response_format = { type: "json_object" };
+    }
+
+    const stream = await this.client.chat.completions.create(params);
+
+    let accumulated = "";
+    let currentModel = model;
+    let finishReason: string | undefined;
+    let promptTokens: number | undefined;
+    let completionTokens: number | undefined;
+    let totalTokens: number | undefined;
+
+    for await (const chunk of stream) {
+      currentModel = chunk.model;
+      const delta = chunk.choices[0]?.delta?.content || "";
+
+      if (delta) {
+        accumulated += delta;
+        yield {
+          delta,
+          accumulated,
+          done: false,
+        };
+      }
+
+      if (chunk.choices[0]?.finish_reason) {
+        finishReason = chunk.choices[0].finish_reason;
+      }
+
+      // OpenAI includes usage in the final chunk for some models
+      if (chunk.usage) {
+        promptTokens = chunk.usage.prompt_tokens;
+        completionTokens = chunk.usage.completion_tokens;
+        totalTokens = chunk.usage.total_tokens;
+      }
+    }
+
+    // Parse JSON response if JSON mode was enabled
+    let structured: AgentStructuredResponse | undefined;
+    if (input.parameters?.jsonMode && accumulated) {
+      try {
+        structured = JSON.parse(accumulated) as AgentStructuredResponse;
+      } catch (error) {
+        console.warn(
+          "[OPENAI] Failed to parse JSON response in stream:",
+          error
+        );
+      }
+    }
+
+    // Yield final chunk
+    yield {
+      delta: "",
+      accumulated,
+      done: true,
+      metadata: {
+        model: currentModel,
+        finishReason,
+        tokensUsed: totalTokens,
+        promptTokens,
+        completionTokens,
+      },
+      structured,
+    };
+  }
 }

package/src/providers/OpenRouterProvider.ts

@@ -11,6 +11,7 @@ import type {
   AgentStructuredResponse,
   GenerateMessageInput,
   GenerateMessageOutput,
+  GenerateMessageStreamChunk,
 } from "../types/ai";
 import { withTimeoutAndRetry } from "../utils/retry";
 
@@ -174,6 +175,12 @@ export class OpenRouterProvider implements AiProvider {
     return this.generateWithBackup(input);
   }
 
+  async *generateMessageStream<TContext = unknown>(
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    yield* this.generateStreamWithBackup(input);
+  }
+
   private async generateWithBackup<TContext = unknown>(
     input: GenerateMessageInput<TContext>
   ): Promise<GenerateMessageOutput> {
@@ -363,4 +370,154 @@ export class OpenRouterProvider implements AiProvider {
       `OpenRouter ${model}`
     );
   }
+
+  private async *generateStreamWithBackup<TContext = unknown>(
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    // Try primary model first
+    try {
+      yield* this.generateStreamWithModel(this.primaryModel, input);
+    } catch (primaryError: unknown) {
+      const primaryErrMsg = getErrorMessage(primaryError);
+      console.warn(
+        `[OPENROUTER] Primary model ${this.primaryModel} failed: ${primaryErrMsg}`
+      );
+
+      if (!shouldUseBackupModel(primaryError)) {
+        throw primaryError;
+      }
+
+      console.log(`[OPENROUTER] Trying backup models for streaming`);
+
+      let lastBackupError: unknown = primaryError;
+
+      for (let i = 0; i < this.backupModels.length; i++) {
+        const backupModel = this.backupModels[i];
+        console.log(
+          `[OPENROUTER] Trying backup model ${i + 1}/${
+            this.backupModels.length
+          }: ${backupModel}`
+        );
+
+        try {
+          yield* this.generateStreamWithModel(backupModel, input);
+          console.log(`[OPENROUTER] Backup model ${backupModel} succeeded`);
+          return;
+        } catch (backupError: unknown) {
+          const backupErrMsg = getErrorMessage(backupError);
+          console.warn(
+            `[OPENROUTER] Backup model ${backupModel} failed: ${backupErrMsg}`
+          );
+          lastBackupError = backupError;
+
+          if (
+            !shouldUseBackupModel(backupError) &&
+            i < this.backupModels.length - 1
+          ) {
+            console.log(
+              `[OPENROUTER] Backup model error doesn't qualify for further attempts`
+            );
+            break;
+          }
+        }
+      }
+
+      const lastBackupErrMsg = getErrorMessage(lastBackupError);
+      console.error(
+        `[OPENROUTER] All models failed. Primary: ${primaryErrMsg}, Last backup: ${lastBackupErrMsg}`
+      );
+      throw lastBackupError;
+    }
+  }
+
+  private async *generateStreamWithModel<TContext = unknown>(
+    model: string,
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk> {
+    const params = {
+      ...this.config,
+      model,
+      messages: [
+        {
+          role: "user" as const,
+          content: input.prompt,
+        },
+      ],
+      stream: true as const,
+    };
+
+    // Override with input parameters if provided
+    if (input.parameters?.maxOutputTokens !== undefined) {
+      params.max_tokens = input.parameters.maxOutputTokens;
+    }
+
+    // Use JSON mode if requested
+    // Note: OpenRouter streaming doesn't support the responses.parse API,
+    // so we use response_format with JSON mode instead
+    if (input.parameters?.jsonMode) {
+      params.response_format = { type: "json_object" };
+    }
+
+    const stream = await this.client.chat.completions.create(params);
+
+    let accumulated = "";
+    let currentModel = model;
+    let finishReason: string | undefined;
+    let promptTokens: number | undefined;
+    let completionTokens: number | undefined;
+    let totalTokens: number | undefined;
+
+    for await (const chunk of stream) {
+      currentModel = chunk.model;
+      const delta = chunk.choices[0]?.delta?.content || "";
+
+      if (delta) {
+        accumulated += delta;
+        yield {
+          delta,
+          accumulated,
+          done: false,
+        };
+      }
+
+      if (chunk.choices[0]?.finish_reason) {
+        finishReason = chunk.choices[0].finish_reason;
+      }
+
+      // OpenRouter may include usage in the final chunk
+      if (chunk.usage) {
+        promptTokens = chunk.usage.prompt_tokens;
+        completionTokens = chunk.usage.completion_tokens;
+        totalTokens = chunk.usage.total_tokens;
+      }
+    }
+
+    // Parse JSON response if JSON mode was enabled
+    let structured: AgentStructuredResponse | undefined;
+    if (input.parameters?.jsonMode && accumulated) {
+      try {
+        structured = JSON.parse(accumulated) as AgentStructuredResponse;
+      } catch (error) {
+        console.warn(
+          "[OPENROUTER] Failed to parse JSON response in stream:",
+          error
+        );
+      }
+    }
+
+    // Yield final chunk
+    yield {
+      delta: "",
+      accumulated,
+      done: true,
+      metadata: {
+        model: currentModel,
+        finishReason,
+        tokensUsed: totalTokens,
+        promptTokens,
+        completionTokens,
+      },
+      structured,
+    };
+  }
 }

package/src/providers/index.ts

@@ -0,0 +1,16 @@
+/**
+ * AI Provider exports
+ * Centralized export point for all AI provider implementations
+ */
+
+export { AnthropicProvider } from "./AnthropicProvider";
+export type { AnthropicProviderOptions } from "./AnthropicProvider";
+
+export { GeminiProvider } from "./GeminiProvider";
+export type { GeminiProviderOptions } from "./GeminiProvider";
+
+export { OpenAIProvider } from "./OpenAIProvider";
+export type { OpenAIProviderOptions } from "./OpenAIProvider";
+
+export { OpenRouterProvider } from "./OpenRouterProvider";
+export type { OpenRouterProviderOptions } from "./OpenRouterProvider";

package/src/types/ai.ts

@@ -94,6 +94,31 @@ export interface GenerateMessageOutput {
   structured?: AgentStructuredResponse;
 }
 
+/**
+ * Stream chunk from AI message generation
+ */
+export interface GenerateMessageStreamChunk {
+  /** The delta/chunk of the message */
+  delta: string;
+  /** Accumulated message so far */
+  accumulated: string;
+  /** Whether this is the final chunk */
+  done: boolean;
+  /** Optional metadata about generation */
+  metadata?: {
+    /** Model used */
+    model?: string;
+    /** Tokens consumed */
+    tokensUsed?: number;
+    /** Finish reason */
+    finishReason?: string;
+    /** Additional provider-specific data */
+    [key: string]: unknown;
+  };
+  /** Structured response data (only available when done=true and JSON mode is enabled) */
+  structured?: AgentStructuredResponse;
+}
+
 /**
  * AI provider interface (strategy pattern)
  */
@@ -107,4 +132,11 @@ export interface AiProvider {
   generateMessage<TContext = unknown>(
     input: GenerateMessageInput<TContext>
   ): Promise<GenerateMessageOutput>;
+
+  /**
+   * Generate a message as a stream based on prompt and context
+   */
+  generateMessageStream<TContext = unknown>(
+    input: GenerateMessageInput<TContext>
+  ): AsyncGenerator<GenerateMessageStreamChunk>;
 }