@vellumai/assistant 0.3.0

package/src/providers/openai/client.ts

@@ -0,0 +1,337 @@
+import OpenAI from 'openai';
+import type {
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  Message,
+  ToolDefinition,
+  ContentBlock,
+} from '../types.js';
+import { ProviderError } from '../../util/errors.js';
+import { createStreamTimeout } from '../stream-timeout.js';
+import { escapeXmlAttr } from '../../util/xml.js';
+
+export interface OpenAICompatibleProviderOptions {
+  baseURL?: string;
+  providerName?: string;
+  providerLabel?: string;
+  streamTimeoutMs?: number;
+}
+
+const OPENAI_SUPPORTED_IMAGE_TYPES = new Set([
+  'image/jpeg', 'image/png', 'image/gif', 'image/webp',
+]);
+
+export class OpenAIProvider implements Provider {
+  public readonly name: string;
+  private readonly providerLabel: string;
+  private client: OpenAI;
+  private model: string;
+  private streamTimeoutMs: number;
+
+  constructor(apiKey: string, model: string, options: OpenAICompatibleProviderOptions = {}) {
+    this.name = options.providerName ?? 'openai';
+    this.providerLabel = options.providerLabel ?? 'OpenAI';
+    this.client = new OpenAI({
+      apiKey,
+      baseURL: options.baseURL,
+    });
+    this.model = model;
+    this.streamTimeoutMs = options.streamTimeoutMs ?? 300_000;
+  }
+
+  async sendMessage(
+    messages: Message[],
+    tools?: ToolDefinition[],
+    systemPrompt?: string,
+    options?: SendMessageOptions,
+  ): Promise<ProviderResponse> {
+    const { config, onEvent, signal } = options ?? {};
+    const configObj = config as Record<string, unknown> | undefined;
+    const maxTokens = configObj?.max_tokens as number | undefined;
+    const modelOverride = configObj?.model as string | undefined;
+
+    try {
+      const openaiMessages = this.toOpenAIMessages(messages, systemPrompt);
+
+      const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
+        model: modelOverride ?? this.model,
+        messages: openaiMessages,
+        stream: true as const,
+        stream_options: { include_usage: true },
+      };
+
+      if (maxTokens) {
+        params.max_completion_tokens = maxTokens;
+      }
+
+      if (tools && tools.length > 0) {
+        params.tools = tools.map((t) => ({
+          type: 'function' as const,
+          function: {
+            name: t.name,
+            description: t.description,
+            parameters: t.input_schema as OpenAI.FunctionParameters,
+          },
+        }));
+      }
+
+      const { signal: timeoutSignal, cleanup: cleanupTimeout } = createStreamTimeout(this.streamTimeoutMs, signal);
+
+      // Accumulate the response from chunks
+      let contentText = '';
+      const toolCallMap = new Map<number, { id: string; name: string; args: string }>();
+      let finishReason = 'unknown';
+      let responseModel = modelOverride ?? this.model;
+      let promptTokens = 0;
+      let completionTokens = 0;
+
+      try {
+        const stream = await this.client.chat.completions.create(params, { signal: timeoutSignal });
+
+        for await (const chunk of stream) {
+          const choice = chunk.choices[0];
+          if (choice) {
+            if (choice.delta.content) {
+              contentText += choice.delta.content;
+              onEvent?.({ type: 'text_delta', text: choice.delta.content });
+            }
+
+            if (choice.delta.tool_calls) {
+              for (const tc of choice.delta.tool_calls) {
+                if (!toolCallMap.has(tc.index)) {
+                  toolCallMap.set(tc.index, { id: '', name: '', args: '' });
+                }
+                const entry = toolCallMap.get(tc.index)!;
+                if (tc.id) entry.id = tc.id;
+                if (tc.function?.name) entry.name += tc.function.name;
+                if (tc.function?.arguments) entry.args += tc.function.arguments;
+              }
+            }
+
+            if (choice.finish_reason) {
+              finishReason = choice.finish_reason;
+            }
+          }
+
+          if (chunk.usage) {
+            promptTokens = chunk.usage.prompt_tokens;
+            completionTokens = chunk.usage.completion_tokens;
+          }
+
+          responseModel = chunk.model;
+        }
+      } finally {
+        cleanupTimeout();
+      }
+
+      // Build content blocks
+      const content: ContentBlock[] = [];
+      if (contentText) {
+        content.push({ type: 'text', text: contentText });
+      }
+      for (const [, tc] of toolCallMap) {
+        let input: Record<string, unknown>;
+        try {
+          input = JSON.parse(tc.args);
+        } catch {
+          input = { _raw: tc.args };
+        }
+        content.push({
+          type: 'tool_use',
+          id: tc.id,
+          name: tc.name,
+          input,
+        });
+      }
+
+      // Build a synthetic response object from accumulated streaming data
+      const rawResponse = {
+        model: responseModel,
+        choices: [{
+          message: {
+            role: 'assistant',
+            content: contentText || null,
+            tool_calls: toolCallMap.size > 0
+              ? Array.from(toolCallMap.values()).map((tc) => ({
+                  id: tc.id,
+                  type: 'function',
+                  function: { name: tc.name, arguments: tc.args },
+                }))
+              : undefined,
+          },
+          finish_reason: finishReason,
+        }],
+        usage: { prompt_tokens: promptTokens, completion_tokens: completionTokens },
+      };
+
+      return {
+        content,
+        model: responseModel,
+        usage: { inputTokens: promptTokens, outputTokens: completionTokens },
+        stopReason: finishReason,
+        rawRequest: params,
+        rawResponse,
+      };
+    } catch (error) {
+      if (error instanceof OpenAI.APIError) {
+        throw new ProviderError(
+          `${this.providerLabel} API error (${error.status}): ${error.message}`,
+          this.name,
+          error.status,
+        );
+      }
+      throw new ProviderError(
+        `${this.providerLabel} request failed: ${error instanceof Error ? error.message : String(error)}`,
+        this.name,
+        undefined,
+        { cause: error },
+      );
+    }
+  }
+
+  /** Convert neutral messages + system prompt to OpenAI message format. */
+  private toOpenAIMessages(
+    messages: Message[],
+    systemPrompt?: string,
+  ): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
+    const result: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [];
+
+    if (systemPrompt) {
+      result.push({ role: 'system', content: systemPrompt });
+    }
+
+    for (const msg of messages) {
+      if (msg.role === 'assistant') {
+        result.push(this.toOpenAIAssistantMessage(msg));
+      } else {
+        // User messages may contain tool_result blocks mixed with text/image
+        const toolResults = msg.content.filter(
+          (b): b is Extract<ContentBlock, { type: 'tool_result' }> => b.type === 'tool_result',
+        );
+        const otherBlocks = msg.content.filter(
+          (b) => b.type !== 'tool_result' && b.type !== 'thinking' && b.type !== 'redacted_thinking',
+        );
+
+        // Emit tool results as separate tool-role messages
+        // OpenAI's API only supports string content in tool messages, so images
+        // from contentBlocks are collected and injected as a user message below.
+        const toolResultImages: ContentBlock[] = [];
+        for (const tr of toolResults) {
+          let textContent = tr.content;
+          if (tr.contentBlocks && tr.contentBlocks.length > 0) {
+            const extraText = tr.contentBlocks
+              .filter((cb): cb is Extract<ContentBlock, { type: 'text' }> => cb.type === 'text')
+              .map((cb) => cb.text);
+            if (extraText.length > 0) {
+              textContent = textContent + '\n' + extraText.join('\n');
+            }
+            for (const cb of tr.contentBlocks) {
+              if (cb.type === 'image') toolResultImages.push(cb);
+            }
+          }
+          result.push({
+            role: 'tool',
+            tool_call_id: tr.tool_use_id,
+            content: tr.is_error ? `[ERROR] ${textContent}` : textContent,
+          });
+        }
+
+        // Emit remaining content + any tool result images as a user message.
+        // Images from tool results (e.g. browser_screenshot) must go in a user
+        // message because OpenAI-compatible APIs don't support images in tool messages.
+        const userContent = [...otherBlocks, ...toolResultImages];
+        if (userContent.length > 0) {
+          result.push(this.toOpenAIUserMessage(userContent));
+        }
+      }
+    }
+
+    return result;
+  }
+
+  /** Convert an assistant message with text + tool_use blocks to OpenAI format. */
+  private toOpenAIAssistantMessage(
+    msg: Message,
+  ): OpenAI.Chat.Completions.ChatCompletionAssistantMessageParam {
+    const textParts: string[] = [];
+    const toolCalls: OpenAI.Chat.Completions.ChatCompletionMessageToolCall[] = [];
+
+    for (const block of msg.content) {
+      switch (block.type) {
+        case 'text':
+          textParts.push(block.text);
+          break;
+        case 'tool_use':
+          toolCalls.push({
+            id: block.id,
+            type: 'function',
+            function: {
+              name: block.name,
+              arguments: JSON.stringify(block.input),
+            },
+          });
+          break;
+        // thinking, redacted_thinking, image — not applicable for OpenAI assistant messages
+      }
+    }
+
+    const result: OpenAI.Chat.Completions.ChatCompletionAssistantMessageParam = {
+      role: 'assistant',
+      content: textParts.length > 0 ? textParts.join('') : null,
+    };
+
+    if (toolCalls.length > 0) {
+      result.tool_calls = toolCalls;
+    }
+
+    return result;
+  }
+
+  /** Convert user content blocks (text, image) to an OpenAI user message. */
+  private toOpenAIUserMessage(
+    blocks: ContentBlock[],
+  ): OpenAI.Chat.Completions.ChatCompletionUserMessageParam {
+    // If only a single text block, use plain string (simpler, fewer tokens)
+    if (blocks.length === 1 && blocks[0].type === 'text') {
+      return { role: 'user', content: blocks[0].text };
+    }
+
+    const parts: OpenAI.Chat.Completions.ChatCompletionContentPart[] = [];
+    for (const block of blocks) {
+      switch (block.type) {
+        case 'text':
+          parts.push({ type: 'text', text: block.text });
+          break;
+        case 'image':
+          if (!OPENAI_SUPPORTED_IMAGE_TYPES.has(block.source.media_type)) {
+            parts.push({ type: 'text', text: `[Image: ${block.source.media_type} — format not supported by this provider]` });
+          } else {
+            parts.push({
+              type: 'image_url',
+              image_url: {
+                url: `data:${block.source.media_type};base64,${block.source.data}`,
+              },
+            });
+          }
+          break;
+        case 'file':
+          parts.push({
+            type: 'text',
+            text: this.fileBlockToText(block),
+          });
+          break;
+      }
+    }
+
+    return { role: 'user', content: parts };
+  }
+
+  private fileBlockToText(block: Extract<ContentBlock, { type: 'file' }>): string {
+    const header = `<attached_file name="${escapeXmlAttr(block.source.filename)}" type="${escapeXmlAttr(block.source.media_type)}" />`;
+    if (block.extracted_text && block.extracted_text.trim().length > 0) {
+      return `${header}\n${block.extracted_text}`;
+    }
+    return `${header}\nNo extracted text available.`;
+  }
+}

package/src/providers/openrouter/client.ts

@@ -0,0 +1,20 @@
+import { OpenAIProvider } from '../openai/client.js';
+
+export interface OpenRouterProviderOptions {
+  apiKey?: string;
+  baseURL?: string;
+  streamTimeoutMs?: number;
+}
+
+const DEFAULT_OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
+
+export class OpenRouterProvider extends OpenAIProvider {
+  constructor(apiKey: string, model: string, options: OpenRouterProviderOptions = {}) {
+    super(apiKey, model, {
+      baseURL: options.baseURL?.trim() || DEFAULT_OPENROUTER_BASE_URL,
+      providerName: 'openrouter',
+      providerLabel: 'OpenRouter',
+      streamTimeoutMs: options.streamTimeoutMs,
+    });
+  }
+}

package/src/providers/ratelimit.ts

@@ -0,0 +1,93 @@
+import type { Provider, ProviderResponse, SendMessageOptions, Message, ToolDefinition } from './types.js';
+import type { RateLimitConfig } from '../config/types.js';
+import { RateLimitError } from '../util/errors.js';
+import { getLogger } from '../util/logger.js';
+
+const log = getLogger('rate-limit');
+
+export class RateLimitProvider implements Provider {
+  public readonly name: string;
+
+  private requestTimestamps: number[];
+  private sessionTokens = 0;
+
+  constructor(
+    private readonly inner: Provider,
+    private readonly config: RateLimitConfig,
+    sharedRequestTimestamps?: number[],
+  ) {
+    this.name = inner.name;
+    this.requestTimestamps = sharedRequestTimestamps ?? [];
+  }
+
+  async sendMessage(
+    messages: Message[],
+    tools?: ToolDefinition[],
+    systemPrompt?: string,
+    options?: SendMessageOptions,
+  ): Promise<ProviderResponse> {
+    this.enforceRequestRate();
+    this.enforceTokenBudget();
+
+    // Record the request timestamp before the await to prevent concurrent
+    // calls from bypassing the rate limit during the async gap.
+    this.recordRequest();
+
+    const response = await this.inner.sendMessage(messages, tools, systemPrompt, options);
+
+    this.recordTokens(response.usage.inputTokens + response.usage.outputTokens);
+
+    return response;
+  }
+
+  private enforceRequestRate(): void {
+    const limit = this.config.maxRequestsPerMinute;
+    if (limit <= 0) return;
+
+    const now = Date.now();
+    const windowStart = now - 60_000;
+    // Prune expired timestamps in-place to preserve the shared array
+    // reference. Single-pass compaction: copy valid entries to the front,
+    // track the oldest surviving entry, and truncate — all in O(n).
+    let write = 0;
+    let oldestInWindow = Infinity;
+    for (let read = 0; read < this.requestTimestamps.length; read++) {
+      if (this.requestTimestamps[read] > windowStart) {
+        if (this.requestTimestamps[read] < oldestInWindow) {
+          oldestInWindow = this.requestTimestamps[read];
+        }
+        this.requestTimestamps[write++] = this.requestTimestamps[read];
+      }
+    }
+    this.requestTimestamps.length = write;
+
+    if (this.requestTimestamps.length >= limit) {
+      const waitSec = Math.ceil((oldestInWindow + 60_000 - now) / 1000);
+      throw new RateLimitError(
+        `Rate limit exceeded: ${limit} requests/minute. Try again in ${waitSec}s.`,
+      );
+    }
+  }
+
+  private enforceTokenBudget(): void {
+    const limit = this.config.maxTokensPerSession;
+    if (limit <= 0) return;
+
+    if (this.sessionTokens >= limit) {
+      throw new RateLimitError(
+        `Session token budget exhausted: ${this.sessionTokens.toLocaleString()}/${limit.toLocaleString()} tokens used. Start a new session to continue.`,
+      );
+    }
+  }
+
+  private recordRequest(): void {
+    if (this.config.maxRequestsPerMinute <= 0) return;
+    this.requestTimestamps.push(Date.now());
+  }
+
+  private recordTokens(tokens: number): void {
+    if (this.config.maxTokensPerSession <= 0) return;
+    this.sessionTokens += tokens;
+    log.debug({ sessionTokens: this.sessionTokens, limit: this.config.maxTokensPerSession }, 'Token usage updated');
+  }
+}

package/src/providers/registry.ts

@@ -0,0 +1,146 @@
+import type { Provider } from "./types.js";
+import { AnthropicProvider } from "./anthropic/client.js";
+import { OpenAIProvider } from "./openai/client.js";
+import { GeminiProvider } from "./gemini/client.js";
+import { OllamaProvider } from "./ollama/client.js";
+import { FireworksProvider } from "./fireworks/client.js";
+import { OpenRouterProvider } from "./openrouter/client.js";
+import { RetryProvider } from "./retry.js";
+import { FailoverProvider } from "./failover.js";
+import { wrapWithLogfire } from "../logfire.js";
+import { ConfigError } from "../util/errors.js";
+
+const DEFAULT_MODELS: Record<string, string> = {
+  anthropic: 'claude-opus-4-6',
+  openai: 'gpt-5.2',
+  gemini: 'gemini-3-flash',
+  ollama: 'llama3.2',
+  fireworks: 'accounts/fireworks/models/kimi-k2p5',
+  openrouter: 'x-ai/grok-4',
+};
+
+const providers = new Map<string, Provider>();
+let cachedFailoverProvider: FailoverProvider | null = null;
+let cachedFailoverKey: string | null = null;
+
+export function registerProvider(name: string, provider: Provider): void {
+  providers.set(name, provider);
+}
+
+export function getProvider(name: string): Provider {
+  const provider = providers.get(name);
+  if (!provider) {
+    throw new ConfigError(
+      `Provider "${name}" not found. Available: ${listProviders().join(", ")}`,
+    );
+  }
+  return provider;
+}
+
+/**
+ * Build a provider that tries the primary provider first, then falls back to
+ * others in the configured order. If providerOrder is empty or only contains
+ * the primary, returns the primary provider directly (no wrapper overhead).
+ * Caches the FailoverProvider instance so health state persists across calls.
+ */
+export function getFailoverProvider(primaryName: string, providerOrder: string[]): Provider {
+  const primary = getProvider(primaryName);
+
+  // Build the ordered list: primary first, then remaining from providerOrder
+  const orderedProviders: Provider[] = [primary];
+  const seen = new Set<string>([primaryName]);
+
+  for (const name of providerOrder) {
+    if (seen.has(name)) continue;
+    const p = providers.get(name);
+    if (p) {
+      orderedProviders.push(p);
+      seen.add(name);
+    }
+  }
+
+  if (orderedProviders.length === 1) {
+    return primary;
+  }
+
+  const cacheKey = `${primaryName}:${providerOrder.join(',')}`;
+  if (cachedFailoverProvider && cachedFailoverKey === cacheKey) {
+    return cachedFailoverProvider;
+  }
+
+  cachedFailoverProvider = new FailoverProvider(orderedProviders);
+  cachedFailoverKey = cacheKey;
+  return cachedFailoverProvider;
+}
+
+export function listProviders(): string[] {
+  return Array.from(providers.keys());
+}
+
+export interface ProvidersConfig {
+  apiKeys: Record<string, string>;
+  provider: string;
+  model: string;
+  webSearchProvider?: string;
+  timeouts?: { providerStreamTimeoutSec?: number };
+}
+
+function resolveModel(config: ProvidersConfig, providerName: keyof typeof DEFAULT_MODELS): string {
+  if (config.provider === providerName) {
+    // If a non-Anthropic provider is selected with the untouched global default
+    // model, use a provider-appropriate fallback instead.
+    if (providerName !== 'anthropic' && config.model === DEFAULT_MODELS.anthropic) {
+      return DEFAULT_MODELS[providerName];
+    }
+    return config.model;
+  }
+  return DEFAULT_MODELS[providerName];
+}
+
+export function initializeProviders(config: ProvidersConfig): void {
+  providers.clear();
+  cachedFailoverProvider = null;
+  cachedFailoverKey = null;
+
+  const streamTimeoutMs = (config.timeouts?.providerStreamTimeoutSec ?? 300) * 1000;
+
+  if (config.apiKeys.anthropic) {
+    const model = resolveModel(config, 'anthropic');
+    registerProvider('anthropic', new RetryProvider(
+      wrapWithLogfire(new AnthropicProvider(config.apiKeys.anthropic, model, {
+        useNativeWebSearch: config.webSearchProvider === 'anthropic-native',
+        streamTimeoutMs,
+      })),
+    ));
+  }
+  if (config.apiKeys.openai) {
+    const model = resolveModel(config, 'openai');
+    registerProvider('openai', new RetryProvider(
+      wrapWithLogfire(new OpenAIProvider(config.apiKeys.openai, model, { streamTimeoutMs })),
+    ));
+  }
+  if (config.apiKeys.gemini) {
+    const model = resolveModel(config, 'gemini');
+    registerProvider('gemini', new RetryProvider(
+      wrapWithLogfire(new GeminiProvider(config.apiKeys.gemini, model, { streamTimeoutMs })),
+    ));
+  }
+  if (config.provider === 'ollama' || config.apiKeys.ollama) {
+    const model = resolveModel(config, 'ollama');
+    registerProvider('ollama', new RetryProvider(
+      wrapWithLogfire(new OllamaProvider(model, { apiKey: config.apiKeys.ollama, streamTimeoutMs })),
+    ));
+  }
+  if (config.apiKeys.fireworks) {
+    const model = resolveModel(config, 'fireworks');
+    registerProvider('fireworks', new RetryProvider(
+      wrapWithLogfire(new FireworksProvider(config.apiKeys.fireworks, model, { streamTimeoutMs })),
+    ));
+  }
+  if (config.apiKeys.openrouter) {
+    const model = resolveModel(config, 'openrouter');
+    registerProvider('openrouter', new RetryProvider(
+      wrapWithLogfire(new OpenRouterProvider(config.apiKeys.openrouter, model, { streamTimeoutMs })),
+    ));
+  }
+}

package/src/providers/retry.ts

@@ -0,0 +1,81 @@
+import type { Provider, ProviderResponse, SendMessageOptions, Message, ToolDefinition } from './types.js';
+import { ProviderError } from '../util/errors.js';
+import { getLogger, isDebug } from '../util/logger.js';
+import {
+  computeRetryDelay,
+  isRetryableNetworkError,
+  sleep,
+  DEFAULT_MAX_RETRIES,
+  DEFAULT_BASE_DELAY_MS,
+} from '../util/retry.js';
+
+const log = getLogger('retry');
+
+function isRetryableError(error: unknown): boolean {
+  if (error instanceof ProviderError && error.statusCode !== undefined) {
+    if (error.statusCode === 429 || error.statusCode >= 500) return true;
+  }
+  return isRetryableNetworkError(error);
+}
+
+export class RetryProvider implements Provider {
+  public readonly name: string;
+
+  constructor(private readonly inner: Provider) {
+    this.name = inner.name;
+  }
+
+  async sendMessage(
+    messages: Message[],
+    tools?: ToolDefinition[],
+    systemPrompt?: string,
+    options?: SendMessageOptions,
+  ): Promise<ProviderResponse> {
+    let lastError: unknown;
+    const debug = isDebug();
+
+    if (debug) {
+      log.debug({
+        provider: this.name,
+        messageCount: messages.length,
+        toolCount: tools?.length ?? 0,
+      }, 'Provider sendMessage start');
+    }
+
+    for (let attempt = 0; attempt <= DEFAULT_MAX_RETRIES; attempt++) {
+      try {
+        const start = Date.now();
+        const result = await this.inner.sendMessage(messages, tools, systemPrompt, options);
+        if (debug) {
+          log.debug({
+            provider: this.name,
+            durationMs: Date.now() - start,
+            attempt: attempt + 1,
+            model: result.model,
+            inputTokens: result.usage.inputTokens,
+            outputTokens: result.usage.outputTokens,
+          }, 'Provider sendMessage success');
+        }
+        return result;
+      } catch (error) {
+        lastError = error;
+
+        if (attempt < DEFAULT_MAX_RETRIES && isRetryableError(error)) {
+          const delay = computeRetryDelay(attempt, DEFAULT_BASE_DELAY_MS);
+          const errorType = error instanceof ProviderError && error.statusCode === 429
+            ? 'rate_limit'
+            : error instanceof ProviderError && error.statusCode !== undefined && error.statusCode >= 500
+              ? `server_error_${error.statusCode}`
+              : 'network_error';
+          log.warn({ attempt: attempt + 1, maxRetries: DEFAULT_MAX_RETRIES, delay, errorType, provider: this.name }, 'Retrying after transient error');
+          await sleep(delay);
+          continue;
+        }
+
+        throw error;
+      }
+    }
+
+    throw lastError;
+  }
+}