ashlrcode 1.0.0

package/src/providers/router.ts

@@ -0,0 +1,159 @@
+/**
+ * Provider router — selects provider, handles failover, tracks costs.
+ */
+
+import { createOpenAICompatibleProvider, createXAIProvider } from "./xai.ts";
+import { createAnthropicProvider } from "./anthropic.ts";
+import { CostTracker } from "./cost-tracker.ts";
+import { CircuitBreaker } from "./retry.ts";
+import type {
+  Provider,
+  ProviderConfig,
+  ProviderRequest,
+  ProviderRouterConfig,
+  StreamEvent,
+  TokenUsage,
+} from "./types.ts";
+
+export { CostTracker };
+
+export class ProviderRouter {
+  private providers: Provider[] = [];
+  private currentIndex = 0;
+  private circuitBreakers = new Map<string, CircuitBreaker>();
+  costTracker = new CostTracker();
+
+  /** @deprecated Use costTracker instead */
+  get costs() {
+    const tracker = this.costTracker;
+    return {
+      get totalInputTokens() { return tracker.totalInputTokens; },
+      get totalOutputTokens() { return tracker.totalOutputTokens; },
+      get totalReasoningTokens() { return tracker.totalReasoningTokens; },
+      get totalCostUSD() { return tracker.totalCostUSD; },
+      perProvider: new Map<string, { inputTokens: number; outputTokens: number; reasoningTokens: number; costUSD: number }>(
+        tracker.getBreakdown().map(e => [
+          `${e.provider}`,
+          { inputTokens: e.usage.inputTokens, outputTokens: e.usage.outputTokens, reasoningTokens: e.usage.reasoningTokens, costUSD: e.costUSD },
+        ])
+      ),
+    };
+  }
+
+  constructor(config: ProviderRouterConfig) {
+    this.providers.push(this.createProvider(config.primary));
+    for (const fallback of config.fallbacks ?? []) {
+      this.providers.push(this.createProvider(fallback));
+    }
+  }
+
+  private createProvider(
+    config: ProviderConfig & { provider: string }
+  ): Provider {
+    switch (config.provider) {
+      case "xai":
+        return createXAIProvider(config);
+      case "anthropic":
+        return createAnthropicProvider(config);
+      case "openai":
+        return createOpenAICompatibleProvider("openai", {
+          ...config,
+          baseURL: config.baseURL ?? "https://api.openai.com/v1",
+        }, [0, 0]);
+      case "ollama":
+        return createOpenAICompatibleProvider("ollama", {
+          ...config,
+          apiKey: config.apiKey || "ollama",
+          baseURL: config.baseURL ?? "http://localhost:11434/v1",
+        }, [0, 0]); // Free — local model
+      case "groq":
+        return createOpenAICompatibleProvider("groq", {
+          ...config,
+          baseURL: config.baseURL ?? "https://api.groq.com/openai/v1",
+        }, [0.05, 0.10]);
+      case "deepseek":
+        return createOpenAICompatibleProvider("deepseek", {
+          ...config,
+          baseURL: config.baseURL ?? "https://api.deepseek.com/v1",
+        }, [0.14, 0.28]);
+      default:
+        throw new Error(`Unknown provider: ${config.provider}`);
+    }
+  }
+
+  get currentProvider(): Provider {
+    return this.providers[this.currentIndex]!;
+  }
+
+  private getCircuitBreaker(providerName: string): CircuitBreaker {
+    let cb = this.circuitBreakers.get(providerName);
+    if (!cb) {
+      cb = new CircuitBreaker(5, 60_000);
+      this.circuitBreakers.set(providerName, cb);
+    }
+    return cb;
+  }
+
+  async *stream(request: ProviderRequest): AsyncGenerator<StreamEvent> {
+    let lastError: Error | null = null;
+
+    for (let i = this.currentIndex; i < this.providers.length; i++) {
+      const provider = this.providers[i]!;
+      const breaker = this.getCircuitBreaker(provider.name);
+
+      // If this provider's circuit is open, skip to the next one
+      if (!breaker.canRequest()) {
+        process.stderr.write(
+          `[router] ${provider.name} circuit breaker open (${breaker.getStatus()}), skipping...\n`,
+        );
+        if (i + 1 < this.providers.length) continue;
+        throw new Error(
+          `All providers unavailable — ${provider.name} circuit breaker is open. Wait and retry.`,
+        );
+      }
+
+      try {
+        for await (const event of provider.stream(request)) {
+          // Track usage
+          if (event.type === "usage" && event.usage) {
+            this.trackUsage(provider, event.usage);
+          }
+          yield event;
+        }
+        breaker.recordSuccess();
+        return; // Success — exit
+      } catch (err) {
+        lastError = err as Error;
+        breaker.recordFailure();
+
+        const isRateLimit =
+          lastError.message.includes("429") ||
+          lastError.message.includes("rate_limit") ||
+          lastError.message.includes("quota");
+
+        if (isRateLimit && i + 1 < this.providers.length) {
+          console.error(
+            `\n⚠ ${provider.name} rate limited, falling back to ${this.providers[i + 1]!.name}...`
+          );
+          this.currentIndex = i + 1;
+          continue;
+        }
+        throw lastError;
+      }
+    }
+
+    throw lastError ?? new Error("No providers available");
+  }
+
+  private trackUsage(provider: Provider, usage: TokenUsage) {
+    this.costTracker.record(provider.name, provider.config.model, {
+      inputTokens: usage.inputTokens,
+      outputTokens: usage.outputTokens,
+      reasoningTokens: usage.reasoningTokens ?? 0,
+    });
+  }
+
+  getCostSummary(): string {
+    return this.costTracker.formatSummary();
+  }
+}

package/src/providers/types.ts

@@ -0,0 +1,79 @@
+/**
+ * Unified provider interface for multi-model support.
+ * Normalizes Claude (Anthropic SDK) and xAI/OpenAI (OpenAI SDK) into a common streaming interface.
+ */
+
+export interface ToolDefinition {
+  name: string;
+  description: string;
+  input_schema: Record<string, unknown>;
+}
+
+export interface ToolCall {
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+}
+
+export interface ToolResult {
+  tool_use_id: string;
+  content: string;
+  is_error?: boolean;
+}
+
+export interface Message {
+  role: "user" | "assistant" | "tool";
+  content: string | ContentBlock[];
+}
+
+export type ContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image_url"; image_url: { url: string } }
+  | { type: "tool_use"; id: string; name: string; input: Record<string, unknown> }
+  | { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean };
+
+export type StopReason = "end_turn" | "tool_use" | "max_tokens";
+
+export interface StreamEvent {
+  type: "text_delta" | "tool_call_start" | "tool_call_delta" | "tool_call_end" | "message_end" | "usage";
+  text?: string;
+  toolCall?: Partial<ToolCall>;
+  stopReason?: StopReason;
+  usage?: TokenUsage;
+}
+
+export interface TokenUsage {
+  inputTokens: number;
+  outputTokens: number;
+  /** xAI-specific: cost in USD ticks (1 tick = $0.000001) */
+  costTicks?: number;
+  /** Reasoning tokens used by the model (separate from output) */
+  reasoningTokens?: number;
+}
+
+export interface ProviderConfig {
+  apiKey: string;
+  model: string;
+  baseURL?: string;
+  maxTokens?: number;
+}
+
+export interface ProviderRequest {
+  systemPrompt: string;
+  messages: Message[];
+  tools: ToolDefinition[];
+  maxTokens?: number;
+}
+
+export interface Provider {
+  name: string;
+  config: ProviderConfig;
+  stream(request: ProviderRequest): AsyncGenerator<StreamEvent>;
+  /** Cost per million tokens [input, output] */
+  pricing: [number, number];
+}
+
+export interface ProviderRouterConfig {
+  primary: ProviderConfig & { provider: "xai" | "anthropic" | "openai" };
+  fallbacks?: Array<ProviderConfig & { provider: "xai" | "anthropic" | "openai" }>;
+}

package/src/providers/xai.ts

@@ -0,0 +1,217 @@
+/**
+ * xAI Grok provider — uses OpenAI SDK with xAI base URL.
+ * Pattern from ashlr-landing/lib/xai-client.ts.
+ */
+
+import OpenAI from "openai";
+import type {
+  Provider,
+  ProviderConfig,
+  ProviderRequest,
+  StreamEvent,
+  ToolCall,
+} from "./types.ts";
+import { withRetry } from "./retry.ts";
+
+export function createXAIProvider(config: ProviderConfig): Provider {
+  return createOpenAICompatibleProvider("xai", config, [0.2, 0.5]);
+}
+
+export function createOpenAICompatibleProvider(
+  name: string,
+  config: ProviderConfig,
+  pricing: [number, number]
+): Provider {
+  const client = new OpenAI({
+    apiKey: config.apiKey,
+    baseURL: config.baseURL ?? "https://api.x.ai/v1",
+  });
+
+  return {
+    name,
+    config,
+    pricing,
+
+    async *stream(request: ProviderRequest): AsyncGenerator<StreamEvent> {
+      const tools: OpenAI.Chat.Completions.ChatCompletionTool[] = request.tools.map(
+        (t) => ({
+          type: "function" as const,
+          function: {
+            name: t.name,
+            description: t.description,
+            parameters: t.input_schema as Record<string, unknown>,
+          },
+        })
+      );
+
+      const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
+        { role: "system", content: request.systemPrompt },
+        ...request.messages.flatMap(convertMessage),
+      ];
+
+      const stream = await withRetry(
+        () =>
+          client.chat.completions.create({
+            model: config.model,
+            messages,
+            tools: tools.length > 0 ? tools : undefined,
+            max_tokens: request.maxTokens ?? config.maxTokens ?? 8192,
+            stream: true,
+            stream_options: { include_usage: true },
+          }),
+        { providerName: name },
+      );
+
+      const toolCalls = new Map<number, { id: string; name: string; args: string }>();
+
+      for await (const chunk of stream) {
+        // Usage comes in a chunk with empty choices — handle it first
+        if (chunk.usage) {
+          const usage = chunk.usage as unknown as Record<string, unknown>;
+          const completionDetails = usage.completion_tokens_details as Record<string, number> | undefined;
+          yield {
+            type: "usage",
+            usage: {
+              inputTokens: chunk.usage.prompt_tokens,
+              outputTokens: chunk.usage.completion_tokens ?? 0,
+              costTicks: usage.cost_in_usd_ticks as number | undefined,
+              reasoningTokens: completionDetails?.reasoning_tokens,
+            },
+          };
+        }
+
+        const choice = chunk.choices[0];
+        if (!choice) continue;
+
+        const delta = choice.delta;
+
+        // Text content
+        if (delta.content) {
+          yield { type: "text_delta", text: delta.content };
+        }
+
+        // Tool calls
+        if (delta.tool_calls) {
+          for (const tc of delta.tool_calls) {
+            const idx = tc.index;
+            if (!toolCalls.has(idx)) {
+              toolCalls.set(idx, { id: tc.id ?? "", name: tc.function?.name ?? "", args: "" });
+              yield {
+                type: "tool_call_start",
+                toolCall: { id: tc.id ?? "", name: tc.function?.name ?? "" },
+              };
+            }
+            const existing = toolCalls.get(idx)!;
+            if (tc.id) existing.id = tc.id;
+            if (tc.function?.name) existing.name = tc.function.name;
+            if (tc.function?.arguments) {
+              existing.args += tc.function.arguments;
+              yield {
+                type: "tool_call_delta",
+                toolCall: { id: existing.id, name: existing.name },
+                text: tc.function.arguments,
+              };
+            }
+          }
+        }
+
+        // Finish
+        if (choice.finish_reason) {
+          // Emit completed tool calls
+          for (const [, tc] of toolCalls) {
+            let input: Record<string, unknown> = {};
+            try {
+              input = JSON.parse(tc.args);
+            } catch {
+              input = { raw: tc.args };
+            }
+            yield {
+              type: "tool_call_end",
+              toolCall: { id: tc.id, name: tc.name, input } as ToolCall,
+            };
+          }
+
+          yield {
+            type: "message_end",
+            stopReason:
+              choice.finish_reason === "tool_calls"
+                ? "tool_use"
+                : choice.finish_reason === "length"
+                  ? "max_tokens"
+                  : "end_turn",
+          };
+        }
+
+      }
+    },
+  };
+}
+
+/**
+ * Convert our unified message format to OpenAI format.
+ * Returns an array because tool results expand into multiple messages.
+ */
+function convertMessage(
+  msg: ProviderRequest["messages"][number]
+): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
+  // Handle tool results — each one becomes a separate "tool" message
+  if (msg.role === "user" && Array.isArray(msg.content)) {
+    const toolResults = msg.content.filter((b) => b.type === "tool_result");
+    if (toolResults.length > 0) {
+      return toolResults.map((b) => {
+        if (b.type !== "tool_result") throw new Error("unreachable");
+        return {
+          role: "tool" as const,
+          content: b.content,
+          tool_call_id: b.tool_use_id,
+        };
+      });
+    }
+  }
+
+  if (typeof msg.content === "string") {
+    return [{ role: msg.role === "tool" ? "user" : msg.role, content: msg.content }];
+  }
+
+  // Handle content blocks — extract text, images, and tool uses
+  const textParts = msg.content
+    .filter((b) => b.type === "text")
+    .map((b) => (b as { type: "text"; text: string }).text)
+    .join("");
+
+  const imageBlocks = msg.content.filter((b) => b.type === "image_url");
+  const toolUses = msg.content.filter((b) => b.type === "tool_use");
+
+  // User message with images — send as multimodal content array
+  if (msg.role === "user" && imageBlocks.length > 0) {
+    const parts: Array<{ type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }> = [];
+    if (textParts) parts.push({ type: "text", text: textParts });
+    for (const img of imageBlocks) {
+      if (img.type === "image_url") {
+        parts.push({ type: "image_url", image_url: { url: img.image_url.url } });
+      }
+    }
+    return [{ role: "user" as const, content: parts }];
+  }
+
+  if (msg.role === "assistant" && toolUses.length > 0) {
+    return [{
+      role: "assistant",
+      content: textParts || null,
+      tool_calls: toolUses.map((tc) => {
+        if (tc.type !== "tool_use") throw new Error("unreachable");
+        return {
+          id: tc.id,
+          type: "function" as const,
+          function: {
+            name: tc.name,
+            arguments: JSON.stringify(tc.input),
+          },
+        };
+      }),
+    }];
+  }
+
+  return [{ role: msg.role === "tool" ? "user" : msg.role, content: textParts }];
+}
+