@hemansubedi/aether-ai 1.0.0

package/src/providers/registry.ts

@@ -0,0 +1,223 @@
+export interface FreeProvider {
+  name: string;
+  baseURL: string;
+  type: "ollama" | "openrouter" | "openai-compatible";
+  models: string[];
+  priority: number;
+  enabled: boolean;
+  maxRetries: number;
+  timeoutMs: number;
+}
+
+export const PROVIDER_REGISTRY: FreeProvider[] = [
+  {
+    name: "ollama-local",
+    baseURL: "http://localhost:11434",
+    type: "ollama",
+    models: ["goekdenizguelmez/JOSIEFIED-Qwen3:8b", "hf.co/OBLITERATUS/Qwen3.8-27B-OBLITERATED:Q4_K_M", "llama3.1:8b", "qwen2.5:7b", "mistral:7b", "phi3:medium"],
+    priority: 1,
+    enabled: true,
+    maxRetries: 2,
+    timeoutMs: 120000,
+  },
+  {
+    name: "openrouter-free",
+    baseURL: "https://openrouter.ai/api/v1",
+    type: "openrouter",
+    models: [
+      "openrouter/auto",
+      "meta-llama/llama-3.1-8b-instruct:free",
+      "google/gemma-2-9b-it:free",
+      "mistralai/mistral-7b-instruct:free",
+      "nvidia/nemotron-70b-instruct:free",
+    ],
+    priority: 2,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "openai-compatible",
+    baseURL: "",
+    type: "openai-compatible",
+    models: ["gpt-4o-mini"],
+    priority: 99,
+    enabled: false,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "groq",
+    baseURL: "https://api.groq.com/openai/v1",
+    type: "openai-compatible",
+    models: ["llama-3.1-8b-instant", "llama-3.1-70b-versatile", "llama-3.2-1b-preview", "llama-3.2-11b-vision-preview", "mixtral-8x7b-32768"],
+    priority: 3,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "mistral",
+    baseURL: "https://api.mistral.com/v1",
+    type: "openai-compatible",
+    models: ["mistral-small-latest", "mistral-medium-latest", "mistral-large-latest", "mixtral-8x7b-latest"],
+    priority: 4,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "cohere",
+    baseURL: "https://api.cohere.com/v1",
+    type: "openai-compatible",
+    models: ["command-r", "command-r-plus", "command-light"],
+    priority: 5,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "huggingface",
+    baseURL: "https://api-inference.huggingface.co",
+    type: "openai-compatible",
+    models: ["HuggingFaceH4/zephyr-7b-beta", "HuggingFaceH4/mistral-7b-it", "bigcode/starcoder2-15b"],
+    priority: 6,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "fireworks",
+    baseURL: "https://api.fireworks.ai/inference/v1",
+    type: "openai-compatible",
+    models: ["accounts/fireworks/models/llama-v3-8b-instruct", "accounts/fireworks/models/llama-v3-70b-instruct", "accounts/fireworks/models/mixtral-8x7b-instruct"],
+    priority: 7,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "together",
+    baseURL: "https://api.together.xyz/v1",
+    type: "openai-compatible",
+    models: ["meta-llama/Llama-3-8b-hf-chat", "meta-llama/Llama-3-70b-hf-chat", "mistralai/Mixtral-8x7B-Instruct-v0.1", "Qwen/Qwen1.5-7B-Chat"],
+    priority: 8,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "deepseek",
+    baseURL: "https://api.deepseek.com/v1",
+    type: "openai-compatible",
+    models: ["deepseek-chat", "deepseek-coder", "deepseek-reasoner"],
+    priority: 9,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "gemini",
+    baseURL: "https://generativelanguage.googleapis.com/v1/openai",
+    type: "openai-compatible",
+    models: ["gemini-1.5-flash", "gemini-1.5-pro", "gemini-2.0-flash-exp"],
+    priority: 10,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "xai",
+    baseURL: "https://api.x.ai/v1",
+    type: "openai-compatible",
+    models: ["grok-2", "grok-2-mini", "grok-beta", "grok-mini"],
+    priority: 11,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "perplexity",
+    baseURL: "https://api.perplexity.ai",
+    type: "openai-compatible",
+    models: ["sonar", "sonar-pro", "sonar-reasoning", "sonar-deep-research"],
+    priority: 12,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "cerebras",
+    baseURL: "https://api.cerebras.ai/v1",
+    type: "openai-compatible",
+    models: ["llama3.1-8b", "llama3.1-70b", "llama3.2-1b", "llama3.2-11b-vision", "qwen-32b"],
+    priority: 13,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "cloudflare",
+    baseURL: "",
+    type: "openai-compatible",
+    models: ["@cf/meta-llama/llama-3.1-8b-instruct", "@cf/qwen/qwen2.5-7b-instruct", "@cf/mistral/mistral-7b-instruct"],
+    priority: 99,
+    enabled: false,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "nvidia",
+    baseURL: "https://integrate.api.nvidia.com/v1",
+    type: "openai-compatible",
+    models: ["meta/llama-3.1-8b-instruct", "meta/llama-3.1-70b-instruct", "mistralai/mistral-7b-instruct", "nvidia/nemotron-4-340b-instruct"],
+    priority: 14,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "voyage",
+    baseURL: "",
+    type: "openai-compatible",
+    models: ["voyage-3-large", "voyage-3", "voyage-code-3"],
+    priority: 99,
+    enabled: false,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "jina",
+    baseURL: "https://api.jina.ai/v1",
+    type: "openai-compatible",
+    models: ["jina-embeddings-v3", "jina-clip-v2"],
+    priority: 15,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "parasail",
+    baseURL: "https://api.parasail.ai/v1",
+    type: "openai-compatible",
+    models: ["parasail-dqn-turbo-70b", "parasail-mixtral-8x7b", "parasail-llama-3-8b"],
+    priority: 16,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+  {
+    name: "featherless",
+    baseURL: "https://api.featherless.ai/v1",
+    type: "openai-compatible",
+    models: ["featherless/qwen-2.5-7b-instruct", "featherless/llama-3.1-8b-instruct", "featherless/mistral-7b-instruct"],
+    priority: 17,
+    enabled: true,
+    maxRetries: 3,
+    timeoutMs: 120000,
+  },
+];
+
+export function getProvider(name: string): FreeProvider | undefined {
+  return PROVIDER_REGISTRY.find((p) => p.name === name);
+}

package/src/router-engine.ts

@@ -0,0 +1,214 @@
+import { HealthTracker } from "./health.js";
+import { createProvider, type Provider, type ProviderConfig } from "./providers/index.js";
+import { PROVIDER_REGISTRY, type FreeProvider } from "./providers/registry.js";
+import { KeyManager, ENV_MAP } from "./keys.js";
+import { type Combo } from "./combos.js";
+
+export interface RouteAttempt {
+  provider: string;
+  model: string;
+  ok: boolean;
+  error?: string;
+  latencyMs: number;
+}
+
+export interface RouteResult {
+  text: string;
+  toolCalls: any[];
+  usage?: { input_tokens: number; output_tokens: number };
+  attempts: RouteAttempt[];
+  provider: string;
+  model: string;
+}
+
+export class RouterEngine {
+  private health: HealthTracker;
+  private cache = new Map<string, Provider>();
+  private configs: ProviderConfig[];
+  private keyManager: KeyManager;
+  activeModel?: string;
+
+  constructor(providers?: FreeProvider[], health?: HealthTracker, keyManager?: KeyManager) {
+    this.health = health ?? new HealthTracker();
+    this.keyManager = keyManager ?? KeyManager.instance();
+    this.configs = (providers ?? PROVIDER_REGISTRY).map((p) => ({
+      name: p.name,
+      type: p.type,
+      baseURL: p.baseURL,
+      apiKey: this.keyManager.get(p.name) ?? process.env[ENV_MAP[p.name] ?? ""],
+      models: p.models,
+      priority: p.priority,
+      enabled: p.enabled,
+      maxRetries: p.maxRetries,
+      timeoutMs: p.timeoutMs,
+    }));
+  }
+
+  get configs_(): ProviderConfig[] { return this.configs; }
+  get healthTracker(): HealthTracker { return this.health; }
+  get keys(): KeyManager { return this.keyManager; }
+
+  /** Update the key for a provider and invalidate the cached provider so the
+   *  new credential takes effect on the next request. */
+  setKey(name: string, key: string): void {
+    this.keyManager.set(name, key);
+    const cfg = this.configs.find((c) => c.name === name);
+    if (cfg) cfg.apiKey = key.trim() || undefined;
+    this.cache.delete(name);
+  }
+
+  /** Activate a combo: rebuild the router's active provider list from the
+   *  combo's providers (in order), set the active model to the combo's first
+   *  model, and invalidate the provider cache so the new providers load. */
+  applyCombo(combo: Combo): string {
+    this.configs = (combo.providers ?? [])
+      .map((name) => {
+        const p = PROVIDER_REGISTRY.find((x: FreeProvider) => x.name === name);
+        if (!p) return undefined;
+        return {
+          name: p.name,
+          type: p.type,
+          baseURL: p.baseURL,
+          apiKey: this.keyManager.get(p.name) ?? process.env[ENV_MAP[p.name] ?? ""],
+          models: p.models,
+          priority: p.priority,
+          enabled: p.enabled,
+          maxRetries: p.maxRetries,
+          timeoutMs: p.timeoutMs,
+        } as ProviderConfig;
+      })
+      .filter((c): c is ProviderConfig => Boolean(c));
+    this.cache.clear();
+    this.activeModel = combo.models?.[0];
+    const provs = this.configs.map((c) => c.name).join(", ") || "(none)";
+    return `Combo "${combo.name}" active: ${this.configs.length} provider(s) [${provs}], model: ${this.activeModel ?? "(none)"}`;
+  }
+
+  private async getProvider(cfg: ProviderConfig): Promise<Provider> {
+    const cached = this.cache.get(cfg.name);
+    if (cached) return cached;
+    const p = await createProvider(cfg);
+    this.cache.set(cfg.name, p);
+    return p;
+  }
+
+  private sorted(): ProviderConfig[] {
+    return [...this.configs].filter((p) => p.enabled).sort((a, b) => a.priority - b.priority);
+  }
+
+  async *chatStream(
+    messages: any[],
+    tools: any[],
+    opts?: { temperature?: number; maxTokens?: number; signal?: AbortSignal; excludeProviders?: string[] }
+  ): AsyncGenerator<any> {
+    const order = this.sorted().filter((p) => !opts?.excludeProviders?.includes(p.name));
+    if (order.length === 0) {
+      yield { type: "error", error: "No providers configured" };
+      return;
+    }
+    let lastError = "";
+    for (const cfg of order) {
+      if (!this.health.isAvailable(cfg.name)) continue;
+      const provider = await this.getProvider(cfg);
+      const model = cfg.models[0] ?? "";
+      const start = Date.now();
+      try {
+        let usage: any;
+        for await (const chunk of provider.chat(messages, tools, {
+          signal: opts?.signal,
+          temperature: opts?.temperature,
+          maxTokens: opts?.maxTokens,
+        })) {
+          if (chunk.type === "done") usage = chunk.usage;
+          if (chunk.type === "error" && chunk.error) throw new Error(chunk.error);
+          yield chunk;
+        }
+        this.health.recordSuccess(cfg.name);
+        if (!usage) yield { type: "done", usage };
+        return;
+      } catch (err) {
+        const msg = (err as Error).message;
+        lastError = msg;
+        this.health.recordFailure(cfg.name, msg);
+        if (opts?.signal?.aborted) {
+          yield { type: "error", error: `Aborted: ${msg}` };
+          return;
+        }
+        yield {
+          type: "error",
+          error: `Provider ${cfg.name} failed (${msg}); failing over...`,
+        };
+      }
+    }
+    yield {
+      type: "error",
+      error: `All ${order.length} providers failed. Last: ${lastError || "unknown"}`,
+    };
+  }
+
+  async chat(messages: any[], tools: any[], opts?: { temperature?: number; maxTokens?: number; signal?: AbortSignal; excludeProviders?: string[] }): Promise<RouteResult> {
+    const order = this.sorted().filter((p) => !opts?.excludeProviders?.includes(p.name));
+    const attempts: RouteAttempt[] = [];
+    let lastError = "";
+    for (const cfg of order) {
+      if (!this.health.isAvailable(cfg.name)) continue;
+      const provider = await this.getProvider(cfg);
+      const model = cfg.models[0] ?? "";
+      const start = Date.now();
+      try {
+        let text = "", toolCalls: any[] = [], usage: any;
+        for await (const chunk of provider.chat(messages, tools, { signal: opts?.signal, temperature: opts?.temperature, maxTokens: opts?.maxTokens })) {
+          if (chunk.type === "text" && chunk.text) text += chunk.text;
+          if (chunk.type === "tool_call" && chunk.tool_call) toolCalls.push(chunk.tool_call);
+          if (chunk.type === "done") usage = chunk.usage;
+          if (chunk.type === "error" && chunk.error) throw new Error(chunk.error);
+        }
+        this.health.recordSuccess(cfg.name);
+        attempts.push({ provider: cfg.name, model, ok: true, latencyMs: Date.now() - start });
+        return { text, toolCalls, usage, attempts, provider: cfg.name, model };
+      } catch (err) {
+        const msg = (err as Error).message;
+        lastError = msg;
+        this.health.recordFailure(cfg.name, msg);
+        attempts.push({ provider: cfg.name, model, ok: false, error: msg, latencyMs: Date.now() - start });
+        if (opts?.signal?.aborted) break;
+      }
+    }
+    throw new Error(`All ${order.length} providers failed. Last: ${lastError}. Attempts: ${JSON.stringify(attempts)}`);
+  }
+
+  async listFreeModels(): Promise<Record<string, string[]>> {
+    const out: Record<string, string[]> = {};
+    for (const cfg of this.sorted()) {
+      try {
+        const p = await this.getProvider(cfg);
+        out[cfg.name] = await p.listModels();
+      } catch { out[cfg.name] = cfg.models; }
+    }
+    return out;
+  }
+
+  async healthAll() {
+    const out: any[] = [];
+    for (const cfg of this.configs) {
+      if (!cfg.enabled) continue;
+      try {
+        const provider = await this.getProvider(cfg);
+        const status = await provider.health();
+        out.push({ provider: cfg.name, ...status });
+      } catch {
+        out.push({
+          provider: cfg.name,
+          healthy: false,
+          failures: 0,
+          lastCheck: 0,
+          circuitOpen: false,
+          cooldownUntil: 0,
+        });
+      }
+    }
+    return out;
+  }
+
+  resetHealth() { this.health.resetAll(); }
+}

package/src/router.ts

@@ -0,0 +1,195 @@
+import type {
+  ChatChunk,
+  Message,
+  ProviderConfig,
+  RouteDecision,
+  ToolDef,
+  HealthStatus,
+} from "./types.js";
+import { HealthTracker } from "./health.js";
+import { createProvider, type Provider } from "./providers/index.js";
+
+export class Router {
+  readonly configs: ProviderConfig[];
+  private health: HealthTracker;
+  private cache = new Map<string, Provider>();
+  private modelCache = new Map<string, Provider>();
+
+  activeProvider?: string;
+  activeModel?: string;
+
+  constructor(providers: ProviderConfig[], health: HealthTracker) {
+    this.configs = providers;
+    this.health = health;
+  }
+
+  private async getProvider(config: ProviderConfig): Promise<Provider> {
+    const cached = this.cache.get(config.name);
+    if (cached) return cached;
+    const p = await createProvider(config);
+    this.cache.set(config.name, p);
+    return p;
+  }
+
+  private sortedConfigs(): ProviderConfig[] {
+    return [...this.configs]
+      .filter((p) => p.enabled)
+      .sort((a, b) => a.priority - b.priority);
+  }
+
+  private findConfig(name?: string): ProviderConfig | undefined {
+    if (!name) return undefined;
+    return this.configs.find((c) => c.name === name);
+  }
+
+  resolveModel(modelId: string): { provider: ProviderConfig; model: string } | null {
+    const exact = this.configs.find((c) => c.models.includes(modelId));
+    if (exact) return { provider: exact, model: modelId };
+    const first = this.sortedConfigs()[0];
+    return first ? { provider: first, model: modelId } : null;
+  }
+
+  async getModelProvider(modelId: string): Promise<Provider> {
+    const cached = this.modelCache.get(modelId);
+    if (cached) return cached;
+    const cfg = this.configs.find((c) => c.models.includes(modelId)) || this.sortedConfigs()[0];
+    if (!cfg) throw new Error(`No provider configured to serve model "${modelId}"`);
+    const override: ProviderConfig = { ...cfg, models: [modelId] };
+    const p = await createProvider(override);
+    this.modelCache.set(modelId, p);
+    return p;
+  }
+
+  select(): RouteDecision | null {
+    const cfg = this.findConfig(this.activeProvider) || this.sortedConfigs()[0];
+    if (cfg && this.health.isAvailable(cfg.name)) {
+      return {
+        provider: cfg.name,
+        model: this.activeModel || cfg.models[0] || "",
+        reason: this.activeProvider ? `selected (active: ${this.activeProvider})` : `selected by priority ${cfg.priority}`,
+      };
+    }
+    for (const c of this.sortedConfigs()) {
+      if (this.health.isAvailable(c.name)) {
+        return { provider: c.name, model: this.activeModel || c.models[0] || "", reason: "fallback" };
+      }
+    }
+    return null;
+  }
+
+  setActiveProvider(name?: string): void {
+    this.activeProvider = name;
+  }
+
+  setActiveModel(model: string): void {
+    this.activeModel = model;
+    const cfg = this.findConfig(this.activeProvider) || this.sortedConfigs()[0];
+    if (cfg) {
+      cfg.models = [model];
+    }
+  }
+
+  getActiveProvider(): string | undefined {
+    return this.activeProvider;
+  }
+
+  getActiveModel(): string | undefined {
+    return this.activeModel;
+  }
+
+  getProviderNames(): string[] {
+    return this.configs.filter((c) => c.enabled).map((c) => c.name);
+  }
+
+  getModelsFor(name?: string): string[] {
+    const cfg = name
+      ? this.configs.find((c) => c.name === name)
+      : this.activeProvider
+        ? this.configs.find((c) => c.name === this.activeProvider)
+        : this.sortedConfigs()[0];
+    return cfg?.models ?? [];
+  }
+
+  async *chat(
+    messages: Message[],
+    tools: ToolDef[],
+    opts?: { temperature?: number; maxTokens?: number; signal?: AbortSignal }
+  ): AsyncIterable<ChatChunk> {
+    const order = this.sortedConfigs();
+    if (order.length === 0) {
+      yield { type: "error", error: "No providers configured" };
+      return;
+    }
+
+    let lastError: Error | null = null;
+    for (const cfg of order) {
+      if (!this.health.isAvailable(cfg.name)) continue;
+      const provider = await this.getProvider(cfg);
+      try {
+        for await (const chunk of provider.chat(messages, tools, {
+          signal: opts?.signal,
+          temperature: opts?.temperature,
+          maxTokens: opts?.maxTokens,
+        })) {
+          if (chunk.type === "error" && chunk.error) {
+            throw new Error(chunk.error);
+          }
+          yield chunk;
+        }
+        this.health.recordSuccess(cfg.name);
+        return;
+      } catch (err) {
+        lastError = err as Error;
+        this.health.recordFailure(cfg.name, lastError?.message);
+        if (opts?.signal?.aborted) {
+          yield { type: "error", error: `Aborted: ${lastError.message}` };
+          return;
+        }
+        yield {
+          type: "error",
+          error: `Provider ${cfg.name} failed (${lastError.message}); failing over...`,
+        };
+        continue;
+      }
+    }
+
+    yield {
+      type: "error",
+      error: `All providers failed. Last error: ${lastError?.message ?? "unknown"}`,
+    };
+  }
+
+  async listAllModels(): Promise<Record<string, string[]>> {
+    const out: Record<string, string[]> = {};
+    for (const cfg of this.sortedConfigs()) {
+      try {
+        const provider = await this.getProvider(cfg);
+        out[cfg.name] = await provider.listModels();
+      } catch {
+        out[cfg.name] = [];
+      }
+    }
+    return out;
+  }
+
+  async healthAll(): Promise<HealthStatus[]> {
+    const out: HealthStatus[] = [];
+    for (const cfg of this.configs) {
+      try {
+        const provider = await this.getProvider(cfg);
+        const status = await provider.health();
+        out.push({ provider: cfg.name, ...status });
+      } catch {
+        out.push({
+          provider: cfg.name,
+          healthy: false,
+          failures: 0,
+          lastCheck: 0,
+          circuitOpen: false,
+          cooldownUntil: 0,
+        });
+      }
+    }
+    return out;
+  }
+}