adaptive-memory-multi-model-router 2.14.59 → 2.15.0

package/src/server/handlers/chatHandler.ts

@@ -0,0 +1,173 @@
+/**
+ * A3M Router - Chat Completions Handler
+ *
+ * Handles POST /v1/chat/completions
+ * Plug-and-play: register with router.registerRoute('POST', /^\/v1\/chat\/completions$/, handleChatCompletions);
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { resolveModel } from '../modelMapper';
+import { costTracker } from '../state';
+import { recordRequest, recordProviderError, recordActiveRequest } from '../metrics';
+
+// ============================================================
+// TYPES (local, matching OpenAI API)
+// ============================================================
+
+interface ChatMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: string;
+}
+
+interface ChatRequest {
+  model: string;
+  messages: ChatMessage[];
+  stream?: boolean;
+  temperature?: number;
+  max_tokens?: number;
+  top_p?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  stop?: string | string[];
+  n?: number;
+}
+
+// ============================================================
+// HELPERS
+// ============================================================
+
+function readBody(req: http.IncomingMessage): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (chunk: Buffer) => chunks.push(chunk));
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+function jsonResponse(res: http.ServerResponse, statusCode: number, body: object): void {
+  res.writeHead(statusCode, {
+    'Content-Type': 'application/json',
+    'Access-Control-Allow-Origin': '*',
+    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+    'Access-Control-Allow-Headers': 'Content-Type, Authorization',
+  });
+  res.end(JSON.stringify(body));
+}
+
+function errorResponse(res: http.ServerResponse, statusCode: number, message: string, errorType = 'server_error'): void {
+  jsonResponse(res, statusCode, { error: { message, type: errorType, code: statusCode } });
+}
+
+// ============================================================
+// HANDLER
+// ============================================================
+
+export async function handleChatCompletions(
+  req: http.IncomingMessage,
+  res: http.ServerResponse,
+  ctx: RouteContext
+): Promise<void> {
+  recordActiveRequest(1);
+
+  const body = await readBody(req);
+  let request: ChatRequest;
+
+  try {
+    request = JSON.parse(body);
+  } catch {
+    recordActiveRequest(-1);
+    errorResponse(res, 400, 'Invalid JSON in request body', 'invalid_request_error');
+    return;
+  }
+
+  if (!request.messages || !Array.isArray(request.messages) || request.messages.length === 0) {
+    recordActiveRequest(-1);
+    errorResponse(res, 400, 'messages is required and must be a non-empty array', 'invalid_request_error');
+    return;
+  }
+
+  const model = request.model || 'auto';
+  const stream = request.stream || false;
+  const requestId = ctx.requestId;
+  const promptForRouting = request.messages.map((m) => m.content).join(' ');
+
+  const mapping = resolveModel(model, promptForRouting);
+  if (!mapping) {
+    recordActiveRequest(-1);
+    errorResponse(res, 503, `No provider available for model "${model}". Configure API keys.`, 'server_error');
+    return;
+  }
+
+  const startTime = Date.now();
+
+  if (stream) {
+    try {
+      const { streamProviderResponse } = await import('../proxyServer');
+      await streamProviderResponse(res, mapping, request.messages,
+        { temperature: request.temperature, max_tokens: request.max_tokens, stop: request.stop },
+        requestId);
+      const latencyMs = Date.now() - startTime;
+      recordRequest({ endpoint: '/v1/chat/completions', provider: mapping.providerId, status: 'success', durationMs: latencyMs, model });
+      recordActiveRequest(-1);
+    } catch (err: unknown) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (!res.headersSent) errorResponse(res, 500, message);
+      recordProviderError(mapping.providerId, 'stream_error');
+      recordActiveRequest(-1);
+    }
+    return;
+  }
+
+  // Non-streaming path
+  try {
+    const { callWithFallback } = await import('../proxyServer');
+    const { result, mapping: usedMapping } = await callWithFallback(
+      model, request.messages,
+      { temperature: request.temperature, max_tokens: request.max_tokens, stop: request.stop },
+      promptForRouting
+    );
+
+    const latencyMs = Date.now() - startTime;
+    const inputCost = (result.usage.prompt_tokens / 1000) * usedMapping.costPerK.input;
+    const outputCost = (result.usage.completion_tokens / 1000) * usedMapping.costPerK.output;
+    const totalCost = inputCost + outputCost;
+
+    costTracker.record(usedMapping.providerId, usedMapping.model, result.usage.prompt_tokens, result.usage.completion_tokens);
+
+    const response = {
+      id: requestId,
+      object: 'chat.completion',
+      created: Math.floor(Date.now() / 1000),
+      model: result.model,
+      choices: [{ index: 0, message: { role: 'assistant' as const, content: result.content }, finish_reason: result.finish_reason }],
+      usage: result.usage,
+    };
+
+    jsonResponse(res, 200, response);
+    recordRequest({
+      endpoint: '/v1/chat/completions',
+      provider: usedMapping.providerId,
+      status: 'success',
+      durationMs: latencyMs,
+      tokensIn: result.usage.prompt_tokens,
+      tokensOut: result.usage.completion_tokens,
+      cost: totalCost,
+      model,
+    });
+
+    console.log(`[a3m-router] ${requestId} chat model=${model}→${usedMapping.providerId}/${usedMapping.model} latency=${latencyMs}ms tokens=${result.usage.total_tokens} cost=$${totalCost.toFixed(6)}`);
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    const latencyMs = Date.now() - startTime;
+    recordRequest({ endpoint: '/v1/chat/completions', provider: mapping.providerId, status: 'error', durationMs: latencyMs, model });
+    recordProviderError(mapping.providerId, message.slice(0, 50));
+    console.error(`[a3m-router] ${requestId} ERROR model=${model}→${mapping.providerId}/${mapping.model} latency=${latencyMs}ms error=${message}`);
+    if (!res.headersSent) errorResponse(res, 502, message, 'upstream_error');
+  }
+
+  recordActiveRequest(-1);
+}
+
+export default handleChatCompletions;

package/src/server/handlers/completionsHandler.ts

@@ -0,0 +1,120 @@
+/**
+ * A3M Router - Completions Handler
+ *
+ * Handles POST /v1/completions
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { resolveModel } from '../modelMapper';
+import { recordRequest, recordActiveRequest } from '../metrics';
+
+interface CompletionRequest {
+  model: string;
+  prompt: string | string[];
+  stream?: boolean;
+  temperature?: number;
+  max_tokens?: number;
+  top_p?: number;
+  echo?: boolean;
+  stop?: string | string[];
+}
+
+function readBody(req: http.IncomingMessage): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (chunk: Buffer) => chunks.push(chunk));
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+function jsonResponse(res: http.ServerResponse, statusCode: number, body: object): void {
+  res.writeHead(statusCode, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
+  res.end(JSON.stringify(body));
+}
+
+function errorResponse(res: http.ServerResponse, statusCode: number, message: string, errorType = 'server_error'): void {
+  jsonResponse(res, statusCode, { error: { message, type: errorType, code: statusCode } });
+}
+
+export async function handleCompletions(
+  req: http.IncomingMessage,
+  res: http.ServerResponse,
+  ctx: RouteContext
+): Promise<void> {
+  recordActiveRequest(1);
+  const body = await readBody(req);
+  let request: CompletionRequest;
+
+  try {
+    request = JSON.parse(body);
+  } catch {
+    recordActiveRequest(-1);
+    errorResponse(res, 400, 'Invalid JSON in request body', 'invalid_request_error');
+    return;
+  }
+
+  const prompts = Array.isArray(request.prompt) ? request.prompt : [request.prompt || ''];
+  const messages = prompts.map((p) => ({ role: 'user' as const, content: p }));
+  const model = request.model || 'auto';
+  const stream = request.stream || false;
+  const requestId = ctx.requestId;
+  const promptForRouting = prompts.join(' ');
+
+  const mapping = resolveModel(model, promptForRouting);
+  if (!mapping) {
+    recordActiveRequest(-1);
+    errorResponse(res, 503, `No provider available for model "${model}".`, 'server_error');
+    return;
+  }
+
+  const startTime = Date.now();
+
+  if (stream) {
+    try {
+      const { streamProviderResponse } = await import('../proxyServer');
+      await streamProviderResponse(res, mapping, messages,
+        { temperature: request.temperature, max_tokens: request.max_tokens, stop: request.stop },
+        requestId);
+      const latencyMs = Date.now() - startTime;
+      recordRequest({ endpoint: '/v1/completions', provider: mapping.providerId, status: 'success', durationMs: latencyMs, model });
+      recordActiveRequest(-1);
+    } catch (err: unknown) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (!res.headersSent) errorResponse(res, 500, message);
+      recordActiveRequest(-1);
+    }
+    return;
+  }
+
+  try {
+    const { callWithFallback } = await import('../proxyServer');
+    const { result, mapping: usedMapping } = await callWithFallback(
+      model, messages,
+      { temperature: request.temperature, max_tokens: request.max_tokens, stop: request.stop },
+      promptForRouting
+    );
+
+    const latencyMs = Date.now() - startTime;
+    const response = {
+      id: requestId,
+      object: 'text_completion',
+      created: Math.floor(Date.now() / 1000),
+      model: result.model,
+      choices: [{ text: result.content, index: 0, finish_reason: result.finish_reason }],
+      usage: result.usage,
+    };
+
+    jsonResponse(res, 200, response);
+    recordRequest({ endpoint: '/v1/completions', provider: usedMapping.providerId, status: 'success', durationMs: latencyMs, tokensIn: result.usage.prompt_tokens, tokensOut: result.usage.completion_tokens, model });
+    console.log(`[a3m-router] ${requestId} completion model=${model}→${usedMapping.providerId}/${usedMapping.model} latency=${latencyMs}ms`);
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!res.headersSent) errorResponse(res, 502, message, 'upstream_error');
+  }
+
+  recordActiveRequest(-1);
+}
+
+export default handleCompletions;

package/src/server/handlers/embeddingsHandler.ts

@@ -0,0 +1,271 @@
+/**
+ * A3M Router - Embeddings Handler
+ *
+ * Handles POST /v1/embeddings
+ * OpenAI-compatible embeddings endpoint.
+ *
+ * Routes embedding requests to the best available provider:
+ * - OpenAI (text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002)
+ * - Cohere (embed-english-v3.0, embed-multilingual-v3.0)
+ * - Google (text-embedding-004)
+ * - Local Ollama (nomic-embed-text, etc.)
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { getAvailableProviders } from '../../providers/providerConfig';
+import { recordRequest, recordProviderError, recordActiveRequest } from '../metrics';
+
+interface EmbeddingsRequest {
+  model?: string;
+  input: string | string[];
+  encoding_format?: 'float' | 'base64';
+  dimensions?: number;
+  user?: string;
+}
+
+interface ProviderAdapter {
+  call: (url: string, apiKey: string, model: string, input: string[], dimensions?: number) => Promise<number[][]>;
+  urlFor: (apiKey: string, model: string) => string;
+  modelMap: (model: string) => string | null;
+}
+
+function readBody(req: http.IncomingMessage): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (chunk: Buffer) => chunks.push(chunk));
+    req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    req.on('error', reject);
+  });
+}
+
+function jsonResponse(res: http.ServerResponse, statusCode: number, body: object): void {
+  res.writeHead(statusCode, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
+  res.end(JSON.stringify(body));
+}
+
+function errorResponse(res: http.ServerResponse, statusCode: number, message: string, errorType = 'server_error'): void {
+  jsonResponse(res, statusCode, { error: { message, type: errorType, code: statusCode } });
+}
+
+/**
+ * Get the best available embeddings provider.
+ */
+function getEmbeddingsProvider(): { providerId: string; model: string; adapter: ProviderAdapter } | null {
+  const providers = getAvailableProviders();
+
+  // Priority: OpenAI > Cohere > Google > Ollama
+  const priority = ['openai', 'cohere', 'google', 'ollama'];
+
+  for (const id of priority) {
+    const provider = providers[id];
+    if (!provider || !provider.apiKey) continue;
+
+    if (id === 'openai') {
+      return {
+        providerId: 'openai',
+        model: 'text-embedding-3-small',
+        adapter: openAIAdapter,
+      };
+    }
+
+    if (id === 'cohere') {
+      return {
+        providerId: 'cohere',
+        model: 'embed-english-v3.0',
+        adapter: cohereAdapter,
+      };
+    }
+
+    if (id === 'google') {
+      return {
+        providerId: 'google',
+        model: 'text-embedding-004',
+        adapter: googleAdapter,
+      };
+    }
+
+    if (id === 'ollama' && provider.baseUrl) {
+      return {
+        providerId: 'ollama',
+        model: 'nomic-embed-text:latest',
+        adapter: ollamaAdapter,
+      };
+    }
+  }
+
+  return null;
+}
+
+// ============================================================
+// PROVIDER ADAPTERS
+// ============================================================
+
+const openAIAdapter: ProviderAdapter = {
+  modelMap(model: string): string | null {
+    const valid = ['text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'];
+    return valid.includes(model) ? model : 'text-embedding-3-small';
+  },
+  urlFor(_apiKey: string, _model: string): string {
+    return 'https://api.openai.com/v1/embeddings';
+  },
+  async call(url, apiKey, model, input, dimensions): Promise<number[][]> {
+    const body: Record<string, unknown> = { model, input };
+    if (dimensions) body.dimensions = dimensions;
+    const resp = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+      body: JSON.stringify(body),
+    });
+    const data = await resp.json() as any;
+    if (data.error) throw new Error(data.error.message || JSON.stringify(data.error));
+    return data.data.map((item: any) => item.embedding as number[]);
+  },
+};
+
+const cohereAdapter: ProviderAdapter = {
+  modelMap(model: string): string | null {
+    const valid = ['embed-english-v3.0', 'embed-multilingual-v3.0', 'embed-english-v2.0'];
+    return valid.includes(model) ? model : 'embed-english-v3.0';
+  },
+  urlFor(_apiKey: string, _model: string): string {
+    return 'https://api.cohere.ai/v2/embed';
+  },
+  async call(url, apiKey, model, input, _dimensions): Promise<number[][]> {
+    const resp = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+      body: JSON.stringify({ model, texts: input }),
+    });
+    const data = await resp.json() as any;
+    if (data.error) throw new Error(data.error.message || JSON.stringify(data.error));
+    return data.embeddings as number[][];
+  },
+};
+
+const googleAdapter: ProviderAdapter = {
+  modelMap(model: string): string | null {
+    return 'text-embedding-004';
+  },
+  urlFor(apiKey: string, _model: string): string {
+    return `https://generativelanguage.googleapis.com/v1beta2/models/text-embedding-004:predict?key=${apiKey}`;
+  },
+  async call(url, _apiKey, model, input, _dimensions): Promise<number[][]> {
+    const resp = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        instances: input.map((text) => ({ content: text })),
+        parameters: { outputDimensionality: 768 },
+      }),
+    });
+    const data = await resp.json() as any;
+    if (data.error) throw new Error(data.error.message || JSON.stringify(data.error));
+    return data.predictions.map((p: any) => p.embeddings.values as number[]);
+  },
+};
+
+const ollamaAdapter: ProviderAdapter = {
+  modelMap(model: string): string | null {
+    return model || 'nomic-embed-text:latest';
+  },
+  urlFor(baseUrl: string): string {
+    return `${baseUrl}/api/embeddings`;
+  },
+  async call(url, _apiKey, model, input): Promise<number[][]> {
+    const results: number[][] = [];
+    for (const text of input) {
+      const resp = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ model, prompt: text }),
+      });
+      const data = await resp.json() as any;
+      if (data.error) throw new Error(data.error);
+      results.push(data.embedding as number[]);
+    }
+    return results;
+  },
+};
+
+// ============================================================
+// HANDLER
+// ============================================================
+
+export async function handleEmbeddings(
+  req: http.IncomingMessage,
+  res: http.ServerResponse,
+  ctx: RouteContext
+): Promise<void> {
+  recordActiveRequest(1);
+  const body = await readBody(req);
+  let request: EmbeddingsRequest;
+
+  try {
+    request = JSON.parse(body);
+  } catch {
+    recordActiveRequest(-1);
+    errorResponse(res, 400, 'Invalid JSON in request body', 'invalid_request_error');
+    return;
+  }
+
+  const inputs = Array.isArray(request.input) ? request.input : [request.input];
+  if (inputs.length === 0) {
+    recordActiveRequest(-1);
+    errorResponse(res, 400, 'input is required', 'invalid_request_error');
+    return;
+  }
+
+  const provider = getEmbeddingsProvider();
+  if (!provider) {
+    recordActiveRequest(-1);
+    errorResponse(res, 503, 'No embeddings provider available. Configure API keys for OpenAI, Cohere, or Google.', 'server_error');
+    return;
+  }
+
+  const model = request.model || provider.model;
+  const adapter = provider.adapter;
+  const providers = getAvailableProviders();
+  const providerConfig = providers[provider.providerId];
+  const resolvedModel = adapter.modelMap(model) || provider.model;
+  const url = adapter.urlFor(providerConfig.apiKey || '', resolvedModel);
+
+  const startTime = Date.now();
+
+  try {
+    const embeddings = await adapter.call(url, providerConfig.apiKey || '', resolvedModel, inputs, request.dimensions);
+    const latencyMs = Date.now() - startTime;
+
+    const response = {
+      object: 'list',
+      data: embeddings.map((embedding, i) => ({
+        object: 'embedding',
+        embedding,
+        index: i,
+      })),
+      model: resolvedModel,
+      usage: {
+        prompt_tokens: 0, // Embedding APIs don't always return token counts
+        total_tokens: 0,
+      },
+    };
+
+    jsonResponse(res, 200, response);
+    recordRequest({
+      endpoint: '/v1/embeddings',
+      provider: provider.providerId,
+      status: 'success',
+      durationMs: latencyMs,
+      model: resolvedModel,
+    });
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!res.headersSent) errorResponse(res, 502, message, 'upstream_error');
+    recordProviderError(provider.providerId, 'embeddings_error');
+    console.error(`[a3m-router] ${ctx.requestId} embeddings ERROR ${provider.providerId}: ${message}`);
+  }
+
+  recordActiveRequest(-1);
+}
+
+export default handleEmbeddings;

package/src/server/handlers/healthHandler.ts

@@ -0,0 +1,57 @@
+/**
+ * A3M Router - Health Handler
+ *
+ * Handles GET /health
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { getAvailableProviders } from '../../providers/providerConfig';
+import { costTracker, requestLogs } from '../state';
+
+function jsonResponse(res: http.ServerResponse, statusCode: number, body: object): void {
+  res.writeHead(statusCode, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
+  res.end(JSON.stringify(body));
+}
+
+export async function handleHealth(
+  _req: http.IncomingMessage,
+  res: http.ServerResponse,
+  _ctx: RouteContext
+): Promise<void> {
+  const available = getAvailableProviders();
+  const providerStatus: Record<string, object> = {};
+  let healthyCount = 0;
+
+  for (const [id, provider] of Object.entries(available)) {
+    const hasKey = !!provider.apiKey;
+    const isAvailable = provider.type !== 'api' || hasKey;
+    providerStatus[id] = {
+      name: provider.name || id,
+      type: provider.type,
+      models: provider.models?.length || 0,
+      available: isAvailable,
+    };
+    if (isAvailable) healthyCount++;
+  }
+
+  const costSummary = costTracker.getSummary();
+
+  jsonResponse(res, 200, {
+    status: 'ok',
+    version: '2.14.60',
+    providers: {
+      total: Object.keys(available).length,
+      healthy: healthyCount,
+      details: providerStatus,
+    },
+    cost: {
+      total: costSummary.total_cost,
+      requests: costSummary.request_count,
+    },
+    uptime: process.uptime(),
+    recentRequests: requestLogs.slice(-20),
+  });
+}
+
+export default handleHealth;

package/src/server/handlers/metricsHandler.ts

@@ -0,0 +1,30 @@
+/**
+ * A3M Router - Prometheus Metrics Handler
+ *
+ * Handles GET /metrics
+ * Returns Prometheus-compatible metrics output.
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { generatePrometheusMetrics } from '../metrics';
+
+function textResponse(res: http.ServerResponse, statusCode: number, body: string): void {
+  res.writeHead(statusCode, {
+    'Content-Type': 'text/plain; version=0.0.4; charset=utf-8',
+    'Access-Control-Allow-Origin': '*',
+    'Cache-Control': 'no-cache',
+  });
+  res.end(body);
+}
+
+export function handleMetrics(
+  _req: http.IncomingMessage,
+  res: http.ServerResponse,
+  _ctx: RouteContext
+): void {
+  const output = generatePrometheusMetrics();
+  textResponse(res, 200, output);
+}
+
+export default handleMetrics;

package/src/server/handlers/modelsHandler.ts

@@ -0,0 +1,25 @@
+/**
+ * A3M Router - Models Handler
+ *
+ * Handles GET /v1/models
+ */
+
+import * as http from 'http';
+import { RouteContext } from '../router';
+import { listAvailableModels } from '../modelMapper';
+
+function jsonResponse(res: http.ServerResponse, statusCode: number, body: object): void {
+  res.writeHead(statusCode, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
+  res.end(JSON.stringify(body));
+}
+
+export function handleModels(
+  _req: http.IncomingMessage,
+  res: http.ServerResponse,
+  _ctx: RouteContext
+): void {
+  const models = listAvailableModels();
+  jsonResponse(res, 200, { object: 'list', data: models });
+}
+
+export default handleModels;