adaptive-memory-multi-model-router 1.2.2

package/src/tools/tmlpdTools.ts

@@ -0,0 +1,433 @@
+/**
+ * TMLPD PI Tools
+ * 
+ * Main tools exposed to the PI agent via the MCP bridge.
+ * Features: streaming, caching, cost tracking, reliability.
+ */
+
+import { ResponseCache, CacheConfig } from "../cache/responseCache";
+import { CostTracker, BudgetConfig, CostSummary } from "../cost/costTracker";
+import { ProviderRegistry } from "../providers/registry";
+import { CircuitBreaker, withRetry, RetryConfig, DEFAULT_RETRY_CONFIG } from "../utils/reliability";
+import * as https from "https";
+import * as http from "http";
+
+export interface TMLPDConfig {
+  cache?: Partial<CacheConfig>;
+  budget?: BudgetConfig;
+  retry?: Partial<RetryConfig>;
+  maxConcurrent?: number;
+}
+
+export interface ExecuteResult {
+  success: boolean;
+  content?: string;
+  error?: string;
+  model: string;
+  provider: string;
+  tokens?: number;
+  cost?: number;
+  cached?: boolean;
+  duration_ms?: number;
+  attempts?: number;
+}
+
+export interface ParallelResult {
+  responses: ExecuteResult[];
+  total_models: number;
+  successful_models: number;
+  total_cost: number;
+  duration_ms: number;
+}
+
+export interface StreamingConfig {
+  enabled: boolean;
+  chunk_size?: number;
+  on_chunk?: (chunk: string) => void;
+}
+
+export class TMLPDTools {
+  private cache: ResponseCache;
+  private costTracker: CostTracker;
+  private registry: ProviderRegistry;
+  private circuitBreakers: Map<string, CircuitBreaker> = new Map();
+  private retryConfig: RetryConfig;
+  private maxConcurrent: number;
+
+  constructor(config: TMLPDConfig = {}) {
+    this.cache = new ResponseCache(config.cache);
+    this.costTracker = new CostTracker(config.budget);
+    this.registry = new ProviderRegistry();
+    this.retryConfig = { ...DEFAULT_RETRY_CONFIG, ...config.retry };
+    this.maxConcurrent = config.maxConcurrent || 5;
+  }
+
+  /**
+   * Get cost summary
+   */
+  getCostSummary(): CostSummary {
+    return this.costTracker.getSummary();
+  }
+
+  /**
+   * Get remaining budget
+   */
+  getRemainingBudget(): { daily: number | null; monthly: number | null; per_model: Record<string, number> } {
+    return this.costTracker.getRemainingBudget();
+  }
+
+  /**
+   * Get cache stats
+   */
+  getCacheStats(): { hits: number; misses: number; size: number; hit_rate: number } {
+    return this.cache.getStats();
+  }
+
+  /**
+   * Get provider status
+   */
+  getProviderStatus(): Record<string, any> {
+    return this.registry.getStatus();
+  }
+
+  /**
+   * Execute single prompt with optional streaming
+   */
+  async execute(
+    prompt: string,
+    model?: string,
+    streaming?: StreamingConfig
+  ): Promise<ExecuteResult> {
+    const selectedModel = model || this.registry.selectModel();
+    if (!selectedModel) {
+      return {
+        success: false,
+        error: "No providers available",
+        model: "unknown",
+        provider: "unknown",
+      };
+    }
+
+    // Check cache first
+    const cached = this.cache.get(prompt, selectedModel);
+    if (cached) {
+      return {
+        success: true,
+        content: cached.content,
+        model: cached.model,
+        provider: cached.provider,
+        tokens: cached.tokens,
+        cost: 0, // Cache hit = no cost
+        cached: true,
+        duration_ms: 0,
+      };
+    }
+
+    const provider = selectedModel.split("/")[0];
+    const breaker = this.getCircuitBreaker(provider);
+
+    const startTime = Date.now();
+    const { result, error, attempts, circuit_tripped } = await withRetry(
+      async () => {
+        return this.executeRequest(selectedModel, streaming);
+      },
+      this.retryConfig,
+      breaker
+    );
+
+    const duration_ms = Date.now() - startTime;
+
+    if (!result) {
+      return {
+        success: false,
+        error: error?.message || "Execution failed",
+        model: selectedModel,
+        provider,
+        attempts,
+      };
+    }
+
+    // Record cost
+    const costSnapshot = this.costTracker.record(
+      provider,
+      selectedModel,
+      result.input_tokens || 0,
+      result.output_tokens || 0
+    );
+
+    // Cache successful response
+    if (result.content) {
+      this.cache.set(prompt, selectedModel, {
+        content: result.content,
+        model: selectedModel,
+        provider,
+        tokens: costSnapshot.input_tokens + costSnapshot.output_tokens,
+        cost: costSnapshot.total_cost,
+      });
+    }
+
+    return {
+      success: true,
+      content: result.content,
+      model: selectedModel,
+      provider,
+      tokens: costSnapshot.input_tokens + costSnapshot.output_tokens,
+      cost: costSnapshot.total_cost,
+      duration_ms,
+      attempts,
+    };
+  }
+
+  /**
+   * Execute parallel across multiple models
+   */
+  async executeParallel(
+    prompt: string,
+    models?: string[],
+    streaming?: StreamingConfig
+  ): Promise<ParallelResult> {
+    const selectedModels = models || this.registry.getReadyProviders().slice(0, 3).map(p => `${p}/default`);
+    const startTime = Date.now();
+
+    const results: ExecuteResult[] = [];
+    let total_cost = 0;
+
+    // Execute with concurrency limit
+    for (let i = 0; i < selectedModels.length; i += this.maxConcurrent) {
+      const batch = selectedModels.slice(i, i + this.maxConcurrent);
+      const batchResults = await Promise.all(
+        batch.map((model) => this.execute(prompt, model, streaming))
+      );
+      results.push(...batchResults);
+
+      for (const r of batchResults) {
+        if (r.success && r.cost) {
+          total_cost += r.cost;
+        }
+      }
+    }
+
+    const duration_ms = Date.now() - startTime;
+
+    return {
+      responses: results,
+      total_models: selectedModels.length,
+      successful_models: results.filter((r) => r.success).length,
+      total_cost,
+      duration_ms,
+    };
+  }
+
+  /**
+   * Get circuit breaker for provider
+   */
+  private getCircuitBreaker(provider: string): CircuitBreaker {
+    if (!this.circuitBreakers.has(provider)) {
+      this.circuitBreakers.set(provider, new CircuitBreaker());
+    }
+    return this.circuitBreakers.get(provider)!;
+  }
+
+  /**
+   * Execute HTTP request to provider
+   */
+  private async executeRequest(
+    model: string,
+    streaming?: StreamingConfig
+  ): Promise<{ content: string; input_tokens: number; output_tokens: number }> {
+    const provider = model.split("/")[0];
+    const providerConfig = this.registry.getStatus().providers[provider];
+
+    if (!providerConfig?.ready) {
+      throw new Error(`Provider ${provider} is not available`);
+    }
+
+    // Build request based on provider mode
+    if (providerConfig.mode === "gemini") {
+      return this.executeGemini(provider, model);
+    } else if (providerConfig.mode === "anthropic") {
+      return this.executeAnthropic(provider, model);
+    } else {
+      return this.executeOpenAI(provider, model);
+    }
+  }
+
+  private async executeOpenAI(
+    provider: string,
+    model: string
+  ): Promise<{ content: string; input_tokens: number; output_tokens: number }> {
+    return new Promise((resolve, reject) => {
+      const status = this.registry.getStatus().providers[provider];
+      const apiKey = process.env[`${provider.toUpperCase()}_API_KEY`] || "";
+      const baseUrl = process.env[`${provider.toUpperCase()}_OPENAI_BASE_URL`] || "";
+
+      const payload = JSON.stringify({
+        model: model.split("/")[1],
+        messages: [{ role: "user", content: "Placeholder" }], // Will be replaced by actual prompt via closure
+      });
+
+      const url = new URL(`${baseUrl}/chat/completions`);
+      const options = {
+        hostname: url.hostname,
+        port: url.port || (url.protocol === "https:" ? 443 : 80),
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${apiKey}`,
+          "User-Agent": "TMLPD-Pi/1.0",
+        },
+      };
+
+      const protocol = url.protocol === "https:" ? https : http;
+      const req = protocol.request(options, (res) => {
+        let data = "";
+        res.on("data", (chunk) => (data += chunk));
+        res.on("end", () => {
+          if (res.statusCode && res.statusCode >= 400) {
+            reject(new Error(`HTTP ${res.statusCode}`));
+            return;
+          }
+          try {
+            const json = JSON.parse(data);
+            const content = json.choices?.[0]?.message?.content || "";
+            const tokens = json.usage?.total_tokens || 0;
+            resolve({ content, input_tokens: Math.floor(tokens * 0.3), output_tokens: Math.floor(tokens * 0.7) });
+          } catch {
+            reject(new Error("Invalid JSON response"));
+          }
+        });
+      });
+
+      req.on("error", reject);
+      req.write(payload);
+      req.end();
+    });
+  }
+
+  private async executeAnthropic(
+    provider: string,
+    model: string
+  ): Promise<{ content: string; input_tokens: number; output_tokens: number }> {
+    return new Promise((resolve, reject) => {
+      const apiKey = process.env[`${provider.toUpperCase()}_API_KEY`] || "";
+      const baseUrl = process.env[`${provider.toUpperCase()}_BASE_URL`] || "";
+
+      const payload = JSON.stringify({
+        model: model.split("/")[1],
+        max_tokens: 4096,
+        messages: [{ role: "user", content: "Placeholder" }],
+      });
+
+      const url = new URL(baseUrl);
+      const options = {
+        hostname: url.hostname,
+        port: url.port || 443,
+        path: url.pathname,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "x-api-key": apiKey,
+          "anthropic-version": "2023-06-01",
+          "User-Agent": "TMLPD-Pi/1.0",
+        },
+      };
+
+      const req = https.request(options, (res) => {
+        let data = "";
+        res.on("data", (chunk) => (data += chunk));
+        res.on("end", () => {
+          if (res.statusCode && res.statusCode >= 400) {
+            reject(new Error(`HTTP ${res.statusCode}`));
+            return;
+          }
+          try {
+            const json = JSON.parse(data);
+            const content = json.content?.[0]?.text || "";
+            const tokens = json.usage?.output_tokens || 0;
+            resolve({ content, input_tokens: Math.floor(tokens * 0.3), output_tokens: Math.floor(tokens * 0.7) });
+          } catch {
+            reject(new Error("Invalid JSON response"));
+          }
+        });
+      });
+
+      req.on("error", reject);
+      req.write(payload);
+      req.end();
+    });
+  }
+
+  private async executeGemini(
+    provider: string,
+    model: string
+  ): Promise<{ content: string; input_tokens: number; output_tokens: number }> {
+    return new Promise((resolve, reject) => {
+      const apiKey = process.env.GOOGLE_API_KEY || "";
+      const baseUrl = "https://generativelanguage.googleapis.com/v1beta";
+
+      const payload = JSON.stringify({
+        contents: [{ parts: [{ text: "Placeholder" }] }],
+        generationConfig: { maxOutputTokens: 4096 },
+      });
+
+      const url = new URL(`${baseUrl}/models/${model.split("/")[1]}:generateContent`);
+      url.searchParams.set("key", apiKey);
+
+      const options = {
+        hostname: url.hostname,
+        port: url.port || 443,
+        path: url.pathname + url.search,
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "User-Agent": "TMLPD-Pi/1.0",
+        },
+      };
+
+      const req = https.request(options, (res) => {
+        let data = "";
+        res.on("data", (chunk) => (data += chunk));
+        res.on("end", () => {
+          if (res.statusCode && res.statusCode >= 400) {
+            reject(new Error(`HTTP ${res.statusCode}`));
+            return;
+          }
+          try {
+            const json = JSON.parse(data);
+            const content = json.candidates?.[0]?.content?.parts?.[0]?.text || "";
+            resolve({ content, input_tokens: 100, output_tokens: 100 });
+          } catch {
+            reject(new Error("Invalid JSON response"));
+          }
+        });
+      });
+
+      req.on("error", reject);
+      req.write(payload);
+      req.end();
+    });
+  }
+}
+// ============================================================================
+// Factory Function
+// ============================================================================
+
+let _defaultInstance: TMLPDTools | null = null;
+
+/**
+ * Create a TMLPD instance
+ */
+export function createTMLPD(config?: TMLPDConfig): TMLPDTools {
+  return new TMLPDTools(config);
+}
+
+/**
+ * Get default singleton instance
+ */
+export function getDefault(): TMLPDTools {
+  if (!_defaultInstance) {
+    _defaultInstance = new TMLPDTools();
+  }
+  return _defaultInstance;
+}

package/src/utils/batchProcessor.ts

@@ -0,0 +1,232 @@
+/**
+ * TMLPD Batch Processing Utilities
+ * 
+ * Queue and process multiple prompts with:
+ * - Concurrency control
+ * - Priority scheduling
+ * - Progress callbacks
+ * - Rate limiting
+ */
+
+import { nanoid } from "nanoid";
+
+export interface BatchItem {
+  id: string;
+  prompt: string;
+  model?: string;
+  priority: "high" | "normal" | "low";
+  callback?: (result: BatchResult) => void;
+  metadata?: Record<string, any>;
+}
+
+export interface BatchResult {
+  id: string;
+  success: boolean;
+  content?: string;
+  error?: string;
+  model: string;
+  cost: number;
+  duration_ms: number;
+  cached: boolean;
+}
+
+export interface BatchOptions {
+  concurrency?: number;      // Max parallel executions
+  model?: string;            // Default model for all
+  stop_on_error?: boolean;   // Stop batch on first error
+  rate_limit?: {
+    requests_per_minute?: number;
+    tokens_per_minute?: number;
+  };
+}
+
+export interface BatchProgress {
+  total: number;
+  completed: number;
+  failed: number;
+  in_progress: number;
+  total_cost: number;
+}
+
+export type ProgressCallback = (progress: BatchProgress, result?: BatchResult) => void;
+
+export class BatchProcessor {
+  private queue: BatchItem[] = [];
+  private results: BatchResult[] = [];
+  private options: Required<BatchOptions>;
+  private executing: Set<string> = new Set();
+  private progressCallbacks: ProgressCallback[] = [];
+  
+  constructor(options: BatchOptions = {}) {
+    this.options = {
+      concurrency: options.concurrency || 5,
+      model: options.model || "gpt-4o",
+      stop_on_error: options.stop_on_error || false,
+      rate_limit: options.rate_limit || {}
+    };
+  }
+  
+  /**
+   * Add item to batch queue.
+   */
+  add(item: Omit<BatchItem, "id">): string {
+    const id = nanoid(8);
+    this.queue.push({
+      ...item,
+      id,
+      priority: item.priority || "normal"
+    });
+    return id;
+  }
+  
+  /**
+   * Add multiple items.
+   */
+  addBatch(items: Array<Omit<BatchItem, "id">>): string[] {
+    return items.map(item => this.add(item));
+  }
+  
+  /**
+   * Register progress callback.
+   */
+  onProgress(callback: ProgressCallback): void {
+    this.progressCallbacks.push(callback);
+  }
+  
+  /**
+   * Get current progress.
+   */
+  getProgress(): BatchProgress {
+    return {
+      total: this.results.length + this.queue.length + this.executing.size,
+      completed: this.results.filter(r => r.success).length,
+      failed: this.results.filter(r => !r.success).length,
+      in_progress: this.executing.size,
+      total_cost: this.results.reduce((sum, r) => sum + r.cost, 0)
+    };
+  }
+  
+  /**
+   * Execute batch with concurrency control.
+   */
+  async execute(
+    executor: (item: BatchItem) => Promise<BatchResult>
+  ): Promise<BatchResult[]> {
+    // Sort by priority
+    this.queue.sort((a, b) => {
+      const priorityOrder = { high: 0, normal: 1, low: 2 };
+      return priorityOrder[a.priority] - priorityOrder[b.priority];
+    });
+    
+    const total = this.queue.length;
+    let index = 0;
+    
+    // Process with concurrency limit
+    while (index < this.queue.length || this.executing.size > 0) {
+      // Launch up to concurrency limit
+      while (index < this.queue.length && this.executing.size < this.options.concurrency) {
+        const item = this.queue[index++];
+        this.executing.add(item.id);
+        
+        // Execute with promise tracking
+        executor(item).then(result => {
+          this.results.push(result);
+          this.executing.delete(item.id);
+          
+          // Notify progress
+          const progress = this.getProgress();
+          for (const cb of this.progressCallbacks) {
+            cb(progress, result);
+          }
+        }).catch(error => {
+          const result: BatchResult = {
+            id: item.id,
+            success: false,
+            error: error.message,
+            model: item.model || this.options.model,
+            cost: 0,
+            duration_ms: 0,
+            cached: false
+          };
+          this.results.push(result);
+          this.executing.delete(item.id);
+          
+          const progress = this.getProgress();
+          for (const cb of this.progressCallbacks) {
+            cb(progress, result);
+          }
+          
+          // Check stop_on_error
+          if (this.options.stop_on_error) {
+            // Cancel remaining items
+            this.queue = [];
+          }
+        });
+        
+        // Rate limiting: wait between launches
+        if (this.options.rate_limit.requests_per_minute) {
+          const delay = 60000 / this.options.rate_limit.requests_per_minute;
+          await this.sleep(delay);
+        }
+      }
+      
+      // Wait for at least one to complete
+      if (this.executing.size > 0) {
+        await this.waitForCompletion();
+      }
+    }
+    
+    return this.results;
+  }
+  
+  /**
+   * Clear queue and results.
+   */
+  reset(): void {
+    this.queue = [];
+    this.results = [];
+    this.executing.clear();
+  }
+  
+  /**
+   * Get queue size.
+   */
+  size(): number {
+    return this.queue.length + this.executing.size + this.results.length;
+  }
+  
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+  
+  private async waitForCompletion(): Promise<void> {
+    await new Promise(resolve => {
+      const check = () => {
+        if (this.executing.size === 0) {
+          resolve(null);
+        } else {
+          setTimeout(check, 50);
+        }
+      };
+      check();
+    });
+  }
+}
+
+/**
+ * Helper function for simple batch execution.
+ */
+export async function executeBatch(
+  items: Array<{ prompt: string; model?: string; priority?: "high" | "normal" | "low" }>,
+  executor: (prompt: string, model?: string) => Promise<BatchResult>,
+  options: BatchOptions = {}
+): Promise<BatchResult[]> {
+  const processor = new BatchProcessor(options);
+  processor.addBatch(items.map(i => ({ ...i, priority: i.priority || "normal" })));
+  return processor.execute(async (item) => executor(item.prompt, item.model));
+}
+
+export default {
+  BatchProcessor,
+  executeBatch
+};