adaptive-memory-multi-model-router 1.2.2 → 1.3.1

package/rust/tmlpd.h

@@ -1,268 +0,0 @@
-/**
- * TMLPD Rust Bindings - FFI Interface
- * 
- * High-performance Rust library for TMLPD operations.
- * Enables zero-overhead integration with Rust projects.
- * 
- * Build: cargo build --release
- * Use:   npm install tmlpd-pi (Rust bindings auto-included)
- */
-
-#include <stdint.h>
-#include <stddef.h>
-#include <stdbool.h>
-
-#ifdef _WIN32
-#define TMLPD_EXPORT __declspec(dllexport)
-#else
-#define TMLPD_EXPORT
-#endif
-
-// Error codes
-typedef enum {
-    TMLPD_OK = 0,
-    TMLPD_ERR_INVALID_INPUT = 1,
-    TMLPD_ERR_PROVIDER_FAILED = 2,
-    TMLPD_ERR_TIMEOUT = 3,
-    TMLPD_ERR_NO_PROVIDERS = 4,
-    TMLPD_ERR_OUT_OF_MEMORY = 5
-} tmlpd_error_t;
-
-// Result structure
-typedef struct {
-    tmlpd_error_t error;
-    char* content;
-    uint32_t tokens_used;
-    double cost_usd;
-    uint64_t duration_ms;
-    bool cached;
-} tmlpd_result_t;
-
-// Config structure
-typedef struct {
-    uint32_t max_concurrent;
-    uint32_t cache_ttl_seconds;
-    double daily_budget_usd;
-    double retry_base_delay_ms;
-    double retry_jitter;
-} tmlpd_config_t;
-
-// Initialize TMLPD with config
-TMLPD_EXPORT tmlpd_error_t tmlpd_init(tmlpd_config_t* config);
-
-// Execute single prompt
-TMLPD_EXPORT tmlpd_result_t* tmlpd_execute(
-    const char* prompt,
-    const char* model,
-    double timeout_ms
-);
-
-// Execute parallel across multiple models
-TMLPD_EXPORT tmlpd_result_t** tmlpd_execute_parallel(
-    const char* prompt,
-    const char** models,
-    uint32_t model_count,
-    double timeout_ms,
-    uint32_t* result_count
-);
-
-// Token counting (no API call needed)
-TMLPD_EXPORT uint32_t tmlpd_count_tokens(
-    const char* text,
-    const char* model
-);
-
-// Estimate cost before execution
-TMLPD_EXPORT double tmlpd_estimate_cost(
-    uint32_t prompt_tokens,
-    uint32_t completion_tokens,
-    const char* model
-);
-
-// ISON compression
-TMLPD_EXPORT char* tmlpd_ison_encode(const char* text);
-TMLPD_EXPORT char* tmlpd_ison_decode(const char* encoded);
-
-// Memory operations
-TMLPD_EXPORT char* tmlpd_store_episode(
-    const char* task_desc,
-    const char* result,
-    const char* model,
-    double cost
-);
-
-TMLPD_EXPORT char* tmlpd_query_similar(
-    const char* task_desc,
-    uint32_t limit
-);
-
-// Cleanup
-TMLPD_EXPORT void tmlpd_free_result(tmlpd_result_t* result);
-TMLPD_EXPORT void tmlpd_free_string(char* str);
-TMLPD_EXPORT void tmlpd_shutdown(void);
-
-// ============================================
-// Implementation stubs (for demonstration)
-// In production, these call actual Rust lib
-// ============================================
-
-#ifdef TMLPD_IMPLEMENTATION
-
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-static tmlpd_config_t g_config = {0};
-static bool g_initialized = false;
-
-TMLPD_EXPORT tmlpd_error_t tmlpd_init(tmlpd_config_t* config) {
-    if (!config) return TMLPD_ERR_INVALID_INPUT;
-    memcpy(&g_config, config, sizeof(tmlpd_config_t));
-    g_initialized = true;
-    return TMLPD_OK;
-}
-
-TMLPD_EXPORT tmlpd_result_t* tmlpd_execute(
-    const char* prompt,
-    const char* model,
-    double timeout_ms
-) {
-    if (!g_initialized || !prompt || !model) return NULL;
-    
-    tmlpd_result_t* result = (tmlpd_result_t*)malloc(sizeof(tmlpd_result_t));
-    if (!result) return NULL;
-    
-    // Simulate execution
-    result->error = TMLPD_OK;
-    result->tokens_used = (uint32_t)(strlen(prompt) / 4);
-    result->cost_usd = result->tokens_used * 0.00001;
-    result->duration_ms = (uint64_t)(rand() % 1000 + 100);
-    result->cached = false;
-    
-    // Allocate and fill content
-    size_t content_len = strlen(prompt) + 20;
-    result->content = (char*)malloc(content_len);
-    snprintf(result->content, content_len, "[TMLPD Rust] Processed: %s", prompt);
-    
-    return result;
-}
-
-TMLPD_EXPORT tmlpd_result_t** tmlpd_execute_parallel(
-    const char* prompt,
-    const char** models,
-    uint32_t model_count,
-    double timeout_ms,
-    uint32_t* result_count
-) {
-    if (!result_count || model_count == 0) return NULL;
-    *result_count = model_count;
-    
-    tmlpd_result_t** results = (tmlpd_result_t**)malloc(
-        sizeof(tmlpd_result_t*) * model_count
-    );
-    
-    for (uint32_t i = 0; i < model_count; i++) {
-        results[i] = tmlpd_execute(prompt, models[i], timeout_ms);
-    }
-    
-    return results;
-}
-
-TMLPD_EXPORT uint32_t tmlpd_count_tokens(const char* text, const char* model) {
-    if (!text) return 0;
-    // Simple word-based approximation
-    uint32_t words = 0;
-    for (const char* p = text; *p; p++) {
-        if (*p == ' ') words++;
-    }
-    return (words + 1) * 13 / 10; // ~1.3 tokens per word
-}
-
-TMLPD_EXPORT double tmlpd_estimate_cost(
-    uint32_t prompt_tokens,
-    uint32_t completion_tokens,
-    const char* model
-) {
-    // Default GPT-4 pricing
-    double input_rate = 0.0025 / 1000;  // $2.50/1M
-    double output_rate = 0.01 / 1000;    // $10/1M
-    
-    if (strstr(model, "claude")) {
-        input_rate = 0.003 / 1000;
-        output_rate = 0.015 / 1000;
-    } else if (strstr(model, "gemini")) {
-        input_rate = 0.000075 / 1000;
-        output_rate = 0.0003 / 1000;
-    }
-    
-    return prompt_tokens * input_rate + completion_tokens * output_rate;
-}
-
-TMLPD_EXPORT char* tmlpd_ison_encode(const char* text) {
-    if (!text) return NULL;
-    
-    // Remove common words
-    size_t len = strlen(text) + 1;
-    char* result = (char*)malloc(len);
-    strcpy(result, text);
-    
-    // Simple ISON: remove articles
-    const char* articles[] = {" the ", " a ", " an ", " The ", " A ", " An "};
-    for (int i = 0; i < 6; i++) {
-        char* pos;
-        while ((pos = strstr(result, articles[i])) != NULL) {
-            memmove(pos, pos + strlen(articles[i]), 
-                    strlen(pos + strlen(articles[i])) + 1);
-        }
-    }
-    
-    return result;
-}
-
-TMLPD_EXPORT char* tmlpd_ison_decode(const char* encoded) {
-    // In production, reverse ISON encoding
-    if (!encoded) return NULL;
-    char* result = (char*)malloc(strlen(encoded) + 10);
-    sprintf(result, "The %s", encoded);
-    return result;
-}
-
-TMLPD_EXPORT char* tmlpd_store_episode(
-    const char* task_desc,
-    const char* result,
-    const char* model,
-    double cost
-) {
-    // In production, store in episodic memory
-    (void)task_desc; (void)result; (void)model; (void)cost;
-    char* id = (char*)malloc(16);
-    snprintf(id, 16, "ep_%ld", time(NULL));
-    return id;
-}
-
-TMLPD_EXPORT char* tmlpd_query_similar(
-    const char* task_desc,
-    uint32_t limit
-) {
-    (void)task_desc; (void)limit;
-    char* result = (char*)malloc(32);
-    strcpy(result, "[]"); // Empty array
-    return result;
-}
-
-TMLPD_EXPORT void tmlpd_free_result(tmlpd_result_t* result) {
-    if (result) {
-        if (result->content) free(result->content);
-        free(result);
-    }
-}
-
-TMLPD_EXPORT void tmlpd_free_string(char* str) {
-    if (str) free(str);
-}
-
-TMLPD_EXPORT void tmlpd_shutdown(void) {
-    g_initialized = false;
-}
-
-#endif // TMLPD_IMPLEMENTATION

package/src/cache/prefixCache.ts

@@ -1,365 +0,0 @@
-/**
- * TMLPD Prefix Cache - RadixAttention Style
- * 
- * Inspired by SGLang's RadixAttention (arXiv:2312.07104)
- * Caches KV states for common prefixes (system prompts, etc.)
- * 5-10x speedup for repeated prompt patterns
- */
-
-export interface CacheEntry {
-  key: string;                    // Hash of the prefix
-  prefix: string;                 // Original prefix text
-  kv_state?: Buffer;              // Cached KV state (if using actual KV cache)
-  response_hash?: string;         // Hash of cached response
-  hit_count: number;               // Times this prefix was used
-  last_used: number;              // Timestamp
-  token_count: number;            // Tokens in this prefix
-  children: Map<string, string>;   // child_key -> child_cache_key
-}
-
-export interface PrefixCacheStats {
-  total_entries: number;
-  total_hits: number;
-  total_misses: number;
-  hit_rate: number;
-  memory_estimate_mb: number;
-  oldest_entry_age_ms: number;
-}
-
-export class PrefixCache {
-  private entries: Map<string, CacheEntry> = new Map();
-  private access_order: string[] = [];  // LRU tracking
-  private max_entries: number;
-  private max_memory_mb: number;
-  
-  constructor(options?: {
-    max_entries?: number;
-    max_memory_mb?: number;
-  }) {
-    this.max_entries = options?.max_entries || 10000;
-    this.max_memory_mb = options?.max_memory_mb || 512;
-  }
-  
-  /**
-   * Generate cache key from text prefix
-   */
-  private generateKey(text: string, model?: string): string {
-    // Simple hash for now - in production use SHA-256
-    const normalized = text.toLowerCase().trim().substring(0, 500);
-    const str = `${model || "default"}:${normalized}`;
-    
-    let hash = 0;
-    for (let i = 0; i < str.length; i++) {
-      const char = str.charCodeAt(i);
-      hash = ((hash << 5) - hash) + char;
-      hash = hash & hash;  // Convert to 32bit integer
-    }
-    
-    return `pc_${Math.abs(hash).toString(16)}`;
-  }
-  
-  /**
-   * Check if prefix is cached
-   */
-  has(prefix: string, model?: string): boolean {
-    const key = this.generateKey(prefix, model);
-    return this.entries.has(key);
-  }
-  
-  /**
-   * Get cached entry
-   */
-  get(prefix: string, model?: string): CacheEntry | undefined {
-    const key = this.generateKey(prefix, model);
-    const entry = this.entries.get(key);
-    
-    if (entry) {
-      // Update LRU
-      this.updateLRU(key);
-      entry.hit_count++;
-      entry.last_used = Date.now();
-    }
-    
-    return entry;
-  }
-  
-  /**
-   * Store a new prefix with its KV state
-   */
-  store(
-    prefix: string,
-    options?: {
-      kv_state?: Buffer;
-      response_hash?: string;
-      model?: string;
-      children?: Map<string, string>;
-    }
-  ): string {
-    const key = this.generateKey(prefix, options?.model);
-    
-    // Check if already exists
-    if (this.entries.has(key)) {
-      const existing = this.entries.get(key)!;
-      existing.hit_count++;
-      existing.last_used = Date.now();
-      return key;
-    }
-    
-    // Estimate memory
-    const token_count = Math.ceil(prefix.split(/\s+/).length * 1.3);
-    const memory_bytes = token_count * 16 * 128 * 2;  // Rough KV estimate
-    const memory_mb = memory_bytes / (1024 * 1024);
-    
-    const entry: CacheEntry = {
-      key,
-      prefix: prefix.substring(0, 1000),  // Store truncated
-      kv_state: options?.kv_state,
-      response_hash: options?.response_hash,
-      hit_count: 1,
-      last_used: Date.now(),
-      token_count,
-      children: options?.children || new Map()
-    };
-    
-    // Evict if necessary
-    while (this.entries.size >= this.max_entries || this.getMemoryUsage() + memory_mb > this.max_memory_mb) {
-      this.evictLRU();
-    }
-    
-    this.entries.set(key, entry);
-    this.access_order.push(key);
-    
-    return key;
-  }
-  
-  /**
-   * Extend cached prefix with completion
-   */
-  extend(
-    prefix: string,
-    completion: string,
-    options?: { model?: string }
-  ): string {
-    const prefix_key = this.generateKey(prefix, options?.model);
-    const parent = this.entries.get(prefix_key);
-    
-    if (!parent) {
-      // No parent - just store completion as new entry
-      return this.store(completion, { model: options?.model });
-    }
-    
-    // Create child entry for the extended sequence
-    const extended = prefix + completion;
-    const child_key = this.store(extended, { model: options?.model });
-    
-    // Link child to parent
-    const completion_key = this.generateKey(completion);
-    parent.children.set(completion_key, child_key);
-    
-    return child_key;
-  }
-  
-  /**
-   * Find common prefix between two texts
-   */
-  findCommonPrefix(text1: string, text2: string): string {
-    const words1 = text1.split(/\s+/);
-    const words2 = text2.split(/\s+/);
-    
-    let common_length = 0;
-    for (let i = 0; i < Math.min(words1.length, words2.length); i++) {
-      if (words1[i].toLowerCase() === words2[i].toLowerCase()) {
-        common_length = i + 1;
-      } else {
-        break;
-      }
-    }
-    
-    return words1.slice(0, common_length).join(" ");
-  }
-  
-  /**
-   * Lookup with prefix matching
-   * Returns cached entry if any prefix is found
-   */
-  lookup(text: string, model?: string): { cached: boolean; prefix?: string; remaining?: string } {
-    // Try exact match first
-    const exact_key = this.generateKey(text, model);
-    if (this.entries.has(exact_key)) {
-      return { cached: true };
-    }
-    
-    // Try progressively shorter prefixes
-    const words = text.split(/\s+/);
-    for (let len = words.length - 1; len >= 5; len--) {  // Min 5 words
-      const prefix = words.slice(0, len).join(" ");
-      const key = this.generateKey(prefix, model);
-      
-      if (this.entries.has(key)) {
-        const remaining = words.slice(len).join(" ");
-        return { cached: true, prefix, remaining };
-      }
-    }
-    
-    return { cached: false };
-  }
-  
-  /**
-   * Batch lookup for multiple texts
-   */
-  lookupBatch(texts: string[], model?: string): Array<{ cached: boolean; prefix?: string; remaining?: string }> {
-    return texts.map(t => this.lookup(t, model));
-  }
-  
-  /**
-   * Get cache statistics
-   */
-  getStats(): PrefixCacheStats {
-    const now = Date.now();
-    let oldest_age = 0;
-    let total_hits = 0;
-    
-    for (const entry of this.entries.values()) {
-      total_hits += entry.hit_count;
-      const age = now - entry.last_used;
-      if (age > oldest_age) oldest_age = age;
-    }
-    
-    const total_requests = total_hits + this.entries.size;  // Approximate
-    const hit_rate = total_requests > 0 ? total_hits / total_requests : 0;
-    
-    return {
-      total_entries: this.entries.size,
-      total_hits: total_hits,
-      total_misses: this.entries.size,  // Approximate
-      hit_rate,
-      memory_estimate_mb: this.getMemoryUsage(),
-      oldest_entry_age_ms: oldest_age
-    };
-  }
-  
-  /**
-   * Get estimated memory usage
-   */
-  private getMemoryUsage(): number {
-    let total_bytes = 0;
-    
-    for (const entry of this.entries.values()) {
-      // Base entry overhead
-      total_bytes += 200;
-      
-      // Prefix text
-      total_bytes += entry.prefix.length * 2;
-      
-      // KV state (if stored)
-      if (entry.kv_state) {
-        total_bytes += entry.kv_state.length;
-      }
-      
-      // Children map
-      total_bytes += entry.children.size * 50;
-    }
-    
-    return total_bytes / (1024 * 1024);
-  }
-  
-  /**
-   * Update LRU order
-   */
-  private updateLRU(key: string): void {
-    const index = this.access_order.indexOf(key);
-    if (index > -1) {
-      this.access_order.splice(index, 1);
-    }
-    this.access_order.push(key);
-  }
-  
-  /**
-   * Evict least recently used entry
-   */
-  private evictLRU(): boolean {
-    if (this.access_order.length === 0) return false;
-    
-    const lru_key = this.access_order.shift()!;
-    const entry = this.entries.get(lru_key);
-    
-    if (entry) {
-      // If has children, re-parent them
-      for (const [child_key, child_cache_key] of entry.children) {
-        const child = this.entries.get(child_cache_key);
-        if (child) {
-          // Promote child to standalone
-          this.access_order.push(child_cache_key);
-        }
-      }
-      
-      this.entries.delete(lru_key);
-      return true;
-    }
-    
-    return false;
-  }
-  
-  /**
-   * Clear all cache
-   */
-  clear(): void {
-    this.entries.clear();
-    this.access_order = [];
-  }
-  
-  /**
-   * Invalidate entries matching pattern
-   */
-  invalidate(pattern?: string): number {
-    let count = 0;
-    
-    if (!pattern) {
-      // Clear all
-      count = this.entries.size;
-      this.clear();
-      return count;
-    }
-    
-    // Pattern-based invalidation
-    for (const [key, entry] of this.entries) {
-      if (entry.prefix.includes(pattern)) {
-        this.entries.delete(key);
-        count++;
-      }
-    }
-    
-    return count;
-  }
-  
-  /**
-   * Warm up cache with common system prompts
-   */
-  warmup(common_prefixes: string[], model?: string): void {
-    for (const prefix of common_prefixes) {
-      this.store(prefix, { model });
-    }
-    console.log(`[PrefixCache] Warmed up with ${common_prefixes.length} common prefixes`);
-  }
-}
-
-// Common system prompts that benefit from prefix caching
-const COMMON_SYSTEM_PROMPTS = [
-  "You are a helpful assistant.",
-  "You are a coding assistant. Help with programming tasks.",
-  "You are an expert data scientist.",
-  "You are a senior software engineer.",
-  "Analyze the following code and provide feedback.",
-  "Explain this concept in simple terms.",
-  "Write clean, well-documented code.",
-  "Think step by step and explain your reasoning."
-];
-
-export default PrefixCache;
-
-// Utility function for creating pre-warmed cache
-export function createWarmedCache(): PrefixCache {
-  const cache = new PrefixCache({ max_entries: 5000 });
-  cache.warmup(COMMON_SYSTEM_PROMPTS);
-  return cache;
-}