adaptive-memory-multi-model-router 2.16.0 → 2.16.2

package/tmlpd-pi-extension/src/tokenOptimization/index.ts

@@ -0,0 +1,197 @@
+/**
+ * Token Optimization Module - Phase 1 Implementation
+ * 
+ * 6 patterns from arXiv:2608.17188 - Token Optimization and Context Window Management
+ * 
+ * Patterns:
+ * 1. SemanticCache - Embedding-based similarity caching
+ * 2. ContextStratifier -分层 context levels
+ * 3. TokenAwareFallback - Token-count based model routing
+ * 4. SchemaContractor - Schema-referenced prompts
+ * 5. FetchOnceProcessor - Fetch once, process locally
+ * 6. InterAgentCompressor - Message compression
+ */
+
+export { SemanticCache, SemanticCacheConfig, SemanticCacheEntry } from "./semanticCache";
+export { ContextStratifier, ContextLevel, ContextStratifierConfig, StratificationResult } from "./contextStratifier";
+export { TokenAwareFallback, TokenAwareFallbackConfig, FallbackDecision, TokenThreshold } from "./tokenAwareFallback";
+export { SchemaContractor, SchemaContractConfig, SchemaInfo } from "./schemaContract";
+export { FetchOnceProcessor, FetchOnceConfig, FetchOnceResult } from "./fetchOnceLocal";
+export { InterAgentCompressor, Message, CompressionConfig, CompressionResult } from "./interAgentCompression";
+
+/**
+ * Create a fully configured TokenOptimizer with all patterns
+ */
+import { SemanticCache } from "./semanticCache";
+import { ContextStratifier } from "./contextStratifier";
+import { TokenAwareFallback } from "./tokenAwareFallback";
+import { SchemaContractor } from "./schemaContract";
+import { FetchOnceProcessor } from "./fetchOnceLocal";
+import { InterAgentCompressor } from "./interAgentCompression";
+
+export interface TokenOptimizerConfig {
+  semanticCache?: boolean;
+  contextStratification?: boolean;
+  tokenAwareFallback?: boolean;
+  schemaContraction?: boolean;
+  fetchOnce?: boolean;
+  agentCompression?: boolean;
+}
+
+export class TokenOptimizer {
+  semanticCache: SemanticCache;
+  contextStratifier: ContextStratifier;
+  tokenAwareFallback: TokenAwareFallback;
+  schemaContractor: SchemaContractor;
+  fetchOnceProcessor: FetchOnceProcessor;
+  agentCompressor: InterAgentCompressor;
+  
+  private config: TokenOptimizerConfig;
+
+  constructor(config: TokenOptimizerConfig = {}) {
+    this.config = config;
+    
+    this.semanticCache = new SemanticCache({
+      enabled: config.semanticCache ?? true,
+    });
+    
+    this.contextStratifier = new ContextStratifier();
+    
+    this.tokenAwareFallback = new TokenAwareFallback();
+    
+    this.schemaContractor = new SchemaContractor();
+    
+    this.fetchOnceProcessor = new FetchOnceProcessor();
+    
+    this.agentCompressor = new InterAgentCompressor();
+  }
+
+  /**
+   * Optimize a query before sending to LLM
+   */
+  async optimizeQuery(
+    query: string,
+    history?: Array<{ role: string; content: string }>
+  ): Promise<{
+    query: string;
+    history: Array<{ role: string; content: string }>;
+    contextLevel: string;
+    recommendedModel: string;
+    cacheHit: boolean;
+    cachedResponse?: string;
+  }> {
+    // 1. Check semantic cache
+    const cacheResult = await this.semanticCache.get(query);
+    if (cacheResult) {
+      return {
+        query,
+        history: [],
+        contextLevel: "CACHED",
+        recommendedModel: "cached",
+        cacheHit: true,
+        cachedResponse: cacheResult.response,
+      };
+    }
+
+    // 2. Classify context level
+    const stratification = this.contextStratifier.classify(query, history);
+    
+    // 3. Get token-aware model recommendation
+    const totalTokens = this.tokenAwareFallback.estimateTotalTokens(query, history);
+    const modelDecision = this.tokenAwareFallback.selectModel(totalTokens);
+    
+    // 4. Stratify history
+    const stratifiedHistory = this.contextStratifier.stratifyHistory(
+      history as Array<{ role: string; content: string }>,
+      stratification.level
+    );
+
+    return {
+      query,
+      history: stratifiedHistory,
+      contextLevel: stratification.level,
+      recommendedModel: modelDecision.targetModel,
+      cacheHit: false,
+    };
+  }
+
+  /**
+   * Record a query-response pair in cache
+   */
+  async recordResponse(
+    query: string,
+    response: string,
+    options: {
+      model?: string;
+      tokens?: number;
+      cost?: number;
+    } = {}
+  ): Promise<void> {
+    await this.semanticCache.set(query, response, {
+      model: options.model,
+      tokensSaved: options.tokens,
+      costSaved: options.cost,
+    });
+  }
+
+  /**
+   * Get optimization statistics
+   */
+  getStats() {
+    return {
+      semanticCache: this.semanticCache.getStats(),
+      tokenAwareFallback: {
+        thresholds: this.tokenAwareFallback.getThresholds(),
+      },
+    };
+  }
+}
+
+/**
+ * Helper to estimate token savings across all patterns
+ */
+export function estimateTotalSavings(
+  patterns: ("semantic" | "context" | "fallback" | "schema" | "fetchOnce" | "compression")[],
+  queryTokens: number,
+  responseTokens: number
+): {
+  tokensSaved: number;
+  percentReduction: string;
+  patterns: string[];
+} {
+  let totalTokens = queryTokens + responseTokens;
+  let saved = 0;
+  const appliedPatterns: string[] = [];
+
+  if (patterns.includes("semantic")) {
+    // ~20% cache hit rate after warmup
+    saved += totalTokens * 0.2;
+    appliedPatterns.push("Semantic Caching (-20%)");
+  }
+
+  if (patterns.includes("context")) {
+    // ~30% reduction from context stratification
+    saved += totalTokens * 0.3;
+    appliedPatterns.push("Context Stratification (-30%)");
+  }
+
+  if (patterns.includes("fallback")) {
+    // ~40% savings from cheap model routing
+    saved += totalTokens * 0.4;
+    appliedPatterns.push("Token-Aware Fallback (-40%)");
+  }
+
+  if (patterns.includes("compression")) {
+    // ~15% from message compression
+    saved += totalTokens * 0.15;
+    appliedPatterns.push("Inter-Agent Compression (-15%)");
+  }
+
+  const percentReduction = totalTokens > 0 ? ((saved / totalTokens) * 100).toFixed(1) : "0%";
+
+  return {
+    tokensSaved: Math.round(saved),
+    percentReduction: percentReduction + "%",
+    patterns: appliedPatterns,
+  };
+}

package/tmlpd-pi-extension/src/tokenOptimization/interAgentCompression.ts

@@ -0,0 +1,157 @@
+/**
+ * Inter-Agent Communication Compression - Token Optimization Pattern #6
+ * 
+ * Compress messages between agents to reduce token usage.
+ * Uses summarization for long messages, preserves key info for short ones.
+ * 
+ * Based on: arXiv:2608.17188 - Token Optimization and Context Window Management
+ */
+
+export interface Message {
+  role: "user" | "assistant" | "system" | "tool";
+  content: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface CompressionConfig {
+  maxMessageLength: number;
+  summarizeThreshold: number;
+  preserveKeywords: string[];
+  summaryModel?: string;
+}
+
+export interface CompressionResult {
+  messages: Message[];
+  originalTokens: number;
+  compressedTokens: number;
+  compressionRatio: number;
+}
+
+/**
+ * Keywords that should be preserved in compression
+ */
+const PRESERVE_KEYWORDS = [
+  "error", "warning", "fail", "success", "api", "url",
+  "id", "token", "key", "config", "path", "file",
+  "function", "class", "method", "variable", "import",
+  "json", "xml", "html", "css", "sql"
+];
+
+/**
+ * Simple summarizer (in production, use actual LLM)
+ */
+function simpleSummarize(text: string, maxLength: number): string {
+  if (text.length <= maxLength) return text;
+  
+  // Truncate and add indicator
+  const summary = text.substring(0, maxLength - 20);
+  return summary + "... [truncated]";
+}
+
+export class InterAgentCompressor {
+  private config: CompressionConfig;
+
+  constructor(config: Partial<CompressionConfig> = {}) {
+    this.config = {
+      maxMessageLength: config.maxMessageLength ?? 500,
+      summarizeThreshold: config.summarizeThreshold ?? 300,
+      preserveKeywords: config.preserveKeywords ?? PRESERVE_KEYWORDS,
+      summaryModel: config.summaryModel ?? "gpt-4o-mini",
+    };
+  }
+
+  /**
+   * Estimate tokens in text (rough)
+   */
+  private estimateTokens(text: string): number {
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * Check if text contains preserved keywords
+   */
+  private containsPreservedKeyword(text: string): boolean {
+    const lower = text.toLowerCase();
+    return this.config.preserveKeywords.some(kw => lower.includes(kw));
+  }
+
+  /**
+   * Compress a single message
+   */
+  compressMessage(message: Message): Message {
+    const content = message.content ?? "";
+    const tokens = this.estimateTokens(content);
+
+    // If below threshold, no compression needed
+    if (tokens <= this.config.summarizeThreshold && !this.containsPreservedKeyword(content)) {
+      return message;
+    }
+
+    // If very long, truncate
+    if (content.length > this.config.maxMessageLength * 2) {
+      return {
+        ...message,
+        content: simpleSummarize(content, this.config.maxMessageLength),
+      };
+    }
+
+    // Otherwise, truncate to threshold
+    return {
+      ...message,
+      content: simpleSummarize(content, this.config.maxMessageLength),
+    };
+  }
+
+  /**
+   * Compress array of messages
+   */
+  compress(messages: Message[]): CompressionResult {
+    const originalTokens = messages.reduce(
+      (sum, msg) => sum + this.estimateTokens(msg.content ?? ""),
+      0
+    );
+
+    const compressed = messages.map(msg => this.compressMessage(msg));
+
+    const compressedTokens = compressed.reduce(
+      (sum, msg) => sum + this.estimateTokens(msg.content ?? ""),
+      0
+    );
+
+    return {
+      messages: compressed,
+      originalTokens,
+      compressedTokens,
+      compressionRatio: originalTokens > 0 ? compressedTokens / originalTokens : 1,
+    };
+  }
+
+  /**
+   * Compress messages in-place (mutates array)
+   */
+  compressInPlace(messages: Message[]): void {
+    for (let i = 0; i < messages.length; i++) {
+      messages[i] = this.compressMessage(messages[i]);
+    }
+  }
+
+  /**
+   * Get compression statistics
+   */
+  getStats(result: CompressionResult): {
+    tokensReduced: number;
+    percentReduction: string;
+  } {
+    const tokensReduced = result.originalTokens - result.compressedTokens;
+    const percentReduction = result.originalTokens > 0
+      ? ((tokensReduced / result.originalTokens) * 100).toFixed(1) + "%"
+      : "0%";
+
+    return {
+      tokensReduced,
+      percentReduction,
+    };
+  }
+}
+
+export default InterAgentCompressor;

package/tmlpd-pi-extension/src/tokenOptimization/schemaContract.ts

@@ -0,0 +1,101 @@
+/**
+ * Schema-Contracted Prompts - Token Optimization Pattern #4
+ * 
+ * Use structured schemas to reduce prompt size.
+ * Instead of verbose descriptions, reference schema directly.
+ * 
+ * Based on: arXiv:2608.17188 - Token Optimization and Context Window Management
+ */
+
+export interface SchemaContractConfig {
+  includeExamples: boolean;
+  maxSchemaSize: number;
+}
+
+export interface SchemaInfo {
+  name: string;
+  description?: string;
+  schema: Record<string, unknown>;
+  examples?: Array<Record<string, unknown>>;
+}
+
+/**
+ * Generate schema instruction for prompt
+ */
+function generateSchemaInstruction(schema: SchemaInfo): string {
+  const parts: string[] = [];
+  
+  parts.push(`Response Schema: ${schema.name}`);
+  
+  if (schema.description) {
+    parts.push(`Description: ${schema.description}`);
+  }
+  
+  parts.push(`Schema: ${JSON.stringify(schema.schema)}`);
+  
+  if (schema.examples && schema.examples.length > 0) {
+    parts.push(`Examples: ${JSON.stringify(schema.examples.slice(0, 2))}`);
+  }
+  
+  return parts.join("\n");
+}
+
+export class SchemaContractor {
+  private config: SchemaContractConfig;
+
+  constructor(config: Partial<SchemaContractConfig> = {}) {
+    this.config = {
+      includeExamples: config.includeExamples ?? true,
+      maxSchemaSize: config.maxSchemaSize ?? 500,
+    };
+  }
+
+  /**
+   * Contract a prompt by injecting schema reference
+   */
+  contract(prompt: string, schema: SchemaInfo): string {
+    const schemaInstruction = generateSchemaInstruction(schema);
+    
+    return `${prompt}
+
+IMPORTANT: Your response MUST conform to this schema:
+${schemaInstruction}
+
+Respond ONLY with valid JSON matching the above schema.`;
+  }
+
+  /**
+   * Contract with inline schema (smaller)
+   */
+  contractInline(prompt: string, schema: SchemaInfo): string {
+    const schemaJson = JSON.stringify(schema.schema).substring(0, this.config.maxSchemaSize);
+    
+    return `${prompt}
+
+Response Schema: ${schema.name}
+${schemaJson}`;
+  }
+
+  /**
+   * Extract schema from Pydantic-style model definition
+   */
+  static extractSchemaFromModel(modelDef: string): SchemaInfo {
+    // Simple parser for TypeScript/JavaScript interface or Python Pydantic
+    const nameMatch = modelDef.match(/class\s+(\w+)/);
+    const name = nameMatch ? nameMatch[1] : "Response";
+    
+    // Extract field types (simplified)
+    const fields: Record<string, string> = {};
+    const fieldMatches = modelDef.matchAll(/(\w+)(\??):\s*(\w+)/g);
+    for (const match of fieldMatches) {
+      fields[match[1]] = match[3];
+    }
+
+    return {
+      name,
+      schema: fields as Record<string, unknown>,
+    };
+  }
+}
+
+export default SchemaContractor;

package/tmlpd-pi-extension/src/tokenOptimization/semanticCache.ts

@@ -0,0 +1,248 @@
+/**
+ * Semantic Cache - Token Optimization Pattern #1
+ * 
+ * Cache semantically similar queries instead of exact-match caching.
+ * Uses embedding similarity (cosine) to find cached responses.
+ * 
+ * Based on: arXiv:2608.17188 - Token Optimization and Context Window Management
+ */
+
+export interface SemanticCacheConfig {
+  enabled: boolean;
+  similarityThreshold: number;  // 0.85 default
+  maxEntries: number;
+  embeddingModel: string;  // e.g., "sentence-transformers/all-MiniLM-L6-v2"
+  ttlSeconds: number;
+  cacheDir?: string;
+}
+
+export interface SemanticCacheEntry {
+  query: string;
+  embedding: number[];
+  response: string;
+  model: string;
+  tokensSaved: number;
+  costSaved: number;
+  cachedAt: number;
+  expiresAt: number;
+}
+
+/**
+ * Compute cosine similarity between two embeddings
+ */
+function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  return denominator === 0 ? 0 : dotProduct / denominator;
+}
+
+/**
+ * Simple embedding generator using TF-IDF style vectors
+ * For production, replace with actual sentence transformer embeddings
+ */
+function simpleHash(str: string): number {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = ((hash << 5) - hash) + str.charCodeAt(i);
+    hash = hash & hash;
+  }
+  return Math.abs(hash);
+}
+
+function generateEmbedding(text: string, dimensions = 384): number[] {
+  const words = text.toLowerCase().split(/\s+/);
+  const embedding: number[] = new Array(dimensions).fill(0);
+  
+  // Simple hash-based embedding for demo
+  // In production, use: sentence-transformers or OpenAI embeddings
+  words.forEach((word, idx) => {
+    const bucket = simpleHash(word) % dimensions;
+    embedding[bucket] += 1 / (idx + 1);  // Position weighting
+  });
+  
+  // Normalize
+  const norm = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
+  return norm > 0 ? embedding.map(v => v / norm) : embedding;
+}
+
+export class SemanticCache {
+  private cache: Map<string, SemanticCacheEntry> = new Map();
+  private config: SemanticCacheConfig;
+  private hits = 0;
+  private misses = 0;
+  private embeddingCache: Map<string, number[]> = new Map();
+
+  constructor(config: Partial<SemanticCacheConfig> = {}) {
+    this.config = {
+      enabled: config.enabled ?? true,
+      similarityThreshold: config.similarityThreshold ?? 0.85,
+      maxEntries: config.maxEntries ?? 1000,
+      embeddingModel: config.embeddingModel ?? "local-tfidf",
+      ttlSeconds: config.ttlSeconds ?? 3600,
+      cacheDir: config.cacheDir,
+    };
+  }
+
+  /**
+   * Get embedding for text (with caching)
+   */
+  private async getEmbedding(text: string): Promise<number[]> {
+    // Check cache first
+    const cached = this.embeddingCache.get(text);
+    if (cached) return cached;
+
+    // Generate embedding
+    const embedding = generateEmbedding(text);
+    this.embeddingCache.set(text, embedding);
+    
+    // Limit cache size
+    if (this.embeddingCache.size > this.config.maxEntries * 2) {
+      const keys = Array.from(this.embeddingCache.keys());
+      keys.slice(0, keys.length - this.config.maxEntries).forEach(k => {
+        this.embeddingCache.delete(k);
+      });
+    }
+
+    return embedding;
+  }
+
+  /**
+   * Generate cache key from query
+   */
+  private generateKey(query: string): string {
+    // Simple hash function instead of crypto
+    const hash = simpleHash(query.toLowerCase().trim());
+    return hash.toString(16).padStart(8, '0') + simpleHash(query).toString(16).padStart(8, '0');
+  }
+
+  /**
+   * Find best matching cached entry using cosine similarity
+   */
+  async get(query: string): Promise<{ response: string; similarity: number; tokensSaved: number; costSaved: number } | null> {
+    if (!this.config.enabled) return null;
+
+    const queryEmbedding = await this.getEmbedding(query);
+    let bestMatch: { key: string; similarity: number } | null = null;
+
+    // Search for most similar cached entry
+    for (const [key, entry] of this.cache.entries()) {
+      // Check expiration
+      if (Date.now() > entry.expiresAt) {
+        this.cache.delete(key);
+        continue;
+      }
+
+      const similarity = cosineSimilarity(queryEmbedding, entry.embedding);
+      if (similarity >= this.config.similarityThreshold) {
+        if (!bestMatch || similarity > bestMatch.similarity) {
+          bestMatch = { key, similarity };
+        }
+      }
+    }
+
+    if (!bestMatch) {
+      this.misses++;
+      return null;
+    }
+
+    const entry = this.cache.get(bestMatch.key)!;
+    this.hits++;
+    
+    return {
+      response: entry.response,
+      similarity: bestMatch.similarity,
+      tokensSaved: entry.tokensSaved,
+      costSaved: entry.costSaved,
+    };
+  }
+
+  /**
+   * Store query-response pair in semantic cache
+   */
+  async set(
+    query: string,
+    response: string,
+    options: {
+      model?: string;
+      tokensSaved?: number;
+      costSaved?: number;
+    } = {}
+  ): Promise<void> {
+    if (!this.config.enabled) return;
+
+    const embedding = await this.getEmbedding(query);
+    const now = Date.now();
+    const key = this.generateKey(query);
+
+    // Evict oldest if at capacity
+    if (this.cache.size >= this.config.maxEntries) {
+      this.evictOldest();
+    }
+
+    this.cache.set(key, {
+      query,
+      embedding,
+      response,
+      model: options.model ?? "unknown",
+      tokensSaved: options.tokensSaved ?? 0,
+      costSaved: options.costSaved ?? 0,
+      cachedAt: now,
+      expiresAt: now + this.config.ttlSeconds * 1000,
+    });
+  }
+
+  /**
+   * Evict oldest entry
+   */
+  private evictOldest(): void {
+    let oldestKey: string | null = null;
+    let oldestTime = Infinity;
+
+    for (const [key, entry] of this.cache.entries()) {
+      if (entry.cachedAt < oldestTime) {
+        oldestTime = entry.cachedAt;
+        oldestKey = key;
+      }
+    }
+
+    if (oldestKey) {
+      this.cache.delete(oldestKey);
+    }
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getStats(): { hits: number; misses: number; size: number; hitRate: number } {
+    const total = this.hits + this.misses;
+    return {
+      hits: this.hits,
+      misses: this.misses,
+      size: this.cache.size,
+      hitRate: total > 0 ? this.hits / total : 0,
+    };
+  }
+
+  /**
+   * Clear cache
+   */
+  clear(): void {
+    this.cache.clear();
+    this.embeddingCache.clear();
+    this.hits = 0;
+    this.misses = 0;
+  }
+}
+
+export default SemanticCache;