adaptive-memory-multi-model-router 2.14.49 → 2.14.51

package/eval/run_eval.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line, idx) => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {
+        throw new Error(`Invalid JSONL at line ${idx + 1}: ${error.message}`);
+      }
+    });
+}
+
+function safeGet(obj, key, fallback = false) {
+  return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(4));
+}
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function evaluateCase(item) {
+  const decision = routeQuery(item.prompt);
+  const expected = item.expected || {};
+  const checks = [];
+
+  if (expected.complexity) {
+    const c = decision.features?.complexity ?? 0;
+    const ok = c >= expected.complexity.min && c <= expected.complexity.max;
+    checks.push({ type: 'complexity', ok, actual: c, expected: expected.complexity });
+  }
+
+  if (expected.flags) {
+    for (const [flag, expectedValue] of Object.entries(expected.flags)) {
+      const actual = safeGet(decision.features || {}, flag, false);
+      checks.push({ type: 'flag', flag, ok: actual === expectedValue, actual, expected: expectedValue });
+    }
+  }
+
+  if (expected.domain) {
+    const actual = decision.features?.domain || '';
+    checks.push({ type: 'domain', ok: actual === expected.domain, actual, expected: expected.domain });
+  }
+
+  if (expected.provider_type) {
+    const actual = decision.provider_type || '';
+    checks.push({ type: 'provider_type', ok: actual === expected.provider_type, actual, expected: expected.provider_type });
+  }
+
+  return {
+    id: item.id,
+    prompt: item.prompt,
+    decision: {
+      primary_model: decision.primary_model,
+      provider_type: decision.provider_type,
+      estimated_cost: decision.estimated_cost,
+      complexity: decision.features?.complexity,
+      detected_domain: decision.features?.detected_domain || ''
+    },
+    checks
+  };
+}
+
+function summarize(results) {
+  const allChecks = results.flatMap((r) => r.checks);
+  const byType = (type) => allChecks.filter((c) => c.type === type);
+  const rate = (arr) => (arr.length ? arr.filter((x) => x.ok).length / arr.length : 1);
+
+  const complexity = rate(byType('complexity'));
+  const flags = rate(byType('flag'));
+  const domain = rate(byType('domain'));
+  const providerType = rate(byType('provider_type'));
+
+  const weighted = [complexity, flags, domain, providerType];
+  const overall = weighted.reduce((a, b) => a + b, 0) / weighted.length;
+
+  return {
+    dataset_size: results.length,
+    checks_count: allChecks.length,
+    complexity_accuracy: toFixed(complexity),
+    flag_accuracy: toFixed(flags),
+    domain_accuracy: toFixed(domain),
+    provider_type_accuracy: toFixed(providerType),
+    overall_score: toFixed(overall)
+  };
+}
+
+function gate(summary, thresholds, baseline) {
+  const failures = [];
+
+  if (summary.dataset_size < thresholds.min_dataset_size) {
+    failures.push(`dataset_size ${summary.dataset_size} < min_dataset_size ${thresholds.min_dataset_size}`);
+  }
+  if (summary.complexity_accuracy < thresholds.min_complexity_accuracy) {
+    failures.push(`complexity_accuracy ${summary.complexity_accuracy} < ${thresholds.min_complexity_accuracy}`);
+  }
+  if (summary.flag_accuracy < thresholds.min_flag_accuracy) {
+    failures.push(`flag_accuracy ${summary.flag_accuracy} < ${thresholds.min_flag_accuracy}`);
+  }
+  if (summary.domain_accuracy < thresholds.min_domain_accuracy) {
+    failures.push(`domain_accuracy ${summary.domain_accuracy} < ${thresholds.min_domain_accuracy}`);
+  }
+  if (summary.provider_type_accuracy < thresholds.min_provider_type_accuracy) {
+    failures.push(`provider_type_accuracy ${summary.provider_type_accuracy} < ${thresholds.min_provider_type_accuracy}`);
+  }
+  if (summary.overall_score < thresholds.min_overall_score) {
+    failures.push(`overall_score ${summary.overall_score} < ${thresholds.min_overall_score}`);
+  }
+
+  if (baseline?.summary) {
+    const delta = baseline.summary.overall_score - summary.overall_score;
+    if (delta > thresholds.max_regression_delta) {
+      failures.push(
+        `overall_score regression ${toFixed(delta)} > max_regression_delta ${thresholds.max_regression_delta}`
+      );
+    }
+  }
+
+  return failures;
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const datasetPath = path.join(evalDir, 'benchmark_dataset.jsonl');
+  const thresholdsPath = path.join(evalDir, 'thresholds.json');
+  const baselinePath = path.join(evalDir, 'baselines', 'main.json');
+  const resultsPath = path.join(evalDir, 'results', 'latest.json');
+
+  const dataset = readJsonl(datasetPath);
+  const thresholds = readJson(thresholdsPath);
+  const results = dataset.map(evaluateCase);
+  const summary = summarize(results);
+  const baseline = fs.existsSync(baselinePath) ? readJson(baselinePath) : null;
+  const failures = gate(summary, thresholds, baseline);
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    commit: process.env.GITHUB_SHA || null,
+    summary,
+    failures,
+    baseline_used: Boolean(baseline),
+    results
+  };
+
+  ensureDir(path.dirname(resultsPath));
+  fs.writeFileSync(resultsPath, JSON.stringify(output, null, 2));
+
+  console.log('\nA3M Routing Eval Summary');
+  console.log('------------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Results file: ${resultsPath}`);
+
+  if (failures.length) {
+    appendExperimentRecord({
+      experiment_id: `routing_eval_${Date.now()}`,
+      dataset_version: 'core_regression_v1',
+      run_type: 'routing_eval',
+      metrics: summary,
+      decision: 'fail',
+      notes: failures
+    });
+    console.error('\nEval gate FAILED:');
+    for (const failure of failures) {
+      console.error(`- ${failure}`);
+    }
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `routing_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'routing_eval',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nEval gate PASSED');
+}
+
+main();

package/eval/run_fault_injection.js

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { ProviderRetryHandler } = require('../dist/routing/providerRetry.js');
+const { ProviderHealthManager } = require('../dist/routing/providerHealth.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+async function scenarioRetryTransientThenSuccess() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 3,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  const result = await handler.executeWithRetry('openai', async () => {
+    callCount += 1;
+    if (callCount < 3) {
+      const err = new Error('temporary timeout');
+      err.code = 'ETIMEDOUT';
+      throw err;
+    }
+    return 'ok';
+  });
+
+  return result === 'ok' && callCount === 3;
+}
+
+async function scenarioNoRetryOnBadRequest() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('openai', async () => {
+      callCount += 1;
+      const err = new Error('bad request');
+      err.status = 400;
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioNoRetryOnChineseQuotaAccountErrors() {
+  const handler = new ProviderRetryHandler({
+    moonshot: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['429', '503', 'ETIMEDOUT']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('moonshot', async () => {
+      callCount += 1;
+      const err = new Error('request reached organization TPD rate limit, current: 1501880, limit: 1500000');
+      err.status = 429;
+      err.code = 'rate_limit_reached_error';
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioCircuitBreakerOpens() {
+  const health = new ProviderHealthManager({
+    circuitBreakerThreshold: 3,
+    cooldownMs: 30000,
+    windowSize: 10
+  });
+
+  const provider = 'mock/provider';
+  health.recordFailure(provider, 'e1');
+  health.recordFailure(provider, 'e2');
+  health.recordFailure(provider, 'e3');
+
+  const state = health.getHealth(provider);
+  return state.isHealthy === false && state.consecutiveErrors >= 3 && state.cooldownUntil > Date.now();
+}
+
+async function scenarioFallbackOrdering() {
+  const health = new ProviderHealthManager({ windowSize: 10 });
+
+  health.recordSuccess('fast', 80);
+  health.recordSuccess('fast', 90);
+
+  health.recordSuccess('slow', 500);
+  health.recordSuccess('slow', 600);
+
+  health.recordFailure('broken', 'x1');
+  health.recordFailure('broken', 'x2');
+  health.recordFailure('broken', 'x3');
+
+  const chain = health.getFallbackChain(['broken', 'slow', 'fast']);
+  return chain[0] === 'fast' && chain[chain.length - 1] === 'broken';
+}
+
+async function main() {
+  const thresholds = readJson(path.join(__dirname, 'fault_injection_thresholds.json'));
+  const scenarios = [
+    ['retry_transient_then_success', scenarioRetryTransientThenSuccess],
+    ['no_retry_on_bad_request', scenarioNoRetryOnBadRequest],
+    ['no_retry_on_chinese_quota_account_errors', scenarioNoRetryOnChineseQuotaAccountErrors],
+    ['circuit_breaker_opens', scenarioCircuitBreakerOpens],
+    ['fallback_ordering', scenarioFallbackOrdering]
+  ];
+
+  const results = [];
+  for (const [name, fn] of scenarios) {
+    try {
+      const ok = await fn();
+      results.push({ name, ok, error: null });
+    } catch (error) {
+      results.push({ name, ok: false, error: String(error) });
+    }
+  }
+
+  const passed = results.filter((r) => r.ok).length;
+  const passRate = results.length ? passed / results.length : 0;
+  const summary = {
+    total: results.length,
+    passed,
+    failed: results.length - passed,
+    pass_rate: Number(passRate.toFixed(4))
+  };
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    thresholds,
+    results
+  };
+
+  const outDir = path.join(__dirname, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(path.join(outDir, 'fault_injection_latest.json'), JSON.stringify(output, null, 2));
+
+  console.log('\nFault Injection Summary');
+  console.log('-----------------------');
+  console.log(JSON.stringify(summary, null, 2));
+
+  if (passRate < thresholds.required_pass_rate) {
+    appendExperimentRecord({
+      experiment_id: `fault_injection_${Date.now()}`,
+      dataset_version: 'fault_scenarios_v1',
+      run_type: 'fault_injection',
+      metrics: summary,
+      decision: 'fail',
+      notes: results.filter((r) => !r.ok).map((r) => `${r.name}: ${r.error || 'failed'}`)
+    });
+    console.error(
+      `\nFault injection gate FAILED: pass_rate ${summary.pass_rate} < ${thresholds.required_pass_rate}`
+    );
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `fault_injection_${Date.now()}`,
+    dataset_version: 'fault_scenarios_v1',
+    run_type: 'fault_injection',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nFault injection gate PASSED');
+}
+
+main();

package/eval/run_shadow_eval.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(6));
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const dataset = readJsonl(path.join(evalDir, 'benchmark_dataset.jsonl'));
+  const candidateBudgetMultiplier = Number(process.env.A3M_SHADOW_BUDGET_MULTIPLIER || '0.85');
+
+  let divergence = 0;
+  let projectedCostDelta = 0;
+  const comparisons = [];
+
+  for (const row of dataset) {
+    const primary = routeQuery(row.prompt);
+    const shadow = routeQuery(row.prompt, undefined, candidateBudgetMultiplier);
+
+    const changed = primary.primary_model !== shadow.primary_model;
+    if (changed) divergence += 1;
+
+    const costDelta = Number((shadow.estimated_cost || 0) - (primary.estimated_cost || 0));
+    projectedCostDelta += costDelta;
+
+    comparisons.push({
+      id: row.id,
+      primary_model: primary.primary_model,
+      shadow_model: shadow.primary_model,
+      changed,
+      primary_cost: primary.estimated_cost,
+      shadow_cost: shadow.estimated_cost,
+      cost_delta: toFixed(costDelta)
+    });
+  }
+
+  const summary = {
+    dataset_size: dataset.length,
+    candidate_budget_multiplier: candidateBudgetMultiplier,
+    divergence_rate: toFixed(divergence / dataset.length),
+    changed_cases: divergence,
+    projected_total_cost_delta: toFixed(projectedCostDelta),
+    projected_avg_cost_delta: toFixed(projectedCostDelta / dataset.length)
+  };
+
+  const out = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    comparisons
+  };
+
+  const outDir = path.join(evalDir, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  const outPath = path.join(outDir, 'shadow_latest.json');
+  fs.writeFileSync(outPath, JSON.stringify(out, null, 2));
+
+  appendExperimentRecord({
+    experiment_id: `shadow_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'shadow_eval',
+    metrics: summary,
+    decision: 'informational',
+    notes: []
+  });
+
+  console.log('\nShadow Eval Summary');
+  console.log('-------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Shadow output: ${outPath}`);
+}
+
+main();

package/eval/thresholds.json

@@ -0,0 +1,9 @@
+{
+  "min_dataset_size": 12,
+  "min_complexity_accuracy": 0.85,
+  "min_flag_accuracy": 0.8,
+  "min_domain_accuracy": 0.8,
+  "min_provider_type_accuracy": 0.75,
+  "min_overall_score": 0.85,
+  "max_regression_delta": 0.03
+}

package/examples/QUICKSTART.md

@@ -0,0 +1,183 @@
+# TMLPD PI Extension - Usage Examples
+
+## Quick Start
+
+```python
+from tmlpd import quick_process
+
+# One-liner
+result = quick_process("What is quantum entanglement?")
+print(result["content"])
+```
+
+## Task Classification
+
+```python
+from tmlpd import TMLPDLite, TaskType
+
+lite = TMLPDLite()
+
+# Automatic task classification
+prompt = "Write Python async HTTP client"
+task_type = lite.classify_task(prompt)  # TaskType.CODING
+
+# Get optimal models
+models = lite.get_optimal_models(task_type, 3)
+# ["codex", "claude-minimax", "claude"]
+```
+
+## Caching
+
+```python
+from tmlpd import TMLPDLite
+
+lite = TMLPDLite()
+
+# First call - not cached
+result1 = lite.process("What is 2+2?", use_cache=True)
+print(f"Cached: {result1['cached']}")  # False
+
+# Second call - from cache
+result2 = lite.process("What is 2+2?", use_cache=True)
+print(f"Cached: {result2['cached']}")  # True
+```
+
+## Async Client (Production)
+
+```python
+import asyncio
+from tmlpd import TMLPDClient, TMLPDConfig
+
+async def main():
+    config = TMLPDConfig(
+        cache_ttl_seconds=3600,
+        daily_budget=10.0,
+        max_concurrent=5
+    )
+    client = TMLPDClient(config)
+    
+    # Single execution
+    result = await client.execute("Explain quantum entanglement")
+    
+    # Parallel execution
+    parallel = await client.execute_parallel(
+        "Compare Python and JavaScript",
+        models=["gpt-4o", "claude", "gemini"]
+    )
+    
+    # Cost summary
+    summary = await client.get_cost_summary()
+    print(f"Total spent: ${summary.total_cost}")
+
+asyncio.run(main())
+```
+
+## Cost Optimization
+
+```python
+import asyncio
+from tmlpd import TMLPDClient
+
+async def cost_optimization():
+    client = TMLPDClient()
+    
+    # Cheap for simple tasks
+    simple = await client.execute("What is 2+2?", model="cerebras/llama-3.3-70b")
+    print(f"Simple task cost: ${simple.cost:.6f}")
+    
+    # Premium for complex tasks
+    complex = await client.execute("Design microservices", model="anthropic/claude-3.5-sonnet")
+    print(f"Complex task cost: ${complex.cost:.6f}")
+
+asyncio.run(cost_optimization())
+```
+
+## LangChain Integration
+
+```python
+from langchain.llms import BaseLLM
+from tmlpd import TMLPDLite
+
+class TMLPDLLM(BaseLLM):
+    def __init__(self, task_type="default"):
+        self.lite = TMLPDLite()
+        self.task_type = task_type
+    
+    def _call(self, prompt: str) -> str:
+        result = self.lite.process(prompt)
+        return result["content"]
+
+# Usage
+llm = TMLPDLLM(task_type="coding")
+response = llm("Write a Python function")
+```
+
+## API Server
+
+```python
+# Run: node node_modules/tmlpd-pi/dist/server.js --port 18791
+# Then use Python client:
+
+import httpx
+
+async def api_example():
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:18791/execute",
+            json={"prompt": "Hello world", "models": ["gpt-4o"]}
+        )
+        print(response.json())
+```
+
+## Task Types
+
+| Task Type | Keywords | Best Models |
+|-----------|----------|-------------|
+| coding | python, javascript, code | codex, claude-minimax |
+| frontend | react, vue, component | codex, claude-minimax |
+| backend | api, server, database | codex, claude-glm |
+| chinese | 中文, 汉语 | claude-glm, claude-minimax |
+| fast | quick, simple | gemini, claude-haiku |
+| premium | advanced, complex | claude-opus, gemini-pro |
+
+## Environment Variables
+
+```bash
+# API Keys
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+export GROQ_API_KEY="gsk_..."
+
+# TMLPD Configuration
+export TMLPD_MAX_CONCURRENT=5
+export TMLPD_DAILY_BUDGET=10.0
+export TMLPD_CACHE_TTL=3600
+```
+
+## Full Example
+
+```python
+import asyncio
+from tmlpd import TMLPDClient, TMLPDLite
+
+async def full_example():
+    # Lite for quick tasks
+    lite = TMLPDLite()
+    result = lite.process("What is Python?", use_cache=True)
+    print(f"Lite: {result['task_type']}")
+    
+    # Full client for production
+    client = TMLPDClient()
+    
+    # Batch processing
+    prompts = ["What is AI?", "What is ML?", "What is DL?"]
+    for prompt in prompts:
+        result = await client.execute(prompt)
+        print(f"Cost: ${result.cost:.6f}")
+    
+    # Final stats
+    summary = await client.get_cost_summary()
+    print(f"Total: ${summary.total_cost:.6f}")
+
+asyncio.run(full_example())
+```