adaptive-memory-multi-model-router 2.14.46 → 2.14.48

package/eval/run_eval.js

@@ -1,197 +0,0 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-const { routeQuery } = require('../dist/index.js');
-const { appendExperimentRecord } = require('./lib/experiment_registry');
-
-function readJson(filePath) {
-  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
-}
-
-function readJsonl(filePath) {
-  return fs
-    .readFileSync(filePath, 'utf8')
-    .split('\n')
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .map((line, idx) => {
-      try {
-        return JSON.parse(line);
-      } catch (error) {
-        throw new Error(`Invalid JSONL at line ${idx + 1}: ${error.message}`);
-      }
-    });
-}
-
-function safeGet(obj, key, fallback = false) {
-  return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
-}
-
-function toFixed(n) {
-  return Number(n.toFixed(4));
-}
-
-function ensureDir(dirPath) {
-  fs.mkdirSync(dirPath, { recursive: true });
-}
-
-function evaluateCase(item) {
-  const decision = routeQuery(item.prompt);
-  const expected = item.expected || {};
-  const checks = [];
-
-  if (expected.complexity) {
-    const c = decision.features?.complexity ?? 0;
-    const ok = c >= expected.complexity.min && c <= expected.complexity.max;
-    checks.push({ type: 'complexity', ok, actual: c, expected: expected.complexity });
-  }
-
-  if (expected.flags) {
-    for (const [flag, expectedValue] of Object.entries(expected.flags)) {
-      const actual = safeGet(decision.features || {}, flag, false);
-      checks.push({ type: 'flag', flag, ok: actual === expectedValue, actual, expected: expectedValue });
-    }
-  }
-
-  if (expected.domain) {
-    const actual = decision.features?.domain || '';
-    checks.push({ type: 'domain', ok: actual === expected.domain, actual, expected: expected.domain });
-  }
-
-  if (expected.provider_type) {
-    const actual = decision.provider_type || '';
-    checks.push({ type: 'provider_type', ok: actual === expected.provider_type, actual, expected: expected.provider_type });
-  }
-
-  return {
-    id: item.id,
-    prompt: item.prompt,
-    decision: {
-      primary_model: decision.primary_model,
-      provider_type: decision.provider_type,
-      estimated_cost: decision.estimated_cost,
-      complexity: decision.features?.complexity,
-      detected_domain: decision.features?.detected_domain || ''
-    },
-    checks
-  };
-}
-
-function summarize(results) {
-  const allChecks = results.flatMap((r) => r.checks);
-  const byType = (type) => allChecks.filter((c) => c.type === type);
-  const rate = (arr) => (arr.length ? arr.filter((x) => x.ok).length / arr.length : 1);
-
-  const complexity = rate(byType('complexity'));
-  const flags = rate(byType('flag'));
-  const domain = rate(byType('domain'));
-  const providerType = rate(byType('provider_type'));
-
-  const weighted = [complexity, flags, domain, providerType];
-  const overall = weighted.reduce((a, b) => a + b, 0) / weighted.length;
-
-  return {
-    dataset_size: results.length,
-    checks_count: allChecks.length,
-    complexity_accuracy: toFixed(complexity),
-    flag_accuracy: toFixed(flags),
-    domain_accuracy: toFixed(domain),
-    provider_type_accuracy: toFixed(providerType),
-    overall_score: toFixed(overall)
-  };
-}
-
-function gate(summary, thresholds, baseline) {
-  const failures = [];
-
-  if (summary.dataset_size < thresholds.min_dataset_size) {
-    failures.push(`dataset_size ${summary.dataset_size} < min_dataset_size ${thresholds.min_dataset_size}`);
-  }
-  if (summary.complexity_accuracy < thresholds.min_complexity_accuracy) {
-    failures.push(`complexity_accuracy ${summary.complexity_accuracy} < ${thresholds.min_complexity_accuracy}`);
-  }
-  if (summary.flag_accuracy < thresholds.min_flag_accuracy) {
-    failures.push(`flag_accuracy ${summary.flag_accuracy} < ${thresholds.min_flag_accuracy}`);
-  }
-  if (summary.domain_accuracy < thresholds.min_domain_accuracy) {
-    failures.push(`domain_accuracy ${summary.domain_accuracy} < ${thresholds.min_domain_accuracy}`);
-  }
-  if (summary.provider_type_accuracy < thresholds.min_provider_type_accuracy) {
-    failures.push(`provider_type_accuracy ${summary.provider_type_accuracy} < ${thresholds.min_provider_type_accuracy}`);
-  }
-  if (summary.overall_score < thresholds.min_overall_score) {
-    failures.push(`overall_score ${summary.overall_score} < ${thresholds.min_overall_score}`);
-  }
-
-  if (baseline?.summary) {
-    const delta = baseline.summary.overall_score - summary.overall_score;
-    if (delta > thresholds.max_regression_delta) {
-      failures.push(
-        `overall_score regression ${toFixed(delta)} > max_regression_delta ${thresholds.max_regression_delta}`
-      );
-    }
-  }
-
-  return failures;
-}
-
-function main() {
-  const evalDir = path.resolve(__dirname);
-  const datasetPath = path.join(evalDir, 'benchmark_dataset.jsonl');
-  const thresholdsPath = path.join(evalDir, 'thresholds.json');
-  const baselinePath = path.join(evalDir, 'baselines', 'main.json');
-  const resultsPath = path.join(evalDir, 'results', 'latest.json');
-
-  const dataset = readJsonl(datasetPath);
-  const thresholds = readJson(thresholdsPath);
-  const results = dataset.map(evaluateCase);
-  const summary = summarize(results);
-  const baseline = fs.existsSync(baselinePath) ? readJson(baselinePath) : null;
-  const failures = gate(summary, thresholds, baseline);
-
-  const output = {
-    timestamp_utc: new Date().toISOString(),
-    commit: process.env.GITHUB_SHA || null,
-    summary,
-    failures,
-    baseline_used: Boolean(baseline),
-    results
-  };
-
-  ensureDir(path.dirname(resultsPath));
-  fs.writeFileSync(resultsPath, JSON.stringify(output, null, 2));
-
-  console.log('\nA3M Routing Eval Summary');
-  console.log('------------------------');
-  console.log(JSON.stringify(summary, null, 2));
-  console.log(`Results file: ${resultsPath}`);
-
-  if (failures.length) {
-    appendExperimentRecord({
-      experiment_id: `routing_eval_${Date.now()}`,
-      dataset_version: 'core_regression_v1',
-      run_type: 'routing_eval',
-      metrics: summary,
-      decision: 'fail',
-      notes: failures
-    });
-    console.error('\nEval gate FAILED:');
-    for (const failure of failures) {
-      console.error(`- ${failure}`);
-    }
-    process.exit(1);
-  }
-
-  appendExperimentRecord({
-    experiment_id: `routing_eval_${Date.now()}`,
-    dataset_version: 'core_regression_v1',
-    run_type: 'routing_eval',
-    metrics: summary,
-    decision: 'pass',
-    notes: []
-  });
-
-  console.log('\nEval gate PASSED');
-}
-
-main();

package/eval/run_fault_injection.js

@@ -1,201 +0,0 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-const { ProviderRetryHandler } = require('../dist/routing/providerRetry.js');
-const { ProviderHealthManager } = require('../dist/routing/providerHealth.js');
-const { appendExperimentRecord } = require('./lib/experiment_registry');
-
-function readJson(filePath) {
-  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
-}
-
-async function scenarioRetryTransientThenSuccess() {
-  const handler = new ProviderRetryHandler({
-    openai: {
-      timeout: 3000,
-      retry: {
-        maxRetries: 3,
-        initialDelayMs: 10,
-        maxDelayMs: 100,
-        backoffMultiplier: 2,
-        retryableErrors: ['ETIMEDOUT', '503']
-      }
-    }
-  });
-
-  let callCount = 0;
-  const result = await handler.executeWithRetry('openai', async () => {
-    callCount += 1;
-    if (callCount < 3) {
-      const err = new Error('temporary timeout');
-      err.code = 'ETIMEDOUT';
-      throw err;
-    }
-    return 'ok';
-  });
-
-  return result === 'ok' && callCount === 3;
-}
-
-async function scenarioNoRetryOnBadRequest() {
-  const handler = new ProviderRetryHandler({
-    openai: {
-      timeout: 3000,
-      retry: {
-        maxRetries: 5,
-        initialDelayMs: 10,
-        maxDelayMs: 100,
-        backoffMultiplier: 2,
-        retryableErrors: ['ETIMEDOUT', '503']
-      }
-    }
-  });
-
-  let callCount = 0;
-  try {
-    await handler.executeWithRetry('openai', async () => {
-      callCount += 1;
-      const err = new Error('bad request');
-      err.status = 400;
-      throw err;
-    });
-    return false;
-  } catch {
-    return callCount === 1;
-  }
-}
-
-async function scenarioNoRetryOnChineseQuotaAccountErrors() {
-  const handler = new ProviderRetryHandler({
-    moonshot: {
-      timeout: 3000,
-      retry: {
-        maxRetries: 5,
-        initialDelayMs: 10,
-        maxDelayMs: 100,
-        backoffMultiplier: 2,
-        retryableErrors: ['429', '503', 'ETIMEDOUT']
-      }
-    }
-  });
-
-  let callCount = 0;
-  try {
-    await handler.executeWithRetry('moonshot', async () => {
-      callCount += 1;
-      const err = new Error('request reached organization TPD rate limit, current: 1501880, limit: 1500000');
-      err.status = 429;
-      err.code = 'rate_limit_reached_error';
-      throw err;
-    });
-    return false;
-  } catch {
-    return callCount === 1;
-  }
-}
-
-async function scenarioCircuitBreakerOpens() {
-  const health = new ProviderHealthManager({
-    circuitBreakerThreshold: 3,
-    cooldownMs: 30000,
-    windowSize: 10
-  });
-
-  const provider = 'mock/provider';
-  health.recordFailure(provider, 'e1');
-  health.recordFailure(provider, 'e2');
-  health.recordFailure(provider, 'e3');
-
-  const state = health.getHealth(provider);
-  return state.isHealthy === false && state.consecutiveErrors >= 3 && state.cooldownUntil > Date.now();
-}
-
-async function scenarioFallbackOrdering() {
-  const health = new ProviderHealthManager({ windowSize: 10 });
-
-  health.recordSuccess('fast', 80);
-  health.recordSuccess('fast', 90);
-
-  health.recordSuccess('slow', 500);
-  health.recordSuccess('slow', 600);
-
-  health.recordFailure('broken', 'x1');
-  health.recordFailure('broken', 'x2');
-  health.recordFailure('broken', 'x3');
-
-  const chain = health.getFallbackChain(['broken', 'slow', 'fast']);
-  return chain[0] === 'fast' && chain[chain.length - 1] === 'broken';
-}
-
-async function main() {
-  const thresholds = readJson(path.join(__dirname, 'fault_injection_thresholds.json'));
-  const scenarios = [
-    ['retry_transient_then_success', scenarioRetryTransientThenSuccess],
-    ['no_retry_on_bad_request', scenarioNoRetryOnBadRequest],
-    ['no_retry_on_chinese_quota_account_errors', scenarioNoRetryOnChineseQuotaAccountErrors],
-    ['circuit_breaker_opens', scenarioCircuitBreakerOpens],
-    ['fallback_ordering', scenarioFallbackOrdering]
-  ];
-
-  const results = [];
-  for (const [name, fn] of scenarios) {
-    try {
-      const ok = await fn();
-      results.push({ name, ok, error: null });
-    } catch (error) {
-      results.push({ name, ok: false, error: String(error) });
-    }
-  }
-
-  const passed = results.filter((r) => r.ok).length;
-  const passRate = results.length ? passed / results.length : 0;
-  const summary = {
-    total: results.length,
-    passed,
-    failed: results.length - passed,
-    pass_rate: Number(passRate.toFixed(4))
-  };
-
-  const output = {
-    timestamp_utc: new Date().toISOString(),
-    summary,
-    thresholds,
-    results
-  };
-
-  const outDir = path.join(__dirname, 'results');
-  fs.mkdirSync(outDir, { recursive: true });
-  fs.writeFileSync(path.join(outDir, 'fault_injection_latest.json'), JSON.stringify(output, null, 2));
-
-  console.log('\nFault Injection Summary');
-  console.log('-----------------------');
-  console.log(JSON.stringify(summary, null, 2));
-
-  if (passRate < thresholds.required_pass_rate) {
-    appendExperimentRecord({
-      experiment_id: `fault_injection_${Date.now()}`,
-      dataset_version: 'fault_scenarios_v1',
-      run_type: 'fault_injection',
-      metrics: summary,
-      decision: 'fail',
-      notes: results.filter((r) => !r.ok).map((r) => `${r.name}: ${r.error || 'failed'}`)
-    });
-    console.error(
-      `\nFault injection gate FAILED: pass_rate ${summary.pass_rate} < ${thresholds.required_pass_rate}`
-    );
-    process.exit(1);
-  }
-
-  appendExperimentRecord({
-    experiment_id: `fault_injection_${Date.now()}`,
-    dataset_version: 'fault_scenarios_v1',
-    run_type: 'fault_injection',
-    metrics: summary,
-    decision: 'pass',
-    notes: []
-  });
-
-  console.log('\nFault injection gate PASSED');
-}
-
-main();

package/eval/run_shadow_eval.js

@@ -1,85 +0,0 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-const { routeQuery } = require('../dist/index.js');
-const { appendExperimentRecord } = require('./lib/experiment_registry');
-
-function readJsonl(filePath) {
-  return fs
-    .readFileSync(filePath, 'utf8')
-    .split('\n')
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .map((line) => JSON.parse(line));
-}
-
-function toFixed(n) {
-  return Number(n.toFixed(6));
-}
-
-function main() {
-  const evalDir = path.resolve(__dirname);
-  const dataset = readJsonl(path.join(evalDir, 'benchmark_dataset.jsonl'));
-  const candidateBudgetMultiplier = Number(process.env.A3M_SHADOW_BUDGET_MULTIPLIER || '0.85');
-
-  let divergence = 0;
-  let projectedCostDelta = 0;
-  const comparisons = [];
-
-  for (const row of dataset) {
-    const primary = routeQuery(row.prompt);
-    const shadow = routeQuery(row.prompt, undefined, candidateBudgetMultiplier);
-
-    const changed = primary.primary_model !== shadow.primary_model;
-    if (changed) divergence += 1;
-
-    const costDelta = Number((shadow.estimated_cost || 0) - (primary.estimated_cost || 0));
-    projectedCostDelta += costDelta;
-
-    comparisons.push({
-      id: row.id,
-      primary_model: primary.primary_model,
-      shadow_model: shadow.primary_model,
-      changed,
-      primary_cost: primary.estimated_cost,
-      shadow_cost: shadow.estimated_cost,
-      cost_delta: toFixed(costDelta)
-    });
-  }
-
-  const summary = {
-    dataset_size: dataset.length,
-    candidate_budget_multiplier: candidateBudgetMultiplier,
-    divergence_rate: toFixed(divergence / dataset.length),
-    changed_cases: divergence,
-    projected_total_cost_delta: toFixed(projectedCostDelta),
-    projected_avg_cost_delta: toFixed(projectedCostDelta / dataset.length)
-  };
-
-  const out = {
-    timestamp_utc: new Date().toISOString(),
-    summary,
-    comparisons
-  };
-
-  const outDir = path.join(evalDir, 'results');
-  fs.mkdirSync(outDir, { recursive: true });
-  const outPath = path.join(outDir, 'shadow_latest.json');
-  fs.writeFileSync(outPath, JSON.stringify(out, null, 2));
-
-  appendExperimentRecord({
-    experiment_id: `shadow_eval_${Date.now()}`,
-    dataset_version: 'core_regression_v1',
-    run_type: 'shadow_eval',
-    metrics: summary,
-    decision: 'informational',
-    notes: []
-  });
-
-  console.log('\nShadow Eval Summary');
-  console.log('-------------------');
-  console.log(JSON.stringify(summary, null, 2));
-  console.log(`Shadow output: ${outPath}`);
-}
-
-main();

package/eval/thresholds.json

@@ -1,9 +0,0 @@
-{
-  "min_dataset_size": 12,
-  "min_complexity_accuracy": 0.85,
-  "min_flag_accuracy": 0.8,
-  "min_domain_accuracy": 0.8,
-  "min_provider_type_accuracy": 0.75,
-  "min_overall_score": 0.85,
-  "max_regression_delta": 0.03
-}

package/examples/QUICKSTART.md

@@ -1,183 +0,0 @@
-# TMLPD PI Extension - Usage Examples
-
-## Quick Start
-
-```python
-from tmlpd import quick_process
-
-# One-liner
-result = quick_process("What is quantum entanglement?")
-print(result["content"])
-```
-
-## Task Classification
-
-```python
-from tmlpd import TMLPDLite, TaskType
-
-lite = TMLPDLite()
-
-# Automatic task classification
-prompt = "Write Python async HTTP client"
-task_type = lite.classify_task(prompt)  # TaskType.CODING
-
-# Get optimal models
-models = lite.get_optimal_models(task_type, 3)
-# ["codex", "claude-minimax", "claude"]
-```
-
-## Caching
-
-```python
-from tmlpd import TMLPDLite
-
-lite = TMLPDLite()
-
-# First call - not cached
-result1 = lite.process("What is 2+2?", use_cache=True)
-print(f"Cached: {result1['cached']}")  # False
-
-# Second call - from cache
-result2 = lite.process("What is 2+2?", use_cache=True)
-print(f"Cached: {result2['cached']}")  # True
-```
-
-## Async Client (Production)
-
-```python
-import asyncio
-from tmlpd import TMLPDClient, TMLPDConfig
-
-async def main():
-    config = TMLPDConfig(
-        cache_ttl_seconds=3600,
-        daily_budget=10.0,
-        max_concurrent=5
-    )
-    client = TMLPDClient(config)
-    
-    # Single execution
-    result = await client.execute("Explain quantum entanglement")
-    
-    # Parallel execution
-    parallel = await client.execute_parallel(
-        "Compare Python and JavaScript",
-        models=["gpt-4o", "claude", "gemini"]
-    )
-    
-    # Cost summary
-    summary = await client.get_cost_summary()
-    print(f"Total spent: ${summary.total_cost}")
-
-asyncio.run(main())
-```
-
-## Cost Optimization
-
-```python
-import asyncio
-from tmlpd import TMLPDClient
-
-async def cost_optimization():
-    client = TMLPDClient()
-    
-    # Cheap for simple tasks
-    simple = await client.execute("What is 2+2?", model="cerebras/llama-3.3-70b")
-    print(f"Simple task cost: ${simple.cost:.6f}")
-    
-    # Premium for complex tasks
-    complex = await client.execute("Design microservices", model="anthropic/claude-3.5-sonnet")
-    print(f"Complex task cost: ${complex.cost:.6f}")
-
-asyncio.run(cost_optimization())
-```
-
-## LangChain Integration
-
-```python
-from langchain.llms import BaseLLM
-from tmlpd import TMLPDLite
-
-class TMLPDLLM(BaseLLM):
-    def __init__(self, task_type="default"):
-        self.lite = TMLPDLite()
-        self.task_type = task_type
-    
-    def _call(self, prompt: str) -> str:
-        result = self.lite.process(prompt)
-        return result["content"]
-
-# Usage
-llm = TMLPDLLM(task_type="coding")
-response = llm("Write a Python function")
-```
-
-## API Server
-
-```python
-# Run: node node_modules/tmlpd-pi/dist/server.js --port 18791
-# Then use Python client:
-
-import httpx
-
-async def api_example():
-    async with httpx.AsyncClient() as client:
-        response = await client.post(
-            "http://localhost:18791/execute",
-            json={"prompt": "Hello world", "models": ["gpt-4o"]}
-        )
-        print(response.json())
-```
-
-## Task Types
-
-| Task Type | Keywords | Best Models |
-|-----------|----------|-------------|
-| coding | python, javascript, code | codex, claude-minimax |
-| frontend | react, vue, component | codex, claude-minimax |
-| backend | api, server, database | codex, claude-glm |
-| chinese | 中文, 汉语 | claude-glm, claude-minimax |
-| fast | quick, simple | gemini, claude-haiku |
-| premium | advanced, complex | claude-opus, gemini-pro |
-
-## Environment Variables
-
-```bash
-# API Keys
-export OPENAI_API_KEY="sk-..."
-export ANTHROPIC_API_KEY="sk-ant-..."
-export GROQ_API_KEY="gsk_..."
-
-# TMLPD Configuration
-export TMLPD_MAX_CONCURRENT=5
-export TMLPD_DAILY_BUDGET=10.0
-export TMLPD_CACHE_TTL=3600
-```
-
-## Full Example
-
-```python
-import asyncio
-from tmlpd import TMLPDClient, TMLPDLite
-
-async def full_example():
-    # Lite for quick tasks
-    lite = TMLPDLite()
-    result = lite.process("What is Python?", use_cache=True)
-    print(f"Lite: {result['task_type']}")
-    
-    # Full client for production
-    client = TMLPDClient()
-    
-    # Batch processing
-    prompts = ["What is AI?", "What is ML?", "What is DL?"]
-    for prompt in prompts:
-        result = await client.execute(prompt)
-        print(f"Cost: ${result.cost:.6f}")
-    
-    # Final stats
-    summary = await client.get_cost_summary()
-    print(f"Total: ${summary.total_cost:.6f}")
-
-asyncio.run(full_example())
-```