adaptive-memory-multi-model-router 2.15.2 → 2.15.3

package/src/routing/shadowSampler.ts

@@ -0,0 +1,427 @@
+/**
+ * ShadowSampler — Value-Proportional Shadow Verification
+ * 
+ * Implements Optimal Defense Theory (Rhoades 1979; McKey 1974; Zangerl & Bazzaz 1992)
+ * from plant ecological economics:
+ * 
+ *   "A plant allocates defensive compounds (defense cost) in proportion to
+ *    tissue value, attack probability, and the marginal cost of defense."
+ * 
+ * Mapped to A3M routing:
+ *   - Tissue value     → query stakes / expected cost of wrong answer
+ *   - Attack probability → probability that the primary provider fails/misbehaves
+ *   - Defense cost    → cost of the shadow verification call
+ *   - Marginal return → expected reduction in error rate from verification
+ * 
+ * Instead of binary always-on (wasteful) or always-off (risky), ODT says:
+ *   Sample shadow verification PROPORTIONALLY to expected value of verification.
+ * 
+ * The sampling probability is:
+ *   P(shadow) = f(expected_error_cost_reduction, query_risk)
+ * 
+ * This is NOT the Zahavi "shared expert" (which always runs on 100% of queries).
+ * ODT shadow runs probabilistically — only on queries where the expected
+ * verification benefit exceeds the verification cost.
+ * 
+ * Usage:
+ *   const sampler = new ShadowSampler();
+ *   const decision = await sampler.routeWithShadow(query, { strategy: 'auto' });
+ *   // decision.hasShadow === true iff we sampled a shadow provider
+ *   // decision.primaryResult and decision.shadowResult are compared automatically
+ */
+
+import { routeQuery, extractQueryFeatures, RouteDecision } from './advancedRouter';
+import { getAvailableProviders, ProviderDefinition } from '../providers/providerConfig';
+import { estimateCost } from '../utils/tokenUtils';
+
+export interface ShadowSamplerConfig {
+  /** 
+   * Minimum query stakes (estimated cost of wrong answer) to consider shadow.
+   * Below this, P(shadow) = 0 regardless of other factors.
+   * In dollars — what a wrong answer costs you.
+   * Default: $0.01 (1 cent — roughly equivalent to a simple API call cost)
+   */
+  minQueryStake?: number;
+  
+  /**
+   * Maximum probability of shadow verification.
+   * Set to 1.0 for always-on (Zahavi-style, expensive).
+   * Default: 0.15 (15% — ODT-proportional sampling)
+   */
+  maxShadowProbability?: number;
+  
+  /**
+   * Risk threshold (from QueryFeatures.risk_profile) above which
+   * shadow probability scales up linearly.
+   * 0.0 = no risk scaling (all queries get same P(shadow))
+   * 0.5 = medium risk scales P(shadow) by 50%
+   * 1.0 = high risk scales P(shadow) by 100%
+   */
+  riskScalingFactor?: number;
+  
+  /**
+   * Cost scaling factor — if query is expensive (large output),
+   * shadow is more worthwhile (higher P(shadow)).
+   * 0.0 = no cost scaling.
+   * Default: 0.3 (expensive queries get moderate boost in P(shadow))
+   */
+  costScalingFactor?: number;
+  
+  /**
+   * Provider to use as shadow (cheapest reliable provider).
+   * If not specified, auto-selected from available providers.
+   */
+  shadowProvider?: string;
+  
+  /**
+   * Always use this shadow provider (overrides auto-selection).
+   */
+  forceShadowProvider?: boolean;
+  
+  /**
+   * If true, use complexity as a signal for verification value.
+   * Complex queries (code, math, reasoning) benefit more from verification.
+   * Default: true
+   */
+  useComplexitySignal?: boolean;
+  
+  /**
+   * Debug mode: always shadow regardless of sampling.
+   * Default: false
+   */
+  forceShadow?: boolean;
+}
+
+export interface ShadowDecision extends RouteDecision {
+  /** Whether a shadow provider was sampled for this query */
+  hasShadow: boolean;
+  /** The sampled shadow provider (null if no shadow) */
+  shadowProvider: string | null;
+  /** Shadow sampling probability that was used */
+  shadowProbability: number;
+  /** The query features that triggered shadow decision */
+  features: ReturnType<typeof extractQueryFeatures>;
+  /** ODT rationale for the shadow decision */
+  shadowReasoning: string;
+}
+
+export interface ShadowedResponse {
+  primary: string;
+  shadow: string | null;
+  winner: 'primary' | 'shadow' | 'tie' | 'no-shadow';
+  confidenceDelta: number;
+}
+
+/**
+ * Optimal Defense Theory Shadow Sampler
+ * 
+ * Allocates shadow verification proportionally to query value and risk.
+ * 
+ * The ODT sampling probability:
+ *   P_shadow = min(maxP, baseP + risk_adj + cost_adj + complexity_adj)
+ * 
+ * where:
+ *   baseP     = baseline verification rate (ODT " constitutive defense")
+ *   risk_adj  = risk_profile scaling (ODT "induced defense")
+ *   cost_adj  = output cost scaling (value of correct answer)
+ *   complexity_adj = query complexity signal
+ */
+export class ShadowSampler {
+  private config: Required<Omit<ShadowSamplerConfig, 'forceShadowProvider'>> & { forceShadowProvider: boolean };
+  
+  // Cached shadow provider (auto-selected on first call)
+  private _shadowProvider: string | null = null;
+  
+  // Counters for sampling statistics
+  private _shadowCount = 0;
+  private _totalCount = 0;
+  
+  constructor(config: ShadowSamplerConfig = {}) {
+    this.config = {
+      minQueryStake: config.minQueryStake ?? 0.01,
+      maxShadowProbability: config.maxShadowProbability ?? 0.15,
+      riskScalingFactor: config.riskScalingFactor ?? 0.5,
+      costScalingFactor: config.costScalingFactor ?? 0.3,
+      shadowProvider: config.shadowProvider ?? null,
+      forceShadowProvider: config.forceShadowProvider ?? false,
+      useComplexitySignal: config.useComplexitySignal ?? true,
+      forceShadow: config.forceShadow ?? false,
+    };
+  }
+  
+  /**
+   * Auto-select the cheapest available provider as the shadow.
+   * Excludes the primary provider to ensure diversity.
+   */
+  private selectShadowProvider(primaryProvider: string): string {
+    if (this.config.forceShadowProvider && this.config.shadowProvider) {
+      return this.config.shadowProvider;
+    }
+    
+    if (this._shadowProvider) return this._shadowProvider;
+    
+    const profiles = getAvailableProviders();
+    const candidates = Object.entries(profiles)
+      .filter(([name, p]: [string, ProviderDefinition]) => {
+        // Exclude primary
+        if (name === primaryProvider) return false;
+        // Must be available (has API key) — cost must be finite
+        const cost = (p.costPerK.input + p.costPerK.output) / 2;
+        return cost < Infinity;
+      })
+      .sort((a, b) => {
+        const costA = (a[1].costPerK.input + a[1].costPerK.output) / 2;
+        const costB = (b[1].costPerK.input + b[1].costPerK.output) / 2;
+        return costA - costB;
+      });
+    
+    if (candidates.length === 0) {
+      // Fallback: pick any non-primary
+      const fallback = Object.keys(profiles).find(n => n !== primaryProvider);
+      this._shadowProvider = fallback || primaryProvider;
+    } else {
+      this._shadowProvider = candidates[0][0];
+    }
+    
+    return this._shadowProvider;
+  }
+  
+  /**
+   * Estimate the query "stakes" — the expected cost of a wrong answer.
+   * 
+   * ODT maps this to "tissue value": how much is this asset worth protecting?
+   * In routing terms: if this query fails, how much does it cost?
+   * 
+   * We approximate this as:
+   *   stake ≈ estimated_output_tokens × cost_per_token × criticality_multiplier
+   * 
+   * where criticality is derived from query complexity and risk_profile.
+   */
+  private estimateQueryStake(features: ReturnType<typeof extractQueryFeatures>): number {
+    const estimatedTokens = features.length * 1.5; // rough estimate
+    const avgCostPerToken = 0.0001; // roughly $0.10/1K tokens
+    const stake = estimatedTokens * avgCostPerToken;
+    
+    // Criticality multiplier from risk profile
+    const riskMultiplier: Record<string, number> = {
+      high: 10.0,    // Wrong answer could cause real harm — worth verifying
+      medium: 2.0,   // Some cost to wrong answer
+      low: 0.5,      // Low stakes — skip verification
+    };
+    const riskMult = riskMultiplier[features.risk_profile || 'medium'] ?? 1.0;
+    
+    // Complexity multiplier — complex queries are harder to verify but more valuable
+    // We use complexity as a proxy for "correctness is harder to judge"
+    const complexityMultiplier = 1 + (features.complexity || 0) * 2;
+    
+    return stake * riskMult * complexityMultiplier;
+  }
+  
+  /**
+   * Compute the ODT sampling probability for this query.
+   * 
+   * ODT principle: defense (shadow) allocation ∝ expected benefit of defense.
+   * Expected benefit = P(failure) × cost_of_failure
+   * 
+   * So P(shadow) scales with:
+   *   1. Query stake (expected cost of wrong answer)
+   *   2. Risk profile (probability of primary failure)
+   *   3. Output cost (verification ROI — cheaper outputs need less verification)
+   *   4. Complexity (complex queries benefit more from verification)
+   */
+  private computeShadowProbability(
+    features: ReturnType<typeof extractQueryFeatures>,
+    stake: number
+  ): number {
+    // Base probability (ODT "constitutive defense" — baseline verification rate)
+    let p = 0.02; // 2% baseline
+    
+    // === STAKES ADJUSTMENT (ODT "tissue value") ===
+    // Higher stake → proportionally higher verification probability
+    // Scale from 2% to maxP as stake goes from minStake to $1.00
+    if (stake >= this.config.minQueryStake) {
+      const stakeAdj = Math.min(
+        (stake - this.config.minQueryStake) / (1.0 - this.config.minQueryStake),
+        1.0
+      ) * (this.config.maxShadowProbability - 0.02);
+      p += stakeAdj;
+    } else {
+      // Below minimum stake — no verification regardless
+      return 0;
+    }
+    
+    // === RISK ADJUSTMENT (ODT "attack probability") ===
+    // High-risk queries are more likely to have primary failures
+    // Scale risk_adj by riskScalingFactor (0.5 = 50% boost for high-risk)
+    const riskAdj: Record<string, number> = {
+      high: 0.10 * this.config.riskScalingFactor,
+      medium: 0.03 * this.config.riskScalingFactor,
+      low: 0.0,
+    };
+    p += riskAdj[features.risk_profile || 'medium'] ?? 0;
+    
+    // === COMPLEXITY ADJUSTMENT (ODT "defense efficacy") ===
+    // Complex queries benefit more from verification (more errors to catch)
+    // But also harder to verify (requires domain knowledge to judge)
+    if (this.config.useComplexitySignal) {
+      if (features.has_code) {
+        p += 0.05; // Code verification has high ROI (bugs are costly)
+      }
+      if (features.requires_reasoning) {
+        p += 0.03; // Reasoning errors are subtle but costly
+      }
+      if (features.has_math) {
+        p += 0.04; // Math has objective ground truth — easy to verify (note: field is has_math)
+      }
+    }
+    
+    // === COST SCALING (ODT "marginal defense cost") ===
+    // Expensive outputs are harder to produce — verify to avoid waste
+    if (this.config.costScalingFactor > 0 && features.length > 500) {
+      const costAdj = Math.min(
+        (features.length - 500) / 10000, // scale up for very long outputs
+        0.05
+      ) * this.config.costScalingFactor;
+      p += costAdj;
+    }
+    
+    // Clamp to [0, maxShadowProbability]
+    return Math.min(Math.max(p, 0), this.config.maxShadowProbability);
+  }
+  
+  /**
+   * Route a query, with optional ODT-proportional shadow verification.
+   * 
+   * @param prompt - The user query
+   * @param options - Routing options (same as routeQuery)
+   * @returns ShadowDecision with shadow metadata
+   */
+  routeWithShadow(
+    prompt: string,
+    options?: { available_models?: string[]; budget_multiplier?: number }
+  ): ShadowDecision {
+    const features = extractQueryFeatures(prompt);
+    const primaryDecision = routeQuery(prompt, options?.available_models, options?.budget_multiplier);
+    
+    // Compute query stakes and shadow probability
+    const stake = this.estimateQueryStake(features);
+    const shadowProb = this.computeShadowProbability(features, stake);
+    
+    // ODT sampling decision
+    const shouldShadow = this.config.forceShadow || Math.random() < shadowProb;
+    const shadowProvider = shouldShadow
+      ? this.selectShadowProvider(primaryDecision.primary_model || '')
+      : null;
+    
+    if (shouldShadow) {
+      this._shadowCount++;
+    }
+    this._totalCount++;
+    
+    const reasoning = shouldShadow
+      ? `ODT shadow: stake=${stake.toFixed(4)}, risk=${features.risk_profile}, P=${shadowProb.toFixed(3)}, complexity=${features.complexity.toFixed(2)}`
+      : `ODT no-shadow: stake=${stake.toFixed(4)} below threshold`;
+    
+    return {
+      ...primaryDecision,
+      hasShadow: shouldShadow,
+      shadowProvider,
+      shadowProbability: shadowProb,
+      features,
+      shadowReasoning: reasoning,
+    };
+  }
+  
+  /**
+   * Compare primary and shadow outputs.
+   * Returns the "better" answer and confidence delta.
+   * 
+   * For production use: this would call both providers in parallel
+   * and compare outputs. For now, returns a stub that signals
+   * the caller should handle comparison.
+   */
+  async compareOutputs(
+    primaryAnswer: string,
+    shadowAnswer: string | null
+  ): Promise<ShadowedResponse> {
+    if (!shadowAnswer) {
+      return {
+        primary: primaryAnswer,
+        shadow: null,
+        winner: 'no-shadow',
+        confidenceDelta: 0,
+      };
+    }
+    
+    // Simple comparison: length + character overlap as proxy for agreement
+    // In production, this would use a proper semantic similarity check
+    const primaryLen = primaryAnswer.length;
+    const shadowLen = shadowAnswer.length;
+    const lengthRatio = Math.min(primaryLen, shadowLen) / Math.max(primaryLen, shadowLen);
+    
+    // Count common bigrams as a simple similarity proxy
+    const primaryBigrams = new Set<string>();
+    const shadowBigrams = new Set<string>();
+    for (let i = 0; i < primaryAnswer.length - 1; i++) {
+      primaryBigrams.add(primaryAnswer.slice(i, i + 2));
+    }
+    for (let i = 0; i < shadowAnswer.length - 1; i++) {
+      shadowBigrams.add(shadowAnswer.slice(i, i + 2));
+    }
+    
+    let intersection = 0;
+    for (const bg of primaryBigrams) {
+      if (shadowBigrams.has(bg)) intersection++;
+    }
+    const union = primaryBigrams.size + shadowBigrams.size - intersection;
+    const jaccard = union > 0 ? intersection / union : 0;
+    
+    // High agreement (jaccard > 0.8) → trust primary
+    // Low agreement → flag for review or prefer primary
+    const winner: ShadowedResponse['winner'] =
+      jaccard > 0.8 ? 'primary'
+      : jaccard > 0.5 ? 'tie'
+      : 'shadow'; // Low agreement: shadow might have caught something
+    
+    const confidenceDelta = jaccard > 0.8 ? 0 : -0.1; // Reduce confidence if they differ
+    
+    return {
+      primary: primaryAnswer,
+      shadow: shadowAnswer,
+      winner,
+      confidenceDelta,
+    };
+  }
+  
+  /**
+   * Get sampling statistics for monitoring.
+   */
+  getStats(): { shadowCount: number; totalCount: number; shadowRate: number } {
+    return {
+      shadowCount: this._shadowCount,
+      totalCount: this._totalCount,
+      shadowRate: this._totalCount > 0 ? this._shadowCount / this._totalCount : 0,
+    };
+  }
+  
+  /**
+   * Reset statistics counters.
+   */
+  resetStats(): void {
+    this._shadowCount = 0;
+    this._totalCount = 0;
+  }
+  
+  /**
+   * Update configuration at runtime.
+   */
+  configure(config: Partial<ShadowSamplerConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+}
+
+// ============================================================
+// NOTE: Named exports are at declaration level above.
+// ShadowSampler, ShadowSamplerConfig, ShadowDecision, ShadowedResponse
+// are all exported via 'export interface' / 'export class'
+// ============================================================

package/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,94 +0,0 @@
----
-name: Bug Report
-about: Report a bug or unexpected behavior to help us improve A3M Router
-title: "[Bug] "
-labels: bug, needs-triage
-assignees: ""
----
-
-## Description
-
-A clear and concise description of the bug.
-
-## Reproduction Steps
-
-Steps to reproduce the behavior:
-
-```bash
-# 1. Set up (if applicable)
-export PROVIDER_API_KEY=sk-...
-
-# 2. Run
-npx a3m-router route "Your query here"
-```
-
-If using the SDK, provide a minimal code snippet:
-
-```typescript
-import { A3MRouter } from "adaptive-memory-multi-model-router";
-
-const router = new A3MRouter({ /* your config */ });
-const result = await router.route("Your query");
-```
-
-## Expected Behavior
-
-What did you expect to happen?
-
-## Actual Behavior
-
-What actually happened? Include error messages, stack traces, or unexpected output.
-
-```
-Paste error output or logs here
-```
-
-## A3M Router Version
-
-- Package: `adaptive-memory-multi-model-router@<version>` (run `npm list adaptive-memory-multi-model-router`)
-- CLI version (if applicable): `npx a3m-router --version`
-
-## Environment
-
-- **OS:** macOS / Linux / Windows
-- **Node.js version:** (run `node --version`)
-- **npm version:** (run `npm --version`)
-- **Python version (if using Python SDK):** (run `python --version`)
-
-## Providers Used
-
-Which provider(s) were involved? (e.g., Groq, OpenAI, Anthropic, NVIDIA, DeepSeek, custom)
-
-## Configuration
-
-Attach or describe relevant config (redact API keys):
-
-```json
-{
-  "providers": { ... },
-  "routing": { ... },
-  "budgets": { ... }
-}
-```
-
-## Logs
-
-If you ran with `DEBUG=*` or `LOG_LEVEL=debug`, include relevant log lines:
-
-```
-[DEBUG] Routing query...
-[ERROR] Provider groq returned 503
-```
-
-## Additional Context
-
-- Does this happen consistently or intermittently?
-- Does it affect all providers or a specific one?
-- Did it work in a previous version? If so, which version?
-- Any recent changes to your setup?
-
-## Checklist
-
-- [ ] I have searched existing issues for duplicates
-- [ ] I have redacted all API keys and secrets from the above
-- [ ] I can reliably reproduce this bug

package/.github/ISSUE_TEMPLATE/checklist.md

@@ -1,35 +0,0 @@
-GitHub Pages Checker Checklist
-
-## Phase 1: Content Ready
-- [x] GitHub Pages workflow configured
-- [x] README.md updated with TL;DR and latest stats
-- [x] popular-boosters.md ready with quick wins
-- [ ] Anim workshops for sponsors
-- [ ] PI-assistant memory setup
-- [ ] Orion assistant memory setup
-
-## Phase 2: Deploy: GitHub Pages
-- [x] /docs/index.html exists (activated and deployed)
-- [ ] Primary landing page URL
-- [ ] Right domain (das-rebel.github.io/a3m-router)
-- [ ] CNAME file if needed
-- [ ] Deployed? -> https://das-rebel.github.io/a3m-router/docs/index.html
-
-## Phase 3: Traffic Greasing
-- [x] Twitter thread
-- [x] Reddit post
-- [x] HackerNews
-- [x] Lint checklist files
-
----
-
-You need to set up a nitro django project (or similar) for tracking usage.
-
-**Key status:**
-- GitHub Pages is working (deployed from /docs)
-- Latest commit was July 7, 2026
-- Repo has 10 stars, 1 fork
-- If you need advanced ORM/batch processing, maybe try something else (like Data Stewards)
-- If you need standard server-side script support, this should work.
-
-Want me to: 1) Create deployment instructions, or 2) Set up script environment?

package/.github/ISSUE_TEMPLATE/config.yml

@@ -1,17 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: "\U0001F4D6 Documentation & Quick Start"
-    url: https://github.com/Das-rebel/a3m-router#readme
-    about: Check the README for setup, SDK usage, API reference, and configuration examples
-  - name: "\U0001F4AC Discussions & Community"
-    url: https://github.com/Das-rebel/a3m-router/discussions
-    about: Ask questions, share ideas, show what you've built, and get help from the community
-  - name: "\U0001F680 NPM Package"
-    url: https://www.npmjs.com/package/adaptive-memory-multi-model-router
-    about: View package details, version history, and download statistics
-  - name: "\U0001F4DC Changelog"
-    url: https://github.com/Das-rebel/a3m-router/blob/main/CHANGELOG.md
-    about: See what's changed in recent releases
-  - name: "\U0001F6A8 Security"
-    url: https://github.com/Das-rebel/a3m-router/security/policy
-    about: Report security vulnerabilities privately

package/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,71 +0,0 @@
----
-name: Feature Request
-about: Suggest an idea or enhancement for A3M Router
-title: "[Feature] "
-labels: enhancement
-assignees: ""
----
-
-## Problem Statement
-
-A clear description of the problem you're trying to solve. What gap or pain point does this feature address?
-
-**Example:** "Currently, A3M Router doesn't support routing based on response language. When I send multilingual queries, I want them routed to providers that perform best in that language."
-
-## Proposed Solution
-
-Describe the feature you'd like to see. Be as specific as possible about behavior, configuration, and API.
-
-```typescript
-// If applicable, sketch the API you envision
-const router = new A3MRouter({
-  languageRouting: {
-    enabled: true,
-    defaultProvider: "openai",
-    languageOverrides: {
-      ja: "anthropic",
-      zh: "deepseek",
-    },
-  },
-});
-```
-
-## Use Case
-
-Describe the real-world scenario that would benefit from this feature.
-
-- Who is the target user? (e.g., solo developer, enterprise team, researcher)
-- What workflow does it enable or simplify?
-- How frequently would this be used?
-
-## Alternatives Considered
-
-List any workarounds or alternative approaches you've explored:
-
-1. Manual provider selection per query
-2. Custom wrapper script
-3. Forking and modifying the router
-4. Using a different tool altogether
-
-Explain why these are insufficient.
-
-## Priority
-
-How important is this to you?
-
-- [ ] **Blocking** — Cannot proceed without this feature
-- [ ] **High** — Important for my workflow
-- [ ] **Medium** — Nice to have
-- [ ] **Low** — Interesting idea, not urgent
-
-## Would You Implement It?
-
-- [ ] **Yes** — I can submit a PR (with guidance)
-- [ ] **Maybe** — Willing to help test or provide requirements
-- [ ] **No** — Just suggesting
-
-## Additional Context
-
-- Links to related discussions, issues, or external references
-- Screenshots or mockups (if applicable)
-- Any constraints or requirements (e.g., must work offline, must not increase bundle size)

package/.well-known/ai-plugin.json

@@ -1,16 +0,0 @@
-{
-  "schema_version": "v1",
-  "name_for_human": "A3M Router",
-  "name_for_model": "a3m_router",
-  "description_for_human": "LLM routing proxy — #1 on RouterArena (0.9404 / 96.77%) at $0.0768/1K. Rule-based, no ML, 47+ providers.",
-  "description_for_model": "A3M Router is an LLM routing service. Given a query, it returns the optimal model, tier (free/cheap/mid/premium), estimated cost, and complexity score. Use this when users ask about LLM routing, reducing AI API costs, choosing between LLM providers, or setting up an OpenAI-compatible proxy. Available as TypeScript SDK, Python SDK, CLI, and REST API. RouterArena rank #1 with 0.9404 / 96.77% at $0.0768 per 1K queries (arXiv:2510.00202).",
-  "api": {
-    "type": "openapi",
-    "url": "https://das-rebel.github.io/a3m-router/docs/openapi.json"
-  },
-  "auth": {
-    "type": "none"
-  },
-  "contact_email": "Sdas22@gmail.com",
-  "legal_info_url": "https://github.com/Das-rebel/a3m-router/blob/main/LICENSE"
-}