adaptive-memory-multi-model-router 2.14.17 → 2.14.18

package/council-votes/coverage-vote.md

@@ -0,0 +1,93 @@
+## Test Coverage Vote
+
+### Finding 1: GuardrailEngine Has Zero Tests (CRITICAL)
+- **Files:** `src/security/guardrails.ts` (500+ lines)
+- **Problem:** Critical security module has NO tests despite handling:
+  - Prompt injection detection (DAN, jailbreak, ignore instructions)
+  - PII redaction (emails, phones, SSN, credit cards, API keys)
+  - Content filtering (violence, hate, self-harm)
+  - Output validation and hallucination detection
+- **Solution:** Add comprehensive tests for:
+  - Each prompt injection pattern detection
+  - PII redaction for all types
+  - Content filter thresholds
+  - GuardrailEngine.checkInput() and checkOutput() full paths
+  - Custom guardrail registration
+  - Blocklist management
+- **Tests gained:** ~25 tests covering security-critical paths
+
+### Finding 2: EnsembleOrchestrator Has Zero Tests (CORE P0)
+- **Files:** `src/ensemble.ts`, `tests/routing/ensembleVoting.test.ts` (partial)
+- **Problem:** Core P0 feature (parallel multi-LLM execution with result merging) has only partial tests in `tests/routing/`. The `EnsembleOrchestrator` class is NOT tested, and these critical paths are missing:
+  - Actual provider calls (mocked in ensembleVoting.test.ts)
+  - Majority voting strategy
+  - Weighted voting strategy  
+  - Conservative strategy with uncertainty detection
+  - All-providers-fail error handling
+- **Solution:** Add integration tests for EnsembleOrchestrator with:
+  - Mock providers returning different answers
+  - Strategy-specific voting logic
+  - Confidence scoring accuracy
+  - Reasoning generation
+- **Tests gained:** ~15 tests for the unique differentiator
+
+### Finding 3: CostAnalytics Has Zero Tests (BUSINESS-CRITICAL)
+- **Files:** `src/analytics/costAnalytics.ts`
+- **Problem:** Cost tracking and savings calculation has no tests:
+  - No verification of savings calculation against known baselines
+  - No tests for BASELINE_COSTS accuracy
+  - No export format tests (JSON/CSV)
+  - No period filtering (hour/day/week/month)
+  - No auto-rotation when maxRecords exceeded
+- **Solution:** Add tests for:
+  - Savings calculation accuracy (compare against known costs)
+  - Period filtering correctness
+  - Export format validation
+  - Provider/query type breakdown accuracy
+- **Tests gained:** ~12 tests for cost accuracy
+
+### Finding 4: SDK Class (A3MRouter) Has Only Structure Tests
+- **Files:** `src/sdk.ts`, `test-council/1-structure-tests.test.ts`
+- **Problem:** SDK wrapper has basic type-checking but no behavioral tests:
+  - No tests for route() behavior
+  - No tests for analyze() feature extraction
+  - No tests for tier classification thresholds
+  - No tests for recommend() method
+- **Solution:** Add behavioral tests verifying:
+  - Route decisions match expected patterns
+  - Feature extraction accuracy
+  - Tier classification boundaries (free/cheap/mid/premium)
+  - Batch routing consistency
+- **Tests gained:** ~10 tests
+
+### Finding 5: Observability Middleware Not Tested
+- **Files:** `src/observability/middleware.ts`
+- **Problem:** Middleware components not tested:
+  - observabilityMiddleware not tested
+  - budgetAlertMiddleware not tested  
+  - Integration with Express/Fastify not tested
+- **Solution:** Add middleware tests verifying:
+  - Request/response interception
+  - Header injection
+  - Budget alert triggers
+  - Error propagation
+- **Tests gained:** ~8 tests
+
+---
+
+## Vote
+
+**I vote for Finding #1 (GuardrailEngine) as highest priority.**
+
+### Rationale:
+1. **Security-critical code without tests = production risk** - Prompt injection and PII handling bugs can leak sensitive data or bypass safety
+2. **Measurable impact** - Every user query goes through guardrails before routing
+3. **Clear scope** - 500+ lines with well-defined functions that are easy to test
+4. **Existing patterns** - The providerRetry.test.ts shows how to structure similar complex tests
+
+### Implementation estimate:
+- **GuardrailEngine:** ~3 hours for comprehensive tests
+- **EnsembleOrchestrator:** ~2 hours  
+- **CostAnalytics:** ~1.5 hours
+- **SDK:** ~1 hour
+- **Middleware:** ~1 hour

package/dist/cost/costTracker.d.ts

@@ -1,22 +1,109 @@
 /**
- * TMLPD Cost Tracker
+ * A3M Router - Budget Enforcer + Cost Tracker
  *
- * Tracks real-time spending across all providers.
- * Supports per-model budgets, spending alerts, and cost analysis.
+ * Hard budget enforcement for API key spend management:
+ * - Track spend per API key with monthly reset
+ * - Check budget before each request
+ * - Emit alerts at configurable thresholds (50%, 80%, 100%)
+ * - Support hard cap (reject requests) or soft cap (warn only)
+ * - In-memory storage (Redis backup can be added later)
+ *
+ * Plus legacy CostTracker for backward compatibility.
  */
+import { EventEmitter } from 'events';
 export interface BudgetConfig {
-    daily_limit?: number;
-    monthly_limit?: number;
-    per_model_limits?: Record<string, number>;
+    apiKey: string;
+    monthlyLimit: number;
+    alertThresholds?: number[];
+    hardCap?: boolean;
+}
+export interface SpendRecord {
+    apiKey: string;
+    spent: number;
+    budget: number;
+    remaining: number;
+    resetDate: Date;
+    alertEmitted: number[] | Set<number>;
+}
+export interface BudgetCheckResult {
+    allowed: boolean;
+    reason?: string;
+    remaining: number;
+}
+export declare class BudgetEnforcer extends EventEmitter {
+    budgets: Map<string, BudgetConfig>;
+    spend: Map<string, SpendRecord>;
+    static DEFAULT_THRESHOLDS: number[];
+    constructor();
+    /**
+     * Set budget for an API key
+     */
+    setBudget(apiKey: string, monthlyLimit: number, options?: {
+        alertThresholds?: number[];
+        hardCap?: boolean;
+    }): void;
+    /**
+     * Get current budget config for an API key
+     */
+    getBudgetConfig(apiKey: string): BudgetConfig | undefined;
+    /**
+     * Remove budget config (stops tracking)
+     */
+    removeBudget(apiKey: string): void;
+    /**
+     * Check if a request is allowed within budget
+     */
+    checkBudget(apiKey: string, additionalCost: number): BudgetCheckResult;
+    /**
+     * Record spend for an API key
+     */
+    recordSpend(apiKey: string, cost: number): void;
+    /**
+     * Get current spend record for an API key
+     */
+    getSpend(apiKey: string): SpendRecord | undefined;
+    /**
+     * Get all spend records
+     */
+    getAllSpend(): SpendRecord[];
+    /**
+     * Reset budget for an API key (manual reset)
+     */
+    resetBudget(apiKey: string): void;
+    /**
+     * Reset all budgets
+     */
+    resetAll(): void;
+    /**
+     * Update monthly limit for an API key
+     */
+    updateLimit(apiKey: string, monthlyLimit: number): void;
+    /**
+     * Check if reset is needed and perform it
+     */
+    checkAndReset(apiKey: string): void;
+    /**
+     * Check if monthly reset is due
+     */
+    isResetDue(resetDate: Date): boolean;
+    /**
+     * Get next monthly reset date
+     */
+    getNextResetDate(): Date;
+    /**
+     * Get days until next reset
+     */
+    getDaysUntilReset(apiKey: string): number | undefined;
 }
-export interface CostAlert {
-    type: "daily" | "monthly" | "model" | "budget";
-    threshold: number;
-    current: number;
-    provider?: string;
-    model?: string;
+export declare function createBudgetEnforcer(): BudgetEnforcer;
+export declare class BudgetExceededError extends Error {
+    apiKey: string;
+    spent: number;
+    budget: number;
+    remaining: number;
+    constructor(apiKey: string, spent: number, budget: number);
 }
-export interface CostSnapshot {
+interface CostSnapshot {
     provider: string;
     model: string;
     input_tokens: number;
@@ -26,7 +113,7 @@ export interface CostSnapshot {
     total_cost: number;
     timestamp: number;
 }
-export interface CostSummary {
+interface CostSummary {
     total_cost: number;
     by_provider: Record<string, number>;
     by_model: Record<string, number>;
@@ -46,50 +133,28 @@ export declare class CostTracker {
     private alerts_callback;
     private daily_reset;
     private monthly_reset;
-    constructor(budgets?: BudgetConfig);
-    /**
-     * Calculate cost for a model based on tokens
-     */
+    constructor(budgets?: {
+        daily_limit?: number;
+        monthly_limit?: number;
+        per_model_limits?: Record<string, number>;
+    });
     calculateCost(model: string, input_tokens: number, output_tokens: number): {
         input: number;
         output: number;
         total: number;
     };
-    /**
-     * Record a request's cost
-     */
     record(provider: string, model: string, input_tokens: number, output_tokens: number): CostSnapshot;
-    /**
-     * Check budgets and trigger alerts
-     */
     private checkBudgets;
-    /**
-     * Emit an alert via callback
-     */
     private emitAlert;
-    /**
-     * Register alert callback
-     */
-    onAlert(callback: (alert: CostAlert) => void): void;
-    /**
-     * Get comprehensive cost summary
-     */
+    onAlert(callback: (alert: any) => void): void;
     getSummary(): CostSummary;
-    /**
-     * Get remaining budget
-     */
     getRemainingBudget(): {
         daily: number | null;
         monthly: number | null;
         per_model: Record<string, number>;
     };
-    /**
-     * Reset cost history
-     */
     reset(): void;
-    /**
-     * Export cost data for analysis
-     */
     export(): CostSnapshot[];
+    getStatus(): CostSummary;
 }
-//# sourceMappingURL=costTracker.d.ts.map
+export {};