adaptive-memory-multi-model-router 2.14.49 → 2.14.52

package/tests/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "a3m-router-tests",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "esbuild": "^0.28.1",
+    "typescript": "^5.8.0",
+    "vitest": "^4.1.9"
+  },
+  "dependencies": {
+    "nanoid": "^5.0.0"
+  }
+}

package/tests/routing/ensembleVoting.test.ts

@@ -0,0 +1,236 @@
+import { describe, it, expect, vi } from 'vitest';
+import {
+  executeEnsemble,
+  mergeComplementary,
+  recordFeedback,
+} from '../../tmlpd-pi-extension/src/routing/ensembleVoting';
+
+describe('executeEnsemble', () => {
+  const defaultExecutors: Record<string, (q: string, s: string, c: string) => Promise<string | null>> = {
+    groq: vi.fn().mockResolvedValue(
+      'Simple answer with no details.'
+    ),
+    nvidia: vi.fn().mockResolvedValue(
+      'Detailed answer including 42 numerical references. The API endpoint app.ts handles requests. ' +
+      '* Point one\n* Point two\n* Point three\n* Point four\n* Point five\n' +
+      'The system uses Docker, Redis, and GCS for infrastructure. npm install is required.'
+    ),
+  };
+
+  it('scores detailed responses higher than short ones', async () => {
+    const result = await executeEnsemble(
+      'test query',
+      'system prompt',
+      '',
+      defaultExecutors,
+      { providers: ['groq', 'nvidia'] }
+    );
+
+    expect(result.scores['nvidia']).toBeGreaterThan(result.scores['groq']);
+    expect(result.winner).toBe('nvidia');
+  });
+
+  it('selects the provider with the highest score as winner', async () => {
+    const executors = {
+      low: vi.fn().mockResolvedValue('Hi'),
+      mid: vi.fn().mockResolvedValue('A moderate answer with some text.'),
+      high: vi.fn().mockResolvedValue(
+        'Excellent comprehensive response. Contains 3 key points. The API integration uses app.ts. ' +
+        '* Point A\n* Point B\n* Point C\n* Point D\n* Point E'
+      ),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['low', 'mid', 'high'] }
+    );
+
+    expect(result.winner).toBe('high');
+  });
+
+  it('handles providers returning null (errors)', async () => {
+    const executors = {
+      good: vi.fn().mockResolvedValue('Valid answer here.'),
+      bad: vi.fn().mockRejectedValue(new Error('API failure')),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['good', 'bad'] }
+    );
+
+    expect(result.allResults['good']).toBe('Valid answer here.');
+    expect(result.allResults['bad']).toBeNull();
+    expect(result.scores['bad']).toBe(0);
+    expect(result.winner).toBe('good');
+  });
+
+  it('handles all providers failing', async () => {
+    const executors = {
+      a: vi.fn().mockRejectedValue(new Error('fail')),
+      b: vi.fn().mockRejectedValue(new Error('fail')),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['a', 'b'] }
+    );
+
+    expect(result.best).toBe('');
+    expect(Object.values(result.scores).every(s => s === 0)).toBe(true);
+  });
+
+  it('applies length penalty for very short responses', async () => {
+    const executors = {
+      short: vi.fn().mockResolvedValue('Hi'),
+      normal: vi.fn().mockResolvedValue('A normal length answer with several words in it.'),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['short', 'normal'] }
+    );
+
+    expect(result.scores["short"]).toBeLessThanOrEqual(result.scores["normal"]);
+  });
+
+  it('applies specificity bonus for responses with numbers', async () => {
+    const executors = {
+      vague: vi.fn().mockResolvedValue('The system works well and is quite good.'),
+      specific: vi.fn().mockResolvedValue('The system processes 42 requests per second with 99.9% uptime.'),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['vague', 'specific'] }
+    );
+
+    expect(result.scores['specific']).toBeGreaterThan(result.scores['vague']);
+  });
+
+  it('applies structure bonus for multi-line responses', async () => {
+    const executors = {
+      oneLine: vi.fn().mockResolvedValue('Just a single line answer here.'),
+      multiLine: vi.fn().mockResolvedValue('Line one\nLine two\nLine three\nLine four\nLine five'),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['oneLine', 'multiLine'] }
+    );
+
+    expect(result.scores['multiLine']).toBeGreaterThan(result.scores['oneLine']);
+  });
+
+  it('includes timing information in result', async () => {
+    const executors = {
+      fast: vi.fn().mockImplementation(
+        () => new Promise(r => setTimeout(() => r('Quick answer.'), 5))
+      ),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['fast'] }
+    );
+
+    expect(result.timing.totalMs).toBeGreaterThanOrEqual(0);
+    expect(result.timing.perProvider['fast']).toBeGreaterThanOrEqual(0);
+  });
+
+  it('produces reasoning string explaining winner selection', async () => {
+    const executors = {
+      a: vi.fn().mockResolvedValue('Answer A with some detail.'),
+      b: vi.fn().mockResolvedValue('Answer B with more specific 42 details and technical API references.'),
+    };
+
+    const result = await executeEnsemble(
+      'query',
+      'sys',
+      '',
+      executors,
+      { providers: ['a', 'b'] }
+    );
+
+    expect(result.reasoning).toBeTruthy();
+    expect(result.reasoning).toContain(result.winner);
+  });
+});
+
+describe('mergeComplementary', () => {
+  it('merges multiple results into sections', () => {
+    const merged = mergeComplementary(['Answer one', 'Answer two']);
+    expect(merged).toContain('### Provider 1');
+    expect(merged).toContain('### Provider 2');
+    expect(merged).toContain('Answer one');
+    expect(merged).toContain('Answer two');
+    expect(merged).toContain('---');
+  });
+
+  it('filters out empty results', () => {
+    const merged = mergeComplementary(['Valid', '', 'Also valid']);
+    expect(merged).toContain('### Provider 1');
+    expect(merged).toContain('### Provider 2');
+    expect(merged).not.toContain('### Provider 3');
+  });
+
+  it('truncates at maxLength', () => {
+    const long = 'A'.repeat(3000);
+    const merged = mergeComplementary([long, long], 500);
+    expect(merged.length).toBeLessThanOrEqual(500);
+  });
+
+  it('returns empty string for all-empty input', () => {
+    expect(mergeComplementary([])).toBe('');
+    expect(mergeComplementary(['', '', null as unknown as string])).toBe('');
+  });
+});
+
+describe('recordFeedback', () => {
+  it('increments good count for helpful feedback', () => {
+    const history: Record<string, { good: number; bad: number }> = {};
+    const updated = recordFeedback('groq', true, history);
+    expect(updated['groq'].good).toBe(1);
+    expect(updated['groq'].bad).toBe(0);
+  });
+
+  it('increments bad count for unhelpful feedback', () => {
+    const history: Record<string, { good: number; bad: number }> = { groq: { good: 1, bad: 0 } };
+    const updated = recordFeedback('groq', false, history);
+    expect(updated['groq'].good).toBe(1);
+    expect(updated['groq'].bad).toBe(1);
+  });
+
+  it('initializes history entry if missing', () => {
+    const history: Record<string, { good: number; bad: number }> = {};
+    const updated = recordFeedback('new-provider', true, history);
+    expect(updated['new-provider']).toEqual({ good: 1, bad: 0 });
+  });
+
+  it('returns the same history object (mutates in place)', () => {
+    const history: Record<string, { good: number; bad: number }> = {};
+    const updated = recordFeedback('groq', true, history);
+    expect(updated).toBe(history);
+  });
+});

package/tests/routing/providerRetry.test.ts

@@ -0,0 +1,360 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// Mock tokenUtils BEFORE importing providerRetry
+vi.mock('../../src/utils/tokenUtils', () => ({
+  countTokens: (text: string) => {
+    if (!text || text.length === 0) return 0;
+    return Math.ceil(text.trim().split(/\s+/).length * 1.3);
+  },
+  estimateTokens: (text: string) => {
+    if (!text || text.length === 0) return 0;
+    return Math.ceil(text.trim().split(/\s+/).length * 1.3);
+  },
+}));
+
+import {
+  ProviderRetryHandler,
+  createRetryHandler,
+  DEFAULT_RETRY_CONFIG,
+  DEFAULT_PROVIDER_CONFIG,
+  PROVIDER_CONTEXT_LIMITS,
+  RetryConfig,
+  ProviderRetryConfig,
+} from '../../src/routing/providerRetry';
+
+// ============================================================
+// HELPERS
+// ============================================================
+
+function expectInRange(actual: number, min: number, max: number, label: string): void {
+  expect(actual >= min && actual <= max).toBe(true);
+}
+
+// ============================================================
+// SUITE
+// ============================================================
+
+describe('ProviderRetryHandler', () => {
+  let handler: ProviderRetryHandler;
+
+  beforeEach(() => {
+    handler = new ProviderRetryHandler();
+  });
+
+  describe('constructor', () => {
+    it('initializes with default configs', () => {
+      expect(handler).toBeInstanceOf(ProviderRetryHandler);
+    });
+
+    it('initializes with custom provider configs', () => {
+      const custom: ProviderRetryConfig = {
+        customProvider: {
+          timeout: 5000,
+          retry: { maxRetries: 2, initialDelayMs: 500, maxDelayMs: 10000, backoffMultiplier: 2 },
+          rateLimitRetries: 3,
+        },
+      };
+      const customHandler = new ProviderRetryHandler(custom);
+      const cfg = customHandler.getConfig('customProvider');
+      expect(cfg.timeout).toBe(5000);
+    });
+  });
+
+  describe('getConfig', () => {
+    it('returns deepseek config with correct values', () => {
+      const cfg = handler.getConfig('deepseek');
+      expect(cfg.timeout).toBe(30000);
+      expect(cfg.retry.maxRetries).toBe(5);
+      expect(cfg.rateLimitRetries).toBe(3);
+    });
+
+    it('returns groq config with short timeout', () => {
+      const cfg = handler.getConfig('groq');
+      expect(cfg.timeout).toBe(10000);
+      expect(cfg.retry.maxRetries).toBe(2);
+      expect(cfg.rateLimitRetries).toBe(1);
+    });
+
+    it('falls back to default for unknown providers', () => {
+      const cfg = handler.getConfig('nonexistent');
+      expect(cfg.timeout).toBe(15000);
+      expect(cfg.retry.maxRetries).toBe(3);
+    });
+  });
+
+  describe('configureProvider', () => {
+    it('adds a new custom provider', () => {
+      handler.configureProvider('my-provider', {
+        timeout: 9999,
+        retry: { maxRetries: 10, initialDelayMs: 100, maxDelayMs: 5000, backoffMultiplier: 1.5 },
+        rateLimitRetries: 5,
+      });
+      const cfg = handler.getConfig('my-provider');
+      expect(cfg.timeout).toBe(9999);
+      expect(cfg.retry.maxRetries).toBe(10);
+      expect(cfg.retry.initialDelayMs).toBe(100);
+      expect(cfg.rateLimitRetries).toBe(5);
+    });
+
+    it('overrides existing provider partially', () => {
+      handler.configureProvider('groq', { timeout: 50000 });
+      const cfg = handler.getConfig('groq');
+      expect(cfg.timeout).toBe(50000);
+      // Other values unchanged
+      expect(cfg.retry.maxRetries).toBe(2);
+      expect(cfg.rateLimitRetries).toBe(1);
+    });
+  });
+
+  describe('isRetryableError', () => {
+    it('returns true for common transient errors', () => {
+      expect(handler.isRetryableError({ code: 'ECONNRESET' })).toBe(true);
+      expect(handler.isRetryableError({ code: 'ETIMEDOUT' })).toBe(true);
+      expect(handler.isRetryableError({ code: 'ECONNREFUSED' })).toBe(true);
+      expect(handler.isRetryableError({ code: 'EAI_AGAIN' })).toBe(true);
+    });
+
+    it('returns true for 5xx status codes', () => {
+      expect(handler.isRetryableError({ status: 500 })).toBe(true);
+      expect(handler.isRetryableError({ status: 502 })).toBe(true);
+      expect(handler.isRetryableError({ status: 503 })).toBe(true);
+      expect(handler.isRetryableError({ status: 504 })).toBe(true);
+    });
+
+    it('returns true for 429 rate limit', () => {
+      expect(handler.isRetryableError({ status: 429 })).toBe(true);
+      expect(handler.isRetryableError({ statusCode: 429 })).toBe(true);
+    });
+
+    it('returns false for 4xx client errors', () => {
+      expect(handler.isRetryableError({ status: 400 })).toBe(false);
+      expect(handler.isRetryableError({ status: 404 })).toBe(false);
+    });
+
+    it('returns false for permanent provider state errors', () => {
+      expect(handler.isRetryableError({ status: 401 })).toBe(false);
+      expect(handler.isRetryableError({ status: 403 })).toBe(false);
+      expect(handler.isRetryableError({ message: 'insufficient balance' })).toBe(false);
+      expect(handler.isRetryableError({ message: 'invalid API key' })).toBe(false);
+      expect(handler.isRetryableError({ message: 'quota exhausted' })).toBe(false);
+    });
+
+    it('returns false for null/undefined error', () => {
+      expect(handler.isRetryableError(null)).toBe(false);
+      expect(handler.isRetryableError(undefined)).toBe(false);
+    });
+  });
+
+  describe('isRateLimitError', () => {
+    it('detects 429 in status field', () => {
+      expect(handler.isRateLimitError({ status: 429 })).toBe(true);
+    });
+
+    it('detects 429 in statusCode field', () => {
+      expect(handler.isRateLimitError({ statusCode: 429 })).toBe(true);
+    });
+
+    it('returns false for non-429 errors', () => {
+      expect(handler.isRateLimitError({ status: 200 })).toBe(false);
+      expect(handler.isRateLimitError({ status: 500 })).toBe(false);
+      expect(handler.isRateLimitError({})).toBe(false);
+    });
+  });
+
+  describe('calculateBackoffDelay', () => {
+    const defaultConfig: RetryConfig = {
+      maxRetries: 3,
+      initialDelayMs: 1000,
+      maxDelayMs: 30000,
+      backoffMultiplier: 2,
+      retryableErrors: ['ECONNRESET'],
+    };
+
+    it('returns delay in range [0.5*base, base] for attempt 0', () => {
+      const delay = handler.calculateBackoffDelay(0, defaultConfig);
+      // base = 1000, jitter = [500, 1000]
+      expectInRange(delay, 500, 1000, 'Attempt 0 delay');
+    });
+
+    it('returns delay in range [0.5*base, base] for attempt 1', () => {
+      const delay = handler.calculateBackoffDelay(1, defaultConfig);
+      // base = 1000 * 2^1 = 2000, jitter = [1000, 2000]
+      expectInRange(delay, 1000, 2000, 'Attempt 1 delay');
+    });
+
+    it('returns delay in range [0.5*base, base] for attempt 2', () => {
+      const delay = handler.calculateBackoffDelay(2, defaultConfig);
+      // base = 1000 * 2^2 = 4000, jitter = [2000, 4000]
+      expectInRange(delay, 2000, 4000, 'Attempt 2 delay');
+    });
+
+    it('caps delay at maxDelayMs', () => {
+      const cappedConfig: RetryConfig = { ...defaultConfig, maxDelayMs: 5000 };
+      const delay = handler.calculateBackoffDelay(10, cappedConfig);
+      expect(delay).toBeLessThanOrEqual(5000);
+    });
+
+    it('respects Retry-After header for 429 errors', () => {
+      const rateLimitError = {
+        status: 429,
+        headers: { 'retry-after': '5' },
+      };
+      const delay = handler.calculateBackoffDelay(0, defaultConfig, rateLimitError);
+      // Retry-After: 5 seconds = 5000ms, with some tolerance
+      expectInRange(delay, 4500, 5500, 'Retry-After delay');
+    });
+  });
+
+  describe('validateContextWindow', () => {
+    it('returns valid for short prompts', () => {
+      const result = handler.validateContextWindow('openai', 'Short prompt');
+      expect(result.valid).toBe(true);
+    });
+
+    it('returns invalid for long prompts on small-context providers', () => {
+      const longText = Array(10000).join('word '); // ~9 chars * 10000 = ~90K chars
+      const result = handler.validateContextWindow('cerebras', longText);
+      expect(result.valid).toBe(false);
+      expect(result.reason).toBeTruthy();
+      expect(result.suggestedProvider).toBeTruthy();
+    });
+
+    it('suggests a provider with larger context when validation fails', () => {
+      const longText = Array(10000).join('word ');
+      const result = handler.validateContextWindow('cerebras', longText);
+      expect(result.suggestedProvider).toBeTruthy();
+      const suggestedLimit = PROVIDER_CONTEXT_LIMITS[result.suggestedProvider!] || 0;
+      const cerebrasLimit = PROVIDER_CONTEXT_LIMITS['cerebras'] || 0;
+      expect(suggestedLimit).toBeGreaterThan(cerebrasLimit);
+    });
+
+    it('returns valid for large prompts on large-context providers', () => {
+      const longText = Array(10000).join('word ');
+      const result = handler.validateContextWindow('minimax', longText);
+      expect(result.valid).toBe(true);
+    });
+
+    it('accepts explicit token count parameter', () => {
+      const result = handler.validateContextWindow('groq', 'test', 100);
+      expect(result.valid).toBe(true);
+    });
+  });
+
+  describe('stats tracking', () => {
+    it('starts with zeroed stats', () => {
+      const stats = handler.getStats('openai');
+      expect(stats.totalRequests).toBe(0);
+      expect(stats.successfulRequests).toBe(0);
+      expect(stats.failedRequests).toBe(0);
+      expect(stats.totalRetries).toBe(0);
+    });
+
+    it('getAllStats returns all providers', () => {
+      const allStats = handler.getAllStats();
+      expect(allStats['openai']).toBeDefined();
+      expect(allStats['deepseek']).toBeDefined();
+      expect(allStats['groq']).toBeDefined();
+      expect(Object.keys(allStats).length).toBeGreaterThan(5);
+    });
+
+    it('resetStats clears single provider', () => {
+      handler.resetStats('openai');
+      const stats = handler.getStats('openai');
+      expect(stats.totalRequests).toBe(0);
+    });
+
+    it('resetStats with no arg clears all', () => {
+      handler.resetStats();
+      const allStats = handler.getAllStats();
+      for (const stats of Object.values(allStats)) {
+        expect(stats.totalRequests).toBe(0);
+      }
+    });
+  });
+
+  describe('executeWithRetry', () => {
+    it('succeeds on first attempt', async () => {
+      const fn = vi.fn().mockResolvedValue('success');
+      const result = await handler.executeWithRetry('groq', fn);
+      expect(result).toBe('success');
+      expect(fn).toHaveBeenCalledTimes(1);
+    });
+
+    it('retries on transient errors then succeeds', async () => {
+      const fn = vi.fn()
+        .mockRejectedValueOnce({ code: 'ECONNRESET', message: 'Connection reset' })
+        .mockRejectedValueOnce({ code: 'ECONNRESET', message: 'Connection reset' })
+        .mockResolvedValue('success after retry');
+
+      const result = await handler.executeWithRetry('groq', fn, {
+        onRetry: vi.fn(),
+      });
+      expect(result).toBe('success after retry');
+      expect(fn).toHaveBeenCalledTimes(3);
+    });
+
+    it('fails after exhausting retries', async () => {
+      const fn = vi.fn().mockRejectedValue({ code: 'ECONNRESET', message: 'Persistent failure' });
+
+      await expect(
+        handler.executeWithRetry('groq', fn)
+      ).rejects.toThrow();
+      expect(fn).toHaveBeenCalledTimes(3); // initial + 2 retries (groq maxRetries=2)
+    });
+
+    it('does not retry on non-retryable errors', async () => {
+      const fn = vi.fn().mockRejectedValue({ status: 401, message: 'Unauthorized' });
+
+      await expect(
+        handler.executeWithRetry('groq', fn)
+      ).rejects.toThrow();
+      expect(fn).toHaveBeenCalledTimes(1);
+    });
+
+    it('calls onRetry callback on each retry', async () => {
+      const fn = vi.fn()
+        .mockRejectedValueOnce({ code: 'ECONNRESET' })
+        .mockResolvedValue('ok');
+      const onRetry = vi.fn();
+
+      await handler.executeWithRetry('groq', fn, { onRetry });
+      expect(onRetry).toHaveBeenCalledTimes(1);
+      expect(onRetry).toHaveBeenCalledWith(1, { code: 'ECONNRESET' }, expect.any(Number));
+    });
+
+    it('respects custom timeout', async () => {
+      const slowFn = vi.fn().mockImplementation(
+        () => new Promise(r => setTimeout(r, 100))
+      );
+
+      // Short timeout should reject
+      await expect(
+        handler.executeWithRetry('groq', slowFn, { timeout: 5 })
+      ).rejects.toThrow(/timed out/i);
+    });
+  });
+
+  describe('PROVIDER_CONTEXT_LIMITS', () => {
+    it('contains expected providers', () => {
+      expect(PROVIDER_CONTEXT_LIMITS['openai']).toBe(128000);
+      expect(PROVIDER_CONTEXT_LIMITS['anthropic']).toBe(200000);
+      expect(PROVIDER_CONTEXT_LIMITS['minimax']).toBe(1000000);
+      expect(PROVIDER_CONTEXT_LIMITS['groq']).toBe(32000);
+      expect(PROVIDER_CONTEXT_LIMITS['default']).toBe(8192);
+    });
+  });
+
+  describe('createRetryHandler', () => {
+    it('creates handler with custom configs', () => {
+      const custom = createRetryHandler({
+        testProv: {
+          timeout: 1000,
+          retry: { maxRetries: 1, initialDelayMs: 100, maxDelayMs: 5000, backoffMultiplier: 2 },
+          rateLimitRetries: 1,
+        },
+      });
+      const cfg = custom.getConfig('testProv');
+      expect(cfg.timeout).toBe(1000);
+    });
+  });
+});