adaptive-memory-multi-model-router 2.14.45 → 2.14.47

package/src/skills/__tests__/skill_manager.test.ts

@@ -1,328 +0,0 @@
-"""
-Tests for SkillManager and TMLEnhancedAgent
-"""
-
-import { describe, it, expect, beforeEach, jest } from '@jest/globals';
-import { SkillManager, Skill } from '../skill_manager';
-import { TMLEnhancedAgent } from '../../agents/skill_enhanced_agent';
-import * as fs from 'fs';
-import * as path from 'path';
-
-// Mock fs operations
-jest.mock('fs');
-
-describe('SkillManager', () => {
-  let skillManager: SkillManager;
-  const mockSkillsDir = '/mock/skills';
-
-  beforeEach(() => {
-    jest.clearAllMocks();
-    skillManager = new SkillManager(mockSkillsDir);
-  });
-
-  describe('initialization', () => {
-    it('should create skill manager instance', () => {
-      expect(skillManager).toBeInstanceOf(SkillManager);
-      expect(skillManager.skills_dir).toBeDefined();
-    });
-
-    it('should have empty skills initially if directory does not exist', () => {
-      expect(skillManager.list_skills()).toEqual([]);
-    });
-  });
-
-  describe('load_skills_metadata', () => {
-    it('should load skill metadata from SKILL.md files', () => {
-      // Mock directory listing and file reading
-      const mockSkillMD = `---
-name: "Test Skill"
-description: "A test skill for testing"
----
-# Test Skill Content`;
-
-      jest.spyOn(fs, 'existsSync').mockReturnValue(true);
-      jest.spyOn(Path.prototype, 'isDirectory').mockReturnValue(true);
-      jest.spyOn(Path.prototype, 'iterdir').mockReturnValue([
-        { name: 'SKILL.md', isFile: () => true }
-      ] as any);
-      jest.spyOn(fs, 'readFileSync').mockReturnValue(mockSkillMD);
-
-      skillManager.reload_skills();
-
-      expect(skillManager.list_skills()).toContain('Test Skill');
-    });
-
-    it('should skip directories without SKILL.md', () => {
-      jest.spyOn(Path.prototype, 'existsSync').mockReturnValue(false);
-
-      skillManager.reload_skills();
-
-      expect(skillManager.list_skills()).toEqual([]);
-    });
-  });
-
-  describe('get_relevant_skills', () => {
-    beforeEach(() => {
-      // Add mock skills
-      skillManager.skills['React Development'] = new Skill(
-        'React Development',
-        'Best practices for React components',
-        Path('/mock/react'),
-        {}
-      );
-
-      skillManager.skills['Node.js API'] = new Skill(
-        'Node.js API',
-        'Building backend APIs with Node.js and Express',
-        Path('/mock/nodejs'),
-        {}
-      );
-
-      skillManager.skills['Python Django'] = new Skill(
-        'Python Django',
-        'Django web framework for Python',
-        Path('/mock/python'),
-        {}
-      );
-    });
-
-    it('should find skills with keyword matching', () => {
-      const relevant = skillManager.get_relevant_skills(
-        'Build a React component for user login',
-        2
-      );
-
-      expect(relevant).toContain('React Development');
-    });
-
-    it('should return skills ordered by relevance', () => {
-      const relevant = skillManager.get_relevant_skills(
-        'Create a React component with Node.js backend',
-        3
-      );
-
-      // React should come first (exact match)
-      expect(relevant[0]).toBe('React Development');
-      expect(relevant).toContain('Node.js API');
-    });
-
-    it('should respect threshold parameter', () => {
-      const relevant = skillManager.get_relevant_skills(
-        'Build a Go microservice',
-        2,
-        0.5  // Higher threshold
-      );
-
-      // Should return fewer or no skills due to high threshold
-      expect(relevant.length).toBeLessThanOrEqual(2);
-    });
-  });
-
-  describe('load_skill', () => {
-    it('should load full skill content on first call', () => {
-      const mockSkill = skillManager.skills['React Development'];
-      mockSkill.content = null;
-
-      const mockContent = '# React Development\n\nBest practices...';
-
-      jest.spyOn(fs, 'readFileSync').mockReturnValue(
-        `---\nname: "React Development"\ndescription: "Best practices"\n---\n${mockContent}`
-      );
-
-      const loaded = skillManager.load_skill('React Development');
-
-      expect(loaded.content).toBe(mockContent);
-      expect(loaded.loaded_at).toBeDefined();
-    });
-
-    it('should return cached content on subsequent calls', () => {
-      const mockSkill = skillManager.skills['React Development'];
-      mockSkill.content = 'Cached content';
-
-      const loaded1 = skillManager.load_skill('React Development');
-      const loaded2 = skillManager.load_skill('React Development');
-
-      expect(loaded1).toBe(loaded2);
-      expect(loaded1.content).toBe('Cached content');
-    });
-
-    it('should throw error for non-existent skill', () => {
-      expect(() => {
-        skillManager.load_skill('Non-existent Skill');
-      }).toThrow("Skill 'Non-existent Skill' not found");
-    });
-  });
-
-  describe('validate_skill', () => {
-    it('should return validation results for existing skill', () => {
-      const mockSkill = skillManager.skills['React Development'];
-
-      jest.spyOn(skillManager, 'list_additional_files').mockReturnValue([]);
-
-      const validation = skillManager.validate_skill('React Development');
-
-      expect(validation).toHaveProperty('exists', true);
-      expect(validation).toHaveProperty('has_name', true);
-      expect(validation).toHaveProperty('has_description', true);
-    });
-
-    it('should return all false for non-existent skill', () => {
-      const validation = skillManager.validate_skill('Non-existent');
-
-      expect(validation.exists).toBe(false);
-      expect(validation.has_skill_md).toBe(false);
-    });
-  });
-});
-
-describe('TMLEnhancedAgent', () => {
-  let agent: TMLEnhancedAgent;
-
-  beforeEach(() => {
-    agent = new TMLEnhancedAgent(
-      'frontend-agent',
-      'anthropic',
-      'claude-sonnet-4',
-      'mock-skills',
-      ['React Frontend Development', 'TypeScript Best Practices']
-    );
-  });
-
-  describe('initialization', () => {
-    it('should create agent with configuration', () => {
-      expect(agent.agent_id).toBe('frontend-agent');
-      expect(agent.provider).toBe('anthropic');
-      expect(agent.model).toBe('claude-sonnet-4');
-    });
-
-    it('should initialize with assigned skills', () => {
-      expect(agent.assigned_skills).toContain('React Frontend Development');
-      expect(agent.assigned_skills).toContain('TypeScript Best Practices');
-    });
-  });
-
-  describe('execute_task', () => {
-    it('should execute task with relevant skills', async () => {
-      const task = {
-        description: 'Build a React login form component',
-        context: 'Must include email and password fields',
-        requirements: 'Use TypeScript and Material-UI'
-      };
-
-      // Mock skill loading
-      jest.spyOn(agent, '_get_relevant_skills').mockReturnValue([]);
-
-      // Mock LLM call
-      jest.spyOn(agent, '_execute_llm_call').mockReturnValue({
-        success: true,
-        output: 'React component code...',
-        tokens_used: 150,
-        cost: 0.015,
-        execution_time: 3.2
-      });
-
-      const result = agent.execute_task(task);
-
-      expect(result.success).toBe(true);
-      expect(result.output).toBeDefined();
-    });
-
-    it('should remember successful patterns', () => {
-      const task = { description: 'Test task' };
-
-      jest.spyOn(agent, '_get_relevant_skills').mockReturnValue([]);
-      jest.spyOn(agent, '_execute_llm_call').mockReturnValue({
-        success: true,
-        output: 'Success'
-      });
-
-      jest.spyOn(agent, '_remember_success_pattern').mockImplementation(() => {});
-
-      agent.execute_task(task);
-
-      expect(agent._remember_success_pattern).toHaveBeenCalled();
-    });
-  });
-
-  describe('skill management', () => {
-    it('should add skill to agent', () => {
-      agent.add_skill('Jest Testing');
-
-      expect(agent.assigned_skills).toContain('Jest Testing');
-    });
-
-    it('should remove skill from agent', () => {
-      agent.remove_skill('TypeScript Best Practices');
-
-      expect(agent.assigned_skills).not.toContain('TypeScript Best Practices');
-    });
-
-    it('should list all available skills', () => {
-      jest.spyOn(agent.skill_manager, 'list_skills').mockReturnValue([
-        'Skill 1',
-        'Skill 2',
-        'Skill 3'
-      ]);
-
-      const skills = agent.list_available_skills();
-
-      expect(skills).toHaveLength(3);
-    });
-  });
-
-  describe('serialization', () => {
-    it('should convert to dictionary', () => {
-      const dict = agent.to_dict();
-
-      expect(dict).toHaveProperty('agent_id', 'frontend-agent');
-      expect(dict).toHaveProperty('provider', 'anthropic');
-      expect(dict).toHaveProperty('model', 'claude-sonnet-4');
-      expect(dict).toHaveProperty('assigned_skills');
-      expect(dict).toHaveProperty('available_skills');
-    });
-  });
-});
-
-describe('TMLEnhancedAgentFactory', () => {
-  describe('create_from_config', () => {
-    it('should create agent from config', () => {
-      const config = {
-        id: 'test-agent',
-        provider: 'openai',
-        model: 'gpt-4-turbo',
-        skills_dir: 'test-skills',
-        skills: ['Test Skill']
-      };
-
-      const agent = TMLEnhancedAgentFactory.create_from_config(config);
-
-      expect(agent).toBeInstanceOf(TMLEnhancedAgent);
-      expect(agent.agent_id).toBe('test-agent');
-    });
-  });
-
-  describe('create_multiple_from_config', () => {
-    it('should create multiple agents from config list', () => {
-      const configs = [
-        {
-          id: 'agent-1',
-          provider: 'anthropic',
-          model: 'claude-sonnet-4',
-          skills: ['Skill A']
-        },
-        {
-          id: 'agent-2',
-          provider: 'openai',
-          model: 'gpt-4-turbo',
-          skills: ['Skill B']
-        }
-      ];
-
-      const agents = TMLEnhancedAgentFactory.create_multiple_from_config(configs);
-
-      expect(agents).toHaveLength(2);
-      expect(agents[0].agent_id).toBe('agent-1');
-      expect(agents[1].agent_id).toBe('agent-2');
-    });
-  });
-});

package/submissions/benchmarks/ALL_PLATFORMS_SUBMISSION.md

@@ -1,94 +0,0 @@
-# A3M Router - Comprehensive Benchmark Submission
-
-## v2.14.23 - Research-Backed Routing
-
-**NPM:** `npm install adaptive-memory-multi-model-router@2.14.23`
-**GitHub:** https://github.com/Das-rebel/a3m-router
-
-### Key Metrics
-
-| Metric | Value |
-|--------|-------|
-| **Exact Tier Accuracy** | 67% (target >50%) |
-| **±1 Tier Accuracy** | 96% (target >85%) |
-| **Cost Savings** | 62.9% vs all-premium |
-| **Over-routing** | 6.5% (very low) |
-| **Under-routing** | 26.5% |
-| **Premium Accuracy** | 57.5% (up from 0%) |
-| **Free Tier Accuracy** | 96% |
-| **RouterArena Score** | 70.32 (v1 evaluated) |
-| **Robustness Score** | 0.8524 (highest) |
-
----
-
-## Benchmark Coverage
-
-### 1. RouterArena
-- **Status:** PR #120 open, awaiting re-evaluation
-- **Score:** 70.32 (v1), 69.12 (v3)
-- **Robustness:** 0.8524 (highest)
-- **Request:** Re-evaluation with v2.14.23
-
-### 2. RouterEval
-- **Status:** ✅ PR #4 merged
-- **Added:** AbstractRouter with cosine similarity + weighted ensemble voting
-
-### 3. LLMRouterBench (ACL'26)
-- **Status:** Not yet submitted
-- **Stars:** 63
-- **Submission:** Needed
-
-### 4. routerbench
-- **Status:** Not yet submitted
-- **Stars:** 165
-- **Submission:** Needed
-
-### 5. MMR-Bench (Multimodal)
-- **Status:** Not yet submitted
-- **Focus:** Multimodal LLM routing
-- **Submission:** Needed for multimodal claim
-
----
-
-## Research-Backed Improvements (v2.14.23)
-
-### 5 Complexity Signals
-1. **Jargon Density (+15%)** - professional terminology
-2. **Task Formality (+10%)** - protocol, audit, brief
-3. **Depth Markers (+8%)** - comprehensive, expert-level
-4. **Stakes Language (+5%)** - critical, liability, regulatory
-5. **Multi-Step Structure (+5%)** - sequential reasoning
-
-### Mathematical Research Implemented
-- **Thompson Sampling** - Bayesian exploration/exploitation
-- **UCB1 Bandits** - Optimal exploration bounds
-- **Pareto Optimization** - Multi-objective routing
-- **Robust Optimization** - Hard constraints for robustness
-
-### Memory Capabilities
-- **Adaptive Memory** - Learns from routing history
-- **EMA Updates** - No retraining needed
-- **MemoryTree** - Hierarchical context storage
-
----
-
-## Features Tested
-
-| Feature | Status |
-|---------|--------|
-| Cost optimization | ✅ 62.9% savings |
-| Robustness | ✅ 0.8524 (highest) |
-| Multimodal | ⚠️ Not benchmarked yet |
-| Memory | ✅ MemoryTree implemented |
-| Parallel ensemble | ✅ Implemented |
-| Fallback chains | ✅ Circuit breaker |
-
----
-
-## Submission Package
-
-```bash
-npm install adaptive-memory-multi-model-router@2.14.23
-```
-
-All research documented in: `research/*.md`

package/submissions/benchmarks/LLMROUTERBENCH_SUBMISSION.md

@@ -1,121 +0,0 @@
-# LLMRouterBench Submission - A3M Router v2.14.41
-
-## ACL'26 Benchmark Submission
-
-**Repository:** https://github.com/ynulihao/LLMRouterBench
-**Stars:** 63 | **Status:** Active research benchmark (ACL 2026 accepted)
-**Submission Date:** 2026-06-06
-**Version:** 2.14.41 (Enhanced Shapley + Multi-Round Dialog)
-
----
-
-## Summary
-
-A3M Router (Adaptive Memory Multi-Model Router) is an open-source LLM gateway featuring **parallel multi-LLM execution with enhanced game-theoretic ensemble voting** - a fundamentally different approach from sequential fallback routers.
-
-**NPM:** `npm install adaptive-memory-multi-model-router@2.14.41`
-
----
-
-## Key Differentiators
-
-### 1. Enhanced Game-Theoretic Credit Assignment (NEW in v2.14.41)
-
-**Ethnocentrism-based Loyalty Matrix:**
-- Models develop trust bonds through successful collaborations
-- Math: L[i,j] = EMA of historical success(i with j) with decay rate
-- Increases marginal contribution of trusted partners
-
-**Handicap Principle (Zahavi, 1975):**
-- Honest signals require costly investment
-- Models spending more tokens despite correctness = reliable
-- Math: H[i] = cost_i × reliability_i (handicap bonus)
-
-**Combined Credit Formula:**
-```
-φ_i* = 0.5·Shapley + 0.3·Ethnocentrism + 0.2·Handicap
-```
-
-### 2. Multi-Round Dialog Optimization (NEW in v2.14.41)
-
-- Track conversation context over multiple turns
-- Topic tracking with dynamic complexity scoring
-- Model performance history per topic
-- Optimized context building for next query
-- Routing hints based on dialog state
-
-### 3. Parallel Multi-LLM Execution (Core USP)
-
-Unlike all other routers (litellm, one-api, LibreChat, gpt-researcher) which use **sequential fallback** (try A → B → C), A3M executes **multiple providers in parallel** and merges results via confidence-weighted voting.
-
----
-
-## Performance Metrics
-
-| Metric | Value | Notes |
-|--------|-------|-------|
-| Exact Tier Accuracy | 67% | +2pp improvement |
-| ±1 Tier Accuracy | 96% | Top-tier routing |
-| Cost Savings | 62.9% | vs all-premium baseline |
-| Robustness Score | 0.8524 | Highest among submissions |
-| Premium Accuracy | 57.5% | +5.5pp improvement |
-| Routing Latency | ~6ms | 40% faster with Quickselect |
-
----
-
-## Algorithm Improvements (v2.14.41)
-
-### 1. Enhanced Shapley Value Calculator
-- Exact calculation for n≤6 models
-- Monte Carlo approximation for larger ensembles
-- Loyalty matrix for ethnocentrism adjustment
-- Handicap calculator for costly signaling
-
-### 2. Multi-Round Dialog Optimizer
-- Conversation state management
-- Topic extraction and tracking
-- Model performance history per topic
-- Adaptive complexity scoring
-- Context-optimized query building
-
-### 3. Wolfram Ruliology Implementation
-- Provider strategies (aggressive/balanced/conservative)
-- Query risk profiles (HIGH/MEDIUM/LOW)
-- Risk-profile to strategy matching
-- Game-theoretic routing
-
----
-
-## How to Evaluate
-
-```bash
-# Install A3M Router
-npm install adaptive-memory-multi-model-router@2.14.41
-
-# Run ensemble with enhanced Shapley
-const { EnsembleOrchestrator } = require('adaptive-memory-multi-model-router');
-const ensemble = new EnsembleOrchestrator(router);
-const result = await ensemble.executeEnsemble(
-  query, 
-  ['deepseek', 'mistral', 'nvidia'],
-  'shapley'  // Use enhanced Shapley voting
-);
-```
-
----
-
-## Submission Checklist
-
-- [x] NPM package published (v2.14.41)
-- [x] GitHub repository with full source
-- [x] Research-backed algorithm description
-- [x] Performance metrics provided
-- [x]ACL 2026 compatible submission format
-
----
-
-## Contact
-
-- **Author:** Subhajit Das
-- **GitHub:** https://github.com/Das-rebel/adaptive-memory-multi-model-router
-- **NPM:** https://www.npmjs.com/package/adaptive-memory-multi-model-router

package/submissions/benchmarks/MMRBENCH_SUBMISSION.md

@@ -1,94 +0,0 @@
-# MMR-Bench Submission - A3M Router v2.14.41 (Multimodal)
-
-## Multimodal LLM Routing Benchmark
-
-**Repository:** https://github.com/Hunter-Wrynn/MMR-Bench
-**Stars:** 4 | **Focus:** Multimodal LLM routing with image + text understanding
-**Submission Date:** 2026-06-06
-**Version:** 2.14.41 (Multimodal-Ready with Enhanced Shapley)
-
----
-
-## Summary
-
-A3M Router extends its adaptive routing capabilities to **multimodal LLM routing**, supporting providers like GPT-4V, Claude Vision, Gemini Pro Vision, and other vision-capable models.
-
-**NPM:** `npm install adaptive-memory-multi-model-router@2.14.41`
-
----
-
-## Multimodal Capabilities
-
-### Supported Providers
-- **GPT-4V** (OpenAI) - Vision understanding
-- **Claude 3 Sonnet** (Anthropic) - Vision + reasoning
-- **Gemini Pro Vision** (Google) - Multimodal
-- **Llava** (Local/Ollama) - Open-source vision
-- **BakLLaVA** (Local/Ollama) - Open-source vision
-
-### Multimodal Routing Strategy
-
-```typescript
-const router = new A3MRouter({
-  multimodal: {
-    enabled: true,
-    imageAnalysis: 'auto',      // Detect if image analysis needed
-    maxImages: 10,             // Max images per request
-    preferredProviders: ['claude-vision', 'gpt-4v', 'gemini-pro-vision']
-  }
-});
-```
-
----
-
-## Key Enhancements in v2.14.41
-
-### Enhanced Shapley for Multimodal
-- Ethnocentrism tracking for vision model collaborations
-- Handicap principle for costly vision signals
-- Combined credit: φ_i* = 0.5·Shapley + 0.3·Ethnocentrism + 0.2·Handicap
-
-### Multi-Round Dialog for Vision
-- Track image context across conversation turns
-- Topic-based model selection for vision tasks
-- Adaptive complexity for multimodal queries
-
----
-
-## How to Evaluate
-
-```bash
-npm install adaptive-memory-multi-model-router@2.14.41
-```
-
-```typescript
-import { createA3MRouter } from 'adaptive-memory-multi-model-router';
-
-const router = createA3MRouter();
-
-// Vision query routing
-const result = await router.route({
-  query: 'What is in this image?',
-  imageUrl: 'https://example.com/image.jpg'
-});
-```
-
----
-
-## Performance on Multimodal Tasks
-
-| Task Type | Preferred Model | Confidence |
-|-----------|-----------------|------------|
-| Image understanding | Claude Vision | 94% |
-| Chart analysis | GPT-4V | 91% |
-| Diagram parsing | Gemini Pro | 88% |
-| Visual QA | Llava | 82% |
-
----
-
-## Submission Checklist
-
-- [x] NPM package published (v2.14.41)
-- [x] Multimodal provider support documented
-- [x] Vision routing strategy explained
-- [x] Performance metrics provided