adaptive-memory-multi-model-router 2.16.2 → 2.16.4

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "adaptive-memory-multi-model-router",
-  "version": "2.16.2",
-  "description": "A3M Router: Parallel LLM routing gateway with ensemble voting — 14% faster than OpenRouter, 92% cheaper | 80+ providers | Fixed: main entry point, TypeScript 5.8, blessed types | npm: 6K+/mo | PyPI: 700+/mo",
+  "version": "2.16.4",
+  "description": "Intelligent LLM routing gateway — 80+ providers, pheromone-trail failover, semantic cache. Saves 70-95% on AI costs. Drop-in OpenAI-compatible proxy. Self-hosted.",
   "main": "dist/index.js",
   "bin": {
     "a3m-router": "./dist/cli.js",
@@ -14,31 +14,40 @@
     "test:coverage": "npx vitest run --coverage",
     "test:watch": "npx vitest",
     "lint": "eslint src/",
-    "build": "tsc -p tsconfig.build.json"
+    "build": "tsc -p tsconfig.build.json && node -e \"const fs=require('fs');fs.mkdirSync('dist/routing/jev/weights',{recursive:true});try{fs.copyFileSync('src/routing/jev/weights/jev-router-weights.json','dist/routing/jev/weights/jev-router-weights.json')}catch(e){}\"",
+    "jev:distill": "node tools/distill.mjs",
+    "jev:train": "python3 tools/train_jev.py && python3 tools/calibrate_temp.py"
   },
   "keywords": [
+    "llm-router",
+    "openrouter-alternative",
+    "litellm-alternative",
+    "llm-failover",
+    "llm-cost-optimization",
+    "multi-provider-llm",
+    "llm-api-gateway",
+    "self-hosted-llm",
+    "anthropic-openai-gateway",
+    "llm-proxy",
+    "circuit-breaker-llm",
+    "semantic-cache-llm",
+    "agentic-ai-gateway",
+    "routellm",
+    "openrouter",
+    "litellm",
     "llm",
     "router",
     "gateway",
-    "openai",
+    "openai-proxy",
     "anthropic",
-    "multi-provider",
-    "cost-optimization",
-    "langchain",
-    "llamaindex",
-    "agentkit",
+    "deepseek",
+    "groq",
+    "mistral",
+    "gemini-api",
+    "ollama",
     "a3m",
-    "openrouter",
-    "litellm",
-    "proxy",
-    "api-gateway",
-    "artificial-intelligence",
-    "machine-learning",
-    "nlp",
-    "transformers",
-    "embeddings",
-    "rag",
-    "vector-search"
+    "langchain",
+    "agentkit"
   ],
   "repository": {
     "type": "git",

package/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "a3m-router"
-version = "2.16.2"
+version = "2.16.3"
 description = "Auto-selects cheapest capable LLM from 47+ providers — 70-95% cost savings"
 readme = "README.md"
 license = {text = "MIT"}

package/src/cli/setupWizard.ts

@@ -1,6 +1,8 @@
 /**
  * A3M Router Setup Wizard
  * Interactive configuration wizard with smart defaults
+ *
+ * Note: Excluded from tsconfig.build.json — type errors here do not affect the build.
  */
 
 const fs = require('fs');
@@ -221,9 +223,9 @@ function createInterface() {
   });
 }
 
-function question(rl, text) {
+function question(rl: readline.Interface, text: string): Promise<string> {
   return new Promise((resolve) => {
-    rl.question(text, (answer) => resolve(answer));
+    rl.question(text, (answer: string) => resolve(answer));
   });
 }
 

package/src/routing/jev/jevRouter.ts

@@ -0,0 +1,251 @@
+/**
+ * JevRouter — System One routing for A3M.
+ *
+ * Replaces the autoregressive/heuristic routing loop with a single-pass
+ * calibrated decision (Jev interface pattern):
+ *
+ *   prompt ──► OptionAttention engine ──► { provider choice, complexity score,
+ *                                            capability noul flags }
+ *
+ * The engine is distilled from A3M's own System 2 router (advancedRouter.ts):
+ * tools/distill.mjs runs routeQuery over a synthetic prompt corpus and
+ * tools/train_jev.py trains the ~25k-parameter option-attention weights.
+ *
+ * Confidence guard: if the top provider probability is below MIN_CONFIDENCE,
+ * we fall back to the full heuristic router (System 2) — Jev decides the easy
+ * 90%, the heavy router handles the hard 10%.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import {
+  routeQuery,
+  type RouteDecision,
+} from "../advancedRouter";
+import {
+  scoreOptions,
+  scoreScalar,
+  scoreNoul,
+  resetQueryCache,
+  type OptionAttentionWeights,
+} from "./optionAttention";
+import { remoteConfigured, remoteDecide } from "./remote";
+import type { JevResponse } from "./types";
+
+const WEIGHTS_PATHS = [
+  path.join(__dirname, "weights", "jev-router-weights.json"),           // dist copy
+  path.join(__dirname, "..", "..", "..", "src", "routing", "jev", "weights", "jev-router-weights.json"), // repo fallback
+];
+
+/** Below this top-probability, defer to the System 2 heuristic router. */
+export const MIN_CONFIDENCE = 0.22;
+
+let cachedWeights: OptionAttentionWeights | null | undefined;
+
+export function loadWeights(): OptionAttentionWeights | null {
+  if (cachedWeights !== undefined) return cachedWeights;
+  try {
+    for (const p of WEIGHTS_PATHS) {
+      if (fs.existsSync(p)) {
+        cachedWeights = JSON.parse(fs.readFileSync(p, "utf8")) as OptionAttentionWeights;
+        return cachedWeights;
+      }
+    }
+    cachedWeights = null;
+  } catch {
+    cachedWeights = null;
+  }
+  return cachedWeights;
+}
+
+/** For tests / hot-reload of retrained weights. */
+export function resetWeightsCache(): void {
+  cachedWeights = undefined;
+  resetQueryCache();
+}
+
+/**
+ * Build the option text for a model profile. Option text is what the engine
+ * embeds — providers unseen at training time are still scored through their
+ * strengths/tier description.
+ */
+export function optionTextForModel(model: string, profile: {
+  providerName?: string;
+  strengths?: string[];
+  quality_score?: number;
+  cost_per_1k_input?: number;
+  cost_per_1k_output?: number;
+  type?: string;
+}): string {
+  const strengths = (profile.strengths || []).slice(0, 6).join(",");
+  const cost =
+    profile.cost_per_1k_input != null
+      ? ((profile.cost_per_1k_input + (profile.cost_per_1k_output || 0)) / 2).toFixed(4)
+      : "?";
+  return `${model} provider:${profile.providerName || "?"} type:${profile.type || "api"} strengths:${strengths} quality:${profile.quality_score ?? "?"} cost:${cost}`;
+}
+
+/**
+ * Single-pass Jev routing decision. Returns a RouteDecision shaped exactly
+ * like advancedRouter.routeQuery so it is a drop-in replacement.
+ */
+export function jevRoute(
+  prompt: string,
+  available_models?: string[],
+  budget_multiplier: number = 1.0
+): RouteDecision & { jev?: JevResponse } {
+  // 1. Remote backend (openjev-sglang / typesafe) takes precedence if configured
+  if (remoteConfigured()) {
+    // remoteDecide is async; jevRoute is sync to match routeQuery's contract.
+    // Remote mode is exposed via jevRouteAsync below.
+  }
+
+  const weights = loadWeights();
+  if (!weights) return fallbackRoute(prompt, available_models, budget_multiplier, "no weights");
+
+  // 2. Candidate pool — mirror routeQuery's candidate selection
+  const profiles = getModelProfilesSafe();
+  const candidateNames = (available_models && available_models.length
+    ? available_models
+    : Object.keys(profiles)
+  ).filter((n) => profiles[n]);
+  if (candidateNames.length === 0)
+    return fallbackRoute(prompt, available_models, budget_multiplier, "no candidates");
+
+  const t0 = Date.now();
+  const options = candidateNames.map((m) => optionTextForModel(m, profiles[m]));
+  const { probs, top } = scoreOptions(prompt, options, weights);
+  const complexity = scoreScalar(prompt, weights);
+  const { value: needsCode, prob: codeProb } = scoreNoul(prompt, "requires code generation", weights);
+  const elapsed = Date.now() - t0;
+
+  const bestIdx = top[0]?.index ?? -1;
+  const bestProb = top[0]?.prob ?? 0;
+
+  const jev: JevResponse = {
+    answers: [
+      {
+        type: "choice",
+        key: "provider",
+        value: bestIdx >= 0 ? candidateNames[bestIdx] : "",
+        prob: bestProb,
+        top: top.map((t) => ({ option: candidateNames[t.index], prob: t.prob })),
+      },
+      { type: "score", key: "complexity", value: complexity },
+      { type: "noul", key: "needs_code", value: needsCode, prob: codeProb },
+    ],
+    elapsed_ms: elapsed,
+    backend: "local-weights",
+  };
+
+  // 3. Confidence guard → System 2 fallback
+  if (bestProb < MIN_CONFIDENCE || bestIdx < 0) {
+    const fb = fallbackRoute(prompt, available_models, budget_multiplier, `low confidence ${bestProb.toFixed(2)}`);
+    return { ...fb, jev };
+  }
+
+  const primary = candidateNames[bestIdx];
+  const profile = profiles[primary];
+  const fallbacks = top.slice(1, 3).map((t) => candidateNames[t.index]);
+
+  return {
+    primary_model: primary,
+    fallback_models: fallbacks,
+    confidence: bestProb,
+    reasoning: `jev single-pass (distilled): p=${bestProb.toFixed(3)} complexity=${complexity.toFixed(2)} needs_code=${needsCode}(${codeProb.toFixed(2)}) [${elapsed}ms]`,
+    estimated_cost:
+      ((profile.cost_per_1k_input || 0) + (profile.cost_per_1k_output || 0)) / 2,
+    estimated_latency_ms: profile.latency_ms || 500,
+    provider_type: profile.type,
+    jev,
+  };
+}
+
+/** Async variant with remote backend support. */
+export async function jevRouteAsync(
+  prompt: string,
+  available_models?: string[],
+  budget_multiplier: number = 1.0
+): Promise<RouteDecision & { jev?: JevResponse }> {
+  if (remoteConfigured()) {
+    const profiles = getModelProfilesSafe();
+    const candidates = (available_models && available_models.length
+      ? available_models
+      : Object.keys(profiles)
+    ).filter((n) => profiles[n]);
+    const remote = await remoteDecide({
+      state: prompt,
+      questions: [
+        {
+          type: "choice",
+          key: "provider",
+          description: "best model for this query",
+          options: candidates.map((m) => optionTextForModel(m, profiles[m])),
+        },
+      ],
+    });
+    if (remote) {
+      const choice = remote.answers[0];
+      if (choice && choice.type === "choice") {
+        // map option text back to model name
+        const idx = candidates.findIndex((m) =>
+          choice.value.startsWith(m + " ")
+        );
+        const primary = idx >= 0 ? candidates[idx] : candidates.find((c) => choice.value.includes(c));
+        if (primary) {
+          const profile = profiles[primary];
+          return {
+            primary_model: primary,
+            fallback_models: [],
+            confidence: choice.prob,
+            reasoning: `jev remote (${remote.backend}) [${remote.elapsed_ms}ms]`,
+            estimated_cost:
+              ((profile.cost_per_1k_input || 0) + (profile.cost_per_1k_output || 0)) / 2,
+            estimated_latency_ms: profile.latency_ms || 500,
+            provider_type: profile.type,
+            jev: remote,
+          };
+        }
+      }
+    }
+  }
+  return jevRoute(prompt, available_models, budget_multiplier);
+}
+
+// ---------------------------------------------------------------------------
+// internals
+// ---------------------------------------------------------------------------
+
+import { MODEL_PROFILES } from "../advancedRouter";
+
+function getModelProfilesSafe(): Record<string, {
+  providerName: string;
+  cost_per_1k_input: number;
+  cost_per_1k_output: number;
+  latency_ms: number;
+  quality_score: number;
+  strengths: string[];
+  type: string;
+}> {
+  // MODEL_PROFILES is populated by the first routeQuery call (profile cache).
+  if (Object.keys(MODEL_PROFILES).length === 0) routeQuery("warmup");
+  return MODEL_PROFILES as Record<string, never>;
+}
+
+function fallbackRoute(
+  prompt: string,
+  available_models: string[] | undefined,
+  budget_multiplier: number,
+  why: string
+): RouteDecision & { jev?: JevResponse } {
+  const d = routeQuery(prompt, available_models, budget_multiplier);
+  return {
+    ...d,
+    reasoning: `system2-fallback (${why}): ${d.reasoning}`,
+    jev: {
+      answers: [],
+      elapsed_ms: 0,
+      backend: "fallback-heuristic",
+    },
+  };
+}

package/src/routing/jev/optionAttention.ts

@@ -0,0 +1,186 @@
+/**
+ * OptionAttention — a pure-TypeScript System One decision engine.
+ *
+ * Port of the architecture popularized by vinnylarouge/jevlike (835★):
+ *
+ *   Each option becomes a query vector. The query assigns attention weights
+ *   to the context tokens; those weights produce one attended context vector
+ *   per option. A shared dot product turns each (option, context) pair into
+ *   one score. A softmax across options yields calibrated probabilities —
+ *   all in a single forward pass, no autoregressive generation.
+ *
+ * Why this architecture for A3M:
+ *   - Dynamic option sets: providers come and go; unseen options are scored
+ *     through their text (name + strengths + tier), so the engine generalizes
+ *     to providers it was never trained on.
+ *   - Zero dependencies: byte-level embeddings, ~25k parameters, weights ship
+ *     as JSON inside the npm package.
+ *   - CPU-fast: single matmul chain, sub-millisecond for 100+ options.
+ */
+
+export interface OptionAttentionWeights {
+  /** Hashed trigram embedding table: VOCAB × D. */
+  emb: number[][];
+  /** Option projection: D × D. */
+  wq: number[][];
+  bq: number[];
+  /** Shared scoring vector: D. */
+  w: number[];
+  b: number[];
+  /** Score head (context pooled → sigmoid): D + 2. */
+  ws: number[];
+  bs: number[];
+  /** Temperature for calibrated softmax. */
+  temperature: number;
+  dim: number;
+}
+
+/** Number of trigram hash buckets (must match tools/train_jev.py). */
+export const VOCAB = 2048;
+
+/**
+ * FNV-1a 32-bit hash of a character trigram → bucket id.
+ * Must match the Python trainer exactly.
+ */
+function trigramHash(b0: number, b1: number, b2: number): number {
+  let h = 0x811c9dc5;
+  h = Math.imul(h ^ (b0 & 0xff), 0x01000193) >>> 0;
+  h = Math.imul(h ^ (b1 & 0xff), 0x01000193) >>> 0;
+  h = Math.imul(h ^ (b2 & 0xff), 0x01000193) >>> 0;
+  return h % VOCAB;
+}
+
+/**
+ * Hashed trigram tokenization — character trigrams give much sharper text
+ * discrimination than raw bytes ("chat" vs "code" produce disjoint buckets).
+ */
+export function tokenize(text: string): number[] {
+  const s = " " + text + " ";
+  const out: number[] = [];
+  const cap = 384;
+  for (let i = 0; i + 2 < s.length && out.length < cap; i++) {
+    out.push(
+      trigramHash(
+        s.charCodeAt(i),
+        s.charCodeAt(i + 1),
+        s.charCodeAt(i + 2)
+      )
+    );
+  }
+  if (out.length === 0) out.push(trigramHash(32, 32, 32));
+  return out;
+}
+
+function meanPool(rows: number[][], weights: OptionAttentionWeights): number[] {
+  const D = weights.dim;
+  const out = new Array<number>(D).fill(0);
+  if (rows.length === 0) return out;
+  for (const r of rows) for (let d = 0; d < D; d++) out[d] += r[d];
+  for (let d = 0; d < D; d++) out[d] /= rows.length;
+  return out;
+}
+
+function embed(tokens: number[], weights: OptionAttentionWeights): number[][] {
+  return tokens.map((t) => weights.emb[t % VOCAB]);
+}
+
+/** Cache: option text → query vector. Option texts are static per provider
+ * profile, so q_i never changes between calls — skip the W_q matmul. */
+const queryCache = new Map<string, number[]>();
+
+export function resetQueryCache(): void {
+  queryCache.clear();
+}
+
+/** Option text → query vector q_i = W_q · meanPool(bytes) + b_q */
+function optionQuery(optionText: string, weights: OptionAttentionWeights): number[] {
+  const hit = queryCache.get(optionText);
+  if (hit) return hit;
+  const D = weights.dim;
+  const h = meanPool(embed(tokenize(optionText), weights), weights);
+  const q = new Array<number>(D).fill(0);
+  for (let i = 0; i < D; i++) {
+    let s = weights.bq[i];
+    for (let j = 0; j < D; j++) s += weights.wq[i][j] * h[j];
+    q[i] = Math.tanh(s);
+  }
+  if (queryCache.size > 512) queryCache.clear();
+  queryCache.set(optionText, q);
+  return q;
+}
+
+export interface ChoiceResult {
+  probs: number[];
+  top: Array<{ index: number; prob: number }>;
+}
+
+/**
+ * Score N options against a context in one pass.
+ * Attention: a_ij = softmax_j( q_i · E_j / √D ), c_i = Σ_j a_ij E_j
+ * Score:     s_i   = w · (q_i ⊙ c_i) + b,  probs = softmax(s / T)
+ */
+export function scoreOptions(
+  context: string,
+  options: string[],
+  weights: OptionAttentionWeights
+): ChoiceResult {
+  const D = weights.dim;
+  const E = embed(tokenize(context), weights);
+  const scale = 1 / Math.sqrt(D);
+
+  const probs: number[] = [];
+  for (const opt of options) {
+    const q = optionQuery(opt, weights);
+    // attention over context tokens
+    const attn = E.map((e) => {
+      let dot = 0;
+      for (let d = 0; d < D; d++) dot += q[d] * e[d];
+      return dot * scale;
+    });
+    const maxA = Math.max(...attn, 0);
+    const exps = attn.map((a) => Math.exp(a - maxA));
+    const sumA = exps.reduce((a, b) => a + b, 0) || 1;
+    // attended context vector c_i
+    const c = new Array<number>(D).fill(0);
+    for (let j = 0; j < E.length; j++) {
+      const a = exps[j] / sumA;
+      if (a < 1e-6) continue;
+      for (let d = 0; d < D; d++) c[d] += a * E[j][d];
+    }
+    // score_i = w · (q ⊙ c) + b
+    let s = weights.b[0];
+    for (let d = 0; d < D; d++) s += weights.w[d] * q[d] * c[d];
+    probs.push(s / weights.temperature);
+  }
+
+  const maxS = Math.max(...probs);
+  const expP = probs.map((p) => Math.exp(p - maxS));
+  const sumP = expP.reduce((a, b) => a + b, 0) || 1;
+  const final = expP.map((e) => e / sumP);
+
+  const top = final
+    .map((prob, index) => ({ index, prob }))
+    .sort((a, b) => b.prob - a.prob)
+    .slice(0, 5);
+  return { probs: final, top };
+}
+
+/** Context → scalar in [0,1] via sigmoid(w_s · meanPool + b_s). */
+export function scoreScalar(context: string, weights: OptionAttentionWeights): number {
+  const D = weights.dim;
+  const p = meanPool(embed(tokenize(context), weights), weights);
+  let s = weights.bs[0];
+  for (let d = 0; d < D; d++) s += weights.ws[d] * p[d];
+  return 1 / (1 + Math.exp(-s));
+}
+
+/** Boolean decision with calibrated probability (2-way choice). */
+export function scoreNoul(
+  context: string,
+  questionText: string,
+  weights: OptionAttentionWeights
+): { value: boolean; prob: number } {
+  const r = scoreOptions(context, [questionText + " — yes", questionText + " — no"], weights);
+  const yesProb = r.probs[0];
+  return { value: yesProb >= 0.5, prob: Math.max(yesProb, 1 - yesProb) };
+}

package/src/routing/jev/remote.ts

@@ -0,0 +1,50 @@
+/**
+ * Remote backend for Jev-compatible endpoints.
+ *
+ * Works with:
+ *   - api.typesafe.ai (commercial Jev, "System One")
+ *   - ekzhang/openjev-sglang (open, Jev HTTP API on SGLang; self-host on Modal)
+ *
+ * Configure via env:
+ *   A3M_JEV_URL      — endpoint base (e.g. https://your-server.modal.direct)
+ *   A3M_JEV_API_KEY  — optional bearer token
+ */
+
+import type { JevRequest, JevResponse } from "./types";
+
+export function remoteConfigured(): boolean {
+  return Boolean(process.env.A3M_JEV_URL);
+}
+
+export async function remoteDecide(req: JevRequest, timeoutMs = 2000): Promise<JevResponse | null> {
+  const base = process.env.A3M_JEV_URL;
+  if (!base) return null;
+  const key = process.env.A3M_JEV_API_KEY || "";
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  const t0 = Date.now();
+  try {
+    const res = await fetch(`${base.replace(/\/$/, "")}/v1/decide`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...(key ? { authorization: `Bearer ${key}` } : {}),
+      },
+      body: JSON.stringify(req),
+      signal: controller.signal,
+    });
+    if (!res.ok) return null;
+    const data = (await res.json()) as { answers?: unknown };
+    if (!Array.isArray(data.answers)) return null;
+    return {
+      answers: data.answers as JevResponse["answers"],
+      elapsed_ms: Date.now() - t0,
+      backend: "remote",
+    };
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/routing/jev/types.ts

@@ -0,0 +1,79 @@
+/**
+ * Jev contract — open interface pattern for System One decision models.
+ *
+ * Follows the interface reproduced by the open community:
+ *   - TheoLeeCJ/SemIf (formerly OpenJev, 1.4k★) — typed option probabilities
+ *   - ekzhang/openjev-sglang — faithful TypeSafe/Jev HTTP API server
+ *   - vinnylarouge/jevlike — trainable context+options→probabilities engine
+ *
+ * Three question types (per TypeSafe's Jev):
+ *   - choice: categorical pick over N options      → calibrated probs
+ *   - score:  numeric value in [0,1]               → sigmoid output
+ *   - noul:   boolean with calibrated probability  → 2-way choice
+ *
+ * A3M usage: one request answers provider routing (choice),
+ * query complexity (score), and capability flags (noul) in a single pass.
+ */
+
+export type JevQuestionType = "choice" | "score" | "noul";
+
+export interface JevChoiceQuestion {
+  type: "choice";
+  key: string;
+  description?: string;
+  /** Dynamic option texts — engine scores unseen options via their text. */
+  options: string[];
+}
+
+export interface JevScoreQuestion {
+  type: "score";
+  key: string;
+  description?: string;
+  min?: number;
+  max?: number;
+}
+
+export interface JevNoulQuestion {
+  type: "noul";
+  key: string;
+  description?: string;
+}
+
+export type JevQuestion = JevChoiceQuestion | JevScoreQuestion | JevNoulQuestion;
+
+export interface JevRequest {
+  /** Unstructured state — the user prompt / conversation context. */
+  state: string;
+  questions: JevQuestion[];
+}
+
+export interface JevChoiceAnswer {
+  type: "choice";
+  key: string;
+  value: string;
+  prob: number;
+  top: Array<{ option: string; prob: number }>;
+}
+
+export interface JevScoreAnswer {
+  type: "score";
+  key: string;
+  value: number;
+}
+
+export interface JevNoulAnswer {
+  type: "noul";
+  key: string;
+  value: boolean;
+  prob: number;
+}
+
+export type JevAnswer = JevChoiceAnswer | JevScoreAnswer | JevNoulAnswer;
+
+export interface JevResponse {
+  answers: JevAnswer[];
+  /** Milliseconds spent in the decision engine (excluding fallbacks). */
+  elapsed_ms: number;
+  /** Which backend produced the answers. */
+  backend: "local-weights" | "remote" | "fallback-heuristic";
+}