jev-layer 0.1.0 → 0.2.0

package/src/model-route-metrics.mjs

@@ -0,0 +1,89 @@
+/**
+ * Read-only analysis for model-route shadow cases. Recommendations stay
+ * advisory: this module joins evidence; it never changes host model settings.
+ */
+export function buildModelRouteReport(records = []) {
+  const routes = new Map();
+  const executions = new Map();
+  for (const record of Array.isArray(records) ? records : []) {
+    if (!record || typeof record !== "object" || typeof record.correlation_id !== "string") continue;
+    if (record.record_type === "routing_case" && record.request?.context?.model_route?.mode === "shadow") routes.set(record.correlation_id, record);
+    if (record.record_type === "execution_receipt") executions.set(record.correlation_id, record);
+  }
+
+  const rows = [...routes.values()].map((route) => toRow(route, executions.get(route.correlation_id))).sort((a, b) => a.recommended_model_id.localeCompare(b.recommended_model_id));
+  return {
+    schema_version: 1,
+    mode: "shadow",
+    generated_at: new Date().toISOString(),
+    summary: summarize(rows),
+    models: groupByRecommendation(rows),
+  };
+}
+
+function toRow(route, execution) {
+  const result = execution?.host?.result?.model_route;
+  const recommended = string(route.decision?.selected, "unrecommended");
+  const actual = string(result?.actual_model_id, null);
+  const outcome = string(result?.outcome, execution ? string(execution.host?.status, "unknown") : null);
+  return {
+    correlation_id: route.correlation_id,
+    recommended_model_id: recommended,
+    actual_model_id: actual,
+    recommendation_match: actual !== null && actual === recommended,
+    outcome,
+    retry_count: boundedInteger(result?.retry_count),
+    host_duration_ms: finite(execution?.host?.duration_ms),
+    jev_latency_ms: finite(route.decision?.receipt?.latency_ms),
+    jev_cost_usd: finite(route.decision?.receipt?.cost_usd),
+  };
+}
+
+function summarize(rows) {
+  const matched = rows.filter((row) => row.recommendation_match).length;
+  const actual = rows.filter((row) => row.actual_model_id !== null).length;
+  return {
+    decisions: rows.length,
+    with_host_outcome: rows.filter((row) => row.outcome !== null).length,
+    with_actual_model: actual,
+    recommendation_matches: matched,
+    recommendation_match_rate: actual ? matched / actual : null,
+    verified_outcomes: rows.filter((row) => row.outcome === "verified" || row.outcome === "completed").length,
+    failed_outcomes: rows.filter((row) => row.outcome === "failed").length,
+    total_retry_count: rows.reduce((sum, row) => sum + (row.retry_count ?? 0), 0),
+    total_jev_cost_usd: sum(rows.map((row) => row.jev_cost_usd)),
+    average_jev_latency_ms: average(rows.map((row) => row.jev_latency_ms)),
+    average_host_duration_ms: average(rows.map((row) => row.host_duration_ms)),
+  };
+}
+
+function groupByRecommendation(rows) {
+  const grouped = new Map();
+  for (const row of rows) {
+    const group = grouped.get(row.recommended_model_id) ?? {
+      recommended_model_id: row.recommended_model_id,
+      decisions: 0,
+      with_host_outcome: 0,
+      recommendation_matches: 0,
+      verified_outcomes: 0,
+      failed_outcomes: 0,
+      total_retry_count: 0,
+      host_actual_model_ids: {},
+    };
+    group.decisions += 1;
+    if (row.outcome !== null) group.with_host_outcome += 1;
+    if (row.recommendation_match) group.recommendation_matches += 1;
+    if (row.outcome === "verified" || row.outcome === "completed") group.verified_outcomes += 1;
+    if (row.outcome === "failed") group.failed_outcomes += 1;
+    group.total_retry_count += row.retry_count ?? 0;
+    if (row.actual_model_id) group.host_actual_model_ids[row.actual_model_id] = (group.host_actual_model_ids[row.actual_model_id] ?? 0) + 1;
+    grouped.set(row.recommended_model_id, group);
+  }
+  return [...grouped.values()].sort((a, b) => a.recommended_model_id.localeCompare(b.recommended_model_id));
+}
+
+function string(value, fallback) { return typeof value === "string" && value.trim() ? value.trim() : fallback; }
+function finite(value) { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; }
+function boundedInteger(value) { return Number.isInteger(value) && value >= 0 ? value : null; }
+function sum(values) { return values.reduce((total, value) => total + (value ?? 0), 0); }
+function average(values) { const present = values.filter((value) => value !== null); return present.length ? sum(present) / present.length : null; }

package/src/model-routing.mjs

@@ -0,0 +1,87 @@
+import { routeRequest } from "./route.mjs";
+
+/**
+ * Recommend one host-declared model profile for the next model call.
+ * This is a shadow decision: it never mutates the host model/provider setting.
+ */
+export async function recommendModelRoute({
+  intent,
+  context = {},
+  models,
+  harness = "unknown",
+  policy = {},
+  provider = "demo",
+  config,
+} = {}) {
+  if (!Array.isArray(models) || models.length === 0) throw new TypeError("models[] is required");
+  const profiles = models.map(normalizeProfile);
+  const profileById = new Map(profiles.map((profile) => [profile.id, profile]));
+  const decision = await routeRequest({
+    harness,
+    intent,
+    context: { ...context, model_route: { mode: "shadow", profiles: profiles.map(publicProfile) } },
+    capabilities: profiles.map(toCapability),
+    policy: {
+      ...policy,
+      // A recommendation must never create an approval boundary by itself.
+      confirmation_risk_levels: [],
+    },
+  }, { provider, config });
+  return {
+    ...decision,
+    route_mode: "shadow",
+    recommended_model: decision.selected ? publicProfile(profileById.get(decision.selected)) : null,
+  };
+}
+
+function normalizeProfile(value, index) {
+  if (!value || typeof value !== "object") throw new TypeError(`models[${index}] must be an object`);
+  const id = required(value.id, `models[${index}].id`);
+  const model = required(value.model ?? value.id, `models[${index}].model`);
+  const provider = string(value.provider, "unknown");
+  const reasoning_effort = string(value.reasoning_effort, "default");
+  const description = string(value.description, `${provider}/${model}; reasoning=${reasoning_effort}`);
+  return {
+    id,
+    provider,
+    model,
+    reasoning_effort,
+    description,
+    available: value.available !== false && value.availability?.available !== false,
+    availability_reason: value.availability?.reason ?? null,
+  };
+}
+
+function toCapability(profile) {
+  return {
+    id: profile.id,
+    kind: "model",
+    name: profile.model,
+    description: profile.description,
+    risk: "low",
+    available: profile.available,
+    availability: { available: profile.available, reason: profile.availability_reason },
+    source: profile.provider,
+    metadata: { model: profile.model },
+  };
+}
+
+function publicProfile(profile) {
+  if (!profile) return null;
+  return {
+    id: profile.id,
+    provider: profile.provider,
+    model: profile.model,
+    reasoning_effort: profile.reasoning_effort,
+    available: profile.available,
+  };
+}
+
+function required(value, label) {
+  if (typeof value !== "string" || !value.trim()) throw new TypeError(`${label} is required`);
+  return value.trim();
+}
+
+function string(value, fallback) {
+  return typeof value === "string" && value.trim() ? value.trim() : fallback;
+}

package/src/providers/demo.mjs

@@ -1,52 +1,52 @@
-import { stableJson } from "../contract.mjs";
-
-const TOKEN_RE = /[\p{L}\p{N}_-]+/gu;
-
-export class DemoProvider {
-  name = "jev-demo";
-
-  async decide({ state, candidates }) {
-    const requestTokens = tokens(`${state.intent} ${stableJson(state.context ?? {})}`);
-    const scored = candidates.map((candidate) => {
-      const candidateTokens = tokens(`${candidate.id} ${candidate.name} ${candidate.description}`);
-      const overlap = [...requestTokens].filter((token) => candidateTokens.has(token)).length;
-      return { id: candidate.id, score: overlap + 0.01 };
-    });
-    const total = scored.reduce((sum, item) => sum + item.score, 0);
-    const probabilities = Object.fromEntries(scored.map(({ id, score }) => [id, score / total]));
-    const choice = [...scored].sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))[0]?.id ?? "";
-    const top = choice ? probabilities[choice] : 0;
-    const confidence = scored.length <= 1
-      ? 1
-      : Math.max(0, Math.min(1, (top - 1 / scored.length) / Math.max(1 - 1 / scored.length, 0.0001)));
-    return {
-      model: "jev-demo",
-      answers: {
-        tool: {
-          type: "choice",
-          choice,
-          probabilities,
-          confidence,
-        },
-      },
-      usage: { input_bytes: Buffer.byteLength(stableJson(state), "utf8"), output_bytes: 0 },
-    };
-  }
-
-  async evaluate({ state, questions }) {
-    const supplied = state.context?.supervision?.judgments ?? {};
-    const answers = Object.fromEntries(Object.keys(questions).map((id) => {
-      const value = typeof supplied[id] === "number" ? Math.max(0, Math.min(1, supplied[id])) : 0.5;
-      return [id, { type: "noul", probability: value, confidence: 1 }];
-    }));
-    return {
-      model: "jev-demo",
-      answers,
-      usage: { input_bytes: Buffer.byteLength(stableJson(state), "utf8"), output_bytes: 0 },
-    };
-  }
-}
-
-function tokens(value) {
-  return new Set(value.toLowerCase().match(TOKEN_RE) ?? []);
-}
+import { stableJson } from "../contract.mjs";
+
+const TOKEN_RE = /[\p{L}\p{N}_-]+/gu;
+
+export class DemoProvider {
+  name = "jev-demo";
+
+  async decide({ state, candidates }) {
+    const requestTokens = tokens(`${state.intent} ${stableJson(state.context ?? {})}`);
+    const scored = candidates.map((candidate) => {
+      const candidateTokens = tokens(`${candidate.id} ${candidate.name} ${candidate.description}`);
+      const overlap = [...requestTokens].filter((token) => candidateTokens.has(token)).length;
+      return { id: candidate.id, score: overlap + 0.01 };
+    });
+    const total = scored.reduce((sum, item) => sum + item.score, 0);
+    const probabilities = Object.fromEntries(scored.map(({ id, score }) => [id, score / total]));
+    const choice = [...scored].sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))[0]?.id ?? "";
+    const top = choice ? probabilities[choice] : 0;
+    const confidence = scored.length <= 1
+      ? 1
+      : Math.max(0, Math.min(1, (top - 1 / scored.length) / Math.max(1 - 1 / scored.length, 0.0001)));
+    return {
+      model: "jev-demo",
+      answers: {
+        tool: {
+          type: "choice",
+          choice,
+          probabilities,
+          confidence,
+        },
+      },
+      usage: { input_bytes: Buffer.byteLength(stableJson(state), "utf8"), output_bytes: 0 },
+    };
+  }
+
+  async evaluate({ state, questions }) {
+    const supplied = state.context?.supervision?.judgments ?? {};
+    const answers = Object.fromEntries(Object.keys(questions).map((id) => {
+      const value = typeof supplied[id] === "number" ? Math.max(0, Math.min(1, supplied[id])) : 0.5;
+      return [id, { type: "noul", probability: value, confidence: 1 }];
+    }));
+    return {
+      model: "jev-demo",
+      answers,
+      usage: { input_bytes: Buffer.byteLength(stableJson(state), "utf8"), output_bytes: 0 },
+    };
+  }
+}
+
+function tokens(value) {
+  return new Set(value.toLowerCase().match(TOKEN_RE) ?? []);
+}

package/src/providers/index.mjs

@@ -0,0 +1,12 @@
+import { configuredProviderOptions } from "../config.mjs";
+import { DemoProvider } from "./demo.mjs";
+import { OpenRouterDecisionsProvider, TypeSafeProvider } from "./typesafe.mjs";
+
+export function resolveProvider(provider, options = {}) {
+  if (provider && typeof provider === "object") return provider;
+  if (provider === "demo") return new DemoProvider();
+  const settings = { ...configuredProviderOptions(options.config, provider), ...options[provider] };
+  if (provider === "openrouter") return new OpenRouterDecisionsProvider(settings);
+  if (provider === "typesafe") return new TypeSafeProvider(settings);
+  throw new Error(`unsupported provider: ${provider}`);
+}

package/src/providers/typesafe.mjs

@@ -1,126 +1,126 @@
-import { stableJson } from "../contract.mjs";
-
-export class TypeSafeProvider {
-  name = "typesafe";
-
-  constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-latest", timeoutMs = 2_000 } = {}) {
-    this.apiKey = apiKey;
-    this.endpoint = endpoint;
-    this.model = model;
-    this.timeoutMs = timeoutMs;
-  }
-
-  async decide({ state, candidates }) {
-    const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
-      name: candidate.name,
-      kind: candidate.kind,
-      description: candidate.description,
-      risk: candidate.risk,
-    }]));
-    return this.evaluate({
-      state,
-      questions: {
-        tool: {
-          type: "choice",
-          instructions: "Which single capability should handle this request? Choose only from the supplied options.",
-          criteria,
-        },
-      },
-    });
-  }
-
-  async evaluate({ state, questions }) {
-    if (!this.apiKey) throw new Error("TYPESAFE_API_KEY is not configured");
-    const response = await fetch(this.endpoint, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${this.apiKey}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ state, model: this.model, questions }),
-      signal: AbortSignal.timeout(this.timeoutMs),
-    }).catch((error) => {
-      if (error?.name === "TimeoutError") throw new Error(`TypeSafe request timed out after ${this.timeoutMs}ms`);
-      throw error;
-    });
-
-    if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`TypeSafe returned HTTP ${response.status}: ${body.slice(0, 240)}`);
-    }
-    const raw = await response.json();
-    if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
-      throw new Error("TypeSafe response has no answers object");
-    }
-    return raw;
-  }
-}
-
-export class OpenRouterDecisionsProvider {
-  name = "openrouter:typesafe/jev-1.13";
-
-  constructor({
-    apiKey = process.env.OPENROUTER_API_KEY,
-    endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions",
-    model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13",
-    timeoutMs = 5_000,
-    fetchImpl = globalThis.fetch,
-    httpReferer = process.env.OPENROUTER_HTTP_REFERER,
-    appTitle = process.env.OPENROUTER_APP_TITLE,
-  } = {}) {
-    this.apiKey = apiKey;
-    this.endpoint = endpoint;
-    this.model = model;
-    this.timeoutMs = timeoutMs;
-    this.fetchImpl = fetchImpl;
-    this.httpReferer = httpReferer;
-  }
-  async decide({ state, candidates }) {
-    const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
-      name: candidate.name,
-      kind: candidate.kind,
-      description: candidate.description,
-      risk: candidate.risk,
-    }]));
-    return this.evaluate({
-      state,
-      questions: {
-        tool: {
-          type: "choice",
-          instructions: "Which single capability should handle this request? Choose only from the supplied options.",
-          criteria,
-        },
-      },
-    });
-  }
-
-  async evaluate({ state, questions }) {
-    if (!this.apiKey) throw new Error("OPENROUTER_API_KEY is not configured");
-    if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable");
-
-    const response = await this.fetchImpl(this.endpoint, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${this.apiKey}`,
-        "Content-Type": "application/json",
-        ...(this.httpReferer ? { "HTTP-Referer": this.httpReferer } : {}),
-        ...(this.appTitle ? { "X-Title": this.appTitle } : {}),
-      },
-      body: JSON.stringify({ model: this.model, questions, state }),
-      signal: AbortSignal.timeout(this.timeoutMs),
-    }).catch((error) => {
-      if (error?.name === "TimeoutError") throw new Error(`OpenRouter Decisions request timed out after ${this.timeoutMs}ms`);
-      throw error;
-    });
-
-    if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`OpenRouter Decisions returned HTTP ${response.status}: ${body.slice(0, 240)}`);
-    }
-    const raw = await response.json();
-    if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
-      throw new Error("OpenRouter Decisions response has no answers object");
-    }
-    return raw;
-  }
-}
+export class TypeSafeProvider {
+  name = "typesafe";
+
+  constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-latest", timeoutMs = 2_000, fetchImpl = globalThis.fetch } = {}) {
+    this.apiKey = apiKey;
+    this.endpoint = endpoint;
+    this.model = model;
+    this.timeoutMs = timeoutMs;
+    this.fetchImpl = fetchImpl;
+  }
+
+  async decide({ state, candidates }) {
+    const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
+      name: candidate.name,
+      kind: candidate.kind,
+      description: candidate.description,
+      risk: candidate.risk,
+    }]));
+    return this.evaluate({
+      state,
+      questions: {
+        tool: {
+          type: "choice",
+          instructions: "Which single capability should handle this request? Choose only from the supplied options.",
+          criteria,
+        },
+      },
+    });
+  }
+
+  async evaluate({ state, questions }) {
+    if (!this.apiKey) throw new Error("TYPESAFE_API_KEY is not configured");
+    if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable");
+    const response = await this.fetchImpl(this.endpoint, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ state, model: this.model, questions }),
+      signal: AbortSignal.timeout(this.timeoutMs),
+    }).catch((error) => {
+      if (error?.name === "TimeoutError") throw new Error(`TypeSafe request timed out after ${this.timeoutMs}ms`);
+      throw error;
+    });
+
+    if (!response.ok) {
+      const body = await response.text();
+      throw new Error(`TypeSafe returned HTTP ${response.status}: ${body.slice(0, 240)}`);
+    }
+    const raw = await response.json();
+    if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
+      throw new Error("TypeSafe response has no answers object");
+    }
+    return raw;
+  }
+}
+
+export class OpenRouterDecisionsProvider {
+  constructor({
+    apiKey = process.env.OPENROUTER_API_KEY,
+    endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions",
+    model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13",
+    timeoutMs = 5_000,
+    fetchImpl = globalThis.fetch,
+    httpReferer = process.env.OPENROUTER_HTTP_REFERER,
+    appTitle = process.env.OPENROUTER_APP_TITLE,
+  } = {}) {
+    this.name = `openrouter:${model}`;
+    this.apiKey = apiKey;
+    this.endpoint = endpoint;
+    this.model = model;
+    this.timeoutMs = timeoutMs;
+    this.fetchImpl = fetchImpl;
+    this.httpReferer = httpReferer;
+    this.appTitle = appTitle;
+  }
+  async decide({ state, candidates }) {
+    const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, {
+      name: candidate.name,
+      kind: candidate.kind,
+      description: candidate.description,
+      risk: candidate.risk,
+    }]));
+    return this.evaluate({
+      state,
+      questions: {
+        tool: {
+          type: "choice",
+          instructions: "Which single capability should handle this request? Choose only from the supplied options.",
+          criteria,
+        },
+      },
+    });
+  }
+
+  async evaluate({ state, questions }) {
+    if (!this.apiKey) throw new Error("OPENROUTER_API_KEY is not configured");
+    if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable");
+
+    const response = await this.fetchImpl(this.endpoint, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.apiKey}`,
+        "Content-Type": "application/json",
+        ...(this.httpReferer ? { "HTTP-Referer": this.httpReferer } : {}),
+        ...(this.appTitle ? { "X-Title": this.appTitle } : {}),
+      },
+      body: JSON.stringify({ model: this.model, questions, state }),
+      signal: AbortSignal.timeout(this.timeoutMs),
+    }).catch((error) => {
+      if (error?.name === "TimeoutError") throw new Error(`OpenRouter Decisions request timed out after ${this.timeoutMs}ms`);
+      throw error;
+    });
+
+    if (!response.ok) {
+      const body = await response.text();
+      throw new Error(`OpenRouter Decisions returned HTTP ${response.status}: ${body.slice(0, 240)}`);
+    }
+    const raw = await response.json();
+    if (!raw || typeof raw !== "object" || !raw.answers || typeof raw.answers !== "object") {
+      throw new Error("OpenRouter Decisions response has no answers object");
+    }
+    return raw;
+  }
+}