@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/model-router/catalog.test.mjs

@@ -1,385 +0,0 @@
-/**
- * Tests for lib/model-router/catalog.mjs — load, authority merge precedence,
- * hot reload, bad-row handling, lookupModel/listModels, overlay tighten-only.
- *
- * Hermetic: most tests inject catalog YAML / patches via opts so they never
- * touch the filesystem. FS-backed paths (bundled rows on disk, hot reload via
- * mtime) write to a temp dir.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, writeFileSync, mkdirSync, utimesSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import {
-  loadCatalog,
-  loadCatalogCached,
-  invalidateCatalogCache,
-  lookupModel,
-  listModels,
-} from "./catalog.mjs";
-
-// A minimal bundled-rows fixture injected via opts.catalogText.
-const ANTHROPIC_YAML = `
-provider: anthropic
-auth_env: ANTHROPIC_API_KEY
-endpoints: { anthropic: https://api.anthropic.com }
-data_residency: us
-models:
-  - id: claude-opus-4-8
-    status: available
-    context_window: 1000000
-    context_tokens: 950000
-    max_tokens: 128000
-    cost: { input: 5.0, output: 25.0, cache_read: 0.5, cache_write: 10.0, tiered: [] }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
-    compat: { cache_control: anthropic, max_tokens_field: max_tokens }
-    tool_reliability: A
-    harness: { session: true, direct: true, batch: true }
-  - id: claude-haiku-4-5
-    status: available
-    context_window: 200000
-    context_tokens: 190000
-    max_tokens: 64000
-    cost: { input: 1.0, output: 5.0, cache_read: 0.1, cache_write: 2.0 }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
-    compat: { cache_control: anthropic }
-    tool_reliability: A
-    harness: { session: true, direct: true, batch: true }
-`;
-
-const DEEPSEEK_YAML = `
-provider: deepseek
-auth_env: DEEPSEEK_API_KEY
-endpoints: { anthropic: https://api.deepseek.com/anthropic, openai: https://api.deepseek.com }
-data_residency: prc
-models:
-  - id: deepseek-v4-flash
-    status: available
-    context_window: 1000000
-    context_tokens: 900000
-    max_tokens: 384000
-    cost: { input: 0.14, output: 0.28, cache_read: 0.0028, cache_write: null }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
-    compat: { cache_control: implicit }
-    tool_reliability: B
-    harness: { session: true, direct: true, batch: false }
-`;
-
-function inject(extra = {}) {
-  return { catalogText: { anthropic: ANTHROPIC_YAML, deepseek: DEEPSEEK_YAML }, ...extra };
-}
-
-test("loadCatalog parses bundled rows and never throws", () => {
-  const res = loadCatalog("/nonexistent-root", inject());
-  assert.equal(res.errors.length, 0, JSON.stringify(res.errors));
-  assert.ok(res.byRef.has("anthropic/claude-opus-4-8"));
-  assert.ok(res.byRef.has("deepseek/deepseek-v4-flash"));
-  assert.equal(res.models.length, 3);
-});
-
-test("rows carry provider metadata (auth_env / endpoints / data_residency / ref)", () => {
-  const res = loadCatalog("/x", inject());
-  const opus = lookupModel(res, "anthropic/claude-opus-4-8");
-  assert.equal(opus.auth_env, "ANTHROPIC_API_KEY");
-  assert.equal(opus.data_residency, "us");
-  assert.equal(opus.ref, "anthropic/claude-opus-4-8");
-  assert.equal(opus.endpoints.anthropic, "https://api.anthropic.com");
-  const ds = lookupModel(res, "deepseek/deepseek-v4-flash");
-  assert.equal(ds.endpoints.openai, "https://api.deepseek.com");
-  assert.equal(ds.cost_provenance.volatile, true);
-});
-
-test("lookupModel returns null for unknown ref, never throws", () => {
-  const res = loadCatalog("/x", inject());
-  assert.equal(lookupModel(res, "nope/nope"), null);
-  assert.equal(lookupModel(res, 123), null);
-  assert.equal(lookupModel(null, "anthropic/claude-opus-4-8"), null);
-});
-
-test("listModels filters by provider / status / minGrade / harness", () => {
-  const res = loadCatalog("/x", inject());
-  assert.equal(listModels(res, { provider: "anthropic" }).length, 2);
-  assert.equal(listModels(res, { provider: "deepseek" }).length, 1);
-  assert.equal(listModels(res, { available: true }).length, 3);
-  // minGrade A excludes the B-grade deepseek row.
-  const gradeA = listModels(res, { minGrade: "A" });
-  assert.ok(gradeA.every((r) => r.tool_reliability === "A"));
-  assert.equal(gradeA.length, 2);
-  // harness:batch excludes deepseek (batch:false).
-  const batch = listModels(res, { harness: "batch" });
-  assert.ok(batch.every((r) => r.harness.batch === true));
-  assert.equal(batch.length, 2);
-});
-
-// ---------------------------------------------------------------------------
-// Authority merge precedence
-// ---------------------------------------------------------------------------
-
-test("pricing cache updates COST fields only (layer 3)", () => {
-  const res = loadCatalog("/x", inject({
-    pricingCache: {
-      "anthropic/claude-opus-4-8": { input: 4.0, output: 20.0, provenance: { source: "litellm", fetched: "2026-06-11", volatile: false } },
-    },
-  }));
-  const opus = lookupModel(res, "anthropic/claude-opus-4-8");
-  assert.equal(opus.cost.input, 4.0);
-  assert.equal(opus.cost.output, 20.0);
-  // capability/grade untouched by pricing layer:
-  assert.equal(opus.tool_reliability, "A");
-  assert.equal(opus.compat.cache_control, "anthropic");
-  assert.equal(opus.cost_provenance.source, "litellm");
-});
-
-test("pricing for an unknown row is ignored with a warning", () => {
-  const res = loadCatalog("/x", inject({
-    pricingCache: { "ghost/model": { input: 1 } },
-  }));
-  assert.ok(res.warnings.some((w) => /unknown row/.test(w.warning)));
-});
-
-test("org overlay catalog_patch is TIGHTEN-ONLY: disable + lower grade allowed", () => {
-  const res = loadCatalog("/x", inject({
-    overlay: {
-      version: "2026.06.11-1",
-      catalog_patch: [
-        { ref: "deepseek/deepseek-v4-flash", status: "disabled" },
-        { ref: "anthropic/claude-haiku-4-5", tool_reliability: "B" }, // A -> B lower
-      ],
-    },
-  }));
-  assert.equal(lookupModel(res, "deepseek/deepseek-v4-flash").status, "disabled");
-  assert.equal(lookupModel(res, "anthropic/claude-haiku-4-5").tool_reliability, "B");
-  assert.equal(res.errors.length, 0, JSON.stringify(res.errors));
-});
-
-test("org overlay CANNOT raise a grade or add a provider (errors, field untouched)", () => {
-  const res = loadCatalog("/x", inject({
-    overlay: {
-      version: "v",
-      catalog_patch: [
-        { ref: "deepseek/deepseek-v4-flash", tool_reliability: "A" }, // B -> A widen
-        { ref: "newprov/new-model", status: "available" }, // add provider
-        { ref: "deepseek/deepseek-v4-flash", status: "available" }, // re-widen status
-      ],
-    },
-  }));
-  // grade stays B
-  assert.equal(lookupModel(res, "deepseek/deepseek-v4-flash").tool_reliability, "B");
-  assert.equal(lookupModel(res, "newprov/new-model"), null);
-  assert.ok(res.errors.some((e) => /cannot raise tool_reliability/.test(e.error)));
-  assert.ok(res.errors.some((e) => /cannot add providers/.test(e.error)));
-  assert.ok(res.errors.some((e) => /cannot widen status/.test(e.error)));
-});
-
-test("models.local.yaml (layer 1) overrides bundled + pricing", () => {
-  const res = loadCatalog("/x", inject({
-    pricingCache: { "anthropic/claude-opus-4-8": { input: 4.0 } },
-    localConfig: {
-      catalog_overrides: [
-        { ref: "anthropic/claude-opus-4-8", cost: { input: 9.99 }, status: "deprecated" },
-      ],
-    },
-  }));
-  const opus = lookupModel(res, "anthropic/claude-opus-4-8");
-  assert.equal(opus.cost.input, 9.99); // local beats pricing-cache
-  assert.equal(opus.status, "deprecated");
-});
-
-test("agent model-routing.yaml catalog_overrides (layer 0) is HIGHEST authority", () => {
-  const res = loadCatalog("/x", inject({
-    localConfig: { catalog_overrides: [{ ref: "anthropic/claude-opus-4-8", cost: { input: 9.99 } }] },
-    agentConfig: { catalog_overrides: [{ ref: "anthropic/claude-opus-4-8", cost: { input: 1.11 }, tool_reliability: "B" }] },
-  }));
-  const opus = lookupModel(res, "anthropic/claude-opus-4-8");
-  assert.equal(opus.cost.input, 1.11); // agent beats local
-  assert.equal(opus.tool_reliability, "B");
-});
-
-test("agent catalog_overrides may add a new row but warns (typo guard)", () => {
-  const res = loadCatalog("/x", inject({
-    agentConfig: {
-      catalog_overrides: [
-        { ref: "moonshot/kimi-k2.6", status: "available", cost: { input: 0.95, output: 4.0 }, tool_reliability: "B", harness: { direct: true } },
-      ],
-    },
-  }));
-  assert.ok(lookupModel(res, "moonshot/kimi-k2.6"));
-  assert.ok(res.warnings.some((w) => /creates new row/.test(w.warning)));
-});
-
-// ---------------------------------------------------------------------------
-// Bad-row handling (strict but never-throw)
-// ---------------------------------------------------------------------------
-
-test("a row missing id is dropped with an error, siblings survive", () => {
-  const bad = `
-provider: anthropic
-auth_env: ANTHROPIC_API_KEY
-models:
-  - status: available
-    cost: { input: 1, output: 2 }
-  - id: claude-haiku-4-5
-    status: available
-    cost: { input: 1, output: 5 }
-`;
-  const res = loadCatalog("/x", { catalogText: { anthropic: bad } });
-  assert.equal(res.byRef.size, 1);
-  assert.ok(lookupModel(res, "anthropic/claude-haiku-4-5"));
-  assert.ok(res.errors.some((e) => /missing string `id`/.test(e.error)));
-});
-
-test("an invalid status drops the row with an error", () => {
-  const bad = `
-provider: anthropic
-auth_env: X
-models:
-  - id: weird
-    status: kinda-available
-    cost: { input: 1, output: 2 }
-`;
-  const res = loadCatalog("/x", { catalogText: { anthropic: bad } });
-  assert.equal(res.byRef.size, 0);
-  assert.ok(res.errors.some((e) => /not in \{available/.test(e.error)));
-});
-
-test("a compat-flag typo surfaces an error but keeps the row", () => {
-  const bad = `
-provider: anthropic
-auth_env: X
-models:
-  - id: m
-    status: available
-    cost: { input: 1, output: 2 }
-    compat: { cach_control: anthropic }
-`;
-  const res = loadCatalog("/x", { catalogText: { anthropic: bad } });
-  assert.ok(lookupModel(res, "anthropic/m"));
-  assert.ok(res.errors.some((e) => /unknown compat flag "cach_control"/.test(e.error)));
-});
-
-test("a bad cache_control value surfaces an error", () => {
-  const bad = `
-provider: anthropic
-auth_env: X
-models:
-  - id: m
-    status: available
-    cost: { input: 1, output: 2 }
-    compat: { cache_control: magic }
-`;
-  const res = loadCatalog("/x", { catalogText: { anthropic: bad } });
-  assert.ok(res.errors.some((e) => /cache_control "magic"/.test(e.error)));
-});
-
-test("malformed YAML for one provider does not crash the load", () => {
-  const res = loadCatalog("/x", { catalogText: { anthropic: ANTHROPIC_YAML, deepseek: ":: not yaml ::\n  - [" } });
-  // anthropic rows still present; deepseek parse error recorded.
-  assert.ok(res.byRef.has("anthropic/claude-opus-4-8"));
-  assert.ok(res.errors.some((e) => /deepseek/.test(e.source) || /parse/.test(e.error)));
-});
-
-test("invalid tool_reliability grade is recorded but does not crash", () => {
-  const bad = `
-provider: anthropic
-auth_env: X
-models:
-  - id: m
-    status: available
-    cost: { input: 1, output: 2 }
-    tool_reliability: Z
-`;
-  const res = loadCatalog("/x", { catalogText: { anthropic: bad } });
-  const row = lookupModel(res, "anthropic/m");
-  assert.ok(row);
-  assert.equal(row.tool_reliability, null); // invalid grade dropped to null
-  assert.ok(res.errors.some((e) => /tool_reliability "Z"/.test(e.error)));
-});
-
-// ---------------------------------------------------------------------------
-// FS-backed: real bundled dir + hot reload via mtime
-// ---------------------------------------------------------------------------
-
-test("loadCatalog reads the real bundled catalog dir on disk", () => {
-  // No injection: hits the shipped lib/model-router/catalog/*.yaml.
-  const res = loadCatalog(process.cwd());
-  assert.equal(res.errors.length, 0, JSON.stringify(res.errors));
-  for (const ref of [
-    "anthropic/claude-opus-4-8",
-    "anthropic/claude-sonnet-4-6",
-    "anthropic/claude-haiku-4-5",
-    "deepseek/deepseek-v4-flash",
-    "deepseek/deepseek-v4-pro",
-    "moonshot/kimi-k2.6",
-    "qwen/qwen3.5-flash",
-  ]) {
-    assert.ok(res.byRef.has(ref), `expected bundled row ${ref}`);
-  }
-});
-
-test("loadCatalogCached hot-reloads when a contributing file's mtime advances", () => {
-  const root = mkdtempSync(join(tmpdir(), "maestro-catalog-"));
-  const dir = join(root, "catalog");
-  mkdirSync(dir, { recursive: true });
-  const file = join(dir, "anthropic.yaml");
-  writeFileSync(file, ANTHROPIC_YAML);
-
-  const cacheKey = `hot-${Date.now()}`;
-  const opts = { bundledDir: dir, cacheKey };
-  const r1 = loadCatalogCached(root, opts);
-  assert.equal(lookupModel(r1, "anthropic/claude-opus-4-8").cost.input, 5.0);
-
-  // Same call again returns the SAME cached object (no mtime change).
-  const r2 = loadCatalogCached(root, opts);
-  assert.strictEqual(r1, r2);
-
-  // Rewrite the file with a different price + bump mtime into the future.
-  writeFileSync(file, ANTHROPIC_YAML.replace("input: 5.0", "input: 7.7"));
-  const future = Date.now() / 1000 + 60;
-  utimesSync(file, future, future);
-
-  const r3 = loadCatalogCached(root, opts);
-  assert.notStrictEqual(r1, r3, "expected a fresh load after mtime advance");
-  assert.equal(lookupModel(r3, "anthropic/claude-opus-4-8").cost.input, 7.7);
-
-  invalidateCatalogCache(cacheKey);
-});
-
-test("invalidateCatalogCache forces a fresh load", () => {
-  const root = mkdtempSync(join(tmpdir(), "maestro-catalog-"));
-  const dir = join(root, "catalog");
-  mkdirSync(dir, { recursive: true });
-  writeFileSync(join(dir, "anthropic.yaml"), ANTHROPIC_YAML);
-  const cacheKey = `inv-${Date.now()}`;
-  const opts = { bundledDir: dir, cacheKey };
-  const a = loadCatalogCached(root, opts);
-  const b = loadCatalogCached(root, opts);
-  assert.strictEqual(a, b);
-  invalidateCatalogCache(cacheKey);
-  const c = loadCatalogCached(root, opts);
-  assert.notStrictEqual(a, c);
-  invalidateCatalogCache(cacheKey);
-});
-
-test("loadCatalogCached force:true bypasses the cache", () => {
-  const cacheKey = `force-${Date.now()}`;
-  const a = loadCatalogCached("/x", inject({ cacheKey }));
-  const b = loadCatalogCached("/x", inject({ cacheKey, force: true }));
-  assert.notStrictEqual(a, b);
-  invalidateCatalogCache(cacheKey);
-});
-
-test("merge-order regression: pricing < overlay-status < local < agent for distinct fields", () => {
-  // pricing sets a price; overlay disables; local re-prices; agent sets grade.
-  const res = loadCatalog("/x", inject({
-    pricingCache: { "deepseek/deepseek-v4-flash": { input: 0.1 } },
-    overlay: { version: "v", catalog_patch: [{ ref: "deepseek/deepseek-v4-flash", status: "disabled" }] },
-    localConfig: { catalog_overrides: [{ ref: "deepseek/deepseek-v4-flash", cost: { input: 0.2 } }] },
-    agentConfig: { catalog_overrides: [{ ref: "deepseek/deepseek-v4-flash", cost: { input: 0.3 } }] },
-  }));
-  const ds = lookupModel(res, "deepseek/deepseek-v4-flash");
-  assert.equal(ds.cost.input, 0.3); // agent wins on price
-  assert.equal(ds.status, "disabled"); // overlay tighten survives (not touched by price/local/agent here)
-});