omp-llm-role 1.0.0 → 1.2.0

package/src/benchmark-sources.ts

@@ -44,6 +44,8 @@ const UA =
 const WRITING_URL = "https://llm-stats.com/research/best-ai-for-writing/evidence.json";
 /** The llm-stats backend that serves one benchmark's leaderboard as JSON. */
 const LLM_STATS_BENCHMARK_URL = "https://api.zeroeval.com/leaderboard/benchmarks";
+/** The catalog cache file under `CACHE_DIR` (shared by the loader and `cachedSourceInfo`). */
+const CATALOG_CACHE_FILE = "benchmark-catalog-fetched-data.json";
 
 /** How a source's ids map to the llm-stats bare id. */
 export type JoinRule = "direct" | "slug-suffix" | "normalized";
@@ -121,6 +123,16 @@ export function catalogMetric(benchmarkId: string): string {
   return SHIPPED_CATALOG_METRICS[benchmarkId] ?? externalMetricKey("bench", benchmarkId);
 }
 
+/** The declaration a catalog benchmark id needs so its metric key resolves on
+ * the next update, or `null` when none is needed: a shipped id (its static
+ * source already resolves) or a dot-free id the generic metric-key fallback
+ * reconstructs. `normalizeMetricKey` folds separators (`.`, case, …), so a
+ * dotted id's raw form survives only here — the declaration must carry it. */
+export function catalogBenchmarkDeclaration(benchmarkId: string): SourceDeclaration | null {
+  if (SHIPPED_CATALOG_METRICS[benchmarkId] !== undefined) return null;
+  return normalizeMetricKey(benchmarkId) !== benchmarkId ? llmStatsBenchmarkDeclaration(benchmarkId) : null;
+}
+
 // ---------------------------------------------------------------------------
 // Parsers
 // ---------------------------------------------------------------------------
@@ -164,6 +176,104 @@ export function parseLlmStatsBenchmark(v: unknown): Record<string, number> | nul
   return out;
 }
 
+/**
+ * The per-entry summary of an llm-stats benchmark payload, read in one pass:
+ * the trust flags (`self_reported`/`verified`) and the `multimodal` flag.
+ *
+ * `trust` is `null` when no parsed entry carried a boolean `self_reported` — a
+ * declared or writing-evidence payload has no entry flags, so the axis degrades
+ * to `unknown` rather than a silent `ok`. `verified` is uniformly `false` today
+ * (the source never sets it); it is kept for when the source starts to.
+ *
+ * `modality` is the per-entry `multimodal` summary: how many of the `entries[]`
+ * carry a boolean `multimodal` (the share denominator) and how many of those are
+ * `true`. It is `null` when no parsed entry carried the flag, so the axis
+ * degrades to `unknown` rather than a silent `ok`.
+ *
+ * `provenance` is the per-entry org distribution: each row's `organization_id`
+ * (falling back to `provider_id` when `organization_id` is not a non-empty
+ * string) is the lab a model's score comes from. `covered` counts the entries
+ * carrying a non-empty org string, `orgs` the distinct org count, and
+ * `dominantOrg`/`dominantShare` the most frequent org and its count / `covered`.
+ * It is `null` when no entry carried an org, so the axis degrades to `unknown`
+ * rather than a silent `ok`.
+ *
+ * `crossSource` is the per-entry **second source** of the price/throughput/
+ * context quantities, keyed by the entry's `model_id`: each row's
+ * `input_cost_per_million`/`output_cost_per_million`/`speed_rps`/`context_window`
+ * (each a finite number, else `null`). It is `null` when no entry carried any of
+ * the four. The focus-metric cross-source axis (scoring.md) compares it against
+ * the plugin's own `price`/`throughput`/`context`; it never feeds a weight.
+ */
+export type BenchmarkPayloadMeta = {
+  trust: { selfReported: number; verified: number; covered: number } | null;
+  modality: { multimodal: number; covered: number } | null;
+  provenance: { dominantOrg: string | null; dominantShare: number; orgs: number; covered: number } | null;
+  crossSource: Record<string, { input: number | null; output: number | null; speed: number | null; context: number | null }> | null;
+};
+
+/**
+ * Pure: the per-entry summary of a raw llm-stats benchmark payload, or `null`
+ * when the payload has no llm-stats entries shape. Junk rows are skipped exactly
+ * like `parseLlmStatsBenchmark`; `verified` counts the covered rows (those
+ * carrying a boolean `self_reported`), so `verified <= covered` holds. The four
+ * summaries have independent denominators (a row may carry one flag and not the
+ * others).
+ */
+export function parseBenchmarkPayloadMeta(v: unknown): BenchmarkPayloadMeta | null {
+  if (!isRecord(v) || !Array.isArray(v.entries)) return null;
+  let selfReported = 0;
+  let verified = 0;
+  let covered = 0;
+  let multimodal = 0;
+  let modalityCovered = 0;
+  const orgCounts = new Map<string, number>();
+  let orgCovered = 0;
+  const crossSource: Record<string, { input: number | null; output: number | null; speed: number | null; context: number | null }> = {};
+  let crossSourceCovered = 0;
+  for (const row of v.entries) {
+    if (!isRecord(row)) continue;
+    if (typeof row.self_reported === "boolean") {
+      covered++;
+      if (row.self_reported) selfReported++;
+      if (row.verified === true) verified++;
+    }
+    if (typeof row.multimodal === "boolean") {
+      modalityCovered++;
+      if (row.multimodal) multimodal++;
+    }
+    const org = typeof row.organization_id === "string" && row.organization_id !== "" ? row.organization_id : typeof row.provider_id === "string" && row.provider_id !== "" ? row.provider_id : null;
+    if (org !== null) {
+      orgCovered++;
+      orgCounts.set(org, (orgCounts.get(org) ?? 0) + 1);
+    }
+    if (typeof row.model_id === "string") {
+      const input = typeof row.input_cost_per_million === "number" && Number.isFinite(row.input_cost_per_million) ? row.input_cost_per_million : null;
+      const output = typeof row.output_cost_per_million === "number" && Number.isFinite(row.output_cost_per_million) ? row.output_cost_per_million : null;
+      const speed = typeof row.speed_rps === "number" && Number.isFinite(row.speed_rps) ? row.speed_rps : null;
+      const context = typeof row.context_window === "number" && Number.isFinite(row.context_window) ? row.context_window : null;
+      if (input !== null || output !== null || speed !== null || context !== null) {
+        crossSource[row.model_id] = { input, output, speed, context };
+        crossSourceCovered++;
+      }
+    }
+  }
+  let dominantOrg: string | null = null;
+  let dominantCount = 0;
+  for (const [org, count] of orgCounts) {
+    if (count > dominantCount) {
+      dominantOrg = org;
+      dominantCount = count;
+    }
+  }
+  return {
+    trust: covered === 0 ? null : { selfReported, verified, covered },
+    modality: modalityCovered === 0 ? null : { multimodal, covered: modalityCovered },
+    provenance: orgCovered === 0 ? null : { dominantOrg, dominantShare: dominantCount / orgCovered, orgs: orgCounts.size, covered: orgCovered },
+    crossSource: crossSourceCovered === 0 ? null : crossSource,
+  };
+}
+
 /** One row of the llm-stats benchmark catalog (`GET /leaderboard/benchmarks`). */
 export type BenchmarkCatalogEntry = {
   id: string;
@@ -171,6 +281,39 @@ export type BenchmarkCatalogEntry = {
   description: string;
   categories: string[];
   modelCount: number;
+  /** The catalog row's `is_community` flag (a community-submitted benchmark);
+   * `false` when absent or non-boolean. The only catalog-level trust signal. */
+  isCommunity: boolean;
+  /** The catalog row's `modality` (e.g. `"text"`, `"image"`, `"audio"`,
+   * `"video"`, `"multimodal"`); `null` when absent or non-string. Kept as an
+   * open string (not a closed enum) so a future value survives. */
+  modality: string | null;
+  /** The catalog row's `updated_at` (when the dataset was last updated); `null`
+   * when absent or non-string. Drives the maintenance axis's dataset age
+   * (distinct from the model-date freshness axis). */
+  updatedAt: string | null;
+  /** The catalog row's `version_count` (how many revisions the dataset has);
+   * `null` when absent or non-finite. Supporting detail for the maintenance
+   * axis (churn). */
+  versionCount: number | null;
+  /** The catalog row's `latest_version_row_count` (the current dataset size);
+   * `null` when absent or non-finite. Supporting detail for the maintenance
+   * axis. */
+  latestVersionRowCount: number | null;
+  /** The catalog row's `star_count`; `null` when absent or non-finite.
+   * Supporting detail for the maintenance axis. */
+  starCount: number | null;
+  /** The catalog row's `dataset_id` (the benchmark dataset's id); `null` when
+   * absent or non-string. Provenance detail (the dataset the benchmark is built
+   * on). */
+  datasetId: string | null;
+  /** The catalog row's `dataset_org_id` (the owning organisation of the
+   * benchmark dataset); `null` when absent or non-string. The provenance axis's
+   * owner — who publishes the benchmark. */
+  datasetOrgId: string | null;
+  /** The catalog row's `dataset_slug`; `null` when absent or non-string. The
+   * provenance axis's owner fallback when `dataset_org_id` is absent. */
+  datasetSlug: string | null;
 };
 
 /** Pure: the catalog payload (a top-level array) -> entries; null when unusable. */
@@ -188,14 +331,54 @@ export function parseBenchmarkCatalog(v: unknown): BenchmarkCatalogEntry[] | nul
       description: typeof row.description === "string" ? row.description : "",
       categories: Array.isArray(row.categories) ? row.categories.filter((c): c is string => typeof c === "string") : [],
       modelCount: typeof row.model_count === "number" && Number.isFinite(row.model_count) ? row.model_count : 0,
+      isCommunity: row.is_community === true,
+      modality: typeof row.modality === "string" && row.modality !== "" ? row.modality : null,
+      updatedAt: typeof row.updated_at === "string" ? row.updated_at : null,
+      versionCount: typeof row.version_count === "number" && Number.isFinite(row.version_count) ? row.version_count : null,
+      latestVersionRowCount:
+        typeof row.latest_version_row_count === "number" && Number.isFinite(row.latest_version_row_count) ? row.latest_version_row_count : null,
+      starCount: typeof row.star_count === "number" && Number.isFinite(row.star_count) ? row.star_count : null,
+      datasetId: typeof row.dataset_id === "string" && row.dataset_id !== "" ? row.dataset_id : null,
+      datasetOrgId: typeof row.dataset_org_id === "string" && row.dataset_org_id !== "" ? row.dataset_org_id : null,
+      datasetSlug: typeof row.dataset_slug === "string" && row.dataset_slug !== "" ? row.dataset_slug : null,
     });
   }
   return out;
 }
 
-/** Pure: run a source's parser over a payload. */
-export function parseBenchmarkPayload(source: BenchmarkSource, payload: unknown): Record<string, number> | null {
-  return source.parse(payload);
+/** The llm-stats per-benchmark endpoint's hard cap on `entries`: it returns at
+ * most this many rows regardless of `limit`/`offset`/`page`/`per_page`, so a
+ * generic `bench:<id>` metric can never load more and no pagination loop can
+ * help. `total_models` still reports the full set, so `loaded < total` marks a
+ * capped load. */
+export const BENCHMARK_ENTRY_CAP = 20;
+
+/** A parsed benchmark payload: the joined scores plus the loadable entry count
+ * and the payload's declared total (`total_models`), so the caller can annotate
+ * a capped load. `meta` carries the payload's per-entry trust summary; it is
+ * `null` when the payload has no llm-stats entries shape (a declared or
+ * writing-evidence payload carries no entry flags). */
+export type BenchmarkPayload = {
+  scores: Record<string, number>;
+  /** Entries actually read from the payload — the loadable count. */
+  loaded: number;
+  /** The payload's declared total model count, when it carries one. */
+  total: number | null;
+  /** The payload's per-entry trust summary; `null` when the payload has no
+   * llm-stats entries shape. */
+  meta: BenchmarkPayloadMeta | null;
+};
+
+/** Pure: run a source's parser over a payload, carrying the loadable count (the
+ * post-fetch coverage count) and the payload's declared total so the caller can
+ * annotate `loaded < total` — never the catalog's `model_count`. The trust meta
+ * is read independently of `source.parse`, so a declared/writing source (whose
+ * parser reads no entry flags) still reports `meta: null`. */
+export function parseBenchmarkPayload(source: BenchmarkSource, payload: unknown): BenchmarkPayload | null {
+  const scores = source.parse(payload);
+  if (scores === null) return null;
+  const total = isRecord(payload) && typeof payload.total_models === "number" && Number.isFinite(payload.total_models) ? payload.total_models : null;
+  return { scores, loaded: Object.keys(scores).length, total, meta: parseBenchmarkPayloadMeta(payload) };
 }
 
 // ---------------------------------------------------------------------------
@@ -587,10 +770,11 @@ export function applyBenchmarkScores(models: Model[], source: BenchmarkSource, s
 // Cache chain
 // ---------------------------------------------------------------------------
 
-type ScoresCacheFile = { fetchedAt: string; source: string; scores: Record<string, number> };
+type ScoresCacheFile = { fetchedAt: string; source: string; scores: Record<string, number>; total?: number | null; meta?: BenchmarkPayloadMeta | null };
 
-/** Daily cache: current when fetchedAt is the current UTC day; requireFresh=false accepts stale. */
-function readScoresCache(path: string, requireFresh: boolean): ScoresCacheFile | null {
+/** Daily cache: current when fetchedAt is the current UTC day; requireFresh=false accepts stale.
+ * Exported for the shape-guard test (a cache predating the payload-meta fields must be rejected). */
+export function readScoresCache(path: string, requireFresh: boolean): ScoresCacheFile | null {
   let parsed: ScoresCacheFile;
   try {
     parsed = JSON.parse(readFileSync(path, "utf8")) as ScoresCacheFile;
@@ -598,19 +782,45 @@ function readScoresCache(path: string, requireFresh: boolean): ScoresCacheFile |
     return null;
   }
   if (typeof parsed?.scores !== "object" || parsed.scores === null) return null;
+  // A cache written before the payload-meta fields (#34/#35/#37/#38) landed lacks
+  // the `meta` key; refetch rather than silently reporting `undefined` (the four
+  // payload-derived focus axes would go inert). `meta` is legitimately `null` for
+  // a declared/writing payload, so the guard keys on the key, not its value.
+  if (!("meta" in parsed)) return null;
   if (requireFresh && parsed.fetchedAt?.slice(0, 10) !== new Date().toISOString().slice(0, 10)) return null;
   return parsed;
 }
 
 /** Pretty-printed with sorted keys for scannable diffs, mirroring the other caches. */
-function writeScoresCache(path: string, fetchedAt: string, source: string, scores: Record<string, number>): void {
-  const cache: ScoresCacheFile = { fetchedAt, source, scores };
+function writeScoresCache(path: string, fetchedAt: string, source: string, scores: Record<string, number>, total: number | null, meta: BenchmarkPayloadMeta | null): void {
+  const cache: ScoresCacheFile = { fetchedAt, source, scores, total, meta };
   mkdirSync(dirname(path), { recursive: true });
   writeFileSync(path, JSON.stringify(cache, null, 2));
   console.error(`wrote ${path}`);
 }
 
-export type BenchmarkScores = { scores: Record<string, number>; label: string };
+/** Annotate a capped load: the llm-stats endpoint returns at most
+ * `BENCHMARK_ENTRY_CAP` entries, so the loadable count (the post-fetch coverage
+ * count) is below the payload's `total_models`. */
+function annotateCappedLoad(source: BenchmarkSource, loaded: number, total: number | null): void {
+  if (total !== null && loaded < total) {
+    console.error(
+      `${source.id}: loaded ${loaded}/${total} models — the endpoint caps entries at ${BENCHMARK_ENTRY_CAP}; ${source.metric} covers only the loadable ${loaded}`,
+    );
+  }
+}
+
+export type BenchmarkScores = {
+  scores: Record<string, number>;
+  label: string;
+  /** Entries actually loaded — the post-fetch coverage count, never the catalog count. */
+  loaded: number;
+  /** The payload's declared total model count, when known. */
+  total: number | null;
+  /** The payload's per-entry trust summary; `null` when the payload carried no
+   * entry flags (a declared/writing source) or the cache predates the field. */
+  meta: BenchmarkPayloadMeta | null;
+};
 
 /** Fetch a URL as JSON (the cache chain and the authoring dry-run). Throws on a
  * non-200 or a non-JSON body. */
@@ -646,16 +856,20 @@ export async function loadBenchmarkScores(source: BenchmarkSource, refresh: bool
     const cached = readScoresCache(path, true);
     if (cached) {
       console.error(`using cache ${path} (fetched ${cached.fetchedAt})`);
-      return { scores: cached.scores, label: source.label };
+      const loaded = Object.keys(cached.scores).length;
+      const total = cached.total ?? null;
+      annotateCappedLoad(source, loaded, total);
+      return { scores: cached.scores, label: source.label, loaded, total, meta: cached.meta ?? null };
     }
   }
   if (source.fetch) {
     try {
       const payload = await fetchJson(source.fetch.url, source.fetch.method, source.fetch.body);
-      const scores = parseBenchmarkPayload(source, payload);
-      if (scores && Object.keys(scores).length > 0) {
-        writeScoresCache(path, new Date().toISOString(), source.fetch.url, scores);
-        return { scores, label: source.labelFromPayload?.(payload) ?? source.label };
+      const parsed = parseBenchmarkPayload(source, payload);
+      if (parsed && parsed.loaded > 0) {
+        writeScoresCache(path, new Date().toISOString(), source.fetch.url, parsed.scores, parsed.total, parsed.meta);
+        annotateCappedLoad(source, parsed.loaded, parsed.total);
+        return { scores: parsed.scores, label: source.labelFromPayload?.(payload) ?? source.label, loaded: parsed.loaded, total: parsed.total, meta: parsed.meta };
       }
       console.error(`${source.id}: unexpected payload shape; skipping`);
     } catch (err) {
@@ -665,15 +879,20 @@ export async function loadBenchmarkScores(source: BenchmarkSource, refresh: bool
   const stale = readScoresCache(path, false);
   if (stale) {
     console.error(`using stale cache ${path} (fetched ${stale.fetchedAt})`);
-    return { scores: stale.scores, label: source.label };
+    const loaded = Object.keys(stale.scores).length;
+    const total = stale.total ?? null;
+    annotateCappedLoad(source, loaded, total);
+    return { scores: stale.scores, label: source.label, loaded, total, meta: stale.meta ?? null };
   }
   return null;
 }
 
 type CatalogCacheFile = { fetchedAt: string; source: string; entries: BenchmarkCatalogEntry[] };
 
-/** Daily cache: current when fetchedAt is the current UTC day; requireFresh=false accepts stale. */
-function readCatalogCache(path: string, requireFresh: boolean): CatalogCacheFile | null {
+/** Daily cache: current when fetchedAt is the current UTC day; requireFresh=false accepts stale.
+ * Exported for the shape-guard test (a cache predating `isCommunity`/`modality`, the
+ * maintenance fields or the provenance fields must be rejected). */
+export function readCatalogCache(path: string, requireFresh: boolean): CatalogCacheFile | null {
   let parsed: CatalogCacheFile;
   try {
     parsed = JSON.parse(readFileSync(path, "utf8")) as CatalogCacheFile;
@@ -681,6 +900,27 @@ function readCatalogCache(path: string, requireFresh: boolean): CatalogCacheFile
     return null;
   }
   if (!Array.isArray(parsed?.entries)) return null;
+  // A cache written before `isCommunity`/`modality` (#34/#35), the maintenance
+  // fields `updatedAt`/`versionCount`/`latestVersionRowCount`/`starCount` (#36)
+  // or the provenance fields `datasetId`/`datasetOrgId`/`datasetSlug` (#37)
+  // landed lacks the field; refetch rather than silently reporting `undefined`
+  // (mirrors `readEndpointsCache`). An empty catalog is still `null` (nothing to
+  // sample, nothing to discover).
+  const sample = parsed.entries[0];
+  if (
+    typeof sample !== "object" ||
+    sample === null ||
+    !("isCommunity" in sample) ||
+    !("modality" in sample) ||
+    !("updatedAt" in sample) ||
+    !("versionCount" in sample) ||
+    !("latestVersionRowCount" in sample) ||
+    !("starCount" in sample) ||
+    !("datasetId" in sample) ||
+    !("datasetOrgId" in sample) ||
+    !("datasetSlug" in sample)
+  )
+    return null;
   if (requireFresh && parsed.fetchedAt?.slice(0, 10) !== new Date().toISOString().slice(0, 10)) return null;
   return parsed;
 }
@@ -700,7 +940,7 @@ function writeCatalogCache(path: string, fetchedAt: string, source: string, entr
  * payload is never cached, so the next run retries.
  */
 export async function loadBenchmarkCatalog(refresh: boolean, cacheDir = CACHE_DIR): Promise<BenchmarkCatalogEntry[] | null> {
-  const path = join(cacheDir, "benchmark-catalog-fetched-data.json");
+  const path = join(cacheDir, CATALOG_CACHE_FILE);
   if (!refresh) {
     const cached = readCatalogCache(path, true);
     if (cached) {
@@ -727,6 +967,31 @@ export async function loadBenchmarkCatalog(refresh: boolean, cacheDir = CACHE_DI
   return null;
 }
 
+/** The cached source inputs for one focus metric: the payload's per-entry
+ * summary (trust/modality/provenance/cross-source, from the source's scores
+ * cache) and the catalog row (from the catalog cache). Either half is `null`
+ * when its cache is absent or predates the field. */
+export type FocusSourceInfo = {
+  payload?: BenchmarkPayloadMeta | null;
+  catalog?: BenchmarkCatalogEntry | null;
+};
+
+/**
+ * The cached source inputs for a focus metric, with no network: the source's
+ * scores cache (if any) for the payload meta, and the catalog cache (if any) for
+ * the matching row. The metric is reverse-mapped to a catalog id through
+ * `catalogMetric` (a shipped key via `SHIPPED_CATALOG_METRICS`, else the
+ * `bench:<local>` form), so a shipped metric and a generic `bench:<id>` both
+ * resolve. `null` for either half when the cache is absent or the metric is
+ * unknown to the catalog.
+ */
+export function cachedSourceInfo(metric: string, declared: readonly SourceDeclaration[] = [], cacheDir = CACHE_DIR): FocusSourceInfo {
+  const source = sourceForMetric(metric, declared);
+  const payload = source === null ? null : readScoresCache(join(cacheDir, source.cacheFile), false)?.meta ?? null;
+  const catalog = readCatalogCache(join(cacheDir, CATALOG_CACHE_FILE), false);
+  return { payload, catalog: catalog?.entries.find((entry) => catalogMetric(entry.id) === metric) ?? null };
+}
+
 // ---------------------------------------------------------------------------
 // Dry run (the authoring step's guardrail)
 // ---------------------------------------------------------------------------

package/src/cli/create-role.ts

@@ -20,6 +20,7 @@
 import { homedir } from "node:os";
 import { join } from "node:path";
 import type { RoleDef, SuffixLevel } from "../engine.ts";
+import { FEATURES, featureById, formatFeatures } from "../features.ts";
 import { writeRoleSettings } from "../role-settings.ts";
 
 type CreateRoleArgs = {
@@ -30,6 +31,8 @@ type CreateRoleArgs = {
   description: string;
   image: boolean;
   lambda: number | undefined;
+  features: Record<string, boolean> | undefined;
+  preferOwnProvider: boolean;
   lockPath: string;
   dryRun: boolean;
   json: boolean;
@@ -45,6 +48,10 @@ const USAGE = [
   "  --description <text>   one-line role description",
   "  --image                require image input (filters.image)",
   "  --lambda <n>           explicit λ override ($ per quality point)",
+  "  --feature <id,...>     capability flag(s) to author on the role (see --list-features);",
+  "                         repeatable, comma-separated, merged",
+  "  --prefer-own-provider  bind the role to each model's best provider route where one exists",
+  "  --list-features        print the capability flags and the settings each applies, then exit",
   "  --lock <path>          settings lock file (default ~/.omp/plugins/omp-plugins.lock.json)",
   "  --dry-run              validate and print without writing",
   "  --json                 print the result as JSON",
@@ -84,6 +91,8 @@ function parseArgs(argv: string[]): CreateRoleArgs {
     description: "",
     image: false,
     lambda: undefined,
+    features: undefined,
+    preferOwnProvider: false,
     lockPath: join(homedir(), ".omp", "plugins", "omp-plugins.lock.json"),
     dryRun: false,
     json: false,
@@ -94,6 +103,10 @@ function parseArgs(argv: string[]): CreateRoleArgs {
       args.image = true;
       continue;
     }
+    if (flag === "--prefer-own-provider") {
+      args.preferOwnProvider = true;
+      continue;
+    }
     if (flag === "--dry-run") {
       args.dryRun = true;
       continue;
@@ -102,6 +115,10 @@ function parseArgs(argv: string[]): CreateRoleArgs {
       args.json = true;
       continue;
     }
+    if (flag === "--list-features") {
+      console.log(formatFeatures());
+      process.exit(0);
+    }
     if (flag === "--help" || flag === "-h") {
       console.log(USAGE);
       process.exit(0);
@@ -114,7 +131,16 @@ function parseArgs(argv: string[]): CreateRoleArgs {
     else if (flag === "--thinking") args.thinking = value as SuffixLevel;
     else if (flag === "--description") args.description = value;
     else if (flag === "--lambda") args.lambda = Number(value);
-    else if (flag === "--lock") args.lockPath = value;
+    else if (flag === "--feature") {
+      const ids = parseList(value);
+      if (ids.length === 0) fail(`--feature: no capability id given\n\n${USAGE}`);
+      const features = args.features ?? {};
+      for (const id of ids) {
+        if (featureById(id) === null) fail(`--feature: unknown capability "${id}" — one of ${FEATURES.map((f) => f.id).join(", ")}`);
+        features[id] = true;
+      }
+      args.features = features;
+    } else if (flag === "--lock") args.lockPath = value;
     else fail(`unknown flag "${flag}"\n\n${USAGE}`);
   }
   if (!args.name) fail(`--name is required\n\n${USAGE}`);
@@ -131,6 +157,8 @@ function buildRoleDef(args: CreateRoleArgs): RoleDef {
   if (args.thinking !== undefined) def.thinking = args.thinking;
   if (args.image) def.filters = { image: true };
   if (args.lambda !== undefined) def.lambda = args.lambda;
+  if (args.features !== undefined) def.features = args.features;
+  if (args.preferOwnProvider) def.preferOwnProvider = true;
   return def;
 }
 

package/src/cli/llm-role-rank.ts

@@ -28,8 +28,11 @@
 import { execFile } from "node:child_process";
 import { writeFileSync } from "node:fs";
 import { promisify } from "node:util";
+import { assessFocusMetric, type FocusMetricAssessment } from "../agent-create.ts";
 import { catalogFromOmpModelsJson, enrichThinkingLevels, resolveVariant, type CatalogEntry } from "../availability.ts";
+import { cachedSourceInfo, loadDeclaredSources } from "../benchmark-sources.ts";
 import { computeRankings, loadRankData, paretoFrontier, roleLambda, thinkingPriceFactor, type Model, type Ranked, type RoleDef } from "../engine.ts";
+import { focusMetricsOf } from "../explorer/explain.ts";
 import { DEFAULT_ROLES as ROLES } from "../settings.ts";
 
 const execFileP = promisify(execFile);
@@ -65,6 +68,37 @@ function metricAbbr(metric: string): string {
   return local.slice(0, 10);
 }
 
+/** One focus-metric annotation line: the nine signals, same rule as the report. */
+function focusLine(metric: string, a: FocusMetricAssessment): string {
+  const c = a.coverage;
+  const coverage =
+    c.status === "unknown" ? "unknown" : `${c.covered}/${c.total} (${c.share === null ? "?" : `${(c.share * 100).toFixed(1)}%`}) ${c.status}`;
+  const dispersion = a.dispersion.status === "unknown" ? "unknown" : `${a.dispersion.value?.toFixed(3)} ${a.dispersion.status}`;
+  const composition =
+    a.composition.status === "unknown" ? "unknown" : a.composition.status === "ok" ? "ok" : `omits ${a.composition.omittedOrgs.join(", ")}`;
+  const freshness = a.freshness.status === "unknown" ? "unknown" : `${a.freshness.monthsBehind?.toFixed(1)}mo ${a.freshness.status}`;
+  const trust =
+    a.trust.status === "unknown"
+      ? a.trust.community
+        ? "community"
+        : "unknown"
+      : `${a.trust.selfReported}/${a.trust.covered} ${a.trust.status}${a.trust.community ? " community" : ""}`;
+  const modality =
+    a.modality.status === "unknown"
+      ? "unknown"
+      : `${a.modality.value ?? "?"}${a.modality.multimodalShare === null ? "" : ` ${(a.modality.multimodalShare * 100).toFixed(0)}% multimodal`}`;
+  const maintenance = a.maintenance.status === "unknown" ? "unknown" : `${a.maintenance.monthsOld?.toFixed(1)}mo ${a.maintenance.status}`;
+  const provenance =
+    a.provenance.status === "unknown"
+      ? "unknown"
+      : `${a.provenance.owner ?? "?"}${a.provenance.dominantShare === null ? "" : ` ${(a.provenance.dominantShare * 100).toFixed(0)}% ${a.provenance.dominantOrg ?? "?"}`} ${a.provenance.compositionAgreement}`;
+  const crossSource =
+    a.crossSource.status === "unknown"
+      ? "unknown"
+      : `${a.crossSource.compared} priceΔ${a.crossSource.priceDivergence === null ? "?" : `${(a.crossSource.priceDivergence * 100).toFixed(0)}%`} ctxΔ${a.crossSource.contextDivergence === null ? "?" : `${(a.crossSource.contextDivergence * 100).toFixed(0)}%`} ${a.crossSource.status}`;
+  return `focus: ${metric} — coverage ${coverage}; dispersion ${dispersion}; composition ${composition}; freshness ${freshness}; trust ${trust}; modality ${modality}; maintenance ${maintenance}; provenance ${provenance}; cross-source ${crossSource}`;
+}
+
 function formatRankings(
   rankings: Record<string, Ranked[]>,
   roles: Record<string, RoleDef>,
@@ -85,7 +119,7 @@ function formatRankings(
     "(no ranks): llm-stats index_* affine (v+20)/80 (interval scale, observed −16..+60),",
     "benchmarks chance-anchored pass rates, throughput log-anchored 10..300 tok/s.",
     "q = Σ weight × metric over the quality metrics (weights renormalized excluding",
-    "price); value = q − λ·$/M sorts each role. λ = price-weight share ÷ $20, per-role",
+    "price); value = q − λ·$/M sorts each role. λ = w/(1−w) ÷ $20 (w = price weight), per-role",
     "override via plugin settings roles.<role>.lambda. Metric columns show weighted",
     "contributions and sum to q; — = metric missing (contributes 0).",
     "Roles with a thinking level rank on the thinking-adjusted price: the billed",
@@ -124,6 +158,7 @@ function formatRankings(
   );
   lines.push("");
 
+  const declared = loadDeclaredSources();
   for (const [role, def] of Object.entries(roles)) {
     const ranked = rankings[role] ?? [];
     lines.push(`## @${role} — ${def.description}`);
@@ -189,6 +224,10 @@ function formatRankings(
       `| ${widths.map((w, c) => (c === 3 || c === 4 ? "-".repeat(w) : `${"-".repeat(w - 1)}:`)).join(" | ")} |`,
     );
     for (let ri = 1; ri < padded.length; ri++) lines.push(`| ${padded[ri].join(" | ")} |`);
+    // A role's focus metrics (the benchmarks it was ranked on) carry the same
+    // nine-axis annotation as the create-agent report. Shipped roles weight index
+    // metrics, so this is empty for them.
+    for (const metric of focusMetricsOf(def, declared)) lines.push(focusLine(metric, assessFocusMetric(models, metric, declared, cachedSourceInfo(metric, declared))));
     lines.push("");
   }
   return lines.join("\n");