omp-llm-role 1.0.0 → 1.2.0

package/src/extension.ts

@@ -10,20 +10,26 @@
  */
 
 import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent";
-import { execFile } from "node:child_process";
-import { readFileSync } from "node:fs";
+import { execFile, execFileSync } from "node:child_process";
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { createAgent, CREATE_AGENT_USAGE, discoverBenchmarks, formatArchetypes, formatBenchmarks, formatCreateAgentReport, parseCreateAgentInput, tokenizeArgs, type DiscoveredBenchmark } from "./agent-create.ts";
+import { assessFocusMetric, countMetricCoverage, createAgent, CREATE_AGENT_USAGE, differentiationWarning, discoverBenchmarks, formatArchetypes, formatBenchmarks, formatCreateAgentReport, parseCreateAgentInput, resolveRole, tokenizeArgs, type DiscoveryOutcome, type FocusAssessor, type FocusMetricAssessment } from "./agent-create.ts";
 import { formatRemoveAgentReport, parseRemoveAgentArgs, removeAgent, REMOVE_AGENT_USAGE } from "./agent-remove.ts";
-import { generateAgentSpec } from "./agent-architect.ts";
+import { generateAgentSpec, generateProjectProfile } from "./agent-architect.ts";
 import { authorBenchmarkSource } from "./benchmark-author.ts";
-import { declarationToSource, declaredSourceForLink, dryRunDeclaration, fetchJson, loadBenchmarkCatalog, loadBenchmarkScores, loadDeclaredSources, resolveBenchmarkSource, saveDeclaredSource, validateDeclaration, type BenchmarkCatalogEntry, type BenchmarkSource, type SourceDeclaration } from "./benchmark-sources.ts";
+import { applyBenchmarkScores, BENCHMARK_ENTRY_CAP, cachedSourceInfo, declarationToSource, declaredSourceForLink, dryRunDeclaration, fetchJson, loadBenchmarkCatalog, loadBenchmarkScores, loadDeclaredSources, resolveBenchmarkSource, saveDeclaredSource, sourceForMetric, validateDeclaration, type BenchmarkCatalogEntry, type BenchmarkPayloadMeta, type BenchmarkSource, type SourceDeclaration } from "./benchmark-sources.ts";
 import { judgeBenchmarkRelevance } from "./benchmark-discovery.ts";
-import { THINKING_LEVELS, catalogFromOmpModelsJson } from "./availability.ts";
-import { loadRankData } from "./engine.ts";
+import { catalogFromOmpModelsJson, fetchKeyAvailability, splitThinkingSuffix, type KeyAvailability } from "./availability.ts";
+import { loadRankData, rankRole, type Model } from "./engine.ts";
+import { defaultErrorReportDeps, reportUnexpectedError, type ErrorReportInput } from "./error-report.ts";
+import { formatFeatures } from "./features.ts";
 import { startExplorer, type ExplorerHandle } from "./explorer/boot.ts";
+import { resolveScopes, unionRoles } from "./explorer/scopes.ts";
 import { isRecord } from "./guards.ts";
-import { KNOWN_METRICS, readPluginSettingsMap, resolveSettings } from "./settings.ts";
+import { applyProviderDisplay, providerPinOf, providerPinOfSelector } from "./provider-display.ts";
+import { applyProfileOverrides, formatProjectRolesReport, parseProjectRolesArgs, PROJECT_ROLES_USAGE, setupProject, type ProjectProfile } from "./project-setup.ts";
+import { DEFAULT_ROLES, findProjectAnchor, KNOWN_METRICS, PLUGIN_SETTINGS_PATH, projectLockPath, readPluginSettingsMap, resolveSettings } from "./settings.ts";
 import { runUpdater, type Deps } from "./updater.ts";
 
 /** SPA directory shipped beside this extension (repo `web/`). */
@@ -66,17 +72,41 @@ function hasEmptyConversation(ctx: ExtContext): boolean {
   return true;
 }
 
-/** True when the live session model resolves from the old default selector (same provider/id, any level). */
-function sessionBootedOnOldDefault(ctx: ExtContext, oldSelector: string): boolean {
+/**
+ * True when `selector` names the live session model: the selector's base
+ * (thinking suffix stripped) resolves to a model with the same `provider` and
+ * `id` as `ctx.models.current()`. Any thinking level or provider pin is
+ * ignored. Used both to detect a session booted on the old default role and to
+ * decide whether a fallback event's selector is authoritative for the session
+ * (advisor fallbacks emit the same events with the advisor's selector).
+ */
+function liveModelIsSelector(ctx: ExtContext, selector: string): boolean {
   const current = ctx.models?.current();
-  const colon = oldSelector.lastIndexOf(":");
-  const level = colon !== -1 && oldSelector.slice(colon + 1) in THINKING_LEVELS ? oldSelector.slice(colon + 1) : null;
-  const base = level === null ? oldSelector : oldSelector.slice(0, oldSelector.length - level.length - 1);
-  const oldModel = ctx.models?.resolve(base);
-  if (current === undefined || current === null || oldModel === undefined || oldModel === null) return false;
-  if (typeof current !== "object" || typeof oldModel !== "object") return false;
-  if (!("provider" in current) || !("id" in current) || !("provider" in oldModel) || !("id" in oldModel)) return false;
-  return current.provider === oldModel.provider && current.id === oldModel.id;
+  const { base } = splitThinkingSuffix(selector);
+  const target = ctx.models?.resolve(base);
+  if (current === undefined || current === null || target === undefined || target === null) return false;
+  if (typeof current !== "object" || typeof target !== "object") return false;
+  if (!("provider" in current) || !("id" in current) || !("provider" in target) || !("id" in target)) return false;
+  return current.provider === target.provider && current.id === target.id;
+}
+
+/**
+ * Apply the provider display suffix to the live session model (spec #77).
+ * `selector` is the event's authoritative selector string when the event
+ * carries one; the live model's routing block is the fallback. The event's
+ * selector is trusted only when it names the live session model — the fallback
+ * events are also emitted for advisor fallbacks, where only the advisor's model
+ * changes and the session model is untouched. Contained: a throw must not tear
+ * down the session or escape into omp's dispatch.
+ */
+function applyProviderDisplayFor(ctx: ExtContext, selector?: string): void {
+  try {
+    const live = ctx.models?.current();
+    const eventPin = typeof selector === "string" && liveModelIsSelector(ctx, selector) ? providerPinOfSelector(selector) : null;
+    applyProviderDisplay(live, eventPin ?? providerPinOf(live));
+  } catch {
+    // display-only; never let a failure escape into omp's dispatch
+  }
 }
 
 /**
@@ -92,10 +122,8 @@ function makeApplySessionModel(pi: ExtensionAPI, ctx: ExtContext, notify: (lines
     if (ctx.agent.kind !== "main") return;
     if (!hasEmptyConversation(ctx)) return;
     if (previous === null) return; // first run ever wrote a fresh default; the session predates no old role
-    if (!sessionBootedOnOldDefault(ctx, previous)) return;
-    const colon = selector.lastIndexOf(":");
-    const level = colon !== -1 && selector.slice(colon + 1) in THINKING_LEVELS ? selector.slice(colon + 1) : null;
-    const base = level === null ? selector : selector.slice(0, selector.length - level.length - 1);
+    if (!liveModelIsSelector(ctx, previous)) return;
+    const { base, level } = splitThinkingSuffix(selector);
     const model = ctx.models?.resolve(base);
     if (model === undefined || model === null) {
       notify([`omp-llm-role: session model not updated — "${base}" matches no available model`]);
@@ -160,12 +188,59 @@ function parseExplorerArgs(args: string): { port: number | undefined; open: bool
   return { port: valid, open };
 }
 
-/** Mirror an explorer line to the UI, and to stderr when the mode has no UI. */
-function notifyLines(ctx: ExtContext, line: string): void {
-  ctx.ui.notify(line, "info");
+/** Mirror a line to the UI, and to stderr when the mode has no UI. */
+function notifyLine(ctx: ExtContext, line: string, level: "info" | "warning"): void {
+  ctx.ui.notify(line, level);
   if (!ctx.hasUI) console.error(line);
 }
 
+/** The explorer/authoring line sink: the shared sink at `info`. */
+function notifyLines(ctx: ExtContext, line: string): void {
+  notifyLine(ctx, line, "info");
+}
+
+/** The `ask` posture's yes/no prompt, built from `ctx.ui.select?` — present only
+ *  in a UI session, so a headless `ask` degrades to notify-plus-URL. */
+function reportConfirm(ctx: ExtContext): ((title: string, payload: string) => Promise<boolean>) | undefined {
+  if (!ctx.hasUI || !ctx.ui.select) return undefined;
+  return async (title, payload) => {
+    const answer = await ctx.ui.select?.(`${title}\n\n${payload}`, [
+      { label: "File", description: "create the GitHub issue with the text above" },
+      { label: "Cancel" },
+    ]);
+    return answer === "File";
+  };
+}
+
+/** The reporting module's input for one failure: the shared line sink and the
+ *  `ask` prompt, both bound to this context. */
+function reportInput(ctx: ExtContext, label: string, error: unknown): ErrorReportInput {
+  return {
+    label,
+    error,
+    hasUI: ctx.hasUI,
+    notify: (line, level) => notifyLine(ctx, line, level),
+    confirm: reportConfirm(ctx),
+  };
+}
+
+/** Route an unexpected command failure through the reporting module. The
+ *  one-line summary is unchanged from the pre-reporting behaviour; the module
+ *  adds its own lines (filed/commented/offered) when reporting is on. */
+function reportCommandError(ctx: ExtContext, label: string, err: unknown, prefix = "llm-role"): void {
+  ctx.ui.notify(`${prefix}: ${err instanceof Error ? err.message : err}`, "warning");
+  void reportUnexpectedError(reportInput(ctx, label, err), defaultErrorReportDeps());
+}
+
+/** Report an updater defect (the abort line is already notified by runUpdater).
+ *  The defect is passed as a string, not wrapped in an `Error`: the module then
+ *  derives the failure site from the message, so each distinct abort gets its
+ *  own fingerprint instead of collapsing onto this construction site. */
+function reportDefect(ctx: ExtContext, label: string, defect: string | undefined): void {
+  if (defect === undefined) return;
+  void reportUnexpectedError(reportInput(ctx, label, defect), defaultErrorReportDeps());
+}
+
 /** External metrics already weighted by a resolved role, for `--list-benchmarks`. */
 function externalMetricsInUse(): string[] {
   const { settings } = resolveSettings(readPluginSettingsMap());
@@ -252,11 +327,23 @@ async function resolveBenchmarkLinks(ctx: ExtContext, links: readonly string[],
       source = await authorLink(ctx, link, yes);
       if (source === null) return null;
     }
+    // A generic llm-stats benchmark's raw id lives only in its declaration (the
+    // metric key folds `.`→`_`), so persist it before the update run — otherwise
+    // the updater's metric-key fallback fetches the wrong, 404-ing URL.
+    const declaration = source.declaration;
+    if (declaration !== undefined && source.labelFromPayload !== undefined && !declared.some((d) => d.metric === declaration.metric)) {
+      const saved = saveDeclaredSource(declaration);
+      if (saved.ok) declared.push(declaration);
+      else notifyLines(ctx, `create-agent: could not persist ${source.metric} source: ${saved.error}`);
+    }
     const loaded = await loadBenchmarkScores(source, false);
     if (loaded === null) {
       notifyLines(ctx, `create-agent: ${source.metric} (${source.label}) — no data fetched; the role will rank on its other weights`);
     } else {
-      notifyLines(ctx, `create-agent: ${source.metric} (${loaded.label}) — ${Object.keys(loaded.scores).length} rows`);
+      notifyLines(
+        ctx,
+        `create-agent: ${source.metric} (${loaded.label}) — ${loaded.loaded} rows${loaded.total !== null && loaded.loaded < loaded.total ? ` of ${loaded.total} (endpoint caps entries at ${BENCHMARK_ENTRY_CAP})` : ""}`,
+      );
     }
     metrics.push(source.metric);
     labels.push(loaded?.label ?? source.label);
@@ -267,9 +354,10 @@ async function resolveBenchmarkLinks(ctx: ExtContext, links: readonly string[],
 /**
  * Discover the catalog benchmarks relevant to `purpose` (non-fatal): a catalog
  * fetch or judge failure just skips discovery, so the command still creates the
- * role. Returns null when discovery could not run.
+ * role. `fieldSize` is the ranking universe's model count (null when unknown),
+ * for the coverage gate. Returns null when discovery could not run.
  */
-async function discoverForPurpose(ctx: ExtContext, purpose: string, exclude: readonly string[]): Promise<DiscoveredBenchmark[] | null> {
+async function discoverForPurpose(ctx: ExtContext, purpose: string, exclude: readonly string[], fieldSize: number | null, assess: FocusAssessor): Promise<DiscoveryOutcome | null> {
   let catalog: BenchmarkCatalogEntry[] | null;
   try {
     catalog = await loadBenchmarkCatalog(false);
@@ -282,13 +370,57 @@ async function discoverForPurpose(ctx: ExtContext, purpose: string, exclude: rea
     return null;
   }
   try {
-    return await discoverBenchmarks(purpose, catalog, (p, candidates) => judgeBenchmarkRelevance(p, candidates, ctx.cwd), exclude);
+    return await discoverBenchmarks(purpose, catalog, (p, candidates) => judgeBenchmarkRelevance(p, candidates, ctx.cwd), exclude, fieldSize, assess);
   } catch (err) {
     notifyLines(ctx, `create-agent: benchmark discovery failed (${err instanceof Error ? err.message : err}) — skipping`);
     return null;
   }
 }
 
+/** Bounded repository artifacts for the `/project-roles` profile architect:
+ * README, package.json, AGENTS.md, a docs listing, the recent git log and a
+ * shallow file tree. Every read is best-effort — a missing file or a non-git cwd
+ * just contributes nothing. */
+function gatherProjectArtifacts(cwd: string): string {
+  const parts: string[] = [];
+  const readBounded = (rel: string, cap = 4000): string | null => {
+    try {
+      return readFileSync(join(cwd, rel), "utf8").slice(0, cap);
+    } catch {
+      return null;
+    }
+  };
+  for (const rel of ["README.md", "package.json", "AGENTS.md", "pyproject.toml", "Cargo.toml", "go.mod"]) {
+    const text = readBounded(rel);
+    if (text !== null) parts.push(`### ${rel}\n${text}`);
+  }
+  try {
+    const docs = readdirSync(join(cwd, "docs"), { recursive: true })
+      .map(String)
+      .filter((p) => p.endsWith(".md"))
+      .slice(0, 40);
+    if (docs.length > 0) parts.push(`### docs/\n${docs.join("\n")}`);
+  } catch {
+    // no docs dir
+  }
+  try {
+    const log = execFileSync("git", ["log", "--oneline", "-20"], { cwd, encoding: "utf8", timeout: 5000 });
+    parts.push(`### git log\n${log}`);
+  } catch {
+    // not a git repo
+  }
+  try {
+    const entries = readdirSync(cwd, { withFileTypes: true })
+      .filter((entry) => !entry.name.startsWith(".") && entry.name !== "node_modules")
+      .map((entry) => (entry.isDirectory() ? `${entry.name}/` : entry.name))
+      .slice(0, 60);
+    parts.push(`### files\n${entries.join("\n")}`);
+  } catch {
+    // unreadable cwd
+  }
+  return parts.join("\n\n");
+}
+
 export default function (pi: ExtensionAPI) {
   /** Loopback explorer owned by this session binding (null until launched). */
   let explorer: ExplorerHandle | null = null;
@@ -302,19 +434,41 @@ export default function (pi: ExtensionAPI) {
     // mid-loop, no write). runUpdater catches its own abort paths; a throw
     // here is contained by handler dispatch (extension error channel).
     try {
-      await runUpdater("session-start", extDeps(pi, ctx));
+      const runResult = await runUpdater("session-start", extDeps(pi, ctx));
+      reportDefect(ctx, "session-start", runResult.defect);
     } catch (err) {
-      ctx.ui.notify(`llm-role: ${err instanceof Error ? err.message : err}`, "warning");
+      reportCommandError(ctx, "session-start", err);
     }
   });
 
+  // Provider display (spec #77): keep the live model's name suffixed with its
+  // provider pin. Never pi.setModel — that would wipe omp's in-flight fallback
+  // state and clobber the thinking level; mutate the live model's name only.
+  // Registered after the updater so the session-start pin reflects the model
+  // the run just selected.
+  pi.on("session_start", async (_event, ctx: ExtContext) => {
+    applyProviderDisplayFor(ctx);
+  });
+  pi.on("retry_fallback_applied", async (event, ctx: ExtContext) => {
+    applyProviderDisplayFor(ctx, event.to);
+  });
+  pi.on("retry_fallback_succeeded", async (event, ctx: ExtContext) => {
+    applyProviderDisplayFor(ctx, event.model);
+  });
+  // Self-healing hook: omp restores the primary after a fallback cooldown with
+  // no session event, so re-derive the pin from the live model before each turn.
+  pi.on("turn_start", async (_event, ctx: ExtContext) => {
+    applyProviderDisplayFor(ctx);
+  });
+
   pi.registerCommand("refresh-roles", {
     description: "Refresh model roles from today's llm-stats/OpenRouter rankings",
     handler: async (_args, ctx: ExtContext) => {
       try {
-        await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        const runResult = await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        reportDefect(ctx, "refresh-roles", runResult.defect);
       } catch (err) {
-        ctx.ui.notify(`llm-role: ${err instanceof Error ? err.message : err}`, "warning");
+        reportCommandError(ctx, "refresh-roles", err);
       }
     },
   });
@@ -333,13 +487,35 @@ export default function (pi: ExtensionAPI) {
           return;
         }
         notifyLines(ctx, "llm-role explorer: loading today's rankings…");
-        const rank = await loadRankData({});
+        // The explorer ranks the resolved roles of every known scope, so its
+        // dataset must carry every metric any of them weights — including
+        // generic llm-stats benchmarks (`bench:<id>`) that only a project role
+        // pulls in. `loadRankData({})` loads declared sources only, so a role
+        // weighting an undeclared benchmark would rank on a dataset missing that
+        // metric and the explorer would disagree with the updater. Pass the
+        // union of every scope's resolved roles plus the shipped keys, so the
+        // dataset stays a superset of any def the UI can rank (the UI can weight
+        // `website`/`writing` even when no role does) and a scope switch is free.
+        const scopeInputs = { userLockPath: PLUGIN_SETTINGS_PATH, cwd: ctx.cwd };
+        const explorerRoles = () => unionRoles(resolveScopes(scopeInputs), PLUGIN_SETTINGS_PATH);
+        const rank = await loadRankData({ roles: explorerRoles(), extraMetrics: Object.keys(KNOWN_METRICS) });
         const catalog = await extDeps(pi, ctx).getCatalog();
+        // Best-effort key-availability overlay: a missing key or a failed fetch
+        // degrades to "unavailable" and never blocks Explorer boot.
+        const loadAvailability = async (): Promise<KeyAvailability> => {
+          const token = await ctx.modelRegistry.getApiKeyForProvider("openrouter").catch(() => undefined);
+          return token
+            ? fetchKeyAvailability(token)
+            : { active: false, reason: "unavailable", allowed: new Set(), blocked: new Set(), publicCount: 0, keyedCount: 0, fetchedAt: new Date().toISOString() };
+        };
         explorer = await startExplorer({
           webDir: WEB_DIR,
+          cwd: ctx.cwd,
           rank,
           catalog,
-          reload: (refresh) => loadRankData({ refresh }),
+          availability: await loadAvailability(),
+          reloadAvailability: loadAvailability,
+          reload: (refresh) => loadRankData({ refresh, roles: explorerRoles(), extraMetrics: Object.keys(KNOWN_METRICS) }),
           port,
           open,
           unref: true,
@@ -347,7 +523,7 @@ export default function (pi: ExtensionAPI) {
         });
         notifyLines(ctx, `llm-role explorer: ${explorer.url}`);
       } catch (err) {
-        ctx.ui.notify(`llm-role explorer: ${err instanceof Error ? err.message : err}`, "warning");
+        reportCommandError(ctx, "explore-roles", err, "llm-role explorer");
       }
     },
   });
@@ -376,6 +552,10 @@ export default function (pi: ExtensionAPI) {
         notifyLines(ctx, formatBenchmarks(undefined, externalMetricsInUse()));
         return;
       }
+      if (parsed.listFeatures) {
+        notifyLines(ctx, formatFeatures());
+        return;
+      }
       if (parsed.bodyFile !== undefined) {
         try {
           parsed.request.body = readFileSync(parsed.bodyFile, "utf8");
@@ -397,15 +577,64 @@ export default function (pi: ExtensionAPI) {
         parsed.request.benchmarkLabels = resolved.labels;
       }
 
+      // 1a. Load today's ranking universe once: the coverage denominator, the
+      //     per-metric coverage of named benchmarks, and the differentiation
+      //     comparison all need it. Cached (the updater re-reads the same daily
+      //     cache), and non-fatal — an unknown field size skips the share rule.
+      let rankModels: Model[] | null = null;
+      try {
+        rankModels = (await loadRankData({})).models;
+      } catch (err) {
+        notifyLines(ctx, `create-agent: ranking universe unavailable (${err instanceof Error ? err.message : err}) — coverage share checks skipped`);
+      }
+      const fieldSize = rankModels === null ? null : rankModels.length;
+
+      // 1a-bis. The focus-metric gate must not run on the score-less create-agent
+      //     pool: it carries no `bench:<id>` scores, so a gate over it would see
+      //     nothing for exactly the benchmarks it targets. Probe-fetch each
+      //     candidate's source, join it onto a copy of the pool, and assess the
+      //     joined pool. A metric whose scores were not loaded reports every new
+      //     axis `unknown` (never a silent `ok`).
+      const declared = loadDeclaredSources();
+      const focusPool: Model[] = rankModels === null ? [] : rankModels.map((m) => ({ ...m, metrics: { ...m.metrics } }));
+      const assess: FocusAssessor = async (metric) => {
+        const source = sourceForMetric(metric, declared);
+        let payload: BenchmarkPayloadMeta | null = null;
+        if (source !== null) {
+          const loaded = await loadBenchmarkScores(source, false);
+          if (loaded !== null) {
+            applyBenchmarkScores(focusPool, source, loaded.scores);
+            payload = loaded.meta;
+          }
+        }
+        return assessFocusMetric(focusPool, metric, declared, { payload, catalog: cachedSourceInfo(metric, declared).catalog });
+      };
+
       // 1b. Discover the catalog benchmarks relevant to the purpose (unless
       //     --no-discover or an explicit --benchmarks list was given). Non-fatal:
-      //     a catalog or judge failure just skips discovery.
+      //     a catalog or judge failure just skips discovery. A candidate whose
+      //     coverage is too low to rank on is dropped, not folded in.
+      const discoveredCovered = new Map<string, number>();
       if (!parsed.noDiscover && !parsed.explicitBenchmarks && parsed.request.purpose.trim() !== "") {
-        const discovered = await discoverForPurpose(ctx, parsed.request.purpose, parsed.request.extraBenchmarks ?? []);
-        if (discovered !== null && discovered.length > 0) {
-          parsed.request.extraBenchmarks = [...(parsed.request.extraBenchmarks ?? []), ...discovered.map((d) => d.metric)];
-          parsed.request.benchmarkLabels = [...(parsed.request.benchmarkLabels ?? []), ...discovered.map((d) => d.label)];
-          notifyLines(ctx, `create-agent: discovered benchmarks: ${discovered.map((d) => `${d.label} (${d.metric})`).join(", ")}`);
+        const outcome = await discoverForPurpose(ctx, parsed.request.purpose, parsed.request.extraBenchmarks ?? [], fieldSize, assess);
+        if (outcome !== null) {
+          if (outcome.dropped.length > 0) {
+            notifyLines(ctx, `create-agent: dropped benchmarks: ${outcome.dropped.map((d) => `${d.label} (${d.metric}) — ${d.reason}`).join(", ")}`);
+          }
+          if (outcome.discovered.length > 0) {
+            parsed.request.extraBenchmarks = [...(parsed.request.extraBenchmarks ?? []), ...outcome.discovered.map((d) => d.metric)];
+            parsed.request.benchmarkLabels = [...(parsed.request.benchmarkLabels ?? []), ...outcome.discovered.map((d) => d.label)];
+            for (const d of outcome.discovered) discoveredCovered.set(d.metric, d.covered);
+            // A dotted id's raw form cannot be reconstructed from its metric key,
+            // so persist its declaration now — otherwise the updater fetches the
+            // normalized (404-ing) URL and the metric is dead weight.
+            for (const d of outcome.discovered) {
+              if (d.declaration === undefined) continue;
+              const saved = saveDeclaredSource(d.declaration);
+              if (!saved.ok) notifyLines(ctx, `create-agent: could not persist ${d.metric} source: ${saved.error}`);
+            }
+            notifyLines(ctx, `create-agent: discovered benchmarks: ${outcome.discovered.map((d) => `${d.label} (${d.metric})`).join(", ")}`);
+          }
         }
       }
 
@@ -441,7 +670,10 @@ export default function (pi: ExtensionAPI) {
       //    user can see what is already in use and avoid duplicates. `--benchmarks`
       //    covers headless runs, where there is no prompt.
       if (ctx.hasUI && ctx.ui.input && parsed.request.extraBenchmarks === undefined) {
-        notifyLines(ctx, formatBenchmarks(undefined, externalMetricsInUse()));
+        // Preview the request's resolved weights so the list marks what the role
+        // already weights (the "already in this role's weights" marker).
+        const preview = resolveRole(parsed.request);
+        notifyLines(ctx, formatBenchmarks("errors" in preview ? undefined : preview.def.weights, externalMetricsInUse()));
         const answer = await ctx.ui.input(
           "Additional benchmarks",
           "comma-separated metric names to add to the weights, or empty",
@@ -454,18 +686,201 @@ export default function (pi: ExtensionAPI) {
         }
       }
 
+      // 3b. Assess the focus metrics: a discovered metric's coverage is the
+      //     catalog's model count; a named metric's is the loaded models that
+      //     carry it. The nine-axis assessment is computed over the joined pool
+      //     (the same object the gate and the report consume).
+      const covered: Record<string, number> = {};
+      const assessments: Record<string, FocusMetricAssessment> = {};
+      for (const metric of parsed.request.extraBenchmarks ?? []) {
+        assessments[metric] = await assess(metric);
+        covered[metric] = discoveredCovered.get(metric) ?? countMetricCoverage(focusPool, metric, declared);
+      }
+      parsed.request.coverage = { total: fieldSize, covered, declared, assessments };
+
       // 4. Write the role + the agent file, then wire `modelRoles.<name>` in-process.
       const result = createAgent(parsed.request);
       if (!result.ok) {
         notifyLines(ctx, `create-agent: ${result.errors.join("\n")}`);
         return;
       }
-      notifyLines(ctx, formatCreateAgentReport(result, "ranking the new role…"));
+      const defaultDef = resolveSettings(readPluginSettingsMap({ global: parsed.request.lockPath, project: null })).settings.roles.default;
+      const differentiation = rankModels !== null && defaultDef !== undefined ? differentiationWarning(result.def, defaultDef, rankModels) : null;
+      if (parsed.json) notifyLines(ctx, JSON.stringify({ ...result, differentiation }, null, 2));
+      else notifyLines(ctx, formatCreateAgentReport(result, "ranking the new role…", { differentiation }));
       if (result.dryRun) return;
       try {
-        await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        const runResult = await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        reportDefect(ctx, "create-agent", runResult.defect);
+      } catch (err) {
+        reportCommandError(ctx, "create-agent", err);
+      }
+    },
+  });
+
+  // Discover the project's usecase from its own artifacts and apply a
+  // project-scoped role set: the project plugin settings, the new roles' agents,
+  // and the project's modelRoles in <cwd>/.omp/config.yml. The project-aware
+  // updater (run in-process at the end) keeps the global config untouched.
+  pi.registerCommand("project-roles", {
+    description: "Discover the project's usecase and apply a project-scoped role set",
+    handler: async (args, ctx: ExtContext) => {
+      const parsed = parseProjectRolesArgs(typeof args === "string" ? args : "");
+      if (!parsed.ok) {
+        notifyLines(ctx, parsed.error);
+        return;
+      }
+      if (parsed.help) {
+        notifyLines(ctx, PROJECT_ROLES_USAGE);
+        return;
+      }
+
+      const cwd = ctx.cwd;
+      const projectDir = join(cwd, ".omp");
+      const lockPath = projectLockPath(cwd);
+      const configPath = join(projectDir, "config.yml");
+
+      // omp reads <cwd>/.omp with no walk-up; the plugin's walk-up anchor may
+      // differ (a session launched in a subdirectory). The written config is what
+      // omp reads, but the anchor would read/write a different .omp — warn.
+      const warnings: string[] = [];
+      const anchor = findProjectAnchor(cwd);
+      if (anchor !== null && anchor !== cwd) {
+        warnings.push(
+          `the session cwd (${cwd}) is not the project anchor (${anchor}); the config is written to ${projectDir}, which is what omp reads — the plugin's walk-up anchor would read a different .omp`,
+        );
+      }
+
+      // 1. Gather the repo artifacts and run the profile architect.
+      notifyLines(ctx, "project-roles: reading the project…");
+      let profile: ProjectProfile;
+      try {
+        profile = await generateProjectProfile({
+          artifacts: gatherProjectArtifacts(cwd),
+          cwd,
+          model: ctx.models?.current(),
+          modelRegistry: ctx.modelRegistry,
+          purpose: parsed.purpose,
+        });
+      } catch (err) {
+        notifyLines(ctx, `project-roles: profile discovery failed: ${err instanceof Error ? err.message : err}`);
+        return;
+      }
+      profile = applyProfileOverrides(profile, { purpose: parsed.purpose, roles: parsed.roles });
+
+      // 2. Load the ranking universe once: the coverage denominator, the focus
+      //    assessment pool and the top picks all need it. Non-fatal.
+      let rankModels: Model[] | null = null;
+      try {
+        rankModels = (await loadRankData({})).models;
+      } catch (err) {
+        notifyLines(ctx, `project-roles: ranking universe unavailable (${err instanceof Error ? err.message : err}) — coverage checks skipped`);
+      }
+      const fieldSize = rankModels === null ? null : rankModels.length;
+      const declared = loadDeclaredSources();
+      const focusPool: Model[] = rankModels === null ? [] : rankModels.map((m) => ({ ...m, metrics: { ...m.metrics } }));
+      const assess: FocusAssessor = async (metric) => {
+        const source = sourceForMetric(metric, declared);
+        let payload: BenchmarkPayloadMeta | null = null;
+        if (source !== null) {
+          const loaded = await loadBenchmarkScores(source, false);
+          if (loaded !== null) {
+            applyBenchmarkScores(focusPool, source, loaded.scores);
+            payload = loaded.meta;
+          }
+        }
+        return assessFocusMetric(focusPool, metric, declared, { payload, catalog: cachedSourceInfo(metric, declared).catalog });
+      };
+
+      // 3. Per new role: discover benchmarks (judge-backed) and author the agent.
+      const covered: Record<string, number> = {};
+      const assessments: Record<string, FocusMetricAssessment> = {};
+      for (const role of profile.roles) {
+        if (role.keep === false) continue;
+        if (role.name.trim() in DEFAULT_ROLES) continue; // shipped role — no new agent
+        if (role.benchmarks === undefined || role.benchmarks.length === 0) {
+          const outcome = await discoverForPurpose(ctx, role.purpose, [], fieldSize, assess);
+          if (outcome !== null) {
+            if (outcome.dropped.length > 0) {
+              notifyLines(ctx, `project-roles: dropped benchmarks (coverage): ${outcome.dropped.map((d) => `${d.label} (${d.metric}) — ${d.reason}`).join(", ")}`);
+            }
+            role.benchmarks = outcome.discovered.map((d) => d.metric);
+            for (const d of outcome.discovered) {
+              if (d.declaration === undefined) continue;
+              const saved = saveDeclaredSource(d.declaration);
+              if (!saved.ok) notifyLines(ctx, `project-roles: could not persist ${d.metric} source: ${saved.error}`);
+            }
+          }
+        }
+        for (const metric of role.benchmarks ?? []) {
+          assessments[metric] = await assess(metric);
+          covered[metric] = rankModels === null ? 0 : countMetricCoverage(focusPool, metric, declared);
+        }
+        try {
+          role.spec = await generateAgentSpec({
+            description: role.purpose,
+            cwd,
+            model: ctx.models?.current(),
+            modelRegistry: ctx.modelRegistry,
+            benchmarks: role.benchmarks,
+          });
+        } catch (err) {
+          notifyLines(ctx, `project-roles: architect failed for "${role.name}": ${err instanceof Error ? err.message : err}`);
+          return;
+        }
+      }
+
+      // 4. Plan first (a dry run of the same all-or-nothing core), print it, then
+      //    apply. `--dry-run` stops after the plan; without `--force` an existing
+      //    project role config is refused by the core.
+      const planOpts = {
+        projectDir,
+        lockPath,
+        configPath,
+        force: parsed.force,
+        yes: parsed.yes,
+        features: parsed.features,
+        coverage: { total: fieldSize, covered, declared, assessments },
+      };
+      const plan = setupProject(profile, { ...planOpts, dryRun: true });
+      if (!plan.ok) {
+        notifyLines(ctx, `project-roles: ${plan.errors.join("\n")}`);
+        return;
+      }
+      const topPicks: Record<string, string> = {};
+      if (rankModels !== null) {
+        for (const name of plan.added) {
+          const ranked = rankRole(plan.roles[name], rankModels);
+          if (ranked.length > 0) topPicks[name] = ranked[0].model.id;
+        }
+      }
+      if (parsed.json) notifyLines(ctx, JSON.stringify({ profile, ...plan, warnings }, null, 2));
+      else notifyLines(ctx, formatProjectRolesReport(profile, plan, { topPicks, warnings }));
+      if (parsed.dryRun) return;
+
+      if (!parsed.yes && ctx.hasUI && ctx.ui.select) {
+        const answer = await ctx.ui.select("Apply this project role set?", [
+          { label: "Apply", description: "write the project settings, agents and modelRoles" },
+          { label: "Cancel" },
+        ]);
+        if (answer !== "Apply") {
+          notifyLines(ctx, "project-roles: cancelled");
+          return;
+        }
+      }
+
+      const result = setupProject(profile, { ...planOpts, dryRun: false });
+      if (!result.ok) {
+        notifyLines(ctx, `project-roles: ${result.errors.join("\n")}`);
+        return;
+      }
+
+      // 5. Run the updater in-process so the project's modelRoles land.
+      try {
+        const runResult = await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        reportDefect(ctx, "project-roles", runResult.defect);
       } catch (err) {
-        ctx.ui.notify(`llm-role: ${err instanceof Error ? err.message : err}`, "warning");
+        reportCommandError(ctx, "project-roles", err);
       }
     },
   });
@@ -503,9 +918,10 @@ export default function (pi: ExtensionAPI) {
       notifyLines(ctx, formatRemoveAgentReport(result, "cleaning up config.yml…"));
       if (result.dryRun) return;
       try {
-        await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        const runResult = await runUpdater("manual", extDeps(pi, ctx), { force: true });
+        reportDefect(ctx, "remove-agent", runResult.defect);
       } catch (err) {
-        ctx.ui.notify(`llm-role: ${err instanceof Error ? err.message : err}`, "warning");
+        reportCommandError(ctx, "remove-agent", err);
       }
     },
   });