agent-doc-system 1.1.0 → 1.3.0

package/evals/harness.mjs

@@ -4,14 +4,8 @@
 // an agent through it retrieve the right context, and less noise than unassisted
 // repository exploration".
 //
-// Two retrieval systems are measured on the same scenarios:
-//
-//   baseline  what an agent gets by exploring the repository the obvious way:
-//             every documentation file, every descriptor, every source file
-//             under the touched component. This is the honest shape of "just
-//             read the repo", and it is what the catalog has to beat.
-//
-//   catalog   what `agentdoc query` returns for the same scenario.
+// Compare catalog routing with a graph-assisted targeted reference and a
+// working-tree search baseline. The full graph is only a size yardstick.
 //
 // Retrieval-set overlap against a per-scenario ground truth gives precision,
 // recall, critical-constraint recall, verification-command recall and noise.
@@ -28,31 +22,35 @@ import { AgentDocError, CODES } from "../core/codes.mjs";
 import { queryContext, ContextIndex } from "../core/query.mjs";
 import { impact } from "../core/impact.mjs";
 
+import { fact, addRelationFacts, addGraphRelations } from "./facts.mjs";
+import { searchContext } from "./baseline-search.mjs";
+
 const HERE = path.dirname(fileURLToPath(import.meta.url));
 
 // A retrieved item is a "fact reference" the agent would have to understand:
 // an entity ref, a constraint path, a verification command, a contract ref.
 export function factsFromContext(ctx) {
   const out = new Set();
-  out.add(ctx.entity.ref);
-  for (const r of ctx.relations.items) out.add(r.type + ":" + r.other);
-  for (const p of ctx.journeyPeers || []) out.add(p);
-  for (const s2 of ctx.placement.systems) out.add(s2);
-  for (const d2 of ctx.placement.domains) out.add(d2);
-  for (const a of ctx.apis.provided) out.add("provides:" + a.api);
-  for (const a of ctx.apis.consumed) out.add("consumes:" + a.api);
-  for (const c of ctx.apis.consumers || []) out.add("consumer:" + c);
-  if (ctx.apis.provider) out.add("provider:" + ctx.apis.provider);
-  for (const r of ctx.resources.used) out.add("uses:" + r);
-  for (const r of ctx.resources.usedBy) out.add("usedBy:" + r);
-  for (const e of ctx.events.items) out.add("event:" + e.pattern);
-  for (const e of ctx.externalDependencies.items) out.add("external:" + e.name);
-  for (const v of ctx.verification.items) out.add("verify:" + v.command);
-  for (const p of ctx.constraints.constraints) out.add("constraint:" + p);
-  for (const d of ctx.constraints.docs) out.add("doc:" + d);
-  for (const r of ctx.constraints.runbooks) out.add("runbook:" + r);
-  for (const c of ctx.authority.conflicts.items) out.add("conflict:" + c.key);
-  for (const o of ctx.authority.observations) out.add("observation:" + o.id);
+  out.add(fact("ref", ctx.entity.ref));
+  for (const n of ctx.neighbours) out.add(fact("ref", n.ref));
+  for (const r of ctx.relations.items) addRelationFacts(out, r.type, r.other, r.direction);
+  for (const p of ctx.journeyPeers || []) out.add(fact("ref", p));
+  for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
+  for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
+  for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
+  for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
+  for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
+  if (ctx.apis.provider) out.add(fact("provider", ctx.apis.provider));
+  for (const r of ctx.resources.used) out.add(fact("uses", r));
+  for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
+  for (const e of ctx.events.items) out.add(fact("event", e.pattern));
+  for (const e of ctx.externalDependencies.items) out.add(fact("external", e.name));
+  for (const v of ctx.verification.items) out.add(fact("verify", v.command));
+  for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
+  for (const d of ctx.constraints.docs) out.add(fact("doc", d));
+  for (const r of ctx.constraints.runbooks) out.add(fact("runbook", r));
+  for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
+  for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
   return out;
 }
 
@@ -61,25 +59,28 @@ export function factsFromContext(ctx) {
 // resource, an API boundary, or an event subject it publishes or consumes.
 export function criticalFromContext(ctx) {
   const out = new Set();
-  for (const e of ctx.events.items) out.add("event:" + e.pattern);
-  for (const p of ctx.constraints.constraints) out.add("constraint:" + p);
-  for (const c of ctx.authority.conflicts.items) out.add("conflict:" + c.key);
-  for (const o of ctx.authority.observations) out.add("observation:" + o.id);
-  for (const r of ctx.resources.used) out.add("uses:" + r);
-  for (const r of ctx.resources.usedBy) out.add("usedBy:" + r);
-  for (const a of ctx.apis.consumed) out.add("consumes:" + a.api);
-  for (const s2 of ctx.placement.systems) out.add(s2);
-  for (const d2 of ctx.placement.domains) out.add(d2);
-  for (const a of ctx.apis.provided) out.add("provides:" + a.api);
-  for (const c of ctx.apis.consumers || []) out.add("consumer:" + c);
+  for (const e of ctx.events.items) out.add(fact("event", e.pattern));
+  for (const p of ctx.constraints.constraints) out.add(fact("constraint", p));
+  for (const c of ctx.authority.conflicts.items) out.add(fact("conflict", c.key));
+  for (const o of ctx.authority.observations) out.add(fact("observation", o.id));
+  for (const r of ctx.resources.used) out.add(fact("uses", r));
+  for (const r of ctx.resources.usedBy) out.add(fact("usedBy", r));
+  for (const a of ctx.apis.consumed) out.add(fact("consumes", a.api));
+  for (const s2 of ctx.placement.systems) out.add(fact("ref", s2));
+  for (const d2 of ctx.placement.domains) out.add(fact("ref", d2));
+  for (const a of ctx.apis.provided) out.add(fact("provides", a.api));
+  for (const c of ctx.apis.consumers || []) out.add(fact("consumer", c));
   return out;
 }
 
 export function verificationFromContext(ctx) {
-  return new Set(ctx.verification.items.map((v) => "verify:" + v.command));
+  // Commands such as `go test ./...` recur in multiple modules. Scenarios can
+  // require a scoped check ID as well as a command to measure dependent checks
+  // without letting the library's identical command satisfy the expectation.
+  return new Set(ctx.verification.items.flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
 }
 
-// ── baseline: unassisted repository exploration ────────────────────────
+// ── baseline-targeted: graph-assisted reference ────────────────────────
 export function baselineContext(graph, touch) {
   const idx = new ContextIndex(graph);
   const seeds = touch.map((t) => idx.resolve(t)).filter(Boolean);
@@ -90,71 +91,73 @@ export function baselineContext(graph, touch) {
     const inSeed = seeds.includes(e);
     const underRoot = roots.some((r) => e.ref && r && d.sourcePath && (d.sourcePath === r || d.sourcePath.startsWith(r + "/")));
     if (!inSeed && !underRoot) continue;
-    out.add(e.ref);
-    for (const r of graph.relations) {
-      if (r.sourceRef === e.ref) out.add(r.type + ":" + r.targetRef);
-      if (r.targetRef === e.ref) out.add(r.type + ":" + r.sourceRef);
-    }
+    out.add(fact("ref", e.ref));
+    if (e.kind === "API") out.add(fact("provides", e.ref));
+    addGraphRelations(out, graph, e.ref);
     for (const ev of graph.interfaces.events) {
-      if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add("event:" + ev.pattern);
-    }
-    for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external:" + x.name);
-    for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify:" + v.command);
-    for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict:" + c.key);
-    for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation:" + o.id);
-    for (const p of d.docs || []) out.add("doc:" + p);
-    for (const p of d.constraints || []) out.add("constraint:" + p);
-    for (const p of d.runbooks || []) out.add("runbook:" + p);
-    for (const r of graph.relations) {
-      if (r.sourceRef === e.ref || r.targetRef === e.ref) out.add(r.type + ":" + (r.sourceRef === e.ref ? r.targetRef : r.sourceRef));
+      if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref)) out.add(fact("event", ev.pattern));
     }
+    for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
+    for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
+    for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
+    for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
+    for (const p of d.docs || []) out.add(fact("doc", p));
+    for (const p of d.constraints || []) out.add(fact("constraint", p));
+    for (const p of d.runbooks || []) out.add(fact("runbook", p));
   }
-  // A repository-wide documentation sweep is what an agent does when the
-  // routing is not available: read everything that looks like documentation.
-  for (const f of new Set([...repoDocPaths(graph)])) out.add("repodoc:" + f);
+  // Preserve the original targeted reference's repository-wide doc sweep,
+  // now using the same labels as query rather than unmatched repodoc labels.
+  for (const f of new Set([...repoDocPaths(graph)])) out.add(fact("doc", f));
   return out;
 }
 
 function repoDocPaths(graph) {
   const out = new Set();
   for (const e of graph.entities) {
-    for (const p of (e.derived || {}).docs || []) out.add(p);
+    for (const p of (e.derived || {}).docs || []) out.add(fact("ref", p));
   }
-  for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(p);
+  for (const cfgp of graph.verification) for (const p of cfgp.configPaths || []) out.add(fact("ref", p));
   return [...out].sort();
 }
 
-// Baseline verification recall: without routing, an agent runs the union of
-// every check in the repository (or nothing). The realistic reading of
-// "unassisted" is the union, since no routing exists to narrow it.
-// The whole-repository baseline: what an agent that has no routing ends up
-// holding. Every entity, every relation, every subject, every check, every
-// document. This is the honest cost of "just read the repository".
+// Whole-graph fact inventory: a size yardstick, not unaided exploration.
 export function fullRepositoryContext(graph) {
   const out = new Set();
   for (const e of graph.entities) {
-    out.add(e.ref);
+    out.add(fact("ref", e.ref));
+    if (e.kind === "API") out.add(fact("provides", e.ref));
     const d = e.derived || {};
-    for (const p of d.docs || []) out.add("doc:" + p);
-    for (const p of d.constraints || []) out.add("constraint:" + p);
-    for (const p of d.runbooks || []) out.add("runbook:" + p);
+    for (const p of d.docs || []) out.add(fact("doc", p));
+    for (const p of d.constraints || []) out.add(fact("constraint", p));
+    for (const p of d.runbooks || []) out.add(fact("runbook", p));
   }
-  for (const r of graph.relations) out.add(r.type + ":" + (r.sourceRef === undefined ? "" : r.targetRef));
-  for (const r of graph.relations) out.add(r.type + ":" + r.sourceRef);
-  for (const ev of graph.interfaces.events) out.add("event:" + ev.pattern);
-  for (const x of graph.externalDependencies) out.add("external:" + x.name);
-  for (const x of graph.capabilities) out.add("capability:" + x.name);
-  for (const v of graph.verification) out.add("verify:" + v.command);
-  for (const c of graph.conflicts) out.add("conflict:" + c.key);
-  for (const o of graph.observations) out.add("observation:" + o.id);
-  for (const p of repoDocPaths(graph)) out.add("repodoc:" + p);
+  for (const e of graph.entities) addGraphRelations(out, graph, e.ref);
+  for (const ev of graph.interfaces.events) out.add(fact("event", ev.pattern));
+  for (const x of graph.externalDependencies) out.add(fact("external", x.name));
+  for (const x of graph.capabilities) out.add(fact("capability", x.name));
+  for (const v of graph.verification) out.add(fact("verify", v.command));
+  for (const c of graph.conflicts) out.add(fact("conflict", c.key));
+  for (const o of graph.observations) out.add(fact("observation", o.id));
+  for (const p of repoDocPaths(graph)) out.add(fact("doc", p));
   return out;
 }
 
-// Baseline verification: without routing, an agent runs the union of every
-// check in the repository, because nothing narrows it.
+// The historical targeted reference runs every registered check and gate.
 function baselineVerification(graph) {
-  return new Set(graph.verification.map((v) => "verify:" + v.command));
+  return new Set([...graph.verification, ...graph.gates].flatMap((v) => [fact("verify", v.command), fact("verify-id", v.id)]));
+}
+
+export function verificationRecall(retrieved, expected) {
+  const { hit, total } = score(retrieved, expected);
+  return total ? hit / total : 1;
+}
+
+// Count scoped check IDs, not commands: identical commands in different
+// components are separate checks. Gates are repository-scoped checks too.
+export function checkSelectivity(graph, recommendedIds) {
+  const all = new Set([...graph.verification, ...graph.gates].map((v) => v.id));
+  const recommended = [...all].filter((id) => recommendedIds.has(id)).length;
+  return all.size ? recommended / all.size : 1;
 }
 
 function score(retrieved, truth) {
@@ -167,13 +170,6 @@ function score(retrieved, truth) {
   return { hit, total: truth.size, missed, precision: truth.size ? hit / truth.size : 1 };
 }
 
-function noiseRatio(retrieved, truth) {
-  if (!retrieved.size) return 0;
-  let n = 0;
-  for (const r of retrieved) if (!truth.has(r)) n++;
-  return n / retrieved.size;
-}
-
 // ── Irrelevance, measured against reachability rather than against the
 // ground truth.
 //
@@ -206,35 +202,25 @@ export function relevantClosure(graph, touch) {
     if (seeds2.has(r.targetRef)) refs.add(r.sourceRef);
   }
   const out = new Set();
-  for (const ref of refs) out.add(ref);
-  for (const r of graph.relations) {
-    if (refs.has(r.sourceRef)) out.add(r.type + ":" + r.targetRef);
-    if (refs.has(r.targetRef)) out.add(r.type + ":" + r.sourceRef);
-  }
+  for (const ref of refs) out.add(fact("ref", ref));
+  for (const ref of refs) addGraphRelations(out, graph, ref);
   for (const e of graph.entities) {
     if (!refs.has(e.ref)) continue;
+    if (e.kind === "API") out.add(fact("provides", e.ref));
     const d = e.derived || {};
-    for (const p of d.docs || []) out.add("doc:" + p);
-    for (const p of d.constraints || []) out.add("constraint:" + p);
-    for (const p of d.runbooks || []) out.add("runbook:" + p);
+    for (const p of d.docs || []) out.add(fact("doc", p));
+    for (const p of d.constraints || []) out.add(fact("constraint", p));
+    for (const p of d.runbooks || []) out.add(fact("runbook", p));
     for (const ev of graph.interfaces.events) {
       if (ev.producers.includes(e.ref) || ev.consumers.includes(e.ref) || (d.sourcePath && ev.contract.ref.startsWith(d.sourcePath + "/"))) {
-        out.add("event:" + ev.pattern);
+        out.add(fact("event", ev.pattern));
       }
     }
-    for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add("external:" + x.name);
-    for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add("capability:" + x.name);
-    for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add("verify:" + v.command);
-    for (const c of graph.conflicts) if (c.subject === e.ref) out.add("conflict:" + c.key);
-    for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add("observation:" + o.id);
-    for (const r of graph.relations) {
-      if (r.type === "usesResource" && r.sourceRef === e.ref) out.add("uses:" + r.targetRef);
-      if (r.type === "usesResource" && r.targetRef === e.ref) out.add("usedBy:" + r.sourceRef);
-      if (r.type === "consumesApi" && r.sourceRef === e.ref) out.add("consumes:" + r.targetRef);
-      if (r.type === "consumesApi" && r.targetRef === e.ref) out.add("consumer:" + r.sourceRef);
-      if (r.type === "providesApi" && r.sourceRef === e.ref) out.add("provides:" + r.targetRef);
-      if (r.type === "providesApi" && r.targetRef === e.ref) out.add("provider:" + r.sourceRef);
-    }
+    for (const x of graph.externalDependencies) if (x.componentRef === e.ref) out.add(fact("external", x.name));
+    for (const x of graph.capabilities) if (x.componentRef === e.ref) out.add(fact("capability", x.name));
+    for (const v of graph.verification) if (v.componentRefs.includes(e.ref)) out.add(fact("verify", v.command));
+    for (const c of graph.conflicts) if (c.subject === e.ref) out.add(fact("conflict", c.key));
+    for (const o of graph.observations) if (o.facts.some((f) => f.subject === e.ref)) out.add(fact("observation", o.id));
   }
   return out;
 }
@@ -271,6 +257,10 @@ export function runRoutingEval(root, { json = false } = {}) {
     const retrieved = new Set();
     const critical = new Set();
     const verif = new Set();
+    const recommendedIds = new Set();
+    const collectChecks = (ctx) => {
+      for (const v of [...ctx.verification.items, ...ctx.globalGates]) recommendedIds.add(v.id);
+    };
     let unresolved = 0;
     for (const t of s.touch) {
       let ctx;
@@ -285,28 +275,39 @@ export function runRoutingEval(root, { json = false } = {}) {
       for (const x of factsFromContext(ctx)) retrieved.add(x);
       for (const x of criticalFromContext(ctx)) critical.add(x);
       for (const x of verificationFromContext(ctx)) verif.add(x);
+      collectChecks(ctx);
     }
     // Change impact is part of routing: it is what tells the agent which other
     // components a change reaches. Its output is legitimately part of context.
     const im = impact(graph, s.touch);
+    for (const v of im.recommendedVerification) recommendedIds.add(v.id);
     for (const c of im.affectedComponents) {
       const ctx = queryContext(graph, c.ref, {});
       for (const x of factsFromContext(ctx)) retrieved.add(x);
+      collectChecks(ctx);
     }
 
     const base = baselineContext(graph, s.touch);
     const baseV = baselineVerification(graph);
     const full = fullRepositoryContext(graph);
+    const search = searchContext(root, s.touch);
+    const searchAll = score(search.facts, new Set(s.truth));
+    // Retrieval has no graph access. Only scoring maps discovered scoped
+    // commands to the registered check IDs for a common selectivity denominator.
+    const searchIds = new Set([...graph.verification, ...graph.gates]
+      .filter((v) => search.checks.some((c) => c.command === v.command &&
+        (c.componentRef === null ? !(v.componentRefs || []).length : (v.componentRefs || []).includes(c.componentRef))))
+      .map((v) => v.id));
+    const searchV = new Set(search.checks.map((c) => fact("verify", c.command)));
+    for (const id of searchIds) searchV.add(fact("verify-id", id));
 
     const closure = relevantClosure(graph, s.touch);
-    const baseClosure = relevantClosure(graph, s.touch);
+
     const all = score(retrieved, new Set(s.truth));
     const baseAll = score(base, new Set(s.truth));
     const crit = score(critical, new Set(s.critical));
-    const vf = score(verif, new Set(s.verification));
-    const baseVf = score(baseV, new Set(s.verification));
     const noise = irrelevanceRatio(retrieved, closure);
-    const baseNoise = irrelevanceRatio(base, baseClosure);
+    const baseNoise = irrelevanceRatio(base, closure);
 
     perScenario.push({
       id: s.id,
@@ -314,13 +315,27 @@ export function runRoutingEval(root, { json = false } = {}) {
       recall: all.hit / (all.total || 1),
       precision: 1 - noise,
       criticalRecall: crit.total ? crit.hit / crit.total : 1,
-      verificationRecall: vf.total ? vf.hit / vf.total : 1,
+      verificationRecall: verificationRecall(verif, new Set(s.verification)),
+      checkSelectivity: checkSelectivity(graph, recommendedIds),
       noise,
       baseline: {
         recall: baseAll.hit / (baseAll.total || 1),
         noise: baseNoise,
-        verificationRecall: baseVf.hit / (baseVf.total || 1),
+        verificationRecall: verificationRecall(baseV, new Set(s.verification)),
+        checkSelectivity: 1, // The baseline runs every verification entry and gate.
       },
+      searchBaseline: {
+        recall: searchAll.hit / (searchAll.total || 1),
+        noise: irrelevanceRatio(search.facts, closure),
+        verificationRecall: verificationRecall(searchV, new Set(s.verification)),
+        checkSelectivity: checkSelectivity(graph, searchIds),
+        missed: searchAll.missed,
+      },
+      truthCount: all.total,
+      recalledCount: all.hit,
+      baselineRecalledCount: baseAll.hit,
+      searchRecalledCount: searchAll.hit,
+      searchCount: search.facts.size,
       missed: all.missed.slice(0, 8),
       offClosure: [...retrieved].filter((r) => !closure.has(r)).slice(0, 8),
       unresolvedTouches: unresolved,
@@ -333,19 +348,34 @@ export function runRoutingEval(root, { json = false } = {}) {
 
   const mean = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b[k], 0) / perScenario.length : 0);
   const meanB = (k) => (perScenario.length ? perScenario.reduce((a, b) => a + b.baseline[k], 0) / perScenario.length : 0);
-  void meanB;
+  const meanS = (k) => perScenario.length ? perScenario.reduce((a, b) => a + b.searchBaseline[k], 0) / perScenario.length : 0;
+  const truthCount = perScenario.reduce((sum, s) => sum + s.truthCount, 0);
+  const micro = (key) => truthCount ? round(perScenario.reduce((sum, s) => sum + s[key], 0) / truthCount) : 0;
   const metrics = {
     scenarios: perScenario.length,
     recall: round(mean("recall")),
+    microRecall: micro("recalledCount"),
     precision: round(mean("precision")),
     criticalRecall: round(mean("criticalRecall")),
     verificationRecall: round(mean("verificationRecall")),
+    checkSelectivity: round(mean("checkSelectivity")),
     noise: round(mean("noise")),
     retrievalReduction: 0,
+    searchBaseline: {
+      recall: round(meanS("recall")),
+      microRecall: micro("searchRecalledCount"),
+      size: round(mean("searchCount")),
+      noise: round(meanS("noise")),
+      verificationRecall: round(meanS("verificationRecall")),
+      checkSelectivity: round(meanS("checkSelectivity")),
+    },
     baseline: {
       recall: round(meanB("recall")),
+      microRecall: micro("baselineRecalledCount"),
+      size: round(mean("baselineCount")),
       noise: round(meanB("noise")),
       verificationRecall: round(meanB("verificationRecall")),
+      checkSelectivity: round(meanB("checkSelectivity")),
     },
   };
   // Size proxy: retrieved fact count versus the whole-repository baseline.

package/evals/scenarios/fixture-b-03-change-shared-lib.json

@@ -5,5 +5,12 @@
   "touch": ["libs/protocol/ledger.pb.go"],
   "truth": ["component:default/protocol", "component:default/ingest", "component:default/gateway", "doc:libs/protocol/README.md"],
   "critical": [],
-  "verification": []
+  "verification": [
+    "verify:go test ./...",
+    "verify:go vet ./...",
+    "verify-id:component:default/gateway:go-test",
+    "verify-id:component:default/gateway:go-vet",
+    "verify-id:component:default/ingest:go-test",
+    "verify-id:component:default/ingest:go-vet"
+  ]
 }

package/evals/scenarios/fixture-c-03-shared-vocabulary.json

@@ -5,5 +5,11 @@
   "touch": ["shared/index.js"],
   "truth": ["component:default/shared", "component:default/edge", "component:default/console", "event:corp.telemetry.ingest.v1"],
   "critical": ["event:corp.telemetry.ingest.v1"],
-  "verification": []
+  "verification": [
+    "verify:vitest run",
+    "verify:tsc --noEmit",
+    "verify-id:component:default/console:test",
+    "verify-id:component:default/console:typecheck",
+    "verify-id:component:default/edge:test"
+  ]
 }

package/evals/scenarios/fixture-e-01-change-shared-lib.json

@@ -0,0 +1,14 @@
+{
+  "id": "e-01",
+  "repository": "fixture-e",
+  "title": "Change the shared engine crate",
+  "touch": ["crates/engine/src/lib.rs"],
+  "truth": ["component:default/engine", "component:default/cli"],
+  "critical": [],
+  "verification": [
+    "verify:cargo test -p engine",
+    "verify:cargo clippy -p engine",
+    "verify:cargo test -p cli",
+    "verify:cargo clippy -p cli"
+  ]
+}

package/evals/scenarios/fixture-e-02-change-manifest.json

@@ -0,0 +1,9 @@
+{
+  "id": "e-02",
+  "repository": "fixture-e",
+  "title": "Change the CLI crate manifest",
+  "touch": ["apps/cli/Cargo.toml"],
+  "truth": ["component:default/cli", "component:default/engine", "component:default/testutil", "external:serde"],
+  "critical": [],
+  "verification": ["verify:cargo test -p cli", "verify:cargo clippy -p cli"]
+}

package/evals/scenarios/fixture-e-03-local-module.json

@@ -0,0 +1,9 @@
+{
+  "id": "e-03",
+  "repository": "fixture-e",
+  "title": "Change CLI code that looks like the logging crate and an ambiguous foo_bar import",
+  "touch": ["apps/cli/src/logging.rs", "apps/cli/src/main.rs"],
+  "truth": ["component:default/cli", "component:default/engine"],
+  "critical": [],
+  "verification": ["verify:cargo test -p cli", "verify:cargo clippy -p cli"]
+}

package/evals/scenarios/fixture-e-04-ambiguous-crate.json

@@ -0,0 +1,14 @@
+{
+  "id": "e-04",
+  "repository": "fixture-e",
+  "title": "Change each crate of the foo-bar / foo_bar pair",
+  "touch": ["crates/foo-bar/src/lib.rs", "crates/foo_bar/src/lib.rs"],
+  "truth": ["component:default/foo-bar", "component:default/foo-u-bar"],
+  "critical": [],
+  "verification": [
+    "verify:cargo test -p foo-bar",
+    "verify:cargo clippy -p foo-bar",
+    "verify:cargo test -p foo_bar",
+    "verify:cargo clippy -p foo_bar"
+  ]
+}

package/evals/scenarios/fixture-f-01-change-shared-lib.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-01",
+  "repository": "fixture-f",
+  "title": "Change the shared ledger-core library",
+  "touch": ["libs/ledger-core/src/ledger_core/models.py"],
+  "truth": ["component:default/ledger-core", "component:default/ledger-api"],
+  "critical": [],
+  "verification": ["verify:uv run pytest", "verify:uv run ruff check", "verify:uv run mypy"]
+}

package/evals/scenarios/fixture-f-02-change-manifest.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-02",
+  "repository": "fixture-f",
+  "title": "Change the ledger API manifest",
+  "touch": ["services/ledger-api/pyproject.toml"],
+  "truth": ["component:default/ledger-api", "component:default/ledger-core", "component:default/ledger-testing", "external:sqlalchemy"],
+  "critical": [],
+  "verification": ["verify:uv run pytest", "verify:uv run ruff check", "verify:uv run mypy"]
+}

package/evals/scenarios/fixture-f-03-shadowed-import.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-03",
+  "repository": "fixture-f",
+  "title": "Change the app that imports its own utils module",
+  "touch": ["services/b-app/b_app/main.py"],
+  "truth": ["component:default/b-app"],
+  "critical": [],
+  "verification": []
+}

package/evals/scenarios/fixture-f-04-nested-workspace.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-04",
+  "repository": "fixture-f",
+  "title": "Change a nested workspace member, not its parent",
+  "touch": ["python/packages/api/src/api_svc/main.py"],
+  "truth": ["component:default/api-svc", "component:default/core-lib"],
+  "critical": [],
+  "verification": []
+}

package/evals/scenarios/fixture-f-05-alembic-database.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-05",
+  "repository": "fixture-f",
+  "title": "Change the ledger API Alembic migration",
+  "touch": ["services/ledger-api/alembic/versions/0001_init.py"],
+  "truth": ["component:default/ledger-api", "uses:resource:default/ledger-api-postgres"],
+  "critical": ["uses:resource:default/ledger-api-postgres"],
+  "verification": ["verify:uv run pytest", "verify:uv run ruff check", "verify:uv run mypy"]
+}

package/evals/scenarios/fixture-f-06-unittest-job.json

@@ -0,0 +1,9 @@
+{
+  "id": "f-06",
+  "repository": "fixture-f",
+  "title": "Change the ledger job exercised by unittest",
+  "touch": ["jobs/ledger-jobs/tests/test_jobs.py"],
+  "truth": ["component:default/ledger-jobs"],
+  "critical": [],
+  "verification": ["verify:python -m unittest discover"]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-doc-system",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "type": "module",
   "description": "Portable documentation and software-catalog system with an explicit authority model, for AI coding agents",
   "keywords": [
@@ -18,7 +18,7 @@
   "homepage": "https://github.com/lbframe/agent-doc-system",
   "bugs": "https://github.com/lbframe/agent-doc-system/issues",
   "bin": {
-    "agentdoc": "./bin/agentdoc.mjs"
+    "agentdoc": "bin/agentdoc.mjs"
   },
   "engines": {
     "node": ">=22"
@@ -37,11 +37,14 @@
     "core",
     "docs",
     "evals/harness.mjs",
+    "evals/facts.mjs",
+    "evals/baseline-search.mjs",
     "evals/scenarios",
     "evals/thresholds.json",
     "schemas",
     "skills",
     "templates",
+    "CHANGELOG.md",
     "INSTALL_FOR_AGENTS.md"
   ],
   "license": "MIT"

package/skills/agent-doc-system/SKILL.md

@@ -83,7 +83,8 @@ not work around it: run `agentdoc compile`, review the diff, commit the graph.
 ## Rules that are never optional
 
 - **Never vendor the CLI.** A project gets `agentdoc/` project data (config,
-  descriptors, observations), `docs/` skeletons, `.agentdoc/graph.json` and a
+  descriptors, observations), documentation skeletons at the configured
+  `docs.*` paths (`agentdoc/docs/` by default), `.agentdoc/graph.json` and a
   CI workflow — never a copy of the agentdoc source tree (`core/`, `bin/`,
   `adapters/`, `schemas/`, `templates/`, `evals/`, `tests/`).
 - **Never promote uncertainty.** Heuristic guesses do not become authored

package/skills/agent-doc-system/references/cli.md

@@ -38,7 +38,12 @@ agentdoc scaffold [--write] # generate descriptors for what can be proven
 subdirectory of a cataloged repository it writes at the catalog root, not the
 current directory. `scaffold` is a dry run without `--write`; with `--write` it
 is a superset of `init` — it also installs anything missing from the config,
-doc skeleton and CI workflow, not only component descriptors. `audit` works
+doc skeleton and CI workflow, not only component descriptors. Both write the
+doc skeleton to the configured `docs.*` paths when a configuration exists, and
+to `agentdoc/docs/` when none does; an existing file is reported as skipped.
+Authored AgentDoc material lives in `agentdoc/`, generated output in
+`.agentdoc/`; `docs/` is left to the product's own user documentation unless a
+configuration declares `docs.*` paths there. `audit` works
 even when the repository cannot compile yet; it reports the compilation
 failures as part of its output.