@tangle-network/agent-knowledge 0.1.1 → 1.1.0

package/AGENTS.md

@@ -23,6 +23,7 @@ agent-knowledge search "query" --json
 agent-knowledge inspect --json
 agent-knowledge explain knowledge/concepts/example.md --json
 agent-knowledge lint --json
+agent-knowledge validate --strict --json
 agent-knowledge viz --json
 ```
 
@@ -49,3 +50,11 @@ The parser rejects absolute paths, `..`, control characters, and writes outside
 ## Eval Boundary
 
 Use `runKnowledgeBaseOptimization()` when comparing candidate knowledge bases on an actual task corpus. It delegates to `@tangle-network/agent-eval` multi-shot optimization, so single-turn and multi-turn agents share the same path.
+
+Use `knowledgeReleaseReportFromOptimization()` before promotion. It projects optimizer traces and `RunRecord` rows into `agent-eval` release confidence evidence.
+
+## Integration Boundaries
+
+- Use `KbStore` for storage. Implement D1 in the consuming app when needed.
+- Use `KnowledgeDiscoveryDispatcher` for research workers. Production apps should wire this to their own swarm/fleet runtime.
+- Do not bypass `lint` or `validate` before using generated knowledge in an agent.

package/README.md

@@ -23,6 +23,8 @@ agent-knowledge inspect --root .
 agent-knowledge explain knowledge/concepts/risk.md --root .
 agent-knowledge graph --root . --format json
 agent-knowledge lint --root .
+agent-knowledge validate --strict --root .
+agent-knowledge export --root . --format json
 agent-knowledge viz --root .
 ```
 
@@ -32,13 +34,20 @@ The default layout is:
 raw/
   sources/
 knowledge/
-  index.md
-  log.md
+  index.md   # scaffold: human-navigation only, excluded from the page index
+  log.md     # scaffold: human-navigation only, excluded from the page index
 .agent-knowledge/
   sources.json
   index.json
 ```
 
+`initKnowledgeBase` writes `knowledge/index.md` and `knowledge/log.md` for
+authors to curate by hand. They are deliberately excluded from
+`buildKnowledgeIndex` / `searchKnowledge` so they do not inflate page counts
+or pollute search hits. Any nested `<dir>/index.md` or `<dir>/log.md` is
+treated the same way. The shared predicate is `isScaffoldPath`, exported
+from `@tangle-network/agent-knowledge`.
+
 ## Design
 
 - Raw sources are immutable evidence.
@@ -47,11 +56,58 @@ knowledge/
 - Lint fails on pages that cite unknown source IDs.
 - Text sources get deterministic anchors (`all`, `l1`, `l51`, ...) for precise citations like `[^src_id#all]`.
 - Agent write proposals can be safely applied with `apply-write-blocks`.
+- `KbStore` keeps storage consumer-owned; use `MemoryKbStore`, `FileSystemKbStore`, or implement D1 in the app.
+- Discovery uses worker/dispatcher contracts, with a local dispatcher for dev and tests.
+- Zod schemas define the stable wire shape.
 - Graph/search/lint are deterministic and fast.
+- `searchKnowledge` returns hits with three score fields. `score` and
+  `rrfScore` are the raw reciprocal-rank-fusion value (typically 0.01–0.05);
+  use them when intent matters or when fusing across queries.
+  `normalizedScore` is the same value scaled into [0, 1] relative to the top
+  hit *in this result set* (top hit = 1, others = score / topScore) — use it
+  when comparing against natural confidence thresholds. The normalization is
+  within-set ranking, not a cross-query absolute confidence.
 - Optimization uses `@tangle-network/agent-eval` internally instead of reimplementing eval gates.
+- `buildEvalKnowledgeBundle()` maps wiki/search evidence into
+  `agent-eval` `KnowledgeRequirement`, `KnowledgeBundle`, and
+  `KnowledgeReadinessReport` contracts so control loops can block, ask, or
+  acquire data before running an agent.
 
 The `/viz` subpath exports graph insight helpers without UI dependencies.
 
 ## Agent-Eval Integration
 
 Use `runKnowledgeBaseOptimization()` when the question is whether a candidate knowledge base actually improves agent task success. The candidate is passed through `runMultiShotOptimization`, so `n=1` single-turn tasks and variable-length multi-turn traces use the same path.
+
+Use `knowledgeReleaseReportFromOptimization()` to turn optimizer output into release confidence evidence using `agent-eval` release gates and `RunRecord` validation.
+
+Use `buildEvalKnowledgeBundle()` before execution when the question is whether
+the agent has enough task-world context to run:
+
+```ts
+import { buildEvalKnowledgeBundle } from '@tangle-network/agent-knowledge'
+
+const readiness = buildEvalKnowledgeBundle({
+  taskId: 'sdk-migration',
+  index,
+  specs: [{
+    id: 'repo-build-command',
+    description: 'Repository build and typecheck command',
+    query: 'build typecheck command',
+    requiredFor: ['coding'],
+    category: 'codebase_specific',
+    acquisitionMode: 'inspect_repo',
+    importance: 'blocking',
+    freshness: 'weekly',
+    sensitivity: 'public',
+    confidenceNeeded: 0.9,
+    minSources: 1,
+  }],
+})
+
+console.log(readiness.report.recommendedAction)
+```
+
+Pass `readiness.report` to `blockingKnowledgeEval()` from
+`@tangle-network/agent-eval`; use `readiness.questions` and
+`readiness.acquisitionPlans` to drive UI or connector workflows.

package/dist/{chunk-XDXIBHPS.js → chunk-BEFAI2HO.js}

@@ -215,6 +215,176 @@ async function applyKnowledgeWriteBlocksFile(root, proposalPath) {
   return applyKnowledgeWriteBlocks(root, await readFile(proposalPath, "utf8"));
 }
 
+// src/schemas.ts
+import { z } from "zod";
+var SourceAnchorSchema = z.object({
+  id: z.string().min(1),
+  sourceId: z.string().min(1),
+  label: z.string().optional(),
+  page: z.number().int().positive().optional(),
+  lineStart: z.number().int().positive().optional(),
+  lineEnd: z.number().int().positive().optional(),
+  charStart: z.number().int().nonnegative().optional(),
+  charEnd: z.number().int().nonnegative().optional(),
+  timestampMs: z.number().nonnegative().optional(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var SourceRecordSchema = z.object({
+  id: z.string().min(1),
+  uri: z.string().min(1),
+  title: z.string().optional(),
+  mediaType: z.string().optional(),
+  contentHash: z.string().min(16),
+  text: z.string().optional(),
+  anchors: z.array(SourceAnchorSchema).optional(),
+  metadata: z.record(z.string(), z.unknown()).optional(),
+  createdAt: z.string().min(1)
+});
+var KnowledgePageSchema = z.object({
+  id: z.string().min(1),
+  path: z.string().min(1),
+  title: z.string().min(1),
+  text: z.string(),
+  frontmatter: z.record(z.string(), z.unknown()),
+  sourceIds: z.array(z.string()),
+  tags: z.array(z.string()),
+  outLinks: z.array(z.string())
+});
+var KnowledgeGraphNodeSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  path: z.string(),
+  tags: z.array(z.string()),
+  sourceIds: z.array(z.string()),
+  outDegree: z.number().int().nonnegative(),
+  inDegree: z.number().int().nonnegative()
+});
+var KnowledgeGraphEdgeSchema = z.object({
+  source: z.string(),
+  target: z.string(),
+  weight: z.number(),
+  reasons: z.array(z.string())
+});
+var KnowledgeIndexSchema = z.object({
+  root: z.string(),
+  generatedAt: z.string(),
+  sources: z.array(SourceRecordSchema),
+  pages: z.array(KnowledgePageSchema),
+  graph: z.object({
+    nodes: z.array(KnowledgeGraphNodeSchema),
+    edges: z.array(KnowledgeGraphEdgeSchema)
+  })
+});
+var KnowledgeEventSchema = z.object({
+  id: z.string().min(1),
+  type: z.enum(["source.added", "proposal.applied", "index.built", "lint.run", "optimization.run", "release.promoted", "release.rejected"]),
+  createdAt: z.string().min(1),
+  actor: z.string().optional(),
+  target: z.string().optional(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var KnowledgeBaseCandidateSchema = z.object({
+  id: z.string().min(1),
+  units: z.array(z.object({
+    id: z.string().min(1),
+    title: z.string().min(1),
+    text: z.string(),
+    claims: z.array(z.object({
+      id: z.string().min(1),
+      text: z.string().min(1),
+      refs: z.array(z.object({
+        sourceId: z.string().min(1),
+        anchorId: z.string().optional(),
+        quote: z.string().optional()
+      })),
+      confidence: z.number().min(0).max(1).optional(),
+      status: z.enum(["draft", "active", "superseded", "rejected"]).optional(),
+      metadata: z.record(z.string(), z.unknown()).optional()
+    })).optional(),
+    relations: z.array(z.object({
+      sourceId: z.string(),
+      targetId: z.string(),
+      predicate: z.string(),
+      weight: z.number().optional(),
+      metadata: z.record(z.string(), z.unknown()).optional()
+    })).optional(),
+    sourceIds: z.array(z.string()).optional(),
+    tags: z.array(z.string()).optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+    updatedAt: z.string().optional()
+  })),
+  retrievalPolicy: z.string().optional(),
+  synthesisPolicy: z.string().optional(),
+  questionPolicy: z.string().optional(),
+  updatePolicy: z.string().optional(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+
+// src/graph.ts
+function buildKnowledgeGraph(pages) {
+  const byId = /* @__PURE__ */ new Map();
+  const bySlug = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    byId.set(page.id, page);
+    bySlug.set(normalizeLinkTarget(page.id), page);
+    bySlug.set(normalizeLinkTarget(page.title), page);
+    bySlug.set(normalizeLinkTarget(page.path.split("/").pop().replace(/\.md$/, "")), page);
+  }
+  const incoming = /* @__PURE__ */ new Map();
+  const outgoing = /* @__PURE__ */ new Map();
+  const edgesByKey = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    outgoing.set(page.id, 0);
+    incoming.set(page.id, 0);
+  }
+  for (const page of pages) {
+    for (const raw of page.outLinks) {
+      const target = bySlug.get(normalizeLinkTarget(raw));
+      if (!target || target.id === page.id) continue;
+      const key = `${page.id}->${target.id}`;
+      const edge = edgesByKey.get(key);
+      if (edge) edge.weight += 1;
+      else edgesByKey.set(key, { source: page.id, target: target.id, weight: 1, reasons: ["wikilink"] });
+      outgoing.set(page.id, (outgoing.get(page.id) ?? 0) + 1);
+      incoming.set(target.id, (incoming.get(target.id) ?? 0) + 1);
+    }
+  }
+  addSourceOverlapEdges(pages, edgesByKey);
+  const nodes = pages.map((page) => ({
+    id: page.id,
+    title: page.title,
+    path: page.path,
+    tags: page.tags,
+    sourceIds: page.sourceIds,
+    outDegree: outgoing.get(page.id) ?? 0,
+    inDegree: incoming.get(page.id) ?? 0
+  }));
+  return { nodes, edges: [...edgesByKey.values()].sort((a, b) => b.weight - a.weight) };
+}
+function addSourceOverlapEdges(pages, edges) {
+  for (let i = 0; i < pages.length; i++) {
+    for (let j = i + 1; j < pages.length; j++) {
+      const a = pages[i];
+      const b = pages[j];
+      const overlap = a.sourceIds.filter((source) => b.sourceIds.includes(source));
+      if (overlap.length === 0) continue;
+      const key = `${a.id}->${b.id}`;
+      const edge = edges.get(key);
+      if (edge) {
+        edge.weight += overlap.length * 0.5;
+        edge.reasons.push("shared-source");
+      } else {
+        edges.set(key, {
+          source: a.id,
+          target: b.id,
+          weight: overlap.length * 0.5,
+          reasons: ["shared-source"]
+        });
+      }
+    }
+  }
+}
+
 // src/store.ts
 import { mkdir as mkdir2, readFile as readFile2, readdir, stat, writeFile as writeFile2 } from "fs/promises";
 import { dirname as dirname2, join as join2, relative } from "path";
@@ -229,6 +399,15 @@ function layoutFor(root) {
     cacheDir: join2(root, ".agent-knowledge")
   };
 }
+var SCAFFOLD_PAGE_BASENAMES = ["index.md", "log.md"];
+function isScaffoldPath(path) {
+  const normalized = path.replace(/\\/g, "/");
+  for (const basename2 of SCAFFOLD_PAGE_BASENAMES) {
+    if (normalized === `knowledge/${basename2}`) return true;
+    if (normalized.endsWith(`/${basename2}`)) return true;
+  }
+  return false;
+}
 async function initKnowledgeBase(root) {
   const layout = layoutFor(root);
   await mkdir2(layout.knowledgeDir, { recursive: true });
@@ -244,9 +423,10 @@ async function loadKnowledgePages(root) {
   const files = await listMarkdownFiles(layout.knowledgeDir);
   const pages = [];
   for (const file of files) {
+    const rel = relative(root, file).replace(/\\/g, "/");
+    if (isScaffoldPath(rel)) continue;
     const content = await readFile2(file, "utf8");
     const { frontmatter, body } = parseFrontmatter(content);
-    const rel = relative(root, file).replace(/\\/g, "/");
     const title = stringField(frontmatter.title) ?? firstHeading(body) ?? rel.split("/").pop().replace(/\.md$/, "");
     const sourceIds = arrayField(frontmatter.sources);
     const tags = arrayField(frontmatter.tags);
@@ -400,71 +580,6 @@ function textPreview(fileName, bytes) {
   return bytes.toString("utf8").slice(0, 2e5);
 }
 
-// src/graph.ts
-function buildKnowledgeGraph(pages) {
-  const byId = /* @__PURE__ */ new Map();
-  const bySlug = /* @__PURE__ */ new Map();
-  for (const page of pages) {
-    byId.set(page.id, page);
-    bySlug.set(normalizeLinkTarget(page.id), page);
-    bySlug.set(normalizeLinkTarget(page.title), page);
-    bySlug.set(normalizeLinkTarget(page.path.split("/").pop().replace(/\.md$/, "")), page);
-  }
-  const incoming = /* @__PURE__ */ new Map();
-  const outgoing = /* @__PURE__ */ new Map();
-  const edgesByKey = /* @__PURE__ */ new Map();
-  for (const page of pages) {
-    outgoing.set(page.id, 0);
-    incoming.set(page.id, 0);
-  }
-  for (const page of pages) {
-    for (const raw of page.outLinks) {
-      const target = bySlug.get(normalizeLinkTarget(raw));
-      if (!target || target.id === page.id) continue;
-      const key = `${page.id}->${target.id}`;
-      const edge = edgesByKey.get(key);
-      if (edge) edge.weight += 1;
-      else edgesByKey.set(key, { source: page.id, target: target.id, weight: 1, reasons: ["wikilink"] });
-      outgoing.set(page.id, (outgoing.get(page.id) ?? 0) + 1);
-      incoming.set(target.id, (incoming.get(target.id) ?? 0) + 1);
-    }
-  }
-  addSourceOverlapEdges(pages, edgesByKey);
-  const nodes = pages.map((page) => ({
-    id: page.id,
-    title: page.title,
-    path: page.path,
-    tags: page.tags,
-    sourceIds: page.sourceIds,
-    outDegree: outgoing.get(page.id) ?? 0,
-    inDegree: incoming.get(page.id) ?? 0
-  }));
-  return { nodes, edges: [...edgesByKey.values()].sort((a, b) => b.weight - a.weight) };
-}
-function addSourceOverlapEdges(pages, edges) {
-  for (let i = 0; i < pages.length; i++) {
-    for (let j = i + 1; j < pages.length; j++) {
-      const a = pages[i];
-      const b = pages[j];
-      const overlap = a.sourceIds.filter((source) => b.sourceIds.includes(source));
-      if (overlap.length === 0) continue;
-      const key = `${a.id}->${b.id}`;
-      const edge = edges.get(key);
-      if (edge) {
-        edge.weight += overlap.length * 0.5;
-        edge.reasons.push("shared-source");
-      } else {
-        edges.set(key, {
-          source: a.id,
-          target: b.id,
-          weight: overlap.length * 0.5,
-          reasons: ["shared-source"]
-        });
-      }
-    }
-  }
-}
-
 // src/search.ts
 var RRF_K = 60;
 var STOP_WORDS = /* @__PURE__ */ new Set(["the", "is", "a", "an", "what", "how", "are", "was", "were", "to", "for", "of", "with", "by", "in", "on", "and"]);
@@ -475,9 +590,13 @@ function searchKnowledge(index, query, limit = 10) {
   const graphRanked = rankByGraph(index.pages, tokenRanked);
   const scores = reciprocalRankFusion([tokenRanked.map((p) => p.id), graphRanked.map((p) => p.id)]);
   const byId = new Map(index.pages.map((page) => [page.id, page]));
-  return [...scores.entries()].map(([id, score]) => ({ page: byId.get(id), score })).filter((item) => Boolean(item.page)).sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)).slice(0, limit).map((item, i) => ({
+  const ranked = [...scores.entries()].map(([id, score]) => ({ page: byId.get(id), score })).filter((item) => Boolean(item.page)).sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)).slice(0, limit);
+  const topScore = ranked[0]?.score ?? 0;
+  return ranked.map((item, i) => ({
     page: item.page,
     score: item.score,
+    rrfScore: item.score,
+    normalizedScore: topScore > 0 ? item.score / topScore : 0,
     rank: i + 1,
     snippet: buildSnippet(item.page.text, trimmed),
     reasons: reasonsFor(item.page, trimmed)
@@ -579,13 +698,19 @@ function lintKnowledgeIndex(index) {
   const titles = /* @__PURE__ */ new Map();
   const sourceIds = new Set(index.sources.map((source) => source.id));
   const anchorIds = new Map(index.sources.map((source) => [source.id, new Set((source.anchors ?? []).map((anchor) => anchor.id))]));
+  const pageIds = /* @__PURE__ */ new Map();
+  const sourceHashes = /* @__PURE__ */ new Map();
   for (const page of index.pages) {
+    pageIds.set(page.id, [...pageIds.get(page.id) ?? [], page.path]);
     byTarget.add(normalizeLinkTarget(page.id));
     byTarget.add(normalizeLinkTarget(page.title));
     byTarget.add(normalizeLinkTarget(page.path.split("/").pop().replace(/\.md$/, "")));
     const titleKey = page.title.toLowerCase();
     titles.set(titleKey, [...titles.get(titleKey) ?? [], page.path]);
   }
+  for (const source of index.sources) {
+    sourceHashes.set(source.contentHash, [...sourceHashes.get(source.contentHash) ?? [], source.id]);
+  }
   const inbound = /* @__PURE__ */ new Map();
   for (const page of index.pages) inbound.set(page.id, 0);
   for (const page of index.pages) {
@@ -624,6 +749,16 @@ function lintKnowledgeIndex(index) {
       findings.push({ type: "duplicate-title", severity: "warning", message: `Duplicate title "${title}" in ${paths.join(", ")}.`, metadata: { paths } });
     }
   }
+  for (const [id, paths] of pageIds) {
+    if (id && paths.length > 1) {
+      findings.push({ type: "duplicate-page-id", severity: "error", message: `Duplicate page id "${id}" in ${paths.join(", ")}.`, metadata: { paths } });
+    }
+  }
+  for (const [hash, ids] of sourceHashes) {
+    if (hash && ids.length > 1) {
+      findings.push({ type: "duplicate-source-hash", severity: "warning", message: `Duplicate source content hash across ${ids.join(", ")}.`, metadata: { sourceIds: ids } });
+    }
+  }
   return findings;
 }
 function isStructural(path) {
@@ -667,6 +802,36 @@ function explainKnowledgeTarget(index, target) {
   };
 }
 
+// src/validate.ts
+function validateKnowledgeIndex(index, options = {}) {
+  const findings = [...lintKnowledgeIndex(index)];
+  const parsed = KnowledgeIndexSchema.safeParse(index);
+  if (!parsed.success) {
+    findings.push({
+      type: "missing-frontmatter",
+      severity: "error",
+      message: parsed.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")
+    });
+  }
+  if (options.strict) {
+    for (const page of index.pages) {
+      if (isStructuralPage(page.path)) continue;
+      if (!page.frontmatter.id || !page.frontmatter.title) {
+        findings.push({
+          type: "missing-frontmatter",
+          severity: "error",
+          page: page.path,
+          message: "Strict mode requires id and title frontmatter."
+        });
+      }
+    }
+  }
+  return { ok: !findings.some((finding) => finding.severity === "error"), findings };
+}
+function isStructuralPage(path) {
+  return path === "knowledge/index.md" || path === "knowledge/log.md" || path.endsWith("/index.md") || path.endsWith("/log.md");
+}
+
 export {
   sha256,
   slugify,
@@ -682,7 +847,18 @@ export {
   mediaTypeFor,
   applyKnowledgeWriteBlocks,
   applyKnowledgeWriteBlocksFile,
+  SourceAnchorSchema,
+  SourceRecordSchema,
+  KnowledgePageSchema,
+  KnowledgeGraphNodeSchema,
+  KnowledgeGraphEdgeSchema,
+  KnowledgeIndexSchema,
+  KnowledgeEventSchema,
+  KnowledgeBaseCandidateSchema,
+  buildKnowledgeGraph,
   layoutFor,
+  SCAFFOLD_PAGE_BASENAMES,
+  isScaffoldPath,
   initKnowledgeBase,
   loadKnowledgePages,
   writeJson,
@@ -690,7 +866,6 @@ export {
   writeSourceRegistry,
   addSourcePath,
   sourceRegistryPath,
-  buildKnowledgeGraph,
   searchKnowledge,
   tokenizeQuery,
   reciprocalRankFusion,
@@ -698,6 +873,7 @@ export {
   writeKnowledgeIndex,
   lintKnowledgeIndex,
   inspectKnowledgeIndex,
-  explainKnowledgeTarget
+  explainKnowledgeTarget,
+  validateKnowledgeIndex
 };
-//# sourceMappingURL=chunk-XDXIBHPS.js.map
+//# sourceMappingURL=chunk-BEFAI2HO.js.map