@sdsrs/llm-wiki 0.2.0

package/src/convert.mjs

@@ -0,0 +1,62 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+export const SUPPORTED_EXTS = ['.md', '.markdown', '.txt', '.html', '.htm', '.pdf', '.docx']
+
+export function slugify(name) {
+  return name.toLowerCase()
+    .replace(/\.[a-z0-9]+$/i, '')
+    .replace(/[^\p{L}\p{N}]+/gu, '-')
+    .replace(/^-+|-+$/g, '')
+}
+
+function titleFrom(markdown, fallback) {
+  const m = markdown.match(/^#\s+(.+)$/m)
+  return m ? m[1].trim() : fallback
+}
+
+export async function convertFile(srcPath) {
+  const ext = path.extname(srcPath).toLowerCase()
+  const base = path.basename(srcPath)
+  const warnings = []
+  try {
+    if (ext === '.md' || ext === '.markdown') {
+      const md = fs.readFileSync(srcPath, 'utf8')
+      return { markdown: md, title: titleFrom(md, base), warnings }
+    }
+    if (ext === '.txt') {
+      const md = fs.readFileSync(srcPath, 'utf8')
+      return { markdown: md, title: titleFrom(md, base), warnings }
+    }
+    if (ext === '.html' || ext === '.htm') {
+      const { JSDOM } = await import('jsdom')
+      const { Readability } = await import('@mozilla/readability')
+      const TurndownService = (await import('turndown')).default
+      const html = fs.readFileSync(srcPath, 'utf8')
+      const dom = new JSDOM(html, { url: 'https://local.invalid/' })
+      const article = new Readability(dom.window.document).parse()
+      if (!article?.content) {
+        warnings.push('readability found no main content; skipped')
+        return { markdown: null, title: base, warnings }
+      }
+      const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })
+      const md = td.turndown(article.content)
+      return { markdown: `# ${article.title || base}\n\n${md}`, title: article.title || base, warnings }
+    }
+    if (ext === '.pdf') {
+      const pdfParse = (await import('pdf-parse/lib/pdf-parse.js')).default
+      const data = await pdfParse(fs.readFileSync(srcPath))
+      return { markdown: data.text, title: titleFrom(data.text, base), warnings }
+    }
+    if (ext === '.docx') {
+      const mammoth = await import('mammoth')
+      const { value } = await mammoth.extractRawText({ path: srcPath })
+      return { markdown: value, title: titleFrom(value, base), warnings }
+    }
+    warnings.push(`unsupported extension: ${ext}`)
+    return { markdown: null, title: base, warnings }
+  } catch (err) {
+    warnings.push(`convert failed: ${err.message}`)
+    return { markdown: null, title: base, warnings }
+  }
+}

package/src/export.mjs

@@ -0,0 +1,176 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+
+const xmlEscape = (s) => String(s)
+  .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+  .replace(/"/g, '&quot;').replace(/'/g, '&apos;')
+
+const cyEscape = (s) => String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'")
+
+// graph.json only lists wiki pages as nodes, but source-type edges point at raw/
+// paths. Exports need every edge endpoint to exist, so synthesize raw nodes.
+export function loadGraph(kbRoot) {
+  const p = kbPaths(kbRoot)
+  if (!fs.existsSync(p.graphJson)) throw new Error('wiki/graph.json not found — run `llm-wiki index` first.')
+  const graph = JSON.parse(fs.readFileSync(p.graphJson, 'utf8'))
+  const ids = new Set(graph.nodes.map(n => n.id))
+  for (const e of graph.edges) {
+    for (const end of [e.source, e.target]) {
+      if (!ids.has(end)) {
+        ids.add(end)
+        graph.nodes.push({ id: end, type: 'raw', title: path.basename(end) })
+      }
+    }
+  }
+  return graph
+}
+
+export function toGraphML(graph) {
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<graphml xmlns="http://graphml.graphdrawing.org/xmlns">',
+    '  <key id="d0" for="node" attr.name="type" attr.type="string"/>',
+    '  <key id="d1" for="node" attr.name="title" attr.type="string"/>',
+    '  <key id="d2" for="node" attr.name="status" attr.type="string"/>',
+    '  <key id="d3" for="edge" attr.name="type" attr.type="string"/>',
+    '  <graph id="llm_wiki" edgedefault="directed">',
+  ]
+  for (const n of graph.nodes) {
+    lines.push(`    <node id="${xmlEscape(n.id)}">`)
+    lines.push(`      <data key="d0">${xmlEscape(n.type ?? '')}</data>`)
+    lines.push(`      <data key="d1">${xmlEscape(n.title ?? '')}</data>`)
+    if (n.status) lines.push(`      <data key="d2">${xmlEscape(n.status)}</data>`)
+    lines.push('    </node>')
+  }
+  graph.edges.forEach((e, i) => {
+    lines.push(`    <edge id="e${i}" source="${xmlEscape(e.source)}" target="${xmlEscape(e.target)}"><data key="d3">${xmlEscape(e.type ?? '')}</data></edge>`)
+  })
+  lines.push('  </graph>', '</graphml>')
+  return lines.join('\n') + '\n'
+}
+
+export function toCypher(graph) {
+  const label = (t) => ({ source: 'Source', entity: 'Entity', concept: 'Concept', comparison: 'Comparison', raw: 'Raw' })[t] ?? 'Page'
+  const lines = graph.nodes.map(n =>
+    `MERGE (n:${label(n.type)} {id: '${cyEscape(n.id)}'}) SET n.title = '${cyEscape(n.title ?? '')}'${n.status ? `, n.status = '${cyEscape(n.status)}'` : ''};`)
+  for (const e of graph.edges) {
+    const rel = String(e.type ?? 'link').replace(/[^a-zA-Z_]/g, '_').toUpperCase()
+    lines.push(`MATCH (a {id: '${cyEscape(e.source)}'}), (b {id: '${cyEscape(e.target)}'}) MERGE (a)-[:${rel}]->(b);`)
+  }
+  return lines.join('\n') + '\n'
+}
+
+export function toHtml(graph) {
+  // <-escape keeps any "</script>" inside titles from terminating the script block.
+  const data = JSON.stringify({ nodes: graph.nodes, edges: graph.edges }).replace(/</g, '\\u003c')
+  return `<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>llm_wiki graph</title>
+<style>
+  html, body { margin: 0; height: 100%; background: #111; color: #ddd; font: 12px system-ui, sans-serif; }
+  #legend { position: fixed; top: 8px; left: 8px; background: #000a; padding: 6px 10px; border-radius: 6px; }
+  #legend span { margin-right: 10px; }
+  canvas { display: block; }
+</style>
+</head>
+<body>
+<div id="legend"></div>
+<canvas id="c"></canvas>
+<script>
+const GRAPH = ${data}
+const COLORS = { source: '#4e9af1', entity: '#5fbf77', concept: '#c98bdb', comparison: '#e0b25c', raw: '#777' }
+const canvas = document.getElementById('c')
+const ctx = canvas.getContext('2d')
+let W, H
+function resize() { W = canvas.width = innerWidth; H = canvas.height = innerHeight }
+resize(); addEventListener('resize', resize)
+
+document.getElementById('legend').innerHTML = Object.entries(COLORS)
+  .map(([t, c]) => '<span><b style="color:' + c + '">●</b> ' + t + '</span>').join('')
+  + '<span>◌ invalidated</span><span>drag: pan · wheel: zoom</span>'
+
+const nodes = GRAPH.nodes.map((n, i) => ({
+  ...n,
+  x: Math.cos(i * 2.399963) * (60 + 14 * Math.sqrt(i)),
+  y: Math.sin(i * 2.399963) * (60 + 14 * Math.sqrt(i)),
+  vx: 0, vy: 0,
+}))
+const byId = new Map(nodes.map(n => [n.id, n]))
+const edges = GRAPH.edges.map(e => ({ ...e, a: byId.get(e.source), b: byId.get(e.target) })).filter(e => e.a && e.b)
+
+let scale = Math.min(2, 500 / (60 + 14 * Math.sqrt(nodes.length))), ox = 0, oy = 0
+let dragging = false, px = 0, py = 0, hover = null
+canvas.onmousedown = (e) => { dragging = true; px = e.clientX; py = e.clientY }
+onmouseup = () => { dragging = false }
+onmousemove = (e) => {
+  if (dragging) { ox += e.clientX - px; oy += e.clientY - py; px = e.clientX; py = e.clientY; return }
+  const mx = (e.clientX - W / 2 - ox) / scale, my = (e.clientY - H / 2 - oy) / scale
+  hover = null
+  for (const n of nodes) if ((n.x - mx) ** 2 + (n.y - my) ** 2 < 100) { hover = n; break }
+}
+canvas.onwheel = (e) => { e.preventDefault(); scale *= e.deltaY < 0 ? 1.1 : 0.9 }
+
+let ticks = 0
+function step() {
+  if (ticks++ < 300) {
+    for (let i = 0; i < nodes.length; i++) for (let j = i + 1; j < nodes.length; j++) {
+      const a = nodes[i], b = nodes[j]
+      let dx = a.x - b.x, dy = a.y - b.y
+      const d2 = dx * dx + dy * dy || 1
+      const f = Math.min(1200 / d2, 4)
+      dx *= f / Math.sqrt(d2); dy *= f / Math.sqrt(d2)
+      a.vx += dx; a.vy += dy; b.vx -= dx; b.vy -= dy
+    }
+    for (const e of edges) {
+      const dx = e.b.x - e.a.x, dy = e.b.y - e.a.y
+      const d = Math.sqrt(dx * dx + dy * dy) || 1
+      const f = (d - 60) * 0.01
+      e.a.vx += dx / d * f; e.a.vy += dy / d * f
+      e.b.vx -= dx / d * f; e.b.vy -= dy / d * f
+    }
+    for (const n of nodes) { n.vx *= 0.85; n.vy *= 0.85; n.x += n.vx; n.y += n.vy }
+  }
+  ctx.clearRect(0, 0, W, H)
+  ctx.save()
+  ctx.translate(W / 2 + ox, H / 2 + oy)
+  ctx.scale(scale, scale)
+  ctx.strokeStyle = '#444'
+  for (const e of edges) {
+    ctx.setLineDash(e.type === 'superseded_by' ? [4, 3] : [])
+    ctx.beginPath(); ctx.moveTo(e.a.x, e.a.y); ctx.lineTo(e.b.x, e.b.y); ctx.stroke()
+  }
+  ctx.setLineDash([])
+  for (const n of nodes) {
+    ctx.fillStyle = COLORS[n.type] ?? '#aaa'
+    ctx.beginPath(); ctx.arc(n.x, n.y, n.type === 'raw' ? 3 : 6, 0, 7); ctx.fill()
+    if (n.status === 'invalidated') {
+      ctx.strokeStyle = '#e05c5c'; ctx.setLineDash([2, 2])
+      ctx.beginPath(); ctx.arc(n.x, n.y, 9, 0, 7); ctx.stroke(); ctx.setLineDash([])
+    }
+  }
+  ctx.fillStyle = '#ddd'
+  if (scale > 0.8) for (const n of nodes) if (n.type !== 'raw') ctx.fillText(n.title ?? n.id, n.x + 8, n.y + 3)
+  if (hover) { ctx.font = 'bold 12px system-ui'; ctx.fillStyle = '#fff'; ctx.fillText(hover.id + (hover.status ? ' [' + hover.status + ']' : ''), hover.x + 8, hover.y - 8); ctx.font = '12px system-ui' }
+  ctx.restore()
+  requestAnimationFrame(step)
+}
+step()
+</script>
+</body>
+</html>
+`
+}
+
+const RENDERERS = { graphml: toGraphML, cypher: toCypher, html: toHtml }
+
+export function exportGraph(kbRoot, { format, out } = {}) {
+  const render = RENDERERS[format]
+  if (!render) throw new Error(`unknown format: ${format} (expected ${Object.keys(RENDERERS).join(' | ')})`)
+  const graph = loadGraph(kbRoot)
+  const outPath = out ?? path.join(kbRoot, `graph.${format === 'graphml' ? 'graphml' : format === 'cypher' ? 'cypher' : 'html'}`)
+  fs.writeFileSync(outPath, render(graph))
+  return { out: outPath, nodeCount: graph.nodes.length, edgeCount: graph.edges.length }
+}

package/src/frontmatter.mjs

@@ -0,0 +1,16 @@
+import YAML from 'yaml'
+
+const FM_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/
+
+export function parseFrontmatter(text) {
+  const m = text.match(FM_RE)
+  if (!m) return null
+  let data
+  try { data = YAML.parse(m[1]) } catch { return { error: 'invalid-yaml' } }
+  if (data === null || typeof data !== 'object') return { error: 'invalid-yaml' }
+  return { data, body: text.slice(m[0].length) }
+}
+
+export function serializeFrontmatter(data, body) {
+  return `---\n${YAML.stringify(data)}---\n\n${body.replace(/^\n+/, '')}`
+}

package/src/hashing.mjs

@@ -0,0 +1,35 @@
+import crypto from 'node:crypto'
+import fs from 'node:fs'
+
+export const sha256Text = (t) => crypto.createHash('sha256').update(t).digest('hex')
+export const sha256File = (p) => crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex')
+
+function fnv1a(str) {
+  let h = 0x811c9dc5
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i)
+    h = Math.imul(h, 0x01000193) >>> 0
+  }
+  return h
+}
+
+export function minhashSignature(text, perms = 32) {
+  const norm = text.toLowerCase().replace(/\s+/g, ' ')
+  const shingles = new Set()
+  for (let i = 0; i <= norm.length - 5; i++) shingles.add(norm.slice(i, i + 5))
+  if (shingles.size === 0) return new Array(perms).fill(0)
+  const sig = new Array(perms).fill(Infinity)
+  for (const s of shingles) {
+    for (let p = 0; p < perms; p++) {
+      const h = fnv1a(p + ' ' + s)
+      if (h < sig[p]) sig[p] = h
+    }
+  }
+  return sig
+}
+
+export function jaccardEstimate(a, b) {
+  let eq = 0
+  for (let i = 0; i < a.length; i++) if (a[i] === b[i]) eq++
+  return eq / a.length
+}

package/src/indexer.mjs

@@ -0,0 +1,87 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+import { DEFAULT_CONFIG } from './templates.mjs'
+import { listWikiPages, isInvalidated } from './pages.mjs'
+
+const WIKILINK_RE = /\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g
+
+export function extractWikilinks(body) {
+  const out = []
+  for (const m of body.matchAll(WIKILINK_RE)) {
+    const target = m[1].trim().replace(/\.md$/, '')
+    if (!out.includes(target)) out.push(target)
+  }
+  return out
+}
+
+const SECTION_TITLES = { source: 'Sources', entity: 'Entities', concept: 'Concepts', comparison: 'Comparisons', other: 'Other' }
+
+export function buildIndex(kbRoot) {
+  const p = kbPaths(kbRoot)
+  const cfg = fs.existsSync(p.config) ? { ...DEFAULT_CONFIG, ...JSON.parse(fs.readFileSync(p.config, 'utf8')) } : DEFAULT_CONFIG
+  const pages = listWikiPages(kbRoot).filter(pg => !pg.error)
+
+  // preserve pending section
+  let pending = '## Pending concepts\n'
+  if (fs.existsSync(p.indexMd)) {
+    const m = fs.readFileSync(p.indexMd, 'utf8').match(/## Pending concepts[\s\S]*$/)
+    if (m) pending = m[0].trimEnd() + '\n'
+  }
+
+  // `other` collects unknown types so index.md and graph.json agree on a page's bucket
+  // (graph nodes still carry the raw type). Rendered as a final `## Other` section, only when non-empty.
+  const byType = { source: [], entity: [], concept: [], comparison: [], other: [] }
+  for (const pg of pages) (byType[pg.data.type] ?? byType.other).push(pg)
+  const line = (pg) => {
+    const base = `- [[${pg.relPath.replace(/\.md$/, '')}]] — ${pg.data.description ?? ''}`
+    if (!isInvalidated(pg)) return base
+    return pg.data.superseded_by
+      ? `${base} ⚠ invalidated, superseded by [[${pg.data.superseded_by}]]`
+      : `${base} ⚠ invalidated`
+  }
+
+  let indexBody = '# Index\n\n> Auto-generated by `llm-wiki index`. Only the "Pending concepts" section may be edited by the LLM.\n'
+  const topicsSplit = pages.length > cfg.indexSplitAt
+  if (topicsSplit) {
+    fs.mkdirSync(p.topics, { recursive: true })
+    for (const [type, list] of Object.entries(byType)) {
+      if (!list.length) continue
+      fs.writeFileSync(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
+      indexBody += `\n## ${SECTION_TITLES[type]}\nSee [[topics/${type}]] (${list.length} pages)\n`
+    }
+  } else {
+    for (const [type, list] of Object.entries(byType)) {
+      if (type === 'other' && !list.length) continue
+      indexBody += `\n## ${SECTION_TITLES[type]}\n${list.map(line).join('\n')}\n`
+    }
+  }
+  fs.writeFileSync(p.indexMd, `${indexBody}\n${pending}`)
+
+  const ids = new Set(pages.map(pg => pg.relPath.replace(/\.md$/, '')))
+  const nodes = pages.map(pg => ({
+    id: pg.relPath.replace(/\.md$/, ''),
+    type: pg.data.type,
+    title: pg.data.title,
+    ...(isInvalidated(pg) ? { status: 'invalidated' } : {}),
+  }))
+  const edges = []
+  for (const pg of pages) {
+    const id = pg.relPath.replace(/\.md$/, '')
+    for (const target of extractWikilinks(pg.body)) {
+      if (ids.has(target)) edges.push({ source: id, target, type: 'wikilink' })
+    }
+    for (const src of pg.data.sources ?? []) edges.push({ source: id, target: String(src), type: 'source' })
+    if (pg.data.superseded_by && ids.has(String(pg.data.superseded_by))) {
+      edges.push({ source: id, target: String(pg.data.superseded_by), type: 'superseded_by' })
+    }
+  }
+  fs.writeFileSync(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
+
+  const kbName = path.basename(path.resolve(kbRoot))
+  const llms = [`# ${kbName}`, '', `> llm_wiki knowledge base. Read wiki/index.md first, then open full pages.`, '', '## Pages',
+    ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${pg.data.title}](wiki/${pg.relPath}): ${pg.data.description ?? ''}`)].join('\n') + '\n'
+  fs.writeFileSync(p.llmsTxt, llms)
+
+  return { pageCount: pages.length, topicsSplit }
+}

package/src/init.mjs

@@ -0,0 +1,43 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+import { DEFAULT_CONFIG, agentsMdTemplate, readmeTemplate } from './templates.mjs'
+
+const INDEX_SKELETON = `# Index
+
+> Auto-generated by \`llm-wiki index\`. Only the "Pending concepts" section may be edited by the LLM.
+
+## Sources
+
+## Entities
+
+## Concepts
+
+## Comparisons
+
+## Pending concepts
+`
+
+export function initKb(root) {
+  const p = kbPaths(root, DEFAULT_CONFIG)
+  const created = []
+  const skipped = []
+  const dir = (d) => {
+    if (fs.existsSync(d)) { skipped.push(d) } else { fs.mkdirSync(d, { recursive: true }); created.push(d) }
+  }
+  const file = (f, content) => {
+    if (fs.existsSync(f)) { skipped.push(f) } else { fs.writeFileSync(f, content); created.push(f) }
+  }
+  dir(root)
+  dir(p.raw)
+  dir(p.sources); dir(p.entities); dir(p.concepts); dir(p.comparisons)
+  const { rawDir, schemaFile, linkStyle, ...emittedConfig } = DEFAULT_CONFIG
+  file(p.config, JSON.stringify(emittedConfig, null, 2) + '\n')
+  file(p.schemaFile, agentsMdTemplate(DEFAULT_CONFIG))
+  file(p.readme, readmeTemplate(path.basename(path.resolve(root))))
+  file(p.manifest, JSON.stringify({ files: {} }, null, 2) + '\n')
+  file(p.indexMd, INDEX_SKELETON)
+  file(p.logMd, '# Log\n')
+  file(p.hotMd, '# Hot\n\n(recent activity snapshot, maintained by the LLM, ~500 chars)\n')
+  return { created, skipped }
+}

package/src/lint.mjs

@@ -0,0 +1,92 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+import { DEFAULT_CONFIG } from './templates.mjs'
+import { listWikiPages, validatePage, isInvalidated, PAGE_STATUSES } from './pages.mjs'
+import { extractWikilinks, buildIndex } from './indexer.mjs'
+import { loadManifest } from './manifest.mjs'
+
+export async function lintKb(kbRoot, { fix = false } = {}) {
+  const p = kbPaths(kbRoot)
+  const cfg = fs.existsSync(p.config) ? { ...DEFAULT_CONFIG, ...JSON.parse(fs.readFileSync(p.config, 'utf8')) } : DEFAULT_CONFIG
+  const pages = listWikiPages(kbRoot)
+  const mechanical = []
+  const semantic = []
+  const autoFixed = []
+  const ids = new Set(pages.filter(pg => !pg.error).map(pg => pg.relPath.replace(/\.md$/, '')))
+  const incoming = new Map()
+
+  for (const pg of pages) {
+    if (pg.error) { mechanical.push({ rule: 'invalid-frontmatter', path: pg.relPath, detail: pg.error }); continue }
+    for (const issue of validatePage(pg)) mechanical.push({ rule: 'missing-field', path: pg.relPath, detail: issue })
+    for (const target of extractWikilinks(pg.body)) {
+      if (target.startsWith('raw/')) {
+        if (!fs.existsSync(path.join(kbRoot, target)) && !fs.existsSync(path.join(kbRoot, target + '.md'))) mechanical.push({ rule: 'broken-raw-link', path: pg.relPath, detail: `-> ${target}` })
+        continue
+      }
+      if (!ids.has(target)) mechanical.push({ rule: 'broken-wikilink', path: pg.relPath, detail: `-> ${target}` })
+      else incoming.set(target, (incoming.get(target) ?? 0) + 1)
+    }
+    for (const src of pg.data.sources ?? []) {
+      if (!fs.existsSync(path.join(kbRoot, String(src)))) mechanical.push({ rule: 'missing-raw-source', path: pg.relPath, detail: String(src) })
+    }
+    if (pg.data.status !== undefined && !PAGE_STATUSES.includes(pg.data.status)) {
+      mechanical.push({ rule: 'invalid-status', path: pg.relPath, detail: `status "${pg.data.status}" (expected ${PAGE_STATUSES.join(' | ')})` })
+    }
+    if (pg.data.superseded_by !== undefined && !ids.has(String(pg.data.superseded_by))) {
+      mechanical.push({ rule: 'superseded-target-missing', path: pg.relPath, detail: `superseded_by -> ${pg.data.superseded_by}` })
+    }
+  }
+  for (const pg of pages) {
+    if (pg.error) continue
+    const id = pg.relPath.replace(/\.md$/, '')
+    if (pg.data.type !== 'source' && pg.data.type !== 'comparison' && !isInvalidated(pg) && !incoming.has(id)) mechanical.push({ rule: 'orphan-page', path: pg.relPath, detail: 'no incoming wikilinks' })
+  }
+
+  if (fs.existsSync(p.indexMd)) {
+    const pendingSection = fs.readFileSync(p.indexMd, 'utf8').match(/## Pending concepts([\s\S]*)$/)?.[1] ?? ''
+    for (const line of pendingSection.split('\n')) {
+      const m = line.match(/^-\s*(.+?)\s*—/)
+      if (!m) continue
+      const refs = (line.match(/\[\[/g) ?? []).length
+      if (refs >= cfg.conceptThreshold) semantic.push({ task: 'promote-concepts', detail: `${m[1]} (${refs} sources)` })
+    }
+  }
+  const byTag = new Map()
+  for (const pg of pages) {
+    if (pg.error) continue
+    for (const tag of pg.data.tags ?? []) {
+      if (!byTag.has(tag)) byTag.set(tag, [])
+      byTag.get(tag).push(pg.relPath)
+    }
+  }
+  for (const [tag, list] of byTag) {
+    // Only flag small shared-tag clusters: 2-5 pages are plausible contradiction candidates.
+    // Larger groups are navigation/topic tags (one tag on dozens of pages) — unactionable noise.
+    if (list.length >= 2 && list.length <= 5) semantic.push({ task: 'contradiction-scan', detail: `tag "${tag}": ${list.join(', ')}` })
+  }
+
+  // stale-scan: a raw file reconverted after a page's last update means the page may
+  // describe an outdated version of its source. Deterministic candidate generation;
+  // the LLM judges whether the page actually needs updating (STALE benchmark: LLMs
+  // detect staleness unaided at only ~55% accuracy — never rely on spontaneous detection).
+  const manifest = loadManifest(kbRoot)
+  const convertedAtByRaw = new Map()
+  for (const entry of Object.values(manifest.files)) {
+    if (entry.raw && entry.convertedAt) convertedAtByRaw.set(entry.raw, entry.convertedAt)
+  }
+  for (const pg of pages) {
+    if (pg.error || isInvalidated(pg)) continue
+    const updated = String(pg.data.updated ?? '')
+    if (!updated) continue
+    for (const src of pg.data.sources ?? []) {
+      const conv = convertedAtByRaw.get(String(src))
+      if (conv && conv > updated) semantic.push({ task: 'stale-scan', detail: `${pg.relPath}: ${src} reconverted ${conv}, page updated ${updated}` })
+    }
+  }
+
+  if (fix) { buildIndex(kbRoot); autoFixed.push('index-rebuilt') }
+  const report = { autoFixed, mechanical, semantic }
+  fs.writeFileSync(path.join(kbRoot, '.lint-report.json'), JSON.stringify(report, null, 2) + '\n')
+  return report
+}

package/src/llm-config.mjs

@@ -0,0 +1,65 @@
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+
+const BUILTIN = {
+  priority: ['openai', 'openrouter'],
+  providers: {
+    openai: { baseURL: 'https://api.openai.com/v1', apiKeyEnv: 'OPENAI_API_KEY', model: 'gpt-4o-mini' },
+    openrouter: { baseURL: 'https://openrouter.ai/api/v1', apiKeyEnv: 'OPENROUTER_API_KEY', model: 'anthropic/claude-sonnet-5' },
+  },
+}
+
+function resolveProviders(fileCfg) {
+  const { priority, providers } = fileCfg.providers ? fileCfg : BUILTIN
+  for (const name of priority ?? Object.keys(providers)) {
+    const prov = providers[name]
+    if (!prov) continue
+    const key = prov.apiKey ?? process.env[prov.apiKeyEnv]
+    if (key) return { baseURL: prov.baseURL, apiKey: key, model: prov.model }
+  }
+  return null
+}
+
+export function loadLlmConfig(kbRoot) {
+  const dir = process.env.LLM_WIKI_CONFIG_DIR ?? path.join(os.homedir(), '.llm-wiki')
+  const globalFile = path.join(dir, 'config.json')
+  const fileCfg = fs.existsSync(globalFile) ? JSON.parse(fs.readFileSync(globalFile, 'utf8')) : {}
+  // flat form: explicit custom endpoint wins outright
+  let cfg = (fileCfg.baseURL && fileCfg.apiKey && fileCfg.model)
+    ? { baseURL: fileCfg.baseURL, apiKey: fileCfg.apiKey, model: fileCfg.model }
+    : resolveProviders(fileCfg)
+  const p = kbPaths(kbRoot)
+  if (fs.existsSync(p.config)) {
+    const kbCfg = JSON.parse(fs.readFileSync(p.config, 'utf8'))
+    if (kbCfg.llm) {
+      if (process.env.LLM_WIKI_ALLOW_KB_LLM_OVERRIDE === '1') {
+        // Opt-in: trust the KB fully (e.g. your own first-party KB).
+        cfg = { ...(cfg ?? {}), ...kbCfg.llm }
+      } else {
+        // Security: a third-party KB must not redirect requests (baseURL) or
+        // inject credentials — that would leak the user's env API key to an
+        // attacker endpoint. Honor only the model name by default.
+        const ignored = Object.keys(kbCfg.llm).filter(key => key !== 'model')
+        if (ignored.length) process.stderr.write(`warning: ignoring kb-level llm.${ignored.join(',')} override (security); set LLM_WIKI_ALLOW_KB_LLM_OVERRIDE=1 to allow\n`)
+        if (kbCfg.llm.model !== undefined) cfg = { ...(cfg ?? {}), model: kbCfg.llm.model }
+      }
+    }
+  }
+  if (cfg && process.env.LLM_WIKI_API_KEY) cfg.apiKey = process.env.LLM_WIKI_API_KEY
+  if (!cfg || !cfg.baseURL || !cfg.apiKey || !cfg.model) return null
+  return cfg
+}
+
+// Node's built-in fetch rejects the npm undici package's dispatcher
+// ("invalid onRequestStart method": handler interface mismatch between
+// undici 8.x and Node's bundled undici), so when a proxy is configured we
+// must use undici's own fetch together with its EnvHttpProxyAgent.
+export async function makeTransport() {
+  const hasProxy = ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY', 'all_proxy']
+    .some(v => process.env[v])
+  if (!hasProxy) return { fetchImpl: globalThis.fetch, dispatcher: undefined }
+  const { fetch: proxyFetch, EnvHttpProxyAgent } = await import('undici')
+  return { fetchImpl: proxyFetch, dispatcher: new EnvHttpProxyAgent() }
+}

package/src/manifest.mjs

@@ -0,0 +1,27 @@
+import fs from 'node:fs'
+import { kbPaths } from './paths.mjs'
+
+export function loadManifest(kbRoot) {
+  const p = kbPaths(kbRoot)
+  if (!fs.existsSync(p.manifest)) return { files: {} }
+  return JSON.parse(fs.readFileSync(p.manifest, 'utf8'))
+}
+
+export function saveManifest(kbRoot, manifest) {
+  const p = kbPaths(kbRoot)
+  fs.writeFileSync(p.manifest, JSON.stringify(manifest, null, 2) + '\n')
+}
+
+export function diffManifest(manifest, entries) {
+  const added = [], changed = [], unchanged = []
+  const seen = new Set()
+  for (const e of entries) {
+    seen.add(e.rel)
+    const prev = manifest.files[e.rel]
+    if (!prev) added.push(e)
+    else if (prev.hash !== e.hash) changed.push(e)
+    else unchanged.push(e)
+  }
+  const removed = Object.keys(manifest.files).filter(rel => !seen.has(rel))
+  return { added, changed, removed, unchanged }
+}

package/src/pages.mjs

@@ -0,0 +1,42 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { kbPaths } from './paths.mjs'
+import { parseFrontmatter } from './frontmatter.mjs'
+
+const PAGE_DIRS = ['sources', 'entities', 'concepts', 'comparisons']
+const REQUIRED = ['type', 'title', 'description', 'tags', 'created', 'updated']
+
+export const PAGE_STATUSES = ['active', 'invalidated']
+
+export function isInvalidated(page) {
+  return page.data?.status === 'invalidated'
+}
+
+export function listWikiPages(kbRoot) {
+  const p = kbPaths(kbRoot)
+  const pages = []
+  for (const dir of PAGE_DIRS) {
+    const abs = path.join(p.wiki, dir)
+    if (!fs.existsSync(abs)) continue
+    for (const f of fs.readdirSync(abs)) {
+      if (!f.endsWith('.md')) continue
+      const fileAbs = path.join(abs, f)
+      const text = fs.readFileSync(fileAbs, 'utf8')
+      const fm = parseFrontmatter(text)
+      if (fm === null) pages.push({ relPath: `${dir}/${f}`, abs: fileAbs, data: {}, body: text, error: 'missing-frontmatter' })
+      else if (fm.error) pages.push({ relPath: `${dir}/${f}`, abs: fileAbs, data: {}, body: text, error: fm.error })
+      else pages.push({ relPath: `${dir}/${f}`, abs: fileAbs, data: fm.data, body: fm.body })
+    }
+  }
+  return pages
+}
+
+export function validatePage(page) {
+  const issues = []
+  if (page.error) return [page.error]
+  for (const k of REQUIRED) {
+    if (page.data[k] === undefined || page.data[k] === null || page.data[k] === '') issues.push(`missing field: ${k}`)
+  }
+  if (!Array.isArray(page.data.sources)) issues.push('missing field: sources (evidence chain)')
+  return issues
+}