llm-wiki-kit 0.1.5 → 0.1.6

package/src/wiki-lint.js

@@ -0,0 +1,271 @@
+import { readdir } from 'fs/promises';
+import { dirname, isAbsolute, join, parse, relative, resolve, sep } from 'path';
+import { exists, readText } from './fs-utils.js';
+import { hasSecretLikeText } from './redaction.js';
+import {
+  buildAliasMap,
+  buildWikiGraph,
+  collectWikiPages,
+  normalizeTarget,
+  resolveWikiLink,
+  wikiRoot,
+} from './wiki-model.js';
+
+const VALID_TYPES = new Set([
+  'source',
+  'concept',
+  'entity',
+  'decision',
+  'architecture',
+  'debugging',
+  'context',
+  'query',
+  'session-log',
+  'convention',
+]);
+const VALID_STATUS = new Set(['draft', 'reviewed', 'stale']);
+const VALID_CONFIDENCE = new Set(['high', 'medium', 'low']);
+const VALID_MEMORY_TYPES = new Set(['semantic', 'episodic', 'procedural']);
+const CORE_PAGES = new Set(['wiki/index.md', 'wiki/log.md', 'wiki/memory.md']);
+
+function issue(severity, code, path, message) {
+  return { severity, code, path, message };
+}
+
+function isDateLike(value) {
+  return value === 'unknown' || /^\d{4}-\d{2}-\d{2}$/.test(String(value || ''));
+}
+
+function relativeMarkdownTarget(page, link) {
+  return resolve(dirname(page.absolutePath), link.path);
+}
+
+function isOutsideProject(projectRoot, absolutePath) {
+  const root = resolve(projectRoot);
+  const target = resolve(absolutePath);
+  const rel = relative(root, target);
+  return rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel);
+}
+
+function invalidSourceId(sourceId) {
+  const id = String(sourceId || '').trim().replace(/\\/g, '/');
+  if (!id) return true;
+  if (id.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(id)) return true;
+  return id.split('/').includes('..');
+}
+
+function sourceCandidates(projectRoot, sourceId) {
+  const id = String(sourceId || '').trim();
+  if (!id) return [];
+  const rawSources = join(projectRoot, 'llm-wiki', 'raw', 'sources');
+  const rawInbox = join(projectRoot, 'llm-wiki', 'raw', 'inbox');
+  const wikiSources = join(projectRoot, 'llm-wiki', 'wiki', 'sources');
+  const candidates = [
+    join(rawSources, id),
+    join(rawSources, `${id}.md`),
+    join(rawSources, `${id}.txt`),
+    join(rawInbox, id),
+    join(rawInbox, `${id}.md`),
+    join(wikiSources, id),
+    join(wikiSources, `${id}.md`),
+  ];
+  if (id.includes('/') || id.includes('\\')) {
+    candidates.push(join(projectRoot, id));
+    candidates.push(join(projectRoot, 'llm-wiki', id));
+  }
+  return candidates.map((candidate) => resolve(candidate))
+    .filter((candidate) => !isOutsideProject(projectRoot, candidate));
+}
+
+async function sourceExists(projectRoot, sourceId) {
+  for (const candidate of sourceCandidates(projectRoot, sourceId)) {
+    if (await pathExistsNormalized(candidate)) return true;
+  }
+  return false;
+}
+
+function titleKey(page) {
+  return normalizeTarget(page.title);
+}
+
+function isOrphanCandidate(page) {
+  if (CORE_PAGES.has(page.rel)) return false;
+  if (page.rel.startsWith('wiki/queries/')) return false;
+  if (page.rel.startsWith('wiki/context/')) return false;
+  return true;
+}
+
+async function pathExistsNormalized(absolutePath) {
+  if (await exists(absolutePath)) return true;
+  const target = resolve(absolutePath);
+  const parsed = parse(target);
+  const segments = target.slice(parsed.root.length).split(sep).filter(Boolean);
+  let current = parsed.root || sep;
+  for (const segment of segments) {
+    let entries = [];
+    try {
+      entries = await readdir(current);
+    } catch {
+      return false;
+    }
+    const match = entries.find((entry) => entry.normalize('NFC') === segment.normalize('NFC'));
+    if (!match) return false;
+    current = join(current, match);
+  }
+  return true;
+}
+
+export async function runLint(projectRoot, options = {}) {
+  const issues = [];
+  const root = wikiRoot(projectRoot);
+  if (!(await exists(root))) {
+    issues.push(issue('error', 'missing-wiki', 'llm-wiki/wiki', 'wiki directory does not exist'));
+    return lintResult(projectRoot, [], issues);
+  }
+
+  const pages = await collectWikiPages(projectRoot, {
+    maxFiles: options.maxFiles || 1000,
+    maxChars: options.maxChars || 75000,
+  });
+  const byRel = new Map(pages.map((page) => [page.rel, page]));
+  const aliasMap = buildAliasMap(pages);
+  const graph = buildWikiGraph(pages);
+
+  for (const rel of ['wiki/index.md', 'wiki/log.md', 'wiki/memory.md']) {
+    if (!byRel.has(rel)) {
+      issues.push(issue(rel === 'wiki/memory.md' ? 'warning' : 'error', 'missing-core-page', rel, 'core wiki page is missing'));
+    }
+  }
+
+  for (const [alias, matches] of aliasMap.entries()) {
+    if (matches.length > 1 && alias.length > 0) {
+      issues.push(issue('warning', 'duplicate-alias', matches.join(', '), `alias "${alias}" maps to multiple pages`));
+    }
+  }
+
+  const titles = new Map();
+  for (const page of pages) {
+    const key = titleKey(page);
+    if (!key) continue;
+    if (!titles.has(key)) titles.set(key, []);
+    titles.get(key).push(page.rel);
+  }
+  for (const [key, matches] of titles.entries()) {
+    if (matches.length > 1) {
+      issues.push(issue('warning', 'duplicate-title', matches.join(', '), `title "${key}" appears on multiple pages`));
+    }
+  }
+
+  for (const page of pages) {
+    if (hasSecretLikeText(page.content)) {
+      issues.push(issue('error', 'secret-like-content', page.rel, 'page contains token, credential, private-key, or secret-like text'));
+    }
+
+    if (!CORE_PAGES.has(page.rel) && !page.hasFrontmatter) {
+      issues.push(issue('warning', 'missing-frontmatter', page.rel, 'curated wiki page should have YAML frontmatter'));
+      continue;
+    }
+
+    if (page.hasFrontmatter) {
+      if (!page.type) issues.push(issue('warning', 'missing-type', page.rel, 'frontmatter is missing type'));
+      if (page.type && !VALID_TYPES.has(page.type)) issues.push(issue('error', 'invalid-type', page.rel, `invalid type: ${page.type}`));
+      if (!page.status) issues.push(issue('warning', 'missing-status', page.rel, 'frontmatter is missing status'));
+      if (page.status && !VALID_STATUS.has(page.status)) issues.push(issue('error', 'invalid-status', page.rel, `invalid status: ${page.status}`));
+      if (!page.confidence) issues.push(issue('warning', 'missing-confidence', page.rel, 'frontmatter is missing confidence'));
+      if (page.confidence && !VALID_CONFIDENCE.has(page.confidence)) issues.push(issue('error', 'invalid-confidence', page.rel, `invalid confidence: ${page.confidence}`));
+      if (page.frontmatter.memory_type && !VALID_MEMORY_TYPES.has(page.frontmatter.memory_type)) {
+        issues.push(issue('error', 'invalid-memory-type', page.rel, `invalid memory_type: ${page.frontmatter.memory_type}`));
+      }
+      if (page.frontmatter.importance !== undefined) {
+        const importance = Number(page.frontmatter.importance);
+        if (!Number.isInteger(importance) || importance < 1 || importance > 5) {
+          issues.push(issue('error', 'invalid-importance', page.rel, 'importance must be an integer from 1 to 5'));
+        }
+      }
+      if (page.frontmatter.last_updated && !isDateLike(page.frontmatter.last_updated)) {
+        issues.push(issue('warning', 'invalid-last-updated', page.rel, `last_updated should be YYYY-MM-DD or unknown: ${page.frontmatter.last_updated}`));
+      }
+      if (page.frontmatter.last_verified && !isDateLike(page.frontmatter.last_verified)) {
+        issues.push(issue('warning', 'invalid-last-verified', page.rel, `last_verified should be YYYY-MM-DD or unknown: ${page.frontmatter.last_verified}`));
+      }
+      if (page.status === 'stale') {
+        issues.push(issue('warning', 'stale-page', page.rel, 'page is marked stale'));
+      }
+    }
+
+    for (const link of page.wikilinks) {
+      const resolved = resolveWikiLink(aliasMap, link.raw, page);
+      const matches = aliasMap.get(normalizeTarget(link.target)) || [];
+      if (!resolved && matches.length === 0) {
+        issues.push(issue('error', 'broken-wikilink', page.rel, `unresolved wikilink: [[${link.raw}]]`));
+      } else if (!resolved && matches.length > 1) {
+        issues.push(issue('error', 'ambiguous-wikilink', page.rel, `ambiguous wikilink: [[${link.raw}]] -> ${matches.join(', ')}`));
+      }
+    }
+
+    for (const link of page.markdownLinks) {
+      const target = relativeMarkdownTarget(page, link);
+      if (isOutsideProject(projectRoot, target)) {
+        issues.push(issue('warning', 'outside-project-link', page.rel, `markdown link points outside project: ${link.raw}`));
+      } else if (!(await pathExistsNormalized(target))) {
+        issues.push(issue('error', 'broken-markdown-link', page.rel, `missing markdown link target: ${link.raw}`));
+      }
+    }
+
+    for (const sourceId of page.sourceIds) {
+      if (invalidSourceId(sourceId)) {
+        issues.push(issue('error', 'invalid-source-id', page.rel, 'source_id points outside the project or uses unsupported syntax'));
+        continue;
+      }
+      if (!(await sourceExists(projectRoot, sourceId))) {
+        issues.push(issue('warning', 'missing-source', page.rel, `source_id has no matching raw/wiki source file: ${sourceId}`));
+      }
+    }
+  }
+
+  const memoryText = await readText(join(projectRoot, 'llm-wiki', 'wiki', 'memory.md'), '');
+  if (Buffer.byteLength(memoryText, 'utf8') > 25000) {
+    issues.push(issue('warning', 'memory-too-large', 'wiki/memory.md', 'memory.md is larger than the hook excerpt budget'));
+  }
+  for (const page of pages.filter(isOrphanCandidate)) {
+    const backlinks = graph.backlinks.get(page.rel);
+    if (!backlinks || backlinks.size === 0) {
+      issues.push(issue('warning', 'orphan-page', page.rel, 'page is not linked from memory/index and has no backlinks'));
+    }
+  }
+
+  return lintResult(projectRoot, pages, issues);
+}
+
+function lintResult(projectRoot, pages, issues) {
+  const errorCount = issues.filter((item) => item.severity === 'error').length;
+  const warningCount = issues.filter((item) => item.severity === 'warning').length;
+  return {
+    workspace: projectRoot,
+    ok: errorCount === 0,
+    pages: pages.length,
+    issueCount: issues.length,
+    errorCount,
+    warningCount,
+    issues,
+  };
+}
+
+export function formatLintResult(result) {
+  const lines = [
+    'llm-wiki lint',
+    `- workspace: ${result.workspace}`,
+    `- pages: ${result.pages}`,
+    `- errors: ${result.errorCount}`,
+    `- warnings: ${result.warningCount}`,
+  ];
+  if (result.issues.length === 0) {
+    lines.push('- result: ok');
+    return lines.join('\n');
+  }
+  lines.push('', 'Issues:');
+  for (const item of result.issues) {
+    lines.push(`- ${item.severity} ${item.code} ${item.path}: ${item.message}`);
+  }
+  return lines.join('\n');
+}

package/src/wiki-model.js

@@ -0,0 +1,263 @@
+import { basename, join, posix, relative } from 'path';
+import { exists, listMarkdownFiles, readText } from './fs-utils.js';
+import { normalizeForStorage } from './redaction.js';
+
+export const MEMORY_LINE_LIMIT = 200;
+export const MEMORY_BYTE_LIMIT = 25 * 1024;
+export const DEFAULT_MAX_WIKI_FILES = 500;
+export const DEFAULT_MAX_PAGE_CHARS = 50000;
+
+export function wikiRoot(projectRoot) {
+  return join(projectRoot, 'llm-wiki', 'wiki');
+}
+
+export function wikiRel(projectRoot, file) {
+  return relative(join(projectRoot, 'llm-wiki'), file).split('\\').join('/');
+}
+
+function stripQuotes(value) {
+  return String(value || '').replace(/^["']|["']$/g, '');
+}
+
+function parseScalar(value) {
+  const trimmed = String(value || '').trim();
+  if (trimmed === '[]') return [];
+  if (/^\[.*\]$/.test(trimmed)) {
+    const inner = trimmed.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner.split(',').map((item) => stripQuotes(item.trim())).filter(Boolean);
+  }
+  return stripQuotes(trimmed);
+}
+
+export function parseFrontmatter(content) {
+  const text = normalizeForStorage(content || '');
+  const match = text.match(/^---\n([\s\S]*?)\n---\n?/);
+  if (!match) return { data: {}, body: text, hasFrontmatter: false };
+
+  const data = {};
+  const lines = match[1].split('\n');
+  let currentArrayKey = null;
+  for (const line of lines) {
+    const arrayItem = line.match(/^\s*-\s+(.+)\s*$/);
+    if (arrayItem && currentArrayKey) {
+      data[currentArrayKey].push(stripQuotes(arrayItem[1].trim()));
+      continue;
+    }
+
+    const keyValue = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
+    if (!keyValue) {
+      currentArrayKey = null;
+      continue;
+    }
+
+    const [, key, value] = keyValue;
+    if (value.trim() === '') {
+      data[key] = [];
+      currentArrayKey = key;
+    } else {
+      data[key] = parseScalar(value);
+      currentArrayKey = Array.isArray(data[key]) ? key : null;
+    }
+  }
+
+  return {
+    data,
+    body: text.slice(match[0].length),
+    hasFrontmatter: true,
+  };
+}
+
+export function firstHeading(body) {
+  return (body.match(/^#\s+(.+)$/m)?.[1] || '').trim();
+}
+
+export function normalizeTarget(value) {
+  return normalizeForStorage(value || '')
+    .split('|')[0]
+    .split('#')[0]
+    .trim()
+    .replace(/\\/g, '/')
+    .replace(/^\.?\//, '')
+    .replace(/\.md$/i, '')
+    .toLowerCase();
+}
+
+export function stripMarkdownCode(content) {
+  return normalizeForStorage(content || '')
+    .replace(/(^|\n)(`{3,}|~{3,})[^\n]*\n[\s\S]*?\n\2[ \t]*(?=\n|$)/g, '$1')
+    .replace(/`+[^`\n]*`+/g, '');
+}
+
+export function extractWikilinks(content) {
+  const links = [];
+  const regex = /\[\[([^\]]+)\]\]/g;
+  const searchable = stripMarkdownCode(content);
+  let match = regex.exec(searchable);
+  while (match) {
+    const raw = match[1].trim();
+    const target = normalizeTarget(raw);
+    if (target) links.push({ raw, target });
+    match = regex.exec(searchable);
+  }
+  return links;
+}
+
+export function extractMarkdownLinks(content) {
+  const links = [];
+  const regex = /(?<!!)\[[^\]]+\]\(([^)]+)\)/g;
+  const searchable = stripMarkdownCode(content);
+  let match = regex.exec(searchable);
+  while (match) {
+    const raw = match[1].trim();
+    const href = raw.replace(/^<|>$/g, '').split(/\s+/)[0];
+    if (!href || href.startsWith('#') || href.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(href)) {
+      match = regex.exec(searchable);
+      continue;
+    }
+    const path = href.split('#')[0].split('?')[0].replace(/\\/g, '/');
+    if (path) links.push({ raw, path });
+    match = regex.exec(searchable);
+  }
+  return links;
+}
+
+export function parseWikiPage(projectRoot, file, content) {
+  const rel = wikiRel(projectRoot, file);
+  const { data, body, hasFrontmatter } = parseFrontmatter(content);
+  const title = String(data.title || firstHeading(body) || basename(file, '.md')).trim();
+  const stem = basename(file, '.md');
+  const relNoExt = rel.replace(/\.md$/i, '');
+  const withoutWiki = rel.replace(/^wiki\//, '');
+  const withoutWikiNoExt = withoutWiki.replace(/\.md$/i, '');
+  const aliases = [
+    stem,
+    title,
+    rel,
+    relNoExt,
+    withoutWiki,
+    withoutWikiNoExt,
+  ].map(normalizeTarget).filter(Boolean);
+
+  return {
+    absolutePath: file,
+    rel,
+    title,
+    stem,
+    frontmatter: data,
+    hasFrontmatter,
+    body,
+    content: normalizeForStorage(content || ''),
+    type: data.type || '',
+    status: data.status || '',
+    confidence: data.confidence || '',
+    memoryType: data.memory_type || '',
+    aliases: [...new Set(aliases)],
+    wikilinks: extractWikilinks(content),
+    markdownLinks: extractMarkdownLinks(content),
+    sourceIds: Array.isArray(data.source_ids) ? data.source_ids : [],
+  };
+}
+
+export async function collectWikiPages(projectRoot, options = {}) {
+  const root = wikiRoot(projectRoot);
+  if (!(await exists(root))) {
+    throw new Error(`llm-wiki wiki directory not found: ${root}`);
+  }
+  const files = (await listMarkdownFiles(root, options.maxFiles || DEFAULT_MAX_WIKI_FILES))
+    .sort();
+  const maxChars = options.maxChars || DEFAULT_MAX_PAGE_CHARS;
+  const pages = [];
+  for (const file of files) {
+    const content = (await readText(file)).slice(0, maxChars);
+    pages.push(parseWikiPage(projectRoot, file, content));
+  }
+  return pages;
+}
+
+export function buildAliasMap(pages) {
+  const aliases = new Map();
+  for (const page of pages) {
+    for (const alias of page.aliases) {
+      if (!aliases.has(alias)) aliases.set(alias, []);
+      aliases.get(alias).push(page.rel);
+    }
+  }
+  return aliases;
+}
+
+function pageRelativeTarget(page, target) {
+  if (!page || !target) return null;
+  const raw = normalizeForStorage(target)
+    .split('|')[0]
+    .split('#')[0]
+    .trim()
+    .replace(/\\/g, '/');
+  if (!raw.startsWith('./') && !raw.startsWith('../')) return null;
+  const pageDir = posix.dirname(page.rel.replace(/^wiki\//, ''));
+  const joined = posix.normalize(posix.join(pageDir, raw));
+  if (joined === '..' || joined.startsWith('../')) return null;
+  return normalizeTarget(joined);
+}
+
+export function resolveWikiLink(aliasMap, target, page = null) {
+  const normalized = pageRelativeTarget(page, target) || normalizeTarget(target);
+  const matches = aliasMap.get(normalized) || [];
+  return matches.length === 1 ? matches[0] : null;
+}
+
+export function resolveMarkdownWikiLink(aliasMap, page, link) {
+  if (!link?.path || !/\.md$/i.test(link.path)) return null;
+  const pageDir = posix.dirname(page.rel.replace(/^wiki\//, ''));
+  const joined = posix.normalize(posix.join(pageDir, link.path));
+  if (joined === '..' || joined.startsWith('../')) return null;
+  return resolveWikiLink(aliasMap, joined);
+}
+
+export function buildWikiGraph(pages) {
+  const aliasMap = buildAliasMap(pages);
+  const outlinks = new Map();
+  const backlinks = new Map();
+  for (const page of pages) {
+    outlinks.set(page.rel, new Set());
+    backlinks.set(page.rel, new Set());
+  }
+
+  for (const page of pages) {
+    for (const link of page.wikilinks) {
+      const resolved = resolveWikiLink(aliasMap, link.raw, page);
+      if (!resolved || resolved === page.rel) continue;
+      outlinks.get(page.rel).add(resolved);
+      backlinks.get(resolved)?.add(page.rel);
+    }
+    for (const link of page.markdownLinks) {
+      const resolved = resolveMarkdownWikiLink(aliasMap, page, link);
+      if (!resolved || resolved === page.rel) continue;
+      outlinks.get(page.rel).add(resolved);
+      backlinks.get(resolved)?.add(page.rel);
+    }
+  }
+
+  return { aliasMap, outlinks, backlinks };
+}
+
+export async function readMemoryExcerpt(projectRoot) {
+  const path = join(projectRoot, 'llm-wiki', 'wiki', 'memory.md');
+  const content = await readText(path, '');
+  if (!content) return '';
+  const lines = content.split('\n').slice(0, MEMORY_LINE_LIMIT).join('\n');
+  if (Buffer.byteLength(lines, 'utf8') <= MEMORY_BYTE_LIMIT) return lines;
+  let bytes = 0;
+  let output = '';
+  for (const char of lines) {
+    const nextBytes = Buffer.byteLength(char, 'utf8');
+    if (bytes + nextBytes > MEMORY_BYTE_LIMIT) break;
+    output += char;
+    bytes += nextBytes;
+  }
+  return output;
+}
+
+export function pageLookup(pages) {
+  return new Map(pages.map((page) => [page.rel, page]));
+}