llm-wiki-kit 0.1.5 → 0.1.7

package/src/wiki-model.js

@@ -0,0 +1,263 @@
+import { basename, join, posix, relative } from 'path';
+import { exists, listMarkdownFiles, readText } from './fs-utils.js';
+import { normalizeForStorage } from './redaction.js';
+
+export const MEMORY_LINE_LIMIT = 200;
+export const MEMORY_BYTE_LIMIT = 25 * 1024;
+export const DEFAULT_MAX_WIKI_FILES = 500;
+export const DEFAULT_MAX_PAGE_CHARS = 50000;
+
+export function wikiRoot(projectRoot) {
+  return join(projectRoot, 'llm-wiki', 'wiki');
+}
+
+export function wikiRel(projectRoot, file) {
+  return relative(join(projectRoot, 'llm-wiki'), file).split('\\').join('/');
+}
+
+function stripQuotes(value) {
+  return String(value || '').replace(/^["']|["']$/g, '');
+}
+
+function parseScalar(value) {
+  const trimmed = String(value || '').trim();
+  if (trimmed === '[]') return [];
+  if (/^\[.*\]$/.test(trimmed)) {
+    const inner = trimmed.slice(1, -1).trim();
+    if (!inner) return [];
+    return inner.split(',').map((item) => stripQuotes(item.trim())).filter(Boolean);
+  }
+  return stripQuotes(trimmed);
+}
+
+export function parseFrontmatter(content) {
+  const text = normalizeForStorage(content || '');
+  const match = text.match(/^---\n([\s\S]*?)\n---\n?/);
+  if (!match) return { data: {}, body: text, hasFrontmatter: false };
+
+  const data = {};
+  const lines = match[1].split('\n');
+  let currentArrayKey = null;
+  for (const line of lines) {
+    const arrayItem = line.match(/^\s*-\s+(.+)\s*$/);
+    if (arrayItem && currentArrayKey) {
+      data[currentArrayKey].push(stripQuotes(arrayItem[1].trim()));
+      continue;
+    }
+
+    const keyValue = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
+    if (!keyValue) {
+      currentArrayKey = null;
+      continue;
+    }
+
+    const [, key, value] = keyValue;
+    if (value.trim() === '') {
+      data[key] = [];
+      currentArrayKey = key;
+    } else {
+      data[key] = parseScalar(value);
+      currentArrayKey = Array.isArray(data[key]) ? key : null;
+    }
+  }
+
+  return {
+    data,
+    body: text.slice(match[0].length),
+    hasFrontmatter: true,
+  };
+}
+
+export function firstHeading(body) {
+  return (body.match(/^#\s+(.+)$/m)?.[1] || '').trim();
+}
+
+export function normalizeTarget(value) {
+  return normalizeForStorage(value || '')
+    .split('|')[0]
+    .split('#')[0]
+    .trim()
+    .replace(/\\/g, '/')
+    .replace(/^\.?\//, '')
+    .replace(/\.md$/i, '')
+    .toLowerCase();
+}
+
+export function stripMarkdownCode(content) {
+  return normalizeForStorage(content || '')
+    .replace(/(^|\n)(`{3,}|~{3,})[^\n]*\n[\s\S]*?\n\2[ \t]*(?=\n|$)/g, '$1')
+    .replace(/`+[^`\n]*`+/g, '');
+}
+
+export function extractWikilinks(content) {
+  const links = [];
+  const regex = /\[\[([^\]]+)\]\]/g;
+  const searchable = stripMarkdownCode(content);
+  let match = regex.exec(searchable);
+  while (match) {
+    const raw = match[1].trim();
+    const target = normalizeTarget(raw);
+    if (target) links.push({ raw, target });
+    match = regex.exec(searchable);
+  }
+  return links;
+}
+
+export function extractMarkdownLinks(content) {
+  const links = [];
+  const regex = /(?<!!)\[[^\]]+\]\(([^)]+)\)/g;
+  const searchable = stripMarkdownCode(content);
+  let match = regex.exec(searchable);
+  while (match) {
+    const raw = match[1].trim();
+    const href = raw.replace(/^<|>$/g, '').split(/\s+/)[0];
+    if (!href || href.startsWith('#') || href.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(href)) {
+      match = regex.exec(searchable);
+      continue;
+    }
+    const path = href.split('#')[0].split('?')[0].replace(/\\/g, '/');
+    if (path) links.push({ raw, path });
+    match = regex.exec(searchable);
+  }
+  return links;
+}
+
+export function parseWikiPage(projectRoot, file, content) {
+  const rel = wikiRel(projectRoot, file);
+  const { data, body, hasFrontmatter } = parseFrontmatter(content);
+  const title = String(data.title || firstHeading(body) || basename(file, '.md')).trim();
+  const stem = basename(file, '.md');
+  const relNoExt = rel.replace(/\.md$/i, '');
+  const withoutWiki = rel.replace(/^wiki\//, '');
+  const withoutWikiNoExt = withoutWiki.replace(/\.md$/i, '');
+  const aliases = [
+    stem,
+    title,
+    rel,
+    relNoExt,
+    withoutWiki,
+    withoutWikiNoExt,
+  ].map(normalizeTarget).filter(Boolean);
+
+  return {
+    absolutePath: file,
+    rel,
+    title,
+    stem,
+    frontmatter: data,
+    hasFrontmatter,
+    body,
+    content: normalizeForStorage(content || ''),
+    type: data.type || '',
+    status: data.status || '',
+    confidence: data.confidence || '',
+    memoryType: data.memory_type || '',
+    aliases: [...new Set(aliases)],
+    wikilinks: extractWikilinks(content),
+    markdownLinks: extractMarkdownLinks(content),
+    sourceIds: Array.isArray(data.source_ids) ? data.source_ids : [],
+  };
+}
+
+export async function collectWikiPages(projectRoot, options = {}) {
+  const root = wikiRoot(projectRoot);
+  if (!(await exists(root))) {
+    throw new Error(`llm-wiki wiki directory not found: ${root}`);
+  }
+  const files = (await listMarkdownFiles(root, options.maxFiles || DEFAULT_MAX_WIKI_FILES))
+    .sort();
+  const maxChars = options.maxChars || DEFAULT_MAX_PAGE_CHARS;
+  const pages = [];
+  for (const file of files) {
+    const content = (await readText(file)).slice(0, maxChars);
+    pages.push(parseWikiPage(projectRoot, file, content));
+  }
+  return pages;
+}
+
+export function buildAliasMap(pages) {
+  const aliases = new Map();
+  for (const page of pages) {
+    for (const alias of page.aliases) {
+      if (!aliases.has(alias)) aliases.set(alias, []);
+      aliases.get(alias).push(page.rel);
+    }
+  }
+  return aliases;
+}
+
+function pageRelativeTarget(page, target) {
+  if (!page || !target) return null;
+  const raw = normalizeForStorage(target)
+    .split('|')[0]
+    .split('#')[0]
+    .trim()
+    .replace(/\\/g, '/');
+  if (!raw.startsWith('./') && !raw.startsWith('../')) return null;
+  const pageDir = posix.dirname(page.rel.replace(/^wiki\//, ''));
+  const joined = posix.normalize(posix.join(pageDir, raw));
+  if (joined === '..' || joined.startsWith('../')) return null;
+  return normalizeTarget(joined);
+}
+
+export function resolveWikiLink(aliasMap, target, page = null) {
+  const normalized = pageRelativeTarget(page, target) || normalizeTarget(target);
+  const matches = aliasMap.get(normalized) || [];
+  return matches.length === 1 ? matches[0] : null;
+}
+
+export function resolveMarkdownWikiLink(aliasMap, page, link) {
+  if (!link?.path || !/\.md$/i.test(link.path)) return null;
+  const pageDir = posix.dirname(page.rel.replace(/^wiki\//, ''));
+  const joined = posix.normalize(posix.join(pageDir, link.path));
+  if (joined === '..' || joined.startsWith('../')) return null;
+  return resolveWikiLink(aliasMap, joined);
+}
+
+export function buildWikiGraph(pages) {
+  const aliasMap = buildAliasMap(pages);
+  const outlinks = new Map();
+  const backlinks = new Map();
+  for (const page of pages) {
+    outlinks.set(page.rel, new Set());
+    backlinks.set(page.rel, new Set());
+  }
+
+  for (const page of pages) {
+    for (const link of page.wikilinks) {
+      const resolved = resolveWikiLink(aliasMap, link.raw, page);
+      if (!resolved || resolved === page.rel) continue;
+      outlinks.get(page.rel).add(resolved);
+      backlinks.get(resolved)?.add(page.rel);
+    }
+    for (const link of page.markdownLinks) {
+      const resolved = resolveMarkdownWikiLink(aliasMap, page, link);
+      if (!resolved || resolved === page.rel) continue;
+      outlinks.get(page.rel).add(resolved);
+      backlinks.get(resolved)?.add(page.rel);
+    }
+  }
+
+  return { aliasMap, outlinks, backlinks };
+}
+
+export async function readMemoryExcerpt(projectRoot) {
+  const path = join(projectRoot, 'llm-wiki', 'wiki', 'memory.md');
+  const content = await readText(path, '');
+  if (!content) return '';
+  const lines = content.split('\n').slice(0, MEMORY_LINE_LIMIT).join('\n');
+  if (Buffer.byteLength(lines, 'utf8') <= MEMORY_BYTE_LIMIT) return lines;
+  let bytes = 0;
+  let output = '';
+  for (const char of lines) {
+    const nextBytes = Buffer.byteLength(char, 'utf8');
+    if (bytes + nextBytes > MEMORY_BYTE_LIMIT) break;
+    output += char;
+    bytes += nextBytes;
+  }
+  return output;
+}
+
+export function pageLookup(pages) {
+  return new Map(pages.map((page) => [page.rel, page]));
+}

package/src/wiki-search.js

@@ -0,0 +1,246 @@
+import { join } from 'path';
+import { readText } from './fs-utils.js';
+import { redactText } from './redaction.js';
+import {
+  buildWikiGraph,
+  collectWikiPages,
+  pageLookup,
+  readMemoryExcerpt,
+} from './wiki-model.js';
+
+const DEFAULT_LIMIT = 5;
+const SNIPPET_CHARS = 350;
+let miniSearchLoader = null;
+
+async function loadMiniSearch() {
+  if (!miniSearchLoader) {
+    miniSearchLoader = import('minisearch')
+      .then((module) => module.default || module.MiniSearch || null)
+      .catch(() => null);
+  }
+  return miniSearchLoader;
+}
+
+function normalizeText(value) {
+  return String(value || '').normalize('NFC').toLowerCase();
+}
+
+export function tokenizeWikiQuery(query) {
+  return normalizeText(query)
+    .replace(/[^\p{Letter}\p{Number}]+/gu, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length >= 2)
+    .slice(0, 80);
+}
+
+function substringScore(page, terms) {
+  if (terms.length === 0) return 0;
+  const haystack = normalizeText(`${page.title}\n${page.rel}\n${page.body}`);
+  let score = 0;
+  for (const term of terms) {
+    if (haystack.includes(term)) score += term.length > 3 ? 2 : 1;
+  }
+  return score;
+}
+
+function snippetFor(page, terms) {
+  const text = page.body.replace(/\s+/g, ' ').trim() || page.content.replace(/\s+/g, ' ').trim();
+  const lower = normalizeText(text);
+  let start = 0;
+  for (const term of terms) {
+    const index = lower.indexOf(term);
+    if (index !== -1) {
+      start = Math.max(0, index - 80);
+      break;
+    }
+  }
+  return text.slice(start, start + SNIPPET_CHARS);
+}
+
+function resultRecord(page, score, fields = {}) {
+  return {
+    path: page.rel,
+    title: page.title,
+    type: page.type || '',
+    memoryType: page.memoryType || '',
+    score,
+    directScore: fields.directScore || 0,
+    linkScore: fields.linkScore || 0,
+    source: fields.source || 'direct',
+    via: fields.via || [],
+    matchedTerms: fields.matchedTerms || [],
+    snippet: fields.snippet || '',
+  };
+}
+
+function sortHits(a, b) {
+  if (b.score !== a.score) return b.score - a.score;
+  if (a.source !== b.source) return a.source === 'direct' ? -1 : 1;
+  return a.path.localeCompare(b.path);
+}
+
+export async function searchWiki(projectRoot, query, options = {}) {
+  return (await performSearch(projectRoot, query, options)).hits;
+}
+
+async function performSearch(projectRoot, query, options = {}) {
+  const opts = typeof options === 'number' ? { limit: options } : options;
+  const limit = Number(opts.limit || DEFAULT_LIMIT);
+  const expand = opts.expand !== false;
+  const terms = tokenizeWikiQuery(query);
+  if (!query || terms.length === 0) {
+    return { hits: [], search: 'none' };
+  }
+
+  let pages = [];
+  try {
+    pages = await collectWikiPages(projectRoot, opts);
+  } catch {
+    return { hits: [], search: 'missing-wiki' };
+  }
+
+  const docs = pages.map((page) => ({
+    id: page.rel,
+    title: page.title,
+    path: page.rel,
+    body: page.body,
+    type: page.type || '',
+    memoryType: page.memoryType || '',
+  }));
+  const MiniSearch = await loadMiniSearch();
+  const byPath = pageLookup(pages);
+  const hits = new Map();
+  let search = 'substring+wikilink';
+
+  if (MiniSearch) {
+    search = 'minisearch+wikilink';
+    const index = new MiniSearch({
+      fields: ['title', 'path', 'body'],
+      storeFields: ['title', 'path', 'type', 'memoryType'],
+      searchOptions: {
+        boost: { title: 3, path: 2 },
+        prefix: true,
+      },
+    });
+    index.addAll(docs);
+
+    for (const item of index.search(query, { prefix: true, boost: { title: 3, path: 2 } })) {
+      const page = byPath.get(item.id);
+      if (!page) continue;
+      const subScore = substringScore(page, terms);
+      const score = item.score + subScore;
+      hits.set(page.rel, resultRecord(page, score, {
+        directScore: score,
+        matchedTerms: item.terms || terms,
+        snippet: snippetFor(page, terms),
+      }));
+    }
+  }
+
+  for (const page of pages) {
+    const score = substringScore(page, terms);
+    if (score <= 0 || hits.has(page.rel)) continue;
+    hits.set(page.rel, resultRecord(page, score, {
+      directScore: score,
+      matchedTerms: terms,
+      snippet: snippetFor(page, terms),
+    }));
+  }
+
+  if (expand && hits.size > 0) {
+    const graph = buildWikiGraph(pages);
+    const direct = [...hits.values()].sort(sortHits).slice(0, Math.max(limit, 5));
+    for (const seed of direct) {
+      const neighbors = new Set([
+        ...(graph.outlinks.get(seed.path) || []),
+        ...(graph.backlinks.get(seed.path) || []),
+      ]);
+      for (const neighborPath of neighbors) {
+        if (hits.has(neighborPath)) continue;
+        const page = byPath.get(neighborPath);
+        if (!page) continue;
+        const linkScore = seed.score * 0.2;
+        hits.set(neighborPath, resultRecord(page, linkScore, {
+          linkScore,
+          source: 'linked',
+          via: [seed.path],
+          matchedTerms: [],
+          snippet: snippetFor(page, terms),
+        }));
+      }
+    }
+  }
+
+  return {
+    hits: [...hits.values()].sort(sortHits).slice(0, limit),
+    search,
+  };
+}
+
+function redactHit(hit) {
+  return {
+    ...hit,
+    path: redactText(hit.path, 300),
+    title: redactText(hit.title, 300),
+    via: Array.isArray(hit.via) ? hit.via.map((item) => redactText(item, 300)) : [],
+    matchedTerms: Array.isArray(hit.matchedTerms) ? hit.matchedTerms.map((item) => redactText(item, 120)) : [],
+    snippet: redactText(hit.snippet, SNIPPET_CHARS),
+  };
+}
+
+export async function buildContextPack(projectRoot, query, options = {}) {
+  const limit = Number(options.limit || DEFAULT_LIMIT);
+  const expand = options.expand !== false;
+  const memoryExcerpt = await readMemoryExcerpt(projectRoot);
+  const indexExcerpt = (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'))).slice(0, 1200).trim();
+  const logExcerpt = options.includeLog
+    ? (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'log.md'))).slice(-1000).trim()
+    : '';
+  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none' };
+  return {
+    workspace: projectRoot,
+    query: redactText(query || '', 1000),
+    limit,
+    expand,
+    search: result.search,
+    memoryExcerpt: redactText(memoryExcerpt.trim(), 30000),
+    indexExcerpt: redactText(indexExcerpt, 2000),
+    logExcerpt: redactText(logExcerpt, 2000),
+    hits: result.hits.map(redactHit),
+  };
+}
+
+export function formatContextPack(pack) {
+  const lines = [
+    'LLM Wiki context from llm-wiki-kit:',
+    '- Treat chat memory as temporary; update project Markdown when knowledge should persist.',
+    '- Preserve raw/wiki separation. Do not store secrets, tokens, .env contents, private keys, or personal/customer identifiers.',
+    '- Prefer updating existing wiki pages over creating duplicate pages.',
+    '- Claude Code/Codex를 평소처럼 사용한다. 사용자가 별도 명령을 실행하지 않아도 agent가 필요한 wiki 조회와 정리를 수행한다.',
+    '- 오래 쓸 지식은 outputs/questions에만 두지 말고 기존 wiki/architecture, wiki/debugging, wiki/decisions, wiki/concepts, procedures 문서에 합친다.',
+    '- wiki/memory.md는 짧은 핵심 기억으로 유지하고, 긴 설명 대신 관련 정식 문서 링크를 둔다.',
+  ];
+  if (pack.query) {
+    lines.push(`- query: "${pack.query}"`);
+    lines.push(`- search: ${pack.search}`);
+  }
+  if (pack.memoryExcerpt) {
+    lines.push('', 'Memory excerpt:', pack.memoryExcerpt);
+  }
+  if (pack.indexExcerpt) {
+    lines.push('', 'Index excerpt:', pack.indexExcerpt);
+  }
+  if (pack.hits.length > 0) {
+    lines.push('', 'Relevant wiki pages:');
+    for (const hit of pack.hits) {
+      const suffix = hit.source === 'linked' && hit.via.length > 0
+        ? `, linked via ${hit.via.join(', ')}`
+        : '';
+      lines.push(`- ${hit.path} (score ${hit.score.toFixed(2)}, ${hit.source}${suffix}): ${hit.snippet}`);
+    }
+  }
+  if (pack.logExcerpt) {
+    lines.push('', 'Recent log excerpt:', pack.logExcerpt);
+  }
+  return lines.join('\n').trim();
+}