llm-wiki-kit 0.2.9 → 0.2.11

package/src/wiki-lint.js

@@ -4,14 +4,24 @@ import { exists, readText } from './fs-utils.js';
 import { maintenanceSummary } from './maintenance.js';
 import { inspectProjectState } from './project-state.js';
 import { hasSecretLikeText } from './redaction.js';
+import { liveQaMaxBytes, liveQaMaxLines } from './live-qa.js';
 import {
   buildAliasMap,
   buildWikiGraph,
   collectWikiPages,
+  DEFAULT_MAX_WIKI_FILES,
+  MEMORY_BYTE_LIMIT,
   normalizeTarget,
   resolveWikiLink,
   wikiRoot,
 } from './wiki-model.js';
+import {
+  hasSupersession,
+  isArchivedPage,
+  isEpisodicLayerPage,
+  isPromotedDurablePage,
+  isStalePage,
+} from './wiki-visibility.js';
 
 const VALID_TYPES = new Set([
   'source',
@@ -29,11 +39,60 @@ const VALID_STATUS = new Set(['draft', 'reviewed', 'stale', 'archived']);
 const VALID_CONFIDENCE = new Set(['high', 'medium', 'low']);
 const VALID_MEMORY_TYPES = new Set(['semantic', 'episodic', 'procedural']);
 const CORE_PAGES = new Set(['wiki/index.md', 'wiki/log.md', 'wiki/memory.md']);
+const MEMORY_NEAR_BUDGET_BYTES = 20 * 1024;
+const HIDDEN_PAGE_WARNING_THRESHOLD = 50;
+const SEARCH_CAP_WARNING_RATIO = 0.8;
+const ARCHIVED_LIVE_QA_MARKER = '<!-- llm-wiki-kit:archived-live-qa -->';
 
 function issue(severity, code, path, message) {
   return { severity, code, path, message };
 }
 
+function countLines(text) {
+  if (!text) return 0;
+  return String(text).split(/\r?\n/).length;
+}
+
+function liveQaOversized(text) {
+  return countLines(text) > liveQaMaxLines() || Buffer.byteLength(text || '', 'utf8') > liveQaMaxBytes();
+}
+
+async function liveQaGrowthIssues(projectRoot) {
+  const issues = [];
+  const root = join(projectRoot, 'llm-wiki', 'outputs', 'questions');
+  let entries = [];
+  try {
+    entries = await readdir(root, { withFileTypes: true });
+  } catch {
+    return issues;
+  }
+  for (const entry of entries) {
+    const full = join(root, entry.name);
+    if (entry.isFile() && /^\d{4}-\d{2}-\d{2}-live-qa\.md$/.test(entry.name)) {
+      const text = await readText(full, '');
+      if (!text.includes(ARCHIVED_LIVE_QA_MARKER) && liveQaOversized(text)) {
+        issues.push(issue('warning', 'live-qa-legacy-oversized', `outputs/questions/${entry.name}`, 'legacy daily live Q&A file is oversized; run archive-questions to split it into chunks'));
+      }
+    }
+    if (entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name)) {
+      let chunkEntries = [];
+      try {
+        chunkEntries = await readdir(full, { withFileTypes: true });
+      } catch {
+        chunkEntries = [];
+      }
+      for (const chunk of chunkEntries) {
+        if (!chunk.isFile() || !/^live-qa-\d+\.md$/.test(chunk.name)) continue;
+        const text = await readText(join(full, chunk.name), '');
+        if (liveQaOversized(text)) {
+          issues.push(issue('warning', 'live-qa-chunk-oversized', `outputs/questions/${entry.name}/${chunk.name}`, 'live Q&A chunk exceeds the configured rollover budget'));
+        }
+      }
+    }
+  }
+  return issues;
+}
+
 function isDateLike(value) {
   return value === 'unknown' || /^\d{4}-\d{2}-\d{2}$/.test(String(value || ''));
 }
@@ -125,8 +184,9 @@ export async function runLint(projectRoot, options = {}) {
     return lintResult(projectRoot, [], issues);
   }
 
+  const maxFiles = options.maxFiles || 1000;
   const pages = await collectWikiPages(projectRoot, {
-    maxFiles: options.maxFiles || 1000,
+    maxFiles,
     maxChars: options.maxChars || 75000,
   });
   const byRel = new Map(pages.map((page) => [page.rel, page]));
@@ -226,9 +286,23 @@ export async function runLint(projectRoot, options = {}) {
   }
 
   const memoryText = await readText(join(projectRoot, 'llm-wiki', 'wiki', 'memory.md'), '');
-  if (Buffer.byteLength(memoryText, 'utf8') > 25000) {
-    issues.push(issue('warning', 'memory-too-large', 'wiki/memory.md', 'memory.md is larger than the hook excerpt budget'));
+  const memoryBytes = Buffer.byteLength(memoryText, 'utf8');
+  if (memoryBytes > MEMORY_BYTE_LIMIT) {
+    issues.push(issue('warning', 'memory-too-large', 'wiki/memory.md', `memory.md is oversized (${memoryBytes} bytes) and larger than the hook excerpt budget`));
+  } else if (memoryBytes >= MEMORY_NEAR_BUDGET_BYTES) {
+    issues.push(issue('warning', 'memory-near-budget', 'wiki/memory.md', `memory.md is near the hook excerpt budget (${memoryBytes} bytes)`));
+  }
+
+  const searchCap = options.maxFiles || DEFAULT_MAX_WIKI_FILES;
+  if (pages.length >= Math.floor(searchCap * SEARCH_CAP_WARNING_RATIO)) {
+    issues.push(issue('warning', 'wiki-page-count-near-search-cap', 'llm-wiki/wiki', `wiki page count ${pages.length} is near the default search cap ${searchCap}`));
+  }
+
+  const hiddenEpisodicPages = pages.filter((page) => isEpisodicLayerPage(page) && !isPromotedDurablePage(page));
+  if (hiddenEpisodicPages.length >= HIDDEN_PAGE_WARNING_THRESHOLD) {
+    issues.push(issue('warning', 'hidden-episodic-growth', 'llm-wiki/wiki', `${hiddenEpisodicPages.length} default-hidden episodic/context/session pages may add wiki growth and search noise`));
   }
+
   for (const page of pages.filter(isOrphanCandidate)) {
     const backlinks = graph.backlinks.get(page.rel);
     if (!backlinks || backlinks.size === 0) {
@@ -236,6 +310,15 @@ export async function runLint(projectRoot, options = {}) {
     }
   }
 
+  for (const page of pages) {
+    if (!isArchivedPage(page) && !isStalePage(page)) continue;
+    const backlinks = graph.backlinks.get(page.rel);
+    const outlinks = graph.outlinks.get(page.rel);
+    if (!hasSupersession(page) && (!backlinks || backlinks.size === 0) && (!outlinks || outlinks.size === 0)) {
+      issues.push(issue('warning', 'stale-archived-discoverability', page.rel, 'stale/archived page has no supersession metadata or wiki links for discoverability'));
+    }
+  }
+
   const projectState = await inspectProjectState(projectRoot).catch(() => null);
   for (const file of projectState?.managedFiles || []) {
     if (file.needsAttention) {
@@ -245,14 +328,18 @@ export async function runLint(projectRoot, options = {}) {
     }
   }
 
-  const maintenance = await maintenanceSummary(projectRoot).catch(() => null);
-  if (maintenance?.tooManyPending) {
-    issues.push(issue('warning', 'maintenance-too-many-pending', 'outputs/maintenance/queue.md', `maintenance queue has ${maintenance.pendingCount} pending items; review and merge durable items into wiki pages`));
-  }
-  if ((maintenance?.stalePending || []).length > 0) {
-    issues.push(issue('warning', 'maintenance-stale-pending', 'outputs/maintenance/queue.md', `${maintenance.stalePending.length} pending maintenance item(s) are older than ${maintenance.stalePendingDays} days`));
+  if (!options.skipMaintenance) {
+    const maintenance = await maintenanceSummary(projectRoot, { skipLint: true }).catch(() => null);
+    if (maintenance?.tooManyPending) {
+      issues.push(issue('warning', 'maintenance-too-many-pending', 'outputs/maintenance/queue.md', `maintenance queue has ${maintenance.pendingCount} pending items; review and merge durable items into wiki pages`));
+    }
+    if ((maintenance?.stalePending || []).length > 0) {
+      issues.push(issue('warning', 'maintenance-stale-pending', 'outputs/maintenance/queue.md', `${maintenance.stalePending.length} pending maintenance item(s) are older than ${maintenance.stalePendingDays} days`));
+    }
   }
 
+  issues.push(...await liveQaGrowthIssues(projectRoot));
+
   return lintResult(projectRoot, pages, issues);
 }
 

package/src/wiki-search.js

@@ -4,10 +4,20 @@ import { redactText } from './redaction.js';
 import {
   buildWikiGraph,
   collectWikiPages,
+  MEMORY_BYTE_LIMIT,
+  MEMORY_LINE_LIMIT,
   pageLookup,
   parseFrontmatter,
   readMemoryExcerpt,
 } from './wiki-model.js';
+import {
+  isArchivedPage,
+  isDefaultSearchCandidate,
+  isEpisodicLayerPage,
+  isStalePage,
+  isSupersededPage,
+  searchWeight,
+} from './wiki-visibility.js';
 
 const DEFAULT_LIMIT = 5;
 const SNIPPET_CHARS = 350;
@@ -65,6 +75,7 @@ function resultRecord(page, score, fields = {}) {
     title: page.title,
     type: page.type || '',
     memoryType: page.memoryType || '',
+    status: page.status || '',
     score,
     directScore: fields.directScore || 0,
     linkScore: fields.linkScore || 0,
@@ -81,22 +92,35 @@ function sortHits(a, b) {
   return a.path.localeCompare(b.path);
 }
 
-function importance(page) {
-  const value = Number(page?.frontmatter?.importance || 0);
-  return Number.isFinite(value) ? value : 0;
+export async function searchWiki(projectRoot, query, options = {}) {
+  return (await performSearch(projectRoot, query, options)).hits;
 }
 
-function isDefaultSearchCandidate(page, options = {}) {
-  if (options.includeEpisodic) return true;
-  const type = String(page.type || '').toLowerCase();
-  if (type === 'query' || page.rel.startsWith('wiki/queries/')) {
-    return ['semantic', 'procedural'].includes(page.memoryType) && importance(page) >= 4;
+function searchVisibility(pages, options = {}) {
+  const metadata = {
+    totalPages: pages.length,
+    searchedPages: 0,
+    hiddenEpisodicPages: 0,
+    hiddenArchivedPages: 0,
+    hiddenSupersededPages: 0,
+    stalePagesSearched: 0,
+    includeEpisodic: Boolean(options.includeEpisodic),
+    includeArchived: Boolean(options.includeArchived),
+  };
+  const visible = [];
+  for (const page of pages) {
+    const hiddenArchived = !options.includeArchived && isArchivedPage(page);
+    const hiddenSuperseded = !options.includeArchived && isSupersededPage(page);
+    const hiddenEpisodic = !options.includeEpisodic && isEpisodicLayerPage(page) && !isDefaultSearchCandidate(page, options);
+    if (hiddenArchived) metadata.hiddenArchivedPages += 1;
+    if (hiddenSuperseded) metadata.hiddenSupersededPages += 1;
+    if (hiddenEpisodic) metadata.hiddenEpisodicPages += 1;
+    if (!isDefaultSearchCandidate(page, options)) continue;
+    if (isStalePage(page)) metadata.stalePagesSearched += 1;
+    visible.push(page);
   }
-  return true;
-}
-
-export async function searchWiki(projectRoot, query, options = {}) {
-  return (await performSearch(projectRoot, query, options)).hits;
+  metadata.searchedPages = visible.length;
+  return { visible, metadata };
 }
 
 async function performSearch(projectRoot, query, options = {}) {
@@ -109,8 +133,21 @@ async function performSearch(projectRoot, query, options = {}) {
   }
 
   let pages = [];
+  let metadata = {
+    totalPages: 0,
+    searchedPages: 0,
+    hiddenEpisodicPages: 0,
+    hiddenArchivedPages: 0,
+    hiddenSupersededPages: 0,
+    stalePagesSearched: 0,
+    includeEpisodic: Boolean(opts.includeEpisodic),
+    includeArchived: Boolean(opts.includeArchived),
+  };
   try {
-    pages = (await collectWikiPages(projectRoot, opts)).filter((page) => isDefaultSearchCandidate(page, opts));
+    const collected = await collectWikiPages(projectRoot, opts);
+    const visibility = searchVisibility(collected, opts);
+    pages = visibility.visible;
+    metadata = visibility.metadata;
   } catch {
     return { hits: [], search: 'missing-wiki' };
   }
@@ -144,7 +181,7 @@ async function performSearch(projectRoot, query, options = {}) {
       const page = byPath.get(item.id);
       if (!page) continue;
       const subScore = substringScore(page, terms);
-      const score = item.score + subScore;
+      const score = (item.score + subScore) * searchWeight(page);
       hits.set(page.rel, resultRecord(page, score, {
         directScore: score,
         matchedTerms: item.terms || terms,
@@ -154,7 +191,7 @@ async function performSearch(projectRoot, query, options = {}) {
   }
 
   for (const page of pages) {
-    const score = substringScore(page, terms);
+    const score = substringScore(page, terms) * searchWeight(page);
     if (score <= 0 || hits.has(page.rel)) continue;
     hits.set(page.rel, resultRecord(page, score, {
       directScore: score,
@@ -175,7 +212,7 @@ async function performSearch(projectRoot, query, options = {}) {
         if (hits.has(neighborPath)) continue;
         const page = byPath.get(neighborPath);
         if (!page) continue;
-        const linkScore = seed.score * 0.2;
+        const linkScore = seed.score * 0.2 * searchWeight(page);
         hits.set(neighborPath, resultRecord(page, linkScore, {
           linkScore,
           source: 'linked',
@@ -190,6 +227,7 @@ async function performSearch(projectRoot, query, options = {}) {
   return {
     hits: [...hits.values()].sort(sortHits).slice(0, limit),
     search,
+    metadata,
   };
 }
 
@@ -282,11 +320,12 @@ export async function buildContextPack(projectRoot, query, options = {}) {
   const limit = Number(options.limit || DEFAULT_LIMIT);
   const expand = options.expand !== false;
   const memoryExcerpt = await readMemoryExcerpt(projectRoot);
-  const indexExcerpt = (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'))).slice(0, 1200).trim();
+  const rawIndexExcerpt = await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'));
+  const indexExcerpt = rawIndexExcerpt.slice(0, 1200).trim();
   const logExcerpt = options.includeLog
     ? (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'log.md'))).slice(-1000).trim()
     : '';
-  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none' };
+  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none', metadata: null };
   return {
     workspace: projectRoot,
     query: redactText(query || '', 1000),
@@ -297,6 +336,32 @@ export async function buildContextPack(projectRoot, query, options = {}) {
     indexExcerpt: redactText(indexExcerpt, 2000),
     logExcerpt: redactText(logExcerpt, 2000),
     hits: result.hits.map(redactHit),
+    metadata: {
+      search: result.metadata,
+      budgets: {
+        memory: {
+          maxLines: MEMORY_LINE_LIMIT,
+          maxBytes: MEMORY_BYTE_LIMIT,
+          returnedBytes: Buffer.byteLength(memoryExcerpt.trim(), 'utf8'),
+          redactionMaxChars: 30000,
+        },
+        index: {
+          maxChars: 1200,
+          sourceChars: rawIndexExcerpt.length,
+          returnedChars: indexExcerpt.length,
+          redactionMaxChars: 2000,
+        },
+        hits: {
+          requestedLimit: limit,
+          returned: result.hits.length,
+          snippetChars: SNIPPET_CHARS,
+          hookSnippetChars: HOOK_SNIPPET_CHARS,
+          expand,
+          includeEpisodic: Boolean(options.includeEpisodic),
+          includeArchived: Boolean(options.includeArchived),
+        },
+      },
+    },
   };
 }
 

package/src/wiki-visibility.js

@@ -0,0 +1,76 @@
+export const DURABLE_MEMORY_TYPES = new Set(['semantic', 'procedural']);
+
+export function frontmatterValues(value) {
+  if (Array.isArray(value)) return value.map((item) => String(item || '').trim()).filter(Boolean);
+  const text = String(value || '').trim();
+  if (!text || text === '[]') return [];
+  return [text];
+}
+
+export function pageImportance(page) {
+  const value = Number(page?.frontmatter?.importance || 0);
+  return Number.isFinite(value) ? value : 0;
+}
+
+export function isArchivedPage(page) {
+  return String(page?.status || '').toLowerCase() === 'archived';
+}
+
+export function isStalePage(page) {
+  return String(page?.status || '').toLowerCase() === 'stale';
+}
+
+export function supersededBy(page) {
+  return frontmatterValues(page?.frontmatter?.superseded_by);
+}
+
+export function supersedes(page) {
+  return frontmatterValues(page?.frontmatter?.supersedes);
+}
+
+export function isSupersededPage(page) {
+  return supersededBy(page).length > 0;
+}
+
+export function hasSupersession(page) {
+  return supersededBy(page).length > 0 || supersedes(page).length > 0;
+}
+
+export function isEpisodicLayerPage(page) {
+  const type = String(page?.type || '').toLowerCase();
+  const rel = String(page?.rel || '');
+  return (
+    type === 'query' ||
+    type === 'context' ||
+    type === 'session-log' ||
+    rel.startsWith('wiki/queries/') ||
+    rel.startsWith('wiki/context/')
+  );
+}
+
+export function isPromotedDurablePage(page) {
+  return DURABLE_MEMORY_TYPES.has(String(page?.memoryType || '').toLowerCase()) && pageImportance(page) >= 4;
+}
+
+export function includeArchivedPages(options = {}) {
+  return Boolean(options.includeArchived);
+}
+
+export function includeEpisodicPages(options = {}) {
+  return Boolean(options.includeEpisodic);
+}
+
+export function isDefaultSearchCandidate(page, options = {}) {
+  if (!includeArchivedPages(options) && (isArchivedPage(page) || isSupersededPage(page))) return false;
+  if (includeEpisodicPages(options)) return true;
+  if (isEpisodicLayerPage(page)) return isPromotedDurablePage(page);
+  return true;
+}
+
+export function searchWeight(page) {
+  let weight = 1;
+  if (DURABLE_MEMORY_TYPES.has(String(page?.memoryType || '').toLowerCase())) weight *= 1.2;
+  if (pageImportance(page) >= 4) weight *= 1.1;
+  if (isStalePage(page)) weight *= 0.35;
+  return weight;
+}