llm-wiki-kit 0.2.8 → 0.2.10

package/src/wiki-search.js

@@ -4,10 +4,20 @@ import { redactText } from './redaction.js';
 import {
   buildWikiGraph,
   collectWikiPages,
+  MEMORY_BYTE_LIMIT,
+  MEMORY_LINE_LIMIT,
   pageLookup,
   parseFrontmatter,
   readMemoryExcerpt,
 } from './wiki-model.js';
+import {
+  isArchivedPage,
+  isDefaultSearchCandidate,
+  isEpisodicLayerPage,
+  isStalePage,
+  isSupersededPage,
+  searchWeight,
+} from './wiki-visibility.js';
 
 const DEFAULT_LIMIT = 5;
 const SNIPPET_CHARS = 350;
@@ -65,6 +75,7 @@ function resultRecord(page, score, fields = {}) {
     title: page.title,
     type: page.type || '',
     memoryType: page.memoryType || '',
+    status: page.status || '',
     score,
     directScore: fields.directScore || 0,
     linkScore: fields.linkScore || 0,
@@ -81,22 +92,35 @@ function sortHits(a, b) {
   return a.path.localeCompare(b.path);
 }
 
-function importance(page) {
-  const value = Number(page?.frontmatter?.importance || 0);
-  return Number.isFinite(value) ? value : 0;
+export async function searchWiki(projectRoot, query, options = {}) {
+  return (await performSearch(projectRoot, query, options)).hits;
 }
 
-function isDefaultSearchCandidate(page, options = {}) {
-  if (options.includeEpisodic) return true;
-  const type = String(page.type || '').toLowerCase();
-  if (type === 'query' || page.rel.startsWith('wiki/queries/')) {
-    return ['semantic', 'procedural'].includes(page.memoryType) && importance(page) >= 4;
+function searchVisibility(pages, options = {}) {
+  const metadata = {
+    totalPages: pages.length,
+    searchedPages: 0,
+    hiddenEpisodicPages: 0,
+    hiddenArchivedPages: 0,
+    hiddenSupersededPages: 0,
+    stalePagesSearched: 0,
+    includeEpisodic: Boolean(options.includeEpisodic),
+    includeArchived: Boolean(options.includeArchived),
+  };
+  const visible = [];
+  for (const page of pages) {
+    const hiddenArchived = !options.includeArchived && isArchivedPage(page);
+    const hiddenSuperseded = !options.includeArchived && isSupersededPage(page);
+    const hiddenEpisodic = !options.includeEpisodic && isEpisodicLayerPage(page) && !isDefaultSearchCandidate(page, options);
+    if (hiddenArchived) metadata.hiddenArchivedPages += 1;
+    if (hiddenSuperseded) metadata.hiddenSupersededPages += 1;
+    if (hiddenEpisodic) metadata.hiddenEpisodicPages += 1;
+    if (!isDefaultSearchCandidate(page, options)) continue;
+    if (isStalePage(page)) metadata.stalePagesSearched += 1;
+    visible.push(page);
   }
-  return true;
-}
-
-export async function searchWiki(projectRoot, query, options = {}) {
-  return (await performSearch(projectRoot, query, options)).hits;
+  metadata.searchedPages = visible.length;
+  return { visible, metadata };
 }
 
 async function performSearch(projectRoot, query, options = {}) {
@@ -109,8 +133,21 @@ async function performSearch(projectRoot, query, options = {}) {
   }
 
   let pages = [];
+  let metadata = {
+    totalPages: 0,
+    searchedPages: 0,
+    hiddenEpisodicPages: 0,
+    hiddenArchivedPages: 0,
+    hiddenSupersededPages: 0,
+    stalePagesSearched: 0,
+    includeEpisodic: Boolean(opts.includeEpisodic),
+    includeArchived: Boolean(opts.includeArchived),
+  };
   try {
-    pages = (await collectWikiPages(projectRoot, opts)).filter((page) => isDefaultSearchCandidate(page, opts));
+    const collected = await collectWikiPages(projectRoot, opts);
+    const visibility = searchVisibility(collected, opts);
+    pages = visibility.visible;
+    metadata = visibility.metadata;
   } catch {
     return { hits: [], search: 'missing-wiki' };
   }
@@ -144,7 +181,7 @@ async function performSearch(projectRoot, query, options = {}) {
       const page = byPath.get(item.id);
       if (!page) continue;
       const subScore = substringScore(page, terms);
-      const score = item.score + subScore;
+      const score = (item.score + subScore) * searchWeight(page);
       hits.set(page.rel, resultRecord(page, score, {
         directScore: score,
         matchedTerms: item.terms || terms,
@@ -154,7 +191,7 @@ async function performSearch(projectRoot, query, options = {}) {
   }
 
   for (const page of pages) {
-    const score = substringScore(page, terms);
+    const score = substringScore(page, terms) * searchWeight(page);
     if (score <= 0 || hits.has(page.rel)) continue;
     hits.set(page.rel, resultRecord(page, score, {
       directScore: score,
@@ -175,7 +212,7 @@ async function performSearch(projectRoot, query, options = {}) {
         if (hits.has(neighborPath)) continue;
         const page = byPath.get(neighborPath);
         if (!page) continue;
-        const linkScore = seed.score * 0.2;
+        const linkScore = seed.score * 0.2 * searchWeight(page);
         hits.set(neighborPath, resultRecord(page, linkScore, {
           linkScore,
           source: 'linked',
@@ -190,6 +227,7 @@ async function performSearch(projectRoot, query, options = {}) {
   return {
     hits: [...hits.values()].sort(sortHits).slice(0, limit),
     search,
+    metadata,
   };
 }
 
@@ -282,11 +320,12 @@ export async function buildContextPack(projectRoot, query, options = {}) {
   const limit = Number(options.limit || DEFAULT_LIMIT);
   const expand = options.expand !== false;
   const memoryExcerpt = await readMemoryExcerpt(projectRoot);
-  const indexExcerpt = (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'))).slice(0, 1200).trim();
+  const rawIndexExcerpt = await readText(join(projectRoot, 'llm-wiki', 'wiki', 'index.md'));
+  const indexExcerpt = rawIndexExcerpt.slice(0, 1200).trim();
   const logExcerpt = options.includeLog
     ? (await readText(join(projectRoot, 'llm-wiki', 'wiki', 'log.md'))).slice(-1000).trim()
     : '';
-  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none' };
+  const result = query ? await performSearch(projectRoot, query, { ...options, limit, expand }) : { hits: [], search: 'none', metadata: null };
   return {
     workspace: projectRoot,
     query: redactText(query || '', 1000),
@@ -297,6 +336,32 @@ export async function buildContextPack(projectRoot, query, options = {}) {
     indexExcerpt: redactText(indexExcerpt, 2000),
     logExcerpt: redactText(logExcerpt, 2000),
     hits: result.hits.map(redactHit),
+    metadata: {
+      search: result.metadata,
+      budgets: {
+        memory: {
+          maxLines: MEMORY_LINE_LIMIT,
+          maxBytes: MEMORY_BYTE_LIMIT,
+          returnedBytes: Buffer.byteLength(memoryExcerpt.trim(), 'utf8'),
+          redactionMaxChars: 30000,
+        },
+        index: {
+          maxChars: 1200,
+          sourceChars: rawIndexExcerpt.length,
+          returnedChars: indexExcerpt.length,
+          redactionMaxChars: 2000,
+        },
+        hits: {
+          requestedLimit: limit,
+          returned: result.hits.length,
+          snippetChars: SNIPPET_CHARS,
+          hookSnippetChars: HOOK_SNIPPET_CHARS,
+          expand,
+          includeEpisodic: Boolean(options.includeEpisodic),
+          includeArchived: Boolean(options.includeArchived),
+        },
+      },
+    },
   };
 }
 

package/src/wiki-visibility.js

@@ -0,0 +1,76 @@
+export const DURABLE_MEMORY_TYPES = new Set(['semantic', 'procedural']);
+
+export function frontmatterValues(value) {
+  if (Array.isArray(value)) return value.map((item) => String(item || '').trim()).filter(Boolean);
+  const text = String(value || '').trim();
+  if (!text || text === '[]') return [];
+  return [text];
+}
+
+export function pageImportance(page) {
+  const value = Number(page?.frontmatter?.importance || 0);
+  return Number.isFinite(value) ? value : 0;
+}
+
+export function isArchivedPage(page) {
+  return String(page?.status || '').toLowerCase() === 'archived';
+}
+
+export function isStalePage(page) {
+  return String(page?.status || '').toLowerCase() === 'stale';
+}
+
+export function supersededBy(page) {
+  return frontmatterValues(page?.frontmatter?.superseded_by);
+}
+
+export function supersedes(page) {
+  return frontmatterValues(page?.frontmatter?.supersedes);
+}
+
+export function isSupersededPage(page) {
+  return supersededBy(page).length > 0;
+}
+
+export function hasSupersession(page) {
+  return supersededBy(page).length > 0 || supersedes(page).length > 0;
+}
+
+export function isEpisodicLayerPage(page) {
+  const type = String(page?.type || '').toLowerCase();
+  const rel = String(page?.rel || '');
+  return (
+    type === 'query' ||
+    type === 'context' ||
+    type === 'session-log' ||
+    rel.startsWith('wiki/queries/') ||
+    rel.startsWith('wiki/context/')
+  );
+}
+
+export function isPromotedDurablePage(page) {
+  return DURABLE_MEMORY_TYPES.has(String(page?.memoryType || '').toLowerCase()) && pageImportance(page) >= 4;
+}
+
+export function includeArchivedPages(options = {}) {
+  return Boolean(options.includeArchived);
+}
+
+export function includeEpisodicPages(options = {}) {
+  return Boolean(options.includeEpisodic);
+}
+
+export function isDefaultSearchCandidate(page, options = {}) {
+  if (!includeArchivedPages(options) && (isArchivedPage(page) || isSupersededPage(page))) return false;
+  if (includeEpisodicPages(options)) return true;
+  if (isEpisodicLayerPage(page)) return isPromotedDurablePage(page);
+  return true;
+}
+
+export function searchWeight(page) {
+  let weight = 1;
+  if (DURABLE_MEMORY_TYPES.has(String(page?.memoryType || '').toLowerCase())) weight *= 1.2;
+  if (pageImportance(page) >= 4) weight *= 1.1;
+  if (isStalePage(page)) weight *= 0.35;
+  return weight;
+}