llm-wiki-kit 0.2.14 → 0.2.15

package/src/wiki-lint.js

@@ -1,9 +1,10 @@
 import { readdir } from 'fs/promises';
 import { dirname, isAbsolute, join, parse, relative, resolve, sep } from 'path';
+import { EVIDENCE_PREFIXES, MAX_CMD_EVIDENCE_CHARS, parseEvidenceRef } from './evidence.js';
 import { exists, readText } from './fs-utils.js';
 import { maintenanceSummary } from './maintenance.js';
 import { inspectProjectState } from './project-state.js';
-import { hasSecretLikeText } from './redaction.js';
+import { hasSecretLikeText, isSensitivePath } from './redaction.js';
 import { liveQaMaxBytes, liveQaMaxLines } from './live-qa.js';
 import {
   buildAliasMap,
@@ -145,6 +146,62 @@ async function sourceExists(projectRoot, sourceId) {
   return false;
 }
 
+function fileEvidencePath(projectRoot, value) {
+  return resolve(projectRoot, String(value || '').replace(/\\/g, '/'));
+}
+
+async function validateEvidenceRef(projectRoot, page, ref) {
+  const issues = [];
+  const parsed = parseEvidenceRef(ref);
+  const raw = parsed.raw;
+  const value = parsed.value;
+  if (!parsed.prefix || !EVIDENCE_PREFIXES.has(parsed.prefix)) {
+    issues.push(issue('error', 'invalid-evidence-ref-prefix', page.rel, `unsupported evidence_refs prefix: ${raw || '(empty)'}`));
+    return issues;
+  }
+  if (hasSecretLikeText(raw)) {
+    issues.push(issue('error', 'secret-like-evidence-ref', page.rel, 'evidence_refs contains token, credential, private-key, or secret-like text'));
+    return issues;
+  }
+  if (parsed.prefix === 'file') {
+    const normalized = value.replace(/\\/g, '/');
+    if (!normalized) {
+      issues.push(issue('error', 'invalid-file-evidence-ref', page.rel, 'file evidence ref is empty'));
+    } else if (isAbsolute(normalized) || /^[a-z][a-z0-9+.-]*:/i.test(normalized) || normalized.split('/').includes('..') || isSensitivePath(normalized)) {
+      issues.push(issue('error', 'invalid-file-evidence-ref', page.rel, `file evidence ref must be safe repo-relative path: ${normalized}`));
+    } else if (!(await pathExistsNormalized(fileEvidencePath(projectRoot, normalized)))) {
+      issues.push(issue('warning', 'missing-file-evidence-ref', page.rel, `file evidence ref has no matching project file: ${normalized}`));
+    }
+  } else if (parsed.prefix === 'raw') {
+    if (invalidSourceId(value)) {
+      issues.push(issue('error', 'invalid-raw-evidence-ref', page.rel, 'raw evidence ref points outside the project or uses unsupported syntax'));
+    } else if (!(await sourceExists(projectRoot, value))) {
+      issues.push(issue('warning', 'missing-raw-evidence-ref', page.rel, `raw evidence ref has no matching source file: ${value}`));
+    }
+  } else if (parsed.prefix === 'url') {
+    try {
+      const url = new URL(value);
+      if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        issues.push(issue('error', 'invalid-url-evidence-ref', page.rel, `url evidence ref must use http or https: ${value}`));
+      }
+      if (url.username || url.password) {
+        issues.push(issue('error', 'credential-url-evidence-ref', page.rel, 'url evidence ref must not include credentials'));
+      }
+    } catch {
+      issues.push(issue('error', 'invalid-url-evidence-ref', page.rel, `url evidence ref is not a valid URL: ${value}`));
+    }
+  } else if (parsed.prefix === 'cmd') {
+    if (!value) {
+      issues.push(issue('error', 'invalid-cmd-evidence-ref', page.rel, 'cmd evidence ref is empty'));
+    } else if (value.length > MAX_CMD_EVIDENCE_CHARS) {
+      issues.push(issue('error', 'invalid-cmd-evidence-ref', page.rel, `cmd evidence ref exceeds ${MAX_CMD_EVIDENCE_CHARS} characters`));
+    } else if (/[\r\n]/.test(value)) {
+      issues.push(issue('error', 'invalid-cmd-evidence-ref', page.rel, 'cmd evidence ref must be a single line'));
+    }
+  }
+  return issues;
+}
+
 function titleKey(page) {
   return normalizeTarget(page.title);
 }
@@ -250,6 +307,14 @@ export async function runLint(projectRoot, options = {}) {
       if (page.frontmatter.last_verified && !isDateLike(page.frontmatter.last_verified)) {
         issues.push(issue('warning', 'invalid-last-verified', page.rel, `last_verified should be YYYY-MM-DD or unknown: ${page.frontmatter.last_verified}`));
       }
+      if (page.frontmatter.evidence_refs !== undefined && !Array.isArray(page.frontmatter.evidence_refs)) {
+        issues.push(issue('warning', 'invalid-evidence-refs', page.rel, 'evidence_refs should be a YAML array'));
+      }
+      if (Array.isArray(page.frontmatter.evidence_refs)) {
+        for (const ref of page.evidenceRefs) {
+          issues.push(...await validateEvidenceRef(projectRoot, page, ref));
+        }
+      }
       if (page.status === 'stale') {
         issues.push(issue('warning', 'stale-page', page.rel, 'page is marked stale'));
       }

package/src/wiki-model.js

@@ -1,4 +1,5 @@
 import { basename, join, posix, relative } from 'path';
+import { normalizeEvidenceRefs } from './evidence.js';
 import { exists, listMarkdownFiles, readText } from './fs-utils.js';
 import { normalizeForStorage } from './redaction.js';
 
@@ -156,6 +157,7 @@ export function parseWikiPage(projectRoot, file, content) {
     wikilinks: extractWikilinks(content),
     markdownLinks: extractMarkdownLinks(content),
     sourceIds: Array.isArray(data.source_ids) ? data.source_ids : [],
+    evidenceRefs: normalizeEvidenceRefs(data.evidence_refs),
   };
 }
 

package/src/wiki-search.js

@@ -17,6 +17,8 @@ import {
   isStalePage,
   isSupersededPage,
   searchWeight,
+  visibleWikiPages,
+  wikiVisibility,
 } from './wiki-visibility.js';
 
 const DEFAULT_LIMIT = 5;
@@ -69,7 +71,41 @@ function snippetFor(page, terms) {
   return text.slice(start, start + SNIPPET_CHARS);
 }
 
+function matchedFieldsFor(page, terms) {
+  const fields = [];
+  const title = normalizeText(page.title);
+  const path = normalizeText(page.rel);
+  const body = normalizeText(page.body);
+  if (terms.some((term) => title.includes(term))) fields.push('title');
+  if (terms.some((term) => path.includes(term))) fields.push('path');
+  if (terms.some((term) => body.includes(term))) fields.push('body');
+  return fields;
+}
+
+function rankReason(page, fields = {}) {
+  const parts = [];
+  if (fields.source === 'linked') {
+    parts.push(`linked from ${fields.via?.join(', ') || 'direct hit'}`);
+  } else if ((fields.matchedFields || []).length > 0) {
+    parts.push(`matched ${fields.matchedFields.join(', ')}`);
+  } else {
+    parts.push('selected by retrieval score');
+  }
+  if (page.memoryType) parts.push(`memory:${page.memoryType}`);
+  if (page.frontmatter?.importance) parts.push(`importance:${page.frontmatter.importance}`);
+  if (page.status) parts.push(`status:${page.status}`);
+  return parts.join('; ');
+}
+
 function resultRecord(page, score, fields = {}) {
+  const matchedFields = fields.matchedFields || [];
+  const scoreBreakdown = {
+    miniSearchScore: fields.miniSearchScore || 0,
+    substringScore: fields.substringScore || 0,
+    directScore: fields.directScore || 0,
+    linkScore: fields.linkScore || 0,
+    weight: fields.weight || searchWeight(page),
+  };
   return {
     path: page.rel,
     title: page.title,
@@ -82,6 +118,11 @@ function resultRecord(page, score, fields = {}) {
     source: fields.source || 'direct',
     via: fields.via || [],
     matchedTerms: fields.matchedTerms || [],
+    matchedFields,
+    scoreBreakdown,
+    rankReason: fields.rankReason || rankReason(page, { ...fields, matchedFields }),
+    visibilityReason: fields.visibilityReason || wikiVisibility(page, fields.visibilityOptions || {}).reason,
+    evidenceRefs: page.evidenceRefs || [],
     snippet: fields.snippet || '',
   };
 }
@@ -108,18 +149,19 @@ function searchVisibility(pages, options = {}) {
     includeArchived: Boolean(options.includeArchived),
   };
   const visible = [];
-  for (const page of pages) {
-    const hiddenArchived = !options.includeArchived && isArchivedPage(page);
-    const hiddenSuperseded = !options.includeArchived && isSupersededPage(page);
-    const hiddenEpisodic = !options.includeEpisodic && isEpisodicLayerPage(page) && !isDefaultSearchCandidate(page, options);
-    if (hiddenArchived) metadata.hiddenArchivedPages += 1;
-    if (hiddenSuperseded) metadata.hiddenSupersededPages += 1;
-    if (hiddenEpisodic) metadata.hiddenEpisodicPages += 1;
-    if (!isDefaultSearchCandidate(page, options)) continue;
-    if (isStalePage(page)) metadata.stalePagesSearched += 1;
-    visible.push(page);
+  const visibility = visibleWikiPages(pages, options);
+  for (const item of visibility.hidden) {
+    const page = item.page;
+    if (!options.includeArchived && isArchivedPage(page)) metadata.hiddenArchivedPages += 1;
+    if (!options.includeArchived && isSupersededPage(page)) metadata.hiddenSupersededPages += 1;
+    if (!options.includeEpisodic && isEpisodicLayerPage(page) && !isDefaultSearchCandidate(page, options)) metadata.hiddenEpisodicPages += 1;
+  }
+  for (const item of visibility.visible) {
+    if (isStalePage(item.page)) metadata.stalePagesSearched += 1;
+    visible.push(item.page);
   }
   metadata.searchedPages = visible.length;
+  metadata.visibilityPolicy = visibility.policy;
   return { visible, metadata };
 }
 
@@ -181,22 +223,34 @@ async function performSearch(projectRoot, query, options = {}) {
       const page = byPath.get(item.id);
       if (!page) continue;
       const subScore = substringScore(page, terms);
-      const score = (item.score + subScore) * searchWeight(page);
+      const weight = searchWeight(page);
+      const score = (item.score + subScore) * weight;
       hits.set(page.rel, resultRecord(page, score, {
         directScore: score,
+        miniSearchScore: item.score,
+        substringScore: subScore,
+        weight,
+        matchedFields: matchedFieldsFor(page, terms),
         matchedTerms: item.terms || terms,
         snippet: snippetFor(page, terms),
+        visibilityOptions: opts,
       }));
     }
   }
 
   for (const page of pages) {
-    const score = substringScore(page, terms) * searchWeight(page);
+    const subScore = substringScore(page, terms);
+    const weight = searchWeight(page);
+    const score = subScore * weight;
     if (score <= 0 || hits.has(page.rel)) continue;
     hits.set(page.rel, resultRecord(page, score, {
       directScore: score,
+      substringScore: subScore,
+      weight,
+      matchedFields: matchedFieldsFor(page, terms),
       matchedTerms: terms,
       snippet: snippetFor(page, terms),
+      visibilityOptions: opts,
     }));
   }
 
@@ -212,13 +266,17 @@ async function performSearch(projectRoot, query, options = {}) {
         if (hits.has(neighborPath)) continue;
         const page = byPath.get(neighborPath);
         if (!page) continue;
-        const linkScore = seed.score * 0.2 * searchWeight(page);
+        const weight = searchWeight(page);
+        const linkScore = seed.score * 0.2 * weight;
         hits.set(neighborPath, resultRecord(page, linkScore, {
           linkScore,
+          weight,
           source: 'linked',
           via: [seed.path],
           matchedTerms: [],
+          matchedFields: [],
           snippet: snippetFor(page, terms),
+          visibilityOptions: opts,
         }));
       }
     }
@@ -238,6 +296,10 @@ function redactHit(hit) {
     title: redactText(hit.title, 300),
     via: Array.isArray(hit.via) ? hit.via.map((item) => redactText(item, 300)) : [],
     matchedTerms: Array.isArray(hit.matchedTerms) ? hit.matchedTerms.map((item) => redactText(item, 120)) : [],
+    matchedFields: Array.isArray(hit.matchedFields) ? hit.matchedFields.map((item) => redactText(item, 120)) : [],
+    rankReason: redactText(hit.rankReason || '', 500),
+    visibilityReason: redactText(hit.visibilityReason || '', 500),
+    evidenceRefs: Array.isArray(hit.evidenceRefs) ? hit.evidenceRefs.map((item) => redactText(item, 700)) : [],
     snippet: redactText(hit.snippet, SNIPPET_CHARS),
   };
 }
@@ -392,6 +454,10 @@ export function formatContextPack(pack) {
         ? `, linked via ${hit.via.join(', ')}`
         : '';
       lines.push(`- ${hit.path} (score ${hit.score.toFixed(2)}, ${hit.source}${suffix}): ${hit.snippet}`);
+      lines.push(`  why selected: ${hit.rankReason || hit.visibilityReason || 'retrieval match'}`);
+      if ((hit.evidenceRefs || []).length > 0) {
+        lines.push(`  evidence_refs: ${hit.evidenceRefs.slice(0, 3).join(', ')}`);
+      }
     }
   }
   if (pack.logExcerpt) {

package/src/wiki-visibility.js

@@ -1,4 +1,5 @@
 export const DURABLE_MEMORY_TYPES = new Set(['semantic', 'procedural']);
+export const DURABLE_VISIBILITY_POLICY = 'durable-default-v1';
 
 export function frontmatterValues(value) {
   if (Array.isArray(value)) return value.map((item) => String(item || '').trim()).filter(Boolean);
@@ -60,11 +61,40 @@ export function includeEpisodicPages(options = {}) {
   return Boolean(options.includeEpisodic);
 }
 
+export function wikiVisibility(page, options = {}) {
+  if (!includeArchivedPages(options) && isArchivedPage(page)) {
+    return { visible: false, reason: 'hidden: archived page; use --include-archived to inspect it' };
+  }
+  if (!includeArchivedPages(options) && isSupersededPage(page)) {
+    return { visible: false, reason: 'hidden: superseded page; use --include-archived to inspect it' };
+  }
+  if (includeEpisodicPages(options)) {
+    return { visible: true, reason: 'visible: --include-episodic includes episodic/context pages' };
+  }
+  if (isEpisodicLayerPage(page) && !isPromotedDurablePage(page)) {
+    return { visible: false, reason: 'hidden: default episodic query/context/session page' };
+  }
+  const memoryType = String(page?.memoryType || '').toLowerCase();
+  if (DURABLE_MEMORY_TYPES.has(memoryType)) {
+    return { visible: true, reason: `visible: durable ${memoryType} wiki page` };
+  }
+  if (isStalePage(page)) return { visible: true, reason: 'visible: stale page kept searchable with lower ranking' };
+  return { visible: true, reason: 'visible: durable non-episodic wiki page' };
+}
+
 export function isDefaultSearchCandidate(page, options = {}) {
-  if (!includeArchivedPages(options) && (isArchivedPage(page) || isSupersededPage(page))) return false;
-  if (includeEpisodicPages(options)) return true;
-  if (isEpisodicLayerPage(page)) return isPromotedDurablePage(page);
-  return true;
+  return wikiVisibility(page, options).visible;
+}
+
+export function visibleWikiPages(pages, options = {}) {
+  const visible = [];
+  const hidden = [];
+  for (const page of pages) {
+    const visibility = wikiVisibility(page, options);
+    if (visibility.visible) visible.push({ page, visibility });
+    else hidden.push({ page, visibility });
+  }
+  return { visible, hidden, policy: DURABLE_VISIBILITY_POLICY };
 }
 
 export function searchWeight(page) {