oh-my-llmwikimode 1.0.0

package/src/wiki/store.js

@@ -0,0 +1,1329 @@
+/**
+ * Wiki storage module for oh-my-llmwikimode
+ *
+ * Markdown + YAML frontmatter persistence with JSON-derived index.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import {
+  VALID_CONFIDENCES as NORMALIZER_VALID_CONFIDENCES,
+  VALID_SOURCES as NORMALIZER_VALID_SOURCES,
+  VALID_STATUSES as NORMALIZER_VALID_STATUSES,
+  normalizeEntryConfidence,
+  normalizeEntrySource,
+  normalizeEntryStatus,
+  normalizeEntryTags,
+  normalizeEntryText as normalizeEntryTextValue,
+  normalizeScalar as normalizeScalarValue,
+  normalizeStringList as normalizeStringListValue,
+  normalizeWikiIndex,
+} from "./entry-normalizer.js";
+import { appendSearchFailure, countSearchFailures } from "./search-feedback.js";
+import { getSemanticIndexPath, getSemanticIndexSize, maybeAutoBuildSemanticIndex } from "./semantic-index.js";
+import { semanticSearch } from "./semantic-search.js";
+
+// ─── Frontmatter Parser ──────────────────────────────────────────────────────
+
+export function parseFrontmatter(content) {
+  const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/);
+  if (!match) return { frontmatter: {}, body: content };
+
+  const lines = match[1].split("\n");
+  const frontmatter = {};
+  let currentKey = null;
+  let currentArray = null;
+
+  for (const line of lines) {
+    const trimmedLine = line.trim();
+
+    // Array item (with potential leading whitespace)
+    if (trimmedLine.startsWith("- ") && currentArray) {
+      currentArray.push(trimmedLine.slice(2));
+      continue;
+    }
+
+    const keyValueMatch = line.match(/^(\w+):\s*(.*)$/);
+    if (keyValueMatch) {
+      const [, key, value] = keyValueMatch;
+      const trimmedValue = value.trim();
+      if (trimmedValue.startsWith('"') && trimmedValue.endsWith('"')) {
+        try {
+          frontmatter[key] = JSON.parse(trimmedValue);
+        } catch {
+          frontmatter[key] = trimmedValue.slice(1, -1);
+        }
+        currentKey = key;
+        currentArray = null;
+      } else if (trimmedValue.startsWith("[") && trimmedValue.endsWith("]")) {
+        frontmatter[key] = trimmedValue.slice(1, -1).split(",").map((s) => s.trim()).filter(Boolean);
+        currentKey = key;
+        currentArray = null;
+      } else if (/^-?\d+\.?\d*$/.test(trimmedValue)) {
+        // Parse as number
+        frontmatter[key] = parseFloat(trimmedValue);
+        currentKey = key;
+        currentArray = null;
+      } else if (trimmedValue) {
+        frontmatter[key] = trimmedValue;
+        currentKey = key;
+        currentArray = null;
+      } else {
+        // Empty value - likely start of array
+        currentKey = key;
+        currentArray = null;
+      }
+    } else if (trimmedLine.startsWith("- ")) {
+      currentArray = [];
+      if (currentKey) {
+        frontmatter[currentKey] = currentArray;
+      }
+      currentArray.push(trimmedLine.slice(2));
+    }
+  }
+
+  return { frontmatter, body: match[2].trim() };
+}
+
+function normalizeScalar(value) {
+  return normalizeScalarValue(value);
+}
+
+function stringifyScalar(value) {
+  if (typeof value === "number") {
+    return String(value);
+  }
+  const scalar = normalizeScalar(value);
+  if (!scalar || scalar.startsWith("[") || scalar.startsWith("{") || scalar.includes(":")) {
+    return JSON.stringify(scalar);
+  }
+  return scalar;
+}
+
+function normalizeEntryText(value) {
+  return normalizeEntryTextValue(value);
+}
+
+function normalizeTags(value) {
+  return normalizeEntryTags(value);
+}
+
+function compareStrings(a, b) {
+  const left = String(a ?? "");
+  const right = String(b ?? "");
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+function normalizeStringList(value) {
+  return normalizeStringListValue(value);
+}
+
+function normalizeLookupLabel(value) {
+  return normalizeScalar(value)
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]/gu, "");
+}
+
+export function stringifyFrontmatter(frontmatter) {
+  const lines = ["---"];
+  for (const [key, value] of Object.entries(frontmatter)) {
+    if (Array.isArray(value)) {
+      lines.push(`${key}:`);
+      for (const item of value) {
+        lines.push(`  - ${normalizeScalar(item)}`);
+      }
+    } else {
+      lines.push(`${key}: ${stringifyScalar(value)}`);
+    }
+  }
+  lines.push("---", "");
+  return lines.join("\n");
+}
+
+// ─── Wiki Paths ──────────────────────────────────────────────────────────────
+
+export function getWikiPaths(wikiRoot) {
+  return {
+    root: wikiRoot,
+    inbox: path.join(wikiRoot, "inbox"),
+    problems: path.join(wikiRoot, "problems"),
+    lessons: path.join(wikiRoot, "editorial", "lessons"),
+    projects: path.join(wikiRoot, "projects"),
+    system: path.join(wikiRoot, ".system"),
+    autoMemoryAuditFile: path.join(wikiRoot, ".system", "auto-memory-audit.jsonl"),
+    indexFile: path.join(wikiRoot, ".system", "index.json"),
+    rawRoot: path.join(wikiRoot, "raw"),
+    rawSources: path.join(wikiRoot, "raw", "sources"),
+    rawNotebooklm: path.join(wikiRoot, "raw", "notebooklm"),
+    rawPacks: path.join(wikiRoot, "raw", "packs"),
+  };
+}
+
+export function ensureWikiStructure(paths) {
+  const dirs = [paths.root, paths.inbox, paths.problems, paths.lessons, paths.projects, paths.system, paths.rawRoot, paths.rawSources, paths.rawNotebooklm, paths.rawPacks];
+  for (const dir of dirs) {
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+function ensureAuditStructure(paths) {
+  for (const dir of [paths.root, paths.system]) {
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+function isInsideDirectory(parentDirectory, candidatePath) {
+  const relativePath = path.relative(parentDirectory, candidatePath);
+  return relativePath !== "" && !relativePath.startsWith("..") && !path.isAbsolute(relativePath);
+}
+
+function resolvePromotableEntryPath(wikiRoot, entryPath) {
+  if (!entryPath || typeof entryPath !== "string") {
+    return { success: false, error: "Entry path is required" };
+  }
+
+  if (path.isAbsolute(entryPath)) {
+    return { success: false, error: "Entry path must be relative" };
+  }
+
+  if (!entryPath.endsWith(".md")) {
+    return { success: false, error: "Only Markdown entries can be promoted" };
+  }
+
+  const normalizedEntryPath = entryPath.replace(/\\/g, "/");
+  if (normalizedEntryPath.split("/").includes("..")) {
+    return { success: false, error: "Path traversal is not allowed" };
+  }
+
+  const isAllowedSource = normalizedEntryPath.startsWith("inbox/") || normalizedEntryPath.startsWith("problems/") || normalizedEntryPath.startsWith("projects/");
+  if (!isAllowedSource) {
+    return { success: false, error: "Only inbox/, problems/, or projects/ entries can be promoted" };
+  }
+
+  const rootPath = path.resolve(wikiRoot);
+  const fullPath = path.resolve(rootPath, entryPath);
+  if (!isInsideDirectory(rootPath, fullPath)) {
+    return { success: false, error: "Entry path escapes wiki root" };
+  }
+
+  const allowedSourceRoots = [
+    path.join(rootPath, "inbox"),
+    path.join(rootPath, "problems"),
+    path.join(rootPath, "projects"),
+  ];
+  if (!allowedSourceRoots.some((sourceRoot) => isInsideDirectory(sourceRoot, fullPath))) {
+    return { success: false, error: "Entry must resolve inside inbox/, problems/, or projects/" };
+  }
+
+  if (!fs.existsSync(fullPath)) {
+    return { success: false, error: "File not found" };
+  }
+
+  const realRoot = fs.realpathSync(rootPath);
+  const realFullPath = fs.realpathSync(fullPath);
+  if (!isInsideDirectory(realRoot, realFullPath)) {
+    return { success: false, error: "Entry path resolves outside wiki root" };
+  }
+
+  if (fs.lstatSync(fullPath).isSymbolicLink()) {
+    return { success: false, error: "Symbolic link entries cannot be promoted" };
+  }
+
+  return { success: true, fullPath, rootPath };
+}
+
+function resolveEditableEntryPath(wikiRoot, entryPath) {
+  if (!entryPath || typeof entryPath !== "string") {
+    return { success: false, error: "Entry path is required" };
+  }
+
+  if (path.isAbsolute(entryPath)) {
+    return { success: false, error: "Entry path must be relative" };
+  }
+
+  if (!entryPath.endsWith(".md")) {
+    return { success: false, error: "Only Markdown entries can be edited" };
+  }
+
+  const normalizedEntryPath = entryPath.replace(/\\/g, "/");
+  if (normalizedEntryPath.split("/").includes("..")) {
+    return { success: false, error: "Path traversal is not allowed" };
+  }
+
+  const allowedSourcePrefixes = ["inbox/", "problems/", "editorial/lessons/", "projects/"];
+  if (!allowedSourcePrefixes.some((prefix) => normalizedEntryPath.startsWith(prefix))) {
+    return { success: false, error: "Only inbox/, problems/, editorial/lessons/, or projects/ entries can be edited" };
+  }
+
+  const rootPath = path.resolve(wikiRoot);
+  const fullPath = path.resolve(rootPath, normalizedEntryPath);
+  if (!isInsideDirectory(rootPath, fullPath)) {
+    return { success: false, error: "Entry path escapes wiki root" };
+  }
+
+  const allowedSourceRoots = [
+    path.join(rootPath, "inbox"),
+    path.join(rootPath, "problems"),
+    path.join(rootPath, "editorial", "lessons"),
+    path.join(rootPath, "projects"),
+  ];
+  if (!allowedSourceRoots.some((sourceRoot) => isInsideDirectory(sourceRoot, fullPath))) {
+    return { success: false, error: "Entry must resolve inside an editable wiki directory" };
+  }
+
+  if (!fs.existsSync(fullPath)) {
+    return { success: false, error: "File not found" };
+  }
+
+  if (fs.lstatSync(fullPath).isSymbolicLink()) {
+    return { success: false, error: "Symbolic link entries cannot be edited" };
+  }
+
+  const realRoot = fs.realpathSync(rootPath);
+  const realFullPath = fs.realpathSync(fullPath);
+  if (!isInsideDirectory(realRoot, realFullPath)) {
+    return { success: false, error: "Entry path resolves outside wiki root" };
+  }
+
+  return { success: true, fullPath, rootPath };
+}
+
+// ─── Store ───────────────────────────────────────────────────────────────────
+
+export const VALID_STATUSES = NORMALIZER_VALID_STATUSES;
+export const VALID_SOURCES = NORMALIZER_VALID_SOURCES;
+export const VALID_CONFIDENCES = NORMALIZER_VALID_CONFIDENCES;
+
+function normalizeStatus(value) {
+  return normalizeEntryStatus(value);
+}
+
+function normalizeSource(value) {
+  return normalizeEntrySource(value);
+}
+
+function normalizeConfidence(value) {
+  return normalizeEntryConfidence(value);
+}
+
+// Statuses that should be excluded from the public graph
+const EXCLUDED_GRAPH_STATUSES = new Set(["rejected", "superseded", "private", "needs-clarification"]);
+
+function isGraphVisibleEntry(entry) {
+  return !EXCLUDED_GRAPH_STATUSES.has(normalizeStatus(entry?.status));
+}
+
+function buildEntryResolver(entries) {
+  const resolver = new Map();
+  const sortedEntries = [...entries].sort((a, b) => compareStrings(a.path, b.path));
+
+  for (const entry of sortedEntries) {
+    const labels = normalizeStringList([entry.title, ...(Array.isArray(entry.aliases) ? entry.aliases : [])]);
+    for (const label of labels) {
+      const normalizedLabel = normalizeLookupLabel(label);
+      if (!normalizedLabel) continue;
+      if (!resolver.has(normalizedLabel)) resolver.set(normalizedLabel, []);
+      resolver.get(normalizedLabel).push(entry.path);
+    }
+  }
+
+  for (const [label, entryPaths] of resolver.entries()) {
+    resolver.set(label, [...new Set(entryPaths)].sort(compareStrings));
+  }
+
+  return resolver;
+}
+
+function resolveEntryPath(resolver, entryPaths, target) {
+  const normalizedTarget = normalizeScalar(target).replace(/\\/g, "/");
+  if (!normalizedTarget) return null;
+  if (entryPaths.has(normalizedTarget)) return normalizedTarget;
+
+  const normalizedLabel = normalizeLookupLabel(normalizedTarget);
+  const resolvedPaths = resolver.get(normalizedLabel);
+  return resolvedPaths?.[0] || null;
+}
+
+function deterministicBuiltAt(entries) {
+  return [...entries]
+    .map((entry) => normalizeScalar(entry.created_at))
+    .filter(Boolean)
+    .sort(compareStrings)
+    .pop() || "1970-01-01T00:00:00.000Z";
+}
+
+function generateGraphId() {
+  return `node-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+function computeContentHash(content) {
+  return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+
+const AUTO_MEMORY_AUDIT_ACTIONS = new Set([
+  "stored",
+  "skipped_privacy",
+  "skipped_oversized",
+  "skipped_injection",
+  "skipped_explicit_request",
+  "skipped_internal",
+  "error",
+]);
+
+export function auditAutoMemory(wikiRoot, action, details = {}) {
+  const paths = getWikiPaths(wikiRoot);
+  ensureAuditStructure(paths);
+
+  const auditAction = AUTO_MEMORY_AUDIT_ACTIONS.has(action) ? action : "error";
+  const entry = {
+    timestamp: new Date().toISOString(),
+    action: auditAction,
+    reason: normalizeScalar(details.reason),
+    entry_id: normalizeScalar(details.entry_id),
+    source: normalizeSource(details.source || "chat.message"),
+  };
+
+  fs.appendFileSync(paths.autoMemoryAuditFile, `${JSON.stringify(entry)}\n`);
+
+  return {
+    success: true,
+    path: paths.autoMemoryAuditFile,
+    entry,
+  };
+}
+
+/**
+ * Store knowledge in the wiki.
+ * @param {string} wikiRoot - Path to the wiki root directory
+ * @param {Object} knowledge - Knowledge entry to store
+ * @param {string} knowledge.summary - Summary/title of the entry
+ * @param {string[]} [knowledge.tags=[]] - Tags for the entry
+ * @param {string} [knowledge.source="auto-memory"] - Source of the entry
+ * @param {string} [knowledge.status="candidate"] - Lifecycle status
+ * @param {string} [knowledge.project] - Project name (optional). When provided, stores under projects/{project}/inbox/
+ * @returns {Object|null} Stored entry info or null if summary is empty
+ */
+export function storeKnowledge(wikiRoot, knowledge) {
+  const paths = getWikiPaths(wikiRoot);
+  ensureWikiStructure(paths);
+
+  const { summary, tags = [], source = "auto-memory", status = "candidate", project } = knowledge;
+  if (!summary) return null;
+
+  // Determine target directory based on project
+  let targetDir = paths.inbox;
+  let category = "inbox";
+  if (project) {
+    const safeProject = String(project).replace(/[^\w\s-]/g, "").trim().replace(/\s+/g, "-");
+    if (safeProject) {
+      targetDir = path.join(paths.projects, safeProject, "inbox");
+      category = "project";
+      if (!fs.existsSync(targetDir)) {
+        fs.mkdirSync(targetDir, { recursive: true });
+      }
+    }
+  }
+
+  // Generate safe filename
+  const safeTitle = summary
+    .slice(0, 50)
+    .replace(/[^\w\s-]/g, "")
+    .trim()
+    .replace(/\s+/g, "-");
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
+  const filename = `${timestamp}-${safeTitle}.md`;
+  let filePath = path.join(targetDir, filename);
+  let collisionIndex = 2;
+  while (fs.existsSync(filePath)) {
+    filePath = path.join(targetDir, `${timestamp}-${safeTitle}-${collisionIndex}.md`);
+    collisionIndex += 1;
+  }
+
+  const frontmatter = {
+    title: summary.slice(0, 100),
+    aliases: Array.isArray(knowledge.aliases) ? knowledge.aliases : [],
+    tags: Array.isArray(tags) ? tags : [tags],
+    status: normalizeStatus(status),
+    source: normalizeSource(source),
+    summary: (knowledge.summary || summary).slice(0, 200),
+    confidence: normalizeConfidence(knowledge.confidence),
+    confidence_score: typeof knowledge.confidence_score === "number" ? knowledge.confidence_score : 1.0,
+    related: Array.isArray(knowledge.related) ? knowledge.related : [],
+    references: Array.isArray(knowledge.references) ? knowledge.references : [],
+    graph_id: knowledge.graph_id || generateGraphId(),
+    community: knowledge.community || "",
+    project: project || "",
+    created_at: new Date().toISOString(),
+    updated_at: new Date().toISOString(),
+    seen_count: 1,
+  };
+
+  const bodyContent = knowledge.details || summary;
+  const content = stringifyFrontmatter(frontmatter) + "\n" + bodyContent;
+  fs.writeFileSync(filePath, content);
+
+  return filePath;
+}
+
+// ─── Tokenize ────────────────────────────────────────────────────────────────
+
+export function tokenize(text) {
+  const normalizedText = normalizeEntryText(text);
+  const tokens = [];
+  const words = normalizedText
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .split(/\s+/)
+    .filter((w) => w.length >= 2);
+
+  tokens.push(...words);
+
+  // Korean bigram
+  const koreanMatches = normalizedText.match(/[\uac00-\ud7af]+/g) || [];
+  for (const word of koreanMatches) {
+    for (let i = 0; i < word.length - 1; i++) {
+      tokens.push(word.slice(i, i + 2));
+    }
+  }
+
+  return tokens;
+}
+
+// ─── Index Builder ───────────────────────────────────────────────────────────
+
+export function buildIndex(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+  ensureWikiStructure(paths);
+
+  const records = [];
+
+  function scanDir(dir, category) {
+    if (!fs.existsSync(dir)) return;
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const entryPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        scanDir(entryPath, category);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        try {
+          const content = fs.readFileSync(entryPath, "utf-8");
+          const { frontmatter, body } = parseFrontmatter(content);
+
+          if (!frontmatter.title) continue;
+
+          const relPath = path.relative(wikiRoot, entryPath).replace(/\\/g, "/");
+          const item = {
+            path: relPath,
+            category,
+            title: normalizeEntryText(frontmatter.title),
+            aliases: normalizeStringList(frontmatter.aliases),
+            tags: normalizeStringList(frontmatter.tags),
+            status: normalizeStatus(frontmatter.status),
+            source: normalizeSource(frontmatter.source),
+            summary: normalizeEntryText(frontmatter.summary || frontmatter.title),
+            confidence: normalizeConfidence(frontmatter.confidence),
+            confidence_score: typeof frontmatter.confidence_score === "number" ? frontmatter.confidence_score : 1.0,
+            related: normalizeStringList(frontmatter.related),
+            community: normalizeScalar(frontmatter.community),
+            project: normalizeScalar(frontmatter.project),
+            created_at: normalizeScalar(frontmatter.created_at),
+            updated_at: normalizeScalar(frontmatter.updated_at),
+          };
+          records.push({ entry: item, body });
+        } catch (err) {
+          // Skip corrupt files
+          console.warn(`[oh-my-llmwikimode] Skipping corrupt file: ${entryPath} - ${err.message}`);
+        }
+      }
+    }
+  }
+
+  scanDir(paths.inbox, "inbox");
+  scanDir(paths.problems, "problems");
+  scanDir(paths.lessons, "lessons");
+  scanDir(paths.projects, "project");
+
+  records.sort((a, b) => compareStrings(a.entry.path, b.entry.path));
+  const entries = records.map((record, index) => ({ ...record.entry, entry_id: index }));
+  const recordsByPath = new Map(records.map((record, index) => [entries[index].path, { entry: entries[index], body: record.body }]));
+
+  // Tokenize and build inverted index after deterministic entry ordering.
+  const invertedIndex = {};
+  const termFrequency = {};
+  for (const entry of entries) {
+    const record = recordsByPath.get(entry.path);
+    const text = [
+      entry.title,
+      entry.summary,
+      ...entry.aliases,
+      ...entry.tags,
+      record?.body || "",
+    ].join(" ").toLowerCase();
+    const allTokens = tokenize(text);
+    const uniqueTokens = [...new Set(allTokens)].sort(compareStrings);
+
+    // Legacy inverted index (unique posting list)
+    for (const token of uniqueTokens) {
+      if (!invertedIndex[token]) {
+        invertedIndex[token] = [];
+      }
+      invertedIndex[token].push(entry.entry_id);
+    }
+
+    // Term frequency for BM25 (count all occurrences)
+    for (const token of allTokens) {
+      if (!termFrequency[token]) {
+        termFrequency[token] = {};
+      }
+      termFrequency[token][entry.entry_id] = (termFrequency[token][entry.entry_id] || 0) + 1;
+    }
+
+    // Document length (total tokens including duplicates)
+    entry.doc_length = allTokens.length;
+
+    // Snippet from body (first 200 chars, cleaned)
+    if (record?.body) {
+      entry.snippet = record.body.replace(/\s+/g, " ").trim().slice(0, 200);
+    }
+  }
+
+  const sortedInvertedIndex = Object.fromEntries(
+    Object.entries(invertedIndex)
+      .sort(([left], [right]) => compareStrings(left, right))
+      .map(([token, postingList]) => [token, [...new Set(postingList)].sort((a, b) => a - b)])
+  );
+
+  const allEntryPaths = new Set(entries.map((entry) => entry.path));
+  const visibleEntries = entries.filter(isGraphVisibleEntry);
+  const visibleEntryPaths = new Set(visibleEntries.map((entry) => entry.path));
+  const allResolver = buildEntryResolver(entries);
+  const visibleResolver = buildEntryResolver(visibleEntries);
+
+  // Extract wikilinks and build backlinks
+  const backlinks = new Map(entries.map((entry) => [entry.path, new Set()]));
+  const edges = [];
+  const warnings = [];
+
+  for (const entry of entries) {
+    const record = recordsByPath.get(entry.path);
+    if (!record) continue;
+
+    const wikilinks = extractWikilinks(record.body).sort(compareStrings);
+
+    for (const link of wikilinks) {
+      const resolvedBacklinkTarget = resolveEntryPath(allResolver, allEntryPaths, link);
+      if (resolvedBacklinkTarget && backlinks.has(resolvedBacklinkTarget)) {
+        backlinks.get(resolvedBacklinkTarget).add(entry.path);
+      }
+
+      if (!isGraphVisibleEntry(entry)) continue;
+
+      const resolvedGraphTarget = resolveEntryPath(visibleResolver, visibleEntryPaths, link);
+      if (resolvedGraphTarget) {
+        edges.push({
+          source: entry.path,
+          target: resolvedGraphTarget,
+          relation: "references",
+          confidence: "EXTRACTED",
+          confidence_score: 1.0,
+        });
+      } else {
+        warnings.push({
+          type: "unresolved_wikilink",
+          source: entry.path,
+          target: link,
+          relation: "references",
+        });
+      }
+    }
+
+    if (!isGraphVisibleEntry(entry)) continue;
+
+    for (const relatedTarget of entry.related) {
+      const resolvedRelatedTarget = resolveEntryPath(visibleResolver, visibleEntryPaths, relatedTarget);
+      if (!resolvedRelatedTarget) continue;
+      edges.push({
+        source: entry.path,
+        target: resolvedRelatedTarget,
+        relation: "related",
+        confidence: "EXTRACTED",
+        confidence_score: 1.0,
+      });
+    }
+  }
+
+  // Attach backlinks to entries
+  for (const entry of entries) {
+    entry.backlinks_from = [...(backlinks.get(entry.path) || [])].sort(compareStrings);
+    entry.backlink_count = entry.backlinks_from.length;
+  }
+
+  // Detect duplicates before writing so both index and graph metadata are stable.
+  const duplicates = detectDuplicates(entries, wikiRoot);
+
+  // Build and write graph
+  const graph = buildGraph(entries, edges, wikiRoot, { warnings });
+  writeGraph(paths.system, graph);
+
+  const index = {
+    entries,
+    inverted_index: sortedInvertedIndex,
+    term_frequency: termFrequency,
+    duplicates: duplicates.length > 0 ? duplicates : undefined,
+    built_at: deterministicBuiltAt(entries),
+  };
+
+  // Atomic write
+  const tmpFile = `${paths.indexFile}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+  fs.writeFileSync(tmpFile, JSON.stringify(index, null, 2));
+  fs.renameSync(tmpFile, paths.indexFile);
+
+  return index;
+}
+
+// ─── Search ──────────────────────────────────────────────────────────────────
+
+// BM25-lite constants
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+const SEMANTIC_FALLBACK_MIN_ENTRIES = 10;
+
+export function search(query, index, maxResults = 3) {
+  const normalizedIndex = normalizeWikiIndex(index);
+  if (!normalizedIndex.inverted_index) return [];
+
+  const queryText = normalizeEntryText(query).toLowerCase();
+  const queryTokens = tokenize(queryText);
+  if (queryTokens.length === 0) return [];
+
+  const scores = new Map();
+  const N = normalizedIndex.entries.length;
+
+  // Compute average document length for BM25
+  const avgDocLength = N > 0
+    ? normalizedIndex.entries.reduce((sum, entry) => sum + (entry.doc_length || 0), 0) / N
+    : 0;
+
+  for (const token of queryTokens) {
+    const postings = normalizedIndex.inverted_index[token] || [];
+    const df = postings.length;
+    if (df === 0) continue;
+
+    // IDF with smoothing
+    const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+    for (const entryId of postings) {
+      const entry = normalizedIndex.entries[entryId];
+      if (!entry) continue;
+
+      // Term frequency for this entry
+      const tf = (normalizedIndex.term_frequency?.[token]?.[entryId]) || 1;
+      const docLength = entry.doc_length || 0;
+
+      // BM25 term saturation and length normalization
+      const lengthNorm = avgDocLength > 0 ? (1 - BM25_B) + BM25_B * (docLength / avgDocLength) : 1;
+      const bm25TermScore = (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * lengthNorm);
+
+      const currentScore = scores.get(entryId) || 0;
+      scores.set(entryId, currentScore + idf * bm25TermScore);
+    }
+  }
+
+  const results = [];
+  for (const [entryId, score] of scores) {
+    const entry = normalizedIndex.entries[entryId];
+    if (!entry) continue;
+    if (entry.status === "rejected" || entry.status === "superseded" || entry.status === "needs-clarification" || entry.status === "private") {
+      continue;
+    }
+
+    // Field boost
+    let boostedScore = score;
+    const queryLower = queryText;
+    const entryTitle = String(normalizeEntryText(entry.title) ?? "");
+    const entryTags = normalizeTags(entry.tags).map((tag) => String(tag));
+    if (entryTitle && entryTitle.toLowerCase().includes(queryLower)) boostedScore *= 2.0;
+    if (entryTags.some((tag) => tag.toLowerCase().includes(queryLower))) boostedScore *= 1.5;
+
+    // Citation metadata
+    const citation = {
+      source_id: entry.source || "manual",
+      path: entry.path,
+      chunk_id: "entry",
+      snippet: entry.snippet || entry.summary || entryTitle,
+    };
+
+    results.push({ ...entry, title: entryTitle, tags: entryTags, score: boostedScore, citation });
+  }
+
+  // Deterministic sorting: score desc, title asc, path asc, entry_id asc
+  results.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    const titleCompare = compareStrings(a.title, b.title);
+    if (titleCompare !== 0) return titleCompare;
+    const pathCompare = compareStrings(a.path, b.path);
+    if (pathCompare !== 0) return pathCompare;
+    return (a.entry_id || 0) - (b.entry_id || 0);
+  });
+
+  return results.slice(0, maxResults);
+}
+
+// Uses data/semantic-index/embeddings.jsonl as the local vector store.
+export async function searchWithSemanticFallback(wikiRoot, query, index, maxResults = 3, options = {}) {
+  const keywordResults = search(query, index, maxResults);
+  if (keywordResults.length > 0) return keywordResults;
+
+  if (process.env.LLMWIKI_DISABLE_SEMANTIC === "1") return [];
+
+  try {
+    await maybeAutoBuildSemanticIndex(wikiRoot, { now: options.now, model: options.model });
+
+    const indexPath = getSemanticIndexPath(wikiRoot);
+    if (!fs.existsSync(indexPath)) return [];
+    if (getSemanticIndexSize(wikiRoot) < SEMANTIC_FALLBACK_MIN_ENTRIES) return [];
+    if (countSearchFailures(wikiRoot) < SEMANTIC_FALLBACK_MIN_ENTRIES) return [];
+
+    const semanticResult = await semanticSearch(wikiRoot, query, {
+      topK: maxResults,
+      threshold: typeof options.threshold === "number" ? options.threshold : 0.6,
+      model: options.model,
+      fallbackFromKeyword: true,
+    });
+    const semanticResults = semanticResult.results;
+    if (semanticResults.length === 0) return [];
+
+    try {
+      appendSearchFailure(
+        wikiRoot,
+        {
+          query,
+          result_count: 0,
+          max_results: maxResults,
+          top_paths: semanticResults.map((result) => result.path),
+          source: "semantic_fallback",
+        },
+        options.now ? { now: options.now } : {}
+      );
+    } catch {
+      // Search feedback logging is best-effort and must not block retrieval.
+    }
+
+    return semanticResults.map((result) => ({
+      path: result.path,
+      title: result.title,
+      similarity: result.similarity,
+      score: result.similarity,
+      fallback_from_keyword: true,
+      source: "semantic",
+      citation: {
+        source_id: "semantic-index",
+        path: result.path,
+        chunk_id: "entry",
+        snippet: result.title,
+      },
+    }));
+  } catch {
+    return [];
+  }
+}
+
+// ─── Promote ─────────────────────────────────────────────────────────────────
+
+export function promoteEntry(wikiRoot, entryPath) {
+  const paths = getWikiPaths(wikiRoot);
+  ensureWikiStructure(paths);
+
+  const resolved = resolvePromotableEntryPath(wikiRoot, entryPath);
+  if (!resolved.success) {
+    return resolved;
+  }
+
+  const { fullPath, rootPath } = resolved;
+
+  const content = fs.readFileSync(fullPath, "utf-8");
+  const { frontmatter, body } = parseFrontmatter(content);
+
+  if (frontmatter.status === "lesson") {
+    return { success: false, error: "Already a lesson" };
+  }
+
+  // Move to editorial/lessons/
+  const filename = path.basename(entryPath);
+  const newPath = path.join(paths.lessons, filename);
+  if (fs.existsSync(newPath)) {
+    return { success: false, error: "Lesson already exists" };
+  }
+
+  frontmatter.status = "lesson";
+  frontmatter.promoted_at = new Date().toISOString();
+
+  const newContent = stringifyFrontmatter(frontmatter) + "\n" + body;
+  fs.writeFileSync(newPath, newContent);
+  fs.unlinkSync(fullPath);
+
+  return { success: true, path: path.relative(rootPath, newPath).replace(/\\/g, "/") };
+}
+
+// ─── Reject ──────────────────────────────────────────────────────────────────
+
+export function rejectEntry(wikiRoot, entryPath) {
+  const paths = getWikiPaths(wikiRoot);
+  ensureWikiStructure(paths);
+
+  const resolved = resolveEditableEntryPath(wikiRoot, entryPath);
+  if (!resolved.success) {
+    return resolved;
+  }
+
+  const { fullPath, rootPath } = resolved;
+  const content = fs.readFileSync(fullPath, "utf-8");
+  const { frontmatter, body } = parseFrontmatter(content);
+  const now = new Date().toISOString();
+
+  frontmatter.status = "rejected";
+  frontmatter.rejected_at = frontmatter.rejected_at || now;
+  frontmatter.updated_at = now;
+
+  const newContent = stringifyFrontmatter(frontmatter) + "\n" + body;
+  const tmpFile = `${fullPath}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+  fs.writeFileSync(tmpFile, newContent);
+  fs.renameSync(tmpFile, fullPath);
+
+  return {
+    success: true,
+    path: path.relative(rootPath, fullPath).replace(/\\/g, "/"),
+    status: "rejected",
+  };
+}
+
+// ─── Format Results ──────────────────────────────────────────────────────────
+
+export function formatResultsForContext(results, wikiRoot, budget = 2500) {
+  if (!results || results.length === 0) return "";
+
+  const lines = ["--- Related Wiki Knowledge ---"];
+  let totalLength = lines[0].length;
+
+  for (const result of results) {
+    const fullPath = path.join(wikiRoot, result.path);
+    let content = "";
+
+    try {
+      content = fs.readFileSync(fullPath, "utf-8");
+    } catch (error) {
+      console.warn(`[oh-my-llmwikimode] Skipping unreadable result: ${fullPath} - ${error.message}`);
+      continue;
+    }
+
+    const { frontmatter, body } = parseFrontmatter(content);
+    const entryTitle = String(normalizeEntryText(frontmatter.title || result.title) ?? "");
+    const entryTags = normalizeTags(frontmatter.tags || result.tags).map((tag) => String(tag));
+    const entry = [
+      `\n# ${entryTitle}`,
+      `Tags: ${entryTags.join(", ")}`,
+      `Status: ${frontmatter.status || result.status || "candidate"}`,
+      "",
+      body.slice(0, 800),
+      "---",
+    ].join("\n");
+
+    if (totalLength + entry.length > budget - 50) break;
+    lines.push(entry);
+    totalLength += entry.length;
+  }
+
+  lines.push("--- End Wiki Knowledge ---");
+  return lines.join("\n");
+}
+
+// ─── Wikilink Extraction ─────────────────────────────────────────────────────
+
+export function extractWikilinks(text) {
+  if (!text || typeof text !== "string") return [];
+  const matches = text.matchAll(/\[\[([^\]|#]+)(?:\|[^\]]*)?(?:#[^\]]*)?\]\]/g);
+  const links = [];
+  for (const match of matches) {
+    const link = match[1].trim();
+    if (link) links.push(link);
+  }
+  return [...new Set(links)];
+}
+
+// ─── Graph Builder ───────────────────────────────────────────────────────────
+
+function normalizeGraphWarnings(warnings, visibleNodeIds) {
+  return (Array.isArray(warnings) ? warnings : [])
+    .filter((warning) => warning && typeof warning === "object")
+    .map((warning) => ({
+      type: normalizeScalar(warning.type),
+      source: normalizeScalar(warning.source),
+      target: normalizeScalar(warning.target),
+      relation: normalizeScalar(warning.relation || "references"),
+    }))
+    .filter((warning) => warning.type && warning.source && warning.target && visibleNodeIds.has(warning.source));
+}
+
+function compareWarnings(a, b) {
+  return compareStrings(
+    `${a.type}\0${a.normalized_label || ""}\0${a.source || ""}\0${a.target || ""}\0${a.relation || ""}`,
+    `${b.type}\0${b.normalized_label || ""}\0${b.source || ""}\0${b.target || ""}\0${b.relation || ""}`
+  );
+}
+
+function buildDuplicateLabelWarnings(entries) {
+  const labelMap = new Map();
+
+  for (const entry of [...entries].sort((a, b) => compareStrings(a.path, b.path))) {
+    const normalizedLabel = normalizeLookupLabel(entry.title);
+    if (!normalizedLabel) continue;
+    if (!labelMap.has(normalizedLabel)) labelMap.set(normalizedLabel, []);
+    labelMap.get(normalizedLabel).push(entry.path);
+  }
+
+  return Array.from(labelMap.entries())
+    .filter(([, entryPaths]) => entryPaths.length > 1)
+    .map(([normalizedLabel, entryPaths]) => ({
+      type: "duplicate_label",
+      normalized_label: normalizedLabel,
+      entries: [...new Set(entryPaths)].sort(compareStrings),
+    }))
+    .sort(compareWarnings);
+}
+
+function dedupeEdges(edges, visibleNodeIds) {
+  const edgeMap = new Map();
+
+  for (const edge of Array.isArray(edges) ? edges : []) {
+    const source = normalizeScalar(edge.source);
+    const target = normalizeScalar(edge.target);
+    if (!visibleNodeIds.has(source) || !visibleNodeIds.has(target)) continue;
+
+    const relation = normalizeScalar(edge.relation || "references") || "references";
+    const key = `${source}\0${target}\0${relation}`;
+    if (edgeMap.has(key)) continue;
+
+    edgeMap.set(key, {
+      source,
+      target,
+      relation,
+      confidence: normalizeConfidence(edge.confidence),
+      confidence_score: typeof edge.confidence_score === "number" ? edge.confidence_score : 1.0,
+    });
+  }
+
+  return Array.from(edgeMap.values()).sort((a, b) => {
+    if (a.source !== b.source) return compareStrings(a.source, b.source);
+    if (a.target !== b.target) return compareStrings(a.target, b.target);
+    return compareStrings(a.relation, b.relation);
+  });
+}
+
+function buildGapHints(nodes) {
+  return nodes
+    .filter((node) => node.degree < 2)
+    .map((node) => ({
+      type: node.degree === 0 ? "isolated_node" : "low_degree_node",
+      node: node.id,
+      label: node.label,
+      degree: node.degree,
+      hint: "Add wikilinks or related frontmatter to connect this entry.",
+    }))
+    .sort((a, b) => {
+      if (a.degree !== b.degree) return a.degree - b.degree;
+      return compareStrings(a.node, b.node);
+    });
+}
+
+export function buildGraph(entries, edges, wikiRoot, options = {}) {
+  // Filter out excluded statuses from graph nodes
+  const visibleEntries = entries
+    .filter(isGraphVisibleEntry)
+    .sort((a, b) => compareStrings(a.path, b.path));
+
+  const nodes = visibleEntries
+    .map((entry) => ({
+      id: entry.path,
+      label: entry.title,
+      path: entry.path,
+      category: entry.category,
+      community: entry.community || "",
+      tags: normalizeStringList(entry.tags),
+      aliases: normalizeStringList(entry.aliases),
+      status: entry.status,
+      source: entry.source,
+      confidence: entry.confidence || "EXTRACTED",
+      confidence_score: typeof entry.confidence_score === "number" ? entry.confidence_score : 1.0,
+      summary: entry.summary || entry.title,
+      created_at: entry.created_at,
+      backlink_count: 0,
+      degree: 0,
+    }))
+    .sort((a, b) => compareStrings(a.id, b.id));
+
+  // Build visible node id set for edge filtering
+  const visibleNodeIds = new Set(nodes.map((n) => n.id));
+
+  // Filter edges to only connect visible nodes
+  const sortedEdges = dedupeEdges(edges, visibleNodeIds);
+
+  // Compute degree and backlink count from visible edges only
+  for (const node of nodes) {
+    node.degree = sortedEdges.filter((e) => e.source === node.id || e.target === node.id).length;
+    node.backlink_count = sortedEdges.filter((e) => e.target === node.id).length;
+  }
+
+  // Build communities from tags
+  const communityMap = new Map();
+  for (const node of nodes) {
+    for (const tag of node.tags) {
+      if (!communityMap.has(tag)) communityMap.set(tag, []);
+      communityMap.get(tag).push(node.id);
+    }
+  }
+
+  const communities = Array.from(communityMap.entries())
+    .sort(([left], [right]) => compareStrings(left, right))
+    .map(([id, members], index) => ({
+      id,
+      label: id,
+      members: [...new Set(members)].sort(compareStrings),
+      color: ["#0F766E", "#2563EB", "#9333EA", "#B45309", "#0E7490", "#0891B2"][index % 6],
+    }));
+
+  const warnings = [
+    ...normalizeGraphWarnings(options.warnings, visibleNodeIds),
+    ...buildDuplicateLabelWarnings(visibleEntries),
+  ].sort(compareWarnings);
+  const gapHints = buildGapHints(nodes);
+
+  return {
+    version: 1,
+    nodes,
+    edges: sortedEdges,
+    communities,
+    meta: {
+      built_at: deterministicBuiltAt(visibleEntries),
+      file_count: visibleEntries.length,
+      warning_count: warnings.length,
+      warnings,
+      gap_hints: gapHints,
+    },
+  };
+}
+
+export function writeGraph(systemDir, graph) {
+  const graphPath = path.join(systemDir, "graph.json");
+  const tmpFile = `${graphPath}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+  fs.writeFileSync(tmpFile, JSON.stringify(graph, null, 2));
+  fs.renameSync(tmpFile, graphPath);
+}
+
+// ─── Dedupe Detection ────────────────────────────────────────────────────────
+
+export function detectDuplicates(entries, wikiRoot) {
+  const titleMap = new Map();
+  const hashMap = new Map();
+
+  for (const entry of [...entries].sort((a, b) => compareStrings(a.path, b.path))) {
+    const normalizedTitle = normalizeLookupLabel(entry.title);
+    if (normalizedTitle) {
+      if (!titleMap.has(normalizedTitle)) titleMap.set(normalizedTitle, []);
+      titleMap.get(normalizedTitle).push(entry.path);
+    }
+
+    // Content hash
+    try {
+      const fullPath = path.join(wikiRoot, entry.path);
+      const content = fs.readFileSync(fullPath, "utf-8");
+      const hash = computeContentHash(content);
+      if (!hashMap.has(hash)) hashMap.set(hash, []);
+      hashMap.get(hash).push(entry.path);
+    } catch {
+      // Skip unreadable
+    }
+  }
+
+  const titleDuplicates = Array.from(titleMap.entries())
+    .filter(([, entryPaths]) => entryPaths.length > 1)
+    .map(([normalizedTitle, entryPaths]) => ({
+      type: "title",
+      normalized_title: normalizedTitle,
+      entries: [...new Set(entryPaths)].sort(compareStrings),
+    }));
+
+  const contentDuplicates = Array.from(hashMap.entries())
+    .filter(([, entryPaths]) => entryPaths.length > 1)
+    .map(([hash, entryPaths]) => ({
+      type: "content",
+      hash,
+      entries: [...new Set(entryPaths)].sort(compareStrings),
+    }));
+
+  return [...titleDuplicates, ...contentDuplicates].sort((a, b) => {
+    if (a.type !== b.type) return compareStrings(a.type, b.type);
+    return compareStrings(a.normalized_title || a.hash, b.normalized_title || b.hash);
+  });
+}
+
+// ─── Wiki Stats ──────────────────────────────────────────────────────────────
+
+export function getWikiStats(wikiRoot) {
+  const index = buildIndex(wikiRoot);
+  const entries = index.entries || [];
+
+  const stats = {
+    total_entries: entries.length,
+    by_category: {},
+    by_status: {},
+    by_source: {},
+    top_tags: [],
+    recent_entries: [],
+    internal_prompt_count: 0,
+  };
+
+  const tagCounts = new Map();
+  const now = Date.now();
+  const oneDayMs = 24 * 60 * 60 * 1000;
+
+  for (const entry of entries) {
+    // Category count
+    const category = entry.category || "unknown";
+    stats.by_category[category] = (stats.by_category[category] || 0) + 1;
+
+    // Status count
+    const status = entry.status || "unknown";
+    stats.by_status[status] = (stats.by_status[status] || 0) + 1;
+
+    // Source count
+    const source = entry.source || "unknown";
+    stats.by_source[source] = (stats.by_source[source] || 0) + 1;
+
+    // Tag counts
+    const tags = Array.isArray(entry.tags) ? entry.tags : [];
+    for (const tag of tags) {
+      const tagStr = String(tag);
+      tagCounts.set(tagStr, (tagCounts.get(tagStr) || 0) + 1);
+    }
+
+    // Recent entries (last 7 days)
+    try {
+      const createdAt = entry.created_at ? new Date(entry.created_at).getTime() : 0;
+      if (createdAt > 0 && (now - createdAt) < 7 * oneDayMs) {
+        stats.recent_entries.push({
+          title: String(entry.title || "").slice(0, 60),
+          path: entry.path,
+          created_at: entry.created_at,
+        });
+      }
+    } catch {
+      // Ignore invalid dates
+    }
+
+    // Internal prompt detection
+    const title = String(entry.title || "").toLowerCase();
+    const summary = String(entry.summary || "").toLowerCase();
+    if (
+      title.includes("[search-mode]") ||
+      title.includes("[analyze-mode]") ||
+      title.includes("maximize search effort") ||
+      title.includes("analysis mode. gather context") ||
+      summary.includes("[search-mode]") ||
+      summary.includes("[analyze-mode]") ||
+      summary.includes("maximize search effort") ||
+      summary.includes("analysis mode. gather context")
+    ) {
+      stats.internal_prompt_count += 1;
+    }
+  }
+
+  // Top 20 tags
+  stats.top_tags = Array.from(tagCounts.entries())
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 20)
+    .map(([tag, count]) => ({ tag, count }));
+
+  // Sort recent entries by date desc
+  stats.recent_entries.sort((a, b) => {
+    try {
+      return new Date(b.created_at).getTime() - new Date(a.created_at).getTime();
+    } catch {
+      return 0;
+    }
+  });
+
+  return stats;
+}
+
+// ─── Wiki Explore ────────────────────────────────────────────────────────────
+
+export function exploreWiki(wikiRoot, query, options = {}) {
+  const { maxResults = 10, excerptLength = 300, sample = false } = options;
+  const index = buildIndex(wikiRoot);
+
+  if (!index || !index.entries || index.entries.length === 0) {
+    return { success: true, entries: [], message: "No entries in wiki." };
+  }
+
+  // Search for matching entries
+  let candidates;
+  if (query && query.trim()) {
+    candidates = search(query, index, Math.min(maxResults * 2, 50));
+  } else {
+    // No query: return recent entries
+    candidates = [...index.entries]
+      .filter((e) => e.status !== "rejected" && e.status !== "superseded" && e.status !== "needs-clarification" && e.status !== "private")
+      .sort((a, b) => {
+        try {
+          return new Date(b.created_at || 0).getTime() - new Date(a.created_at || 0).getTime();
+        } catch {
+          return 0;
+        }
+      });
+  }
+
+  // Sample mode: shuffle for diversity
+  if (sample) {
+    candidates = candidates.sort(() => Math.random() - 0.5);
+  }
+
+  const results = [];
+  for (const entry of candidates.slice(0, maxResults)) {
+    const fullPath = path.join(wikiRoot, entry.path);
+    let excerpt = "";
+
+    try {
+      const content = fs.readFileSync(fullPath, "utf-8");
+      const { body } = parseFrontmatter(content);
+      excerpt = body.slice(0, excerptLength);
+      if (body.length > excerptLength) excerpt += "...";
+    } catch {
+      excerpt = "(unable to read content)";
+    }
+
+    results.push({
+      title: String(entry.title || "").slice(0, 100),
+      path: entry.path,
+      category: entry.category,
+      status: entry.status,
+      tags: Array.isArray(entry.tags) ? entry.tags.map((t) => String(t)) : [],
+      created_at: entry.created_at,
+      excerpt,
+    });
+  }
+
+  return {
+    success: true,
+    query: query || "(recent entries)",
+    count: results.length,
+    entries: results,
+  };
+}