oh-my-llmwikimode 1.0.0

package/src/wiki/query.js

@@ -0,0 +1,304 @@
+import fs from "node:fs";
+import path from "node:path";
+import { wrapWithBoundary } from "../security.js";
+import { buildIndex, getWikiPaths, search } from "./store.js";
+import {
+  normalizeConfidenceScore,
+  normalizeEntryStatus,
+  normalizeEntryTags,
+  normalizeEntryText,
+  normalizeIndexEntry,
+  normalizeWikiIndex,
+} from "./entry-normalizer.js";
+
+const DEFAULT_MAX_RESULTS = 3;
+const DEFAULT_MAX_NEIGHBORS = 6;
+const EXCLUDED_QUERY_STATUSES = new Set([
+  "rejected",
+  "superseded",
+  "private",
+  "needs-clarification",
+]);
+const EXCLUDED_GRAPH_STATUSES = new Set(["rejected", "superseded", "private"]);
+
+function normalizeStatus(status) {
+  return normalizeEntryStatus(status);
+}
+
+function isQueryable(entry) {
+  return entry && !EXCLUDED_QUERY_STATUSES.has(normalizeStatus(entry.status));
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+}
+
+function tryReadJson(filePath) {
+  try {
+    if (!fs.existsSync(filePath)) return null;
+    return readJson(filePath);
+  } catch {
+    return null;
+  }
+}
+
+function expectedGraphNodeIds(index) {
+  return new Set(
+    (index.entries || [])
+      .filter((entry) => !EXCLUDED_GRAPH_STATUSES.has(normalizeStatus(entry.status)))
+      .map((entry) => entry.path)
+  );
+}
+
+function isGraphStale(index, graph) {
+  if (!index || !Array.isArray(index.entries)) return true;
+  if (!graph || graph.version !== 1) return true;
+  if (!Array.isArray(graph.nodes) || !Array.isArray(graph.edges)) return true;
+
+  const expectedNodeIds = expectedGraphNodeIds(index);
+  const actualNodeIds = new Set(graph.nodes.map((node) => node.id));
+
+  if (expectedNodeIds.size !== actualNodeIds.size) return true;
+  for (const nodeId of expectedNodeIds) {
+    if (!actualNodeIds.has(nodeId)) return true;
+  }
+  return false;
+}
+
+function loadQueryArtifacts(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+  const graphPath = path.join(paths.system, "graph.json");
+  let index = tryReadJson(paths.indexFile);
+  let graph = tryReadJson(graphPath);
+  let rebuilt = false;
+
+  if (index) {
+    index = normalizeWikiIndex(index);
+  }
+
+  if (!index || isGraphStale(index, graph)) {
+    index = normalizeWikiIndex(buildIndex(wikiRoot));
+    graph = tryReadJson(graphPath);
+    rebuilt = true;
+  }
+
+  if (!graph) {
+    graph = { version: 1, nodes: [], edges: [], communities: [] };
+  }
+
+  return {
+    index,
+    graph,
+    rebuilt,
+    indexPath: paths.indexFile,
+    graphPath,
+  };
+}
+
+function mapEntriesByPath(entries) {
+  const entriesByPath = new Map();
+  for (const entry of entries || []) {
+    entriesByPath.set(entry.path, entry);
+  }
+  return entriesByPath;
+}
+
+function mapNodesById(nodes) {
+  const nodesById = new Map();
+  for (const node of nodes || []) {
+    nodesById.set(node.id, node);
+  }
+  return nodesById;
+}
+
+function confidenceScore(value, fallback = 1.0) {
+  return normalizeConfidenceScore(value, fallback);
+}
+
+function normalizeEntry(entry, node, extras = {}) {
+  const normalizedEntry = normalizeIndexEntry(entry, node?.path || node?.id || "");
+  const nodeTitle = normalizeEntryText(node?.label || node?.title);
+  const entryTags = normalizedEntry.tags.length > 0 ? normalizedEntry.tags : normalizeEntryTags(node?.tags);
+
+  return {
+    path: normalizedEntry.path,
+    title: normalizedEntry.title || nodeTitle || normalizedEntry.path,
+    category: normalizedEntry.category || normalizeEntryText(node?.category) || "wiki",
+    tags: entryTags,
+    status: normalizeStatus(normalizedEntry.status || node?.status),
+    score: typeof entry.score === "number" ? entry.score : undefined,
+    confidence: normalizeEntryText(node?.confidence || normalizedEntry.confidence || "EXTRACTED"),
+    confidence_score: confidenceScore(node?.confidence_score ?? normalizedEntry.confidence_score),
+    ...extras,
+  };
+}
+
+function buildDirectMatches(query, index, nodesById, maxResults) {
+  return search(query, index, maxResults)
+    .filter(isQueryable)
+    .map((entry) => normalizeEntry(entry, nodesById.get(entry.path)));
+}
+
+function addNeighbor(neighborsByPath, neighbor, relation) {
+  const existing = neighborsByPath.get(neighbor.path);
+  if (!existing) {
+    neighborsByPath.set(neighbor.path, neighbor);
+    return;
+  }
+
+  existing.relations.push(relation);
+  existing.relation_labels = [...new Set(existing.relations.map((item) => item.relation_label))];
+}
+
+function buildGraphNeighbors(directMatches, graph, entriesByPath, nodesById, maxNeighbors) {
+  const directPaths = new Set(directMatches.map((entry) => entry.path));
+  const neighborsByPath = new Map();
+
+  for (const edge of graph.edges || []) {
+    const sourceIsDirect = directPaths.has(edge.source);
+    const targetIsDirect = directPaths.has(edge.target);
+    if (!sourceIsDirect && !targetIsDirect) continue;
+
+    const matchedPath = sourceIsDirect ? edge.source : edge.target;
+    const neighborPath = sourceIsDirect ? edge.target : edge.source;
+    if (neighborPath === matchedPath) continue;
+
+    const entry = entriesByPath.get(neighborPath);
+    const node = nodesById.get(neighborPath);
+    if (!entry || !isQueryable(entry)) continue;
+
+    const relation = {
+      relation_label: edge.relation || "related",
+      source_path: edge.source,
+      target_path: edge.target,
+      via_path: matchedPath,
+      confidence: edge.confidence || "EXTRACTED",
+      confidence_score: confidenceScore(edge.confidence_score),
+    };
+
+    const neighbor = normalizeEntry(entry, node, {
+      relation_label: relation.relation_label,
+      relation_labels: [relation.relation_label],
+      relation_source_path: relation.source_path,
+      relation_target_path: relation.target_path,
+      via_path: relation.via_path,
+      confidence: relation.confidence,
+      confidence_score: relation.confidence_score,
+      relations: [relation],
+    });
+
+    addNeighbor(neighborsByPath, neighbor, relation);
+  }
+
+  return [...neighborsByPath.values()]
+    .sort((a, b) => {
+      if (b.confidence_score !== a.confidence_score) return b.confidence_score - a.confidence_score;
+      return String(a.title ?? "").localeCompare(String(b.title ?? ""));
+    })
+    .slice(0, maxNeighbors);
+}
+
+function buildDuplicateWarnings(duplicates, returnedPaths) {
+  return (duplicates || [])
+    .filter((warning) => Array.isArray(warning.entries) && warning.entries.some((entryPath) => returnedPaths.has(entryPath)))
+    .map((warning) => ({
+      ...warning,
+      relevant_entries: warning.entries.filter((entryPath) => returnedPaths.has(entryPath)),
+    }));
+}
+
+function formatEntryLine(entry) {
+  const parts = [`- ${entry.title}`, `path: ${entry.path}`, `status: ${entry.status}`];
+  if (entry.relation_label) parts.push(`relation: ${entry.relation_label}`);
+  if (entry.via_path) parts.push(`via: ${entry.via_path}`);
+  parts.push(`confidence: ${entry.confidence} ${entry.confidence_score}`);
+  return parts.join(" | ");
+}
+
+export function formatQueryForContext(result, budget = 2500) {
+  const lines = [
+    "--- Graph-Expanded Wiki Query ---",
+    `Query: ${result.query}`,
+    "",
+    "## Direct matches",
+  ];
+
+  if (result.direct_matches.length === 0) {
+    lines.push("- None");
+  } else {
+    for (const entry of result.direct_matches) {
+      lines.push(formatEntryLine(entry));
+    }
+  }
+
+  lines.push("", "## One-hop graph neighbors");
+  if (result.graph_neighbors.length === 0) {
+    lines.push("- None");
+  } else {
+    for (const entry of result.graph_neighbors) {
+      lines.push(formatEntryLine(entry));
+    }
+  }
+
+  if (result.duplicate_warnings.length > 0) {
+    lines.push("", "## Duplicate warnings");
+    for (const warning of result.duplicate_warnings) {
+      lines.push(`- ${warning.type}: ${warning.entries.join(", ")}`);
+    }
+  }
+
+  let content = lines.join("\n");
+  if (content.length > budget) {
+    content = `${content.slice(0, Math.max(0, budget - 32))}\n... [truncated]`;
+  }
+
+  return wrapWithBoundary(content);
+}
+
+export function queryWiki(wikiRoot, query, options = {}) {
+  const searchText = String(query || "").trim();
+  if (!searchText) {
+    return { success: false, error: "query requires search text" };
+  }
+
+  const maxResults = Math.max(1, Math.min(Number(options.maxResults) || DEFAULT_MAX_RESULTS, 20));
+  const maxNeighbors = Math.max(
+    0,
+    Math.min(Number(options.maxNeighbors) || Math.max(maxResults, DEFAULT_MAX_NEIGHBORS), 40)
+  );
+  const artifacts = loadQueryArtifacts(wikiRoot);
+  const entriesByPath = mapEntriesByPath(artifacts.index.entries);
+  const nodesById = mapNodesById(artifacts.graph.nodes);
+  const directMatches = buildDirectMatches(searchText, artifacts.index, nodesById, maxResults);
+  const graphNeighbors = buildGraphNeighbors(directMatches, artifacts.graph, entriesByPath, nodesById, maxNeighbors);
+  const returnedPaths = new Set([
+    ...directMatches.map((entry) => entry.path),
+    ...graphNeighbors.map((entry) => entry.path),
+  ]);
+  const duplicateWarnings = buildDuplicateWarnings(artifacts.index.duplicates, returnedPaths);
+
+  const result = {
+    success: true,
+    query: searchText,
+    wikiRoot,
+    rebuilt: artifacts.rebuilt,
+    direct_matches: directMatches,
+    graph_neighbors: graphNeighbors,
+    duplicate_warnings: duplicateWarnings,
+    meta: {
+      indexPath: artifacts.indexPath,
+      graphPath: artifacts.graphPath,
+      direct_count: directMatches.length,
+      neighbor_count: graphNeighbors.length,
+    },
+    message: `Found ${directMatches.length} direct match(es) and ${graphNeighbors.length} graph neighbor(s).`,
+  };
+
+  if (options.format === "context") {
+    result.format = "context";
+    result.context = formatQueryForContext(result, options.contextBudget);
+    result.message = result.context;
+  }
+
+  return result;
+}

package/src/wiki/raw-manager.js

@@ -0,0 +1,400 @@
+/**
+ * Raw folder manager for oh-my-llmwikimode
+ *
+ * Unified management of raw/ subsystems: sources, notebooklm, packs.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { getWikiPaths } from "./store.js";
+import { readSourceRecords } from "./source-ledger.js";
+
+function compareStrings(left, right) {
+  return String(left ?? "").localeCompare(String(right ?? ""));
+}
+
+function statOrNull(filePath) {
+  try {
+    return fs.statSync(filePath);
+  } catch {
+    return null;
+  }
+}
+
+function readJsonFile(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Ensure only raw/ base directories exist.
+ * Does NOT create inbox/, problems/, editorial/lessons/.
+ */
+function ensureRawStructure(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+  const dirs = [paths.rawRoot, paths.rawSources, paths.rawNotebooklm, paths.rawPacks];
+  for (const dir of dirs) {
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+  }
+}
+
+/**
+ * List all raw contents categorized by subsystem.
+ * Pure read function — does not create curated directories.
+ */
+export function listRawContents(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+  // Only ensure raw dirs exist if we're writing; for read-only, skip.
+  // But if raw/ doesn't exist at all, we still need the paths object.
+
+  const result = {
+    sources: [],
+    notebooklm: [],
+    packs: [],
+  };
+
+  // Sources: read from ledger
+  const ledgerResult = readSourceRecords(wikiRoot);
+  if (ledgerResult.success && Array.isArray(ledgerResult.records)) {
+    result.sources = ledgerResult.records.map((r) => ({
+      source_id: r.source_id,
+      title: r.title,
+      kind: r.kind,
+      imported_at: r.imported_at,
+      stored_path: r.stored_path,
+      redaction: r.redaction,
+    }));
+  }
+
+  // NotebookLM: scan imports directory
+  const notebooklmImportsDir = path.join(paths.rawNotebooklm, "imports");
+  if (fs.existsSync(notebooklmImportsDir)) {
+    const runDirs = fs.readdirSync(notebooklmImportsDir, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+
+    for (const runId of runDirs) {
+      const runDir = path.join(notebooklmImportsDir, runId);
+      const manifestPath = path.join(runDir, "manifest.json");
+      const manifest = readJsonFile(manifestPath);
+
+      if (manifest) {
+        result.notebooklm.push({
+          run_id: manifest.run_id || runId,
+          original_query: manifest.original_query || "",
+          created_at: manifest.created_at || "",
+          file_count: Array.isArray(manifest.files) ? manifest.files.length : 0,
+          raw_path: path.relative(wikiRoot, runDir).replace(/\\/g, "/"),
+        });
+      } else {
+        // Fallback: count files directly
+        const files = fs.readdirSync(runDir, { withFileTypes: true })
+          .filter((e) => e.isFile())
+          .map((e) => e.name);
+        result.notebooklm.push({
+          run_id: runId,
+          original_query: "",
+          created_at: "",
+          file_count: files.length,
+          raw_path: path.relative(wikiRoot, runDir).replace(/\\/g, "/"),
+        });
+      }
+    }
+  }
+
+  // Packs: scan packs directory for {pack_id}/{version}/ structure
+  if (fs.existsSync(paths.rawPacks)) {
+    const packIdDirs = fs.readdirSync(paths.rawPacks, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name);
+
+    for (const packId of packIdDirs) {
+      const packIdPath = path.join(paths.rawPacks, packId);
+      const versionDirs = fs.readdirSync(packIdPath, { withFileTypes: true })
+        .filter((d) => d.isDirectory())
+        .map((d) => d.name);
+
+      for (const version of versionDirs) {
+        const versionPath = path.join(packIdPath, version);
+        const manifestPath = path.join(versionPath, "manifest.json");
+        const manifest = readJsonFile(manifestPath);
+
+        if (manifest) {
+          result.packs.push({
+            pack_id: manifest.id || packId,
+            version: manifest.version || version,
+            title: manifest.title || packId,
+            entry_count: Array.isArray(manifest.entries) ? manifest.entries.length : 0,
+            raw_path: path.relative(wikiRoot, versionPath).replace(/\\/g, "/"),
+          });
+        } else {
+          // Fallback: count .md files in version dir
+          const entries = fs.readdirSync(versionPath, { withFileTypes: true })
+            .filter((e) => e.isFile() && e.name.endsWith(".md"))
+            .length;
+          result.packs.push({
+            pack_id: packId,
+            version,
+            title: packId,
+            entry_count: entries,
+            raw_path: path.relative(wikiRoot, versionPath).replace(/\\/g, "/"),
+          });
+        }
+      }
+    }
+  }
+
+  // Sort each category
+  result.sources.sort((a, b) => compareStrings(a.source_id, b.source_id));
+  result.notebooklm.sort((a, b) => compareStrings(a.run_id, b.run_id));
+  result.packs.sort((a, b) => compareStrings(a.pack_id, b.pack_id) || compareStrings(a.version, b.version));
+
+  return result;
+}
+
+/**
+ * Get summary statistics for raw/ folder.
+ * Pure read function — does not create curated directories.
+ */
+export function getRawSummary(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+
+  const categories = {
+    sources: { count: 0, total_bytes: 0, last_updated: null },
+    notebooklm: { count: 0, total_bytes: 0, last_updated: null },
+    packs: { count: 0, total_bytes: 0, last_updated: null },
+  };
+
+  // Sources
+  const ledgerResult = readSourceRecords(wikiRoot);
+  if (ledgerResult.success && Array.isArray(ledgerResult.records)) {
+    categories.sources.count = ledgerResult.records.length;
+    for (const r of ledgerResult.records) {
+      if (r.stored_path) {
+        const fullPath = path.join(wikiRoot, r.stored_path);
+        const st = statOrNull(fullPath);
+        if (st) {
+          categories.sources.total_bytes += st.size;
+        }
+      }
+      if (r.imported_at) {
+        if (!categories.sources.last_updated || r.imported_at > categories.sources.last_updated) {
+          categories.sources.last_updated = r.imported_at;
+        }
+      }
+    }
+  }
+
+  // NotebookLM
+  const notebooklmImportsDir = path.join(paths.rawNotebooklm, "imports");
+  if (fs.existsSync(notebooklmImportsDir)) {
+    const runDirs = fs.readdirSync(notebooklmImportsDir, { withFileTypes: true })
+      .filter((d) => d.isDirectory());
+    categories.notebooklm.count = runDirs.length;
+
+    for (const runDir of runDirs) {
+      const runPath = path.join(notebooklmImportsDir, runDir.name);
+      const files = fs.readdirSync(runPath, { withFileTypes: true })
+        .filter((e) => e.isFile());
+      for (const file of files) {
+        const st = statOrNull(path.join(runPath, file.name));
+        if (st) {
+          categories.notebooklm.total_bytes += st.size;
+        }
+      }
+      // Check manifest for created_at
+      const manifest = readJsonFile(path.join(runPath, "manifest.json"));
+      if (manifest?.created_at) {
+        if (!categories.notebooklm.last_updated || manifest.created_at > categories.notebooklm.last_updated) {
+          categories.notebooklm.last_updated = manifest.created_at;
+        }
+      }
+    }
+  }
+
+  // Packs: {pack_id}/{version}/ structure
+  if (fs.existsSync(paths.rawPacks)) {
+    const packIdDirs = fs.readdirSync(paths.rawPacks, { withFileTypes: true })
+      .filter((d) => d.isDirectory());
+
+    for (const packIdDir of packIdDirs) {
+      const packIdPath = path.join(paths.rawPacks, packIdDir.name);
+      const versionDirs = fs.readdirSync(packIdPath, { withFileTypes: true })
+        .filter((d) => d.isDirectory());
+
+      categories.packs.count += versionDirs.length;
+
+      for (const versionDir of versionDirs) {
+        const versionPath = path.join(packIdPath, versionDir.name);
+        // Use string paths (not Dirent) when recursive: true, because Dirent.name is just basename
+        const entries = fs.readdirSync(versionPath, { recursive: true })
+          .filter((relPath) => fs.statSync(path.join(versionPath, relPath)).isFile());
+        for (const relPath of entries) {
+          const fullPath = path.join(versionPath, relPath);
+          const st = statOrNull(fullPath);
+          if (st) {
+            categories.packs.total_bytes += st.size;
+          }
+        }
+        // Check manifest for updated_at or created_at
+        const manifest = readJsonFile(path.join(versionPath, "manifest.json"));
+        const ts = manifest?.updated_at || manifest?.created_at || null;
+        if (ts) {
+          if (!categories.packs.last_updated || ts > categories.packs.last_updated) {
+            categories.packs.last_updated = ts;
+          }
+        }
+      }
+    }
+  }
+
+  const totalFiles = categories.sources.count + categories.notebooklm.count + categories.packs.count;
+  const totalBytes = categories.sources.total_bytes + categories.notebooklm.total_bytes + categories.packs.total_bytes;
+
+  return {
+    wiki_root: wikiRoot,
+    raw_root: "raw/",
+    categories,
+    total_files: totalFiles,
+    total_bytes: totalBytes,
+  };
+}
+
+/**
+ * Validate raw/ directory structure.
+ * Does NOT modify the filesystem.
+ */
+function pathType(filePath) {
+  try {
+    const st = fs.statSync(filePath);
+    if (st.isDirectory()) return "directory";
+    if (st.isFile()) return "file";
+    return "other";
+  } catch {
+    return "missing";
+  }
+}
+
+export function validateRawStructure(wikiRoot) {
+  const paths = getWikiPaths(wikiRoot);
+
+  const checks = [];
+  let warnings = 0;
+
+  function checkPath(label, filePath, required) {
+    const type = pathType(filePath);
+    const exists = type === "directory";
+    let message;
+    if (exists) {
+      message = "OK";
+    } else if (type === "file") {
+      message = `ERROR: ${label} is a file, expected directory`;
+      warnings++;
+    } else if (type === "missing") {
+      message = required ? `Missing ${label}` : `No ${label} yet`;
+    } else {
+      message = `ERROR: ${label} has unexpected type`;
+      warnings++;
+    }
+    checks.push({
+      path: label,
+      exists,
+      required,
+      message,
+      type,
+    });
+    return exists;
+  }
+
+  // Check raw root exists and is a directory
+  const rawRootExists = checkPath("raw/", paths.rawRoot, true);
+
+  // Check sources structure
+  checkPath("raw/sources/", paths.rawSources, false);
+
+  // Check notebooklm structure
+  checkPath("raw/notebooklm/", paths.rawNotebooklm, false);
+
+  // Check packs structure
+  const packsExists = checkPath("raw/packs/", paths.rawPacks, false);
+
+  // Validate pack substructure if packs exist as directory
+  if (packsExists) {
+    const packChildren = fs.readdirSync(paths.rawPacks, { withFileTypes: true });
+
+    // Check for non-directory children in raw/packs/
+    for (const child of packChildren) {
+      if (!child.isDirectory()) {
+        checks.push({
+          path: `raw/packs/${child.name}`,
+          exists: true,
+          required: false,
+          message: `WARNING: Expected pack directory, found ${child.isFile() ? "file" : "other"}`,
+          warning: true,
+        });
+        warnings++;
+      }
+    }
+
+    const packIdDirs = packChildren.filter((d) => d.isDirectory());
+
+    for (const packIdDir of packIdDirs) {
+      const packIdPath = path.join(paths.rawPacks, packIdDir.name);
+      const versionDirs = fs.readdirSync(packIdPath, { withFileTypes: true })
+        .filter((d) => d.isDirectory());
+
+      if (versionDirs.length === 0) {
+        checks.push({
+          path: `raw/packs/${packIdDir.name}/`,
+          exists: true,
+          required: false,
+          message: "WARNING: No version subdirectory",
+          warning: true,
+        });
+        warnings++;
+      } else {
+        for (const versionDir of versionDirs) {
+          const versionPath = path.join(packIdPath, versionDir.name);
+          const manifestPath = path.join(versionPath, "manifest.json");
+          const manifestType = pathType(manifestPath);
+          const hasManifest = manifestType === "file";
+          if (!hasManifest) {
+            checks.push({
+              path: `raw/packs/${packIdDir.name}/${versionDir.name}/manifest.json`,
+              exists: false,
+              required: false,
+              message: manifestType === "directory"
+                ? "WARNING: manifest.json is a directory"
+                : "WARNING: Missing manifest.json",
+              warning: true,
+            });
+            warnings++;
+          }
+        }
+        checks.push({
+          path: `raw/packs/${packIdDir.name}/`,
+          exists: true,
+          required: false,
+          message: `${versionDirs.length} version subdirectory(s)`,
+        });
+      }
+    }
+  }
+
+  const requiredOk = checks.every((c) => !c.required || c.exists);
+  const valid = requiredOk && warnings === 0;
+
+  return {
+    valid,
+    wiki_root: wikiRoot,
+    checks,
+    warnings,
+  };
+}
+
+export { ensureRawStructure };