oh-my-llmwikimode 1.0.0

package/src/wiki/lint.js

@@ -0,0 +1,294 @@
+/**
+ * Runtime lint engine for oh-my-llmwikimode wiki entries.
+ *
+ * This module is intentionally read-only: it reports validation findings but
+ * never rewrites, merges, deletes, or indexes wiki files.
+ */
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { containsPromptInjection, redactSecrets } from "../security.js";
+import {
+  parseFrontmatter,
+  VALID_CONFIDENCES,
+  VALID_SOURCES,
+  VALID_STATUSES,
+} from "./store.js";
+
+const ENTRY_ROOTS = [
+  { category: "inbox", segments: ["inbox"] },
+  { category: "problems", segments: ["problems"] },
+  { category: "lessons", segments: ["editorial", "lessons"] },
+];
+
+const REQUIRED_FIELDS = ["title", "status", "source"];
+const EXCLUDED_STATUSES = new Set(["rejected", "superseded", "private", "needs-clarification"]);
+
+function normalizeScalar(value) {
+  return String(value ?? "")
+    .replace(/\r?\n/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function normalizeDisplayText(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => normalizeDisplayText(item)).filter(Boolean).join(" ");
+  }
+  return normalizeScalar(value);
+}
+
+function normalizeTitleKey(value) {
+  return normalizeDisplayText(value).toLowerCase().replace(/[^a-z0-9\p{L}\p{N}]/gu, "");
+}
+
+function contentHash(content) {
+  return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+
+function toRelativePath(wikiRoot, fullPath) {
+  return path.relative(wikiRoot, fullPath).replace(/\\/g, "/");
+}
+
+function collectMarkdownFiles(rootDir) {
+  if (!fs.existsSync(rootDir)) return [];
+
+  const files = [];
+  function walk(dir) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  walk(rootDir);
+  return files;
+}
+
+function createFinding(pathName, code, field, message, value) {
+  const finding = { path: pathName, code, field, message };
+  if (value !== undefined) finding.value = value;
+  return finding;
+}
+
+function isMissingRequiredValue(value) {
+  if (value === undefined || value === null) return true;
+  if (typeof value === "string") return value.trim() === "";
+  if (Array.isArray(value)) return value.length === 0 || normalizeDisplayText(value) === "";
+  return false;
+}
+
+function validateRequiredFields(frontmatter, relPath, errors) {
+  for (const field of REQUIRED_FIELDS) {
+    if (isMissingRequiredValue(frontmatter[field])) {
+      errors.push(createFinding(relPath, "missing_required_field", field, `${field} is required`));
+    }
+  }
+}
+
+function validateTitle(frontmatter, relPath, errors) {
+  const title = frontmatter.title;
+  if (!isMissingRequiredValue(title) && typeof title !== "string") {
+    errors.push(createFinding(relPath, "invalid_field_type", "title", "title must be a string", title));
+  }
+}
+
+function validateEnum(frontmatter, relPath, errors, field, validValues, normalizeValue) {
+  const value = frontmatter[field];
+  if (value === undefined || value === null || value === "") return;
+
+  if (typeof value !== "string") {
+    errors.push(createFinding(relPath, "invalid_field_type", field, `${field} must be a string`, value));
+    return;
+  }
+
+  const normalized = normalizeValue(value);
+  if (!validValues.includes(normalized)) {
+    errors.push(
+      createFinding(
+        relPath,
+        "invalid_enum_value",
+        field,
+        `${field} must be one of: ${validValues.join(", ")}`,
+        value
+      )
+    );
+  }
+}
+
+function isUnsafeRelativePath(value) {
+  if (typeof value !== "string" || value.trim() === "") return false;
+  const normalizedPath = value.replace(/\\/g, "/");
+  if (path.isAbsolute(value) || path.posix.isAbsolute(normalizedPath)) return true;
+  return normalizedPath.split("/").includes("..");
+}
+
+function validateRawPath(frontmatter, relPath, errors) {
+  if (frontmatter.raw_path === undefined) return;
+
+  if (typeof frontmatter.raw_path !== "string") {
+    errors.push(createFinding(relPath, "invalid_field_type", "raw_path", "raw_path must be a relative string", frontmatter.raw_path));
+    return;
+  }
+
+  if (isUnsafeRelativePath(frontmatter.raw_path)) {
+    errors.push(
+      createFinding(
+        relPath,
+        "unsafe_raw_path",
+        "raw_path",
+        "raw_path must be relative and must not contain path traversal",
+        frontmatter.raw_path
+      )
+    );
+  }
+}
+
+function validatePrivacy(content, relPath, warnings) {
+  if (redactSecrets(content) !== content) {
+    warnings.push(createFinding(relPath, "secret_detected", "content", "entry contains text that matches a secret pattern"));
+  }
+
+  if (containsPromptInjection(content)) {
+    warnings.push(createFinding(relPath, "prompt_injection_detected", "content", "entry contains a prompt-injection-like pattern"));
+  }
+}
+
+function addDuplicateWarnings(groups, warnings, duplicates) {
+  for (const group of groups) {
+    if (group.entries.length < 2) continue;
+
+    duplicates.push(group);
+    warnings.push({
+      code: group.type === "title" ? "duplicate_title" : "duplicate_content_hash",
+      type: group.type,
+      message: group.type === "title"
+        ? `Duplicate normalized title: ${group.normalized_title}`
+        : `Duplicate content hash: ${group.hash}`,
+      entries: group.entries,
+    });
+  }
+}
+
+function buildDuplicateFindings(titleMap, hashMap, warnings) {
+  const duplicates = [];
+  const titleGroups = Array.from(titleMap.entries())
+    .filter(([, entries]) => entries.length > 1)
+    .map(([normalizedTitle, entries]) => ({
+      type: "title",
+      normalized_title: normalizedTitle,
+      entries,
+    }));
+  const hashGroups = Array.from(hashMap.entries())
+    .filter(([, entries]) => entries.length > 1)
+    .map(([hash, entries]) => ({
+      type: "content",
+      hash,
+      entries,
+    }));
+
+  addDuplicateWarnings(titleGroups, warnings, duplicates);
+  addDuplicateWarnings(hashGroups, warnings, duplicates);
+  return duplicates;
+}
+
+function scanEntryFile({ wikiRoot, fullPath, category, errors, warnings, titleMap, hashMap }) {
+  const relPath = toRelativePath(wikiRoot, fullPath);
+  const content = fs.readFileSync(fullPath, "utf-8");
+  const { frontmatter, body } = parseFrontmatter(content);
+  const title = normalizeDisplayText(frontmatter.title);
+  const status = typeof frontmatter.status === "string" ? frontmatter.status.toLowerCase() : frontmatter.status;
+  const source = typeof frontmatter.source === "string" ? frontmatter.source.toLowerCase() : frontmatter.source;
+  const confidence = typeof frontmatter.confidence === "string" ? frontmatter.confidence.toUpperCase() : frontmatter.confidence;
+  const excluded = typeof status === "string" && EXCLUDED_STATUSES.has(status);
+
+  validateRequiredFields(frontmatter, relPath, errors);
+  validateTitle(frontmatter, relPath, errors);
+  validateEnum(frontmatter, relPath, errors, "status", VALID_STATUSES, (value) => value.toLowerCase());
+  validateEnum(frontmatter, relPath, errors, "source", VALID_SOURCES, (value) => value.toLowerCase());
+  validateEnum(frontmatter, relPath, errors, "confidence", VALID_CONFIDENCES, (value) => value.toUpperCase());
+  validateRawPath(frontmatter, relPath, errors);
+  validatePrivacy(content, relPath, warnings);
+
+  const titleKey = normalizeTitleKey(frontmatter.title);
+  if (titleKey) {
+    const entries = titleMap.get(titleKey) || [];
+    entries.push(relPath);
+    titleMap.set(titleKey, entries);
+  }
+
+  const hash = contentHash(content);
+  const hashEntries = hashMap.get(hash) || [];
+  hashEntries.push(relPath);
+  hashMap.set(hash, hashEntries);
+
+  return {
+    path: relPath,
+    category,
+    title,
+    status,
+    source,
+    confidence,
+    excluded,
+    body_hash: contentHash(body),
+    content_hash: hash,
+    raw_path: frontmatter.raw_path,
+  };
+}
+
+export function lintWiki(wikiRoot) {
+  const rootPath = path.resolve(wikiRoot);
+  const errors = [];
+  const warnings = [];
+  const entries = [];
+  const titleMap = new Map();
+  const hashMap = new Map();
+
+  if (!fs.existsSync(rootPath)) {
+    errors.push({ code: "wiki_root_missing", path: rootPath, message: "wiki root does not exist" });
+    return { ok: false, errors, warnings, entries, duplicates: [] };
+  }
+
+  for (const entryRoot of ENTRY_ROOTS) {
+    const dir = path.join(rootPath, ...entryRoot.segments);
+    for (const fullPath of collectMarkdownFiles(dir)) {
+      try {
+        entries.push(
+          scanEntryFile({
+            wikiRoot: rootPath,
+            fullPath,
+            category: entryRoot.category,
+            errors,
+            warnings,
+            titleMap,
+            hashMap,
+          })
+        );
+      } catch (error) {
+        errors.push({
+          path: toRelativePath(rootPath, fullPath),
+          code: "unreadable_entry",
+          message: `entry could not be read: ${error.message}`,
+        });
+      }
+    }
+  }
+
+  const duplicates = buildDuplicateFindings(titleMap, hashMap, warnings);
+
+  return {
+    ok: errors.length === 0,
+    errors,
+    warnings,
+    entries,
+    duplicates,
+  };
+}
+
+export { EXCLUDED_STATUSES };

package/src/wiki/notebooklm-adapter.js

@@ -0,0 +1,264 @@
+/**
+ * NotebookLM artifact adapter for oh-my-llmwikimode
+ *
+ * Imports local NotebookLM export files into raw storage and creates
+ * searchable candidate wiki entries.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import {
+  getWikiPaths,
+  ensureWikiStructure,
+  stringifyFrontmatter,
+  storeKnowledge,
+} from "./store.js";
+import { redactSecrets } from "../security.js";
+
+/**
+ * Generate a stable run_id from query and current date.
+ */
+export function generateRunId(query, dateStr) {
+  const input = `${query}:${dateStr || new Date().toISOString().slice(0, 10)}`;
+  return crypto.createHash("sha256").update(input).digest("hex").slice(0, 16);
+}
+
+/**
+ * Compute SHA-256 hash of a file.
+ */
+export function hashFile(filePath) {
+  const content = fs.readFileSync(filePath);
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Validate NotebookLM import input path.
+ */
+function validateInputPath(inputPath) {
+  if (!inputPath || typeof inputPath !== "string") {
+    return { success: false, error: "Input path is required" };
+  }
+
+  if (!fs.existsSync(inputPath)) {
+    return { success: false, error: "Input path does not exist" };
+  }
+
+  const stat = fs.lstatSync(inputPath);
+  if (stat.isSymbolicLink()) {
+    return { success: false, error: "Symbolic link directories are not allowed" };
+  }
+
+  if (!stat.isDirectory()) {
+    return { success: false, error: "Input path must be a directory" };
+  }
+
+  return { success: true };
+}
+
+const ALLOWED_EXTENSIONS = new Set([".md", ".txt", ".json", ".yaml", ".yml", ".csv"]);
+const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB
+const MAX_FILE_COUNT = 50;
+
+/**
+ * Import NotebookLM artifacts from a local export directory.
+ */
+export function importNotebookLmArtifacts(options) {
+  const {
+    inputPath,
+    wikiRoot,
+    originalQuery = "",
+    title,
+    tags = [],
+    runId: providedRunId,
+  } = options;
+
+  // Validate input path
+  const pathValidation = validateInputPath(inputPath);
+  if (!pathValidation.success) {
+    return pathValidation;
+  }
+
+  const paths = getWikiPaths(wikiRoot);
+  ensureWikiStructure(paths);
+
+  const runId = providedRunId || generateRunId(originalQuery);
+  const rawDir = path.join(paths.root, "raw", "notebooklm", "imports", runId);
+
+  // Check for run-id collision
+  if (fs.existsSync(rawDir)) {
+    return { success: false, error: `Run ID collision: ${runId} already exists` };
+  }
+
+  fs.mkdirSync(rawDir, { recursive: true });
+
+  // Collect artifacts with validation
+  const artifacts = [];
+  const files = fs.readdirSync(inputPath);
+
+  if (files.length > MAX_FILE_COUNT) {
+    return { success: false, error: `Too many files: ${files.length} > ${MAX_FILE_COUNT}` };
+  }
+
+  for (const file of files) {
+    const sourcePath = path.join(inputPath, file);
+
+    // Skip symlinks
+    if (fs.lstatSync(sourcePath).isSymbolicLink()) {
+      continue;
+    }
+
+    if (!fs.statSync(sourcePath).isFile()) {
+      continue;
+    }
+
+    // Check extension
+    const ext = path.extname(file).toLowerCase();
+    if (!ALLOWED_EXTENSIONS.has(ext)) {
+      continue;
+    }
+
+    // Check size
+    const size = fs.statSync(sourcePath).size;
+    if (size > MAX_FILE_SIZE) {
+      continue;
+    }
+
+    const destPath = path.join(rawDir, file);
+
+    // Read and redact before writing
+    const content = fs.readFileSync(sourcePath, "utf-8");
+    const redactedContent = redactSecrets(content);
+    fs.writeFileSync(destPath, redactedContent);
+
+    artifacts.push({
+      relative_path: file,
+      sha256: hashFile(destPath),
+      bytes: fs.statSync(destPath).size,
+    });
+  }
+
+  if (artifacts.length === 0) {
+    // Clean up empty raw dir
+    try {
+      fs.rmSync(rawDir, { recursive: true });
+    } catch {
+      // ignore cleanup error
+    }
+    return { success: false, error: "No valid artifacts found in input path" };
+  }
+
+  // Write manifest
+  const manifest = {
+    source: "notebooklm",
+    run_id: runId,
+    original_query: originalQuery,
+    created_at: new Date().toISOString(),
+    files: artifacts,
+  };
+
+  const manifestPath = path.join(rawDir, "manifest.json");
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+
+  // Create candidate entry
+  const reportFile = artifacts.find((a) => a.relative_path === "report.md");
+  const candidatePaths = [];
+
+  if (reportFile) {
+    const reportPath = path.join(rawDir, "report.md");
+    const reportContent = fs.readFileSync(reportPath, "utf-8");
+    const safeContent = redactSecrets(reportContent);
+
+    const candidateTitle = title || (originalQuery ? originalQuery.slice(0, 80) : "NotebookLM Import");
+    const candidateTags = [...tags, "notebooklm", "research", "candidate"];
+
+    const candidatePath = storeKnowledge(wikiRoot, {
+      summary: candidateTitle,
+      details: safeContent,
+      tags: candidateTags,
+      status: "candidate",
+      source: "notebooklm",
+    });
+
+    if (candidatePath) {
+      // Append provenance metadata to the candidate file
+      const candidateContent = fs.readFileSync(candidatePath, "utf-8");
+      const { frontmatter, body } = parseCandidateFrontmatter(candidateContent);
+
+      frontmatter.original_query = originalQuery;
+      frontmatter.run_id = runId;
+      frontmatter.raw_path = path.relative(wikiRoot, rawDir).replace(/\\/g, "/");
+      frontmatter.confidence = "AMBIGUOUS";
+      frontmatter.imported_at = new Date().toISOString();
+
+      const updatedContent = stringifyFrontmatter(frontmatter) + "\n" + body;
+      fs.writeFileSync(candidatePath, updatedContent);
+      candidatePaths.push(candidatePath);
+    }
+  }
+
+  return {
+    success: true,
+    runId,
+    rawPath: rawDir,
+    manifestPath,
+    candidatePaths,
+    artifactCount: artifacts.length,
+  };
+}
+
+/**
+ * Parse frontmatter from a candidate file (lightweight wrapper).
+ */
+function parseCandidateFrontmatter(content) {
+  const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/);
+  if (!match) return { frontmatter: {}, body: content };
+
+  const lines = match[1].split("\n");
+  const frontmatter = {};
+  let currentKey = null;
+  let currentArray = null;
+
+  for (const line of lines) {
+    const trimmedLine = line.trim();
+
+    if (trimmedLine.startsWith("- ") && currentArray) {
+      currentArray.push(trimmedLine.slice(2));
+      continue;
+    }
+
+    const keyValueMatch = line.match(/^(\w+):\s*(.*)$/);
+    if (keyValueMatch) {
+      const [, key, value] = keyValueMatch;
+      const trimmedValue = value.trim();
+      if (trimmedValue.startsWith('"') && trimmedValue.endsWith('"')) {
+        try {
+          frontmatter[key] = JSON.parse(trimmedValue);
+        } catch {
+          frontmatter[key] = trimmedValue.slice(1, -1);
+        }
+        currentKey = key;
+        currentArray = null;
+      } else if (trimmedValue.startsWith("[") && trimmedValue.endsWith("]")) {
+        frontmatter[key] = trimmedValue.slice(1, -1).split(",").map((s) => s.trim());
+        currentKey = key;
+        currentArray = null;
+      } else if (trimmedValue) {
+        frontmatter[key] = trimmedValue;
+        currentKey = key;
+        currentArray = null;
+      } else {
+        currentKey = key;
+        currentArray = null;
+      }
+    } else if (trimmedLine.startsWith("- ")) {
+      currentArray = [];
+      if (currentKey) {
+        frontmatter[currentKey] = currentArray;
+      }
+      currentArray.push(trimmedLine.slice(2));
+    }
+  }
+
+  return { frontmatter, body: match[2].trim() };
+}