oh-my-llmwikimode 1.0.0

package/src/wiki/entry-normalizer.js

@@ -0,0 +1,139 @@
+export const VALID_STATUSES = ["candidate", "lesson", "problem", "rejected", "superseded", "private", "needs-clarification"];
+export const VALID_SOURCES = ["auto-memory", "manual", "import", "notebooklm", "chat.message"];
+export const VALID_CONFIDENCES = ["EXTRACTED", "INFERRED", "AMBIGUOUS"];
+
+function compareStrings(a, b) {
+  const left = String(a ?? "");
+  const right = String(b ?? "");
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringifyRecord(value) {
+  try {
+    return JSON.stringify(value) || "";
+  } catch (error) {
+    return Object.prototype.toString.call(value);
+  }
+}
+
+export function normalizeScalar(value) {
+  const scalar = isRecord(value) ? stringifyRecord(value) : String(value ?? "");
+  return scalar
+    .replace(/\r?\n/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function normalizeEntryText(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => normalizeScalar(item)).filter(Boolean).join(" ");
+  }
+  return normalizeScalar(value);
+}
+
+export function normalizeEntryTags(value) {
+  const values = Array.isArray(value) ? value : typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? [value] : [];
+  return [...new Set(values.map((item) => normalizeScalar(item)).filter(Boolean))].sort(compareStrings);
+}
+
+export function normalizeStringList(value) {
+  const values = Array.isArray(value) ? value : typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? [value] : [];
+  return [...new Set(values.map((item) => normalizeScalar(item)).filter(Boolean))].sort(compareStrings);
+}
+
+export function normalizeEntryStatus(value) {
+  const normalized = normalizeScalar(value || "candidate").toLowerCase();
+  return VALID_STATUSES.includes(normalized) ? normalized : "candidate";
+}
+
+export function normalizeEntrySource(value) {
+  const normalized = normalizeScalar(value || "manual").toLowerCase();
+  return VALID_SOURCES.includes(normalized) ? normalized : "manual";
+}
+
+export function normalizeEntryConfidence(value) {
+  const normalized = normalizeScalar(value || "EXTRACTED").toUpperCase();
+  return VALID_CONFIDENCES.includes(normalized) ? normalized : "EXTRACTED";
+}
+
+export function normalizeConfidenceScore(value, fallback = 1.0) {
+  return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+
+export function normalizeIndexEntry(entry, fallbackPath = "") {
+  const rawEntry = isRecord(entry) ? entry : {};
+  const normalizedPath = normalizeEntryText(rawEntry.path) || normalizeEntryText(fallbackPath);
+  const normalizedTitle = normalizeEntryText(rawEntry.title) || normalizeEntryText(rawEntry.summary) || normalizedPath;
+
+  return {
+    ...rawEntry,
+    path: normalizedPath,
+    category: normalizeEntryText(rawEntry.category) || "wiki",
+    title: normalizedTitle,
+    aliases: normalizeStringList(rawEntry.aliases),
+    tags: normalizeEntryTags(rawEntry.tags),
+    status: normalizeEntryStatus(rawEntry.status),
+    source: normalizeEntrySource(rawEntry.source),
+    summary: normalizeEntryText(rawEntry.summary || normalizedTitle),
+    confidence: normalizeEntryConfidence(rawEntry.confidence),
+    confidence_score: normalizeConfidenceScore(rawEntry.confidence_score),
+    related: normalizeStringList(rawEntry.related),
+  };
+}
+
+function normalizePostingList(value) {
+  if (!Array.isArray(value)) return [];
+  return [...new Set(value.map((item) => Number.parseInt(item, 10)).filter((item) => Number.isInteger(item) && item >= 0))]
+    .sort((left, right) => left - right);
+}
+
+function normalizeTermFrequency(value) {
+  if (!isRecord(value)) return {};
+  const normalized = {};
+  const tokenEntries = Object.entries(value)
+    .map(([token, entryCounts]) => {
+      const normalizedToken = normalizeScalar(token).toLowerCase();
+      if (!normalizedToken) return null;
+      if (!isRecord(entryCounts)) return null;
+      const normalizedCounts = Object.fromEntries(
+        Object.entries(entryCounts)
+          .map(([id, count]) => [Number.parseInt(id, 10), count])
+          .filter(([id, count]) => Number.isInteger(id) && id >= 0 && typeof count === "number" && count > 0)
+          .sort(([a], [b]) => a - b)
+      );
+      if (Object.keys(normalizedCounts).length === 0) return null;
+      return [normalizedToken, normalizedCounts];
+    })
+    .filter(Boolean)
+    .sort(([left], [right]) => compareStrings(left, right));
+  for (const [token, counts] of tokenEntries) {
+    normalized[token] = counts;
+  }
+  return normalized;
+}
+
+function normalizeInvertedIndex(value) {
+  if (!isRecord(value)) return {};
+  const normalizedEntries = Object.entries(value)
+    .map(([token, postings]) => [normalizeScalar(token).toLowerCase(), normalizePostingList(postings)])
+    .filter(([token, postings]) => token && postings.length > 0)
+    .sort(([left], [right]) => compareStrings(left, right));
+  return Object.fromEntries(normalizedEntries);
+}
+
+export function normalizeWikiIndex(index) {
+  const rawIndex = isRecord(index) ? index : {};
+  const rawEntries = Array.isArray(rawIndex.entries) ? rawIndex.entries : [];
+  return {
+    ...rawIndex,
+    entries: rawEntries.map((entry, entryIndex) => normalizeIndexEntry(entry, `entry-${entryIndex}.md`)),
+    inverted_index: normalizeInvertedIndex(rawIndex.inverted_index),
+    term_frequency: normalizeTermFrequency(rawIndex.term_frequency),
+  };
+}

package/src/wiki/ingest.js

@@ -0,0 +1,443 @@
+/**
+ * Runtime ingest engine for oh-my-llmwikimode.
+ *
+ * Creates review-first candidate entries in inbox/ after privacy gates,
+ * path validation, and secret redaction.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { hasPrivacyOptOut, hasOversizedMessage } from "../hooks.js";
+import { containsPromptInjection, redactSecrets } from "../security.js";
+import {
+  buildIndex,
+  ensureWikiStructure,
+  getWikiPaths,
+  storeKnowledge,
+} from "./store.js";
+import { importNotebookLmArtifacts } from "./notebooklm-adapter.js";
+
+const ALLOWED_SOURCES = new Set(["manual", "notebooklm", "import", "auto-memory"]);
+const ALLOWED_EXTENSIONS = new Set([".md", ".markdown", ".txt", ".json", ".yaml", ".yml", ".csv"]);
+
+function normalizeSource(source) {
+  const normalized = String(source || "manual").toLowerCase();
+  return ALLOWED_SOURCES.has(normalized) ? normalized : null;
+}
+
+function normalizeTags(tags, fallbackSource) {
+  if (Array.isArray(tags)) {
+    return tags.map((tag) => String(tag || "").trim()).filter(Boolean);
+  }
+
+  if (typeof tags === "string") {
+    return tags
+      .split(",")
+      .map((tag) => tag.trim())
+      .filter(Boolean);
+  }
+
+  return [fallbackSource].filter(Boolean);
+}
+
+function isInsideOrEqual(parentDirectory, candidatePath) {
+  const relativePath = path.relative(parentDirectory, candidatePath);
+  return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
+}
+
+function relativeWikiPath(wikiRoot, filePath) {
+  return path.relative(wikiRoot, filePath).replace(/\\/g, "/");
+}
+
+function baseResult({ wikiRoot, source }) {
+  return {
+    success: true,
+    wikiRoot,
+    source,
+    candidatePaths: [],
+    rejections: [],
+    duplicateWarnings: [],
+    duplicates: [],
+  };
+}
+
+function failResult({ wikiRoot, source, error, rejections = [] }) {
+  return {
+    ...baseResult({ wikiRoot, source }),
+    success: false,
+    error,
+    rejections,
+  };
+}
+
+function rejection(input, reason, message) {
+  return { input, reason, message };
+}
+
+function privacyGate(text, inputLabel) {
+  if (!text || !String(text).trim()) {
+    return rejection(inputLabel, "empty", "Input is empty");
+  }
+
+  if (hasPrivacyOptOut(text)) {
+    return rejection(inputLabel, "privacy-opt-out", "Privacy opt-out detected; refusing to store content");
+  }
+
+  if (hasOversizedMessage(text)) {
+    return rejection(inputLabel, "oversized", "Input exceeds the safe ingest size limit");
+  }
+
+  if (containsPromptInjection(text)) {
+    return rejection(inputLabel, "prompt-injection", "Prompt injection pattern detected; refusing to store content");
+  }
+
+  return null;
+}
+
+function ensureSafeRoot(wikiRoot) {
+  const rootPath = path.resolve(wikiRoot);
+  const paths = getWikiPaths(rootPath);
+  ensureWikiStructure(paths);
+  return { rootPath, realRoot: fs.realpathSync(rootPath) };
+}
+
+function validateInputPath({ wikiRoot, inputPath, allowOutsideWikiRoot = false }) {
+  if (!inputPath || typeof inputPath !== "string") {
+    return { success: false, error: "Input path is required" };
+  }
+
+  const resolvedPath = path.resolve(inputPath);
+  if (!fs.existsSync(resolvedPath)) {
+    return { success: false, error: "Input path does not exist" };
+  }
+
+  const stat = fs.lstatSync(resolvedPath);
+  if (stat.isSymbolicLink()) {
+    return { success: false, error: "Symbolic link input paths are not allowed", reason: "symlink" };
+  }
+
+  if (!allowOutsideWikiRoot) {
+    const { realRoot } = ensureSafeRoot(wikiRoot);
+    const realInput = fs.realpathSync(resolvedPath);
+    if (!isInsideOrEqual(realRoot, realInput)) {
+      return { success: false, error: "Input path resolves outside wiki root", reason: "outside-wiki-root" };
+    }
+  }
+
+  return { success: true, path: resolvedPath, stat };
+}
+
+function validateExtension(filePath) {
+  const extension = path.extname(filePath).toLowerCase();
+  return ALLOWED_EXTENSIONS.has(extension);
+}
+
+function collectFilesFromDirectory({ wikiRoot, directoryPath }) {
+  const files = [];
+  const rejections = [];
+  const { realRoot } = ensureSafeRoot(wikiRoot);
+
+  function visit(currentDirectory) {
+    const entries = fs.readdirSync(currentDirectory, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = path.join(currentDirectory, entry.name);
+      const inputLabel = relativeWikiPath(wikiRoot, fullPath);
+      const stat = fs.lstatSync(fullPath);
+
+      if (stat.isSymbolicLink()) {
+        rejections.push(rejection(inputLabel, "symlink", "Symbolic links are not allowed"));
+        continue;
+      }
+
+      const realPath = fs.realpathSync(fullPath);
+      if (!isInsideOrEqual(realRoot, realPath)) {
+        rejections.push(rejection(inputLabel, "outside-wiki-root", "File resolves outside wiki root"));
+        continue;
+      }
+
+      if (stat.isDirectory()) {
+        visit(fullPath);
+        continue;
+      }
+
+      if (!stat.isFile()) {
+        rejections.push(rejection(inputLabel, "not-file", "Only regular files can be ingested"));
+        continue;
+      }
+
+      if (!validateExtension(fullPath)) {
+        rejections.push(rejection(inputLabel, "invalid-extension", "File extension is not allowed"));
+        continue;
+      }
+
+      files.push(fullPath);
+    }
+  }
+
+  visit(directoryPath);
+  return { files, rejections };
+}
+
+function titleFromInput({ providedTitle, filePath, text }) {
+  if (providedTitle && String(providedTitle).trim()) {
+    return String(providedTitle).trim();
+  }
+
+  if (filePath) {
+    return path.basename(filePath, path.extname(filePath)).replace(/[-_]+/g, " ").trim() || "Imported Knowledge";
+  }
+
+  const firstLine = String(text || "")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .find(Boolean);
+  return (firstLine || "Manual Ingest").slice(0, 100);
+}
+
+function duplicateWarningsForCandidates(wikiRoot, candidatePaths) {
+  if (candidatePaths.length === 0) {
+    return { duplicateWarnings: [], duplicates: [] };
+  }
+
+  const index = buildIndex(wikiRoot);
+  const candidateRelPaths = new Set(candidatePaths.map((candidatePath) => relativeWikiPath(wikiRoot, candidatePath)));
+  const duplicates = (index.duplicates || []).filter((duplicate) => (
+    Array.isArray(duplicate.entries)
+      && duplicate.entries.some((entryPath) => candidateRelPaths.has(entryPath))
+  ));
+
+  return {
+    duplicates,
+    duplicateWarnings: duplicates.map((duplicate) => (
+      `Possible duplicate ${duplicate.type}: ${duplicate.entries.join(", ")}`
+    )),
+  };
+}
+
+function ingestTextItems({ wikiRoot, source, title, tags, items, project, initialRejections = [] }) {
+  const result = baseResult({ wikiRoot, source });
+  result.rejections.push(...initialRejections);
+  const safeTags = normalizeTags(tags, source).map((tag) => redactSecrets(tag));
+
+  for (const item of items) {
+    const gateRejection = privacyGate(item.text, item.input);
+    if (gateRejection) {
+      result.rejections.push(gateRejection);
+      continue;
+    }
+
+    const safeText = redactSecrets(item.text);
+    const safeTitle = redactSecrets(titleFromInput({ providedTitle: item.title || title, filePath: item.filePath, text: item.text }));
+    const candidatePath = storeKnowledge(wikiRoot, {
+      summary: safeTitle,
+      details: safeText,
+      tags: safeTags,
+      status: "candidate",
+      source,
+      project,
+    });
+
+    if (candidatePath) {
+      result.candidatePaths.push(candidatePath);
+    }
+  }
+
+  const duplicateResult = duplicateWarningsForCandidates(wikiRoot, result.candidatePaths);
+  result.duplicates = duplicateResult.duplicates;
+  result.duplicateWarnings = duplicateResult.duplicateWarnings;
+
+  if (result.candidatePaths.length === 0) {
+    const firstRejection = result.rejections[0];
+    return {
+      ...result,
+      success: false,
+      error: firstRejection?.message || "No valid input was ingested",
+    };
+  }
+
+  result.message = `Ingested ${result.candidatePaths.length} candidate${result.candidatePaths.length === 1 ? "" : "s"}.`;
+  return result;
+}
+
+function notebookLmGate(inputPath) {
+  const files = [];
+  const entries = fs.readdirSync(inputPath);
+
+  for (const entry of entries) {
+    const fullPath = path.join(inputPath, entry);
+    const stat = fs.lstatSync(fullPath);
+    if (stat.isSymbolicLink() || !stat.isFile() || !validateExtension(fullPath)) {
+      continue;
+    }
+    files.push(fullPath);
+  }
+
+  for (const filePath of files) {
+    const text = fs.readFileSync(filePath, "utf-8");
+    const gateRejection = privacyGate(text, path.basename(filePath));
+    if (gateRejection) {
+      return gateRejection;
+    }
+  }
+
+  return null;
+}
+
+function ingestNotebookLm({ wikiRoot, inputPath, title, tags, originalQuery }) {
+  const source = "notebooklm";
+  const validation = validateInputPath({ wikiRoot, inputPath, allowOutsideWikiRoot: true });
+  if (!validation.success) {
+    return failResult({
+      wikiRoot,
+      source,
+      error: validation.error,
+      rejections: [rejection(inputPath, validation.reason || "invalid-path", validation.error)],
+    });
+  }
+
+  if (!validation.stat.isDirectory()) {
+    return failResult({
+      wikiRoot,
+      source,
+      error: "NotebookLM input path must be a directory",
+      rejections: [rejection(inputPath, "not-directory", "NotebookLM input path must be a directory")],
+    });
+  }
+
+  const gateRejection = notebookLmGate(validation.path);
+  if (gateRejection) {
+    return failResult({ wikiRoot, source, error: gateRejection.message, rejections: [gateRejection] });
+  }
+
+  const importResult = importNotebookLmArtifacts({
+    inputPath: validation.path,
+    wikiRoot,
+    originalQuery: originalQuery || "",
+    title,
+    tags: normalizeTags(tags, null),
+  });
+
+  if (!importResult.success) {
+    return failResult({
+      wikiRoot,
+      source,
+      error: importResult.error,
+      rejections: [rejection(inputPath, "notebooklm-import", importResult.error)],
+    });
+  }
+
+  const duplicateResult = duplicateWarningsForCandidates(wikiRoot, importResult.candidatePaths || []);
+  return {
+    ...baseResult({ wikiRoot, source }),
+    ...importResult,
+    candidatePaths: importResult.candidatePaths || [],
+    rejections: [],
+    duplicates: duplicateResult.duplicates,
+    duplicateWarnings: duplicateResult.duplicateWarnings,
+    message: `Imported ${importResult.artifactCount} NotebookLM artifact${importResult.artifactCount === 1 ? "" : "s"}.`,
+  };
+}
+
+export function ingestKnowledge(options = {}) {
+  const wikiRoot = path.resolve(options.wikiRoot || "");
+  const source = normalizeSource(options.source);
+
+  if (!source) {
+    return failResult({
+      wikiRoot,
+      source: String(options.source || ""),
+      error: "Invalid source; expected manual, notebooklm, import, or auto-memory",
+    });
+  }
+
+  ensureSafeRoot(wikiRoot);
+
+  if (source === "notebooklm") {
+    return ingestNotebookLm({
+      wikiRoot,
+      inputPath: options.inputPath,
+      title: options.title,
+      tags: options.tags,
+      originalQuery: options.originalQuery,
+    });
+  }
+
+  const hasInputText = typeof options.inputText === "string";
+  const hasInputPath = typeof options.inputPath === "string" && options.inputPath.length > 0;
+
+  if (hasInputText && hasInputPath) {
+    return failResult({ wikiRoot, source, error: "Provide either stdin text or input-path, not both" });
+  }
+
+  if (!hasInputText && !hasInputPath) {
+    return failResult({ wikiRoot, source, error: "No ingest input provided" });
+  }
+
+  if (hasInputText) {
+    return ingestTextItems({
+      wikiRoot,
+      source,
+      title: options.title,
+      tags: options.tags,
+      project: options.project,
+      items: [{ input: "stdin", text: options.inputText }],
+    });
+  }
+
+  const validation = validateInputPath({ wikiRoot, inputPath: options.inputPath });
+  if (!validation.success) {
+    return failResult({
+      wikiRoot,
+      source,
+      error: validation.error,
+      rejections: [rejection(options.inputPath, validation.reason || "invalid-path", validation.error)],
+    });
+  }
+
+  if (validation.stat.isFile()) {
+    if (!validateExtension(validation.path)) {
+      const item = rejection(relativeWikiPath(wikiRoot, validation.path), "invalid-extension", "File extension is not allowed");
+      return failResult({ wikiRoot, source, error: item.message, rejections: [item] });
+    }
+
+    return ingestTextItems({
+      wikiRoot,
+      source,
+      title: options.title,
+      tags: options.tags,
+      project: options.project,
+      items: [{
+        input: relativeWikiPath(wikiRoot, validation.path),
+        filePath: validation.path,
+        text: fs.readFileSync(validation.path, "utf-8"),
+      }],
+    });
+  }
+
+  if (validation.stat.isDirectory()) {
+    const collected = collectFilesFromDirectory({ wikiRoot, directoryPath: validation.path });
+    const items = collected.files.map((filePath) => ({
+      input: relativeWikiPath(wikiRoot, filePath),
+      filePath,
+      text: fs.readFileSync(filePath, "utf-8"),
+    }));
+
+    return ingestTextItems({
+      wikiRoot,
+      source,
+      title: options.title,
+      tags: options.tags,
+      project: options.project,
+      items,
+      initialRejections: collected.rejections,
+    });
+  }
+
+  return failResult({
+    wikiRoot,
+    source,
+    error: "Input path must be a regular file or directory",
+    rejections: [rejection(options.inputPath, "not-file-or-directory", "Input path must be a regular file or directory")],
+  });
+}