oh-my-llmwikimode 1.0.0

package/src/wiki/source-ledger.js

@@ -0,0 +1,370 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { redactSecrets } from "../security.js";
+
+const DEFAULT_IMPORTED_AT = "1970-01-01T00:00:00.000Z";
+const LEDGER_RELATIVE_PATH = "raw/sources/ledger.jsonl";
+const SOURCE_FILES_RELATIVE_ROOT = "raw/sources/files";
+const AUDIT_RELATIVE_PATH = ".system/source-ledger/audit.jsonl";
+
+function result(success, payload = {}) {
+  return { success, ...payload };
+}
+
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function nowIso(options = {}) {
+  return options.now || new Date().toISOString();
+}
+
+function compareStrings(left, right) {
+  return String(left ?? "").localeCompare(String(right ?? ""));
+}
+
+function normalizeScalar(value, maxLength = 500) {
+  return redactSecrets(String(value ?? "")
+    .replace(/\r?\n/g, " ")
+    .replace(/\s+/g, " ")
+    .trim())
+    .slice(0, maxLength);
+}
+
+function normalizeId(value, label = "id") {
+  const id = normalizeScalar(value, 120).toLowerCase();
+  if (!/^[a-z0-9][a-z0-9_-]{0,119}$/.test(id)) {
+    throw new Error(`Invalid ${label}: ${value}`);
+  }
+  return id;
+}
+
+function normalizeToken(value, fallback, label) {
+  const token = normalizeScalar(value || fallback, 120).toLowerCase();
+  if (!token) throw new Error(`${label} is required`);
+  return token;
+}
+
+function normalizeContentHash(value) {
+  const hash = normalizeScalar(value, 80).toLowerCase();
+  if (!/^[a-f0-9]{64}$/.test(hash)) {
+    throw new Error("content_sha256 is required");
+  }
+  return hash;
+}
+
+function normalizeStringList(value, maxLength = 260) {
+  const values = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
+  return [...new Set(values.map((item) => normalizeScalar(item, maxLength)).filter(Boolean))]
+    .sort(compareStrings);
+}
+
+function normalizeBooleanFalse(value, label) {
+  if (value === undefined || value === null || value === false) return false;
+  throw new Error(`${label} must remain false`);
+}
+
+function isAbsolutePath(value) {
+  return path.isAbsolute(value) || path.win32.isAbsolute(value) || path.posix.isAbsolute(value);
+}
+
+function isInsideOrEqual(parentDirectory, candidatePath) {
+  const relativePath = path.relative(parentDirectory, candidatePath);
+  return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
+}
+
+function toSlashPath(value) {
+  return value.replace(/\\/g, "/");
+}
+
+function normalizeWikiRelativePath(wikiRoot, value, label = "path", maxLength = 500) {
+  const rawPath = normalizeScalar(value, maxLength);
+  if (!rawPath) throw new Error(`Unsafe ${label}: path is required`);
+
+  if (/^[a-z]:($|[^/\\])/i.test(rawPath)) {
+    throw new Error(`Unsafe ${label}: drive-qualified paths are not allowed`);
+  }
+
+  if (isAbsolutePath(rawPath)) {
+    const absolutePath = path.resolve(rawPath);
+    const absoluteRoot = path.resolve(wikiRoot);
+    if (!isInsideOrEqual(absoluteRoot, absolutePath)) {
+      throw new Error(`Unsafe ${label}: absolute path is outside wiki root`);
+    }
+    const relativePath = toSlashPath(path.relative(absoluteRoot, absolutePath));
+    if (!relativePath) throw new Error(`Unsafe ${label}: path is required`);
+    if (relativePath.split("/").some((segment) => segment === "..")) {
+      throw new Error(`Unsafe ${label}: traversal is not allowed`);
+    }
+    return relativePath;
+  }
+
+  const normalized = toSlashPath(rawPath);
+  if (normalized.startsWith("/") || normalized.startsWith("//")) {
+    throw new Error(`Unsafe ${label}: absolute path is outside wiki root`);
+  }
+  if (normalized.split("/").some((segment) => segment === "..")) {
+    throw new Error(`Unsafe ${label}: traversal is not allowed`);
+  }
+
+  const resolved = path.resolve(wikiRoot, normalized);
+  if (!isInsideOrEqual(path.resolve(wikiRoot), resolved)) {
+    throw new Error(`Unsafe ${label}: path is outside wiki root`);
+  }
+  return normalized;
+}
+
+function normalizeStoredPath(wikiRoot, sourceId, value) {
+  const storedPath = normalizeWikiRelativePath(wikiRoot, value, "stored path");
+  const requiredPrefix = `${SOURCE_FILES_RELATIVE_ROOT}/${sourceId}/`;
+  if (!storedPath.startsWith(requiredPrefix)) {
+    throw new Error(`Unsafe stored path: must stay under ${requiredPrefix}`);
+  }
+  return storedPath;
+}
+
+function validatedJsonLine(value) {
+  const line = JSON.stringify(value);
+  JSON.parse(line);
+  return `${line}\n`;
+}
+
+function validationResult(builder) {
+  try {
+    return { valid: true, value: builder(), errors: [] };
+  } catch (error) {
+    return { valid: false, value: null, errors: [error.message] };
+  }
+}
+
+function validateSourceLedgerAuditRecord(wikiRoot, input) {
+  return validationResult(() => {
+    if (!isRecord(input)) throw new Error("Source ledger audit record must be an object");
+    const action = normalizeToken(input.action, "", "Source ledger audit action");
+    return {
+      timestamp: normalizeScalar(input.timestamp || DEFAULT_IMPORTED_AT, 80),
+      action,
+      actor: normalizeScalar(input.actor || "system", 120),
+      subject_id: normalizeId(input.subject_id || input.subjectId, "audit subject id"),
+      artifact_path: normalizeWikiRelativePath(
+        wikiRoot,
+        input.artifact_path || input.artifactPath || LEDGER_RELATIVE_PATH,
+        "artifact path"
+      ),
+      source_paths: normalizeStringList(input.source_paths || input.sourcePaths, 500)
+        .map((item) => normalizeWikiRelativePath(wikiRoot, item, "source path")),
+      executed: normalizeBooleanFalse(input.executed, "executed"),
+    };
+  });
+}
+
+export function getSourceLedgerPaths(wikiRoot, sourceId = "") {
+  const rawSourcesDir = path.join(wikiRoot, "raw", "sources");
+  const filesRoot = path.join(rawSourcesDir, "files");
+  const safeSourceId = sourceId ? normalizeId(sourceId, "source id") : "";
+  return {
+    root: wikiRoot,
+    rawSourcesDir,
+    ledgerFile: path.join(wikiRoot, ...LEDGER_RELATIVE_PATH.split("/")),
+    filesRoot,
+    sourceFilesDir: safeSourceId ? path.join(filesRoot, safeSourceId) : filesRoot,
+    systemDir: path.join(wikiRoot, ".system"),
+    auditDir: path.join(wikiRoot, ".system", "source-ledger"),
+    auditFile: path.join(wikiRoot, ...AUDIT_RELATIVE_PATH.split("/")),
+  };
+}
+
+export function normalizeSourceRecord(wikiRoot, input, options = {}) {
+  if (!isRecord(input)) throw new Error("Source ledger record must be an object");
+  const sourceId = normalizeId(input.source_id || input.sourceId, "source id");
+  const sourcePath = normalizeWikiRelativePath(wikiRoot, input.source_path || input.sourcePath, "source path");
+  const title = normalizeScalar(input.title || sourcePath, 180);
+  if (!title) throw new Error("Source ledger title is required");
+
+  return {
+    version: 1,
+    source_id: sourceId,
+    kind: normalizeToken(input.kind, "unknown", "kind"),
+    title,
+    source_path: sourcePath,
+    stored_path: normalizeStoredPath(wikiRoot, sourceId, input.stored_path || input.storedPath),
+    content_sha256: normalizeContentHash(input.content_sha256 || input.contentSha256),
+    imported_at: normalizeScalar(input.imported_at || input.importedAt || nowIso(options), 80),
+    visibility: normalizeToken(input.visibility, "local", "visibility"),
+    redaction: normalizeToken(input.redaction, "metadata_only", "redaction"),
+    creates_wiki_entry: false,
+  };
+}
+
+export function appendSourceRecord(wikiRoot, input, options = {}) {
+  try {
+    const record = normalizeSourceRecord(wikiRoot, input, options);
+    const paths = getSourceLedgerPaths(wikiRoot, record.source_id);
+    fs.mkdirSync(path.dirname(paths.ledgerFile), { recursive: true });
+    fs.mkdirSync(paths.sourceFilesDir, { recursive: true });
+    fs.appendFileSync(paths.ledgerFile, validatedJsonLine(record), { encoding: "utf-8", flag: "a" });
+    return result(true, {
+      record,
+      path: LEDGER_RELATIVE_PATH,
+      source_files_dir: `${SOURCE_FILES_RELATIVE_ROOT}/${record.source_id}`
+    });
+  } catch (error) {
+    return result(false, { error: error.message });
+  }
+}
+
+export function readSourceRecords(wikiRoot) {
+  try {
+    const paths = getSourceLedgerPaths(wikiRoot);
+    if (!fs.existsSync(paths.ledgerFile)) return result(true, { records: [] });
+    const lines = fs.readFileSync(paths.ledgerFile, "utf-8")
+      .split(/\r?\n/)
+      .filter((line) => line.trim());
+    const records = lines.map((line) => normalizeSourceRecord(wikiRoot, JSON.parse(line)));
+    return result(true, { records });
+  } catch (error) {
+    return result(false, { error: error.message, records: [] });
+  }
+}
+
+export function appendSourceLedgerAudit(wikiRoot, record, options = {}) {
+  const validation = validateSourceLedgerAuditRecord(wikiRoot, {
+    timestamp: nowIso(options),
+    actor: options.actor || "system",
+    executed: false,
+    ...record,
+  });
+  if (!validation.valid) return result(false, { error: validation.errors.join("; ") });
+  const paths = getSourceLedgerPaths(wikiRoot);
+  fs.mkdirSync(paths.auditDir, { recursive: true });
+  fs.appendFileSync(paths.auditFile, validatedJsonLine(validation.value), { encoding: "utf-8", flag: "a" });
+  return result(true, { audit: validation.value });
+}
+
+const TEXT_EXTENSIONS = new Set([
+  ".txt", ".md", ".markdown", ".json", ".js", ".ts", ".jsx", ".tsx",
+  ".css", ".scss", ".less", ".html", ".htm", ".xml", ".yaml", ".yml",
+  ".csv", ".log", ".ini", ".conf", ".cfg", ".sh", ".bash", ".ps1",
+  ".py", ".rb", ".go", ".rs", ".java", ".c", ".cpp", ".h", ".hpp",
+  ".php", ".sql", ".r", ".swift", ".kt", ".scala", ".clj", ".ex",
+  ".exs", ".elm", ".erl", ".hrl", ".fs", ".fsx", ".gd", ".gdscript",
+  ".nim", ".nims", ".ml", ".mli", ".opam", ".re", ".rei", ".v",
+  ".vhd", ".vhdl", ".zig", ".wasm", ".wat", ".coffee", ".litcoffee",
+  ".dart", ".flutter", ".groovy", ".gvy", ".gy", ".gsh", ".julia",
+  ".jl", ".lua", ".moon", ".nuspec", ".purs", ".purescript", ".rkt",
+  ".scheme", ".scm", ".ss", ".sls", ".sld", ".sps", ".st", ".smalltalk",
+  ".tcl", ".tk", ".wasm", ".wat", ".wren", ".xquery", ".xq", ".xsl",
+  ".xslt", ".axd", ".coffee", ".litcoffee", ".iced", ".cjsx",
+]);
+
+function isTextFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext) return TEXT_EXTENSIONS.has(ext);
+  // If no extension, try to read a small sample
+  try {
+    const sample = fs.readFileSync(filePath, "utf-8").slice(0, 1024);
+    // If it contains null bytes, it's likely binary
+    return !sample.includes("\0");
+  } catch {
+    return false;
+  }
+}
+
+export function importSourceFile(wikiRoot, filePath, options = {}) {
+  try {
+    const absPath = path.resolve(String(filePath || "").trim());
+    if (!absPath || !fs.existsSync(absPath)) {
+      return result(false, { error: `Source file not found: ${filePath}` });
+    }
+
+    const filename = path.basename(absPath);
+    const isText = isTextFile(absPath);
+    const importedAt = nowIso(options);
+    const sourceIdInput = options.source_id || options.sourceId || filename.replace(/\.[^.]+$/, "").replace(/[^a-zA-Z0-9_-]/g, "_").toLowerCase();
+    const sourceId = normalizeId(sourceIdInput, "source id");
+    const paths = getSourceLedgerPaths(wikiRoot, sourceId);
+    const title = options.title || filename;
+
+    let contentSha256;
+    let storedRel;
+    let redactionType;
+    let hasContentFile = false;
+
+    if (isText) {
+      const content = fs.readFileSync(absPath, "utf-8");
+      contentSha256 = crypto.createHash("sha256").update(content, "utf-8").digest("hex");
+      const redactedContent = redactSecrets(content);
+      redactionType = content !== redactedContent ? "content_redacted" : "content_clean";
+      storedRel = ["raw", "sources", "files", sourceId, "content.txt"].join("/");
+      const storedAbs = path.join(wikiRoot, storedRel);
+      fs.mkdirSync(path.dirname(storedAbs), { recursive: true });
+      fs.writeFileSync(storedAbs, redactedContent, "utf-8");
+      hasContentFile = true;
+    } else {
+      // Non-text: metadata only, compute hash from binary
+      const buffer = fs.readFileSync(absPath);
+      contentSha256 = crypto.createHash("sha256").update(buffer).digest("hex");
+      redactionType = "metadata_only";
+      storedRel = ["raw", "sources", "files", sourceId, "manifest.json"].join("/");
+    }
+
+    const manifest = {
+      source_id: sourceId,
+      original_filename: filename,
+      stored_path: storedRel,
+      content_sha256: contentSha256,
+      imported_at: importedAt,
+      kind: options.kind || (isText ? "document" : "binary"),
+      title: title,
+      size_bytes: fs.statSync(absPath).size,
+      is_text: isText,
+      redaction: redactionType,
+    };
+
+    const manifestPath = path.join(paths.sourceFilesDir, "manifest.json");
+    fs.mkdirSync(paths.sourceFilesDir, { recursive: true });
+    fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
+
+    const ledgerResult = appendSourceRecord(wikiRoot, {
+      source_id: sourceId,
+      kind: manifest.kind,
+      title: title,
+      source_path: storedRel,
+      stored_path: storedRel,
+      content_sha256: contentSha256,
+      imported_at: importedAt,
+      visibility: options.visibility || "local",
+      redaction: redactionType,
+    }, { now: importedAt });
+
+    if (!ledgerResult.success) {
+      return result(false, { error: "Ledger append failed: " + ledgerResult.error });
+    }
+
+    const auditResult = appendSourceLedgerAudit(wikiRoot, {
+      action: "import_source_file",
+      subject_id: sourceId,
+      artifact_path: storedRel,
+      source_paths: [storedRel],
+    }, { actor: options.actor || "cli", now: importedAt });
+
+    if (!auditResult.success) {
+      return result(false, { error: "Audit append failed: " + auditResult.error });
+    }
+
+    return result(true, {
+      source_id: sourceId,
+      filename: filename,
+      stored_path: storedRel,
+      manifest_path: ["raw", "sources", "files", sourceId, "manifest.json"].join("/"),
+      ledger_path: LEDGER_RELATIVE_PATH,
+      audit_path: AUDIT_RELATIVE_PATH,
+      content_sha256: contentSha256,
+      is_text: isText,
+      redacted: redactionType === "content_redacted",
+    });
+  } catch (error) {
+    return result(false, { error: error.message });
+  }
+}