logseq-graph-living-atlas 0.1.0

package/server/logseq/parser.mjs

@@ -0,0 +1,121 @@
+import path from "node:path";
+
+const WIKILINK_RE = /\[\[([^\]]+?)\]\]/g;
+const PROP_RE = /^([a-zA-Z][\w-]*?)::\s*(.*?)\s*$/;
+const TYPED_LINK_RE = /^\s*(?:[-*]\s*)?(?:\*\*)?([a-zA-Z][\w\s-]{1,40}?):(?:\*\*)?\s*\[\[([^\]]+?)\]\]/gm;
+const CODE_FENCE_RE = /```[\s\S]*?```/g;
+const INLINE_CODE_RE = /`[^`]*`/g;
+
+export function slugify(name) {
+  return String(name || "")
+    .normalize("NFC")
+    .trim()
+    .replace(/\.md$/i, "")
+    .replace(/\//g, "___")
+    .toLowerCase();
+}
+
+export function displayNameFromFile(filePath, root = "") {
+  const normalizedRoot = root ? path.resolve(root) : "";
+  const normalizedFile = path.resolve(filePath);
+  if (!normalizedRoot) return path.basename(filePath, ".md").replace(/___/g, "/");
+  const relative = path.relative(normalizedRoot, normalizedFile);
+  const parts = relative.split(path.sep);
+  if (parts[0] === "pages" || parts[0] === "journals") parts.shift();
+  return parts.join("/").replace(/\.md$/i, "").replace(/___/g, "/");
+}
+
+export function stripCode(text) {
+  return String(text || "").replace(CODE_FENCE_RE, "").replace(INLINE_CODE_RE, "");
+}
+
+export function extractWikilinks(text) {
+  const clean = stripCode(text);
+  const links = new Set();
+  for (const match of clean.matchAll(WIKILINK_RE)) {
+    const target = match[1].trim();
+    if (!target || /^https?:/i.test(target)) continue;
+    links.add(slugify(target));
+  }
+  return [...links];
+}
+
+export function parseProperties(text) {
+  const props = {};
+  for (const line of String(text || "").split(/\r?\n/)) {
+    const match = line.match(PROP_RE);
+    if (!match) continue;
+    props[match[1].toLowerCase()] = match[2].trim();
+  }
+  return props;
+}
+
+export function parsePageRecord(filePath, text, stat = undefined, options = {}) {
+  const root = typeof options === "string" ? options : options.root || "";
+  const name = displayNameFromFile(filePath, root);
+  const props = parseProperties(text);
+  const type = props.type || "note";
+  const tags = parseTags(props.tags || "");
+  const out = extractWikilinks(text).filter((target) => target !== slugify(name));
+  const mtimeMs = stat?.mtimeMs ?? Date.now();
+  const relations = extractTypedRelations(text, props);
+  return {
+    id: slugify(name),
+    name,
+    path: filePath,
+    type,
+    tags,
+    status: props.status || "",
+    source: props.source || "",
+    confidence: props.confidence || "",
+    lastContacted: props["last-contacted"] || "",
+    updatedAt: new Date(mtimeMs).toISOString(),
+    mtimeMs,
+    out,
+    relations,
+    props
+  };
+}
+
+export function extractTypedRelations(text, props = {}) {
+  const relations = [];
+  for (const [key, value] of Object.entries(props || {})) {
+    const links = extractWikilinks(String(value || ""));
+    for (const target of links) {
+      relations.push({ kind: normalizeRelationKind(key), target, evidence: `${key}:: ${value}` });
+    }
+  }
+  const clean = stripCode(text);
+  for (const match of clean.matchAll(TYPED_LINK_RE)) {
+    const kind = normalizeRelationKind(match[1]);
+    const target = slugify(match[2]);
+    if (!kind || !target) continue;
+    relations.push({ kind, target, evidence: match[0].trim() });
+  }
+  const seen = new Set();
+  return relations.filter((relation) => {
+    const key = `${relation.kind}:${relation.target}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+export function parseTags(raw) {
+  if (!raw) return [];
+  const tags = [];
+  for (const wikilink of raw.matchAll(WIKILINK_RE)) tags.push(wikilink[1].trim());
+  const cleaned = raw.replace(WIKILINK_RE, " ");
+  for (const token of cleaned.split(/[\s,]+/)) {
+    const value = token.trim();
+    if (value) tags.push(value);
+  }
+  return [...new Set(tags)];
+}
+
+function normalizeRelationKind(value) {
+  return String(value || "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .toLowerCase();
+}

package/server/logseq/source-adapter.mjs

@@ -0,0 +1,147 @@
+import fs from "node:fs";
+import crypto from "node:crypto";
+import os from "node:os";
+import path from "node:path";
+import { parsePageRecord } from "./parser.mjs";
+
+export const DEFAULT_LOGSEQ_ROOT = process.env.LOGSEQ_ROOT || "";
+const LOGSEQ_SOURCE_DIRS = ["pages", "journals"];
+const DEFAULT_MAX_MARKDOWN_FILES = 250000;
+const DEFAULT_MAX_MARKDOWN_FILE_BYTES = 2 * 1024 * 1024;
+
+export function createLogseqSourceAdapter(root = DEFAULT_LOGSEQ_ROOT) {
+  return {
+    kind: "logseq-markdown",
+    root,
+    readManifest: () => readGraphManifest(root),
+    readRecords: () => readPageRecords(root),
+    watchDirectories: () => LOGSEQ_SOURCE_DIRS
+      .map((sourceDir) => ({ sourceDir, path: path.join(root, sourceDir) }))
+      .filter((entry) => fs.existsSync(entry.path))
+  };
+}
+
+export function readPageRecords(root = DEFAULT_LOGSEQ_ROOT) {
+  if (!root) throw new Error("LOGSEQ_ROOT or --root is required. Point it at a Logseq graph folder containing pages/.");
+  const records = [];
+  for (const filePath of readLogseqMarkdownFiles(root)) {
+    const stat = fs.statSync(filePath);
+    const text = fs.readFileSync(filePath, "utf8");
+    records.push(parsePageRecord(filePath, text, stat, { root }));
+  }
+  assertUniqueRecordIds(records);
+  records.sort((a, b) => a.name.localeCompare(b.name));
+  return records;
+}
+
+export function readGraphManifest(root = DEFAULT_LOGSEQ_ROOT) {
+  if (!root) throw new Error("LOGSEQ_ROOT or --root is required. Point it at a Logseq graph folder containing pages/.");
+  const files = [];
+  for (const filePath of readLogseqMarkdownFiles(root)) {
+    const stat = fs.statSync(filePath);
+    files.push({
+      name: path.relative(root, filePath),
+      size: stat.size,
+      mtimeMs: Math.round(stat.mtimeMs),
+      contentHash: fileContentHash(filePath)
+    });
+  }
+  files.sort((a, b) => a.name.localeCompare(b.name));
+  const fingerprint = crypto.createHash("sha256").update(JSON.stringify(files)).digest("hex");
+  return {
+    pages: files.length,
+    graphId: graphIdentity(root),
+    fingerprint,
+    maxMtimeMs: Math.max(0, ...files.map((file) => file.mtimeMs))
+  };
+}
+
+export function readLogseqMarkdownFiles(root = DEFAULT_LOGSEQ_ROOT) {
+  if (!root) throw new Error("LOGSEQ_ROOT or --root is required. Point it at a Logseq graph folder containing pages/.");
+  const pagesDir = path.join(root, "pages");
+  if (!fs.existsSync(pagesDir)) {
+    throw new Error("Logseq pages directory not found under the configured graph root.");
+  }
+  const files = [];
+  for (const sourceDir of LOGSEQ_SOURCE_DIRS) {
+    const directory = path.join(root, sourceDir);
+    if (!fs.existsSync(directory)) continue;
+    collectMarkdownFiles(directory, files, root);
+  }
+  const maxFiles = positiveInteger(process.env.LIVING_ATLAS_MAX_FILES, DEFAULT_MAX_MARKDOWN_FILES);
+  if (files.length > maxFiles) {
+    throw new Error(`Logseq graph has ${files.length} markdown files, above LIVING_ATLAS_MAX_FILES=${maxFiles}.`);
+  }
+  files.sort((a, b) => path.relative(root, a).localeCompare(path.relative(root, b)));
+  return files;
+}
+
+function collectMarkdownFiles(directory, files, root) {
+  for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+    const filePath = path.join(directory, entry.name);
+    if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
+    if (entry.isDirectory()) {
+      collectMarkdownFiles(filePath, files, root);
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".md")) {
+      const maxBytes = positiveInteger(process.env.LIVING_ATLAS_MAX_FILE_BYTES, DEFAULT_MAX_MARKDOWN_FILE_BYTES);
+      const stat = fs.statSync(filePath);
+      if (stat.size > maxBytes) {
+        throw new Error(`Logseq markdown file is too large for indexing: ${path.relative(root, filePath)}. Set LIVING_ATLAS_MAX_FILE_BYTES to override or use --debug-paths for full diagnostics.`);
+      }
+      files.push(filePath);
+    }
+  }
+}
+
+function assertUniqueRecordIds(records) {
+  const seen = new Map();
+  const duplicates = [];
+  for (const record of records) {
+    const previous = seen.get(record.id);
+    if (previous) duplicates.push(`${previous.name} <-> ${record.name}`);
+    seen.set(record.id, record);
+  }
+  if (duplicates.length) {
+    throw new Error(`Duplicate Logseq page identities found: ${duplicates.slice(0, 5).join(", ")}. Rename or merge duplicate namespace pages before indexing.`);
+  }
+}
+
+function positiveInteger(value, fallback) {
+  const parsed = Number(value || 0);
+  if (Number.isFinite(parsed) && parsed > 0) return Math.floor(parsed);
+  return fallback;
+}
+
+function graphIdentity(root) {
+  const realRoot = fs.realpathSync(root);
+  const stat = fs.statSync(realRoot);
+  return crypto
+    .createHmac("sha256", installSecret())
+    .update(`${realRoot}:${stat.dev}:${stat.ino}`)
+    .digest("hex")
+    .slice(0, 32);
+}
+
+function installSecret() {
+  const secretPath = path.join(defaultCacheDirectory(), "install-secret");
+  if (fs.existsSync(secretPath)) return fs.readFileSync(secretPath, "utf8").trim();
+  fs.mkdirSync(path.dirname(secretPath), { recursive: true, mode: 0o700 });
+  const installKey = crypto.randomBytes(32).toString("hex");
+  fs.writeFileSync(secretPath, installKey, { mode: 0o600 });
+  return installKey;
+}
+
+function defaultCacheDirectory() {
+  if (process.env.XDG_CACHE_HOME) return path.join(process.env.XDG_CACHE_HOME, "logseq-graph-living-atlas");
+  if (process.platform === "darwin") return path.join(os.homedir(), "Library", "Caches", "logseq-graph-living-atlas");
+  if (process.platform === "win32") {
+    return path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), "AppData", "Local"), "logseq-graph-living-atlas");
+  }
+  return path.join(os.homedir(), ".cache", "logseq-graph-living-atlas");
+}
+
+function fileContentHash(filePath) {
+  return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
+}

package/server/redaction.mjs

@@ -0,0 +1,89 @@
+const SENSITIVE_KEYS = new Set([
+  "id",
+  "name",
+  "label",
+  "title",
+  "detail",
+  "target",
+  "source",
+  "cluster",
+  "clusterLabel",
+  "nodeId",
+  "nodeIds",
+  "graphId",
+  "fingerprint",
+  "nodeName",
+  "sourceId",
+  "targetId",
+  "linkId",
+  "tags",
+  "relativePath",
+  "preview",
+  "query",
+  "evidence",
+  "rationale",
+  "nextStep"
+]);
+
+const PRESERVED_KEYS = new Set([
+  "type",
+  "kind",
+  "severity",
+  "status",
+  "confidence",
+  "updatedAt",
+  "generatedAt",
+  "observedAt",
+  "actor",
+  "reason",
+  "ok",
+  "version"
+]);
+
+export function redactPayload(payload) {
+  const labels = new Map();
+  let next = 0;
+  const labelFor = (value, parentKey = "") => {
+    if (!labels.has(value)) {
+      next += 1;
+      const prefix = parentKey.toLowerCase().includes("path") ? "path" : "entity";
+      labels.set(value, `${prefix}-${String(next).padStart(4, "0")}`);
+    }
+    return labels.get(value);
+  };
+
+  const visit = (value, parentKey = "") => {
+    if (Array.isArray(value)) return value.map((item) => visit(item, parentKey));
+    if (!value || typeof value !== "object") {
+      if (typeof value === "string" && SENSITIVE_KEYS.has(parentKey)) return labelFor(value, parentKey);
+      return value;
+    }
+    const output = {};
+    for (const [key, item] of Object.entries(value)) {
+      if (PRESERVED_KEYS.has(key)) {
+        output[key] = item;
+      } else if (SENSITIVE_KEYS.has(key)) {
+        output[key] = typeof item === "string" ? labelFor(item, key) : visit(item, key);
+      } else if (key === "properties" || key === "provenance" || key === "action") {
+        output[key] = redactPropertyBag(item, labelFor);
+      } else {
+        output[key] = visit(item, key);
+      }
+    }
+    return output;
+  };
+
+  return visit(payload);
+}
+
+function redactPropertyBag(value, labelFor) {
+  if (Array.isArray(value)) return value.map((item) => redactPropertyBag(item, labelFor));
+  if (!value || typeof value !== "object") {
+    return typeof value === "string" ? labelFor(value) : value;
+  }
+  const output = {};
+  for (const [key, item] of Object.entries(value)) {
+    output[key] = redactPropertyBag(item, labelFor);
+  }
+  return output;
+}