@staix/vault-mem 0.2.2 → 0.2.4

package/src/workspace-graph.ts

@@ -0,0 +1,139 @@
+import { AnalysisYamlScalar, parseAnalysisYaml, yamlTruthy } from "./analysis-yaml.ts";
+import type { AnalysisSnapshot } from "./analysis-types.ts";
+import { buildGraph, finalizeGraph, pythonTitle, pyString, type GraphNode, type GraphEdge, type VaultGraph } from "./vault-graph.ts";
+import { posix } from "node:path";
+
+export interface WorkspaceSnapshot {
+  files: { path: string; text: string }[];
+  buSlugs: string[];
+  skipRoots: string[];
+  assertUnchanged?: () => void;
+  sourceTargets: Record<string, "file" | "directory" | "missing" | "denied">;
+}
+export const WORK_SKIP_DIRS = new Set([".git", "node_modules", ".venv", ".sync-conflicts", "temp", ".maru", ".pnpm-store", ".conductor", ".dotfiles", ".omx", ".omc", ".gstack", ".frontend-slides", ".claude", ".archive", "archive", "vault", "inbox", "sites", "dev", "shared", "_templates", "templates"]);
+const FIELDS = ["projects", "project", "vault_note", "topics", "attendees", "relatedMeetings", "relatedTasks"];
+const FRONTMATTER = /^---\s*\n([\s\S]*?)\n---\s*\n/;
+const WIKI = /\[\[([^\]|]+?)(?:\|[^\]]+?)?\]\]/g;
+const PRIORITY = { wiki_link: 0, fm_ref: 1, bu_member: 2, source_ref: 3, related: 4, supersedes: 5 };
+export type WorkCounters = Record<"work_docs" | "yaml_fail" | "template_skip" | "related_unresolved" | "related_ambiguous" | "related_dir_skip" | "related_root_escape" | "related_home_escape" | "source_stub" | "source_dead" | "source_nonmd" | "source_skiproot" | "collisions" | "dup_edges", number>;
+export function workSkipped(path: string, roots: string[]): boolean {
+  const normalized = path.normalize("NFC").replace(/^\/+|\/+$/g, "");
+  return roots.some(root => normalized === root || normalized.startsWith(root + "/"));
+}
+function mapping(value: unknown): value is Record<string, unknown> { return value !== null && typeof value === "object" && !Array.isArray(value) && !(value instanceof AnalysisYamlScalar); }
+export function workspaceFrontmatter(text: string): Record<string, unknown> {
+  const match = FRONTMATTER.exec(text);
+  if (!match) return {};
+  try { const value = parseAnalysisYaml(match[1]!); return mapping(value) ? value : {}; }
+  catch { return {}; } // The reference skips rejected YAML and counts work-doc failures.
+}
+function links(value: unknown): string[] { return [...pyString(value).matchAll(WIKI)].map(match => match[1]!.trim().normalize("NFC").toLowerCase().replaceAll(" ", "-")); }
+function coerce(value: unknown): string {
+  return value instanceof AnalysisYamlScalar && value.kind === "timestamp" ? value.text.replace(" ", "T") : pyString(value);
+}
+function label(path: string): string {
+  let stem = posix.basename(path, ".md").normalize("NFC");
+  const parent = posix.basename(posix.dirname(path));
+  if (["readme", "index"].includes(stem.toLowerCase()) && parent !== ".") stem = parent + "-" + stem;
+  return pythonTitle(stem.replaceAll("-", " "));
+}
+/** Reference source spelling: keep its ID spelling, including path segments, rather than silently remap it. */
+export function workspaceSource(fm: Record<string, unknown>): string | undefined {
+  if (typeof fm.source !== "string" || !fm.source.trim()) return undefined;
+  let raw = fm.source.trim().replace(/^"+|"+$/g, "").replace(/^'+|'+$/g, "");
+  if (!raw || raw.startsWith("http://") || raw.startsWith("https://")) return undefined;
+  if (raw.startsWith("work/")) raw = raw.slice(5);
+  return raw.normalize("NFC");
+}
+export function buildWorkspaceGraph(snapshot: AnalysisSnapshot, work: WorkspaceSnapshot): { graph: VaultGraph; counters: WorkCounters; relations: Record<string, number> } {
+  const graph = buildGraph(snapshot);
+  const counters: WorkCounters = { work_docs: 0, yaml_fail: 0, template_skip: 0, related_unresolved: 0, related_ambiguous: 0, related_dir_skip: 0, related_root_escape: 0, related_home_escape: 0, source_stub: 0, source_dead: 0, source_nonmd: 0, source_skiproot: 0, collisions: 0, dup_edges: 0 };
+  const vaultIds = new Set(graph.nodes.map(node => node.id));
+  const docs = new Map<string, { path: string; fm: Record<string, unknown>; body: string }>(), stems = new Map<string, string[]>();
+  for (const file of work.files) {
+    const match = FRONTMATTER.exec(file.text);
+    if (!match) continue;
+    if (match[1]!.includes("{{")) { counters.template_skip++; continue; }
+    const fm = workspaceFrontmatter(file.text);
+    if (!Object.keys(fm).length) { if (match[1]!.trim()) counters.yaml_fail++; continue; }
+    if (!yamlTruthy(fm.document_type) && !FIELDS.some(field => yamlTruthy(fm[field])) && !yamlTruthy(fm.related)) continue;
+    const path = file.path.normalize("NFC"), id = "work:" + path.slice(0, -3);
+    docs.set(id, { path, fm, body: file.text.slice(match[0].length) });
+    const key = posix.basename(path, ".md").toLowerCase().replaceAll(" ", "-");
+    stems.set(key, [...stems.get(key) ?? [], id]); counters.work_docs++;
+  }
+  const validBu = new Set(work.buSlugs), usedBu = new Set<string>();
+  const businessUnit = (fm: Record<string, unknown>) => pyString(fm.business_unit ?? "").replaceAll("[[", "").replaceAll("]]", "").trim();
+  for (const doc of docs.values()) { const bu = businessUnit(doc.fm); if (validBu.has(bu)) usedBu.add(bu); }
+  const node = (value: Omit<GraphNode, "degree" | "community">): GraphNode => ({ ...value, degree: 0, community: null });
+  for (const slug of [...usedBu].sort()) graph.nodes.push(node({ id: "bu:" + slug, type: "bu", domain: "bu", label: slug, source_file: "" }));
+  for (const [id, doc] of docs) graph.nodes.push(node({ id, label: label(doc.path), type: coerce(yamlTruthy(doc.fm.document_type) ? doc.fm.document_type : "work-doc"), domain: doc.path.split("/")[0]!, description: Array.from(coerce(yamlTruthy(doc.fm.title) ? doc.fm.title : yamlTruthy(doc.fm.description) ? doc.fm.description : "")).slice(0, 200).join(""), source_file: doc.path }));
+  if (graph.nodes.length > 20000) throw new Error("workspace graph: exceeds 20000 nodes");
+  const candidates: GraphEdge[] = [];
+  const add = (relation: GraphEdge["relation"], source: string, target: string, attrs: { rel?: string; field?: string } = {}) => {
+    if (candidates.length >= 200000) throw new Error("workspace graph: exceeds 200000 edge candidates");
+    candidates.push({ source, target, relation, confidence_tag: "EXTRACTED", confidence: 1, ...attrs });
+  };
+  for (const [id, doc] of docs) {
+    const related = doc.fm.related;
+    if (related !== null && related !== undefined) for (const entry of Array.isArray(related) ? related : [related]) {
+      const rtype = mapping(entry) ? entry.rel : undefined;
+      let value = pyString(mapping(entry) ? entry.doc ?? "" : entry).trim();
+      if (!value) continue;
+      value = value.replace(/\s*\([^)]*\)\s*$/, "").trim();
+      const wiki = links(value);
+      let target: string | undefined;
+      if (wiki.length) {
+        const targets = stems.get(wiki[0]!);
+        if (!targets?.length) counters.related_unresolved++;
+        else if (targets.length > 1) counters.related_ambiguous++;
+        else target = targets[0];
+      } else if (value.startsWith("~")) counters.related_home_escape++;
+      else {
+        const joined = posix.isAbsolute(value) ? posix.normalize(value) : posix.normalize(posix.join(posix.dirname(doc.path), value));
+        if (joined.startsWith("..")) counters.related_root_escape++;
+        else if (value.endsWith("/") || !joined.endsWith(".md")) counters.related_dir_skip++;
+        else if (docs.has("work:" + joined.slice(0, -3))) target = "work:" + joined.slice(0, -3);
+        else counters.related_unresolved++;
+      }
+      if (target) add("related", id, target, yamlTruthy(rtype) ? { rel: coerce(rtype) } : {});
+    }
+    for (const field of FIELDS) if (field in doc.fm) for (const key of links(doc.fm[field])) if (vaultIds.has(key)) add("fm_ref", id, key, { field });
+    for (const key of links(doc.body)) if (vaultIds.has(key)) add("wiki_link", id, key);
+    const bu = businessUnit(doc.fm); if (usedBu.has(bu)) add("bu_member", id, "bu:" + bu);
+  }
+  const stubs = new Set<string>();
+  for (const note of [...snapshot.notes].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) {
+    const fm = workspaceFrontmatter(note.source), id = posix.basename(note.path, ".md"), raw = workspaceSource(fm);
+    if (raw !== undefined) {
+      const ext = posix.extname(raw), norm = raw.endsWith(".md") ? raw.slice(0, -3) : raw, target = "work:" + norm;
+      if (raw.endsWith("/") || (ext && ext !== ".md")) counters.source_nonmd++;
+      else if (docs.has(target)) add("source_ref", id, target);
+      else if (norm.split("/").some(part => WORK_SKIP_DIRS.has(part)) || workSkipped(norm, work.skipRoots) || work.sourceTargets[norm] === "denied") counters.source_skiproot++;
+      else if (work.sourceTargets[norm] === "file") {
+        if (!stubs.has(target)) { stubs.add(target); counters.source_stub++; graph.nodes.push(node({ id: target, type: "file", label: label(norm + ".md"), domain: norm.split("/")[0]!, source_file: norm + ".md" })); }
+        add("source_ref", id, target);
+      } else if (work.sourceTargets[norm] === "directory") counters.source_nonmd++;
+      else counters.source_dead++;
+    }
+    for (const field of ["supersedes", "superseded_by"]) if (yamlTruthy(fm[field])) for (const key of links(fm[field])) if (vaultIds.has(key)) add("supersedes", id, key, { field });
+  }
+  const known = new Set(graph.nodes.map(node => node.id)), best = new Map<string, GraphEdge>();
+  const pair = (edge: GraphEdge) => JSON.stringify([edge.source, edge.target].sort());
+  for (const edge of candidates) {
+    if (edge.source === edge.target || !known.has(edge.source) || !known.has(edge.target)) continue;
+    const key = pair(edge), current = best.get(key);
+    if (current) {
+      if (current.relation !== edge.relation) counters.collisions++; else counters.dup_edges++;
+    }
+    if (!current || PRIORITY[edge.relation] > PRIORITY[current.relation]) best.set(key, edge);
+  }
+  const edges = new Map(graph.edges.map(edge => [pair(edge), edge]));
+  for (const [key, edge] of best) edges.set(key, { ...edges.get(key), ...edge });
+  graph.edges = [...edges.values()];
+  if (graph.nodes.length > 20000 || graph.edges.length > 100000) throw new Error("workspace graph: node or edge bound exceeded");
+  finalizeGraph(graph);
+  const relations: Record<string, number> = {};
+  for (const edge of graph.edges) relations[edge.relation] = (relations[edge.relation] ?? 0) + 1;
+  return { graph, counters, relations };
+}

package/src/workspace-input.ts

@@ -0,0 +1,150 @@
+import { closeSync, constants, fstatSync, lstatSync, opendirSync, openSync, readSync, realpathSync } from "node:fs";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { homedir } from "node:os";
+import type { VaultSpec, VmemConfig } from "./config.ts";
+import type { AnalysisSnapshot } from "./analysis-types.ts";
+import { analysisRead } from "./analysis-input.ts";
+import { parseAnalysisYaml } from "./analysis-yaml.ts";
+import { isDenied } from "./platform.ts";
+import { sourceWorkspaceRoot } from "./source.ts";
+import { WORK_SKIP_DIRS, workspaceFrontmatter, workspaceSource, workSkipped, type WorkspaceSnapshot } from "./workspace-graph.ts";
+
+const CONFIG_SKIP = new Set([".git", "node_modules", ".venv", ".sync-conflicts", "archive", ".archive", "vault", "temp"]);
+export const WORK_MAX_FILES = 50000;
+/** Per-file bound for work docs that open with frontmatter (the only ones that can become nodes). */
+export const WORK_MAX_FILE_BYTES = 2 * 1024 * 1024;
+export const WORK_MAX_TOTAL_BYTES = 128 * 1024 * 1024;
+/** Bytes probed from every `.md` file to decide whether it opens with frontmatter; plain files are never read further. */
+const PROBE_BYTES = 4;
+/** Directory entries visited per walk (symlinks included), so a runaway tree fails explicitly instead of stalling. */
+export const WORK_MAX_ENTRIES = 2_000_000;
+const DEFAULT_ROOTS = ["scratchpad/memos", "scratchpad/temp"];
+function object(value: unknown): Record<string, unknown> { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {}; }
+/** Filesystem-only snapshot. No directory or file symlinks are followed, including source stubs. */
+/** Test-only overrides of the reader bounds; production callers pass nothing and get the exported constants. */
+export interface WorkspaceLimits { files?: number; fileBytes?: number; bytes?: number; entries?: number }
+export function workspaceInput(snapshot: AnalysisSnapshot, vault: VaultSpec, config: VmemConfig, limits: WorkspaceLimits = {}): WorkspaceSnapshot {
+  const maxFiles = limits.files ?? WORK_MAX_FILES, maxFileBytes = limits.fileBytes ?? WORK_MAX_FILE_BYTES;
+  const maxBytes = limits.bytes ?? WORK_MAX_TOTAL_BYTES, maxEntries = limits.entries ?? WORK_MAX_ENTRIES;
+  const found = sourceWorkspaceRoot(vault);
+  if (!found) throw new Error("workspace graph: vault requires a registry");
+  const root = realpathSync(found);
+  if (root !== found || lstatSync(found).isSymbolicLink() || !lstatSync(found).isDirectory()) throw new Error("workspace graph: work root must be a plain directory");
+  const denied = (file: string) => isDenied(file, config.denyPaths ?? []);
+  if (denied(root)) throw new Error("workspace graph: work root is denied");
+  let filesRead = 0, bytesRead = 0, entriesSeen = 0;
+  const stamps = new Map<string, string>();
+  const stamp = (file: string) => { const stat = lstatSync(file, { bigint: true }); return [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs, stat.mode].join(":"); };
+  const remember = (file: string) => {
+    const current = stamp(file), prior = stamps.get(file);
+    if (prior !== undefined && prior !== current) throw new Error("workspace graph: input changed during snapshot");
+    stamps.set(file, current);
+  };
+  /**
+   * FRONTMATTER_RE needs `---` then whitespace at offset 0; a file that cannot match is skipped unread, as in the reference.
+   * Every probed file counts once toward WORK_MAX_FILES and is stamped, so one that later gains frontmatter fails assertUnchanged.
+   */
+  const probed = new Set<string>();
+  const opensWithFrontmatter = (file: string): boolean => {
+    if (++filesRead > maxFiles) throw new Error("workspace graph: file count exceeds bound");
+    probed.add(file);
+    remember(file);
+    let fd: number;
+    try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; }
+    try {
+      if (!fstatSync(fd).isFile()) return false;
+      const head = Buffer.alloc(PROBE_BYTES), n = readSync(fd, head, 0, PROBE_BYTES, 0);
+      return n === PROBE_BYTES && head.subarray(0, 3).toString("latin1") === "---" && /\s/.test(String.fromCharCode(head[3]!));
+    } finally { closeSync(fd); }
+  };
+  const read = (file: string, maximum = maxFileBytes): string => {
+    // A probed work doc was already counted; other reads (configs) count here.
+    if (!probed.has(file) && ++filesRead > maxFiles) throw new Error("workspace graph: file count exceeds bound");
+    remember(file);
+    const text = analysisRead(file, maximum);
+    if (text === null) throw new Error("workspace graph: input disappeared");
+    remember(file);
+    bytesRead += Buffer.byteLength(text);
+    if (bytesRead > maxBytes) throw new Error("workspace graph: total bytes exceed bound");
+    return text;
+  };
+  let skipRoots = DEFAULT_ROOTS;
+  const workspaceConfig = join(root, "workspace.config.yaml");
+  try {
+    const stat = lstatSync(workspaceConfig);
+    if (stat.isFile() && !stat.isSymbolicLink() && !denied(workspaceConfig)) {
+      const text = read(workspaceConfig, 1024 * 1024);
+      let data: Record<string, unknown> = {};
+      try { data = object(parseAnalysisYaml(text)); } catch { /* Reference falls back on malformed config. */ }
+      const paths = object(data.paths), scratchpad = object(data.scratchpad), configured = String(paths.scratchpad ?? "").trim();
+      let scratchRoot = "scratchpad";
+      if (configured) {
+        const expanded = configured === "~" ? homedir() : configured.startsWith("~/") ? join(homedir(), configured.slice(2)) : configured;
+        const absolute = resolve(expanded), rel = relative(root, absolute);
+        if (rel !== ".." && !rel.startsWith(".." + sep) && !isAbsolute(rel)) scratchRoot = rel.normalize("NFC").replace(/^\/+|\/+$/g, "");
+        else scratchRoot = "";
+      }
+      if (scratchRoot) skipRoots = [String(scratchpad.memos_subdir || "memos"), String(scratchpad.temp_subdir || "temp")].map(dir => scratchRoot + "/" + dir.normalize("NFC").replace(/^\/+|\/+$/g, ""));
+    }
+  } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+  const walk = (dir: string, skip: Set<string>, roots: string[], visit: (file: string, rel: string) => void): void => {
+    if (realpathSync(dir) !== dir || denied(dir)) return;
+    remember(dir);
+    const handle = opendirSync(dir), children: { name: string; directory: boolean; file: boolean }[] = [];
+    try {
+      for (;;) {
+        const entry = handle.readSync(); if (!entry) break;
+        if (++entriesSeen > maxEntries) throw new Error("workspace graph: directory entry count exceeds bound");
+        if (entry.isSymbolicLink()) continue;
+        children.push({ name: entry.name, directory: entry.isDirectory(), file: entry.isFile() });
+      }
+    } finally { handle.closeSync(); }
+    for (const entry of children) {
+      const file = join(dir, entry.name);
+      if (entry.file && !denied(file)) visit(file, relative(root, file).normalize("NFC"));
+    }
+    for (const entry of children) {
+      const file = join(dir, entry.name), rel = relative(root, file).normalize("NFC");
+      if (entry.directory && !skip.has(entry.name) && !workSkipped(rel, roots) && !denied(file)) walk(file, skip, roots, visit);
+    }
+    remember(dir);
+  };
+  const files: WorkspaceSnapshot["files"] = [], buSlugs = new Set<string>();
+  walk(root, WORK_SKIP_DIRS, skipRoots, (file, path) => { if (path.endsWith(".md") && opensWithFrontmatter(file)) files.push({ path, text: read(file) }); });
+  // The bu-config walk prunes fewer trees (it must see `.maru/`), like the reference's rglob, so it gets its own entry budget.
+  entriesSeen = 0;
+  walk(root, CONFIG_SKIP, [], (file, path) => {
+    if (!path.endsWith("/bu-config.yaml") && path !== "bu-config.yaml") return;
+    const text = read(file, 1024 * 1024);
+    try {
+      const data = object(parseAnalysisYaml(text)), slug = String(data.bu_id ?? "").replaceAll("[[", "").replaceAll("]]", "").trim();
+      if (/^[a-z0-9][a-z0-9-]*$/.test(slug)) buSlugs.add(slug);
+    } catch { /* Invalid bu config does not declare a hub. */ }
+  });
+  const plain = (candidate: string): "file" | "directory" | "missing" | "denied" => {
+    const rel = relative(root, candidate);
+    if (isAbsolute(rel) || rel === ".." || rel.startsWith(".." + sep)) return "missing";
+    let current = root;
+    for (const part of rel.split(sep)) {
+      current = join(current, part);
+      if (denied(current)) return "denied";
+      try { const stat = lstatSync(current); if (stat.isSymbolicLink()) return "missing"; remember(current); }
+      catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT" || (error as NodeJS.ErrnoException).code === "ENOTDIR") return "missing"; throw error; }
+    }
+    const stat = lstatSync(candidate);
+    return stat.isFile() ? "file" : stat.isDirectory() ? "directory" : "missing";
+  };
+  const sourceTargets: WorkspaceSnapshot["sourceTargets"] = {};
+  for (const note of snapshot.notes) {
+    const raw = workspaceSource(workspaceFrontmatter(note.source));
+    if (raw === undefined) continue;
+    const norm = raw.endsWith(".md") ? raw.slice(0, -3) : raw;
+    if (norm.split("/").some(part => WORK_SKIP_DIRS.has(part)) || workSkipped(norm, skipRoots)) continue;
+    const file = plain(resolve(root, norm + ".md"));
+    sourceTargets[norm] = file === "file" || file === "denied" ? file : plain(resolve(root, norm)) === "directory" ? "directory" : "missing";
+  }
+  const assertUnchanged = () => { for (const [file, expected] of stamps) if (stamp(file) !== expected) throw new Error("workspace graph: input changed during build"); };
+  assertUnchanged();
+  return { files, buSlugs: [...buSlugs].sort(), skipRoots, sourceTargets, assertUnchanged };
+}