@series-a/cli 0.1.0 → 0.6.0

package/src/commands/knowledge-sync.mjs

@@ -0,0 +1,519 @@
+// AdvisorPro → Obsidian vault: ONE-WAY PUSH (pull into the vault).
+//
+// One vault per workspace. The platform is the source of truth; the vault is a
+// read-only mirror of the workspace Knowledge Base and its knowledge graph:
+// plain Markdown notes with YAML frontmatter and [[wikilinks]].
+//
+// Nothing written inside Obsidian ever flows back to AdvisorPro. Local edits are
+// overwritten on the next pull unless --preserve-local is passed, in which case
+// the fresh remote copy lands beside the note as <slug>.remote.md.
+//
+// Sync state lives in <vault>/.advisoros/state.json.
+
+import { createHash } from "node:crypto";
+import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { call } from "../api.mjs";
+import { emit, fail, ctxFrom, withCommonOpts, table, truncate } from "../ui.mjs";
+
+const STATE_DIR = ".advisoros";
+const STATE_FILE = "state.json";
+const TRASH_DIR = "trash";
+const ENTITY_DIR = "Entities";
+
+/* ------------------------------------------------------------------ utils */
+
+const sha256 = (s) => createHash("sha256").update(s ?? "", "utf8").digest("hex");
+
+function slugify(s) {
+  return String(s || "untitled")
+    .normalize("NFKD")
+    .replace(/[^\w\s-]/g, "")
+    .trim()
+    .replace(/\s+/g, "-")
+    .slice(0, 80)
+    .toLowerCase() || "untitled";
+}
+
+function titleCase(s) {
+  return String(s || "General")
+    .split(/[-_\s]+/)
+    .filter(Boolean)
+    .map((w) => w[0].toUpperCase() + w.slice(1))
+    .join(" ");
+}
+
+/* ------------------------------------------------- frontmatter (tiny YAML) */
+
+function parseFrontmatter(text) {
+  if (!text.startsWith("---")) return { data: {}, body: text };
+  const end = text.indexOf("\n---", 3);
+  if (end === -1) return { data: {}, body: text };
+  const raw = text.slice(text.indexOf("\n") + 1, end);
+  const body = text.slice(end + 4).replace(/^\r?\n/, "");
+  const data = {};
+  for (const line of raw.split(/\r?\n/)) {
+    const m = line.match(/^([A-Za-z0-9_]+):\s*(.*)$/);
+    if (!m) continue;
+    let [, key, value] = m;
+    value = value.trim();
+    if (value.startsWith("[") && value.endsWith("]")) {
+      data[key] = value.slice(1, -1).split(",").map((v) => v.trim().replace(/^["']|["']$/g, "")).filter(Boolean);
+    } else {
+      data[key] = value.replace(/^["']|["']$/g, "");
+    }
+  }
+  return { data, body };
+}
+
+function serializeNote(data, body) {
+  const lines = ["---"];
+  for (const [k, v] of Object.entries(data)) {
+    if (v === undefined || v === null || v === "") continue;
+    if (Array.isArray(v)) lines.push(`${k}: [${v.map((x) => String(x)).join(", ")}]`);
+    else lines.push(`${k}: ${String(v).replace(/\n/g, " ")}`);
+  }
+  lines.push("---", "");
+  return lines.join("\n") + (body ?? "").replace(/^\n+/, "") + "\n";
+}
+
+/* -------------------------------------------------------------- rendering */
+
+function linkSection(heading, titles) {
+  if (!titles?.length) return "";
+  return ["", `## ${heading}`, ...titles.map((t) => `- [[${t}]]`), ""].join("\n");
+}
+
+function remoteToNote(item, titleById, entityTitlesByItem) {
+  const bodyRaw = item.extracted_text ?? item.summary ?? "";
+  const links = Array.isArray(item.metadata?.links) ? item.metadata.links : [];
+  const linkedTitles = links
+    .map((l) => titleById.get(l.id) ?? l.title ?? l.id)
+    .filter(Boolean);
+  const entityTitles = entityTitlesByItem.get(item.id) ?? [];
+
+  let body = bodyRaw;
+  if (linkedTitles.length && !/\[\[[^\]]+\]\]/.test(bodyRaw ?? "")) {
+    body += "\n" + linkSection("Linked", linkedTitles);
+  }
+  if (entityTitles.length) body += "\n" + linkSection("Entities", entityTitles);
+
+  const rel = path.join(titleCase(item.category || "General"), `${slugify(item.name)}.md`);
+  const fm = {
+    advisoros_id: item.id,
+    workspace: item.workspace_id,
+    title: item.name,
+    category: item.category || "general",
+    tags: item.tags ?? [],
+    source_url: item.content_url ?? "",
+    updated_at: item.updated_at,
+    content_hash: sha256(body.trim()),
+    source: "advisorpro (read-only mirror)",
+  };
+  return { rel, text: serializeNote(fm, body), hash: sha256(body.trim()) };
+}
+
+function entityToNote(entity, ctxIndex) {
+  const type = titleCase(entity.entity_type || "Concept");
+  const rel = path.join(ENTITY_DIR, type, `${slugify(entity.name)}.md`);
+
+  const lines = [];
+  if (entity.description) lines.push(entity.description.trim(), "");
+
+  const rels = ctxIndex.edgesByEntity.get(entity.id) ?? [];
+  if (rels.length) {
+    lines.push("## Relationships");
+    for (const r of rels) {
+      const other = ctxIndex.entityNameById.get(r.otherId);
+      if (!other) continue;
+      lines.push(`- ${r.direction === "out" ? "" : "← "}${r.edge_type} → [[${other}]]`);
+    }
+    lines.push("");
+  }
+
+  const docs = ctxIndex.itemTitlesByEntity.get(entity.id) ?? [];
+  if (docs.length) {
+    lines.push("## Mentioned in");
+    for (const d of docs) lines.push(`- [[${d}]]`);
+    lines.push("");
+  }
+
+  const body = lines.join("\n");
+  const fm = {
+    advisoros_entity_id: entity.id,
+    workspace: entity.workspace_id ?? ctxIndex.workspaceId,
+    title: entity.name,
+    type: entity.entity_type || "concept",
+    aliases: Array.isArray(entity.aliases) ? entity.aliases : [],
+    mention_count: entity.mention_count ?? 0,
+    updated_at: entity.updated_at,
+    content_hash: sha256(body.trim()),
+    source: "advisorpro (read-only mirror)",
+  };
+  return { rel, text: serializeNote(fm, body), hash: sha256(body.trim()) };
+}
+
+/* ------------------------------------------------------------------ state */
+
+const statePath = (vault) => path.join(vault, STATE_DIR, STATE_FILE);
+
+async function readState(vault) {
+  try {
+    return JSON.parse(await readFile(statePath(vault), "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+async function writeState(vault, state) {
+  await mkdir(path.join(vault, STATE_DIR), { recursive: true });
+  await writeFile(statePath(vault), JSON.stringify(state, null, 2) + "\n", "utf8");
+}
+
+async function requireLinkedVault(vaultArg, cmdWorkspace) {
+  const vault = path.resolve(vaultArg || ".");
+  if (!existsSync(vault)) throw new Error(`Vault not found: ${vault}`);
+  const state = await readState(vault);
+  if (!state) {
+    throw new Error(`Vault is not linked. Run: advisoros kb link "${vault}" --workspace <id>`);
+  }
+  if (cmdWorkspace && state.workspace_id !== cmdWorkspace) {
+    throw new Error(
+      `Vault is linked to workspace ${state.workspace_id}, but --workspace ${cmdWorkspace} was given. Refusing to sync.`,
+    );
+  }
+  return { vault, state };
+}
+
+/* ------------------------------------------------------------ vault scans */
+
+async function walkMarkdown(dir, base = dir, acc = []) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const e of entries) {
+    if (e.name.startsWith(".")) continue;
+    const full = path.join(dir, e.name);
+    if (e.isDirectory()) await walkMarkdown(full, base, acc);
+    else if (e.isFile() && e.name.endsWith(".md") && !e.name.endsWith(".remote.md")) {
+      acc.push(path.relative(base, full));
+    }
+  }
+  return acc;
+}
+
+async function localHash(vault, rel) {
+  try {
+    const text = await readFile(path.join(vault, rel), "utf8");
+    const { body } = parseFrontmatter(text);
+    return sha256(body.trim());
+  } catch {
+    return null;
+  }
+}
+
+async function trash(vault, rel, report) {
+  const src = path.join(vault, rel);
+  if (!existsSync(src)) return;
+  const dest = path.join(vault, STATE_DIR, TRASH_DIR, rel.replace(/[\\/]/g, "__"));
+  await mkdir(path.dirname(dest), { recursive: true });
+  await writeFile(dest, await readFile(src, "utf8"), "utf8");
+  await rm(src);
+  report.trashed.push(rel);
+}
+
+/* ------------------------------------------------------------------- pull */
+
+async function fetchGraph(ctx) {
+  const entities = [];
+  let edges = [];
+  let mentions = [];
+  let offset = 0;
+  for (;;) {
+    const page = await call("knowledge", "graph_sync_list", { limit: 1000, offset }, ctx);
+    entities.push(...(page.entities ?? []));
+    if (offset === 0) {
+      edges = page.edges ?? [];
+      mentions = page.mentions ?? [];
+    }
+    if (!page.next_offset) break;
+    offset = page.next_offset;
+  }
+  return { entities, edges, mentions };
+}
+
+async function runPull(vault, state, ctx, opts) {
+  const report = {
+    workspace_id: state.workspace_id,
+    vault,
+    pulled: [], entities: [], preserved: [], unchanged: 0, trashed: [], skipped: [],
+    dry_run: !!opts.dryRun,
+    direction: "advisorpro → obsidian (one-way)",
+  };
+
+  // ---- fetch remote knowledge items --------------------------------------
+  const remote = [];
+  let offset = 0;
+  for (;;) {
+    const page = await call("knowledge", "sync_list", { limit: 500, offset }, ctx);
+    remote.push(...(page.items ?? []));
+    if (!page.next_offset) break;
+    offset = page.next_offset;
+  }
+  const titleById = new Map(remote.map((r) => [r.id, r.name]));
+
+  // ---- fetch the graph (best effort: an older backend may not expose it) --
+  let graph = { entities: [], edges: [], mentions: [] };
+  if (opts.graph !== false) {
+    try {
+      graph = await fetchGraph(ctx);
+    } catch (e) {
+      report.skipped.push({ path: "Entities/", reason: `graph unavailable — ${e.message}` });
+    }
+  }
+
+  const entityNameById = new Map(graph.entities.map((e) => [e.id, e.name]));
+  const entityTitlesByItem = new Map();
+  const itemTitlesByEntity = new Map();
+  for (const m of graph.mentions) {
+    const name = entityNameById.get(m.entity_id);
+    const docTitle = titleById.get(m.item_id);
+    if (name) {
+      const arr = entityTitlesByItem.get(m.item_id) ?? [];
+      if (!arr.includes(name)) arr.push(name);
+      entityTitlesByItem.set(m.item_id, arr);
+    }
+    if (docTitle) {
+      const arr = itemTitlesByEntity.get(m.entity_id) ?? [];
+      if (!arr.includes(docTitle)) arr.push(docTitle);
+      itemTitlesByEntity.set(m.entity_id, arr);
+    }
+  }
+  const edgesByEntity = new Map();
+  const addEdge = (id, rec) => {
+    const arr = edgesByEntity.get(id) ?? [];
+    arr.push(rec);
+    edgesByEntity.set(id, arr);
+  };
+  for (const e of graph.edges) {
+    if (e.source_entity_id && e.target_entity_id) {
+      addEdge(e.source_entity_id, { edge_type: e.edge_type, otherId: e.target_entity_id, direction: "out" });
+      addEdge(e.target_entity_id, { edge_type: e.edge_type, otherId: e.source_entity_id, direction: "in" });
+    }
+  }
+  const ctxIndex = {
+    entityNameById, itemTitlesByEntity, edgesByEntity, workspaceId: state.workspace_id,
+  };
+
+  // ---- writer ------------------------------------------------------------
+  const tracked = state.notes ?? {};
+  const trackedEntities = state.entities ?? {};
+
+  const writeMirror = async (key, store, rendered, remoteUpdatedAt) => {
+    const known = store[key];
+    const currentPath = known?.path ?? rendered.rel;
+    const onDisk = await localHash(vault, currentPath);
+
+    if (onDisk && known && onDisk === known.hash && known.hash === rendered.hash &&
+        known.updated_at === remoteUpdatedAt && currentPath === rendered.rel) {
+      report.unchanged++;
+      return;
+    }
+
+    const locallyModified = !!onDisk && !!known && onDisk !== known.hash;
+    let target = rendered.rel;
+    if (locallyModified && opts.preserveLocal) {
+      target = rendered.rel.replace(/\.md$/, ".remote.md");
+      report.preserved.push(currentPath);
+    }
+
+    if (!opts.dryRun) {
+      // The note moved (renamed / recategorised) → drop the stale copy.
+      if (known?.path && known.path !== target && existsSync(path.join(vault, known.path)) &&
+          !(locallyModified && opts.preserveLocal)) {
+        await rm(path.join(vault, known.path));
+      }
+      const full = path.join(vault, target);
+      await mkdir(path.dirname(full), { recursive: true });
+      await writeFile(full, rendered.text, "utf8");
+      if (!(locallyModified && opts.preserveLocal)) {
+        store[key] = { path: target, hash: rendered.hash, updated_at: remoteUpdatedAt };
+      }
+    }
+    return target;
+  };
+
+  // ---- knowledge items ---------------------------------------------------
+  const remoteIds = new Set();
+  for (const item of remote) {
+    if (item.is_active === false) {
+      const known = tracked[item.id];
+      if (known?.path && !opts.dryRun) await trash(vault, known.path, report);
+      else if (known?.path) report.trashed.push(known.path);
+      delete tracked[item.id];
+      continue;
+    }
+    remoteIds.add(item.id);
+    const rendered = remoteToNote(item, titleById, entityTitlesByItem);
+    const written = await writeMirror(item.id, tracked, rendered, item.updated_at);
+    if (written) report.pulled.push(written);
+  }
+
+  // Items that vanished from the workspace entirely.
+  for (const [id, meta] of Object.entries(tracked)) {
+    if (remoteIds.has(id)) continue;
+    if (!opts.dryRun) await trash(vault, meta.path, report);
+    else report.trashed.push(meta.path);
+    delete tracked[id];
+  }
+
+  // ---- entities ----------------------------------------------------------
+  const entityIds = new Set();
+  for (const entity of graph.entities) {
+    entityIds.add(entity.id);
+    const rendered = entityToNote(entity, ctxIndex);
+    const written = await writeMirror(entity.id, trackedEntities, rendered, entity.updated_at);
+    if (written) report.entities.push(written);
+  }
+  if (graph.entities.length || Object.keys(trackedEntities).length) {
+    for (const [id, meta] of Object.entries(trackedEntities)) {
+      if (entityIds.has(id)) continue;
+      if (!opts.dryRun) await trash(vault, meta.path, report);
+      else report.trashed.push(meta.path);
+      delete trackedEntities[id];
+    }
+  }
+
+  if (!opts.dryRun) {
+    state.notes = tracked;
+    state.entities = trackedEntities;
+    state.last_sync_at = new Date().toISOString();
+    state.direction = "pull";
+    await writeState(vault, state);
+  }
+
+  report.summary = {
+    notes: report.pulled.length,
+    entities: report.entities.length,
+    unchanged: report.unchanged,
+    preserved: report.preserved.length,
+    trashed: report.trashed.length,
+    skipped: report.skipped.length,
+  };
+  return report;
+}
+
+function renderReport(r) {
+  const s = r.summary;
+  process.stdout.write(
+    `AdvisorPro → Obsidian (one-way mirror). Local edits are overwritten` +
+    ` unless --preserve-local is used.\n\n`,
+  );
+  process.stdout.write(
+    table(
+      ["Notes", "Entities", "Unchanged", "Preserved", "Trashed", "Skipped"],
+      [[s.notes, s.entities, s.unchanged, s.preserved, s.trashed, s.skipped]],
+    ) + "\n",
+  );
+  if (r.dry_run) process.stdout.write("(dry run — nothing was written)\n");
+  if (r.preserved.length) {
+    process.stdout.write("\nLocally modified (remote copy written as *.remote.md):\n");
+    for (const p of r.preserved) process.stdout.write(`  • ${p}\n`);
+  }
+  if (r.skipped.length) {
+    process.stdout.write("\nSkipped:\n");
+    for (const x of r.skipped) process.stdout.write(`  • ${truncate(x.path, 60)} — ${x.reason}\n`);
+  }
+}
+
+/* -------------------------------------------------------------- commands */
+
+export function registerVaultSync(program) {
+  const kb = program.command("kb")
+    .description("Mirror the workspace Knowledge Base into an Obsidian vault (one-way)");
+
+  withCommonOpts(kb.command("link <vault>"))
+    .description("Link a local Obsidian vault to a workspace (one vault per workspace)")
+    .action(async (vaultArg, _o, cmd) => {
+      try {
+        const ctx = ctxFrom(cmd);
+        const vault = path.resolve(vaultArg);
+        await mkdir(vault, { recursive: true });
+        const existing = await readState(vault);
+        const me = await call("workspaces", "list", {}, ctx).catch(() => null);
+        const wsId = ctx.workspaceId ||
+          process.env.ADVISOROS_WORKSPACE_ID ||
+          existing?.workspace_id ||
+          me?.workspaces?.[0]?.id;
+        if (!wsId) throw new Error("Pass --workspace <id> to link this vault.");
+        if (existing && existing.workspace_id !== wsId) {
+          throw new Error(
+            `Vault already linked to ${existing.workspace_id}. Run \`advisoros kb unlink\` first.`,
+          );
+        }
+        const state = existing ??
+          { workspace_id: wsId, notes: {}, entities: {}, created_at: new Date().toISOString() };
+        state.workspace_id = wsId;
+        state.entities = state.entities ?? {};
+        await writeState(vault, state);
+        emit(cmd, { vault, workspace_id: wsId }, (d) =>
+          process.stdout.write(
+            `✔ Vault linked: ${d.vault} → workspace ${d.workspace_id}\n` +
+            `  This vault is a read-only mirror. Edits made in Obsidian are not sent back.\n`,
+          ),
+        );
+      } catch (e) { fail(e); }
+    });
+
+  const pullOpts = (c) =>
+    withCommonOpts(c)
+      .option("--dry-run", "show what would change without writing anything")
+      .option("--preserve-local", "never overwrite a locally edited note; write <name>.remote.md instead")
+      .option("--no-graph", "skip the Entities/ graph mirror, notes only");
+
+  pullOpts(kb.command("pull [vault]"))
+    .description("Mirror the workspace Knowledge Base and graph into the vault (one-way)")
+    .action(async (vaultArg, o, cmd) => {
+      try {
+        const ctx = ctxFrom(cmd);
+        const { vault, state } = await requireLinkedVault(vaultArg, ctx.workspaceId);
+        const report = await runPull(vault, state, { ...ctx, workspaceId: state.workspace_id }, o);
+        emit(cmd, report, renderReport);
+      } catch (e) { fail(e); }
+    });
+
+  pullOpts(kb.command("sync [vault]"))
+    .description("Alias of `kb pull` (sync is one-way: AdvisorPro → Obsidian)")
+    .action(async (vaultArg, o, cmd) => {
+      try {
+        const ctx = ctxFrom(cmd);
+        const { vault, state } = await requireLinkedVault(vaultArg, ctx.workspaceId);
+        const report = await runPull(vault, state, { ...ctx, workspaceId: state.workspace_id }, o);
+        emit(cmd, report, renderReport);
+      } catch (e) { fail(e); }
+    });
+
+  withCommonOpts(kb.command("status [vault]"))
+    .description("Preview what the next pull would change (same as pull --dry-run)")
+    .action(async (vaultArg, _o, cmd) => {
+      try {
+        const ctx = ctxFrom(cmd);
+        const { vault, state } = await requireLinkedVault(vaultArg, ctx.workspaceId);
+        const report = await runPull(
+          vault, state, { ...ctx, workspaceId: state.workspace_id }, { dryRun: true },
+        );
+        emit(cmd, report, renderReport);
+      } catch (e) { fail(e); }
+    });
+
+  kb.command("unlink [vault]")
+    .description("Remove the sync state from a vault (notes are left untouched)")
+    .action(async (vaultArg) => {
+      try {
+        const vault = path.resolve(vaultArg || ".");
+        const p = statePath(vault);
+        if (!existsSync(p)) throw new Error("Vault is not linked.");
+        await rm(p);
+        process.stdout.write(`✔ Unlinked ${vault}\n`);
+      } catch (e) { fail(e); }
+    });
+}