@openparachute/vault 0.4.9-rc.4 → 0.4.9-rc.6

package/core/src/portable-md.ts

@@ -72,7 +72,7 @@
  * See vault#308.
  */
 
-import { readdirSync, readFileSync, statSync, mkdirSync, writeFileSync, copyFileSync, existsSync, rmSync } from "fs";
+import { readdirSync, readFileSync, realpathSync, statSync, mkdirSync, writeFileSync, copyFileSync, existsSync, rmSync } from "fs";
 import { basename, join, relative, extname, dirname, resolve as resolvePath, sep as pathSep } from "path";
 import type { Store, Note, Link, Attachment } from "./types.js";
 import type { TagRecord } from "./tag-schemas.js";
@@ -1001,7 +1001,375 @@ export async function exportVaultToDir(
   };
 }
 
-function hasSchemaContent(tag: TagRecord): boolean {
+// ---------------------------------------------------------------------------
+// Orphan sweep — for git-mirror's delete propagation
+// ---------------------------------------------------------------------------
+
+/**
+ * Result of a `pruneOrphans` pass.
+ */
+export interface PruneOrphansStats {
+  /** Note content files removed (frontmatter id not in `validNoteIds`). */
+  notes_removed: number;
+  /** Sidecar metadata files removed (under `.parachute/notes-meta/`). */
+  sidecars_removed: number;
+  /** Schema sidecars removed (under `.parachute/schemas/`). */
+  schemas_removed: number;
+  /** Attachment directories removed (under `.parachute/attachments/`). */
+  attachment_dirs_removed: number;
+  /**
+   * Files we couldn't parse / classify. Surfaced for operator audit;
+   * the sweep doesn't touch them. Empty when everything classified
+   * cleanly.
+   */
+  unparseable_files: Array<{ path: string; reason: string }>;
+}
+
+/**
+ * Options for the orphan-sweep pass. Run periodically (the mirror manager
+ * arms a safety-net poll, default 1h) and after operator-visible deletions
+ * (the mirror manager's targeted-deletion fast path also calls this for the
+ * touched-set).
+ *
+ * The sweep is the bookkeeping cousin of the event-driven fast path: events
+ * cover the common case (a single note deletion fires "deleted" → mirror
+ * removes that file); the sweep covers anything the fast path missed
+ * (direct SQL writes, app crashes between dispatch and handler, restart
+ * gaps).
+ */
+export interface PruneOrphansOptions {
+  /** Directory to sweep — same shape as an `exportVaultToDir` outDir. */
+  outDir: string;
+  /** Note IDs that should be kept (everything else under the export gets removed). */
+  validNoteIds: Set<string>;
+  /** Tag names that should be kept (other schema sidecars under `.parachute/schemas/` get removed). */
+  validTagNames: Set<string>;
+  /** Attachment IDs that should be kept (other dirs under `.parachute/attachments/` get removed). */
+  validAttachmentIds: Set<string>;
+  /**
+   * Override `extension`-based content-file extension recognition. The
+   * default treats `.md` and `.mdx` as having inline frontmatter (with
+   * `id:` reachable via the file head); everything else needs a sidecar
+   * lookup to know what id owns it.
+   */
+  supportsInlineFrontmatter?: (ext: string) => boolean;
+}
+
+/**
+ * Sweep the export directory for files belonging to notes / tags /
+ * attachments that no longer exist in the vault. Removes them so the
+ * mirror's `git diff` reflects what vault actually has.
+ *
+ * Strategy:
+ *   1. Walk content files. For each `.md` / `.mdx`, parse the frontmatter
+ *      `id` and compare against `validNoteIds`. Mismatch → remove the
+ *      file. Files we can't parse are recorded in `unparseable_files`
+ *      and left alone (best-effort, no destructive guesses).
+ *   2. For non-inline-frontmatter extensions (`.csv`, `.yaml`, etc.),
+ *      check the matching `.parachute/notes-meta/<id>.yaml` sidecar — the
+ *      sidecar carries the canonical `id` + `path` + `extension` triple.
+ *      An orphaned sidecar (no matching content file) is also removed,
+ *      and an orphaned content file (no matching sidecar) without a
+ *      parseable frontmatter is left as unparseable.
+ *   3. Walk `.parachute/schemas/`. Each file is `<tag>.yaml` (after
+ *      filename sanitization). Parse the `name:` field; compare against
+ *      `validTagNames`. Mismatch → remove.
+ *   4. Walk `.parachute/attachments/`. Each subdir name IS the
+ *      attachment id (per `exportVaultToDir`'s layout). Compare against
+ *      `validAttachmentIds`. Mismatch → recursive remove.
+ *
+ * Returns counts so callers can log + decide whether to commit.
+ *
+ * Safe to call on a directory that's never been exported to (returns
+ * zero counts; doesn't create anything).
+ */
+export function pruneOrphans(opts: PruneOrphansOptions): PruneOrphansStats {
+  const stats: PruneOrphansStats = {
+    notes_removed: 0,
+    sidecars_removed: 0,
+    schemas_removed: 0,
+    attachment_dirs_removed: 0,
+    unparseable_files: [],
+  };
+
+  const outDir = opts.outDir;
+  if (!existsSync(outDir)) return stats;
+
+  const supportsInline = opts.supportsInlineFrontmatter ?? supportsInlineFrontmatter;
+  const sidecarRoot = join(outDir, SIDECAR_DIR);
+  const notesMetaRoot = join(sidecarRoot, NOTES_META_DIR);
+  const schemasRoot = join(sidecarRoot, "schemas");
+  const attachmentsRoot = join(sidecarRoot, "attachments");
+
+  // Path-traversal guard. `walkContentFiles` uses `statSync` which
+  // follows symlinks — a symlink inside the mirror pointing OUTSIDE
+  // `outDir` would resurface its target's files in the prune sweep,
+  // and a bare `rmSync(filepath)` would delete them off-tree. Every
+  // deletion in this function routes through `safeRm`, which calls
+  // `realpathSync` (resolves through symlinks, unlike syntactic-only
+  // `path.resolve`) and refuses to delete anything that isn't `outDir`
+  // or beneath it after symlink resolution. Refusals get recorded in
+  // `unparseable_files` so an operator can see what was skipped.
+  // Reviewer-flagged on vault#382 (Critical #2).
+  const outDirReal = realpathSync(outDir);
+  const safeRm = (
+    candidate: string,
+    onSuccess: () => void,
+    options: { recursive?: boolean } = {},
+  ): void => {
+    let real: string;
+    try {
+      real = realpathSync(candidate);
+    } catch (err) {
+      // realpathSync throws if the path doesn't exist or is unreadable.
+      // Don't delete what we can't fully resolve.
+      stats.unparseable_files.push({
+        path: candidate,
+        reason: `realpath failed: ${(err as Error).message ?? err}`,
+      });
+      return;
+    }
+    if (!isWithinDir(real, outDirReal)) {
+      stats.unparseable_files.push({
+        path: candidate,
+        reason: "real path resolved outside mirror outDir — refusing to delete (symlink?)",
+      });
+      return;
+    }
+    try {
+      // Delete via the resolved real path; never via the unresolved
+      // candidate (which could route through a symlink we already
+      // determined would escape).
+      rmSync(real, { force: true, recursive: options.recursive ?? false });
+      onSuccess();
+    } catch (err) {
+      stats.unparseable_files.push({
+        path: candidate,
+        reason: `unlink failed: ${(err as Error).message ?? err}`,
+      });
+    }
+  };
+
+  // ---- 1 + 2. Notes + sidecars ----
+  //
+  // First pass: build the sidecar id → { path, extension } map so we can
+  // resolve non-inline-frontmatter content files via their sidecar.
+  // Sidecars whose claimed (path, extension) doesn't map to an existing
+  // content file are tracked as "orphaned sidecar" candidates.
+  const sidecarById = new Map<string, { path: string | null; extension: string | null }>();
+  const sidecarFilesById = new Map<string, string>(); // id → absolute filepath
+  if (existsSync(notesMetaRoot)) {
+    try {
+      for (const entry of readdirSync(notesMetaRoot)) {
+        if (!entry.endsWith(".yaml")) continue;
+        const id = entry.slice(0, -5);
+        const full = join(notesMetaRoot, entry);
+        sidecarFilesById.set(id, full);
+        try {
+          const text = readFileSync(full, "utf-8");
+          const { frontmatter } = parseFrontmatter(`---\n${text}---\n`);
+          sidecarById.set(id, {
+            path: typeof frontmatter.path === "string" ? (frontmatter.path as string) : null,
+            extension: typeof frontmatter.extension === "string" ? (frontmatter.extension as string) : null,
+          });
+        } catch (err) {
+          stats.unparseable_files.push({
+            path: full,
+            reason: `failed to parse sidecar: ${(err as Error).message ?? err}`,
+          });
+        }
+      }
+    } catch (err) {
+      // Notes-meta dir read-error is non-fatal — record + carry on.
+      stats.unparseable_files.push({
+        path: notesMetaRoot,
+        reason: `notes-meta walk failed: ${(err as Error).message ?? err}`,
+      });
+    }
+  }
+
+  // Now walk content files (everything outside the .parachute sidecar).
+  const contentFiles = walkContentFiles(outDir);
+  // Track which sidecars matched a content file so we can also remove
+  // orphaned sidecars (sidecar present but content file gone).
+  const pairedSidecarIds = new Set<string>();
+  for (const filepath of contentFiles) {
+    const ext = extname(filepath).slice(1).toLowerCase();
+    if (supportsInline(ext)) {
+      // Parse frontmatter, read id.
+      try {
+        const raw = readFileSync(filepath, "utf-8");
+        const { frontmatter } = parseFrontmatter(raw);
+        const id = typeof frontmatter.id === "string" ? (frontmatter.id as string) : null;
+        if (!id) {
+          stats.unparseable_files.push({
+            path: filepath,
+            reason: "no `id` in frontmatter",
+          });
+          continue;
+        }
+        if (!opts.validNoteIds.has(id)) {
+          safeRm(filepath, () => {
+            stats.notes_removed++;
+          });
+        }
+      } catch (err) {
+        stats.unparseable_files.push({
+          path: filepath,
+          reason: `read failed: ${(err as Error).message ?? err}`,
+        });
+      }
+    } else {
+      // Sidecar-required extension. Find the matching sidecar by
+      // (path, extension) — sidecars are keyed by id, so we sweep the
+      // sidecarById map.
+      const relPath = relative(outDir, filepath).replace(/\\/g, "/");
+      // Strip extension to get the canonical path stored in the sidecar
+      // (vault paths don't carry extensions).
+      const pathNoExt = relPath.slice(0, -(ext.length + 1));
+      let foundId: string | null = null;
+      for (const [id, info] of sidecarById.entries()) {
+        if (
+          info.path === pathNoExt &&
+          (info.extension ?? "md").toLowerCase() === ext
+        ) {
+          foundId = id;
+          break;
+        }
+      }
+      if (!foundId) {
+        // Content file with no sidecar — can't tell which note owns it.
+        // Conservative: leave alone, record as unparseable.
+        stats.unparseable_files.push({
+          path: filepath,
+          reason: "no sidecar metadata could be matched by (path, extension)",
+        });
+        continue;
+      }
+      pairedSidecarIds.add(foundId);
+      if (!opts.validNoteIds.has(foundId)) {
+        // Note is orphaned — remove both content and sidecar.
+        safeRm(filepath, () => {
+          stats.notes_removed++;
+        });
+        const sidecarPath = sidecarFilesById.get(foundId);
+        if (sidecarPath) {
+          safeRm(sidecarPath, () => {
+            stats.sidecars_removed++;
+          });
+        }
+      }
+    }
+  }
+
+  // Sweep up orphaned sidecars (sidecar exists but no content file
+  // matched OR sidecar's id isn't in validNoteIds).
+  for (const [id, sidecarPath] of sidecarFilesById.entries()) {
+    if (opts.validNoteIds.has(id) && pairedSidecarIds.has(id)) continue;
+    if (opts.validNoteIds.has(id) && !pairedSidecarIds.has(id)) {
+      // Sidecar refers to a valid note but the content file is gone —
+      // that's an inconsistency, not an orphan. Leave the sidecar so
+      // the next export can rewrite the content file alongside it.
+      continue;
+    }
+    safeRm(sidecarPath, () => {
+      stats.sidecars_removed++;
+    });
+  }
+
+  // ---- 3. Schema sidecars ----
+  if (existsSync(schemasRoot)) {
+    try {
+      for (const entry of readdirSync(schemasRoot)) {
+        if (!entry.endsWith(".yaml")) continue;
+        const full = join(schemasRoot, entry);
+        try {
+          const text = readFileSync(full, "utf-8");
+          const { frontmatter } = parseFrontmatter(`---\n${text}---\n`);
+          const name = typeof frontmatter.name === "string" ? (frontmatter.name as string) : null;
+          if (!name) {
+            // Fall back to filename — sanitizeTagFilename replaces `/`
+            // with `__`, so reverse for the lookup.
+            const fromFilename = entry.slice(0, -5).replace(/__/g, "/");
+            if (!opts.validTagNames.has(fromFilename)) {
+              safeRm(full, () => {
+                stats.schemas_removed++;
+              });
+            }
+            continue;
+          }
+          if (!opts.validTagNames.has(name)) {
+            safeRm(full, () => {
+              stats.schemas_removed++;
+            });
+          }
+        } catch (err) {
+          stats.unparseable_files.push({
+            path: full,
+            reason: `schema sweep failed: ${(err as Error).message ?? err}`,
+          });
+        }
+      }
+    } catch (err) {
+      stats.unparseable_files.push({
+        path: schemasRoot,
+        reason: `schemas walk failed: ${(err as Error).message ?? err}`,
+      });
+    }
+  }
+
+  // ---- 4. Attachment directories ----
+  if (existsSync(attachmentsRoot)) {
+    try {
+      for (const entry of readdirSync(attachmentsRoot)) {
+        const full = join(attachmentsRoot, entry);
+        let stat;
+        try {
+          stat = statSync(full);
+        } catch (err) {
+          stats.unparseable_files.push({
+            path: full,
+            reason: `stat failed: ${(err as Error).message ?? err}`,
+          });
+          continue;
+        }
+        if (!stat.isDirectory()) continue;
+        // The directory name IS the attachment id (per the export layout).
+        if (!opts.validAttachmentIds.has(entry)) {
+          safeRm(
+            full,
+            () => {
+              stats.attachment_dirs_removed++;
+            },
+            { recursive: true },
+          );
+        }
+      }
+    } catch (err) {
+      stats.unparseable_files.push({
+        path: attachmentsRoot,
+        reason: `attachments walk failed: ${(err as Error).message ?? err}`,
+      });
+    }
+  }
+
+  return stats;
+}
+
+/**
+ * True iff the tag carries content the export writer will emit as a
+ * schema sidecar (`.parachute/schemas/<tag>.yaml`). Bare-name tags
+ * (the `tags` table has a row but description/fields/relationships/
+ * parents are all empty) get no sidecar — and crucially, after
+ * `deleteTagSchema` clears those fields the row persists with the
+ * bare name. Callers building the `validTagNames` set for
+ * `pruneOrphans` MUST filter through this predicate, otherwise the
+ * stale sidecar lingers indefinitely.
+ *
+ * Reviewer-flagged on vault#382: without this filter, a cleared
+ * schema's sidecar never gets pruned.
+ */
+export function hasSchemaContent(tag: TagRecord): boolean {
   if (tag.description !== undefined && tag.description.length > 0) return true;
   if (tag.fields && Object.keys(tag.fields).length > 0) return true;
   if (tag.relationships && Object.keys(tag.relationships).length > 0) return true;

package/core/src/store.ts

@@ -217,10 +217,23 @@ export class BunSqliteStore implements Store {
   }
 
   async deleteNote(id: string): Promise<void> {
-    // Read before delete so we can invalidate config caches on the way out.
+    // Read before delete so we can invalidate config caches on the way out
+    // AND so the post-delete hook dispatch carries the minimum payload
+    // ({ id, path }). The full note can't be reconstructed post-delete —
+    // by design, hooks subscribing to "deleted" receive a DeletedNoteRef,
+    // not a Note.
     const existing = noteOps.getNote(this.db, id);
     noteOps.deleteNote(this.db, id);
     if (existing?.path) this.invalidateConfigCachesForPath(existing.path);
+    // Dispatch even when `existing` was null — the caller asked for a
+    // deletion, and downstream consumers (e.g. the mirror) reconcile via
+    // id. Path is undefined in that case; the mirror sweep will catch
+    // any orphans missed by the targeted-removal fast path.
+    this.hooks.dispatch(
+      "deleted",
+      { id, ...(existing?.path ? { path: existing.path } : {}) },
+      this,
+    );
   }
 
   async queryNotes(opts: QueryOpts): Promise<Note[]> {
@@ -340,6 +353,10 @@ export class BunSqliteStore implements Store {
     // and may have declared `fields` powering schema validation.
     this._tagHierarchy = null;
     this._schemaConfig = null;
+    // Fire "deleted" only when SOMETHING happened (the underlying
+    // deleteTag returns `deleted: false` when the tag didn't exist).
+    // The git-mirror reacts to this by sweeping the schema sidecar.
+    if (result.deleted) this.hooks.dispatchTag("deleted", name, this);
     return result;
   }
 
@@ -352,6 +369,16 @@ export class BunSqliteStore implements Store {
     // the schema-config by parent_names + fields content.
     this._tagHierarchy = null;
     this._schemaConfig = null;
+    // Rename = delete-then-upsert from the perspective of any consumer
+    // that keys schema artifacts on the tag name (e.g. the git-mirror's
+    // `.parachute/schemas/<tag>.yaml` sidecar file). Fire both events
+    // so the consumer drops the old artifact and writes the new one.
+    // Only dispatch when the rename actually happened — error returns
+    // ({ error: ... }) shouldn't notify subscribers about phantom moves.
+    if ("renamed" in result) {
+      this.hooks.dispatchTag("deleted", oldName, this);
+      this.hooks.dispatchTag("upserted", newName, this);
+    }
     return result;
   }
 
@@ -365,6 +392,15 @@ export class BunSqliteStore implements Store {
     // bust the schema cache — `fields` declarations follow tag identity.
     this._tagHierarchy = null;
     this._schemaConfig = null;
+    // Each merged source vanishes from the tag set; the target's
+    // schema may have absorbed fields/relationships from the sources.
+    // Fire "deleted" for each source and "upserted" for the target so
+    // the mirror sweeps the source sidecars and rewrites the target.
+    for (const source of sources) {
+      if (source === target) continue;
+      this.hooks.dispatchTag("deleted", source, this);
+    }
+    this.hooks.dispatchTag("upserted", target, this);
     return result;
   }
 
@@ -440,12 +476,26 @@ export class BunSqliteStore implements Store {
     // `fields` drives validation — bust the schema cache so the next
     // create/update sees the new declarations.
     this._schemaConfig = null;
+    // The tag schema sidecar in the mirror needs to track this. Fire
+    // "upserted" regardless of whether the row was created or modified
+    // — the mirror writes the sidecar fresh either way.
+    this.hooks.dispatchTag("upserted", tag, this);
     return result;
   }
 
   async deleteTagSchema(tag: string) {
     const result = tagSchemaOps.deleteTagSchema(this.db, tag);
-    if (result) this._schemaConfig = null;
+    if (result) {
+      this._schemaConfig = null;
+      // Schema-only delete: the tag may still exist as a name in the
+      // hierarchy, but the sidecar lost its content. Mirror reacts by
+      // sweeping the sidecar file. (If the underlying row was reduced
+      // to a bare name with no schema content, hasSchemaContent() in
+      // exportVaultToDir already wouldn't have written it on the next
+      // export pass — the targeted delete is the fast path; the sweep
+      // is the safety net.)
+      this.hooks.dispatchTag("deleted", tag, this);
+    }
     return result;
   }
 
@@ -494,6 +544,11 @@ export class BunSqliteStore implements Store {
       this._tagHierarchy = null;
       this._schemaConfig = null;
     }
+    // Tag-mutation event for the git-mirror and any other downstream
+    // consumer. Fire "upserted" on every successful tag-record write —
+    // schema/relationship/parent-name mutations all alter the sidecar
+    // contents the mirror persists.
+    this.hooks.dispatchTag("upserted", tag, this);
     return result;
   }
 
@@ -599,6 +654,17 @@ export class BunSqliteStore implements Store {
     const other = this.db.prepare(
       "SELECT 1 FROM attachments WHERE path = ? LIMIT 1",
     ).get(row.path);
+
+    // Post-delete event for downstream consumers (e.g. the git-mirror's
+    // sweep of `.parachute/attachments/<id>/...`). Payload is the
+    // DeletedAttachmentRef — the row is gone, so we pass only id /
+    // note_id / path.
+    this.hooks.dispatchAttachment(
+      "deleted",
+      { id: attachmentId, noteId, path: row.path },
+      this,
+    );
+
     return { deleted: true, path: row.path, orphaned: !other };
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/vault",
-  "version": "0.4.9-rc.4",
+  "version": "0.4.9-rc.6",
   "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
   "module": "src/cli.ts",
   "type": "module",