@openparachute/vault 0.5.1 → 0.5.2-rc.1

package/src/usage.ts

@@ -0,0 +1,318 @@
+/**
+ * Per-vault usage monitoring — a cheap, read-scoped "how big is this vault"
+ * report (data footprint). Built for running many vaults for many users: the
+ * operator needs to see each vault's size, and a vault's own user should be
+ * able to see their own vault's footprint.
+ *
+ * The endpoint that consumes these helpers lives in `routing.ts` at
+ * `GET /vault/<name>/.parachute/usage` (read-scoped). This module owns the
+ * filesystem arithmetic and the dir-walk cache.
+ *
+ * Cost model:
+ *   - counts + contentBytes come from `getVaultStats` (a handful of indexed
+ *     COUNT/SUM queries) — cheap, computed on every request.
+ *   - `dbBytes` is three `statSync` calls (the WAL trio) — cheap, every request.
+ *   - the two recursive directory walks (`assets`, `mirror`) can traverse
+ *     thousands of files, so they go through a 60s TTL cache keyed by vault
+ *     name. `?fresh=1` bypasses the cache; an attachment upload invalidates it.
+ *
+ * Everything I/O-touching goes through an injectable `UsageFs` seam + an
+ * injectable clock so tests can assert call counts and TTL behavior without
+ * touching the real disk.
+ */
+
+import {
+  statSync as realStatSync,
+  readdirSync as realReaddirSync,
+  type Dirent,
+} from "fs";
+import { join } from "path";
+import { vaultDir, assetsDir } from "./config.ts";
+import { readMirrorConfigForVault, resolveMirrorPath } from "./mirror-config.ts";
+import type { VaultStats } from "../core/src/types.ts";
+
+// ---------------------------------------------------------------------------
+// Injectable seams
+// ---------------------------------------------------------------------------
+
+/**
+ * The minimal filesystem surface the usage helpers depend on. Injecting it
+ * lets tests count how many times the dir-walk runs (to prove the cache
+ * actually skips it) and synthesize a tree without writing real files.
+ *
+ * Symlink safety lives in the WALK, not in `statFile`: `dirSize` never
+ * descends into or sizes a symlink entry (it filters on
+ * `Dirent.isSymbolicLink()` before ever calling `statFile`), so a symlink
+ * loop can't hang the walk and a symlink pointing at a huge tree elsewhere
+ * can't inflate the count. `statFile` is therefore only ever called on real
+ * files / the WAL trio (which are never symlinks).
+ */
+export interface UsageFs {
+  /**
+   * `stat` (symlink-FOLLOWING, like `statSync`): returns the size of a file
+   * (or throws if absent — callers wrap in try/catch → 0). Following symlinks
+   * is safe here because dir descent already filters symlinks via
+   * `Dirent.isSymbolicLink()`, and the WAL files (`vault.db{,-wal,-shm}`) are
+   * never symlinks.
+   */
+  statFile(path: string): { size: number; isDirectory(): boolean; isSymbolicLink(): boolean };
+  /** `readdirSync(path, { withFileTypes: true })`. */
+  readDir(path: string): Dirent[];
+}
+
+/** Monotonic-enough clock seam: returns epoch millis. */
+export type Clock = () => number;
+
+/** Real filesystem implementation (production default). */
+export const realUsageFs: UsageFs = {
+  statFile: (path) => realStatSync(path),
+  readDir: (path) => realReaddirSync(path, { withFileTypes: true }),
+};
+
+// ---------------------------------------------------------------------------
+// Primitive byte counters
+// ---------------------------------------------------------------------------
+
+/**
+ * Physical bytes of a vault's SQLite database, WAL-aware: `vault.db` +
+ * `vault.db-wal` + `vault.db-shm`. The WAL (write-ahead log) and shared-memory
+ * index files hold committed-but-not-yet-checkpointed pages; ignoring them
+ * undercounts a busy vault. A missing `-wal`/`-shm` (the common case at rest,
+ * after a checkpoint) contributes 0 rather than erroring.
+ *
+ * This is the PHYSICAL DB-file size — it includes SQLite page overhead, free
+ * pages from deletes, etc. It is intentionally distinct from the LOGICAL
+ * `contentBytes` in `VaultStats` (sum of note-content UTF-8 bytes).
+ */
+export function dbBytes(vaultName: string, fs: UsageFs = realUsageFs): number {
+  const base = join(vaultDir(vaultName), "vault.db");
+  const candidates = [base, `${base}-wal`, `${base}-shm`];
+  let total = 0;
+  for (const path of candidates) {
+    total += safeFileSize(path, fs);
+  }
+  return total;
+}
+
+/** Size of a single file, or 0 if it's missing / unreadable. */
+function safeFileSize(path: string, fs: UsageFs): number {
+  try {
+    return fs.statFile(path).size;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Recursively sum the size of every regular file under `dirPath`.
+ *
+ *   - Missing directory → 0 (never throws; a vault may have no assets / no
+ *     mirror yet).
+ *   - Symlinks are NOT followed — a symlink (file or dir) is skipped entirely.
+ *     This guards against symlink loops (which would hang an infinite walk)
+ *     and against a symlink that points at a large tree outside the vault
+ *     inflating the reported footprint.
+ *
+ * Uses an explicit stack (not recursion) so a deep tree can't blow the call
+ * stack, and reads dirents with types so we avoid an extra `stat` per entry
+ * just to learn directory-ness.
+ */
+export function dirSize(dirPath: string, fs: UsageFs = realUsageFs): number {
+  let total = 0;
+  const stack: string[] = [dirPath];
+
+  while (stack.length > 0) {
+    const current = stack.pop()!;
+    let entries: Dirent[];
+    try {
+      entries = fs.readDir(current);
+    } catch {
+      // Missing/unreadable dir (including the root) → contributes 0.
+      continue;
+    }
+    for (const entry of entries) {
+      // Never follow symlinks — neither symlinked files nor symlinked dirs.
+      if (entry.isSymbolicLink()) continue;
+      const childPath = join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(childPath);
+      } else if (entry.isFile()) {
+        total += safeFileSize(childPath, fs);
+      }
+      // Sockets/FIFOs/devices: ignored.
+    }
+  }
+  return total;
+}
+
+/**
+ * Resolve a vault's mirror directory on disk, or `null` when no mirror is
+ * configured / resolvable (never configured, or external-without-path). The
+ * usage endpoint reports `mirror: 0` (and omits it from the total) in that
+ * case.
+ */
+export function resolveVaultMirrorDir(vaultName: string): string | null {
+  const config = readMirrorConfigForVault(vaultName);
+  if (!config) return null;
+  return resolveMirrorPath(vaultDir(vaultName), config);
+}
+
+// ---------------------------------------------------------------------------
+// Dir-walk cache (the only expensive part)
+// ---------------------------------------------------------------------------
+
+export interface DirWalkResult {
+  /** Bytes under the vault's assets directory. */
+  assets: number;
+  /**
+   * Bytes under the vault's mirror directory, or `null` when no mirror is
+   * configured. `null` → omit `mirror` from the response (or report 0).
+   */
+  mirror: number | null;
+}
+
+interface CacheEntry {
+  value: DirWalkResult;
+  /** Epoch millis when this entry was computed. */
+  computedAt: number;
+}
+
+/** Default cache lifetime for the dir-walk numbers. */
+export const DIR_WALK_TTL_MS = 60_000;
+
+/**
+ * In-process, per-vault cache of the two dir-walk numbers. Module-level so it
+ * survives across requests within a server process. Tests construct their own
+ * `UsageCache` to get isolation + a controllable clock.
+ */
+export class UsageCache {
+  private readonly entries = new Map<string, CacheEntry>();
+
+  constructor(
+    private readonly fs: UsageFs = realUsageFs,
+    private readonly now: Clock = Date.now,
+    private readonly ttlMs: number = DIR_WALK_TTL_MS,
+  ) {}
+
+  /**
+   * Get the dir-walk numbers for a vault. Returns whether the numbers came
+   * from cache (`cached: true`) or were freshly walked (`cached: false`).
+   *
+   *   - `fresh: true` always recomputes (and refreshes the cache entry).
+   *   - Otherwise a non-expired entry is returned as-is (no walk); an absent
+   *     or expired entry triggers a walk + store.
+   */
+  get(vaultName: string, opts: { fresh?: boolean } = {}): { result: DirWalkResult; cached: boolean } {
+    const existing = this.entries.get(vaultName);
+    const ts = this.now();
+    if (!opts.fresh && existing && ts - existing.computedAt < this.ttlMs) {
+      return { result: existing.value, cached: true };
+    }
+    const value = this.compute(vaultName);
+    this.entries.set(vaultName, { value, computedAt: ts });
+    return { result: value, cached: false };
+  }
+
+  /** Drop a vault's cached dir-walk so the next read recomputes. */
+  invalidate(vaultName: string): void {
+    this.entries.delete(vaultName);
+  }
+
+  /** Run the (expensive) walks. */
+  private compute(vaultName: string): DirWalkResult {
+    const assets = dirSize(assetsDir(vaultName), this.fs);
+    const mirrorDir = resolveVaultMirrorDir(vaultName);
+    const mirror = mirrorDir === null ? null : dirSize(mirrorDir, this.fs);
+    return { assets, mirror };
+  }
+}
+
+/**
+ * The process-wide cache the live HTTP route uses. A single instance shared
+ * across requests gives the 60s TTL its meaning. `invalidateUsageCache` is the
+ * write-path hook (attachment upload, post-export).
+ */
+export const usageCache = new UsageCache();
+
+/** Invalidate the process-wide usage cache for a vault (write-path hook). */
+export function invalidateUsageCache(vaultName: string): void {
+  usageCache.invalidate(vaultName);
+}
+
+// ---------------------------------------------------------------------------
+// Report assembly (consumed by the HTTP route)
+// ---------------------------------------------------------------------------
+
+export interface UsageReport {
+  counts: {
+    notes: number;
+    attachments: number;
+    links: number;
+    tags: number;
+  };
+  bytes: {
+    /** Logical: sum of note-content UTF-8 bytes (from VaultStats). */
+    content: number;
+    /** Physical: vault.db + WAL + shm. */
+    db: number;
+    /** Bytes under the vault's assets directory (dir-walk, cached). */
+    assets: number;
+    /**
+     * Bytes under the vault's mirror directory (dir-walk, cached), present
+     * only when a mirror is configured. The mirror is a git PROJECTION of the
+     * same notes/attachments — it is NOT added to `total` (that would
+     * double-count the data); it's reported as a separate line item.
+     */
+    mirror?: number;
+    /**
+     * Physical on-disk footprint (db + assets); excludes content (inside db)
+     * and mirror (separate projection). This is the number an operator sizes a
+     * vault by.
+     */
+    total: number;
+  };
+  /** ISO-8601 timestamp of when this report was assembled. */
+  computedAt: string;
+  /** Whether the dir-walk numbers (assets/mirror) came from cache. */
+  cached: boolean;
+}
+
+/**
+ * Assemble the usage report for a vault. `stats` is the already-computed
+ * `getVaultStats()` result (counts + contentBytes); the cache supplies the two
+ * dir-walk numbers and reports whether they were cached.
+ *
+ * `total = db + assets` — the physical footprint. Mirror is a separate line
+ * item (projection of the same data; adding it double-counts). Content is the
+ * logical note size, already inside `db` physically.
+ */
+export function buildUsageReport(
+  vaultName: string,
+  stats: VaultStats,
+  opts: { fresh?: boolean; cache?: UsageCache; fs?: UsageFs; now?: Clock } = {},
+): UsageReport {
+  const cache = opts.cache ?? usageCache;
+  const { result, cached } = cache.get(vaultName, { fresh: opts.fresh });
+  const db = dbBytes(vaultName, opts.fs ?? realUsageFs);
+  const total = db + result.assets;
+
+  const bytes: UsageReport["bytes"] = {
+    content: stats.contentBytes,
+    db,
+    assets: result.assets,
+    total,
+  };
+  if (result.mirror !== null) bytes.mirror = result.mirror;
+
+  return {
+    counts: {
+      notes: stats.totalNotes,
+      attachments: stats.attachmentCount,
+      links: stats.linkCount,
+      tags: stats.tagCount,
+    },
+    bytes,
+    computedAt: new Date((opts.now ?? Date.now)()).toISOString(),
+    cached,
+  };
+}

package/src/vault-create.test.ts

@@ -10,7 +10,15 @@
 
 import { describe, test, expect, beforeEach, afterEach } from "bun:test";
 import { resolve } from "path";
-import { mkdtempSync, rmSync, existsSync, readFileSync } from "fs";
+import {
+  mkdtempSync,
+  rmSync,
+  existsSync,
+  readFileSync,
+  mkdirSync,
+  writeFileSync,
+  symlinkSync,
+} from "fs";
 import { tmpdir } from "os";
 import { join } from "path";
 
@@ -33,6 +41,23 @@ function runCli(
   };
 }
 
+/** Path to a vault's per-vault mirror-config file inside a temp PARACHUTE_HOME. */
+function mirrorConfigPath(home: string, name: string): string {
+  return join(home, "vault", "data", name, "mirror-config.yaml");
+}
+
+/** Path to a vault's internal mirror git working tree. */
+function mirrorRepoPath(home: string, name: string): string {
+  return join(home, "vault", "data", name, "mirror");
+}
+
+/** Write a minimal config.yaml carrying a `default_mirror` knob value. */
+function writeDefaultMirrorKnob(home: string, value: "internal" | "off"): void {
+  const dir = join(home, "vault");
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, "config.yaml"), `port: 1940\ndefault_mirror: ${value}\n`);
+}
+
 let home: string;
 
 beforeEach(() => {
@@ -107,7 +132,20 @@ describe("vault create --json", () => {
     );
     expect(exitCode).not.toBe(0);
     expect(stdout).toBe("");
-    expect(stderr).toContain("letters, numbers");
+    expect(stderr).toContain("lowercase alphanumeric");
+  });
+
+  test("UPPERCASE vault name is rejected (security review — audience case-drift)", () => {
+    // An uppercase name would flip the JWT audience case (vault.<Name> vs
+    // vault.<name>) and drift from hub/init lowercasing. cmdCreate must
+    // reject it the same way `init` does.
+    const { exitCode, stdout, stderr } = runCli(
+      ["create", "MyVault", "--json"],
+      { PARACHUTE_HOME: home },
+    );
+    expect(exitCode).not.toBe(0);
+    expect(stdout).toBe("");
+    expect(stderr).toContain("lowercase");
   });
 
   test("duplicate name in --json mode errors on stderr and exits non-zero", () => {
@@ -199,3 +237,157 @@ describe("vault create (human mode)", () => {
     expect(() => JSON.parse(stdout.trim())).toThrow();
   });
 });
+
+/**
+ * Create-time default backup posture: new vaults default to an internal live
+ * mirror (local git backup of the markdown projection). The History preset
+ * `{enabled:true, location:internal, sync_mode:events, auto_commit:true,
+ * auto_push:false}` is written to `data/<vault>/mirror-config.yaml` and (when
+ * git is present) the internal mirror dir is git-bootstrapped. The behavior is
+ * controlled by the server-wide `default_mirror` knob (default `internal`) and
+ * overridable per-create by `--no-mirror`.
+ *
+ * Critically: create-time ONLY — already-created vaults are NOT retroactively
+ * migrated, and the git-less box stays a successful create (best-effort
+ * bootstrap, config still written, actionable log).
+ */
+describe("vault create — default internal live mirror", () => {
+  test("default (no knob) → History-preset mirror config written + git-bootstrapped", () => {
+    const { exitCode } = runCli(["create", "backed", "--json"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(0);
+
+    // Mirror config exists with the exact History preset.
+    const cfgPath = mirrorConfigPath(home, "backed");
+    expect(existsSync(cfgPath)).toBe(true);
+    const cfg = readFileSync(cfgPath, "utf-8");
+    expect(cfg).toContain("enabled: true");
+    expect(cfg).toContain("location: internal");
+    expect(cfg).toContain("sync_mode: events");
+    expect(cfg).toContain("auto_commit: true");
+    expect(cfg).toContain("auto_push: false");
+
+    // Git present on the test host → the internal mirror dir is a real repo
+    // with the seed commit (bootstrap ran). counts/usage reflect it: the
+    // mirror working tree exists under the vault data dir.
+    const repo = mirrorRepoPath(home, "backed");
+    expect(existsSync(join(repo, ".git"))).toBe(true);
+    const log = Bun.spawnSync({
+      cmd: ["git", "-C", repo, "log", "--oneline"],
+      stdout: "pipe",
+    });
+    expect(new TextDecoder().decode(log.stdout)).toContain("initial mirror bootstrap");
+  });
+
+  test("default_mirror: off knob → no mirror config written", () => {
+    writeDefaultMirrorKnob(home, "off");
+    const { exitCode } = runCli(["create", "bare", "--json"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(0);
+
+    // Vault is created…
+    expect(existsSync(join(home, "vault", "data", "bare", "vault.db"))).toBe(true);
+    // …but NO mirror config + NO mirror dir.
+    expect(existsSync(mirrorConfigPath(home, "bare"))).toBe(false);
+    expect(existsSync(mirrorRepoPath(home, "bare"))).toBe(false);
+  });
+
+  test("default_mirror: internal knob (explicit) → mirror config written", () => {
+    writeDefaultMirrorKnob(home, "internal");
+    const { exitCode } = runCli(["create", "explicit", "--json"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(0);
+    expect(existsSync(mirrorConfigPath(home, "explicit"))).toBe(true);
+  });
+
+  test("--no-mirror → no mirror config even when knob is internal", () => {
+    writeDefaultMirrorKnob(home, "internal");
+    const { exitCode } = runCli(["create", "optout", "--no-mirror", "--json"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(0);
+
+    expect(existsSync(join(home, "vault", "data", "optout", "vault.db"))).toBe(true);
+    expect(existsSync(mirrorConfigPath(home, "optout"))).toBe(false);
+    expect(existsSync(mirrorRepoPath(home, "optout"))).toBe(false);
+  });
+
+  test("--no-mirror in human mode keeps the rest of the create working", () => {
+    const { exitCode, stdout } = runCli(["create", "humanbare", "--no-mirror"], {
+      PARACHUTE_HOME: home,
+    });
+    expect(exitCode).toBe(0);
+    expect(stdout).toContain('Vault "humanbare" created.');
+    expect(existsSync(mirrorConfigPath(home, "humanbare"))).toBe(false);
+  });
+
+  test("git missing → vault still creates, config written, mirror NOT bootstrapped, actionable log, no crash", () => {
+    // Simulate a git-less server: spawn the CLI with a PATH that resolves
+    // `bun` (so the test can run) but NOT `git`. `bootstrapInternalMirror`'s
+    // `Bun.which("git")` preflight then returns null → friendly,
+    // best-effort failure. The vault create MUST still succeed.
+    const bunBin = Bun.which("bun");
+    expect(bunBin).toBeTruthy();
+    const fakeBin = mkdtempSync(join(tmpdir(), "vault-create-nogit-bin-"));
+    symlinkSync(bunBin!, join(fakeBin, "bun"));
+    try {
+      const proc = Bun.spawnSync({
+        cmd: [bunBin!, CLI, "create", "nogit", "--json"],
+        stdout: "pipe",
+        stderr: "pipe",
+        // Replace PATH entirely so `git` is unresolvable — only our fake bin
+        // (bun only) is on the path.
+        env: { ...process.env, PARACHUTE_HOME: home, PATH: fakeBin },
+      });
+      const exitCode = proc.exitCode ?? -1;
+      const stdout = new TextDecoder().decode(proc.stdout);
+      const stderr = new TextDecoder().decode(proc.stderr);
+
+      // No crash — the vault create succeeds on a git-less box.
+      expect(exitCode).toBe(0);
+      // The vault itself was created.
+      expect(existsSync(join(home, "vault", "data", "nogit", "vault.db"))).toBe(true);
+      // The mirror CONFIG was still written (intent persists for when git
+      // lands later).
+      expect(existsSync(mirrorConfigPath(home, "nogit"))).toBe(true);
+      // But the mirror was NOT git-bootstrapped (no .git — git was absent).
+      expect(existsSync(join(mirrorRepoPath(home, "nogit"), ".git"))).toBe(false);
+      // And the operator got an actionable, clear log on stderr.
+      expect(stderr).toContain("local git backup configured but not yet active");
+      expect(stderr).toContain("Install git");
+      // JSON stdout stays clean + parseable (the note went to stderr).
+      const payload = JSON.parse(stdout.trim());
+      expect(payload.name).toBe("nogit");
+    } finally {
+      rmSync(fakeBin, { recursive: true, force: true });
+    }
+  });
+
+  test("EXISTING vault (created before this change) is NOT auto-migrated on a later create", () => {
+    // Simulate a vault that predates the default-mirror behavior: create it
+    // with the knob OFF so it gets no mirror config (stand-in for "created by
+    // an older vault build").
+    writeDefaultMirrorKnob(home, "off");
+    const first = runCli(["create", "legacy", "--json"], { PARACHUTE_HOME: home });
+    expect(first.exitCode).toBe(0);
+    expect(existsSync(mirrorConfigPath(home, "legacy"))).toBe(false);
+
+    // Now flip the knob ON and create a SECOND, different vault. The act of
+    // creating "fresh" (which DOES get a mirror) must not retroactively write
+    // a mirror config onto the pre-existing "legacy" vault — create-time only,
+    // never a sweep over existing vaults.
+    writeDefaultMirrorKnob(home, "internal");
+    const second = runCli(["create", "fresh", "--json"], { PARACHUTE_HOME: home });
+    expect(second.exitCode).toBe(0);
+
+    // The new vault is backed…
+    expect(existsSync(mirrorConfigPath(home, "fresh"))).toBe(true);
+    // …but the pre-existing vault is left exactly as it was — no surprise
+    // mirror config, no surprise disk-doubling mirror repo.
+    expect(existsSync(mirrorConfigPath(home, "legacy"))).toBe(false);
+    expect(existsSync(mirrorRepoPath(home, "legacy"))).toBe(false);
+  });
+});