@openparachute/vault 0.3.3 → 0.4.0

package/core/src/schema.ts

@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
 import { normalizePath } from "./paths.js";
 import { rebuildIndexes } from "./indexed-fields.js";
 
-export const SCHEMA_VERSION = 12;
+export const SCHEMA_VERSION = 16;
 
 export const SCHEMA_SQL = `
 -- Notes: the universal record
@@ -15,9 +15,27 @@ CREATE TABLE IF NOT EXISTS notes (
   updated_at TEXT
 );
 
--- Tags: flat labels
+-- Tags: first-class identity carrying schema, hierarchy, and typed-link
+-- declarations. One row per tag; no notes-as-config sidecars for these
+-- concerns. See parachute-patterns/patterns/tag-data-model.md.
+--
+-- description    — human-readable blurb (markdown).
+-- fields         — JSON: indexed metadata field declarations per
+--                  query-operators.md. Replaces v6-era tag_schemas.fields.
+-- relationships  — JSON: typed-link declarations
+--                  ({ "rel": { target_tag, cardinality, description? } }).
+--                  Cardinality vocabulary: one | optional | many | many-required.
+--                  Phase 1 informational — declared but not enforced at write.
+-- parent_names   — JSON array of parent tag names. Replaces the v6-era
+--                  _tags/NAME config-note hierarchy.
 CREATE TABLE IF NOT EXISTS tags (
-  name TEXT PRIMARY KEY
+  name TEXT PRIMARY KEY,
+  description TEXT,
+  fields TEXT,
+  relationships TEXT,
+  parent_names TEXT,
+  created_at TEXT,
+  updated_at TEXT
 );
 
 -- Note-Tag join
@@ -47,11 +65,40 @@ CREATE TABLE IF NOT EXISTS links (
   UNIQUE(source_id, target_id, relationship)
 );
 
--- Tag schemas: optional metadata schema per tag
-CREATE TABLE IF NOT EXISTS tag_schemas (
-  tag_name TEXT PRIMARY KEY REFERENCES tags(name) ON DELETE CASCADE,
+-- tag_schemas (v6) was retired in v14; description + fields lifted onto the
+-- tags row directly. The CREATE TABLE was removed from SCHEMA_SQL after the
+-- v14 data migration drops the table; existing v6+ vaults pick up the
+-- migration on next boot. See migrateToV14.
+
+-- Note schemas (v15): schema definitions used to validate notes by path
+-- prefix or tag. Replaces the v6-era _schemas/NAME notes-as-config
+-- convention. Validation is non-blocking — schemas surface warnings on
+-- create/update responses, never reject the write. See
+-- core/src/schema-defaults.ts and patterns/tag-data-model.md §Note schemas.
+--
+-- name        — primary key; the schema identifier referenced by mappings.
+-- description — human-readable blurb (markdown).
+-- fields      — JSON: { fieldName: { type?, enum?, description? } }.
+-- required    — JSON: string[] of required field names.
+CREATE TABLE IF NOT EXISTS note_schemas (
+  name TEXT PRIMARY KEY,
   description TEXT,
-  fields TEXT -- JSON: { "field_name": { "type": "string", "description": "..." }, ... }
+  fields TEXT,
+  required TEXT,
+  created_at TEXT,
+  updated_at TEXT
+);
+
+-- Schema mappings (v15): replaces the singleton _schema_defaults note. One
+-- row per match rule; the resolver walks the table at note-write time.
+-- match_kind is constrained to 'path_prefix' or 'tag'. Composite PK so
+-- (schema, kind, value) is naturally unique without an extra surrogate id.
+-- ON DELETE CASCADE: dropping a schema cleans up its mappings.
+CREATE TABLE IF NOT EXISTS schema_mappings (
+  schema_name TEXT NOT NULL REFERENCES note_schemas(name) ON DELETE CASCADE,
+  match_kind TEXT NOT NULL CHECK (match_kind IN ('path_prefix', 'tag')),
+  match_value TEXT NOT NULL,
+  PRIMARY KEY (schema_name, match_kind, match_value)
 );
 
 -- Indexed fields: SSOT for generated columns and indexes on notes derived
@@ -73,6 +120,19 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
 -- one-release-cycle back-compat window and will be dropped in a future
 -- migration.
 --
+-- scoped_tags is a JSON-encoded array of root tag names that constrain the
+-- token's effective access (intersection with the scopes column). NULL
+-- means unscoped — full vault access per scopes. Introduced in v13 per
+-- patterns/tag-scoped-tokens.md. Hierarchy expansion is applied at auth
+-- time via getTagDescendants; the column stores root names only.
+--
+-- vault_name (v16) binds the token to a single vault. NULL means the
+-- token is server-wide / legacy (pre-v16 rows backfill to NULL on the
+-- migration; auth treats NULL as accept-any-vault for back-compat).
+-- New tokens minted via per-vault routes write the column explicitly so
+-- cross-vault presentation rejects in authenticateVaultRequest. See
+-- vault#257.
+--
 -- scope_tag / scope_path_prefix are deprecated Phase-0 columns — never
 -- enforced at runtime, kept only for schema stability.
 CREATE TABLE IF NOT EXISTS tokens (
@@ -80,11 +140,13 @@ CREATE TABLE IF NOT EXISTS tokens (
   label TEXT NOT NULL,
   permission TEXT NOT NULL DEFAULT 'admin',
   scopes TEXT,
+  scoped_tags TEXT,
   scope_tag TEXT,
   scope_path_prefix TEXT,
   expires_at TEXT,
   created_at TEXT NOT NULL,
-  last_used_at TEXT
+  last_used_at TEXT,
+  vault_name TEXT
 );
 
 -- OAuth: registered clients (Dynamic Client Registration)
@@ -147,6 +209,13 @@ CREATE INDEX IF NOT EXISTS idx_note_tags_tag ON note_tags(tag_name, note_id);
 CREATE INDEX IF NOT EXISTS idx_attachments_note ON attachments(note_id);
 CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_id);
 CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_id);
+CREATE INDEX IF NOT EXISTS idx_schema_mappings_match ON schema_mappings(match_kind, match_value);
+-- idx_tokens_vault_name is created in migrateToV16, not here. SCHEMA_SQL
+-- runs BEFORE migrations; an upgrading v15 vault doesn't yet have the
+-- vault_name column when this section evaluates, so the index has to
+-- live downstream of the ALTER TABLE that adds the column. Fresh vaults
+-- (column already present from this CREATE TABLE) still get the index
+-- because migrateToV16 also runs the unconditional CREATE INDEX path.
 `;
 
 /**
@@ -194,6 +263,28 @@ export function initSchema(db: Database): void {
   // Migrate v11 → v12: add `scopes` column to tokens for Phase 2 enforcement.
   migrateToV12(db);
 
+  // Migrate v12 → v13: add `scoped_tags` column to tokens for tag-scoped tokens.
+  migrateToV13(db);
+
+  // Migrate v13 → v14: tag-data-model reshape. Augment `tags` row with
+  // description/fields/relationships/parent_names/timestamps; copy data
+  // from the v6-era tag_schemas sidecar and from `_tags/<name>` config
+  // notes; drop tag_schemas after copy. See patterns/tag-data-model.md.
+  migrateToV14(db);
+
+  // Migrate v14 → v15: retire the `_schemas/<name>` and `_schema_defaults`
+  // notes-as-config sidecars. Copy each `_schemas/<name>` note into the
+  // new `note_schemas` table and the `_schema_defaults` mappings into
+  // `schema_mappings`. The legacy notes are LEFT IN PLACE — they are
+  // inert post-v15 (no resolver reads them) and serve as audit trail.
+  migrateToV15(db);
+
+  // Migrate v15 → v16: add `vault_name` column to tokens. Existing rows
+  // backfill to NULL ("server-wide / legacy" semantic) — auth accepts
+  // NULL for any vault, so today's pvt_* tokens keep working unchanged.
+  // New mints via per-vault routes write the column explicitly. See vault#257.
+  migrateToV16(db);
+
   // Rebuild any generated columns + indexes declared in indexed_fields.
   // No-op for a fresh vault; idempotent on existing vaults.
   rebuildIndexes(db);
@@ -258,7 +349,7 @@ function migrateToV5(db: Database): void {
     // Keep first, rename the rest
     for (let i = 1; i < ids.length; i++) {
       const newPath = `${dupe.path}-${i}`;
-      db.prepare("UPDATE notes SET path = ? WHERE id = ?").run(newPath, ids[i]);
+      db.prepare("UPDATE notes SET path = ? WHERE id = ?").run(newPath, ids[i]!);
     }
   }
 
@@ -337,6 +428,299 @@ function migrateToV12(db: Database): void {
   }
 }
 
+/**
+ * Migrate v12 → v13: add `scoped_tags` column to tokens. NULL means unscoped
+ * (current full-vault behavior); a JSON array of root tag names narrows the
+ * token's access to notes carrying one of those tags or a sub-tag thereof
+ * (hierarchy expansion via getTagDescendants at auth time). See
+ * parachute-patterns/patterns/tag-scoped-tokens.md.
+ */
+function migrateToV13(db: Database): void {
+  if (hasTable(db, "tokens") && !hasColumn(db, "tokens", "scoped_tags")) {
+    db.exec("ALTER TABLE tokens ADD COLUMN scoped_tags TEXT");
+  }
+}
+
+/**
+ * Migrate v13 → v14: tag-data-model reshape (patterns/tag-data-model.md).
+ *
+ * Augments the `tags` table with five new columns and one timestamp pair,
+ * then copies pre-existing data from two notes-as-config sidecars:
+ *
+ *   - tag_schemas (v6 sidecar) → tags.{description,fields}
+ *   - notes at path `_tags/<name>` → tags.parent_names (from metadata.parents)
+ *
+ * After the copy lands, `tag_schemas` is dropped. The `_tags/<name>` notes
+ * are LEFT IN PLACE — they're harmless historical record and a user might
+ * have other content there. Future writes go to the tags row directly.
+ *
+ * Wrapped in BEGIN IMMEDIATE / COMMIT (with a try/catch ROLLBACK) so a
+ * crash mid-migration leaves the DB in either pre-v14 or post-v14 state,
+ * never half-migrated. Each step remains individually idempotent — the
+ * transaction wrap is belt-and-suspenders, not load-bearing — so a future
+ * reader who removes the `hasColumn` / `hasTable` guards still gets correct
+ * behavior on retry.
+ */
+function migrateToV14(db: Database): void {
+  if (!hasTable(db, "tags")) return;
+
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    // 1. ALTER TABLE — additive, idempotent.
+    const cols: [string, string][] = [
+      ["description", "TEXT"],
+      ["fields", "TEXT"],
+      ["relationships", "TEXT"],
+      ["parent_names", "TEXT"],
+      ["created_at", "TEXT"],
+      ["updated_at", "TEXT"],
+    ];
+    for (const [col, type] of cols) {
+      if (!hasColumn(db, "tags", col)) {
+        db.exec(`ALTER TABLE tags ADD COLUMN ${col} ${type}`);
+      }
+    }
+
+    const now = new Date().toISOString();
+    let copiedSchemas = 0;
+    let copiedHierarchy = 0;
+
+    // 2. Copy tag_schemas → tags.{description,fields}.
+    if (hasTable(db, "tag_schemas")) {
+      const rows = db.prepare(
+        "SELECT tag_name, description, fields FROM tag_schemas",
+      ).all() as { tag_name: string; description: string | null; fields: string | null }[];
+      const upsert = db.prepare(
+        "INSERT OR IGNORE INTO tags (name, created_at, updated_at) VALUES (?, ?, ?)",
+      );
+      const update = db.prepare(
+        "UPDATE tags SET description = ?, fields = ?, updated_at = ? WHERE name = ?",
+      );
+      for (const row of rows) {
+        upsert.run(row.tag_name, now, now);
+        update.run(row.description, row.fields, now, row.tag_name);
+        copiedSchemas++;
+      }
+    }
+
+    // 3. Copy `_tags/<name>` notes' metadata.parents → tags.parent_names.
+    // Only runs if the notes table exists (it always does post-SCHEMA_SQL,
+    // but stay defensive — initSchema runs SCHEMA_SQL first so this is true).
+    if (hasTable(db, "notes")) {
+      const tagNotes = db.prepare(
+        "SELECT path, metadata FROM notes WHERE path GLOB '_tags/*'",
+      ).all() as { path: string; metadata: string | null }[];
+      const upsert = db.prepare(
+        "INSERT OR IGNORE INTO tags (name, created_at, updated_at) VALUES (?, ?, ?)",
+      );
+      const update = db.prepare(
+        "UPDATE tags SET parent_names = ?, updated_at = ? WHERE name = ?",
+      );
+      for (const note of tagNotes) {
+        const tagName = note.path.slice("_tags/".length);
+        if (!tagName) continue;
+        let parents: string[] | null = null;
+        try {
+          const meta = note.metadata ? JSON.parse(note.metadata) : {};
+          const raw = meta?.parents;
+          if (Array.isArray(raw) && raw.length > 0) {
+            const cleaned = raw.filter((p: unknown): p is string => typeof p === "string" && p.length > 0);
+            if (cleaned.length > 0) parents = cleaned;
+          }
+        } catch {
+          // Malformed metadata — skip; the note is left untouched.
+          continue;
+        }
+        if (!parents) continue;
+        upsert.run(tagName, now, now);
+        update.run(JSON.stringify(parents), now, tagName);
+        copiedHierarchy++;
+      }
+    }
+
+    // 4. Backfill timestamps for rows the copies didn't touch.
+    db.exec(`UPDATE tags SET created_at = '${now}' WHERE created_at IS NULL`);
+
+    // 5. Drop the sidecar after the copy is complete.
+    if (hasTable(db, "tag_schemas")) {
+      db.exec("DROP TABLE tag_schemas");
+    }
+
+    db.exec("COMMIT");
+
+    if (copiedSchemas > 0 || copiedHierarchy > 0) {
+      console.log(
+        `[vault] migrated to schema v14: copied ${copiedSchemas} tag_schemas + ${copiedHierarchy} _tags/* hierarchies onto tags rows`,
+      );
+    }
+  } catch (err) {
+    db.exec("ROLLBACK");
+    throw err;
+  }
+}
+
+/**
+ * Migrate v14 → v15: retire `_schemas/<name>` + `_schema_defaults` notes
+ * as the canonical source for schema definitions and mapping rules. After
+ * this migration the resolver reads from `note_schemas` and
+ * `schema_mappings` tables. The legacy notes are LEFT IN PLACE — they're
+ * harmless historical record and a user might have other content there.
+ *
+ * Idempotent: SCHEMA_SQL creates the tables before this runs (CREATE TABLE
+ * IF NOT EXISTS); the data copy uses INSERT OR IGNORE so re-running on a
+ * post-v15 DB is a no-op. Wrapped in BEGIN/COMMIT so a crash mid-migration
+ * leaves the DB in either pre-v15 or post-v15 state, never partial.
+ */
+function migrateToV15(db: Database): void {
+  if (!hasTable(db, "note_schemas") || !hasTable(db, "notes")) return;
+
+  // Short-circuit: if either destination table already has data, the
+  // migration has run before. `||` not `&&` — a vault with schemas but zero
+  // mappings (or mappings but zero schemas) is a valid post-v15 state, and
+  // re-scanning notes on every boot would be wasted I/O.
+  const hasSchemas = (db.prepare(
+    "SELECT 1 FROM note_schemas LIMIT 1",
+  ).get()) !== null;
+  const hasMappings = (db.prepare(
+    "SELECT 1 FROM schema_mappings LIMIT 1",
+  ).get()) !== null;
+  if (hasSchemas || hasMappings) return;
+
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    const now = new Date().toISOString();
+    let copiedSchemas = 0;
+    let copiedMappings = 0;
+
+    // 1. Copy `_schemas/<name>` notes → note_schemas.
+    const defRows = db.prepare(
+      "SELECT path, metadata FROM notes WHERE path GLOB '_schemas/*'",
+    ).all() as { path: string; metadata: string | null }[];
+    const insertSchema = db.prepare(
+      "INSERT OR IGNORE INTO note_schemas (name, description, fields, required, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)",
+    );
+    for (const row of defRows) {
+      const name = row.path.slice("_schemas/".length);
+      if (!name) continue;
+      let description: string | null = null;
+      let fields: string | null = null;
+      let required: string | null = null;
+      try {
+        const meta = row.metadata ? JSON.parse(row.metadata) : {};
+        if (typeof meta?.description === "string") description = meta.description;
+        if (meta?.fields && typeof meta.fields === "object" && !Array.isArray(meta.fields)) {
+          fields = JSON.stringify(meta.fields);
+        }
+        if (Array.isArray(meta?.required)) {
+          const cleaned = meta.required.filter((x: unknown): x is string => typeof x === "string");
+          if (cleaned.length > 0) required = JSON.stringify(cleaned);
+        }
+      } catch {
+        // Malformed metadata — skip; the note is left alone.
+        continue;
+      }
+      insertSchema.run(name, description, fields, required, now, now);
+      copiedSchemas++;
+    }
+
+    // 2. Copy `_schema_defaults` note → schema_mappings.
+    const mappingNote = db.prepare(
+      "SELECT metadata FROM notes WHERE path = '_schema_defaults'",
+    ).get() as { metadata: string | null } | undefined;
+    if (mappingNote?.metadata) {
+      const insertMapping = db.prepare(
+        "INSERT OR IGNORE INTO schema_mappings (schema_name, match_kind, match_value) VALUES (?, ?, ?)",
+      );
+      const ensureSchemaRow = db.prepare(
+        "INSERT OR IGNORE INTO note_schemas (name, created_at, updated_at) VALUES (?, ?, ?)",
+      );
+      try {
+        const meta = JSON.parse(mappingNote.metadata);
+        const pathPrefixes = meta?.path_prefixes;
+        if (pathPrefixes && typeof pathPrefixes === "object" && !Array.isArray(pathPrefixes)) {
+          for (const [prefix, schema] of Object.entries(pathPrefixes as Record<string, unknown>)) {
+            if (typeof schema === "string" && schema.length > 0 && prefix.length > 0) {
+              // Foreign key requires the schema row to exist; create a stub
+              // if the user mapped to a name with no _schemas/<name> note.
+              ensureSchemaRow.run(schema, now, now);
+              insertMapping.run(schema, "path_prefix", prefix);
+              copiedMappings++;
+            }
+          }
+        }
+        const tags = meta?.tags;
+        if (tags && typeof tags === "object" && !Array.isArray(tags)) {
+          for (const [tag, schema] of Object.entries(tags as Record<string, unknown>)) {
+            if (typeof schema === "string" && schema.length > 0 && tag.length > 0) {
+              ensureSchemaRow.run(schema, now, now);
+              insertMapping.run(schema, "tag", tag);
+              copiedMappings++;
+            }
+          }
+        }
+      } catch {
+        // Malformed _schema_defaults — leave both note and table empty;
+        // user can fix and re-run.
+      }
+    }
+
+    db.exec("COMMIT");
+
+    if (copiedSchemas > 0 || copiedMappings > 0) {
+      console.log(
+        `[vault] migrated to schema v15: copied ${copiedSchemas} _schemas/* + ${copiedMappings} _schema_defaults mappings into note_schemas/schema_mappings`,
+      );
+    }
+  } catch (err) {
+    db.exec("ROLLBACK");
+    throw err;
+  }
+}
+
+/**
+ * Migrate v15 → v16: per-vault token storage (vault#257).
+ *
+ * Adds `tokens.vault_name TEXT` (nullable). Existing rows stay NULL —
+ * "server-wide / legacy" semantic — and `authenticateVaultRequest`
+ * accepts NULL for any vault, so today's pvt_* tokens keep working
+ * unchanged. New mints via `/vault/<name>/tokens` write the column
+ * explicitly; cross-vault presentation rejects on the row's vault_name
+ * mismatch. The complementary index speeds the per-vault listTokens
+ * filter in the admin SPA.
+ *
+ * Wrapped in BEGIN IMMEDIATE / COMMIT (with try/catch ROLLBACK) per the
+ * v14/v15 wrap pattern from vault#251 — the column add and index create
+ * are individually idempotent (`hasColumn` / IF NOT EXISTS), but the
+ * transaction means a crash mid-migration leaves either pre-v16 or
+ * post-v16 state, never partial.
+ */
+function migrateToV16(db: Database): void {
+  if (!hasTable(db, "tokens")) return;
+
+  // Two responsibilities, separated so the index lands for both fresh
+  // vaults (SCHEMA_SQL already created the column) and upgrading v15
+  // vaults (column missing). Index creation lives outside the wrapped
+  // ALTER block so a fresh vault — where the column exists but the index
+  // doesn't — still gets it.
+  if (!hasColumn(db, "tokens", "vault_name")) {
+    db.exec("BEGIN IMMEDIATE");
+    try {
+      db.exec("ALTER TABLE tokens ADD COLUMN vault_name TEXT");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_tokens_vault_name ON tokens(vault_name)");
+      db.exec("COMMIT");
+    } catch (err) {
+      db.exec("ROLLBACK");
+      throw err;
+    }
+    return;
+  }
+
+  // Column exists (fresh vault from SCHEMA_SQL, or post-upgrade vault).
+  // Make sure the index exists too. IF NOT EXISTS makes this a no-op on
+  // the steady-state path.
+  db.exec("CREATE INDEX IF NOT EXISTS idx_tokens_vault_name ON tokens(vault_name)");
+}
+
 function hasTable(db: Database, name: string): boolean {
   const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
   return !!row;