@openparachute/vault 0.2.3 → 0.3.0-rc.1

package/core/src/indexed-fields.test.ts

@@ -0,0 +1,285 @@
+import { describe, it, expect, beforeEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { SqliteStore } from "./store.js";
+import { generateMcpTools, type McpToolDef } from "./mcp.js";
+import {
+  IndexedFieldError,
+  declareField,
+  getIndexedField,
+  listIndexedFields,
+  rebuildIndexes,
+  releaseField,
+  TYPE_MAP,
+  validateFieldName,
+} from "./indexed-fields.js";
+
+let db: Database;
+let store: SqliteStore;
+let tools: Record<string, McpToolDef>;
+
+function findTool(name: string): McpToolDef {
+  const t = tools[name];
+  if (!t) throw new Error(`tool not found: ${name}`);
+  return t;
+}
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  store = new SqliteStore(db);
+  tools = Object.fromEntries(generateMcpTools(store).map((t) => [t.name, t]));
+});
+
+function notesColumns(): string[] {
+  return (db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map(
+    (r) => r.name,
+  );
+}
+
+function notesIndexes(): string[] {
+  return (
+    db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='notes'").all() as {
+      name: string;
+    }[]
+  ).map((r) => r.name);
+}
+
+describe("indexed-fields: module", () => {
+  it("schema creates the indexed_fields table", () => {
+    const row = db
+      .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='indexed_fields'")
+      .get();
+    expect(row).toBeTruthy();
+  });
+
+  it("validateFieldName accepts safe identifiers", () => {
+    expect(() => validateFieldName("status")).not.toThrow();
+    expect(() => validateFieldName("first_seen_at")).not.toThrow();
+    expect(() => validateFieldName("_private")).not.toThrow();
+  });
+
+  it("validateFieldName rejects unsafe names", () => {
+    expect(() => validateFieldName("has-dash")).toThrow(IndexedFieldError);
+    expect(() => validateFieldName("1leading_digit")).toThrow(IndexedFieldError);
+    expect(() => validateFieldName("has space")).toThrow(IndexedFieldError);
+    expect(() => validateFieldName("'; DROP TABLE notes; --")).toThrow(IndexedFieldError);
+  });
+
+  it("TYPE_MAP covers string/integer/boolean", () => {
+    expect(TYPE_MAP.string).toBe("TEXT");
+    expect(TYPE_MAP.integer).toBe("INTEGER");
+    expect(TYPE_MAP.boolean).toBe("INTEGER");
+  });
+
+  it("declareField creates column + index on first declaration", () => {
+    declareField(db, "status", "TEXT", "project");
+    expect(notesColumns()).toContain("meta_status");
+    expect(notesIndexes()).toContain("idx_meta_status");
+    const row = getIndexedField(db, "status");
+    expect(row?.sqliteType).toBe("TEXT");
+    expect(row?.declarerTags).toEqual(["project"]);
+  });
+
+  it("declareField adds second declarer without duplicating the column", () => {
+    declareField(db, "status", "TEXT", "project");
+    declareField(db, "status", "TEXT", "ticket");
+    const row = getIndexedField(db, "status");
+    expect(row?.declarerTags).toEqual(["project", "ticket"]);
+    const statusColumns = notesColumns().filter((c) => c === "meta_status");
+    expect(statusColumns).toHaveLength(1);
+  });
+
+  it("declareField is idempotent when the tag is already a declarer", () => {
+    declareField(db, "status", "TEXT", "project");
+    declareField(db, "status", "TEXT", "project");
+    const row = getIndexedField(db, "status");
+    expect(row?.declarerTags).toEqual(["project"]);
+  });
+
+  it("declareField throws on type mismatch with other declarers", () => {
+    declareField(db, "priority", "INTEGER", "project");
+    expect(() => declareField(db, "priority", "TEXT", "ticket")).toThrow(IndexedFieldError);
+    const row = getIndexedField(db, "priority");
+    expect(row?.sqliteType).toBe("INTEGER");
+    expect(row?.declarerTags).toEqual(["project"]);
+  });
+
+  it("declareField allows a sole declarer to change type (drops + recreates)", () => {
+    declareField(db, "priority", "TEXT", "project");
+    declareField(db, "priority", "INTEGER", "project");
+    const row = getIndexedField(db, "priority");
+    expect(row?.sqliteType).toBe("INTEGER");
+  });
+
+  it("releaseField removes a single declarer but keeps column while others remain", () => {
+    declareField(db, "status", "TEXT", "project");
+    declareField(db, "status", "TEXT", "ticket");
+    const dropped = releaseField(db, "status", "project");
+    expect(dropped).toBe(false);
+    expect(notesColumns()).toContain("meta_status");
+    const row = getIndexedField(db, "status");
+    expect(row?.declarerTags).toEqual(["ticket"]);
+  });
+
+  it("releaseField drops column + index when last declarer leaves", () => {
+    declareField(db, "status", "TEXT", "project");
+    const dropped = releaseField(db, "status", "project");
+    expect(dropped).toBe(true);
+    expect(notesColumns()).not.toContain("meta_status");
+    expect(notesIndexes()).not.toContain("idx_meta_status");
+    expect(getIndexedField(db, "status")).toBeNull();
+  });
+
+  it("releaseField is a no-op for unknown field", () => {
+    expect(releaseField(db, "nonexistent", "t")).toBe(false);
+  });
+
+  it("indexed column reflects json_extract of metadata at query time", async () => {
+    declareField(db, "priority", "INTEGER", "project");
+    await store.createNote("a", { metadata: { priority: 1 } });
+    await store.createNote("b", { metadata: { priority: 5 } });
+    await store.createNote("c", { metadata: { priority: 3 } });
+    const rows = db
+      .prepare("SELECT id, meta_priority FROM notes ORDER BY meta_priority")
+      .all() as { id: string; meta_priority: number }[];
+    expect(rows.map((r) => r.meta_priority)).toEqual([1, 3, 5]);
+  });
+
+  it("listIndexedFields returns rows ordered by field", () => {
+    declareField(db, "zeta", "TEXT", "t");
+    declareField(db, "alpha", "TEXT", "t");
+    declareField(db, "mu", "TEXT", "t");
+    expect(listIndexedFields(db).map((f) => f.field)).toEqual(["alpha", "mu", "zeta"]);
+  });
+
+  it("rebuildIndexes restores columns that are missing from notes", () => {
+    declareField(db, "status", "TEXT", "project");
+    // Simulate a stale DB: drop the column while leaving the row intact.
+    db.exec('DROP INDEX IF EXISTS "idx_meta_status"');
+    db.exec('ALTER TABLE notes DROP COLUMN "meta_status"');
+    expect(notesColumns()).not.toContain("meta_status");
+    rebuildIndexes(db);
+    expect(notesColumns()).toContain("meta_status");
+    expect(notesIndexes()).toContain("idx_meta_status");
+  });
+});
+
+describe("update-tag: indexed flag", () => {
+  it("declaring indexed field creates column + index", async () => {
+    await findTool("update-tag").execute({
+      tag: "project",
+      fields: { status: { type: "string", indexed: true } },
+    });
+    expect(notesColumns()).toContain("meta_status");
+    expect(notesIndexes()).toContain("idx_meta_status");
+    expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project"]);
+  });
+
+  it("second declarer with matching type joins the declarer set", async () => {
+    const t = findTool("update-tag");
+    await t.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
+    await t.execute({ tag: "ticket", fields: { status: { type: "string", indexed: true } } });
+    expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project", "ticket"]);
+  });
+
+  it("type conflict across declarers throws and names the other tag", async () => {
+    const t = findTool("update-tag");
+    await t.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
+    expect(() =>
+      t.execute({ tag: "ticket", fields: { status: { type: "integer", indexed: true } } }),
+    ).toThrow(/tag "project".*"string"/);
+  });
+
+  it("indexed-flag conflict across declarers throws", async () => {
+    const t = findTool("update-tag");
+    await t.execute({ tag: "project", fields: { priority: { type: "integer", indexed: true } } });
+    expect(() =>
+      t.execute({ tag: "ticket", fields: { priority: { type: "integer", indexed: false } } }),
+    ).toThrow(/indexed-flag conflict/);
+  });
+
+  it("unsupported field type for indexing throws", async () => {
+    expect(() =>
+      findTool("update-tag").execute({
+        tag: "project",
+        fields: { weird: { type: "date", indexed: true } },
+      }),
+    ).toThrow(/unsupported type "date"/);
+  });
+
+  it("invalid field name for indexing throws", async () => {
+    expect(() =>
+      findTool("update-tag").execute({
+        tag: "project",
+        fields: { "bad-name": { type: "string", indexed: true } },
+      }),
+    ).toThrow(IndexedFieldError);
+  });
+
+  it("rejects non-atomic indexed-flag change while other declarers hold it true", async () => {
+    const t = findTool("update-tag");
+    await t.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
+    await t.execute({ tag: "ticket", fields: { status: { type: "string", indexed: true } } });
+    expect(() =>
+      t.execute({
+        tag: "project",
+        fields: { status: { type: "string", indexed: false } },
+      }),
+    ).toThrow(/indexed-flag conflict.*tag "ticket"/);
+    expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project", "ticket"]);
+  });
+
+  it("last declarer releasing drops column + index", async () => {
+    const t = findTool("update-tag");
+    await t.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
+    await t.execute({ tag: "project", fields: { status: { type: "string", indexed: false } } });
+    expect(getIndexedField(db, "status")).toBeNull();
+    expect(notesColumns()).not.toContain("meta_status");
+  });
+
+  it("indexing a boolean field maps to INTEGER storage", async () => {
+    await findTool("update-tag").execute({
+      tag: "project",
+      fields: { archived: { type: "boolean", indexed: true } },
+    });
+    expect(getIndexedField(db, "archived")?.sqliteType).toBe("INTEGER");
+  });
+
+  it("non-indexed fields are not tracked", async () => {
+    await findTool("update-tag").execute({
+      tag: "project",
+      fields: { notes: { type: "string" } },
+    });
+    expect(listIndexedFields(db)).toEqual([]);
+    expect(notesColumns()).not.toContain("meta_notes");
+  });
+});
+
+describe("delete-tag: indexed fields", () => {
+  it("releases indexed fields the tag declared", async () => {
+    const update = findTool("update-tag");
+    const del = findTool("delete-tag");
+    await update.execute({
+      tag: "project",
+      fields: { status: { type: "string", indexed: true } },
+    });
+    await del.execute({ tag: "project" });
+    expect(getIndexedField(db, "status")).toBeNull();
+    expect(notesColumns()).not.toContain("meta_status");
+  });
+
+  it("keeps the column if another tag still declares the field", async () => {
+    const update = findTool("update-tag");
+    const del = findTool("delete-tag");
+    await update.execute({
+      tag: "project",
+      fields: { status: { type: "string", indexed: true } },
+    });
+    await update.execute({
+      tag: "ticket",
+      fields: { status: { type: "string", indexed: true } },
+    });
+    await del.execute({ tag: "project" });
+    expect(getIndexedField(db, "status")?.declarerTags).toEqual(["ticket"]);
+    expect(notesColumns()).toContain("meta_status");
+  });
+});

package/core/src/indexed-fields.ts

@@ -0,0 +1,238 @@
+/**
+ * Indexed-field lifecycle — manages generated columns + indexes on `notes`
+ * for metadata fields declared `indexed: true` by one or more tag schemas.
+ *
+ * Semantic: the tag authorizes the index, but the index is universal across
+ * all notes (not partitioned by tag). A generated column mirrors
+ * `json_extract(metadata, '$.<field>')` and a B-tree index on that column
+ * makes operator queries (eq/gt/lt/in/...) and `order_by` fast.
+ *
+ * Lifetime is tied to the declarer set: the column + index exist as long as
+ * at least one tag schema declares `indexed: true` for the field. When the
+ * last declarer releases (schema update or `delete-tag`), the column + index
+ * are dropped. The `indexed_fields` table is the single source of truth and
+ * is used by `rebuildIndexes` on vault init for idempotent reconstruction.
+ *
+ * See Parachute/Decisions/2026-04-19-metadata-indexing-via-tag-schemas.
+ */
+
+import { Database } from "bun:sqlite";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type SqliteType = "TEXT" | "INTEGER";
+export type FieldType = "string" | "integer" | "boolean";
+
+export const TYPE_MAP: Record<FieldType, SqliteType> = {
+  string: "TEXT",
+  integer: "INTEGER",
+  boolean: "INTEGER",
+};
+
+export interface IndexedField {
+  field: string;
+  sqliteType: SqliteType;
+  declarerTags: string[];
+}
+
+interface IndexedFieldRow {
+  field: string;
+  sqlite_type: string;
+  declarer_tags: string;
+}
+
+export class IndexedFieldError extends Error {
+  override name = "IndexedFieldError";
+}
+
+// Restrict field names to safe SQL identifiers. This also bounds the
+// generated column and index names, which are derived from the field.
+const FIELD_NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]{0,62}$/;
+
+export function validateFieldName(field: string): void {
+  if (!FIELD_NAME_RE.test(field)) {
+    throw new IndexedFieldError(
+      `invalid field name "${field}": must start with a letter or underscore and contain only [A-Za-z0-9_] (max 63 chars)`,
+    );
+  }
+}
+
+/** Map a tag-schema field type to the backing SQLite storage class. */
+export function mapFieldType(type: string): SqliteType | null {
+  return TYPE_MAP[type as FieldType] ?? null;
+}
+
+function columnName(field: string): string {
+  return `meta_${field}`;
+}
+
+function indexName(field: string): string {
+  return `idx_meta_${field}`;
+}
+
+function rowToField(row: IndexedFieldRow): IndexedField {
+  let declarerTags: string[] = [];
+  try {
+    const parsed = JSON.parse(row.declarer_tags);
+    if (Array.isArray(parsed)) {
+      declarerTags = parsed.filter((t): t is string => typeof t === "string");
+    }
+  } catch {}
+  return {
+    field: row.field,
+    sqliteType: row.sqlite_type as SqliteType,
+    declarerTags,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Queries
+// ---------------------------------------------------------------------------
+
+export function listIndexedFields(db: Database): IndexedField[] {
+  const rows = db
+    .prepare("SELECT field, sqlite_type, declarer_tags FROM indexed_fields ORDER BY field")
+    .all() as IndexedFieldRow[];
+  return rows.map(rowToField);
+}
+
+export function getIndexedField(db: Database, field: string): IndexedField | null {
+  const row = db
+    .prepare("SELECT field, sqlite_type, declarer_tags FROM indexed_fields WHERE field = ?")
+    .get(field) as IndexedFieldRow | undefined;
+  return row ? rowToField(row) : null;
+}
+
+// ---------------------------------------------------------------------------
+// DDL helpers
+// ---------------------------------------------------------------------------
+
+function hasNotesColumn(db: Database, col: string): boolean {
+  // `table_xinfo` includes generated (VIRTUAL) columns, which `table_info`
+  // omits. We use xinfo here so re-declaration paths detect the existing
+  // generated column and stay idempotent.
+  const rows = db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[];
+  return rows.some((r) => r.name === col);
+}
+
+function createColumnAndIndex(db: Database, field: string, sqliteType: SqliteType): void {
+  const col = columnName(field);
+  if (!hasNotesColumn(db, col)) {
+    // SQLite requires VIRTUAL (not STORED) for generated columns added via
+    // ALTER TABLE. VIRTUAL is also what we want here: the value isn't stored
+    // twice (just indexed), so writes stay cheap.
+    db.exec(
+      `ALTER TABLE notes ADD COLUMN "${col}" ${sqliteType} GENERATED ALWAYS AS (json_extract(metadata, '$."${field}"')) VIRTUAL`,
+    );
+  }
+  db.exec(`CREATE INDEX IF NOT EXISTS "${indexName(field)}" ON notes("${col}")`);
+}
+
+function dropColumnAndIndex(db: Database, field: string): void {
+  db.exec(`DROP INDEX IF EXISTS "${indexName(field)}"`);
+  if (hasNotesColumn(db, columnName(field))) {
+    db.exec(`ALTER TABLE notes DROP COLUMN "${columnName(field)}"`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Lifecycle operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Register `tag` as a declarer of indexed `field` with the given storage type.
+ *
+ * - First declarer: inserts the row, creates the generated column + index.
+ * - Additional declarer (matching type): adds tag to the set. Idempotent if
+ *   the tag is already a declarer.
+ * - Type mismatch with an existing *other* declarer: throws. This is a
+ *   defensive check; the MCP layer should have already rejected the call
+ *   with a more descriptive cross-tag error message.
+ * - Type mismatch when this tag is the sole declarer: drops + recreates the
+ *   column with the new type. Allowed because no other schema's contract
+ *   depends on the prior type.
+ */
+export function declareField(
+  db: Database,
+  field: string,
+  sqliteType: SqliteType,
+  tag: string,
+): void {
+  validateFieldName(field);
+  const existing = getIndexedField(db, field);
+  if (!existing) {
+    db.prepare(
+      "INSERT INTO indexed_fields (field, sqlite_type, declarer_tags) VALUES (?, ?, ?)",
+    ).run(field, sqliteType, JSON.stringify([tag]));
+    createColumnAndIndex(db, field, sqliteType);
+    return;
+  }
+  if (existing.sqliteType !== sqliteType) {
+    const others = existing.declarerTags.filter((t) => t !== tag);
+    if (others.length > 0) {
+      throw new IndexedFieldError(
+        `field "${field}" is declared by tag(s) [${others.join(", ")}] with sqlite type ${existing.sqliteType}; tag "${tag}" requested ${sqliteType}`,
+      );
+    }
+    dropColumnAndIndex(db, field);
+    db.prepare(
+      "UPDATE indexed_fields SET sqlite_type = ?, declarer_tags = ? WHERE field = ?",
+    ).run(sqliteType, JSON.stringify([tag]), field);
+    createColumnAndIndex(db, field, sqliteType);
+    return;
+  }
+  if (existing.declarerTags.includes(tag)) {
+    createColumnAndIndex(db, field, sqliteType);
+    return;
+  }
+  const next = [...existing.declarerTags, tag];
+  db.prepare("UPDATE indexed_fields SET declarer_tags = ? WHERE field = ?").run(
+    JSON.stringify(next),
+    field,
+  );
+  createColumnAndIndex(db, field, sqliteType);
+}
+
+/**
+ * Remove `tag` from `field`'s declarer set. If the set becomes empty, drop
+ * the row, the generated column, and the index. Returns true if the column
+ * was dropped.
+ */
+export function releaseField(db: Database, field: string, tag: string): boolean {
+  const existing = getIndexedField(db, field);
+  if (!existing) return false;
+  const next = existing.declarerTags.filter((t) => t !== tag);
+  if (next.length === 0) {
+    db.prepare("DELETE FROM indexed_fields WHERE field = ?").run(field);
+    dropColumnAndIndex(db, field);
+    return true;
+  }
+  db.prepare("UPDATE indexed_fields SET declarer_tags = ? WHERE field = ?").run(
+    JSON.stringify(next),
+    field,
+  );
+  return false;
+}
+
+/**
+ * Reconcile the generated columns + indexes with `indexed_fields` rows.
+ * Idempotent — safe to call on every vault init.
+ *
+ * `indexed_fields` is authoritative. Any row without its column/index gets
+ * one created. Extras (columns beginning with `meta_` but not backed by a
+ * row) are not touched — cleanup happens through the normal release path.
+ */
+export function rebuildIndexes(db: Database): void {
+  for (const f of listIndexedFields(db)) {
+    try {
+      createColumnAndIndex(db, f.field, f.sqliteType);
+    } catch (err) {
+      console.error(
+        `[indexed-fields] could not rebuild column for "${f.field}":`,
+        err,
+      );
+    }
+  }
+}