@openparachute/vault 0.6.4-rc.1 → 0.6.4-rc.11

package/core/src/conformance.ts

@@ -0,0 +1,214 @@
+/**
+ * Schema conformance check — count how many EXISTING notes would violate a
+ * PROPOSED field spec for a tag, before the operator commits a tightening
+ * (vault#283, the MW2/#314 tie-in).
+ *
+ * Motivation: marking a field `strict`/`required`, narrowing an `enum`, or
+ * changing a `type` is a backwards-incompatible move — notes written under
+ * the old contract may now reject on their next write
+ * (`enforceStrictSchema` → `SchemaValidationError`). The Schema editor in
+ * the admin SPA calls this BEFORE save so the operator sees "N existing
+ * notes violate this — they'll reject on next write" and is pointed at the
+ * `parachute-vault schema migrate-field` CLI (#314) rather than silently
+ * shipping a contract their own data breaks.
+ *
+ * What "violate" means here: we overlay the PROPOSED `fields` for `tag` on
+ * top of the live schema config, treating every proposed field as if it
+ * were `strict:true` for the purpose of the count (so the count is "notes
+ * that would hard-reject IF this field were strict" — the worst case the
+ * operator is being warned about). We then walk every note carrying `tag`
+ * (descendants included, per the hierarchy) and run the SAME `validateNote`
+ * resolver the write path uses, counting notes with ≥1 violation on a field
+ * the proposal touches.
+ *
+ * This is a READ — no mutation. It's deliberately scoped to the proposed
+ * tag's own fields (the columns the operator is editing); inherited fields
+ * from ancestors are already enforced and aren't what the operator is
+ * tightening in this edit.
+ */
+
+import { Database } from "bun:sqlite";
+import {
+  loadSchemaConfig,
+  validateNote,
+  type SchemaField,
+  type ResolvedSchemas,
+} from "./schema-defaults.ts";
+import type { TagFieldSchema } from "./tag-schemas.ts";
+import { getTagDescendants, loadTagHierarchy } from "./tag-hierarchy.ts";
+import { queryNotes, getNoteTagsForNotes } from "./notes.ts";
+
+export interface ConformanceViolation {
+  /** Note id of a non-conforming note. */
+  id: string;
+  /** Note path, when set (helps the operator find it). */
+  path?: string | null;
+  /** The proposed-field violations on this note (reasons + messages). */
+  fields: { field: string; reason: string; message: string }[];
+}
+
+export interface ConformanceReport {
+  tag: string;
+  /** Total notes carrying the tag (descendants included). */
+  total_notes: number;
+  /** Count of notes that would violate the proposed spec. */
+  violating_notes: number;
+  /** The field names the proposal touches (the ones checked). */
+  checked_fields: string[];
+  /**
+   * A bounded sample of violating notes (id + path + which fields), so the
+   * SPA can show examples without paging the whole vault. Capped at
+   * `sampleLimit` (default 20).
+   */
+  sample: ConformanceViolation[];
+}
+
+/**
+ * Convert the on-disk `TagFieldSchema` shape to the resolver's `SchemaField`
+ * shape, FORCING `strict:true` so `validateNote` flags every constraint as a
+ * (strict) violation we can count. `indexed` is dropped — it's not a
+ * note-data constraint. `type` is narrowed to the resolver's union; an
+ * unknown type string is dropped (no type check) rather than throwing.
+ */
+function toStrictSchemaField(spec: TagFieldSchema): SchemaField {
+  const out: SchemaField = { strict: true };
+  if (
+    spec.type === "string" ||
+    spec.type === "number" ||
+    spec.type === "integer" ||
+    spec.type === "boolean" ||
+    spec.type === "array" ||
+    spec.type === "object"
+  ) {
+    out.type = spec.type;
+  }
+  if (Array.isArray(spec.enum)) out.enum = spec.enum;
+  if (spec.required === true) out.required = true;
+  if (spec.cardinality === "one" || spec.cardinality === "many") {
+    out.cardinality = spec.cardinality;
+  }
+  if (typeof spec.description === "string") out.description = spec.description;
+  return out;
+}
+
+/**
+ * Build a `ResolvedSchemas` snapshot with the proposed fields overlaid on
+ * `tag`'s OWN field map. We start from the live config (so inheritance +
+ * other tags are intact) and replace just `tag`'s field declarations with
+ * the strict-forced proposed set.
+ */
+function overlayProposedSchema(
+  base: ResolvedSchemas,
+  tag: string,
+  proposedFields: Record<string, TagFieldSchema>,
+): ResolvedSchemas {
+  const tagToFields = new Map(base.tagToFields);
+  const forced: Record<string, SchemaField> = {};
+  for (const [name, spec] of Object.entries(proposedFields)) {
+    forced[name] = toStrictSchemaField(spec);
+  }
+  if (Object.keys(forced).length > 0) {
+    tagToFields.set(tag, forced);
+  } else {
+    tagToFields.delete(tag);
+  }
+  return {
+    allTags: new Set(base.allTags).add(tag),
+    tagToFields,
+    tagToParents: base.tagToParents,
+  };
+}
+
+/**
+ * Count existing notes that would violate the PROPOSED field spec for `tag`.
+ *
+ * `proposedFields` is the full merged field map the operator intends to save
+ * (the same object the PUT body carries). Only the fields in this map are
+ * checked — the violation count answers "if I tighten THESE fields, how many
+ * notes break?" The check runs against every note carrying `tag` or any of
+ * its descendants.
+ *
+ * Pure read; no mutation. Returns 0 violating notes when `proposedFields` is
+ * empty (nothing to enforce).
+ */
+export function countConformanceViolations(
+  db: Database,
+  tag: string,
+  proposedFields: Record<string, TagFieldSchema>,
+  opts: { sampleLimit?: number } = {},
+): ConformanceReport {
+  const sampleLimit = opts.sampleLimit ?? 20;
+  const checkedFields = Object.keys(proposedFields);
+
+  const empty: ConformanceReport = {
+    tag,
+    total_notes: 0,
+    violating_notes: 0,
+    checked_fields: checkedFields,
+    sample: [],
+  };
+
+  // Resolve the tag's descendant set so we count notes on subtype tags too
+  // (a strict field on `#dev` applies to `#dev/log` notes via inheritance).
+  const hierarchy = loadTagHierarchy(db);
+  const tagSet = Array.from(getTagDescendants(hierarchy, tag));
+  if (tagSet.length === 0) return empty;
+
+  // All notes carrying the tag (or a descendant). `tagMatch: "any"` over the
+  // expanded set — a note on ANY of these tags is in scope. We hydrate tags
+  // ourselves (batched) so the resolver sees the full ancestor set per note.
+  const notes = queryNotes(db, { tags: tagSet, tagMatch: "any" });
+  if (notes.length === 0) return empty;
+
+  if (checkedFields.length === 0) {
+    return { ...empty, total_notes: notes.length };
+  }
+
+  const base = loadSchemaConfig(db);
+  const resolved = overlayProposedSchema(base, tag, proposedFields);
+  const checkedSet = new Set(checkedFields);
+
+  const tagsById = getNoteTagsForNotes(
+    db,
+    notes.map((n) => n.id),
+  );
+
+  let violating = 0;
+  const sample: ConformanceViolation[] = [];
+
+  for (const note of notes) {
+    const tags = tagsById.get(note.id) ?? note.tags ?? [];
+    const status = validateNote(resolved, {
+      path: note.path ?? null,
+      tags,
+      metadata: note.metadata ?? {},
+    });
+    if (!status) continue;
+    // Keep only violations on a field the PROPOSAL touches (ignore conflicts
+    // + violations on inherited/other fields the operator isn't editing).
+    const relevant = status.warnings.filter(
+      (w) => w.reason !== "schema_conflict" && checkedSet.has(w.field),
+    );
+    if (relevant.length === 0) continue;
+    violating++;
+    if (sample.length < sampleLimit) {
+      sample.push({
+        id: note.id,
+        path: note.path ?? null,
+        fields: relevant.map((w) => ({
+          field: w.field,
+          reason: w.reason,
+          message: w.message,
+        })),
+      });
+    }
+  }
+
+  return {
+    tag,
+    total_notes: notes.length,
+    violating_notes: violating,
+    checked_fields: checkedFields,
+    sample,
+  };
+}

package/core/src/core.test.ts

@@ -5,6 +5,7 @@ import { generateMcpTools } from "./mcp.js";
 import { initSchema } from "./schema.js";
 import { decodeCursor } from "./cursor.js";
 import { traverseLinks } from "./links.js";
+import * as indexedFieldOps from "./indexed-fields.js";
 
 let store: SqliteStore;
 let db: Database;
@@ -2169,6 +2170,50 @@ describe("MCP tools", async () => {
     expect(result.extension).toBe("csv");
   });
 
+  // ---- Metadata key-deletion via null tombstone (vault#478 / #479) ---------
+
+  it("update-note deletes a metadata key when its value is null (RFC 7386)", async () => {
+    const note = await store.createNote("m", {
+      metadata: { keep: "yes", drop: "old", other: 1 },
+    });
+    const tools = generateMcpTools(store);
+    const updateNote = tools.find((t) => t.name === "update-note")!;
+    await updateNote.execute({ id: note.id, metadata: { drop: null }, force: true });
+
+    const fresh = await store.getNote(note.id);
+    // The deleted key is GONE — not a literal JSON null.
+    expect(fresh!.metadata).not.toHaveProperty("drop");
+    // Other keys are untouched.
+    expect(fresh!.metadata).toMatchObject({ keep: "yes", other: 1 });
+  });
+
+  it("update-note metadata key-rename in one call (new key set, old key deleted)", async () => {
+    const note = await store.createNote("m", {
+      metadata: { "old-key": "v", stable: true },
+    });
+    const tools = generateMcpTools(store);
+    const updateNote = tools.find((t) => t.name === "update-note")!;
+    await updateNote.execute({
+      id: note.id,
+      metadata: { new_key: "v", "old-key": null },
+      force: true,
+    });
+
+    const fresh = await store.getNote(note.id);
+    expect(fresh!.metadata).not.toHaveProperty("old-key");
+    expect(fresh!.metadata).toMatchObject({ new_key: "v", stable: true });
+  });
+
+  it("deleting an absent metadata key is a no-op (round-trips cleanly)", async () => {
+    const note = await store.createNote("m", { metadata: { a: 1 } });
+    const tools = generateMcpTools(store);
+    const updateNote = tools.find((t) => t.name === "update-note")!;
+    await updateNote.execute({ id: note.id, metadata: { ghost: null }, force: true });
+    const fresh = await store.getNote(note.id);
+    expect(fresh!.metadata).toEqual({ a: 1 });
+    expect(fresh!.metadata).not.toHaveProperty("ghost");
+  });
+
   it("query-notes filters by extension (vault#328)", async () => {
     await store.createNote("md note", { path: "a" });
     await store.createNote("csv note", { path: "b", extension: "csv" });
@@ -5301,6 +5346,87 @@ describe("tag record API (patterns/tag-data-model.md)", async () => {
     expect((await store.queryNotes({ tags: ["manual"] })).length).toBe(0);
     expect(await store.getTagRecord("voice")).toBeNull();
   });
+
+  // ---- Indexed-field atomicity (vault#478) ----------------------------------
+
+  it("upsertTagRecord with a bad indexed-field name throws IndexedFieldError and leaves the schema unchanged", async () => {
+    // kebab-case field name violates [A-Za-z0-9_] / no-leading-digit. The
+    // declaration must NOT persist, and no orphan/lying index may be created.
+    await expect(
+      store.upsertTagRecord("meeting", {
+        description: "meetings",
+        fields: { "meeting-type": { type: "string", indexed: true } },
+      }),
+    ).rejects.toThrow(indexedFieldOps.IndexedFieldError);
+
+    // Schema is untouched — the tag row may not even exist, and definitely
+    // doesn't claim the poisoned field.
+    const record = await store.getTagRecord("meeting");
+    expect(record?.fields?.["meeting-type"]).toBeUndefined();
+
+    // No backing index row was created for the bad field.
+    expect(indexedFieldOps.getIndexedField(db, "meeting-type")).toBeNull();
+    // And the generated column doesn't exist either.
+    const cols = (db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map(
+      (c) => c.name,
+    );
+    expect(cols).not.toContain("meta_meeting-type");
+  });
+
+  it("upsertTagRecord with an unindexable type throws and leaves the schema unchanged", async () => {
+    await expect(
+      store.upsertTagRecord("meeting", {
+        fields: { attendees: { type: "json", indexed: true } },
+      }),
+    ).rejects.toThrow(indexedFieldOps.IndexedFieldError);
+    const record = await store.getTagRecord("meeting");
+    expect(record?.fields?.attendees).toBeUndefined();
+    expect(indexedFieldOps.getIndexedField(db, "attendees")).toBeNull();
+  });
+
+  it("a bad indexed field does NOT corrupt a prior valid declaration on the same tag", async () => {
+    // First, a clean valid indexed field.
+    await store.upsertTagRecord("meeting", {
+      fields: { held_on: { type: "string", indexed: true } },
+    });
+    expect(indexedFieldOps.getIndexedField(db, "held_on")?.declarerTags).toContain("meeting");
+
+    // Now an update that adds a poisoned field. The whole write must roll
+    // back — the prior valid field survives untouched.
+    await expect(
+      store.upsertTagRecord("meeting", {
+        fields: {
+          held_on: { type: "string", indexed: true },
+          "bad-field": { type: "string", indexed: true },
+        },
+      }),
+    ).rejects.toThrow(indexedFieldOps.IndexedFieldError);
+
+    const record = await store.getTagRecord("meeting");
+    expect(record?.fields?.held_on?.indexed).toBe(true);
+    expect(record?.fields?.["bad-field"]).toBeUndefined();
+    expect(indexedFieldOps.getIndexedField(db, "held_on")?.declarerTags).toContain("meeting");
+    expect(indexedFieldOps.getIndexedField(db, "bad-field")).toBeNull();
+  });
+
+  it("upsertTagRecord with fields:null releases indexed fields (transaction commits the release path)", async () => {
+    // Declare an indexed field, then clear all fields — the release path runs
+    // inside the same transaction as the persist. It must commit cleanly:
+    // the row, generated column, and index all drop.
+    await store.upsertTagRecord("meeting", {
+      fields: { held_on: { type: "string", indexed: true } },
+    });
+    expect(indexedFieldOps.getIndexedField(db, "held_on")?.declarerTags).toContain("meeting");
+
+    await store.upsertTagRecord("meeting", { fields: null });
+    const record = await store.getTagRecord("meeting");
+    expect(record?.fields).toBeUndefined();
+    expect(indexedFieldOps.getIndexedField(db, "held_on")).toBeNull();
+    const cols = (db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map(
+      (c) => c.name,
+    );
+    expect(cols).not.toContain("meta_held_on");
+  });
 });
 
 // ---------------------------------------------------------------------------

package/core/src/cursor.ts

@@ -160,6 +160,14 @@ export interface QueryHashInputs {
   extension?: string | string[];
   ids?: string[];
   metadata?: Record<string, unknown>;
+  // Write-attribution filters (vault#298) — part of the result-set-affecting
+  // opts, so they're bound into the cursor: changing "who/via" between polls
+  // invalidates the cursor with cursor_query_mismatch rather than silently
+  // continuing the prior filter's watermark.
+  createdBy?: string;
+  lastUpdatedBy?: string;
+  createdVia?: string;
+  lastUpdatedVia?: string;
   dateFrom?: string;
   dateTo?: string;
   dateFilter?: { field?: string; from?: string; to?: string };