@openparachute/vault 0.7.0-rc.4 → 0.7.0-rc.6

package/core/src/schema-defaults.ts

@@ -60,6 +60,20 @@ export interface SchemaField {
   type?: "string" | "number" | "integer" | "boolean" | "array" | "object";
   enum?: string[];
   description?: string;
+  /**
+   * Mirrors `TagFieldSchema.indexed` (core/src/tag-schemas.ts) — this field
+   * has a generated column + B-tree index maintained on `notes` (see
+   * core/src/indexed-fields.ts). Parsed here (not just there) because
+   * `validateNote` uses it: an indexed field is a QUERY CONTRACT, not just a
+   * storage hint, so a `type_mismatch` on an indexed field is always
+   * enforced (`strict: true` on that ONE warning) regardless of this field's
+   * own `strict` flag (vault#553 Decision A). Every other constraint
+   * (enum/required/cardinality) on an indexed field stays governed by
+   * `strict` as before — only the TYPE contract is unconditional, because
+   * that's the one a type-mismatched write can silently poison range
+   * queries with (SQLite's TEXT-sorts-above-INTEGER affinity ordering).
+   */
+  indexed?: boolean;
   /**
    * Strict enforcement opt-in (vault#299, Part A). Default `false` — when
    * unset/false, ALL constraints on this field are advisory: violations
@@ -169,6 +183,7 @@ function parseFieldsJson(raw: string | null): Record<string, SchemaField> {
     if (typeof f.type === "string") field.type = f.type as SchemaField["type"];
     if (Array.isArray(f.enum)) field.enum = f.enum.filter((x): x is string => typeof x === "string");
     if (typeof f.description === "string") field.description = f.description;
+    if (f.indexed === true) field.indexed = true;
     if (f.strict === true) field.strict = true;
     if (f.required === true) field.required = true;
     if (f.cardinality === "one" || f.cardinality === "many") field.cardinality = f.cardinality;
@@ -319,6 +334,20 @@ function stringArraysEqual(a: string[] | undefined, b: string[] | undefined): bo
   return true;
 }
 
+/**
+ * Human-readable JSON-shape name for a value, used to name the "got" side of
+ * a `type_mismatch` message (vault#553 — "message names field + expected
+ * type + got type"). Distinct from SQLite's `json_type()` vocabulary
+ * (integer/real/true/false/text/...) — this is the JSON-Schema-ish vocabulary
+ * `SchemaField.type` already uses, so the message reads as "expected X, got
+ * Y" in the SAME words the schema declares.
+ */
+function jsonTypeOf(value: unknown): string {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  return typeof value; // "string" | "number" | "boolean" | "object" | ...
+}
+
 function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
   if (type === undefined) return true;
   switch (type) {
@@ -357,11 +386,15 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
  *   → `cardinality_mismatch`
  *
  * Every violation carries `strict: true` iff its field declared `strict:true`
- * (vault#299). The list itself is the SAME whether or not a field is strict —
- * the difference is only the per-warning `strict` flag, which the write path
- * uses to decide block-vs-warn. Under `strict:false` this is byte-identical to
- * the historical advisory behavior PLUS the new `required`/`cardinality`
- * advisory reasons (which fire for any field declaring those, strict or not).
+ * (vault#299) — EXCEPT a `type_mismatch` on an `indexed: true` field, which is
+ * ALWAYS `strict: true` regardless of the field's own `strict` setting
+ * (vault#553 Decision A — an indexed field's type is a query contract, not
+ * just guidance). The list itself is the SAME whether or not a field is
+ * strict — the difference is only the per-warning `strict` flag, which the
+ * write path uses to decide block-vs-warn. Under `strict:false` (and
+ * `indexed:false`) this is byte-identical to the historical advisory
+ * behavior PLUS the new `required`/`cardinality` advisory reasons (which
+ * fire for any field declaring those, strict or not).
  *
  * Fields not declared by any ancestor's schema are ignored entirely.
  */
@@ -397,12 +430,20 @@ export function validateNote(
     if (absent) continue;
 
     if (spec.type && !valueMatchesType(value, spec.type)) {
+      // Decision A (vault#553): an INDEXED field's type is a query
+      // contract — a type-mismatched write poisons range-query ordering
+      // (SQLite's TEXT-sorts-above-INTEGER affinity) regardless of whether
+      // this field opted into `strict`. Force `strict: true` on THIS
+      // warning alone when the field is indexed; every other constraint
+      // (enum/required/cardinality) stays governed by `spec.strict` as
+      // before.
+      const indexedTypeStrict = spec.indexed === true;
       warnings.push({
         field: fieldName,
         schema: sourceTag,
         reason: "type_mismatch",
-        message: `'${fieldName}' should be ${spec.type} (tag '${sourceTag}')`,
-        ...strictFlag,
+        message: `'${fieldName}' should be ${spec.type} (tag '${sourceTag}'), got ${jsonTypeOf(value)}${indexedTypeStrict ? " — indexed fields reject type-mismatched writes (vault#553)" : ""}`,
+        ...(strictFlag.strict || indexedTypeStrict ? { strict: true } : {}),
       });
     }
 

package/core/src/schema.ts

@@ -1,9 +1,10 @@
 import { Database } from "bun:sqlite";
 import { normalizePath } from "./paths.js";
-import { rebuildIndexes } from "./indexed-fields.js";
+import { rebuildIndexes, listIndexedFields } from "./indexed-fields.js";
+import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
 import { transaction } from "./txn.js";
 
-export const SCHEMA_VERSION = 23;
+export const SCHEMA_VERSION = 24;
 
 export const SCHEMA_SQL = `
 -- Notes: the universal record.
@@ -502,6 +503,10 @@ export function initSchema(db: Database): void {
   // vault#298.
   migrateToV23(db);
 
+  // Migrate v23 → v24: coerce existing typed-index poison where lossless,
+  // leave the rest in place for `doctor` to surface. See vault#553.
+  migrateToV24(db);
+
   // Rebuild any generated columns + indexes declared in indexed_fields.
   // No-op for a fresh vault; idempotent on existing vaults.
   rebuildIndexes(db);
@@ -1187,6 +1192,146 @@ function migrateToV23(db: Database): void {
   }
 }
 
+/** Sentinel returned by {@link coerceIndexedValue} when no lossless conversion exists. */
+const NOT_COERCIBLE = Symbol("not-coercible");
+
+/** Full JSON-number grammar — used to gate string→number coercion to values that round-trip exactly. */
+const JSON_NUMBER_RE = /^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$/;
+
+/**
+ * Decide the lossless coercion (if any) for a single poisoned indexed-field
+ * value, given `jt` — the SQLite `json_type()` of its CURRENT value (the
+ * SAME vocabulary `doctor`'s `findMixedTypeIndexedFieldNotes` compares
+ * against: "text", "integer", "real", "true", "false", "array", "object")
+ * — and `targetSqliteType`, the field's declared storage class ("INTEGER"
+ * or "TEXT"). Returns the coerced value, or {@link NOT_COERCIBLE} when no
+ * exact round-trip conversion exists — the caller leaves the value
+ * untouched (never deletes or nulls note data).
+ *
+ * Coercible cases (vault#553 Decision D):
+ *   - INTEGER target, TEXT source: a clean numeric string ("5", "5.5",
+ *     "-3") → the JSON number, ONLY when `String(Number(str)) === str`
+ *     (exact round-trip — rejects "5.50", scientific-notation reformats,
+ *     and anything outside safe double precision that wouldn't survive the
+ *     trip). "true"/"false" (exact string match) → the JSON boolean.
+ *   - TEXT target, INTEGER/REAL/boolean source: a number or boolean → its
+ *     JSON string form via `String(value)` — always exact for finite JS
+ *     numbers and booleans within IEEE-754 double precision.
+ * Never coercible: array/object/null values in either direction, or a TEXT
+ * value that isn't a clean number/boolean string (e.g. "four", "5,000").
+ */
+function coerceIndexedValue(
+  value: unknown,
+  jt: string,
+  targetSqliteType: string,
+): unknown | typeof NOT_COERCIBLE {
+  if (targetSqliteType === "INTEGER") {
+    if (jt !== "text" || typeof value !== "string") return NOT_COERCIBLE;
+    if (value === "true") return true;
+    if (value === "false") return false;
+    if (JSON_NUMBER_RE.test(value)) {
+      const n = Number(value);
+      if (Number.isFinite(n) && String(n) === value) return n;
+    }
+    return NOT_COERCIBLE;
+  }
+  if (targetSqliteType === "TEXT") {
+    if ((jt === "true" || jt === "false") && typeof value === "boolean") {
+      return jt; // "true" / "false" — the json_type string IS the target text
+    }
+    if ((jt === "integer" || jt === "real") && typeof value === "number") {
+      const s = String(value);
+      if (Number(s) === value) return s;
+    }
+    return NOT_COERCIBLE;
+  }
+  return NOT_COERCIBLE;
+}
+
+/**
+ * Migrate v23 → v24: typed-index poison coercion (vault#553 Decision D).
+ *
+ * For every declared indexed field (`indexed_fields` — the SSOT from
+ * `core/src/indexed-fields.ts`) and every note whose `metadata.<field>` JSON
+ * type disagrees with the field's declared sqlite storage class — reusing
+ * `doctor`'s `findMixedTypeIndexedFieldNotes` (`core/src/doctor.ts`) as the
+ * DETECTOR, the exact same query that powers the `mixed_type_indexed_field`
+ * finding, so migration and diagnostic can never disagree on what counts as
+ * poisoned:
+ *
+ *   - COERCE where {@link coerceIndexedValue} finds a lossless conversion —
+ *     rewrites `notes.metadata` in place (the generated `meta_<field>`
+ *     column re-derives via `json_extract` automatically; nothing else to
+ *     touch).
+ *   - LEAVE IN PLACE everything else. NEVER DELETE OR NULL NOTE DATA in a
+ *     startup migration — a non-coercible value (e.g. `"hello"` in an
+ *     integer field) stays exactly as written and remains visible via
+ *     `doctor`'s `mixed_type_indexed_field` finding for the operator to
+ *     clean up deliberately.
+ *
+ * Rewrites happen at the JS level — `SELECT metadata` → `JSON.parse` →
+ * mutate the one field → `JSON.stringify` → single-row `UPDATE ... WHERE id
+ * = ?` (two bound params) — rather than SQL `json_set`, so the migration
+ * stays portable to a narrower json1 surface (Cloudflare DO SQLite; flagged
+ * for the wire reviewer to confirm). Detection still uses
+ * `json_extract`/`json_type`/`json_valid`, the same functions the indexed
+ * generated columns and `doctor` already depend on everywhere in this
+ * codebase — not a new cross-runtime risk. One row at a time also means this
+ * never approaches SQLite's bound-parameter ceiling regardless of how many
+ * notes are poisoned (no batched `WHERE id IN (...)`).
+ *
+ * Idempotent: every step is naturally re-runnable (a coerced value now
+ * agrees with the declared type and is never re-detected as a mismatch; a
+ * left-in-place value re-detects identically and is re-skipped identically).
+ * Safe on a vault with zero indexed fields (the field-list guard short-
+ * circuits before any table scan) and safe to run on every boot, matching
+ * the unconditional-migration-function idiom the rest of this file uses.
+ */
+function migrateToV24(db: Database): void {
+  if (!hasTable(db, "notes") || !hasTable(db, "indexed_fields")) return;
+  const fields = listIndexedFields(db);
+  if (fields.length === 0) return;
+
+  let coerced = 0;
+  let left = 0;
+
+  transaction(db, () => {
+    const readStmt = db.prepare("SELECT metadata FROM notes WHERE id = ?");
+    const updateStmt = db.prepare("UPDATE notes SET metadata = ? WHERE id = ?");
+
+    for (const f of fields) {
+      const mismatches = findMixedTypeIndexedFieldNotes(db, f.field, f.sqliteType);
+      for (const { id, jt } of mismatches) {
+        const row = readStmt.get(id) as { metadata: string | null } | null;
+        if (!row?.metadata) continue;
+        let meta: unknown;
+        try {
+          meta = JSON.parse(row.metadata);
+        } catch {
+          continue; // shouldn't happen — findMixedTypeIndexedFieldNotes already filters to json_valid rows
+        }
+        if (!meta || typeof meta !== "object" || Array.isArray(meta)) continue;
+        const obj = meta as Record<string, unknown>;
+        if (!(f.field in obj)) continue;
+        const coercedValue = coerceIndexedValue(obj[f.field], jt, f.sqliteType);
+        if (coercedValue === NOT_COERCIBLE) {
+          left++;
+          continue;
+        }
+        obj[f.field] = coercedValue;
+        updateStmt.run(JSON.stringify(obj), id);
+        coerced++;
+      }
+    }
+  });
+
+  if (coerced > 0 || left > 0) {
+    console.log(
+      `[vault] migrated to schema v24 (vault#553): typed-index poison scan — coerced ${coerced} value(s) to their declared indexed type; left ${left} non-coercible value(s) in place (see \`doctor\`'s mixed_type_indexed_field finding for cleanup).`,
+    );
+  }
+}
+
 function hasTable(db: Database, name: string): boolean {
   const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
   return !!row;

package/core/src/seed-packs.ts

@@ -468,9 +468,9 @@ update-tag {
   tag: "meeting",
   description: "A meeting with notes",
   fields: {
-    held_on: { type: "string", indexed: true },              // queryable with operators
-    status:  { type: "string", enum: ["scheduled", "done"] }, // first enum value is the default
-    rating:  { type: "integer" }
+    held_on: { type: "string", indexed: true },                                         // queryable with operators
+    status:  { type: "string", enum: ["scheduled", "done"], default: "scheduled" },      // explicit default — declare one to auto-fill
+    rating:  { type: "integer" }                                                        // no default — stays unset until written
   }
 }
 \`\`\`
@@ -490,16 +490,23 @@ A few behaviors worth knowing before you write at scale:
   conflict error (re-read, reconcile, retry). For bulk/scripted writes where
   concurrency is known-safe, pass \`force: true\` to waive the *requirement to
   supply* it. \`append\`/\`prepend\`-only updates are exempt (no-conflict-by-design).
-- **A schema field's default is filled in on write, so it shows up even when you
-  didn't set it.** When a note gets a tag whose schema declares a field, the
-  missing field is back-filled: an \`enum\` field → its **first listed value**, an
-  \`integer\` → \`0\`, a \`boolean\` → \`false\`, a plain string → \`""\`. So a
-  \`rating: { type: "integer" }\` reads as \`0\` on notes nobody rated — that \`0\`
-  is "unset," not "rated zero." Order an \`enum\`'s values so the first is a sane
-  default, and don't read a back-filled \`0\`/\`""\`/\`false\` as a real value.
-- **Validation is advisory, never blocking.** A type/enum mismatch comes back as
-  a \`validation_status\` warning on the write response — the write still lands.
-  Read those warnings and self-correct on the next turn.
+- **A schema field only back-fills when you declare an explicit \`default\`.**
+  When a note gets a tag whose schema declares a field with a \`default\`, an
+  unset field is filled with THAT value. A field with no \`default\` stays
+  genuinely absent — \`query-notes { metadata: { rating: { exists: false } } }\`
+  reliably finds notes that never set \`rating\`. Declare \`default\` on a field
+  only when "unset" and "explicitly set to X" should read the same; leave it
+  off when you need to tell them apart.
+- **Validation is advisory, never blocking — EXCEPT an \`indexed: true\`
+  field's type.** A type/enum mismatch on a non-indexed field (or a
+  non-\`strict\` constraint on any field) comes back as a \`validation_status\`
+  warning and the write still lands — read those warnings and self-correct on
+  the next turn. An \`indexed: true\` field's TYPE is always enforced, though:
+  writing a string into an indexed \`integer\` field is REJECTED with a
+  \`schema_validation\` error, because a bad-typed value would otherwise
+  silently poison range queries (\`gt\`/\`gte\`/\`lt\`/\`lte\`) on that field. Mark a
+  field \`strict: true\` to enforce its OTHER constraints
+  (enum/required/cardinality) the same hard way.
 
 (Full design guide, with copy-paste examples: https://parachute.computer/scripting/)
 

package/core/src/store.ts

@@ -35,6 +35,7 @@ import {
   type ConformanceReport,
 } from "./conformance.js";
 import type { SearchMode } from "./search-query.js";
+import { runDoctorScan, type DoctorReport, type DoctorScanOpts } from "./doctor.js";
 
 /**
  * bun:sqlite-backed Store implementation. Internally everything is
@@ -457,10 +458,15 @@ export class BunSqliteStore implements Store {
     return noteOps.listTags(this.db);
   }
 
-  async deleteTag(name: string): Promise<{ deleted: boolean; notes_untagged: number }> {
-    const result = noteOps.deleteTag(this.db, name);
+  async deleteTag(name: string, opts?: noteOps.DeleteTagOpts): Promise<noteOps.DeleteTagResult> {
+    const result = noteOps.deleteTag(this.db, name, opts);
+    // Referential-integrity refusal (vault#552) — nothing was written;
+    // caches and hooks are untouched.
+    if ("error" in result) return result;
     // The deleted tag may have been a parent or child in the hierarchy
-    // and may have declared `fields` powering schema validation.
+    // and may have declared `fields` powering schema validation. A
+    // cascade/detach repair also rewrites OTHER tags' parent_names, so
+    // busting the hierarchy cache covers both cases.
     this._tagHierarchy = null;
     this._schemaConfig = null;
     // Fire "deleted" only when SOMETHING happened (the underlying
@@ -644,6 +650,14 @@ export class BunSqliteStore implements Store {
     return count;
   }
 
+  /**
+   * Read-only taxonomy/metadata integrity scan (vault#552). Pure read — no
+   * cache invalidation needed since nothing is written.
+   */
+  async doctor(opts?: DoctorScanOpts): Promise<DoctorReport> {
+    return runDoctorScan(this.db, opts);
+  }
+
   // ---- Tag Records (post-v14: full identity row) ----
 
   async listTagRecords() {
@@ -727,6 +741,24 @@ export class BunSqliteStore implements Store {
         // Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
         indexedFieldOps.validateFieldName(fieldName);
       }
+      // Default-conformance pre-validate (vault#553 Decision B) — mirrors the
+      // indexed-type/name checks above: fail BEFORE any persistence so a bad
+      // `default` never gets written. Runs over EVERY field in the full
+      // (already-merged) `nextFields` map, not just indexed ones — a default
+      // that doesn't match its own field's type/enum is a tag-schema error
+      // regardless of queryability. This is the REST parity path: MCP's
+      // `update-tag` tool ALSO calls `collectTagFieldViolations` before
+      // reaching here (bundling every violation into one `TagFieldConflictError`
+      // — see that function's doc comment), so a conforming MCP call never
+      // trips this; a non-conforming one throws there first. REST has no such
+      // pre-check, so this is REST's only gate — same asymmetry as the
+      // indexed-type/name checks it sits beside.
+      for (const [fieldName, spec] of Object.entries(nextFields ?? {})) {
+        const violation = tagSchemaOps.validateFieldDefault(fieldName, spec);
+        if (violation) {
+          throw new tagSchemaOps.InvalidFieldDefaultError(fieldName, violation.message);
+        }
+      }
     }
 
     // Persist the record + reconcile the indexed-field lifecycle atomically.

package/core/src/tag-hierarchy.ts

@@ -388,3 +388,69 @@ export function findHierarchyCycles(h: TagHierarchy): string[] {
   }
   return cycles;
 }
+
+/**
+ * Cycle guard for a PROSPECTIVE `parent_names` write (vault#552). Setting
+ * `tag`'s parents to include `p` adds the edge `childrenOf[p] += tag` (tag
+ * becomes a child of `p`). That closes a cycle iff `tag` can ALREADY reach
+ * `p` via the existing hierarchy — i.e. `p` is already in `tag`'s descendant
+ * set — because the new edge would then let `p` reach back to itself through
+ * `tag`. Self-parent (`parentNames` containing `tag` itself) is caught by the
+ * same check: `getTagDescendants` always includes `tag` in its own result.
+ *
+ * Returns the offending cycle path (`[tag, ...existing chain down to p, p,
+ * tag]`, closing the loop) for the FIRST conflicting parent found, or `null`
+ * when no cycle would result. `h` must reflect the hierarchy BEFORE this
+ * write — the check only depends on OTHER tags' parent_names (edges pointing
+ * away from `tag`), never `tag`'s own current parent_names, so passing the
+ * pre-write hierarchy is always correct regardless of write ordering.
+ */
+export function findParentCycle(
+  h: TagHierarchy,
+  tag: string,
+  parentNames: string[],
+): string[] | null {
+  const descendants = getTagDescendants(h, tag);
+  for (const p of parentNames) {
+    if (descendants.has(p)) {
+      return [...findChildPath(h, tag, p), tag];
+    }
+  }
+  return null;
+}
+
+/**
+ * BFS shortest path from `from` to `to` following `childrenOf` edges
+ * (parent → child). Used only to render a human-legible cycle path for
+ * {@link findParentCycle}'s error — the existence of a path is already
+ * established by the caller (`to` is a known member of `from`'s descendant
+ * set) before this runs, so the fallback return is unreachable in practice
+ * and exists only to keep the function total.
+ */
+function findChildPath(h: TagHierarchy, from: string, to: string): string[] {
+  if (from === to) return [from];
+  const queue: string[] = [from];
+  const cameFrom = new Map<string, string>();
+  const visited = new Set<string>([from]);
+  while (queue.length > 0) {
+    const current = queue.shift()!;
+    const children = h.childrenOf.get(current);
+    if (!children) continue;
+    for (const child of children) {
+      if (visited.has(child)) continue;
+      visited.add(child);
+      cameFrom.set(child, current);
+      if (child === to) {
+        const path = [to];
+        let node = to;
+        while (cameFrom.has(node)) {
+          node = cameFrom.get(node)!;
+          path.unshift(node);
+        }
+        return path;
+      }
+      queue.push(child);
+    }
+  }
+  return [from, to];
+}