@openparachute/vault 0.7.2-rc.4 → 0.7.2-rc.6

package/core/src/notes.ts

@@ -905,6 +905,50 @@ function validateIsoDateValue(field: string, value: string): void {
   }
 }
 
+/**
+ * Resolve a `date_filter` bound on the `updated_at` field to a millisecond
+ * epoch, for comparison against the integer `n.updated_at_ms` mirror column
+ * instead of the TEXT `n.updated_at` (vault#585 — the same TEXT-vs-ms class
+ * of bug vault#586 closed for the cursor keyset: a non-canonical stored
+ * timestamp — space-form, offset, no-`Z` — sorts/compares WRONG
+ * lexicographically).
+ *
+ * Runs `validateIsoDateValue` first — same loud `INVALID_QUERY` contract a
+ * caller already gets for a malformed bound — then converts with
+ * `timestampToMs` (the UTC-correct parse `updated_at_ms` itself was
+ * backfilled with). `timestampToMs` returning `null` here would mean the
+ * bound matched `Date.parse`'s more permissive grammar (e.g. a non-ISO
+ * locale string) but not `timestampToMs`'s stricter one — a defensive
+ * fallback, not an expected path for any bound `validateIsoDateValue`'s own
+ * error message advertises as valid ("2026-07-09" / full ISO timestamps).
+ * Surfaces the SAME `INVALID_QUERY` shape rather than binding `NaN`.
+ *
+ * One deliberate tightening for `updated_at` only: an ISO *shorthand* bound
+ * like `"2024"` or `"2024-06"` passes `Date.parse` (so `validateIsoDateValue`
+ * accepted it, and on a canonical vault it compared correctly via lexicographic
+ * prefix) but fails `timestampToMs`'s stricter `TIMESTAMP_RE`, so it now
+ * returns `INVALID_QUERY` here — loud, not silent. `date_filter` on
+ * `created_at` (no ms mirror) still accepts the shorthand; the asymmetry is
+ * intentional (ambiguous precision on a range bound).
+ */
+function resolveUpdatedAtBoundMs(field: string, value: string): number {
+  validateIsoDateValue(field, value);
+  const ms = timestampToMs(value);
+  if (ms === null) {
+    throw new QueryError(
+      `invalid date value for "${field}": ${JSON.stringify(value)} — must be an ISO-8601 date/timestamp (e.g. "2026-07-09" or "2026-07-09T00:00:00.000Z").`,
+      "INVALID_QUERY",
+      {
+        error_type: "invalid_query",
+        field,
+        got: value,
+        hint: "pass an ISO-8601 date or timestamp",
+      },
+    );
+  }
+  return ms;
+}
+
 /**
  * Build the shared WHERE-clause conditions + bound params for the filter
  * surface both `queryNotes` and `aggregateNotes` apply: tag membership
@@ -1169,15 +1213,17 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
     const filter = opts.dateFilter!;
     const field = filter.field ?? "created_at";
     let column: string;
+    // vault#585: `updated_at` compares against the integer `updated_at_ms`
+    // mirror (vault#586), not the TEXT `n.updated_at` — a TEXT compare
+    // mis-filters non-canonical rows (space-form / offset / no-`Z`) the same
+    // way the pre-v26 cursor keyset did. `created_at` has no ms mirror and
+    // stays a TEXT compare (out of scope — tracked separately, see vault#585
+    // follow-up note in the fix PR).
+    const isUpdatedAt = field === "updated_at";
     if (field === "created_at") {
       column = "n.created_at";
-    } else if (field === "updated_at") {
-      // NOTE (vault#586 follow-up): this range filter compares the TEXT
-      // `updated_at`, which is inconsistent on non-canonical rows the same way
-      // the cursor keyset was pre-v26. Cursor pagination moved to the integer
-      // `updated_at_ms` column; `date_filter`/`order_by`/export `--since` still
-      // read the string and are tracked separately. Behavior unchanged here.
-      column = "n.updated_at";
+    } else if (isUpdatedAt) {
+      column = "n.updated_at_ms";
     } else {
       // Re-uses the same indexed-field gate as `metadata` operator queries
       // and `orderBy` so the error message and contract are consistent.
@@ -1185,14 +1231,24 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
       column = `"meta_${field}"`;
     }
     if (filter.from !== undefined) {
-      validateIsoDateValue(field === "created_at" ? "date_filter.from" : `date_filter.from (${field})`, filter.from);
+      const label = field === "created_at" ? "date_filter.from" : `date_filter.from (${field})`;
       conditions.push(`${column} >= ?`);
-      params.push(filter.from);
+      if (isUpdatedAt) {
+        params.push(resolveUpdatedAtBoundMs(label, filter.from));
+      } else {
+        validateIsoDateValue(label, filter.from);
+        params.push(filter.from);
+      }
     }
     if (filter.to !== undefined) {
-      validateIsoDateValue(field === "created_at" ? "date_filter.to" : `date_filter.to (${field})`, filter.to);
+      const label = field === "created_at" ? "date_filter.to" : `date_filter.to (${field})`;
       conditions.push(`${column} < ?`);
-      params.push(filter.to);
+      if (isUpdatedAt) {
+        params.push(resolveUpdatedAtBoundMs(label, filter.to));
+      } else {
+        validateIsoDateValue(label, filter.to);
+        params.push(filter.to);
+      }
     }
   } else if (hasLegacyDate) {
     if (opts.dateFrom) {
@@ -1316,6 +1372,18 @@ export function queryNotes(db: Database, opts: QueryOpts, _outUpdatedAtMs?: Map<
     // without it, two notes sharing an `updated_at_ms` would be at the mercy
     // of SQLite's row order and the next page could miss or duplicate one.
     orderBy = "n.updated_at_ms ASC, n.id ASC";
+  } else if (opts.orderBy === "updated_at") {
+    // `updated_at` is a pseudo-field like `link_count` below — it's a native
+    // notes column, not a declared-indexed metadata field, so it bypasses
+    // `requireIndexedField` (vault#585). Prior to this, `order_by:
+    // "updated_at"` always 400'd FIELD_NOT_INDEXED (a documented, intentional
+    // reject — see the 2026-06 MCP-tool-description-audit CHANGELOG entry);
+    // it never silently sorted on the TEXT column. Ordering on the integer
+    // `updated_at_ms` mirror (vault#586) from the start means this new
+    // capability is UTC-correct on day one — no TEXT-lexicographic pass ever
+    // shipped for it. `id` is the explicit tiebreak for stable ordering when
+    // two notes share a millisecond, matching the cursor keyset's tiebreak.
+    orderBy = `n.updated_at_ms ${direction}, n.id ${direction}`;
   } else if (opts.orderBy === "link_count") {
     // `link_count` is a pseudo-field — like `created_at`/`updated_at` in the
     // dateFilter block above, it bypasses `requireIndexedField` (it's not a

package/core/src/portable-md.test.ts

@@ -24,6 +24,7 @@ import { join } from "path";
 import { tmpdir } from "os";
 
 import { SqliteStore } from "./store.js";
+import { timestampToMs } from "./cursor.js";
 import { getIndexedField } from "./indexed-fields.js";
 import { buildVaultProjection } from "./vault-projection.js";
 import {
@@ -593,6 +594,48 @@ describe("exportVaultToDir", async () => {
     }
   });
 
+  it("respects --since correctly on non-canonical `updated_at` (vault#585)", async () => {
+    // Independent db/store pair (not the describe's shared `store`) so the
+    // test can reach the raw `updated_at` / `updated_at_ms` columns
+    // directly — same pattern used elsewhere in this file (pruneOrphans,
+    // case-collision describes).
+    const db = new Database(":memory:");
+    const localStore = new SqliteStore(db);
+
+    const included = await localStore.createNote("space-form, after since", { id: "inc", path: "inc" });
+    const excluded = await localStore.createNote("canonical, before since", { id: "exc", path: "exc" });
+
+    // "2024-11-02 14:30:00" (space-form, no zone) is UTC-correct 14:30 on
+    // Nov 2 per timestampToMs — AFTER the 10:00 `since` bound below, so it
+    // MUST be exported. A TEXT `>=` comparison (the pre-vault#585 behavior)
+    // disagrees: the space (0x20) sorts BEFORE 'T' (0x54), so this string
+    // compares as LESS than the canonical `since` bound on the same
+    // calendar day — the old code wrongly EXCLUDED it from the incremental
+    // export.
+    const includedRaw = "2024-11-02 14:30:00";
+    const excludedRaw = "2024-11-01T00:00:00.000Z";
+    const includedMs = timestampToMs(includedRaw);
+    const excludedMs = timestampToMs(excludedRaw);
+    if (includedMs === null || excludedMs === null) {
+      throw new Error("test fixture bug: unparseable timestamp");
+    }
+    db.prepare("UPDATE notes SET updated_at = ?, updated_at_ms = ? WHERE id = ?")
+      .run(includedRaw, includedMs, included.id);
+    db.prepare("UPDATE notes SET updated_at = ?, updated_at_ms = ? WHERE id = ?")
+      .run(excludedRaw, excludedMs, excluded.id);
+
+    const outDir = join(tmpBase, "since-non-canonical");
+    const stats = await exportVaultToDir(localStore, {
+      outDir,
+      since: "2024-11-02T10:00:00.000Z",
+      exportedAt: "2026-05-12T00:00:00.000Z",
+    });
+
+    expect(stats.filtered_by_since).toBe(true);
+    expect(existsSync(join(outDir, "inc.md"))).toBe(true);
+    expect(existsSync(join(outDir, "exc.md"))).toBe(false);
+  });
+
   it("re-export with same exportedAt produces byte-identical output (idempotency)", async () => {
     await store.createNote("body", { id: "n1", path: "Inbox/x", tags: ["b", "a"] });
     await store.upsertTagSchema("a", { description: "tag-a" });

package/core/src/portable-md.ts

@@ -78,6 +78,7 @@ import type { Store, Note, Link, Attachment } from "./types.js";
 import type { TagRecord } from "./tag-schemas.js";
 import { ParentCycleError } from "./tag-schemas.js";
 import { transactionAsync } from "./txn.js";
+import { timestampToMs } from "./cursor.js";
 
 // ---------------------------------------------------------------------------
 // Format constants
@@ -1627,9 +1628,40 @@ function disambiguateFilename(path: string, extension: string, noteId: string):
   return `${path}__${idShort}.${extension}`;
 }
 
+/**
+ * Incremental-export inclusion test (vault#585). Compares millisecond
+ * epochs via {@link timestampToMs} — the SAME UTC-correct parse the
+ * `notes.updated_at_ms` column mirror was backfilled with (vault#586
+ * `migrateToV26`) — instead of a TEXT `>=` comparison on the stored ISO
+ * string. Import stores frontmatter timestamps VERBATIM, so an
+ * aged/imported vault routinely carries non-canonical `updated_at`
+ * (space-separated `2024-11-02 14:30:00`, a `+02:00` offset, no trailing
+ * `Z`); those sort WRONG against a canonical `.toISOString()` bound under
+ * plain string comparison, silently including or excluding notes from an
+ * incremental export.
+ *
+ * The fallback chain mirrors `migrateToV26`'s exactly — `updatedAt` →
+ * `createdAt` → the `0` (epoch) sentinel — so this function's answer for
+ * any given note is identical to what filtering on the real
+ * `updated_at_ms` column would produce; a note whose stored timestamp is
+ * genuinely unparseable sorts as "ancient" and drops out of an
+ * incremental export rather than being (wrongly) always-included or
+ * always-excluded by an arbitrary string comparison.
+ *
+ * `since` is caller input (CLI `--since`), already validated upstream
+ * (`Date.parse`-checked in `src/cli.ts` before this is ever reached from
+ * the CLI) — `timestampToMs` is tried first for UTC-correctness on the
+ * same non-canonical shapes, falling back to `Date.parse` for any exotic
+ * shape `timestampToMs`'s stricter grammar doesn't cover. A genuinely
+ * unparseable `since` (only reachable by calling the engine directly,
+ * bypassing the CLI guard) yields `NaN`, against which `>=` is always
+ * `false` — every note is excluded rather than the prior TEXT compare's
+ * arbitrary (and possibly all-inclusive) result.
+ */
 function shouldIncludeForSince(note: Note, since: string): boolean {
-  const stamp = note.updatedAt ?? note.createdAt;
-  return stamp >= since;
+  const stampMs = timestampToMs(note.updatedAt) ?? timestampToMs(note.createdAt) ?? 0;
+  const sinceMs = timestampToMs(since) ?? Date.parse(since);
+  return stampMs >= sinceMs;
 }
 
 // ---------------------------------------------------------------------------

package/core/src/schema-defaults.ts

@@ -348,7 +348,7 @@ function jsonTypeOf(value: unknown): string {
   return typeof value; // "string" | "number" | "boolean" | "object" | ...
 }
 
-function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
+function valueMatchesType(value: unknown, type: SchemaField["type"], cardinality?: SchemaField["cardinality"]): boolean {
   if (type === undefined) return true;
   switch (type) {
     case "string":
@@ -373,7 +373,22 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
     // the write path (core/src/store.ts) separately resolves this value to
     // a note and maintains a graph link; see tag-schemas.ts's
     // `VALID_FIELD_TYPES` doc comment for the full contract.
+    //
+    // `cardinality: "many"` (vault#typed-reference-field gap #2) is a
+    // one-to-MANY reference field — the value is an ARRAY of reference
+    // strings, one per linked note, not a single string. Validate the
+    // shape a "many" reference actually takes: an array whose elements are
+    // ALL non-empty-typed strings (per-item type check — the ARRAY shape
+    // itself is separately covered by the `cardinality_mismatch` check
+    // below in `validateNote`, so this only needs to judge element types).
+    // Without this branch, EVERY valid `cardinality:"many"` reference write
+    // fired a self-contradictory `type_mismatch` ("should be reference, got
+    // array") — an array is exactly what "many" asks for; only a
+    // non-string ELEMENT should ever fail this check.
     case "reference":
+      if (cardinality === "many") {
+        return Array.isArray(value) && value.every((item) => typeof item === "string");
+      }
       return typeof value === "string";
   }
 }
@@ -435,7 +450,7 @@ export function validateNote(
 
     if (absent) continue;
 
-    if (spec.type && !valueMatchesType(value, spec.type)) {
+    if (spec.type && !valueMatchesType(value, spec.type, spec.cardinality)) {
       // Decision A (vault#553): an INDEXED field's type is a query
       // contract — a type-mismatched write poisons range-query ordering
       // (SQLite's TEXT-sorts-above-INTEGER affinity) regardless of whether