@openparachute/vault 0.7.0-rc.9 → 0.7.2-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/core/src/aggregate.test.ts +260 -0
  2. package/core/src/contract-taxonomy.test.ts +26 -2
  3. package/core/src/contract-typed-index.test.ts +4 -2
  4. package/core/src/core.test.ts +737 -2
  5. package/core/src/cursor-keyset-ms.test.ts +537 -0
  6. package/core/src/cursor.ts +76 -6
  7. package/core/src/doctor.ts +23 -1
  8. package/core/src/hooks.ts +9 -0
  9. package/core/src/indexed-fields.test.ts +4 -1
  10. package/core/src/indexed-fields.ts +6 -1
  11. package/core/src/links.ts +22 -0
  12. package/core/src/mcp.ts +322 -53
  13. package/core/src/notes.ts +518 -67
  14. package/core/src/paths.ts +4 -0
  15. package/core/src/portable-md.test.ts +161 -0
  16. package/core/src/portable-md.ts +140 -9
  17. package/core/src/query-warnings.ts +60 -12
  18. package/core/src/schema-defaults.ts +7 -1
  19. package/core/src/schema.ts +128 -2
  20. package/core/src/seed-packs.ts +55 -15
  21. package/core/src/store.ts +141 -7
  22. package/core/src/tag-schemas.ts +20 -7
  23. package/core/src/txn.test.ts +100 -4
  24. package/core/src/txn.ts +119 -24
  25. package/core/src/types.ts +89 -0
  26. package/core/src/ulid.test.ts +56 -0
  27. package/core/src/ulid.ts +116 -0
  28. package/core/src/vault-projection.ts +19 -1
  29. package/core/src/wikilinks.test.ts +205 -1
  30. package/core/src/wikilinks.ts +200 -35
  31. package/package.json +1 -1
  32. package/src/aggregate-routes.test.ts +230 -0
  33. package/src/cli.ts +6 -0
  34. package/src/contract-errors.test.ts +2 -1
  35. package/src/contract-search.test.ts +17 -0
  36. package/src/mcp-link-warnings-scope.test.ts +144 -0
  37. package/src/mcp-query-notes-aggregate-scope.test.ts +183 -0
  38. package/src/mcp-tools.ts +76 -17
  39. package/src/mirror-import.ts +5 -0
  40. package/src/routes.ts +298 -24
  41. package/src/routing.test.ts +167 -0
  42. package/src/routing.ts +47 -11
  43. package/src/tag-integrity-mcp.test.ts +45 -6
  44. package/src/tag-scope.ts +10 -1
  45. package/src/vault.test.ts +557 -21
  46. package/web/ui/dist/assets/{index-B8pGeQam.js → index-CJ6NtIwh.js} +1 -1
  47. package/web/ui/dist/index.html +1 -1
package/core/src/paths.ts CHANGED
@@ -19,6 +19,10 @@ export function normalizePath(path: string | null | undefined): string | null {
19
19
  if (path === null || path === undefined) return null;
20
20
 
21
21
  let p = path
22
+ .replace(/\0/g, "") // strip NUL bytes — never valid in a path;
23
+ // belt for legacy/imported rows so a stored
24
+ // NUL can't reach the filesystem (writes
25
+ // reject NUL up front via validatePath)
22
26
  .trim()
23
27
  .replace(/\\/g, "/") // backslash → forward slash
24
28
  .replace(/\.md$/i, "") // strip .md extension
@@ -29,6 +29,7 @@ import { buildVaultProjection } from "./vault-projection.js";
29
29
  import {
30
30
  CaseCollisionError,
31
31
  emitYamlDoc,
32
+ exportVault,
32
33
  exportVaultToDir,
33
34
  importPortableVault,
34
35
  noteToPortable,
@@ -41,7 +42,9 @@ import {
41
42
  supportsInlineFrontmatter,
42
43
  toPortableMarkdown,
43
44
  toSidecarYaml,
45
+ type ExportSink,
44
46
  type PortableNote,
47
+ type SinkWriteResult,
45
48
  } from "./portable-md.js";
46
49
 
47
50
  // ---------------------------------------------------------------------------
@@ -416,6 +419,86 @@ describe("exportVaultToDir", async () => {
416
419
  expect(vault).toContain("name: test");
417
420
  });
418
421
 
422
+ // FIX 2(b) (vault#589) — a NUL-in-path note (a pre-fix poisoned row a
423
+ // write-capable token could have planted) must be SKIPPED with a warning +
424
+ // stat, never abort the whole export. Before the sink hardening the
425
+ // uncaught `writeFileSync` throw aborted the entire vault export, so one bad
426
+ // note broke backup/export for everyone.
427
+ //
428
+ // RED-without-fix: drop the try/catch from `FsExportSink.writeText` and this
429
+ // test's `exportVaultToDir` call throws (whole export aborts) instead of
430
+ // returning a partial-with-skip result. Verified manually during development.
431
+ it("hardened export sink: a NUL-in-path note is skipped, not fatal (vault#589)", async () => {
432
+ await store.createNote("clean one", { id: "c1", path: "clean/one" });
433
+ await store.createNote("clean two", { id: "c2", path: "clean/two" });
434
+ await store.createNote("poison", { id: "p1", path: "temp" });
435
+ // Simulate a pre-fix vault: inject a real NUL directly into the stored
436
+ // path, bypassing normalizePath (which now strips NUL). `char(0)` yields a
437
+ // NUL byte in the TEXT column that survives read-back into note.path.
438
+ store.db.prepare("UPDATE notes SET path = 'bad' || char(0) || 'path' WHERE id = ?").run("p1");
439
+
440
+ const outDir = join(tmpBase, "poisoned-out");
441
+ const stats = await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
442
+
443
+ // The export completes and every OTHER note is written ...
444
+ expect(existsSync(join(outDir, "clean/one.md"))).toBe(true);
445
+ expect(existsSync(join(outDir, "clean/two.md"))).toBe(true);
446
+ expect(stats.notes).toBe(2); // only the two clean notes counted as written
447
+
448
+ // ... while the poisoned note is skipped-with-reason (fs write failure),
449
+ // surfaced as a stat rather than a thrown, aborted export.
450
+ expect(stats.skipped_traversal).toBeGreaterThanOrEqual(1);
451
+ const poisonSkip = stats.skipped_notes.find((s) => s.reason.includes("fs write failed"));
452
+ expect(poisonSkip).toBeTruthy();
453
+ });
454
+
455
+ // BLOCKER (vault#589 review) — the skip-with-warn export belt (FIX 2b) is
456
+ // correct ONLY for per-NOTE writes. The STRUCTURAL writes (the vault.yaml
457
+ // manifest + per-tag schema sidecars) are global: a dir missing
458
+ // `.parachute/vault.yaml` is hard-rejected at restore/import, so a discarded
459
+ // `{ok:false}` there would produce a manifest-less "success" — a silently
460
+ // broken backup. Those must FAIL LOUDLY.
461
+ //
462
+ // RED-without-fix: with the structural writes discarding the sink result,
463
+ // both `exportVault` calls RESOLVE (no throw) and the manifest/schema is
464
+ // silently absent. Verified manually during development.
465
+ it("aborts loudly when the manifest write fails (structural, not skip-with-warn)", async () => {
466
+ await store.createNote("n", { id: "n1", path: "p" });
467
+ // Stub sink that fails ONLY the manifest write; everything else succeeds.
468
+ const sink: ExportSink = {
469
+ caseSensitive: true,
470
+ attachmentsEnabled: false,
471
+ writeText(relPath: string): SinkWriteResult {
472
+ if (relPath === join(SIDECAR_DIR, "vault.yaml")) {
473
+ return { ok: false, reason: "simulated EACCES on .parachute/" };
474
+ }
475
+ return { ok: true };
476
+ },
477
+ copyAttachment(): SinkWriteResult { return { ok: true }; },
478
+ };
479
+ await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
480
+ .rejects.toThrow(/manifest/i);
481
+ });
482
+
483
+ it("aborts loudly when a schema sidecar write fails (structural, not skip-with-warn)", async () => {
484
+ await store.upsertTagSchema("task", { description: "A unit of work" });
485
+ await store.createNote("x", { id: "x1", tags: ["task"] });
486
+ // Fail ONLY the schema write; the manifest (and everything else) succeeds.
487
+ const sink: ExportSink = {
488
+ caseSensitive: true,
489
+ attachmentsEnabled: false,
490
+ writeText(relPath: string): SinkWriteResult {
491
+ if (relPath.startsWith(join(SIDECAR_DIR, "schemas"))) {
492
+ return { ok: false, reason: "simulated disk full" };
493
+ }
494
+ return { ok: true };
495
+ },
496
+ copyAttachment(): SinkWriteResult { return { ok: true }; },
497
+ };
498
+ await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
499
+ .rejects.toThrow(/schema sidecar/i);
500
+ });
501
+
419
502
  it("writes per-tag schemas to .parachute/schemas/", async () => {
420
503
  await store.upsertTagSchema("task", {
421
504
  description: "A unit of work",
@@ -710,6 +793,84 @@ describe("importPortableVault", async () => {
710
793
  expect(await targetStore.getNote("k1")).toBeTruthy();
711
794
  });
712
795
 
796
+ // FIX 1 (vault#589) — blow-away must be all-or-nothing. Before the fix, the
797
+ // wipe + replay ran with NO enclosing transaction: a mid-replay throw (here a
798
+ // PathConflictError from two source files declaring the same `path`) left the
799
+ // vault WIPED + partially replayed, originals gone for good. The wrap rolls
800
+ // the whole thing back to the exact pre-import vault.
801
+ //
802
+ // RED-without-fix: temporarily change `importVault` to always call
803
+ // `importVaultReplay` unwrapped (drop the `transactionAsync`) and this test
804
+ // ends with 1 note (`n-first`), all three originals gone — the assertions
805
+ // below fail. Verified manually during development.
806
+ it("--blow-away is atomic — a mid-replay throw rolls back to the original vault (vault#589)", async () => {
807
+ // Target vault holds originals that MUST survive a failed restore.
808
+ const target = new SqliteStore(new Database(":memory:"));
809
+ await target.createNote("original one", { id: "orig1", path: "keep/one" });
810
+ await target.createNote("original two", { id: "orig2", path: "keep/two" });
811
+ await target.createNote("original three", { id: "orig3", path: "keep/three" });
812
+
813
+ // Hand-build a poisoned export: two notes declaring the SAME frontmatter
814
+ // `path` → the 2nd createNote throws PathConflictError mid-replay. Files
815
+ // walk in sorted order (first.md before second.md), so first.md lands, then
816
+ // second.md collides.
817
+ const inDir = join(tmpBase, "poison-collide");
818
+ mkdirSync(join(inDir, ".parachute"), { recursive: true });
819
+ writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: poison\nexport_format_version: 1\n");
820
+ writeFileSync(join(inDir, "first.md"), "---\nid: nfirst\npath: collide\n---\nfirst body\n");
821
+ writeFileSync(join(inDir, "second.md"), "---\nid: nsecond\npath: collide\n---\nsecond body\n");
822
+
823
+ // The blow-away import propagates the collision (rejects) ...
824
+ await expect(importPortableVault(target, { inDir, blowAway: true })).rejects.toThrow();
825
+
826
+ // ... and rolls back: all three ORIGINALS survive, and NONE of the
827
+ // poisoned import's notes landed. The vault is exactly as it was.
828
+ expect(await target.getNote("orig1")).toBeTruthy();
829
+ expect(await target.getNote("orig2")).toBeTruthy();
830
+ expect(await target.getNote("orig3")).toBeTruthy();
831
+ expect(await target.getNote("nfirst")).toBeNull();
832
+ expect(await target.getNote("nsecond")).toBeNull();
833
+ const all = await target.queryNotes({ limit: 100 });
834
+ expect(all.length).toBe(3);
835
+ });
836
+
837
+ // FIX 3 (vault#589) — duplicate-id and blank-id content files must be
838
+ // reported + skipped, not silently clobber (last-wins) or seed a bogus key.
839
+ it("reports duplicate + blank note ids instead of silently clobbering (vault#589)", async () => {
840
+ // Hand-build a tree: two files share id "dup" (distinct content); a third
841
+ // carries a whitespace-only id. Files walk sorted: aaa < bbb < ccc.
842
+ const inDir = join(tmpBase, "dup-ids");
843
+ mkdirSync(join(inDir, ".parachute"), { recursive: true });
844
+ writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: dup\nexport_format_version: 1\n");
845
+ writeFileSync(join(inDir, "aaa.md"), "---\nid: dup\npath: pa\n---\nfrom aaa\n");
846
+ writeFileSync(join(inDir, "bbb.md"), "---\nid: dup\npath: pb\n---\nfrom bbb\n");
847
+ writeFileSync(join(inDir, "ccc.md"), "---\nid: ' '\npath: pc\n---\nfrom ccc\n");
848
+
849
+ const target = new SqliteStore(new Database(":memory:"));
850
+ const stats = await importPortableVault(target, { inDir });
851
+
852
+ // Exactly ONE note created — the FIRST of the id collision (aaa) — and the
853
+ // collision is NOT folded into "updated".
854
+ expect(stats.notes_created).toBe(1);
855
+ expect(stats.notes_updated).toBe(0);
856
+ const kept = await target.getNote("dup");
857
+ expect(kept).toBeTruthy();
858
+ expect(kept!.content.trimEnd()).toBe("from aaa"); // first-wins, deterministic
859
+
860
+ // Both the duplicate (bbb) and the blank-id (ccc) file are reported as
861
+ // skipped — surfaced in stats, not absorbed into the created/updated count.
862
+ expect(stats.skipped_duplicate_ids.length).toBe(2);
863
+ const dupEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("duplicate id"));
864
+ expect(dupEntry).toBeTruthy();
865
+ expect(dupEntry!.path).toBe("pb");
866
+ const blankEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("blank"));
867
+ expect(blankEntry).toBeTruthy();
868
+
869
+ // The whitespace-id file clobbered nothing — no bogus key, only "dup" exists.
870
+ const allNotes = await target.queryNotes({ limit: 100 });
871
+ expect(allNotes.length).toBe(1);
872
+ });
873
+
713
874
  it("restores tag schemas (description + fields)", async () => {
714
875
  await store.upsertTagSchema("task", {
715
876
  description: "A unit of work",
@@ -77,6 +77,7 @@ import { basename, join, relative, extname, dirname, resolve as resolvePath, sep
77
77
  import type { Store, Note, Link, Attachment } from "./types.js";
78
78
  import type { TagRecord } from "./tag-schemas.js";
79
79
  import { ParentCycleError } from "./tag-schemas.js";
80
+ import { transactionAsync } from "./txn.js";
80
81
 
81
82
  // ---------------------------------------------------------------------------
82
83
  // Format constants
@@ -786,8 +787,23 @@ export class FsExportSink implements ExportSink {
786
787
  reason: `path-traversal: resolved write target "${fullResolved}" escapes export root "${this.rootResolved}"`,
787
788
  };
788
789
  }
789
- mkdirSync(dirname(full), { recursive: true });
790
- writeFileSync(full, content);
790
+ // Belt for already-poisoned vaults (vault#589 / FIX 2). A NUL-in-path note
791
+ // resolves WITHIN the export root (passing the traversal guard above) but
792
+ // then makes `writeFileSync` throw ("path contains null byte") — an
793
+ // uncaught throw here would abort the ENTIRE export, so one bad note breaks
794
+ // backup for every note. Any fs failure (NUL, EACCES, ENAMETOOLONG, …)
795
+ // becomes a skip-with-reason the caller records as a `skipped_notes` stat +
796
+ // a warning, exactly like the traversal skip, so the rest of the vault
797
+ // still exports.
798
+ try {
799
+ mkdirSync(dirname(full), { recursive: true });
800
+ writeFileSync(full, content);
801
+ } catch (err) {
802
+ return {
803
+ ok: false,
804
+ reason: `fs write failed for "${relPath}": ${(err as Error)?.message ?? String(err)}`,
805
+ };
806
+ }
791
807
  return { ok: true };
792
808
  }
793
809
 
@@ -812,8 +828,17 @@ export class FsExportSink implements ExportSink {
812
828
  reason: `path-traversal: dest "${destResolved}" escapes export root "${this.rootResolved}"`,
813
829
  };
814
830
  }
815
- mkdirSync(dirname(destFull), { recursive: true });
816
- copyFileSync(srcResolved, destResolved);
831
+ // Same belt as `writeText` (vault#589 / FIX 2): a NUL/otherwise-unwritable
832
+ // attachment dest must skip-with-reason, not abort the whole export.
833
+ try {
834
+ mkdirSync(dirname(destFull), { recursive: true });
835
+ copyFileSync(srcResolved, destResolved);
836
+ } catch (err) {
837
+ return {
838
+ ok: false,
839
+ reason: `fs copy failed for "${destRelPath}": ${(err as Error)?.message ?? String(err)}`,
840
+ };
841
+ }
817
842
  return { ok: true };
818
843
  }
819
844
  }
@@ -878,11 +903,29 @@ export async function exportVault(
878
903
  ...(opts.vaultName ? { name: opts.vaultName } : {}),
879
904
  ...(opts.vaultDescription ? { description: opts.vaultDescription } : {}),
880
905
  };
881
- sink.writeText(join(SIDECAR_DIR, "vault.yaml"), emitYamlDoc(vaultMeta as unknown as Record<string, unknown>));
906
+ // The manifest is STRUCTURAL, not per-note: a portable-md export WITHOUT its
907
+ // `.parachute/vault.yaml` is unusable — restore (`importPortableVault`), CLI
908
+ // autodetect, and mirror-import all hard-reject a dir that lacks it. So the
909
+ // skip-with-warn belt that hardens per-note writes (vault#589 / FIX 2b) is
910
+ // exactly WRONG here: a discarded `{ok:false}` would produce a manifest-less
911
+ // "success" — a silently-broken backup. Fail LOUDLY instead. (Per-note
912
+ // content/attachment writes below stay skip-with-warn — one poisoned note
913
+ // must not abort the whole export.)
914
+ const manifestPath = join(SIDECAR_DIR, "vault.yaml");
915
+ const manifestWrite = sink.writeText(manifestPath, emitYamlDoc(vaultMeta as unknown as Record<string, unknown>));
916
+ if (!manifestWrite.ok) {
917
+ throw new Error(
918
+ `export aborted: failed to write the vault manifest "${manifestPath}": ${manifestWrite.reason}. ` +
919
+ `A portable-md export without its manifest is unusable (restore/import hard-rejects it), so this fails loudly rather than producing a manifest-less "success".`,
920
+ );
921
+ }
882
922
 
883
923
  // 2. Per-tag schemas. Only tags carrying at least one schema-shaped
884
924
  // field (description, fields, relationships, parent_names) get a file;
885
- // tags that are just-a-name don't pollute the sidecar.
925
+ // tags that are just-a-name don't pollute the sidecar. Schemas are
926
+ // structural too — a dropped schema sidecar silently loses a tag's
927
+ // definition from the backup — so a write failure here also fails loudly
928
+ // (and never increments `schemasWritten` on a failed write).
886
929
  const tagRecords = await store.listTagRecords();
887
930
  let schemasWritten = 0;
888
931
  for (const tag of tagRecords) {
@@ -895,7 +938,14 @@ export async function exportVault(
895
938
  if (tag.parent_names !== undefined && tag.parent_names.length > 0) {
896
939
  doc.parent_names = tag.parent_names;
897
940
  }
898
- sink.writeText(join(SIDECAR_DIR, "schemas", filename), emitYamlDoc(doc));
941
+ const schemaPath = join(SIDECAR_DIR, "schemas", filename);
942
+ const schemaWrite = sink.writeText(schemaPath, emitYamlDoc(doc));
943
+ if (!schemaWrite.ok) {
944
+ throw new Error(
945
+ `export aborted: failed to write schema sidecar "${schemaPath}" for tag "${tag.tag}": ${schemaWrite.reason}. ` +
946
+ `Tag schemas are structural — a partial export would silently drop a schema definition — so this fails loudly.`,
947
+ );
948
+ }
899
949
  schemasWritten++;
900
950
  }
901
951
 
@@ -1643,6 +1693,17 @@ export interface ImportStats {
1643
1693
  * imports — the cycle is warned, not fatal. Empty in the common case.
1644
1694
  */
1645
1695
  skipped_schema_parents: Array<{ tag: string; parent_names: string[]; reason: string }>;
1696
+ /**
1697
+ * Content files skipped because their frontmatter `id` couldn't be used as
1698
+ * a note key (vault#589 / FIX 3): a DUPLICATE id (a later file shares an id
1699
+ * with one already imported — the first wins, deterministically, since
1700
+ * `listContentFiles()` walks in sorted path order) or a BLANK id
1701
+ * (whitespace-only, e.g. `" "`). Without this the later duplicate silently
1702
+ * clobbered the earlier note (data loss folded into "updated N") and a
1703
+ * blank id seeded a bogus map key. Each entry names the offending id, the
1704
+ * skipped file's path, and why. Empty in the common case.
1705
+ */
1706
+ skipped_duplicate_ids: Array<{ id: string; path: string | undefined; reason: string }>;
1646
1707
  /** Set when the caller passed `blowAway: true`; counts notes removed. */
1647
1708
  notes_wiped: number;
1648
1709
  /**
@@ -1815,10 +1876,49 @@ export async function importVault(
1815
1876
  skipped_attachments: [],
1816
1877
  skipped_sidecars: [],
1817
1878
  skipped_schema_parents: [],
1879
+ skipped_duplicate_ids: [],
1818
1880
  notes_wiped: 0,
1819
1881
  indexes_declared: 0,
1820
1882
  };
1821
1883
 
1884
+ // Blow-away is all-or-nothing (vault#589 / FIX 1). A blow-away import wipes
1885
+ // the vault (step 1) then replays every schema/note/link/attachment. If any
1886
+ // replay step throws mid-flight — a `PathConflictError` from two source
1887
+ // files sharing a path, a malformed record, a crash — an UNWRAPPED run would
1888
+ // leave the vault wiped-and-partial with the originals gone for good. Wrap
1889
+ // the wipe + the whole replay in ONE transaction so a failure rolls back to
1890
+ // the exact pre-import vault. The transaction seam is re-entrant (SAVEPOINTs,
1891
+ // vault#589) so the replay's own transactional writes (`upsertTagRecord`,
1892
+ // `deleteTag` cascade, …) compose inside it rather than hitting SQLite's
1893
+ // "cannot start a transaction within a transaction". Additive (non-blow-away)
1894
+ // and dry-run imports are never destructive, so they replay unwrapped —
1895
+ // exactly as before. Hook dispatch stays best-effort/reconciled: no
1896
+ // registered hook does synchronous destructive work (the mirror arms a
1897
+ // debounce; transcription only fires on attachment `created` and kicks an
1898
+ // idempotent worker), so a rollback can't leave a half-applied side effect
1899
+ // the sweep won't reconcile.
1900
+ if (opts.blowAway && !opts.dryRun) {
1901
+ await transactionAsync(store.db, () => importVaultReplay(store, source, opts, stats));
1902
+ } else {
1903
+ await importVaultReplay(store, source, opts, stats);
1904
+ }
1905
+
1906
+ return stats;
1907
+ }
1908
+
1909
+ /**
1910
+ * The replay body of {@link importVault} — wipe (blow-away only) then restore
1911
+ * schemas → notes → links → attachments → wikilinks → indexes, mutating the
1912
+ * shared `stats`. Split out so `importVault` can run it either bare or inside a
1913
+ * single `transactionAsync` (the atomic blow-away path); the restoration order
1914
+ * and every skip/warn policy are unchanged from the pre-split code.
1915
+ */
1916
+ async function importVaultReplay(
1917
+ store: Store,
1918
+ source: ImportSource,
1919
+ opts: ImportEngineOptions,
1920
+ stats: ImportStats,
1921
+ ): Promise<void> {
1822
1922
  // 1. Optional wipe. Notes are deleted via the public Store API so hooks
1823
1923
  // fire (callers depend on `attachment.deleted` hooks for assets-dir
1824
1924
  // cleanup; we don't bypass that on blow-away). Deleted in bounded batches
@@ -2049,6 +2149,39 @@ export async function importVault(
2049
2149
  ...(links ? { links: links as PortableLink[] } : {}),
2050
2150
  ...(attachments ? { attachments: attachments as PortableAttachmentRef[] } : {}),
2051
2151
  };
2152
+
2153
+ // FIX 3 (vault#589) — guard the `seenNotes` key before it's set:
2154
+ // - A whitespace-only id (" ") slips past the `!id` truthiness guard
2155
+ // above (non-empty string) but is not a real note key. Skip it rather
2156
+ // than seeding a bogus map entry / clobbering a real note.
2157
+ // - A DUPLICATE id means a LATER content file carries an id already
2158
+ // imported from an earlier file. The old `Map.set` silently
2159
+ // last-wins-clobbered the first note (data loss, folded into
2160
+ // "updated N"). Keep the FIRST (deterministic — `listContentFiles()`
2161
+ // returns sorted paths) and skip the later, recording the collision.
2162
+ // Both cases land in `skipped_duplicate_ids` + a per-file `console.warn`
2163
+ // so the operator sees the collision instead of it vanishing.
2164
+ if (id.trim() === "") {
2165
+ stats.skipped_duplicate_ids.push({
2166
+ id,
2167
+ path: portable.path,
2168
+ reason: "blank (whitespace-only) id is not a valid note key",
2169
+ });
2170
+ // eslint-disable-next-line no-console
2171
+ console.warn(`[import] skipped "${relPath}": blank (whitespace-only) \`id\` in frontmatter`);
2172
+ continue;
2173
+ }
2174
+ if (seenNotes.has(id)) {
2175
+ const kept = seenNotes.get(id)!;
2176
+ stats.skipped_duplicate_ids.push({
2177
+ id,
2178
+ path: portable.path,
2179
+ reason: `duplicate id "${id}" — already imported from an earlier file${kept.path ? ` (path="${kept.path}")` : ""}; kept the first, skipped this one`,
2180
+ });
2181
+ // eslint-disable-next-line no-console
2182
+ console.warn(`[import] skipped "${relPath}": duplicate id "${id}" (kept the first file's note; this file's content is NOT imported)`);
2183
+ continue;
2184
+ }
2052
2185
  seenNotes.set(id, portable);
2053
2186
 
2054
2187
  if (opts.dryRun) {
@@ -2232,8 +2365,6 @@ export async function importVault(
2232
2365
  }
2233
2366
  }
2234
2367
  }
2235
-
2236
- return stats;
2237
2368
  }
2238
2369
 
2239
2370
  /**
@@ -34,6 +34,7 @@ import {
34
34
  type TagHierarchy,
35
35
  } from "./tag-hierarchy.js";
36
36
  import { chunkForInClause } from "./sql-in.js";
37
+ import { escapeFtsToken } from "./search-query.js";
37
38
 
38
39
  export interface QueryWarning {
39
40
  code: string;
@@ -231,13 +232,15 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
231
232
  * - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
232
233
  * created here LAZILY and BEST-EFFORT (not part of the schema/
233
234
  * migration — see the try/catch below) — the terms actually indexed,
234
- * POST-tokenization/stemming (porter). A suggestion therefore
235
- * sometimes reads as a stemmed form ("propoli" rather than "propolis")
236
- * rather than the original dictionary word an accepted tradeoff
237
- * (documented in docs/HTTP_API.md) rather than maintaining a second,
238
- * unstemmed index just for spelling suggestions.
235
+ * POST-tokenization/stemming (porter). The closest CANDIDATE is found
236
+ * against this stemmed vocabulary (small, deduped, cheap to
237
+ * range-scan), but a stemmed candidate is never returned verbatim
238
+ * see `resolveSurfaceForm` below, which maps it back to a real word
239
+ * before it's handed to the caller (vault#570 — a raw stem like
240
+ * "propoli" or "cactu" isn't a word anyone would type).
239
241
  * - tag names, when the caller passes `tagNames` (already cached by the
240
- * caller's tag hierarchy — free to include).
242
+ * caller's tag hierarchy — free to include, and already real words —
243
+ * no de-stemming needed).
241
244
  *
242
245
  * `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
243
246
  * itself (not a separately-enabled module) so it's expected to be
@@ -263,6 +266,8 @@ export function computeSearchDidYouMean(
263
266
  const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
264
267
  if (tokens.length === 0) return undefined;
265
268
 
269
+ const tagNameSet = tagNames ? new Set(tagNames) : undefined;
270
+
266
271
  db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
267
272
  const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
268
273
  const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
@@ -284,14 +289,19 @@ export function computeSearchDidYouMean(
284
289
  const hi = clean.length + 2;
285
290
  const rows = rangeStmt.all(lo, hi) as { term: string }[];
286
291
  const candidates = new Set<string>(rows.map((r) => r.term));
287
- if (tagNames) for (const t of tagNames) candidates.add(t);
292
+ if (tagNameSet) for (const t of tagNameSet) candidates.add(t);
288
293
 
289
294
  const suggestion = suggestSimilarTag(candidates, lower);
290
- if (suggestion && suggestion.toLowerCase() !== lower) {
291
- changed = true;
292
- return suggestion;
293
- }
294
- return tok;
295
+ if (!suggestion || suggestion.toLowerCase() === lower) return tok;
296
+
297
+ changed = true;
298
+ // A literal tag name is already a real word — return it verbatim. A
299
+ // vocabulary hit, though, came out of the PORTER-STEMMED index, so
300
+ // it's usually a truncated stem ("cactu" for "cactus") rather than a
301
+ // word a user would ever type (vault#570) — recover the real surface
302
+ // form before handing it back.
303
+ if (tagNameSet?.has(suggestion)) return suggestion;
304
+ return resolveSurfaceForm(db, suggestion) ?? suggestion;
295
305
  });
296
306
 
297
307
  if (!changed) return undefined;
@@ -301,6 +311,44 @@ export function computeSearchDidYouMean(
301
311
  }
302
312
  }
303
313
 
314
+ /**
315
+ * Map a porter-stemmed vocabulary term ("cactu") back to the real
316
+ * dictionary word a note actually contains ("cactus") — vault#570 (issue
317
+ * #570's search polish item). `notes_fts` is porter-tokenized, so MATCHing
318
+ * the bare stem finds any note carrying a word that stems to it (the stem
319
+ * came FROM `notes_fts_vocab` in the first place, so at least one match is
320
+ * guaranteed). From there, tokenize that note's own path/content text —
321
+ * unstemmed, done here in JS with the same punctuation-stripping rule used
322
+ * above — and return the first token that literally STARTS WITH the stem.
323
+ * This works because Porter's rules are almost entirely suffix-stripping:
324
+ * the stem is near-always a prefix of the original word. Irregular cases
325
+ * (a stem that isn't a literal prefix) are a known, accepted limitation —
326
+ * same class as the porter tokenizer's irregular-plural gap documented in
327
+ * docs/HTTP_API.md — and simply fall back to the raw stem via the caller's
328
+ * `?? suggestion`, never worse than the pre-fix behavior.
329
+ *
330
+ * Bounded to a handful of rows (`LIMIT 5`) and wrapped in try/catch same as
331
+ * the caller — a nicety, never worth risking the search response itself.
332
+ */
333
+ function resolveSurfaceForm(db: Database, stem: string): string | undefined {
334
+ try {
335
+ const rows = db
336
+ .prepare("SELECT path, content FROM notes_fts WHERE notes_fts MATCH ? LIMIT 5")
337
+ .all(escapeFtsToken(stem)) as { path: string | null; content: string | null }[];
338
+ for (const row of rows) {
339
+ const text = `${row.path ?? ""} ${row.content ?? ""}`;
340
+ for (const raw of text.split(/[^\p{L}\p{N}]+/gu)) {
341
+ if (!raw) continue;
342
+ const lower = raw.toLowerCase();
343
+ if (lower.length > stem.length && lower.startsWith(stem)) return lower;
344
+ }
345
+ }
346
+ return undefined;
347
+ } catch {
348
+ return undefined;
349
+ }
350
+ }
351
+
304
352
  /**
305
353
  * `search_did_you_mean` warning (vault#551 WS2B) — wraps
306
354
  * {@link computeSearchDidYouMean}'s suggestion in the standard warning
@@ -57,7 +57,7 @@ export interface SchemaField {
57
57
  * `type_mismatch` warning that previously fired on every integer-shaped
58
58
  * field because the validator had no `"integer"` case. See vault#310.
59
59
  */
60
- type?: "string" | "number" | "integer" | "boolean" | "array" | "object";
60
+ type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
61
61
  enum?: string[];
62
62
  description?: string;
63
63
  /**
@@ -369,6 +369,12 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
369
369
  return Array.isArray(value);
370
370
  case "object":
371
371
  return !!value && typeof value === "object" && !Array.isArray(value);
372
+ // `reference` (vault#typed-reference-field) validates like `string` —
373
+ // the write path (core/src/store.ts) separately resolves this value to
374
+ // a note and maintains a graph link; see tag-schemas.ts's
375
+ // `VALID_FIELD_TYPES` doc comment for the full contract.
376
+ case "reference":
377
+ return typeof value === "string";
372
378
  }
373
379
  }
374
380