@openparachute/vault 0.7.0-rc.9 → 0.7.2-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/aggregate.test.ts +260 -0
- package/core/src/contract-taxonomy.test.ts +26 -2
- package/core/src/contract-typed-index.test.ts +4 -2
- package/core/src/core.test.ts +737 -2
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/doctor.ts +23 -1
- package/core/src/hooks.ts +9 -0
- package/core/src/indexed-fields.test.ts +4 -1
- package/core/src/indexed-fields.ts +6 -1
- package/core/src/links.ts +22 -0
- package/core/src/mcp.ts +322 -53
- package/core/src/notes.ts +518 -67
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/query-warnings.ts +60 -12
- package/core/src/schema-defaults.ts +7 -1
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +55 -15
- package/core/src/store.ts +141 -7
- package/core/src/tag-schemas.ts +20 -7
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/core/src/types.ts +89 -0
- package/core/src/ulid.test.ts +56 -0
- package/core/src/ulid.ts +116 -0
- package/core/src/vault-projection.ts +19 -1
- package/core/src/wikilinks.test.ts +205 -1
- package/core/src/wikilinks.ts +200 -35
- package/package.json +1 -1
- package/src/aggregate-routes.test.ts +230 -0
- package/src/cli.ts +6 -0
- package/src/contract-errors.test.ts +2 -1
- package/src/contract-search.test.ts +17 -0
- package/src/mcp-link-warnings-scope.test.ts +144 -0
- package/src/mcp-query-notes-aggregate-scope.test.ts +183 -0
- package/src/mcp-tools.ts +76 -17
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +298 -24
- package/src/routing.test.ts +167 -0
- package/src/routing.ts +47 -11
- package/src/tag-integrity-mcp.test.ts +45 -6
- package/src/tag-scope.ts +10 -1
- package/src/vault.test.ts +557 -21
- package/web/ui/dist/assets/{index-B8pGeQam.js → index-CJ6NtIwh.js} +1 -1
- package/web/ui/dist/index.html +1 -1
package/core/src/paths.ts
CHANGED
|
@@ -19,6 +19,10 @@ export function normalizePath(path: string | null | undefined): string | null {
|
|
|
19
19
|
if (path === null || path === undefined) return null;
|
|
20
20
|
|
|
21
21
|
let p = path
|
|
22
|
+
.replace(/\0/g, "") // strip NUL bytes — never valid in a path;
|
|
23
|
+
// belt for legacy/imported rows so a stored
|
|
24
|
+
// NUL can't reach the filesystem (writes
|
|
25
|
+
// reject NUL up front via validatePath)
|
|
22
26
|
.trim()
|
|
23
27
|
.replace(/\\/g, "/") // backslash → forward slash
|
|
24
28
|
.replace(/\.md$/i, "") // strip .md extension
|
|
@@ -29,6 +29,7 @@ import { buildVaultProjection } from "./vault-projection.js";
|
|
|
29
29
|
import {
|
|
30
30
|
CaseCollisionError,
|
|
31
31
|
emitYamlDoc,
|
|
32
|
+
exportVault,
|
|
32
33
|
exportVaultToDir,
|
|
33
34
|
importPortableVault,
|
|
34
35
|
noteToPortable,
|
|
@@ -41,7 +42,9 @@ import {
|
|
|
41
42
|
supportsInlineFrontmatter,
|
|
42
43
|
toPortableMarkdown,
|
|
43
44
|
toSidecarYaml,
|
|
45
|
+
type ExportSink,
|
|
44
46
|
type PortableNote,
|
|
47
|
+
type SinkWriteResult,
|
|
45
48
|
} from "./portable-md.js";
|
|
46
49
|
|
|
47
50
|
// ---------------------------------------------------------------------------
|
|
@@ -416,6 +419,86 @@ describe("exportVaultToDir", async () => {
|
|
|
416
419
|
expect(vault).toContain("name: test");
|
|
417
420
|
});
|
|
418
421
|
|
|
422
|
+
// FIX 2(b) (vault#589) — a NUL-in-path note (a pre-fix poisoned row a
|
|
423
|
+
// write-capable token could have planted) must be SKIPPED with a warning +
|
|
424
|
+
// stat, never abort the whole export. Before the sink hardening the
|
|
425
|
+
// uncaught `writeFileSync` throw aborted the entire vault export, so one bad
|
|
426
|
+
// note broke backup/export for everyone.
|
|
427
|
+
//
|
|
428
|
+
// RED-without-fix: drop the try/catch from `FsExportSink.writeText` and this
|
|
429
|
+
// test's `exportVaultToDir` call throws (whole export aborts) instead of
|
|
430
|
+
// returning a partial-with-skip result. Verified manually during development.
|
|
431
|
+
it("hardened export sink: a NUL-in-path note is skipped, not fatal (vault#589)", async () => {
|
|
432
|
+
await store.createNote("clean one", { id: "c1", path: "clean/one" });
|
|
433
|
+
await store.createNote("clean two", { id: "c2", path: "clean/two" });
|
|
434
|
+
await store.createNote("poison", { id: "p1", path: "temp" });
|
|
435
|
+
// Simulate a pre-fix vault: inject a real NUL directly into the stored
|
|
436
|
+
// path, bypassing normalizePath (which now strips NUL). `char(0)` yields a
|
|
437
|
+
// NUL byte in the TEXT column that survives read-back into note.path.
|
|
438
|
+
store.db.prepare("UPDATE notes SET path = 'bad' || char(0) || 'path' WHERE id = ?").run("p1");
|
|
439
|
+
|
|
440
|
+
const outDir = join(tmpBase, "poisoned-out");
|
|
441
|
+
const stats = await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
442
|
+
|
|
443
|
+
// The export completes and every OTHER note is written ...
|
|
444
|
+
expect(existsSync(join(outDir, "clean/one.md"))).toBe(true);
|
|
445
|
+
expect(existsSync(join(outDir, "clean/two.md"))).toBe(true);
|
|
446
|
+
expect(stats.notes).toBe(2); // only the two clean notes counted as written
|
|
447
|
+
|
|
448
|
+
// ... while the poisoned note is skipped-with-reason (fs write failure),
|
|
449
|
+
// surfaced as a stat rather than a thrown, aborted export.
|
|
450
|
+
expect(stats.skipped_traversal).toBeGreaterThanOrEqual(1);
|
|
451
|
+
const poisonSkip = stats.skipped_notes.find((s) => s.reason.includes("fs write failed"));
|
|
452
|
+
expect(poisonSkip).toBeTruthy();
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
// BLOCKER (vault#589 review) — the skip-with-warn export belt (FIX 2b) is
|
|
456
|
+
// correct ONLY for per-NOTE writes. The STRUCTURAL writes (the vault.yaml
|
|
457
|
+
// manifest + per-tag schema sidecars) are global: a dir missing
|
|
458
|
+
// `.parachute/vault.yaml` is hard-rejected at restore/import, so a discarded
|
|
459
|
+
// `{ok:false}` there would produce a manifest-less "success" — a silently
|
|
460
|
+
// broken backup. Those must FAIL LOUDLY.
|
|
461
|
+
//
|
|
462
|
+
// RED-without-fix: with the structural writes discarding the sink result,
|
|
463
|
+
// both `exportVault` calls RESOLVE (no throw) and the manifest/schema is
|
|
464
|
+
// silently absent. Verified manually during development.
|
|
465
|
+
it("aborts loudly when the manifest write fails (structural, not skip-with-warn)", async () => {
|
|
466
|
+
await store.createNote("n", { id: "n1", path: "p" });
|
|
467
|
+
// Stub sink that fails ONLY the manifest write; everything else succeeds.
|
|
468
|
+
const sink: ExportSink = {
|
|
469
|
+
caseSensitive: true,
|
|
470
|
+
attachmentsEnabled: false,
|
|
471
|
+
writeText(relPath: string): SinkWriteResult {
|
|
472
|
+
if (relPath === join(SIDECAR_DIR, "vault.yaml")) {
|
|
473
|
+
return { ok: false, reason: "simulated EACCES on .parachute/" };
|
|
474
|
+
}
|
|
475
|
+
return { ok: true };
|
|
476
|
+
},
|
|
477
|
+
copyAttachment(): SinkWriteResult { return { ok: true }; },
|
|
478
|
+
};
|
|
479
|
+
await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
|
|
480
|
+
.rejects.toThrow(/manifest/i);
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
it("aborts loudly when a schema sidecar write fails (structural, not skip-with-warn)", async () => {
|
|
484
|
+
await store.upsertTagSchema("task", { description: "A unit of work" });
|
|
485
|
+
await store.createNote("x", { id: "x1", tags: ["task"] });
|
|
486
|
+
// Fail ONLY the schema write; the manifest (and everything else) succeeds.
|
|
487
|
+
const sink: ExportSink = {
|
|
488
|
+
caseSensitive: true,
|
|
489
|
+
attachmentsEnabled: false,
|
|
490
|
+
writeText(relPath: string): SinkWriteResult {
|
|
491
|
+
if (relPath.startsWith(join(SIDECAR_DIR, "schemas"))) {
|
|
492
|
+
return { ok: false, reason: "simulated disk full" };
|
|
493
|
+
}
|
|
494
|
+
return { ok: true };
|
|
495
|
+
},
|
|
496
|
+
copyAttachment(): SinkWriteResult { return { ok: true }; },
|
|
497
|
+
};
|
|
498
|
+
await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
|
|
499
|
+
.rejects.toThrow(/schema sidecar/i);
|
|
500
|
+
});
|
|
501
|
+
|
|
419
502
|
it("writes per-tag schemas to .parachute/schemas/", async () => {
|
|
420
503
|
await store.upsertTagSchema("task", {
|
|
421
504
|
description: "A unit of work",
|
|
@@ -710,6 +793,84 @@ describe("importPortableVault", async () => {
|
|
|
710
793
|
expect(await targetStore.getNote("k1")).toBeTruthy();
|
|
711
794
|
});
|
|
712
795
|
|
|
796
|
+
// FIX 1 (vault#589) — blow-away must be all-or-nothing. Before the fix, the
|
|
797
|
+
// wipe + replay ran with NO enclosing transaction: a mid-replay throw (here a
|
|
798
|
+
// PathConflictError from two source files declaring the same `path`) left the
|
|
799
|
+
// vault WIPED + partially replayed, originals gone for good. The wrap rolls
|
|
800
|
+
// the whole thing back to the exact pre-import vault.
|
|
801
|
+
//
|
|
802
|
+
// RED-without-fix: temporarily change `importVault` to always call
|
|
803
|
+
// `importVaultReplay` unwrapped (drop the `transactionAsync`) and this test
|
|
804
|
+
// ends with 1 note (`n-first`), all three originals gone — the assertions
|
|
805
|
+
// below fail. Verified manually during development.
|
|
806
|
+
it("--blow-away is atomic — a mid-replay throw rolls back to the original vault (vault#589)", async () => {
|
|
807
|
+
// Target vault holds originals that MUST survive a failed restore.
|
|
808
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
809
|
+
await target.createNote("original one", { id: "orig1", path: "keep/one" });
|
|
810
|
+
await target.createNote("original two", { id: "orig2", path: "keep/two" });
|
|
811
|
+
await target.createNote("original three", { id: "orig3", path: "keep/three" });
|
|
812
|
+
|
|
813
|
+
// Hand-build a poisoned export: two notes declaring the SAME frontmatter
|
|
814
|
+
// `path` → the 2nd createNote throws PathConflictError mid-replay. Files
|
|
815
|
+
// walk in sorted order (first.md before second.md), so first.md lands, then
|
|
816
|
+
// second.md collides.
|
|
817
|
+
const inDir = join(tmpBase, "poison-collide");
|
|
818
|
+
mkdirSync(join(inDir, ".parachute"), { recursive: true });
|
|
819
|
+
writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: poison\nexport_format_version: 1\n");
|
|
820
|
+
writeFileSync(join(inDir, "first.md"), "---\nid: nfirst\npath: collide\n---\nfirst body\n");
|
|
821
|
+
writeFileSync(join(inDir, "second.md"), "---\nid: nsecond\npath: collide\n---\nsecond body\n");
|
|
822
|
+
|
|
823
|
+
// The blow-away import propagates the collision (rejects) ...
|
|
824
|
+
await expect(importPortableVault(target, { inDir, blowAway: true })).rejects.toThrow();
|
|
825
|
+
|
|
826
|
+
// ... and rolls back: all three ORIGINALS survive, and NONE of the
|
|
827
|
+
// poisoned import's notes landed. The vault is exactly as it was.
|
|
828
|
+
expect(await target.getNote("orig1")).toBeTruthy();
|
|
829
|
+
expect(await target.getNote("orig2")).toBeTruthy();
|
|
830
|
+
expect(await target.getNote("orig3")).toBeTruthy();
|
|
831
|
+
expect(await target.getNote("nfirst")).toBeNull();
|
|
832
|
+
expect(await target.getNote("nsecond")).toBeNull();
|
|
833
|
+
const all = await target.queryNotes({ limit: 100 });
|
|
834
|
+
expect(all.length).toBe(3);
|
|
835
|
+
});
|
|
836
|
+
|
|
837
|
+
// FIX 3 (vault#589) — duplicate-id and blank-id content files must be
|
|
838
|
+
// reported + skipped, not silently clobber (last-wins) or seed a bogus key.
|
|
839
|
+
it("reports duplicate + blank note ids instead of silently clobbering (vault#589)", async () => {
|
|
840
|
+
// Hand-build a tree: two files share id "dup" (distinct content); a third
|
|
841
|
+
// carries a whitespace-only id. Files walk sorted: aaa < bbb < ccc.
|
|
842
|
+
const inDir = join(tmpBase, "dup-ids");
|
|
843
|
+
mkdirSync(join(inDir, ".parachute"), { recursive: true });
|
|
844
|
+
writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: dup\nexport_format_version: 1\n");
|
|
845
|
+
writeFileSync(join(inDir, "aaa.md"), "---\nid: dup\npath: pa\n---\nfrom aaa\n");
|
|
846
|
+
writeFileSync(join(inDir, "bbb.md"), "---\nid: dup\npath: pb\n---\nfrom bbb\n");
|
|
847
|
+
writeFileSync(join(inDir, "ccc.md"), "---\nid: ' '\npath: pc\n---\nfrom ccc\n");
|
|
848
|
+
|
|
849
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
850
|
+
const stats = await importPortableVault(target, { inDir });
|
|
851
|
+
|
|
852
|
+
// Exactly ONE note created — the FIRST of the id collision (aaa) — and the
|
|
853
|
+
// collision is NOT folded into "updated".
|
|
854
|
+
expect(stats.notes_created).toBe(1);
|
|
855
|
+
expect(stats.notes_updated).toBe(0);
|
|
856
|
+
const kept = await target.getNote("dup");
|
|
857
|
+
expect(kept).toBeTruthy();
|
|
858
|
+
expect(kept!.content.trimEnd()).toBe("from aaa"); // first-wins, deterministic
|
|
859
|
+
|
|
860
|
+
// Both the duplicate (bbb) and the blank-id (ccc) file are reported as
|
|
861
|
+
// skipped — surfaced in stats, not absorbed into the created/updated count.
|
|
862
|
+
expect(stats.skipped_duplicate_ids.length).toBe(2);
|
|
863
|
+
const dupEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("duplicate id"));
|
|
864
|
+
expect(dupEntry).toBeTruthy();
|
|
865
|
+
expect(dupEntry!.path).toBe("pb");
|
|
866
|
+
const blankEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("blank"));
|
|
867
|
+
expect(blankEntry).toBeTruthy();
|
|
868
|
+
|
|
869
|
+
// The whitespace-id file clobbered nothing — no bogus key, only "dup" exists.
|
|
870
|
+
const allNotes = await target.queryNotes({ limit: 100 });
|
|
871
|
+
expect(allNotes.length).toBe(1);
|
|
872
|
+
});
|
|
873
|
+
|
|
713
874
|
it("restores tag schemas (description + fields)", async () => {
|
|
714
875
|
await store.upsertTagSchema("task", {
|
|
715
876
|
description: "A unit of work",
|
package/core/src/portable-md.ts
CHANGED
|
@@ -77,6 +77,7 @@ import { basename, join, relative, extname, dirname, resolve as resolvePath, sep
|
|
|
77
77
|
import type { Store, Note, Link, Attachment } from "./types.js";
|
|
78
78
|
import type { TagRecord } from "./tag-schemas.js";
|
|
79
79
|
import { ParentCycleError } from "./tag-schemas.js";
|
|
80
|
+
import { transactionAsync } from "./txn.js";
|
|
80
81
|
|
|
81
82
|
// ---------------------------------------------------------------------------
|
|
82
83
|
// Format constants
|
|
@@ -786,8 +787,23 @@ export class FsExportSink implements ExportSink {
|
|
|
786
787
|
reason: `path-traversal: resolved write target "${fullResolved}" escapes export root "${this.rootResolved}"`,
|
|
787
788
|
};
|
|
788
789
|
}
|
|
789
|
-
|
|
790
|
-
|
|
790
|
+
// Belt for already-poisoned vaults (vault#589 / FIX 2). A NUL-in-path note
|
|
791
|
+
// resolves WITHIN the export root (passing the traversal guard above) but
|
|
792
|
+
// then makes `writeFileSync` throw ("path contains null byte") — an
|
|
793
|
+
// uncaught throw here would abort the ENTIRE export, so one bad note breaks
|
|
794
|
+
// backup for every note. Any fs failure (NUL, EACCES, ENAMETOOLONG, …)
|
|
795
|
+
// becomes a skip-with-reason the caller records as a `skipped_notes` stat +
|
|
796
|
+
// a warning, exactly like the traversal skip, so the rest of the vault
|
|
797
|
+
// still exports.
|
|
798
|
+
try {
|
|
799
|
+
mkdirSync(dirname(full), { recursive: true });
|
|
800
|
+
writeFileSync(full, content);
|
|
801
|
+
} catch (err) {
|
|
802
|
+
return {
|
|
803
|
+
ok: false,
|
|
804
|
+
reason: `fs write failed for "${relPath}": ${(err as Error)?.message ?? String(err)}`,
|
|
805
|
+
};
|
|
806
|
+
}
|
|
791
807
|
return { ok: true };
|
|
792
808
|
}
|
|
793
809
|
|
|
@@ -812,8 +828,17 @@ export class FsExportSink implements ExportSink {
|
|
|
812
828
|
reason: `path-traversal: dest "${destResolved}" escapes export root "${this.rootResolved}"`,
|
|
813
829
|
};
|
|
814
830
|
}
|
|
815
|
-
|
|
816
|
-
|
|
831
|
+
// Same belt as `writeText` (vault#589 / FIX 2): a NUL/otherwise-unwritable
|
|
832
|
+
// attachment dest must skip-with-reason, not abort the whole export.
|
|
833
|
+
try {
|
|
834
|
+
mkdirSync(dirname(destFull), { recursive: true });
|
|
835
|
+
copyFileSync(srcResolved, destResolved);
|
|
836
|
+
} catch (err) {
|
|
837
|
+
return {
|
|
838
|
+
ok: false,
|
|
839
|
+
reason: `fs copy failed for "${destRelPath}": ${(err as Error)?.message ?? String(err)}`,
|
|
840
|
+
};
|
|
841
|
+
}
|
|
817
842
|
return { ok: true };
|
|
818
843
|
}
|
|
819
844
|
}
|
|
@@ -878,11 +903,29 @@ export async function exportVault(
|
|
|
878
903
|
...(opts.vaultName ? { name: opts.vaultName } : {}),
|
|
879
904
|
...(opts.vaultDescription ? { description: opts.vaultDescription } : {}),
|
|
880
905
|
};
|
|
881
|
-
|
|
906
|
+
// The manifest is STRUCTURAL, not per-note: a portable-md export WITHOUT its
|
|
907
|
+
// `.parachute/vault.yaml` is unusable — restore (`importPortableVault`), CLI
|
|
908
|
+
// autodetect, and mirror-import all hard-reject a dir that lacks it. So the
|
|
909
|
+
// skip-with-warn belt that hardens per-note writes (vault#589 / FIX 2b) is
|
|
910
|
+
// exactly WRONG here: a discarded `{ok:false}` would produce a manifest-less
|
|
911
|
+
// "success" — a silently-broken backup. Fail LOUDLY instead. (Per-note
|
|
912
|
+
// content/attachment writes below stay skip-with-warn — one poisoned note
|
|
913
|
+
// must not abort the whole export.)
|
|
914
|
+
const manifestPath = join(SIDECAR_DIR, "vault.yaml");
|
|
915
|
+
const manifestWrite = sink.writeText(manifestPath, emitYamlDoc(vaultMeta as unknown as Record<string, unknown>));
|
|
916
|
+
if (!manifestWrite.ok) {
|
|
917
|
+
throw new Error(
|
|
918
|
+
`export aborted: failed to write the vault manifest "${manifestPath}": ${manifestWrite.reason}. ` +
|
|
919
|
+
`A portable-md export without its manifest is unusable (restore/import hard-rejects it), so this fails loudly rather than producing a manifest-less "success".`,
|
|
920
|
+
);
|
|
921
|
+
}
|
|
882
922
|
|
|
883
923
|
// 2. Per-tag schemas. Only tags carrying at least one schema-shaped
|
|
884
924
|
// field (description, fields, relationships, parent_names) get a file;
|
|
885
|
-
// tags that are just-a-name don't pollute the sidecar.
|
|
925
|
+
// tags that are just-a-name don't pollute the sidecar. Schemas are
|
|
926
|
+
// structural too — a dropped schema sidecar silently loses a tag's
|
|
927
|
+
// definition from the backup — so a write failure here also fails loudly
|
|
928
|
+
// (and never increments `schemasWritten` on a failed write).
|
|
886
929
|
const tagRecords = await store.listTagRecords();
|
|
887
930
|
let schemasWritten = 0;
|
|
888
931
|
for (const tag of tagRecords) {
|
|
@@ -895,7 +938,14 @@ export async function exportVault(
|
|
|
895
938
|
if (tag.parent_names !== undefined && tag.parent_names.length > 0) {
|
|
896
939
|
doc.parent_names = tag.parent_names;
|
|
897
940
|
}
|
|
898
|
-
|
|
941
|
+
const schemaPath = join(SIDECAR_DIR, "schemas", filename);
|
|
942
|
+
const schemaWrite = sink.writeText(schemaPath, emitYamlDoc(doc));
|
|
943
|
+
if (!schemaWrite.ok) {
|
|
944
|
+
throw new Error(
|
|
945
|
+
`export aborted: failed to write schema sidecar "${schemaPath}" for tag "${tag.tag}": ${schemaWrite.reason}. ` +
|
|
946
|
+
`Tag schemas are structural — a partial export would silently drop a schema definition — so this fails loudly.`,
|
|
947
|
+
);
|
|
948
|
+
}
|
|
899
949
|
schemasWritten++;
|
|
900
950
|
}
|
|
901
951
|
|
|
@@ -1643,6 +1693,17 @@ export interface ImportStats {
|
|
|
1643
1693
|
* imports — the cycle is warned, not fatal. Empty in the common case.
|
|
1644
1694
|
*/
|
|
1645
1695
|
skipped_schema_parents: Array<{ tag: string; parent_names: string[]; reason: string }>;
|
|
1696
|
+
/**
|
|
1697
|
+
* Content files skipped because their frontmatter `id` couldn't be used as
|
|
1698
|
+
* a note key (vault#589 / FIX 3): a DUPLICATE id (a later file shares an id
|
|
1699
|
+
* with one already imported — the first wins, deterministically, since
|
|
1700
|
+
* `listContentFiles()` walks in sorted path order) or a BLANK id
|
|
1701
|
+
* (whitespace-only, e.g. `" "`). Without this the later duplicate silently
|
|
1702
|
+
* clobbered the earlier note (data loss folded into "updated N") and a
|
|
1703
|
+
* blank id seeded a bogus map key. Each entry names the offending id, the
|
|
1704
|
+
* skipped file's path, and why. Empty in the common case.
|
|
1705
|
+
*/
|
|
1706
|
+
skipped_duplicate_ids: Array<{ id: string; path: string | undefined; reason: string }>;
|
|
1646
1707
|
/** Set when the caller passed `blowAway: true`; counts notes removed. */
|
|
1647
1708
|
notes_wiped: number;
|
|
1648
1709
|
/**
|
|
@@ -1815,10 +1876,49 @@ export async function importVault(
|
|
|
1815
1876
|
skipped_attachments: [],
|
|
1816
1877
|
skipped_sidecars: [],
|
|
1817
1878
|
skipped_schema_parents: [],
|
|
1879
|
+
skipped_duplicate_ids: [],
|
|
1818
1880
|
notes_wiped: 0,
|
|
1819
1881
|
indexes_declared: 0,
|
|
1820
1882
|
};
|
|
1821
1883
|
|
|
1884
|
+
// Blow-away is all-or-nothing (vault#589 / FIX 1). A blow-away import wipes
|
|
1885
|
+
// the vault (step 1) then replays every schema/note/link/attachment. If any
|
|
1886
|
+
// replay step throws mid-flight — a `PathConflictError` from two source
|
|
1887
|
+
// files sharing a path, a malformed record, a crash — an UNWRAPPED run would
|
|
1888
|
+
// leave the vault wiped-and-partial with the originals gone for good. Wrap
|
|
1889
|
+
// the wipe + the whole replay in ONE transaction so a failure rolls back to
|
|
1890
|
+
// the exact pre-import vault. The transaction seam is re-entrant (SAVEPOINTs,
|
|
1891
|
+
// vault#589) so the replay's own transactional writes (`upsertTagRecord`,
|
|
1892
|
+
// `deleteTag` cascade, …) compose inside it rather than hitting SQLite's
|
|
1893
|
+
// "cannot start a transaction within a transaction". Additive (non-blow-away)
|
|
1894
|
+
// and dry-run imports are never destructive, so they replay unwrapped —
|
|
1895
|
+
// exactly as before. Hook dispatch stays best-effort/reconciled: no
|
|
1896
|
+
// registered hook does synchronous destructive work (the mirror arms a
|
|
1897
|
+
// debounce; transcription only fires on attachment `created` and kicks an
|
|
1898
|
+
// idempotent worker), so a rollback can't leave a half-applied side effect
|
|
1899
|
+
// the sweep won't reconcile.
|
|
1900
|
+
if (opts.blowAway && !opts.dryRun) {
|
|
1901
|
+
await transactionAsync(store.db, () => importVaultReplay(store, source, opts, stats));
|
|
1902
|
+
} else {
|
|
1903
|
+
await importVaultReplay(store, source, opts, stats);
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
return stats;
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
/**
|
|
1910
|
+
* The replay body of {@link importVault} — wipe (blow-away only) then restore
|
|
1911
|
+
* schemas → notes → links → attachments → wikilinks → indexes, mutating the
|
|
1912
|
+
* shared `stats`. Split out so `importVault` can run it either bare or inside a
|
|
1913
|
+
* single `transactionAsync` (the atomic blow-away path); the restoration order
|
|
1914
|
+
* and every skip/warn policy are unchanged from the pre-split code.
|
|
1915
|
+
*/
|
|
1916
|
+
async function importVaultReplay(
|
|
1917
|
+
store: Store,
|
|
1918
|
+
source: ImportSource,
|
|
1919
|
+
opts: ImportEngineOptions,
|
|
1920
|
+
stats: ImportStats,
|
|
1921
|
+
): Promise<void> {
|
|
1822
1922
|
// 1. Optional wipe. Notes are deleted via the public Store API so hooks
|
|
1823
1923
|
// fire (callers depend on `attachment.deleted` hooks for assets-dir
|
|
1824
1924
|
// cleanup; we don't bypass that on blow-away). Deleted in bounded batches
|
|
@@ -2049,6 +2149,39 @@ export async function importVault(
|
|
|
2049
2149
|
...(links ? { links: links as PortableLink[] } : {}),
|
|
2050
2150
|
...(attachments ? { attachments: attachments as PortableAttachmentRef[] } : {}),
|
|
2051
2151
|
};
|
|
2152
|
+
|
|
2153
|
+
// FIX 3 (vault#589) — guard the `seenNotes` key before it's set:
|
|
2154
|
+
// - A whitespace-only id (" ") slips past the `!id` truthiness guard
|
|
2155
|
+
// above (non-empty string) but is not a real note key. Skip it rather
|
|
2156
|
+
// than seeding a bogus map entry / clobbering a real note.
|
|
2157
|
+
// - A DUPLICATE id means a LATER content file carries an id already
|
|
2158
|
+
// imported from an earlier file. The old `Map.set` silently
|
|
2159
|
+
// last-wins-clobbered the first note (data loss, folded into
|
|
2160
|
+
// "updated N"). Keep the FIRST (deterministic — `listContentFiles()`
|
|
2161
|
+
// returns sorted paths) and skip the later, recording the collision.
|
|
2162
|
+
// Both cases land in `skipped_duplicate_ids` + a per-file `console.warn`
|
|
2163
|
+
// so the operator sees the collision instead of it vanishing.
|
|
2164
|
+
if (id.trim() === "") {
|
|
2165
|
+
stats.skipped_duplicate_ids.push({
|
|
2166
|
+
id,
|
|
2167
|
+
path: portable.path,
|
|
2168
|
+
reason: "blank (whitespace-only) id is not a valid note key",
|
|
2169
|
+
});
|
|
2170
|
+
// eslint-disable-next-line no-console
|
|
2171
|
+
console.warn(`[import] skipped "${relPath}": blank (whitespace-only) \`id\` in frontmatter`);
|
|
2172
|
+
continue;
|
|
2173
|
+
}
|
|
2174
|
+
if (seenNotes.has(id)) {
|
|
2175
|
+
const kept = seenNotes.get(id)!;
|
|
2176
|
+
stats.skipped_duplicate_ids.push({
|
|
2177
|
+
id,
|
|
2178
|
+
path: portable.path,
|
|
2179
|
+
reason: `duplicate id "${id}" — already imported from an earlier file${kept.path ? ` (path="${kept.path}")` : ""}; kept the first, skipped this one`,
|
|
2180
|
+
});
|
|
2181
|
+
// eslint-disable-next-line no-console
|
|
2182
|
+
console.warn(`[import] skipped "${relPath}": duplicate id "${id}" (kept the first file's note; this file's content is NOT imported)`);
|
|
2183
|
+
continue;
|
|
2184
|
+
}
|
|
2052
2185
|
seenNotes.set(id, portable);
|
|
2053
2186
|
|
|
2054
2187
|
if (opts.dryRun) {
|
|
@@ -2232,8 +2365,6 @@ export async function importVault(
|
|
|
2232
2365
|
}
|
|
2233
2366
|
}
|
|
2234
2367
|
}
|
|
2235
|
-
|
|
2236
|
-
return stats;
|
|
2237
2368
|
}
|
|
2238
2369
|
|
|
2239
2370
|
/**
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
type TagHierarchy,
|
|
35
35
|
} from "./tag-hierarchy.js";
|
|
36
36
|
import { chunkForInClause } from "./sql-in.js";
|
|
37
|
+
import { escapeFtsToken } from "./search-query.js";
|
|
37
38
|
|
|
38
39
|
export interface QueryWarning {
|
|
39
40
|
code: string;
|
|
@@ -231,13 +232,15 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
|
|
|
231
232
|
* - the FTS5 vocabulary — a `notes_fts_vocab` `fts5vocab('row')` table
|
|
232
233
|
* created here LAZILY and BEST-EFFORT (not part of the schema/
|
|
233
234
|
* migration — see the try/catch below) — the terms actually indexed,
|
|
234
|
-
* POST-tokenization/stemming (porter).
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
*
|
|
235
|
+
* POST-tokenization/stemming (porter). The closest CANDIDATE is found
|
|
236
|
+
* against this stemmed vocabulary (small, deduped, cheap to
|
|
237
|
+
* range-scan), but a stemmed candidate is never returned verbatim —
|
|
238
|
+
* see `resolveSurfaceForm` below, which maps it back to a real word
|
|
239
|
+
* before it's handed to the caller (vault#570 — a raw stem like
|
|
240
|
+
* "propoli" or "cactu" isn't a word anyone would type).
|
|
239
241
|
* - tag names, when the caller passes `tagNames` (already cached by the
|
|
240
|
-
* caller's tag hierarchy — free to include
|
|
242
|
+
* caller's tag hierarchy — free to include, and already real words —
|
|
243
|
+
* no de-stemming needed).
|
|
241
244
|
*
|
|
242
245
|
* `fts5vocab` is registered by the same FTS5 extension init as `notes_fts`
|
|
243
246
|
* itself (not a separately-enabled module) so it's expected to be
|
|
@@ -263,6 +266,8 @@ export function computeSearchDidYouMean(
|
|
|
263
266
|
const tokens = rawQuery.trim().split(/\s+/).filter(Boolean);
|
|
264
267
|
if (tokens.length === 0) return undefined;
|
|
265
268
|
|
|
269
|
+
const tagNameSet = tagNames ? new Set(tagNames) : undefined;
|
|
270
|
+
|
|
266
271
|
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts_vocab USING fts5vocab(notes_fts, 'row')");
|
|
267
272
|
const exactStmt = db.prepare("SELECT 1 FROM notes_fts_vocab WHERE term = ? LIMIT 1");
|
|
268
273
|
const rangeStmt = db.prepare("SELECT term FROM notes_fts_vocab WHERE length(term) BETWEEN ? AND ?");
|
|
@@ -284,14 +289,19 @@ export function computeSearchDidYouMean(
|
|
|
284
289
|
const hi = clean.length + 2;
|
|
285
290
|
const rows = rangeStmt.all(lo, hi) as { term: string }[];
|
|
286
291
|
const candidates = new Set<string>(rows.map((r) => r.term));
|
|
287
|
-
if (
|
|
292
|
+
if (tagNameSet) for (const t of tagNameSet) candidates.add(t);
|
|
288
293
|
|
|
289
294
|
const suggestion = suggestSimilarTag(candidates, lower);
|
|
290
|
-
if (suggestion
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
+
if (!suggestion || suggestion.toLowerCase() === lower) return tok;
|
|
296
|
+
|
|
297
|
+
changed = true;
|
|
298
|
+
// A literal tag name is already a real word — return it verbatim. A
|
|
299
|
+
// vocabulary hit, though, came out of the PORTER-STEMMED index, so
|
|
300
|
+
// it's usually a truncated stem ("cactu" for "cactus") rather than a
|
|
301
|
+
// word a user would ever type (vault#570) — recover the real surface
|
|
302
|
+
// form before handing it back.
|
|
303
|
+
if (tagNameSet?.has(suggestion)) return suggestion;
|
|
304
|
+
return resolveSurfaceForm(db, suggestion) ?? suggestion;
|
|
295
305
|
});
|
|
296
306
|
|
|
297
307
|
if (!changed) return undefined;
|
|
@@ -301,6 +311,44 @@ export function computeSearchDidYouMean(
|
|
|
301
311
|
}
|
|
302
312
|
}
|
|
303
313
|
|
|
314
|
+
/**
|
|
315
|
+
* Map a porter-stemmed vocabulary term ("cactu") back to the real
|
|
316
|
+
* dictionary word a note actually contains ("cactus") — vault#570 (issue
|
|
317
|
+
* #570's search polish item). `notes_fts` is porter-tokenized, so MATCHing
|
|
318
|
+
* the bare stem finds any note carrying a word that stems to it (the stem
|
|
319
|
+
* came FROM `notes_fts_vocab` in the first place, so at least one match is
|
|
320
|
+
* guaranteed). From there, tokenize that note's own path/content text —
|
|
321
|
+
* unstemmed, done here in JS with the same punctuation-stripping rule used
|
|
322
|
+
* above — and return the first token that literally STARTS WITH the stem.
|
|
323
|
+
* This works because Porter's rules are almost entirely suffix-stripping:
|
|
324
|
+
* the stem is near-always a prefix of the original word. Irregular cases
|
|
325
|
+
* (a stem that isn't a literal prefix) are a known, accepted limitation —
|
|
326
|
+
* same class as the porter tokenizer's irregular-plural gap documented in
|
|
327
|
+
* docs/HTTP_API.md — and simply fall back to the raw stem via the caller's
|
|
328
|
+
* `?? suggestion`, never worse than the pre-fix behavior.
|
|
329
|
+
*
|
|
330
|
+
* Bounded to a handful of rows (`LIMIT 5`) and wrapped in try/catch same as
|
|
331
|
+
* the caller — a nicety, never worth risking the search response itself.
|
|
332
|
+
*/
|
|
333
|
+
function resolveSurfaceForm(db: Database, stem: string): string | undefined {
|
|
334
|
+
try {
|
|
335
|
+
const rows = db
|
|
336
|
+
.prepare("SELECT path, content FROM notes_fts WHERE notes_fts MATCH ? LIMIT 5")
|
|
337
|
+
.all(escapeFtsToken(stem)) as { path: string | null; content: string | null }[];
|
|
338
|
+
for (const row of rows) {
|
|
339
|
+
const text = `${row.path ?? ""} ${row.content ?? ""}`;
|
|
340
|
+
for (const raw of text.split(/[^\p{L}\p{N}]+/gu)) {
|
|
341
|
+
if (!raw) continue;
|
|
342
|
+
const lower = raw.toLowerCase();
|
|
343
|
+
if (lower.length > stem.length && lower.startsWith(stem)) return lower;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
return undefined;
|
|
347
|
+
} catch {
|
|
348
|
+
return undefined;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
304
352
|
/**
|
|
305
353
|
* `search_did_you_mean` warning (vault#551 WS2B) — wraps
|
|
306
354
|
* {@link computeSearchDidYouMean}'s suggestion in the standard warning
|
|
@@ -57,7 +57,7 @@ export interface SchemaField {
|
|
|
57
57
|
* `type_mismatch` warning that previously fired on every integer-shaped
|
|
58
58
|
* field because the validator had no `"integer"` case. See vault#310.
|
|
59
59
|
*/
|
|
60
|
-
type?: "string" | "number" | "integer" | "boolean" | "array" | "object";
|
|
60
|
+
type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
|
|
61
61
|
enum?: string[];
|
|
62
62
|
description?: string;
|
|
63
63
|
/**
|
|
@@ -369,6 +369,12 @@ function valueMatchesType(value: unknown, type: SchemaField["type"]): boolean {
|
|
|
369
369
|
return Array.isArray(value);
|
|
370
370
|
case "object":
|
|
371
371
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
372
|
+
// `reference` (vault#typed-reference-field) validates like `string` —
|
|
373
|
+
// the write path (core/src/store.ts) separately resolves this value to
|
|
374
|
+
// a note and maintains a graph link; see tag-schemas.ts's
|
|
375
|
+
// `VALID_FIELD_TYPES` doc comment for the full contract.
|
|
376
|
+
case "reference":
|
|
377
|
+
return typeof value === "string";
|
|
372
378
|
}
|
|
373
379
|
}
|
|
374
380
|
|