@openparachute/vault 0.7.0-rc.4 → 0.7.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/attribution.test.ts +11 -3
- package/core/src/contract-taxonomy.test.ts +193 -17
- package/core/src/contract-typed-index.test.ts +228 -11
- package/core/src/core.test.ts +67 -14
- package/core/src/doctor-scope.test.ts +117 -0
- package/core/src/doctor.ts +360 -0
- package/core/src/mcp.ts +164 -28
- package/core/src/notes.ts +98 -24
- package/core/src/portable-md.test.ts +107 -0
- package/core/src/portable-md.ts +51 -5
- package/core/src/schema-defaults.ts +48 -7
- package/core/src/schema.ts +147 -2
- package/core/src/seed-packs.ts +20 -13
- package/core/src/store.ts +35 -3
- package/core/src/tag-hierarchy.ts +66 -0
- package/core/src/tag-schemas.ts +166 -6
- package/core/src/types.ts +23 -1
- package/package.json +1 -1
- package/src/mcp-http.ts +13 -0
- package/src/mcp-tools.ts +90 -5
- package/src/mirror-import.ts +5 -0
- package/src/onboarding-seed.test.ts +5 -2
- package/src/routes.ts +91 -52
- package/src/routing.ts +29 -0
- package/src/tag-integrity-mcp.test.ts +288 -0
- package/src/tag-integrity-scope.test.ts +97 -0
- package/src/tag-scope.ts +43 -0
- package/src/vault.test.ts +58 -35
package/core/src/core.test.ts
CHANGED
|
@@ -2030,6 +2030,12 @@ describe("MCP tools", async () => {
|
|
|
2030
2030
|
// prune-schema (admin) — drops orphaned indexed-field columns whose
|
|
2031
2031
|
// declaring tags are gone. The gitcoin orphaned-fields fix.
|
|
2032
2032
|
expect(names).toContain("prune-schema");
|
|
2033
|
+
// rename-tag / merge-tags (vault#552) — MCP parity with the
|
|
2034
|
+
// pre-existing REST rename/merge engine. doctor (admin, vault#552) —
|
|
2035
|
+
// read-only taxonomy/metadata integrity scan.
|
|
2036
|
+
expect(names).toContain("rename-tag");
|
|
2037
|
+
expect(names).toContain("merge-tags");
|
|
2038
|
+
expect(names).toContain("doctor");
|
|
2033
2039
|
// Six note-schema tools (list/update/delete-note-schema +
|
|
2034
2040
|
// list/set/delete-schema-mapping) retired in v17 — the standalone
|
|
2035
2041
|
// note_schemas + schema_mappings subsystem was a parallel path to
|
|
@@ -2043,7 +2049,7 @@ describe("MCP tools", async () => {
|
|
|
2043
2049
|
// synthesize-notes retired in v17 — replicable with query-notes(near=) +
|
|
2044
2050
|
// find-path + agent-side aggregation. See vault#268.
|
|
2045
2051
|
expect(names).not.toContain("synthesize-notes");
|
|
2046
|
-
expect(tools).toHaveLength(
|
|
2052
|
+
expect(tools).toHaveLength(13);
|
|
2047
2053
|
});
|
|
2048
2054
|
|
|
2049
2055
|
it("create-note tool works", async () => {
|
|
@@ -2075,8 +2081,11 @@ describe("MCP tools", async () => {
|
|
|
2075
2081
|
// schema-default-filled field was missing from the returned note even
|
|
2076
2082
|
// though it had just been written to disk.
|
|
2077
2083
|
it("create-note response reflects post-applySchemaDefaults state (vault#316)", async () => {
|
|
2084
|
+
// vault#553 Decision B: backfill is explicit-`default`-only now — declare
|
|
2085
|
+
// one so this test still exercises the post-defaults re-read mechanism
|
|
2086
|
+
// (the thing vault#316 is actually about).
|
|
2078
2087
|
await store.upsertTagSchema("task", {
|
|
2079
|
-
fields: { priority: { type: "string", enum: ["high", "low"] } },
|
|
2088
|
+
fields: { priority: { type: "string", enum: ["high", "low"], default: "high" } },
|
|
2080
2089
|
});
|
|
2081
2090
|
const tools = generateMcpTools(store);
|
|
2082
2091
|
const createNote = tools.find((t) => t.name === "create-note")!;
|
|
@@ -2087,7 +2096,7 @@ describe("MCP tools", async () => {
|
|
|
2087
2096
|
path: "Inbox/task-1",
|
|
2088
2097
|
tags: ["task"],
|
|
2089
2098
|
}) as any;
|
|
2090
|
-
expect(single.metadata?.priority).toBe("high"); //
|
|
2099
|
+
expect(single.metadata?.priority).toBe("high"); // explicit schema default
|
|
2091
2100
|
// Disk and response agree.
|
|
2092
2101
|
const onDisk = await store.getNoteByPath("Inbox/task-1");
|
|
2093
2102
|
expect((onDisk!.metadata as any)?.priority).toBe("high");
|
|
@@ -3450,7 +3459,7 @@ describe("MCP tools", async () => {
|
|
|
3450
3459
|
expect(links.some((l) => l.relationship === "wikilink")).toBe(true);
|
|
3451
3460
|
});
|
|
3452
3461
|
|
|
3453
|
-
it("create-note with schema tag
|
|
3462
|
+
it("create-note with schema tag leaves undeclared fields ABSENT (vault#553 Decision B — no implicit enum[0]/zero-value backfill)", async () => {
|
|
3454
3463
|
await store.upsertTagSchema("person", {
|
|
3455
3464
|
description: "A person",
|
|
3456
3465
|
fields: {
|
|
@@ -3466,10 +3475,36 @@ describe("MCP tools", async () => {
|
|
|
3466
3475
|
|
|
3467
3476
|
const result = await createNote.execute({ content: "Alice", tags: ["person"] }) as any;
|
|
3468
3477
|
const fresh = await query.execute({ id: result.id }) as any;
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3478
|
+
// None of these fields declares a `default` — the pre-0.7.0 behavior
|
|
3479
|
+
// (enum[0] / type zero-value backfill) is retired. "Never set" now stays
|
|
3480
|
+
// genuinely absent instead of masquerading as "explicitly set to X" — in
|
|
3481
|
+
// fact NO field got backfilled, so `metadata` itself stays empty/absent.
|
|
3482
|
+
expect(fresh.metadata?.first_appeared).toBeUndefined();
|
|
3483
|
+
expect(fresh.metadata?.active).toBeUndefined();
|
|
3484
|
+
expect(fresh.metadata?.priority).toBeUndefined();
|
|
3485
|
+
expect(fresh.metadata?.status).toBeUndefined();
|
|
3486
|
+
});
|
|
3487
|
+
|
|
3488
|
+
it("create-note with schema tag applies EXPLICIT `default:` values only (vault#553 Decision B)", async () => {
|
|
3489
|
+
await store.upsertTagSchema("employee", {
|
|
3490
|
+
description: "An employee",
|
|
3491
|
+
fields: {
|
|
3492
|
+
active: { type: "boolean", default: true },
|
|
3493
|
+
priority: { type: "integer", default: 3 },
|
|
3494
|
+
status: { type: "string", enum: ["active", "archived"], default: "active" },
|
|
3495
|
+
nickname: { type: "string" }, // no default — stays absent
|
|
3496
|
+
},
|
|
3497
|
+
});
|
|
3498
|
+
const tools = generateMcpTools(store);
|
|
3499
|
+
const createNote = tools.find((t) => t.name === "create-note")!;
|
|
3500
|
+
const query = tools.find((t) => t.name === "query-notes")!;
|
|
3501
|
+
|
|
3502
|
+
const result = await createNote.execute({ content: "Bob", tags: ["employee"] }) as any;
|
|
3503
|
+
const fresh = await query.execute({ id: result.id }) as any;
|
|
3504
|
+
expect(fresh.metadata.active).toBe(true);
|
|
3505
|
+
expect(fresh.metadata.priority).toBe(3);
|
|
3472
3506
|
expect(fresh.metadata.status).toBe("active");
|
|
3507
|
+
expect(fresh.metadata.nickname).toBeUndefined();
|
|
3473
3508
|
});
|
|
3474
3509
|
|
|
3475
3510
|
// ---- query-notes input-shape tolerance (vault#214) ----
|
|
@@ -4170,7 +4205,17 @@ describe("tag hierarchy (tags.parent_names)", async () => {
|
|
|
4170
4205
|
|
|
4171
4206
|
it("tolerates a cycle without infinite-looping", async () => {
|
|
4172
4207
|
await store.upsertTagRecord("a", { parent_names: ["b"] });
|
|
4173
|
-
await store.upsertTagRecord("b", {
|
|
4208
|
+
await store.upsertTagRecord("b", {}); // bare row, no parent yet
|
|
4209
|
+
// vault#552: upsertTagRecord now REJECTS a parent_names write that would
|
|
4210
|
+
// close a cycle, so the second leg of the cycle can no longer be built
|
|
4211
|
+
// via the public API — simulate pre-existing cyclic data (written
|
|
4212
|
+
// before the guard shipped, or via any path that bypasses it) with a
|
|
4213
|
+
// direct DB write, same technique as "malformed parent_names JSON is
|
|
4214
|
+
// ignored silently" below. Traversal must stay cycle-safe regardless of
|
|
4215
|
+
// how the cycle got there.
|
|
4216
|
+
(store as any).db.prepare("UPDATE tags SET parent_names = ? WHERE name = ?")
|
|
4217
|
+
.run(JSON.stringify(["a"]), "b");
|
|
4218
|
+
(store as any)._tagHierarchy = null;
|
|
4174
4219
|
await store.createNote("note-a", { tags: ["a"] });
|
|
4175
4220
|
|
|
4176
4221
|
// Both a and b should resolve without hanging; both reach the same set {a, b}.
|
|
@@ -4600,8 +4645,9 @@ describe("update-note if_missing=create (vault#309)", async () => {
|
|
|
4600
4645
|
});
|
|
4601
4646
|
|
|
4602
4647
|
it("create branch applies tag-schema defaults when the new tag declares fields", async () => {
|
|
4648
|
+
// vault#553 Decision B: backfill is explicit-`default`-only.
|
|
4603
4649
|
await store.upsertTagSchema("task", {
|
|
4604
|
-
fields: { priority: { type: "string", enum: ["high", "low"] } },
|
|
4650
|
+
fields: { priority: { type: "string", enum: ["high", "low"], default: "high" } },
|
|
4605
4651
|
});
|
|
4606
4652
|
const tools = generateMcpTools(store);
|
|
4607
4653
|
const update = tools.find((t) => t.name === "update-note")!;
|
|
@@ -4613,7 +4659,7 @@ describe("update-note if_missing=create (vault#309)", async () => {
|
|
|
4613
4659
|
}) as any;
|
|
4614
4660
|
expect(result.created).toBe(true);
|
|
4615
4661
|
// Schema defaults populated metadata.priority on insert.
|
|
4616
|
-
expect(result.metadata?.priority).
|
|
4662
|
+
expect(result.metadata?.priority).toBe("high");
|
|
4617
4663
|
});
|
|
4618
4664
|
|
|
4619
4665
|
it("create branch surfaces validation warnings just like create-note", async () => {
|
|
@@ -4803,14 +4849,21 @@ describe("schema inheritance via parent_names (vault#270)", async () => {
|
|
|
4803
4849
|
});
|
|
4804
4850
|
|
|
4805
4851
|
it("cycle: A→B, B→A — no infinite loop, both fields visible", async () => {
|
|
4852
|
+
await store.upsertTagRecord("B", {
|
|
4853
|
+
fields: { b_field: { type: "string" } },
|
|
4854
|
+
});
|
|
4806
4855
|
await store.upsertTagRecord("A", {
|
|
4807
4856
|
parent_names: ["B"],
|
|
4808
4857
|
fields: { a_field: { type: "string" } },
|
|
4809
4858
|
});
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4859
|
+
// vault#552: upsertTagRecord now REJECTS the write that would close this
|
|
4860
|
+
// cycle (B declaring A as parent, when A already declares B) — simulate
|
|
4861
|
+
// pre-existing cyclic data with a direct DB write, same technique as the
|
|
4862
|
+
// "tolerates a cycle without infinite-looping" test above.
|
|
4863
|
+
(store as any).db.prepare("UPDATE tags SET parent_names = ? WHERE name = ?")
|
|
4864
|
+
.run(JSON.stringify(["A"]), "B");
|
|
4865
|
+
(store as any)._tagHierarchy = null;
|
|
4866
|
+
(store as any)._schemaConfig = null;
|
|
4814
4867
|
|
|
4815
4868
|
const tools = generateMcpTools(store);
|
|
4816
4869
|
const create = tools.find((t) => t.name === "create-note")!;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tag-scope confidentiality for the `vault doctor` integrity scan (vault#552
|
|
3
|
+
* auth fold). Two finding types query notes VAULT-WIDE and previously leaked
|
|
4
|
+
* out-of-scope data to a tag-scoped caller:
|
|
5
|
+
*
|
|
6
|
+
* - `mixed_type_indexed_field`: the note query + mismatch count + exemplars
|
|
7
|
+
* + the "Declared by: <tags>" detail were all unscoped — a caller scoped
|
|
8
|
+
* to `mine` could see an out-of-scope declarer tag name AND an
|
|
9
|
+
* out-of-scope note id.
|
|
10
|
+
* - `dead_tag_metadata_reference`: the example live value (the `e.g.
|
|
11
|
+
* "<value>"` in detail) was computed vault-wide — leaking an out-of-scope
|
|
12
|
+
* tag NAME — and (unscoped) exemplars could include out-of-scope note ids.
|
|
13
|
+
*
|
|
14
|
+
* Each test asserts the out-of-scope tag name / note id appears NOWHERE in
|
|
15
|
+
* the serialized findings when scoped, and pairs with an UNSCOPED positive
|
|
16
|
+
* control proving the fixture genuinely carries that leak-able data (so the
|
|
17
|
+
* scoped assertion isn't vacuous — remove the scope filter and the control's
|
|
18
|
+
* leak reappears in the scoped run).
|
|
19
|
+
*/
|
|
20
|
+
import { describe, it, expect, beforeEach } from "bun:test";
|
|
21
|
+
import { Database } from "bun:sqlite";
|
|
22
|
+
import { SqliteStore } from "./store.js";
|
|
23
|
+
import { runDoctorScan } from "./doctor.js";
|
|
24
|
+
|
|
25
|
+
const OUT_OF_SCOPE_TAG = "theirs-secret";
|
|
26
|
+
|
|
27
|
+
let store: SqliteStore;
|
|
28
|
+
let db: Database;
|
|
29
|
+
|
|
30
|
+
beforeEach(() => {
|
|
31
|
+
db = new Database(":memory:");
|
|
32
|
+
store = new SqliteStore(db);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe("doctor mixed_type_indexed_field × tag scope (vault#552 MUST-FIX 3)", () => {
|
|
36
|
+
beforeEach(async () => {
|
|
37
|
+
// Field `n` is indexed INTEGER, co-declared by an in-scope tag (`mine`)
|
|
38
|
+
// and an out-of-scope one (`theirs-secret`). Both carry a note whose
|
|
39
|
+
// metadata.n is a JSON string (type mismatch), not strict so the write
|
|
40
|
+
// lands the poison value.
|
|
41
|
+
await store.upsertTagRecord("mine", { fields: { n: { type: "integer", indexed: true } } });
|
|
42
|
+
await store.upsertTagRecord(OUT_OF_SCOPE_TAG, { fields: { n: { type: "integer", indexed: true } } });
|
|
43
|
+
await store.createNote("in-scope mismatch", { id: "mine-mm", tags: ["mine"], metadata: { n: "bad-mine" } });
|
|
44
|
+
await store.createNote("out-of-scope mismatch", { id: "theirs-mm", tags: [OUT_OF_SCOPE_TAG], metadata: { n: "bad-theirs" } });
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("scoped caller: the out-of-scope declarer name and out-of-scope note id appear NOWHERE", async () => {
|
|
48
|
+
const report = await store.doctor({ allowedTags: new Set(["mine"]) });
|
|
49
|
+
const finding = report.findings.find((f) => f.type === "mixed_type_indexed_field");
|
|
50
|
+
expect(finding).toBeDefined(); // the in-scope mismatch still fires it
|
|
51
|
+
expect(finding!.subject).toBe("n");
|
|
52
|
+
|
|
53
|
+
const serialized = JSON.stringify(report.findings);
|
|
54
|
+
expect(serialized).not.toContain(OUT_OF_SCOPE_TAG);
|
|
55
|
+
expect(serialized).not.toContain("theirs-mm");
|
|
56
|
+
// The in-scope exemplar IS present (proves the finding is real, not empty).
|
|
57
|
+
expect(serialized).toContain("mine-mm");
|
|
58
|
+
// Count reflects the single in-scope mismatch, not both.
|
|
59
|
+
expect(finding!.detail).toContain("1 note(s)");
|
|
60
|
+
expect(finding!.detail).toContain("Declared by: mine.");
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("unscoped control: the same fixture DOES surface the out-of-scope declarer + note (mutation-check)", async () => {
|
|
64
|
+
const report = await runDoctorScan(db); // no allowedTags
|
|
65
|
+
const serialized = JSON.stringify(report.findings);
|
|
66
|
+
// Absent the scope filter, both the out-of-scope declarer name and the
|
|
67
|
+
// out-of-scope note id leak — this is exactly what the scoped run above
|
|
68
|
+
// must suppress. If the scope filter were removed, the scoped assertions
|
|
69
|
+
// would see these too and fail.
|
|
70
|
+
expect(serialized).toContain(OUT_OF_SCOPE_TAG);
|
|
71
|
+
expect(serialized).toContain("theirs-mm");
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe("doctor dead_tag_metadata_reference × tag scope (vault#552 MUST-FIX 4)", () => {
|
|
76
|
+
beforeEach(async () => {
|
|
77
|
+
await store.upsertTagRecord("mine", {});
|
|
78
|
+
await store.upsertTagRecord(OUT_OF_SCOPE_TAG, {});
|
|
79
|
+
// Insertion ORDER matters: the out-of-scope live-tag value is created
|
|
80
|
+
// FIRST so the vault-wide `liveValues[0]` example would be
|
|
81
|
+
// "theirs-secret" absent the scope fix.
|
|
82
|
+
await store.createNote("oos live", { id: "theirs-live", tags: [OUT_OF_SCOPE_TAG], metadata: { epic: OUT_OF_SCOPE_TAG } });
|
|
83
|
+
await store.createNote("in-scope live", { id: "mine-live", tags: ["mine"], metadata: { epic: "mine" } });
|
|
84
|
+
await store.createNote("in-scope dead", { id: "mine-dead", tags: ["mine"], metadata: { epic: "ghost-epic" } });
|
|
85
|
+
await store.createNote("oos dead", { id: "theirs-dead", tags: [OUT_OF_SCOPE_TAG], metadata: { epic: "ghost-epic" } });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("scoped caller: the out-of-scope example tag name and out-of-scope note id appear NOWHERE", async () => {
|
|
89
|
+
const report = await store.doctor({ allowedTags: new Set(["mine"]) });
|
|
90
|
+
const finding = report.findings.find((f) => f.type === "dead_tag_metadata_reference");
|
|
91
|
+
expect(finding).toBeDefined();
|
|
92
|
+
expect(finding!.subject).toBe("metadata.epic");
|
|
93
|
+
expect(finding!.heuristic).toBe(true);
|
|
94
|
+
|
|
95
|
+
const serialized = JSON.stringify(report.findings);
|
|
96
|
+
// No out-of-scope tag NAME (as the `e.g. "<value>"` live example) and no
|
|
97
|
+
// out-of-scope note id (as an exemplar of the dead value).
|
|
98
|
+
expect(serialized).not.toContain(OUT_OF_SCOPE_TAG);
|
|
99
|
+
expect(serialized).not.toContain("theirs-dead");
|
|
100
|
+
expect(serialized).not.toContain("theirs-live");
|
|
101
|
+
// The in-scope live example + in-scope exemplar + dead value ARE present.
|
|
102
|
+
expect(finding!.detail).toContain('e.g. "mine"');
|
|
103
|
+
expect(serialized).toContain("mine-dead");
|
|
104
|
+
expect(serialized).toContain("ghost-epic");
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("unscoped control: the same fixture leaks the out-of-scope example name + note id (mutation-check)", async () => {
|
|
108
|
+
const report = await runDoctorScan(db); // no allowedTags
|
|
109
|
+
const finding = report.findings.find((f) => f.type === "dead_tag_metadata_reference")!;
|
|
110
|
+
const serialized = JSON.stringify(report.findings);
|
|
111
|
+
// Vault-wide, the example live value is the first-inserted live tag
|
|
112
|
+
// ("theirs-secret"), and the out-of-scope note id is an exemplar — both
|
|
113
|
+
// the leaks the scoped run suppresses.
|
|
114
|
+
expect(finding.detail).toContain(`e.g. "${OUT_OF_SCOPE_TAG}"`);
|
|
115
|
+
expect(serialized).toContain("theirs-dead");
|
|
116
|
+
});
|
|
117
|
+
});
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vault doctor` — read-only taxonomy/metadata integrity scan (vault#552).
|
|
3
|
+
*
|
|
4
|
+
* NOT to be confused with `parachute-vault doctor` (the CLI's install/daemon
|
|
5
|
+
* diagnostic in `src/cli.ts` — port reachability, config sanity, launchd/
|
|
6
|
+
* systemd registration). This module answers a different question: "did my
|
|
7
|
+
* tag reorg (rename/merge/delete) leak?" It never auto-fixes — every finding
|
|
8
|
+
* carries a suggested `remedy` the caller applies deliberately (usually via
|
|
9
|
+
* `rename-tag`/`merge-tags`/`prune-schema`/another `update-tag` call).
|
|
10
|
+
*
|
|
11
|
+
* Scans:
|
|
12
|
+
* - `dangling_parent_name` — a `parent_names` entry naming a tag with no
|
|
13
|
+
* identity row.
|
|
14
|
+
* - `parent_names_cycle` — a tag that reaches itself through its
|
|
15
|
+
* declared ancestor chain (traversal is cycle-safe; this just SURFACES
|
|
16
|
+
* data that predates the vault#552 write-time cycle guard, or was
|
|
17
|
+
* written by a path that bypasses it).
|
|
18
|
+
* - `mixed_type_indexed_field` — a note whose `metadata.<field>` JSON type
|
|
19
|
+
* disagrees with the field's declared indexed sqlite type. This is the
|
|
20
|
+
* WS4 typed-index "poison precursor": inconsistent column affinity
|
|
21
|
+
* coercion silently breaks ordering/comparison across rows. The finding
|
|
22
|
+
* shape here is intentionally reusable as WS4's migration pre-flight.
|
|
23
|
+
* - `orphaned_indexed_field_declarer` — an `indexed_fields` row naming a
|
|
24
|
+
* declarer tag with no `tags` row (overlaps `prune-schema`, which is the
|
|
25
|
+
* suggested remedy).
|
|
26
|
+
* - `dead_tag_metadata_reference` — HEURISTIC, always labeled `heuristic:
|
|
27
|
+
* true`. A metadata key is used elsewhere in the vault with values that
|
|
28
|
+
* ARE live tag names (establishing "this key holds tag-shaped values");
|
|
29
|
+
* a value under the same key that does NOT match any live tag is
|
|
30
|
+
* flagged as a possible stale reference to a renamed/merged/deleted tag
|
|
31
|
+
* — the PM's `metadata.epic` drift class from the #552 write-up. Vault
|
|
32
|
+
* has no history of past tag names, so this can never be certain —
|
|
33
|
+
* hence heuristic.
|
|
34
|
+
*
|
|
35
|
+
* Tag-scope (vault#552): pass `allowedTags` (the caller's EXPANDED
|
|
36
|
+
* allowlist, e.g. from `expandTokenTagScope`) to restrict every finding to
|
|
37
|
+
* in-scope tags/fields/notes. `null` (or omitted) — the default — scans the
|
|
38
|
+
* whole vault. The scan is re-run with the allowlist rather than filtering
|
|
39
|
+
* an unscoped result afterward, so the `summary` counts never leak
|
|
40
|
+
* out-of-scope activity.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
import { Database } from "bun:sqlite";
|
|
44
|
+
import { loadTagHierarchy, findHierarchyCycles } from "./tag-hierarchy.js";
|
|
45
|
+
import { listIndexedFields } from "./indexed-fields.js";
|
|
46
|
+
import { pruneOrphanedIndexedFields } from "./indexed-fields.js";
|
|
47
|
+
|
|
48
|
+
export type DoctorFindingType =
|
|
49
|
+
| "dangling_parent_name"
|
|
50
|
+
| "parent_names_cycle"
|
|
51
|
+
| "mixed_type_indexed_field"
|
|
52
|
+
| "orphaned_indexed_field_declarer"
|
|
53
|
+
| "dead_tag_metadata_reference";
|
|
54
|
+
|
|
55
|
+
export type DoctorSeverity = "error" | "warning" | "info";
|
|
56
|
+
|
|
57
|
+
export interface DoctorFinding {
|
|
58
|
+
type: DoctorFindingType;
|
|
59
|
+
severity: DoctorSeverity;
|
|
60
|
+
/** The tag / field / metadata-key this finding is about. */
|
|
61
|
+
subject: string;
|
|
62
|
+
detail: string;
|
|
63
|
+
remedy: string;
|
|
64
|
+
/**
|
|
65
|
+
* True ONLY for `dead_tag_metadata_reference` — flags the finding as a
|
|
66
|
+
* best-effort inference (no tag-rename history exists to confirm it),
|
|
67
|
+
* never present (not even `false`) on the other, structurally-certain
|
|
68
|
+
* finding types.
|
|
69
|
+
*/
|
|
70
|
+
heuristic?: true;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface DoctorReport {
|
|
74
|
+
findings: DoctorFinding[];
|
|
75
|
+
summary: string;
|
|
76
|
+
scanned_at: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface DoctorScanOpts {
|
|
80
|
+
/**
|
|
81
|
+
* The caller's expanded tag allowlist (root ∪ descendants). `null`
|
|
82
|
+
* (default) scans the whole vault. See the module doc comment for how
|
|
83
|
+
* each finding type is scoped.
|
|
84
|
+
*/
|
|
85
|
+
allowedTags?: Set<string> | null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Cap on exemplar note IDs embedded in a finding's `detail` (vault#552 — keep report bodies bounded). */
|
|
89
|
+
const MAX_EXEMPLARS = 5;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Build a per-note in-scope predicate for a tag-scoped doctor run (vault#552
|
|
93
|
+
* auth fold). A note is in-scope iff at least one of its tags is in
|
|
94
|
+
* `allowedTags`; an unscoped run (`allowedTags === null`) treats every note
|
|
95
|
+
* as in-scope. Caches each note's tag lookup so repeated checks across scans
|
|
96
|
+
* cost one query per note. Shared by the mixed-type-indexed-field and
|
|
97
|
+
* dead-tag-metadata scans, both of which query notes vault-wide and must NOT
|
|
98
|
+
* surface an out-of-scope note id (or count, or exemplar) to a scoped caller.
|
|
99
|
+
*/
|
|
100
|
+
function makeNoteInScope(db: Database, allowedTags: Set<string> | null): (id: string) => boolean {
|
|
101
|
+
if (!allowedTags) return () => true;
|
|
102
|
+
const cache = new Map<string, boolean>();
|
|
103
|
+
return (id: string): boolean => {
|
|
104
|
+
const cached = cache.get(id);
|
|
105
|
+
if (cached !== undefined) return cached;
|
|
106
|
+
const tags = (db.prepare("SELECT tag_name FROM note_tags WHERE note_id = ?").all(id) as { tag_name: string }[]).map((r) => r.tag_name);
|
|
107
|
+
const inScope = tags.some((t) => allowedTags.has(t));
|
|
108
|
+
cache.set(id, inScope);
|
|
109
|
+
return inScope;
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function runDoctorScan(db: Database, opts?: DoctorScanOpts): DoctorReport {
|
|
114
|
+
const allowedTags = opts?.allowedTags ?? null;
|
|
115
|
+
const findings: DoctorFinding[] = [
|
|
116
|
+
...scanDanglingParentNames(db, allowedTags),
|
|
117
|
+
...scanParentNamesCycles(db, allowedTags),
|
|
118
|
+
...scanMixedTypeIndexedFields(db, allowedTags),
|
|
119
|
+
...scanOrphanedIndexedFieldDeclarers(db, allowedTags),
|
|
120
|
+
...scanDeadTagMetadataReferences(db, allowedTags),
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
const errors = findings.filter((f) => f.severity === "error").length;
|
|
124
|
+
const warnings = findings.filter((f) => f.severity === "warning").length;
|
|
125
|
+
const infos = findings.filter((f) => f.severity === "info").length;
|
|
126
|
+
const summary =
|
|
127
|
+
findings.length === 0
|
|
128
|
+
? "clean — no integrity findings"
|
|
129
|
+
: `${findings.length} finding(s): ${errors} error(s), ${warnings} warning(s), ${infos} info(s)`;
|
|
130
|
+
|
|
131
|
+
return { findings, summary, scanned_at: new Date().toISOString() };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// A. dangling parent_names
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
function scanDanglingParentNames(db: Database, allowedTags: Set<string> | null): DoctorFinding[] {
|
|
139
|
+
const rows = db
|
|
140
|
+
.prepare("SELECT name, parent_names FROM tags WHERE parent_names IS NOT NULL")
|
|
141
|
+
.all() as { name: string; parent_names: string }[];
|
|
142
|
+
const allTags = new Set(
|
|
143
|
+
(db.prepare("SELECT name FROM tags").all() as { name: string }[]).map((r) => r.name),
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
const findings: DoctorFinding[] = [];
|
|
147
|
+
for (const row of rows) {
|
|
148
|
+
if (allowedTags && !allowedTags.has(row.name)) continue;
|
|
149
|
+
let parsed: unknown;
|
|
150
|
+
try { parsed = JSON.parse(row.parent_names); } catch { continue; }
|
|
151
|
+
if (!Array.isArray(parsed)) continue;
|
|
152
|
+
for (const parent of parsed) {
|
|
153
|
+
if (typeof parent !== "string" || parent.length === 0) continue;
|
|
154
|
+
if (allTags.has(parent)) continue;
|
|
155
|
+
findings.push({
|
|
156
|
+
type: "dangling_parent_name",
|
|
157
|
+
severity: "warning",
|
|
158
|
+
subject: row.name,
|
|
159
|
+
detail: `tag "${row.name}" declares parent "${parent}", which has no identity row — a typo, a tag that was never created, or a rename/delete that didn't update this reference.`,
|
|
160
|
+
remedy: `remove "${parent}" from "${row.name}"'s parent_names (update-tag), or create the missing "${parent}" tag if it's meant to exist`,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return findings;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
// B. parent_names cycles (pre-existing data — the write-time guard blocks
|
|
169
|
+
// NEW ones; this surfaces anything already on disk)
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
function scanParentNamesCycles(db: Database, allowedTags: Set<string> | null): DoctorFinding[] {
|
|
173
|
+
const hierarchy = loadTagHierarchy(db);
|
|
174
|
+
const cycles = findHierarchyCycles(hierarchy);
|
|
175
|
+
const findings: DoctorFinding[] = [];
|
|
176
|
+
for (const tag of cycles) {
|
|
177
|
+
if (allowedTags && !allowedTags.has(tag)) continue;
|
|
178
|
+
findings.push({
|
|
179
|
+
type: "parent_names_cycle",
|
|
180
|
+
severity: "error",
|
|
181
|
+
subject: tag,
|
|
182
|
+
detail: `tag "${tag}" reaches itself through its declared parent_names ancestor chain. Query expansion tolerates this (a visited-set stops it looping), but it's confusing hierarchy state.`,
|
|
183
|
+
remedy: `clear or correct parent_names on one of the tags in "${tag}"'s ancestor chain (update-tag) to break the cycle`,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
return findings;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// ---------------------------------------------------------------------------
|
|
190
|
+
// C. mixed-type values in an indexed column (WS4 poison precursor)
|
|
191
|
+
// ---------------------------------------------------------------------------
|
|
192
|
+
|
|
193
|
+
/** `json_type()` results consistent with each declared indexed sqlite storage class. */
|
|
194
|
+
const EXPECTED_JSON_TYPES: Record<string, Set<string>> = {
|
|
195
|
+
INTEGER: new Set(["integer", "real", "true", "false"]),
|
|
196
|
+
TEXT: new Set(["text"]),
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
/** One vault-wide-scan result row from {@link findMixedTypeIndexedFieldNotes}. */
|
|
200
|
+
export interface MixedTypeIndexedFieldRow {
|
|
201
|
+
id: string;
|
|
202
|
+
/** SQLite `json_type()` of the CURRENT `metadata.<field>` value — "text", "integer", "real", "true", "false", "array", "object" (never "null": absent/null keys are excluded upstream). */
|
|
203
|
+
jt: string;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Vault-wide detector: every note whose `metadata.<field>` JSON type
|
|
208
|
+
* disagrees with `sqliteType` (the field's declared indexed storage class —
|
|
209
|
+
* "INTEGER" or "TEXT"). Returns `[]` for an unknown `sqliteType` or when
|
|
210
|
+
* nothing mismatches. UNSCOPED — no tag-scope filtering (that's the caller's
|
|
211
|
+
* job; see `scanMixedTypeIndexedFields` below for the tag-scoped doctor
|
|
212
|
+
* finding, and `migrateToV24` in `core/src/schema.ts` for the unscoped
|
|
213
|
+
* migration consumer). `json_type(metadata, ?)` returns NULL when the key is
|
|
214
|
+
* absent — those notes simply don't declare the field and aren't a mismatch.
|
|
215
|
+
*
|
|
216
|
+
* Extracted (vault#553 Decision D) so the SAME detection query backs both
|
|
217
|
+
* the `mixed_type_indexed_field` doctor finding AND the `migrateToV24`
|
|
218
|
+
* startup migration's poison scan — one source of truth for "what counts as
|
|
219
|
+
* mismatched," so a fix to one can't silently diverge from the other.
|
|
220
|
+
*/
|
|
221
|
+
export function findMixedTypeIndexedFieldNotes(
|
|
222
|
+
db: Database,
|
|
223
|
+
field: string,
|
|
224
|
+
sqliteType: string,
|
|
225
|
+
): MixedTypeIndexedFieldRow[] {
|
|
226
|
+
const expected = EXPECTED_JSON_TYPES[sqliteType];
|
|
227
|
+
if (!expected) return []; // unknown sqlite_type — nothing to compare against
|
|
228
|
+
const path = `$."${field}"`;
|
|
229
|
+
const rows = db
|
|
230
|
+
.prepare(
|
|
231
|
+
`SELECT id, json_type(metadata, ?) as jt FROM notes WHERE metadata IS NOT NULL AND metadata != '' AND json_valid(metadata)`,
|
|
232
|
+
)
|
|
233
|
+
.all(path) as { id: string; jt: string | null }[];
|
|
234
|
+
return rows.filter((r): r is MixedTypeIndexedFieldRow => r.jt !== null && !expected.has(r.jt));
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function scanMixedTypeIndexedFields(db: Database, allowedTags: Set<string> | null): DoctorFinding[] {
|
|
238
|
+
const findings: DoctorFinding[] = [];
|
|
239
|
+
const noteInScope = makeNoteInScope(db, allowedTags);
|
|
240
|
+
for (const f of listIndexedFields(db)) {
|
|
241
|
+
if (allowedTags && !f.declarerTags.some((t) => allowedTags.has(t))) continue;
|
|
242
|
+
|
|
243
|
+
const rows = findMixedTypeIndexedFieldNotes(db, f.field, f.sqliteType);
|
|
244
|
+
if (rows.length === 0) continue;
|
|
245
|
+
|
|
246
|
+
// Tag-scope (vault#552 auth fold): the note query above is vault-wide —
|
|
247
|
+
// filter mismatches to IN-SCOPE notes so a scoped caller never sees an
|
|
248
|
+
// out-of-scope note id (nor a count/exemplar reflecting one). If every
|
|
249
|
+
// mismatch is on an out-of-scope note, the finding is dropped entirely.
|
|
250
|
+
const mismatches = rows.filter((r) => noteInScope(r.id));
|
|
251
|
+
if (mismatches.length === 0) continue;
|
|
252
|
+
|
|
253
|
+
// Also generalize the "Declared by" list to in-scope declarers only —
|
|
254
|
+
// the field can be co-declared by an out-of-scope tag whose name must
|
|
255
|
+
// not leak (the caller sees this field only via its in-scope declarer).
|
|
256
|
+
const visibleDeclarers = allowedTags
|
|
257
|
+
? f.declarerTags.filter((t) => allowedTags.has(t))
|
|
258
|
+
: f.declarerTags;
|
|
259
|
+
const exemplars = mismatches.slice(0, MAX_EXEMPLARS).map((r) => `${r.id} (${r.jt})`);
|
|
260
|
+
findings.push({
|
|
261
|
+
type: "mixed_type_indexed_field",
|
|
262
|
+
severity: "error",
|
|
263
|
+
subject: f.field,
|
|
264
|
+
detail: `field "${f.field}" is indexed as ${f.sqliteType} but ${mismatches.length} note(s) carry a metadata."${f.field}" value of a disagreeing JSON type — the generated column's affinity coercion makes ordering/filtering across these rows inconsistent. Declared by: ${visibleDeclarers.join(", ") || "(no live declarer)"}. Example note(s): ${exemplars.join(", ")}${mismatches.length > exemplars.length ? ", …" : ""}`,
|
|
265
|
+
remedy: `backfill each listed note's metadata."${f.field}" to a value matching the declared type, or relax the field's declared type (update-tag) if the mixed values are intentional`,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
return findings;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ---------------------------------------------------------------------------
|
|
272
|
+
// D. orphaned indexed_fields declarers (overlaps prune-schema)
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
function scanOrphanedIndexedFieldDeclarers(db: Database, allowedTags: Set<string> | null): DoctorFinding[] {
|
|
276
|
+
const plan = pruneOrphanedIndexedFields(db, { dryRun: true });
|
|
277
|
+
const findings: DoctorFinding[] = [];
|
|
278
|
+
for (const p of plan) {
|
|
279
|
+
if (allowedTags && !p.deadDeclarers.some((t) => allowedTags.has(t))) {
|
|
280
|
+
// Dead declarer names never survive `allowedTags` (it's derived from
|
|
281
|
+
// LIVE tags only) — this branch exists for completeness; in practice
|
|
282
|
+
// a scoped caller only sees this finding when the report is unscoped.
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
findings.push({
|
|
286
|
+
type: "orphaned_indexed_field_declarer",
|
|
287
|
+
severity: p.dropped ? "warning" : "info",
|
|
288
|
+
subject: p.field,
|
|
289
|
+
detail: p.dropped
|
|
290
|
+
? `field "${p.field}" has NO surviving live declarer (dead: ${p.deadDeclarers.join(", ")}) — the generated column + index are stale and will be dropped on next prune (metadata values are never lost, only the index).`
|
|
291
|
+
: `field "${p.field}" has dead declarer tag(s) [${p.deadDeclarers.join(", ")}] still listed alongside live ones — cosmetic, no functional impact.`,
|
|
292
|
+
remedy: `run the prune-schema tool (apply: true) to drop the dead declarer(s)${p.dropped ? " and the now-unreferenced column/index" : ""}`,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
return findings;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// ---------------------------------------------------------------------------
|
|
299
|
+
// E. metadata values equal to a tag name that no longer exists (heuristic)
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
|
|
302
|
+
function scanDeadTagMetadataReferences(db: Database, allowedTags: Set<string> | null): DoctorFinding[] {
|
|
303
|
+
const liveTags = new Set(
|
|
304
|
+
(db.prepare("SELECT name FROM tags").all() as { name: string }[]).map((r) => r.name),
|
|
305
|
+
);
|
|
306
|
+
|
|
307
|
+
const rows = db
|
|
308
|
+
.prepare("SELECT id, metadata FROM notes WHERE metadata IS NOT NULL AND metadata != ''")
|
|
309
|
+
.all() as { id: string; metadata: string }[];
|
|
310
|
+
|
|
311
|
+
// key -> value -> note ids carrying that (key, value) pair.
|
|
312
|
+
const byKey = new Map<string, Map<string, string[]>>();
|
|
313
|
+
for (const row of rows) {
|
|
314
|
+
let parsed: unknown;
|
|
315
|
+
try { parsed = JSON.parse(row.metadata); } catch { continue; }
|
|
316
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) continue;
|
|
317
|
+
for (const [key, value] of Object.entries(parsed as Record<string, unknown>)) {
|
|
318
|
+
if (typeof value !== "string" || value.length === 0) continue;
|
|
319
|
+
let valueMap = byKey.get(key);
|
|
320
|
+
if (!valueMap) { valueMap = new Map(); byKey.set(key, valueMap); }
|
|
321
|
+
let ids = valueMap.get(value);
|
|
322
|
+
if (!ids) { ids = []; valueMap.set(value, ids); }
|
|
323
|
+
ids.push(row.id);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// Tag-scope (vault#552 auth fold): a note is in-scope iff at least one of
|
|
328
|
+
// its tags is in `allowedTags` (unscoped → every note in-scope).
|
|
329
|
+
const noteInScope = makeNoteInScope(db, allowedTags);
|
|
330
|
+
|
|
331
|
+
const findings: DoctorFinding[] = [];
|
|
332
|
+
for (const [key, valueMap] of byKey) {
|
|
333
|
+
// Signal: does this metadata key hold tag-shaped values on IN-SCOPE
|
|
334
|
+
// notes? Computed over in-scope notes ONLY (not vault-wide) so the
|
|
335
|
+
// example live value quoted in `detail` can never leak an out-of-scope
|
|
336
|
+
// tag name — a scoped caller must not learn a tag exists just because a
|
|
337
|
+
// note it can't see uses it as a metadata value. Only flag mismatches
|
|
338
|
+
// under a key that demonstrably holds tag-shaped values in-scope.
|
|
339
|
+
const liveValuesInScope = [...valueMap.entries()]
|
|
340
|
+
.filter(([v, ids]) => liveTags.has(v) && ids.some(noteInScope))
|
|
341
|
+
.map(([v]) => v);
|
|
342
|
+
if (liveValuesInScope.length === 0) continue;
|
|
343
|
+
|
|
344
|
+
for (const [value, noteIds] of valueMap) {
|
|
345
|
+
if (liveTags.has(value)) continue;
|
|
346
|
+
const inScopeIds = noteIds.filter(noteInScope);
|
|
347
|
+
if (inScopeIds.length === 0) continue;
|
|
348
|
+
const exemplars = inScopeIds.slice(0, MAX_EXEMPLARS);
|
|
349
|
+
findings.push({
|
|
350
|
+
type: "dead_tag_metadata_reference",
|
|
351
|
+
severity: "info",
|
|
352
|
+
subject: `metadata.${key}`,
|
|
353
|
+
detail: `${inScopeIds.length} note(s) carry metadata.${key} = "${value}", which matches no current tag — other notes' metadata.${key} values ARE live tags (e.g. "${liveValuesInScope[0]}"), suggesting "${value}" may be a stale reference to a tag that was renamed, merged, or deleted. Example note(s): ${exemplars.join(", ")}${inScopeIds.length > exemplars.length ? ", …" : ""}`,
|
|
354
|
+
remedy: `if "${value}" was renamed/merged/deleted, update these notes' metadata.${key} to the current tag name; if "${value}" was never a tag, ignore this finding`,
|
|
355
|
+
heuristic: true,
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return findings;
|
|
360
|
+
}
|