@openparachute/vault 0.4.9-rc.9 → 0.5.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -54
- package/core/src/core.test.ts +4 -1
- package/core/src/indexed-fields.test.ts +151 -0
- package/core/src/indexed-fields.ts +98 -0
- package/core/src/mcp.ts +66 -43
- package/core/src/notes.ts +26 -2
- package/core/src/portable-md.test.ts +52 -0
- package/core/src/portable-md.ts +48 -0
- package/core/src/schema.ts +87 -14
- package/core/src/store.ts +117 -0
- package/core/src/types.ts +28 -0
- package/package.json +2 -2
- package/src/auth-hub-jwt.test.ts +191 -11
- package/src/auth-status.ts +12 -5
- package/src/auth.test.ts +135 -219
- package/src/auth.ts +158 -107
- package/src/cli.ts +306 -224
- package/src/config.ts +12 -4
- package/src/export-watch.test.ts +23 -0
- package/src/export-watch.ts +14 -0
- package/src/git-preflight.test.ts +70 -0
- package/src/git-preflight.ts +68 -0
- package/src/hub-jwt.test.ts +27 -2
- package/src/hub-jwt.ts +10 -0
- package/src/init-summary.test.ts +4 -4
- package/src/init-summary.ts +36 -10
- package/src/mcp-config.test.ts +4 -2
- package/src/mcp-http.ts +24 -3
- package/src/mcp-install-interactive.test.ts +33 -71
- package/src/mcp-install-interactive.ts +23 -76
- package/src/mcp-install.test.ts +156 -55
- package/src/mcp-install.ts +109 -3
- package/src/mcp-tools.ts +249 -74
- package/src/mirror-config.test.ts +107 -0
- package/src/mirror-config.ts +275 -9
- package/src/mirror-credentials.test.ts +168 -17
- package/src/mirror-credentials.ts +155 -32
- package/src/mirror-deps.ts +25 -16
- package/src/mirror-import.test.ts +122 -16
- package/src/mirror-import.ts +50 -16
- package/src/mirror-manager.test.ts +51 -0
- package/src/mirror-manager.ts +116 -22
- package/src/mirror-per-vault.test.ts +519 -0
- package/src/mirror-registry.ts +91 -14
- package/src/mirror-routes.test.ts +81 -21
- package/src/mirror-routes.ts +90 -16
- package/src/routes.ts +39 -2
- package/src/routing.test.ts +203 -118
- package/src/routing.ts +46 -59
- package/src/scopes.test.ts +0 -86
- package/src/scopes.ts +9 -97
- package/src/server.ts +102 -34
- package/src/storage.test.ts +132 -7
- package/src/token-store.test.ts +88 -169
- package/src/token-store.ts +123 -249
- package/src/vault-create.test.ts +12 -4
- package/src/vault.test.ts +408 -103
- package/web/ui/dist/assets/index-DDRo6F4u.js +60 -0
- package/web/ui/dist/index.html +1 -1
- package/src/tokens-routes.test.ts +0 -727
- package/src/tokens-routes.ts +0 -392
- package/web/ui/dist/assets/index-Degr8snN.js +0 -60
package/core/src/mcp.ts
CHANGED
|
@@ -23,8 +23,8 @@ export interface McpToolDef {
|
|
|
23
23
|
/**
|
|
24
24
|
* Minimum scope verb the caller must hold for THIS vault to see + invoke
|
|
25
25
|
* the tool. `read` for pure queries, `write` for mutations, `admin` for
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* operator-only surfaces (`prune-schema` in core; `manage-token` in the
|
|
27
|
+
* server layer). The MCP HTTP layer filters
|
|
28
28
|
* `tools/list` by this field and verb-gates `tools/call` against it; the
|
|
29
29
|
* filter is the primary defense, the inner gate is defense-in-depth.
|
|
30
30
|
*
|
|
@@ -100,9 +100,9 @@ function removeWikilinkBrackets(content: string, targetPath: string): string {
|
|
|
100
100
|
// ---------------------------------------------------------------------------
|
|
101
101
|
|
|
102
102
|
/**
|
|
103
|
-
* Generate the consolidated MCP tools for a vault.
|
|
103
|
+
* Generate the consolidated MCP tools for a vault. Surface (10):
|
|
104
104
|
* query-notes, create-note, update-note, delete-note, list-tags, update-tag,
|
|
105
|
-
* delete-tag, find-path, vault-info.
|
|
105
|
+
* delete-tag, find-path, vault-info, prune-schema (admin).
|
|
106
106
|
*/
|
|
107
107
|
export function generateMcpTools(store: Store): McpToolDef[] {
|
|
108
108
|
const db: Database = (store as any).db;
|
|
@@ -1074,12 +1074,23 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1074
1074
|
const tag = params.tag as string;
|
|
1075
1075
|
const existing = tagSchemaOps.getTagRecord(db, tag);
|
|
1076
1076
|
|
|
1077
|
-
// ---- fields:
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1077
|
+
// ---- fields: three-way semantics, distinguishing `null` from
|
|
1078
|
+
// `undefined` (do NOT collapse with `?? {}` — that silently turns an
|
|
1079
|
+
// explicit clear-all into a no-op, the gitcoin orphaned-fields bug).
|
|
1080
|
+
// - undefined → no change. Preserve every existing field; declare
|
|
1081
|
+
// nothing new. mergedFields === existing.fields.
|
|
1082
|
+
// - null → clear ALL of this tag's field schemas.
|
|
1083
|
+
// mergedFields = {} so the diff below releases every
|
|
1084
|
+
// indexed field this tag exclusively declares.
|
|
1085
|
+
// - object → shallow-merge into existing (preserves prior keys).
|
|
1086
|
+
const incomingFields =
|
|
1087
|
+
params.fields === null || params.fields === undefined
|
|
1088
|
+
? {}
|
|
1089
|
+
: (params.fields as Record<string, TagFieldSchema>);
|
|
1090
|
+
const mergedFields: Record<string, TagFieldSchema> =
|
|
1091
|
+
params.fields === null
|
|
1092
|
+
? {}
|
|
1093
|
+
: { ...(existing?.fields ?? {}), ...incomingFields };
|
|
1083
1094
|
|
|
1084
1095
|
// Validate cross-tag consistency on fields being (re)declared in this
|
|
1085
1096
|
// call. `type` and `indexed` are global — all declarers must agree.
|
|
@@ -1146,6 +1157,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1146
1157
|
: (params.fields !== undefined ? null : undefined);
|
|
1147
1158
|
const descriptionPatch =
|
|
1148
1159
|
params.description === undefined ? undefined : (params.description as string);
|
|
1160
|
+
// The indexed-field lifecycle (declareField for added indexed fields,
|
|
1161
|
+
// releaseField for removed ones, with the co-declaration guard) is
|
|
1162
|
+
// reconciled inside store.upsertTagRecord — the single chokepoint all
|
|
1163
|
+
// callers (MCP, REST PUT /tags/:name, import) share — so it can't be
|
|
1164
|
+
// bypassed. The cross-tag validation above stays here to surface a
|
|
1165
|
+
// clean error before persisting. See the gitcoin orphaned-fields bug.
|
|
1149
1166
|
const result = await store.upsertTagRecord(tag, {
|
|
1150
1167
|
...(descriptionPatch !== undefined ? { description: descriptionPatch } : {}),
|
|
1151
1168
|
...(fieldsPatch !== undefined ? { fields: fieldsPatch } : {}),
|
|
@@ -1153,28 +1170,6 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1153
1170
|
...(parentNamesPatch !== undefined ? { parent_names: parentNamesPatch } : {}),
|
|
1154
1171
|
});
|
|
1155
1172
|
|
|
1156
|
-
// ---- Reconcile indexed-field lifecycle for this tag.
|
|
1157
|
-
const priorIndexed = new Set(
|
|
1158
|
-
Object.entries(existing?.fields ?? {})
|
|
1159
|
-
.filter(([, v]) => v.indexed === true)
|
|
1160
|
-
.map(([k]) => k),
|
|
1161
|
-
);
|
|
1162
|
-
const nextIndexed = new Set(
|
|
1163
|
-
Object.entries(mergedFields)
|
|
1164
|
-
.filter(([, v]) => v.indexed === true)
|
|
1165
|
-
.map(([k]) => k),
|
|
1166
|
-
);
|
|
1167
|
-
for (const fieldName of nextIndexed) {
|
|
1168
|
-
const spec = mergedFields[fieldName]!;
|
|
1169
|
-
const mapped = indexedFieldOps.mapFieldType(spec.type)!;
|
|
1170
|
-
indexedFieldOps.declareField(db, fieldName, mapped, tag);
|
|
1171
|
-
}
|
|
1172
|
-
for (const fieldName of priorIndexed) {
|
|
1173
|
-
if (!nextIndexed.has(fieldName)) {
|
|
1174
|
-
indexedFieldOps.releaseField(db, fieldName, tag);
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
1173
|
return result;
|
|
1179
1174
|
},
|
|
1180
1175
|
},
|
|
@@ -1198,19 +1193,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1198
1193
|
},
|
|
1199
1194
|
execute: async (params) => {
|
|
1200
1195
|
const tag = params.tag as string;
|
|
1201
|
-
// Release any indexed fields this tag declared before the row
|
|
1202
|
-
// drops. releaseField drops the generated column + index when the
|
|
1203
|
-
// declarer set empties.
|
|
1204
|
-
const schema = tagSchemaOps.getTagSchema(db, tag);
|
|
1205
|
-
if (schema?.fields) {
|
|
1206
|
-
for (const [fieldName, spec] of Object.entries(schema.fields)) {
|
|
1207
|
-
if (spec.indexed === true) {
|
|
1208
|
-
indexedFieldOps.releaseField(db, fieldName, tag);
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
1196
|
// Drop the row outright — description/fields/relationships/parents
|
|
1213
1197
|
// travel with it. (No more sidecar table to clear separately.)
|
|
1198
|
+
// Indexed-field release is handled inside store.deleteTag →
|
|
1199
|
+
// noteOps.deleteTag so every entry point (MCP, REST, import sweep)
|
|
1200
|
+
// releases consistently with the co-declaration guard. See the
|
|
1201
|
+
// gitcoin orphaned-fields bug report.
|
|
1214
1202
|
return await store.deleteTag(tag);
|
|
1215
1203
|
},
|
|
1216
1204
|
},
|
|
@@ -1266,6 +1254,41 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1266
1254
|
},
|
|
1267
1255
|
},
|
|
1268
1256
|
|
|
1257
|
+
// =====================================================================
|
|
1258
|
+
// 10. prune-schema — drop orphaned indexed-field columns
|
|
1259
|
+
// =====================================================================
|
|
1260
|
+
{
|
|
1261
|
+
name: "prune-schema",
|
|
1262
|
+
// `admin` — a destructive schema-maintenance op, same tier as
|
|
1263
|
+
// manage-token. Operator-only; hidden from read/write sessions.
|
|
1264
|
+
requiredVerb: "admin",
|
|
1265
|
+
description:
|
|
1266
|
+
"Drop orphaned indexed-field columns + indexes whose declaring tags no longer exist (the result of a deleted tag never releasing its fields). Dry-run by default — returns the drop plan without mutating. Pass `apply: true` to execute. A field co-declared by a still-live tag is never dropped; only the dead declarers are trimmed from its set. Generated columns are derived from notes.metadata JSON, so a drop loses only the index, never source data — declare the field again to rebuild it.",
|
|
1267
|
+
inputSchema: {
|
|
1268
|
+
type: "object",
|
|
1269
|
+
properties: {
|
|
1270
|
+
apply: {
|
|
1271
|
+
type: "boolean",
|
|
1272
|
+
description: "Execute the prune. Default false (dry-run — report what would be dropped without changing anything).",
|
|
1273
|
+
},
|
|
1274
|
+
},
|
|
1275
|
+
},
|
|
1276
|
+
execute: async (params) => {
|
|
1277
|
+
const apply = params.apply === true;
|
|
1278
|
+
const plan = await store.pruneIndexedFields({ dryRun: !apply });
|
|
1279
|
+
const dropped = plan.filter((p) => p.dropped);
|
|
1280
|
+
const trimmed = plan.filter((p) => !p.dropped);
|
|
1281
|
+
return {
|
|
1282
|
+
dry_run: !apply,
|
|
1283
|
+
fields_dropped: dropped.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
|
|
1284
|
+
fields_trimmed: trimmed.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
|
|
1285
|
+
summary: apply
|
|
1286
|
+
? `pruned ${dropped.length} orphaned field(s); trimmed dead declarers on ${trimmed.length} co-declared field(s)`
|
|
1287
|
+
: `would prune ${dropped.length} orphaned field(s); would trim dead declarers on ${trimmed.length} co-declared field(s) — pass apply:true to execute`,
|
|
1288
|
+
};
|
|
1289
|
+
},
|
|
1290
|
+
},
|
|
1291
|
+
|
|
1269
1292
|
];
|
|
1270
1293
|
}
|
|
1271
1294
|
|
package/core/src/notes.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
type CursorPayload,
|
|
19
19
|
type QueryHashInputs,
|
|
20
20
|
} from "./cursor.js";
|
|
21
|
+
import { releaseField } from "./indexed-fields.js";
|
|
21
22
|
|
|
22
23
|
let idCounter = 0;
|
|
23
24
|
|
|
@@ -943,12 +944,35 @@ export function listTags(db: Database): { name: string; count: number }[] {
|
|
|
943
944
|
}
|
|
944
945
|
|
|
945
946
|
export function deleteTag(db: Database, name: string): { deleted: boolean; notes_untagged: number } {
|
|
946
|
-
const
|
|
947
|
-
|
|
947
|
+
const row = db.prepare("SELECT fields FROM tags WHERE name = ?").get(name) as
|
|
948
|
+
| { fields: string | null }
|
|
949
|
+
| undefined;
|
|
950
|
+
if (!row) return { deleted: false, notes_untagged: 0 };
|
|
948
951
|
|
|
949
952
|
const countRow = db.prepare("SELECT COUNT(*) as c FROM note_tags WHERE tag_name = ?").get(name) as { c: number };
|
|
950
953
|
const notesUntagged = countRow.c;
|
|
951
954
|
|
|
955
|
+
// Release any indexed fields this tag declared BEFORE the row drops.
|
|
956
|
+
// `releaseField` only drops the generated column + index when this tag is
|
|
957
|
+
// the last live declarer (co-declaration guard) — a field co-declared by
|
|
958
|
+
// another live tag keeps its column and just loses this tag from the set.
|
|
959
|
+
// This lives in the store-level delete (not the MCP layer) so every caller
|
|
960
|
+
// — MCP delete-tag, the REST DELETE /tags/:name route, the import
|
|
961
|
+
// blow-away sweep — releases consistently. See the gitcoin orphaned-fields
|
|
962
|
+
// bug report.
|
|
963
|
+
if (row.fields) {
|
|
964
|
+
try {
|
|
965
|
+
const fields = JSON.parse(row.fields) as Record<string, { indexed?: boolean }>;
|
|
966
|
+
for (const [fieldName, spec] of Object.entries(fields)) {
|
|
967
|
+
if (spec?.indexed === true) {
|
|
968
|
+
releaseField(db, fieldName, name);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
} catch {
|
|
972
|
+
// Malformed fields JSON — nothing to release; proceed with the delete.
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
952
976
|
db.prepare("DELETE FROM note_tags WHERE tag_name = ?").run(name);
|
|
953
977
|
db.prepare("DELETE FROM tags WHERE name = ?").run(name);
|
|
954
978
|
|
|
@@ -24,6 +24,8 @@ import { join } from "path";
|
|
|
24
24
|
import { tmpdir } from "os";
|
|
25
25
|
|
|
26
26
|
import { SqliteStore } from "./store.js";
|
|
27
|
+
import { getIndexedField } from "./indexed-fields.js";
|
|
28
|
+
import { buildVaultProjection } from "./vault-projection.js";
|
|
27
29
|
import {
|
|
28
30
|
CaseCollisionError,
|
|
29
31
|
emitYamlDoc,
|
|
@@ -725,6 +727,56 @@ describe("importPortableVault", async () => {
|
|
|
725
727
|
expect(schema!.description).toBe("A unit of work");
|
|
726
728
|
});
|
|
727
729
|
|
|
730
|
+
// Fix 2 — import must re-declare indexed fields. The import writes
|
|
731
|
+
// tags.fields via upsertTagRecord but historically never materialized the
|
|
732
|
+
// backing generated columns + indexes, so a fresh import advertised
|
|
733
|
+
// `indexed: true` while queries silently full-scanned. Regression: export a
|
|
734
|
+
// vault with an indexed field → fresh import → the generated column + index
|
|
735
|
+
// exist and the field shows in the vault-info indexed_fields catalog.
|
|
736
|
+
it("re-declares indexed fields on import (column + index + vault-info catalog)", async () => {
|
|
737
|
+
await store.upsertTagRecord("project", {
|
|
738
|
+
fields: { status: { type: "string", indexed: true } },
|
|
739
|
+
});
|
|
740
|
+
await store.createNote("p", { id: "p", tags: ["project"], metadata: { status: "active" } });
|
|
741
|
+
const outDir = join(tmpBase, "out");
|
|
742
|
+
await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
743
|
+
|
|
744
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
745
|
+
const targetDb = target.db;
|
|
746
|
+
// Pre-condition: a fresh store has no backing column yet.
|
|
747
|
+
const colsBefore = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
748
|
+
expect(colsBefore).not.toContain("meta_status");
|
|
749
|
+
|
|
750
|
+
const stats = await importPortableVault(target, { inDir: outDir });
|
|
751
|
+
expect(stats.schemas_restored).toBe(1);
|
|
752
|
+
expect(stats.indexes_declared).toBe(1);
|
|
753
|
+
|
|
754
|
+
// The generated column + index now exist — same introspection vault-info
|
|
755
|
+
// uses to advertise the queryable-field catalog.
|
|
756
|
+
const colsAfter = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
757
|
+
expect(colsAfter).toContain("meta_status");
|
|
758
|
+
const idxs = (targetDb.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='notes'").all() as { name: string }[]).map((r) => r.name);
|
|
759
|
+
expect(idxs).toContain("idx_meta_status");
|
|
760
|
+
expect(getIndexedField(targetDb, "status")?.declarerTags).toEqual(["project"]);
|
|
761
|
+
// And vault-info lists it.
|
|
762
|
+
expect(buildVaultProjection(targetDb).indexed_fields.map((f) => f.name)).toContain("status");
|
|
763
|
+
});
|
|
764
|
+
|
|
765
|
+
it("dry-run import counts indexes_declared without materializing columns", async () => {
|
|
766
|
+
await store.upsertTagRecord("project", {
|
|
767
|
+
fields: { status: { type: "string", indexed: true } },
|
|
768
|
+
});
|
|
769
|
+
const outDir = join(tmpBase, "out");
|
|
770
|
+
await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
771
|
+
|
|
772
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
773
|
+
const stats = await importPortableVault(target, { inDir: outDir, dryRun: true });
|
|
774
|
+
expect(stats.indexes_declared).toBe(1);
|
|
775
|
+
// Dry-run touches nothing.
|
|
776
|
+
const cols = (target.db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
777
|
+
expect(cols).not.toContain("meta_status");
|
|
778
|
+
});
|
|
779
|
+
|
|
728
780
|
it("restores typed links (non-wikilink relationships)", async () => {
|
|
729
781
|
await store.createNote("src body", { id: "src", path: "src" });
|
|
730
782
|
await store.createNote("tgt body", { id: "tgt", path: "tgt" });
|
package/core/src/portable-md.ts
CHANGED
|
@@ -1535,6 +1535,14 @@ export interface ImportStats {
|
|
|
1535
1535
|
skipped_sidecars: Array<{ sidecar_id: string; expected_path: string | null; expected_extension: string | null; reason: string }>;
|
|
1536
1536
|
/** Set when the caller passed `blowAway: true`; counts notes removed. */
|
|
1537
1537
|
notes_wiped: number;
|
|
1538
|
+
/**
|
|
1539
|
+
* (tag, field) indexed-field declarations replayed after restoring tag
|
|
1540
|
+
* schemas — materializes the generated columns + indexes a live vault
|
|
1541
|
+
* would have. Without this an imported vault's schemas say `indexed: true`
|
|
1542
|
+
* but the backing columns don't exist until each tag is next `update-tag`'d
|
|
1543
|
+
* (queries fall back to full scans). See the import re-declare fix.
|
|
1544
|
+
*/
|
|
1545
|
+
indexes_declared: number;
|
|
1538
1546
|
}
|
|
1539
1547
|
|
|
1540
1548
|
/**
|
|
@@ -1582,6 +1590,7 @@ export async function importPortableVault(
|
|
|
1582
1590
|
skipped_attachments: [],
|
|
1583
1591
|
skipped_sidecars: [],
|
|
1584
1592
|
notes_wiped: 0,
|
|
1593
|
+
indexes_declared: 0,
|
|
1585
1594
|
};
|
|
1586
1595
|
|
|
1587
1596
|
// 1. Optional wipe. Notes are deleted via the public Store API so
|
|
@@ -2019,6 +2028,45 @@ export async function importPortableVault(
|
|
|
2019
2028
|
await store.syncAllWikilinks();
|
|
2020
2029
|
}
|
|
2021
2030
|
|
|
2031
|
+
// 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
|
|
2032
|
+
// Step 2 restored tag schemas via `store.upsertTagRecord`, which — now that
|
|
2033
|
+
// the indexed-field lifecycle is centralized in the store — already
|
|
2034
|
+
// materializes the backing generated columns + indexes as it persists each
|
|
2035
|
+
// schema. This explicit reconcile is therefore idempotent on the happy path;
|
|
2036
|
+
// it stays as a safety net (covers any schema written through a path that
|
|
2037
|
+
// skipped the lifecycle) and gives the authoritative `indexes_declared`
|
|
2038
|
+
// count. Without it, a regression in step 2 would silently leave the
|
|
2039
|
+
// imported schemas advertising `indexed: true` while queries full-scan.
|
|
2040
|
+
if (!opts.dryRun) {
|
|
2041
|
+
stats.indexes_declared = await store.reconcileDeclaredIndexes();
|
|
2042
|
+
} else {
|
|
2043
|
+
// Dry-run: count what WOULD be declared without touching the DB. Both
|
|
2044
|
+
// paths count per (tag, field) declaration (a co-declared field counts
|
|
2045
|
+
// once per declaring tag). The one asymmetry: this dry-run counts every
|
|
2046
|
+
// `indexed: true` field including unsupported types, whereas the applied
|
|
2047
|
+
// `reconcileDeclaredIndexes` skips fields whose type can't be indexed —
|
|
2048
|
+
// so the dry-run can over-count by the number of mis-typed indexed
|
|
2049
|
+
// fields. It's a "how much indexing work" signal, not a row-exact promise.
|
|
2050
|
+
const schemasDir2 = join(sidecar, "schemas");
|
|
2051
|
+
if (existsSync(schemasDir2)) {
|
|
2052
|
+
for (const entry of readdirSync(schemasDir2)) {
|
|
2053
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
2054
|
+
const fullPath = join(schemasDir2, entry);
|
|
2055
|
+
const resolved = resolvePath(fullPath);
|
|
2056
|
+
if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
|
|
2057
|
+
const text = readFileSync(fullPath, "utf-8");
|
|
2058
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
2059
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
2060
|
+
const fields = frontmatter.fields;
|
|
2061
|
+
if (fields && typeof fields === "object" && !Array.isArray(fields)) {
|
|
2062
|
+
for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
|
|
2063
|
+
if (spec?.indexed === true) stats.indexes_declared++;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2022
2070
|
return stats;
|
|
2023
2071
|
}
|
|
2024
2072
|
|
package/core/src/schema.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
|
|
|
2
2
|
import { normalizePath } from "./paths.js";
|
|
3
3
|
import { rebuildIndexes } from "./indexed-fields.js";
|
|
4
4
|
|
|
5
|
-
export const SCHEMA_VERSION =
|
|
5
|
+
export const SCHEMA_VERSION = 20;
|
|
6
6
|
|
|
7
7
|
export const SCHEMA_SQL = `
|
|
8
8
|
-- Notes: the universal record.
|
|
@@ -96,6 +96,17 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
|
|
|
96
96
|
|
|
97
97
|
-- Tokens: API authentication with OAuth-standard scopes.
|
|
98
98
|
--
|
|
99
|
+
-- VESTIGIAL as of 0.5.0 (vault#282 Stage 2). Vault is a pure hub
|
|
100
|
+
-- resource-server: it no longer mints (pvt_*) or validates rows in this
|
|
101
|
+
-- table — auth runs through hub-issued JWTs + VAULT_AUTH_TOKEN + legacy YAML
|
|
102
|
+
-- api_keys only. The table is KEPT (not dropped) because migrateVaultKeys
|
|
103
|
+
-- raw-INSERTs legacy YAML api_keys here as its import landing zone, and
|
|
104
|
+
-- dropping it would trip an upgrade on a missing column for operators with
|
|
105
|
+
-- leftover rows. A future cosmetic migration may drop it alongside
|
|
106
|
+
-- oauth_clients/oauth_codes. 'tokens list' / 'tokens revoke' (CLI) still
|
|
107
|
+
-- read/delete here for cleanup of leftover rows. See the field docs below for
|
|
108
|
+
-- the historical (pre-0.5.0) semantics.
|
|
109
|
+
--
|
|
99
110
|
-- scopes is a whitespace-separated list of granted scopes (OAuth 2.0 §3.3)
|
|
100
111
|
-- — e.g. "vault:read vault:write". Introduced in v12 alongside enforcement;
|
|
101
112
|
-- NULL rows are pre-v12 tokens which fall back to deriving scopes from the
|
|
@@ -128,10 +139,9 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
|
|
|
128
139
|
-- minted this one, or the hub-JWT jti claim when minted from a hub
|
|
129
140
|
-- session. Session-pinned list+revoke in manage-token filters on this.
|
|
130
141
|
--
|
|
131
|
-
-- revoked_at (v19)
|
|
132
|
-
--
|
|
133
|
-
--
|
|
134
|
-
-- resolveToken treats a revoked_at-set row as not-found.
|
|
142
|
+
-- revoked_at (v19) marked soft-revocation of vault-DB tokens. Vestigial
|
|
143
|
+
-- post-0.5.0 (vault#282 Stage 2) — the validation path that read it
|
|
144
|
+
-- (resolveToken) was removed alongside the pvt_* mint.
|
|
135
145
|
CREATE TABLE IF NOT EXISTS tokens (
|
|
136
146
|
token_hash TEXT PRIMARY KEY,
|
|
137
147
|
label TEXT NOT NULL,
|
|
@@ -149,6 +159,30 @@ CREATE TABLE IF NOT EXISTS tokens (
|
|
|
149
159
|
revoked_at TEXT
|
|
150
160
|
);
|
|
151
161
|
|
|
162
|
+
-- mcp_mint_ledger (v20) — session-pinned record of HUB JWTs minted by the
|
|
163
|
+
-- manage-token MCP tool (vault#403, MGT). After the auth-unification arc,
|
|
164
|
+
-- manage-token mints hub JWTs (via hub mint-token attenuation proxy), not
|
|
165
|
+
-- pvt_* vault-DB tokens — so the mints no longer live in the tokens table.
|
|
166
|
+
-- This ledger is a lightweight local index of (parent_jti to minted hub jti)
|
|
167
|
+
-- so the tool list/revoke surface can stay session-scoped: a session sees
|
|
168
|
+
-- and revokes only the hub JWTs it minted. Rows are NOT credentials — the
|
|
169
|
+
-- signed JWT is never stored, only its jti (the revocation handle) plus
|
|
170
|
+
-- display metadata. revoked_at mirrors the tokens-table soft-revoke marker
|
|
171
|
+
-- so a second revoke is idempotent even if the hub round-trip is skipped.
|
|
172
|
+
-- The authoritative revocation state lives in hub token registry; this is
|
|
173
|
+
-- the attribution index only.
|
|
174
|
+
CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
|
|
175
|
+
jti TEXT PRIMARY KEY,
|
|
176
|
+
parent_jti TEXT NOT NULL,
|
|
177
|
+
vault_name TEXT NOT NULL,
|
|
178
|
+
label TEXT NOT NULL,
|
|
179
|
+
scopes TEXT,
|
|
180
|
+
scoped_tags TEXT,
|
|
181
|
+
created_at TEXT NOT NULL,
|
|
182
|
+
expires_at TEXT,
|
|
183
|
+
revoked_at TEXT
|
|
184
|
+
);
|
|
185
|
+
|
|
152
186
|
-- OAuth: registered clients (Dynamic Client Registration)
|
|
153
187
|
-- VESTIGIAL after vault 0.4.x workstream E (2026-05-25). The standalone
|
|
154
188
|
-- OAuth issuer that wrote these rows was retired (hub is the issuer now;
|
|
@@ -214,6 +248,10 @@ CREATE INDEX IF NOT EXISTS idx_note_tags_tag ON note_tags(tag_name, note_id);
|
|
|
214
248
|
CREATE INDEX IF NOT EXISTS idx_attachments_note ON attachments(note_id);
|
|
215
249
|
CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_id);
|
|
216
250
|
CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_id);
|
|
251
|
+
-- Session-pinned manage-token ledger lookup (v20): list/revoke scope on
|
|
252
|
+
-- (parent_jti, vault_name). The mcp_mint_ledger table is created
|
|
253
|
+
-- unconditionally above, so this index is safe in SCHEMA_SQL.
|
|
254
|
+
CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name);
|
|
217
255
|
-- idx_tokens_vault_name is created in migrateToV16, not here. SCHEMA_SQL
|
|
218
256
|
-- runs BEFORE migrations; an upgrading v15 vault doesn't yet have the
|
|
219
257
|
-- vault_name column when this section evaluates, so the index has to
|
|
@@ -380,9 +418,10 @@ export function initSchema(db: Database): void {
|
|
|
380
418
|
migrateToV15(db);
|
|
381
419
|
|
|
382
420
|
// Migrate v15 → v16: add `vault_name` column to tokens. Existing rows
|
|
383
|
-
//
|
|
384
|
-
// NULL for any vault
|
|
385
|
-
//
|
|
421
|
+
// backfilled to NULL ("server-wide / legacy" semantic) — at the time auth
|
|
422
|
+
// accepted NULL for any vault so pre-v16 pvt_* tokens kept working. (pvt_*
|
|
423
|
+
// validation was dropped at 0.5.0 / vault#282 Stage 2; the column is now
|
|
424
|
+
// vestigial.) See vault#257.
|
|
386
425
|
migrateToV16(db);
|
|
387
426
|
|
|
388
427
|
// Migrate v16 → v17: rip the standalone `note_schemas` + `schema_mappings`
|
|
@@ -403,6 +442,13 @@ export function initSchema(db: Database): void {
|
|
|
403
442
|
// "non-MCP-minted, not revoked" — identical pre-v19 semantics.
|
|
404
443
|
migrateToV19(db);
|
|
405
444
|
|
|
445
|
+
// Migrate v19 → v20: the `mcp_mint_ledger` table (session-pinned record of
|
|
446
|
+
// hub JWTs minted by the manage-token MCP tool). Created by SCHEMA_SQL's
|
|
447
|
+
// `CREATE TABLE IF NOT EXISTS` above, so this is a no-op confirmation hook
|
|
448
|
+
// — present for symmetry with the other migration steps and to anchor the
|
|
449
|
+
// version bump. See vault#403 (MGT — manage-token mints hub JWTs).
|
|
450
|
+
migrateToV20(db);
|
|
451
|
+
|
|
406
452
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
407
453
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
408
454
|
rebuildIndexes(db);
|
|
@@ -803,12 +849,11 @@ function migrateToV15(db: Database): void {
|
|
|
803
849
|
* Migrate v15 → v16: per-vault token storage (vault#257).
|
|
804
850
|
*
|
|
805
851
|
* Adds `tokens.vault_name TEXT` (nullable). Existing rows stay NULL —
|
|
806
|
-
* "server-wide / legacy" semantic
|
|
807
|
-
*
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
*
|
|
811
|
-
* filter in the admin SPA.
|
|
852
|
+
* "server-wide / legacy" semantic. At the time, `authenticateVaultRequest`
|
|
853
|
+
* accepted NULL for any vault so pre-v16 pvt_* tokens kept working. (pvt_*
|
|
854
|
+
* validation + the `/vault/<name>/tokens` mint route were both removed at
|
|
855
|
+
* 0.5.0 / vault#282 Stage 2 — the column + index are now vestigial; the
|
|
856
|
+
* index still speeds the per-vault `listTokens` cleanup listing.)
|
|
812
857
|
*
|
|
813
858
|
* Wrapped in BEGIN IMMEDIATE / COMMIT (with try/catch ROLLBACK) per the
|
|
814
859
|
* v14/v15 wrap pattern from vault#251 — the column add and index create
|
|
@@ -1001,6 +1046,34 @@ function migrateToV19(db: Database): void {
|
|
|
1001
1046
|
}
|
|
1002
1047
|
}
|
|
1003
1048
|
|
|
1049
|
+
/**
|
|
1050
|
+
* Migrate v19 → v20: ensure the `mcp_mint_ledger` table exists. SCHEMA_SQL's
|
|
1051
|
+
* `CREATE TABLE IF NOT EXISTS` already covers fresh vaults AND upgrading
|
|
1052
|
+
* vaults (SCHEMA_SQL runs unconditionally on every open before the migration
|
|
1053
|
+
* steps), so this is a defensive no-op confirmation — there is no ALTER to
|
|
1054
|
+
* perform. Kept as a named step so the version-bump arc reads cleanly and a
|
|
1055
|
+
* future column addition has an obvious home. See vault#403 (MGT).
|
|
1056
|
+
*/
|
|
1057
|
+
function migrateToV20(db: Database): void {
|
|
1058
|
+
if (hasTable(db, "mcp_mint_ledger")) return;
|
|
1059
|
+
db.exec(`
|
|
1060
|
+
CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
|
|
1061
|
+
jti TEXT PRIMARY KEY,
|
|
1062
|
+
parent_jti TEXT NOT NULL,
|
|
1063
|
+
vault_name TEXT NOT NULL,
|
|
1064
|
+
label TEXT NOT NULL,
|
|
1065
|
+
scopes TEXT,
|
|
1066
|
+
scoped_tags TEXT,
|
|
1067
|
+
created_at TEXT NOT NULL,
|
|
1068
|
+
expires_at TEXT,
|
|
1069
|
+
revoked_at TEXT
|
|
1070
|
+
)
|
|
1071
|
+
`);
|
|
1072
|
+
db.exec(
|
|
1073
|
+
"CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name)",
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1004
1077
|
function hasTable(db: Database, name: string): boolean {
|
|
1005
1078
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1006
1079
|
return !!row;
|
package/core/src/store.ts
CHANGED
|
@@ -4,6 +4,12 @@ import { initSchema } from "./schema.js";
|
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
5
|
import * as linkOps from "./links.js";
|
|
6
6
|
import * as tagSchemaOps from "./tag-schemas.js";
|
|
7
|
+
import * as indexedFieldOps from "./indexed-fields.js";
|
|
8
|
+
import {
|
|
9
|
+
pruneOrphanedIndexedFields,
|
|
10
|
+
reconcileDeclaredIndexes,
|
|
11
|
+
type PrunedField,
|
|
12
|
+
} from "./indexed-fields.js";
|
|
7
13
|
import { syncWikilinks, resolveUnresolvedWikilinks } from "./wikilinks.js";
|
|
8
14
|
import { pathTitle } from "./paths.js";
|
|
9
15
|
import { HookRegistry } from "./hooks.js";
|
|
@@ -503,6 +509,37 @@ export class BunSqliteStore implements Store {
|
|
|
503
509
|
return tagSchemaOps.getTagSchemaMap(this.db);
|
|
504
510
|
}
|
|
505
511
|
|
|
512
|
+
// ---- Indexed-field lifecycle (generated columns + indexes) ----
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Prune orphaned `indexed_fields` declarers — declarer tags that no longer
|
|
516
|
+
* have a `tags` row. Fields with no surviving live declarer are dropped
|
|
517
|
+
* wholesale (row + generated column + index); co-declared fields keep their
|
|
518
|
+
* column and just lose the dead declarers. `dryRun` (default true) returns
|
|
519
|
+
* the plan without mutating. See the gitcoin orphaned-fields bug.
|
|
520
|
+
*/
|
|
521
|
+
async pruneIndexedFields(opts?: { dryRun?: boolean }): Promise<PrunedField[]> {
|
|
522
|
+
const plan = pruneOrphanedIndexedFields(this.db, opts);
|
|
523
|
+
// A drop changes the queryable-field catalog vault-info advertises; bust
|
|
524
|
+
// the schema cache so the next read reflects it.
|
|
525
|
+
if (opts?.dryRun === false && plan.length > 0) this._schemaConfig = null;
|
|
526
|
+
return plan;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Replay `declareField` for every `indexed: true` field across all current
|
|
531
|
+
* tag records, materializing the generated columns + indexes. Idempotent —
|
|
532
|
+
* used by the portable-md import path so a fresh import ends with the same
|
|
533
|
+
* backing columns a live vault would have. Returns the count of (tag, field)
|
|
534
|
+
* declarations replayed.
|
|
535
|
+
*/
|
|
536
|
+
async reconcileDeclaredIndexes(): Promise<number> {
|
|
537
|
+
const schemas = await this.listTagRecords();
|
|
538
|
+
const count = reconcileDeclaredIndexes(this.db, schemas);
|
|
539
|
+
if (count > 0) this._schemaConfig = null;
|
|
540
|
+
return count;
|
|
541
|
+
}
|
|
542
|
+
|
|
506
543
|
// ---- Tag Records (post-v14: full identity row) ----
|
|
507
544
|
|
|
508
545
|
async listTagRecords() {
|
|
@@ -517,6 +554,18 @@ export class BunSqliteStore implements Store {
|
|
|
517
554
|
* Partial upsert of the full tag record. Any patch field left undefined
|
|
518
555
|
* is preserved; pass null to clear. Invalidates the tag-hierarchy cache
|
|
519
556
|
* when `parent_names` is touched.
|
|
557
|
+
*
|
|
558
|
+
* Indexed-field lifecycle is reconciled HERE — at the single store
|
|
559
|
+
* chokepoint every caller (MCP update-tag, REST PUT /tags/:name, import)
|
|
560
|
+
* funnels through — so no caller can persist a `fields` change without the
|
|
561
|
+
* matching declareField/releaseField. When `patch.fields` is touched
|
|
562
|
+
* (object or explicit `null`), the prior-vs-next indexed-field set is
|
|
563
|
+
* diffed: added indexed fields get `declareField`, removed ones get
|
|
564
|
+
* `releaseField` (which drops the generated column + index only when this
|
|
565
|
+
* tag is the last live declarer — the co-declaration guard). `patch.fields
|
|
566
|
+
* === undefined` (no-touch) skips reconciliation entirely. Centralizing
|
|
567
|
+
* here is the same discipline as moving delete-release into noteOps.deleteTag
|
|
568
|
+
* — it closes the REST PUT orphaned-column leak. See the gitcoin bug.
|
|
520
569
|
*/
|
|
521
570
|
async upsertTagRecord(
|
|
522
571
|
tag: string,
|
|
@@ -527,7 +576,43 @@ export class BunSqliteStore implements Store {
|
|
|
527
576
|
parent_names?: string[] | null;
|
|
528
577
|
},
|
|
529
578
|
) {
|
|
579
|
+
// Snapshot the prior indexed-field set BEFORE the write so the diff below
|
|
580
|
+
// sees what this tag declared going in. Only needed when `fields` changes.
|
|
581
|
+
const priorRecord =
|
|
582
|
+
patch.fields !== undefined ? tagSchemaOps.getTagRecord(this.db, tag) : null;
|
|
583
|
+
|
|
530
584
|
const result = tagSchemaOps.upsertTagRecord(this.db, tag, patch);
|
|
585
|
+
|
|
586
|
+
if (patch.fields !== undefined) {
|
|
587
|
+
const indexedSet = (fields: Record<string, tagSchemaOps.TagFieldSchema> | null | undefined) =>
|
|
588
|
+
new Set(
|
|
589
|
+
Object.entries(fields ?? {})
|
|
590
|
+
.filter(([, v]) => v.indexed === true)
|
|
591
|
+
.map(([k]) => k),
|
|
592
|
+
);
|
|
593
|
+
const nextFields = patch.fields; // object | null
|
|
594
|
+
const priorIndexed = indexedSet(priorRecord?.fields);
|
|
595
|
+
const nextIndexed = indexedSet(nextFields);
|
|
596
|
+
for (const fieldName of nextIndexed) {
|
|
597
|
+
const spec = nextFields![fieldName]!;
|
|
598
|
+
const mapped = indexedFieldOps.mapFieldType(spec.type);
|
|
599
|
+
// Unmappable type for indexing is a caller error; surface it rather
|
|
600
|
+
// than silently skipping. MCP/REST validate up-front for a cleaner
|
|
601
|
+
// message, but this is the backstop at the chokepoint.
|
|
602
|
+
if (!mapped) {
|
|
603
|
+
throw new indexedFieldOps.IndexedFieldError(
|
|
604
|
+
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
indexedFieldOps.declareField(this.db, fieldName, mapped, tag);
|
|
608
|
+
}
|
|
609
|
+
for (const fieldName of priorIndexed) {
|
|
610
|
+
if (!nextIndexed.has(fieldName)) {
|
|
611
|
+
indexedFieldOps.releaseField(this.db, fieldName, tag);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
531
616
|
if (patch.parent_names !== undefined) {
|
|
532
617
|
// parent_names drives both query expansion (tag hierarchy) AND, post
|
|
533
618
|
// vault#270, schema inheritance — bust both caches.
|
|
@@ -687,6 +772,38 @@ export class BunSqliteStore implements Store {
|
|
|
687
772
|
};
|
|
688
773
|
}
|
|
689
774
|
|
|
775
|
+
/**
|
|
776
|
+
* Reverse-lookup: every attachment row whose `path` column equals the given
|
|
777
|
+
* vault-internal relative path (`<date>/<filename>`). A single on-disk asset
|
|
778
|
+
* can be referenced by more than one attachment row (the orphan check in
|
|
779
|
+
* `deleteAttachment` accounts for that), so this returns an array. Used by
|
|
780
|
+
* the raw `/api/storage/<date>/<file>` byte-serve path to map a requested
|
|
781
|
+
* file back to its owning note(s) for tag-scope enforcement — without this,
|
|
782
|
+
* a tag-scoped token could fetch an out-of-scope note's attachment bytes
|
|
783
|
+
* directly by path (the path-secrecy-only bypass; see the C0 adversarial
|
|
784
|
+
* audit finding).
|
|
785
|
+
*/
|
|
786
|
+
async getAttachmentsByPath(path: string): Promise<Attachment[]> {
|
|
787
|
+
const rows = this.db.prepare(
|
|
788
|
+
"SELECT * FROM attachments WHERE path = ? ORDER BY created_at",
|
|
789
|
+
).all(path) as { id: string; note_id: string; path: string; mime_type: string; metadata: string | null; created_at: string }[];
|
|
790
|
+
|
|
791
|
+
return rows.map((r) => {
|
|
792
|
+
let metadata: Record<string, unknown> | undefined;
|
|
793
|
+
if (r.metadata && r.metadata !== "{}") {
|
|
794
|
+
try { metadata = JSON.parse(r.metadata); } catch {}
|
|
795
|
+
}
|
|
796
|
+
return {
|
|
797
|
+
id: r.id,
|
|
798
|
+
noteId: r.note_id,
|
|
799
|
+
path: r.path,
|
|
800
|
+
mimeType: r.mime_type,
|
|
801
|
+
metadata,
|
|
802
|
+
createdAt: r.created_at,
|
|
803
|
+
};
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
|
|
690
807
|
/**
|
|
691
808
|
* Replace the attachment's metadata JSON blob. The caller passes the full
|
|
692
809
|
* merged object — this is a set, not a patch, so partial-field updates
|