@openparachute/vault 0.4.9-rc.9 → 0.5.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +51 -54
  2. package/core/src/core.test.ts +4 -1
  3. package/core/src/indexed-fields.test.ts +151 -0
  4. package/core/src/indexed-fields.ts +98 -0
  5. package/core/src/mcp.ts +66 -43
  6. package/core/src/notes.ts +26 -2
  7. package/core/src/portable-md.test.ts +52 -0
  8. package/core/src/portable-md.ts +48 -0
  9. package/core/src/schema.ts +87 -14
  10. package/core/src/store.ts +117 -0
  11. package/core/src/types.ts +28 -0
  12. package/package.json +2 -2
  13. package/src/auth-hub-jwt.test.ts +191 -11
  14. package/src/auth-status.ts +12 -5
  15. package/src/auth.test.ts +135 -219
  16. package/src/auth.ts +158 -107
  17. package/src/cli.ts +306 -224
  18. package/src/config.ts +12 -4
  19. package/src/export-watch.test.ts +23 -0
  20. package/src/export-watch.ts +14 -0
  21. package/src/git-preflight.test.ts +70 -0
  22. package/src/git-preflight.ts +68 -0
  23. package/src/hub-jwt.test.ts +27 -2
  24. package/src/hub-jwt.ts +10 -0
  25. package/src/init-summary.test.ts +4 -4
  26. package/src/init-summary.ts +36 -10
  27. package/src/mcp-config.test.ts +4 -2
  28. package/src/mcp-http.ts +24 -3
  29. package/src/mcp-install-interactive.test.ts +33 -71
  30. package/src/mcp-install-interactive.ts +23 -76
  31. package/src/mcp-install.test.ts +156 -55
  32. package/src/mcp-install.ts +109 -3
  33. package/src/mcp-tools.ts +249 -74
  34. package/src/mirror-config.test.ts +107 -0
  35. package/src/mirror-config.ts +275 -9
  36. package/src/mirror-credentials.test.ts +168 -17
  37. package/src/mirror-credentials.ts +155 -32
  38. package/src/mirror-deps.ts +25 -16
  39. package/src/mirror-import.test.ts +122 -16
  40. package/src/mirror-import.ts +50 -16
  41. package/src/mirror-manager.test.ts +51 -0
  42. package/src/mirror-manager.ts +116 -22
  43. package/src/mirror-per-vault.test.ts +519 -0
  44. package/src/mirror-registry.ts +91 -14
  45. package/src/mirror-routes.test.ts +81 -21
  46. package/src/mirror-routes.ts +90 -16
  47. package/src/routes.ts +39 -2
  48. package/src/routing.test.ts +203 -118
  49. package/src/routing.ts +46 -59
  50. package/src/scopes.test.ts +0 -86
  51. package/src/scopes.ts +9 -97
  52. package/src/server.ts +102 -34
  53. package/src/storage.test.ts +132 -7
  54. package/src/token-store.test.ts +88 -169
  55. package/src/token-store.ts +123 -249
  56. package/src/vault-create.test.ts +12 -4
  57. package/src/vault.test.ts +408 -103
  58. package/web/ui/dist/assets/index-DDRo6F4u.js +60 -0
  59. package/web/ui/dist/index.html +1 -1
  60. package/src/tokens-routes.test.ts +0 -727
  61. package/src/tokens-routes.ts +0 -392
  62. package/web/ui/dist/assets/index-Degr8snN.js +0 -60
package/core/src/mcp.ts CHANGED
@@ -23,8 +23,8 @@ export interface McpToolDef {
23
23
  /**
24
24
  * Minimum scope verb the caller must hold for THIS vault to see + invoke
25
25
  * the tool. `read` for pure queries, `write` for mutations, `admin` for
26
- * token-management surfaces (only `manage-token` in the current set —
27
- * core's nine tools cap at `write`). The MCP HTTP layer filters
26
+ * operator-only surfaces (`prune-schema` in core; `manage-token` in the
27
+ * server layer). The MCP HTTP layer filters
28
28
  * `tools/list` by this field and verb-gates `tools/call` against it; the
29
29
  * filter is the primary defense, the inner gate is defense-in-depth.
30
30
  *
@@ -100,9 +100,9 @@ function removeWikilinkBrackets(content: string, targetPath: string): string {
100
100
  // ---------------------------------------------------------------------------
101
101
 
102
102
  /**
103
- * Generate the consolidated MCP tools for a vault. Post-v17 surface (9):
103
+ * Generate the consolidated MCP tools for a vault. Surface (10):
104
104
  * query-notes, create-note, update-note, delete-note, list-tags, update-tag,
105
- * delete-tag, find-path, vault-info.
105
+ * delete-tag, find-path, vault-info, prune-schema (admin).
106
106
  */
107
107
  export function generateMcpTools(store: Store): McpToolDef[] {
108
108
  const db: Database = (store as any).db;
@@ -1074,12 +1074,23 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1074
1074
  const tag = params.tag as string;
1075
1075
  const existing = tagSchemaOps.getTagRecord(db, tag);
1076
1076
 
1077
- // ---- fields: shallow-merge into existing (preserves prior keys).
1078
- const incomingFields = (params.fields as Record<string, TagFieldSchema> | undefined) ?? {};
1079
- const mergedFields: Record<string, TagFieldSchema> = {
1080
- ...(existing?.fields ?? {}),
1081
- ...incomingFields,
1082
- };
1077
+ // ---- fields: three-way semantics, distinguishing `null` from
1078
+ // `undefined` (do NOT collapse with `?? {}` that silently turns an
1079
+ // explicit clear-all into a no-op, the gitcoin orphaned-fields bug).
1080
+ // - undefined → no change. Preserve every existing field; declare
1081
+ // nothing new. mergedFields === existing.fields.
1082
+ // - null → clear ALL of this tag's field schemas.
1083
+ // mergedFields = {} so the diff below releases every
1084
+ // indexed field this tag exclusively declares.
1085
+ // - object → shallow-merge into existing (preserves prior keys).
1086
+ const incomingFields =
1087
+ params.fields === null || params.fields === undefined
1088
+ ? {}
1089
+ : (params.fields as Record<string, TagFieldSchema>);
1090
+ const mergedFields: Record<string, TagFieldSchema> =
1091
+ params.fields === null
1092
+ ? {}
1093
+ : { ...(existing?.fields ?? {}), ...incomingFields };
1083
1094
 
1084
1095
  // Validate cross-tag consistency on fields being (re)declared in this
1085
1096
  // call. `type` and `indexed` are global — all declarers must agree.
@@ -1146,6 +1157,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1146
1157
  : (params.fields !== undefined ? null : undefined);
1147
1158
  const descriptionPatch =
1148
1159
  params.description === undefined ? undefined : (params.description as string);
1160
+ // The indexed-field lifecycle (declareField for added indexed fields,
1161
+ // releaseField for removed ones, with the co-declaration guard) is
1162
+ // reconciled inside store.upsertTagRecord — the single chokepoint all
1163
+ // callers (MCP, REST PUT /tags/:name, import) share — so it can't be
1164
+ // bypassed. The cross-tag validation above stays here to surface a
1165
+ // clean error before persisting. See the gitcoin orphaned-fields bug.
1149
1166
  const result = await store.upsertTagRecord(tag, {
1150
1167
  ...(descriptionPatch !== undefined ? { description: descriptionPatch } : {}),
1151
1168
  ...(fieldsPatch !== undefined ? { fields: fieldsPatch } : {}),
@@ -1153,28 +1170,6 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1153
1170
  ...(parentNamesPatch !== undefined ? { parent_names: parentNamesPatch } : {}),
1154
1171
  });
1155
1172
 
1156
- // ---- Reconcile indexed-field lifecycle for this tag.
1157
- const priorIndexed = new Set(
1158
- Object.entries(existing?.fields ?? {})
1159
- .filter(([, v]) => v.indexed === true)
1160
- .map(([k]) => k),
1161
- );
1162
- const nextIndexed = new Set(
1163
- Object.entries(mergedFields)
1164
- .filter(([, v]) => v.indexed === true)
1165
- .map(([k]) => k),
1166
- );
1167
- for (const fieldName of nextIndexed) {
1168
- const spec = mergedFields[fieldName]!;
1169
- const mapped = indexedFieldOps.mapFieldType(spec.type)!;
1170
- indexedFieldOps.declareField(db, fieldName, mapped, tag);
1171
- }
1172
- for (const fieldName of priorIndexed) {
1173
- if (!nextIndexed.has(fieldName)) {
1174
- indexedFieldOps.releaseField(db, fieldName, tag);
1175
- }
1176
- }
1177
-
1178
1173
  return result;
1179
1174
  },
1180
1175
  },
@@ -1198,19 +1193,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1198
1193
  },
1199
1194
  execute: async (params) => {
1200
1195
  const tag = params.tag as string;
1201
- // Release any indexed fields this tag declared before the row
1202
- // drops. releaseField drops the generated column + index when the
1203
- // declarer set empties.
1204
- const schema = tagSchemaOps.getTagSchema(db, tag);
1205
- if (schema?.fields) {
1206
- for (const [fieldName, spec] of Object.entries(schema.fields)) {
1207
- if (spec.indexed === true) {
1208
- indexedFieldOps.releaseField(db, fieldName, tag);
1209
- }
1210
- }
1211
- }
1212
1196
  // Drop the row outright — description/fields/relationships/parents
1213
1197
  // travel with it. (No more sidecar table to clear separately.)
1198
+ // Indexed-field release is handled inside store.deleteTag →
1199
+ // noteOps.deleteTag so every entry point (MCP, REST, import sweep)
1200
+ // releases consistently with the co-declaration guard. See the
1201
+ // gitcoin orphaned-fields bug report.
1214
1202
  return await store.deleteTag(tag);
1215
1203
  },
1216
1204
  },
@@ -1266,6 +1254,41 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1266
1254
  },
1267
1255
  },
1268
1256
 
1257
+ // =====================================================================
1258
+ // 10. prune-schema — drop orphaned indexed-field columns
1259
+ // =====================================================================
1260
+ {
1261
+ name: "prune-schema",
1262
+ // `admin` — a destructive schema-maintenance op, same tier as
1263
+ // manage-token. Operator-only; hidden from read/write sessions.
1264
+ requiredVerb: "admin",
1265
+ description:
1266
+ "Drop orphaned indexed-field columns + indexes whose declaring tags no longer exist (the result of a deleted tag never releasing its fields). Dry-run by default — returns the drop plan without mutating. Pass `apply: true` to execute. A field co-declared by a still-live tag is never dropped; only the dead declarers are trimmed from its set. Generated columns are derived from notes.metadata JSON, so a drop loses only the index, never source data — declare the field again to rebuild it.",
1267
+ inputSchema: {
1268
+ type: "object",
1269
+ properties: {
1270
+ apply: {
1271
+ type: "boolean",
1272
+ description: "Execute the prune. Default false (dry-run — report what would be dropped without changing anything).",
1273
+ },
1274
+ },
1275
+ },
1276
+ execute: async (params) => {
1277
+ const apply = params.apply === true;
1278
+ const plan = await store.pruneIndexedFields({ dryRun: !apply });
1279
+ const dropped = plan.filter((p) => p.dropped);
1280
+ const trimmed = plan.filter((p) => !p.dropped);
1281
+ return {
1282
+ dry_run: !apply,
1283
+ fields_dropped: dropped.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
1284
+ fields_trimmed: trimmed.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
1285
+ summary: apply
1286
+ ? `pruned ${dropped.length} orphaned field(s); trimmed dead declarers on ${trimmed.length} co-declared field(s)`
1287
+ : `would prune ${dropped.length} orphaned field(s); would trim dead declarers on ${trimmed.length} co-declared field(s) — pass apply:true to execute`,
1288
+ };
1289
+ },
1290
+ },
1291
+
1269
1292
  ];
1270
1293
  }
1271
1294
 
package/core/src/notes.ts CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  type CursorPayload,
19
19
  type QueryHashInputs,
20
20
  } from "./cursor.js";
21
+ import { releaseField } from "./indexed-fields.js";
21
22
 
22
23
  let idCounter = 0;
23
24
 
@@ -943,12 +944,35 @@ export function listTags(db: Database): { name: string; count: number }[] {
943
944
  }
944
945
 
945
946
  export function deleteTag(db: Database, name: string): { deleted: boolean; notes_untagged: number } {
946
- const exists = db.prepare("SELECT 1 FROM tags WHERE name = ?").get(name);
947
- if (!exists) return { deleted: false, notes_untagged: 0 };
947
+ const row = db.prepare("SELECT fields FROM tags WHERE name = ?").get(name) as
948
+ | { fields: string | null }
949
+ | undefined;
950
+ if (!row) return { deleted: false, notes_untagged: 0 };
948
951
 
949
952
  const countRow = db.prepare("SELECT COUNT(*) as c FROM note_tags WHERE tag_name = ?").get(name) as { c: number };
950
953
  const notesUntagged = countRow.c;
951
954
 
955
+ // Release any indexed fields this tag declared BEFORE the row drops.
956
+ // `releaseField` only drops the generated column + index when this tag is
957
+ // the last live declarer (co-declaration guard) — a field co-declared by
958
+ // another live tag keeps its column and just loses this tag from the set.
959
+ // This lives in the store-level delete (not the MCP layer) so every caller
960
+ // — MCP delete-tag, the REST DELETE /tags/:name route, the import
961
+ // blow-away sweep — releases consistently. See the gitcoin orphaned-fields
962
+ // bug report.
963
+ if (row.fields) {
964
+ try {
965
+ const fields = JSON.parse(row.fields) as Record<string, { indexed?: boolean }>;
966
+ for (const [fieldName, spec] of Object.entries(fields)) {
967
+ if (spec?.indexed === true) {
968
+ releaseField(db, fieldName, name);
969
+ }
970
+ }
971
+ } catch {
972
+ // Malformed fields JSON — nothing to release; proceed with the delete.
973
+ }
974
+ }
975
+
952
976
  db.prepare("DELETE FROM note_tags WHERE tag_name = ?").run(name);
953
977
  db.prepare("DELETE FROM tags WHERE name = ?").run(name);
954
978
 
@@ -24,6 +24,8 @@ import { join } from "path";
24
24
  import { tmpdir } from "os";
25
25
 
26
26
  import { SqliteStore } from "./store.js";
27
+ import { getIndexedField } from "./indexed-fields.js";
28
+ import { buildVaultProjection } from "./vault-projection.js";
27
29
  import {
28
30
  CaseCollisionError,
29
31
  emitYamlDoc,
@@ -725,6 +727,56 @@ describe("importPortableVault", async () => {
725
727
  expect(schema!.description).toBe("A unit of work");
726
728
  });
727
729
 
730
+ // Fix 2 — import must re-declare indexed fields. The import writes
731
+ // tags.fields via upsertTagRecord but historically never materialized the
732
+ // backing generated columns + indexes, so a fresh import advertised
733
+ // `indexed: true` while queries silently full-scanned. Regression: export a
734
+ // vault with an indexed field → fresh import → the generated column + index
735
+ // exist and the field shows in the vault-info indexed_fields catalog.
736
+ it("re-declares indexed fields on import (column + index + vault-info catalog)", async () => {
737
+ await store.upsertTagRecord("project", {
738
+ fields: { status: { type: "string", indexed: true } },
739
+ });
740
+ await store.createNote("p", { id: "p", tags: ["project"], metadata: { status: "active" } });
741
+ const outDir = join(tmpBase, "out");
742
+ await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
743
+
744
+ const target = new SqliteStore(new Database(":memory:"));
745
+ const targetDb = target.db;
746
+ // Pre-condition: a fresh store has no backing column yet.
747
+ const colsBefore = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
748
+ expect(colsBefore).not.toContain("meta_status");
749
+
750
+ const stats = await importPortableVault(target, { inDir: outDir });
751
+ expect(stats.schemas_restored).toBe(1);
752
+ expect(stats.indexes_declared).toBe(1);
753
+
754
+ // The generated column + index now exist — same introspection vault-info
755
+ // uses to advertise the queryable-field catalog.
756
+ const colsAfter = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
757
+ expect(colsAfter).toContain("meta_status");
758
+ const idxs = (targetDb.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='notes'").all() as { name: string }[]).map((r) => r.name);
759
+ expect(idxs).toContain("idx_meta_status");
760
+ expect(getIndexedField(targetDb, "status")?.declarerTags).toEqual(["project"]);
761
+ // And vault-info lists it.
762
+ expect(buildVaultProjection(targetDb).indexed_fields.map((f) => f.name)).toContain("status");
763
+ });
764
+
765
+ it("dry-run import counts indexes_declared without materializing columns", async () => {
766
+ await store.upsertTagRecord("project", {
767
+ fields: { status: { type: "string", indexed: true } },
768
+ });
769
+ const outDir = join(tmpBase, "out");
770
+ await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
771
+
772
+ const target = new SqliteStore(new Database(":memory:"));
773
+ const stats = await importPortableVault(target, { inDir: outDir, dryRun: true });
774
+ expect(stats.indexes_declared).toBe(1);
775
+ // Dry-run touches nothing.
776
+ const cols = (target.db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
777
+ expect(cols).not.toContain("meta_status");
778
+ });
779
+
728
780
  it("restores typed links (non-wikilink relationships)", async () => {
729
781
  await store.createNote("src body", { id: "src", path: "src" });
730
782
  await store.createNote("tgt body", { id: "tgt", path: "tgt" });
@@ -1535,6 +1535,14 @@ export interface ImportStats {
1535
1535
  skipped_sidecars: Array<{ sidecar_id: string; expected_path: string | null; expected_extension: string | null; reason: string }>;
1536
1536
  /** Set when the caller passed `blowAway: true`; counts notes removed. */
1537
1537
  notes_wiped: number;
1538
+ /**
1539
+ * (tag, field) indexed-field declarations replayed after restoring tag
1540
+ * schemas — materializes the generated columns + indexes a live vault
1541
+ * would have. Without this an imported vault's schemas say `indexed: true`
1542
+ * but the backing columns don't exist until each tag is next `update-tag`'d
1543
+ * (queries fall back to full scans). See the import re-declare fix.
1544
+ */
1545
+ indexes_declared: number;
1538
1546
  }
1539
1547
 
1540
1548
  /**
@@ -1582,6 +1590,7 @@ export async function importPortableVault(
1582
1590
  skipped_attachments: [],
1583
1591
  skipped_sidecars: [],
1584
1592
  notes_wiped: 0,
1593
+ indexes_declared: 0,
1585
1594
  };
1586
1595
 
1587
1596
  // 1. Optional wipe. Notes are deleted via the public Store API so
@@ -2019,6 +2028,45 @@ export async function importPortableVault(
2019
2028
  await store.syncAllWikilinks();
2020
2029
  }
2021
2030
 
2031
+ // 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
2032
+ // Step 2 restored tag schemas via `store.upsertTagRecord`, which — now that
2033
+ // the indexed-field lifecycle is centralized in the store — already
2034
+ // materializes the backing generated columns + indexes as it persists each
2035
+ // schema. This explicit reconcile is therefore idempotent on the happy path;
2036
+ // it stays as a safety net (covers any schema written through a path that
2037
+ // skipped the lifecycle) and gives the authoritative `indexes_declared`
2038
+ // count. Without it, a regression in step 2 would silently leave the
2039
+ // imported schemas advertising `indexed: true` while queries full-scan.
2040
+ if (!opts.dryRun) {
2041
+ stats.indexes_declared = await store.reconcileDeclaredIndexes();
2042
+ } else {
2043
+ // Dry-run: count what WOULD be declared without touching the DB. Both
2044
+ // paths count per (tag, field) declaration (a co-declared field counts
2045
+ // once per declaring tag). The one asymmetry: this dry-run counts every
2046
+ // `indexed: true` field including unsupported types, whereas the applied
2047
+ // `reconcileDeclaredIndexes` skips fields whose type can't be indexed —
2048
+ // so the dry-run can over-count by the number of mis-typed indexed
2049
+ // fields. It's a "how much indexing work" signal, not a row-exact promise.
2050
+ const schemasDir2 = join(sidecar, "schemas");
2051
+ if (existsSync(schemasDir2)) {
2052
+ for (const entry of readdirSync(schemasDir2)) {
2053
+ if (!entry.endsWith(".yaml")) continue;
2054
+ const fullPath = join(schemasDir2, entry);
2055
+ const resolved = resolvePath(fullPath);
2056
+ if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
2057
+ const text = readFileSync(fullPath, "utf-8");
2058
+ const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
2059
+ const { frontmatter } = parseFrontmatter(wrapped);
2060
+ const fields = frontmatter.fields;
2061
+ if (fields && typeof fields === "object" && !Array.isArray(fields)) {
2062
+ for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
2063
+ if (spec?.indexed === true) stats.indexes_declared++;
2064
+ }
2065
+ }
2066
+ }
2067
+ }
2068
+ }
2069
+
2022
2070
  return stats;
2023
2071
  }
2024
2072
 
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
2
2
  import { normalizePath } from "./paths.js";
3
3
  import { rebuildIndexes } from "./indexed-fields.js";
4
4
 
5
- export const SCHEMA_VERSION = 19;
5
+ export const SCHEMA_VERSION = 20;
6
6
 
7
7
  export const SCHEMA_SQL = `
8
8
  -- Notes: the universal record.
@@ -96,6 +96,17 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
96
96
 
97
97
  -- Tokens: API authentication with OAuth-standard scopes.
98
98
  --
99
+ -- VESTIGIAL as of 0.5.0 (vault#282 Stage 2). Vault is a pure hub
100
+ -- resource-server: it no longer mints (pvt_*) or validates rows in this
101
+ -- table — auth runs through hub-issued JWTs + VAULT_AUTH_TOKEN + legacy YAML
102
+ -- api_keys only. The table is KEPT (not dropped) because migrateVaultKeys
103
+ -- raw-INSERTs legacy YAML api_keys here as its import landing zone, and
104
+ -- dropping it would trip an upgrade on a missing column for operators with
105
+ -- leftover rows. A future cosmetic migration may drop it alongside
106
+ -- oauth_clients/oauth_codes. 'tokens list' / 'tokens revoke' (CLI) still
107
+ -- read/delete here for cleanup of leftover rows. See the field docs below for
108
+ -- the historical (pre-0.5.0) semantics.
109
+ --
99
110
  -- scopes is a whitespace-separated list of granted scopes (OAuth 2.0 §3.3)
100
111
  -- — e.g. "vault:read vault:write". Introduced in v12 alongside enforcement;
101
112
  -- NULL rows are pre-v12 tokens which fall back to deriving scopes from the
@@ -128,10 +139,9 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
128
139
  -- minted this one, or the hub-JWT jti claim when minted from a hub
129
140
  -- session. Session-pinned list+revoke in manage-token filters on this.
130
141
  --
131
- -- revoked_at (v19) marks soft-revocation. Revoke from manage-token sets
132
- -- this rather than deleting the row, so the audit trail stays intact and
133
- -- the second revoke of the same jti is idempotent (returns ok=true).
134
- -- resolveToken treats a revoked_at-set row as not-found.
142
+ -- revoked_at (v19) marked soft-revocation of vault-DB tokens. Vestigial
143
+ -- post-0.5.0 (vault#282 Stage 2) the validation path that read it
144
+ -- (resolveToken) was removed alongside the pvt_* mint.
135
145
  CREATE TABLE IF NOT EXISTS tokens (
136
146
  token_hash TEXT PRIMARY KEY,
137
147
  label TEXT NOT NULL,
@@ -149,6 +159,30 @@ CREATE TABLE IF NOT EXISTS tokens (
149
159
  revoked_at TEXT
150
160
  );
151
161
 
162
+ -- mcp_mint_ledger (v20) — session-pinned record of HUB JWTs minted by the
163
+ -- manage-token MCP tool (vault#403, MGT). After the auth-unification arc,
164
+ -- manage-token mints hub JWTs (via hub mint-token attenuation proxy), not
165
+ -- pvt_* vault-DB tokens — so the mints no longer live in the tokens table.
166
+ -- This ledger is a lightweight local index of (parent_jti to minted hub jti)
167
+ -- so the tool list/revoke surface can stay session-scoped: a session sees
168
+ -- and revokes only the hub JWTs it minted. Rows are NOT credentials — the
169
+ -- signed JWT is never stored, only its jti (the revocation handle) plus
170
+ -- display metadata. revoked_at mirrors the tokens-table soft-revoke marker
171
+ -- so a second revoke is idempotent even if the hub round-trip is skipped.
172
+ -- The authoritative revocation state lives in hub token registry; this is
173
+ -- the attribution index only.
174
+ CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
175
+ jti TEXT PRIMARY KEY,
176
+ parent_jti TEXT NOT NULL,
177
+ vault_name TEXT NOT NULL,
178
+ label TEXT NOT NULL,
179
+ scopes TEXT,
180
+ scoped_tags TEXT,
181
+ created_at TEXT NOT NULL,
182
+ expires_at TEXT,
183
+ revoked_at TEXT
184
+ );
185
+
152
186
  -- OAuth: registered clients (Dynamic Client Registration)
153
187
  -- VESTIGIAL after vault 0.4.x workstream E (2026-05-25). The standalone
154
188
  -- OAuth issuer that wrote these rows was retired (hub is the issuer now;
@@ -214,6 +248,10 @@ CREATE INDEX IF NOT EXISTS idx_note_tags_tag ON note_tags(tag_name, note_id);
214
248
  CREATE INDEX IF NOT EXISTS idx_attachments_note ON attachments(note_id);
215
249
  CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_id);
216
250
  CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_id);
251
+ -- Session-pinned manage-token ledger lookup (v20): list/revoke scope on
252
+ -- (parent_jti, vault_name). The mcp_mint_ledger table is created
253
+ -- unconditionally above, so this index is safe in SCHEMA_SQL.
254
+ CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name);
217
255
  -- idx_tokens_vault_name is created in migrateToV16, not here. SCHEMA_SQL
218
256
  -- runs BEFORE migrations; an upgrading v15 vault doesn't yet have the
219
257
  -- vault_name column when this section evaluates, so the index has to
@@ -380,9 +418,10 @@ export function initSchema(db: Database): void {
380
418
  migrateToV15(db);
381
419
 
382
420
  // Migrate v15 → v16: add `vault_name` column to tokens. Existing rows
383
- // backfill to NULL ("server-wide / legacy" semantic) — auth accepts
384
- // NULL for any vault, so today's pvt_* tokens keep working unchanged.
385
- // New mints via per-vault routes write the column explicitly. See vault#257.
421
+ // backfilled to NULL ("server-wide / legacy" semantic) — at the time auth
422
+ // accepted NULL for any vault so pre-v16 pvt_* tokens kept working. (pvt_*
423
+ // validation was dropped at 0.5.0 / vault#282 Stage 2; the column is now
424
+ // vestigial.) See vault#257.
386
425
  migrateToV16(db);
387
426
 
388
427
  // Migrate v16 → v17: rip the standalone `note_schemas` + `schema_mappings`
@@ -403,6 +442,13 @@ export function initSchema(db: Database): void {
403
442
  // "non-MCP-minted, not revoked" — identical pre-v19 semantics.
404
443
  migrateToV19(db);
405
444
 
445
+ // Migrate v19 → v20: the `mcp_mint_ledger` table (session-pinned record of
446
+ // hub JWTs minted by the manage-token MCP tool). Created by SCHEMA_SQL's
447
+ // `CREATE TABLE IF NOT EXISTS` above, so this is a no-op confirmation hook
448
+ // — present for symmetry with the other migration steps and to anchor the
449
+ // version bump. See vault#403 (MGT — manage-token mints hub JWTs).
450
+ migrateToV20(db);
451
+
406
452
  // Rebuild any generated columns + indexes declared in indexed_fields.
407
453
  // No-op for a fresh vault; idempotent on existing vaults.
408
454
  rebuildIndexes(db);
@@ -803,12 +849,11 @@ function migrateToV15(db: Database): void {
803
849
  * Migrate v15 → v16: per-vault token storage (vault#257).
804
850
  *
805
851
  * Adds `tokens.vault_name TEXT` (nullable). Existing rows stay NULL —
806
- * "server-wide / legacy" semantic and `authenticateVaultRequest`
807
- * accepts NULL for any vault, so today's pvt_* tokens keep working
808
- * unchanged. New mints via `/vault/<name>/tokens` write the column
809
- * explicitly; cross-vault presentation rejects on the row's vault_name
810
- * mismatch. The complementary index speeds the per-vault listTokens
811
- * filter in the admin SPA.
852
+ * "server-wide / legacy" semantic. At the time, `authenticateVaultRequest`
853
+ * accepted NULL for any vault so pre-v16 pvt_* tokens kept working. (pvt_*
854
+ * validation + the `/vault/<name>/tokens` mint route were both removed at
855
+ * 0.5.0 / vault#282 Stage 2 the column + index are now vestigial; the
856
+ * index still speeds the per-vault `listTokens` cleanup listing.)
812
857
  *
813
858
  * Wrapped in BEGIN IMMEDIATE / COMMIT (with try/catch ROLLBACK) per the
814
859
  * v14/v15 wrap pattern from vault#251 — the column add and index create
@@ -1001,6 +1046,34 @@ function migrateToV19(db: Database): void {
1001
1046
  }
1002
1047
  }
1003
1048
 
1049
+ /**
1050
+ * Migrate v19 → v20: ensure the `mcp_mint_ledger` table exists. SCHEMA_SQL's
1051
+ * `CREATE TABLE IF NOT EXISTS` already covers fresh vaults AND upgrading
1052
+ * vaults (SCHEMA_SQL runs unconditionally on every open before the migration
1053
+ * steps), so this is a defensive no-op confirmation — there is no ALTER to
1054
+ * perform. Kept as a named step so the version-bump arc reads cleanly and a
1055
+ * future column addition has an obvious home. See vault#403 (MGT).
1056
+ */
1057
+ function migrateToV20(db: Database): void {
1058
+ if (hasTable(db, "mcp_mint_ledger")) return;
1059
+ db.exec(`
1060
+ CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
1061
+ jti TEXT PRIMARY KEY,
1062
+ parent_jti TEXT NOT NULL,
1063
+ vault_name TEXT NOT NULL,
1064
+ label TEXT NOT NULL,
1065
+ scopes TEXT,
1066
+ scoped_tags TEXT,
1067
+ created_at TEXT NOT NULL,
1068
+ expires_at TEXT,
1069
+ revoked_at TEXT
1070
+ )
1071
+ `);
1072
+ db.exec(
1073
+ "CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name)",
1074
+ );
1075
+ }
1076
+
1004
1077
  function hasTable(db: Database, name: string): boolean {
1005
1078
  const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
1006
1079
  return !!row;
package/core/src/store.ts CHANGED
@@ -4,6 +4,12 @@ import { initSchema } from "./schema.js";
4
4
  import * as noteOps from "./notes.js";
5
5
  import * as linkOps from "./links.js";
6
6
  import * as tagSchemaOps from "./tag-schemas.js";
7
+ import * as indexedFieldOps from "./indexed-fields.js";
8
+ import {
9
+ pruneOrphanedIndexedFields,
10
+ reconcileDeclaredIndexes,
11
+ type PrunedField,
12
+ } from "./indexed-fields.js";
7
13
  import { syncWikilinks, resolveUnresolvedWikilinks } from "./wikilinks.js";
8
14
  import { pathTitle } from "./paths.js";
9
15
  import { HookRegistry } from "./hooks.js";
@@ -503,6 +509,37 @@ export class BunSqliteStore implements Store {
503
509
  return tagSchemaOps.getTagSchemaMap(this.db);
504
510
  }
505
511
 
512
+ // ---- Indexed-field lifecycle (generated columns + indexes) ----
513
+
514
+ /**
515
+ * Prune orphaned `indexed_fields` declarers — declarer tags that no longer
516
+ * have a `tags` row. Fields with no surviving live declarer are dropped
517
+ * wholesale (row + generated column + index); co-declared fields keep their
518
+ * column and just lose the dead declarers. `dryRun` (default true) returns
519
+ * the plan without mutating. See the gitcoin orphaned-fields bug.
520
+ */
521
+ async pruneIndexedFields(opts?: { dryRun?: boolean }): Promise<PrunedField[]> {
522
+ const plan = pruneOrphanedIndexedFields(this.db, opts);
523
+ // A drop changes the queryable-field catalog vault-info advertises; bust
524
+ // the schema cache so the next read reflects it.
525
+ if (opts?.dryRun === false && plan.length > 0) this._schemaConfig = null;
526
+ return plan;
527
+ }
528
+
529
+ /**
530
+ * Replay `declareField` for every `indexed: true` field across all current
531
+ * tag records, materializing the generated columns + indexes. Idempotent —
532
+ * used by the portable-md import path so a fresh import ends with the same
533
+ * backing columns a live vault would have. Returns the count of (tag, field)
534
+ * declarations replayed.
535
+ */
536
+ async reconcileDeclaredIndexes(): Promise<number> {
537
+ const schemas = await this.listTagRecords();
538
+ const count = reconcileDeclaredIndexes(this.db, schemas);
539
+ if (count > 0) this._schemaConfig = null;
540
+ return count;
541
+ }
542
+
506
543
  // ---- Tag Records (post-v14: full identity row) ----
507
544
 
508
545
  async listTagRecords() {
@@ -517,6 +554,18 @@ export class BunSqliteStore implements Store {
517
554
  * Partial upsert of the full tag record. Any patch field left undefined
518
555
  * is preserved; pass null to clear. Invalidates the tag-hierarchy cache
519
556
  * when `parent_names` is touched.
557
+ *
558
+ * Indexed-field lifecycle is reconciled HERE — at the single store
559
+ * chokepoint every caller (MCP update-tag, REST PUT /tags/:name, import)
560
+ * funnels through — so no caller can persist a `fields` change without the
561
+ * matching declareField/releaseField. When `patch.fields` is touched
562
+ * (object or explicit `null`), the prior-vs-next indexed-field set is
563
+ * diffed: added indexed fields get `declareField`, removed ones get
564
+ * `releaseField` (which drops the generated column + index only when this
565
+ * tag is the last live declarer — the co-declaration guard). `patch.fields
566
+ * === undefined` (no-touch) skips reconciliation entirely. Centralizing
567
+ * here is the same discipline as moving delete-release into noteOps.deleteTag
568
+ * — it closes the REST PUT orphaned-column leak. See the gitcoin bug.
520
569
  */
521
570
  async upsertTagRecord(
522
571
  tag: string,
@@ -527,7 +576,43 @@ export class BunSqliteStore implements Store {
527
576
  parent_names?: string[] | null;
528
577
  },
529
578
  ) {
579
+ // Snapshot the prior indexed-field set BEFORE the write so the diff below
580
+ // sees what this tag declared going in. Only needed when `fields` changes.
581
+ const priorRecord =
582
+ patch.fields !== undefined ? tagSchemaOps.getTagRecord(this.db, tag) : null;
583
+
530
584
  const result = tagSchemaOps.upsertTagRecord(this.db, tag, patch);
585
+
586
+ if (patch.fields !== undefined) {
587
+ const indexedSet = (fields: Record<string, tagSchemaOps.TagFieldSchema> | null | undefined) =>
588
+ new Set(
589
+ Object.entries(fields ?? {})
590
+ .filter(([, v]) => v.indexed === true)
591
+ .map(([k]) => k),
592
+ );
593
+ const nextFields = patch.fields; // object | null
594
+ const priorIndexed = indexedSet(priorRecord?.fields);
595
+ const nextIndexed = indexedSet(nextFields);
596
+ for (const fieldName of nextIndexed) {
597
+ const spec = nextFields![fieldName]!;
598
+ const mapped = indexedFieldOps.mapFieldType(spec.type);
599
+ // Unmappable type for indexing is a caller error; surface it rather
600
+ // than silently skipping. MCP/REST validate up-front for a cleaner
601
+ // message, but this is the backstop at the chokepoint.
602
+ if (!mapped) {
603
+ throw new indexedFieldOps.IndexedFieldError(
604
+ `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
605
+ );
606
+ }
607
+ indexedFieldOps.declareField(this.db, fieldName, mapped, tag);
608
+ }
609
+ for (const fieldName of priorIndexed) {
610
+ if (!nextIndexed.has(fieldName)) {
611
+ indexedFieldOps.releaseField(this.db, fieldName, tag);
612
+ }
613
+ }
614
+ }
615
+
531
616
  if (patch.parent_names !== undefined) {
532
617
  // parent_names drives both query expansion (tag hierarchy) AND, post
533
618
  // vault#270, schema inheritance — bust both caches.
@@ -687,6 +772,38 @@ export class BunSqliteStore implements Store {
687
772
  };
688
773
  }
689
774
 
775
+ /**
776
+ * Reverse-lookup: every attachment row whose `path` column equals the given
777
+ * vault-internal relative path (`<date>/<filename>`). A single on-disk asset
778
+ * can be referenced by more than one attachment row (the orphan check in
779
+ * `deleteAttachment` accounts for that), so this returns an array. Used by
780
+ * the raw `/api/storage/<date>/<file>` byte-serve path to map a requested
781
+ * file back to its owning note(s) for tag-scope enforcement — without this,
782
+ * a tag-scoped token could fetch an out-of-scope note's attachment bytes
783
+ * directly by path (the path-secrecy-only bypass; see the C0 adversarial
784
+ * audit finding).
785
+ */
786
+ async getAttachmentsByPath(path: string): Promise<Attachment[]> {
787
+ const rows = this.db.prepare(
788
+ "SELECT * FROM attachments WHERE path = ? ORDER BY created_at",
789
+ ).all(path) as { id: string; note_id: string; path: string; mime_type: string; metadata: string | null; created_at: string }[];
790
+
791
+ return rows.map((r) => {
792
+ let metadata: Record<string, unknown> | undefined;
793
+ if (r.metadata && r.metadata !== "{}") {
794
+ try { metadata = JSON.parse(r.metadata); } catch {}
795
+ }
796
+ return {
797
+ id: r.id,
798
+ noteId: r.note_id,
799
+ path: r.path,
800
+ mimeType: r.mime_type,
801
+ metadata,
802
+ createdAt: r.created_at,
803
+ };
804
+ });
805
+ }
806
+
690
807
  /**
691
808
  * Replace the attachment's metadata JSON blob. The caller passes the full
692
809
  * merged object — this is a set, not a patch, so partial-field updates