@openparachute/vault 0.4.9-rc.10 → 0.4.9-rc.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/core/src/core.test.ts +4 -1
  2. package/core/src/indexed-fields.test.ts +151 -0
  3. package/core/src/indexed-fields.ts +98 -0
  4. package/core/src/mcp.ts +66 -43
  5. package/core/src/notes.ts +26 -2
  6. package/core/src/portable-md.test.ts +52 -0
  7. package/core/src/portable-md.ts +48 -0
  8. package/core/src/schema.ts +64 -1
  9. package/core/src/store.ts +117 -0
  10. package/core/src/types.ts +28 -0
  11. package/package.json +2 -2
  12. package/src/auth-hub-jwt.test.ts +262 -0
  13. package/src/auth.ts +137 -7
  14. package/src/cli.ts +131 -36
  15. package/src/config.ts +12 -4
  16. package/src/hub-jwt.test.ts +27 -2
  17. package/src/hub-jwt.ts +10 -0
  18. package/src/mcp-http.ts +24 -3
  19. package/src/mcp-install-interactive.test.ts +10 -21
  20. package/src/mcp-install-interactive.ts +12 -21
  21. package/src/mcp-install.test.ts +141 -30
  22. package/src/mcp-install.ts +109 -3
  23. package/src/mcp-tools.ts +246 -73
  24. package/src/mirror-config.test.ts +93 -0
  25. package/src/mirror-config.ts +264 -9
  26. package/src/mirror-credentials.test.ts +168 -17
  27. package/src/mirror-credentials.ts +155 -32
  28. package/src/mirror-deps.ts +25 -16
  29. package/src/mirror-import.test.ts +16 -16
  30. package/src/mirror-import.ts +6 -3
  31. package/src/mirror-manager.ts +43 -11
  32. package/src/mirror-per-vault.test.ts +519 -0
  33. package/src/mirror-registry.ts +91 -14
  34. package/src/mirror-routes.test.ts +21 -21
  35. package/src/mirror-routes.ts +28 -15
  36. package/src/routes.ts +38 -1
  37. package/src/routing.ts +37 -28
  38. package/src/server.ts +102 -34
  39. package/src/storage.test.ts +132 -7
  40. package/src/token-store.ts +142 -0
  41. package/src/vault.test.ts +393 -93
  42. package/web/ui/dist/assets/index-Cn-PPMRv.js +60 -0
  43. package/web/ui/dist/index.html +1 -1
  44. package/web/ui/dist/assets/index-DE18QJMx.js +0 -60
@@ -1944,6 +1944,9 @@ describe("MCP tools", async () => {
1944
1944
  expect(names).toContain("delete-tag");
1945
1945
  expect(names).toContain("find-path");
1946
1946
  expect(names).toContain("vault-info");
1947
+ // prune-schema (admin) — drops orphaned indexed-field columns whose
1948
+ // declaring tags are gone. The gitcoin orphaned-fields fix.
1949
+ expect(names).toContain("prune-schema");
1947
1950
  // Six note-schema tools (list/update/delete-note-schema +
1948
1951
  // list/set/delete-schema-mapping) retired in v17 — the standalone
1949
1952
  // note_schemas + schema_mappings subsystem was a parallel path to
@@ -1957,7 +1960,7 @@ describe("MCP tools", async () => {
1957
1960
  // synthesize-notes retired in v17 — replicable with query-notes(near=) +
1958
1961
  // find-path + agent-side aggregation. See vault#268.
1959
1962
  expect(names).not.toContain("synthesize-notes");
1960
- expect(tools).toHaveLength(9);
1963
+ expect(tools).toHaveLength(10);
1961
1964
  });
1962
1965
 
1963
1966
  it("create-note tool works", async () => {
@@ -7,11 +7,13 @@ import {
7
7
  declareField,
8
8
  getIndexedField,
9
9
  listIndexedFields,
10
+ pruneOrphanedIndexedFields,
10
11
  rebuildIndexes,
11
12
  releaseField,
12
13
  TYPE_MAP,
13
14
  validateFieldName,
14
15
  } from "./indexed-fields.js";
16
+ import { buildVaultProjection } from "./vault-projection.js";
15
17
 
16
18
  let db: Database;
17
19
  let store: SqliteStore;
@@ -282,4 +284,153 @@ describe("delete-tag: indexed fields", () => {
282
284
  expect(getIndexedField(db, "status")?.declarerTags).toEqual(["ticket"]);
283
285
  expect(notesColumns()).toContain("meta_status");
284
286
  });
287
+
288
+ // Bug 1b — the release lives in store.deleteTag now (not the MCP layer), so
289
+ // every delete entry point releases. Co-declaration sequencing: deleting the
290
+ // FIRST co-declarer keeps the column; deleting the SECOND drops it.
291
+ it("co-declaration: delete A keeps column (B holds), then delete B drops it", async () => {
292
+ const update = findTool("update-tag");
293
+ const del = findTool("delete-tag");
294
+ await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
295
+ await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
296
+
297
+ await del.execute({ tag: "asset" });
298
+ expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
299
+ expect(notesColumns()).toContain("meta_aspect_ratio");
300
+
301
+ await del.execute({ tag: "storyboard" });
302
+ expect(getIndexedField(db, "aspect_ratio")).toBeNull();
303
+ expect(notesColumns()).not.toContain("meta_aspect_ratio");
304
+ expect(notesIndexes()).not.toContain("idx_meta_aspect_ratio");
305
+ });
306
+ });
307
+
308
+ // ===========================================================================
309
+ // Bug 1a — update-tag {fields: null} clears all of this tag's field schemas,
310
+ // dropping the exclusively-declared columns + indexes. `null` (clear-all) must
311
+ // be distinguished from `undefined` (no change). The gitcoin orphaned-fields
312
+ // bug was that `?? {}` collapsed null to a no-op.
313
+ // ===========================================================================
314
+ describe("update-tag: fields null vs undefined", () => {
315
+ it("fields:null drops the tag's exclusively-declared columns + indexed_fields rows", async () => {
316
+ const update = findTool("update-tag");
317
+ await update.execute({
318
+ tag: "project",
319
+ fields: { status: { type: "string", indexed: true }, priority: { type: "integer", indexed: true } },
320
+ });
321
+ expect(notesColumns()).toContain("meta_status");
322
+ expect(notesColumns()).toContain("meta_priority");
323
+
324
+ await update.execute({ tag: "project", fields: null });
325
+
326
+ expect(getIndexedField(db, "status")).toBeNull();
327
+ expect(getIndexedField(db, "priority")).toBeNull();
328
+ expect(notesColumns()).not.toContain("meta_status");
329
+ expect(notesColumns()).not.toContain("meta_priority");
330
+ expect(notesIndexes()).not.toContain("idx_meta_status");
331
+ // The tag's fields column is cleared too.
332
+ const rec = await store.getTagRecord("project");
333
+ expect(rec?.fields ?? null).toBeNull();
334
+ });
335
+
336
+ it("fields:null no longer lists the fields in the vault-info indexed_fields catalog", async () => {
337
+ const update = findTool("update-tag");
338
+ await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
339
+ expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).toContain("status");
340
+
341
+ await update.execute({ tag: "project", fields: null });
342
+ expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).not.toContain("status");
343
+ });
344
+
345
+ it("fields:undefined is a no-op — preserves existing field schemas + columns", async () => {
346
+ const update = findTool("update-tag");
347
+ await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
348
+ // Update only the description; omit fields entirely.
349
+ await update.execute({ tag: "project", description: "a project" });
350
+ expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project"]);
351
+ expect(notesColumns()).toContain("meta_status");
352
+ });
353
+
354
+ it("fields:null respects co-declaration — keeps a field another live tag still declares", async () => {
355
+ const update = findTool("update-tag");
356
+ await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
357
+ await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
358
+
359
+ await update.execute({ tag: "asset", fields: null });
360
+ expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
361
+ expect(notesColumns()).toContain("meta_aspect_ratio");
362
+ });
363
+ });
364
+
365
+ // ===========================================================================
366
+ // Bug 1c — prune for ALREADY-orphaned fields. The gitcoin case: an
367
+ // indexed_fields row whose every declarer tag has no `tags` row (orphaned by a
368
+ // pre-fix delete/clear that never released). prune finds + drops them; it must
369
+ // NOT touch fields with a live declarer.
370
+ // ===========================================================================
371
+ describe("pruneOrphanedIndexedFields", () => {
372
+ // Seed an orphaned field directly: declare via the API (creates the tag
373
+ // row + column), then delete the tag row out from under it WITHOUT going
374
+ // through the release path — exactly the pre-fix orphaned state.
375
+ function orphanField(field: string, type: "TEXT" | "INTEGER", tag: string) {
376
+ declareField(db, field, type, tag);
377
+ // Drop the tags row directly — simulating the pre-fix delete-tag that
378
+ // never released. The indexed_fields row + column survive (orphaned).
379
+ db.prepare("DELETE FROM tags WHERE name = ?").run(tag);
380
+ }
381
+
382
+ it("dry-run reports the orphan without mutating", async () => {
383
+ orphanField("legacy_status", "TEXT", "ghost");
384
+ expect(notesColumns()).toContain("meta_legacy_status");
385
+
386
+ const plan = pruneOrphanedIndexedFields(db, { dryRun: true });
387
+ expect(plan).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
388
+ // Nothing changed.
389
+ expect(getIndexedField(db, "legacy_status")).not.toBeNull();
390
+ expect(notesColumns()).toContain("meta_legacy_status");
391
+ });
392
+
393
+ it("apply drops the orphaned column + index + row", async () => {
394
+ orphanField("legacy_status", "TEXT", "ghost");
395
+ const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
396
+ expect(plan).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
397
+ expect(getIndexedField(db, "legacy_status")).toBeNull();
398
+ expect(notesColumns()).not.toContain("meta_legacy_status");
399
+ expect(notesIndexes()).not.toContain("idx_meta_legacy_status");
400
+ // No longer advertised by vault-info.
401
+ expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).not.toContain("legacy_status");
402
+ });
403
+
404
+ it("does NOT touch a field with a live declarer", async () => {
405
+ const update = findTool("update-tag");
406
+ await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
407
+ const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
408
+ expect(plan).toEqual([]);
409
+ expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project"]);
410
+ expect(notesColumns()).toContain("meta_status");
411
+ });
412
+
413
+ it("trims dead declarers but keeps the column when a live co-declarer remains", async () => {
414
+ const update = findTool("update-tag");
415
+ // Two declarers, then orphan only one by deleting its tag row directly.
416
+ await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
417
+ await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
418
+ db.prepare("DELETE FROM tags WHERE name = ?").run("asset"); // orphan one declarer
419
+
420
+ const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
421
+ expect(plan).toEqual([{ field: "aspect_ratio", deadDeclarers: ["asset"], dropped: false }]);
422
+ // Column kept; storyboard still declares it; asset trimmed from the set.
423
+ expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
424
+ expect(notesColumns()).toContain("meta_aspect_ratio");
425
+ });
426
+
427
+ it("store.pruneIndexedFields surfaces the same plan", async () => {
428
+ orphanField("legacy_status", "TEXT", "ghost");
429
+ const dry = await store.pruneIndexedFields({ dryRun: true });
430
+ expect(dry).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
431
+ expect(getIndexedField(db, "legacy_status")).not.toBeNull(); // dry-run didn't mutate
432
+ const applied = await store.pruneIndexedFields({ dryRun: false });
433
+ expect(applied).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
434
+ expect(getIndexedField(db, "legacy_status")).toBeNull();
435
+ });
285
436
  });
@@ -236,3 +236,101 @@ export function rebuildIndexes(db: Database): void {
236
236
  }
237
237
  }
238
238
  }
239
+
240
+ export interface PrunedField {
241
+ /** The field whose `indexed_fields` row was affected. */
242
+ field: string;
243
+ /** Declarer tags that no longer have a `tags` row (removed from the set). */
244
+ deadDeclarers: string[];
245
+ /**
246
+ * True when the field had NO surviving live declarer and was fully dropped
247
+ * (row + generated column + index). False when at least one live declarer
248
+ * remained and only the dead declarers were pruned from the set.
249
+ */
250
+ dropped: boolean;
251
+ }
252
+
253
+ /**
254
+ * Prune orphaned `indexed_fields` declarers — the gitcoin defect.
255
+ *
256
+ * A declarer tag is "dead" when no `tags` row carries that name (the tag was
257
+ * deleted, or its schema was cleared, without releasing the field). For every
258
+ * `indexed_fields` row:
259
+ *
260
+ * - Drop dead declarers from the set.
261
+ * - If NO live declarer remains, drop the whole field — row + generated
262
+ * column + index. This is the only data-loss-free drop: the generated
263
+ * column is `json_extract(metadata, …)` so the source values stay in
264
+ * `notes.metadata`; only the (now-dead) index is lost.
265
+ * - If at least one live declarer remains (co-declaration), keep the column
266
+ * and just trim the dead names from the declarer set.
267
+ *
268
+ * `dryRun` (default) computes the plan without mutating; pass `dryRun: false`
269
+ * to apply. Returns the per-field plan either way so the CLI / MCP surface can
270
+ * print what it would (or did) drop.
271
+ */
272
+ export function pruneOrphanedIndexedFields(
273
+ db: Database,
274
+ opts: { dryRun?: boolean } = {},
275
+ ): PrunedField[] {
276
+ const dryRun = opts.dryRun ?? true;
277
+ const liveTags = new Set(
278
+ (db.prepare("SELECT name FROM tags").all() as { name: string }[]).map((r) => r.name),
279
+ );
280
+ const plan: PrunedField[] = [];
281
+ for (const f of listIndexedFields(db)) {
282
+ const deadDeclarers = f.declarerTags.filter((t) => !liveTags.has(t));
283
+ if (deadDeclarers.length === 0) continue; // every declarer is live — leave it
284
+ const liveDeclarers = f.declarerTags.filter((t) => liveTags.has(t));
285
+ const dropped = liveDeclarers.length === 0;
286
+ plan.push({ field: f.field, deadDeclarers, dropped });
287
+ if (dryRun) continue;
288
+ if (dropped) {
289
+ db.prepare("DELETE FROM indexed_fields WHERE field = ?").run(f.field);
290
+ dropColumnAndIndex(db, f.field);
291
+ } else {
292
+ db.prepare("UPDATE indexed_fields SET declarer_tags = ? WHERE field = ?").run(
293
+ JSON.stringify(liveDeclarers),
294
+ f.field,
295
+ );
296
+ }
297
+ }
298
+ return plan;
299
+ }
300
+
301
+ /**
302
+ * Replay `declareField` for every field a tag schema marks `indexed: true`.
303
+ * Idempotent — used by the portable-md import path so a fresh import ends with
304
+ * the same generated columns a live vault would have (the import writes
305
+ * `tags.fields` via `upsertTagRecord` but never materializes the backing
306
+ * columns). Without this, an imported vault's schemas say `indexed: true` but
307
+ * queries fall back to full scans until each tag is next `update-tag`'d.
308
+ *
309
+ * `tagSchemas` is the post-import set of (tag, fields) pairs. Returns the
310
+ * number of (tag, field) declarations replayed.
311
+ */
312
+ export function reconcileDeclaredIndexes(
313
+ db: Database,
314
+ tagSchemas: { tag: string; fields?: Record<string, { type: string; indexed?: boolean }> }[],
315
+ ): number {
316
+ let declared = 0;
317
+ for (const schema of tagSchemas) {
318
+ if (!schema.fields) continue;
319
+ for (const [fieldName, spec] of Object.entries(schema.fields)) {
320
+ if (spec.indexed !== true) continue;
321
+ const mapped = mapFieldType(spec.type);
322
+ if (!mapped) continue; // unsupported type for indexing — skip, don't throw
323
+ try {
324
+ validateFieldName(fieldName);
325
+ declareField(db, fieldName, mapped, schema.tag);
326
+ declared++;
327
+ } catch (err) {
328
+ console.error(
329
+ `[indexed-fields] could not re-declare "${fieldName}" for tag "${schema.tag}" on import:`,
330
+ err,
331
+ );
332
+ }
333
+ }
334
+ }
335
+ return declared;
336
+ }
package/core/src/mcp.ts CHANGED
@@ -23,8 +23,8 @@ export interface McpToolDef {
23
23
  /**
24
24
  * Minimum scope verb the caller must hold for THIS vault to see + invoke
25
25
  * the tool. `read` for pure queries, `write` for mutations, `admin` for
26
- * token-management surfaces (only `manage-token` in the current set —
27
- * core's nine tools cap at `write`). The MCP HTTP layer filters
26
+ * operator-only surfaces (`prune-schema` in core; `manage-token` in the
27
+ * server layer). The MCP HTTP layer filters
28
28
  * `tools/list` by this field and verb-gates `tools/call` against it; the
29
29
  * filter is the primary defense, the inner gate is defense-in-depth.
30
30
  *
@@ -100,9 +100,9 @@ function removeWikilinkBrackets(content: string, targetPath: string): string {
100
100
  // ---------------------------------------------------------------------------
101
101
 
102
102
  /**
103
- * Generate the consolidated MCP tools for a vault. Post-v17 surface (9):
103
+ * Generate the consolidated MCP tools for a vault. Surface (10):
104
104
  * query-notes, create-note, update-note, delete-note, list-tags, update-tag,
105
- * delete-tag, find-path, vault-info.
105
+ * delete-tag, find-path, vault-info, prune-schema (admin).
106
106
  */
107
107
  export function generateMcpTools(store: Store): McpToolDef[] {
108
108
  const db: Database = (store as any).db;
@@ -1074,12 +1074,23 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1074
1074
  const tag = params.tag as string;
1075
1075
  const existing = tagSchemaOps.getTagRecord(db, tag);
1076
1076
 
1077
- // ---- fields: shallow-merge into existing (preserves prior keys).
1078
- const incomingFields = (params.fields as Record<string, TagFieldSchema> | undefined) ?? {};
1079
- const mergedFields: Record<string, TagFieldSchema> = {
1080
- ...(existing?.fields ?? {}),
1081
- ...incomingFields,
1082
- };
1077
+ // ---- fields: three-way semantics, distinguishing `null` from
1078
+ // `undefined` (do NOT collapse with `?? {}` that silently turns an
1079
+ // explicit clear-all into a no-op, the gitcoin orphaned-fields bug).
1080
+ // - undefined → no change. Preserve every existing field; declare
1081
+ // nothing new. mergedFields === existing.fields.
1082
+ // - null → clear ALL of this tag's field schemas.
1083
+ // mergedFields = {} so the diff below releases every
1084
+ // indexed field this tag exclusively declares.
1085
+ // - object → shallow-merge into existing (preserves prior keys).
1086
+ const incomingFields =
1087
+ params.fields === null || params.fields === undefined
1088
+ ? {}
1089
+ : (params.fields as Record<string, TagFieldSchema>);
1090
+ const mergedFields: Record<string, TagFieldSchema> =
1091
+ params.fields === null
1092
+ ? {}
1093
+ : { ...(existing?.fields ?? {}), ...incomingFields };
1083
1094
 
1084
1095
  // Validate cross-tag consistency on fields being (re)declared in this
1085
1096
  // call. `type` and `indexed` are global — all declarers must agree.
@@ -1146,6 +1157,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1146
1157
  : (params.fields !== undefined ? null : undefined);
1147
1158
  const descriptionPatch =
1148
1159
  params.description === undefined ? undefined : (params.description as string);
1160
+ // The indexed-field lifecycle (declareField for added indexed fields,
1161
+ // releaseField for removed ones, with the co-declaration guard) is
1162
+ // reconciled inside store.upsertTagRecord — the single chokepoint all
1163
+ // callers (MCP, REST PUT /tags/:name, import) share — so it can't be
1164
+ // bypassed. The cross-tag validation above stays here to surface a
1165
+ // clean error before persisting. See the gitcoin orphaned-fields bug.
1149
1166
  const result = await store.upsertTagRecord(tag, {
1150
1167
  ...(descriptionPatch !== undefined ? { description: descriptionPatch } : {}),
1151
1168
  ...(fieldsPatch !== undefined ? { fields: fieldsPatch } : {}),
@@ -1153,28 +1170,6 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1153
1170
  ...(parentNamesPatch !== undefined ? { parent_names: parentNamesPatch } : {}),
1154
1171
  });
1155
1172
 
1156
- // ---- Reconcile indexed-field lifecycle for this tag.
1157
- const priorIndexed = new Set(
1158
- Object.entries(existing?.fields ?? {})
1159
- .filter(([, v]) => v.indexed === true)
1160
- .map(([k]) => k),
1161
- );
1162
- const nextIndexed = new Set(
1163
- Object.entries(mergedFields)
1164
- .filter(([, v]) => v.indexed === true)
1165
- .map(([k]) => k),
1166
- );
1167
- for (const fieldName of nextIndexed) {
1168
- const spec = mergedFields[fieldName]!;
1169
- const mapped = indexedFieldOps.mapFieldType(spec.type)!;
1170
- indexedFieldOps.declareField(db, fieldName, mapped, tag);
1171
- }
1172
- for (const fieldName of priorIndexed) {
1173
- if (!nextIndexed.has(fieldName)) {
1174
- indexedFieldOps.releaseField(db, fieldName, tag);
1175
- }
1176
- }
1177
-
1178
1173
  return result;
1179
1174
  },
1180
1175
  },
@@ -1198,19 +1193,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1198
1193
  },
1199
1194
  execute: async (params) => {
1200
1195
  const tag = params.tag as string;
1201
- // Release any indexed fields this tag declared before the row
1202
- // drops. releaseField drops the generated column + index when the
1203
- // declarer set empties.
1204
- const schema = tagSchemaOps.getTagSchema(db, tag);
1205
- if (schema?.fields) {
1206
- for (const [fieldName, spec] of Object.entries(schema.fields)) {
1207
- if (spec.indexed === true) {
1208
- indexedFieldOps.releaseField(db, fieldName, tag);
1209
- }
1210
- }
1211
- }
1212
1196
  // Drop the row outright — description/fields/relationships/parents
1213
1197
  // travel with it. (No more sidecar table to clear separately.)
1198
+ // Indexed-field release is handled inside store.deleteTag →
1199
+ // noteOps.deleteTag so every entry point (MCP, REST, import sweep)
1200
+ // releases consistently with the co-declaration guard. See the
1201
+ // gitcoin orphaned-fields bug report.
1214
1202
  return await store.deleteTag(tag);
1215
1203
  },
1216
1204
  },
@@ -1266,6 +1254,41 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
1266
1254
  },
1267
1255
  },
1268
1256
 
1257
+ // =====================================================================
1258
+ // 10. prune-schema — drop orphaned indexed-field columns
1259
+ // =====================================================================
1260
+ {
1261
+ name: "prune-schema",
1262
+ // `admin` — a destructive schema-maintenance op, same tier as
1263
+ // manage-token. Operator-only; hidden from read/write sessions.
1264
+ requiredVerb: "admin",
1265
+ description:
1266
+ "Drop orphaned indexed-field columns + indexes whose declaring tags no longer exist (the result of a deleted tag never releasing its fields). Dry-run by default — returns the drop plan without mutating. Pass `apply: true` to execute. A field co-declared by a still-live tag is never dropped; only the dead declarers are trimmed from its set. Generated columns are derived from notes.metadata JSON, so a drop loses only the index, never source data — declare the field again to rebuild it.",
1267
+ inputSchema: {
1268
+ type: "object",
1269
+ properties: {
1270
+ apply: {
1271
+ type: "boolean",
1272
+ description: "Execute the prune. Default false (dry-run — report what would be dropped without changing anything).",
1273
+ },
1274
+ },
1275
+ },
1276
+ execute: async (params) => {
1277
+ const apply = params.apply === true;
1278
+ const plan = await store.pruneIndexedFields({ dryRun: !apply });
1279
+ const dropped = plan.filter((p) => p.dropped);
1280
+ const trimmed = plan.filter((p) => !p.dropped);
1281
+ return {
1282
+ dry_run: !apply,
1283
+ fields_dropped: dropped.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
1284
+ fields_trimmed: trimmed.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
1285
+ summary: apply
1286
+ ? `pruned ${dropped.length} orphaned field(s); trimmed dead declarers on ${trimmed.length} co-declared field(s)`
1287
+ : `would prune ${dropped.length} orphaned field(s); would trim dead declarers on ${trimmed.length} co-declared field(s) — pass apply:true to execute`,
1288
+ };
1289
+ },
1290
+ },
1291
+
1269
1292
  ];
1270
1293
  }
1271
1294
 
package/core/src/notes.ts CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  type CursorPayload,
19
19
  type QueryHashInputs,
20
20
  } from "./cursor.js";
21
+ import { releaseField } from "./indexed-fields.js";
21
22
 
22
23
  let idCounter = 0;
23
24
 
@@ -943,12 +944,35 @@ export function listTags(db: Database): { name: string; count: number }[] {
943
944
  }
944
945
 
945
946
  export function deleteTag(db: Database, name: string): { deleted: boolean; notes_untagged: number } {
946
- const exists = db.prepare("SELECT 1 FROM tags WHERE name = ?").get(name);
947
- if (!exists) return { deleted: false, notes_untagged: 0 };
947
+ const row = db.prepare("SELECT fields FROM tags WHERE name = ?").get(name) as
948
+ | { fields: string | null }
949
+ | undefined;
950
+ if (!row) return { deleted: false, notes_untagged: 0 };
948
951
 
949
952
  const countRow = db.prepare("SELECT COUNT(*) as c FROM note_tags WHERE tag_name = ?").get(name) as { c: number };
950
953
  const notesUntagged = countRow.c;
951
954
 
955
+ // Release any indexed fields this tag declared BEFORE the row drops.
956
+ // `releaseField` only drops the generated column + index when this tag is
957
+ // the last live declarer (co-declaration guard) — a field co-declared by
958
+ // another live tag keeps its column and just loses this tag from the set.
959
+ // This lives in the store-level delete (not the MCP layer) so every caller
960
+ // — MCP delete-tag, the REST DELETE /tags/:name route, the import
961
+ // blow-away sweep — releases consistently. See the gitcoin orphaned-fields
962
+ // bug report.
963
+ if (row.fields) {
964
+ try {
965
+ const fields = JSON.parse(row.fields) as Record<string, { indexed?: boolean }>;
966
+ for (const [fieldName, spec] of Object.entries(fields)) {
967
+ if (spec?.indexed === true) {
968
+ releaseField(db, fieldName, name);
969
+ }
970
+ }
971
+ } catch {
972
+ // Malformed fields JSON — nothing to release; proceed with the delete.
973
+ }
974
+ }
975
+
952
976
  db.prepare("DELETE FROM note_tags WHERE tag_name = ?").run(name);
953
977
  db.prepare("DELETE FROM tags WHERE name = ?").run(name);
954
978
 
@@ -24,6 +24,8 @@ import { join } from "path";
24
24
  import { tmpdir } from "os";
25
25
 
26
26
  import { SqliteStore } from "./store.js";
27
+ import { getIndexedField } from "./indexed-fields.js";
28
+ import { buildVaultProjection } from "./vault-projection.js";
27
29
  import {
28
30
  CaseCollisionError,
29
31
  emitYamlDoc,
@@ -725,6 +727,56 @@ describe("importPortableVault", async () => {
725
727
  expect(schema!.description).toBe("A unit of work");
726
728
  });
727
729
 
730
+ // Fix 2 — import must re-declare indexed fields. The import writes
731
+ // tags.fields via upsertTagRecord but historically never materialized the
732
+ // backing generated columns + indexes, so a fresh import advertised
733
+ // `indexed: true` while queries silently full-scanned. Regression: export a
734
+ // vault with an indexed field → fresh import → the generated column + index
735
+ // exist and the field shows in the vault-info indexed_fields catalog.
736
+ it("re-declares indexed fields on import (column + index + vault-info catalog)", async () => {
737
+ await store.upsertTagRecord("project", {
738
+ fields: { status: { type: "string", indexed: true } },
739
+ });
740
+ await store.createNote("p", { id: "p", tags: ["project"], metadata: { status: "active" } });
741
+ const outDir = join(tmpBase, "out");
742
+ await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
743
+
744
+ const target = new SqliteStore(new Database(":memory:"));
745
+ const targetDb = target.db;
746
+ // Pre-condition: a fresh store has no backing column yet.
747
+ const colsBefore = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
748
+ expect(colsBefore).not.toContain("meta_status");
749
+
750
+ const stats = await importPortableVault(target, { inDir: outDir });
751
+ expect(stats.schemas_restored).toBe(1);
752
+ expect(stats.indexes_declared).toBe(1);
753
+
754
+ // The generated column + index now exist — same introspection vault-info
755
+ // uses to advertise the queryable-field catalog.
756
+ const colsAfter = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
757
+ expect(colsAfter).toContain("meta_status");
758
+ const idxs = (targetDb.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='notes'").all() as { name: string }[]).map((r) => r.name);
759
+ expect(idxs).toContain("idx_meta_status");
760
+ expect(getIndexedField(targetDb, "status")?.declarerTags).toEqual(["project"]);
761
+ // And vault-info lists it.
762
+ expect(buildVaultProjection(targetDb).indexed_fields.map((f) => f.name)).toContain("status");
763
+ });
764
+
765
+ it("dry-run import counts indexes_declared without materializing columns", async () => {
766
+ await store.upsertTagRecord("project", {
767
+ fields: { status: { type: "string", indexed: true } },
768
+ });
769
+ const outDir = join(tmpBase, "out");
770
+ await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
771
+
772
+ const target = new SqliteStore(new Database(":memory:"));
773
+ const stats = await importPortableVault(target, { inDir: outDir, dryRun: true });
774
+ expect(stats.indexes_declared).toBe(1);
775
+ // Dry-run touches nothing.
776
+ const cols = (target.db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
777
+ expect(cols).not.toContain("meta_status");
778
+ });
779
+
728
780
  it("restores typed links (non-wikilink relationships)", async () => {
729
781
  await store.createNote("src body", { id: "src", path: "src" });
730
782
  await store.createNote("tgt body", { id: "tgt", path: "tgt" });
@@ -1535,6 +1535,14 @@ export interface ImportStats {
1535
1535
  skipped_sidecars: Array<{ sidecar_id: string; expected_path: string | null; expected_extension: string | null; reason: string }>;
1536
1536
  /** Set when the caller passed `blowAway: true`; counts notes removed. */
1537
1537
  notes_wiped: number;
1538
+ /**
1539
+ * (tag, field) indexed-field declarations replayed after restoring tag
1540
+ * schemas — materializes the generated columns + indexes a live vault
1541
+ * would have. Without this an imported vault's schemas say `indexed: true`
1542
+ * but the backing columns don't exist until each tag is next `update-tag`'d
1543
+ * (queries fall back to full scans). See the import re-declare fix.
1544
+ */
1545
+ indexes_declared: number;
1538
1546
  }
1539
1547
 
1540
1548
  /**
@@ -1582,6 +1590,7 @@ export async function importPortableVault(
1582
1590
  skipped_attachments: [],
1583
1591
  skipped_sidecars: [],
1584
1592
  notes_wiped: 0,
1593
+ indexes_declared: 0,
1585
1594
  };
1586
1595
 
1587
1596
  // 1. Optional wipe. Notes are deleted via the public Store API so
@@ -2019,6 +2028,45 @@ export async function importPortableVault(
2019
2028
  await store.syncAllWikilinks();
2020
2029
  }
2021
2030
 
2031
+ // 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
2032
+ // Step 2 restored tag schemas via `store.upsertTagRecord`, which — now that
2033
+ // the indexed-field lifecycle is centralized in the store — already
2034
+ // materializes the backing generated columns + indexes as it persists each
2035
+ // schema. This explicit reconcile is therefore idempotent on the happy path;
2036
+ // it stays as a safety net (covers any schema written through a path that
2037
+ // skipped the lifecycle) and gives the authoritative `indexes_declared`
2038
+ // count. Without it, a regression in step 2 would silently leave the
2039
+ // imported schemas advertising `indexed: true` while queries full-scan.
2040
+ if (!opts.dryRun) {
2041
+ stats.indexes_declared = await store.reconcileDeclaredIndexes();
2042
+ } else {
2043
+ // Dry-run: count what WOULD be declared without touching the DB. Both
2044
+ // paths count per (tag, field) declaration (a co-declared field counts
2045
+ // once per declaring tag). The one asymmetry: this dry-run counts every
2046
+ // `indexed: true` field including unsupported types, whereas the applied
2047
+ // `reconcileDeclaredIndexes` skips fields whose type can't be indexed —
2048
+ // so the dry-run can over-count by the number of mis-typed indexed
2049
+ // fields. It's a "how much indexing work" signal, not a row-exact promise.
2050
+ const schemasDir2 = join(sidecar, "schemas");
2051
+ if (existsSync(schemasDir2)) {
2052
+ for (const entry of readdirSync(schemasDir2)) {
2053
+ if (!entry.endsWith(".yaml")) continue;
2054
+ const fullPath = join(schemasDir2, entry);
2055
+ const resolved = resolvePath(fullPath);
2056
+ if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
2057
+ const text = readFileSync(fullPath, "utf-8");
2058
+ const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
2059
+ const { frontmatter } = parseFrontmatter(wrapped);
2060
+ const fields = frontmatter.fields;
2061
+ if (fields && typeof fields === "object" && !Array.isArray(fields)) {
2062
+ for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
2063
+ if (spec?.indexed === true) stats.indexes_declared++;
2064
+ }
2065
+ }
2066
+ }
2067
+ }
2068
+ }
2069
+
2022
2070
  return stats;
2023
2071
  }
2024
2072