@openparachute/vault 0.4.9-rc.10 → 0.4.9-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/core.test.ts +4 -1
- package/core/src/indexed-fields.test.ts +151 -0
- package/core/src/indexed-fields.ts +98 -0
- package/core/src/mcp.ts +66 -43
- package/core/src/notes.ts +26 -2
- package/core/src/portable-md.test.ts +52 -0
- package/core/src/portable-md.ts +48 -0
- package/core/src/schema.ts +64 -1
- package/core/src/store.ts +117 -0
- package/core/src/types.ts +28 -0
- package/package.json +2 -2
- package/src/auth-hub-jwt.test.ts +262 -0
- package/src/auth.ts +137 -7
- package/src/cli.ts +131 -36
- package/src/config.ts +12 -4
- package/src/hub-jwt.test.ts +27 -2
- package/src/hub-jwt.ts +10 -0
- package/src/mcp-http.ts +24 -3
- package/src/mcp-install-interactive.test.ts +10 -21
- package/src/mcp-install-interactive.ts +12 -21
- package/src/mcp-install.test.ts +141 -30
- package/src/mcp-install.ts +109 -3
- package/src/mcp-tools.ts +246 -73
- package/src/mirror-config.test.ts +93 -0
- package/src/mirror-config.ts +264 -9
- package/src/mirror-credentials.test.ts +168 -17
- package/src/mirror-credentials.ts +155 -32
- package/src/mirror-deps.ts +25 -16
- package/src/mirror-import.test.ts +16 -16
- package/src/mirror-import.ts +6 -3
- package/src/mirror-manager.ts +43 -11
- package/src/mirror-per-vault.test.ts +519 -0
- package/src/mirror-registry.ts +91 -14
- package/src/mirror-routes.test.ts +21 -21
- package/src/mirror-routes.ts +28 -15
- package/src/routes.ts +38 -1
- package/src/routing.ts +37 -28
- package/src/server.ts +102 -34
- package/src/storage.test.ts +132 -7
- package/src/token-store.ts +142 -0
- package/src/vault.test.ts +393 -93
- package/web/ui/dist/assets/index-Cn-PPMRv.js +60 -0
- package/web/ui/dist/index.html +1 -1
- package/web/ui/dist/assets/index-DE18QJMx.js +0 -60
package/core/src/core.test.ts
CHANGED
|
@@ -1944,6 +1944,9 @@ describe("MCP tools", async () => {
|
|
|
1944
1944
|
expect(names).toContain("delete-tag");
|
|
1945
1945
|
expect(names).toContain("find-path");
|
|
1946
1946
|
expect(names).toContain("vault-info");
|
|
1947
|
+
// prune-schema (admin) — drops orphaned indexed-field columns whose
|
|
1948
|
+
// declaring tags are gone. The gitcoin orphaned-fields fix.
|
|
1949
|
+
expect(names).toContain("prune-schema");
|
|
1947
1950
|
// Six note-schema tools (list/update/delete-note-schema +
|
|
1948
1951
|
// list/set/delete-schema-mapping) retired in v17 — the standalone
|
|
1949
1952
|
// note_schemas + schema_mappings subsystem was a parallel path to
|
|
@@ -1957,7 +1960,7 @@ describe("MCP tools", async () => {
|
|
|
1957
1960
|
// synthesize-notes retired in v17 — replicable with query-notes(near=) +
|
|
1958
1961
|
// find-path + agent-side aggregation. See vault#268.
|
|
1959
1962
|
expect(names).not.toContain("synthesize-notes");
|
|
1960
|
-
expect(tools).toHaveLength(
|
|
1963
|
+
expect(tools).toHaveLength(10);
|
|
1961
1964
|
});
|
|
1962
1965
|
|
|
1963
1966
|
it("create-note tool works", async () => {
|
|
@@ -7,11 +7,13 @@ import {
|
|
|
7
7
|
declareField,
|
|
8
8
|
getIndexedField,
|
|
9
9
|
listIndexedFields,
|
|
10
|
+
pruneOrphanedIndexedFields,
|
|
10
11
|
rebuildIndexes,
|
|
11
12
|
releaseField,
|
|
12
13
|
TYPE_MAP,
|
|
13
14
|
validateFieldName,
|
|
14
15
|
} from "./indexed-fields.js";
|
|
16
|
+
import { buildVaultProjection } from "./vault-projection.js";
|
|
15
17
|
|
|
16
18
|
let db: Database;
|
|
17
19
|
let store: SqliteStore;
|
|
@@ -282,4 +284,153 @@ describe("delete-tag: indexed fields", () => {
|
|
|
282
284
|
expect(getIndexedField(db, "status")?.declarerTags).toEqual(["ticket"]);
|
|
283
285
|
expect(notesColumns()).toContain("meta_status");
|
|
284
286
|
});
|
|
287
|
+
|
|
288
|
+
// Bug 1b — the release lives in store.deleteTag now (not the MCP layer), so
|
|
289
|
+
// every delete entry point releases. Co-declaration sequencing: deleting the
|
|
290
|
+
// FIRST co-declarer keeps the column; deleting the SECOND drops it.
|
|
291
|
+
it("co-declaration: delete A keeps column (B holds), then delete B drops it", async () => {
|
|
292
|
+
const update = findTool("update-tag");
|
|
293
|
+
const del = findTool("delete-tag");
|
|
294
|
+
await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
295
|
+
await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
296
|
+
|
|
297
|
+
await del.execute({ tag: "asset" });
|
|
298
|
+
expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
|
|
299
|
+
expect(notesColumns()).toContain("meta_aspect_ratio");
|
|
300
|
+
|
|
301
|
+
await del.execute({ tag: "storyboard" });
|
|
302
|
+
expect(getIndexedField(db, "aspect_ratio")).toBeNull();
|
|
303
|
+
expect(notesColumns()).not.toContain("meta_aspect_ratio");
|
|
304
|
+
expect(notesIndexes()).not.toContain("idx_meta_aspect_ratio");
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
// ===========================================================================
|
|
309
|
+
// Bug 1a — update-tag {fields: null} clears all of this tag's field schemas,
|
|
310
|
+
// dropping the exclusively-declared columns + indexes. `null` (clear-all) must
|
|
311
|
+
// be distinguished from `undefined` (no change). The gitcoin orphaned-fields
|
|
312
|
+
// bug was that `?? {}` collapsed null to a no-op.
|
|
313
|
+
// ===========================================================================
|
|
314
|
+
describe("update-tag: fields null vs undefined", () => {
|
|
315
|
+
it("fields:null drops the tag's exclusively-declared columns + indexed_fields rows", async () => {
|
|
316
|
+
const update = findTool("update-tag");
|
|
317
|
+
await update.execute({
|
|
318
|
+
tag: "project",
|
|
319
|
+
fields: { status: { type: "string", indexed: true }, priority: { type: "integer", indexed: true } },
|
|
320
|
+
});
|
|
321
|
+
expect(notesColumns()).toContain("meta_status");
|
|
322
|
+
expect(notesColumns()).toContain("meta_priority");
|
|
323
|
+
|
|
324
|
+
await update.execute({ tag: "project", fields: null });
|
|
325
|
+
|
|
326
|
+
expect(getIndexedField(db, "status")).toBeNull();
|
|
327
|
+
expect(getIndexedField(db, "priority")).toBeNull();
|
|
328
|
+
expect(notesColumns()).not.toContain("meta_status");
|
|
329
|
+
expect(notesColumns()).not.toContain("meta_priority");
|
|
330
|
+
expect(notesIndexes()).not.toContain("idx_meta_status");
|
|
331
|
+
// The tag's fields column is cleared too.
|
|
332
|
+
const rec = await store.getTagRecord("project");
|
|
333
|
+
expect(rec?.fields ?? null).toBeNull();
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
it("fields:null no longer lists the fields in the vault-info indexed_fields catalog", async () => {
|
|
337
|
+
const update = findTool("update-tag");
|
|
338
|
+
await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
|
|
339
|
+
expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).toContain("status");
|
|
340
|
+
|
|
341
|
+
await update.execute({ tag: "project", fields: null });
|
|
342
|
+
expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).not.toContain("status");
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
it("fields:undefined is a no-op — preserves existing field schemas + columns", async () => {
|
|
346
|
+
const update = findTool("update-tag");
|
|
347
|
+
await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
|
|
348
|
+
// Update only the description; omit fields entirely.
|
|
349
|
+
await update.execute({ tag: "project", description: "a project" });
|
|
350
|
+
expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project"]);
|
|
351
|
+
expect(notesColumns()).toContain("meta_status");
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
it("fields:null respects co-declaration — keeps a field another live tag still declares", async () => {
|
|
355
|
+
const update = findTool("update-tag");
|
|
356
|
+
await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
357
|
+
await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
358
|
+
|
|
359
|
+
await update.execute({ tag: "asset", fields: null });
|
|
360
|
+
expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
|
|
361
|
+
expect(notesColumns()).toContain("meta_aspect_ratio");
|
|
362
|
+
});
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
// ===========================================================================
|
|
366
|
+
// Bug 1c — prune for ALREADY-orphaned fields. The gitcoin case: an
|
|
367
|
+
// indexed_fields row whose every declarer tag has no `tags` row (orphaned by a
|
|
368
|
+
// pre-fix delete/clear that never released). prune finds + drops them; it must
|
|
369
|
+
// NOT touch fields with a live declarer.
|
|
370
|
+
// ===========================================================================
|
|
371
|
+
describe("pruneOrphanedIndexedFields", () => {
|
|
372
|
+
// Seed an orphaned field directly: declare via the API (creates the tag
|
|
373
|
+
// row + column), then delete the tag row out from under it WITHOUT going
|
|
374
|
+
// through the release path — exactly the pre-fix orphaned state.
|
|
375
|
+
function orphanField(field: string, type: "TEXT" | "INTEGER", tag: string) {
|
|
376
|
+
declareField(db, field, type, tag);
|
|
377
|
+
// Drop the tags row directly — simulating the pre-fix delete-tag that
|
|
378
|
+
// never released. The indexed_fields row + column survive (orphaned).
|
|
379
|
+
db.prepare("DELETE FROM tags WHERE name = ?").run(tag);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
it("dry-run reports the orphan without mutating", async () => {
|
|
383
|
+
orphanField("legacy_status", "TEXT", "ghost");
|
|
384
|
+
expect(notesColumns()).toContain("meta_legacy_status");
|
|
385
|
+
|
|
386
|
+
const plan = pruneOrphanedIndexedFields(db, { dryRun: true });
|
|
387
|
+
expect(plan).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
|
|
388
|
+
// Nothing changed.
|
|
389
|
+
expect(getIndexedField(db, "legacy_status")).not.toBeNull();
|
|
390
|
+
expect(notesColumns()).toContain("meta_legacy_status");
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
it("apply drops the orphaned column + index + row", async () => {
|
|
394
|
+
orphanField("legacy_status", "TEXT", "ghost");
|
|
395
|
+
const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
|
|
396
|
+
expect(plan).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
|
|
397
|
+
expect(getIndexedField(db, "legacy_status")).toBeNull();
|
|
398
|
+
expect(notesColumns()).not.toContain("meta_legacy_status");
|
|
399
|
+
expect(notesIndexes()).not.toContain("idx_meta_legacy_status");
|
|
400
|
+
// No longer advertised by vault-info.
|
|
401
|
+
expect(buildVaultProjection(db).indexed_fields.map((f) => f.name)).not.toContain("legacy_status");
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
it("does NOT touch a field with a live declarer", async () => {
|
|
405
|
+
const update = findTool("update-tag");
|
|
406
|
+
await update.execute({ tag: "project", fields: { status: { type: "string", indexed: true } } });
|
|
407
|
+
const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
|
|
408
|
+
expect(plan).toEqual([]);
|
|
409
|
+
expect(getIndexedField(db, "status")?.declarerTags).toEqual(["project"]);
|
|
410
|
+
expect(notesColumns()).toContain("meta_status");
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
it("trims dead declarers but keeps the column when a live co-declarer remains", async () => {
|
|
414
|
+
const update = findTool("update-tag");
|
|
415
|
+
// Two declarers, then orphan only one by deleting its tag row directly.
|
|
416
|
+
await update.execute({ tag: "asset", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
417
|
+
await update.execute({ tag: "storyboard", fields: { aspect_ratio: { type: "string", indexed: true } } });
|
|
418
|
+
db.prepare("DELETE FROM tags WHERE name = ?").run("asset"); // orphan one declarer
|
|
419
|
+
|
|
420
|
+
const plan = pruneOrphanedIndexedFields(db, { dryRun: false });
|
|
421
|
+
expect(plan).toEqual([{ field: "aspect_ratio", deadDeclarers: ["asset"], dropped: false }]);
|
|
422
|
+
// Column kept; storyboard still declares it; asset trimmed from the set.
|
|
423
|
+
expect(getIndexedField(db, "aspect_ratio")?.declarerTags).toEqual(["storyboard"]);
|
|
424
|
+
expect(notesColumns()).toContain("meta_aspect_ratio");
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
it("store.pruneIndexedFields surfaces the same plan", async () => {
|
|
428
|
+
orphanField("legacy_status", "TEXT", "ghost");
|
|
429
|
+
const dry = await store.pruneIndexedFields({ dryRun: true });
|
|
430
|
+
expect(dry).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
|
|
431
|
+
expect(getIndexedField(db, "legacy_status")).not.toBeNull(); // dry-run didn't mutate
|
|
432
|
+
const applied = await store.pruneIndexedFields({ dryRun: false });
|
|
433
|
+
expect(applied).toEqual([{ field: "legacy_status", deadDeclarers: ["ghost"], dropped: true }]);
|
|
434
|
+
expect(getIndexedField(db, "legacy_status")).toBeNull();
|
|
435
|
+
});
|
|
285
436
|
});
|
|
@@ -236,3 +236,101 @@ export function rebuildIndexes(db: Database): void {
|
|
|
236
236
|
}
|
|
237
237
|
}
|
|
238
238
|
}
|
|
239
|
+
|
|
240
|
+
export interface PrunedField {
|
|
241
|
+
/** The field whose `indexed_fields` row was affected. */
|
|
242
|
+
field: string;
|
|
243
|
+
/** Declarer tags that no longer have a `tags` row (removed from the set). */
|
|
244
|
+
deadDeclarers: string[];
|
|
245
|
+
/**
|
|
246
|
+
* True when the field had NO surviving live declarer and was fully dropped
|
|
247
|
+
* (row + generated column + index). False when at least one live declarer
|
|
248
|
+
* remained and only the dead declarers were pruned from the set.
|
|
249
|
+
*/
|
|
250
|
+
dropped: boolean;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Prune orphaned `indexed_fields` declarers — the gitcoin defect.
|
|
255
|
+
*
|
|
256
|
+
* A declarer tag is "dead" when no `tags` row carries that name (the tag was
|
|
257
|
+
* deleted, or its schema was cleared, without releasing the field). For every
|
|
258
|
+
* `indexed_fields` row:
|
|
259
|
+
*
|
|
260
|
+
* - Drop dead declarers from the set.
|
|
261
|
+
* - If NO live declarer remains, drop the whole field — row + generated
|
|
262
|
+
* column + index. This is the only data-loss-free drop: the generated
|
|
263
|
+
* column is `json_extract(metadata, …)` so the source values stay in
|
|
264
|
+
* `notes.metadata`; only the (now-dead) index is lost.
|
|
265
|
+
* - If at least one live declarer remains (co-declaration), keep the column
|
|
266
|
+
* and just trim the dead names from the declarer set.
|
|
267
|
+
*
|
|
268
|
+
* `dryRun` (default) computes the plan without mutating; pass `dryRun: false`
|
|
269
|
+
* to apply. Returns the per-field plan either way so the CLI / MCP surface can
|
|
270
|
+
* print what it would (or did) drop.
|
|
271
|
+
*/
|
|
272
|
+
export function pruneOrphanedIndexedFields(
|
|
273
|
+
db: Database,
|
|
274
|
+
opts: { dryRun?: boolean } = {},
|
|
275
|
+
): PrunedField[] {
|
|
276
|
+
const dryRun = opts.dryRun ?? true;
|
|
277
|
+
const liveTags = new Set(
|
|
278
|
+
(db.prepare("SELECT name FROM tags").all() as { name: string }[]).map((r) => r.name),
|
|
279
|
+
);
|
|
280
|
+
const plan: PrunedField[] = [];
|
|
281
|
+
for (const f of listIndexedFields(db)) {
|
|
282
|
+
const deadDeclarers = f.declarerTags.filter((t) => !liveTags.has(t));
|
|
283
|
+
if (deadDeclarers.length === 0) continue; // every declarer is live — leave it
|
|
284
|
+
const liveDeclarers = f.declarerTags.filter((t) => liveTags.has(t));
|
|
285
|
+
const dropped = liveDeclarers.length === 0;
|
|
286
|
+
plan.push({ field: f.field, deadDeclarers, dropped });
|
|
287
|
+
if (dryRun) continue;
|
|
288
|
+
if (dropped) {
|
|
289
|
+
db.prepare("DELETE FROM indexed_fields WHERE field = ?").run(f.field);
|
|
290
|
+
dropColumnAndIndex(db, f.field);
|
|
291
|
+
} else {
|
|
292
|
+
db.prepare("UPDATE indexed_fields SET declarer_tags = ? WHERE field = ?").run(
|
|
293
|
+
JSON.stringify(liveDeclarers),
|
|
294
|
+
f.field,
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return plan;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Replay `declareField` for every field a tag schema marks `indexed: true`.
|
|
303
|
+
* Idempotent — used by the portable-md import path so a fresh import ends with
|
|
304
|
+
* the same generated columns a live vault would have (the import writes
|
|
305
|
+
* `tags.fields` via `upsertTagRecord` but never materializes the backing
|
|
306
|
+
* columns). Without this, an imported vault's schemas say `indexed: true` but
|
|
307
|
+
* queries fall back to full scans until each tag is next `update-tag`'d.
|
|
308
|
+
*
|
|
309
|
+
* `tagSchemas` is the post-import set of (tag, fields) pairs. Returns the
|
|
310
|
+
* number of (tag, field) declarations replayed.
|
|
311
|
+
*/
|
|
312
|
+
export function reconcileDeclaredIndexes(
|
|
313
|
+
db: Database,
|
|
314
|
+
tagSchemas: { tag: string; fields?: Record<string, { type: string; indexed?: boolean }> }[],
|
|
315
|
+
): number {
|
|
316
|
+
let declared = 0;
|
|
317
|
+
for (const schema of tagSchemas) {
|
|
318
|
+
if (!schema.fields) continue;
|
|
319
|
+
for (const [fieldName, spec] of Object.entries(schema.fields)) {
|
|
320
|
+
if (spec.indexed !== true) continue;
|
|
321
|
+
const mapped = mapFieldType(spec.type);
|
|
322
|
+
if (!mapped) continue; // unsupported type for indexing — skip, don't throw
|
|
323
|
+
try {
|
|
324
|
+
validateFieldName(fieldName);
|
|
325
|
+
declareField(db, fieldName, mapped, schema.tag);
|
|
326
|
+
declared++;
|
|
327
|
+
} catch (err) {
|
|
328
|
+
console.error(
|
|
329
|
+
`[indexed-fields] could not re-declare "${fieldName}" for tag "${schema.tag}" on import:`,
|
|
330
|
+
err,
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
return declared;
|
|
336
|
+
}
|
package/core/src/mcp.ts
CHANGED
|
@@ -23,8 +23,8 @@ export interface McpToolDef {
|
|
|
23
23
|
/**
|
|
24
24
|
* Minimum scope verb the caller must hold for THIS vault to see + invoke
|
|
25
25
|
* the tool. `read` for pure queries, `write` for mutations, `admin` for
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* operator-only surfaces (`prune-schema` in core; `manage-token` in the
|
|
27
|
+
* server layer). The MCP HTTP layer filters
|
|
28
28
|
* `tools/list` by this field and verb-gates `tools/call` against it; the
|
|
29
29
|
* filter is the primary defense, the inner gate is defense-in-depth.
|
|
30
30
|
*
|
|
@@ -100,9 +100,9 @@ function removeWikilinkBrackets(content: string, targetPath: string): string {
|
|
|
100
100
|
// ---------------------------------------------------------------------------
|
|
101
101
|
|
|
102
102
|
/**
|
|
103
|
-
* Generate the consolidated MCP tools for a vault.
|
|
103
|
+
* Generate the consolidated MCP tools for a vault. Surface (10):
|
|
104
104
|
* query-notes, create-note, update-note, delete-note, list-tags, update-tag,
|
|
105
|
-
* delete-tag, find-path, vault-info.
|
|
105
|
+
* delete-tag, find-path, vault-info, prune-schema (admin).
|
|
106
106
|
*/
|
|
107
107
|
export function generateMcpTools(store: Store): McpToolDef[] {
|
|
108
108
|
const db: Database = (store as any).db;
|
|
@@ -1074,12 +1074,23 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1074
1074
|
const tag = params.tag as string;
|
|
1075
1075
|
const existing = tagSchemaOps.getTagRecord(db, tag);
|
|
1076
1076
|
|
|
1077
|
-
// ---- fields:
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1077
|
+
// ---- fields: three-way semantics, distinguishing `null` from
|
|
1078
|
+
// `undefined` (do NOT collapse with `?? {}` — that silently turns an
|
|
1079
|
+
// explicit clear-all into a no-op, the gitcoin orphaned-fields bug).
|
|
1080
|
+
// - undefined → no change. Preserve every existing field; declare
|
|
1081
|
+
// nothing new. mergedFields === existing.fields.
|
|
1082
|
+
// - null → clear ALL of this tag's field schemas.
|
|
1083
|
+
// mergedFields = {} so the diff below releases every
|
|
1084
|
+
// indexed field this tag exclusively declares.
|
|
1085
|
+
// - object → shallow-merge into existing (preserves prior keys).
|
|
1086
|
+
const incomingFields =
|
|
1087
|
+
params.fields === null || params.fields === undefined
|
|
1088
|
+
? {}
|
|
1089
|
+
: (params.fields as Record<string, TagFieldSchema>);
|
|
1090
|
+
const mergedFields: Record<string, TagFieldSchema> =
|
|
1091
|
+
params.fields === null
|
|
1092
|
+
? {}
|
|
1093
|
+
: { ...(existing?.fields ?? {}), ...incomingFields };
|
|
1083
1094
|
|
|
1084
1095
|
// Validate cross-tag consistency on fields being (re)declared in this
|
|
1085
1096
|
// call. `type` and `indexed` are global — all declarers must agree.
|
|
@@ -1146,6 +1157,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1146
1157
|
: (params.fields !== undefined ? null : undefined);
|
|
1147
1158
|
const descriptionPatch =
|
|
1148
1159
|
params.description === undefined ? undefined : (params.description as string);
|
|
1160
|
+
// The indexed-field lifecycle (declareField for added indexed fields,
|
|
1161
|
+
// releaseField for removed ones, with the co-declaration guard) is
|
|
1162
|
+
// reconciled inside store.upsertTagRecord — the single chokepoint all
|
|
1163
|
+
// callers (MCP, REST PUT /tags/:name, import) share — so it can't be
|
|
1164
|
+
// bypassed. The cross-tag validation above stays here to surface a
|
|
1165
|
+
// clean error before persisting. See the gitcoin orphaned-fields bug.
|
|
1149
1166
|
const result = await store.upsertTagRecord(tag, {
|
|
1150
1167
|
...(descriptionPatch !== undefined ? { description: descriptionPatch } : {}),
|
|
1151
1168
|
...(fieldsPatch !== undefined ? { fields: fieldsPatch } : {}),
|
|
@@ -1153,28 +1170,6 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1153
1170
|
...(parentNamesPatch !== undefined ? { parent_names: parentNamesPatch } : {}),
|
|
1154
1171
|
});
|
|
1155
1172
|
|
|
1156
|
-
// ---- Reconcile indexed-field lifecycle for this tag.
|
|
1157
|
-
const priorIndexed = new Set(
|
|
1158
|
-
Object.entries(existing?.fields ?? {})
|
|
1159
|
-
.filter(([, v]) => v.indexed === true)
|
|
1160
|
-
.map(([k]) => k),
|
|
1161
|
-
);
|
|
1162
|
-
const nextIndexed = new Set(
|
|
1163
|
-
Object.entries(mergedFields)
|
|
1164
|
-
.filter(([, v]) => v.indexed === true)
|
|
1165
|
-
.map(([k]) => k),
|
|
1166
|
-
);
|
|
1167
|
-
for (const fieldName of nextIndexed) {
|
|
1168
|
-
const spec = mergedFields[fieldName]!;
|
|
1169
|
-
const mapped = indexedFieldOps.mapFieldType(spec.type)!;
|
|
1170
|
-
indexedFieldOps.declareField(db, fieldName, mapped, tag);
|
|
1171
|
-
}
|
|
1172
|
-
for (const fieldName of priorIndexed) {
|
|
1173
|
-
if (!nextIndexed.has(fieldName)) {
|
|
1174
|
-
indexedFieldOps.releaseField(db, fieldName, tag);
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
1173
|
return result;
|
|
1179
1174
|
},
|
|
1180
1175
|
},
|
|
@@ -1198,19 +1193,12 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1198
1193
|
},
|
|
1199
1194
|
execute: async (params) => {
|
|
1200
1195
|
const tag = params.tag as string;
|
|
1201
|
-
// Release any indexed fields this tag declared before the row
|
|
1202
|
-
// drops. releaseField drops the generated column + index when the
|
|
1203
|
-
// declarer set empties.
|
|
1204
|
-
const schema = tagSchemaOps.getTagSchema(db, tag);
|
|
1205
|
-
if (schema?.fields) {
|
|
1206
|
-
for (const [fieldName, spec] of Object.entries(schema.fields)) {
|
|
1207
|
-
if (spec.indexed === true) {
|
|
1208
|
-
indexedFieldOps.releaseField(db, fieldName, tag);
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
1196
|
// Drop the row outright — description/fields/relationships/parents
|
|
1213
1197
|
// travel with it. (No more sidecar table to clear separately.)
|
|
1198
|
+
// Indexed-field release is handled inside store.deleteTag →
|
|
1199
|
+
// noteOps.deleteTag so every entry point (MCP, REST, import sweep)
|
|
1200
|
+
// releases consistently with the co-declaration guard. See the
|
|
1201
|
+
// gitcoin orphaned-fields bug report.
|
|
1214
1202
|
return await store.deleteTag(tag);
|
|
1215
1203
|
},
|
|
1216
1204
|
},
|
|
@@ -1266,6 +1254,41 @@ Link expansion: pass \`expand_links: true\` to inline [[wikilinks]] from returne
|
|
|
1266
1254
|
},
|
|
1267
1255
|
},
|
|
1268
1256
|
|
|
1257
|
+
// =====================================================================
|
|
1258
|
+
// 10. prune-schema — drop orphaned indexed-field columns
|
|
1259
|
+
// =====================================================================
|
|
1260
|
+
{
|
|
1261
|
+
name: "prune-schema",
|
|
1262
|
+
// `admin` — a destructive schema-maintenance op, same tier as
|
|
1263
|
+
// manage-token. Operator-only; hidden from read/write sessions.
|
|
1264
|
+
requiredVerb: "admin",
|
|
1265
|
+
description:
|
|
1266
|
+
"Drop orphaned indexed-field columns + indexes whose declaring tags no longer exist (the result of a deleted tag never releasing its fields). Dry-run by default — returns the drop plan without mutating. Pass `apply: true` to execute. A field co-declared by a still-live tag is never dropped; only the dead declarers are trimmed from its set. Generated columns are derived from notes.metadata JSON, so a drop loses only the index, never source data — declare the field again to rebuild it.",
|
|
1267
|
+
inputSchema: {
|
|
1268
|
+
type: "object",
|
|
1269
|
+
properties: {
|
|
1270
|
+
apply: {
|
|
1271
|
+
type: "boolean",
|
|
1272
|
+
description: "Execute the prune. Default false (dry-run — report what would be dropped without changing anything).",
|
|
1273
|
+
},
|
|
1274
|
+
},
|
|
1275
|
+
},
|
|
1276
|
+
execute: async (params) => {
|
|
1277
|
+
const apply = params.apply === true;
|
|
1278
|
+
const plan = await store.pruneIndexedFields({ dryRun: !apply });
|
|
1279
|
+
const dropped = plan.filter((p) => p.dropped);
|
|
1280
|
+
const trimmed = plan.filter((p) => !p.dropped);
|
|
1281
|
+
return {
|
|
1282
|
+
dry_run: !apply,
|
|
1283
|
+
fields_dropped: dropped.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
|
|
1284
|
+
fields_trimmed: trimmed.map((p) => ({ field: p.field, dead_declarers: p.deadDeclarers })),
|
|
1285
|
+
summary: apply
|
|
1286
|
+
? `pruned ${dropped.length} orphaned field(s); trimmed dead declarers on ${trimmed.length} co-declared field(s)`
|
|
1287
|
+
: `would prune ${dropped.length} orphaned field(s); would trim dead declarers on ${trimmed.length} co-declared field(s) — pass apply:true to execute`,
|
|
1288
|
+
};
|
|
1289
|
+
},
|
|
1290
|
+
},
|
|
1291
|
+
|
|
1269
1292
|
];
|
|
1270
1293
|
}
|
|
1271
1294
|
|
package/core/src/notes.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
type CursorPayload,
|
|
19
19
|
type QueryHashInputs,
|
|
20
20
|
} from "./cursor.js";
|
|
21
|
+
import { releaseField } from "./indexed-fields.js";
|
|
21
22
|
|
|
22
23
|
let idCounter = 0;
|
|
23
24
|
|
|
@@ -943,12 +944,35 @@ export function listTags(db: Database): { name: string; count: number }[] {
|
|
|
943
944
|
}
|
|
944
945
|
|
|
945
946
|
export function deleteTag(db: Database, name: string): { deleted: boolean; notes_untagged: number } {
|
|
946
|
-
const
|
|
947
|
-
|
|
947
|
+
const row = db.prepare("SELECT fields FROM tags WHERE name = ?").get(name) as
|
|
948
|
+
| { fields: string | null }
|
|
949
|
+
| undefined;
|
|
950
|
+
if (!row) return { deleted: false, notes_untagged: 0 };
|
|
948
951
|
|
|
949
952
|
const countRow = db.prepare("SELECT COUNT(*) as c FROM note_tags WHERE tag_name = ?").get(name) as { c: number };
|
|
950
953
|
const notesUntagged = countRow.c;
|
|
951
954
|
|
|
955
|
+
// Release any indexed fields this tag declared BEFORE the row drops.
|
|
956
|
+
// `releaseField` only drops the generated column + index when this tag is
|
|
957
|
+
// the last live declarer (co-declaration guard) — a field co-declared by
|
|
958
|
+
// another live tag keeps its column and just loses this tag from the set.
|
|
959
|
+
// This lives in the store-level delete (not the MCP layer) so every caller
|
|
960
|
+
// — MCP delete-tag, the REST DELETE /tags/:name route, the import
|
|
961
|
+
// blow-away sweep — releases consistently. See the gitcoin orphaned-fields
|
|
962
|
+
// bug report.
|
|
963
|
+
if (row.fields) {
|
|
964
|
+
try {
|
|
965
|
+
const fields = JSON.parse(row.fields) as Record<string, { indexed?: boolean }>;
|
|
966
|
+
for (const [fieldName, spec] of Object.entries(fields)) {
|
|
967
|
+
if (spec?.indexed === true) {
|
|
968
|
+
releaseField(db, fieldName, name);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
} catch {
|
|
972
|
+
// Malformed fields JSON — nothing to release; proceed with the delete.
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
952
976
|
db.prepare("DELETE FROM note_tags WHERE tag_name = ?").run(name);
|
|
953
977
|
db.prepare("DELETE FROM tags WHERE name = ?").run(name);
|
|
954
978
|
|
|
@@ -24,6 +24,8 @@ import { join } from "path";
|
|
|
24
24
|
import { tmpdir } from "os";
|
|
25
25
|
|
|
26
26
|
import { SqliteStore } from "./store.js";
|
|
27
|
+
import { getIndexedField } from "./indexed-fields.js";
|
|
28
|
+
import { buildVaultProjection } from "./vault-projection.js";
|
|
27
29
|
import {
|
|
28
30
|
CaseCollisionError,
|
|
29
31
|
emitYamlDoc,
|
|
@@ -725,6 +727,56 @@ describe("importPortableVault", async () => {
|
|
|
725
727
|
expect(schema!.description).toBe("A unit of work");
|
|
726
728
|
});
|
|
727
729
|
|
|
730
|
+
// Fix 2 — import must re-declare indexed fields. The import writes
|
|
731
|
+
// tags.fields via upsertTagRecord but historically never materialized the
|
|
732
|
+
// backing generated columns + indexes, so a fresh import advertised
|
|
733
|
+
// `indexed: true` while queries silently full-scanned. Regression: export a
|
|
734
|
+
// vault with an indexed field → fresh import → the generated column + index
|
|
735
|
+
// exist and the field shows in the vault-info indexed_fields catalog.
|
|
736
|
+
it("re-declares indexed fields on import (column + index + vault-info catalog)", async () => {
|
|
737
|
+
await store.upsertTagRecord("project", {
|
|
738
|
+
fields: { status: { type: "string", indexed: true } },
|
|
739
|
+
});
|
|
740
|
+
await store.createNote("p", { id: "p", tags: ["project"], metadata: { status: "active" } });
|
|
741
|
+
const outDir = join(tmpBase, "out");
|
|
742
|
+
await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
743
|
+
|
|
744
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
745
|
+
const targetDb = target.db;
|
|
746
|
+
// Pre-condition: a fresh store has no backing column yet.
|
|
747
|
+
const colsBefore = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
748
|
+
expect(colsBefore).not.toContain("meta_status");
|
|
749
|
+
|
|
750
|
+
const stats = await importPortableVault(target, { inDir: outDir });
|
|
751
|
+
expect(stats.schemas_restored).toBe(1);
|
|
752
|
+
expect(stats.indexes_declared).toBe(1);
|
|
753
|
+
|
|
754
|
+
// The generated column + index now exist — same introspection vault-info
|
|
755
|
+
// uses to advertise the queryable-field catalog.
|
|
756
|
+
const colsAfter = (targetDb.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
757
|
+
expect(colsAfter).toContain("meta_status");
|
|
758
|
+
const idxs = (targetDb.prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='notes'").all() as { name: string }[]).map((r) => r.name);
|
|
759
|
+
expect(idxs).toContain("idx_meta_status");
|
|
760
|
+
expect(getIndexedField(targetDb, "status")?.declarerTags).toEqual(["project"]);
|
|
761
|
+
// And vault-info lists it.
|
|
762
|
+
expect(buildVaultProjection(targetDb).indexed_fields.map((f) => f.name)).toContain("status");
|
|
763
|
+
});
|
|
764
|
+
|
|
765
|
+
it("dry-run import counts indexes_declared without materializing columns", async () => {
|
|
766
|
+
await store.upsertTagRecord("project", {
|
|
767
|
+
fields: { status: { type: "string", indexed: true } },
|
|
768
|
+
});
|
|
769
|
+
const outDir = join(tmpBase, "out");
|
|
770
|
+
await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
771
|
+
|
|
772
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
773
|
+
const stats = await importPortableVault(target, { inDir: outDir, dryRun: true });
|
|
774
|
+
expect(stats.indexes_declared).toBe(1);
|
|
775
|
+
// Dry-run touches nothing.
|
|
776
|
+
const cols = (target.db.prepare("PRAGMA table_xinfo(notes)").all() as { name: string }[]).map((r) => r.name);
|
|
777
|
+
expect(cols).not.toContain("meta_status");
|
|
778
|
+
});
|
|
779
|
+
|
|
728
780
|
it("restores typed links (non-wikilink relationships)", async () => {
|
|
729
781
|
await store.createNote("src body", { id: "src", path: "src" });
|
|
730
782
|
await store.createNote("tgt body", { id: "tgt", path: "tgt" });
|
package/core/src/portable-md.ts
CHANGED
|
@@ -1535,6 +1535,14 @@ export interface ImportStats {
|
|
|
1535
1535
|
skipped_sidecars: Array<{ sidecar_id: string; expected_path: string | null; expected_extension: string | null; reason: string }>;
|
|
1536
1536
|
/** Set when the caller passed `blowAway: true`; counts notes removed. */
|
|
1537
1537
|
notes_wiped: number;
|
|
1538
|
+
/**
|
|
1539
|
+
* (tag, field) indexed-field declarations replayed after restoring tag
|
|
1540
|
+
* schemas — materializes the generated columns + indexes a live vault
|
|
1541
|
+
* would have. Without this an imported vault's schemas say `indexed: true`
|
|
1542
|
+
* but the backing columns don't exist until each tag is next `update-tag`'d
|
|
1543
|
+
* (queries fall back to full scans). See the import re-declare fix.
|
|
1544
|
+
*/
|
|
1545
|
+
indexes_declared: number;
|
|
1538
1546
|
}
|
|
1539
1547
|
|
|
1540
1548
|
/**
|
|
@@ -1582,6 +1590,7 @@ export async function importPortableVault(
|
|
|
1582
1590
|
skipped_attachments: [],
|
|
1583
1591
|
skipped_sidecars: [],
|
|
1584
1592
|
notes_wiped: 0,
|
|
1593
|
+
indexes_declared: 0,
|
|
1585
1594
|
};
|
|
1586
1595
|
|
|
1587
1596
|
// 1. Optional wipe. Notes are deleted via the public Store API so
|
|
@@ -2019,6 +2028,45 @@ export async function importPortableVault(
|
|
|
2019
2028
|
await store.syncAllWikilinks();
|
|
2020
2029
|
}
|
|
2021
2030
|
|
|
2031
|
+
// 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
|
|
2032
|
+
// Step 2 restored tag schemas via `store.upsertTagRecord`, which — now that
|
|
2033
|
+
// the indexed-field lifecycle is centralized in the store — already
|
|
2034
|
+
// materializes the backing generated columns + indexes as it persists each
|
|
2035
|
+
// schema. This explicit reconcile is therefore idempotent on the happy path;
|
|
2036
|
+
// it stays as a safety net (covers any schema written through a path that
|
|
2037
|
+
// skipped the lifecycle) and gives the authoritative `indexes_declared`
|
|
2038
|
+
// count. Without it, a regression in step 2 would silently leave the
|
|
2039
|
+
// imported schemas advertising `indexed: true` while queries full-scan.
|
|
2040
|
+
if (!opts.dryRun) {
|
|
2041
|
+
stats.indexes_declared = await store.reconcileDeclaredIndexes();
|
|
2042
|
+
} else {
|
|
2043
|
+
// Dry-run: count what WOULD be declared without touching the DB. Both
|
|
2044
|
+
// paths count per (tag, field) declaration (a co-declared field counts
|
|
2045
|
+
// once per declaring tag). The one asymmetry: this dry-run counts every
|
|
2046
|
+
// `indexed: true` field including unsupported types, whereas the applied
|
|
2047
|
+
// `reconcileDeclaredIndexes` skips fields whose type can't be indexed —
|
|
2048
|
+
// so the dry-run can over-count by the number of mis-typed indexed
|
|
2049
|
+
// fields. It's a "how much indexing work" signal, not a row-exact promise.
|
|
2050
|
+
const schemasDir2 = join(sidecar, "schemas");
|
|
2051
|
+
if (existsSync(schemasDir2)) {
|
|
2052
|
+
for (const entry of readdirSync(schemasDir2)) {
|
|
2053
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
2054
|
+
const fullPath = join(schemasDir2, entry);
|
|
2055
|
+
const resolved = resolvePath(fullPath);
|
|
2056
|
+
if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
|
|
2057
|
+
const text = readFileSync(fullPath, "utf-8");
|
|
2058
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
2059
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
2060
|
+
const fields = frontmatter.fields;
|
|
2061
|
+
if (fields && typeof fields === "object" && !Array.isArray(fields)) {
|
|
2062
|
+
for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
|
|
2063
|
+
if (spec?.indexed === true) stats.indexes_declared++;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2022
2070
|
return stats;
|
|
2023
2071
|
}
|
|
2024
2072
|
|