@openparachute/vault 0.4.8 → 0.4.9-rc.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/core.test.ts +4 -1
- package/core/src/hooks.test.ts +320 -1
- package/core/src/hooks.ts +243 -38
- package/core/src/indexed-fields.test.ts +151 -0
- package/core/src/indexed-fields.ts +98 -0
- package/core/src/mcp.ts +99 -41
- package/core/src/notes.ts +26 -2
- package/core/src/portable-md.test.ts +304 -1
- package/core/src/portable-md.ts +418 -2
- package/core/src/schema.ts +114 -2
- package/core/src/store.ts +185 -2
- package/core/src/types.ts +28 -0
- package/package.json +2 -2
- package/src/auth-hub-jwt.test.ts +147 -0
- package/src/auth.ts +121 -1
- package/src/auto-transcribe.test.ts +7 -2
- package/src/auto-transcribe.ts +6 -2
- package/src/cli.ts +131 -36
- package/src/config.ts +12 -4
- package/src/export-watch.test.ts +74 -0
- package/src/export-watch.ts +108 -7
- package/src/github-device-flow.test.ts +404 -0
- package/src/github-device-flow.ts +415 -0
- package/src/hub-jwt.test.ts +27 -2
- package/src/hub-jwt.ts +10 -0
- package/src/mcp-http.ts +48 -39
- package/src/mcp-install-interactive.test.ts +10 -21
- package/src/mcp-install-interactive.ts +12 -21
- package/src/mcp-install.test.ts +141 -30
- package/src/mcp-install.ts +109 -3
- package/src/mcp-tools.ts +460 -3
- package/src/mirror-config.test.ts +277 -14
- package/src/mirror-config.ts +482 -31
- package/src/mirror-credentials.test.ts +601 -0
- package/src/mirror-credentials.ts +700 -0
- package/src/mirror-deps.ts +67 -17
- package/src/mirror-import.test.ts +550 -0
- package/src/mirror-import.ts +487 -0
- package/src/mirror-manager.test.ts +423 -12
- package/src/mirror-manager.ts +621 -72
- package/src/mirror-per-vault.test.ts +519 -0
- package/src/mirror-registry.ts +91 -14
- package/src/mirror-routes.test.ts +966 -10
- package/src/mirror-routes.ts +1111 -7
- package/src/module-config.ts +11 -5
- package/src/routes.ts +38 -1
- package/src/routing.test.ts +92 -1
- package/src/routing.ts +193 -20
- package/src/server.ts +116 -35
- package/src/storage.test.ts +132 -7
- package/src/token-store.ts +300 -5
- package/src/transcription-worker.ts +9 -4
- package/src/triggers.ts +16 -3
- package/src/vault.test.ts +681 -2
- package/web/ui/dist/assets/index-Cn-PPMRv.js +60 -0
- package/web/ui/dist/assets/{index-BOa-JJtV.css → index-DBe8Xiah.css} +1 -1
- package/web/ui/dist/index.html +2 -2
- package/web/ui/dist/assets/index-BzA5LgE3.js +0 -60
package/core/src/portable-md.ts
CHANGED
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
* See vault#308.
|
|
73
73
|
*/
|
|
74
74
|
|
|
75
|
-
import { readdirSync, readFileSync, statSync, mkdirSync, writeFileSync, copyFileSync, existsSync, rmSync } from "fs";
|
|
75
|
+
import { readdirSync, readFileSync, realpathSync, statSync, mkdirSync, writeFileSync, copyFileSync, existsSync, rmSync } from "fs";
|
|
76
76
|
import { basename, join, relative, extname, dirname, resolve as resolvePath, sep as pathSep } from "path";
|
|
77
77
|
import type { Store, Note, Link, Attachment } from "./types.js";
|
|
78
78
|
import type { TagRecord } from "./tag-schemas.js";
|
|
@@ -1001,7 +1001,375 @@ export async function exportVaultToDir(
|
|
|
1001
1001
|
};
|
|
1002
1002
|
}
|
|
1003
1003
|
|
|
1004
|
-
|
|
1004
|
+
// ---------------------------------------------------------------------------
|
|
1005
|
+
// Orphan sweep — for git-mirror's delete propagation
|
|
1006
|
+
// ---------------------------------------------------------------------------
|
|
1007
|
+
|
|
1008
|
+
/**
|
|
1009
|
+
* Result of a `pruneOrphans` pass.
|
|
1010
|
+
*/
|
|
1011
|
+
export interface PruneOrphansStats {
|
|
1012
|
+
/** Note content files removed (frontmatter id not in `validNoteIds`). */
|
|
1013
|
+
notes_removed: number;
|
|
1014
|
+
/** Sidecar metadata files removed (under `.parachute/notes-meta/`). */
|
|
1015
|
+
sidecars_removed: number;
|
|
1016
|
+
/** Schema sidecars removed (under `.parachute/schemas/`). */
|
|
1017
|
+
schemas_removed: number;
|
|
1018
|
+
/** Attachment directories removed (under `.parachute/attachments/`). */
|
|
1019
|
+
attachment_dirs_removed: number;
|
|
1020
|
+
/**
|
|
1021
|
+
* Files we couldn't parse / classify. Surfaced for operator audit;
|
|
1022
|
+
* the sweep doesn't touch them. Empty when everything classified
|
|
1023
|
+
* cleanly.
|
|
1024
|
+
*/
|
|
1025
|
+
unparseable_files: Array<{ path: string; reason: string }>;
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
/**
|
|
1029
|
+
* Options for the orphan-sweep pass. Run periodically (the mirror manager
|
|
1030
|
+
* arms a safety-net poll, default 1h) and after operator-visible deletions
|
|
1031
|
+
* (the mirror manager's targeted-deletion fast path also calls this for the
|
|
1032
|
+
* touched-set).
|
|
1033
|
+
*
|
|
1034
|
+
* The sweep is the bookkeeping cousin of the event-driven fast path: events
|
|
1035
|
+
* cover the common case (a single note deletion fires "deleted" → mirror
|
|
1036
|
+
* removes that file); the sweep covers anything the fast path missed
|
|
1037
|
+
* (direct SQL writes, app crashes between dispatch and handler, restart
|
|
1038
|
+
* gaps).
|
|
1039
|
+
*/
|
|
1040
|
+
export interface PruneOrphansOptions {
|
|
1041
|
+
/** Directory to sweep — same shape as an `exportVaultToDir` outDir. */
|
|
1042
|
+
outDir: string;
|
|
1043
|
+
/** Note IDs that should be kept (everything else under the export gets removed). */
|
|
1044
|
+
validNoteIds: Set<string>;
|
|
1045
|
+
/** Tag names that should be kept (other schema sidecars under `.parachute/schemas/` get removed). */
|
|
1046
|
+
validTagNames: Set<string>;
|
|
1047
|
+
/** Attachment IDs that should be kept (other dirs under `.parachute/attachments/` get removed). */
|
|
1048
|
+
validAttachmentIds: Set<string>;
|
|
1049
|
+
/**
|
|
1050
|
+
* Override `extension`-based content-file extension recognition. The
|
|
1051
|
+
* default treats `.md` and `.mdx` as having inline frontmatter (with
|
|
1052
|
+
* `id:` reachable via the file head); everything else needs a sidecar
|
|
1053
|
+
* lookup to know what id owns it.
|
|
1054
|
+
*/
|
|
1055
|
+
supportsInlineFrontmatter?: (ext: string) => boolean;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* Sweep the export directory for files belonging to notes / tags /
|
|
1060
|
+
* attachments that no longer exist in the vault. Removes them so the
|
|
1061
|
+
* mirror's `git diff` reflects what vault actually has.
|
|
1062
|
+
*
|
|
1063
|
+
* Strategy:
|
|
1064
|
+
* 1. Walk content files. For each `.md` / `.mdx`, parse the frontmatter
|
|
1065
|
+
* `id` and compare against `validNoteIds`. Mismatch → remove the
|
|
1066
|
+
* file. Files we can't parse are recorded in `unparseable_files`
|
|
1067
|
+
* and left alone (best-effort, no destructive guesses).
|
|
1068
|
+
* 2. For non-inline-frontmatter extensions (`.csv`, `.yaml`, etc.),
|
|
1069
|
+
* check the matching `.parachute/notes-meta/<id>.yaml` sidecar — the
|
|
1070
|
+
* sidecar carries the canonical `id` + `path` + `extension` triple.
|
|
1071
|
+
* An orphaned sidecar (no matching content file) is also removed,
|
|
1072
|
+
* and an orphaned content file (no matching sidecar) without a
|
|
1073
|
+
* parseable frontmatter is left as unparseable.
|
|
1074
|
+
* 3. Walk `.parachute/schemas/`. Each file is `<tag>.yaml` (after
|
|
1075
|
+
* filename sanitization). Parse the `name:` field; compare against
|
|
1076
|
+
* `validTagNames`. Mismatch → remove.
|
|
1077
|
+
* 4. Walk `.parachute/attachments/`. Each subdir name IS the
|
|
1078
|
+
* attachment id (per `exportVaultToDir`'s layout). Compare against
|
|
1079
|
+
* `validAttachmentIds`. Mismatch → recursive remove.
|
|
1080
|
+
*
|
|
1081
|
+
* Returns counts so callers can log + decide whether to commit.
|
|
1082
|
+
*
|
|
1083
|
+
* Safe to call on a directory that's never been exported to (returns
|
|
1084
|
+
* zero counts; doesn't create anything).
|
|
1085
|
+
*/
|
|
1086
|
+
export function pruneOrphans(opts: PruneOrphansOptions): PruneOrphansStats {
|
|
1087
|
+
const stats: PruneOrphansStats = {
|
|
1088
|
+
notes_removed: 0,
|
|
1089
|
+
sidecars_removed: 0,
|
|
1090
|
+
schemas_removed: 0,
|
|
1091
|
+
attachment_dirs_removed: 0,
|
|
1092
|
+
unparseable_files: [],
|
|
1093
|
+
};
|
|
1094
|
+
|
|
1095
|
+
const outDir = opts.outDir;
|
|
1096
|
+
if (!existsSync(outDir)) return stats;
|
|
1097
|
+
|
|
1098
|
+
const supportsInline = opts.supportsInlineFrontmatter ?? supportsInlineFrontmatter;
|
|
1099
|
+
const sidecarRoot = join(outDir, SIDECAR_DIR);
|
|
1100
|
+
const notesMetaRoot = join(sidecarRoot, NOTES_META_DIR);
|
|
1101
|
+
const schemasRoot = join(sidecarRoot, "schemas");
|
|
1102
|
+
const attachmentsRoot = join(sidecarRoot, "attachments");
|
|
1103
|
+
|
|
1104
|
+
// Path-traversal guard. `walkContentFiles` uses `statSync` which
|
|
1105
|
+
// follows symlinks — a symlink inside the mirror pointing OUTSIDE
|
|
1106
|
+
// `outDir` would resurface its target's files in the prune sweep,
|
|
1107
|
+
// and a bare `rmSync(filepath)` would delete them off-tree. Every
|
|
1108
|
+
// deletion in this function routes through `safeRm`, which calls
|
|
1109
|
+
// `realpathSync` (resolves through symlinks, unlike syntactic-only
|
|
1110
|
+
// `path.resolve`) and refuses to delete anything that isn't `outDir`
|
|
1111
|
+
// or beneath it after symlink resolution. Refusals get recorded in
|
|
1112
|
+
// `unparseable_files` so an operator can see what was skipped.
|
|
1113
|
+
// Reviewer-flagged on vault#382 (Critical #2).
|
|
1114
|
+
const outDirReal = realpathSync(outDir);
|
|
1115
|
+
const safeRm = (
|
|
1116
|
+
candidate: string,
|
|
1117
|
+
onSuccess: () => void,
|
|
1118
|
+
options: { recursive?: boolean } = {},
|
|
1119
|
+
): void => {
|
|
1120
|
+
let real: string;
|
|
1121
|
+
try {
|
|
1122
|
+
real = realpathSync(candidate);
|
|
1123
|
+
} catch (err) {
|
|
1124
|
+
// realpathSync throws if the path doesn't exist or is unreadable.
|
|
1125
|
+
// Don't delete what we can't fully resolve.
|
|
1126
|
+
stats.unparseable_files.push({
|
|
1127
|
+
path: candidate,
|
|
1128
|
+
reason: `realpath failed: ${(err as Error).message ?? err}`,
|
|
1129
|
+
});
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
if (!isWithinDir(real, outDirReal)) {
|
|
1133
|
+
stats.unparseable_files.push({
|
|
1134
|
+
path: candidate,
|
|
1135
|
+
reason: "real path resolved outside mirror outDir — refusing to delete (symlink?)",
|
|
1136
|
+
});
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
try {
|
|
1140
|
+
// Delete via the resolved real path; never via the unresolved
|
|
1141
|
+
// candidate (which could route through a symlink we already
|
|
1142
|
+
// determined would escape).
|
|
1143
|
+
rmSync(real, { force: true, recursive: options.recursive ?? false });
|
|
1144
|
+
onSuccess();
|
|
1145
|
+
} catch (err) {
|
|
1146
|
+
stats.unparseable_files.push({
|
|
1147
|
+
path: candidate,
|
|
1148
|
+
reason: `unlink failed: ${(err as Error).message ?? err}`,
|
|
1149
|
+
});
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
|
|
1153
|
+
// ---- 1 + 2. Notes + sidecars ----
|
|
1154
|
+
//
|
|
1155
|
+
// First pass: build the sidecar id → { path, extension } map so we can
|
|
1156
|
+
// resolve non-inline-frontmatter content files via their sidecar.
|
|
1157
|
+
// Sidecars whose claimed (path, extension) doesn't map to an existing
|
|
1158
|
+
// content file are tracked as "orphaned sidecar" candidates.
|
|
1159
|
+
const sidecarById = new Map<string, { path: string | null; extension: string | null }>();
|
|
1160
|
+
const sidecarFilesById = new Map<string, string>(); // id → absolute filepath
|
|
1161
|
+
if (existsSync(notesMetaRoot)) {
|
|
1162
|
+
try {
|
|
1163
|
+
for (const entry of readdirSync(notesMetaRoot)) {
|
|
1164
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
1165
|
+
const id = entry.slice(0, -5);
|
|
1166
|
+
const full = join(notesMetaRoot, entry);
|
|
1167
|
+
sidecarFilesById.set(id, full);
|
|
1168
|
+
try {
|
|
1169
|
+
const text = readFileSync(full, "utf-8");
|
|
1170
|
+
const { frontmatter } = parseFrontmatter(`---\n${text}---\n`);
|
|
1171
|
+
sidecarById.set(id, {
|
|
1172
|
+
path: typeof frontmatter.path === "string" ? (frontmatter.path as string) : null,
|
|
1173
|
+
extension: typeof frontmatter.extension === "string" ? (frontmatter.extension as string) : null,
|
|
1174
|
+
});
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
stats.unparseable_files.push({
|
|
1177
|
+
path: full,
|
|
1178
|
+
reason: `failed to parse sidecar: ${(err as Error).message ?? err}`,
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
} catch (err) {
|
|
1183
|
+
// Notes-meta dir read-error is non-fatal — record + carry on.
|
|
1184
|
+
stats.unparseable_files.push({
|
|
1185
|
+
path: notesMetaRoot,
|
|
1186
|
+
reason: `notes-meta walk failed: ${(err as Error).message ?? err}`,
|
|
1187
|
+
});
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// Now walk content files (everything outside the .parachute sidecar).
|
|
1192
|
+
const contentFiles = walkContentFiles(outDir);
|
|
1193
|
+
// Track which sidecars matched a content file so we can also remove
|
|
1194
|
+
// orphaned sidecars (sidecar present but content file gone).
|
|
1195
|
+
const pairedSidecarIds = new Set<string>();
|
|
1196
|
+
for (const filepath of contentFiles) {
|
|
1197
|
+
const ext = extname(filepath).slice(1).toLowerCase();
|
|
1198
|
+
if (supportsInline(ext)) {
|
|
1199
|
+
// Parse frontmatter, read id.
|
|
1200
|
+
try {
|
|
1201
|
+
const raw = readFileSync(filepath, "utf-8");
|
|
1202
|
+
const { frontmatter } = parseFrontmatter(raw);
|
|
1203
|
+
const id = typeof frontmatter.id === "string" ? (frontmatter.id as string) : null;
|
|
1204
|
+
if (!id) {
|
|
1205
|
+
stats.unparseable_files.push({
|
|
1206
|
+
path: filepath,
|
|
1207
|
+
reason: "no `id` in frontmatter",
|
|
1208
|
+
});
|
|
1209
|
+
continue;
|
|
1210
|
+
}
|
|
1211
|
+
if (!opts.validNoteIds.has(id)) {
|
|
1212
|
+
safeRm(filepath, () => {
|
|
1213
|
+
stats.notes_removed++;
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
} catch (err) {
|
|
1217
|
+
stats.unparseable_files.push({
|
|
1218
|
+
path: filepath,
|
|
1219
|
+
reason: `read failed: ${(err as Error).message ?? err}`,
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
} else {
|
|
1223
|
+
// Sidecar-required extension. Find the matching sidecar by
|
|
1224
|
+
// (path, extension) — sidecars are keyed by id, so we sweep the
|
|
1225
|
+
// sidecarById map.
|
|
1226
|
+
const relPath = relative(outDir, filepath).replace(/\\/g, "/");
|
|
1227
|
+
// Strip extension to get the canonical path stored in the sidecar
|
|
1228
|
+
// (vault paths don't carry extensions).
|
|
1229
|
+
const pathNoExt = relPath.slice(0, -(ext.length + 1));
|
|
1230
|
+
let foundId: string | null = null;
|
|
1231
|
+
for (const [id, info] of sidecarById.entries()) {
|
|
1232
|
+
if (
|
|
1233
|
+
info.path === pathNoExt &&
|
|
1234
|
+
(info.extension ?? "md").toLowerCase() === ext
|
|
1235
|
+
) {
|
|
1236
|
+
foundId = id;
|
|
1237
|
+
break;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
if (!foundId) {
|
|
1241
|
+
// Content file with no sidecar — can't tell which note owns it.
|
|
1242
|
+
// Conservative: leave alone, record as unparseable.
|
|
1243
|
+
stats.unparseable_files.push({
|
|
1244
|
+
path: filepath,
|
|
1245
|
+
reason: "no sidecar metadata could be matched by (path, extension)",
|
|
1246
|
+
});
|
|
1247
|
+
continue;
|
|
1248
|
+
}
|
|
1249
|
+
pairedSidecarIds.add(foundId);
|
|
1250
|
+
if (!opts.validNoteIds.has(foundId)) {
|
|
1251
|
+
// Note is orphaned — remove both content and sidecar.
|
|
1252
|
+
safeRm(filepath, () => {
|
|
1253
|
+
stats.notes_removed++;
|
|
1254
|
+
});
|
|
1255
|
+
const sidecarPath = sidecarFilesById.get(foundId);
|
|
1256
|
+
if (sidecarPath) {
|
|
1257
|
+
safeRm(sidecarPath, () => {
|
|
1258
|
+
stats.sidecars_removed++;
|
|
1259
|
+
});
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// Sweep up orphaned sidecars (sidecar exists but no content file
|
|
1266
|
+
// matched OR sidecar's id isn't in validNoteIds).
|
|
1267
|
+
for (const [id, sidecarPath] of sidecarFilesById.entries()) {
|
|
1268
|
+
if (opts.validNoteIds.has(id) && pairedSidecarIds.has(id)) continue;
|
|
1269
|
+
if (opts.validNoteIds.has(id) && !pairedSidecarIds.has(id)) {
|
|
1270
|
+
// Sidecar refers to a valid note but the content file is gone —
|
|
1271
|
+
// that's an inconsistency, not an orphan. Leave the sidecar so
|
|
1272
|
+
// the next export can rewrite the content file alongside it.
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
safeRm(sidecarPath, () => {
|
|
1276
|
+
stats.sidecars_removed++;
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
// ---- 3. Schema sidecars ----
|
|
1281
|
+
if (existsSync(schemasRoot)) {
|
|
1282
|
+
try {
|
|
1283
|
+
for (const entry of readdirSync(schemasRoot)) {
|
|
1284
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
1285
|
+
const full = join(schemasRoot, entry);
|
|
1286
|
+
try {
|
|
1287
|
+
const text = readFileSync(full, "utf-8");
|
|
1288
|
+
const { frontmatter } = parseFrontmatter(`---\n${text}---\n`);
|
|
1289
|
+
const name = typeof frontmatter.name === "string" ? (frontmatter.name as string) : null;
|
|
1290
|
+
if (!name) {
|
|
1291
|
+
// Fall back to filename — sanitizeTagFilename replaces `/`
|
|
1292
|
+
// with `__`, so reverse for the lookup.
|
|
1293
|
+
const fromFilename = entry.slice(0, -5).replace(/__/g, "/");
|
|
1294
|
+
if (!opts.validTagNames.has(fromFilename)) {
|
|
1295
|
+
safeRm(full, () => {
|
|
1296
|
+
stats.schemas_removed++;
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
continue;
|
|
1300
|
+
}
|
|
1301
|
+
if (!opts.validTagNames.has(name)) {
|
|
1302
|
+
safeRm(full, () => {
|
|
1303
|
+
stats.schemas_removed++;
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
} catch (err) {
|
|
1307
|
+
stats.unparseable_files.push({
|
|
1308
|
+
path: full,
|
|
1309
|
+
reason: `schema sweep failed: ${(err as Error).message ?? err}`,
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
} catch (err) {
|
|
1314
|
+
stats.unparseable_files.push({
|
|
1315
|
+
path: schemasRoot,
|
|
1316
|
+
reason: `schemas walk failed: ${(err as Error).message ?? err}`,
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
// ---- 4. Attachment directories ----
|
|
1322
|
+
if (existsSync(attachmentsRoot)) {
|
|
1323
|
+
try {
|
|
1324
|
+
for (const entry of readdirSync(attachmentsRoot)) {
|
|
1325
|
+
const full = join(attachmentsRoot, entry);
|
|
1326
|
+
let stat;
|
|
1327
|
+
try {
|
|
1328
|
+
stat = statSync(full);
|
|
1329
|
+
} catch (err) {
|
|
1330
|
+
stats.unparseable_files.push({
|
|
1331
|
+
path: full,
|
|
1332
|
+
reason: `stat failed: ${(err as Error).message ?? err}`,
|
|
1333
|
+
});
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
if (!stat.isDirectory()) continue;
|
|
1337
|
+
// The directory name IS the attachment id (per the export layout).
|
|
1338
|
+
if (!opts.validAttachmentIds.has(entry)) {
|
|
1339
|
+
safeRm(
|
|
1340
|
+
full,
|
|
1341
|
+
() => {
|
|
1342
|
+
stats.attachment_dirs_removed++;
|
|
1343
|
+
},
|
|
1344
|
+
{ recursive: true },
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
} catch (err) {
|
|
1349
|
+
stats.unparseable_files.push({
|
|
1350
|
+
path: attachmentsRoot,
|
|
1351
|
+
reason: `attachments walk failed: ${(err as Error).message ?? err}`,
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
return stats;
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
/**
|
|
1360
|
+
* True iff the tag carries content the export writer will emit as a
|
|
1361
|
+
* schema sidecar (`.parachute/schemas/<tag>.yaml`). Bare-name tags
|
|
1362
|
+
* (the `tags` table has a row but description/fields/relationships/
|
|
1363
|
+
* parents are all empty) get no sidecar — and crucially, after
|
|
1364
|
+
* `deleteTagSchema` clears those fields the row persists with the
|
|
1365
|
+
* bare name. Callers building the `validTagNames` set for
|
|
1366
|
+
* `pruneOrphans` MUST filter through this predicate, otherwise the
|
|
1367
|
+
* stale sidecar lingers indefinitely.
|
|
1368
|
+
*
|
|
1369
|
+
* Reviewer-flagged on vault#382: without this filter, a cleared
|
|
1370
|
+
* schema's sidecar never gets pruned.
|
|
1371
|
+
*/
|
|
1372
|
+
export function hasSchemaContent(tag: TagRecord): boolean {
|
|
1005
1373
|
if (tag.description !== undefined && tag.description.length > 0) return true;
|
|
1006
1374
|
if (tag.fields && Object.keys(tag.fields).length > 0) return true;
|
|
1007
1375
|
if (tag.relationships && Object.keys(tag.relationships).length > 0) return true;
|
|
@@ -1167,6 +1535,14 @@ export interface ImportStats {
|
|
|
1167
1535
|
skipped_sidecars: Array<{ sidecar_id: string; expected_path: string | null; expected_extension: string | null; reason: string }>;
|
|
1168
1536
|
/** Set when the caller passed `blowAway: true`; counts notes removed. */
|
|
1169
1537
|
notes_wiped: number;
|
|
1538
|
+
/**
|
|
1539
|
+
* (tag, field) indexed-field declarations replayed after restoring tag
|
|
1540
|
+
* schemas — materializes the generated columns + indexes a live vault
|
|
1541
|
+
* would have. Without this an imported vault's schemas say `indexed: true`
|
|
1542
|
+
* but the backing columns don't exist until each tag is next `update-tag`'d
|
|
1543
|
+
* (queries fall back to full scans). See the import re-declare fix.
|
|
1544
|
+
*/
|
|
1545
|
+
indexes_declared: number;
|
|
1170
1546
|
}
|
|
1171
1547
|
|
|
1172
1548
|
/**
|
|
@@ -1214,6 +1590,7 @@ export async function importPortableVault(
|
|
|
1214
1590
|
skipped_attachments: [],
|
|
1215
1591
|
skipped_sidecars: [],
|
|
1216
1592
|
notes_wiped: 0,
|
|
1593
|
+
indexes_declared: 0,
|
|
1217
1594
|
};
|
|
1218
1595
|
|
|
1219
1596
|
// 1. Optional wipe. Notes are deleted via the public Store API so
|
|
@@ -1651,6 +2028,45 @@ export async function importPortableVault(
|
|
|
1651
2028
|
await store.syncAllWikilinks();
|
|
1652
2029
|
}
|
|
1653
2030
|
|
|
2031
|
+
// 7. Re-declare indexed fields (belt-and-suspenders + authoritative count).
|
|
2032
|
+
// Step 2 restored tag schemas via `store.upsertTagRecord`, which — now that
|
|
2033
|
+
// the indexed-field lifecycle is centralized in the store — already
|
|
2034
|
+
// materializes the backing generated columns + indexes as it persists each
|
|
2035
|
+
// schema. This explicit reconcile is therefore idempotent on the happy path;
|
|
2036
|
+
// it stays as a safety net (covers any schema written through a path that
|
|
2037
|
+
// skipped the lifecycle) and gives the authoritative `indexes_declared`
|
|
2038
|
+
// count. Without it, a regression in step 2 would silently leave the
|
|
2039
|
+
// imported schemas advertising `indexed: true` while queries full-scan.
|
|
2040
|
+
if (!opts.dryRun) {
|
|
2041
|
+
stats.indexes_declared = await store.reconcileDeclaredIndexes();
|
|
2042
|
+
} else {
|
|
2043
|
+
// Dry-run: count what WOULD be declared without touching the DB. Both
|
|
2044
|
+
// paths count per (tag, field) declaration (a co-declared field counts
|
|
2045
|
+
// once per declaring tag). The one asymmetry: this dry-run counts every
|
|
2046
|
+
// `indexed: true` field including unsupported types, whereas the applied
|
|
2047
|
+
// `reconcileDeclaredIndexes` skips fields whose type can't be indexed —
|
|
2048
|
+
// so the dry-run can over-count by the number of mis-typed indexed
|
|
2049
|
+
// fields. It's a "how much indexing work" signal, not a row-exact promise.
|
|
2050
|
+
const schemasDir2 = join(sidecar, "schemas");
|
|
2051
|
+
if (existsSync(schemasDir2)) {
|
|
2052
|
+
for (const entry of readdirSync(schemasDir2)) {
|
|
2053
|
+
if (!entry.endsWith(".yaml")) continue;
|
|
2054
|
+
const fullPath = join(schemasDir2, entry);
|
|
2055
|
+
const resolved = resolvePath(fullPath);
|
|
2056
|
+
if (!isWithinDir(resolved, resolvePath(schemasDir2))) continue;
|
|
2057
|
+
const text = readFileSync(fullPath, "utf-8");
|
|
2058
|
+
const wrapped = `---\n${text}${text.endsWith("\n") ? "" : "\n"}---\n`;
|
|
2059
|
+
const { frontmatter } = parseFrontmatter(wrapped);
|
|
2060
|
+
const fields = frontmatter.fields;
|
|
2061
|
+
if (fields && typeof fields === "object" && !Array.isArray(fields)) {
|
|
2062
|
+
for (const spec of Object.values(fields as Record<string, { indexed?: boolean }>)) {
|
|
2063
|
+
if (spec?.indexed === true) stats.indexes_declared++;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
1654
2070
|
return stats;
|
|
1655
2071
|
}
|
|
1656
2072
|
|
package/core/src/schema.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
|
|
|
2
2
|
import { normalizePath } from "./paths.js";
|
|
3
3
|
import { rebuildIndexes } from "./indexed-fields.js";
|
|
4
4
|
|
|
5
|
-
export const SCHEMA_VERSION =
|
|
5
|
+
export const SCHEMA_VERSION = 20;
|
|
6
6
|
|
|
7
7
|
export const SCHEMA_SQL = `
|
|
8
8
|
-- Notes: the universal record.
|
|
@@ -118,6 +118,20 @@ CREATE TABLE IF NOT EXISTS indexed_fields (
|
|
|
118
118
|
--
|
|
119
119
|
-- scope_tag / scope_path_prefix are deprecated Phase-0 columns — never
|
|
120
120
|
-- enforced at runtime, kept only for schema stability.
|
|
121
|
+
-- created_via (v19) records the provenance of a token. NULL means the
|
|
122
|
+
-- legacy/unspecified path (CLI, REST mint, YAML import); 'mcp_mint' means
|
|
123
|
+
-- the token was minted by the manage-token MCP tool, which lets the
|
|
124
|
+
-- list/revoke surface of that tool restrict itself to its own session's
|
|
125
|
+
-- mints (no cross-session token management from inside MCP).
|
|
126
|
+
--
|
|
127
|
+
-- parent_jti (v19) is the display id (t_hashprefix) of the token that
|
|
128
|
+
-- minted this one, or the hub-JWT jti claim when minted from a hub
|
|
129
|
+
-- session. Session-pinned list+revoke in manage-token filters on this.
|
|
130
|
+
--
|
|
131
|
+
-- revoked_at (v19) marks soft-revocation. Revoke from manage-token sets
|
|
132
|
+
-- this rather than deleting the row, so the audit trail stays intact and
|
|
133
|
+
-- the second revoke of the same jti is idempotent (returns ok=true).
|
|
134
|
+
-- resolveToken treats a revoked_at-set row as not-found.
|
|
121
135
|
CREATE TABLE IF NOT EXISTS tokens (
|
|
122
136
|
token_hash TEXT PRIMARY KEY,
|
|
123
137
|
label TEXT NOT NULL,
|
|
@@ -129,7 +143,34 @@ CREATE TABLE IF NOT EXISTS tokens (
|
|
|
129
143
|
expires_at TEXT,
|
|
130
144
|
created_at TEXT NOT NULL,
|
|
131
145
|
last_used_at TEXT,
|
|
132
|
-
vault_name TEXT
|
|
146
|
+
vault_name TEXT,
|
|
147
|
+
created_via TEXT,
|
|
148
|
+
parent_jti TEXT,
|
|
149
|
+
revoked_at TEXT
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
-- mcp_mint_ledger (v20) — session-pinned record of HUB JWTs minted by the
|
|
153
|
+
-- manage-token MCP tool (vault#403, MGT). After the auth-unification arc,
|
|
154
|
+
-- manage-token mints hub JWTs (via hub mint-token attenuation proxy), not
|
|
155
|
+
-- pvt_* vault-DB tokens — so the mints no longer live in the tokens table.
|
|
156
|
+
-- This ledger is a lightweight local index of (parent_jti to minted hub jti)
|
|
157
|
+
-- so the tool list/revoke surface can stay session-scoped: a session sees
|
|
158
|
+
-- and revokes only the hub JWTs it minted. Rows are NOT credentials — the
|
|
159
|
+
-- signed JWT is never stored, only its jti (the revocation handle) plus
|
|
160
|
+
-- display metadata. revoked_at mirrors the tokens-table soft-revoke marker
|
|
161
|
+
-- so a second revoke is idempotent even if the hub round-trip is skipped.
|
|
162
|
+
-- The authoritative revocation state lives in hub token registry; this is
|
|
163
|
+
-- the attribution index only.
|
|
164
|
+
CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
|
|
165
|
+
jti TEXT PRIMARY KEY,
|
|
166
|
+
parent_jti TEXT NOT NULL,
|
|
167
|
+
vault_name TEXT NOT NULL,
|
|
168
|
+
label TEXT NOT NULL,
|
|
169
|
+
scopes TEXT,
|
|
170
|
+
scoped_tags TEXT,
|
|
171
|
+
created_at TEXT NOT NULL,
|
|
172
|
+
expires_at TEXT,
|
|
173
|
+
revoked_at TEXT
|
|
133
174
|
);
|
|
134
175
|
|
|
135
176
|
-- OAuth: registered clients (Dynamic Client Registration)
|
|
@@ -197,6 +238,10 @@ CREATE INDEX IF NOT EXISTS idx_note_tags_tag ON note_tags(tag_name, note_id);
|
|
|
197
238
|
CREATE INDEX IF NOT EXISTS idx_attachments_note ON attachments(note_id);
|
|
198
239
|
CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_id);
|
|
199
240
|
CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_id);
|
|
241
|
+
-- Session-pinned manage-token ledger lookup (v20): list/revoke scope on
|
|
242
|
+
-- (parent_jti, vault_name). The mcp_mint_ledger table is created
|
|
243
|
+
-- unconditionally above, so this index is safe in SCHEMA_SQL.
|
|
244
|
+
CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name);
|
|
200
245
|
-- idx_tokens_vault_name is created in migrateToV16, not here. SCHEMA_SQL
|
|
201
246
|
-- runs BEFORE migrations; an upgrading v15 vault doesn't yet have the
|
|
202
247
|
-- vault_name column when this section evaluates, so the index has to
|
|
@@ -380,6 +425,19 @@ export function initSchema(db: Database): void {
|
|
|
380
425
|
// (markdown), unchanged in meaning. See vault#328.
|
|
381
426
|
migrateToV18(db);
|
|
382
427
|
|
|
428
|
+
// Migrate v18 → v19: add MCP-mint provenance columns to `tokens`
|
|
429
|
+
// (created_via, parent_jti, revoked_at) for vault#376's manage-token tool.
|
|
430
|
+
// All three are nullable; existing rows backfill to NULL which means
|
|
431
|
+
// "non-MCP-minted, not revoked" — identical pre-v19 semantics.
|
|
432
|
+
migrateToV19(db);
|
|
433
|
+
|
|
434
|
+
// Migrate v19 → v20: the `mcp_mint_ledger` table (session-pinned record of
|
|
435
|
+
// hub JWTs minted by the manage-token MCP tool). Created by SCHEMA_SQL's
|
|
436
|
+
// `CREATE TABLE IF NOT EXISTS` above, so this is a no-op confirmation hook
|
|
437
|
+
// — present for symmetry with the other migration steps and to anchor the
|
|
438
|
+
// version bump. See vault#403 (MGT — manage-token mints hub JWTs).
|
|
439
|
+
migrateToV20(db);
|
|
440
|
+
|
|
383
441
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
384
442
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
385
443
|
rebuildIndexes(db);
|
|
@@ -952,6 +1010,60 @@ function migrateToV18(db: Database): void {
|
|
|
952
1010
|
}
|
|
953
1011
|
}
|
|
954
1012
|
|
|
1013
|
+
/**
|
|
1014
|
+
* Migrate v18 → v19: add `created_via`, `parent_jti`, `revoked_at` columns
|
|
1015
|
+
* to `tokens` so manage-token can attribute mints to the MCP session that
|
|
1016
|
+
* minted them, scope its list+revoke to those tokens, and soft-revoke
|
|
1017
|
+
* (preserving the audit trail).
|
|
1018
|
+
*
|
|
1019
|
+
* All three columns are nullable; existing rows backfill to NULL with
|
|
1020
|
+
* back-compat semantics — `created_via IS NULL` matches the CLI/REST-mint
|
|
1021
|
+
* provenance, `revoked_at IS NULL` matches "still active". Idempotent —
|
|
1022
|
+
* the column-existence guard means the migration is safe to re-run on a
|
|
1023
|
+
* post-v19 vault. See vault#376.
|
|
1024
|
+
*/
|
|
1025
|
+
function migrateToV19(db: Database): void {
|
|
1026
|
+
if (!hasTable(db, "tokens")) return;
|
|
1027
|
+
const cols: [string, string][] = [
|
|
1028
|
+
["created_via", "TEXT"],
|
|
1029
|
+
["parent_jti", "TEXT"],
|
|
1030
|
+
["revoked_at", "TEXT"],
|
|
1031
|
+
];
|
|
1032
|
+
for (const [col, type] of cols) {
|
|
1033
|
+
if (!hasColumn(db, "tokens", col)) {
|
|
1034
|
+
db.exec(`ALTER TABLE tokens ADD COLUMN ${col} ${type}`);
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
/**
|
|
1040
|
+
* Migrate v19 → v20: ensure the `mcp_mint_ledger` table exists. SCHEMA_SQL's
|
|
1041
|
+
* `CREATE TABLE IF NOT EXISTS` already covers fresh vaults AND upgrading
|
|
1042
|
+
* vaults (SCHEMA_SQL runs unconditionally on every open before the migration
|
|
1043
|
+
* steps), so this is a defensive no-op confirmation — there is no ALTER to
|
|
1044
|
+
* perform. Kept as a named step so the version-bump arc reads cleanly and a
|
|
1045
|
+
* future column addition has an obvious home. See vault#403 (MGT).
|
|
1046
|
+
*/
|
|
1047
|
+
function migrateToV20(db: Database): void {
|
|
1048
|
+
if (hasTable(db, "mcp_mint_ledger")) return;
|
|
1049
|
+
db.exec(`
|
|
1050
|
+
CREATE TABLE IF NOT EXISTS mcp_mint_ledger (
|
|
1051
|
+
jti TEXT PRIMARY KEY,
|
|
1052
|
+
parent_jti TEXT NOT NULL,
|
|
1053
|
+
vault_name TEXT NOT NULL,
|
|
1054
|
+
label TEXT NOT NULL,
|
|
1055
|
+
scopes TEXT,
|
|
1056
|
+
scoped_tags TEXT,
|
|
1057
|
+
created_at TEXT NOT NULL,
|
|
1058
|
+
expires_at TEXT,
|
|
1059
|
+
revoked_at TEXT
|
|
1060
|
+
)
|
|
1061
|
+
`);
|
|
1062
|
+
db.exec(
|
|
1063
|
+
"CREATE INDEX IF NOT EXISTS idx_mcp_mint_ledger_session ON mcp_mint_ledger(parent_jti, vault_name)",
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
955
1067
|
function hasTable(db: Database, name: string): boolean {
|
|
956
1068
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
957
1069
|
return !!row;
|