@openparachute/vault 0.6.0-rc.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.parachute/module.json +14 -3
- package/README.md +7 -7
- package/core/src/core.test.ts +279 -26
- package/core/src/expand-visibility.test.ts +102 -0
- package/core/src/expand.ts +31 -3
- package/core/src/indexed-fields.ts +1 -1
- package/core/src/link-count.test.ts +301 -0
- package/core/src/links.ts +97 -2
- package/core/src/mcp.ts +201 -33
- package/core/src/notes.ts +44 -8
- package/core/src/obsidian-alignment.test.ts +375 -0
- package/core/src/obsidian.ts +234 -14
- package/core/src/portable-md.test.ts +40 -0
- package/core/src/portable-md.ts +142 -16
- package/core/src/schema.ts +58 -11
- package/core/src/store.ts +69 -22
- package/core/src/tag-expand-axis.test.ts +301 -0
- package/core/src/tag-hierarchy.ts +80 -0
- package/core/src/tag-schemas.ts +61 -46
- package/core/src/triggers-store.test.ts +100 -0
- package/core/src/triggers-store.ts +165 -0
- package/core/src/types.ts +68 -4
- package/core/src/vault-projection.ts +20 -0
- package/core/src/wikilinks.ts +2 -2
- package/package.json +2 -3
- package/src/admin-spa.test.ts +100 -10
- package/src/admin-spa.ts +48 -3
- package/src/auth-hub-jwt.test.ts +8 -1
- package/src/auth-status.ts +2 -2
- package/src/auth.test.ts +39 -3
- package/src/auth.ts +31 -2
- package/src/auto-transcribe.test.ts +51 -0
- package/src/auto-transcribe.ts +24 -6
- package/src/autostart.test.ts +75 -0
- package/src/autostart.ts +84 -0
- package/src/cli.ts +434 -140
- package/src/config.test.ts +109 -0
- package/src/config.ts +157 -10
- package/src/export-watch.test.ts +23 -0
- package/src/export-watch.ts +14 -0
- package/src/git-preflight.test.ts +70 -0
- package/src/git-preflight.ts +68 -0
- package/src/hub-jwt.test.ts +75 -2
- package/src/hub-jwt.ts +43 -6
- package/src/init-summary.test.ts +120 -5
- package/src/init-summary.ts +67 -25
- package/src/live-match.test.ts +198 -0
- package/src/live-match.ts +310 -0
- package/src/mcp-install.test.ts +93 -0
- package/src/mcp-install.ts +106 -0
- package/src/mcp-tools.ts +80 -7
- package/src/mirror-config.test.ts +14 -0
- package/src/mirror-config.ts +11 -0
- package/src/mirror-import.test.ts +110 -0
- package/src/mirror-import.ts +71 -13
- package/src/mirror-manager.test.ts +51 -0
- package/src/mirror-manager.ts +73 -11
- package/src/mirror-routes.test.ts +463 -1
- package/src/mirror-routes.ts +474 -4
- package/src/oauth-discovery.test.ts +55 -0
- package/src/oauth-discovery.ts +24 -5
- package/src/routes.ts +696 -121
- package/src/routing.test.ts +451 -5
- package/src/routing.ts +113 -5
- package/src/scopes.ts +1 -1
- package/src/server.ts +66 -4
- package/src/storage.test.ts +162 -0
- package/src/subscribe.test.ts +588 -0
- package/src/subscribe.ts +248 -0
- package/src/subscriptions.ts +295 -0
- package/src/tag-expand-routes.test.ts +45 -0
- package/src/tag-scope.ts +68 -1
- package/src/token-store.ts +7 -7
- package/src/transcription-worker.test.ts +471 -5
- package/src/transcription-worker.ts +212 -44
- package/src/triggers-api.test.ts +533 -0
- package/src/triggers-api.ts +295 -0
- package/src/triggers.ts +93 -7
- package/src/usage.test.ts +362 -0
- package/src/usage.ts +318 -0
- package/src/vault-create.test.ts +340 -12
- package/src/vault-name.test.ts +61 -3
- package/src/vault-name.ts +62 -14
- package/src/vault-remove.test.ts +187 -0
- package/src/vault-store.ts +10 -3
- package/src/vault.test.ts +1353 -62
- package/web/ui/dist/assets/index-CGL256oe.js +60 -0
- package/web/ui/dist/assets/index-J0pVP7I-.css +1 -0
- package/web/ui/dist/index.html +2 -2
- package/web/ui/dist/assets/index-DBe8Xiah.css +0 -1
- package/web/ui/dist/assets/index-DDRo6F4u.js +0 -60
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the usage helpers (src/usage.ts).
|
|
3
|
+
*
|
|
4
|
+
* Everything here runs against the injectable `UsageFs` seam — no real disk
|
|
5
|
+
* I/O — so we can (a) synthesize trees with symlinks/missing dirs and (b)
|
|
6
|
+
* count how many times the dir-walk actually runs, which is how we prove the
|
|
7
|
+
* TTL cache skips the walk on a hit.
|
|
8
|
+
*
|
|
9
|
+
* The path helpers (`vaultDir`, `assetsDir`, mirror resolution) DO read
|
|
10
|
+
* `process.env.PARACHUTE_HOME`; we point it at a tmp dir so the resolved paths
|
|
11
|
+
* are deterministic, but no files are written there — the fake fs intercepts
|
|
12
|
+
* every stat/readdir.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, test, expect, beforeEach } from "bun:test";
|
|
16
|
+
import { join } from "path";
|
|
17
|
+
import { tmpdir } from "os";
|
|
18
|
+
|
|
19
|
+
const testDir = join(
|
|
20
|
+
tmpdir(),
|
|
21
|
+
`vault-usage-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
|
|
22
|
+
);
|
|
23
|
+
process.env.PARACHUTE_HOME = testDir;
|
|
24
|
+
|
|
25
|
+
const {
|
|
26
|
+
dbBytes,
|
|
27
|
+
dirSize,
|
|
28
|
+
UsageCache,
|
|
29
|
+
buildUsageReport,
|
|
30
|
+
} = await import("./usage.ts");
|
|
31
|
+
const { vaultDir, assetsDir } = await import("./config.ts");
|
|
32
|
+
|
|
33
|
+
import type { UsageFs } from "./usage.ts";
|
|
34
|
+
import type { VaultStats } from "../core/src/types.ts";
|
|
35
|
+
import type { Dirent } from "fs";
|
|
36
|
+
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// Fake filesystem builder
|
|
39
|
+
//
|
|
40
|
+
// A node is either a file (number = size in bytes), a dir (object mapping
|
|
41
|
+
// names → nodes), or a symlink (special marker, never followed).
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
type FileNode = { kind: "file"; size: number };
|
|
45
|
+
type DirNode = { kind: "dir"; children: Record<string, FsNode> };
|
|
46
|
+
type LinkNode = { kind: "link" };
|
|
47
|
+
type FsNode = FileNode | DirNode | LinkNode;
|
|
48
|
+
|
|
49
|
+
const file = (size: number): FileNode => ({ kind: "file", size });
|
|
50
|
+
const dir = (children: Record<string, FsNode>): DirNode => ({ kind: "dir", children });
|
|
51
|
+
const link = (): LinkNode => ({ kind: "link" });
|
|
52
|
+
|
|
53
|
+
function makeDirent(name: string, node: FsNode): Dirent {
|
|
54
|
+
return {
|
|
55
|
+
name,
|
|
56
|
+
isFile: () => node.kind === "file",
|
|
57
|
+
isDirectory: () => node.kind === "dir",
|
|
58
|
+
isSymbolicLink: () => node.kind === "link",
|
|
59
|
+
isBlockDevice: () => false,
|
|
60
|
+
isCharacterDevice: () => false,
|
|
61
|
+
isFIFO: () => false,
|
|
62
|
+
isSocket: () => false,
|
|
63
|
+
} as unknown as Dirent;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Build a fake `UsageFs` rooted at a set of absolute paths. `roots` maps an
|
|
68
|
+
* absolute path → the node that lives there. Lookups resolve a requested
|
|
69
|
+
* absolute path by walking from the matching root prefix. `readCount` exposes
|
|
70
|
+
* how many `readDir` calls happened (for cache assertions).
|
|
71
|
+
*/
|
|
72
|
+
function makeFakeFs(roots: Record<string, FsNode>): UsageFs & { readCount: number } {
|
|
73
|
+
function resolve(path: string): FsNode | undefined {
|
|
74
|
+
// Exact root match first.
|
|
75
|
+
if (roots[path]) return roots[path];
|
|
76
|
+
// Otherwise find the root that's a prefix and descend by segment.
|
|
77
|
+
for (const [rootPath, rootNode] of Object.entries(roots)) {
|
|
78
|
+
if (path === rootPath) return rootNode;
|
|
79
|
+
if (path.startsWith(rootPath + "/")) {
|
|
80
|
+
const rest = path.slice(rootPath.length + 1).split("/");
|
|
81
|
+
let cur: FsNode | undefined = rootNode;
|
|
82
|
+
for (const seg of rest) {
|
|
83
|
+
if (!cur || cur.kind !== "dir") return undefined;
|
|
84
|
+
cur = cur.children[seg];
|
|
85
|
+
}
|
|
86
|
+
return cur;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const fs = {
|
|
93
|
+
readCount: 0,
|
|
94
|
+
statFile(path: string) {
|
|
95
|
+
const node = resolve(path);
|
|
96
|
+
if (!node) throw new Error(`ENOENT: ${path}`);
|
|
97
|
+
return {
|
|
98
|
+
size: node.kind === "file" ? node.size : 0,
|
|
99
|
+
isDirectory: () => node.kind === "dir",
|
|
100
|
+
isSymbolicLink: () => node.kind === "link",
|
|
101
|
+
};
|
|
102
|
+
},
|
|
103
|
+
readDir(path: string): Dirent[] {
|
|
104
|
+
fs.readCount++;
|
|
105
|
+
const node = resolve(path);
|
|
106
|
+
if (!node || node.kind !== "dir") throw new Error(`ENOTDIR: ${path}`);
|
|
107
|
+
return Object.entries(node.children).map(([name, child]) => makeDirent(name, child));
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
return fs;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
// dbBytes — sums the WAL trio (vault.db + -wal + -shm)
|
|
115
|
+
// ---------------------------------------------------------------------------
|
|
116
|
+
|
|
117
|
+
describe("dbBytes (WAL-aware DB file sizing)", () => {
|
|
118
|
+
const VAULT = "journal";
|
|
119
|
+
const dbBase = join(vaultDir(VAULT), "vault.db");
|
|
120
|
+
|
|
121
|
+
test("sums vault.db + vault.db-wal + vault.db-shm", () => {
|
|
122
|
+
const fs = makeFakeFs({
|
|
123
|
+
[dbBase]: file(4096),
|
|
124
|
+
[`${dbBase}-wal`]: file(800),
|
|
125
|
+
[`${dbBase}-shm`]: file(32),
|
|
126
|
+
});
|
|
127
|
+
expect(dbBytes(VAULT, fs)).toBe(4096 + 800 + 32);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("tolerates missing -wal/-shm (checkpointed at rest)", () => {
|
|
131
|
+
const fs = makeFakeFs({ [dbBase]: file(4096) });
|
|
132
|
+
// -wal and -shm absent → contribute 0, not an error.
|
|
133
|
+
expect(dbBytes(VAULT, fs)).toBe(4096);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test("missing DB entirely → 0", () => {
|
|
137
|
+
const fs = makeFakeFs({});
|
|
138
|
+
expect(dbBytes(VAULT, fs)).toBe(0);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
// dirSize — recursive, missing-dir tolerant, symlink-safe
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
describe("dirSize (recursive directory byte sum)", () => {
|
|
147
|
+
const ROOT = "/fake/assets";
|
|
148
|
+
|
|
149
|
+
test("sums files across nested directories", () => {
|
|
150
|
+
const fs = makeFakeFs({
|
|
151
|
+
[ROOT]: dir({
|
|
152
|
+
"a.png": file(100),
|
|
153
|
+
"2026-06-03": dir({
|
|
154
|
+
"x.jpg": file(250),
|
|
155
|
+
nested: dir({ "y.pdf": file(50) }),
|
|
156
|
+
}),
|
|
157
|
+
}),
|
|
158
|
+
});
|
|
159
|
+
expect(dirSize(ROOT, fs)).toBe(100 + 250 + 50);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
test("empty directory → 0", () => {
|
|
163
|
+
const fs = makeFakeFs({ [ROOT]: dir({}) });
|
|
164
|
+
expect(dirSize(ROOT, fs)).toBe(0);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("missing directory → 0 (no throw)", () => {
|
|
168
|
+
const fs = makeFakeFs({});
|
|
169
|
+
expect(dirSize(ROOT, fs)).toBe(0);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("does NOT follow symlinks (file or dir)", () => {
|
|
173
|
+
const fs = makeFakeFs({
|
|
174
|
+
[ROOT]: dir({
|
|
175
|
+
"real.png": file(100),
|
|
176
|
+
"linked-file": link(), // would be a file if followed
|
|
177
|
+
"linked-dir": link(), // would be a dir if followed
|
|
178
|
+
}),
|
|
179
|
+
// A target tree the symlink "points at" — if dirSize followed the link
|
|
180
|
+
// it would walk this and add 9999. It must NOT.
|
|
181
|
+
[join(ROOT, "linked-dir")]: dir({ "huge.bin": file(9999) }),
|
|
182
|
+
});
|
|
183
|
+
expect(dirSize(ROOT, fs)).toBe(100);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test("symlink loop does not hang (link is skipped, never descended)", () => {
|
|
187
|
+
// The classic infinite-walk trap: a dir containing a symlink to itself.
|
|
188
|
+
// Because we skip symlinks outright, this terminates immediately.
|
|
189
|
+
const fs = makeFakeFs({
|
|
190
|
+
[ROOT]: dir({
|
|
191
|
+
"f.png": file(10),
|
|
192
|
+
loop: link(),
|
|
193
|
+
}),
|
|
194
|
+
});
|
|
195
|
+
expect(dirSize(ROOT, fs)).toBe(10);
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
// ---------------------------------------------------------------------------
|
|
200
|
+
// UsageCache — 60s TTL, fresh bypass, invalidation, call-count proof
|
|
201
|
+
// ---------------------------------------------------------------------------
|
|
202
|
+
|
|
203
|
+
describe("UsageCache (dir-walk TTL cache)", () => {
|
|
204
|
+
const VAULT = "journal";
|
|
205
|
+
const assets = assetsDir(VAULT);
|
|
206
|
+
|
|
207
|
+
function fsWith(assetsBytes: number) {
|
|
208
|
+
return makeFakeFs({ [assets]: dir({ "a.png": file(assetsBytes) }) });
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
test("first read walks (cached:false); second read within TTL is cached (no walk)", () => {
|
|
212
|
+
const fs = fsWith(500);
|
|
213
|
+
let clock = 1_000;
|
|
214
|
+
const cache = new UsageCache(fs, () => clock, 60_000);
|
|
215
|
+
|
|
216
|
+
const first = cache.get(VAULT);
|
|
217
|
+
expect(first.cached).toBe(false);
|
|
218
|
+
expect(first.result.assets).toBe(500);
|
|
219
|
+
const afterFirst = fs.readCount;
|
|
220
|
+
expect(afterFirst).toBeGreaterThan(0);
|
|
221
|
+
|
|
222
|
+
clock += 30_000; // within the 60s TTL
|
|
223
|
+
const second = cache.get(VAULT);
|
|
224
|
+
expect(second.cached).toBe(true);
|
|
225
|
+
expect(second.result.assets).toBe(500);
|
|
226
|
+
// The cache MUST NOT have re-walked — call count is unchanged.
|
|
227
|
+
expect(fs.readCount).toBe(afterFirst);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("entry expires after TTL → re-walks (cached:false)", () => {
|
|
231
|
+
const fs = fsWith(500);
|
|
232
|
+
let clock = 1_000;
|
|
233
|
+
const cache = new UsageCache(fs, () => clock, 60_000);
|
|
234
|
+
|
|
235
|
+
cache.get(VAULT); // prime
|
|
236
|
+
const afterPrime = fs.readCount;
|
|
237
|
+
|
|
238
|
+
clock += 60_001; // just past TTL
|
|
239
|
+
const stale = cache.get(VAULT);
|
|
240
|
+
expect(stale.cached).toBe(false);
|
|
241
|
+
expect(fs.readCount).toBeGreaterThan(afterPrime);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
test("fresh:true bypasses a valid cache entry and re-walks", () => {
|
|
245
|
+
const fs = fsWith(500);
|
|
246
|
+
let clock = 1_000;
|
|
247
|
+
const cache = new UsageCache(fs, () => clock, 60_000);
|
|
248
|
+
|
|
249
|
+
cache.get(VAULT); // prime
|
|
250
|
+
const afterPrime = fs.readCount;
|
|
251
|
+
|
|
252
|
+
clock += 1_000; // well within TTL — a normal read would be cached
|
|
253
|
+
const forced = cache.get(VAULT, { fresh: true });
|
|
254
|
+
expect(forced.cached).toBe(false);
|
|
255
|
+
expect(fs.readCount).toBeGreaterThan(afterPrime);
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
test("invalidate() forces the next read to re-walk", () => {
|
|
259
|
+
const fs = fsWith(500);
|
|
260
|
+
let clock = 1_000;
|
|
261
|
+
const cache = new UsageCache(fs, () => clock, 60_000);
|
|
262
|
+
|
|
263
|
+
cache.get(VAULT); // prime
|
|
264
|
+
const afterPrime = fs.readCount;
|
|
265
|
+
|
|
266
|
+
cache.invalidate(VAULT);
|
|
267
|
+
clock += 1_000; // within TTL, but the entry is gone
|
|
268
|
+
const after = cache.get(VAULT);
|
|
269
|
+
expect(after.cached).toBe(false);
|
|
270
|
+
expect(fs.readCount).toBeGreaterThan(afterPrime);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
test("no mirror configured → mirror:null (omitted from report)", () => {
|
|
274
|
+
// No mirror-config.yaml written for this vault → resolveVaultMirrorDir
|
|
275
|
+
// returns null → mirror is null.
|
|
276
|
+
const fs = fsWith(500);
|
|
277
|
+
const cache = new UsageCache(fs, () => 1_000, 60_000);
|
|
278
|
+
const { result } = cache.get(VAULT);
|
|
279
|
+
expect(result.mirror).toBeNull();
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
// buildUsageReport — shape + total math + mirror handling
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
|
|
287
|
+
describe("buildUsageReport", () => {
|
|
288
|
+
const VAULT = "journal";
|
|
289
|
+
const dbBase = join(vaultDir(VAULT), "vault.db");
|
|
290
|
+
const assets = assetsDir(VAULT);
|
|
291
|
+
|
|
292
|
+
function makeStats(overrides: Partial<VaultStats> = {}): VaultStats {
|
|
293
|
+
return {
|
|
294
|
+
totalNotes: 12,
|
|
295
|
+
earliestNote: null,
|
|
296
|
+
latestNote: null,
|
|
297
|
+
notesByMonth: [],
|
|
298
|
+
topTags: [],
|
|
299
|
+
tagCount: 4,
|
|
300
|
+
attachmentCount: 3,
|
|
301
|
+
linkCount: 7,
|
|
302
|
+
contentBytes: 1234,
|
|
303
|
+
...overrides,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
test("full shape: counts, bytes, total = db + assets, mirror omitted when none", () => {
|
|
308
|
+
const fs = makeFakeFs({
|
|
309
|
+
[dbBase]: file(4096),
|
|
310
|
+
[`${dbBase}-wal`]: file(900),
|
|
311
|
+
[assets]: dir({ "a.png": file(2000) }),
|
|
312
|
+
});
|
|
313
|
+
const cache = new UsageCache(fs, () => 1_000, 60_000);
|
|
314
|
+
const report = buildUsageReport(VAULT, makeStats(), { cache, fs, now: () => 1_700_000_000_000 });
|
|
315
|
+
|
|
316
|
+
expect(report.counts).toEqual({ notes: 12, attachments: 3, links: 7, tags: 4 });
|
|
317
|
+
expect(report.bytes.content).toBe(1234);
|
|
318
|
+
expect(report.bytes.db).toBe(4096 + 900);
|
|
319
|
+
expect(report.bytes.assets).toBe(2000);
|
|
320
|
+
// total = db + assets only. NOT content (logical, already inside db) and
|
|
321
|
+
// NOT mirror (projection).
|
|
322
|
+
expect(report.bytes.total).toBe(4096 + 900 + 2000);
|
|
323
|
+
expect(report.bytes).not.toHaveProperty("mirror");
|
|
324
|
+
expect(report.cached).toBe(false);
|
|
325
|
+
expect(report.computedAt).toBe(new Date(1_700_000_000_000).toISOString());
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test("mirror is a separate line item, NOT added to total", () => {
|
|
329
|
+
// Configure an internal mirror so resolveVaultMirrorDir returns a dir.
|
|
330
|
+
const { writeMirrorConfigForVault, defaultMirrorConfig } = require("./mirror-config.ts");
|
|
331
|
+
writeMirrorConfigForVault(VAULT, { ...defaultMirrorConfig(), location: "internal", enabled: true });
|
|
332
|
+
const mirrorDir = join(vaultDir(VAULT), "mirror");
|
|
333
|
+
|
|
334
|
+
const fs = makeFakeFs({
|
|
335
|
+
[dbBase]: file(1000),
|
|
336
|
+
[assets]: dir({ "a.png": file(500) }),
|
|
337
|
+
[mirrorDir]: dir({ "note.md": file(8000) }),
|
|
338
|
+
});
|
|
339
|
+
const cache = new UsageCache(fs, () => 1_000, 60_000);
|
|
340
|
+
const report = buildUsageReport(VAULT, makeStats(), { cache, fs });
|
|
341
|
+
|
|
342
|
+
expect(report.bytes.mirror).toBe(8000);
|
|
343
|
+
// total stays db + assets — the 8000-byte mirror does not inflate it.
|
|
344
|
+
expect(report.bytes.total).toBe(1000 + 500);
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
test("cached flag reflects a cache hit", () => {
|
|
348
|
+
const fs = makeFakeFs({
|
|
349
|
+
[dbBase]: file(100),
|
|
350
|
+
[assets]: dir({}),
|
|
351
|
+
});
|
|
352
|
+
let clock = 1_000;
|
|
353
|
+
const cache = new UsageCache(fs, () => clock, 60_000);
|
|
354
|
+
|
|
355
|
+
const first = buildUsageReport(VAULT, makeStats(), { cache, fs });
|
|
356
|
+
expect(first.cached).toBe(false);
|
|
357
|
+
|
|
358
|
+
clock += 5_000;
|
|
359
|
+
const second = buildUsageReport(VAULT, makeStats(), { cache, fs });
|
|
360
|
+
expect(second.cached).toBe(true);
|
|
361
|
+
});
|
|
362
|
+
});
|
package/src/usage.ts
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-vault usage monitoring — a cheap, read-scoped "how big is this vault"
|
|
3
|
+
* report (data footprint). Built for running many vaults for many users: the
|
|
4
|
+
* operator needs to see each vault's size, and a vault's own user should be
|
|
5
|
+
* able to see their own vault's footprint.
|
|
6
|
+
*
|
|
7
|
+
* The endpoint that consumes these helpers lives in `routing.ts` at
|
|
8
|
+
* `GET /vault/<name>/.parachute/usage` (read-scoped). This module owns the
|
|
9
|
+
* filesystem arithmetic and the dir-walk cache.
|
|
10
|
+
*
|
|
11
|
+
* Cost model:
|
|
12
|
+
* - counts + contentBytes come from `getVaultStats` (a handful of indexed
|
|
13
|
+
* COUNT/SUM queries) — cheap, computed on every request.
|
|
14
|
+
* - `dbBytes` is three `statSync` calls (the WAL trio) — cheap, every request.
|
|
15
|
+
* - the two recursive directory walks (`assets`, `mirror`) can traverse
|
|
16
|
+
* thousands of files, so they go through a 60s TTL cache keyed by vault
|
|
17
|
+
* name. `?fresh=1` bypasses the cache; an attachment upload invalidates it.
|
|
18
|
+
*
|
|
19
|
+
* Everything I/O-touching goes through an injectable `UsageFs` seam + an
|
|
20
|
+
* injectable clock so tests can assert call counts and TTL behavior without
|
|
21
|
+
* touching the real disk.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
statSync as realStatSync,
|
|
26
|
+
readdirSync as realReaddirSync,
|
|
27
|
+
type Dirent,
|
|
28
|
+
} from "fs";
|
|
29
|
+
import { join } from "path";
|
|
30
|
+
import { vaultDir, assetsDir } from "./config.ts";
|
|
31
|
+
import { readMirrorConfigForVault, resolveMirrorPath } from "./mirror-config.ts";
|
|
32
|
+
import type { VaultStats } from "../core/src/types.ts";
|
|
33
|
+
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
// Injectable seams
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The minimal filesystem surface the usage helpers depend on. Injecting it
|
|
40
|
+
* lets tests count how many times the dir-walk runs (to prove the cache
|
|
41
|
+
* actually skips it) and synthesize a tree without writing real files.
|
|
42
|
+
*
|
|
43
|
+
* Symlink safety lives in the WALK, not in `statFile`: `dirSize` never
|
|
44
|
+
* descends into or sizes a symlink entry (it filters on
|
|
45
|
+
* `Dirent.isSymbolicLink()` before ever calling `statFile`), so a symlink
|
|
46
|
+
* loop can't hang the walk and a symlink pointing at a huge tree elsewhere
|
|
47
|
+
* can't inflate the count. `statFile` is therefore only ever called on real
|
|
48
|
+
* files / the WAL trio (which are never symlinks).
|
|
49
|
+
*/
|
|
50
|
+
export interface UsageFs {
|
|
51
|
+
/**
|
|
52
|
+
* `stat` (symlink-FOLLOWING, like `statSync`): returns the size of a file
|
|
53
|
+
* (or throws if absent — callers wrap in try/catch → 0). Following symlinks
|
|
54
|
+
* is safe here because dir descent already filters symlinks via
|
|
55
|
+
* `Dirent.isSymbolicLink()`, and the WAL files (`vault.db{,-wal,-shm}`) are
|
|
56
|
+
* never symlinks.
|
|
57
|
+
*/
|
|
58
|
+
statFile(path: string): { size: number; isDirectory(): boolean; isSymbolicLink(): boolean };
|
|
59
|
+
/** `readdirSync(path, { withFileTypes: true })`. */
|
|
60
|
+
readDir(path: string): Dirent[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Monotonic-enough clock seam: returns epoch millis. */
|
|
64
|
+
export type Clock = () => number;
|
|
65
|
+
|
|
66
|
+
/** Real filesystem implementation (production default). */
|
|
67
|
+
export const realUsageFs: UsageFs = {
|
|
68
|
+
statFile: (path) => realStatSync(path),
|
|
69
|
+
readDir: (path) => realReaddirSync(path, { withFileTypes: true }),
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
// Primitive byte counters
|
|
74
|
+
// ---------------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Physical bytes of a vault's SQLite database, WAL-aware: `vault.db` +
|
|
78
|
+
* `vault.db-wal` + `vault.db-shm`. The WAL (write-ahead log) and shared-memory
|
|
79
|
+
* index files hold committed-but-not-yet-checkpointed pages; ignoring them
|
|
80
|
+
* undercounts a busy vault. A missing `-wal`/`-shm` (the common case at rest,
|
|
81
|
+
* after a checkpoint) contributes 0 rather than erroring.
|
|
82
|
+
*
|
|
83
|
+
* This is the PHYSICAL DB-file size — it includes SQLite page overhead, free
|
|
84
|
+
* pages from deletes, etc. It is intentionally distinct from the LOGICAL
|
|
85
|
+
* `contentBytes` in `VaultStats` (sum of note-content UTF-8 bytes).
|
|
86
|
+
*/
|
|
87
|
+
export function dbBytes(vaultName: string, fs: UsageFs = realUsageFs): number {
|
|
88
|
+
const base = join(vaultDir(vaultName), "vault.db");
|
|
89
|
+
const candidates = [base, `${base}-wal`, `${base}-shm`];
|
|
90
|
+
let total = 0;
|
|
91
|
+
for (const path of candidates) {
|
|
92
|
+
total += safeFileSize(path, fs);
|
|
93
|
+
}
|
|
94
|
+
return total;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Size of a single file, or 0 if it's missing / unreadable. */
|
|
98
|
+
function safeFileSize(path: string, fs: UsageFs): number {
|
|
99
|
+
try {
|
|
100
|
+
return fs.statFile(path).size;
|
|
101
|
+
} catch {
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Recursively sum the size of every regular file under `dirPath`.
|
|
108
|
+
*
|
|
109
|
+
* - Missing directory → 0 (never throws; a vault may have no assets / no
|
|
110
|
+
* mirror yet).
|
|
111
|
+
* - Symlinks are NOT followed — a symlink (file or dir) is skipped entirely.
|
|
112
|
+
* This guards against symlink loops (which would hang an infinite walk)
|
|
113
|
+
* and against a symlink that points at a large tree outside the vault
|
|
114
|
+
* inflating the reported footprint.
|
|
115
|
+
*
|
|
116
|
+
* Uses an explicit stack (not recursion) so a deep tree can't blow the call
|
|
117
|
+
* stack, and reads dirents with types so we avoid an extra `stat` per entry
|
|
118
|
+
* just to learn directory-ness.
|
|
119
|
+
*/
|
|
120
|
+
export function dirSize(dirPath: string, fs: UsageFs = realUsageFs): number {
|
|
121
|
+
let total = 0;
|
|
122
|
+
const stack: string[] = [dirPath];
|
|
123
|
+
|
|
124
|
+
while (stack.length > 0) {
|
|
125
|
+
const current = stack.pop()!;
|
|
126
|
+
let entries: Dirent[];
|
|
127
|
+
try {
|
|
128
|
+
entries = fs.readDir(current);
|
|
129
|
+
} catch {
|
|
130
|
+
// Missing/unreadable dir (including the root) → contributes 0.
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
for (const entry of entries) {
|
|
134
|
+
// Never follow symlinks — neither symlinked files nor symlinked dirs.
|
|
135
|
+
if (entry.isSymbolicLink()) continue;
|
|
136
|
+
const childPath = join(current, entry.name);
|
|
137
|
+
if (entry.isDirectory()) {
|
|
138
|
+
stack.push(childPath);
|
|
139
|
+
} else if (entry.isFile()) {
|
|
140
|
+
total += safeFileSize(childPath, fs);
|
|
141
|
+
}
|
|
142
|
+
// Sockets/FIFOs/devices: ignored.
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return total;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Resolve a vault's mirror directory on disk, or `null` when no mirror is
|
|
150
|
+
* configured / resolvable (never configured, or external-without-path). The
|
|
151
|
+
* usage endpoint reports `mirror: 0` (and omits it from the total) in that
|
|
152
|
+
* case.
|
|
153
|
+
*/
|
|
154
|
+
export function resolveVaultMirrorDir(vaultName: string): string | null {
|
|
155
|
+
const config = readMirrorConfigForVault(vaultName);
|
|
156
|
+
if (!config) return null;
|
|
157
|
+
return resolveMirrorPath(vaultDir(vaultName), config);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
// Dir-walk cache (the only expensive part)
|
|
162
|
+
// ---------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
export interface DirWalkResult {
|
|
165
|
+
/** Bytes under the vault's assets directory. */
|
|
166
|
+
assets: number;
|
|
167
|
+
/**
|
|
168
|
+
* Bytes under the vault's mirror directory, or `null` when no mirror is
|
|
169
|
+
* configured. `null` → omit `mirror` from the response (or report 0).
|
|
170
|
+
*/
|
|
171
|
+
mirror: number | null;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
interface CacheEntry {
|
|
175
|
+
value: DirWalkResult;
|
|
176
|
+
/** Epoch millis when this entry was computed. */
|
|
177
|
+
computedAt: number;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Default cache lifetime for the dir-walk numbers. */
|
|
181
|
+
export const DIR_WALK_TTL_MS = 60_000;
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* In-process, per-vault cache of the two dir-walk numbers. Module-level so it
|
|
185
|
+
* survives across requests within a server process. Tests construct their own
|
|
186
|
+
* `UsageCache` to get isolation + a controllable clock.
|
|
187
|
+
*/
|
|
188
|
+
export class UsageCache {
|
|
189
|
+
private readonly entries = new Map<string, CacheEntry>();
|
|
190
|
+
|
|
191
|
+
constructor(
|
|
192
|
+
private readonly fs: UsageFs = realUsageFs,
|
|
193
|
+
private readonly now: Clock = Date.now,
|
|
194
|
+
private readonly ttlMs: number = DIR_WALK_TTL_MS,
|
|
195
|
+
) {}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Get the dir-walk numbers for a vault. Returns whether the numbers came
|
|
199
|
+
* from cache (`cached: true`) or were freshly walked (`cached: false`).
|
|
200
|
+
*
|
|
201
|
+
* - `fresh: true` always recomputes (and refreshes the cache entry).
|
|
202
|
+
* - Otherwise a non-expired entry is returned as-is (no walk); an absent
|
|
203
|
+
* or expired entry triggers a walk + store.
|
|
204
|
+
*/
|
|
205
|
+
get(vaultName: string, opts: { fresh?: boolean } = {}): { result: DirWalkResult; cached: boolean } {
|
|
206
|
+
const existing = this.entries.get(vaultName);
|
|
207
|
+
const ts = this.now();
|
|
208
|
+
if (!opts.fresh && existing && ts - existing.computedAt < this.ttlMs) {
|
|
209
|
+
return { result: existing.value, cached: true };
|
|
210
|
+
}
|
|
211
|
+
const value = this.compute(vaultName);
|
|
212
|
+
this.entries.set(vaultName, { value, computedAt: ts });
|
|
213
|
+
return { result: value, cached: false };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Drop a vault's cached dir-walk so the next read recomputes. */
|
|
217
|
+
invalidate(vaultName: string): void {
|
|
218
|
+
this.entries.delete(vaultName);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** Run the (expensive) walks. */
|
|
222
|
+
private compute(vaultName: string): DirWalkResult {
|
|
223
|
+
const assets = dirSize(assetsDir(vaultName), this.fs);
|
|
224
|
+
const mirrorDir = resolveVaultMirrorDir(vaultName);
|
|
225
|
+
const mirror = mirrorDir === null ? null : dirSize(mirrorDir, this.fs);
|
|
226
|
+
return { assets, mirror };
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* The process-wide cache the live HTTP route uses. A single instance shared
|
|
232
|
+
* across requests gives the 60s TTL its meaning. `invalidateUsageCache` is the
|
|
233
|
+
* write-path hook (attachment upload, post-export).
|
|
234
|
+
*/
|
|
235
|
+
export const usageCache = new UsageCache();
|
|
236
|
+
|
|
237
|
+
/** Invalidate the process-wide usage cache for a vault (write-path hook). */
|
|
238
|
+
export function invalidateUsageCache(vaultName: string): void {
|
|
239
|
+
usageCache.invalidate(vaultName);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
// Report assembly (consumed by the HTTP route)
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
|
|
246
|
+
export interface UsageReport {
|
|
247
|
+
counts: {
|
|
248
|
+
notes: number;
|
|
249
|
+
attachments: number;
|
|
250
|
+
links: number;
|
|
251
|
+
tags: number;
|
|
252
|
+
};
|
|
253
|
+
bytes: {
|
|
254
|
+
/** Logical: sum of note-content UTF-8 bytes (from VaultStats). */
|
|
255
|
+
content: number;
|
|
256
|
+
/** Physical: vault.db + WAL + shm. */
|
|
257
|
+
db: number;
|
|
258
|
+
/** Bytes under the vault's assets directory (dir-walk, cached). */
|
|
259
|
+
assets: number;
|
|
260
|
+
/**
|
|
261
|
+
* Bytes under the vault's mirror directory (dir-walk, cached), present
|
|
262
|
+
* only when a mirror is configured. The mirror is a git PROJECTION of the
|
|
263
|
+
* same notes/attachments — it is NOT added to `total` (that would
|
|
264
|
+
* double-count the data); it's reported as a separate line item.
|
|
265
|
+
*/
|
|
266
|
+
mirror?: number;
|
|
267
|
+
/**
|
|
268
|
+
* Physical on-disk footprint (db + assets); excludes content (inside db)
|
|
269
|
+
* and mirror (separate projection). This is the number an operator sizes a
|
|
270
|
+
* vault by.
|
|
271
|
+
*/
|
|
272
|
+
total: number;
|
|
273
|
+
};
|
|
274
|
+
/** ISO-8601 timestamp of when this report was assembled. */
|
|
275
|
+
computedAt: string;
|
|
276
|
+
/** Whether the dir-walk numbers (assets/mirror) came from cache. */
|
|
277
|
+
cached: boolean;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Assemble the usage report for a vault. `stats` is the already-computed
|
|
282
|
+
* `getVaultStats()` result (counts + contentBytes); the cache supplies the two
|
|
283
|
+
* dir-walk numbers and reports whether they were cached.
|
|
284
|
+
*
|
|
285
|
+
* `total = db + assets` — the physical footprint. Mirror is a separate line
|
|
286
|
+
* item (projection of the same data; adding it double-counts). Content is the
|
|
287
|
+
* logical note size, already inside `db` physically.
|
|
288
|
+
*/
|
|
289
|
+
export function buildUsageReport(
|
|
290
|
+
vaultName: string,
|
|
291
|
+
stats: VaultStats,
|
|
292
|
+
opts: { fresh?: boolean; cache?: UsageCache; fs?: UsageFs; now?: Clock } = {},
|
|
293
|
+
): UsageReport {
|
|
294
|
+
const cache = opts.cache ?? usageCache;
|
|
295
|
+
const { result, cached } = cache.get(vaultName, { fresh: opts.fresh });
|
|
296
|
+
const db = dbBytes(vaultName, opts.fs ?? realUsageFs);
|
|
297
|
+
const total = db + result.assets;
|
|
298
|
+
|
|
299
|
+
const bytes: UsageReport["bytes"] = {
|
|
300
|
+
content: stats.contentBytes,
|
|
301
|
+
db,
|
|
302
|
+
assets: result.assets,
|
|
303
|
+
total,
|
|
304
|
+
};
|
|
305
|
+
if (result.mirror !== null) bytes.mirror = result.mirror;
|
|
306
|
+
|
|
307
|
+
return {
|
|
308
|
+
counts: {
|
|
309
|
+
notes: stats.totalNotes,
|
|
310
|
+
attachments: stats.attachmentCount,
|
|
311
|
+
links: stats.linkCount,
|
|
312
|
+
tags: stats.tagCount,
|
|
313
|
+
},
|
|
314
|
+
bytes,
|
|
315
|
+
computedAt: new Date((opts.now ?? Date.now)()).toISOString(),
|
|
316
|
+
cached,
|
|
317
|
+
};
|
|
318
|
+
}
|