@indigoai-us/hq-cloud 6.14.44 → 6.14.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/watch-roots.d.ts +72 -0
- package/dist/watch-roots.d.ts.map +1 -0
- package/dist/watch-roots.js +115 -0
- package/dist/watch-roots.js.map +1 -0
- package/dist/watch-roots.test.d.ts +18 -0
- package/dist/watch-roots.test.d.ts.map +1 -0
- package/dist/watch-roots.test.js +236 -0
- package/dist/watch-roots.test.js.map +1 -0
- package/dist/watcher-event-gate.test.d.ts +16 -0
- package/dist/watcher-event-gate.test.d.ts.map +1 -0
- package/dist/watcher-event-gate.test.js +191 -0
- package/dist/watcher-event-gate.test.js.map +1 -0
- package/dist/watcher.d.ts +21 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +312 -54
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/src/watch-roots.test.ts +278 -0
- package/src/watch-roots.ts +162 -0
- package/src/watcher-event-gate.test.ts +212 -0
- package/src/watcher.ts +363 -65
- package/test/e2e/watcher-scoped-coverage.test.ts +381 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the watch-root planner.
|
|
3
|
+
*
|
|
4
|
+
* The invariant under test is the one the whole change exists for: the watcher
|
|
5
|
+
* must only register OS watches over paths that can actually upload. Before
|
|
6
|
+
* this planner, macOS/Windows placed ONE recursive `fs.watch` on hqRoot, so the
|
|
7
|
+
* OS delivered every event in the tree — including `repos/` (~26k dirs on a
|
|
8
|
+
* real HQ) and `workspace/worktrees/` (~62k dirs) — and each one was filtered
|
|
9
|
+
* out only AFTER it had already cost a syscall and an allocation.
|
|
10
|
+
*
|
|
11
|
+
* The planner walks the exclusion filter up front and returns the minimum set
|
|
12
|
+
* of watch roots that covers exactly the in-scope tree:
|
|
13
|
+
* - `recursive`: the dir's entire subtree is in scope → one recursive watch.
|
|
14
|
+
* - `shallow`: the dir contains an excluded child → watch the dir itself
|
|
15
|
+
* non-recursively and descend into its in-scope children.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { mkdtemp, mkdir, rm } from "node:fs/promises";
|
|
19
|
+
import { tmpdir } from "node:os";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
|
|
22
|
+
import { describe, expect, it } from "vitest";
|
|
23
|
+
|
|
24
|
+
import { isCoveredByRecursiveRoot, planWatchRoots } from "./watch-roots.js";
|
|
25
|
+
import type { WatchPathFilter } from "./watcher.js";
|
|
26
|
+
|
|
27
|
+
const ROOT = path.resolve("/hq");
|
|
28
|
+
|
|
29
|
+
/** Build the `listChildDirs` seam over a literal `absDir -> childNames` map. */
|
|
30
|
+
function treeLister(tree: Record<string, string[]>) {
|
|
31
|
+
return (dir: string): string[] =>
|
|
32
|
+
(tree[dir] ?? []).map((name) => path.join(dir, name));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Build a filter shaped like the real one: `prefixes` are exact vault-relative
|
|
37
|
+
* prefixes (`repos`, `workspace/worktrees` — position-sensitive), `anySegment`
|
|
38
|
+
* names are excluded at any depth (`node_modules`, `.git`).
|
|
39
|
+
*/
|
|
40
|
+
function makeFilter(opts: {
|
|
41
|
+
prefixes?: string[];
|
|
42
|
+
anySegment?: string[];
|
|
43
|
+
}): WatchPathFilter {
|
|
44
|
+
const prefixes = opts.prefixes ?? [];
|
|
45
|
+
const anySegment = new Set(opts.anySegment ?? []);
|
|
46
|
+
return (absolutePath: string): boolean => {
|
|
47
|
+
const rel = path.relative(ROOT, absolutePath).split(path.sep).join("/");
|
|
48
|
+
if (rel === "" || rel.startsWith("..")) return false;
|
|
49
|
+
if (prefixes.some((p) => rel === p || rel.startsWith(p + "/"))) return false;
|
|
50
|
+
if (rel.split("/").some((seg) => anySegment.has(seg))) return false;
|
|
51
|
+
return true;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Every dir the plan asks the OS to watch, recursive roots and shallow alike. */
|
|
56
|
+
function allWatched(plan: { recursive: string[]; shallow: string[] }): string[] {
|
|
57
|
+
return [...plan.recursive, ...plan.shallow];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
describe("planWatchRoots — minimal cover of the in-scope tree", () => {
|
|
61
|
+
it("collapses a fully in-scope tree to ONE recursive watch at the root", () => {
|
|
62
|
+
const plan = planWatchRoots(ROOT, makeFilter({}), {
|
|
63
|
+
listChildDirs: treeLister({
|
|
64
|
+
[ROOT]: ["companies", "core"],
|
|
65
|
+
[path.join(ROOT, "companies")]: ["acme"],
|
|
66
|
+
}),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
expect(plan.recursive).toEqual([ROOT]);
|
|
70
|
+
expect(plan.shallow).toEqual([]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("never registers a watch on an excluded dir, and never covers one from a recursive root", () => {
|
|
74
|
+
const filter = makeFilter({
|
|
75
|
+
prefixes: ["repos"],
|
|
76
|
+
anySegment: ["node_modules"],
|
|
77
|
+
});
|
|
78
|
+
const plan = planWatchRoots(ROOT, filter, {
|
|
79
|
+
listChildDirs: treeLister({
|
|
80
|
+
[ROOT]: ["companies", "repos", "node_modules"],
|
|
81
|
+
[path.join(ROOT, "companies")]: ["acme"],
|
|
82
|
+
[path.join(ROOT, "repos")]: ["hq-cloud"],
|
|
83
|
+
}),
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
// The root has excluded children, so it cannot be a recursive root.
|
|
87
|
+
expect(plan.shallow).toContain(ROOT);
|
|
88
|
+
expect(plan.recursive).not.toContain(ROOT);
|
|
89
|
+
// `companies/` is clean, so it collapses to one recursive watch.
|
|
90
|
+
expect(plan.recursive).toContain(path.join(ROOT, "companies"));
|
|
91
|
+
|
|
92
|
+
// The point of the whole change: excluded trees are never watched at all.
|
|
93
|
+
for (const watched of allWatched(plan)) {
|
|
94
|
+
expect(watched).not.toContain(`${path.sep}repos`);
|
|
95
|
+
expect(watched).not.toContain("node_modules");
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("descends through a partially-excluded dir to reach its in-scope children", () => {
|
|
100
|
+
// `workspace/` is in scope but `workspace/worktrees/` (the 62k-dir bucket)
|
|
101
|
+
// is not — so `workspace` must be shallow and its siblings recursive.
|
|
102
|
+
const filter = makeFilter({ prefixes: ["workspace/worktrees"] });
|
|
103
|
+
const plan = planWatchRoots(ROOT, filter, {
|
|
104
|
+
listChildDirs: treeLister({
|
|
105
|
+
[ROOT]: ["workspace"],
|
|
106
|
+
[path.join(ROOT, "workspace")]: ["agency", "threads", "worktrees"],
|
|
107
|
+
[path.join(ROOT, "workspace", "worktrees")]: ["feature-a"],
|
|
108
|
+
}),
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
expect(plan.shallow).toEqual(
|
|
112
|
+
expect.arrayContaining([ROOT, path.join(ROOT, "workspace")]),
|
|
113
|
+
);
|
|
114
|
+
expect(plan.recursive).toEqual(
|
|
115
|
+
expect.arrayContaining([
|
|
116
|
+
path.join(ROOT, "workspace", "agency"),
|
|
117
|
+
path.join(ROOT, "workspace", "threads"),
|
|
118
|
+
]),
|
|
119
|
+
);
|
|
120
|
+
expect(allWatched(plan)).not.toContain(
|
|
121
|
+
path.join(ROOT, "workspace", "worktrees"),
|
|
122
|
+
);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("holds the invariant: no excluded path lies under any recursive root", () => {
|
|
126
|
+
const filter = makeFilter({
|
|
127
|
+
prefixes: ["repos", "workspace/worktrees"],
|
|
128
|
+
anySegment: ["node_modules", ".git"],
|
|
129
|
+
});
|
|
130
|
+
const tree: Record<string, string[]> = {
|
|
131
|
+
[ROOT]: [
|
|
132
|
+
"companies",
|
|
133
|
+
"core",
|
|
134
|
+
"workspace",
|
|
135
|
+
"repos",
|
|
136
|
+
".git",
|
|
137
|
+
"node_modules",
|
|
138
|
+
],
|
|
139
|
+
[path.join(ROOT, "companies")]: ["acme", "indigo"],
|
|
140
|
+
[path.join(ROOT, "core")]: ["skills"],
|
|
141
|
+
[path.join(ROOT, "workspace")]: ["agency", "worktrees"],
|
|
142
|
+
[path.join(ROOT, "workspace", "worktrees")]: ["wt-1"],
|
|
143
|
+
[path.join(ROOT, "repos")]: ["hq-cloud"],
|
|
144
|
+
};
|
|
145
|
+
const plan = planWatchRoots(ROOT, filter, {
|
|
146
|
+
listChildDirs: treeLister(tree),
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
const excludedDirs = Object.keys(tree)
|
|
150
|
+
.flatMap((dir) => (tree[dir] ?? []).map((n) => path.join(dir, n)))
|
|
151
|
+
.filter((p) => !filter(p, true));
|
|
152
|
+
expect(excludedDirs.length).toBeGreaterThan(0);
|
|
153
|
+
|
|
154
|
+
for (const excluded of excludedDirs) {
|
|
155
|
+
// Not watched directly...
|
|
156
|
+
expect(allWatched(plan)).not.toContain(excluded);
|
|
157
|
+
// ...and not swept in as a descendant of some recursive root either.
|
|
158
|
+
for (const root of plan.recursive) {
|
|
159
|
+
expect(excluded.startsWith(root + path.sep)).toBe(false);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("stops descending at maxDepth and takes a recursive watch there", () => {
|
|
165
|
+
const filter = makeFilter({ anySegment: ["node_modules"] });
|
|
166
|
+
const plan = planWatchRoots(ROOT, filter, {
|
|
167
|
+
maxDepth: 1,
|
|
168
|
+
listChildDirs: treeLister({
|
|
169
|
+
[ROOT]: ["a", "node_modules"],
|
|
170
|
+
[path.join(ROOT, "a")]: ["b", "node_modules"],
|
|
171
|
+
[path.join(ROOT, "a", "b")]: ["c"],
|
|
172
|
+
}),
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
// Depth 0 has an excluded child → shallow. Depth 1 is the bound, so `a`
|
|
176
|
+
// takes a recursive watch even though it still contains node_modules; the
|
|
177
|
+
// per-event filter drops that residue. This bounds handle count on trees
|
|
178
|
+
// with deeply scattered exclusions.
|
|
179
|
+
expect(plan.shallow).toEqual([ROOT]);
|
|
180
|
+
expect(plan.recursive).toEqual([path.join(ROOT, "a")]);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("skips a directory it cannot read instead of throwing", () => {
|
|
184
|
+
const plan = planWatchRoots(ROOT, makeFilter({ prefixes: ["repos"] }), {
|
|
185
|
+
listChildDirs: (dir: string) => {
|
|
186
|
+
if (dir === path.join(ROOT, "denied")) {
|
|
187
|
+
throw Object.assign(new Error("EACCES"), { code: "EACCES" });
|
|
188
|
+
}
|
|
189
|
+
return dir === ROOT
|
|
190
|
+
? [path.join(ROOT, "denied"), path.join(ROOT, "repos")]
|
|
191
|
+
: [];
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
// The unreadable dir is still watched (we cannot see inside it to decide
|
|
196
|
+
// better) and the walk completes rather than crashing start().
|
|
197
|
+
expect(allWatched(plan)).toContain(path.join(ROOT, "denied"));
|
|
198
|
+
expect(allWatched(plan)).not.toContain(path.join(ROOT, "repos"));
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("reports every in-scope directory through onDirectory, and no excluded one", () => {
|
|
202
|
+
// TreeWatcher builds its known-kinds index from this callback instead of
|
|
203
|
+
// walking the same tree a second time, so the callback must cover every
|
|
204
|
+
// in-scope directory — including ones inside a collapsed recursive root.
|
|
205
|
+
const filter = makeFilter({ prefixes: ["repos"] });
|
|
206
|
+
const seen: string[] = [];
|
|
207
|
+
planWatchRoots(ROOT, filter, {
|
|
208
|
+
onDirectory: (dir) => seen.push(dir),
|
|
209
|
+
listChildDirs: treeLister({
|
|
210
|
+
[ROOT]: ["companies", "repos"],
|
|
211
|
+
[path.join(ROOT, "companies")]: ["acme"],
|
|
212
|
+
[path.join(ROOT, "companies", "acme")]: ["knowledge"],
|
|
213
|
+
[path.join(ROOT, "repos")]: ["hq-cloud"],
|
|
214
|
+
}),
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
expect(seen).toEqual([
|
|
218
|
+
path.join(ROOT, "companies"),
|
|
219
|
+
path.join(ROOT, "companies", "acme"),
|
|
220
|
+
path.join(ROOT, "companies", "acme", "knowledge"),
|
|
221
|
+
]);
|
|
222
|
+
expect(seen).not.toContain(path.join(ROOT, "repos"));
|
|
223
|
+
expect(seen).not.toContain(path.join(ROOT, "repos", "hq-cloud"));
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
it("resolves coverage by walking ancestors, not by scanning every root", () => {
|
|
227
|
+
// This runs on EVERY rename event, in the hot path of a change whose whole
|
|
228
|
+
// point is cutting per-event cost. A linear scan is O(number of roots) —
|
|
229
|
+
// 685 on a real HQ tree — where an ancestor walk is O(path depth), ~10.
|
|
230
|
+
// The behavior must be identical, including the prefix trap below.
|
|
231
|
+
const roots = new Set([path.join(ROOT, "a", "b"), path.join(ROOT, "c")]);
|
|
232
|
+
|
|
233
|
+
expect(isCoveredByRecursiveRoot(path.join(ROOT, "a", "b"), roots)).toBe(true);
|
|
234
|
+
expect(
|
|
235
|
+
isCoveredByRecursiveRoot(path.join(ROOT, "a", "b", "deep", "x.md"), roots),
|
|
236
|
+
).toBe(true);
|
|
237
|
+
expect(isCoveredByRecursiveRoot(path.join(ROOT, "a"), roots)).toBe(false);
|
|
238
|
+
expect(isCoveredByRecursiveRoot(path.join(ROOT, "d"), roots)).toBe(false);
|
|
239
|
+
// The prefix trap: `/hq/a/bc` shares a string prefix with root `/hq/a/b`
|
|
240
|
+
// but is NOT under it. A naive startsWith without the separator would say
|
|
241
|
+
// covered, and the whole subtree would go unwatched.
|
|
242
|
+
expect(isCoveredByRecursiveRoot(path.join(ROOT, "a", "bc"), roots)).toBe(
|
|
243
|
+
false,
|
|
244
|
+
);
|
|
245
|
+
expect(
|
|
246
|
+
isCoveredByRecursiveRoot(path.join(ROOT, "a", "bc", "x.md"), roots),
|
|
247
|
+
).toBe(false);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it("reports nothing as covered when there are no recursive roots", () => {
|
|
251
|
+
expect(
|
|
252
|
+
isCoveredByRecursiveRoot(path.join(ROOT, "a", "b"), new Set<string>()),
|
|
253
|
+
).toBe(false);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
it("plans over a real directory tree using the default reader", async () => {
|
|
257
|
+
const root = await mkdtemp(path.join(tmpdir(), "hqcloud-planroots-"));
|
|
258
|
+
try {
|
|
259
|
+
await mkdir(path.join(root, "companies", "acme"), { recursive: true });
|
|
260
|
+
await mkdir(path.join(root, "repos", "hq-cloud", "src"), {
|
|
261
|
+
recursive: true,
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
const filter: WatchPathFilter = (abs) => {
|
|
265
|
+
const rel = path.relative(root, abs).split(path.sep).join("/");
|
|
266
|
+
if (rel === "" || rel.startsWith("..")) return false;
|
|
267
|
+
return rel !== "repos" && !rel.startsWith("repos/");
|
|
268
|
+
};
|
|
269
|
+
const plan = planWatchRoots(root, filter);
|
|
270
|
+
|
|
271
|
+
expect(plan.shallow).toContain(root);
|
|
272
|
+
expect(plan.recursive).toContain(path.join(root, "companies"));
|
|
273
|
+
expect(allWatched(plan)).not.toContain(path.join(root, "repos"));
|
|
274
|
+
} finally {
|
|
275
|
+
await rm(root, { recursive: true, force: true });
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
});
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watch-root planner — reduce the in-scope tree to the smallest set of OS
|
|
3
|
+
* watches that covers exactly the paths that can upload.
|
|
4
|
+
*
|
|
5
|
+
* Why this exists: on macOS/Windows the watcher used to place ONE recursive
|
|
6
|
+
* `fs.watch` on hqRoot. That is cheap in handles but the OS then delivers every
|
|
7
|
+
* event in the tree, including the buckets sync never uploads — on a real HQ
|
|
8
|
+
* root that is `repos/` (~26k directories) and `workspace/worktrees/` (~62k),
|
|
9
|
+
* where agent builds, installs, and checkouts churn constantly. Every one of
|
|
10
|
+
* those events woke the runner, cost a `path.resolve` (plus a synchronous
|
|
11
|
+
* `lstat` for renames), and was then dropped by the emit filter. The result was
|
|
12
|
+
* a node process pinned above 100% CPU doing nothing but allocating and
|
|
13
|
+
* garbage-collecting discarded paths.
|
|
14
|
+
*
|
|
15
|
+
* The planner walks the exclusion filter ONCE at start and returns:
|
|
16
|
+
* - `recursive`: directories whose entire subtree is in scope. One recursive
|
|
17
|
+
* watch each; the OS never reports an excluded path under them.
|
|
18
|
+
* - `shallow`: directories that contain an excluded descendant. Watched
|
|
19
|
+
* non-recursively so their own files still fire, with their in-scope
|
|
20
|
+
* children planned separately.
|
|
21
|
+
*
|
|
22
|
+
* The walk is post-order: whether a directory can collapse to a single
|
|
23
|
+
* recursive watch depends on its whole subtree, not just its direct children
|
|
24
|
+
* (`workspace/` looks clean until you reach `workspace/worktrees/`).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import * as fs from "fs";
|
|
28
|
+
import * as path from "path";
|
|
29
|
+
|
|
30
|
+
import type { WatchPathFilter } from "./watcher.js";
|
|
31
|
+
|
|
32
|
+
export interface WatchRootPlan {
|
|
33
|
+
/** Directories to watch recursively — their whole subtree is in scope. */
|
|
34
|
+
recursive: string[];
|
|
35
|
+
/** Directories to watch non-recursively — they contain excluded children. */
|
|
36
|
+
shallow: string[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface PlanWatchRootsOptions {
|
|
40
|
+
/**
|
|
41
|
+
* Optional depth at which the walk stops splitting and takes a recursive
|
|
42
|
+
* watch even if the subtree still holds exclusions. Unbounded by default:
|
|
43
|
+
* the walk costs about what {@link TreeWatcher}'s known-kinds seed walk
|
|
44
|
+
* already cost on every start, and the two are fused via `onDirectory`, so
|
|
45
|
+
* paying for full pruning is free. A bound trades pruning for walk time on
|
|
46
|
+
* pathologically deep trees; the per-event filter drops whatever residue it
|
|
47
|
+
* admits, so this is a performance dial, never a correctness one.
|
|
48
|
+
*/
|
|
49
|
+
maxDepth?: number;
|
|
50
|
+
/** Directory reader seam — defaults to a real `readdirSync`. */
|
|
51
|
+
listChildDirs?: (dir: string) => string[];
|
|
52
|
+
/**
|
|
53
|
+
* Invoked once for every in-scope directory the walk visits. Lets a caller
|
|
54
|
+
* populate its own directory index in the SAME pass instead of walking the
|
|
55
|
+
* tree a second time.
|
|
56
|
+
*/
|
|
57
|
+
onDirectory?: (absolutePath: string) => void;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function defaultListChildDirs(dir: string): string[] {
|
|
61
|
+
return fs
|
|
62
|
+
.readdirSync(dir, { withFileTypes: true })
|
|
63
|
+
.filter((entry) => entry.isDirectory())
|
|
64
|
+
.map((entry) => path.join(dir, entry.name));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
interface VisitResult {
|
|
68
|
+
/** True when every descendant of this directory is in scope. */
|
|
69
|
+
clean: boolean;
|
|
70
|
+
plan: WatchRootPlan;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Plan the watch roots for `hqRoot` under `shouldEmit`.
|
|
75
|
+
*
|
|
76
|
+
* `shouldEmit(dir, true)` is the same directory predicate the watcher uses at
|
|
77
|
+
* event time, so the plan and the emit filter can never disagree about what is
|
|
78
|
+
* in scope.
|
|
79
|
+
*/
|
|
80
|
+
export function planWatchRoots(
|
|
81
|
+
hqRoot: string,
|
|
82
|
+
shouldEmit: WatchPathFilter,
|
|
83
|
+
opts: PlanWatchRootsOptions = {},
|
|
84
|
+
): WatchRootPlan {
|
|
85
|
+
const maxDepth = opts.maxDepth ?? Number.POSITIVE_INFINITY;
|
|
86
|
+
const listChildDirs = opts.listChildDirs ?? defaultListChildDirs;
|
|
87
|
+
const onDirectory = opts.onDirectory;
|
|
88
|
+
const root = path.resolve(hqRoot);
|
|
89
|
+
|
|
90
|
+
const visit = (dir: string, depth: number): VisitResult => {
|
|
91
|
+
let children: string[];
|
|
92
|
+
try {
|
|
93
|
+
children = listChildDirs(dir);
|
|
94
|
+
} catch {
|
|
95
|
+
// Unreadable (permissions, or a race with a concurrent delete). We cannot
|
|
96
|
+
// prove the subtree is clean, but dropping it would silently stop syncing
|
|
97
|
+
// it — take the recursive watch and let the per-event filter sort it out.
|
|
98
|
+
return { clean: true, plan: { recursive: [dir], shallow: [] } };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const included: string[] = [];
|
|
102
|
+
let hasExcludedChild = false;
|
|
103
|
+
for (const child of children) {
|
|
104
|
+
if (shouldEmit(child, true)) {
|
|
105
|
+
included.push(child);
|
|
106
|
+
onDirectory?.(path.resolve(child));
|
|
107
|
+
} else hasExcludedChild = true;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// At the depth bound we stop splitting. `clean` still reports what we can
|
|
111
|
+
// see one level down so an ancestor does not collapse over a known
|
|
112
|
+
// exclusion it could have pruned.
|
|
113
|
+
if (depth >= maxDepth) {
|
|
114
|
+
return {
|
|
115
|
+
clean: !hasExcludedChild,
|
|
116
|
+
plan: { recursive: [dir], shallow: [] },
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const childResults = included.map((child) => visit(child, depth + 1));
|
|
121
|
+
const clean =
|
|
122
|
+
!hasExcludedChild && childResults.every((result) => result.clean);
|
|
123
|
+
|
|
124
|
+
// A clean subtree collapses to a single recursive watch here, discarding
|
|
125
|
+
// the per-child watches the recursion built below it.
|
|
126
|
+
if (clean) return { clean: true, plan: { recursive: [dir], shallow: [] } };
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
clean: false,
|
|
130
|
+
plan: {
|
|
131
|
+
recursive: childResults.flatMap((result) => result.plan.recursive),
|
|
132
|
+
shallow: [dir, ...childResults.flatMap((result) => result.plan.shallow)],
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
return visit(root, 0).plan;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* True when `absolutePath` already falls under one of `recursiveRoots`.
|
|
142
|
+
*
|
|
143
|
+
* Walks the path's own ancestors rather than scanning the root set: this runs
|
|
144
|
+
* on every rename event, in the hot path of a backend whose entire purpose is
|
|
145
|
+
* cutting per-event cost, and a real HQ tree plans ~685 roots. Ancestor-walking
|
|
146
|
+
* is O(path depth) — about ten set lookups — instead of O(roots) string
|
|
147
|
+
* comparisons. It also cannot fall into the shared-prefix trap, where `/hq/a/bc`
|
|
148
|
+
* looks like it sits under the root `/hq/a/b`.
|
|
149
|
+
*/
|
|
150
|
+
export function isCoveredByRecursiveRoot(
|
|
151
|
+
absolutePath: string,
|
|
152
|
+
recursiveRoots: ReadonlySet<string>,
|
|
153
|
+
): boolean {
|
|
154
|
+
if (recursiveRoots.size === 0) return false;
|
|
155
|
+
let current = path.resolve(absolutePath);
|
|
156
|
+
for (;;) {
|
|
157
|
+
if (recursiveRoots.has(current)) return true;
|
|
158
|
+
const parent = path.dirname(current);
|
|
159
|
+
if (parent === current) return false;
|
|
160
|
+
current = parent;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The emit filter must run BEFORE handleEvent pays for a stat.
|
|
3
|
+
*
|
|
4
|
+
* The recursive backend hands every OS event to handleEvent, and a `rename`
|
|
5
|
+
* used to be lstat'd before the exclusion filter was consulted. On a real HQ
|
|
6
|
+
* that is one synchronous syscall for every file touched anywhere under
|
|
7
|
+
* `repos/` or `workspace/worktrees/` — build, install, and checkout churn the
|
|
8
|
+
* watcher immediately throws away. Scoped watch roots (see watch-roots.ts) keep
|
|
9
|
+
* most of that traffic from being delivered at all; this gate makes whatever
|
|
10
|
+
* still arrives cost a string compare instead of a syscall.
|
|
11
|
+
*
|
|
12
|
+
* This lives in its own file because it mocks `fs` module-wide, which would
|
|
13
|
+
* otherwise perturb the 53 tests in watcher.test.ts.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import * as os from "os";
|
|
17
|
+
import * as path from "path";
|
|
18
|
+
|
|
19
|
+
import { describe, expect, it, vi } from "vitest";
|
|
20
|
+
|
|
21
|
+
const { lstatCalls } = vi.hoisted(() => ({ lstatCalls: vi.fn() }));
|
|
22
|
+
|
|
23
|
+
vi.mock("fs", async (importOriginal) => {
|
|
24
|
+
const actual = await importOriginal<typeof import("fs")>();
|
|
25
|
+
return {
|
|
26
|
+
...actual,
|
|
27
|
+
lstatSync: (...args: Parameters<typeof actual.lstatSync>) => {
|
|
28
|
+
lstatCalls(...args);
|
|
29
|
+
return actual.lstatSync(...args);
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const { FakeClock, TreeWatcher } = await import("./watcher.js");
|
|
35
|
+
|
|
36
|
+
const hqRoot = path.join(os.tmpdir(), "hqcloud-event-gate-root");
|
|
37
|
+
/** Everything under `repos/` is out of scope, as file AND as directory. */
|
|
38
|
+
const outOfScope = (abs: string) => !abs.split(path.sep).includes("repos");
|
|
39
|
+
|
|
40
|
+
function makeWatcher() {
|
|
41
|
+
return new TreeWatcher({
|
|
42
|
+
hqRoot,
|
|
43
|
+
clock: new FakeClock(),
|
|
44
|
+
debounceMs: 100,
|
|
45
|
+
pathFilter: outOfScope,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe("TreeWatcher.handleEvent — an excluded path costs no syscall", () => {
|
|
50
|
+
it("does NOT stat an out-of-scope rename before dropping it", () => {
|
|
51
|
+
const watcher = makeWatcher();
|
|
52
|
+
lstatCalls.mockClear();
|
|
53
|
+
try {
|
|
54
|
+
watcher.handleEvent(
|
|
55
|
+
path.join(hqRoot, "repos", "hq-cloud", "src", "a.ts"),
|
|
56
|
+
"rename",
|
|
57
|
+
);
|
|
58
|
+
expect(lstatCalls).not.toHaveBeenCalled();
|
|
59
|
+
} finally {
|
|
60
|
+
watcher.dispose();
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("still stats an in-scope rename, so kind classification is unchanged", () => {
|
|
65
|
+
const watcher = makeWatcher();
|
|
66
|
+
lstatCalls.mockClear();
|
|
67
|
+
try {
|
|
68
|
+
watcher.handleEvent(
|
|
69
|
+
path.join(hqRoot, "companies", "acme", "note.md"),
|
|
70
|
+
"rename",
|
|
71
|
+
);
|
|
72
|
+
expect(lstatCalls).toHaveBeenCalled();
|
|
73
|
+
} finally {
|
|
74
|
+
watcher.dispose();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("skips its own seed walk when the backend already indexed directories", async () => {
|
|
79
|
+
// The scoped backend indexes every in-scope directory while planning its
|
|
80
|
+
// watch roots. TreeWatcher must NOT then walk the same tree again — on a
|
|
81
|
+
// real HQ root that walk is ~3.6s, and doing it twice per runner start was
|
|
82
|
+
// a straight doubling of startup cost for an identical index.
|
|
83
|
+
const fsp = await import("node:fs/promises");
|
|
84
|
+
const realRoot = await fsp.mkdtemp(
|
|
85
|
+
path.join(os.tmpdir(), "hqcloud-seed-fusion-"),
|
|
86
|
+
);
|
|
87
|
+
try {
|
|
88
|
+
await fsp.mkdir(path.join(realRoot, "companies", "acme", "knowledge"), {
|
|
89
|
+
recursive: true,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
const watcher = new TreeWatcher({
|
|
93
|
+
hqRoot: realRoot,
|
|
94
|
+
clock: new FakeClock(),
|
|
95
|
+
pathFilter: () => true,
|
|
96
|
+
backendFactory: (opts) => {
|
|
97
|
+
opts.onDirectorySeen?.(path.join(realRoot, "companies"));
|
|
98
|
+
return {
|
|
99
|
+
close: () => {},
|
|
100
|
+
needsKnownKinds: true,
|
|
101
|
+
seededKnownKinds: true,
|
|
102
|
+
watchedPathCount: () => 1,
|
|
103
|
+
};
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
watcher.start();
|
|
107
|
+
// Exactly the one directory the backend reported — not the three that a
|
|
108
|
+
// second full walk of this tree would have found.
|
|
109
|
+
expect(watcher.knownDirectoryCount()).toBe(1);
|
|
110
|
+
watcher.dispose();
|
|
111
|
+
} finally {
|
|
112
|
+
await fsp.rm(realRoot, { recursive: true, force: true });
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("still walks itself when the backend did NOT index directories", async () => {
|
|
117
|
+
const fsp = await import("node:fs/promises");
|
|
118
|
+
const realRoot = await fsp.mkdtemp(
|
|
119
|
+
path.join(os.tmpdir(), "hqcloud-seed-legacy-"),
|
|
120
|
+
);
|
|
121
|
+
try {
|
|
122
|
+
await fsp.mkdir(path.join(realRoot, "companies", "acme", "knowledge"), {
|
|
123
|
+
recursive: true,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const watcher = new TreeWatcher({
|
|
127
|
+
hqRoot: realRoot,
|
|
128
|
+
clock: new FakeClock(),
|
|
129
|
+
pathFilter: () => true,
|
|
130
|
+
backendFactory: () => ({
|
|
131
|
+
close: () => {},
|
|
132
|
+
needsKnownKinds: true,
|
|
133
|
+
watchedPathCount: () => 1,
|
|
134
|
+
}),
|
|
135
|
+
});
|
|
136
|
+
watcher.start();
|
|
137
|
+
expect(watcher.knownDirectoryCount()).toBe(3);
|
|
138
|
+
watcher.dispose();
|
|
139
|
+
} finally {
|
|
140
|
+
await fsp.rm(realRoot, { recursive: true, force: true });
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it("does not let a duplicate delete downgrade unlinkDir to unlink", () => {
|
|
145
|
+
// One path can reach handleEvent twice when two watches overlap (the
|
|
146
|
+
// scoped backend briefly runs its bootstrap watch alongside the planned
|
|
147
|
+
// ones so in-flight events are not dropped). The first unlinkDir clears
|
|
148
|
+
// the path's known-kind hint, so without this guard the duplicate lands as
|
|
149
|
+
// a plain `unlink` — narrowing the deletion scope the vault applies and
|
|
150
|
+
// discarding the captured descendant snapshots.
|
|
151
|
+
const clock = new FakeClock();
|
|
152
|
+
const w = new TreeWatcher({
|
|
153
|
+
hqRoot,
|
|
154
|
+
clock,
|
|
155
|
+
debounceMs: 100,
|
|
156
|
+
pathFilter: () => true,
|
|
157
|
+
captureLocalDeleteSnapshots: (relativePath, kind) =>
|
|
158
|
+
kind === "unlinkDir"
|
|
159
|
+
? [
|
|
160
|
+
{
|
|
161
|
+
journalSlug: "acme",
|
|
162
|
+
journalPath: `${relativePath}/child.md`,
|
|
163
|
+
absolutePath: path.join(hqRoot, relativePath, "child.md"),
|
|
164
|
+
remoteEtag: "etag",
|
|
165
|
+
localHash: "hash",
|
|
166
|
+
localKind: "file" as const,
|
|
167
|
+
},
|
|
168
|
+
]
|
|
169
|
+
: [],
|
|
170
|
+
});
|
|
171
|
+
const batches: Array<Map<string, { kind: string }> | undefined> = [];
|
|
172
|
+
w.onChange((_first, batch) => batches.push(batch?.changes as never));
|
|
173
|
+
try {
|
|
174
|
+
const dir = path.join(hqRoot, "companies", "acme", "moved");
|
|
175
|
+
w.handleEvent(dir, "unlinkDir");
|
|
176
|
+
w.handleEvent(dir, "unlink"); // duplicate delivery of the same delete
|
|
177
|
+
clock.advance(200);
|
|
178
|
+
|
|
179
|
+
const change = batches[0]?.get(dir) as
|
|
180
|
+
| { kind: string; deleteSnapshots?: unknown[] }
|
|
181
|
+
| undefined;
|
|
182
|
+
expect(change?.kind).toBe("unlinkDir");
|
|
183
|
+
expect(change?.deleteSnapshots).toHaveLength(1);
|
|
184
|
+
} finally {
|
|
185
|
+
w.dispose();
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("keeps emitting in-scope changes after the gate", () => {
|
|
190
|
+
const watcher = makeWatcher();
|
|
191
|
+
const clock = new FakeClock();
|
|
192
|
+
const w = new TreeWatcher({
|
|
193
|
+
hqRoot,
|
|
194
|
+
clock,
|
|
195
|
+
debounceMs: 100,
|
|
196
|
+
pathFilter: outOfScope,
|
|
197
|
+
});
|
|
198
|
+
watcher.dispose();
|
|
199
|
+
const rels: string[] = [];
|
|
200
|
+
w.onChange((_first, batch) => {
|
|
201
|
+
if (batch) rels.push(...batch.paths.values());
|
|
202
|
+
});
|
|
203
|
+
try {
|
|
204
|
+
w.handleEvent(path.join(hqRoot, "companies", "acme", "note.md"), "change");
|
|
205
|
+
w.handleEvent(path.join(hqRoot, "repos", "x", "code.ts"), "change");
|
|
206
|
+
clock.advance(200);
|
|
207
|
+
expect(rels).toEqual(["companies/acme/note.md"]);
|
|
208
|
+
} finally {
|
|
209
|
+
w.dispose();
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
});
|