@indigoai-us/hq-cloud 5.25.0 → 5.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +34 -0
- package/dist/bin/sync-runner.d.ts +138 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +288 -16
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +372 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/sync/feature-flags.d.ts +136 -0
- package/dist/sync/feature-flags.d.ts.map +1 -0
- package/dist/sync/feature-flags.js +160 -0
- package/dist/sync/feature-flags.js.map +1 -0
- package/dist/sync/feature-flags.test.d.ts +24 -0
- package/dist/sync/feature-flags.test.d.ts.map +1 -0
- package/dist/sync/feature-flags.test.js +330 -0
- package/dist/sync/feature-flags.test.js.map +1 -0
- package/dist/sync/index.d.ts +19 -0
- package/dist/sync/index.d.ts.map +1 -0
- package/dist/sync/index.js +13 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/logger.d.ts +61 -0
- package/dist/sync/logger.d.ts.map +1 -0
- package/dist/sync/logger.js +51 -0
- package/dist/sync/logger.js.map +1 -0
- package/dist/sync/logger.test.d.ts +19 -0
- package/dist/sync/logger.test.d.ts.map +1 -0
- package/dist/sync/logger.test.js +199 -0
- package/dist/sync/logger.test.js.map +1 -0
- package/dist/sync/metrics.d.ts +89 -0
- package/dist/sync/metrics.d.ts.map +1 -0
- package/dist/sync/metrics.js +105 -0
- package/dist/sync/metrics.js.map +1 -0
- package/dist/sync/metrics.test.d.ts +19 -0
- package/dist/sync/metrics.test.d.ts.map +1 -0
- package/dist/sync/metrics.test.js +280 -0
- package/dist/sync/metrics.test.js.map +1 -0
- package/dist/sync/push-event.d.ts +110 -0
- package/dist/sync/push-event.d.ts.map +1 -0
- package/dist/sync/push-event.js +153 -0
- package/dist/sync/push-event.js.map +1 -0
- package/dist/sync/push-event.test.d.ts +15 -0
- package/dist/sync/push-event.test.d.ts.map +1 -0
- package/dist/sync/push-event.test.js +188 -0
- package/dist/sync/push-event.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +442 -0
- package/dist/sync/push-receiver.d.ts.map +1 -0
- package/dist/sync/push-receiver.js +782 -0
- package/dist/sync/push-receiver.js.map +1 -0
- package/dist/sync/push-receiver.test.d.ts +25 -0
- package/dist/sync/push-receiver.test.d.ts.map +1 -0
- package/dist/sync/push-receiver.test.js +477 -0
- package/dist/sync/push-receiver.test.js.map +1 -0
- package/dist/sync/push-transport.d.ts +150 -0
- package/dist/sync/push-transport.d.ts.map +1 -0
- package/dist/sync/push-transport.js +150 -0
- package/dist/sync/push-transport.js.map +1 -0
- package/dist/watcher.d.ts +271 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +480 -3
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.d.ts +2 -0
- package/dist/watcher.test.d.ts.map +1 -0
- package/dist/watcher.test.js +334 -0
- package/dist/watcher.test.js.map +1 -0
- package/package.json +10 -5
- package/src/bin/sync-runner.test.ts +487 -1
- package/src/bin/sync-runner.ts +406 -9
- package/src/index.ts +38 -0
- package/src/sync/feature-flags.test.ts +392 -0
- package/src/sync/feature-flags.ts +229 -0
- package/src/sync/index.ts +74 -0
- package/src/sync/logger.test.ts +241 -0
- package/src/sync/logger.ts +79 -0
- package/src/sync/metrics.test.ts +380 -0
- package/src/sync/metrics.ts +158 -0
- package/src/sync/push-event.test.ts +224 -0
- package/src/sync/push-event.ts +208 -0
- package/src/sync/push-receiver.test.ts +545 -0
- package/src/sync/push-receiver.ts +1077 -0
- package/src/sync/push-transport.ts +231 -0
- package/src/watcher.test.ts +388 -0
- package/src/watcher.ts +672 -4
- package/test/e2e/sync/cross-tenant-isolation.test.ts +502 -0
- package/test/e2e/watcher-real-chokidar.test.ts +105 -0
package/src/watcher.ts
CHANGED
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import * as fs from "fs";
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import { readFile, stat } from "node:fs/promises";
|
|
11
13
|
import * as path from "path";
|
|
12
14
|
import { watch } from "chokidar";
|
|
13
15
|
import type { FSWatcher } from "chokidar";
|
|
@@ -16,9 +18,188 @@ import { createIgnoreFilter, isWithinSizeLimit } from "./ignore.js";
|
|
|
16
18
|
import { readJournal, writeJournal, hashFile, updateEntry } from "./journal.js";
|
|
17
19
|
import { uploadFile, deleteRemoteFile } from "./s3.js";
|
|
18
20
|
import type { UploadAuthor } from "./s3.js";
|
|
21
|
+
import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
|
|
22
|
+
import { PERSONAL_VAULT_EXCLUDED_TOP_LEVEL } from "./personal-vault.js";
|
|
23
|
+
import type { PushEvent } from "./sync/push-event.js";
|
|
24
|
+
import type { PushTransport } from "./sync/push-transport.js";
|
|
25
|
+
import type { EventDrivenPushFlagProvider } from "./sync/feature-flags.js";
|
|
19
26
|
|
|
20
27
|
const DEBOUNCE_MS = 2000;
|
|
21
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Injectable clock seam (US-001).
|
|
31
|
+
*
|
|
32
|
+
* Production code uses the host timers; tests inject a {@link FakeClock} so the
|
|
33
|
+
* debounce window can be advanced deterministically without real wall-clock
|
|
34
|
+
* sleeps. US-002 (real chokidar watcher) and US-003 (runner wiring) build on
|
|
35
|
+
* this same seam — keep the surface minimal and stable.
|
|
36
|
+
*/
|
|
37
|
+
export interface Clock {
|
|
38
|
+
/** Schedule `fn` to run after `ms`. Returns an opaque handle. */
|
|
39
|
+
setTimeout(fn: () => void, ms: number): unknown;
|
|
40
|
+
/** Cancel a previously scheduled timeout. Safe to call with a stale handle. */
|
|
41
|
+
clearTimeout(handle: unknown): void;
|
|
42
|
+
/** Current epoch milliseconds. */
|
|
43
|
+
now(): number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Real clock backed by host timers + Date.now. The production default. */
|
|
47
|
+
export const systemClock: Clock = {
|
|
48
|
+
setTimeout: (fn, ms) => setTimeout(fn, ms),
|
|
49
|
+
clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
50
|
+
now: () => Date.now(),
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
interface FakeTimer {
|
|
54
|
+
id: number;
|
|
55
|
+
fireAt: number;
|
|
56
|
+
fn: () => void;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Deterministic clock for tests. Advance virtual time with {@link advance};
|
|
61
|
+
* any timers whose deadline has passed fire in scheduled order. No real timers
|
|
62
|
+
* are ever created, so a test using only this clock leaks nothing.
|
|
63
|
+
*/
|
|
64
|
+
export class FakeClock implements Clock {
|
|
65
|
+
private current = 0;
|
|
66
|
+
private nextId = 1;
|
|
67
|
+
private timers = new Map<number, FakeTimer>();
|
|
68
|
+
|
|
69
|
+
now(): number {
|
|
70
|
+
return this.current;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
setTimeout(fn: () => void, ms: number): unknown {
|
|
74
|
+
const id = this.nextId++;
|
|
75
|
+
this.timers.set(id, { id, fireAt: this.current + Math.max(0, ms), fn });
|
|
76
|
+
return id;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
clearTimeout(handle: unknown): void {
|
|
80
|
+
if (typeof handle === "number") this.timers.delete(handle);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Advance virtual time by `ms`, firing every timer whose deadline falls in
|
|
85
|
+
* the interval (in deadline order). Timers scheduled by a firing callback are
|
|
86
|
+
* honored within the same advance if their new deadline is still within the
|
|
87
|
+
* advanced window.
|
|
88
|
+
*/
|
|
89
|
+
advance(ms: number): void {
|
|
90
|
+
const target = this.current + ms;
|
|
91
|
+
while (true) {
|
|
92
|
+
const due = [...this.timers.values()]
|
|
93
|
+
.filter((t) => t.fireAt <= target)
|
|
94
|
+
.sort((a, b) => a.fireAt - b.fireAt || a.id - b.id);
|
|
95
|
+
if (due.length === 0) break;
|
|
96
|
+
const next = due[0];
|
|
97
|
+
this.timers.delete(next.id);
|
|
98
|
+
this.current = Math.max(this.current, next.fireAt);
|
|
99
|
+
next.fn();
|
|
100
|
+
}
|
|
101
|
+
this.current = target;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Number of timers still pending — a leak check for tests. */
|
|
105
|
+
pendingTimerCount(): number {
|
|
106
|
+
return this.timers.size;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** A push pass to run when a debounced change settles. May be async. */
|
|
111
|
+
export type PushFn = () => void | Promise<void>;
|
|
112
|
+
|
|
113
|
+
export interface WatchPushDriverOptions {
|
|
114
|
+
/** Quiet window (ms) before a settled change triggers a push. */
|
|
115
|
+
debounceMs?: number;
|
|
116
|
+
/** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
|
|
117
|
+
clock?: Clock;
|
|
118
|
+
/** Push pass to invoke when the window settles. */
|
|
119
|
+
push: PushFn;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The reusable debounce + coalesce + concurrency-guard core of event-driven
|
|
124
|
+
* push (US-001 seam).
|
|
125
|
+
*
|
|
126
|
+
* It is intentionally decoupled from chokidar and from S3: callers feed it
|
|
127
|
+
* synthetic or real change notifications via {@link notifyChange}, and it
|
|
128
|
+
* invokes the injected `push` fn at most once per quiet window. A push that is
|
|
129
|
+
* still in flight is never overlapped — a change arriving mid-push is collapsed
|
|
130
|
+
* and re-triggers a single follow-up pass after the next quiet window.
|
|
131
|
+
*
|
|
132
|
+
* US-002 wires a real chokidar watcher's events into {@link notifyChange};
|
|
133
|
+
* US-003 supplies the targeted-push `push` fn. Tests drive it directly with a
|
|
134
|
+
* {@link FakeClock} and a spy `push` fn (no real S3, no 10-minute sleep).
|
|
135
|
+
*/
|
|
136
|
+
export class WatchPushDriver {
|
|
137
|
+
private readonly debounceMs: number;
|
|
138
|
+
private readonly clock: Clock;
|
|
139
|
+
private readonly push: PushFn;
|
|
140
|
+
private timer: unknown = null;
|
|
141
|
+
private pushing = false;
|
|
142
|
+
private pendingWhilePushing = false;
|
|
143
|
+
private disposed = false;
|
|
144
|
+
|
|
145
|
+
constructor(opts: WatchPushDriverOptions) {
|
|
146
|
+
this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
|
|
147
|
+
this.clock = opts.clock ?? systemClock;
|
|
148
|
+
this.push = opts.push;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Register a change. Resets the quiet window; the push fires `debounceMs`
|
|
153
|
+
* after the LAST change in a burst, coalescing the burst to one push.
|
|
154
|
+
*/
|
|
155
|
+
notifyChange(): void {
|
|
156
|
+
if (this.disposed) return;
|
|
157
|
+
if (this.timer !== null) {
|
|
158
|
+
this.clock.clearTimeout(this.timer);
|
|
159
|
+
this.timer = null;
|
|
160
|
+
}
|
|
161
|
+
this.timer = this.clock.setTimeout(() => {
|
|
162
|
+
this.timer = null;
|
|
163
|
+
void this.fire();
|
|
164
|
+
}, this.debounceMs);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
private async fire(): Promise<void> {
|
|
168
|
+
if (this.disposed) return;
|
|
169
|
+
if (this.pushing) {
|
|
170
|
+
// A pass is in flight — collapse this trigger; re-arm after it settles.
|
|
171
|
+
this.pendingWhilePushing = true;
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
this.pushing = true;
|
|
175
|
+
try {
|
|
176
|
+
await this.push();
|
|
177
|
+
} finally {
|
|
178
|
+
this.pushing = false;
|
|
179
|
+
if (this.pendingWhilePushing && !this.disposed) {
|
|
180
|
+
this.pendingWhilePushing = false;
|
|
181
|
+
// Re-arm a fresh quiet window for the change(s) seen mid-push.
|
|
182
|
+
this.notifyChange();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** True while a push pass is executing. */
|
|
188
|
+
isPushing(): boolean {
|
|
189
|
+
return this.pushing;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Cancel any pending debounce timer; idempotent. Leaves no timers behind. */
|
|
193
|
+
dispose(): void {
|
|
194
|
+
this.disposed = true;
|
|
195
|
+
if (this.timer !== null) {
|
|
196
|
+
this.clock.clearTimeout(this.timer);
|
|
197
|
+
this.timer = null;
|
|
198
|
+
}
|
|
199
|
+
this.pendingWhilePushing = false;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
22
203
|
interface PendingChange {
|
|
23
204
|
type: "add" | "change" | "unlink";
|
|
24
205
|
absolutePath: string;
|
|
@@ -46,10 +227,11 @@ export class SyncWatcher {
|
|
|
46
227
|
if (this.watcher) return;
|
|
47
228
|
|
|
48
229
|
this.watcher = watch(this.hqRoot, {
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
|
|
52
|
-
|
|
230
|
+
// See toChokidarIgnored: chokidar's pre-stat descent probe has no stats
|
|
231
|
+
// hint, so a naive file-verdict prunes intermediate allowlist dirs
|
|
232
|
+
// before descending to their in-scope leaves. Keep a statless probe
|
|
233
|
+
// when EITHER the file or directory reading would survive the filter.
|
|
234
|
+
ignored: toChokidarIgnored(this.shouldSync, this.hqRoot),
|
|
53
235
|
persistent: true,
|
|
54
236
|
ignoreInitial: true,
|
|
55
237
|
awaitWriteFinish: {
|
|
@@ -146,3 +328,489 @@ export class SyncWatcher {
|
|
|
146
328
|
}
|
|
147
329
|
}
|
|
148
330
|
}
|
|
331
|
+
|
|
332
|
+
// ---------------------------------------------------------------------------
|
|
333
|
+
// US-002 — debounced, ignore-aware, exclusion-aware tree watcher.
|
|
334
|
+
//
|
|
335
|
+
// Unlike SyncWatcher (which does S3 work inline), TreeWatcher is a pure change
|
|
336
|
+
// detector: it emits a single debounced `changed` callback after a quiet window
|
|
337
|
+
// and never touches S3 itself. US-003 wires that callback to a targeted push.
|
|
338
|
+
//
|
|
339
|
+
// The emit decision composes the SAME filter stack the push walk uses, so a
|
|
340
|
+
// path that the push would skip never wakes the watcher:
|
|
341
|
+
// 1. createIgnoreFilter — .hqignore/.gitignore/DEFAULT_IGNORES
|
|
342
|
+
// 2. PERSONAL_VAULT_DEFAULT_EXCLUSIONS (personalMode) — .env/output/.beads/…
|
|
343
|
+
// 3. PERSONAL_VAULT_EXCLUDED_TOP_LEVEL (personalMode) — .git/companies/repos/workspace
|
|
344
|
+
// ---------------------------------------------------------------------------
|
|
345
|
+
|
|
346
|
+
/** Decision for a single path: emit a change for it, or ignore it. */
|
|
347
|
+
export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Translate an emit-filter into chokidar's `ignored` predicate (true = SKIP).
|
|
351
|
+
*
|
|
352
|
+
* The subtlety this encodes: chokidar probes a candidate path for the descent
|
|
353
|
+
* decision BEFORE it has stat()ed it, so `stats` is `undefined` on that first
|
|
354
|
+
* call — and if we say "ignore" there, chokidar prunes the subtree and never
|
|
355
|
+
* descends. In allowlist mode (`.hqinclude`) the in-scope leaves
|
|
356
|
+
* (`companies/*\/knowledge/`) live UNDER intermediate dirs (`companies/`,
|
|
357
|
+
* `companies/<slug>/`) that only survive the filter when queried as a
|
|
358
|
+
* directory (the ancestor matcher fires for `isDir=true` only). With no stats
|
|
359
|
+
* hint we don't know if the candidate is that ancestor dir or an in-scope
|
|
360
|
+
* file, so we must NOT prune when EITHER reading would keep it. The real
|
|
361
|
+
* isDir-accurate verdict is reapplied at event time (TreeWatcher.handleEvent
|
|
362
|
+
* re-checks with the known kind), so this only widens descent, never emit.
|
|
363
|
+
*/
|
|
364
|
+
function toChokidarIgnored(
|
|
365
|
+
shouldEmit: WatchPathFilter,
|
|
366
|
+
hqRoot: string,
|
|
367
|
+
): (filePath: string, stats?: fs.Stats) => boolean {
|
|
368
|
+
const root = path.resolve(hqRoot);
|
|
369
|
+
return (filePath: string, stats?: fs.Stats): boolean => {
|
|
370
|
+
// The watched root itself must NEVER be ignored, or chokidar tears down
|
|
371
|
+
// the whole watch. (shouldEmit returns false for it — it's not a valid
|
|
372
|
+
// emit target — so we special-case it here.)
|
|
373
|
+
if (path.resolve(filePath) === root) return false;
|
|
374
|
+
const isDir = stats?.isDirectory();
|
|
375
|
+
if (isDir === undefined) {
|
|
376
|
+
// Pre-stat descent probe: keep the path if it could be an in-scope file
|
|
377
|
+
// OR an ancestor directory of one.
|
|
378
|
+
return !(shouldEmit(filePath, false) || shouldEmit(filePath, true));
|
|
379
|
+
}
|
|
380
|
+
return !shouldEmit(filePath, isDir);
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Build the composite emit-decision predicate. Returns true when a change to
|
|
386
|
+
* `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
|
|
387
|
+
* layer). Pure and chokidar-free so the matching logic is unit-testable
|
|
388
|
+
* directly.
|
|
389
|
+
*
|
|
390
|
+
* @param hqRoot sync root (== personal-vault root in personalMode).
|
|
391
|
+
* @param personalMode when true, also applies the personal-vault default
|
|
392
|
+
* exclusions and the excluded-top-level buckets.
|
|
393
|
+
*/
|
|
394
|
+
export function createWatchPathFilter(
|
|
395
|
+
hqRoot: string,
|
|
396
|
+
personalMode = false,
|
|
397
|
+
): WatchPathFilter {
|
|
398
|
+
const ignoreFilter = createIgnoreFilter(hqRoot);
|
|
399
|
+
const excludedTopLevel = new Set(PERSONAL_VAULT_EXCLUDED_TOP_LEVEL);
|
|
400
|
+
|
|
401
|
+
return (absolutePath: string, isDir = false): boolean => {
|
|
402
|
+
const rel = path.relative(hqRoot, absolutePath).split(path.sep).join("/");
|
|
403
|
+
// The root itself, or anything outside it, is never an emit target.
|
|
404
|
+
if (rel === "" || rel.startsWith("..")) return false;
|
|
405
|
+
|
|
406
|
+
// Layer 1: shared ignore stack (.hqignore/.gitignore/DEFAULT_IGNORES).
|
|
407
|
+
// createIgnoreFilter returns true = "sync this"; false = "ignored".
|
|
408
|
+
if (!ignoreFilter(absolutePath, isDir)) return false;
|
|
409
|
+
|
|
410
|
+
if (personalMode) {
|
|
411
|
+
// Layer 3: excluded top-level buckets (.git/companies/repos/workspace).
|
|
412
|
+
const topLevel = rel.split("/")[0];
|
|
413
|
+
if (excludedTopLevel.has(topLevel)) return false;
|
|
414
|
+
|
|
415
|
+
// Layer 2: personal-vault default exclusions (.env/output/.beads/…).
|
|
416
|
+
if (isPersonalVaultExcluded(rel, isDir)) return false;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return true;
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
export interface TreeWatcherOptions {
|
|
424
|
+
/** Sync root to watch (== personal-vault root in personalMode). */
|
|
425
|
+
hqRoot: string;
|
|
426
|
+
/** Quiet window (ms) before a settled burst emits one `changed` call. */
|
|
427
|
+
debounceMs?: number;
|
|
428
|
+
/** Apply personal-vault default + top-level exclusions. */
|
|
429
|
+
personalMode?: boolean;
|
|
430
|
+
/** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
|
|
431
|
+
clock?: Clock;
|
|
432
|
+
/**
|
|
433
|
+
* Pre-built path filter override (test seam). When omitted, one is built
|
|
434
|
+
* from {@link createWatchPathFilter}.
|
|
435
|
+
*/
|
|
436
|
+
pathFilter?: WatchPathFilter;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Chokidar-backed file watcher that emits a single debounced `changed` signal
|
|
441
|
+
* after a {@link debounceMs} quiet window, coalescing bursts. It honors the
|
|
442
|
+
* full exclusion stack via {@link createWatchPathFilter}, so excluded paths
|
|
443
|
+
* (`.env`, `output/`, `.git/`, `companies/` in personalMode, …) never emit.
|
|
444
|
+
*
|
|
445
|
+
* Lifecycle: {@link start} (idempotent), {@link stop}, {@link dispose}. Stop
|
|
446
|
+
* closes the chokidar watcher (releasing fds) and cancels any pending debounce
|
|
447
|
+
* timer. dispose() is stop() + permanent shutdown.
|
|
448
|
+
*/
|
|
449
|
+
/**
|
|
450
|
+
* One settled change-burst, handed to {@link TreeWatcher} listeners. Carries
|
|
451
|
+
* the set of relative paths that changed during the quiet window so a listener
|
|
452
|
+
* (e.g. {@link PushEventEmitter}) can build one PushEvent per path. `paths` is
|
|
453
|
+
* absolute-path → relative-path; both are needed (relative for the wire shape,
|
|
454
|
+
* absolute for hashing/statting the file on disk).
|
|
455
|
+
*/
|
|
456
|
+
export interface TreeChangeBatch {
|
|
457
|
+
/** Map of absolutePath → relativePath for every path in the settled burst. */
|
|
458
|
+
paths: Map<string, string>;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Listener invoked once per settled debounce window.
|
|
463
|
+
*
|
|
464
|
+
* Backwards compatible with the US-003 `WatcherSurface` contract: the first
|
|
465
|
+
* argument is the OPTIONAL changed relative path the loop routes its targeted
|
|
466
|
+
* push to (the first path of the burst; undefined when the window settled with
|
|
467
|
+
* no captured path). US-008's {@link PushEventEmitter} consumes the SECOND
|
|
468
|
+
* argument — the full {@link TreeChangeBatch} of every path in the burst — to
|
|
469
|
+
* build one PushEvent per path. Listeners are free to ignore either argument.
|
|
470
|
+
*/
|
|
471
|
+
export type TreeChangeListener = (
|
|
472
|
+
changedRelPath?: string,
|
|
473
|
+
batch?: TreeChangeBatch,
|
|
474
|
+
) => void;
|
|
475
|
+
|
|
476
|
+
export class TreeWatcher {
|
|
477
|
+
private readonly hqRoot: string;
|
|
478
|
+
private readonly debounceMs: number;
|
|
479
|
+
private readonly clock: Clock;
|
|
480
|
+
private readonly shouldEmit: WatchPathFilter;
|
|
481
|
+
private watcher: FSWatcher | null = null;
|
|
482
|
+
private timer: unknown = null;
|
|
483
|
+
private listeners = new Set<TreeChangeListener>();
|
|
484
|
+
/** Paths accumulated for the current (in-flight) debounce window. */
|
|
485
|
+
private pending = new Map<string, string>();
|
|
486
|
+
private disposed = false;
|
|
487
|
+
|
|
488
|
+
constructor(opts: TreeWatcherOptions) {
|
|
489
|
+
this.hqRoot = opts.hqRoot;
|
|
490
|
+
this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
|
|
491
|
+
this.clock = opts.clock ?? systemClock;
|
|
492
|
+
this.shouldEmit =
|
|
493
|
+
opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/**
|
|
497
|
+
* Register a debounced-`changed` listener. Returns an unsubscribe fn.
|
|
498
|
+
*
|
|
499
|
+
* Listeners receive a {@link TreeChangeBatch} of the paths that changed in
|
|
500
|
+
* the settled window. Existing US-003 callers that only need the "something
|
|
501
|
+
* changed" signal can ignore the argument — the contract is backwards
|
|
502
|
+
* compatible (a zero-arg callback still type-checks).
|
|
503
|
+
*/
|
|
504
|
+
onChange(listener: TreeChangeListener): () => void {
|
|
505
|
+
this.listeners.add(listener);
|
|
506
|
+
return () => this.listeners.delete(listener);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Begin watching. Idempotent — a second call while already running is a
|
|
511
|
+
* no-op (no second chokidar instance, no leaked fds).
|
|
512
|
+
*/
|
|
513
|
+
start(): void {
|
|
514
|
+
if (this.disposed || this.watcher) return;
|
|
515
|
+
|
|
516
|
+
this.watcher = watch(this.hqRoot, {
|
|
517
|
+
// `ignored` returns true to SKIP. chokidar probes a candidate for the
|
|
518
|
+
// descent decision BEFORE stat()ing it (stats undefined on that call),
|
|
519
|
+
// so a naive `!shouldEmit(path, stats?.isDirectory())` treats the
|
|
520
|
+
// statless probe as a file and prunes intermediate dirs (`companies/`)
|
|
521
|
+
// that only survive the allowlist filter as directories — starving the
|
|
522
|
+
// in-scope leaves below them. toChokidarIgnored keeps a statless probe
|
|
523
|
+
// when EITHER the file or the directory reading would; the accurate
|
|
524
|
+
// isDir verdict is reapplied at event time in handleEvent.
|
|
525
|
+
ignored: toChokidarIgnored(this.shouldEmit, this.hqRoot),
|
|
526
|
+
persistent: true,
|
|
527
|
+
ignoreInitial: true,
|
|
528
|
+
awaitWriteFinish: {
|
|
529
|
+
stabilityThreshold: 500,
|
|
530
|
+
pollInterval: 100,
|
|
531
|
+
},
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
const onEvent = (absolutePath: string) => this.handleEvent(absolutePath);
|
|
535
|
+
this.watcher
|
|
536
|
+
.on("add", onEvent)
|
|
537
|
+
.on("change", onEvent)
|
|
538
|
+
.on("unlink", onEvent)
|
|
539
|
+
.on("addDir", onEvent)
|
|
540
|
+
.on("unlinkDir", onEvent)
|
|
541
|
+
.on("error", (err) => console.error("TreeWatcher error:", err));
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Test/seam entry point: feed a raw filesystem path as if chokidar reported
|
|
546
|
+
* it. Applies the emit filter then arms the debounce. Real chokidar events
|
|
547
|
+
* route through here too.
|
|
548
|
+
*/
|
|
549
|
+
handleEvent(absolutePath: string): void {
|
|
550
|
+
if (this.disposed) return;
|
|
551
|
+
// Defensive: chokidar's `ignored` already filtered, but a path that slips
|
|
552
|
+
// through (or a synthetic test event) is re-checked here.
|
|
553
|
+
if (!this.shouldEmit(absolutePath, false)) return;
|
|
554
|
+
const abs = path.resolve(absolutePath);
|
|
555
|
+
const rel = path.relative(this.hqRoot, abs).split(path.sep).join("/");
|
|
556
|
+
this.pending.set(abs, rel);
|
|
557
|
+
this.arm();
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
private arm(): void {
|
|
561
|
+
if (this.timer !== null) {
|
|
562
|
+
this.clock.clearTimeout(this.timer);
|
|
563
|
+
this.timer = null;
|
|
564
|
+
}
|
|
565
|
+
this.timer = this.clock.setTimeout(() => {
|
|
566
|
+
this.timer = null;
|
|
567
|
+
this.emit();
|
|
568
|
+
}, this.debounceMs);
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
private emit(): void {
|
|
572
|
+
if (this.disposed) return;
|
|
573
|
+
// Snapshot + clear the accumulated paths so the next window starts fresh
|
|
574
|
+
// even if a listener re-enters synchronously.
|
|
575
|
+
const batch: TreeChangeBatch = { paths: new Map(this.pending) };
|
|
576
|
+
this.pending.clear();
|
|
577
|
+
// First changed relative path of the burst — the US-003 routing argument.
|
|
578
|
+
// undefined when the window settled with no captured path (e.g. a synthetic
|
|
579
|
+
// arm() with no handleEvent).
|
|
580
|
+
const firstRel = [...batch.paths.values()][0];
|
|
581
|
+
for (const l of this.listeners) {
|
|
582
|
+
try {
|
|
583
|
+
l(firstRel, batch);
|
|
584
|
+
} catch (err) {
|
|
585
|
+
console.error("TreeWatcher listener error:", err);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/** True while the chokidar watcher is active. */
|
|
591
|
+
isWatching(): boolean {
|
|
592
|
+
return this.watcher !== null;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** Number of pending debounce timers — a leak check for tests. */
|
|
596
|
+
pendingTimerCount(): number {
|
|
597
|
+
return this.timer === null ? 0 : 1;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Stop watching: close the chokidar watcher (releasing fds) and cancel any
|
|
602
|
+
* pending debounce timer. Idempotent. The instance can be restarted with
|
|
603
|
+
* {@link start} unless {@link dispose} was called.
|
|
604
|
+
*/
|
|
605
|
+
stop(): void {
|
|
606
|
+
if (this.timer !== null) {
|
|
607
|
+
this.clock.clearTimeout(this.timer);
|
|
608
|
+
this.timer = null;
|
|
609
|
+
}
|
|
610
|
+
this.pending.clear();
|
|
611
|
+
if (this.watcher) {
|
|
612
|
+
void this.watcher.close();
|
|
613
|
+
this.watcher = null;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/** Permanent shutdown: stop() + drop listeners; further events are no-ops. */
|
|
618
|
+
dispose(): void {
|
|
619
|
+
this.stop();
|
|
620
|
+
this.disposed = true;
|
|
621
|
+
this.listeners.clear();
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// ---------------------------------------------------------------------------
|
|
626
|
+
// US-008 — PushEventEmitter: bridge the TreeWatcher to the Phase 2 transport.
|
|
627
|
+
//
|
|
628
|
+
// On each settled debounce window, build one PushEvent per changed path
|
|
629
|
+
// (sha-256 contentHash, monotonic per-device sequenceNumber, originTenantId)
|
|
630
|
+
// and hand it to the injected PushTransport. Feature-flag gated: when the
|
|
631
|
+
// originTenantId is NOT enabled the emitter is DORMANT — it subscribes to
|
|
632
|
+
// nothing and ships nothing, so the Phase 1 poll-only behavior is unchanged.
|
|
633
|
+
//
|
|
634
|
+
// Failure posture: a transport publish that throws (network/non-2xx/timeout)
|
|
635
|
+
// MUST NOT crash the daemon. The emitter catches per-event publish errors and
|
|
636
|
+
// routes them to `onError` (default: console.error); the existing cadence
|
|
637
|
+
// poll remains the safety net that eventually ships the change.
|
|
638
|
+
// ---------------------------------------------------------------------------
|
|
639
|
+
|
|
640
|
+
/** Computes the canonical contentHash for a file: `sha256:<64-hex>`. */
|
|
641
|
+
async function computeContentHash(absolutePath: string): Promise<string> {
|
|
642
|
+
const bytes = await readFile(absolutePath);
|
|
643
|
+
const hex = createHash("sha256").update(bytes).digest("hex");
|
|
644
|
+
return `sha256:${hex}`;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
export interface PushEventEmitterOptions {
|
|
648
|
+
/** Tenant identifier stamped onto every PushEvent + checked against the flag. */
|
|
649
|
+
originTenantId: string;
|
|
650
|
+
/** Device identifier stamped onto every PushEvent. */
|
|
651
|
+
originDeviceId: string;
|
|
652
|
+
/** Transport that ships each PushEvent (US-007 NoopPushTransport / HttpPushTransport). */
|
|
653
|
+
transport: PushTransport;
|
|
654
|
+
/**
|
|
655
|
+
* Feature-flag seam. When `isEnabled(originTenantId)` is false the emitter is
|
|
656
|
+
* dormant: {@link attach} subscribes nothing and {@link emitForBatch} is a
|
|
657
|
+
* no-op. Defaults are NOT supplied here — the caller injects an
|
|
658
|
+
* EventDrivenPushFlagProvider so dormancy is explicit.
|
|
659
|
+
*/
|
|
660
|
+
flagProvider: EventDrivenPushFlagProvider;
|
|
661
|
+
/**
|
|
662
|
+
* Returns the next monotonic sequence number for this device. Default: an
|
|
663
|
+
* internal counter starting at 0. Inject to persist across daemon restarts.
|
|
664
|
+
*/
|
|
665
|
+
getSequenceNumber?: () => number;
|
|
666
|
+
/** Clock for eventTimestamp. Default `() => new Date()`. */
|
|
667
|
+
now?: () => Date;
|
|
668
|
+
/**
|
|
669
|
+
* Where publish failures + hash/stat errors go. Default `console.error`.
|
|
670
|
+
* Receives the offending PushEvent (when known) so callers can correlate.
|
|
671
|
+
*/
|
|
672
|
+
onError?: (err: Error, ctx: { relativePath?: string }) => void;
|
|
673
|
+
/**
|
|
674
|
+
* Optional structured logger for the US-011 3-log diagnostic chain. When
|
|
675
|
+
* supplied, the emitter logs `event=watcher.emit` (the 1st correlated link)
|
|
676
|
+
* carrying the PushEvent's `sequenceNumber` — the same join key stamped by
|
|
677
|
+
* the server `push.receive` log and the client `fanout.receive` log. Default:
|
|
678
|
+
* no log (the daemon stays quiet unless wired with a logger).
|
|
679
|
+
*/
|
|
680
|
+
logger?: EmitterLogger;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/**
|
|
684
|
+
* Minimal structured logger surface for {@link PushEventEmitter}. A pino
|
|
685
|
+
* `Logger` (from `./sync/logger.ts`) satisfies this; tests inject a fake.
|
|
686
|
+
*/
|
|
687
|
+
export interface EmitterLogger {
|
|
688
|
+
info(obj: Record<string, unknown>, msg?: string): void;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Bridges {@link TreeWatcher} change batches to a {@link PushTransport} as
|
|
693
|
+
* typed PushEvents. Construct once per daemon, then {@link attach} to a
|
|
694
|
+
* running TreeWatcher (returns an unsubscribe fn). Flag-gated + failure-safe.
|
|
695
|
+
*/
|
|
696
|
+
export class PushEventEmitter {
|
|
697
|
+
private readonly originTenantId: string;
|
|
698
|
+
private readonly originDeviceId: string;
|
|
699
|
+
private readonly transport: PushTransport;
|
|
700
|
+
private readonly flagProvider: EventDrivenPushFlagProvider;
|
|
701
|
+
private readonly now: () => Date;
|
|
702
|
+
private readonly onError: (err: Error, ctx: { relativePath?: string }) => void;
|
|
703
|
+
private readonly logger: EmitterLogger | undefined;
|
|
704
|
+
private internalSeq = 0;
|
|
705
|
+
private readonly nextSeq: () => number;
|
|
706
|
+
|
|
707
|
+
constructor(opts: PushEventEmitterOptions) {
|
|
708
|
+
this.originTenantId = opts.originTenantId;
|
|
709
|
+
this.originDeviceId = opts.originDeviceId;
|
|
710
|
+
this.transport = opts.transport;
|
|
711
|
+
this.flagProvider = opts.flagProvider;
|
|
712
|
+
this.now = opts.now ?? (() => new Date());
|
|
713
|
+
this.logger = opts.logger;
|
|
714
|
+
this.onError =
|
|
715
|
+
opts.onError ??
|
|
716
|
+
((err, ctx) =>
|
|
717
|
+
console.error("PushEventEmitter error:", ctx.relativePath ?? "", err));
|
|
718
|
+
this.nextSeq =
|
|
719
|
+
opts.getSequenceNumber ??
|
|
720
|
+
(() => {
|
|
721
|
+
const n = this.internalSeq;
|
|
722
|
+
this.internalSeq += 1;
|
|
723
|
+
return n;
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/** True iff event-driven push is enabled for this emitter's tenant. */
|
|
728
|
+
get enabled(): boolean {
|
|
729
|
+
return this.flagProvider.isEnabled(this.originTenantId);
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Subscribe to a TreeWatcher's change batches. Dormant (no subscription)
|
|
734
|
+
* when the flag is OFF for this tenant. Returns an unsubscribe fn (a no-op
|
|
735
|
+
* when dormant).
|
|
736
|
+
*/
|
|
737
|
+
attach(watcher: TreeWatcher): () => void {
|
|
738
|
+
if (!this.enabled) return () => {};
|
|
739
|
+
return watcher.onChange((_changedRelPath, batch) => {
|
|
740
|
+
if (batch) void this.emitForBatch(batch);
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/**
|
|
745
|
+
* Build + ship one PushEvent per changed path in the batch. No-op when the
|
|
746
|
+
* flag is OFF. Each path is independent: a hash/stat failure or a transport
|
|
747
|
+
* publish rejection for one path is caught + surfaced via `onError` and does
|
|
748
|
+
* NOT abort the others or propagate (the daemon must not crash; the cadence
|
|
749
|
+
* poll covers any miss).
|
|
750
|
+
*/
|
|
751
|
+
async emitForBatch(batch: TreeChangeBatch): Promise<void> {
|
|
752
|
+
if (!this.enabled) return;
|
|
753
|
+
const entries = [...batch.paths.entries()];
|
|
754
|
+
await Promise.all(
|
|
755
|
+
entries.map(([absolutePath, relativePath]) =>
|
|
756
|
+
this.emitOne(absolutePath, relativePath),
|
|
757
|
+
),
|
|
758
|
+
);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
private async emitOne(
|
|
762
|
+
absolutePath: string,
|
|
763
|
+
relativePath: string,
|
|
764
|
+
): Promise<void> {
|
|
765
|
+
let event: PushEvent | undefined;
|
|
766
|
+
try {
|
|
767
|
+
const [contentHash, st] = await Promise.all([
|
|
768
|
+
computeContentHash(absolutePath),
|
|
769
|
+
stat(absolutePath),
|
|
770
|
+
]);
|
|
771
|
+
event = {
|
|
772
|
+
relativePath,
|
|
773
|
+
contentHash,
|
|
774
|
+
mtime: st.mtime.toISOString(),
|
|
775
|
+
originDeviceId: this.originDeviceId,
|
|
776
|
+
originTenantId: this.originTenantId,
|
|
777
|
+
sequenceNumber: this.nextSeq(),
|
|
778
|
+
eventTimestamp: this.now().toISOString(),
|
|
779
|
+
};
|
|
780
|
+
} catch (err) {
|
|
781
|
+
// File deleted between debounce fire and hash read (legal race), or stat
|
|
782
|
+
// failed. Surface, don't crash.
|
|
783
|
+
this.onError(err instanceof Error ? err : new Error(String(err)), {
|
|
784
|
+
relativePath,
|
|
785
|
+
});
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// US-011: 1st link of the 3-log diagnostic chain. Stamps the same
|
|
790
|
+
// `sequenceNumber` the server `push.receive` log and the client
|
|
791
|
+
// `fanout.receive` log carry, so an operator can walk one event
|
|
792
|
+
// end-to-end. Logged after the event is built, before the publish.
|
|
793
|
+
this.logger?.info(
|
|
794
|
+
{
|
|
795
|
+
event: "watcher.emit",
|
|
796
|
+
originTenantId: event.originTenantId,
|
|
797
|
+
originDeviceId: event.originDeviceId,
|
|
798
|
+
relativePath: event.relativePath,
|
|
799
|
+
contentHash: event.contentHash,
|
|
800
|
+
sequenceNumber: event.sequenceNumber,
|
|
801
|
+
eventTimestamp: event.eventTimestamp,
|
|
802
|
+
},
|
|
803
|
+
"watcher emitted push event",
|
|
804
|
+
);
|
|
805
|
+
|
|
806
|
+
try {
|
|
807
|
+
await this.transport.publish(event);
|
|
808
|
+
} catch (err) {
|
|
809
|
+
// Push failure (network / non-2xx / timeout). The cadence poll is the
|
|
810
|
+
// safety net — log + continue, never throw.
|
|
811
|
+
this.onError(err instanceof Error ? err : new Error(String(err)), {
|
|
812
|
+
relativePath,
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
}
|