@indigoai-us/hq-cloud 6.15.0 → 6.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-mutation.d.ts +16 -0
- package/dist/bin/sync-mutation.d.ts.map +1 -0
- package/dist/bin/sync-mutation.js +60 -0
- package/dist/bin/sync-mutation.js.map +1 -0
- package/dist/bin/sync-mutation.test.d.ts +2 -0
- package/dist/bin/sync-mutation.test.d.ts.map +1 -0
- package/dist/bin/sync-mutation.test.js +165 -0
- package/dist/bin/sync-mutation.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +8 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +16 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.d.ts +2 -0
- package/dist/bin/sync-runner-company.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.test.js +36 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +98 -8
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +17 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +109 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/conflict-recovery.test.d.ts +2 -0
- package/dist/cli/conflict-recovery.test.d.ts.map +1 -0
- package/dist/cli/conflict-recovery.test.js +201 -0
- package/dist/cli/conflict-recovery.test.js.map +1 -0
- package/dist/cli/conflict.d.ts +60 -0
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +333 -0
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/sync.d.ts +35 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +100 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +85 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/skill-telemetry.d.ts +6 -0
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +14 -2
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +79 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/candidate-uploader.d.ts +88 -0
- package/dist/sync/candidate-uploader.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.js +212 -0
- package/dist/sync/candidate-uploader.js.map +1 -0
- package/dist/sync/candidate-uploader.test.d.ts +2 -0
- package/dist/sync/candidate-uploader.test.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.test.js +132 -0
- package/dist/sync/candidate-uploader.test.js.map +1 -0
- package/dist/sync/delta-client.d.ts +73 -0
- package/dist/sync/delta-client.d.ts.map +1 -0
- package/dist/sync/delta-client.js +201 -0
- package/dist/sync/delta-client.js.map +1 -0
- package/dist/sync/delta-client.test.d.ts +2 -0
- package/dist/sync/delta-client.test.d.ts.map +1 -0
- package/dist/sync/delta-client.test.js +97 -0
- package/dist/sync/delta-client.test.js.map +1 -0
- package/dist/sync/durable-apply.d.ts +76 -0
- package/dist/sync/durable-apply.d.ts.map +1 -0
- package/dist/sync/durable-apply.js +530 -0
- package/dist/sync/durable-apply.js.map +1 -0
- package/dist/sync/durable-apply.test.d.ts +2 -0
- package/dist/sync/durable-apply.test.d.ts.map +1 -0
- package/dist/sync/durable-apply.test.js +180 -0
- package/dist/sync/durable-apply.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +33 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +149 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +142 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +2 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/multipart-uploader.d.ts +99 -0
- package/dist/sync/multipart-uploader.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.js +447 -0
- package/dist/sync/multipart-uploader.js.map +1 -0
- package/dist/sync/multipart-uploader.test.d.ts +2 -0
- package/dist/sync/multipart-uploader.test.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.test.js +119 -0
- package/dist/sync/multipart-uploader.test.js.map +1 -0
- package/dist/sync/mutation-client.d.ts +85 -0
- package/dist/sync/mutation-client.d.ts.map +1 -0
- package/dist/sync/mutation-client.js +245 -0
- package/dist/sync/mutation-client.js.map +1 -0
- package/dist/sync/mutation-client.test.d.ts +2 -0
- package/dist/sync/mutation-client.test.d.ts.map +1 -0
- package/dist/sync/mutation-client.test.js +51 -0
- package/dist/sync/mutation-client.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +45 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +101 -0
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +54 -2
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/scope-inventory-client.d.ts +69 -0
- package/dist/sync/scope-inventory-client.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.js +210 -0
- package/dist/sync/scope-inventory-client.js.map +1 -0
- package/dist/sync/scope-inventory-client.test.d.ts +2 -0
- package/dist/sync/scope-inventory-client.test.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.test.js +94 -0
- package/dist/sync/scope-inventory-client.test.js.map +1 -0
- package/dist/sync/snapshot-client.d.ts +98 -0
- package/dist/sync/snapshot-client.d.ts.map +1 -0
- package/dist/sync/snapshot-client.js +402 -0
- package/dist/sync/snapshot-client.js.map +1 -0
- package/dist/sync/snapshot-client.test.d.ts +2 -0
- package/dist/sync/snapshot-client.test.d.ts.map +1 -0
- package/dist/sync/snapshot-client.test.js +169 -0
- package/dist/sync/snapshot-client.test.js.map +1 -0
- package/dist/sync/uploader-finalization.d.ts +97 -0
- package/dist/sync/uploader-finalization.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.js +273 -0
- package/dist/sync/uploader-finalization.js.map +1 -0
- package/dist/sync/uploader-finalization.test.d.ts +2 -0
- package/dist/sync/uploader-finalization.test.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.test.js +92 -0
- package/dist/sync/uploader-finalization.test.js.map +1 -0
- package/dist/telemetry.d.ts +11 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +21 -2
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +80 -0
- package/dist/telemetry.test.js.map +1 -1
- package/package.json +6 -1
- package/.claude/policies/hq-cloud-esm-cannot-spy-fs-builtins.md +0 -30
- package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +0 -22
- package/.github/workflows/ci.yml +0 -84
- package/.github/workflows/publish.yml +0 -56
- package/.github/workflows/unreleased-commits-nag.yml +0 -256
- package/eslint.config.js +0 -67
- package/pnpm-workspace.yaml +0 -2
- package/scripts/presign-transport-e2e.mjs +0 -250
- package/scripts/vault-rebaseline.sh +0 -323
- package/scripts/vault-rescue.sh +0 -332
- package/src/active-company.test.ts +0 -188
- package/src/active-company.ts +0 -168
- package/src/agent-codex-instructions.test.ts +0 -332
- package/src/agent-codex-instructions.ts +0 -309
- package/src/auth.ts +0 -146
- package/src/backup-prune.test.ts +0 -98
- package/src/backup-prune.ts +0 -182
- package/src/bin/backup-prune-runner.ts +0 -33
- package/src/bin/rescue-runner.ts +0 -25
- package/src/bin/sync-runner-company.ts +0 -695
- package/src/bin/sync-runner-events.test.ts +0 -143
- package/src/bin/sync-runner-events.ts +0 -55
- package/src/bin/sync-runner-planning.test.ts +0 -311
- package/src/bin/sync-runner-planning.ts +0 -258
- package/src/bin/sync-runner-rollup.test.ts +0 -37
- package/src/bin/sync-runner-rollup.ts +0 -97
- package/src/bin/sync-runner-telemetry.ts +0 -15
- package/src/bin/sync-runner-watch-loop.ts +0 -1235
- package/src/bin/sync-runner-watch-routes.test.ts +0 -71
- package/src/bin/sync-runner-watch-routes.ts +0 -184
- package/src/bin/sync-runner.test.ts +0 -8767
- package/src/bin/sync-runner.ts +0 -2190
- package/src/cli/accept.ts +0 -124
- package/src/cli/conflict.ts +0 -119
- package/src/cli/doctor.test.ts +0 -581
- package/src/cli/doctor.ts +0 -642
- package/src/cli/index.ts +0 -49
- package/src/cli/invite.test.ts +0 -250
- package/src/cli/invite.ts +0 -214
- package/src/cli/promote.ts +0 -157
- package/src/cli/reindex-knowledge.test.ts +0 -307
- package/src/cli/reindex-knowledge.ts +0 -450
- package/src/cli/reindex.test.ts +0 -957
- package/src/cli/reindex.ts +0 -979
- package/src/cli/rescue-classify-ordering.test.ts +0 -548
- package/src/cli/rescue-clone-diagnostics.test.ts +0 -120
- package/src/cli/rescue-core.ts +0 -3011
- package/src/cli/rescue-drift-reconcile.test.ts +0 -179
- package/src/cli/rescue-drop-dir-symlink.test.ts +0 -224
- package/src/cli/rescue-exec-bit-preserve.test.ts +0 -187
- package/src/cli/rescue-hq-root-guard.test.ts +0 -232
- package/src/cli/rescue-journal-reconcile.test.ts +0 -215
- package/src/cli/rescue-mtime-preserve.test.ts +0 -203
- package/src/cli/rescue-settings-reconcile.test.ts +0 -637
- package/src/cli/rescue-snapshot.test.ts +0 -57
- package/src/cli/rescue-snapshot.ts +0 -51
- package/src/cli/rescue.reindex.test.ts +0 -63
- package/src/cli/rescue.test.ts +0 -131
- package/src/cli/rescue.ts +0 -182
- package/src/cli/share.test.ts +0 -7843
- package/src/cli/share.ts +0 -3663
- package/src/cli/sync-scope.test.ts +0 -652
- package/src/cli/sync.test.ts +0 -5207
- package/src/cli/sync.ts +0 -3470
- package/src/cli/tombstones.ts +0 -106
- package/src/cli/watch-event-push-conflict.test.ts +0 -234
- package/src/client-info.test.ts +0 -214
- package/src/client-info.ts +0 -121
- package/src/cognito-auth.test.ts +0 -712
- package/src/cognito-auth.ts +0 -1422
- package/src/company-resolver.test.ts +0 -618
- package/src/company-resolver.ts +0 -521
- package/src/context.test.ts +0 -583
- package/src/context.ts +0 -378
- package/src/daemon-worker.ts +0 -26
- package/src/daemon.ts +0 -99
- package/src/entity-resolver.test.ts +0 -315
- package/src/entity-resolver.ts +0 -180
- package/src/ignore.test.ts +0 -466
- package/src/ignore.ts +0 -469
- package/src/index.ts +0 -439
- package/src/journal.test.ts +0 -968
- package/src/journal.ts +0 -765
- package/src/lib/cloud-authoritative.test.ts +0 -45
- package/src/lib/cloud-authoritative.ts +0 -59
- package/src/lib/conflict-file.ts +0 -86
- package/src/lib/conflict-index.ts +0 -289
- package/src/lib/conflict.test.ts +0 -348
- package/src/lib/describe-error.test.ts +0 -100
- package/src/lib/describe-error.ts +0 -58
- package/src/lib/exit-codes.ts +0 -24
- package/src/lib/machine-id.test.ts +0 -231
- package/src/lib/machine-id.ts +0 -175
- package/src/lib/net-errors.test.ts +0 -65
- package/src/lib/net-errors.ts +0 -86
- package/src/lib/readlink-safe.test.ts +0 -43
- package/src/lib/readlink-safe.ts +0 -29
- package/src/local-path-codec.test.ts +0 -138
- package/src/local-path-codec.ts +0 -161
- package/src/machine-auth.test.ts +0 -1323
- package/src/manifest-reconcile.test.ts +0 -1123
- package/src/manifest-reconcile.ts +0 -518
- package/src/object-io.test.ts +0 -1221
- package/src/object-io.ts +0 -1306
- package/src/operation-lock.test.ts +0 -484
- package/src/operation-lock.ts +0 -680
- package/src/outcome-telemetry.test.ts +0 -498
- package/src/outcome-telemetry.ts +0 -639
- package/src/personal-vault-exclusions.test.ts +0 -308
- package/src/personal-vault-exclusions.ts +0 -354
- package/src/personal-vault.test.ts +0 -756
- package/src/personal-vault.ts +0 -496
- package/src/prefix-coalesce.test.ts +0 -240
- package/src/prefix-coalesce.ts +0 -273
- package/src/public-surface.test.ts +0 -117
- package/src/qmd-reindex.test.ts +0 -877
- package/src/qmd-reindex.ts +0 -842
- package/src/read-only-state-dir.test.ts +0 -188
- package/src/remote-pull.test.ts +0 -1130
- package/src/remote-pull.ts +0 -618
- package/src/s3.symlink-materialize.test.ts +0 -492
- package/src/s3.test.ts +0 -1789
- package/src/s3.ts +0 -1532
- package/src/schemas/signal-types.test.ts +0 -82
- package/src/schemas/signal-types.ts +0 -38
- package/src/schemas/source-channels.test.ts +0 -82
- package/src/schemas/source-channels.ts +0 -53
- package/src/scope-shrink.test.ts +0 -633
- package/src/scope-shrink.ts +0 -481
- package/src/signals/get.test.ts +0 -310
- package/src/signals/get.ts +0 -75
- package/src/signals/internals.ts +0 -195
- package/src/signals/list.test.ts +0 -420
- package/src/signals/list.ts +0 -79
- package/src/signals/parse.ts +0 -8
- package/src/signals/types.ts +0 -91
- package/src/skill-telemetry.test.ts +0 -1825
- package/src/skill-telemetry.ts +0 -1439
- package/src/sources/get.test.ts +0 -293
- package/src/sources/get.ts +0 -66
- package/src/sources/internals.ts +0 -198
- package/src/sources/list.test.ts +0 -402
- package/src/sources/list.ts +0 -84
- package/src/sources/parse.ts +0 -43
- package/src/sources/types.ts +0 -84
- package/src/sync/event-sync.test.ts +0 -594
- package/src/sync/event-sync.ts +0 -545
- package/src/sync/feature-flags.test.ts +0 -378
- package/src/sync/feature-flags.ts +0 -62
- package/src/sync/index.ts +0 -76
- package/src/sync/lease-client.test.ts +0 -128
- package/src/sync/lease-client.ts +0 -207
- package/src/sync/logger.test.ts +0 -242
- package/src/sync/logger.ts +0 -79
- package/src/sync/metrics.test.ts +0 -462
- package/src/sync/metrics.ts +0 -213
- package/src/sync/pull-scope.ts +0 -265
- package/src/sync/push-event.test.ts +0 -266
- package/src/sync/push-event.ts +0 -224
- package/src/sync/push-receiver.test.ts +0 -566
- package/src/sync/push-receiver.ts +0 -1048
- package/src/sync/push-transport.ts +0 -231
- package/src/sync/realtime-rollout.test.ts +0 -86
- package/src/sync/realtime-rollout.ts +0 -262
- package/src/sync/state-store.test.ts +0 -194
- package/src/sync/state-store.ts +0 -727
- package/src/sync-core.ts +0 -58
- package/src/sync-progress.test.ts +0 -94
- package/src/sync-progress.ts +0 -140
- package/src/telemetry-events.test.ts +0 -88
- package/src/telemetry-events.ts +0 -205
- package/src/telemetry.test.ts +0 -1280
- package/src/telemetry.ts +0 -1109
- package/src/types.ts +0 -314
- package/src/vault-client.test.ts +0 -1380
- package/src/vault-client.ts +0 -1694
- package/src/version.ts +0 -24
- package/src/watch-roots.test.ts +0 -278
- package/src/watch-roots.ts +0 -162
- package/src/watcher-event-gate.test.ts +0 -212
- package/src/watcher.test.ts +0 -1079
- package/src/watcher.ts +0 -1741
- package/test/e2e/sync/cross-tenant-isolation.test.ts +0 -630
- package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +0 -124
- package/test/e2e/sync/transient-company-leg.test.ts +0 -384
- package/test/e2e/sync/windows-unreadable-link-leg.test.ts +0 -191
- package/test/e2e/watcher-real-chokidar.test.ts +0 -165
- package/test/e2e/watcher-recursive-backend.test.ts +0 -181
- package/test/e2e/watcher-scoped-coverage.test.ts +0 -381
- package/test/invite-flow.integration.test.ts +0 -244
- package/test/joiner-manifest-reconcile.integration.test.ts +0 -322
- package/test/share-sync.integration.test.ts +0 -213
- package/tsconfig.json +0 -19
- package/vitest.config.ts +0 -22
package/src/watcher.ts
DELETED
|
@@ -1,1741 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* File watcher — monitors HQ directory for changes
|
|
3
|
-
* Uses chokidar with debounced batching
|
|
4
|
-
*
|
|
5
|
-
* Active watcher path: TreeWatcher detects local changes, WatchPushDriver
|
|
6
|
-
* schedules pushes, and PushEventEmitter publishes typed push events.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import * as fs from "fs";
|
|
10
|
-
import { createHash } from "node:crypto";
|
|
11
|
-
import { readFile, stat } from "node:fs/promises";
|
|
12
|
-
import * as path from "path";
|
|
13
|
-
import { watch } from "chokidar";
|
|
14
|
-
import { createIgnoreFilter } from "./ignore.js";
|
|
15
|
-
import {
|
|
16
|
-
isCoveredByRecursiveRoot,
|
|
17
|
-
planWatchRoots,
|
|
18
|
-
type WatchRootPlan,
|
|
19
|
-
} from "./watch-roots.js";
|
|
20
|
-
import { vaultKeyForLocalPath } from "./local-path-codec.js";
|
|
21
|
-
import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
|
|
22
|
-
import {
|
|
23
|
-
AGENCY_SYNC_REL,
|
|
24
|
-
CONTINUITY_POINTER_REL,
|
|
25
|
-
SESSION_LOGS_SYNC_REL,
|
|
26
|
-
PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,
|
|
27
|
-
} from "./personal-vault.js";
|
|
28
|
-
import type { PushEvent } from "./sync/push-event.js";
|
|
29
|
-
import type { PushTransport } from "./sync/push-transport.js";
|
|
30
|
-
import type { EventDrivenPushFlagProvider } from "./sync/feature-flags.js";
|
|
31
|
-
import {
|
|
32
|
-
emitCloudTelemetry,
|
|
33
|
-
type CloudTelemetryClient,
|
|
34
|
-
type TelemetryClaims,
|
|
35
|
-
} from "./telemetry-events.js";
|
|
36
|
-
|
|
37
|
-
const DEBOUNCE_MS = 2000;
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* A single Linux chokidar watcher must leave room for editors, dev servers,
|
|
41
|
-
* and other HQ runners on the same uid. This is deliberately far below the
|
|
42
|
-
* common 250k kernel maximum. The cap is on distinct directories admitted over
|
|
43
|
-
* this watcher's lifetime, which also prevents a high-churn tree from slowly
|
|
44
|
-
* accumulating an unbounded set of kernel watches.
|
|
45
|
-
*/
|
|
46
|
-
export const DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS = 20_000;
|
|
47
|
-
export const TREE_WATCHER_MAX_WATCHED_PATHS_ENV = "HQ_SYNC_MAX_WATCHED_PATHS";
|
|
48
|
-
|
|
49
|
-
function resolveMaxWatchedPaths(override: number | undefined): number {
|
|
50
|
-
const configured = override ?? Number(process.env[TREE_WATCHER_MAX_WATCHED_PATHS_ENV]);
|
|
51
|
-
if (override === undefined && process.env[TREE_WATCHER_MAX_WATCHED_PATHS_ENV] === undefined) {
|
|
52
|
-
return DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS;
|
|
53
|
-
}
|
|
54
|
-
if (Number.isFinite(configured) && configured > 0) return Math.trunc(configured);
|
|
55
|
-
console.warn(
|
|
56
|
-
`TreeWatcher ignoring invalid ${TREE_WATCHER_MAX_WATCHED_PATHS_ENV}; using ${DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS}`,
|
|
57
|
-
);
|
|
58
|
-
return DEFAULT_TREE_WATCHER_MAX_WATCHED_PATHS;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Injectable clock seam (US-001).
|
|
63
|
-
*
|
|
64
|
-
* Production code uses the host timers; tests inject a {@link FakeClock} so the
|
|
65
|
-
* debounce window can be advanced deterministically without real wall-clock
|
|
66
|
-
* sleeps. US-002 (real chokidar watcher) and US-003 (runner wiring) build on
|
|
67
|
-
* this same seam — keep the surface minimal and stable.
|
|
68
|
-
*/
|
|
69
|
-
export interface Clock {
|
|
70
|
-
/** Schedule `fn` to run after `ms`. Returns an opaque handle. */
|
|
71
|
-
setTimeout(fn: () => void, ms: number): unknown;
|
|
72
|
-
/** Cancel a previously scheduled timeout. Safe to call with a stale handle. */
|
|
73
|
-
clearTimeout(handle: unknown): void;
|
|
74
|
-
/** Current epoch milliseconds. */
|
|
75
|
-
now(): number;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/** Real clock backed by host timers + Date.now. The production default. */
|
|
79
|
-
export const systemClock: Clock = {
|
|
80
|
-
setTimeout: (fn, ms) => setTimeout(fn, ms),
|
|
81
|
-
clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
82
|
-
now: () => Date.now(),
|
|
83
|
-
};
|
|
84
|
-
|
|
85
|
-
interface FakeTimer {
|
|
86
|
-
id: number;
|
|
87
|
-
fireAt: number;
|
|
88
|
-
fn: () => void;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* Deterministic clock for tests. Advance virtual time with {@link advance};
|
|
93
|
-
* any timers whose deadline has passed fire in scheduled order. No real timers
|
|
94
|
-
* are ever created, so a test using only this clock leaks nothing.
|
|
95
|
-
*/
|
|
96
|
-
export class FakeClock implements Clock {
|
|
97
|
-
private current = 0;
|
|
98
|
-
private nextId = 1;
|
|
99
|
-
private timers = new Map<number, FakeTimer>();
|
|
100
|
-
|
|
101
|
-
now(): number {
|
|
102
|
-
return this.current;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
setTimeout(fn: () => void, ms: number): unknown {
|
|
106
|
-
const id = this.nextId++;
|
|
107
|
-
this.timers.set(id, { id, fireAt: this.current + Math.max(0, ms), fn });
|
|
108
|
-
return id;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
clearTimeout(handle: unknown): void {
|
|
112
|
-
if (typeof handle === "number") this.timers.delete(handle);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* Advance virtual time by `ms`, firing every timer whose deadline falls in
|
|
117
|
-
* the interval (in deadline order). Timers scheduled by a firing callback are
|
|
118
|
-
* honored within the same advance if their new deadline is still within the
|
|
119
|
-
* advanced window.
|
|
120
|
-
*/
|
|
121
|
-
advance(ms: number): void {
|
|
122
|
-
const target = this.current + ms;
|
|
123
|
-
while (true) {
|
|
124
|
-
const due = [...this.timers.values()]
|
|
125
|
-
.filter((t) => t.fireAt <= target)
|
|
126
|
-
.sort((a, b) => a.fireAt - b.fireAt || a.id - b.id);
|
|
127
|
-
if (due.length === 0) break;
|
|
128
|
-
const next = due[0];
|
|
129
|
-
this.timers.delete(next.id);
|
|
130
|
-
this.current = Math.max(this.current, next.fireAt);
|
|
131
|
-
next.fn();
|
|
132
|
-
}
|
|
133
|
-
this.current = target;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/** Number of timers still pending — a leak check for tests. */
|
|
137
|
-
pendingTimerCount(): number {
|
|
138
|
-
return this.timers.size;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/** A push pass to run when a debounced change settles. May be async. */
|
|
143
|
-
export type PushFn = () => void | Promise<void>;
|
|
144
|
-
|
|
145
|
-
export interface WatchPushDriverOptions {
|
|
146
|
-
/** Quiet window (ms) before a settled change triggers a push. */
|
|
147
|
-
debounceMs?: number;
|
|
148
|
-
/** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
|
|
149
|
-
clock?: Clock;
|
|
150
|
-
/** Push pass to invoke when the window settles. */
|
|
151
|
-
push: PushFn;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* The reusable debounce + coalesce + concurrency-guard core of event-driven
|
|
156
|
-
* push (US-001 seam).
|
|
157
|
-
*
|
|
158
|
-
* It is intentionally decoupled from chokidar and from S3: callers feed it
|
|
159
|
-
* synthetic or real change notifications via {@link notifyChange}, and it
|
|
160
|
-
* invokes the injected `push` fn at most once per quiet window. A push that is
|
|
161
|
-
* still in flight is never overlapped — a change arriving mid-push is collapsed
|
|
162
|
-
* and re-triggers a single follow-up pass after the next quiet window.
|
|
163
|
-
*
|
|
164
|
-
* US-002 wires a real chokidar watcher's events into {@link notifyChange};
|
|
165
|
-
* US-003 supplies the targeted-push `push` fn. Tests drive it directly with a
|
|
166
|
-
* {@link FakeClock} and a spy `push` fn (no real S3, no 10-minute sleep).
|
|
167
|
-
*/
|
|
168
|
-
export class WatchPushDriver {
|
|
169
|
-
private readonly debounceMs: number;
|
|
170
|
-
private readonly clock: Clock;
|
|
171
|
-
private readonly push: PushFn;
|
|
172
|
-
private timer: unknown = null;
|
|
173
|
-
private pushing = false;
|
|
174
|
-
private pendingWhilePushing = false;
|
|
175
|
-
private disposed = false;
|
|
176
|
-
|
|
177
|
-
constructor(opts: WatchPushDriverOptions) {
|
|
178
|
-
this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
|
|
179
|
-
this.clock = opts.clock ?? systemClock;
|
|
180
|
-
this.push = opts.push;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Register a change. Resets the quiet window; the push fires `debounceMs`
|
|
185
|
-
* after the LAST change in a burst, coalescing the burst to one push.
|
|
186
|
-
*/
|
|
187
|
-
notifyChange(): void {
|
|
188
|
-
if (this.disposed) return;
|
|
189
|
-
if (this.timer !== null) {
|
|
190
|
-
this.clock.clearTimeout(this.timer);
|
|
191
|
-
this.timer = null;
|
|
192
|
-
}
|
|
193
|
-
this.timer = this.clock.setTimeout(() => {
|
|
194
|
-
this.timer = null;
|
|
195
|
-
void this.fire().catch((err) => {
|
|
196
|
-
console.error("WatchPushDriver push failed:", err);
|
|
197
|
-
});
|
|
198
|
-
}, this.debounceMs);
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
private async fire(): Promise<void> {
|
|
202
|
-
if (this.disposed) return;
|
|
203
|
-
if (this.pushing) {
|
|
204
|
-
// A pass is in flight — collapse this trigger; re-arm after it settles.
|
|
205
|
-
this.pendingWhilePushing = true;
|
|
206
|
-
return;
|
|
207
|
-
}
|
|
208
|
-
this.pushing = true;
|
|
209
|
-
try {
|
|
210
|
-
await this.push();
|
|
211
|
-
} finally {
|
|
212
|
-
this.pushing = false;
|
|
213
|
-
if (this.pendingWhilePushing && !this.disposed) {
|
|
214
|
-
this.pendingWhilePushing = false;
|
|
215
|
-
// Re-arm a fresh quiet window for the change(s) seen mid-push.
|
|
216
|
-
this.notifyChange();
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/** True while a push pass is executing. */
|
|
222
|
-
isPushing(): boolean {
|
|
223
|
-
return this.pushing;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/** Cancel any pending debounce timer; idempotent. Leaves no timers behind. */
|
|
227
|
-
dispose(): void {
|
|
228
|
-
this.disposed = true;
|
|
229
|
-
if (this.timer !== null) {
|
|
230
|
-
this.clock.clearTimeout(this.timer);
|
|
231
|
-
this.timer = null;
|
|
232
|
-
}
|
|
233
|
-
this.pendingWhilePushing = false;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
// ---------------------------------------------------------------------------
|
|
238
|
-
// US-002 — debounced, ignore-aware, exclusion-aware tree watcher.
|
|
239
|
-
//
|
|
240
|
-
// TreeWatcher is a pure change detector: it emits a single debounced `changed`
|
|
241
|
-
// callback after a quiet window and never touches S3 itself. US-003 wires that
|
|
242
|
-
// callback to a targeted push.
|
|
243
|
-
//
|
|
244
|
-
// The emit decision composes the SAME filter stack the push walk uses, so a
|
|
245
|
-
// path that the push would skip never wakes the watcher:
|
|
246
|
-
// 1. createIgnoreFilter — .hqignore/.gitignore/DEFAULT_IGNORES
|
|
247
|
-
// 2. PERSONAL_VAULT_DEFAULT_EXCLUSIONS (personalMode) — .env/output/.beads/…
|
|
248
|
-
// 3. PERSONAL_VAULT_EXCLUDED_TOP_LEVEL (personalMode) — .git/companies/repos/workspace
|
|
249
|
-
// ---------------------------------------------------------------------------
|
|
250
|
-
|
|
251
|
-
/** Decision for a single path: emit a change for it, or ignore it. */
|
|
252
|
-
export type WatchPathFilter = (absolutePath: string, isDir?: boolean) => boolean;
|
|
253
|
-
|
|
254
|
-
export interface TreeWatcherWatchBudgetExceeded {
|
|
255
|
-
maxWatchedPaths: number;
|
|
256
|
-
watchedPaths: number;
|
|
257
|
-
/** Absolute directory whose admission would exceed the cap. */
|
|
258
|
-
offendingPath: string;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
/**
|
|
262
|
-
* Tracks directories admitted to chokidar. Chokidar asks its ignored callback
|
|
263
|
-
* twice: before stat (the descent gate) and after stat (where we can safely
|
|
264
|
-
* account only for directories). This class is intentionally lifetime-bounded:
|
|
265
|
-
* a directory that is deleted and recreated cannot turn one long-running
|
|
266
|
-
* runner into an unbounded consumer.
|
|
267
|
-
*/
|
|
268
|
-
export class ChokidarWatchBudget {
|
|
269
|
-
private readonly admitted = new Set<string>();
|
|
270
|
-
private exhausted = false;
|
|
271
|
-
|
|
272
|
-
constructor(
|
|
273
|
-
private readonly maxWatchedPaths: number,
|
|
274
|
-
private readonly onExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
|
|
275
|
-
) {}
|
|
276
|
-
|
|
277
|
-
admit(absolutePath: string): boolean {
|
|
278
|
-
const resolved = path.resolve(absolutePath);
|
|
279
|
-
if (this.admitted.has(resolved)) return true;
|
|
280
|
-
if (this.exhausted || this.admitted.size >= this.maxWatchedPaths) {
|
|
281
|
-
if (!this.exhausted) {
|
|
282
|
-
this.exhausted = true;
|
|
283
|
-
this.onExceeded({
|
|
284
|
-
maxWatchedPaths: this.maxWatchedPaths,
|
|
285
|
-
watchedPaths: this.admitted.size,
|
|
286
|
-
offendingPath: resolved,
|
|
287
|
-
});
|
|
288
|
-
}
|
|
289
|
-
return false;
|
|
290
|
-
}
|
|
291
|
-
this.admitted.add(resolved);
|
|
292
|
-
return true;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
count(): number {
|
|
296
|
-
return this.admitted.size;
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/**
|
|
301
|
-
* Translate the emit filter into chokidar's descent predicate.
|
|
302
|
-
*
|
|
303
|
-
* Chokidar first calls this with no stats, before it has chosen whether to
|
|
304
|
-
* recurse. That call is a DIRECTORY question, not an event-emission question:
|
|
305
|
-
* consulting `shouldEmit(path, false)` here lets directory-only ignores such
|
|
306
|
-
* as `node_modules/` masquerade as files and opens their whole subtree. The
|
|
307
|
-
* directory branch of createIgnoreFilter already contains the exact
|
|
308
|
-
* `.hqinclude` ancestor matcher, so it is both safe and sufficient to consult
|
|
309
|
-
* `shouldEmit(path, true)` at this gate.
|
|
310
|
-
*/
|
|
311
|
-
export function toChokidarIgnored(
|
|
312
|
-
shouldEmit: WatchPathFilter,
|
|
313
|
-
hqRoot: string,
|
|
314
|
-
budget?: ChokidarWatchBudget,
|
|
315
|
-
): (filePath: string, stats?: fs.Stats) => boolean {
|
|
316
|
-
const root = path.resolve(hqRoot);
|
|
317
|
-
return (filePath: string, stats?: fs.Stats): boolean => {
|
|
318
|
-
// The watched root itself must NEVER be ignored, or chokidar tears down
|
|
319
|
-
// the whole watch. (shouldEmit returns false for it — it's not a valid
|
|
320
|
-
// emit target — so we special-case it here.)
|
|
321
|
-
if (path.resolve(filePath) === root) {
|
|
322
|
-
return stats?.isDirectory() && budget !== undefined
|
|
323
|
-
? !budget.admit(filePath)
|
|
324
|
-
: false;
|
|
325
|
-
}
|
|
326
|
-
const isDir = stats?.isDirectory();
|
|
327
|
-
if (isDir === undefined) {
|
|
328
|
-
// This is chokidar's descent decision. The `isDir=true` branch includes
|
|
329
|
-
// only precise .hqinclude ancestors, rather than widening descent for
|
|
330
|
-
// every path that could happen to be an allowed file.
|
|
331
|
-
return !shouldEmit(filePath, true);
|
|
332
|
-
}
|
|
333
|
-
if (isDir) {
|
|
334
|
-
return (
|
|
335
|
-
!shouldEmit(filePath, true) ||
|
|
336
|
-
(budget !== undefined && !budget.admit(filePath))
|
|
337
|
-
);
|
|
338
|
-
}
|
|
339
|
-
return !shouldEmit(filePath, isDir);
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
/** A started watch backend. `close()` releases its OS handle(s); idempotent. */
|
|
344
|
-
export interface WatchBackend {
|
|
345
|
-
close(): void;
|
|
346
|
-
/** Native recursive watches need rename-kind state; chokidar does not. */
|
|
347
|
-
needsKnownKinds: boolean;
|
|
348
|
-
/**
|
|
349
|
-
* True when the backend already populated those known kinds through
|
|
350
|
-
* `onDirectorySeen` while planning its watch roots, so {@link TreeWatcher}
|
|
351
|
-
* must NOT walk the tree a second time to seed them.
|
|
352
|
-
*/
|
|
353
|
-
seededKnownKinds?: boolean;
|
|
354
|
-
/** Directories/OS handles admitted by this backend so degradation is auditable. */
|
|
355
|
-
watchedPathCount(): number;
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
export interface TreeWatcherDegradation {
|
|
359
|
-
reason: "watch_budget_exhausted" | "inotify_enospc";
|
|
360
|
-
maxWatchedPaths: number;
|
|
361
|
-
watchedPaths: number;
|
|
362
|
-
/** Full local path is deliberately log-only; telemetry receives its top level. */
|
|
363
|
-
offendingPath?: string;
|
|
364
|
-
offendingTopLevel?: string;
|
|
365
|
-
errorCode?: string;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
export interface TreeWatchBackendOptions {
|
|
369
|
-
hqRoot: string;
|
|
370
|
-
shouldEmit: WatchPathFilter;
|
|
371
|
-
onEvent: (absolutePath: string, kind: BackendChangeKind) => void;
|
|
372
|
-
onError: (err: unknown) => void;
|
|
373
|
-
maxWatchedPaths: number;
|
|
374
|
-
onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void;
|
|
375
|
-
/**
|
|
376
|
-
* Reports every in-scope directory a backend encounters while planning its
|
|
377
|
-
* watch roots, so the known-kinds index can be built in that same pass.
|
|
378
|
-
*/
|
|
379
|
-
onDirectorySeen?: (absolutePath: string) => void;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
export type TreeWatchBackendFactory = (
|
|
383
|
-
opts: TreeWatchBackendOptions,
|
|
384
|
-
) => WatchBackend;
|
|
385
|
-
|
|
386
|
-
function isInotifyLimitError(err: unknown): boolean {
|
|
387
|
-
return (
|
|
388
|
-
typeof err === "object" &&
|
|
389
|
-
err !== null &&
|
|
390
|
-
"code" in err &&
|
|
391
|
-
(err as { code?: unknown }).code === "ENOSPC"
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
function errorPath(err: unknown): string | undefined {
|
|
396
|
-
if (
|
|
397
|
-
typeof err === "object" &&
|
|
398
|
-
err !== null &&
|
|
399
|
-
"path" in err &&
|
|
400
|
-
typeof (err as { path?: unknown }).path === "string"
|
|
401
|
-
) {
|
|
402
|
-
return (err as { path: string }).path;
|
|
403
|
-
}
|
|
404
|
-
return undefined;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
export type TreeChangeKind =
|
|
408
|
-
| "add"
|
|
409
|
-
| "change"
|
|
410
|
-
| "unlink"
|
|
411
|
-
| "addDir"
|
|
412
|
-
| "unlinkDir";
|
|
413
|
-
|
|
414
|
-
/** Exact journal revision observed when a live watcher reports a deletion. */
|
|
415
|
-
export interface LocalDeleteSnapshot {
|
|
416
|
-
journalSlug: string;
|
|
417
|
-
journalPath: string;
|
|
418
|
-
/** Exact vault-relative root authorized by the originating watcher event. */
|
|
419
|
-
deleteScopeRoot?: string;
|
|
420
|
-
absolutePath: string;
|
|
421
|
-
remoteEtag: string;
|
|
422
|
-
localHash: string;
|
|
423
|
-
localKind: "file" | "symlink";
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
export interface TreeChange {
|
|
427
|
-
kind: TreeChangeKind;
|
|
428
|
-
deleteSnapshots?: LocalDeleteSnapshot[];
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
type BackendChangeKind = TreeChangeKind | "rename";
|
|
432
|
-
|
|
433
|
-
/**
|
|
434
|
-
* Platforms where Node's `fs.watch(dir, { recursive: true })` is natively
|
|
435
|
-
* supported by a SINGLE OS handle: macOS (FSEvents) and Windows
|
|
436
|
-
* (ReadDirectoryChangesW). On Linux, recursive `fs.watch` is NOT implemented
|
|
437
|
-
* (it throws `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM`), so we fall back to
|
|
438
|
-
* chokidar there.
|
|
439
|
-
*/
|
|
440
|
-
function supportsRecursiveWatch(): boolean {
|
|
441
|
-
return process.platform === "darwin" || process.platform === "win32";
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
function startChokidarTreeWatch(
|
|
445
|
-
hqRoot: string,
|
|
446
|
-
shouldEmit: WatchPathFilter,
|
|
447
|
-
onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
|
|
448
|
-
onError: (err: unknown) => void,
|
|
449
|
-
maxWatchedPaths: number,
|
|
450
|
-
onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
|
|
451
|
-
): WatchBackend {
|
|
452
|
-
const budget = new ChokidarWatchBudget(
|
|
453
|
-
maxWatchedPaths,
|
|
454
|
-
onWatchBudgetExceeded,
|
|
455
|
-
);
|
|
456
|
-
let cw: ReturnType<typeof watch>;
|
|
457
|
-
try {
|
|
458
|
-
cw = watch(hqRoot, {
|
|
459
|
-
// On Linux chokidar adds one fs.watch/inotify watch per descended dir.
|
|
460
|
-
// The predicate therefore enforces both scope and the hard per-runner cap.
|
|
461
|
-
ignored: toChokidarIgnored(shouldEmit, hqRoot, budget),
|
|
462
|
-
persistent: true,
|
|
463
|
-
ignoreInitial: true,
|
|
464
|
-
awaitWriteFinish: {
|
|
465
|
-
stabilityThreshold: 500,
|
|
466
|
-
pollInterval: 100,
|
|
467
|
-
},
|
|
468
|
-
});
|
|
469
|
-
} catch (err) {
|
|
470
|
-
// Some chokidar/fs.watch versions surface ENOSPC synchronously. Route it
|
|
471
|
-
// through the same degradation path as an asynchronous backend error.
|
|
472
|
-
onError(err);
|
|
473
|
-
return {
|
|
474
|
-
close: () => {},
|
|
475
|
-
needsKnownKinds: false,
|
|
476
|
-
watchedPathCount: () => budget.count(),
|
|
477
|
-
};
|
|
478
|
-
}
|
|
479
|
-
cw.on("add", (p) => onEvent(p, "add"))
|
|
480
|
-
.on("change", (p) => onEvent(p, "change"))
|
|
481
|
-
.on("unlink", (p) => onEvent(p, "unlink"))
|
|
482
|
-
.on("addDir", (p) => onEvent(p, "addDir"))
|
|
483
|
-
.on("unlinkDir", (p) => onEvent(p, "unlinkDir"))
|
|
484
|
-
.on("error", onError);
|
|
485
|
-
return {
|
|
486
|
-
close: () => {
|
|
487
|
-
void cw.close();
|
|
488
|
-
},
|
|
489
|
-
needsKnownKinds: false,
|
|
490
|
-
watchedPathCount: () => budget.count(),
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
/**
|
|
495
|
-
* macOS/Windows backend: recursive `fs.watch` handles placed over the in-scope
|
|
496
|
-
* tree ONLY, as planned by {@link planWatchRoots}.
|
|
497
|
-
*
|
|
498
|
-
* The previous shape was a single recursive watch on hqRoot. That is one handle
|
|
499
|
-
* regardless of tree size — but the OS then reports the WHOLE tree, including
|
|
500
|
-
* the buckets sync never uploads. On a real HQ root that is `repos/` (~26k
|
|
501
|
-
* directories) and `workspace/worktrees/` (~62k), where agent builds, installs,
|
|
502
|
-
* and git checkouts churn constantly. Each of those events woke the runner and
|
|
503
|
-
* was dropped only after the emit filter ran, which was enough allocation churn
|
|
504
|
-
* to pin the process above 100% CPU in GC.
|
|
505
|
-
*
|
|
506
|
-
* So instead: plan the minimum cover of the uploadable tree and watch that.
|
|
507
|
-
* A directory whose entire subtree is in scope takes one recursive watch; a
|
|
508
|
-
* directory containing an excluded child is watched non-recursively and its
|
|
509
|
-
* in-scope children are planned separately. Excluded subtrees get no watch at
|
|
510
|
-
* all, so the OS never reports them.
|
|
511
|
-
*
|
|
512
|
-
* Handle count stays small (tens, not thousands — well clear of the kqueue fd
|
|
513
|
-
* exhaustion that made chokidar 4 unusable here), and the emit filter still
|
|
514
|
-
* runs on every delivered event, so a depth-bounded plan admitting some residue
|
|
515
|
-
* is a performance concern, never a correctness one.
|
|
516
|
-
*/
|
|
517
|
-
/**
|
|
518
|
-
* How long the temporary whole-tree bootstrap watch is held open after the
|
|
519
|
-
* scoped plan goes live, so events the OS observed during the planning walk
|
|
520
|
-
* still drain instead of being dropped with the handle.
|
|
521
|
-
*/
|
|
522
|
-
const BOOTSTRAP_DRAIN_MS = 2000;
|
|
523
|
-
|
|
524
|
-
export interface ScopedWatchBackend extends WatchBackend {
|
|
525
|
-
/**
|
|
526
|
-
* Planned watches that could not be attached during the initial plan. Any
|
|
527
|
-
* failure here means the scoped plan has a permanent hole, so the caller
|
|
528
|
-
* discards it and falls back to chokidar rather than running with partial
|
|
529
|
-
* coverage.
|
|
530
|
-
*/
|
|
531
|
-
attachFailures: number;
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
export function startScopedRecursiveWatch(
|
|
535
|
-
hqRoot: string,
|
|
536
|
-
shouldEmit: WatchPathFilter,
|
|
537
|
-
onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
|
|
538
|
-
onError: (err: unknown) => void,
|
|
539
|
-
onDirectorySeen?: (absolutePath: string) => void,
|
|
540
|
-
): ScopedWatchBackend {
|
|
541
|
-
const root = path.resolve(hqRoot);
|
|
542
|
-
const watchers = new Map<string, fs.FSWatcher>();
|
|
543
|
-
const recursiveRoots = new Set<string>();
|
|
544
|
-
const recursiveFlag = new Map<string, boolean>();
|
|
545
|
-
let closed = false;
|
|
546
|
-
let attachFailures = 0;
|
|
547
|
-
/** Retires the bootstrap watch — replaced once that watch exists. */
|
|
548
|
-
let closeBootstrap: () => void = () => {};
|
|
549
|
-
|
|
550
|
-
/** lstat that answers "is this a directory right now", never throwing. */
|
|
551
|
-
function isDirectoryNow(absolutePath: string): boolean {
|
|
552
|
-
try {
|
|
553
|
-
return fs.lstatSync(absolutePath).isDirectory();
|
|
554
|
-
} catch {
|
|
555
|
-
return false;
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
function exists(absolutePath: string): boolean {
|
|
560
|
-
try {
|
|
561
|
-
fs.lstatSync(absolutePath);
|
|
562
|
-
return true;
|
|
563
|
-
} catch {
|
|
564
|
-
return false;
|
|
565
|
-
}
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
/**
|
|
569
|
-
* Release the watch on `dir` AND on everything beneath it.
|
|
570
|
-
*
|
|
571
|
-
* When a directory is removed the OS reports that directory, not each of its
|
|
572
|
-
* descendants — so dropping only the exact path leaks every watch underneath
|
|
573
|
-
* it. Those handles stay open forever, and a leaked recursive root keeps
|
|
574
|
-
* marking its (now absent) subtree as covered, so a later recreation is never
|
|
575
|
-
* re-attached. These runners live for many hours, so the leak accumulates
|
|
576
|
-
* across every directory the tree churns through.
|
|
577
|
-
*/
|
|
578
|
-
function dropWatch(dir: string): void {
|
|
579
|
-
const prefix = dir + path.sep;
|
|
580
|
-
for (const watched of [...watchers.keys()]) {
|
|
581
|
-
if (watched !== dir && !watched.startsWith(prefix)) continue;
|
|
582
|
-
const handle = watchers.get(watched);
|
|
583
|
-
if (handle) {
|
|
584
|
-
try {
|
|
585
|
-
handle.close();
|
|
586
|
-
} catch {
|
|
587
|
-
/* already closed */
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
watchers.delete(watched);
|
|
591
|
-
recursiveRoots.delete(watched);
|
|
592
|
-
recursiveFlag.delete(watched);
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
function addWatch(dir: string, recursive: boolean): void {
|
|
597
|
-
if (closed || watchers.has(dir)) return;
|
|
598
|
-
let handle: fs.FSWatcher;
|
|
599
|
-
try {
|
|
600
|
-
handle = fs.watch(
|
|
601
|
-
dir,
|
|
602
|
-
{ recursive, persistent: true },
|
|
603
|
-
(eventType, filename) => {
|
|
604
|
-
// `filename` is relative to THIS watch's directory (or null/empty if
|
|
605
|
-
// the platform couldn't provide it — nothing actionable then).
|
|
606
|
-
// `String()` coerces both the string and (older @types/node) Buffer
|
|
607
|
-
// shapes.
|
|
608
|
-
if (filename == null) return;
|
|
609
|
-
const rel = String(filename);
|
|
610
|
-
if (rel === "") return;
|
|
611
|
-
const abs = path.resolve(dir, rel);
|
|
612
|
-
const kind: BackendChangeKind =
|
|
613
|
-
eventType === "change" ? "change" : "rename";
|
|
614
|
-
if (kind === "rename") reconcileCoverage(abs);
|
|
615
|
-
onEvent(abs, kind);
|
|
616
|
-
},
|
|
617
|
-
);
|
|
618
|
-
} catch (err) {
|
|
619
|
-
// A planned watch we could not attach. Report it and record the hole so
|
|
620
|
-
// the caller can reject the whole plan instead of running half-blind.
|
|
621
|
-
attachFailures += 1;
|
|
622
|
-
onError(err);
|
|
623
|
-
return;
|
|
624
|
-
}
|
|
625
|
-
handle.on("error", (err) => {
|
|
626
|
-
onError(err);
|
|
627
|
-
if (closed) return;
|
|
628
|
-
// Rebuild this root rather than leaving it permanently unwatched. The
|
|
629
|
-
// onError above has already asked TreeWatcher for a reconcile, so a gap
|
|
630
|
-
// between the drop and the re-attach cannot lose data.
|
|
631
|
-
const wasRecursive = recursiveFlag.get(dir) ?? recursive;
|
|
632
|
-
dropWatch(dir);
|
|
633
|
-
if (exists(dir) && shouldEmit(dir, true)) addWatch(dir, wasRecursive);
|
|
634
|
-
});
|
|
635
|
-
watchers.set(dir, handle);
|
|
636
|
-
recursiveFlag.set(dir, recursive);
|
|
637
|
-
if (recursive) recursiveRoots.add(dir);
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
function applyPlan(plan: WatchRootPlan): void {
|
|
641
|
-
for (const dir of plan.recursive) addWatch(dir, true);
|
|
642
|
-
for (const dir of plan.shallow) addWatch(dir, false);
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
function planAndAttach(dir: string): void {
|
|
646
|
-
applyPlan(planWatchRoots(dir, shouldEmit, { onDirectory: onDirectorySeen }));
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Keep the live watch set honest about a path that just appeared, moved, or
|
|
651
|
-
* vanished. Three cases, all of which silently break coverage if ignored:
|
|
652
|
-
*
|
|
653
|
-
* 1. A watched directory was deleted (or deleted and recreated, which gives
|
|
654
|
-
* the replacement a new inode). Its stale handle would otherwise make the
|
|
655
|
-
* path look covered forever, so drop it and let the recreate re-plan.
|
|
656
|
-
* 2. An EXCLUDED directory was created under a recursive root — e.g.
|
|
657
|
-
* `workspace/worktrees/` or a fresh `node_modules/` that did not exist at
|
|
658
|
-
* plan time. The root now covers a bucket we must never watch, which is
|
|
659
|
-
* exactly the firehose this backend exists to prevent, so re-plan it.
|
|
660
|
-
* 3. An in-scope directory was created under a SHALLOW watch, so nothing
|
|
661
|
-
* covers its subtree yet. Attach before the event is handed on.
|
|
662
|
-
*/
|
|
663
|
-
function reconcileCoverage(absolutePath: string): void {
|
|
664
|
-
if (closed) return;
|
|
665
|
-
|
|
666
|
-
// (1) stale handle for a path that no longer exists.
|
|
667
|
-
if (watchers.has(absolutePath) && !exists(absolutePath)) {
|
|
668
|
-
dropWatch(absolutePath);
|
|
669
|
-
return;
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
const inScope = shouldEmit(absolutePath, true);
|
|
673
|
-
const covered = isCoveredByRecursiveRoot(absolutePath, recursiveRoots);
|
|
674
|
-
|
|
675
|
-
// (2) newly-created exclusion swallowed by an existing recursive root. The
|
|
676
|
-
// lstat is paid only for excluded paths under a recursive root — rare, and
|
|
677
|
-
// strictly cheaper than the unconditional stat this backend replaced.
|
|
678
|
-
if (!inScope) {
|
|
679
|
-
if (covered && isDirectoryNow(absolutePath)) {
|
|
680
|
-
for (const rootDir of [...recursiveRoots]) {
|
|
681
|
-
if (
|
|
682
|
-
absolutePath === rootDir ||
|
|
683
|
-
absolutePath.startsWith(rootDir + path.sep)
|
|
684
|
-
) {
|
|
685
|
-
dropWatch(rootDir);
|
|
686
|
-
planAndAttach(rootDir);
|
|
687
|
-
}
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
return;
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
// (3) in-scope directory with no coverage yet.
|
|
694
|
-
if (covered || watchers.has(absolutePath)) return;
|
|
695
|
-
if (!isDirectoryNow(absolutePath)) return;
|
|
696
|
-
onDirectorySeen?.(absolutePath);
|
|
697
|
-
planAndAttach(absolutePath);
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
// Cover the whole tree BEFORE planning. The plan walk below is synchronous
|
|
701
|
-
// and takes seconds on a real HQ root; without this, every change during that
|
|
702
|
-
// window would be invisible to event-driven push and would wait for the next
|
|
703
|
-
// cadence poll. This temporary watch is exactly the old backend's shape, so
|
|
704
|
-
// the bootstrap window is no worse than the previous behavior — and it is
|
|
705
|
-
// dropped the moment the scoped plan is live.
|
|
706
|
-
let bootstrap: fs.FSWatcher | null = null;
|
|
707
|
-
try {
|
|
708
|
-
bootstrap = fs.watch(
|
|
709
|
-
root,
|
|
710
|
-
{ recursive: true, persistent: true },
|
|
711
|
-
(eventType, filename) => {
|
|
712
|
-
if (filename == null) return;
|
|
713
|
-
const rel = String(filename);
|
|
714
|
-
if (rel === "") return;
|
|
715
|
-
onEvent(
|
|
716
|
-
path.resolve(root, rel),
|
|
717
|
-
eventType === "change" ? "change" : "rename",
|
|
718
|
-
);
|
|
719
|
-
},
|
|
720
|
-
);
|
|
721
|
-
bootstrap.on("error", onError);
|
|
722
|
-
} catch (err) {
|
|
723
|
-
// Recursive watch is unavailable at all — the plan below cannot work
|
|
724
|
-
// either, so record the hole and let the caller fall back.
|
|
725
|
-
attachFailures += 1;
|
|
726
|
-
onError(err);
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
// Unbounded on purpose: this walk replaces TreeWatcher's separate known-kinds
|
|
730
|
-
// seed walk (same tree, same filter, ~3.6s on a real HQ root), so planning
|
|
731
|
-
// the tree exhaustively costs nothing over the status quo and leaves no
|
|
732
|
-
// excluded subtree covered by a recursive root.
|
|
733
|
-
planAndAttach(root);
|
|
734
|
-
|
|
735
|
-
// The scoped plan is live, so narrow from whole-tree to in-scope-only — but
|
|
736
|
-
// NOT synchronously. Events observed during the planning walk can still be
|
|
737
|
-
// sitting in the OS queue undelivered, and closing the handle here drops
|
|
738
|
-
// them, reopening the very gap the bootstrap watch exists to close. Hold it
|
|
739
|
-
// open for one drain window instead; the overlap only costs duplicate events
|
|
740
|
-
// for excluded paths, which the emit filter drops and the debounce coalesces.
|
|
741
|
-
if (bootstrap) {
|
|
742
|
-
const handle = bootstrap;
|
|
743
|
-
bootstrap = null;
|
|
744
|
-
const timer = setTimeout(() => {
|
|
745
|
-
try {
|
|
746
|
-
handle.close();
|
|
747
|
-
} catch {
|
|
748
|
-
/* already closed */
|
|
749
|
-
}
|
|
750
|
-
closeBootstrap = () => {};
|
|
751
|
-
}, BOOTSTRAP_DRAIN_MS);
|
|
752
|
-
// Never hold the process open just to retire a temporary watch.
|
|
753
|
-
(timer as { unref?: () => void }).unref?.();
|
|
754
|
-
closeBootstrap = () => {
|
|
755
|
-
clearTimeout(timer);
|
|
756
|
-
try {
|
|
757
|
-
handle.close();
|
|
758
|
-
} catch {
|
|
759
|
-
/* already closed */
|
|
760
|
-
}
|
|
761
|
-
closeBootstrap = () => {};
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
return {
|
|
766
|
-
close: () => {
|
|
767
|
-
closed = true;
|
|
768
|
-
closeBootstrap();
|
|
769
|
-
for (const handle of watchers.values()) {
|
|
770
|
-
try {
|
|
771
|
-
handle.close();
|
|
772
|
-
} catch {
|
|
773
|
-
/* already closed */
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
watchers.clear();
|
|
777
|
-
recursiveRoots.clear();
|
|
778
|
-
recursiveFlag.clear();
|
|
779
|
-
},
|
|
780
|
-
needsKnownKinds: true,
|
|
781
|
-
seededKnownKinds: onDirectorySeen !== undefined,
|
|
782
|
-
watchedPathCount: () => watchers.size,
|
|
783
|
-
attachFailures,
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
/**
|
|
788
|
-
* Start watching `hqRoot`, calling `onEvent(absolutePath)` for every change.
|
|
789
|
-
*
|
|
790
|
-
* Backend selection (the whole point of this helper):
|
|
791
|
-
* - macOS / Windows → scoped recursive `fs.watch` handles over the in-scope
|
|
792
|
-
* tree only ({@link startScopedRecursiveWatch}). chokidar 4 dropped its
|
|
793
|
-
* `fsevents` backend, so on macOS it watches via kqueue, which costs ~1
|
|
794
|
-
* open fd PER watched path — ~11k fds over a real HQ tree, which EMFILEs
|
|
795
|
-
* under the default soft `ulimit -n` (256) and silently kills the watcher
|
|
796
|
-
* (instant sync then falls back to the poll).
|
|
797
|
-
* - Linux / other → chokidar (recursive `fs.watch` is unsupported there).
|
|
798
|
-
*
|
|
799
|
-
* The recursive backend does no descent-time filtering WITHIN a watched
|
|
800
|
-
* subtree, so changes there are funneled to `onEvent`, which re-applies the
|
|
801
|
-
* emit filter via {@link TreeWatcher.handleEvent}. That filter remains the
|
|
802
|
-
* authoritative gate; the watch plan is a performance optimization on top of
|
|
803
|
-
* it. This is also why the recursive backend is immune to the `.hqinclude`
|
|
804
|
-
* ancestor-descent pruning that the chokidar backend needs
|
|
805
|
-
* {@link toChokidarIgnored} to avoid.
|
|
806
|
-
*/
|
|
807
|
-
function startTreeWatch(
|
|
808
|
-
hqRoot: string,
|
|
809
|
-
shouldEmit: WatchPathFilter,
|
|
810
|
-
onEvent: (absolutePath: string, kind: BackendChangeKind) => void,
|
|
811
|
-
onError: (err: unknown) => void,
|
|
812
|
-
maxWatchedPaths: number,
|
|
813
|
-
onWatchBudgetExceeded: (info: TreeWatcherWatchBudgetExceeded) => void,
|
|
814
|
-
onDirectorySeen?: (absolutePath: string) => void,
|
|
815
|
-
): WatchBackend {
|
|
816
|
-
if (supportsRecursiveWatch()) {
|
|
817
|
-
try {
|
|
818
|
-
const scoped = startScopedRecursiveWatch(
|
|
819
|
-
hqRoot,
|
|
820
|
-
shouldEmit,
|
|
821
|
-
onEvent,
|
|
822
|
-
onError,
|
|
823
|
-
onDirectorySeen,
|
|
824
|
-
);
|
|
825
|
-
// Partial coverage is worse than a different backend: a planned root we
|
|
826
|
-
// could not attach is silently unwatched forever, and the operator only
|
|
827
|
-
// sees it as "instant sync stopped working for that subtree". Accept the
|
|
828
|
-
// scoped plan only when EVERY planned watch attached.
|
|
829
|
-
if (scoped.attachFailures === 0 && scoped.watchedPathCount() > 0) {
|
|
830
|
-
return scoped;
|
|
831
|
-
}
|
|
832
|
-
scoped.close();
|
|
833
|
-
} catch (err) {
|
|
834
|
-
// Recursive watch unexpectedly unavailable — fall back to chokidar
|
|
835
|
-
// rather than leaving the daemon with no watcher at all.
|
|
836
|
-
onError(err);
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
return startChokidarTreeWatch(
|
|
841
|
-
hqRoot,
|
|
842
|
-
shouldEmit,
|
|
843
|
-
onEvent,
|
|
844
|
-
onError,
|
|
845
|
-
maxWatchedPaths,
|
|
846
|
-
onWatchBudgetExceeded,
|
|
847
|
-
);
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
/**
|
|
851
|
-
* Build the composite emit-decision predicate. Returns true when a change to
|
|
852
|
-
* `absolutePath` SHOULD wake the watcher (i.e. it survives every exclusion
|
|
853
|
-
* layer). Pure and chokidar-free so the matching logic is unit-testable
|
|
854
|
-
* directly.
|
|
855
|
-
*
|
|
856
|
-
* @param hqRoot sync root (== personal-vault root in personalMode).
|
|
857
|
-
* @param personalMode when true, also applies the personal-vault default
|
|
858
|
-
* exclusions and the excluded-top-level buckets.
|
|
859
|
-
*/
|
|
860
|
-
export function createWatchPathFilter(
|
|
861
|
-
hqRoot: string,
|
|
862
|
-
personalMode = false,
|
|
863
|
-
): WatchPathFilter {
|
|
864
|
-
const ignoreFilter = createIgnoreFilter(hqRoot);
|
|
865
|
-
const excludedTopLevel = new Set(PERSONAL_VAULT_EXCLUDED_TOP_LEVEL);
|
|
866
|
-
|
|
867
|
-
return (absolutePath: string, isDir = false): boolean => {
|
|
868
|
-
const rel = path.relative(hqRoot, absolutePath).split(path.sep).join("/");
|
|
869
|
-
// The root itself, or anything outside it, is never an emit target.
|
|
870
|
-
if (rel === "" || rel.startsWith("..")) return false;
|
|
871
|
-
|
|
872
|
-
// Layer 1: shared ignore stack (.hqignore/.gitignore/DEFAULT_IGNORES).
|
|
873
|
-
// createIgnoreFilter returns true = "sync this"; false = "ignored".
|
|
874
|
-
if (!ignoreFilter(absolutePath, isDir)) return false;
|
|
875
|
-
|
|
876
|
-
// Never-synced heavy buckets — pruned in BOTH modes. The local git worktrees
|
|
877
|
-
// (`workspace/worktrees/`, where `/worktree` and agent coding runs check out
|
|
878
|
-
// repos — often 100k+ transient source/build files) and the on-box `repos/`
|
|
879
|
-
// checkout are never vault-synced in any mode. In personalMode the
|
|
880
|
-
// excluded-top-level layer below already drops them; in `--companies` mode
|
|
881
|
-
// that layer is skipped, so without this the watcher descends into every
|
|
882
|
-
// worktree source tree — tens of thousands of wasted inotify watches (the
|
|
883
|
-
// outpost sat at ~78% of the OS limit) plus CPU spent processing build churn
|
|
884
|
-
// it immediately discards, and enough backlog to trip the
|
|
885
|
-
// overflow → full-reconcile fallback. Returning false for the dir probe too
|
|
886
|
-
// prunes the whole subtree at descent instead of per-file. `companies/` and
|
|
887
|
-
// the synced `workspace/agency/` + continuity-pointer paths are untouched.
|
|
888
|
-
const topSegments = rel.split("/");
|
|
889
|
-
if (topSegments[0] === "repos") return false;
|
|
890
|
-
if (topSegments[0] === "workspace" && topSegments[1] === "worktrees") {
|
|
891
|
-
return false;
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
if (personalMode) {
|
|
895
|
-
// Continuity-pointer carve-out: the session pointer lives under the
|
|
896
|
-
// otherwise-excluded `workspace/`, so it must re-include BEFORE the
|
|
897
|
-
// top-level bucket rejection below. Two cases:
|
|
898
|
-
// (a) the pointer FILE itself → emit. Waking on the pointer is
|
|
899
|
-
// sufficient: `/handoff` rewrites handoff.json AFTER its
|
|
900
|
-
// (immutable) thread file, and the resulting personal push
|
|
901
|
-
// re-enumerates via computePersonalVaultPaths, which sweeps the
|
|
902
|
-
// pointer + its active thread file together.
|
|
903
|
-
// (b) an ANCESTOR DIR of the pointer (`workspace`, `workspace/threads`)
|
|
904
|
-
// queried as a directory → allow descent. The Linux chokidar
|
|
905
|
-
// backend prunes a subtree the instant its dir probe says "ignore",
|
|
906
|
-
// so without this it would prune `workspace/` before ever reaching
|
|
907
|
-
// the leaf (the same ancestor-descent subtlety `.hqinclude` handles
|
|
908
|
-
// via its ancestor matcher). The native recursive backend ignores
|
|
909
|
-
// descent decisions and re-checks each path at event time, so this
|
|
910
|
-
// only matters for chokidar but is harmless everywhere.
|
|
911
|
-
// See computeContinuityPointerPaths for the matching push-side carve-out.
|
|
912
|
-
if (rel === CONTINUITY_POINTER_REL && !isDir) return true;
|
|
913
|
-
if (isDir && CONTINUITY_POINTER_REL.startsWith(rel + "/")) return true;
|
|
914
|
-
|
|
915
|
-
// Agency-workspace carve-out: `workspace/agency/` (the whole subtree) IS
|
|
916
|
-
// synced via computeAgencySyncPaths (mirrored on the push side), so it must
|
|
917
|
-
// re-include BEFORE the top-level `workspace/` rejection below — same
|
|
918
|
-
// reason as the continuity pointer above, but for an entire subtree rather
|
|
919
|
-
// than one file. Without this, a `--personal --watch` runner never emits
|
|
920
|
-
// agency changes and they wait for the 60s poll. Paths at/under agency
|
|
921
|
-
// emit (handleEvent always probes isDir=false); agency itself and its
|
|
922
|
-
// ancestor dir (`workspace`) return true for the dir probe so chokidar
|
|
923
|
-
// descends to reach them instead of pruning `workspace/`.
|
|
924
|
-
if (rel === AGENCY_SYNC_REL || rel.startsWith(AGENCY_SYNC_REL + "/")) {
|
|
925
|
-
return true;
|
|
926
|
-
}
|
|
927
|
-
if (isDir && AGENCY_SYNC_REL.startsWith(rel + "/")) return true;
|
|
928
|
-
|
|
929
|
-
// Session-logs carve-out: `workspace/.session-logs/` (the whole subtree)
|
|
930
|
-
// IS synced via computeSessionLogsSyncPaths (mirrored on the push side),
|
|
931
|
-
// so it must re-include BEFORE the top-level `workspace/` rejection below
|
|
932
|
-
// — identical shape to the agency carve-out just above. Without this, a
|
|
933
|
-
// `--personal --watch` runner never emits reindex-captured session-log
|
|
934
|
-
// changes and they wait for the 60s poll. Paths at/under it emit; the
|
|
935
|
-
// subtree root and its ancestor dir (`workspace`) return true for the dir
|
|
936
|
-
// probe so chokidar descends to reach them instead of pruning `workspace/`.
|
|
937
|
-
if (
|
|
938
|
-
rel === SESSION_LOGS_SYNC_REL ||
|
|
939
|
-
rel.startsWith(SESSION_LOGS_SYNC_REL + "/")
|
|
940
|
-
) {
|
|
941
|
-
return true;
|
|
942
|
-
}
|
|
943
|
-
if (isDir && SESSION_LOGS_SYNC_REL.startsWith(rel + "/")) return true;
|
|
944
|
-
|
|
945
|
-
// Layer 3: excluded top-level buckets (.git/companies/repos/workspace).
|
|
946
|
-
const topLevel = rel.split("/")[0];
|
|
947
|
-
if (excludedTopLevel.has(topLevel)) return false;
|
|
948
|
-
|
|
949
|
-
// Layer 2: personal-vault default exclusions (.env/output/.beads/…).
|
|
950
|
-
if (isPersonalVaultExcluded(rel, isDir)) return false;
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
return true;
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
export const DEFAULT_TREE_WATCHER_MAX_PENDING_PATHS = 4096;
|
|
958
|
-
export const DEFAULT_TREE_WATCHER_MAX_PENDING_BYTES = 1024 * 1024;
|
|
959
|
-
|
|
960
|
-
export interface TreeWatcherBacklogOverflow {
|
|
961
|
-
pendingPaths: number;
|
|
962
|
-
pendingBytes: number;
|
|
963
|
-
maxPendingPaths: number;
|
|
964
|
-
maxPendingBytes: number;
|
|
965
|
-
droppedPaths: number;
|
|
966
|
-
droppedBytes: number;
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
function estimatePendingEntryBytes(
|
|
970
|
-
absolutePath: string,
|
|
971
|
-
relativePath: string,
|
|
972
|
-
): number {
|
|
973
|
-
return Buffer.byteLength(absolutePath) + Buffer.byteLength(relativePath) + 64;
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
export interface TreeWatcherOptions {
|
|
977
|
-
/** Sync root to watch (== personal-vault root in personalMode). */
|
|
978
|
-
hqRoot: string;
|
|
979
|
-
/** Quiet window (ms) before a settled burst emits one `changed` call. */
|
|
980
|
-
debounceMs?: number;
|
|
981
|
-
/** Apply personal-vault default + top-level exclusions. */
|
|
982
|
-
personalMode?: boolean;
|
|
983
|
-
/** Clock seam — defaults to {@link systemClock}; tests inject {@link FakeClock}. */
|
|
984
|
-
clock?: Clock;
|
|
985
|
-
/**
|
|
986
|
-
* Pre-built path filter override (test seam). When omitted, one is built
|
|
987
|
-
* from {@link createWatchPathFilter}.
|
|
988
|
-
*/
|
|
989
|
-
pathFilter?: WatchPathFilter;
|
|
990
|
-
/** Maximum distinct paths retained in one debounce window. */
|
|
991
|
-
maxPendingPaths?: number;
|
|
992
|
-
/** Approximate maximum path-string bytes retained in one debounce window. */
|
|
993
|
-
maxPendingBytes?: number;
|
|
994
|
-
/**
|
|
995
|
-
* Hard cap on distinct directories admitted to the chokidar backend for the
|
|
996
|
-
* lifetime of this instance. Linux uses one inotify watch per such directory.
|
|
997
|
-
* Overrides `HQ_SYNC_MAX_WATCHED_PATHS` for callers that construct a watcher.
|
|
998
|
-
*/
|
|
999
|
-
maxWatchedPaths?: number;
|
|
1000
|
-
/** Backlog overflow signal. Defaults to a console warning. */
|
|
1001
|
-
onBacklogOverflow?: (info: TreeWatcherBacklogOverflow) => void;
|
|
1002
|
-
/**
|
|
1003
|
-
* Structured degradation/metric seam. It fires exactly once when the watch
|
|
1004
|
-
* cap or the kernel's inotify limit disables event watching; cadence polling
|
|
1005
|
-
* remains the correctness path.
|
|
1006
|
-
*/
|
|
1007
|
-
onDegraded?: (info: TreeWatcherDegradation) => void;
|
|
1008
|
-
/** Optional cloud telemetry sink for the same degradation signal. */
|
|
1009
|
-
telemetryClient?: CloudTelemetryClient | null;
|
|
1010
|
-
telemetryClaims?: TelemetryClaims | null;
|
|
1011
|
-
/** Test seam for a watch backend; production uses native/chokidar selection. */
|
|
1012
|
-
backendFactory?: TreeWatchBackendFactory;
|
|
1013
|
-
captureLocalDeleteSnapshots?: (
|
|
1014
|
-
relativePath: string,
|
|
1015
|
-
kind: "unlink" | "unlinkDir",
|
|
1016
|
-
) => LocalDeleteSnapshot[];
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
/**
|
|
1020
|
-
* Chokidar-backed file watcher that emits a single debounced `changed` signal
|
|
1021
|
-
* after a {@link debounceMs} quiet window, coalescing bursts. It honors the
|
|
1022
|
-
* full exclusion stack via {@link createWatchPathFilter}, so excluded paths
|
|
1023
|
-
* (`.env`, `output/`, `.git/`, `companies/` in personalMode, …) never emit.
|
|
1024
|
-
*
|
|
1025
|
-
* Lifecycle: {@link start} (idempotent), {@link stop}, {@link dispose}. Stop
|
|
1026
|
-
* closes the chokidar watcher (releasing fds) and cancels any pending debounce
|
|
1027
|
-
* timer. dispose() is stop() + permanent shutdown.
|
|
1028
|
-
*/
|
|
1029
|
-
/**
|
|
1030
|
-
* One settled change-burst, handed to {@link TreeWatcher} listeners. Carries
|
|
1031
|
-
* the set of relative paths that changed during the quiet window so a listener
|
|
1032
|
-
* (e.g. {@link PushEventEmitter}) can build one PushEvent per path. `paths` is
|
|
1033
|
-
* absolute-path → relative-path; both are needed (relative for the wire shape,
|
|
1034
|
-
* absolute for hashing/statting the file on disk).
|
|
1035
|
-
*/
|
|
1036
|
-
export interface TreeChangeBatch {
|
|
1037
|
-
/** Map of absolutePath → relativePath for every path in the settled burst. */
|
|
1038
|
-
paths: Map<string, string>;
|
|
1039
|
-
/** Per-path watcher event and the captured pre-delete journal revision. */
|
|
1040
|
-
changes?: Map<string, TreeChange>;
|
|
1041
|
-
/** True when path detail was dropped after the watcher backlog cap was hit. */
|
|
1042
|
-
overflowed?: boolean;
|
|
1043
|
-
/** Count of paths dropped from the detailed batch after overflow. */
|
|
1044
|
-
droppedPaths?: number;
|
|
1045
|
-
/** Approximate path-string bytes dropped from the detailed batch. */
|
|
1046
|
-
droppedBytes?: number;
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
/**
|
|
1050
|
-
* Listener invoked once per settled debounce window.
|
|
1051
|
-
*
|
|
1052
|
-
* Backwards compatible with the US-003 `WatcherSurface` contract: the first
|
|
1053
|
-
* argument is the OPTIONAL changed relative path the loop routes its targeted
|
|
1054
|
-
* push to (the first path of the burst; undefined when the window settled with
|
|
1055
|
-
* no captured path). US-008's {@link PushEventEmitter} consumes the SECOND
|
|
1056
|
-
* argument — the full {@link TreeChangeBatch} of every path in the burst — to
|
|
1057
|
-
* build one PushEvent per path. Listeners are free to ignore either argument.
|
|
1058
|
-
*/
|
|
1059
|
-
export type TreeChangeListener = (
|
|
1060
|
-
changedRelPath?: string,
|
|
1061
|
-
batch?: TreeChangeBatch,
|
|
1062
|
-
) => void;
|
|
1063
|
-
|
|
1064
|
-
export class TreeWatcher {
|
|
1065
|
-
private readonly hqRoot: string;
|
|
1066
|
-
private readonly debounceMs: number;
|
|
1067
|
-
private readonly clock: Clock;
|
|
1068
|
-
private readonly shouldEmit: WatchPathFilter;
|
|
1069
|
-
private readonly maxPendingPaths: number;
|
|
1070
|
-
private readonly maxPendingBytes: number;
|
|
1071
|
-
private readonly maxWatchedPaths: number;
|
|
1072
|
-
private readonly onBacklogOverflow: (info: TreeWatcherBacklogOverflow) => void;
|
|
1073
|
-
private readonly onDegraded: (info: TreeWatcherDegradation) => void;
|
|
1074
|
-
private readonly telemetryClient: CloudTelemetryClient | null | undefined;
|
|
1075
|
-
private readonly telemetryClaims: TelemetryClaims | null | undefined;
|
|
1076
|
-
private readonly backendFactory?: TreeWatchBackendFactory;
|
|
1077
|
-
private readonly captureLocalDeleteSnapshots?: TreeWatcherOptions["captureLocalDeleteSnapshots"];
|
|
1078
|
-
private backend: WatchBackend | null = null;
|
|
1079
|
-
private timer: unknown = null;
|
|
1080
|
-
private listeners = new Set<TreeChangeListener>();
|
|
1081
|
-
/** Paths accumulated for the current (in-flight) debounce window. */
|
|
1082
|
-
private pending = new Map<string, string>();
|
|
1083
|
-
private pendingChanges = new Map<string, TreeChange>();
|
|
1084
|
-
private knownKinds = new Map<string, "file" | "directory">();
|
|
1085
|
-
private pendingBytes = 0;
|
|
1086
|
-
private overflowed = false;
|
|
1087
|
-
private overflowLogged = false;
|
|
1088
|
-
private droppedPaths = 0;
|
|
1089
|
-
private droppedBytes = 0;
|
|
1090
|
-
private degraded = false;
|
|
1091
|
-
private disposed = false;
|
|
1092
|
-
|
|
1093
|
-
constructor(opts: TreeWatcherOptions) {
|
|
1094
|
-
this.hqRoot = opts.hqRoot;
|
|
1095
|
-
this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
|
|
1096
|
-
this.clock = opts.clock ?? systemClock;
|
|
1097
|
-
this.shouldEmit =
|
|
1098
|
-
opts.pathFilter ?? createWatchPathFilter(opts.hqRoot, opts.personalMode ?? false);
|
|
1099
|
-
this.maxPendingPaths = Math.max(
|
|
1100
|
-
1,
|
|
1101
|
-
opts.maxPendingPaths ?? DEFAULT_TREE_WATCHER_MAX_PENDING_PATHS,
|
|
1102
|
-
);
|
|
1103
|
-
this.maxPendingBytes = Math.max(
|
|
1104
|
-
1,
|
|
1105
|
-
opts.maxPendingBytes ?? DEFAULT_TREE_WATCHER_MAX_PENDING_BYTES,
|
|
1106
|
-
);
|
|
1107
|
-
this.maxWatchedPaths = resolveMaxWatchedPaths(opts.maxWatchedPaths);
|
|
1108
|
-
this.onBacklogOverflow =
|
|
1109
|
-
opts.onBacklogOverflow ??
|
|
1110
|
-
((info) => {
|
|
1111
|
-
console.warn(
|
|
1112
|
-
`TreeWatcher backlog cap exceeded; dropping ${info.droppedPaths} path(s) until the next resync`,
|
|
1113
|
-
);
|
|
1114
|
-
});
|
|
1115
|
-
this.onDegraded =
|
|
1116
|
-
opts.onDegraded ??
|
|
1117
|
-
((info) => {
|
|
1118
|
-
console.error(
|
|
1119
|
-
JSON.stringify({ event: "watcher.degraded", ...info }),
|
|
1120
|
-
);
|
|
1121
|
-
});
|
|
1122
|
-
this.telemetryClient = opts.telemetryClient;
|
|
1123
|
-
this.telemetryClaims = opts.telemetryClaims;
|
|
1124
|
-
this.backendFactory = opts.backendFactory;
|
|
1125
|
-
this.captureLocalDeleteSnapshots = opts.captureLocalDeleteSnapshots;
|
|
1126
|
-
}
|
|
1127
|
-
|
|
1128
|
-
/**
|
|
1129
|
-
* Register a debounced-`changed` listener. Returns an unsubscribe fn.
|
|
1130
|
-
*
|
|
1131
|
-
* Listeners receive a {@link TreeChangeBatch} of the paths that changed in
|
|
1132
|
-
* the settled window. Existing US-003 callers that only need the "something
|
|
1133
|
-
* changed" signal can ignore the argument — the contract is backwards
|
|
1134
|
-
* compatible (a zero-arg callback still type-checks).
|
|
1135
|
-
*/
|
|
1136
|
-
onChange(listener: TreeChangeListener): () => void {
|
|
1137
|
-
this.listeners.add(listener);
|
|
1138
|
-
return () => this.listeners.delete(listener);
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
/**
|
|
1142
|
-
* Begin watching. Idempotent — a second call while already running is a
|
|
1143
|
-
* no-op (no second watch backend, no leaked handles).
|
|
1144
|
-
*
|
|
1145
|
-
* Uses a SINGLE recursive `fs.watch` on macOS/Windows (1 OS handle for the
|
|
1146
|
-
* whole tree) and falls back to chokidar on Linux. See {@link startTreeWatch}
|
|
1147
|
-
* for why per-path watching is avoided (kqueue fd exhaustion → EMFILE).
|
|
1148
|
-
*/
|
|
1149
|
-
start(): void {
|
|
1150
|
-
if (this.disposed || this.degraded || this.backend) return;
|
|
1151
|
-
const backendOpts: TreeWatchBackendOptions = {
|
|
1152
|
-
hqRoot: this.hqRoot,
|
|
1153
|
-
shouldEmit: this.shouldEmit,
|
|
1154
|
-
onEvent: (absolutePath, kind) => this.handleEvent(absolutePath, kind),
|
|
1155
|
-
onError: (err) => this.handleBackendError(err),
|
|
1156
|
-
maxWatchedPaths: this.maxWatchedPaths,
|
|
1157
|
-
onWatchBudgetExceeded: (info) => this.degrade({
|
|
1158
|
-
reason: "watch_budget_exhausted",
|
|
1159
|
-
maxWatchedPaths: info.maxWatchedPaths,
|
|
1160
|
-
watchedPaths: info.watchedPaths,
|
|
1161
|
-
offendingPath: info.offendingPath,
|
|
1162
|
-
offendingTopLevel: this.topLevelForTelemetry(info.offendingPath),
|
|
1163
|
-
}),
|
|
1164
|
-
onDirectorySeen: (absolutePath) =>
|
|
1165
|
-
this.knownKinds.set(path.resolve(absolutePath), "directory"),
|
|
1166
|
-
};
|
|
1167
|
-
const backend = this.backendFactory
|
|
1168
|
-
? this.backendFactory(backendOpts)
|
|
1169
|
-
: startTreeWatch(
|
|
1170
|
-
backendOpts.hqRoot,
|
|
1171
|
-
backendOpts.shouldEmit,
|
|
1172
|
-
backendOpts.onEvent,
|
|
1173
|
-
backendOpts.onError,
|
|
1174
|
-
backendOpts.maxWatchedPaths,
|
|
1175
|
-
backendOpts.onWatchBudgetExceeded,
|
|
1176
|
-
backendOpts.onDirectorySeen,
|
|
1177
|
-
);
|
|
1178
|
-
// A test backend (or a future eager backend) can discover exhaustion while
|
|
1179
|
-
// it is being constructed. Never retain that just-created handle after the
|
|
1180
|
-
// degradation path has deliberately disabled event watching.
|
|
1181
|
-
if (this.degraded) {
|
|
1182
|
-
backend.close();
|
|
1183
|
-
return;
|
|
1184
|
-
}
|
|
1185
|
-
this.backend = backend;
|
|
1186
|
-
// The scoped backend already indexed every in-scope directory while
|
|
1187
|
-
// planning its watch roots — walking the tree again here would double a
|
|
1188
|
-
// multi-second startup cost for an identical result.
|
|
1189
|
-
if (backend.needsKnownKinds && !backend.seededKnownKinds) {
|
|
1190
|
-
this.seedKnownKinds();
|
|
1191
|
-
}
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
|
-
private handleBackendError(err: unknown): void {
|
|
1195
|
-
if (isInotifyLimitError(err)) {
|
|
1196
|
-
const offendingPath = errorPath(err);
|
|
1197
|
-
this.degrade({
|
|
1198
|
-
reason: "inotify_enospc",
|
|
1199
|
-
maxWatchedPaths: this.maxWatchedPaths,
|
|
1200
|
-
watchedPaths: this.backend?.watchedPathCount() ?? 0,
|
|
1201
|
-
offendingPath,
|
|
1202
|
-
offendingTopLevel: offendingPath
|
|
1203
|
-
? this.topLevelForTelemetry(offendingPath)
|
|
1204
|
-
: undefined,
|
|
1205
|
-
errorCode: "ENOSPC",
|
|
1206
|
-
});
|
|
1207
|
-
return;
|
|
1208
|
-
}
|
|
1209
|
-
console.error("TreeWatcher error:", err);
|
|
1210
|
-
this.signalBackendResync();
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
private topLevelForTelemetry(absolutePath: string): string | undefined {
|
|
1214
|
-
const rel = path.relative(this.hqRoot, absolutePath).split(path.sep);
|
|
1215
|
-
const topLevel = rel[0];
|
|
1216
|
-
return topLevel && topLevel !== ".." ? topLevel : undefined;
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
private degrade(info: TreeWatcherDegradation): void {
|
|
1220
|
-
if (this.disposed || this.degraded) return;
|
|
1221
|
-
this.degraded = true;
|
|
1222
|
-
// Closing synchronously prevents further directory admissions while
|
|
1223
|
-
// chokidar's initial walk is still in flight.
|
|
1224
|
-
this.backend?.close();
|
|
1225
|
-
this.backend = null;
|
|
1226
|
-
try {
|
|
1227
|
-
this.onDegraded(info);
|
|
1228
|
-
} catch (err) {
|
|
1229
|
-
console.error("TreeWatcher degradation reporter error:", err);
|
|
1230
|
-
}
|
|
1231
|
-
void emitCloudTelemetry(
|
|
1232
|
-
this.telemetryClient,
|
|
1233
|
-
{
|
|
1234
|
-
eventName: "watcher_degraded",
|
|
1235
|
-
source: "watcher",
|
|
1236
|
-
properties: {
|
|
1237
|
-
reason: info.reason,
|
|
1238
|
-
maxWatchedPaths: info.maxWatchedPaths,
|
|
1239
|
-
watchedPaths: info.watchedPaths,
|
|
1240
|
-
...(info.offendingTopLevel
|
|
1241
|
-
? { offendingTopLevel: info.offendingTopLevel }
|
|
1242
|
-
: {}),
|
|
1243
|
-
},
|
|
1244
|
-
},
|
|
1245
|
-
{ claims: this.telemetryClaims },
|
|
1246
|
-
);
|
|
1247
|
-
// The existing overflow/resync batch causes the runner to use its scoped
|
|
1248
|
-
// drain while the cadence poll continues covering everything else.
|
|
1249
|
-
this.signalBackendResync();
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
private signalBackendResync(): void {
|
|
1253
|
-
if (this.disposed) return;
|
|
1254
|
-
this.overflowed = true;
|
|
1255
|
-
this.arm();
|
|
1256
|
-
}
|
|
1257
|
-
|
|
1258
|
-
/**
|
|
1259
|
-
* Test/seam entry point: feed a raw filesystem path as if the backend
|
|
1260
|
-
* reported it. Applies the emit filter then arms the debounce. Real watch
|
|
1261
|
-
* events route through here too — and for the recursive backend this is the
|
|
1262
|
-
* ONLY place filtering happens, so out-of-scope paths are dropped here.
|
|
1263
|
-
*/
|
|
1264
|
-
handleEvent(
|
|
1265
|
-
absolutePath: string,
|
|
1266
|
-
backendKind: BackendChangeKind = "change",
|
|
1267
|
-
): void {
|
|
1268
|
-
if (this.disposed) return;
|
|
1269
|
-
const abs = path.resolve(absolutePath);
|
|
1270
|
-
// Cheap pre-gate. When a path is out of scope BOTH as a file and as a
|
|
1271
|
-
// directory, no stat can change the answer — so drop it before paying the
|
|
1272
|
-
// syscall below. Scoped watch roots (see planWatchRoots) keep most excluded
|
|
1273
|
-
// traffic from being delivered at all; this covers the residue a
|
|
1274
|
-
// depth-bounded plan still admits, at the cost of two string matches
|
|
1275
|
-
// instead of one synchronous lstat.
|
|
1276
|
-
if (!this.shouldEmit(abs, false) && !this.shouldEmit(abs, true)) return;
|
|
1277
|
-
let kind: TreeChangeKind;
|
|
1278
|
-
if (backendKind === "rename") {
|
|
1279
|
-
try {
|
|
1280
|
-
kind = fs.lstatSync(abs).isDirectory() ? "addDir" : "add";
|
|
1281
|
-
} catch (err: unknown) {
|
|
1282
|
-
const code =
|
|
1283
|
-
err && typeof err === "object" && "code" in err
|
|
1284
|
-
? (err as { code?: string }).code
|
|
1285
|
-
: undefined;
|
|
1286
|
-
if (code !== "ENOENT") return;
|
|
1287
|
-
kind = this.knownKinds.get(abs) === "directory" ? "unlinkDir" : "unlink";
|
|
1288
|
-
}
|
|
1289
|
-
} else {
|
|
1290
|
-
kind = backendKind;
|
|
1291
|
-
}
|
|
1292
|
-
const isDir = kind === "addDir" || kind === "unlinkDir";
|
|
1293
|
-
// The recursive backend does no descent-time filtering, so this is the
|
|
1294
|
-
// authoritative emit gate; the chokidar backend's `ignored` pre-filters
|
|
1295
|
-
// but a slipped-through (or synthetic test) path is re-checked here too.
|
|
1296
|
-
if (!this.shouldEmit(abs, isDir)) return;
|
|
1297
|
-
// Codec boundary: the batch's relative path is a WIRE identifier — it
|
|
1298
|
-
// becomes `PushEvent.relativePath` and the targeted-push route key — so
|
|
1299
|
-
// a win32 local farm name (`indigo%3Ax`) must decode back to its
|
|
1300
|
-
// canonical vault-key spelling (`indigo:x`) here. Emitting the encoded
|
|
1301
|
-
// local name verbatim would mint a fresh junk key family on the wire
|
|
1302
|
-
// (the mixed-version amplifier). The emit filter above still sees the
|
|
1303
|
-
// raw local path: its exclusion patterns live in local-name space and
|
|
1304
|
-
// none of them contain codec-reserved characters.
|
|
1305
|
-
const rel = vaultKeyForLocalPath(this.hqRoot, abs);
|
|
1306
|
-
if (!this.pending.has(abs)) {
|
|
1307
|
-
const entryBytes = estimatePendingEntryBytes(abs, rel);
|
|
1308
|
-
if (
|
|
1309
|
-
this.pending.size >= this.maxPendingPaths ||
|
|
1310
|
-
this.pendingBytes + entryBytes > this.maxPendingBytes
|
|
1311
|
-
) {
|
|
1312
|
-
this.recordBacklogOverflow(entryBytes);
|
|
1313
|
-
this.arm();
|
|
1314
|
-
return;
|
|
1315
|
-
}
|
|
1316
|
-
this.pendingBytes += entryBytes;
|
|
1317
|
-
}
|
|
1318
|
-
this.pending.set(abs, rel);
|
|
1319
|
-
if (kind === "unlink" || kind === "unlinkDir") {
|
|
1320
|
-
// A SECOND delete event for the same path in one window must never
|
|
1321
|
-
// downgrade a directory delete to a file delete. The first `unlinkDir`
|
|
1322
|
-
// clears this path's known-kind hint, so a duplicate event — the OS can
|
|
1323
|
-
// report one path through two overlapping watches — would re-classify it
|
|
1324
|
-
// as `unlink` and, with it, narrow the deletion scope the vault applies
|
|
1325
|
-
// and drop the captured descendant snapshots.
|
|
1326
|
-
if (kind === "unlink" && this.pendingChanges.get(abs)?.kind === "unlinkDir") {
|
|
1327
|
-
this.pending.set(abs, rel);
|
|
1328
|
-
this.arm();
|
|
1329
|
-
return;
|
|
1330
|
-
}
|
|
1331
|
-
const deleteSnapshots =
|
|
1332
|
-
this.captureLocalDeleteSnapshots?.(rel, kind) ?? [];
|
|
1333
|
-
this.pendingChanges.set(abs, {
|
|
1334
|
-
kind,
|
|
1335
|
-
...(deleteSnapshots.length > 0 ? { deleteSnapshots } : {}),
|
|
1336
|
-
});
|
|
1337
|
-
this.knownKinds.delete(abs);
|
|
1338
|
-
} else {
|
|
1339
|
-
this.pendingChanges.set(abs, { kind });
|
|
1340
|
-
// Only directories are needed to disambiguate a native recursive
|
|
1341
|
-
// rename-after-delete. Unknown paths already correctly default to an
|
|
1342
|
-
// unlink file, so retaining every file ever touched made this map grow
|
|
1343
|
-
// without bound in a long-lived runner.
|
|
1344
|
-
if (isDir) this.knownKinds.set(abs, "directory");
|
|
1345
|
-
else this.knownKinds.delete(abs);
|
|
1346
|
-
}
|
|
1347
|
-
this.arm();
|
|
1348
|
-
}
|
|
1349
|
-
|
|
1350
|
-
private seedKnownKinds(): void {
|
|
1351
|
-
const visit = (dir: string): void => {
|
|
1352
|
-
let entries: fs.Dirent[];
|
|
1353
|
-
try {
|
|
1354
|
-
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
1355
|
-
} catch {
|
|
1356
|
-
return;
|
|
1357
|
-
}
|
|
1358
|
-
for (const entry of entries) {
|
|
1359
|
-
const absolutePath = path.join(dir, entry.name);
|
|
1360
|
-
if (entry.isDirectory()) {
|
|
1361
|
-
if (!this.shouldEmit(absolutePath, true)) continue;
|
|
1362
|
-
this.knownKinds.set(path.resolve(absolutePath), "directory");
|
|
1363
|
-
visit(absolutePath);
|
|
1364
|
-
}
|
|
1365
|
-
}
|
|
1366
|
-
};
|
|
1367
|
-
visit(this.hqRoot);
|
|
1368
|
-
}
|
|
1369
|
-
|
|
1370
|
-
private recordBacklogOverflow(entryBytes: number): void {
|
|
1371
|
-
this.overflowed = true;
|
|
1372
|
-
this.droppedPaths += 1;
|
|
1373
|
-
this.droppedBytes += entryBytes;
|
|
1374
|
-
if (this.overflowLogged) return;
|
|
1375
|
-
this.overflowLogged = true;
|
|
1376
|
-
try {
|
|
1377
|
-
this.onBacklogOverflow({
|
|
1378
|
-
pendingPaths: this.pending.size,
|
|
1379
|
-
pendingBytes: this.pendingBytes,
|
|
1380
|
-
maxPendingPaths: this.maxPendingPaths,
|
|
1381
|
-
maxPendingBytes: this.maxPendingBytes,
|
|
1382
|
-
droppedPaths: this.droppedPaths,
|
|
1383
|
-
droppedBytes: this.droppedBytes,
|
|
1384
|
-
});
|
|
1385
|
-
} catch (err) {
|
|
1386
|
-
console.error("TreeWatcher backlog overflow logger error:", err);
|
|
1387
|
-
}
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
private arm(): void {
|
|
1391
|
-
if (this.timer !== null) {
|
|
1392
|
-
this.clock.clearTimeout(this.timer);
|
|
1393
|
-
this.timer = null;
|
|
1394
|
-
}
|
|
1395
|
-
this.timer = this.clock.setTimeout(() => {
|
|
1396
|
-
this.timer = null;
|
|
1397
|
-
this.emit();
|
|
1398
|
-
}, this.debounceMs);
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
private emit(): void {
|
|
1402
|
-
if (this.disposed) return;
|
|
1403
|
-
// Snapshot + clear the accumulated paths so the next window starts fresh
|
|
1404
|
-
// even if a listener re-enters synchronously.
|
|
1405
|
-
const batch: TreeChangeBatch = {
|
|
1406
|
-
paths: new Map(this.pending),
|
|
1407
|
-
changes: new Map(this.pendingChanges),
|
|
1408
|
-
};
|
|
1409
|
-
if (this.overflowed) {
|
|
1410
|
-
batch.overflowed = true;
|
|
1411
|
-
batch.droppedPaths = this.droppedPaths;
|
|
1412
|
-
batch.droppedBytes = this.droppedBytes;
|
|
1413
|
-
}
|
|
1414
|
-
this.clearPending();
|
|
1415
|
-
// First changed relative path of the burst — the US-003 routing argument.
|
|
1416
|
-
// undefined when the window settled with no captured path (e.g. a synthetic
|
|
1417
|
-
// arm() with no handleEvent).
|
|
1418
|
-
const firstRel = [...batch.paths.values()][0];
|
|
1419
|
-
for (const l of this.listeners) {
|
|
1420
|
-
try {
|
|
1421
|
-
l(firstRel, batch);
|
|
1422
|
-
} catch (err) {
|
|
1423
|
-
console.error("TreeWatcher listener error:", err);
|
|
1424
|
-
}
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
private clearPending(): void {
|
|
1429
|
-
this.pending.clear();
|
|
1430
|
-
this.pendingChanges.clear();
|
|
1431
|
-
this.pendingBytes = 0;
|
|
1432
|
-
this.overflowed = false;
|
|
1433
|
-
this.overflowLogged = false;
|
|
1434
|
-
this.droppedPaths = 0;
|
|
1435
|
-
this.droppedBytes = 0;
|
|
1436
|
-
}
|
|
1437
|
-
|
|
1438
|
-
/** True while the watch backend is active. */
|
|
1439
|
-
isWatching(): boolean {
|
|
1440
|
-
return this.backend !== null;
|
|
1441
|
-
}
|
|
1442
|
-
|
|
1443
|
-
/** Number of pending debounce timers — a leak check for tests. */
|
|
1444
|
-
pendingTimerCount(): number {
|
|
1445
|
-
return this.timer === null ? 0 : 1;
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
/** Number of native-rename directory hints retained for deletion handling. */
|
|
1449
|
-
knownDirectoryCount(): number {
|
|
1450
|
-
return this.knownKinds.size;
|
|
1451
|
-
}
|
|
1452
|
-
|
|
1453
|
-
/**
|
|
1454
|
-
* Stop watching: close the watch backend (releasing its OS handle) and
|
|
1455
|
-
* cancel any pending debounce timer. Idempotent. The instance can be
|
|
1456
|
-
* restarted with {@link start} unless {@link dispose} was called.
|
|
1457
|
-
*/
|
|
1458
|
-
stop(): void {
|
|
1459
|
-
if (this.timer !== null) {
|
|
1460
|
-
this.clock.clearTimeout(this.timer);
|
|
1461
|
-
this.timer = null;
|
|
1462
|
-
}
|
|
1463
|
-
this.clearPending();
|
|
1464
|
-
if (this.backend) {
|
|
1465
|
-
this.backend.close();
|
|
1466
|
-
this.backend = null;
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
|
|
1470
|
-
/** Permanent shutdown: stop() + drop listeners; further events are no-ops. */
|
|
1471
|
-
dispose(): void {
|
|
1472
|
-
this.stop();
|
|
1473
|
-
this.disposed = true;
|
|
1474
|
-
this.listeners.clear();
|
|
1475
|
-
}
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
|
-
// ---------------------------------------------------------------------------
|
|
1479
|
-
// US-008 — PushEventEmitter: bridge the TreeWatcher to the Phase 2 transport.
|
|
1480
|
-
//
|
|
1481
|
-
// On each settled debounce window, build one PushEvent per changed path
|
|
1482
|
-
// (sha-256 contentHash, monotonic per-device sequenceNumber, originTenantId)
|
|
1483
|
-
// and hand it to the injected PushTransport. Feature-flag gated: when the
|
|
1484
|
-
// originTenantId is NOT enabled the emitter is DORMANT — it subscribes to
|
|
1485
|
-
// nothing and ships nothing, so the Phase 1 poll-only behavior is unchanged.
|
|
1486
|
-
//
|
|
1487
|
-
// Failure posture: a transport publish that throws (network/non-2xx/timeout)
|
|
1488
|
-
// MUST NOT crash the daemon. The emitter catches per-event publish errors and
|
|
1489
|
-
// routes them to `onError` (default: console.error); the existing cadence
|
|
1490
|
-
// poll remains the safety net that eventually ships the change.
|
|
1491
|
-
// ---------------------------------------------------------------------------
|
|
1492
|
-
|
|
1493
|
-
/** Computes the canonical contentHash for a file: `sha256:<64-hex>`. */
|
|
1494
|
-
async function computeContentHash(absolutePath: string): Promise<string> {
|
|
1495
|
-
const bytes = await readFile(absolutePath);
|
|
1496
|
-
const hex = createHash("sha256").update(bytes).digest("hex");
|
|
1497
|
-
return `sha256:${hex}`;
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
export interface PushEventEmitterOptions {
|
|
1501
|
-
/** Tenant identifier stamped onto every PushEvent + checked against the flag. */
|
|
1502
|
-
originTenantId: string;
|
|
1503
|
-
/** Device identifier stamped onto every PushEvent. */
|
|
1504
|
-
originDeviceId: string;
|
|
1505
|
-
/** Transport that ships each PushEvent (US-007 NoopPushTransport / HttpPushTransport). */
|
|
1506
|
-
transport: PushTransport;
|
|
1507
|
-
/**
|
|
1508
|
-
* Feature-flag seam. When `isEnabled(originTenantId)` is false the emitter is
|
|
1509
|
-
* dormant: {@link attach} subscribes nothing and {@link emitForBatch} is a
|
|
1510
|
-
* no-op. Defaults are NOT supplied here — the caller injects an
|
|
1511
|
-
* EventDrivenPushFlagProvider so dormancy is explicit.
|
|
1512
|
-
*/
|
|
1513
|
-
flagProvider: EventDrivenPushFlagProvider;
|
|
1514
|
-
/**
|
|
1515
|
-
* Returns the next monotonic sequence number for this device. Default: an
|
|
1516
|
-
* internal counter starting at 0. Inject to persist across daemon restarts.
|
|
1517
|
-
*/
|
|
1518
|
-
getSequenceNumber?: () => number;
|
|
1519
|
-
/** Clock for eventTimestamp. Default `() => new Date()`. */
|
|
1520
|
-
now?: () => Date;
|
|
1521
|
-
/**
|
|
1522
|
-
* Where publish failures + hash/stat errors go. Default `console.error`.
|
|
1523
|
-
* Receives the offending PushEvent (when known) so callers can correlate.
|
|
1524
|
-
*/
|
|
1525
|
-
onError?: (err: Error, ctx: { relativePath?: string }) => void;
|
|
1526
|
-
/**
|
|
1527
|
-
* Optional structured logger for the US-011 3-log diagnostic chain. When
|
|
1528
|
-
* supplied, the emitter logs `event=watcher.emit` (the 1st correlated link)
|
|
1529
|
-
* carrying the PushEvent's `sequenceNumber` — the same join key stamped by
|
|
1530
|
-
* the server `push.receive` log and the client `fanout.receive` log. Default:
|
|
1531
|
-
* no log (the daemon stays quiet unless wired with a logger).
|
|
1532
|
-
*/
|
|
1533
|
-
logger?: EmitterLogger;
|
|
1534
|
-
/** Optional hq-pro ACTION telemetry sink. Best-effort; never affects publish. */
|
|
1535
|
-
telemetryClient?: CloudTelemetryClient | null;
|
|
1536
|
-
telemetryClaims?: TelemetryClaims | null;
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
/**
|
|
1540
|
-
* Minimal structured logger surface for {@link PushEventEmitter}. A pino
|
|
1541
|
-
* `Logger` (from `./sync/logger.ts`) satisfies this; tests inject a fake.
|
|
1542
|
-
*/
|
|
1543
|
-
export interface EmitterLogger {
|
|
1544
|
-
info(obj: Record<string, unknown>, msg?: string): void;
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
/**
|
|
1548
|
-
* Bridges {@link TreeWatcher} change batches to a {@link PushTransport} as
|
|
1549
|
-
* typed PushEvents. Construct once per daemon, then {@link attach} to a
|
|
1550
|
-
* running TreeWatcher (returns an unsubscribe fn). Flag-gated + failure-safe.
|
|
1551
|
-
*/
|
|
1552
|
-
export class PushEventEmitter {
|
|
1553
|
-
private static readonly MAX_CONCURRENT_PUBLISHES = 16;
|
|
1554
|
-
private readonly originTenantId: string;
|
|
1555
|
-
private readonly originDeviceId: string;
|
|
1556
|
-
private readonly transport: PushTransport;
|
|
1557
|
-
private readonly flagProvider: EventDrivenPushFlagProvider;
|
|
1558
|
-
private readonly now: () => Date;
|
|
1559
|
-
private readonly onError: (err: Error, ctx: { relativePath?: string }) => void;
|
|
1560
|
-
private readonly logger: EmitterLogger | undefined;
|
|
1561
|
-
private readonly telemetryClient: CloudTelemetryClient | null | undefined;
|
|
1562
|
-
private readonly telemetryClaims: TelemetryClaims | null | undefined;
|
|
1563
|
-
private internalSeq = 0;
|
|
1564
|
-
private readonly nextSeq: () => number;
|
|
1565
|
-
private publishTail: Promise<void> = Promise.resolve();
|
|
1566
|
-
|
|
1567
|
-
constructor(opts: PushEventEmitterOptions) {
|
|
1568
|
-
this.originTenantId = opts.originTenantId;
|
|
1569
|
-
this.originDeviceId = opts.originDeviceId;
|
|
1570
|
-
this.transport = opts.transport;
|
|
1571
|
-
this.flagProvider = opts.flagProvider;
|
|
1572
|
-
this.now = opts.now ?? (() => new Date());
|
|
1573
|
-
this.logger = opts.logger;
|
|
1574
|
-
this.telemetryClient = opts.telemetryClient;
|
|
1575
|
-
this.telemetryClaims = opts.telemetryClaims;
|
|
1576
|
-
this.onError =
|
|
1577
|
-
opts.onError ??
|
|
1578
|
-
((err, ctx) =>
|
|
1579
|
-
console.error("PushEventEmitter error:", ctx.relativePath ?? "", err));
|
|
1580
|
-
this.nextSeq =
|
|
1581
|
-
opts.getSequenceNumber ??
|
|
1582
|
-
(() => {
|
|
1583
|
-
const n = this.internalSeq;
|
|
1584
|
-
this.internalSeq += 1;
|
|
1585
|
-
return n;
|
|
1586
|
-
});
|
|
1587
|
-
}
|
|
1588
|
-
|
|
1589
|
-
/** True iff event-driven push is enabled for this emitter's tenant. */
|
|
1590
|
-
get enabled(): boolean {
|
|
1591
|
-
return this.flagProvider.isEnabled(this.originTenantId);
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
/**
|
|
1595
|
-
* Subscribe to a TreeWatcher's change batches. Dormant (no subscription)
|
|
1596
|
-
* when the flag is OFF for this tenant. Returns an unsubscribe fn (a no-op
|
|
1597
|
-
* when dormant).
|
|
1598
|
-
*/
|
|
1599
|
-
attach(watcher: TreeWatcher): () => void {
|
|
1600
|
-
if (!this.enabled) return () => {};
|
|
1601
|
-
return watcher.onChange((_changedRelPath, batch) => {
|
|
1602
|
-
if (batch) void this.emitForBatch(batch);
|
|
1603
|
-
});
|
|
1604
|
-
}
|
|
1605
|
-
|
|
1606
|
-
/**
|
|
1607
|
-
* Build + ship one PushEvent per changed path in the batch. No-op when the
|
|
1608
|
-
* flag is OFF. Each path is independent: a hash/stat failure or a transport
|
|
1609
|
-
* publish rejection for one path is caught + surfaced via `onError` and does
|
|
1610
|
-
* NOT abort the others or propagate (the daemon must not crash; the cadence
|
|
1611
|
-
* poll covers any miss).
|
|
1612
|
-
*/
|
|
1613
|
-
async emitForBatch(batch: TreeChangeBatch): Promise<void> {
|
|
1614
|
-
if (!this.enabled) return;
|
|
1615
|
-
const entriesByRel = new Map<string, [string, string]>();
|
|
1616
|
-
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
1617
|
-
entriesByRel.set(relativePath, [absolutePath, relativePath]);
|
|
1618
|
-
}
|
|
1619
|
-
const entries = [...entriesByRel.values()];
|
|
1620
|
-
const run = this.publishTail.then(
|
|
1621
|
-
() => this.emitEntries(entries),
|
|
1622
|
-
() => this.emitEntries(entries),
|
|
1623
|
-
);
|
|
1624
|
-
this.publishTail = run.catch(() => undefined);
|
|
1625
|
-
await run;
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
|
-
private async emitEntries(entries: Array<[string, string]>): Promise<void> {
|
|
1629
|
-
for (
|
|
1630
|
-
let i = 0;
|
|
1631
|
-
i < entries.length;
|
|
1632
|
-
i += PushEventEmitter.MAX_CONCURRENT_PUBLISHES
|
|
1633
|
-
) {
|
|
1634
|
-
const chunk = entries.slice(
|
|
1635
|
-
i,
|
|
1636
|
-
i + PushEventEmitter.MAX_CONCURRENT_PUBLISHES,
|
|
1637
|
-
);
|
|
1638
|
-
await Promise.all(
|
|
1639
|
-
chunk.map(([absolutePath, relativePath]) =>
|
|
1640
|
-
this.emitOne(absolutePath, relativePath),
|
|
1641
|
-
),
|
|
1642
|
-
);
|
|
1643
|
-
}
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
private async emitOne(
|
|
1647
|
-
absolutePath: string,
|
|
1648
|
-
relativePath: string,
|
|
1649
|
-
): Promise<void> {
|
|
1650
|
-
let event: PushEvent | undefined;
|
|
1651
|
-
try {
|
|
1652
|
-
const st = await stat(absolutePath);
|
|
1653
|
-
if (st.isDirectory()) return;
|
|
1654
|
-
|
|
1655
|
-
const contentHash = await computeContentHash(absolutePath);
|
|
1656
|
-
event = {
|
|
1657
|
-
kind: "upsert",
|
|
1658
|
-
relativePath,
|
|
1659
|
-
contentHash,
|
|
1660
|
-
mtime: st.mtime.toISOString(),
|
|
1661
|
-
originDeviceId: this.originDeviceId,
|
|
1662
|
-
originTenantId: this.originTenantId,
|
|
1663
|
-
sequenceNumber: this.nextSeq(),
|
|
1664
|
-
eventTimestamp: this.now().toISOString(),
|
|
1665
|
-
};
|
|
1666
|
-
} catch (err) {
|
|
1667
|
-
const code =
|
|
1668
|
-
err && typeof err === "object" && "code" in err
|
|
1669
|
-
? (err as { code?: string }).code
|
|
1670
|
-
: undefined;
|
|
1671
|
-
if (code !== "ENOENT") {
|
|
1672
|
-
// Genuine stat/read failure. Surface, don't crash.
|
|
1673
|
-
this.onError(err instanceof Error ? err : new Error(String(err)), {
|
|
1674
|
-
relativePath,
|
|
1675
|
-
});
|
|
1676
|
-
this.emitPublishFailureTelemetry("unknown", "capture");
|
|
1677
|
-
return;
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
// Deleted before/during capture: publish a tombstone so peers run a
|
|
1681
|
-
// targeted pull and let the vault-confirmed tombstone path remove it.
|
|
1682
|
-
event = {
|
|
1683
|
-
kind: "delete",
|
|
1684
|
-
relativePath,
|
|
1685
|
-
originDeviceId: this.originDeviceId,
|
|
1686
|
-
originTenantId: this.originTenantId,
|
|
1687
|
-
sequenceNumber: this.nextSeq(),
|
|
1688
|
-
eventTimestamp: this.now().toISOString(),
|
|
1689
|
-
};
|
|
1690
|
-
}
|
|
1691
|
-
|
|
1692
|
-
// US-011: 1st link of the 3-log diagnostic chain. Stamps the same
|
|
1693
|
-
// `sequenceNumber` the server `push.receive` log and the client
|
|
1694
|
-
// `fanout.receive` log carry, so an operator can walk one event
|
|
1695
|
-
// end-to-end. Logged after the event is built, before the publish.
|
|
1696
|
-
this.logger?.info(
|
|
1697
|
-
{
|
|
1698
|
-
event: "watcher.emit",
|
|
1699
|
-
kind: event.kind,
|
|
1700
|
-
originTenantId: event.originTenantId,
|
|
1701
|
-
originDeviceId: event.originDeviceId,
|
|
1702
|
-
relativePath: event.relativePath,
|
|
1703
|
-
contentHash: event.contentHash,
|
|
1704
|
-
sequenceNumber: event.sequenceNumber,
|
|
1705
|
-
eventTimestamp: event.eventTimestamp,
|
|
1706
|
-
},
|
|
1707
|
-
"watcher emitted push event",
|
|
1708
|
-
);
|
|
1709
|
-
|
|
1710
|
-
try {
|
|
1711
|
-
await this.transport.publish(event);
|
|
1712
|
-
} catch (err) {
|
|
1713
|
-
// Push failure (network / non-2xx / timeout). The cadence poll is the
|
|
1714
|
-
// safety net — log + continue, never throw.
|
|
1715
|
-
this.onError(err instanceof Error ? err : new Error(String(err)), {
|
|
1716
|
-
relativePath,
|
|
1717
|
-
});
|
|
1718
|
-
this.emitPublishFailureTelemetry(event.kind, "publish");
|
|
1719
|
-
}
|
|
1720
|
-
}
|
|
1721
|
-
|
|
1722
|
-
private emitPublishFailureTelemetry(
|
|
1723
|
-
kind: string,
|
|
1724
|
-
stage: "capture" | "publish",
|
|
1725
|
-
): void {
|
|
1726
|
-
void emitCloudTelemetry(
|
|
1727
|
-
this.telemetryClient,
|
|
1728
|
-
{
|
|
1729
|
-
eventName: "push_event_failed",
|
|
1730
|
-
source: "watcher",
|
|
1731
|
-
properties: {
|
|
1732
|
-
kind,
|
|
1733
|
-
count: 1,
|
|
1734
|
-
status: "failure",
|
|
1735
|
-
stage,
|
|
1736
|
-
},
|
|
1737
|
-
},
|
|
1738
|
-
{ claims: this.telemetryClaims },
|
|
1739
|
-
);
|
|
1740
|
-
}
|
|
1741
|
-
}
|