@indigoai-us/hq-cloud 6.16.30 → 6.16.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-post-sync-flags.d.ts +22 -11
- package/dist/bin/sync-runner-post-sync-flags.d.ts.map +1 -1
- package/dist/bin/sync-runner-post-sync-flags.js +31 -44
- package/dist/bin/sync-runner-post-sync-flags.js.map +1 -1
- package/dist/bin/sync-runner-post-sync-flags.test.js +37 -77
- package/dist/bin/sync-runner-post-sync-flags.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -19
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +0 -4
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.test.js +44 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/watch-roots.d.ts +11 -0
- package/dist/watch-roots.d.ts.map +1 -1
- package/dist/watch-roots.js +16 -7
- package/dist/watch-roots.js.map +1 -1
- package/dist/watch-roots.test.js +97 -0
- package/dist/watch-roots.test.js.map +1 -1
- package/dist/watcher.d.ts +43 -4
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +180 -36
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +292 -1
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
package/dist/watcher.js
CHANGED
|
@@ -346,14 +346,28 @@ function errorPath(err) {
|
|
|
346
346
|
return undefined;
|
|
347
347
|
}
|
|
348
348
|
/**
|
|
349
|
-
*
|
|
350
|
-
*
|
|
351
|
-
* (
|
|
352
|
-
*
|
|
353
|
-
*
|
|
349
|
+
* Which backend a platform gets.
|
|
350
|
+
*
|
|
351
|
+
* - macOS (FSEvents) and Windows (ReadDirectoryChangesW) watch a directory
|
|
352
|
+
* recursively with ONE OS handle, so the scoped planner places a few
|
|
353
|
+
* recursive watches over the in-scope tree.
|
|
354
|
+
* - Linux has no recursive handle: a directory watch reports only its own
|
|
355
|
+
* direct contents. The same scoped backend runs with one shallow inotify
|
|
356
|
+
* watch per in-scope directory instead — and NONE per file, because the
|
|
357
|
+
* directory watch already reports content changes to the files directly
|
|
358
|
+
* inside it. chokidar's Linux mode took a watch per file as well, which
|
|
359
|
+
* on a real HQ root meant ~950k watches for ~74k directories.
|
|
360
|
+
* - Anything else falls back to chokidar; a forced polling rebuild always
|
|
361
|
+
* goes through chokidar's polling backend.
|
|
354
362
|
*/
|
|
355
|
-
function
|
|
356
|
-
|
|
363
|
+
export function selectTreeWatchBackend(platform, forcePolling) {
|
|
364
|
+
if (forcePolling)
|
|
365
|
+
return "polling";
|
|
366
|
+
if (platform === "darwin" || platform === "win32")
|
|
367
|
+
return "native-recursive";
|
|
368
|
+
if (platform === "linux")
|
|
369
|
+
return "native-shallow";
|
|
370
|
+
return "chokidar";
|
|
357
371
|
}
|
|
358
372
|
function startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatchedPaths, onWatchBudgetExceeded, forcePolling = false) {
|
|
359
373
|
const budget = new ChokidarWatchBudget(maxWatchedPaths, onWatchBudgetExceeded);
|
|
@@ -445,13 +459,35 @@ function startChokidarTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatched
|
|
|
445
459
|
* still drain instead of being dropped with the handle.
|
|
446
460
|
*/
|
|
447
461
|
const BOOTSTRAP_DRAIN_MS = 2000;
|
|
448
|
-
|
|
462
|
+
/** ENOSPC (inotify watch limit) and EMFILE (fd limit) mean the box, not the path. */
|
|
463
|
+
function isSystemicWatchError(err) {
|
|
464
|
+
const code = typeof err === "object" && err !== null && "code" in err
|
|
465
|
+
? err.code
|
|
466
|
+
: undefined;
|
|
467
|
+
return code === "ENOSPC" || code === "EMFILE" || code === "ENFILE";
|
|
468
|
+
}
|
|
469
|
+
export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError, onDirectorySeen, opts = {}) {
|
|
449
470
|
const root = path.resolve(hqRoot);
|
|
471
|
+
const recursiveWatches = opts.recursiveWatches ?? true;
|
|
472
|
+
const maxWatchedPaths = opts.maxWatchedPaths;
|
|
450
473
|
const watchers = new Map();
|
|
451
474
|
const recursiveRoots = new Set();
|
|
452
475
|
const recursiveFlag = new Map();
|
|
476
|
+
/**
|
|
477
|
+
* Which directory each watch was opened on — `dev:ino`, captured just before
|
|
478
|
+
* the handle is taken. inotify follows the inode, not the pathname: when a
|
|
479
|
+
* watched directory is renamed away and a replacement lands at the same
|
|
480
|
+
* path, the handle keeps reporting the moved-away tree and the replacement
|
|
481
|
+
* is silent. Comparing identity on the next rename event is what catches
|
|
482
|
+
* that. (Captured BEFORE the handle so a swap racing the attach can only make
|
|
483
|
+
* us re-attach once more, never trust a stale handle.)
|
|
484
|
+
*/
|
|
485
|
+
const identity = new Map();
|
|
453
486
|
let closed = false;
|
|
454
487
|
let attachFailures = 0;
|
|
488
|
+
let budgetExhausted = false;
|
|
489
|
+
/** True once the initial plan is live; later attaches announce their files. */
|
|
490
|
+
let planned = false;
|
|
455
491
|
/** Retires the bootstrap watch — replaced once that watch exists. */
|
|
456
492
|
let closeBootstrap = () => { };
|
|
457
493
|
/** lstat that answers "is this a directory right now", never throwing. */
|
|
@@ -472,6 +508,16 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
472
508
|
return false;
|
|
473
509
|
}
|
|
474
510
|
}
|
|
511
|
+
/** `dev:ino` of the directory at `absolutePath`; undefined if absent or not a directory. */
|
|
512
|
+
function directoryIdentity(absolutePath) {
|
|
513
|
+
try {
|
|
514
|
+
const st = fs.lstatSync(absolutePath);
|
|
515
|
+
return st.isDirectory() ? `${st.dev}:${st.ino}` : undefined;
|
|
516
|
+
}
|
|
517
|
+
catch {
|
|
518
|
+
return undefined;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
475
521
|
/**
|
|
476
522
|
* Release the watch on `dir` AND on everything beneath it.
|
|
477
523
|
*
|
|
@@ -499,11 +545,28 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
499
545
|
watchers.delete(watched);
|
|
500
546
|
recursiveRoots.delete(watched);
|
|
501
547
|
recursiveFlag.delete(watched);
|
|
548
|
+
identity.delete(watched);
|
|
502
549
|
}
|
|
503
550
|
}
|
|
504
|
-
function addWatch(dir,
|
|
551
|
+
function addWatch(dir, wantRecursive) {
|
|
505
552
|
if (closed || watchers.has(dir))
|
|
506
553
|
return;
|
|
554
|
+
const recursive = wantRecursive && recursiveWatches;
|
|
555
|
+
// The cap counts live OS watches — the thing that costs kernel and heap
|
|
556
|
+
// memory — so it binds in shallow mode exactly when the tree is too big,
|
|
557
|
+
// instead of reading a directory count that is a fraction of the truth.
|
|
558
|
+
if (maxWatchedPaths !== undefined && watchers.size >= maxWatchedPaths) {
|
|
559
|
+
if (!budgetExhausted) {
|
|
560
|
+
budgetExhausted = true;
|
|
561
|
+
opts.onWatchBudgetExceeded?.({
|
|
562
|
+
maxWatchedPaths,
|
|
563
|
+
watchedPaths: watchers.size,
|
|
564
|
+
offendingPath: dir,
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
569
|
+
const openedOn = directoryIdentity(dir);
|
|
507
570
|
let handle;
|
|
508
571
|
try {
|
|
509
572
|
handle = fs.watch(dir, { recursive, persistent: true }, (eventType, filename) => {
|
|
@@ -526,7 +589,11 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
526
589
|
catch (err) {
|
|
527
590
|
// A planned watch we could not attach. Report it and record the hole so
|
|
528
591
|
// the caller can reject the whole plan instead of running half-blind.
|
|
529
|
-
|
|
592
|
+
// Shallow mode attaches tens of thousands of watches, so one directory
|
|
593
|
+
// we cannot open (permissions, a delete racing the walk) is reported and
|
|
594
|
+
// skipped; only the kernel running out of watches rejects the plan.
|
|
595
|
+
if (recursiveWatches || isSystemicWatchError(err))
|
|
596
|
+
attachFailures += 1;
|
|
530
597
|
onError(err);
|
|
531
598
|
return;
|
|
532
599
|
}
|
|
@@ -544,6 +611,8 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
544
611
|
});
|
|
545
612
|
watchers.set(dir, handle);
|
|
546
613
|
recursiveFlag.set(dir, recursive);
|
|
614
|
+
if (openedOn !== undefined)
|
|
615
|
+
identity.set(dir, openedOn);
|
|
547
616
|
if (recursive)
|
|
548
617
|
recursiveRoots.add(dir);
|
|
549
618
|
}
|
|
@@ -553,16 +622,73 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
553
622
|
for (const dir of plan.shallow)
|
|
554
623
|
addWatch(dir, false);
|
|
555
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* Plan `dir` and attach the result. In shallow mode every in-scope
|
|
627
|
+
* directory is attached the moment the walk discovers it — BEFORE the walk
|
|
628
|
+
* lists that directory — rather than after the whole plan is computed. The
|
|
629
|
+
* plan is a snapshot: a child created after its parent was listed is in
|
|
630
|
+
* neither the snapshot nor, with the parent's watch attached only later,
|
|
631
|
+
* any inotify report, so it would stay unwatched until restart. With the
|
|
632
|
+
* watch live first, the kernel reports the late child and
|
|
633
|
+
* {@link reconcileCoverage} attaches it. Recursive mode keeps the
|
|
634
|
+
* plan-then-attach order: which directories become recursive roots is only
|
|
635
|
+
* known once their subtree has been walked, and the bootstrap watch covers
|
|
636
|
+
* the walk there.
|
|
637
|
+
*/
|
|
556
638
|
function planAndAttach(dir) {
|
|
557
|
-
|
|
639
|
+
const eager = !recursiveWatches;
|
|
640
|
+
if (eager)
|
|
641
|
+
addWatch(dir, false);
|
|
642
|
+
applyPlan(planWatchRoots(dir, shouldEmit, {
|
|
643
|
+
onDirectory: eager
|
|
644
|
+
? (absolutePath) => {
|
|
645
|
+
onDirectorySeen?.(absolutePath);
|
|
646
|
+
addWatch(absolutePath, false);
|
|
647
|
+
}
|
|
648
|
+
: onDirectorySeen,
|
|
649
|
+
recursiveWatches,
|
|
650
|
+
listChildDirs: opts.listChildDirs,
|
|
651
|
+
}));
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* Announce every in-scope file already inside a directory that was just
|
|
655
|
+
* attached. A shallow parent watch reports only that the directory
|
|
656
|
+
* appeared: files moved in with it (a `rename` of a populated tree, an
|
|
657
|
+
* extract, a clone) never generate their own events, and a directory batch
|
|
658
|
+
* entry pushes nothing by itself. chokidar walked such a directory on
|
|
659
|
+
* `addDir`; this is the same walk. Recursive roots never need it — the OS
|
|
660
|
+
* reports each file for them.
|
|
661
|
+
*/
|
|
662
|
+
function announceExisting(dir) {
|
|
663
|
+
let entries;
|
|
664
|
+
try {
|
|
665
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
for (const entry of entries) {
|
|
671
|
+
const absolutePath = path.join(dir, entry.name);
|
|
672
|
+
if (entry.isDirectory()) {
|
|
673
|
+
if (shouldEmit(absolutePath, true))
|
|
674
|
+
announceExisting(absolutePath);
|
|
675
|
+
}
|
|
676
|
+
else if (shouldEmit(absolutePath, false)) {
|
|
677
|
+
onEvent(absolutePath, "rename");
|
|
678
|
+
}
|
|
679
|
+
}
|
|
558
680
|
}
|
|
559
681
|
/**
|
|
560
682
|
* Keep the live watch set honest about a path that just appeared, moved, or
|
|
561
683
|
* vanished. Three cases, all of which silently break coverage if ignored:
|
|
562
684
|
*
|
|
563
|
-
* 1. A watched directory was deleted
|
|
564
|
-
* the
|
|
565
|
-
*
|
|
685
|
+
* 1. A watched directory was deleted, or replaced by a different directory
|
|
686
|
+
* at the same path (a rename-away plus a rename-in that both land before
|
|
687
|
+
* this callback runs). Its stale handle would otherwise make the path
|
|
688
|
+
* look covered forever — and on Linux keep reporting the moved-away
|
|
689
|
+
* tree under this path — so drop it. A deletion waits for the recreate
|
|
690
|
+
* to re-plan; a replacement is re-planned right here, because no further
|
|
691
|
+
* event will announce it.
|
|
566
692
|
* 2. An EXCLUDED directory was created under a recursive root — e.g.
|
|
567
693
|
* `workspace/worktrees/` or a fresh `node_modules/` that did not exist at
|
|
568
694
|
* plan time. The root now covers a bucket we must never watch, which is
|
|
@@ -573,10 +699,17 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
573
699
|
function reconcileCoverage(absolutePath) {
|
|
574
700
|
if (closed)
|
|
575
701
|
return;
|
|
576
|
-
// (1) stale handle
|
|
577
|
-
if (watchers.has(absolutePath)
|
|
702
|
+
// (1) stale handle: the path is gone, or it is a different directory now.
|
|
703
|
+
if (watchers.has(absolutePath)) {
|
|
704
|
+
const now = directoryIdentity(absolutePath);
|
|
705
|
+
if (now === undefined) {
|
|
706
|
+
dropWatch(absolutePath);
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
if (now === identity.get(absolutePath))
|
|
710
|
+
return;
|
|
578
711
|
dropWatch(absolutePath);
|
|
579
|
-
|
|
712
|
+
// Fall through: the replacement is an uncovered in-scope directory (3).
|
|
580
713
|
}
|
|
581
714
|
const inScope = shouldEmit(absolutePath, true);
|
|
582
715
|
const covered = isCoveredByRecursiveRoot(absolutePath, recursiveRoots);
|
|
@@ -602,6 +735,8 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
602
735
|
return;
|
|
603
736
|
onDirectorySeen?.(absolutePath);
|
|
604
737
|
planAndAttach(absolutePath);
|
|
738
|
+
if (!recursiveWatches && planned)
|
|
739
|
+
announceExisting(absolutePath);
|
|
605
740
|
}
|
|
606
741
|
// Cover the whole tree BEFORE planning. The plan walk below is synchronous
|
|
607
742
|
// and takes seconds on a real HQ root; without this, every change during that
|
|
@@ -609,29 +744,35 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
609
744
|
// cadence poll. This temporary watch is exactly the old backend's shape, so
|
|
610
745
|
// the bootstrap window is no worse than the previous behavior — and it is
|
|
611
746
|
// dropped the moment the scoped plan is live.
|
|
747
|
+
// No such watch exists in shallow mode: Node's Linux "recursive" shim would
|
|
748
|
+
// walk the WHOLE tree — worktrees, node_modules and all — to emulate it.
|
|
749
|
+
// The runner's initial sync pass covers the planning window there instead.
|
|
612
750
|
let bootstrap = null;
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
751
|
+
if (recursiveWatches) {
|
|
752
|
+
try {
|
|
753
|
+
bootstrap = fs.watch(root, { recursive: true, persistent: true }, (eventType, filename) => {
|
|
754
|
+
if (filename == null)
|
|
755
|
+
return;
|
|
756
|
+
const rel = String(filename);
|
|
757
|
+
if (rel === "")
|
|
758
|
+
return;
|
|
759
|
+
onEvent(path.resolve(root, rel), eventType === "change" ? "change" : "rename");
|
|
760
|
+
});
|
|
761
|
+
bootstrap.on("error", onError);
|
|
762
|
+
}
|
|
763
|
+
catch (err) {
|
|
764
|
+
// Recursive watch is unavailable at all — the plan below cannot work
|
|
765
|
+
// either, so record the hole and let the caller fall back.
|
|
766
|
+
attachFailures += 1;
|
|
767
|
+
onError(err);
|
|
768
|
+
}
|
|
629
769
|
}
|
|
630
770
|
// Unbounded on purpose: this walk replaces TreeWatcher's separate known-kinds
|
|
631
771
|
// seed walk (same tree, same filter, ~3.6s on a real HQ root), so planning
|
|
632
772
|
// the tree exhaustively costs nothing over the status quo and leaves no
|
|
633
773
|
// excluded subtree covered by a recursive root.
|
|
634
774
|
planAndAttach(root);
|
|
775
|
+
planned = true;
|
|
635
776
|
// The scoped plan is live, so narrow from whole-tree to in-scope-only — but
|
|
636
777
|
// NOT synchronously. Events observed during the planning walk can still be
|
|
637
778
|
// sitting in the OS queue undelivered, and closing the handle here drops
|
|
@@ -707,9 +848,12 @@ export function startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError,
|
|
|
707
848
|
* {@link toChokidarIgnored} to avoid.
|
|
708
849
|
*/
|
|
709
850
|
function startTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatchedPaths, onWatchBudgetExceeded, onDirectorySeen, forcePolling = false) {
|
|
710
|
-
|
|
851
|
+
const choice = selectTreeWatchBackend(process.platform, forcePolling);
|
|
852
|
+
if (choice === "native-recursive" || choice === "native-shallow") {
|
|
711
853
|
try {
|
|
712
|
-
const scoped = startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError, onDirectorySeen
|
|
854
|
+
const scoped = startScopedRecursiveWatch(hqRoot, shouldEmit, onEvent, onError, onDirectorySeen, choice === "native-shallow"
|
|
855
|
+
? { recursiveWatches: false, maxWatchedPaths, onWatchBudgetExceeded }
|
|
856
|
+
: {});
|
|
713
857
|
// Partial coverage is worse than a different backend: a planned root we
|
|
714
858
|
// could not attach is silently unwatched forever, and the operator only
|
|
715
859
|
// sees it as "instant sync stopped working for that subtree". Accept the
|
|
@@ -720,7 +864,7 @@ function startTreeWatch(hqRoot, shouldEmit, onEvent, onError, maxWatchedPaths, o
|
|
|
720
864
|
scoped.close();
|
|
721
865
|
}
|
|
722
866
|
catch (err) {
|
|
723
|
-
//
|
|
867
|
+
// Native watch unexpectedly unavailable — fall back to chokidar
|
|
724
868
|
// rather than leaving the daemon with no watcher at all.
|
|
725
869
|
onError(err);
|
|
726
870
|
}
|