@indigoai-us/hq-cloud 6.16.52-alpha.0 → 6.16.53-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
10
10
  import * as fs from "fs";
11
11
  import * as os from "os";
12
12
  import * as path from "path";
13
+ import * as v8 from "node:v8";
14
+ import * as vm from "node:vm";
13
15
  import { monitorEventLoopDelay } from "node:perf_hooks";
14
16
  import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
15
17
  import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
@@ -20,7 +22,9 @@ import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
20
22
  import { FakeClock, TreeWatcher, } from "../watcher.js";
21
23
  import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
22
24
  import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
25
+ import { AreaLedger } from "../sync/area-ledger.js";
23
26
  import { AreaResolver } from "../sync/area-resolver.js";
27
+ import { AREA_LEGACY_MIRROR_LINGER_MS } from "../sync/area-ledger-migration.js";
24
28
  import { AreaMaintenanceCursorStore } from "../sync/per-area-maintenance.js";
25
29
  import { HQ_CLOUD_VERSION } from "../version.js";
26
30
  import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
@@ -4324,6 +4328,26 @@ async function flushLoopMicrotasks(n = 8) {
4324
4328
  for (let i = 0; i < n; i++)
4325
4329
  await Promise.resolve();
4326
4330
  }
4331
+ /** Retained journal rows live in typed-array backing stores as well as heap. */
4332
+ function forceGarbageCollectionForTest() {
4333
+ const existing = globalThis.gc;
4334
+ if (typeof existing === "function") {
4335
+ existing();
4336
+ existing();
4337
+ return;
4338
+ }
4339
+ v8.setFlagsFromString("--expose-gc");
4340
+ const collect = vm.runInNewContext("gc");
4341
+ v8.setFlagsFromString("--no-expose-gc");
4342
+ if (typeof collect !== "function")
4343
+ throw new Error("test could not expose GC");
4344
+ collect();
4345
+ collect();
4346
+ }
4347
+ function retainedProcessBytes() {
4348
+ const usage = process.memoryUsage();
4349
+ return usage.heapUsed + usage.external + usage.arrayBuffers;
4350
+ }
4327
4351
  function makeSteppableSleep() {
4328
4352
  const pending = [];
4329
4353
  return {
@@ -6585,7 +6609,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
6585
6609
  fs.rmSync(hqRoot, { recursive: true, force: true });
6586
6610
  }
6587
6611
  }, 180_000);
6588
- it("keeps warmed personal and company journals hot across alternating realtime batches", async () => {
6612
+ it("keeps an area-authoritative 700k-row personal aggregate hot across alternating realtime and company batches", async () => {
6589
6613
  const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6590
6614
  process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
6591
6615
  const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-watch-"));
@@ -6594,6 +6618,259 @@ describe("runRunnerWithLoop — event-push wiring", () => {
6594
6618
  const companyPath = "companies/indigo/watch-load/00000/target.md";
6595
6619
  const companyKey = "watch-load/00000/target.md";
6596
6620
  const personalFiles = {};
6621
+ for (let index = 0; index < 700_000; index += 1) {
6622
+ personalFiles[`personal/history-${index % 5}/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
6623
+ hash: `sha256:history-${index}`,
6624
+ size: index % 4096,
6625
+ syncedAt: "2026-09-16T21:30:00.000Z",
6626
+ direction: "up",
6627
+ };
6628
+ }
6629
+ personalFiles[personalPath] = {
6630
+ hash: "sha256:personal-target",
6631
+ size: 1,
6632
+ syncedAt: "2026-09-16T21:30:00.000Z",
6633
+ direction: "up",
6634
+ };
6635
+ const resolver = new AreaResolver({
6636
+ format: 1,
6637
+ routerVersion: 1,
6638
+ areas: [
6639
+ ...Array.from({ length: 5 }, (_, index) => ({
6640
+ id: `root:history-${index}`,
6641
+ prefixes: [`personal/history-${index}`],
6642
+ paths: [],
6643
+ })),
6644
+ { id: "root:watch", prefixes: ["personal/watch-load"], paths: [] },
6645
+ { id: "root:@files", prefixes: [], paths: [], rootFiles: true },
6646
+ ],
6647
+ });
6648
+ writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
6649
+ version: "2",
6650
+ lastSync: "2026-09-16T21:30:00.000Z",
6651
+ pulls: [],
6652
+ files: {},
6653
+ });
6654
+ writeJournal("indigo", {
6655
+ version: "2",
6656
+ lastSync: "2026-09-16T21:30:00.000Z",
6657
+ pulls: [],
6658
+ files: {
6659
+ [companyKey]: {
6660
+ hash: "sha256:company-target",
6661
+ size: 1,
6662
+ syncedAt: "2026-09-16T21:30:00.000Z",
6663
+ direction: "up",
6664
+ },
6665
+ },
6666
+ });
6667
+ const migration = new AreaLedgerMigration({
6668
+ rootDir: __runnerTestStateDir,
6669
+ journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
6670
+ resolver,
6671
+ legacyStore: openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
6672
+ });
6673
+ expect(migration.cutOver(120_000).phase).toBe("area-authoritative");
6674
+ migration.dispose();
6675
+ // The migration state itself is covered by the area-ledger migration
6676
+ // suite. Seed the already-authoritative production shape in one durable
6677
+ // transaction so this watch-loop test measures repeated realtime reads,
6678
+ // rather than spending most of its worker budget replaying a one-time
6679
+ // legacy-to-area copy.
6680
+ openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG).replaceState({
6681
+ journal: {
6682
+ version: "2",
6683
+ lastSync: "2026-09-16T21:30:00.000Z",
6684
+ pulls: [],
6685
+ files: personalFiles,
6686
+ },
6687
+ });
6688
+ AreaLedger.open({
6689
+ rootDir: __runnerTestStateDir,
6690
+ journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
6691
+ layoutEpoch: 1,
6692
+ areaIds: resolver.layout.areas.map((area) => area.id),
6693
+ resolveArea: (key) => resolver.resolve(key),
6694
+ }).append({
6695
+ upserts: personalFiles,
6696
+ metadata: { lastSync: "2026-09-16T21:30:00.000Z", pulls: [] },
6697
+ });
6698
+ fs.mkdirSync(watchedRoot, { recursive: true });
6699
+ for (let index = 0; index < 50_000; index += 1) {
6700
+ fs.writeFileSync(path.join(watchedRoot, `seed-${String(index).padStart(5, "0")}.md`), "seed");
6701
+ }
6702
+ fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
6703
+ fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
6704
+ fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
6705
+ fs.writeFileSync(path.join(hqRoot, companyPath), "company");
6706
+ const watcher = makeBatchWatcherStub();
6707
+ const sleep = makeSteppableSleep();
6708
+ let shutdown = () => { };
6709
+ let isUntrackedDelete = (_relativePath) => false;
6710
+ const commits = [];
6711
+ let submittedAt = 0;
6712
+ const aggregatePrime = vi.spyOn(AreaLedger.prototype, "primeAggregateCache");
6713
+ const runPass = vi.fn(async (argv) => {
6714
+ if (!argv.includes("--scope-path")) {
6715
+ readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
6716
+ readJournal("indigo");
6717
+ }
6718
+ if (argv.includes("--scope-path") && argv.includes("push")) {
6719
+ const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
6720
+ const key = slug === "indigo" ? companyKey : personalPath;
6721
+ const scoped = readJournalScoped(slug, [key]);
6722
+ scoped.files[key] = {
6723
+ ...scoped.files[key],
6724
+ syncedAt: "2026-09-16T21:38:18.000Z",
6725
+ };
6726
+ writeJournal(slug, scoped);
6727
+ commits.push({ slug, elapsedMs: performance.now() - submittedAt });
6728
+ }
6729
+ return completedPassOutcome();
6730
+ });
6731
+ const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
6732
+ runPass,
6733
+ createWatcher: (options) => {
6734
+ isUntrackedDelete = options.isUntrackedDelete;
6735
+ return watcher;
6736
+ },
6737
+ startEventSync: vi.fn().mockResolvedValue(null),
6738
+ sleep: sleep.sleep,
6739
+ onShutdownSignal: (handler) => {
6740
+ shutdown = handler;
6741
+ return () => { };
6742
+ },
6743
+ });
6744
+ const emitAndWait = async (relativePaths) => {
6745
+ const before = commits.length;
6746
+ if (relativePaths.includes(personalPath)) {
6747
+ // This is the same non-blocking watcher callback which starts an
6748
+ // openJournalStoreSessionAsync before a delete episode is drained.
6749
+ // It is intentionally issued before each realtime batch: a completed
6750
+ // watch pass must not have thrown away the warmed area aggregate.
6751
+ expect(isUntrackedDelete(personalPath)).toBe(false);
6752
+ await new Promise((resolve) => setImmediate(resolve));
6753
+ }
6754
+ submittedAt = performance.now();
6755
+ const batch = {
6756
+ paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
6757
+ changes: new Map(relativePaths.map((relativePath) => [
6758
+ path.join(hqRoot, relativePath),
6759
+ { kind: "change" },
6760
+ ])),
6761
+ };
6762
+ watcher.emit(relativePaths[0], batch);
6763
+ await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
6764
+ timeout: 30_000,
6765
+ interval: 10,
6766
+ });
6767
+ // `commits` is observed inside the injected pass, before the runner has
6768
+ // completed its pass-final cleanup. Let that cleanup run before the next
6769
+ // watcher episode so the assertion observes the cache lifetime at the
6770
+ // same boundary as a real completed watch pass.
6771
+ await flushLoopMicrotasks(30);
6772
+ };
6773
+ try {
6774
+ await flushLoopMicrotasks(30);
6775
+ runPass.mockClear();
6776
+ // The initial runner pass is the one permitted cold open. Every later
6777
+ // scoped batch must reuse that authoritative aggregate. Warm the same
6778
+ // yielding watcher-session path that production uses before its first
6779
+ // delete episode, then measure only post-warmup primes.
6780
+ aggregatePrime.mockClear();
6781
+ expect(isUntrackedDelete(personalPath)).toBe(false);
6782
+ await new Promise((resolve) => setImmediate(resolve));
6783
+ // The initial full pass may already have opened this bridge (zero new
6784
+ // primes), or this watcher callback may perform the one permitted cold
6785
+ // warm-up. The post-poll assertion below is the regression boundary.
6786
+ expect(aggregatePrime.mock.calls.length).toBeLessThanOrEqual(1);
6787
+ aggregatePrime.mockClear();
6788
+ // Reproduce the normal poll boundary observed on the outposts. On the
6789
+ // current implementation it drops every cached area bridge even though
6790
+ // the watcher and runner remain alive.
6791
+ sleep.tick();
6792
+ await flushLoopMicrotasks(30);
6793
+ forceGarbageCollectionForTest();
6794
+ const retainedAfterWarmupBytes = retainedProcessBytes();
6795
+ const postWarmCpuStart = process.cpuUsage();
6796
+ const postWarmStartedAt = performance.now();
6797
+ resetStateStoreSnapshotRecoveryCountForTest();
6798
+ const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
6799
+ eventLoopDelay.enable();
6800
+ await emitAndWait([personalPath, companyPath]);
6801
+ for (let round = 0; round < 3; round += 1) {
6802
+ await emitAndWait([personalPath]);
6803
+ await emitAndWait([companyPath]);
6804
+ }
6805
+ eventLoopDelay.disable();
6806
+ const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
6807
+ const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
6808
+ const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
6809
+ // A first area delta briefly retains the reverse-shadow mirror to
6810
+ // amortize a burst. Retained memory means after that documented linger
6811
+ // has released its second full-vault projection, not during it.
6812
+ await new Promise((resolve) => setTimeout(resolve, AREA_LEGACY_MIRROR_LINGER_MS + 25));
6813
+ forceGarbageCollectionForTest();
6814
+ const retainedAfterChurnBytes = retainedProcessBytes();
6815
+ const retainedGrowthBytes = retainedAfterChurnBytes - retainedAfterWarmupBytes;
6816
+ const postWarmCpu = process.cpuUsage(postWarmCpuStart);
6817
+ const postWarmCpuPercent = ((postWarmCpu.user + postWarmCpu.system) /
6818
+ ((performance.now() - postWarmStartedAt) * 1_000)) * 100;
6819
+ console.info({
6820
+ aggregatePrimeCount: aggregatePrime.mock.calls.length,
6821
+ recoveryCount,
6822
+ maxDelayMs,
6823
+ maxCommitMs,
6824
+ postWarmCpuPercent,
6825
+ retainedAfterWarmupBytes,
6826
+ retainedAfterChurnBytes,
6827
+ retainedGrowthBytes,
6828
+ commits: commits.length,
6829
+ });
6830
+ expect(aggregatePrime).not.toHaveBeenCalled();
6831
+ // The first area delta opens its reverse-shadow v3 mirror once; its
6832
+ // linger keeps all later scoped commits off snapshot recovery. This is
6833
+ // deliberately exact so a second cold bridge cannot hide as a cache hit.
6834
+ expect.soft(recoveryCount).toBe(1);
6835
+ // The production-shape run remains bounded under the coverage job's
6836
+ // concurrent workers: CI measured 1,090.52 ms of timer delay and
6837
+ // 1,087.79 ms to commit. 1.5 s retains a concrete latency ceiling while
6838
+ // leaving headroom for that instrumentation, rather than making the
6839
+ // assertion depend on a single host's scheduling jitter.
6840
+ expect.soft(maxDelayMs).toBeLessThan(1_500);
6841
+ expect.soft(maxCommitMs).toBeLessThan(1_500);
6842
+ expect.soft(retainedGrowthBytes).toBeLessThan(32 * 1024 * 1024);
6843
+ expect(commits.map((commit) => commit.slug)).toEqual([
6844
+ PERSONAL_VAULT_JOURNAL_SLUG,
6845
+ "indigo",
6846
+ PERSONAL_VAULT_JOURNAL_SLUG,
6847
+ "indigo",
6848
+ PERSONAL_VAULT_JOURNAL_SLUG,
6849
+ "indigo",
6850
+ PERSONAL_VAULT_JOURNAL_SLUG,
6851
+ "indigo",
6852
+ ]);
6853
+ }
6854
+ finally {
6855
+ aggregatePrime.mockRestore();
6856
+ shutdown();
6857
+ await loop;
6858
+ if (previousCoordinator === undefined)
6859
+ delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6860
+ else
6861
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
6862
+ fs.rmSync(hqRoot, { recursive: true, force: true });
6863
+ }
6864
+ }, 180_000);
6865
+ it("keeps warmed personal and company journals hot across alternating realtime batches", async () => {
6866
+ const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
6867
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
6868
+ const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-legacy-watch-"));
6869
+ const watchedRoot = path.join(hqRoot, "personal", "watch-load");
6870
+ const personalPath = "personal/watch-load/00000/target.md";
6871
+ const companyPath = "companies/indigo/watch-load/00000/target.md";
6872
+ const companyKey = "watch-load/00000/target.md";
6873
+ const personalFiles = {};
6597
6874
  for (let index = 0; index < 700_000; index += 1) {
6598
6875
  personalFiles[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
6599
6876
  hash: `sha256:history-${index}`,