@indigoai-us/hq-cloud 6.16.52-alpha.0 → 6.16.53-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +7 -4
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +278 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/journal.d.ts +6 -3
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +6 -3
- package/dist/journal.js.map +1 -1
- package/package.json +1 -1
|
@@ -10,6 +10,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
|
10
10
|
import * as fs from "fs";
|
|
11
11
|
import * as os from "os";
|
|
12
12
|
import * as path from "path";
|
|
13
|
+
import * as v8 from "node:v8";
|
|
14
|
+
import * as vm from "node:vm";
|
|
13
15
|
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
14
16
|
import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
|
|
15
17
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
@@ -20,7 +22,9 @@ import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
|
20
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
21
23
|
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
22
24
|
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
25
|
+
import { AreaLedger } from "../sync/area-ledger.js";
|
|
23
26
|
import { AreaResolver } from "../sync/area-resolver.js";
|
|
27
|
+
import { AREA_LEGACY_MIRROR_LINGER_MS } from "../sync/area-ledger-migration.js";
|
|
24
28
|
import { AreaMaintenanceCursorStore } from "../sync/per-area-maintenance.js";
|
|
25
29
|
import { HQ_CLOUD_VERSION } from "../version.js";
|
|
26
30
|
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
@@ -4324,6 +4328,26 @@ async function flushLoopMicrotasks(n = 8) {
|
|
|
4324
4328
|
for (let i = 0; i < n; i++)
|
|
4325
4329
|
await Promise.resolve();
|
|
4326
4330
|
}
|
|
4331
|
+
/** Retained journal rows live in typed-array backing stores as well as heap. */
|
|
4332
|
+
function forceGarbageCollectionForTest() {
|
|
4333
|
+
const existing = globalThis.gc;
|
|
4334
|
+
if (typeof existing === "function") {
|
|
4335
|
+
existing();
|
|
4336
|
+
existing();
|
|
4337
|
+
return;
|
|
4338
|
+
}
|
|
4339
|
+
v8.setFlagsFromString("--expose-gc");
|
|
4340
|
+
const collect = vm.runInNewContext("gc");
|
|
4341
|
+
v8.setFlagsFromString("--no-expose-gc");
|
|
4342
|
+
if (typeof collect !== "function")
|
|
4343
|
+
throw new Error("test could not expose GC");
|
|
4344
|
+
collect();
|
|
4345
|
+
collect();
|
|
4346
|
+
}
|
|
4347
|
+
function retainedProcessBytes() {
|
|
4348
|
+
const usage = process.memoryUsage();
|
|
4349
|
+
return usage.heapUsed + usage.external + usage.arrayBuffers;
|
|
4350
|
+
}
|
|
4327
4351
|
function makeSteppableSleep() {
|
|
4328
4352
|
const pending = [];
|
|
4329
4353
|
return {
|
|
@@ -6585,7 +6609,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6585
6609
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6586
6610
|
}
|
|
6587
6611
|
}, 180_000);
|
|
6588
|
-
it("keeps
|
|
6612
|
+
it("keeps an area-authoritative 700k-row personal aggregate hot across alternating realtime and company batches", async () => {
|
|
6589
6613
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6590
6614
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6591
6615
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-watch-"));
|
|
@@ -6594,6 +6618,259 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6594
6618
|
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
6595
6619
|
const companyKey = "watch-load/00000/target.md";
|
|
6596
6620
|
const personalFiles = {};
|
|
6621
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
6622
|
+
personalFiles[`personal/history-${index % 5}/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6623
|
+
hash: `sha256:history-${index}`,
|
|
6624
|
+
size: index % 4096,
|
|
6625
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6626
|
+
direction: "up",
|
|
6627
|
+
};
|
|
6628
|
+
}
|
|
6629
|
+
personalFiles[personalPath] = {
|
|
6630
|
+
hash: "sha256:personal-target",
|
|
6631
|
+
size: 1,
|
|
6632
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6633
|
+
direction: "up",
|
|
6634
|
+
};
|
|
6635
|
+
const resolver = new AreaResolver({
|
|
6636
|
+
format: 1,
|
|
6637
|
+
routerVersion: 1,
|
|
6638
|
+
areas: [
|
|
6639
|
+
...Array.from({ length: 5 }, (_, index) => ({
|
|
6640
|
+
id: `root:history-${index}`,
|
|
6641
|
+
prefixes: [`personal/history-${index}`],
|
|
6642
|
+
paths: [],
|
|
6643
|
+
})),
|
|
6644
|
+
{ id: "root:watch", prefixes: ["personal/watch-load"], paths: [] },
|
|
6645
|
+
{ id: "root:@files", prefixes: [], paths: [], rootFiles: true },
|
|
6646
|
+
],
|
|
6647
|
+
});
|
|
6648
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
6649
|
+
version: "2",
|
|
6650
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6651
|
+
pulls: [],
|
|
6652
|
+
files: {},
|
|
6653
|
+
});
|
|
6654
|
+
writeJournal("indigo", {
|
|
6655
|
+
version: "2",
|
|
6656
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6657
|
+
pulls: [],
|
|
6658
|
+
files: {
|
|
6659
|
+
[companyKey]: {
|
|
6660
|
+
hash: "sha256:company-target",
|
|
6661
|
+
size: 1,
|
|
6662
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6663
|
+
direction: "up",
|
|
6664
|
+
},
|
|
6665
|
+
},
|
|
6666
|
+
});
|
|
6667
|
+
const migration = new AreaLedgerMigration({
|
|
6668
|
+
rootDir: __runnerTestStateDir,
|
|
6669
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6670
|
+
resolver,
|
|
6671
|
+
legacyStore: openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
|
|
6672
|
+
});
|
|
6673
|
+
expect(migration.cutOver(120_000).phase).toBe("area-authoritative");
|
|
6674
|
+
migration.dispose();
|
|
6675
|
+
// The migration state itself is covered by the area-ledger migration
|
|
6676
|
+
// suite. Seed the already-authoritative production shape in one durable
|
|
6677
|
+
// transaction so this watch-loop test measures repeated realtime reads,
|
|
6678
|
+
// rather than spending most of its worker budget replaying a one-time
|
|
6679
|
+
// legacy-to-area copy.
|
|
6680
|
+
openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG).replaceState({
|
|
6681
|
+
journal: {
|
|
6682
|
+
version: "2",
|
|
6683
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6684
|
+
pulls: [],
|
|
6685
|
+
files: personalFiles,
|
|
6686
|
+
},
|
|
6687
|
+
});
|
|
6688
|
+
AreaLedger.open({
|
|
6689
|
+
rootDir: __runnerTestStateDir,
|
|
6690
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6691
|
+
layoutEpoch: 1,
|
|
6692
|
+
areaIds: resolver.layout.areas.map((area) => area.id),
|
|
6693
|
+
resolveArea: (key) => resolver.resolve(key),
|
|
6694
|
+
}).append({
|
|
6695
|
+
upserts: personalFiles,
|
|
6696
|
+
metadata: { lastSync: "2026-09-16T21:30:00.000Z", pulls: [] },
|
|
6697
|
+
});
|
|
6698
|
+
fs.mkdirSync(watchedRoot, { recursive: true });
|
|
6699
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
6700
|
+
fs.writeFileSync(path.join(watchedRoot, `seed-${String(index).padStart(5, "0")}.md`), "seed");
|
|
6701
|
+
}
|
|
6702
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
|
|
6703
|
+
fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
|
|
6704
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
|
|
6705
|
+
fs.writeFileSync(path.join(hqRoot, companyPath), "company");
|
|
6706
|
+
const watcher = makeBatchWatcherStub();
|
|
6707
|
+
const sleep = makeSteppableSleep();
|
|
6708
|
+
let shutdown = () => { };
|
|
6709
|
+
let isUntrackedDelete = (_relativePath) => false;
|
|
6710
|
+
const commits = [];
|
|
6711
|
+
let submittedAt = 0;
|
|
6712
|
+
const aggregatePrime = vi.spyOn(AreaLedger.prototype, "primeAggregateCache");
|
|
6713
|
+
const runPass = vi.fn(async (argv) => {
|
|
6714
|
+
if (!argv.includes("--scope-path")) {
|
|
6715
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
6716
|
+
readJournal("indigo");
|
|
6717
|
+
}
|
|
6718
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
6719
|
+
const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
|
|
6720
|
+
const key = slug === "indigo" ? companyKey : personalPath;
|
|
6721
|
+
const scoped = readJournalScoped(slug, [key]);
|
|
6722
|
+
scoped.files[key] = {
|
|
6723
|
+
...scoped.files[key],
|
|
6724
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
6725
|
+
};
|
|
6726
|
+
writeJournal(slug, scoped);
|
|
6727
|
+
commits.push({ slug, elapsedMs: performance.now() - submittedAt });
|
|
6728
|
+
}
|
|
6729
|
+
return completedPassOutcome();
|
|
6730
|
+
});
|
|
6731
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6732
|
+
runPass,
|
|
6733
|
+
createWatcher: (options) => {
|
|
6734
|
+
isUntrackedDelete = options.isUntrackedDelete;
|
|
6735
|
+
return watcher;
|
|
6736
|
+
},
|
|
6737
|
+
startEventSync: vi.fn().mockResolvedValue(null),
|
|
6738
|
+
sleep: sleep.sleep,
|
|
6739
|
+
onShutdownSignal: (handler) => {
|
|
6740
|
+
shutdown = handler;
|
|
6741
|
+
return () => { };
|
|
6742
|
+
},
|
|
6743
|
+
});
|
|
6744
|
+
const emitAndWait = async (relativePaths) => {
|
|
6745
|
+
const before = commits.length;
|
|
6746
|
+
if (relativePaths.includes(personalPath)) {
|
|
6747
|
+
// This is the same non-blocking watcher callback which starts an
|
|
6748
|
+
// openJournalStoreSessionAsync before a delete episode is drained.
|
|
6749
|
+
// It is intentionally issued before each realtime batch: a completed
|
|
6750
|
+
// watch pass must not have thrown away the warmed area aggregate.
|
|
6751
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
6752
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
6753
|
+
}
|
|
6754
|
+
submittedAt = performance.now();
|
|
6755
|
+
const batch = {
|
|
6756
|
+
paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
6757
|
+
changes: new Map(relativePaths.map((relativePath) => [
|
|
6758
|
+
path.join(hqRoot, relativePath),
|
|
6759
|
+
{ kind: "change" },
|
|
6760
|
+
])),
|
|
6761
|
+
};
|
|
6762
|
+
watcher.emit(relativePaths[0], batch);
|
|
6763
|
+
await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
|
|
6764
|
+
timeout: 30_000,
|
|
6765
|
+
interval: 10,
|
|
6766
|
+
});
|
|
6767
|
+
// `commits` is observed inside the injected pass, before the runner has
|
|
6768
|
+
// completed its pass-final cleanup. Let that cleanup run before the next
|
|
6769
|
+
// watcher episode so the assertion observes the cache lifetime at the
|
|
6770
|
+
// same boundary as a real completed watch pass.
|
|
6771
|
+
await flushLoopMicrotasks(30);
|
|
6772
|
+
};
|
|
6773
|
+
try {
|
|
6774
|
+
await flushLoopMicrotasks(30);
|
|
6775
|
+
runPass.mockClear();
|
|
6776
|
+
// The initial runner pass is the one permitted cold open. Every later
|
|
6777
|
+
// scoped batch must reuse that authoritative aggregate. Warm the same
|
|
6778
|
+
// yielding watcher-session path that production uses before its first
|
|
6779
|
+
// delete episode, then measure only post-warmup primes.
|
|
6780
|
+
aggregatePrime.mockClear();
|
|
6781
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
6782
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
6783
|
+
// The initial full pass may already have opened this bridge (zero new
|
|
6784
|
+
// primes), or this watcher callback may perform the one permitted cold
|
|
6785
|
+
// warm-up. The post-poll assertion below is the regression boundary.
|
|
6786
|
+
expect(aggregatePrime.mock.calls.length).toBeLessThanOrEqual(1);
|
|
6787
|
+
aggregatePrime.mockClear();
|
|
6788
|
+
// Reproduce the normal poll boundary observed on the outposts. On the
|
|
6789
|
+
// current implementation it drops every cached area bridge even though
|
|
6790
|
+
// the watcher and runner remain alive.
|
|
6791
|
+
sleep.tick();
|
|
6792
|
+
await flushLoopMicrotasks(30);
|
|
6793
|
+
forceGarbageCollectionForTest();
|
|
6794
|
+
const retainedAfterWarmupBytes = retainedProcessBytes();
|
|
6795
|
+
const postWarmCpuStart = process.cpuUsage();
|
|
6796
|
+
const postWarmStartedAt = performance.now();
|
|
6797
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
6798
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
6799
|
+
eventLoopDelay.enable();
|
|
6800
|
+
await emitAndWait([personalPath, companyPath]);
|
|
6801
|
+
for (let round = 0; round < 3; round += 1) {
|
|
6802
|
+
await emitAndWait([personalPath]);
|
|
6803
|
+
await emitAndWait([companyPath]);
|
|
6804
|
+
}
|
|
6805
|
+
eventLoopDelay.disable();
|
|
6806
|
+
const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
|
|
6807
|
+
const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
|
|
6808
|
+
const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
|
|
6809
|
+
// A first area delta briefly retains the reverse-shadow mirror to
|
|
6810
|
+
// amortize a burst. Retained memory means after that documented linger
|
|
6811
|
+
// has released its second full-vault projection, not during it.
|
|
6812
|
+
await new Promise((resolve) => setTimeout(resolve, AREA_LEGACY_MIRROR_LINGER_MS + 25));
|
|
6813
|
+
forceGarbageCollectionForTest();
|
|
6814
|
+
const retainedAfterChurnBytes = retainedProcessBytes();
|
|
6815
|
+
const retainedGrowthBytes = retainedAfterChurnBytes - retainedAfterWarmupBytes;
|
|
6816
|
+
const postWarmCpu = process.cpuUsage(postWarmCpuStart);
|
|
6817
|
+
const postWarmCpuPercent = ((postWarmCpu.user + postWarmCpu.system) /
|
|
6818
|
+
((performance.now() - postWarmStartedAt) * 1_000)) * 100;
|
|
6819
|
+
console.info({
|
|
6820
|
+
aggregatePrimeCount: aggregatePrime.mock.calls.length,
|
|
6821
|
+
recoveryCount,
|
|
6822
|
+
maxDelayMs,
|
|
6823
|
+
maxCommitMs,
|
|
6824
|
+
postWarmCpuPercent,
|
|
6825
|
+
retainedAfterWarmupBytes,
|
|
6826
|
+
retainedAfterChurnBytes,
|
|
6827
|
+
retainedGrowthBytes,
|
|
6828
|
+
commits: commits.length,
|
|
6829
|
+
});
|
|
6830
|
+
expect(aggregatePrime).not.toHaveBeenCalled();
|
|
6831
|
+
// The first area delta opens its reverse-shadow v3 mirror once; its
|
|
6832
|
+
// linger keeps all later scoped commits off snapshot recovery. This is
|
|
6833
|
+
// deliberately exact so a second cold bridge cannot hide as a cache hit.
|
|
6834
|
+
expect.soft(recoveryCount).toBe(1);
|
|
6835
|
+
// The production-shape run remains bounded under the coverage job's
|
|
6836
|
+
// concurrent workers: CI measured 1,090.52 ms of timer delay and
|
|
6837
|
+
// 1,087.79 ms to commit. 1.5 s retains a concrete latency ceiling while
|
|
6838
|
+
// leaving headroom for that instrumentation, rather than making the
|
|
6839
|
+
// assertion depend on a single host's scheduling jitter.
|
|
6840
|
+
expect.soft(maxDelayMs).toBeLessThan(1_500);
|
|
6841
|
+
expect.soft(maxCommitMs).toBeLessThan(1_500);
|
|
6842
|
+
expect.soft(retainedGrowthBytes).toBeLessThan(32 * 1024 * 1024);
|
|
6843
|
+
expect(commits.map((commit) => commit.slug)).toEqual([
|
|
6844
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6845
|
+
"indigo",
|
|
6846
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6847
|
+
"indigo",
|
|
6848
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6849
|
+
"indigo",
|
|
6850
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6851
|
+
"indigo",
|
|
6852
|
+
]);
|
|
6853
|
+
}
|
|
6854
|
+
finally {
|
|
6855
|
+
aggregatePrime.mockRestore();
|
|
6856
|
+
shutdown();
|
|
6857
|
+
await loop;
|
|
6858
|
+
if (previousCoordinator === undefined)
|
|
6859
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6860
|
+
else
|
|
6861
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6862
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6863
|
+
}
|
|
6864
|
+
}, 180_000);
|
|
6865
|
+
it("keeps warmed personal and company journals hot across alternating realtime batches", async () => {
|
|
6866
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6867
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6868
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-legacy-watch-"));
|
|
6869
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
6870
|
+
const personalPath = "personal/watch-load/00000/target.md";
|
|
6871
|
+
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
6872
|
+
const companyKey = "watch-load/00000/target.md";
|
|
6873
|
+
const personalFiles = {};
|
|
6597
6874
|
for (let index = 0; index < 700_000; index += 1) {
|
|
6598
6875
|
personalFiles[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6599
6876
|
hash: `sha256:history-${index}`,
|