@indigoai-us/hq-cloud 6.16.64 → 6.16.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +62 -7
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +52 -3
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +443 -167
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +64 -6
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1577 -169
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/push-scan-exclusions.test.js +4 -4
- package/dist/cli/push-scan-exclusions.test.js.map +1 -1
- package/dist/cli/share.d.ts +31 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +136 -6
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +295 -5
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +302 -123
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/hq-root-recovery.d.ts +30 -0
- package/dist/hq-root-recovery.d.ts.map +1 -0
- package/dist/hq-root-recovery.js +131 -0
- package/dist/hq-root-recovery.js.map +1 -0
- package/dist/hq-root-recovery.test.d.ts +2 -0
- package/dist/hq-root-recovery.test.d.ts.map +1 -0
- package/dist/hq-root-recovery.test.js +65 -0
- package/dist/hq-root-recovery.test.js.map +1 -0
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watch-coverage.d.ts +24 -0
- package/dist/watch-coverage.d.ts.map +1 -0
- package/dist/watch-coverage.js +46 -0
- package/dist/watch-coverage.js.map +1 -0
- package/dist/watch-coverage.test.d.ts +2 -0
- package/dist/watch-coverage.test.d.ts.map +1 -0
- package/dist/watch-coverage.test.js +56 -0
- package/dist/watch-coverage.test.js.map +1 -0
- package/dist/watcher.d.ts +31 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +172 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +78 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
|
@@ -10,6 +10,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
|
10
10
|
import * as fs from "fs";
|
|
11
11
|
import * as os from "os";
|
|
12
12
|
import * as path from "path";
|
|
13
|
+
import * as v8 from "node:v8";
|
|
14
|
+
import * as vm from "node:vm";
|
|
15
|
+
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
13
16
|
import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
|
|
14
17
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
15
18
|
import { AUTH_WAIT_INITIAL_RECHECK_MS, AUTH_WAIT_RESUME_MESSAGE, AUTH_WAIT_START_MESSAGE, AUTH_WAIT_STEADY_RECHECK_MS, } from "./sync-runner-auth-wait.js";
|
|
@@ -17,17 +20,20 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
|
|
|
17
20
|
import { setBandwidthGovernorForTesting } from "../bandwidth.js";
|
|
18
21
|
import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
19
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
20
|
-
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
21
|
-
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
|
|
23
|
+
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
24
|
+
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
25
|
+
import { AreaLedger } from "../sync/area-ledger.js";
|
|
22
26
|
import { AreaResolver } from "../sync/area-resolver.js";
|
|
27
|
+
import { AREA_LEGACY_MIRROR_LINGER_MS } from "../sync/area-ledger-migration.js";
|
|
23
28
|
import { AreaMaintenanceCursorStore } from "../sync/per-area-maintenance.js";
|
|
24
29
|
import { HQ_CLOUD_VERSION } from "../version.js";
|
|
25
30
|
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
26
31
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
27
32
|
import { getStateStoreSnapshotRecoveryCountForTest, resetStateStoreSnapshotRecoveryCountForTest, StateStoreLockError, } from "../sync/state-store.js";
|
|
28
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
33
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
29
34
|
import { isCoveredByAny } from "../prefix-coalesce.js";
|
|
30
35
|
import { VaultAuthError, VaultNotFoundError } from "../vault-client.js";
|
|
36
|
+
import { buildRoutePullArgv } from "./sync-runner-watch-routes.js";
|
|
31
37
|
import { CognitoAuthError, CognitoRefreshError } from "../cognito-auth.js";
|
|
32
38
|
import { reconcileCompanyManifest, } from "../manifest-reconcile.js";
|
|
33
39
|
import yaml from "js-yaml";
|
|
@@ -166,6 +172,7 @@ function makeVaultStub(opts = {}) {
|
|
|
166
172
|
bucketName: `bucket-${uid}`,
|
|
167
173
|
status: "active",
|
|
168
174
|
})),
|
|
175
|
+
findInMyNamespace: opts.findInMyNamespace ?? (() => Promise.resolve(null)),
|
|
169
176
|
listByType: opts.listPersons ??
|
|
170
177
|
(() => Promise.resolve([])),
|
|
171
178
|
},
|
|
@@ -958,6 +965,164 @@ describe("target resolution", () => {
|
|
|
958
965
|
const call = deps.sync.mock.calls[0][0];
|
|
959
966
|
expect(call.company).toBe("cmp_abc");
|
|
960
967
|
});
|
|
968
|
+
it("returns the unresolved-company exit for an empty single-company plan instead of emitting a clean all-complete", async () => {
|
|
969
|
+
const deps = makeDeps({
|
|
970
|
+
createVaultClient: () => makeVaultStub({
|
|
971
|
+
entityGet: () => Promise.resolve(undefined),
|
|
972
|
+
}),
|
|
973
|
+
});
|
|
974
|
+
expect(await runRunner(["--company", "cmp_missing"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
975
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
976
|
+
type: "error",
|
|
977
|
+
company: "cmp_missing",
|
|
978
|
+
path: "(company-resolution)",
|
|
979
|
+
message: "company entity lookup returned no entity",
|
|
980
|
+
});
|
|
981
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
982
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
983
|
+
});
|
|
984
|
+
it("returns the unresolved-company exit when a cmp-prefixed route gets a 404", async () => {
|
|
985
|
+
const client = makeVaultStub({
|
|
986
|
+
entityGet: () => Promise.reject(new VaultNotFoundError("company deleted")),
|
|
987
|
+
});
|
|
988
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
989
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
990
|
+
const deps = makeDeps({
|
|
991
|
+
createVaultClient: () => client,
|
|
992
|
+
});
|
|
993
|
+
expect(await runRunner(["--company", "cmp_ghost"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
994
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_ghost");
|
|
995
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
996
|
+
type: "error",
|
|
997
|
+
company: "cmp_ghost",
|
|
998
|
+
path: "(company-resolution)",
|
|
999
|
+
message: "company entity lookup returned 404",
|
|
1000
|
+
});
|
|
1001
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
1002
|
+
});
|
|
1003
|
+
it("reports an intentional skipCompanies skip without treating it as a failed pass", async () => {
|
|
1004
|
+
const previous = process.env.HQ_SYNC_SKIP_COMPANIES;
|
|
1005
|
+
process.env.HQ_SYNC_SKIP_COMPANIES = "paused";
|
|
1006
|
+
try {
|
|
1007
|
+
const deps = makeDeps({
|
|
1008
|
+
createVaultClient: () => makeVaultStub({
|
|
1009
|
+
entityGet: (uid) => Promise.resolve({
|
|
1010
|
+
uid,
|
|
1011
|
+
slug: "paused",
|
|
1012
|
+
type: "company",
|
|
1013
|
+
status: "active",
|
|
1014
|
+
}),
|
|
1015
|
+
}),
|
|
1016
|
+
});
|
|
1017
|
+
expect(await runRunner(["--company", "cmp_paused"], deps)).toBe(0);
|
|
1018
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1019
|
+
type: "company-skipped",
|
|
1020
|
+
company: "paused",
|
|
1021
|
+
reason: "skip-companies",
|
|
1022
|
+
});
|
|
1023
|
+
expect(deps.stderr.events().some((event) => event.type === "error")).toBe(false);
|
|
1024
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
1025
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
1026
|
+
}
|
|
1027
|
+
finally {
|
|
1028
|
+
if (previous === undefined)
|
|
1029
|
+
delete process.env.HQ_SYNC_SKIP_COMPANIES;
|
|
1030
|
+
else
|
|
1031
|
+
process.env.HQ_SYNC_SKIP_COMPANIES = previous;
|
|
1032
|
+
}
|
|
1033
|
+
});
|
|
1034
|
+
it("resolves a slug emitted by the company watch route to its namespace UID", async () => {
|
|
1035
|
+
const company = {
|
|
1036
|
+
uid: "cmp_indigo",
|
|
1037
|
+
slug: "indigo",
|
|
1038
|
+
type: "company",
|
|
1039
|
+
status: "active",
|
|
1040
|
+
bucketName: "bucket-cmp_indigo",
|
|
1041
|
+
};
|
|
1042
|
+
const client = makeVaultStub({
|
|
1043
|
+
entityGet: (uid) => Promise.resolve(uid === company.uid ? company : undefined),
|
|
1044
|
+
});
|
|
1045
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(company);
|
|
1046
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1047
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1048
|
+
const argv = buildRoutePullArgv({ kind: "company", slug: "indigo" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
1049
|
+
expect(argv).toEqual([
|
|
1050
|
+
"--company",
|
|
1051
|
+
"indigo",
|
|
1052
|
+
"--direction",
|
|
1053
|
+
"pull",
|
|
1054
|
+
"--hq-root",
|
|
1055
|
+
"/tmp/hq",
|
|
1056
|
+
]);
|
|
1057
|
+
expect(await runRunner(argv, deps)).toBe(0);
|
|
1058
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "indigo");
|
|
1059
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: "cmp_indigo" }));
|
|
1060
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1061
|
+
type: "fanout-plan",
|
|
1062
|
+
companies: [
|
|
1063
|
+
expect.objectContaining({ uid: "cmp_indigo", slug: "indigo" }),
|
|
1064
|
+
],
|
|
1065
|
+
});
|
|
1066
|
+
});
|
|
1067
|
+
it("resolves a cmp-prefixed directory slug in the caller namespace before treating it as a UID", async () => {
|
|
1068
|
+
const company = {
|
|
1069
|
+
uid: "cmp_01JTEAM000000000000000000",
|
|
1070
|
+
slug: "cmp_team",
|
|
1071
|
+
type: "company",
|
|
1072
|
+
status: "active",
|
|
1073
|
+
bucketName: "bucket-cmp-team",
|
|
1074
|
+
};
|
|
1075
|
+
const get = vi.fn().mockImplementation((uid) => Promise.resolve(uid === company.uid ? company : undefined));
|
|
1076
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1077
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(company);
|
|
1078
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1079
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1080
|
+
const argv = buildRoutePullArgv({ kind: "company", slug: "cmp_team" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
1081
|
+
expect(await runRunner(argv, deps)).toBe(0);
|
|
1082
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_team");
|
|
1083
|
+
expect(get).toHaveBeenCalledWith(company.uid);
|
|
1084
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: company.uid }));
|
|
1085
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1086
|
+
type: "fanout-plan",
|
|
1087
|
+
companies: [expect.objectContaining({ uid: company.uid, slug: "cmp_team" })],
|
|
1088
|
+
});
|
|
1089
|
+
});
|
|
1090
|
+
it("falls back to a literal company UID after caller-namespace slug resolution misses", async () => {
|
|
1091
|
+
const uid = "cmp_01JUID0000000000000000000";
|
|
1092
|
+
const company = {
|
|
1093
|
+
uid,
|
|
1094
|
+
slug: "operator-company",
|
|
1095
|
+
type: "company",
|
|
1096
|
+
status: "active",
|
|
1097
|
+
bucketName: "bucket-operator-company",
|
|
1098
|
+
};
|
|
1099
|
+
const get = vi.fn().mockResolvedValue(company);
|
|
1100
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1101
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
1102
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1103
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1104
|
+
expect(await runRunner(["--company", uid], deps)).toBe(0);
|
|
1105
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", uid);
|
|
1106
|
+
expect(get).toHaveBeenCalledWith(uid);
|
|
1107
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: uid }));
|
|
1108
|
+
});
|
|
1109
|
+
it("reports a foreign company slug when the literal UID fallback finds nothing", async () => {
|
|
1110
|
+
const get = vi.fn().mockRejectedValue(new VaultNotFoundError("company not found"));
|
|
1111
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1112
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
1113
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1114
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1115
|
+
expect(await runRunner(["--company", "indigo"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
1116
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "indigo");
|
|
1117
|
+
expect(get).toHaveBeenCalledWith("indigo");
|
|
1118
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
1119
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
1120
|
+
type: "error",
|
|
1121
|
+
company: "indigo",
|
|
1122
|
+
path: "(company-resolution)",
|
|
1123
|
+
message: "company entity lookup returned 404",
|
|
1124
|
+
});
|
|
1125
|
+
});
|
|
961
1126
|
});
|
|
962
1127
|
// ---------------------------------------------------------------------------
|
|
963
1128
|
// --personal mode (personal-vault-only, skip company fanout)
|
|
@@ -1555,6 +1720,43 @@ describe("automatic journal maintenance", () => {
|
|
|
1555
1720
|
},
|
|
1556
1721
|
]);
|
|
1557
1722
|
});
|
|
1723
|
+
it("signals watch readiness after journal preparation and before fanout", async () => {
|
|
1724
|
+
const order = [];
|
|
1725
|
+
let releaseFanout = () => { };
|
|
1726
|
+
const fanout = new Promise((resolve) => { releaseFanout = resolve; });
|
|
1727
|
+
const deps = makeDeps({
|
|
1728
|
+
createVaultClient: () => makeVaultStub({
|
|
1729
|
+
memberships: [{ companyUid: "cmp_indigo" }],
|
|
1730
|
+
entityGet: (uid) => Promise.resolve({ uid, slug: "indigo" }),
|
|
1731
|
+
}),
|
|
1732
|
+
repairJournalStateIfNeeded: vi.fn((slug) => {
|
|
1733
|
+
order.push(`repair:${slug}`);
|
|
1734
|
+
return { status: "not-needed", slug };
|
|
1735
|
+
}),
|
|
1736
|
+
migratePersonalVaultJournal: vi.fn(() => {
|
|
1737
|
+
order.push("seed:personal");
|
|
1738
|
+
}),
|
|
1739
|
+
onJournalsReady: () => {
|
|
1740
|
+
order.push("journals-ready");
|
|
1741
|
+
},
|
|
1742
|
+
sync: vi.fn(async () => {
|
|
1743
|
+
order.push("fanout");
|
|
1744
|
+
await fanout;
|
|
1745
|
+
return defaultSyncResult();
|
|
1746
|
+
}),
|
|
1747
|
+
});
|
|
1748
|
+
const running = runRunner(["--companies", "--skip-personal"], deps);
|
|
1749
|
+
await vi.waitFor(() => expect(order).toContain("fanout"));
|
|
1750
|
+
expect(order).toEqual([
|
|
1751
|
+
"repair:indigo",
|
|
1752
|
+
"repair:personal",
|
|
1753
|
+
"seed:personal",
|
|
1754
|
+
"journals-ready",
|
|
1755
|
+
"fanout",
|
|
1756
|
+
]);
|
|
1757
|
+
releaseFanout();
|
|
1758
|
+
await expect(running).resolves.toBe(0);
|
|
1759
|
+
});
|
|
1558
1760
|
it("continues the pass when one journal repair is transiently lock-contended", async () => {
|
|
1559
1761
|
const repaired = [];
|
|
1560
1762
|
const sync = vi.fn().mockResolvedValue(defaultSyncResult());
|
|
@@ -4178,6 +4380,17 @@ describe("routeChangeToTarget", () => {
|
|
|
4178
4380
|
});
|
|
4179
4381
|
expect(routeChangeToTarget("companies/acme/projects/x/prd.json")).toEqual({ kind: "company", slug: "acme" });
|
|
4180
4382
|
});
|
|
4383
|
+
it("keeps a company-root directory delete attributable without routing company-root files", () => {
|
|
4384
|
+
// A whole-company removal has no child path, but it still owns every
|
|
4385
|
+
// journal row below the slug and must reach the delete-intent guard.
|
|
4386
|
+
expect(routeChangeToTarget("companies/indigo", "unlinkDir")).toEqual({
|
|
4387
|
+
kind: "company",
|
|
4388
|
+
slug: "indigo",
|
|
4389
|
+
});
|
|
4390
|
+
// A file directly under companies/ is shared metadata, never a tenant.
|
|
4391
|
+
expect(routeChangeToTarget("companies/manifest.yaml", "change")).toBeNull();
|
|
4392
|
+
expect(routeChangeToTarget("companies/.DS_Store", "add")).toBeNull();
|
|
4393
|
+
});
|
|
4181
4394
|
it("routes non-company top-levels to the personal target", () => {
|
|
4182
4395
|
expect(routeChangeToTarget("personal/notes.md")).toEqual({
|
|
4183
4396
|
kind: "personal",
|
|
@@ -4202,6 +4415,12 @@ describe("routeChangeToTarget", () => {
|
|
|
4202
4415
|
// edit always lands at companies/<slug>/..., never at bare "companies".
|
|
4203
4416
|
expect(routeChangeToTarget("companies")).toBeNull();
|
|
4204
4417
|
expect(routeChangeToTarget("companies/")).toBeNull();
|
|
4418
|
+
// The companies directory itself contains shared manifest and scaffold
|
|
4419
|
+
// files. They do not belong to a company journal, so a direct child must
|
|
4420
|
+
// never become a bogus `--company <filename>` pass.
|
|
4421
|
+
expect(routeChangeToTarget("companies/manifest.yaml")).toBeNull();
|
|
4422
|
+
expect(routeChangeToTarget("companies/.DS_Store")).toBeNull();
|
|
4423
|
+
expect(routeChangeToTarget("companies/../personal/notes.md")).toBeNull();
|
|
4205
4424
|
});
|
|
4206
4425
|
it("does not route the company scaffold to cloud sync", () => {
|
|
4207
4426
|
expect(routeChangeToTarget("companies/_template/knowledge/README.md")).toBeNull();
|
|
@@ -4283,6 +4502,26 @@ async function flushLoopMicrotasks(n = 8) {
|
|
|
4283
4502
|
for (let i = 0; i < n; i++)
|
|
4284
4503
|
await Promise.resolve();
|
|
4285
4504
|
}
|
|
4505
|
+
/** Retained journal rows live in typed-array backing stores as well as heap. */
|
|
4506
|
+
function forceGarbageCollectionForTest() {
|
|
4507
|
+
const existing = globalThis.gc;
|
|
4508
|
+
if (typeof existing === "function") {
|
|
4509
|
+
existing();
|
|
4510
|
+
existing();
|
|
4511
|
+
return;
|
|
4512
|
+
}
|
|
4513
|
+
v8.setFlagsFromString("--expose-gc");
|
|
4514
|
+
const collect = vm.runInNewContext("gc");
|
|
4515
|
+
v8.setFlagsFromString("--no-expose-gc");
|
|
4516
|
+
if (typeof collect !== "function")
|
|
4517
|
+
throw new Error("test could not expose GC");
|
|
4518
|
+
collect();
|
|
4519
|
+
collect();
|
|
4520
|
+
}
|
|
4521
|
+
function retainedProcessBytes() {
|
|
4522
|
+
const usage = process.memoryUsage();
|
|
4523
|
+
return usage.heapUsed + usage.external + usage.arrayBuffers;
|
|
4524
|
+
}
|
|
4286
4525
|
function makeSteppableSleep() {
|
|
4287
4526
|
const pending = [];
|
|
4288
4527
|
return {
|
|
@@ -4295,8 +4534,19 @@ function makeSteppableSleep() {
|
|
|
4295
4534
|
throw new Error("poll sleep was not pending");
|
|
4296
4535
|
resolve();
|
|
4297
4536
|
},
|
|
4537
|
+
pendingCount() {
|
|
4538
|
+
return pending.length;
|
|
4539
|
+
},
|
|
4298
4540
|
};
|
|
4299
4541
|
}
|
|
4542
|
+
async function waitForPollSleep(sleep) {
|
|
4543
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
4544
|
+
if (sleep.pendingCount() > 0)
|
|
4545
|
+
return;
|
|
4546
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4547
|
+
}
|
|
4548
|
+
throw new Error("timed out waiting for a poll sleep");
|
|
4549
|
+
}
|
|
4300
4550
|
// ---------------------------------------------------------------------------
|
|
4301
4551
|
// Adaptive remote-poll backoff — whole-box CPU aware (no --poll-remote-ms)
|
|
4302
4552
|
// ---------------------------------------------------------------------------
|
|
@@ -5285,6 +5535,52 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5285
5535
|
vi.useRealTimers();
|
|
5286
5536
|
}
|
|
5287
5537
|
});
|
|
5538
|
+
it("wakes a full reconcile when coverage probe finds a vanished HQ root", async () => {
|
|
5539
|
+
vi.useFakeTimers();
|
|
5540
|
+
const watcher = makeWatcherStub();
|
|
5541
|
+
watcher.isWatching = () => true;
|
|
5542
|
+
watcher.watchedPathCount = () => 121;
|
|
5543
|
+
watcher.watchMode = () => "native";
|
|
5544
|
+
watcher.probeCoverage = () => ({
|
|
5545
|
+
ok: false,
|
|
5546
|
+
reason: "root-missing",
|
|
5547
|
+
staleWatches: 121,
|
|
5548
|
+
});
|
|
5549
|
+
const rebuild = vi.fn();
|
|
5550
|
+
watcher.rebuildCoverage = rebuild;
|
|
5551
|
+
const stderrWrite = vi
|
|
5552
|
+
.spyOn(process.stderr, "write")
|
|
5553
|
+
.mockImplementation(() => true);
|
|
5554
|
+
const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
|
|
5555
|
+
let triggerShutdown = () => { };
|
|
5556
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
5557
|
+
runPass,
|
|
5558
|
+
createWatcher: () => watcher,
|
|
5559
|
+
sleep: () => new Promise(() => { }),
|
|
5560
|
+
onShutdownSignal: (handler) => {
|
|
5561
|
+
triggerShutdown = handler;
|
|
5562
|
+
return () => { };
|
|
5563
|
+
},
|
|
5564
|
+
});
|
|
5565
|
+
try {
|
|
5566
|
+
await flushLoopMicrotasks();
|
|
5567
|
+
runPass.mockClear();
|
|
5568
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
5569
|
+
await flushLoopMicrotasks();
|
|
5570
|
+
const stderr = stderrWrite.mock.calls.map(([chunk]) => String(chunk)).join("");
|
|
5571
|
+
expect(stderr).toContain("event-push watcher coverage stale");
|
|
5572
|
+
expect(stderr).toContain("reason=root-missing");
|
|
5573
|
+
expect(stderr).toContain("not pushing deletes");
|
|
5574
|
+
expect(rebuild).toHaveBeenCalled();
|
|
5575
|
+
expect(runPass).toHaveBeenCalled();
|
|
5576
|
+
}
|
|
5577
|
+
finally {
|
|
5578
|
+
triggerShutdown();
|
|
5579
|
+
await loop;
|
|
5580
|
+
stderrWrite.mockRestore();
|
|
5581
|
+
vi.useRealTimers();
|
|
5582
|
+
}
|
|
5583
|
+
});
|
|
5288
5584
|
it("does not dispose a populated watcher after liveness reports inactive", async () => {
|
|
5289
5585
|
vi.useFakeTimers();
|
|
5290
5586
|
const watcher = makeWatcherStub();
|
|
@@ -5388,6 +5684,83 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5388
5684
|
triggerShutdown();
|
|
5389
5685
|
await loop;
|
|
5390
5686
|
});
|
|
5687
|
+
it.each([
|
|
5688
|
+
["serialized", "0"],
|
|
5689
|
+
["parallel coordinator", "1"],
|
|
5690
|
+
])("starts event ingress once after journals are ready while the first %s full pass is unfinished", async (_mode, coordinator) => {
|
|
5691
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5692
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = coordinator;
|
|
5693
|
+
const watcher = makeWatcherStub();
|
|
5694
|
+
const createWatcher = vi.fn(() => watcher);
|
|
5695
|
+
let releaseFullPass = () => { };
|
|
5696
|
+
const fullPass = new Promise((resolve) => {
|
|
5697
|
+
releaseFullPass = resolve;
|
|
5698
|
+
});
|
|
5699
|
+
let activeDurableWriters = 0;
|
|
5700
|
+
let maxDurableWriters = 0;
|
|
5701
|
+
const runPass = vi.fn(async (...args) => {
|
|
5702
|
+
const journalsReady = args[2];
|
|
5703
|
+
activeDurableWriters += 1;
|
|
5704
|
+
maxDurableWriters = Math.max(maxDurableWriters, activeDurableWriters);
|
|
5705
|
+
journalsReady?.();
|
|
5706
|
+
await fullPass;
|
|
5707
|
+
activeDurableWriters -= 1;
|
|
5708
|
+
return completedPassOutcome();
|
|
5709
|
+
});
|
|
5710
|
+
let shutdown = () => { };
|
|
5711
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
5712
|
+
runPass: runPass,
|
|
5713
|
+
runPassSupportsCooperativeCheckpoint: true,
|
|
5714
|
+
createWatcher,
|
|
5715
|
+
sleep: () => new Promise(() => { }),
|
|
5716
|
+
onShutdownSignal: (handler) => {
|
|
5717
|
+
shutdown = handler;
|
|
5718
|
+
return () => { };
|
|
5719
|
+
},
|
|
5720
|
+
});
|
|
5721
|
+
try {
|
|
5722
|
+
await flushLoopMicrotasks(30);
|
|
5723
|
+
expect(runPass).toHaveBeenCalledOnce();
|
|
5724
|
+
expect(createWatcher).toHaveBeenCalledOnce();
|
|
5725
|
+
expect(watcher.started).toBe(true);
|
|
5726
|
+
expect(maxDurableWriters).toBe(1);
|
|
5727
|
+
await flushLoopMicrotasks();
|
|
5728
|
+
expect(runPass).toHaveBeenCalledOnce();
|
|
5729
|
+
}
|
|
5730
|
+
finally {
|
|
5731
|
+
releaseFullPass();
|
|
5732
|
+
shutdown();
|
|
5733
|
+
await loop;
|
|
5734
|
+
if (previousCoordinator === undefined)
|
|
5735
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5736
|
+
else
|
|
5737
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
5738
|
+
}
|
|
5739
|
+
});
|
|
5740
|
+
it.each([
|
|
5741
|
+
["auth-required", AUTH_REQUIRED_PASS_EXIT],
|
|
5742
|
+
["journal initialization failure", 1],
|
|
5743
|
+
])("does not arm event ingress when the first pass exits %s before journal readiness", async (_reason, exitCode) => {
|
|
5744
|
+
const watcher = makeWatcherStub();
|
|
5745
|
+
let shutdown = () => { };
|
|
5746
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
5747
|
+
runPass: vi.fn().mockResolvedValue(exitCode),
|
|
5748
|
+
createWatcher: () => watcher,
|
|
5749
|
+
sleep: () => new Promise(() => { }),
|
|
5750
|
+
onShutdownSignal: (handler) => {
|
|
5751
|
+
shutdown = handler;
|
|
5752
|
+
return () => { };
|
|
5753
|
+
},
|
|
5754
|
+
});
|
|
5755
|
+
try {
|
|
5756
|
+
await flushLoopMicrotasks(30);
|
|
5757
|
+
expect(watcher.started).toBe(false);
|
|
5758
|
+
}
|
|
5759
|
+
finally {
|
|
5760
|
+
shutdown();
|
|
5761
|
+
await expect(loop).resolves.toBe(0);
|
|
5762
|
+
}
|
|
5763
|
+
});
|
|
5391
5764
|
it("pushes a file created before watcher ready through the warm-up catch-up", async () => {
|
|
5392
5765
|
const watcher = makeBatchWatcherStub();
|
|
5393
5766
|
let markReady = () => { };
|
|
@@ -5706,7 +6079,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5706
6079
|
});
|
|
5707
6080
|
await expect(superseded).resolves.toBeUndefined();
|
|
5708
6081
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
|
|
5709
|
-
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":
|
|
6082
|
+
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":2,"source":"receiver","direction":"pull","outcome":"refused"');
|
|
5710
6083
|
releaseHeld();
|
|
5711
6084
|
await heldReceiverWork;
|
|
5712
6085
|
await replacement;
|
|
@@ -6019,7 +6392,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6019
6392
|
triggerShutdown();
|
|
6020
6393
|
await loop;
|
|
6021
6394
|
});
|
|
6022
|
-
it("fast lane: a small watcher batch
|
|
6395
|
+
it("fast lane: a small watcher batch waits for a full pass that has not reached a hand-off checkpoint", async () => {
|
|
6023
6396
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-watch-"));
|
|
6024
6397
|
const relativePath = "companies/indigo/realtime.md";
|
|
6025
6398
|
const absolutePath = path.join(hqRoot, relativePath);
|
|
@@ -6059,15 +6432,17 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6059
6432
|
expect(order).toEqual(["bulk"]);
|
|
6060
6433
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
6061
6434
|
await flushLoopMicrotasks(30);
|
|
6062
|
-
//
|
|
6063
|
-
|
|
6435
|
+
// A full pass that has not reached its durable checkpoint remains the sole
|
|
6436
|
+
// owner. Realtime is promoted at the checkpoint, never run concurrently.
|
|
6437
|
+
expect(order).toEqual(["bulk"]);
|
|
6064
6438
|
releaseBulk();
|
|
6065
|
-
await flushLoopMicrotasks();
|
|
6439
|
+
await flushLoopMicrotasks(30);
|
|
6440
|
+
expect(order).toEqual(["bulk", "fast", "fast"]);
|
|
6066
6441
|
triggerShutdown();
|
|
6067
6442
|
await loop;
|
|
6068
6443
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6069
6444
|
});
|
|
6070
|
-
it("parallel coordinator: a watcher push
|
|
6445
|
+
it("parallel coordinator: a watcher push waits for a full chore without a hand-off checkpoint", async () => {
|
|
6071
6446
|
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6072
6447
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6073
6448
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-watch-"));
|
|
@@ -6110,9 +6485,10 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6110
6485
|
expect(order).toEqual(["chore"]);
|
|
6111
6486
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
6112
6487
|
await flushLoopMicrotasks(30);
|
|
6113
|
-
expect(order).toEqual(["chore"
|
|
6488
|
+
expect(order).toEqual(["chore"]);
|
|
6114
6489
|
releaseBulk();
|
|
6115
|
-
await flushLoopMicrotasks();
|
|
6490
|
+
await flushLoopMicrotasks(30);
|
|
6491
|
+
expect(order).toEqual(["chore", "realtime"]);
|
|
6116
6492
|
triggerShutdown();
|
|
6117
6493
|
await loop;
|
|
6118
6494
|
}
|
|
@@ -6197,7 +6573,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6197
6573
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6198
6574
|
}
|
|
6199
6575
|
});
|
|
6200
|
-
it("parallel coordinator partitions receiver events into fast and slow realtime lanes", async () => {
|
|
6576
|
+
it("parallel coordinator partitions receiver events into fast and slow realtime lanes without overlapping their durable writes", async () => {
|
|
6201
6577
|
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6202
6578
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6203
6579
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-receiver-lanes-"));
|
|
@@ -6268,7 +6644,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6268
6644
|
signal: new AbortController().signal,
|
|
6269
6645
|
});
|
|
6270
6646
|
await flushLoopMicrotasks(20);
|
|
6271
|
-
expect(order).toEqual(["slow-held"
|
|
6647
|
+
expect(order).toEqual(["slow-held"]);
|
|
6272
6648
|
releaseSlow();
|
|
6273
6649
|
await held;
|
|
6274
6650
|
await partitioned;
|
|
@@ -6434,64 +6810,562 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6434
6810
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6435
6811
|
}
|
|
6436
6812
|
});
|
|
6437
|
-
it("
|
|
6438
|
-
const
|
|
6813
|
+
it("bounds a 32-path realtime personal commit on a 700k-row journal and 50k-path tree", async () => {
|
|
6814
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6439
6815
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6440
|
-
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-
|
|
6816
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-production-scale-watch-"));
|
|
6817
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
6818
|
+
const targetPaths = Array.from({ length: 32 }, (_, index) => `personal/watch-load/${String(index).padStart(5, "0")}/target.md`);
|
|
6819
|
+
const files = {};
|
|
6820
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
6821
|
+
files[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6822
|
+
hash: `sha256:history-${index}`,
|
|
6823
|
+
size: index % 4096,
|
|
6824
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6825
|
+
direction: "up",
|
|
6826
|
+
};
|
|
6827
|
+
}
|
|
6828
|
+
for (const target of targetPaths) {
|
|
6829
|
+
files[target] = {
|
|
6830
|
+
hash: `sha256:${target}`,
|
|
6831
|
+
size: 1,
|
|
6832
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6833
|
+
direction: "up",
|
|
6834
|
+
};
|
|
6835
|
+
}
|
|
6836
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
6837
|
+
version: "2",
|
|
6838
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6839
|
+
pulls: [],
|
|
6840
|
+
files,
|
|
6841
|
+
});
|
|
6842
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
6843
|
+
const directory = path.join(watchedRoot, String(index).padStart(5, "0"));
|
|
6844
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
6845
|
+
fs.writeFileSync(path.join(directory, "seed.md"), "seed");
|
|
6846
|
+
}
|
|
6441
6847
|
const watcher = makeBatchWatcherStub();
|
|
6442
|
-
let
|
|
6443
|
-
let
|
|
6444
|
-
const
|
|
6445
|
-
|
|
6446
|
-
|
|
6447
|
-
|
|
6448
|
-
|
|
6449
|
-
|
|
6450
|
-
|
|
6451
|
-
|
|
6452
|
-
|
|
6453
|
-
|
|
6848
|
+
let shutdown = () => { };
|
|
6849
|
+
let committedAt;
|
|
6850
|
+
const runPass = vi.fn(async (argv) => {
|
|
6851
|
+
if (!argv.includes("--scope-path")) {
|
|
6852
|
+
// Production opens the packed journal during the initial full pass,
|
|
6853
|
+
// before the watcher accepts realtime work. That recovered projection
|
|
6854
|
+
// is the one a scoped commit must reuse.
|
|
6855
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
6856
|
+
}
|
|
6857
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
6858
|
+
const scoped = readJournalScoped(PERSONAL_VAULT_JOURNAL_SLUG, targetPaths);
|
|
6859
|
+
const target = targetPaths[0];
|
|
6860
|
+
scoped.files[target] = {
|
|
6861
|
+
...scoped.files[target],
|
|
6862
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
6863
|
+
};
|
|
6864
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, scoped);
|
|
6865
|
+
committedAt = performance.now();
|
|
6866
|
+
}
|
|
6867
|
+
return completedPassOutcome();
|
|
6454
6868
|
});
|
|
6455
|
-
const
|
|
6456
|
-
|
|
6457
|
-
hash: "sha256:journal-small",
|
|
6458
|
-
size: 16,
|
|
6459
|
-
syncedAt: "2026-08-28T00:00:00.000Z",
|
|
6460
|
-
direction: "down",
|
|
6461
|
-
};
|
|
6462
|
-
writeJournal("indigo", journal);
|
|
6463
|
-
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
|
|
6464
|
-
runPass: vi.fn(async () => completedPassOutcome()),
|
|
6869
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6870
|
+
runPass,
|
|
6465
6871
|
createWatcher: () => watcher,
|
|
6466
|
-
createReceiver: ({ syncBatchFn }) => {
|
|
6467
|
-
receiverSync = syncBatchFn;
|
|
6468
|
-
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6469
|
-
},
|
|
6470
6872
|
sleep: () => new Promise(() => { }),
|
|
6471
6873
|
onShutdownSignal: (handler) => {
|
|
6472
|
-
|
|
6874
|
+
shutdown = handler;
|
|
6473
6875
|
return () => { };
|
|
6474
6876
|
},
|
|
6475
6877
|
});
|
|
6476
6878
|
try {
|
|
6477
6879
|
await flushLoopMicrotasks(30);
|
|
6478
|
-
|
|
6479
|
-
|
|
6480
|
-
|
|
6481
|
-
|
|
6482
|
-
|
|
6483
|
-
|
|
6484
|
-
|
|
6485
|
-
|
|
6486
|
-
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
],
|
|
6493
|
-
|
|
6494
|
-
|
|
6880
|
+
runPass.mockClear();
|
|
6881
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
6882
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
6883
|
+
eventLoopDelay.enable();
|
|
6884
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
6885
|
+
const submittedAt = performance.now();
|
|
6886
|
+
let timerDelayMs;
|
|
6887
|
+
const eventLoopTurn = new Promise((resolve) => {
|
|
6888
|
+
setTimeout(() => {
|
|
6889
|
+
timerDelayMs = performance.now() - submittedAt;
|
|
6890
|
+
resolve();
|
|
6891
|
+
}, 0);
|
|
6892
|
+
});
|
|
6893
|
+
const batch = {
|
|
6894
|
+
paths: new Map(targetPaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
6895
|
+
changes: new Map(targetPaths.map((relativePath) => [
|
|
6896
|
+
path.join(hqRoot, relativePath),
|
|
6897
|
+
{ kind: "change" },
|
|
6898
|
+
])),
|
|
6899
|
+
};
|
|
6900
|
+
watcher.emit(targetPaths[0], batch);
|
|
6901
|
+
await vi.waitFor(() => expect(committedAt).toBeDefined(), { timeout: 30_000, interval: 10 });
|
|
6902
|
+
await eventLoopTurn;
|
|
6903
|
+
eventLoopDelay.disable();
|
|
6904
|
+
expect(committedAt - submittedAt).toBeLessThan(1_000);
|
|
6905
|
+
expect(Number(eventLoopDelay.max) / 1_000_000).toBeLessThan(100);
|
|
6906
|
+
expect(timerDelayMs).toBeLessThan(100);
|
|
6907
|
+
expect(getStateStoreSnapshotRecoveryCountForTest()).toBe(0);
|
|
6908
|
+
expect(runPass.mock.calls.filter(([argv]) => argv.includes("--scope-path"))).toHaveLength(1);
|
|
6909
|
+
}
|
|
6910
|
+
finally {
|
|
6911
|
+
shutdown();
|
|
6912
|
+
await loop;
|
|
6913
|
+
if (previousCoordinator === undefined)
|
|
6914
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6915
|
+
else
|
|
6916
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6917
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6918
|
+
}
|
|
6919
|
+
}, 180_000);
|
|
6920
|
+
it("keeps an area-authoritative 700k-row personal aggregate hot across alternating realtime and company batches", async () => {
|
|
6921
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6922
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6923
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-watch-"));
|
|
6924
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
6925
|
+
const personalPath = "personal/watch-load/00000/target.md";
|
|
6926
|
+
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
6927
|
+
const companyKey = "watch-load/00000/target.md";
|
|
6928
|
+
const personalFiles = {};
|
|
6929
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
6930
|
+
personalFiles[`personal/history-${index % 5}/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6931
|
+
hash: `sha256:history-${index}`,
|
|
6932
|
+
size: index % 4096,
|
|
6933
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6934
|
+
direction: "up",
|
|
6935
|
+
};
|
|
6936
|
+
}
|
|
6937
|
+
personalFiles[personalPath] = {
|
|
6938
|
+
hash: "sha256:personal-target",
|
|
6939
|
+
size: 1,
|
|
6940
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6941
|
+
direction: "up",
|
|
6942
|
+
};
|
|
6943
|
+
const resolver = new AreaResolver({
|
|
6944
|
+
format: 1,
|
|
6945
|
+
routerVersion: 1,
|
|
6946
|
+
areas: [
|
|
6947
|
+
...Array.from({ length: 5 }, (_, index) => ({
|
|
6948
|
+
id: `root:history-${index}`,
|
|
6949
|
+
prefixes: [`personal/history-${index}`],
|
|
6950
|
+
paths: [],
|
|
6951
|
+
})),
|
|
6952
|
+
{ id: "root:watch", prefixes: ["personal/watch-load"], paths: [] },
|
|
6953
|
+
{ id: "root:@files", prefixes: [], paths: [], rootFiles: true },
|
|
6954
|
+
],
|
|
6955
|
+
});
|
|
6956
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
6957
|
+
version: "2",
|
|
6958
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6959
|
+
pulls: [],
|
|
6960
|
+
files: {},
|
|
6961
|
+
});
|
|
6962
|
+
writeJournal("indigo", {
|
|
6963
|
+
version: "2",
|
|
6964
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6965
|
+
pulls: [],
|
|
6966
|
+
files: {
|
|
6967
|
+
[companyKey]: {
|
|
6968
|
+
hash: "sha256:company-target",
|
|
6969
|
+
size: 1,
|
|
6970
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6971
|
+
direction: "up",
|
|
6972
|
+
},
|
|
6973
|
+
},
|
|
6974
|
+
});
|
|
6975
|
+
const migration = new AreaLedgerMigration({
|
|
6976
|
+
rootDir: __runnerTestStateDir,
|
|
6977
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6978
|
+
resolver,
|
|
6979
|
+
legacyStore: openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
|
|
6980
|
+
});
|
|
6981
|
+
expect(migration.cutOver(120_000).phase).toBe("area-authoritative");
|
|
6982
|
+
migration.dispose();
|
|
6983
|
+
// The migration state itself is covered by the area-ledger migration
|
|
6984
|
+
// suite. Seed the already-authoritative production shape in one durable
|
|
6985
|
+
// transaction so this watch-loop test measures repeated realtime reads,
|
|
6986
|
+
// rather than spending most of its worker budget replaying a one-time
|
|
6987
|
+
// legacy-to-area copy.
|
|
6988
|
+
openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG).replaceState({
|
|
6989
|
+
journal: {
|
|
6990
|
+
version: "2",
|
|
6991
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6992
|
+
pulls: [],
|
|
6993
|
+
files: personalFiles,
|
|
6994
|
+
},
|
|
6995
|
+
});
|
|
6996
|
+
AreaLedger.open({
|
|
6997
|
+
rootDir: __runnerTestStateDir,
|
|
6998
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6999
|
+
layoutEpoch: 1,
|
|
7000
|
+
areaIds: resolver.layout.areas.map((area) => area.id),
|
|
7001
|
+
resolveArea: (key) => resolver.resolve(key),
|
|
7002
|
+
}).append({
|
|
7003
|
+
upserts: personalFiles,
|
|
7004
|
+
metadata: { lastSync: "2026-09-16T21:30:00.000Z", pulls: [] },
|
|
7005
|
+
});
|
|
7006
|
+
fs.mkdirSync(watchedRoot, { recursive: true });
|
|
7007
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
7008
|
+
fs.writeFileSync(path.join(watchedRoot, `seed-${String(index).padStart(5, "0")}.md`), "seed");
|
|
7009
|
+
}
|
|
7010
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
|
|
7011
|
+
fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
|
|
7012
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
|
|
7013
|
+
fs.writeFileSync(path.join(hqRoot, companyPath), "company");
|
|
7014
|
+
const watcher = makeBatchWatcherStub();
|
|
7015
|
+
const sleep = makeSteppableSleep();
|
|
7016
|
+
let shutdown = () => { };
|
|
7017
|
+
let isUntrackedDelete = (_relativePath) => false;
|
|
7018
|
+
const commits = [];
|
|
7019
|
+
let submittedAt = 0;
|
|
7020
|
+
const aggregatePrime = vi.spyOn(AreaLedger.prototype, "primeAggregateCache");
|
|
7021
|
+
const runPass = vi.fn(async (argv) => {
|
|
7022
|
+
if (!argv.includes("--scope-path")) {
|
|
7023
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
7024
|
+
readJournal("indigo");
|
|
7025
|
+
}
|
|
7026
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
7027
|
+
const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
|
|
7028
|
+
const key = slug === "indigo" ? companyKey : personalPath;
|
|
7029
|
+
const scoped = readJournalScoped(slug, [key]);
|
|
7030
|
+
scoped.files[key] = {
|
|
7031
|
+
...scoped.files[key],
|
|
7032
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
7033
|
+
};
|
|
7034
|
+
writeJournal(slug, scoped);
|
|
7035
|
+
commits.push({ slug, elapsedMs: performance.now() - submittedAt });
|
|
7036
|
+
}
|
|
7037
|
+
return completedPassOutcome();
|
|
7038
|
+
});
|
|
7039
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
7040
|
+
runPass,
|
|
7041
|
+
createWatcher: (options) => {
|
|
7042
|
+
isUntrackedDelete = options.isUntrackedDelete;
|
|
7043
|
+
return watcher;
|
|
7044
|
+
},
|
|
7045
|
+
startEventSync: vi.fn().mockResolvedValue(null),
|
|
7046
|
+
sleep: sleep.sleep,
|
|
7047
|
+
onShutdownSignal: (handler) => {
|
|
7048
|
+
shutdown = handler;
|
|
7049
|
+
return () => { };
|
|
7050
|
+
},
|
|
7051
|
+
});
|
|
7052
|
+
const emitAndWait = async (relativePaths) => {
|
|
7053
|
+
const before = commits.length;
|
|
7054
|
+
if (relativePaths.includes(personalPath)) {
|
|
7055
|
+
// This is the same non-blocking watcher callback which starts an
|
|
7056
|
+
// openJournalStoreSessionAsync before a delete episode is drained.
|
|
7057
|
+
// It is intentionally issued before each realtime batch: a completed
|
|
7058
|
+
// watch pass must not have thrown away the warmed area aggregate.
|
|
7059
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
7060
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
7061
|
+
}
|
|
7062
|
+
submittedAt = performance.now();
|
|
7063
|
+
const batch = {
|
|
7064
|
+
paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
7065
|
+
changes: new Map(relativePaths.map((relativePath) => [
|
|
7066
|
+
path.join(hqRoot, relativePath),
|
|
7067
|
+
{ kind: "change" },
|
|
7068
|
+
])),
|
|
7069
|
+
};
|
|
7070
|
+
watcher.emit(relativePaths[0], batch);
|
|
7071
|
+
await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
|
|
7072
|
+
timeout: 30_000,
|
|
7073
|
+
interval: 10,
|
|
7074
|
+
});
|
|
7075
|
+
// `commits` is observed inside the injected pass, before the runner has
|
|
7076
|
+
// completed its pass-final cleanup. Let that cleanup run before the next
|
|
7077
|
+
// watcher episode so the assertion observes the cache lifetime at the
|
|
7078
|
+
// same boundary as a real completed watch pass.
|
|
7079
|
+
await flushLoopMicrotasks(30);
|
|
7080
|
+
};
|
|
7081
|
+
try {
|
|
7082
|
+
await flushLoopMicrotasks(30);
|
|
7083
|
+
runPass.mockClear();
|
|
7084
|
+
// The initial runner pass is the one permitted cold open. Every later
|
|
7085
|
+
// scoped batch must reuse that authoritative aggregate. Warm the same
|
|
7086
|
+
// yielding watcher-session path that production uses before its first
|
|
7087
|
+
// delete episode, then measure only post-warmup primes.
|
|
7088
|
+
aggregatePrime.mockClear();
|
|
7089
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
7090
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
7091
|
+
// The initial full pass may already have opened this bridge (zero new
|
|
7092
|
+
// primes), or this watcher callback may perform the one permitted cold
|
|
7093
|
+
// warm-up. The post-poll assertion below is the regression boundary.
|
|
7094
|
+
expect(aggregatePrime.mock.calls.length).toBeLessThanOrEqual(1);
|
|
7095
|
+
aggregatePrime.mockClear();
|
|
7096
|
+
// Reproduce the normal poll boundary observed on the outposts. On the
|
|
7097
|
+
// current implementation it drops every cached area bridge even though
|
|
7098
|
+
// the watcher and runner remain alive.
|
|
7099
|
+
sleep.tick();
|
|
7100
|
+
await flushLoopMicrotasks(30);
|
|
7101
|
+
forceGarbageCollectionForTest();
|
|
7102
|
+
const retainedAfterWarmupBytes = retainedProcessBytes();
|
|
7103
|
+
const postWarmCpuStart = process.cpuUsage();
|
|
7104
|
+
const postWarmStartedAt = performance.now();
|
|
7105
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
7106
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
7107
|
+
eventLoopDelay.enable();
|
|
7108
|
+
await emitAndWait([personalPath, companyPath]);
|
|
7109
|
+
for (let round = 0; round < 3; round += 1) {
|
|
7110
|
+
await emitAndWait([personalPath]);
|
|
7111
|
+
await emitAndWait([companyPath]);
|
|
7112
|
+
}
|
|
7113
|
+
eventLoopDelay.disable();
|
|
7114
|
+
const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
|
|
7115
|
+
const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
|
|
7116
|
+
const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
|
|
7117
|
+
// A first area delta briefly retains the reverse-shadow mirror to
|
|
7118
|
+
// amortize a burst. Retained memory means after that documented linger
|
|
7119
|
+
// has released its second full-vault projection, not during it.
|
|
7120
|
+
await new Promise((resolve) => setTimeout(resolve, AREA_LEGACY_MIRROR_LINGER_MS + 25));
|
|
7121
|
+
forceGarbageCollectionForTest();
|
|
7122
|
+
const retainedAfterChurnBytes = retainedProcessBytes();
|
|
7123
|
+
const retainedGrowthBytes = retainedAfterChurnBytes - retainedAfterWarmupBytes;
|
|
7124
|
+
const postWarmCpu = process.cpuUsage(postWarmCpuStart);
|
|
7125
|
+
const postWarmCpuPercent = ((postWarmCpu.user + postWarmCpu.system) /
|
|
7126
|
+
((performance.now() - postWarmStartedAt) * 1_000)) * 100;
|
|
7127
|
+
console.info({
|
|
7128
|
+
aggregatePrimeCount: aggregatePrime.mock.calls.length,
|
|
7129
|
+
recoveryCount,
|
|
7130
|
+
maxDelayMs,
|
|
7131
|
+
maxCommitMs,
|
|
7132
|
+
postWarmCpuPercent,
|
|
7133
|
+
retainedAfterWarmupBytes,
|
|
7134
|
+
retainedAfterChurnBytes,
|
|
7135
|
+
retainedGrowthBytes,
|
|
7136
|
+
commits: commits.length,
|
|
7137
|
+
});
|
|
7138
|
+
expect(aggregatePrime).not.toHaveBeenCalled();
|
|
7139
|
+
// The first area delta opens its reverse-shadow v3 mirror once; its
|
|
7140
|
+
// linger keeps all later scoped commits off snapshot recovery. This is
|
|
7141
|
+
// deliberately exact so a second cold bridge cannot hide as a cache hit.
|
|
7142
|
+
expect.soft(recoveryCount).toBe(1);
|
|
7143
|
+
// The production-shape run remains bounded under the coverage job's
|
|
7144
|
+
// concurrent workers: CI measured 1,090.52 ms of timer delay and
|
|
7145
|
+
// 1,087.79 ms to commit. 1.5 s retains a concrete latency ceiling while
|
|
7146
|
+
// leaving headroom for that instrumentation, rather than making the
|
|
7147
|
+
// assertion depend on a single host's scheduling jitter.
|
|
7148
|
+
expect.soft(maxDelayMs).toBeLessThan(1_500);
|
|
7149
|
+
expect.soft(maxCommitMs).toBeLessThan(1_500);
|
|
7150
|
+
expect.soft(retainedGrowthBytes).toBeLessThan(32 * 1024 * 1024);
|
|
7151
|
+
expect(commits.map((commit) => commit.slug)).toEqual([
|
|
7152
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7153
|
+
"indigo",
|
|
7154
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7155
|
+
"indigo",
|
|
7156
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7157
|
+
"indigo",
|
|
7158
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7159
|
+
"indigo",
|
|
7160
|
+
]);
|
|
7161
|
+
}
|
|
7162
|
+
finally {
|
|
7163
|
+
aggregatePrime.mockRestore();
|
|
7164
|
+
shutdown();
|
|
7165
|
+
await loop;
|
|
7166
|
+
if (previousCoordinator === undefined)
|
|
7167
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7168
|
+
else
|
|
7169
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
7170
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
7171
|
+
}
|
|
7172
|
+
}, 180_000);
|
|
7173
|
+
it("keeps warmed personal and company journals hot across alternating realtime batches", async () => {
|
|
7174
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7175
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
7176
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-legacy-watch-"));
|
|
7177
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
7178
|
+
const personalPath = "personal/watch-load/00000/target.md";
|
|
7179
|
+
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
7180
|
+
const companyKey = "watch-load/00000/target.md";
|
|
7181
|
+
const personalFiles = {};
|
|
7182
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
7183
|
+
personalFiles[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
7184
|
+
hash: `sha256:history-${index}`,
|
|
7185
|
+
size: index % 4096,
|
|
7186
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7187
|
+
direction: "up",
|
|
7188
|
+
};
|
|
7189
|
+
}
|
|
7190
|
+
personalFiles[personalPath] = {
|
|
7191
|
+
hash: "sha256:personal-target",
|
|
7192
|
+
size: 1,
|
|
7193
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7194
|
+
direction: "up",
|
|
7195
|
+
};
|
|
7196
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
7197
|
+
version: "2",
|
|
7198
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
7199
|
+
pulls: [],
|
|
7200
|
+
files: personalFiles,
|
|
7201
|
+
});
|
|
7202
|
+
writeJournal("indigo", {
|
|
7203
|
+
version: "2",
|
|
7204
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
7205
|
+
pulls: [],
|
|
7206
|
+
files: {
|
|
7207
|
+
[companyKey]: {
|
|
7208
|
+
hash: "sha256:company-target",
|
|
7209
|
+
size: 1,
|
|
7210
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7211
|
+
direction: "up",
|
|
7212
|
+
},
|
|
7213
|
+
},
|
|
7214
|
+
});
|
|
7215
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
7216
|
+
const directory = path.join(watchedRoot, String(index).padStart(5, "0"));
|
|
7217
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
7218
|
+
fs.writeFileSync(path.join(directory, "seed.md"), "seed");
|
|
7219
|
+
}
|
|
7220
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
|
|
7221
|
+
fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
|
|
7222
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
|
|
7223
|
+
fs.writeFileSync(path.join(hqRoot, companyPath), "company");
|
|
7224
|
+
const watcher = makeBatchWatcherStub();
|
|
7225
|
+
let shutdown = () => { };
|
|
7226
|
+
const commits = [];
|
|
7227
|
+
let submittedAt = 0;
|
|
7228
|
+
const runPass = vi.fn(async (argv) => {
|
|
7229
|
+
if (!argv.includes("--scope-path")) {
|
|
7230
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
7231
|
+
readJournal("indigo");
|
|
7232
|
+
}
|
|
7233
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
7234
|
+
const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
|
|
7235
|
+
const key = slug === "indigo" ? companyKey : personalPath;
|
|
7236
|
+
const scoped = readJournalScoped(slug, [key]);
|
|
7237
|
+
scoped.files[key] = {
|
|
7238
|
+
...scoped.files[key],
|
|
7239
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
7240
|
+
};
|
|
7241
|
+
writeJournal(slug, scoped);
|
|
7242
|
+
commits.push({ slug, elapsedMs: performance.now() - submittedAt });
|
|
7243
|
+
}
|
|
7244
|
+
return completedPassOutcome();
|
|
7245
|
+
});
|
|
7246
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
7247
|
+
runPass,
|
|
7248
|
+
createWatcher: () => watcher,
|
|
7249
|
+
sleep: () => new Promise(() => { }),
|
|
7250
|
+
onShutdownSignal: (handler) => {
|
|
7251
|
+
shutdown = handler;
|
|
7252
|
+
return () => { };
|
|
7253
|
+
},
|
|
7254
|
+
});
|
|
7255
|
+
const emitAndWait = async (relativePaths) => {
|
|
7256
|
+
const before = commits.length;
|
|
7257
|
+
submittedAt = performance.now();
|
|
7258
|
+
const batch = {
|
|
7259
|
+
paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
7260
|
+
changes: new Map(relativePaths.map((relativePath) => [
|
|
7261
|
+
path.join(hqRoot, relativePath),
|
|
7262
|
+
{ kind: "change" },
|
|
7263
|
+
])),
|
|
7264
|
+
};
|
|
7265
|
+
watcher.emit(relativePaths[0], batch);
|
|
7266
|
+
await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
|
|
7267
|
+
timeout: 30_000,
|
|
7268
|
+
interval: 10,
|
|
7269
|
+
});
|
|
7270
|
+
};
|
|
7271
|
+
try {
|
|
7272
|
+
await flushLoopMicrotasks(30);
|
|
7273
|
+
runPass.mockClear();
|
|
7274
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
7275
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
7276
|
+
eventLoopDelay.enable();
|
|
7277
|
+
await emitAndWait([personalPath, companyPath]);
|
|
7278
|
+
for (let round = 0; round < 3; round += 1) {
|
|
7279
|
+
await emitAndWait([personalPath]);
|
|
7280
|
+
await emitAndWait([companyPath]);
|
|
7281
|
+
}
|
|
7282
|
+
eventLoopDelay.disable();
|
|
7283
|
+
const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
|
|
7284
|
+
const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
|
|
7285
|
+
const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
|
|
7286
|
+
console.info({ recoveryCount, maxDelayMs, maxCommitMs, commits: commits.length });
|
|
7287
|
+
expect.soft(recoveryCount).toBe(0);
|
|
7288
|
+
expect.soft(maxDelayMs).toBeLessThan(100);
|
|
7289
|
+
expect.soft(maxCommitMs).toBeLessThan(1_000);
|
|
7290
|
+
expect(commits.map((commit) => commit.slug)).toEqual([
|
|
7291
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7292
|
+
"indigo",
|
|
7293
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7294
|
+
"indigo",
|
|
7295
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7296
|
+
"indigo",
|
|
7297
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7298
|
+
"indigo",
|
|
7299
|
+
]);
|
|
7300
|
+
}
|
|
7301
|
+
finally {
|
|
7302
|
+
shutdown();
|
|
7303
|
+
await loop;
|
|
7304
|
+
if (previousCoordinator === undefined)
|
|
7305
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7306
|
+
else
|
|
7307
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
7308
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
7309
|
+
}
|
|
7310
|
+
}, 180_000);
|
|
7311
|
+
it("fast lane: prefers declared receiver sizes and falls back to the journal", async () => {
|
|
7312
|
+
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7313
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
7314
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-receiver-declared-size-"));
|
|
7315
|
+
const watcher = makeBatchWatcherStub();
|
|
7316
|
+
let receiverSync;
|
|
7317
|
+
let triggerShutdown = () => { };
|
|
7318
|
+
const event = (relativePath, sizeBytes) => ({
|
|
7319
|
+
kind: "upsert",
|
|
7320
|
+
relativePath,
|
|
7321
|
+
contentHash: "sha256:remote",
|
|
7322
|
+
mtime: "2026-08-28T00:00:00.000Z",
|
|
7323
|
+
...(sizeBytes === undefined ? {} : { sizeBytes }),
|
|
7324
|
+
originDeviceId: "peer-device",
|
|
7325
|
+
originTenantId: "tenant-indigo",
|
|
7326
|
+
sequenceNumber: 1,
|
|
7327
|
+
eventTimestamp: "2026-08-28T00:00:00.000Z",
|
|
7328
|
+
});
|
|
7329
|
+
const journal = readJournal("indigo");
|
|
7330
|
+
journal.files["journal-small.md"] = {
|
|
7331
|
+
hash: "sha256:journal-small",
|
|
7332
|
+
size: 16,
|
|
7333
|
+
syncedAt: "2026-08-28T00:00:00.000Z",
|
|
7334
|
+
direction: "down",
|
|
7335
|
+
};
|
|
7336
|
+
writeJournal("indigo", journal);
|
|
7337
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
|
|
7338
|
+
runPass: vi.fn(async () => completedPassOutcome()),
|
|
7339
|
+
createWatcher: () => watcher,
|
|
7340
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
7341
|
+
receiverSync = syncBatchFn;
|
|
7342
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
7343
|
+
},
|
|
7344
|
+
sleep: () => new Promise(() => { }),
|
|
7345
|
+
onShutdownSignal: (handler) => {
|
|
7346
|
+
triggerShutdown = handler;
|
|
7347
|
+
return () => { };
|
|
7348
|
+
},
|
|
7349
|
+
});
|
|
7350
|
+
try {
|
|
7351
|
+
await flushLoopMicrotasks(30);
|
|
7352
|
+
const batches = receiverSync.partition([
|
|
7353
|
+
event("companies/indigo/declared-small.md", 16),
|
|
7354
|
+
event("companies/indigo/declared-large.md", DEFAULT_FAST_LANE_MAX_FILE_BYTES + 1),
|
|
7355
|
+
event("companies/indigo/journal-small.md"),
|
|
7356
|
+
event("companies/indigo/unknown.md"),
|
|
7357
|
+
]);
|
|
7358
|
+
expect(batches.map((batch) => batch.map((event) => event.relativePath))).toEqual([
|
|
7359
|
+
[
|
|
7360
|
+
"companies/indigo/declared-small.md",
|
|
7361
|
+
"companies/indigo/journal-small.md",
|
|
7362
|
+
],
|
|
7363
|
+
[
|
|
7364
|
+
"companies/indigo/declared-large.md",
|
|
7365
|
+
"companies/indigo/unknown.md",
|
|
7366
|
+
],
|
|
7367
|
+
]);
|
|
7368
|
+
}
|
|
6495
7369
|
finally {
|
|
6496
7370
|
triggerShutdown();
|
|
6497
7371
|
await loop;
|
|
@@ -6609,19 +7483,19 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6609
7483
|
]),
|
|
6610
7484
|
});
|
|
6611
7485
|
await flushLoopMicrotasks(30);
|
|
6612
|
-
//
|
|
6613
|
-
//
|
|
6614
|
-
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(
|
|
6615
|
-
expect(calls.flat().join(" ")).toContain("realtime.md");
|
|
7486
|
+
// The full pass owns durable state until a checkpoint. Its queued realtime
|
|
7487
|
+
// residue is promoted at that hand-off instead of overlapping the pass.
|
|
7488
|
+
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(0);
|
|
6616
7489
|
releaseBulk();
|
|
6617
7490
|
await flushLoopMicrotasks(30);
|
|
6618
7491
|
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(2);
|
|
7492
|
+
expect(calls.flat().join(" ")).toContain("realtime.md");
|
|
6619
7493
|
expect(calls.flat().join(" ")).toContain("session-0.log");
|
|
6620
7494
|
triggerShutdown();
|
|
6621
7495
|
await loop;
|
|
6622
7496
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6623
7497
|
});
|
|
6624
|
-
it("fast lane: a small receiver pull
|
|
7498
|
+
it("fast lane: a small receiver pull waits for a slow full pass without a hand-off checkpoint", async () => {
|
|
6625
7499
|
const watcher = makeWatcherStub();
|
|
6626
7500
|
let triggerShutdown = () => { };
|
|
6627
7501
|
let receiverSync;
|
|
@@ -6666,7 +7540,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6666
7540
|
watcher.emit();
|
|
6667
7541
|
await flushLoopMicrotasks();
|
|
6668
7542
|
expect(order).toEqual(["bulk"]);
|
|
6669
|
-
|
|
7543
|
+
const receiverPull = receiverSync({
|
|
6670
7544
|
events: [{
|
|
6671
7545
|
kind: "upsert", relativePath: "companies/acme/realtime.md", contentHash: "sha256:x",
|
|
6672
7546
|
mtime: "2026-08-26T00:00:00.000Z", originDeviceId: "peer", originTenantId: "acme",
|
|
@@ -6674,8 +7548,11 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6674
7548
|
}],
|
|
6675
7549
|
signal: new AbortController().signal,
|
|
6676
7550
|
});
|
|
6677
|
-
|
|
7551
|
+
await flushLoopMicrotasks(20);
|
|
7552
|
+
expect(order).toEqual(["bulk"]);
|
|
6678
7553
|
releaseBulk();
|
|
7554
|
+
await receiverPull;
|
|
7555
|
+
expect(order).toEqual(["bulk", "fast-pull"]);
|
|
6679
7556
|
await flushLoopMicrotasks();
|
|
6680
7557
|
triggerShutdown();
|
|
6681
7558
|
await loop;
|
|
@@ -6719,12 +7596,14 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6719
7596
|
await flushLoopMicrotasks();
|
|
6720
7597
|
watcher.emit();
|
|
6721
7598
|
await flushLoopMicrotasks();
|
|
7599
|
+
releaseBulk();
|
|
7600
|
+
// The full chore has no explicit safe checkpoint in this stub, so the
|
|
7601
|
+
// realtime pass begins only after the chore has released ownership.
|
|
6722
7602
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
6723
7603
|
await fastStarted;
|
|
6724
|
-
releaseBulk();
|
|
6725
7604
|
await flushLoopMicrotasks(20);
|
|
6726
|
-
// The
|
|
6727
|
-
// sync/rescue must not acquire the
|
|
7605
|
+
// The realtime pass owns the root lock until its own durable work ends. A
|
|
7606
|
+
// second sync/rescue must not acquire the lock in this interval.
|
|
6728
7607
|
expect(fs.existsSync(lockPathFor(hqRoot))).toBe(true);
|
|
6729
7608
|
releaseFast();
|
|
6730
7609
|
await flushLoopMicrotasks(20);
|
|
@@ -6899,59 +7778,68 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6899
7778
|
return () => { };
|
|
6900
7779
|
},
|
|
6901
7780
|
});
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
const
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
6933
|
-
|
|
6934
|
-
|
|
6935
|
-
|
|
6936
|
-
|
|
7781
|
+
try {
|
|
7782
|
+
await flushLoopMicrotasks();
|
|
7783
|
+
watcher.emit();
|
|
7784
|
+
await flushLoopMicrotasks();
|
|
7785
|
+
runPass.mockClear();
|
|
7786
|
+
const events = [
|
|
7787
|
+
...Array.from({ length: 17 }, (_, index) => ({
|
|
7788
|
+
kind: "upsert",
|
|
7789
|
+
relativePath: `companies/acme/a-${index}.md`,
|
|
7790
|
+
contentHash: "sha256:x",
|
|
7791
|
+
mtime: "2026-08-26T00:00:00.000Z",
|
|
7792
|
+
originDeviceId: "peer",
|
|
7793
|
+
originTenantId: "acme",
|
|
7794
|
+
sequenceNumber: index,
|
|
7795
|
+
eventTimestamp: "2026-08-26T00:00:00.000Z",
|
|
7796
|
+
})),
|
|
7797
|
+
...Array.from({ length: 16 }, (_, index) => ({
|
|
7798
|
+
kind: "upsert",
|
|
7799
|
+
relativePath: `companies/beta/b-${index}.md`,
|
|
7800
|
+
contentHash: "sha256:x",
|
|
7801
|
+
mtime: "2026-08-26T00:00:00.000Z",
|
|
7802
|
+
originDeviceId: "peer",
|
|
7803
|
+
originTenantId: "beta",
|
|
7804
|
+
sequenceNumber: index + 17,
|
|
7805
|
+
eventTimestamp: "2026-08-26T00:00:00.000Z",
|
|
7806
|
+
})),
|
|
7807
|
+
];
|
|
7808
|
+
for (const [slug, prefix, count] of [["acme", "a", 17], ["beta", "b", 16]]) {
|
|
7809
|
+
const journal = readJournal(slug);
|
|
7810
|
+
for (let index = 0; index < count; index++) {
|
|
7811
|
+
journal.files[`${prefix}-${index}.md`] = {
|
|
7812
|
+
hash: "sha256:x",
|
|
7813
|
+
size: 16,
|
|
7814
|
+
syncedAt: "2026-08-26T00:00:00.000Z",
|
|
7815
|
+
direction: "down",
|
|
7816
|
+
};
|
|
7817
|
+
}
|
|
7818
|
+
writeJournal(slug, journal);
|
|
6937
7819
|
}
|
|
6938
|
-
|
|
7820
|
+
const receiverDrain = receiverSync({
|
|
7821
|
+
events,
|
|
7822
|
+
signal: new AbortController().signal,
|
|
7823
|
+
});
|
|
7824
|
+
await flushLoopMicrotasks(20);
|
|
7825
|
+
// The running full chore has not yet yielded a durable hand-off, so no
|
|
7826
|
+
// receiver work may overlap it. Once it releases ownership, the
|
|
7827
|
+
// scheduler admits each route's bounded work before the residual slow
|
|
7828
|
+
// beta batch (acme fast, beta fast, then beta slow).
|
|
7829
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
7830
|
+
releaseBulk();
|
|
7831
|
+
await receiverDrain;
|
|
7832
|
+
const scopedCalls = runPass.mock.calls.filter(([argv]) => argv.includes("--scope-path"));
|
|
7833
|
+
expect(scopedCalls).toHaveLength(3);
|
|
7834
|
+
expect(scopedCalls.flat().join(" ")).toContain("a-16.md");
|
|
7835
|
+
expect(scopedCalls.flat().join(" ")).toContain("b-14.md");
|
|
7836
|
+
}
|
|
7837
|
+
finally {
|
|
7838
|
+
releaseBulk();
|
|
7839
|
+
triggerShutdown();
|
|
7840
|
+
await loop;
|
|
7841
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6939
7842
|
}
|
|
6940
|
-
const receiverDrain = receiverSync({
|
|
6941
|
-
events,
|
|
6942
|
-
signal: new AbortController().signal,
|
|
6943
|
-
});
|
|
6944
|
-
await flushLoopMicrotasks(20);
|
|
6945
|
-
// The 33-event mixed delivery peels its first 32 bounded paths into the
|
|
6946
|
-
// fast lane (17 acme + 15 beta); the final beta event remains slow.
|
|
6947
|
-
expect(runPass).toHaveBeenCalledTimes(2);
|
|
6948
|
-
expect(runPass.mock.calls.flat().join(" ")).toContain("a-16.md");
|
|
6949
|
-
expect(runPass.mock.calls.flat().join(" ")).toContain("b-14.md");
|
|
6950
|
-
releaseBulk();
|
|
6951
|
-
await receiverDrain;
|
|
6952
|
-
triggerShutdown();
|
|
6953
|
-
await loop;
|
|
6954
|
-
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6955
7843
|
});
|
|
6956
7844
|
it("fast lane: queues slow work behind a receiver pass that owns the root lock", async () => {
|
|
6957
7845
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-slow-queue-"));
|
|
@@ -7123,9 +8011,12 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7123
8011
|
await flushLoopMicrotasks();
|
|
7124
8012
|
watcher.emit(paths[0][1], { paths: new Map(paths) });
|
|
7125
8013
|
await flushLoopMicrotasks(30);
|
|
7126
|
-
|
|
8014
|
+
// The full pass has not reached a durable checkpoint, so it remains the
|
|
8015
|
+
// only journal writer. The scoped realtime pass is admitted immediately
|
|
8016
|
+
// after this safe hand-off, preserving both sets of rows.
|
|
8017
|
+
expect(Object.keys(readJournal("indigo").files)).toHaveLength(0);
|
|
7127
8018
|
releaseBulk();
|
|
7128
|
-
await flushLoopMicrotasks();
|
|
8019
|
+
await flushLoopMicrotasks(30);
|
|
7129
8020
|
const journal = readJournal("indigo").files;
|
|
7130
8021
|
expect(Object.keys(journal)).toHaveLength(paths.length * 2);
|
|
7131
8022
|
for (let index = 0; index < paths.length; index++) {
|
|
@@ -7177,10 +8068,12 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7177
8068
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
7178
8069
|
}
|
|
7179
8070
|
await flushLoopMicrotasks(60);
|
|
7180
|
-
|
|
8071
|
+
// A full pass without a checkpoint remains the sole mutable-state owner.
|
|
8072
|
+
// The queued realtime batches run immediately after it completes.
|
|
8073
|
+
expect(completed).toEqual([]);
|
|
7181
8074
|
releaseBulk();
|
|
7182
|
-
await flushLoopMicrotasks();
|
|
7183
|
-
expect(completed).
|
|
8075
|
+
await flushLoopMicrotasks(60);
|
|
8076
|
+
expect(completed).toEqual(["bulk", "fast", "fast", "fast", "fast", "fast"]);
|
|
7184
8077
|
triggerShutdown();
|
|
7185
8078
|
await loop;
|
|
7186
8079
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
@@ -7414,18 +8307,33 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7414
8307
|
// No pending FakeClock timers left behind by the driver's debounce.
|
|
7415
8308
|
expect(clock.pendingTimerCount()).toBe(0);
|
|
7416
8309
|
});
|
|
7417
|
-
it("a
|
|
8310
|
+
it("a non-auth pass error keeps the watch loop alive for the next poll", async () => {
|
|
7418
8311
|
const watcher = makeWatcherStub();
|
|
7419
8312
|
const runPass = vi.fn().mockResolvedValue(1);
|
|
7420
|
-
const
|
|
8313
|
+
const sleep = makeSteppableSleep();
|
|
8314
|
+
let triggerShutdown = () => { };
|
|
8315
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7421
8316
|
runPass,
|
|
7422
8317
|
clock: new FakeClock(),
|
|
7423
8318
|
createWatcher: () => watcher,
|
|
7424
|
-
sleep:
|
|
7425
|
-
onShutdownSignal: () =>
|
|
8319
|
+
sleep: sleep.sleep,
|
|
8320
|
+
onShutdownSignal: (handler) => {
|
|
8321
|
+
triggerShutdown = handler;
|
|
8322
|
+
return () => { };
|
|
8323
|
+
},
|
|
7426
8324
|
});
|
|
7427
|
-
|
|
7428
|
-
|
|
8325
|
+
try {
|
|
8326
|
+
await flushLoopMicrotasks();
|
|
8327
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
8328
|
+
sleep.tick();
|
|
8329
|
+
await flushLoopMicrotasks();
|
|
8330
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8331
|
+
}
|
|
8332
|
+
finally {
|
|
8333
|
+
triggerShutdown();
|
|
8334
|
+
await expect(loop).resolves.toBe(0);
|
|
8335
|
+
expect(watcher.started).toBe(false);
|
|
8336
|
+
}
|
|
7429
8337
|
});
|
|
7430
8338
|
it("an auth-required watch pass parks in-process instead of exiting", async () => {
|
|
7431
8339
|
const watcher = makeWatcherStub();
|
|
@@ -7474,29 +8382,92 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7474
8382
|
it("a TRANSIENT_NETWORK_EXIT poll pass does NOT surface as a crash — the loop retries (HQ-SYNC-1W)", async () => {
|
|
7475
8383
|
// A transient offline blip must keep the watcher alive: the loop logs and
|
|
7476
8384
|
// polls again instead of returning the code and letting the process exit
|
|
7477
|
-
// (which the menubar reports as "watcher exited unexpectedly"). A
|
|
7478
|
-
//
|
|
8385
|
+
// (which the menubar reports as "watcher exited unexpectedly"). A later
|
|
8386
|
+
// generic failure is also retried rather than becoming a daemon exit.
|
|
7479
8387
|
const watcher = makeWatcherStub();
|
|
7480
8388
|
let n = 0;
|
|
7481
8389
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7482
8390
|
n++;
|
|
7483
|
-
// First pass: machine is offline (transient). Second pass: a
|
|
7484
|
-
//
|
|
8391
|
+
// First pass: machine is offline (transient). Second pass: a generic
|
|
8392
|
+
// failed pass, which must take the same non-fatal retry path.
|
|
7485
8393
|
return n === 1 ? TRANSIENT_NETWORK_EXIT : 1;
|
|
7486
8394
|
});
|
|
7487
|
-
const
|
|
8395
|
+
const sleep = makeSteppableSleep();
|
|
8396
|
+
let triggerShutdown = () => { };
|
|
8397
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7488
8398
|
runPass,
|
|
7489
8399
|
clock: new FakeClock(),
|
|
7490
8400
|
createWatcher: () => watcher,
|
|
7491
|
-
sleep:
|
|
7492
|
-
onShutdownSignal: () =>
|
|
8401
|
+
sleep: sleep.sleep,
|
|
8402
|
+
onShutdownSignal: (handler) => {
|
|
8403
|
+
triggerShutdown = handler;
|
|
8404
|
+
return () => { };
|
|
8405
|
+
},
|
|
7493
8406
|
});
|
|
7494
|
-
|
|
8407
|
+
await flushLoopMicrotasks();
|
|
8408
|
+
sleep.tick();
|
|
8409
|
+
await flushLoopMicrotasks();
|
|
8410
|
+
// The transient pass did NOT end the loop — it ran again, and neither
|
|
8411
|
+
// error causes the daemon itself to exit.
|
|
7495
8412
|
expect(runPass).toHaveBeenCalledTimes(2);
|
|
7496
|
-
|
|
7497
|
-
expect(
|
|
7498
|
-
expect(
|
|
8413
|
+
triggerShutdown();
|
|
8414
|
+
await expect(loop).resolves.toBe(0);
|
|
8415
|
+
expect(watcher.disposed).toBe(false);
|
|
8416
|
+
});
|
|
8417
|
+
it("keeps the watcher alive for an unresolved company watch route (HQ-SYNC-COMPANY-ROUTE-UID)", async () => {
|
|
8418
|
+
const watcher = makeWatcherStub();
|
|
8419
|
+
const client = makeVaultStub({
|
|
8420
|
+
entityGet: () => Promise.reject(new VaultNotFoundError("company deleted")),
|
|
8421
|
+
});
|
|
8422
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
8423
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
8424
|
+
const routedPass = makeDeps({ createVaultClient: () => client });
|
|
8425
|
+
const routedArgv = buildRoutePullArgv({ kind: "company", slug: "cmp_ghost" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
8426
|
+
let attempts = 0;
|
|
8427
|
+
let routedExit;
|
|
8428
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
8429
|
+
attempts += 1;
|
|
8430
|
+
if (attempts === 1) {
|
|
8431
|
+
routedExit = await runRunner(routedArgv, routedPass);
|
|
8432
|
+
return routedExit;
|
|
8433
|
+
}
|
|
8434
|
+
return UNRESOLVED_COMPANY_EXIT;
|
|
8435
|
+
});
|
|
8436
|
+
const sleep = makeSteppableSleep();
|
|
8437
|
+
let triggerShutdown = () => { };
|
|
8438
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
8439
|
+
runPass,
|
|
8440
|
+
clock: new FakeClock(),
|
|
8441
|
+
createWatcher: () => watcher,
|
|
8442
|
+
sleep: sleep.sleep,
|
|
8443
|
+
onShutdownSignal: (handler) => {
|
|
8444
|
+
triggerShutdown = handler;
|
|
8445
|
+
return () => { };
|
|
8446
|
+
},
|
|
8447
|
+
});
|
|
8448
|
+
await flushLoopMicrotasks();
|
|
8449
|
+
sleep.tick();
|
|
8450
|
+
await flushLoopMicrotasks();
|
|
8451
|
+
expect(routedArgv).toEqual([
|
|
8452
|
+
"--company",
|
|
8453
|
+
"cmp_ghost",
|
|
8454
|
+
"--direction",
|
|
8455
|
+
"pull",
|
|
8456
|
+
"--hq-root",
|
|
8457
|
+
"/tmp/hq",
|
|
8458
|
+
]);
|
|
8459
|
+
expect(routedExit).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
8460
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8461
|
+
triggerShutdown();
|
|
8462
|
+
await expect(loop).resolves.toBe(0);
|
|
7499
8463
|
expect(watcher.disposed).toBe(false);
|
|
8464
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_ghost");
|
|
8465
|
+
expect(routedPass.stderr.events()).toContainEqual({
|
|
8466
|
+
type: "error",
|
|
8467
|
+
company: "cmp_ghost",
|
|
8468
|
+
path: "(company-resolution)",
|
|
8469
|
+
message: "company entity lookup returned 404",
|
|
8470
|
+
});
|
|
7500
8471
|
});
|
|
7501
8472
|
it("keeps the watch loop alive when a company leg returns transient exit 75 (HQ-SYNC-X)", async () => {
|
|
7502
8473
|
const watcher = makeWatcherStub();
|
|
@@ -7513,24 +8484,36 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7513
8484
|
let passes = 0;
|
|
7514
8485
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7515
8486
|
passes += 1;
|
|
7516
|
-
//
|
|
7517
|
-
//
|
|
8487
|
+
// The first pass uses the real company-leg fanout path, not a synthetic
|
|
8488
|
+
// exit code. A later generic failure must also leave the daemon alive.
|
|
7518
8489
|
return passes === 1 ? runRunner(["--companies"], transientPass) : 1;
|
|
7519
8490
|
});
|
|
7520
|
-
const
|
|
8491
|
+
const sleep = makeSteppableSleep();
|
|
8492
|
+
let triggerShutdown = () => { };
|
|
8493
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7521
8494
|
runPass,
|
|
7522
8495
|
clock: new FakeClock(),
|
|
7523
8496
|
createWatcher: () => watcher,
|
|
7524
|
-
sleep:
|
|
7525
|
-
onShutdownSignal: () =>
|
|
8497
|
+
sleep: sleep.sleep,
|
|
8498
|
+
onShutdownSignal: (handler) => {
|
|
8499
|
+
triggerShutdown = handler;
|
|
8500
|
+
return () => { };
|
|
8501
|
+
},
|
|
7526
8502
|
});
|
|
7527
|
-
|
|
7528
|
-
|
|
7529
|
-
|
|
7530
|
-
|
|
7531
|
-
.
|
|
7532
|
-
|
|
7533
|
-
|
|
8503
|
+
try {
|
|
8504
|
+
await waitForPollSleep(sleep);
|
|
8505
|
+
sleep.tick();
|
|
8506
|
+
await flushLoopMicrotasks();
|
|
8507
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8508
|
+
expect(transientPass.stderr
|
|
8509
|
+
.events()
|
|
8510
|
+
.some((event) => event.type === "error" && event.path === "(company)")).toBe(false);
|
|
8511
|
+
expect(watcher.disposed).toBe(false);
|
|
8512
|
+
}
|
|
8513
|
+
finally {
|
|
8514
|
+
triggerShutdown();
|
|
8515
|
+
await expect(loop).resolves.toBe(0);
|
|
8516
|
+
}
|
|
7534
8517
|
});
|
|
7535
8518
|
it("a PARTIAL_SYNC_EXIT poll pass does NOT surface as a crash — the loop retries", async () => {
|
|
7536
8519
|
// A partial pass (some per-file transfers threw: oversized object on an
|
|
@@ -7544,22 +8527,32 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7544
8527
|
let n = 0;
|
|
7545
8528
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7546
8529
|
n++;
|
|
7547
|
-
// Two partial passes prove a
|
|
7548
|
-
// the loop either. A
|
|
8530
|
+
// Two partial passes prove a deterministic per-file failure doesn't kill
|
|
8531
|
+
// the loop either. A generic failure must also leave it alive.
|
|
7549
8532
|
return n <= 2 ? PARTIAL_SYNC_EXIT : 1;
|
|
7550
8533
|
});
|
|
7551
|
-
const
|
|
8534
|
+
const sleep = makeSteppableSleep();
|
|
8535
|
+
let triggerShutdown = () => { };
|
|
8536
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7552
8537
|
runPass,
|
|
7553
8538
|
clock: new FakeClock(),
|
|
7554
8539
|
createWatcher: () => watcher,
|
|
7555
|
-
sleep:
|
|
7556
|
-
onShutdownSignal: () =>
|
|
8540
|
+
sleep: sleep.sleep,
|
|
8541
|
+
onShutdownSignal: (handler) => {
|
|
8542
|
+
triggerShutdown = handler;
|
|
8543
|
+
return () => { };
|
|
8544
|
+
},
|
|
7557
8545
|
});
|
|
8546
|
+
await flushLoopMicrotasks();
|
|
8547
|
+
sleep.tick();
|
|
8548
|
+
await flushLoopMicrotasks();
|
|
8549
|
+
sleep.tick();
|
|
8550
|
+
await flushLoopMicrotasks();
|
|
7558
8551
|
// Both partial passes stayed alive — the loop ran a third time…
|
|
7559
8552
|
expect(runPass).toHaveBeenCalledTimes(3);
|
|
7560
|
-
// …and
|
|
7561
|
-
|
|
7562
|
-
expect(
|
|
8553
|
+
// …and neither partial nor generic pass failure exits the daemon.
|
|
8554
|
+
triggerShutdown();
|
|
8555
|
+
await expect(loop).resolves.toBe(0);
|
|
7563
8556
|
expect(watcher.disposed).toBe(false);
|
|
7564
8557
|
});
|
|
7565
8558
|
it("alerts when a partial pass failed before a company's download leg", async () => {
|
|
@@ -7587,24 +8580,33 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7587
8580
|
return 1;
|
|
7588
8581
|
});
|
|
7589
8582
|
// Every clock read advances a full cadence interval, so the tick after
|
|
7590
|
-
// the partial pass is a scheduled full reconcile
|
|
7591
|
-
// code this test asserts on.
|
|
8583
|
+
// the partial pass is a scheduled full reconcile with a generic failure.
|
|
7592
8584
|
let monotonic = 0;
|
|
8585
|
+
const sleep = makeSteppableSleep();
|
|
8586
|
+
let triggerShutdown = () => { };
|
|
7593
8587
|
try {
|
|
7594
|
-
const
|
|
8588
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7595
8589
|
runPass,
|
|
7596
8590
|
clock: new FakeClock(),
|
|
7597
8591
|
createWatcher: () => watcher,
|
|
7598
8592
|
monotonicNow: () => (monotonic += DEFAULT_FULL_RECONCILE_MS),
|
|
7599
|
-
sleep:
|
|
7600
|
-
onShutdownSignal: () =>
|
|
8593
|
+
sleep: sleep.sleep,
|
|
8594
|
+
onShutdownSignal: (handler) => {
|
|
8595
|
+
triggerShutdown = handler;
|
|
8596
|
+
return () => { };
|
|
8597
|
+
},
|
|
7601
8598
|
});
|
|
7602
|
-
|
|
8599
|
+
await flushLoopMicrotasks();
|
|
8600
|
+
sleep.tick();
|
|
8601
|
+
await flushLoopMicrotasks();
|
|
8602
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
7603
8603
|
const stderr = stderrWrite.mock.calls
|
|
7604
8604
|
.map(([chunk]) => String(chunk))
|
|
7605
8605
|
.join("");
|
|
7606
8606
|
expect(stderr).toContain("hq-sync-runner: COMPANY PASS FAILED company=personal message=RangeError Maximum call stack size exceeded (downloads for this company did not run)");
|
|
7607
8607
|
expect(stderr).toContain("hq-sync-runner: watch pass completed with per-file errors; failed-companies=personal; staying alive, retrying next poll");
|
|
8608
|
+
triggerShutdown();
|
|
8609
|
+
await expect(loop).resolves.toBe(0);
|
|
7608
8610
|
}
|
|
7609
8611
|
finally {
|
|
7610
8612
|
stderrWrite.mockRestore();
|
|
@@ -7791,6 +8793,85 @@ describe("runRunnerWithLoop — watch-mode auth-wait", () => {
|
|
|
7791
8793
|
});
|
|
7792
8794
|
});
|
|
7793
8795
|
describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
|
|
8796
|
+
it("hands a wake realtime drain to the guard before a long-running full chore completes", async () => {
|
|
8797
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
8798
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
8799
|
+
const watcher = makeWatcherStub();
|
|
8800
|
+
let triggerShutdown = () => { };
|
|
8801
|
+
let releaseCheckpoint = () => { };
|
|
8802
|
+
const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
8803
|
+
let releaseRealtime = () => { };
|
|
8804
|
+
const realtimeGate = new Promise((resolve) => { releaseRealtime = resolve; });
|
|
8805
|
+
const order = [];
|
|
8806
|
+
let passes = 0;
|
|
8807
|
+
let runRealtime = null;
|
|
8808
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
8809
|
+
runPass: vi.fn().mockImplementation(async (_argv, checkpoint) => {
|
|
8810
|
+
passes += 1;
|
|
8811
|
+
if (passes === 2) {
|
|
8812
|
+
order.push("chore");
|
|
8813
|
+
// The fake pass has completed its current durable unit and is now
|
|
8814
|
+
// at the same explicit hand-off seam production uses between
|
|
8815
|
+
// fully-settled fanout targets.
|
|
8816
|
+
await checkpointGate;
|
|
8817
|
+
await checkpoint?.();
|
|
8818
|
+
order.push("chore-complete");
|
|
8819
|
+
}
|
|
8820
|
+
return completedPassOutcome();
|
|
8821
|
+
}),
|
|
8822
|
+
runPassSupportsCooperativeCheckpoint: true,
|
|
8823
|
+
clock: new FakeClock(),
|
|
8824
|
+
createWatcher: () => watcher,
|
|
8825
|
+
sleep: makeSteppableSleep().sleep,
|
|
8826
|
+
onShutdownSignal: (handler) => {
|
|
8827
|
+
triggerShutdown = handler;
|
|
8828
|
+
return () => { };
|
|
8829
|
+
},
|
|
8830
|
+
startRealtimeScheduler: async (opts) => {
|
|
8831
|
+
// This is the same wake path used by the realtime scheduler: it
|
|
8832
|
+
// enters runGuardedTask behind the coordinator worker.
|
|
8833
|
+
runRealtime = () => opts.runGuarded(async () => {
|
|
8834
|
+
order.push("realtime");
|
|
8835
|
+
await realtimeGate;
|
|
8836
|
+
order.push("realtime-complete");
|
|
8837
|
+
});
|
|
8838
|
+
return {
|
|
8839
|
+
signal: () => undefined,
|
|
8840
|
+
checkHighWater: async () => undefined,
|
|
8841
|
+
dispose: async () => undefined,
|
|
8842
|
+
};
|
|
8843
|
+
},
|
|
8844
|
+
});
|
|
8845
|
+
try {
|
|
8846
|
+
await flushLoopMicrotasks(30);
|
|
8847
|
+
watcher.emit();
|
|
8848
|
+
await flushLoopMicrotasks(20);
|
|
8849
|
+
expect(order).toEqual(["chore"]);
|
|
8850
|
+
const realtime = runRealtime();
|
|
8851
|
+
releaseCheckpoint();
|
|
8852
|
+
await flushLoopMicrotasks(30);
|
|
8853
|
+
expect(order).toEqual(["chore", "realtime"]);
|
|
8854
|
+
releaseRealtime();
|
|
8855
|
+
await realtime;
|
|
8856
|
+
await flushLoopMicrotasks(20);
|
|
8857
|
+
expect(order).toEqual([
|
|
8858
|
+
"chore",
|
|
8859
|
+
"realtime",
|
|
8860
|
+
"realtime-complete",
|
|
8861
|
+
"chore-complete",
|
|
8862
|
+
]);
|
|
8863
|
+
}
|
|
8864
|
+
finally {
|
|
8865
|
+
releaseCheckpoint();
|
|
8866
|
+
releaseRealtime();
|
|
8867
|
+
triggerShutdown();
|
|
8868
|
+
await loop;
|
|
8869
|
+
if (previousCoordinator === undefined)
|
|
8870
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
8871
|
+
else
|
|
8872
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
8873
|
+
}
|
|
8874
|
+
});
|
|
7794
8875
|
it("feeds a watcher-originated V2 drain through the same single-flight guard as the poll", async () => {
|
|
7795
8876
|
const watcher = makeWatcherStub();
|
|
7796
8877
|
let triggerShutdown = () => { };
|
|
@@ -7903,6 +8984,173 @@ describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
|
|
|
7903
8984
|
await loop;
|
|
7904
8985
|
});
|
|
7905
8986
|
});
|
|
8987
|
+
describe("CooperativePassScheduler", () => {
|
|
8988
|
+
it("admits an explicitly safe fast realtime pass while a slow scoped realtime pass remains in flight", async () => {
|
|
8989
|
+
const scheduler = new CooperativePassScheduler(() => 0);
|
|
8990
|
+
const order = [];
|
|
8991
|
+
let releaseSlow = () => { };
|
|
8992
|
+
const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
|
|
8993
|
+
let markSlowStarted = () => { };
|
|
8994
|
+
const slowStarted = new Promise((resolve) => { markSlowStarted = resolve; });
|
|
8995
|
+
const slow = scheduler.submit(async () => {
|
|
8996
|
+
order.push("slow-start");
|
|
8997
|
+
markSlowStarted();
|
|
8998
|
+
await slowGate;
|
|
8999
|
+
order.push("slow-complete");
|
|
9000
|
+
return 0;
|
|
9001
|
+
}, "realtime", "slow");
|
|
9002
|
+
await slowStarted;
|
|
9003
|
+
const fast = scheduler.submit(async () => {
|
|
9004
|
+
order.push("fast");
|
|
9005
|
+
return 0;
|
|
9006
|
+
}, "realtime", "fast", true);
|
|
9007
|
+
try {
|
|
9008
|
+
await Promise.resolve();
|
|
9009
|
+
await Promise.resolve();
|
|
9010
|
+
expect([...order]).toEqual(["slow-start", "fast"]);
|
|
9011
|
+
}
|
|
9012
|
+
finally {
|
|
9013
|
+
releaseSlow();
|
|
9014
|
+
await Promise.all([slow, fast]);
|
|
9015
|
+
}
|
|
9016
|
+
});
|
|
9017
|
+
it("preserves journal order and resumes full work across repeated randomized hand-offs", async () => {
|
|
9018
|
+
// These deterministic seeds deliberately vary the number and enqueue
|
|
9019
|
+
// ordering of realtime tasks. Every append asserts one durable-state owner
|
|
9020
|
+
// at a time; the full continuation must resume after every realtime burst.
|
|
9021
|
+
for (let seed = 1; seed <= 32; seed += 1) {
|
|
9022
|
+
let now = 0;
|
|
9023
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
9024
|
+
const journal = [];
|
|
9025
|
+
const expected = new Set();
|
|
9026
|
+
let writer = null;
|
|
9027
|
+
let maximumWriters = 0;
|
|
9028
|
+
const append = async (entry) => {
|
|
9029
|
+
expect(writer).toBeNull();
|
|
9030
|
+
writer = entry;
|
|
9031
|
+
maximumWriters = Math.max(maximumWriters, 1);
|
|
9032
|
+
// A journal append crosses an async durable-write boundary. If the
|
|
9033
|
+
// scheduler admitted another pass here, its ownership assertion would
|
|
9034
|
+
// fail rather than merely observing JavaScript's synchronous turn.
|
|
9035
|
+
await Promise.resolve();
|
|
9036
|
+
expect(writer).toBe(entry);
|
|
9037
|
+
journal.push(entry);
|
|
9038
|
+
writer = null;
|
|
9039
|
+
};
|
|
9040
|
+
let releaseCheckpoint = () => { };
|
|
9041
|
+
const checkpointReady = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
9042
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
9043
|
+
expected.add(`chore-${seed}-before`);
|
|
9044
|
+
await append(`chore-${seed}-before`);
|
|
9045
|
+
await checkpointReady;
|
|
9046
|
+
await checkpoint();
|
|
9047
|
+
expected.add(`chore-${seed}-after`);
|
|
9048
|
+
await append(`chore-${seed}-after`);
|
|
9049
|
+
return 0;
|
|
9050
|
+
}, "chore");
|
|
9051
|
+
await Promise.resolve();
|
|
9052
|
+
const realtimeCount = (seed % 7) + 1;
|
|
9053
|
+
const shuffledIndexes = Array.from({ length: realtimeCount }, (_, index) => index);
|
|
9054
|
+
let randomState = seed;
|
|
9055
|
+
for (let index = shuffledIndexes.length - 1; index > 0; index -= 1) {
|
|
9056
|
+
randomState = (randomState * 1_664_525 + 1_013_904_223) >>> 0;
|
|
9057
|
+
const swapIndex = randomState % (index + 1);
|
|
9058
|
+
[shuffledIndexes[index], shuffledIndexes[swapIndex]] = [
|
|
9059
|
+
shuffledIndexes[swapIndex],
|
|
9060
|
+
shuffledIndexes[index],
|
|
9061
|
+
];
|
|
9062
|
+
}
|
|
9063
|
+
const realtime = shuffledIndexes.map((index) => {
|
|
9064
|
+
const entry = `realtime-${seed}-${(index * 17 + seed) % realtimeCount}`;
|
|
9065
|
+
expected.add(entry);
|
|
9066
|
+
return scheduler.submit(async () => {
|
|
9067
|
+
// Shuffle the microtask timing without ever allowing a second writer.
|
|
9068
|
+
if ((seed + index) % 3 === 0)
|
|
9069
|
+
await Promise.resolve();
|
|
9070
|
+
await append(entry);
|
|
9071
|
+
now += 1;
|
|
9072
|
+
return 0;
|
|
9073
|
+
}, "realtime");
|
|
9074
|
+
});
|
|
9075
|
+
releaseCheckpoint();
|
|
9076
|
+
await Promise.all([chore, ...realtime]);
|
|
9077
|
+
expect(maximumWriters).toBe(1);
|
|
9078
|
+
expect(new Set(journal)).toEqual(expected);
|
|
9079
|
+
expect(journal).toHaveLength(expected.size);
|
|
9080
|
+
expect(journal[0]).toBe(`chore-${seed}-before`);
|
|
9081
|
+
expect(journal.at(-1)).toBe(`chore-${seed}-after`);
|
|
9082
|
+
}
|
|
9083
|
+
});
|
|
9084
|
+
it("lets realtime win at 15 minutes minus one millisecond, then gives the chore the exact-boundary admission", async () => {
|
|
9085
|
+
const runCase = async (elapsedMs) => {
|
|
9086
|
+
let now = 0;
|
|
9087
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
9088
|
+
const order = [];
|
|
9089
|
+
let releaseActive = () => { };
|
|
9090
|
+
const activeGate = new Promise((resolve) => { releaseActive = resolve; });
|
|
9091
|
+
const activeRealtime = scheduler.submit(async () => {
|
|
9092
|
+
order.push("active-realtime");
|
|
9093
|
+
await activeGate;
|
|
9094
|
+
return 0;
|
|
9095
|
+
}, "realtime");
|
|
9096
|
+
await Promise.resolve();
|
|
9097
|
+
const chore = scheduler.submit(async () => {
|
|
9098
|
+
order.push("chore");
|
|
9099
|
+
return 0;
|
|
9100
|
+
}, "chore");
|
|
9101
|
+
const queuedRealtime = scheduler.submit(async () => {
|
|
9102
|
+
order.push("queued-realtime");
|
|
9103
|
+
return 0;
|
|
9104
|
+
}, "realtime");
|
|
9105
|
+
now = elapsedMs;
|
|
9106
|
+
releaseActive();
|
|
9107
|
+
await Promise.all([activeRealtime, chore, queuedRealtime]);
|
|
9108
|
+
return order;
|
|
9109
|
+
};
|
|
9110
|
+
await expect(runCase(CHORE_STARVATION_CAP_MS - 1)).resolves.toEqual([
|
|
9111
|
+
"active-realtime",
|
|
9112
|
+
"queued-realtime",
|
|
9113
|
+
"chore",
|
|
9114
|
+
]);
|
|
9115
|
+
await expect(runCase(CHORE_STARVATION_CAP_MS)).resolves.toEqual([
|
|
9116
|
+
"active-realtime",
|
|
9117
|
+
"chore",
|
|
9118
|
+
"queued-realtime",
|
|
9119
|
+
]);
|
|
9120
|
+
});
|
|
9121
|
+
it("resumes a handed-off chore when a continuous realtime burst reaches the 15-minute cap", async () => {
|
|
9122
|
+
let now = 0;
|
|
9123
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
9124
|
+
const order = [];
|
|
9125
|
+
let releaseCheckpoint = () => { };
|
|
9126
|
+
const checkpointReady = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
9127
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
9128
|
+
order.push("chore-before");
|
|
9129
|
+
await checkpointReady;
|
|
9130
|
+
await checkpoint();
|
|
9131
|
+
order.push("chore-after");
|
|
9132
|
+
return 0;
|
|
9133
|
+
}, "chore");
|
|
9134
|
+
await Promise.resolve();
|
|
9135
|
+
const firstRealtime = scheduler.submit(async () => {
|
|
9136
|
+
order.push("realtime-before-cap");
|
|
9137
|
+
now = CHORE_STARVATION_CAP_MS;
|
|
9138
|
+
return 0;
|
|
9139
|
+
}, "realtime");
|
|
9140
|
+
const laterRealtime = scheduler.submit(async () => {
|
|
9141
|
+
order.push("realtime-after-cap");
|
|
9142
|
+
return 0;
|
|
9143
|
+
}, "realtime");
|
|
9144
|
+
releaseCheckpoint();
|
|
9145
|
+
await Promise.all([chore, firstRealtime, laterRealtime]);
|
|
9146
|
+
expect(order).toEqual([
|
|
9147
|
+
"chore-before",
|
|
9148
|
+
"realtime-before-cap",
|
|
9149
|
+
"chore-after",
|
|
9150
|
+
"realtime-after-cap",
|
|
9151
|
+
]);
|
|
9152
|
+
});
|
|
9153
|
+
});
|
|
7906
9154
|
describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
7907
9155
|
const watchArgv = [
|
|
7908
9156
|
"--companies",
|
|
@@ -8031,6 +9279,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
|
8031
9279
|
"watcher-warmup-catch-up": matched.includes("watcher-warmup-catch-up"),
|
|
8032
9280
|
"watcher-warmup-catch-up-failed": matched.includes("watcher-warmup-catch-up-failed"),
|
|
8033
9281
|
"scheduled-interval": matched.includes("scheduled-interval"),
|
|
9282
|
+
"root-coverage-stale": matched.includes("root-coverage-stale"),
|
|
8034
9283
|
});
|
|
8035
9284
|
function expectFullReconcileDecision(record, mode, reasons) {
|
|
8036
9285
|
expect(record).toMatchObject({
|
|
@@ -10066,6 +11315,9 @@ describe("runRunnerWithLoop — Phase 3 event-sync wiring (US-017/018/019)", ()
|
|
|
10066
11315
|
"/tmp/hq",
|
|
10067
11316
|
], {
|
|
10068
11317
|
runPass: runPass,
|
|
11318
|
+
...(opts.supportsCooperativeCheckpoint
|
|
11319
|
+
? { runPassSupportsCooperativeCheckpoint: true }
|
|
11320
|
+
: {}),
|
|
10069
11321
|
clock,
|
|
10070
11322
|
createWatcher: (options) => {
|
|
10071
11323
|
opts.onCreateWatcher?.({
|
|
@@ -10178,6 +11430,123 @@ describe("runRunnerWithLoop — Phase 3 event-sync wiring (US-017/018/019)", ()
|
|
|
10178
11430
|
shutdown();
|
|
10179
11431
|
await loop;
|
|
10180
11432
|
});
|
|
11433
|
+
it("records initial-pull writes before its cooperative checkpoint can admit watcher work", async () => {
|
|
11434
|
+
const markContentPublished = vi.fn();
|
|
11435
|
+
const startEventSync = vi.fn().mockResolvedValue({
|
|
11436
|
+
publishBatch: vi.fn(),
|
|
11437
|
+
markContentPublished,
|
|
11438
|
+
receiver: { start: async () => { }, dispose: async () => { }, connected: true },
|
|
11439
|
+
ownDeviceId: "dev-test",
|
|
11440
|
+
dispose: vi.fn().mockResolvedValue(undefined),
|
|
11441
|
+
});
|
|
11442
|
+
const watcher = makeBatchWatcherStub();
|
|
11443
|
+
let suppressionWasInstalledWhenRealtimeStarted;
|
|
11444
|
+
const runPass = vi.fn(async (argv, checkpoint, onJournalsReady, onAppliedUpserts) => {
|
|
11445
|
+
if (argv.includes("--scope-path")) {
|
|
11446
|
+
suppressionWasInstalledWhenRealtimeStarted = markContentPublished
|
|
11447
|
+
.mock.calls
|
|
11448
|
+
.some(([relativePath]) => relativePath === "companies/indigo/initial-pull.md");
|
|
11449
|
+
return completedPassOutcome();
|
|
11450
|
+
}
|
|
11451
|
+
onJournalsReady?.();
|
|
11452
|
+
await microtasks();
|
|
11453
|
+
watcher.emit(undefined, {
|
|
11454
|
+
paths: new Map([[
|
|
11455
|
+
"/tmp/hq/companies/indigo/watcher-arrived-during-pull.md",
|
|
11456
|
+
"companies/indigo/watcher-arrived-during-pull.md",
|
|
11457
|
+
]]),
|
|
11458
|
+
changes: new Map([[
|
|
11459
|
+
"/tmp/hq/companies/indigo/watcher-arrived-during-pull.md",
|
|
11460
|
+
{ kind: "change" },
|
|
11461
|
+
]]),
|
|
11462
|
+
});
|
|
11463
|
+
// Let the watcher submit its realtime work before the full pass reaches
|
|
11464
|
+
// its explicit hand-off. No timer is involved: the submission is a
|
|
11465
|
+
// promise-only scheduling boundary.
|
|
11466
|
+
await microtasks();
|
|
11467
|
+
onAppliedUpserts?.([{
|
|
11468
|
+
relativePath: "companies/indigo/initial-pull.md",
|
|
11469
|
+
contentHash: "sha256:initial-pull",
|
|
11470
|
+
}]);
|
|
11471
|
+
await checkpoint?.();
|
|
11472
|
+
return {
|
|
11473
|
+
exitCode: 0,
|
|
11474
|
+
result: {
|
|
11475
|
+
pushPathResults: [],
|
|
11476
|
+
appliedUpserts: [{
|
|
11477
|
+
relativePath: "companies/indigo/initial-pull.md",
|
|
11478
|
+
contentHash: "sha256:initial-pull",
|
|
11479
|
+
}],
|
|
11480
|
+
},
|
|
11481
|
+
};
|
|
11482
|
+
});
|
|
11483
|
+
const { loop, shutdown } = runLoop({
|
|
11484
|
+
claims: EMAIL_CLAIMS,
|
|
11485
|
+
startEventSync,
|
|
11486
|
+
runPass,
|
|
11487
|
+
watcher,
|
|
11488
|
+
supportsCooperativeCheckpoint: true,
|
|
11489
|
+
});
|
|
11490
|
+
await microtasks(16);
|
|
11491
|
+
expect(suppressionWasInstalledWhenRealtimeStarted).toBe(true);
|
|
11492
|
+
shutdown();
|
|
11493
|
+
await loop;
|
|
11494
|
+
});
|
|
11495
|
+
it("does not let a yielding warm-up catch-up overwrite a newer watcher unlink", async () => {
|
|
11496
|
+
const watcher = makeBatchWatcherStub();
|
|
11497
|
+
let resolveCatchup;
|
|
11498
|
+
watcher.collectWarmupCatchupAsync = vi.fn(() => new Promise((resolve) => {
|
|
11499
|
+
resolveCatchup = resolve;
|
|
11500
|
+
}));
|
|
11501
|
+
const runPass = vi.fn(async (_argv, _checkpoint, onJournalsReady) => {
|
|
11502
|
+
onJournalsReady?.();
|
|
11503
|
+
return completedPassOutcome();
|
|
11504
|
+
});
|
|
11505
|
+
const { loop, tick, shutdown } = runLoop({
|
|
11506
|
+
claims: EMAIL_CLAIMS,
|
|
11507
|
+
watcher,
|
|
11508
|
+
runPass,
|
|
11509
|
+
supportsCooperativeCheckpoint: true,
|
|
11510
|
+
});
|
|
11511
|
+
await microtasks(12);
|
|
11512
|
+
const absolutePath = "/tmp/hq/companies/indigo/recreated.md";
|
|
11513
|
+
const relativePath = "companies/indigo/recreated.md";
|
|
11514
|
+
watcher.emit(undefined, {
|
|
11515
|
+
paths: new Map([[absolutePath, relativePath]]),
|
|
11516
|
+
changes: new Map([[absolutePath, {
|
|
11517
|
+
kind: "unlink",
|
|
11518
|
+
deleteSnapshots: [{
|
|
11519
|
+
journalSlug: "indigo",
|
|
11520
|
+
journalPath: "recreated.md",
|
|
11521
|
+
deleteScopeRoot: "recreated.md",
|
|
11522
|
+
absolutePath,
|
|
11523
|
+
remoteEtag: "etag-before-recreate",
|
|
11524
|
+
localHash: "hash-before-recreate",
|
|
11525
|
+
localKind: "file",
|
|
11526
|
+
}],
|
|
11527
|
+
}]]),
|
|
11528
|
+
});
|
|
11529
|
+
// This is the older filesystem state collected before the live unlink.
|
|
11530
|
+
resolveCatchup?.({
|
|
11531
|
+
batch: {
|
|
11532
|
+
paths: new Map([[absolutePath, relativePath]]),
|
|
11533
|
+
changes: new Map([[absolutePath, { kind: "change" }]]),
|
|
11534
|
+
},
|
|
11535
|
+
scannedPaths: 1,
|
|
11536
|
+
});
|
|
11537
|
+
await microtasks(16);
|
|
11538
|
+
tick();
|
|
11539
|
+
await microtasks(16);
|
|
11540
|
+
const scopedPush = runPass.mock.calls.find((call) => call[0].includes("--scope-path"))?.[0];
|
|
11541
|
+
expect(scopedPush).toEqual(expect.arrayContaining([
|
|
11542
|
+
"--scope-path",
|
|
11543
|
+
"recreated.md",
|
|
11544
|
+
"--delete-scope-path",
|
|
11545
|
+
"recreated.md",
|
|
11546
|
+
]));
|
|
11547
|
+
shutdown();
|
|
11548
|
+
await loop;
|
|
11549
|
+
});
|
|
10181
11550
|
it("replays pull upsert marks after event-sync startup becomes live", async () => {
|
|
10182
11551
|
const markContentPublished = vi.fn();
|
|
10183
11552
|
let resolveStart;
|
|
@@ -12568,6 +13937,45 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
12568
13937
|
expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
|
|
12569
13938
|
await stopHarness(h);
|
|
12570
13939
|
});
|
|
13940
|
+
it("attributes a company-root unlinkDir to its journal while rejecting company-root metadata files", async () => {
|
|
13941
|
+
const keys = Array.from({ length: 12 }, (_, i) => `knowledge/company-root/f-${i}.md`);
|
|
13942
|
+
const pushPasses = [];
|
|
13943
|
+
const h = startDeleteHarness(keys, {
|
|
13944
|
+
direction: "push",
|
|
13945
|
+
onPass: (argv) => {
|
|
13946
|
+
if (argv[argv.indexOf("--direction") + 1] === "push") {
|
|
13947
|
+
pushPasses.push(argv);
|
|
13948
|
+
}
|
|
13949
|
+
},
|
|
13950
|
+
});
|
|
13951
|
+
await settle();
|
|
13952
|
+
pushPasses.length = 0;
|
|
13953
|
+
const childKey = keys[0];
|
|
13954
|
+
const childPath = path.join(h.companyRoot, childKey);
|
|
13955
|
+
fs.unlinkSync(childPath);
|
|
13956
|
+
expect(h.capture()(`companies/indigo/${childKey}`, "unlink")).toHaveLength(1);
|
|
13957
|
+
expect(readJournal("indigo").files[childKey]?.localDeleteIntent).toBeDefined();
|
|
13958
|
+
// The child intent is safe only until the enclosing whole-company removal
|
|
13959
|
+
// is classified. At that scope the exact denominator is the whole shard,
|
|
13960
|
+
// so the existing bulk guard must revoke it rather than lose attribution.
|
|
13961
|
+
fs.rmSync(h.companyRoot, { recursive: true, force: true });
|
|
13962
|
+
const rootSnapshots = h.capture()("companies/indigo", "unlinkDir");
|
|
13963
|
+
expect(rootSnapshots).toHaveLength(0);
|
|
13964
|
+
expect(Object.values(readJournal("indigo").files).filter((entry) => entry.localDeleteIntent)).toHaveLength(0);
|
|
13965
|
+
h.watcher.emit("companies/indigo", {
|
|
13966
|
+
paths: new Map([[h.companyRoot, "companies/indigo"]]),
|
|
13967
|
+
changes: new Map([[h.companyRoot, {
|
|
13968
|
+
kind: "unlinkDir",
|
|
13969
|
+
deleteSnapshots: rootSnapshots,
|
|
13970
|
+
}]]),
|
|
13971
|
+
});
|
|
13972
|
+
h.tick();
|
|
13973
|
+
await settle();
|
|
13974
|
+
expect(pushPasses).toEqual([
|
|
13975
|
+
expect.arrayContaining(["--company", "indigo", "--direction", "push"]),
|
|
13976
|
+
]);
|
|
13977
|
+
await stopHarness(h);
|
|
13978
|
+
});
|
|
12571
13979
|
// The native macOS/Windows backends make no child-before-parent guarantee,
|
|
12572
13980
|
// so the refusal has to hold for descendants that arrive after it too.
|
|
12573
13981
|
it("a descendant unlink arriving after a refused unlinkDir records no intent", async () => {
|