@indigoai-us/hq-cloud 6.16.53 → 6.16.55-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +1 -23
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +8 -92
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +0 -36
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +1 -2
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +57 -126
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +403 -164
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -4
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +59 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -40
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +162 -1454
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -4
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +1 -20
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +1 -16
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +150 -467
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +95 -608
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +13 -15
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +259 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +43 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +297 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +0 -7
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +0 -11
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +66 -34
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +90 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -26
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +20 -43
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +0 -32
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -334
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.js +0 -627
- package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
- package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
|
@@ -10,6 +10,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
|
10
10
|
import * as fs from "fs";
|
|
11
11
|
import * as os from "os";
|
|
12
12
|
import * as path from "path";
|
|
13
|
+
import * as v8 from "node:v8";
|
|
14
|
+
import * as vm from "node:vm";
|
|
15
|
+
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
13
16
|
import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal, resolveSkipCompanies, selectObjectIOFactory, routeChangeToTarget, buildTargetedPushArgv, resolvePullScope, readPinnedPrefixes, SESSIONS_SCOPE_PREFIX, defaultCollectTelemetry, AUTH_REQUIRED_PASS_EXIT, } from "./sync-runner.js";
|
|
14
17
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
15
18
|
import { AUTH_WAIT_INITIAL_RECHECK_MS, AUTH_WAIT_RESUME_MESSAGE, AUTH_WAIT_START_MESSAGE, AUTH_WAIT_STEADY_RECHECK_MS, } from "./sync-runner-auth-wait.js";
|
|
@@ -17,17 +20,20 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
|
|
|
17
20
|
import { setBandwidthGovernorForTesting } from "../bandwidth.js";
|
|
18
21
|
import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
19
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
20
|
-
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
21
|
-
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
|
|
23
|
+
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
24
|
+
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
25
|
+
import { AreaLedger } from "../sync/area-ledger.js";
|
|
22
26
|
import { AreaResolver } from "../sync/area-resolver.js";
|
|
27
|
+
import { AREA_LEGACY_MIRROR_LINGER_MS } from "../sync/area-ledger-migration.js";
|
|
23
28
|
import { AreaMaintenanceCursorStore } from "../sync/per-area-maintenance.js";
|
|
24
29
|
import { HQ_CLOUD_VERSION } from "../version.js";
|
|
25
30
|
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
26
31
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
27
32
|
import { getStateStoreSnapshotRecoveryCountForTest, resetStateStoreSnapshotRecoveryCountForTest, StateStoreLockError, } from "../sync/state-store.js";
|
|
28
|
-
import { PARTIAL_SYNC_EXIT } from "../lib/exit-codes.js";
|
|
33
|
+
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
29
34
|
import { isCoveredByAny } from "../prefix-coalesce.js";
|
|
30
35
|
import { VaultAuthError, VaultNotFoundError } from "../vault-client.js";
|
|
36
|
+
import { buildRoutePullArgv } from "./sync-runner-watch-routes.js";
|
|
31
37
|
import { CognitoAuthError, CognitoRefreshError } from "../cognito-auth.js";
|
|
32
38
|
import { reconcileCompanyManifest, } from "../manifest-reconcile.js";
|
|
33
39
|
import yaml from "js-yaml";
|
|
@@ -162,6 +168,7 @@ function makeVaultStub(opts = {}) {
|
|
|
162
168
|
bucketName: `bucket-${uid}`,
|
|
163
169
|
status: "active",
|
|
164
170
|
})),
|
|
171
|
+
findInMyNamespace: opts.findInMyNamespace ?? (() => Promise.resolve(null)),
|
|
165
172
|
listByType: opts.listPersons ??
|
|
166
173
|
(() => Promise.resolve([])),
|
|
167
174
|
},
|
|
@@ -954,6 +961,164 @@ describe("target resolution", () => {
|
|
|
954
961
|
const call = deps.sync.mock.calls[0][0];
|
|
955
962
|
expect(call.company).toBe("cmp_abc");
|
|
956
963
|
});
|
|
964
|
+
it("returns the unresolved-company exit for an empty single-company plan instead of emitting a clean all-complete", async () => {
|
|
965
|
+
const deps = makeDeps({
|
|
966
|
+
createVaultClient: () => makeVaultStub({
|
|
967
|
+
entityGet: () => Promise.resolve(undefined),
|
|
968
|
+
}),
|
|
969
|
+
});
|
|
970
|
+
expect(await runRunner(["--company", "cmp_missing"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
971
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
972
|
+
type: "error",
|
|
973
|
+
company: "cmp_missing",
|
|
974
|
+
path: "(company-resolution)",
|
|
975
|
+
message: "company entity lookup returned no entity",
|
|
976
|
+
});
|
|
977
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
978
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
979
|
+
});
|
|
980
|
+
it("returns the unresolved-company exit when a cmp-prefixed route gets a 404", async () => {
|
|
981
|
+
const client = makeVaultStub({
|
|
982
|
+
entityGet: () => Promise.reject(new VaultNotFoundError("company deleted")),
|
|
983
|
+
});
|
|
984
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
985
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
986
|
+
const deps = makeDeps({
|
|
987
|
+
createVaultClient: () => client,
|
|
988
|
+
});
|
|
989
|
+
expect(await runRunner(["--company", "cmp_ghost"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
990
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_ghost");
|
|
991
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
992
|
+
type: "error",
|
|
993
|
+
company: "cmp_ghost",
|
|
994
|
+
path: "(company-resolution)",
|
|
995
|
+
message: "company entity lookup returned 404",
|
|
996
|
+
});
|
|
997
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
998
|
+
});
|
|
999
|
+
it("reports an intentional skipCompanies skip without treating it as a failed pass", async () => {
|
|
1000
|
+
const previous = process.env.HQ_SYNC_SKIP_COMPANIES;
|
|
1001
|
+
process.env.HQ_SYNC_SKIP_COMPANIES = "paused";
|
|
1002
|
+
try {
|
|
1003
|
+
const deps = makeDeps({
|
|
1004
|
+
createVaultClient: () => makeVaultStub({
|
|
1005
|
+
entityGet: (uid) => Promise.resolve({
|
|
1006
|
+
uid,
|
|
1007
|
+
slug: "paused",
|
|
1008
|
+
type: "company",
|
|
1009
|
+
status: "active",
|
|
1010
|
+
}),
|
|
1011
|
+
}),
|
|
1012
|
+
});
|
|
1013
|
+
expect(await runRunner(["--company", "cmp_paused"], deps)).toBe(0);
|
|
1014
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1015
|
+
type: "company-skipped",
|
|
1016
|
+
company: "paused",
|
|
1017
|
+
reason: "skip-companies",
|
|
1018
|
+
});
|
|
1019
|
+
expect(deps.stderr.events().some((event) => event.type === "error")).toBe(false);
|
|
1020
|
+
expect(deps.stdout.events().some((event) => event.type === "all-complete")).toBe(false);
|
|
1021
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
1022
|
+
}
|
|
1023
|
+
finally {
|
|
1024
|
+
if (previous === undefined)
|
|
1025
|
+
delete process.env.HQ_SYNC_SKIP_COMPANIES;
|
|
1026
|
+
else
|
|
1027
|
+
process.env.HQ_SYNC_SKIP_COMPANIES = previous;
|
|
1028
|
+
}
|
|
1029
|
+
});
|
|
1030
|
+
it("resolves a slug emitted by the company watch route to its namespace UID", async () => {
|
|
1031
|
+
const company = {
|
|
1032
|
+
uid: "cmp_indigo",
|
|
1033
|
+
slug: "indigo",
|
|
1034
|
+
type: "company",
|
|
1035
|
+
status: "active",
|
|
1036
|
+
bucketName: "bucket-cmp_indigo",
|
|
1037
|
+
};
|
|
1038
|
+
const client = makeVaultStub({
|
|
1039
|
+
entityGet: (uid) => Promise.resolve(uid === company.uid ? company : undefined),
|
|
1040
|
+
});
|
|
1041
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(company);
|
|
1042
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1043
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1044
|
+
const argv = buildRoutePullArgv({ kind: "company", slug: "indigo" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
1045
|
+
expect(argv).toEqual([
|
|
1046
|
+
"--company",
|
|
1047
|
+
"indigo",
|
|
1048
|
+
"--direction",
|
|
1049
|
+
"pull",
|
|
1050
|
+
"--hq-root",
|
|
1051
|
+
"/tmp/hq",
|
|
1052
|
+
]);
|
|
1053
|
+
expect(await runRunner(argv, deps)).toBe(0);
|
|
1054
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "indigo");
|
|
1055
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: "cmp_indigo" }));
|
|
1056
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1057
|
+
type: "fanout-plan",
|
|
1058
|
+
companies: [
|
|
1059
|
+
expect.objectContaining({ uid: "cmp_indigo", slug: "indigo" }),
|
|
1060
|
+
],
|
|
1061
|
+
});
|
|
1062
|
+
});
|
|
1063
|
+
it("resolves a cmp-prefixed directory slug in the caller namespace before treating it as a UID", async () => {
|
|
1064
|
+
const company = {
|
|
1065
|
+
uid: "cmp_01JTEAM000000000000000000",
|
|
1066
|
+
slug: "cmp_team",
|
|
1067
|
+
type: "company",
|
|
1068
|
+
status: "active",
|
|
1069
|
+
bucketName: "bucket-cmp-team",
|
|
1070
|
+
};
|
|
1071
|
+
const get = vi.fn().mockImplementation((uid) => Promise.resolve(uid === company.uid ? company : undefined));
|
|
1072
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1073
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(company);
|
|
1074
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1075
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1076
|
+
const argv = buildRoutePullArgv({ kind: "company", slug: "cmp_team" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
1077
|
+
expect(await runRunner(argv, deps)).toBe(0);
|
|
1078
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_team");
|
|
1079
|
+
expect(get).toHaveBeenCalledWith(company.uid);
|
|
1080
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: company.uid }));
|
|
1081
|
+
expect(deps.stdout.events()).toContainEqual({
|
|
1082
|
+
type: "fanout-plan",
|
|
1083
|
+
companies: [expect.objectContaining({ uid: company.uid, slug: "cmp_team" })],
|
|
1084
|
+
});
|
|
1085
|
+
});
|
|
1086
|
+
it("falls back to a literal company UID after caller-namespace slug resolution misses", async () => {
|
|
1087
|
+
const uid = "cmp_01JUID0000000000000000000";
|
|
1088
|
+
const company = {
|
|
1089
|
+
uid,
|
|
1090
|
+
slug: "operator-company",
|
|
1091
|
+
type: "company",
|
|
1092
|
+
status: "active",
|
|
1093
|
+
bucketName: "bucket-operator-company",
|
|
1094
|
+
};
|
|
1095
|
+
const get = vi.fn().mockResolvedValue(company);
|
|
1096
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1097
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
1098
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1099
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1100
|
+
expect(await runRunner(["--company", uid], deps)).toBe(0);
|
|
1101
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", uid);
|
|
1102
|
+
expect(get).toHaveBeenCalledWith(uid);
|
|
1103
|
+
expect(deps.sync).toHaveBeenCalledWith(expect.objectContaining({ company: uid }));
|
|
1104
|
+
});
|
|
1105
|
+
it("reports a foreign company slug when the literal UID fallback finds nothing", async () => {
|
|
1106
|
+
const get = vi.fn().mockRejectedValue(new VaultNotFoundError("company not found"));
|
|
1107
|
+
const client = makeVaultStub({ entityGet: get });
|
|
1108
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
1109
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
1110
|
+
const deps = makeDeps({ createVaultClient: () => client });
|
|
1111
|
+
expect(await runRunner(["--company", "indigo"], deps)).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
1112
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "indigo");
|
|
1113
|
+
expect(get).toHaveBeenCalledWith("indigo");
|
|
1114
|
+
expect(deps.sync).not.toHaveBeenCalled();
|
|
1115
|
+
expect(deps.stderr.events()).toContainEqual({
|
|
1116
|
+
type: "error",
|
|
1117
|
+
company: "indigo",
|
|
1118
|
+
path: "(company-resolution)",
|
|
1119
|
+
message: "company entity lookup returned 404",
|
|
1120
|
+
});
|
|
1121
|
+
});
|
|
957
1122
|
});
|
|
958
1123
|
// ---------------------------------------------------------------------------
|
|
959
1124
|
// --personal mode (personal-vault-only, skip company fanout)
|
|
@@ -1551,6 +1716,43 @@ describe("automatic journal maintenance", () => {
|
|
|
1551
1716
|
},
|
|
1552
1717
|
]);
|
|
1553
1718
|
});
|
|
1719
|
+
it("signals watch readiness after journal preparation and before fanout", async () => {
|
|
1720
|
+
const order = [];
|
|
1721
|
+
let releaseFanout = () => { };
|
|
1722
|
+
const fanout = new Promise((resolve) => { releaseFanout = resolve; });
|
|
1723
|
+
const deps = makeDeps({
|
|
1724
|
+
createVaultClient: () => makeVaultStub({
|
|
1725
|
+
memberships: [{ companyUid: "cmp_indigo" }],
|
|
1726
|
+
entityGet: (uid) => Promise.resolve({ uid, slug: "indigo" }),
|
|
1727
|
+
}),
|
|
1728
|
+
repairJournalStateIfNeeded: vi.fn((slug) => {
|
|
1729
|
+
order.push(`repair:${slug}`);
|
|
1730
|
+
return { status: "not-needed", slug };
|
|
1731
|
+
}),
|
|
1732
|
+
migratePersonalVaultJournal: vi.fn(() => {
|
|
1733
|
+
order.push("seed:personal");
|
|
1734
|
+
}),
|
|
1735
|
+
onJournalsReady: () => {
|
|
1736
|
+
order.push("journals-ready");
|
|
1737
|
+
},
|
|
1738
|
+
sync: vi.fn(async () => {
|
|
1739
|
+
order.push("fanout");
|
|
1740
|
+
await fanout;
|
|
1741
|
+
return defaultSyncResult();
|
|
1742
|
+
}),
|
|
1743
|
+
});
|
|
1744
|
+
const running = runRunner(["--companies", "--skip-personal"], deps);
|
|
1745
|
+
await vi.waitFor(() => expect(order).toContain("fanout"));
|
|
1746
|
+
expect(order).toEqual([
|
|
1747
|
+
"repair:indigo",
|
|
1748
|
+
"repair:personal",
|
|
1749
|
+
"seed:personal",
|
|
1750
|
+
"journals-ready",
|
|
1751
|
+
"fanout",
|
|
1752
|
+
]);
|
|
1753
|
+
releaseFanout();
|
|
1754
|
+
await expect(running).resolves.toBe(0);
|
|
1755
|
+
});
|
|
1554
1756
|
it("continues the pass when one journal repair is transiently lock-contended", async () => {
|
|
1555
1757
|
const repaired = [];
|
|
1556
1758
|
const sync = vi.fn().mockResolvedValue(defaultSyncResult());
|
|
@@ -4058,6 +4260,17 @@ describe("routeChangeToTarget", () => {
|
|
|
4058
4260
|
});
|
|
4059
4261
|
expect(routeChangeToTarget("companies/acme/projects/x/prd.json")).toEqual({ kind: "company", slug: "acme" });
|
|
4060
4262
|
});
|
|
4263
|
+
it("keeps a company-root directory delete attributable without routing company-root files", () => {
|
|
4264
|
+
// A whole-company removal has no child path, but it still owns every
|
|
4265
|
+
// journal row below the slug and must reach the delete-intent guard.
|
|
4266
|
+
expect(routeChangeToTarget("companies/indigo", "unlinkDir")).toEqual({
|
|
4267
|
+
kind: "company",
|
|
4268
|
+
slug: "indigo",
|
|
4269
|
+
});
|
|
4270
|
+
// A file directly under companies/ is shared metadata, never a tenant.
|
|
4271
|
+
expect(routeChangeToTarget("companies/manifest.yaml", "change")).toBeNull();
|
|
4272
|
+
expect(routeChangeToTarget("companies/.DS_Store", "add")).toBeNull();
|
|
4273
|
+
});
|
|
4061
4274
|
it("routes non-company top-levels to the personal target", () => {
|
|
4062
4275
|
expect(routeChangeToTarget("personal/notes.md")).toEqual({
|
|
4063
4276
|
kind: "personal",
|
|
@@ -4082,6 +4295,12 @@ describe("routeChangeToTarget", () => {
|
|
|
4082
4295
|
// edit always lands at companies/<slug>/..., never at bare "companies".
|
|
4083
4296
|
expect(routeChangeToTarget("companies")).toBeNull();
|
|
4084
4297
|
expect(routeChangeToTarget("companies/")).toBeNull();
|
|
4298
|
+
// The companies directory itself contains shared manifest and scaffold
|
|
4299
|
+
// files. They do not belong to a company journal, so a direct child must
|
|
4300
|
+
// never become a bogus `--company <filename>` pass.
|
|
4301
|
+
expect(routeChangeToTarget("companies/manifest.yaml")).toBeNull();
|
|
4302
|
+
expect(routeChangeToTarget("companies/.DS_Store")).toBeNull();
|
|
4303
|
+
expect(routeChangeToTarget("companies/../personal/notes.md")).toBeNull();
|
|
4085
4304
|
});
|
|
4086
4305
|
it("does not route the company scaffold to cloud sync", () => {
|
|
4087
4306
|
expect(routeChangeToTarget("companies/_template/knowledge/README.md")).toBeNull();
|
|
@@ -4163,6 +4382,26 @@ async function flushLoopMicrotasks(n = 8) {
|
|
|
4163
4382
|
for (let i = 0; i < n; i++)
|
|
4164
4383
|
await Promise.resolve();
|
|
4165
4384
|
}
|
|
4385
|
+
/** Retained journal rows live in typed-array backing stores as well as heap. */
|
|
4386
|
+
function forceGarbageCollectionForTest() {
|
|
4387
|
+
const existing = globalThis.gc;
|
|
4388
|
+
if (typeof existing === "function") {
|
|
4389
|
+
existing();
|
|
4390
|
+
existing();
|
|
4391
|
+
return;
|
|
4392
|
+
}
|
|
4393
|
+
v8.setFlagsFromString("--expose-gc");
|
|
4394
|
+
const collect = vm.runInNewContext("gc");
|
|
4395
|
+
v8.setFlagsFromString("--no-expose-gc");
|
|
4396
|
+
if (typeof collect !== "function")
|
|
4397
|
+
throw new Error("test could not expose GC");
|
|
4398
|
+
collect();
|
|
4399
|
+
collect();
|
|
4400
|
+
}
|
|
4401
|
+
function retainedProcessBytes() {
|
|
4402
|
+
const usage = process.memoryUsage();
|
|
4403
|
+
return usage.heapUsed + usage.external + usage.arrayBuffers;
|
|
4404
|
+
}
|
|
4166
4405
|
function makeSteppableSleep() {
|
|
4167
4406
|
const pending = [];
|
|
4168
4407
|
return {
|
|
@@ -4175,8 +4414,19 @@ function makeSteppableSleep() {
|
|
|
4175
4414
|
throw new Error("poll sleep was not pending");
|
|
4176
4415
|
resolve();
|
|
4177
4416
|
},
|
|
4417
|
+
pendingCount() {
|
|
4418
|
+
return pending.length;
|
|
4419
|
+
},
|
|
4178
4420
|
};
|
|
4179
4421
|
}
|
|
4422
|
+
async function waitForPollSleep(sleep) {
|
|
4423
|
+
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
4424
|
+
if (sleep.pendingCount() > 0)
|
|
4425
|
+
return;
|
|
4426
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4427
|
+
}
|
|
4428
|
+
throw new Error("timed out waiting for a poll sleep");
|
|
4429
|
+
}
|
|
4180
4430
|
// ---------------------------------------------------------------------------
|
|
4181
4431
|
// Adaptive remote-poll backoff — whole-box CPU aware (no --poll-remote-ms)
|
|
4182
4432
|
// ---------------------------------------------------------------------------
|
|
@@ -5268,6 +5518,83 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5268
5518
|
triggerShutdown();
|
|
5269
5519
|
await loop;
|
|
5270
5520
|
});
|
|
5521
|
+
it.each([
|
|
5522
|
+
["serialized", "0"],
|
|
5523
|
+
["parallel coordinator", "1"],
|
|
5524
|
+
])("starts event ingress once after journals are ready while the first %s full pass is unfinished", async (_mode, coordinator) => {
|
|
5525
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5526
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = coordinator;
|
|
5527
|
+
const watcher = makeWatcherStub();
|
|
5528
|
+
const createWatcher = vi.fn(() => watcher);
|
|
5529
|
+
let releaseFullPass = () => { };
|
|
5530
|
+
const fullPass = new Promise((resolve) => {
|
|
5531
|
+
releaseFullPass = resolve;
|
|
5532
|
+
});
|
|
5533
|
+
let activeDurableWriters = 0;
|
|
5534
|
+
let maxDurableWriters = 0;
|
|
5535
|
+
const runPass = vi.fn(async (...args) => {
|
|
5536
|
+
const journalsReady = args[2];
|
|
5537
|
+
activeDurableWriters += 1;
|
|
5538
|
+
maxDurableWriters = Math.max(maxDurableWriters, activeDurableWriters);
|
|
5539
|
+
journalsReady?.();
|
|
5540
|
+
await fullPass;
|
|
5541
|
+
activeDurableWriters -= 1;
|
|
5542
|
+
return completedPassOutcome();
|
|
5543
|
+
});
|
|
5544
|
+
let shutdown = () => { };
|
|
5545
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
5546
|
+
runPass: runPass,
|
|
5547
|
+
runPassSupportsCooperativeCheckpoint: true,
|
|
5548
|
+
createWatcher,
|
|
5549
|
+
sleep: () => new Promise(() => { }),
|
|
5550
|
+
onShutdownSignal: (handler) => {
|
|
5551
|
+
shutdown = handler;
|
|
5552
|
+
return () => { };
|
|
5553
|
+
},
|
|
5554
|
+
});
|
|
5555
|
+
try {
|
|
5556
|
+
await flushLoopMicrotasks(30);
|
|
5557
|
+
expect(runPass).toHaveBeenCalledOnce();
|
|
5558
|
+
expect(createWatcher).toHaveBeenCalledOnce();
|
|
5559
|
+
expect(watcher.started).toBe(true);
|
|
5560
|
+
expect(maxDurableWriters).toBe(1);
|
|
5561
|
+
await flushLoopMicrotasks();
|
|
5562
|
+
expect(runPass).toHaveBeenCalledOnce();
|
|
5563
|
+
}
|
|
5564
|
+
finally {
|
|
5565
|
+
releaseFullPass();
|
|
5566
|
+
shutdown();
|
|
5567
|
+
await loop;
|
|
5568
|
+
if (previousCoordinator === undefined)
|
|
5569
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5570
|
+
else
|
|
5571
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
5572
|
+
}
|
|
5573
|
+
});
|
|
5574
|
+
it.each([
|
|
5575
|
+
["auth-required", AUTH_REQUIRED_PASS_EXIT],
|
|
5576
|
+
["journal initialization failure", 1],
|
|
5577
|
+
])("does not arm event ingress when the first pass exits %s before journal readiness", async (_reason, exitCode) => {
|
|
5578
|
+
const watcher = makeWatcherStub();
|
|
5579
|
+
let shutdown = () => { };
|
|
5580
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
5581
|
+
runPass: vi.fn().mockResolvedValue(exitCode),
|
|
5582
|
+
createWatcher: () => watcher,
|
|
5583
|
+
sleep: () => new Promise(() => { }),
|
|
5584
|
+
onShutdownSignal: (handler) => {
|
|
5585
|
+
shutdown = handler;
|
|
5586
|
+
return () => { };
|
|
5587
|
+
},
|
|
5588
|
+
});
|
|
5589
|
+
try {
|
|
5590
|
+
await flushLoopMicrotasks(30);
|
|
5591
|
+
expect(watcher.started).toBe(false);
|
|
5592
|
+
}
|
|
5593
|
+
finally {
|
|
5594
|
+
shutdown();
|
|
5595
|
+
await expect(loop).resolves.toBe(0);
|
|
5596
|
+
}
|
|
5597
|
+
});
|
|
5271
5598
|
it("pushes a file created before watcher ready through the warm-up catch-up", async () => {
|
|
5272
5599
|
const watcher = makeBatchWatcherStub();
|
|
5273
5600
|
let markReady = () => { };
|
|
@@ -5586,7 +5913,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5586
5913
|
});
|
|
5587
5914
|
await expect(superseded).resolves.toBeUndefined();
|
|
5588
5915
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
|
|
5589
|
-
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":
|
|
5916
|
+
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":2,"source":"receiver","direction":"pull","outcome":"refused"');
|
|
5590
5917
|
releaseHeld();
|
|
5591
5918
|
await heldReceiverWork;
|
|
5592
5919
|
await replacement;
|
|
@@ -5899,7 +6226,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5899
6226
|
triggerShutdown();
|
|
5900
6227
|
await loop;
|
|
5901
6228
|
});
|
|
5902
|
-
it("fast lane: a small watcher batch
|
|
6229
|
+
it("fast lane: a small watcher batch waits for a full pass that has not reached a hand-off checkpoint", async () => {
|
|
5903
6230
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-watch-"));
|
|
5904
6231
|
const relativePath = "companies/indigo/realtime.md";
|
|
5905
6232
|
const absolutePath = path.join(hqRoot, relativePath);
|
|
@@ -5939,15 +6266,17 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5939
6266
|
expect(order).toEqual(["bulk"]);
|
|
5940
6267
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
5941
6268
|
await flushLoopMicrotasks(30);
|
|
5942
|
-
//
|
|
5943
|
-
|
|
6269
|
+
// A full pass that has not reached its durable checkpoint remains the sole
|
|
6270
|
+
// owner. Realtime is promoted at the checkpoint, never run concurrently.
|
|
6271
|
+
expect(order).toEqual(["bulk"]);
|
|
5944
6272
|
releaseBulk();
|
|
5945
|
-
await flushLoopMicrotasks();
|
|
6273
|
+
await flushLoopMicrotasks(30);
|
|
6274
|
+
expect(order).toEqual(["bulk", "fast", "fast"]);
|
|
5946
6275
|
triggerShutdown();
|
|
5947
6276
|
await loop;
|
|
5948
6277
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
5949
6278
|
});
|
|
5950
|
-
it("parallel coordinator: a watcher push
|
|
6279
|
+
it("parallel coordinator: a watcher push waits for a full chore without a hand-off checkpoint", async () => {
|
|
5951
6280
|
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
5952
6281
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
5953
6282
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-watch-"));
|
|
@@ -5990,9 +6319,10 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
5990
6319
|
expect(order).toEqual(["chore"]);
|
|
5991
6320
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
5992
6321
|
await flushLoopMicrotasks(30);
|
|
5993
|
-
expect(order).toEqual(["chore"
|
|
6322
|
+
expect(order).toEqual(["chore"]);
|
|
5994
6323
|
releaseBulk();
|
|
5995
|
-
await flushLoopMicrotasks();
|
|
6324
|
+
await flushLoopMicrotasks(30);
|
|
6325
|
+
expect(order).toEqual(["chore", "realtime"]);
|
|
5996
6326
|
triggerShutdown();
|
|
5997
6327
|
await loop;
|
|
5998
6328
|
}
|
|
@@ -6077,7 +6407,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6077
6407
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6078
6408
|
}
|
|
6079
6409
|
});
|
|
6080
|
-
it("parallel coordinator partitions receiver events into fast and slow realtime lanes", async () => {
|
|
6410
|
+
it("parallel coordinator partitions receiver events into fast and slow realtime lanes without overlapping their durable writes", async () => {
|
|
6081
6411
|
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6082
6412
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6083
6413
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-receiver-lanes-"));
|
|
@@ -6148,7 +6478,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6148
6478
|
signal: new AbortController().signal,
|
|
6149
6479
|
});
|
|
6150
6480
|
await flushLoopMicrotasks(20);
|
|
6151
|
-
expect(order).toEqual(["slow-held"
|
|
6481
|
+
expect(order).toEqual(["slow-held"]);
|
|
6152
6482
|
releaseSlow();
|
|
6153
6483
|
await held;
|
|
6154
6484
|
await partitioned;
|
|
@@ -6314,96 +6644,594 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6314
6644
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6315
6645
|
}
|
|
6316
6646
|
});
|
|
6317
|
-
it("
|
|
6318
|
-
const
|
|
6647
|
+
it("bounds a 32-path realtime personal commit on a 700k-row journal and 50k-path tree", async () => {
|
|
6648
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6319
6649
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6320
|
-
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-
|
|
6650
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-production-scale-watch-"));
|
|
6651
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
6652
|
+
const targetPaths = Array.from({ length: 32 }, (_, index) => `personal/watch-load/${String(index).padStart(5, "0")}/target.md`);
|
|
6653
|
+
const files = {};
|
|
6654
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
6655
|
+
files[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6656
|
+
hash: `sha256:history-${index}`,
|
|
6657
|
+
size: index % 4096,
|
|
6658
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6659
|
+
direction: "up",
|
|
6660
|
+
};
|
|
6661
|
+
}
|
|
6662
|
+
for (const target of targetPaths) {
|
|
6663
|
+
files[target] = {
|
|
6664
|
+
hash: `sha256:${target}`,
|
|
6665
|
+
size: 1,
|
|
6666
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6667
|
+
direction: "up",
|
|
6668
|
+
};
|
|
6669
|
+
}
|
|
6670
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
6671
|
+
version: "2",
|
|
6672
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6673
|
+
pulls: [],
|
|
6674
|
+
files,
|
|
6675
|
+
});
|
|
6676
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
6677
|
+
const directory = path.join(watchedRoot, String(index).padStart(5, "0"));
|
|
6678
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
6679
|
+
fs.writeFileSync(path.join(directory, "seed.md"), "seed");
|
|
6680
|
+
}
|
|
6321
6681
|
const watcher = makeBatchWatcherStub();
|
|
6322
|
-
let
|
|
6323
|
-
let
|
|
6324
|
-
const
|
|
6325
|
-
|
|
6326
|
-
|
|
6327
|
-
|
|
6328
|
-
|
|
6329
|
-
|
|
6330
|
-
|
|
6331
|
-
|
|
6332
|
-
|
|
6333
|
-
|
|
6682
|
+
let shutdown = () => { };
|
|
6683
|
+
let committedAt;
|
|
6684
|
+
const runPass = vi.fn(async (argv) => {
|
|
6685
|
+
if (!argv.includes("--scope-path")) {
|
|
6686
|
+
// Production opens the packed journal during the initial full pass,
|
|
6687
|
+
// before the watcher accepts realtime work. That recovered projection
|
|
6688
|
+
// is the one a scoped commit must reuse.
|
|
6689
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
6690
|
+
}
|
|
6691
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
6692
|
+
const scoped = readJournalScoped(PERSONAL_VAULT_JOURNAL_SLUG, targetPaths);
|
|
6693
|
+
const target = targetPaths[0];
|
|
6694
|
+
scoped.files[target] = {
|
|
6695
|
+
...scoped.files[target],
|
|
6696
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
6697
|
+
};
|
|
6698
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, scoped);
|
|
6699
|
+
committedAt = performance.now();
|
|
6700
|
+
}
|
|
6701
|
+
return completedPassOutcome();
|
|
6334
6702
|
});
|
|
6335
|
-
const
|
|
6336
|
-
|
|
6337
|
-
hash: "sha256:journal-small",
|
|
6338
|
-
size: 16,
|
|
6339
|
-
syncedAt: "2026-08-28T00:00:00.000Z",
|
|
6340
|
-
direction: "down",
|
|
6341
|
-
};
|
|
6342
|
-
writeJournal("indigo", journal);
|
|
6343
|
-
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
|
|
6344
|
-
runPass: vi.fn(async () => completedPassOutcome()),
|
|
6703
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6704
|
+
runPass,
|
|
6345
6705
|
createWatcher: () => watcher,
|
|
6346
|
-
createReceiver: ({ syncBatchFn }) => {
|
|
6347
|
-
receiverSync = syncBatchFn;
|
|
6348
|
-
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6349
|
-
},
|
|
6350
6706
|
sleep: () => new Promise(() => { }),
|
|
6351
6707
|
onShutdownSignal: (handler) => {
|
|
6352
|
-
|
|
6708
|
+
shutdown = handler;
|
|
6353
6709
|
return () => { };
|
|
6354
6710
|
},
|
|
6355
6711
|
});
|
|
6356
6712
|
try {
|
|
6357
6713
|
await flushLoopMicrotasks(30);
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
],
|
|
6373
|
-
|
|
6714
|
+
runPass.mockClear();
|
|
6715
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
6716
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
6717
|
+
eventLoopDelay.enable();
|
|
6718
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
6719
|
+
const submittedAt = performance.now();
|
|
6720
|
+
let timerDelayMs;
|
|
6721
|
+
const eventLoopTurn = new Promise((resolve) => {
|
|
6722
|
+
setTimeout(() => {
|
|
6723
|
+
timerDelayMs = performance.now() - submittedAt;
|
|
6724
|
+
resolve();
|
|
6725
|
+
}, 0);
|
|
6726
|
+
});
|
|
6727
|
+
const batch = {
|
|
6728
|
+
paths: new Map(targetPaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
6729
|
+
changes: new Map(targetPaths.map((relativePath) => [
|
|
6730
|
+
path.join(hqRoot, relativePath),
|
|
6731
|
+
{ kind: "change" },
|
|
6732
|
+
])),
|
|
6733
|
+
};
|
|
6734
|
+
watcher.emit(targetPaths[0], batch);
|
|
6735
|
+
await vi.waitFor(() => expect(committedAt).toBeDefined(), { timeout: 30_000, interval: 10 });
|
|
6736
|
+
await eventLoopTurn;
|
|
6737
|
+
eventLoopDelay.disable();
|
|
6738
|
+
expect(committedAt - submittedAt).toBeLessThan(1_000);
|
|
6739
|
+
expect(Number(eventLoopDelay.max) / 1_000_000).toBeLessThan(100);
|
|
6740
|
+
expect(timerDelayMs).toBeLessThan(100);
|
|
6741
|
+
expect(getStateStoreSnapshotRecoveryCountForTest()).toBe(0);
|
|
6742
|
+
expect(runPass.mock.calls.filter(([argv]) => argv.includes("--scope-path"))).toHaveLength(1);
|
|
6374
6743
|
}
|
|
6375
6744
|
finally {
|
|
6376
|
-
|
|
6745
|
+
shutdown();
|
|
6377
6746
|
await loop;
|
|
6378
|
-
if (
|
|
6747
|
+
if (previousCoordinator === undefined)
|
|
6379
6748
|
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6380
6749
|
else
|
|
6381
|
-
process.env.HQ_SYNC_PARALLEL_COORDINATOR =
|
|
6750
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6382
6751
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6383
6752
|
}
|
|
6384
|
-
});
|
|
6385
|
-
it("
|
|
6753
|
+
}, 180_000);
|
|
6754
|
+
it("keeps an area-authoritative 700k-row personal aggregate hot across alternating realtime and company batches", async () => {
|
|
6386
6755
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6387
|
-
const previousFastFiles = process.env[FAST_LANE_MAX_FILES_ENV];
|
|
6388
6756
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6389
|
-
|
|
6390
|
-
const
|
|
6391
|
-
const
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
const
|
|
6395
|
-
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
|
|
6399
|
-
|
|
6400
|
-
|
|
6401
|
-
|
|
6402
|
-
|
|
6403
|
-
|
|
6404
|
-
|
|
6405
|
-
|
|
6406
|
-
|
|
6757
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-watch-"));
|
|
6758
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
6759
|
+
const personalPath = "personal/watch-load/00000/target.md";
|
|
6760
|
+
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
6761
|
+
const companyKey = "watch-load/00000/target.md";
|
|
6762
|
+
const personalFiles = {};
|
|
6763
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
6764
|
+
personalFiles[`personal/history-${index % 5}/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
6765
|
+
hash: `sha256:history-${index}`,
|
|
6766
|
+
size: index % 4096,
|
|
6767
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6768
|
+
direction: "up",
|
|
6769
|
+
};
|
|
6770
|
+
}
|
|
6771
|
+
personalFiles[personalPath] = {
|
|
6772
|
+
hash: "sha256:personal-target",
|
|
6773
|
+
size: 1,
|
|
6774
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6775
|
+
direction: "up",
|
|
6776
|
+
};
|
|
6777
|
+
const resolver = new AreaResolver({
|
|
6778
|
+
format: 1,
|
|
6779
|
+
routerVersion: 1,
|
|
6780
|
+
areas: [
|
|
6781
|
+
...Array.from({ length: 5 }, (_, index) => ({
|
|
6782
|
+
id: `root:history-${index}`,
|
|
6783
|
+
prefixes: [`personal/history-${index}`],
|
|
6784
|
+
paths: [],
|
|
6785
|
+
})),
|
|
6786
|
+
{ id: "root:watch", prefixes: ["personal/watch-load"], paths: [] },
|
|
6787
|
+
{ id: "root:@files", prefixes: [], paths: [], rootFiles: true },
|
|
6788
|
+
],
|
|
6789
|
+
});
|
|
6790
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
6791
|
+
version: "2",
|
|
6792
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6793
|
+
pulls: [],
|
|
6794
|
+
files: {},
|
|
6795
|
+
});
|
|
6796
|
+
writeJournal("indigo", {
|
|
6797
|
+
version: "2",
|
|
6798
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6799
|
+
pulls: [],
|
|
6800
|
+
files: {
|
|
6801
|
+
[companyKey]: {
|
|
6802
|
+
hash: "sha256:company-target",
|
|
6803
|
+
size: 1,
|
|
6804
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
6805
|
+
direction: "up",
|
|
6806
|
+
},
|
|
6807
|
+
},
|
|
6808
|
+
});
|
|
6809
|
+
const migration = new AreaLedgerMigration({
|
|
6810
|
+
rootDir: __runnerTestStateDir,
|
|
6811
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6812
|
+
resolver,
|
|
6813
|
+
legacyStore: openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
|
|
6814
|
+
});
|
|
6815
|
+
expect(migration.cutOver(120_000).phase).toBe("area-authoritative");
|
|
6816
|
+
migration.dispose();
|
|
6817
|
+
// The migration state itself is covered by the area-ledger migration
|
|
6818
|
+
// suite. Seed the already-authoritative production shape in one durable
|
|
6819
|
+
// transaction so this watch-loop test measures repeated realtime reads,
|
|
6820
|
+
// rather than spending most of its worker budget replaying a one-time
|
|
6821
|
+
// legacy-to-area copy.
|
|
6822
|
+
openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG).replaceState({
|
|
6823
|
+
journal: {
|
|
6824
|
+
version: "2",
|
|
6825
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
6826
|
+
pulls: [],
|
|
6827
|
+
files: personalFiles,
|
|
6828
|
+
},
|
|
6829
|
+
});
|
|
6830
|
+
AreaLedger.open({
|
|
6831
|
+
rootDir: __runnerTestStateDir,
|
|
6832
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6833
|
+
layoutEpoch: 1,
|
|
6834
|
+
areaIds: resolver.layout.areas.map((area) => area.id),
|
|
6835
|
+
resolveArea: (key) => resolver.resolve(key),
|
|
6836
|
+
}).append({
|
|
6837
|
+
upserts: personalFiles,
|
|
6838
|
+
metadata: { lastSync: "2026-09-16T21:30:00.000Z", pulls: [] },
|
|
6839
|
+
});
|
|
6840
|
+
fs.mkdirSync(watchedRoot, { recursive: true });
|
|
6841
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
6842
|
+
fs.writeFileSync(path.join(watchedRoot, `seed-${String(index).padStart(5, "0")}.md`), "seed");
|
|
6843
|
+
}
|
|
6844
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
|
|
6845
|
+
fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
|
|
6846
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
|
|
6847
|
+
fs.writeFileSync(path.join(hqRoot, companyPath), "company");
|
|
6848
|
+
const watcher = makeBatchWatcherStub();
|
|
6849
|
+
const sleep = makeSteppableSleep();
|
|
6850
|
+
let shutdown = () => { };
|
|
6851
|
+
let isUntrackedDelete = (_relativePath) => false;
|
|
6852
|
+
const commits = [];
|
|
6853
|
+
let submittedAt = 0;
|
|
6854
|
+
const aggregatePrime = vi.spyOn(AreaLedger.prototype, "primeAggregateCache");
|
|
6855
|
+
const runPass = vi.fn(async (argv) => {
|
|
6856
|
+
if (!argv.includes("--scope-path")) {
|
|
6857
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
6858
|
+
readJournal("indigo");
|
|
6859
|
+
}
|
|
6860
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
6861
|
+
const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
|
|
6862
|
+
const key = slug === "indigo" ? companyKey : personalPath;
|
|
6863
|
+
const scoped = readJournalScoped(slug, [key]);
|
|
6864
|
+
scoped.files[key] = {
|
|
6865
|
+
...scoped.files[key],
|
|
6866
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
6867
|
+
};
|
|
6868
|
+
writeJournal(slug, scoped);
|
|
6869
|
+
commits.push({ slug, elapsedMs: performance.now() - submittedAt });
|
|
6870
|
+
}
|
|
6871
|
+
return completedPassOutcome();
|
|
6872
|
+
});
|
|
6873
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
6874
|
+
runPass,
|
|
6875
|
+
createWatcher: (options) => {
|
|
6876
|
+
isUntrackedDelete = options.isUntrackedDelete;
|
|
6877
|
+
return watcher;
|
|
6878
|
+
},
|
|
6879
|
+
startEventSync: vi.fn().mockResolvedValue(null),
|
|
6880
|
+
sleep: sleep.sleep,
|
|
6881
|
+
onShutdownSignal: (handler) => {
|
|
6882
|
+
shutdown = handler;
|
|
6883
|
+
return () => { };
|
|
6884
|
+
},
|
|
6885
|
+
});
|
|
6886
|
+
const emitAndWait = async (relativePaths) => {
|
|
6887
|
+
const before = commits.length;
|
|
6888
|
+
if (relativePaths.includes(personalPath)) {
|
|
6889
|
+
// This is the same non-blocking watcher callback which starts an
|
|
6890
|
+
// openJournalStoreSessionAsync before a delete episode is drained.
|
|
6891
|
+
// It is intentionally issued before each realtime batch: a completed
|
|
6892
|
+
// watch pass must not have thrown away the warmed area aggregate.
|
|
6893
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
6894
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
6895
|
+
}
|
|
6896
|
+
submittedAt = performance.now();
|
|
6897
|
+
const batch = {
|
|
6898
|
+
paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
6899
|
+
changes: new Map(relativePaths.map((relativePath) => [
|
|
6900
|
+
path.join(hqRoot, relativePath),
|
|
6901
|
+
{ kind: "change" },
|
|
6902
|
+
])),
|
|
6903
|
+
};
|
|
6904
|
+
watcher.emit(relativePaths[0], batch);
|
|
6905
|
+
await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
|
|
6906
|
+
timeout: 30_000,
|
|
6907
|
+
interval: 10,
|
|
6908
|
+
});
|
|
6909
|
+
// `commits` is observed inside the injected pass, before the runner has
|
|
6910
|
+
// completed its pass-final cleanup. Let that cleanup run before the next
|
|
6911
|
+
// watcher episode so the assertion observes the cache lifetime at the
|
|
6912
|
+
// same boundary as a real completed watch pass.
|
|
6913
|
+
await flushLoopMicrotasks(30);
|
|
6914
|
+
};
|
|
6915
|
+
try {
|
|
6916
|
+
await flushLoopMicrotasks(30);
|
|
6917
|
+
runPass.mockClear();
|
|
6918
|
+
// The initial runner pass is the one permitted cold open. Every later
|
|
6919
|
+
// scoped batch must reuse that authoritative aggregate. Warm the same
|
|
6920
|
+
// yielding watcher-session path that production uses before its first
|
|
6921
|
+
// delete episode, then measure only post-warmup primes.
|
|
6922
|
+
aggregatePrime.mockClear();
|
|
6923
|
+
expect(isUntrackedDelete(personalPath)).toBe(false);
|
|
6924
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
6925
|
+
// The initial full pass may already have opened this bridge (zero new
|
|
6926
|
+
// primes), or this watcher callback may perform the one permitted cold
|
|
6927
|
+
// warm-up. The post-poll assertion below is the regression boundary.
|
|
6928
|
+
expect(aggregatePrime.mock.calls.length).toBeLessThanOrEqual(1);
|
|
6929
|
+
aggregatePrime.mockClear();
|
|
6930
|
+
// Reproduce the normal poll boundary observed on the outposts. On the
|
|
6931
|
+
// current implementation it drops every cached area bridge even though
|
|
6932
|
+
// the watcher and runner remain alive.
|
|
6933
|
+
sleep.tick();
|
|
6934
|
+
await flushLoopMicrotasks(30);
|
|
6935
|
+
forceGarbageCollectionForTest();
|
|
6936
|
+
const retainedAfterWarmupBytes = retainedProcessBytes();
|
|
6937
|
+
const postWarmCpuStart = process.cpuUsage();
|
|
6938
|
+
const postWarmStartedAt = performance.now();
|
|
6939
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
6940
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
6941
|
+
eventLoopDelay.enable();
|
|
6942
|
+
await emitAndWait([personalPath, companyPath]);
|
|
6943
|
+
for (let round = 0; round < 3; round += 1) {
|
|
6944
|
+
await emitAndWait([personalPath]);
|
|
6945
|
+
await emitAndWait([companyPath]);
|
|
6946
|
+
}
|
|
6947
|
+
eventLoopDelay.disable();
|
|
6948
|
+
const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
|
|
6949
|
+
const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
|
|
6950
|
+
const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
|
|
6951
|
+
// A first area delta briefly retains the reverse-shadow mirror to
|
|
6952
|
+
// amortize a burst. Retained memory means after that documented linger
|
|
6953
|
+
// has released its second full-vault projection, not during it.
|
|
6954
|
+
await new Promise((resolve) => setTimeout(resolve, AREA_LEGACY_MIRROR_LINGER_MS + 25));
|
|
6955
|
+
forceGarbageCollectionForTest();
|
|
6956
|
+
const retainedAfterChurnBytes = retainedProcessBytes();
|
|
6957
|
+
const retainedGrowthBytes = retainedAfterChurnBytes - retainedAfterWarmupBytes;
|
|
6958
|
+
const postWarmCpu = process.cpuUsage(postWarmCpuStart);
|
|
6959
|
+
const postWarmCpuPercent = ((postWarmCpu.user + postWarmCpu.system) /
|
|
6960
|
+
((performance.now() - postWarmStartedAt) * 1_000)) * 100;
|
|
6961
|
+
console.info({
|
|
6962
|
+
aggregatePrimeCount: aggregatePrime.mock.calls.length,
|
|
6963
|
+
recoveryCount,
|
|
6964
|
+
maxDelayMs,
|
|
6965
|
+
maxCommitMs,
|
|
6966
|
+
postWarmCpuPercent,
|
|
6967
|
+
retainedAfterWarmupBytes,
|
|
6968
|
+
retainedAfterChurnBytes,
|
|
6969
|
+
retainedGrowthBytes,
|
|
6970
|
+
commits: commits.length,
|
|
6971
|
+
});
|
|
6972
|
+
expect(aggregatePrime).not.toHaveBeenCalled();
|
|
6973
|
+
// The first area delta opens its reverse-shadow v3 mirror once; its
|
|
6974
|
+
// linger keeps all later scoped commits off snapshot recovery. This is
|
|
6975
|
+
// deliberately exact so a second cold bridge cannot hide as a cache hit.
|
|
6976
|
+
expect.soft(recoveryCount).toBe(1);
|
|
6977
|
+
// The production-shape run remains bounded under the coverage job's
|
|
6978
|
+
// concurrent workers: CI measured 1,090.52 ms of timer delay and
|
|
6979
|
+
// 1,087.79 ms to commit. 1.5 s retains a concrete latency ceiling while
|
|
6980
|
+
// leaving headroom for that instrumentation, rather than making the
|
|
6981
|
+
// assertion depend on a single host's scheduling jitter.
|
|
6982
|
+
expect.soft(maxDelayMs).toBeLessThan(1_500);
|
|
6983
|
+
expect.soft(maxCommitMs).toBeLessThan(1_500);
|
|
6984
|
+
expect.soft(retainedGrowthBytes).toBeLessThan(32 * 1024 * 1024);
|
|
6985
|
+
expect(commits.map((commit) => commit.slug)).toEqual([
|
|
6986
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6987
|
+
"indigo",
|
|
6988
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6989
|
+
"indigo",
|
|
6990
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6991
|
+
"indigo",
|
|
6992
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
6993
|
+
"indigo",
|
|
6994
|
+
]);
|
|
6995
|
+
}
|
|
6996
|
+
finally {
|
|
6997
|
+
aggregatePrime.mockRestore();
|
|
6998
|
+
shutdown();
|
|
6999
|
+
await loop;
|
|
7000
|
+
if (previousCoordinator === undefined)
|
|
7001
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7002
|
+
else
|
|
7003
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
7004
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
7005
|
+
}
|
|
7006
|
+
}, 180_000);
|
|
7007
|
+
it("keeps warmed personal and company journals hot across alternating realtime batches", async () => {
|
|
7008
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7009
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
7010
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-multi-journal-legacy-watch-"));
|
|
7011
|
+
const watchedRoot = path.join(hqRoot, "personal", "watch-load");
|
|
7012
|
+
const personalPath = "personal/watch-load/00000/target.md";
|
|
7013
|
+
const companyPath = "companies/indigo/watch-load/00000/target.md";
|
|
7014
|
+
const companyKey = "watch-load/00000/target.md";
|
|
7015
|
+
const personalFiles = {};
|
|
7016
|
+
for (let index = 0; index < 700_000; index += 1) {
|
|
7017
|
+
personalFiles[`personal/history/${String(index % 50_000).padStart(5, "0")}/entry-${index}.md`] = {
|
|
7018
|
+
hash: `sha256:history-${index}`,
|
|
7019
|
+
size: index % 4096,
|
|
7020
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7021
|
+
direction: "up",
|
|
7022
|
+
};
|
|
7023
|
+
}
|
|
7024
|
+
personalFiles[personalPath] = {
|
|
7025
|
+
hash: "sha256:personal-target",
|
|
7026
|
+
size: 1,
|
|
7027
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7028
|
+
direction: "up",
|
|
7029
|
+
};
|
|
7030
|
+
writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
|
|
7031
|
+
version: "2",
|
|
7032
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
7033
|
+
pulls: [],
|
|
7034
|
+
files: personalFiles,
|
|
7035
|
+
});
|
|
7036
|
+
writeJournal("indigo", {
|
|
7037
|
+
version: "2",
|
|
7038
|
+
lastSync: "2026-09-16T21:30:00.000Z",
|
|
7039
|
+
pulls: [],
|
|
7040
|
+
files: {
|
|
7041
|
+
[companyKey]: {
|
|
7042
|
+
hash: "sha256:company-target",
|
|
7043
|
+
size: 1,
|
|
7044
|
+
syncedAt: "2026-09-16T21:30:00.000Z",
|
|
7045
|
+
direction: "up",
|
|
7046
|
+
},
|
|
7047
|
+
},
|
|
7048
|
+
});
|
|
7049
|
+
for (let index = 0; index < 50_000; index += 1) {
|
|
7050
|
+
const directory = path.join(watchedRoot, String(index).padStart(5, "0"));
|
|
7051
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
7052
|
+
fs.writeFileSync(path.join(directory, "seed.md"), "seed");
|
|
7053
|
+
}
|
|
7054
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, personalPath)), { recursive: true });
|
|
7055
|
+
fs.writeFileSync(path.join(hqRoot, personalPath), "personal");
|
|
7056
|
+
fs.mkdirSync(path.dirname(path.join(hqRoot, companyPath)), { recursive: true });
|
|
7057
|
+
fs.writeFileSync(path.join(hqRoot, companyPath), "company");
|
|
7058
|
+
const watcher = makeBatchWatcherStub();
|
|
7059
|
+
let shutdown = () => { };
|
|
7060
|
+
const commits = [];
|
|
7061
|
+
let submittedAt = 0;
|
|
7062
|
+
const runPass = vi.fn(async (argv) => {
|
|
7063
|
+
if (!argv.includes("--scope-path")) {
|
|
7064
|
+
readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
|
|
7065
|
+
readJournal("indigo");
|
|
7066
|
+
}
|
|
7067
|
+
if (argv.includes("--scope-path") && argv.includes("push")) {
|
|
7068
|
+
const slug = argv.includes("--company") ? "indigo" : PERSONAL_VAULT_JOURNAL_SLUG;
|
|
7069
|
+
const key = slug === "indigo" ? companyKey : personalPath;
|
|
7070
|
+
const scoped = readJournalScoped(slug, [key]);
|
|
7071
|
+
scoped.files[key] = {
|
|
7072
|
+
...scoped.files[key],
|
|
7073
|
+
syncedAt: "2026-09-16T21:38:18.000Z",
|
|
7074
|
+
};
|
|
7075
|
+
writeJournal(slug, scoped);
|
|
7076
|
+
commits.push({ slug, elapsedMs: performance.now() - submittedAt });
|
|
7077
|
+
}
|
|
7078
|
+
return completedPassOutcome();
|
|
7079
|
+
});
|
|
7080
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
|
|
7081
|
+
runPass,
|
|
7082
|
+
createWatcher: () => watcher,
|
|
7083
|
+
sleep: () => new Promise(() => { }),
|
|
7084
|
+
onShutdownSignal: (handler) => {
|
|
7085
|
+
shutdown = handler;
|
|
7086
|
+
return () => { };
|
|
7087
|
+
},
|
|
7088
|
+
});
|
|
7089
|
+
const emitAndWait = async (relativePaths) => {
|
|
7090
|
+
const before = commits.length;
|
|
7091
|
+
submittedAt = performance.now();
|
|
7092
|
+
const batch = {
|
|
7093
|
+
paths: new Map(relativePaths.map((relativePath) => [path.join(hqRoot, relativePath), relativePath])),
|
|
7094
|
+
changes: new Map(relativePaths.map((relativePath) => [
|
|
7095
|
+
path.join(hqRoot, relativePath),
|
|
7096
|
+
{ kind: "change" },
|
|
7097
|
+
])),
|
|
7098
|
+
};
|
|
7099
|
+
watcher.emit(relativePaths[0], batch);
|
|
7100
|
+
await vi.waitFor(() => expect(commits.length).toBe(before + relativePaths.length), {
|
|
7101
|
+
timeout: 30_000,
|
|
7102
|
+
interval: 10,
|
|
7103
|
+
});
|
|
7104
|
+
};
|
|
7105
|
+
try {
|
|
7106
|
+
await flushLoopMicrotasks(30);
|
|
7107
|
+
runPass.mockClear();
|
|
7108
|
+
resetStateStoreSnapshotRecoveryCountForTest();
|
|
7109
|
+
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
|
|
7110
|
+
eventLoopDelay.enable();
|
|
7111
|
+
await emitAndWait([personalPath, companyPath]);
|
|
7112
|
+
for (let round = 0; round < 3; round += 1) {
|
|
7113
|
+
await emitAndWait([personalPath]);
|
|
7114
|
+
await emitAndWait([companyPath]);
|
|
7115
|
+
}
|
|
7116
|
+
eventLoopDelay.disable();
|
|
7117
|
+
const recoveryCount = getStateStoreSnapshotRecoveryCountForTest();
|
|
7118
|
+
const maxDelayMs = Number(eventLoopDelay.max) / 1_000_000;
|
|
7119
|
+
const maxCommitMs = Math.max(...commits.map((commit) => commit.elapsedMs));
|
|
7120
|
+
console.info({ recoveryCount, maxDelayMs, maxCommitMs, commits: commits.length });
|
|
7121
|
+
expect.soft(recoveryCount).toBe(0);
|
|
7122
|
+
expect.soft(maxDelayMs).toBeLessThan(100);
|
|
7123
|
+
expect.soft(maxCommitMs).toBeLessThan(1_000);
|
|
7124
|
+
expect(commits.map((commit) => commit.slug)).toEqual([
|
|
7125
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7126
|
+
"indigo",
|
|
7127
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7128
|
+
"indigo",
|
|
7129
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7130
|
+
"indigo",
|
|
7131
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
7132
|
+
"indigo",
|
|
7133
|
+
]);
|
|
7134
|
+
}
|
|
7135
|
+
finally {
|
|
7136
|
+
shutdown();
|
|
7137
|
+
await loop;
|
|
7138
|
+
if (previousCoordinator === undefined)
|
|
7139
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7140
|
+
else
|
|
7141
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
7142
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
7143
|
+
}
|
|
7144
|
+
}, 180_000);
|
|
7145
|
+
it("fast lane: prefers declared receiver sizes and falls back to the journal", async () => {
|
|
7146
|
+
const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7147
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
7148
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-receiver-declared-size-"));
|
|
7149
|
+
const watcher = makeBatchWatcherStub();
|
|
7150
|
+
let receiverSync;
|
|
7151
|
+
let triggerShutdown = () => { };
|
|
7152
|
+
const event = (relativePath, sizeBytes) => ({
|
|
7153
|
+
kind: "upsert",
|
|
7154
|
+
relativePath,
|
|
7155
|
+
contentHash: "sha256:remote",
|
|
7156
|
+
mtime: "2026-08-28T00:00:00.000Z",
|
|
7157
|
+
...(sizeBytes === undefined ? {} : { sizeBytes }),
|
|
7158
|
+
originDeviceId: "peer-device",
|
|
7159
|
+
originTenantId: "tenant-indigo",
|
|
7160
|
+
sequenceNumber: 1,
|
|
7161
|
+
eventTimestamp: "2026-08-28T00:00:00.000Z",
|
|
7162
|
+
});
|
|
7163
|
+
const journal = readJournal("indigo");
|
|
7164
|
+
journal.files["journal-small.md"] = {
|
|
7165
|
+
hash: "sha256:journal-small",
|
|
7166
|
+
size: 16,
|
|
7167
|
+
syncedAt: "2026-08-28T00:00:00.000Z",
|
|
7168
|
+
direction: "down",
|
|
7169
|
+
};
|
|
7170
|
+
writeJournal("indigo", journal);
|
|
7171
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
|
|
7172
|
+
runPass: vi.fn(async () => completedPassOutcome()),
|
|
7173
|
+
createWatcher: () => watcher,
|
|
7174
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
7175
|
+
receiverSync = syncBatchFn;
|
|
7176
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
7177
|
+
},
|
|
7178
|
+
sleep: () => new Promise(() => { }),
|
|
7179
|
+
onShutdownSignal: (handler) => {
|
|
7180
|
+
triggerShutdown = handler;
|
|
7181
|
+
return () => { };
|
|
7182
|
+
},
|
|
7183
|
+
});
|
|
7184
|
+
try {
|
|
7185
|
+
await flushLoopMicrotasks(30);
|
|
7186
|
+
const batches = receiverSync.partition([
|
|
7187
|
+
event("companies/indigo/declared-small.md", 16),
|
|
7188
|
+
event("companies/indigo/declared-large.md", DEFAULT_FAST_LANE_MAX_FILE_BYTES + 1),
|
|
7189
|
+
event("companies/indigo/journal-small.md"),
|
|
7190
|
+
event("companies/indigo/unknown.md"),
|
|
7191
|
+
]);
|
|
7192
|
+
expect(batches.map((batch) => batch.map((event) => event.relativePath))).toEqual([
|
|
7193
|
+
[
|
|
7194
|
+
"companies/indigo/declared-small.md",
|
|
7195
|
+
"companies/indigo/journal-small.md",
|
|
7196
|
+
],
|
|
7197
|
+
[
|
|
7198
|
+
"companies/indigo/declared-large.md",
|
|
7199
|
+
"companies/indigo/unknown.md",
|
|
7200
|
+
],
|
|
7201
|
+
]);
|
|
7202
|
+
}
|
|
7203
|
+
finally {
|
|
7204
|
+
triggerShutdown();
|
|
7205
|
+
await loop;
|
|
7206
|
+
if (previous === undefined)
|
|
7207
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7208
|
+
else
|
|
7209
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previous;
|
|
7210
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
7211
|
+
}
|
|
7212
|
+
});
|
|
7213
|
+
it("fast lane: does not charge declared sizes from a skipped personal route", async () => {
|
|
7214
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
7215
|
+
const previousFastFiles = process.env[FAST_LANE_MAX_FILES_ENV];
|
|
7216
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
7217
|
+
process.env[FAST_LANE_MAX_FILES_ENV] = "1";
|
|
7218
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-receiver-skipped-route-size-"));
|
|
7219
|
+
const watcher = makeBatchWatcherStub();
|
|
7220
|
+
let receiverSync;
|
|
7221
|
+
let triggerShutdown = () => { };
|
|
7222
|
+
const event = (relativePath) => ({
|
|
7223
|
+
kind: "upsert",
|
|
7224
|
+
relativePath,
|
|
7225
|
+
contentHash: "sha256:remote",
|
|
7226
|
+
mtime: "2026-08-28T00:00:00.000Z",
|
|
7227
|
+
sizeBytes: 64,
|
|
7228
|
+
originDeviceId: "peer-device",
|
|
7229
|
+
originTenantId: "tenant-indigo",
|
|
7230
|
+
sequenceNumber: 1,
|
|
7231
|
+
eventTimestamp: "2026-08-28T00:00:00.000Z",
|
|
7232
|
+
});
|
|
7233
|
+
const loop = runRunnerWithLoop([
|
|
7234
|
+
"--companies", "--skip-personal", "--watch", "--event-push", "--direction", "pull",
|
|
6407
7235
|
"--hq-root", hqRoot,
|
|
6408
7236
|
], {
|
|
6409
7237
|
runPass: vi.fn(async () => completedPassOutcome()),
|
|
@@ -6489,19 +7317,19 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6489
7317
|
]),
|
|
6490
7318
|
});
|
|
6491
7319
|
await flushLoopMicrotasks(30);
|
|
6492
|
-
//
|
|
6493
|
-
//
|
|
6494
|
-
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(
|
|
6495
|
-
expect(calls.flat().join(" ")).toContain("realtime.md");
|
|
7320
|
+
// The full pass owns durable state until a checkpoint. Its queued realtime
|
|
7321
|
+
// residue is promoted at that hand-off instead of overlapping the pass.
|
|
7322
|
+
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(0);
|
|
6496
7323
|
releaseBulk();
|
|
6497
7324
|
await flushLoopMicrotasks(30);
|
|
6498
7325
|
expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(2);
|
|
7326
|
+
expect(calls.flat().join(" ")).toContain("realtime.md");
|
|
6499
7327
|
expect(calls.flat().join(" ")).toContain("session-0.log");
|
|
6500
7328
|
triggerShutdown();
|
|
6501
7329
|
await loop;
|
|
6502
7330
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6503
7331
|
});
|
|
6504
|
-
it("fast lane: a small receiver pull
|
|
7332
|
+
it("fast lane: a small receiver pull waits for a slow full pass without a hand-off checkpoint", async () => {
|
|
6505
7333
|
const watcher = makeWatcherStub();
|
|
6506
7334
|
let triggerShutdown = () => { };
|
|
6507
7335
|
let receiverSync;
|
|
@@ -6546,7 +7374,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6546
7374
|
watcher.emit();
|
|
6547
7375
|
await flushLoopMicrotasks();
|
|
6548
7376
|
expect(order).toEqual(["bulk"]);
|
|
6549
|
-
|
|
7377
|
+
const receiverPull = receiverSync({
|
|
6550
7378
|
events: [{
|
|
6551
7379
|
kind: "upsert", relativePath: "companies/acme/realtime.md", contentHash: "sha256:x",
|
|
6552
7380
|
mtime: "2026-08-26T00:00:00.000Z", originDeviceId: "peer", originTenantId: "acme",
|
|
@@ -6554,8 +7382,11 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6554
7382
|
}],
|
|
6555
7383
|
signal: new AbortController().signal,
|
|
6556
7384
|
});
|
|
6557
|
-
|
|
7385
|
+
await flushLoopMicrotasks(20);
|
|
7386
|
+
expect(order).toEqual(["bulk"]);
|
|
6558
7387
|
releaseBulk();
|
|
7388
|
+
await receiverPull;
|
|
7389
|
+
expect(order).toEqual(["bulk", "fast-pull"]);
|
|
6559
7390
|
await flushLoopMicrotasks();
|
|
6560
7391
|
triggerShutdown();
|
|
6561
7392
|
await loop;
|
|
@@ -6599,12 +7430,14 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6599
7430
|
await flushLoopMicrotasks();
|
|
6600
7431
|
watcher.emit();
|
|
6601
7432
|
await flushLoopMicrotasks();
|
|
7433
|
+
releaseBulk();
|
|
7434
|
+
// The full chore has no explicit safe checkpoint in this stub, so the
|
|
7435
|
+
// realtime pass begins only after the chore has released ownership.
|
|
6602
7436
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
6603
7437
|
await fastStarted;
|
|
6604
|
-
releaseBulk();
|
|
6605
7438
|
await flushLoopMicrotasks(20);
|
|
6606
|
-
// The
|
|
6607
|
-
// sync/rescue must not acquire the
|
|
7439
|
+
// The realtime pass owns the root lock until its own durable work ends. A
|
|
7440
|
+
// second sync/rescue must not acquire the lock in this interval.
|
|
6608
7441
|
expect(fs.existsSync(lockPathFor(hqRoot))).toBe(true);
|
|
6609
7442
|
releaseFast();
|
|
6610
7443
|
await flushLoopMicrotasks(20);
|
|
@@ -6779,59 +7612,68 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6779
7612
|
return () => { };
|
|
6780
7613
|
},
|
|
6781
7614
|
});
|
|
6782
|
-
|
|
6783
|
-
|
|
6784
|
-
|
|
6785
|
-
|
|
6786
|
-
|
|
6787
|
-
|
|
6788
|
-
|
|
6789
|
-
|
|
6790
|
-
|
|
6791
|
-
|
|
6792
|
-
|
|
6793
|
-
|
|
6794
|
-
|
|
6795
|
-
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6799
|
-
|
|
6800
|
-
|
|
6801
|
-
|
|
6802
|
-
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
const
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
7615
|
+
try {
|
|
7616
|
+
await flushLoopMicrotasks();
|
|
7617
|
+
watcher.emit();
|
|
7618
|
+
await flushLoopMicrotasks();
|
|
7619
|
+
runPass.mockClear();
|
|
7620
|
+
const events = [
|
|
7621
|
+
...Array.from({ length: 17 }, (_, index) => ({
|
|
7622
|
+
kind: "upsert",
|
|
7623
|
+
relativePath: `companies/acme/a-${index}.md`,
|
|
7624
|
+
contentHash: "sha256:x",
|
|
7625
|
+
mtime: "2026-08-26T00:00:00.000Z",
|
|
7626
|
+
originDeviceId: "peer",
|
|
7627
|
+
originTenantId: "acme",
|
|
7628
|
+
sequenceNumber: index,
|
|
7629
|
+
eventTimestamp: "2026-08-26T00:00:00.000Z",
|
|
7630
|
+
})),
|
|
7631
|
+
...Array.from({ length: 16 }, (_, index) => ({
|
|
7632
|
+
kind: "upsert",
|
|
7633
|
+
relativePath: `companies/beta/b-${index}.md`,
|
|
7634
|
+
contentHash: "sha256:x",
|
|
7635
|
+
mtime: "2026-08-26T00:00:00.000Z",
|
|
7636
|
+
originDeviceId: "peer",
|
|
7637
|
+
originTenantId: "beta",
|
|
7638
|
+
sequenceNumber: index + 17,
|
|
7639
|
+
eventTimestamp: "2026-08-26T00:00:00.000Z",
|
|
7640
|
+
})),
|
|
7641
|
+
];
|
|
7642
|
+
for (const [slug, prefix, count] of [["acme", "a", 17], ["beta", "b", 16]]) {
|
|
7643
|
+
const journal = readJournal(slug);
|
|
7644
|
+
for (let index = 0; index < count; index++) {
|
|
7645
|
+
journal.files[`${prefix}-${index}.md`] = {
|
|
7646
|
+
hash: "sha256:x",
|
|
7647
|
+
size: 16,
|
|
7648
|
+
syncedAt: "2026-08-26T00:00:00.000Z",
|
|
7649
|
+
direction: "down",
|
|
7650
|
+
};
|
|
7651
|
+
}
|
|
7652
|
+
writeJournal(slug, journal);
|
|
6817
7653
|
}
|
|
6818
|
-
|
|
7654
|
+
const receiverDrain = receiverSync({
|
|
7655
|
+
events,
|
|
7656
|
+
signal: new AbortController().signal,
|
|
7657
|
+
});
|
|
7658
|
+
await flushLoopMicrotasks(20);
|
|
7659
|
+
// The running full chore has not yet yielded a durable hand-off, so no
|
|
7660
|
+
// receiver work may overlap it. Once it releases ownership, the
|
|
7661
|
+
// scheduler admits each route's bounded work before the residual slow
|
|
7662
|
+
// beta batch (acme fast, beta fast, then beta slow).
|
|
7663
|
+
expect(runPass).not.toHaveBeenCalled();
|
|
7664
|
+
releaseBulk();
|
|
7665
|
+
await receiverDrain;
|
|
7666
|
+
const scopedCalls = runPass.mock.calls.filter(([argv]) => argv.includes("--scope-path"));
|
|
7667
|
+
expect(scopedCalls).toHaveLength(3);
|
|
7668
|
+
expect(scopedCalls.flat().join(" ")).toContain("a-16.md");
|
|
7669
|
+
expect(scopedCalls.flat().join(" ")).toContain("b-14.md");
|
|
7670
|
+
}
|
|
7671
|
+
finally {
|
|
7672
|
+
releaseBulk();
|
|
7673
|
+
triggerShutdown();
|
|
7674
|
+
await loop;
|
|
7675
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6819
7676
|
}
|
|
6820
|
-
const receiverDrain = receiverSync({
|
|
6821
|
-
events,
|
|
6822
|
-
signal: new AbortController().signal,
|
|
6823
|
-
});
|
|
6824
|
-
await flushLoopMicrotasks(20);
|
|
6825
|
-
// The 33-event mixed delivery peels its first 32 bounded paths into the
|
|
6826
|
-
// fast lane (17 acme + 15 beta); the final beta event remains slow.
|
|
6827
|
-
expect(runPass).toHaveBeenCalledTimes(2);
|
|
6828
|
-
expect(runPass.mock.calls.flat().join(" ")).toContain("a-16.md");
|
|
6829
|
-
expect(runPass.mock.calls.flat().join(" ")).toContain("b-14.md");
|
|
6830
|
-
releaseBulk();
|
|
6831
|
-
await receiverDrain;
|
|
6832
|
-
triggerShutdown();
|
|
6833
|
-
await loop;
|
|
6834
|
-
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6835
7677
|
});
|
|
6836
7678
|
it("fast lane: queues slow work behind a receiver pass that owns the root lock", async () => {
|
|
6837
7679
|
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-slow-queue-"));
|
|
@@ -7003,9 +7845,12 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7003
7845
|
await flushLoopMicrotasks();
|
|
7004
7846
|
watcher.emit(paths[0][1], { paths: new Map(paths) });
|
|
7005
7847
|
await flushLoopMicrotasks(30);
|
|
7006
|
-
|
|
7848
|
+
// The full pass has not reached a durable checkpoint, so it remains the
|
|
7849
|
+
// only journal writer. The scoped realtime pass is admitted immediately
|
|
7850
|
+
// after this safe hand-off, preserving both sets of rows.
|
|
7851
|
+
expect(Object.keys(readJournal("indigo").files)).toHaveLength(0);
|
|
7007
7852
|
releaseBulk();
|
|
7008
|
-
await flushLoopMicrotasks();
|
|
7853
|
+
await flushLoopMicrotasks(30);
|
|
7009
7854
|
const journal = readJournal("indigo").files;
|
|
7010
7855
|
expect(Object.keys(journal)).toHaveLength(paths.length * 2);
|
|
7011
7856
|
for (let index = 0; index < paths.length; index++) {
|
|
@@ -7057,10 +7902,12 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7057
7902
|
watcher.emit(relativePath, { paths: new Map([[absolutePath, relativePath]]) });
|
|
7058
7903
|
}
|
|
7059
7904
|
await flushLoopMicrotasks(60);
|
|
7060
|
-
|
|
7905
|
+
// A full pass without a checkpoint remains the sole mutable-state owner.
|
|
7906
|
+
// The queued realtime batches run immediately after it completes.
|
|
7907
|
+
expect(completed).toEqual([]);
|
|
7061
7908
|
releaseBulk();
|
|
7062
|
-
await flushLoopMicrotasks();
|
|
7063
|
-
expect(completed).
|
|
7909
|
+
await flushLoopMicrotasks(60);
|
|
7910
|
+
expect(completed).toEqual(["bulk", "fast", "fast", "fast", "fast", "fast"]);
|
|
7064
7911
|
triggerShutdown();
|
|
7065
7912
|
await loop;
|
|
7066
7913
|
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
@@ -7294,18 +8141,33 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7294
8141
|
// No pending FakeClock timers left behind by the driver's debounce.
|
|
7295
8142
|
expect(clock.pendingTimerCount()).toBe(0);
|
|
7296
8143
|
});
|
|
7297
|
-
it("a
|
|
8144
|
+
it("a non-auth pass error keeps the watch loop alive for the next poll", async () => {
|
|
7298
8145
|
const watcher = makeWatcherStub();
|
|
7299
8146
|
const runPass = vi.fn().mockResolvedValue(1);
|
|
7300
|
-
const
|
|
8147
|
+
const sleep = makeSteppableSleep();
|
|
8148
|
+
let triggerShutdown = () => { };
|
|
8149
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7301
8150
|
runPass,
|
|
7302
8151
|
clock: new FakeClock(),
|
|
7303
8152
|
createWatcher: () => watcher,
|
|
7304
|
-
sleep:
|
|
7305
|
-
onShutdownSignal: () =>
|
|
8153
|
+
sleep: sleep.sleep,
|
|
8154
|
+
onShutdownSignal: (handler) => {
|
|
8155
|
+
triggerShutdown = handler;
|
|
8156
|
+
return () => { };
|
|
8157
|
+
},
|
|
7306
8158
|
});
|
|
7307
|
-
|
|
7308
|
-
|
|
8159
|
+
try {
|
|
8160
|
+
await flushLoopMicrotasks();
|
|
8161
|
+
expect(runPass).toHaveBeenCalledTimes(1);
|
|
8162
|
+
sleep.tick();
|
|
8163
|
+
await flushLoopMicrotasks();
|
|
8164
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8165
|
+
}
|
|
8166
|
+
finally {
|
|
8167
|
+
triggerShutdown();
|
|
8168
|
+
await expect(loop).resolves.toBe(0);
|
|
8169
|
+
expect(watcher.started).toBe(false);
|
|
8170
|
+
}
|
|
7309
8171
|
});
|
|
7310
8172
|
it("an auth-required watch pass parks in-process instead of exiting", async () => {
|
|
7311
8173
|
const watcher = makeWatcherStub();
|
|
@@ -7354,29 +8216,92 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7354
8216
|
it("a TRANSIENT_NETWORK_EXIT poll pass does NOT surface as a crash — the loop retries (HQ-SYNC-1W)", async () => {
|
|
7355
8217
|
// A transient offline blip must keep the watcher alive: the loop logs and
|
|
7356
8218
|
// polls again instead of returning the code and letting the process exit
|
|
7357
|
-
// (which the menubar reports as "watcher exited unexpectedly"). A
|
|
7358
|
-
//
|
|
8219
|
+
// (which the menubar reports as "watcher exited unexpectedly"). A later
|
|
8220
|
+
// generic failure is also retried rather than becoming a daemon exit.
|
|
7359
8221
|
const watcher = makeWatcherStub();
|
|
7360
8222
|
let n = 0;
|
|
7361
8223
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7362
8224
|
n++;
|
|
7363
|
-
// First pass: machine is offline (transient). Second pass: a
|
|
7364
|
-
//
|
|
8225
|
+
// First pass: machine is offline (transient). Second pass: a generic
|
|
8226
|
+
// failed pass, which must take the same non-fatal retry path.
|
|
7365
8227
|
return n === 1 ? TRANSIENT_NETWORK_EXIT : 1;
|
|
7366
8228
|
});
|
|
7367
|
-
const
|
|
8229
|
+
const sleep = makeSteppableSleep();
|
|
8230
|
+
let triggerShutdown = () => { };
|
|
8231
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7368
8232
|
runPass,
|
|
7369
8233
|
clock: new FakeClock(),
|
|
7370
8234
|
createWatcher: () => watcher,
|
|
7371
|
-
sleep:
|
|
7372
|
-
onShutdownSignal: () =>
|
|
8235
|
+
sleep: sleep.sleep,
|
|
8236
|
+
onShutdownSignal: (handler) => {
|
|
8237
|
+
triggerShutdown = handler;
|
|
8238
|
+
return () => { };
|
|
8239
|
+
},
|
|
7373
8240
|
});
|
|
7374
|
-
|
|
8241
|
+
await flushLoopMicrotasks();
|
|
8242
|
+
sleep.tick();
|
|
8243
|
+
await flushLoopMicrotasks();
|
|
8244
|
+
// The transient pass did NOT end the loop — it ran again, and neither
|
|
8245
|
+
// error causes the daemon itself to exit.
|
|
7375
8246
|
expect(runPass).toHaveBeenCalledTimes(2);
|
|
7376
|
-
|
|
7377
|
-
expect(
|
|
7378
|
-
expect(
|
|
8247
|
+
triggerShutdown();
|
|
8248
|
+
await expect(loop).resolves.toBe(0);
|
|
8249
|
+
expect(watcher.disposed).toBe(false);
|
|
8250
|
+
});
|
|
8251
|
+
it("keeps the watcher alive for an unresolved company watch route (HQ-SYNC-COMPANY-ROUTE-UID)", async () => {
|
|
8252
|
+
const watcher = makeWatcherStub();
|
|
8253
|
+
const client = makeVaultStub({
|
|
8254
|
+
entityGet: () => Promise.reject(new VaultNotFoundError("company deleted")),
|
|
8255
|
+
});
|
|
8256
|
+
const findInMyNamespace = vi.fn().mockResolvedValue(null);
|
|
8257
|
+
client.entity.findInMyNamespace = findInMyNamespace;
|
|
8258
|
+
const routedPass = makeDeps({ createVaultClient: () => client });
|
|
8259
|
+
const routedArgv = buildRoutePullArgv({ kind: "company", slug: "cmp_ghost" }, ["--companies", "--watch", "--hq-root", "/tmp/hq"]);
|
|
8260
|
+
let attempts = 0;
|
|
8261
|
+
let routedExit;
|
|
8262
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
8263
|
+
attempts += 1;
|
|
8264
|
+
if (attempts === 1) {
|
|
8265
|
+
routedExit = await runRunner(routedArgv, routedPass);
|
|
8266
|
+
return routedExit;
|
|
8267
|
+
}
|
|
8268
|
+
return UNRESOLVED_COMPANY_EXIT;
|
|
8269
|
+
});
|
|
8270
|
+
const sleep = makeSteppableSleep();
|
|
8271
|
+
let triggerShutdown = () => { };
|
|
8272
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
8273
|
+
runPass,
|
|
8274
|
+
clock: new FakeClock(),
|
|
8275
|
+
createWatcher: () => watcher,
|
|
8276
|
+
sleep: sleep.sleep,
|
|
8277
|
+
onShutdownSignal: (handler) => {
|
|
8278
|
+
triggerShutdown = handler;
|
|
8279
|
+
return () => { };
|
|
8280
|
+
},
|
|
8281
|
+
});
|
|
8282
|
+
await flushLoopMicrotasks();
|
|
8283
|
+
sleep.tick();
|
|
8284
|
+
await flushLoopMicrotasks();
|
|
8285
|
+
expect(routedArgv).toEqual([
|
|
8286
|
+
"--company",
|
|
8287
|
+
"cmp_ghost",
|
|
8288
|
+
"--direction",
|
|
8289
|
+
"pull",
|
|
8290
|
+
"--hq-root",
|
|
8291
|
+
"/tmp/hq",
|
|
8292
|
+
]);
|
|
8293
|
+
expect(routedExit).toBe(UNRESOLVED_COMPANY_EXIT);
|
|
8294
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8295
|
+
triggerShutdown();
|
|
8296
|
+
await expect(loop).resolves.toBe(0);
|
|
7379
8297
|
expect(watcher.disposed).toBe(false);
|
|
8298
|
+
expect(findInMyNamespace).toHaveBeenCalledWith("company", "cmp_ghost");
|
|
8299
|
+
expect(routedPass.stderr.events()).toContainEqual({
|
|
8300
|
+
type: "error",
|
|
8301
|
+
company: "cmp_ghost",
|
|
8302
|
+
path: "(company-resolution)",
|
|
8303
|
+
message: "company entity lookup returned 404",
|
|
8304
|
+
});
|
|
7380
8305
|
});
|
|
7381
8306
|
it("keeps the watch loop alive when a company leg returns transient exit 75 (HQ-SYNC-X)", async () => {
|
|
7382
8307
|
const watcher = makeWatcherStub();
|
|
@@ -7393,24 +8318,36 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7393
8318
|
let passes = 0;
|
|
7394
8319
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7395
8320
|
passes += 1;
|
|
7396
|
-
//
|
|
7397
|
-
//
|
|
8321
|
+
// The first pass uses the real company-leg fanout path, not a synthetic
|
|
8322
|
+
// exit code. A later generic failure must also leave the daemon alive.
|
|
7398
8323
|
return passes === 1 ? runRunner(["--companies"], transientPass) : 1;
|
|
7399
8324
|
});
|
|
7400
|
-
const
|
|
8325
|
+
const sleep = makeSteppableSleep();
|
|
8326
|
+
let triggerShutdown = () => { };
|
|
8327
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7401
8328
|
runPass,
|
|
7402
8329
|
clock: new FakeClock(),
|
|
7403
8330
|
createWatcher: () => watcher,
|
|
7404
|
-
sleep:
|
|
7405
|
-
onShutdownSignal: () =>
|
|
8331
|
+
sleep: sleep.sleep,
|
|
8332
|
+
onShutdownSignal: (handler) => {
|
|
8333
|
+
triggerShutdown = handler;
|
|
8334
|
+
return () => { };
|
|
8335
|
+
},
|
|
7406
8336
|
});
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
.
|
|
7412
|
-
|
|
7413
|
-
|
|
8337
|
+
try {
|
|
8338
|
+
await waitForPollSleep(sleep);
|
|
8339
|
+
sleep.tick();
|
|
8340
|
+
await flushLoopMicrotasks();
|
|
8341
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
8342
|
+
expect(transientPass.stderr
|
|
8343
|
+
.events()
|
|
8344
|
+
.some((event) => event.type === "error" && event.path === "(company)")).toBe(false);
|
|
8345
|
+
expect(watcher.disposed).toBe(false);
|
|
8346
|
+
}
|
|
8347
|
+
finally {
|
|
8348
|
+
triggerShutdown();
|
|
8349
|
+
await expect(loop).resolves.toBe(0);
|
|
8350
|
+
}
|
|
7414
8351
|
});
|
|
7415
8352
|
it("a PARTIAL_SYNC_EXIT poll pass does NOT surface as a crash — the loop retries", async () => {
|
|
7416
8353
|
// A partial pass (some per-file transfers threw: oversized object on an
|
|
@@ -7424,22 +8361,32 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7424
8361
|
let n = 0;
|
|
7425
8362
|
const runPass = vi.fn().mockImplementation(async () => {
|
|
7426
8363
|
n++;
|
|
7427
|
-
// Two partial passes prove a
|
|
7428
|
-
// the loop either. A
|
|
8364
|
+
// Two partial passes prove a deterministic per-file failure doesn't kill
|
|
8365
|
+
// the loop either. A generic failure must also leave it alive.
|
|
7429
8366
|
return n <= 2 ? PARTIAL_SYNC_EXIT : 1;
|
|
7430
8367
|
});
|
|
7431
|
-
const
|
|
8368
|
+
const sleep = makeSteppableSleep();
|
|
8369
|
+
let triggerShutdown = () => { };
|
|
8370
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7432
8371
|
runPass,
|
|
7433
8372
|
clock: new FakeClock(),
|
|
7434
8373
|
createWatcher: () => watcher,
|
|
7435
|
-
sleep:
|
|
7436
|
-
onShutdownSignal: () =>
|
|
8374
|
+
sleep: sleep.sleep,
|
|
8375
|
+
onShutdownSignal: (handler) => {
|
|
8376
|
+
triggerShutdown = handler;
|
|
8377
|
+
return () => { };
|
|
8378
|
+
},
|
|
7437
8379
|
});
|
|
8380
|
+
await flushLoopMicrotasks();
|
|
8381
|
+
sleep.tick();
|
|
8382
|
+
await flushLoopMicrotasks();
|
|
8383
|
+
sleep.tick();
|
|
8384
|
+
await flushLoopMicrotasks();
|
|
7438
8385
|
// Both partial passes stayed alive — the loop ran a third time…
|
|
7439
8386
|
expect(runPass).toHaveBeenCalledTimes(3);
|
|
7440
|
-
// …and
|
|
7441
|
-
|
|
7442
|
-
expect(
|
|
8387
|
+
// …and neither partial nor generic pass failure exits the daemon.
|
|
8388
|
+
triggerShutdown();
|
|
8389
|
+
await expect(loop).resolves.toBe(0);
|
|
7443
8390
|
expect(watcher.disposed).toBe(false);
|
|
7444
8391
|
});
|
|
7445
8392
|
it("alerts when a partial pass failed before a company's download leg", async () => {
|
|
@@ -7467,24 +8414,33 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
7467
8414
|
return 1;
|
|
7468
8415
|
});
|
|
7469
8416
|
// Every clock read advances a full cadence interval, so the tick after
|
|
7470
|
-
// the partial pass is a scheduled full reconcile
|
|
7471
|
-
// code this test asserts on.
|
|
8417
|
+
// the partial pass is a scheduled full reconcile with a generic failure.
|
|
7472
8418
|
let monotonic = 0;
|
|
8419
|
+
const sleep = makeSteppableSleep();
|
|
8420
|
+
let triggerShutdown = () => { };
|
|
7473
8421
|
try {
|
|
7474
|
-
const
|
|
8422
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
7475
8423
|
runPass,
|
|
7476
8424
|
clock: new FakeClock(),
|
|
7477
8425
|
createWatcher: () => watcher,
|
|
7478
8426
|
monotonicNow: () => (monotonic += DEFAULT_FULL_RECONCILE_MS),
|
|
7479
|
-
sleep:
|
|
7480
|
-
onShutdownSignal: () =>
|
|
8427
|
+
sleep: sleep.sleep,
|
|
8428
|
+
onShutdownSignal: (handler) => {
|
|
8429
|
+
triggerShutdown = handler;
|
|
8430
|
+
return () => { };
|
|
8431
|
+
},
|
|
7481
8432
|
});
|
|
7482
|
-
|
|
8433
|
+
await flushLoopMicrotasks();
|
|
8434
|
+
sleep.tick();
|
|
8435
|
+
await flushLoopMicrotasks();
|
|
8436
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
7483
8437
|
const stderr = stderrWrite.mock.calls
|
|
7484
8438
|
.map(([chunk]) => String(chunk))
|
|
7485
8439
|
.join("");
|
|
7486
8440
|
expect(stderr).toContain("hq-sync-runner: COMPANY PASS FAILED company=personal message=RangeError Maximum call stack size exceeded (downloads for this company did not run)");
|
|
7487
8441
|
expect(stderr).toContain("hq-sync-runner: watch pass completed with per-file errors; failed-companies=personal; staying alive, retrying next poll");
|
|
8442
|
+
triggerShutdown();
|
|
8443
|
+
await expect(loop).resolves.toBe(0);
|
|
7488
8444
|
}
|
|
7489
8445
|
finally {
|
|
7490
8446
|
stderrWrite.mockRestore();
|
|
@@ -7671,6 +8627,85 @@ describe("runRunnerWithLoop — watch-mode auth-wait", () => {
|
|
|
7671
8627
|
});
|
|
7672
8628
|
});
|
|
7673
8629
|
describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
|
|
8630
|
+
it("hands a wake realtime drain to the guard before a long-running full chore completes", async () => {
|
|
8631
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
8632
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
8633
|
+
const watcher = makeWatcherStub();
|
|
8634
|
+
let triggerShutdown = () => { };
|
|
8635
|
+
let releaseCheckpoint = () => { };
|
|
8636
|
+
const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
8637
|
+
let releaseRealtime = () => { };
|
|
8638
|
+
const realtimeGate = new Promise((resolve) => { releaseRealtime = resolve; });
|
|
8639
|
+
const order = [];
|
|
8640
|
+
let passes = 0;
|
|
8641
|
+
let runRealtime = null;
|
|
8642
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
|
|
8643
|
+
runPass: vi.fn().mockImplementation(async (_argv, checkpoint) => {
|
|
8644
|
+
passes += 1;
|
|
8645
|
+
if (passes === 2) {
|
|
8646
|
+
order.push("chore");
|
|
8647
|
+
// The fake pass has completed its current durable unit and is now
|
|
8648
|
+
// at the same explicit hand-off seam production uses between
|
|
8649
|
+
// fully-settled fanout targets.
|
|
8650
|
+
await checkpointGate;
|
|
8651
|
+
await checkpoint?.();
|
|
8652
|
+
order.push("chore-complete");
|
|
8653
|
+
}
|
|
8654
|
+
return completedPassOutcome();
|
|
8655
|
+
}),
|
|
8656
|
+
runPassSupportsCooperativeCheckpoint: true,
|
|
8657
|
+
clock: new FakeClock(),
|
|
8658
|
+
createWatcher: () => watcher,
|
|
8659
|
+
sleep: makeSteppableSleep().sleep,
|
|
8660
|
+
onShutdownSignal: (handler) => {
|
|
8661
|
+
triggerShutdown = handler;
|
|
8662
|
+
return () => { };
|
|
8663
|
+
},
|
|
8664
|
+
startRealtimeScheduler: async (opts) => {
|
|
8665
|
+
// This is the same wake path used by the realtime scheduler: it
|
|
8666
|
+
// enters runGuardedTask behind the coordinator worker.
|
|
8667
|
+
runRealtime = () => opts.runGuarded(async () => {
|
|
8668
|
+
order.push("realtime");
|
|
8669
|
+
await realtimeGate;
|
|
8670
|
+
order.push("realtime-complete");
|
|
8671
|
+
});
|
|
8672
|
+
return {
|
|
8673
|
+
signal: () => undefined,
|
|
8674
|
+
checkHighWater: async () => undefined,
|
|
8675
|
+
dispose: async () => undefined,
|
|
8676
|
+
};
|
|
8677
|
+
},
|
|
8678
|
+
});
|
|
8679
|
+
try {
|
|
8680
|
+
await flushLoopMicrotasks(30);
|
|
8681
|
+
watcher.emit();
|
|
8682
|
+
await flushLoopMicrotasks(20);
|
|
8683
|
+
expect(order).toEqual(["chore"]);
|
|
8684
|
+
const realtime = runRealtime();
|
|
8685
|
+
releaseCheckpoint();
|
|
8686
|
+
await flushLoopMicrotasks(30);
|
|
8687
|
+
expect(order).toEqual(["chore", "realtime"]);
|
|
8688
|
+
releaseRealtime();
|
|
8689
|
+
await realtime;
|
|
8690
|
+
await flushLoopMicrotasks(20);
|
|
8691
|
+
expect(order).toEqual([
|
|
8692
|
+
"chore",
|
|
8693
|
+
"realtime",
|
|
8694
|
+
"realtime-complete",
|
|
8695
|
+
"chore-complete",
|
|
8696
|
+
]);
|
|
8697
|
+
}
|
|
8698
|
+
finally {
|
|
8699
|
+
releaseCheckpoint();
|
|
8700
|
+
releaseRealtime();
|
|
8701
|
+
triggerShutdown();
|
|
8702
|
+
await loop;
|
|
8703
|
+
if (previousCoordinator === undefined)
|
|
8704
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
8705
|
+
else
|
|
8706
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
8707
|
+
}
|
|
8708
|
+
});
|
|
7674
8709
|
it("feeds a watcher-originated V2 drain through the same single-flight guard as the poll", async () => {
|
|
7675
8710
|
const watcher = makeWatcherStub();
|
|
7676
8711
|
let triggerShutdown = () => { };
|
|
@@ -7783,6 +8818,173 @@ describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
|
|
|
7783
8818
|
await loop;
|
|
7784
8819
|
});
|
|
7785
8820
|
});
|
|
8821
|
+
describe("CooperativePassScheduler", () => {
|
|
8822
|
+
it("admits an explicitly safe fast realtime pass while a slow scoped realtime pass remains in flight", async () => {
|
|
8823
|
+
const scheduler = new CooperativePassScheduler(() => 0);
|
|
8824
|
+
const order = [];
|
|
8825
|
+
let releaseSlow = () => { };
|
|
8826
|
+
const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
|
|
8827
|
+
let markSlowStarted = () => { };
|
|
8828
|
+
const slowStarted = new Promise((resolve) => { markSlowStarted = resolve; });
|
|
8829
|
+
const slow = scheduler.submit(async () => {
|
|
8830
|
+
order.push("slow-start");
|
|
8831
|
+
markSlowStarted();
|
|
8832
|
+
await slowGate;
|
|
8833
|
+
order.push("slow-complete");
|
|
8834
|
+
return 0;
|
|
8835
|
+
}, "realtime", "slow");
|
|
8836
|
+
await slowStarted;
|
|
8837
|
+
const fast = scheduler.submit(async () => {
|
|
8838
|
+
order.push("fast");
|
|
8839
|
+
return 0;
|
|
8840
|
+
}, "realtime", "fast", true);
|
|
8841
|
+
try {
|
|
8842
|
+
await Promise.resolve();
|
|
8843
|
+
await Promise.resolve();
|
|
8844
|
+
expect([...order]).toEqual(["slow-start", "fast"]);
|
|
8845
|
+
}
|
|
8846
|
+
finally {
|
|
8847
|
+
releaseSlow();
|
|
8848
|
+
await Promise.all([slow, fast]);
|
|
8849
|
+
}
|
|
8850
|
+
});
|
|
8851
|
+
it("preserves journal order and resumes full work across repeated randomized hand-offs", async () => {
|
|
8852
|
+
// These deterministic seeds deliberately vary the number and enqueue
|
|
8853
|
+
// ordering of realtime tasks. Every append asserts one durable-state owner
|
|
8854
|
+
// at a time; the full continuation must resume after every realtime burst.
|
|
8855
|
+
for (let seed = 1; seed <= 32; seed += 1) {
|
|
8856
|
+
let now = 0;
|
|
8857
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
8858
|
+
const journal = [];
|
|
8859
|
+
const expected = new Set();
|
|
8860
|
+
let writer = null;
|
|
8861
|
+
let maximumWriters = 0;
|
|
8862
|
+
const append = async (entry) => {
|
|
8863
|
+
expect(writer).toBeNull();
|
|
8864
|
+
writer = entry;
|
|
8865
|
+
maximumWriters = Math.max(maximumWriters, 1);
|
|
8866
|
+
// A journal append crosses an async durable-write boundary. If the
|
|
8867
|
+
// scheduler admitted another pass here, its ownership assertion would
|
|
8868
|
+
// fail rather than merely observing JavaScript's synchronous turn.
|
|
8869
|
+
await Promise.resolve();
|
|
8870
|
+
expect(writer).toBe(entry);
|
|
8871
|
+
journal.push(entry);
|
|
8872
|
+
writer = null;
|
|
8873
|
+
};
|
|
8874
|
+
let releaseCheckpoint = () => { };
|
|
8875
|
+
const checkpointReady = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
8876
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
8877
|
+
expected.add(`chore-${seed}-before`);
|
|
8878
|
+
await append(`chore-${seed}-before`);
|
|
8879
|
+
await checkpointReady;
|
|
8880
|
+
await checkpoint();
|
|
8881
|
+
expected.add(`chore-${seed}-after`);
|
|
8882
|
+
await append(`chore-${seed}-after`);
|
|
8883
|
+
return 0;
|
|
8884
|
+
}, "chore");
|
|
8885
|
+
await Promise.resolve();
|
|
8886
|
+
const realtimeCount = (seed % 7) + 1;
|
|
8887
|
+
const shuffledIndexes = Array.from({ length: realtimeCount }, (_, index) => index);
|
|
8888
|
+
let randomState = seed;
|
|
8889
|
+
for (let index = shuffledIndexes.length - 1; index > 0; index -= 1) {
|
|
8890
|
+
randomState = (randomState * 1_664_525 + 1_013_904_223) >>> 0;
|
|
8891
|
+
const swapIndex = randomState % (index + 1);
|
|
8892
|
+
[shuffledIndexes[index], shuffledIndexes[swapIndex]] = [
|
|
8893
|
+
shuffledIndexes[swapIndex],
|
|
8894
|
+
shuffledIndexes[index],
|
|
8895
|
+
];
|
|
8896
|
+
}
|
|
8897
|
+
const realtime = shuffledIndexes.map((index) => {
|
|
8898
|
+
const entry = `realtime-${seed}-${(index * 17 + seed) % realtimeCount}`;
|
|
8899
|
+
expected.add(entry);
|
|
8900
|
+
return scheduler.submit(async () => {
|
|
8901
|
+
// Shuffle the microtask timing without ever allowing a second writer.
|
|
8902
|
+
if ((seed + index) % 3 === 0)
|
|
8903
|
+
await Promise.resolve();
|
|
8904
|
+
await append(entry);
|
|
8905
|
+
now += 1;
|
|
8906
|
+
return 0;
|
|
8907
|
+
}, "realtime");
|
|
8908
|
+
});
|
|
8909
|
+
releaseCheckpoint();
|
|
8910
|
+
await Promise.all([chore, ...realtime]);
|
|
8911
|
+
expect(maximumWriters).toBe(1);
|
|
8912
|
+
expect(new Set(journal)).toEqual(expected);
|
|
8913
|
+
expect(journal).toHaveLength(expected.size);
|
|
8914
|
+
expect(journal[0]).toBe(`chore-${seed}-before`);
|
|
8915
|
+
expect(journal.at(-1)).toBe(`chore-${seed}-after`);
|
|
8916
|
+
}
|
|
8917
|
+
});
|
|
8918
|
+
it("lets realtime win at 15 minutes minus one millisecond, then gives the chore the exact-boundary admission", async () => {
|
|
8919
|
+
const runCase = async (elapsedMs) => {
|
|
8920
|
+
let now = 0;
|
|
8921
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
8922
|
+
const order = [];
|
|
8923
|
+
let releaseActive = () => { };
|
|
8924
|
+
const activeGate = new Promise((resolve) => { releaseActive = resolve; });
|
|
8925
|
+
const activeRealtime = scheduler.submit(async () => {
|
|
8926
|
+
order.push("active-realtime");
|
|
8927
|
+
await activeGate;
|
|
8928
|
+
return 0;
|
|
8929
|
+
}, "realtime");
|
|
8930
|
+
await Promise.resolve();
|
|
8931
|
+
const chore = scheduler.submit(async () => {
|
|
8932
|
+
order.push("chore");
|
|
8933
|
+
return 0;
|
|
8934
|
+
}, "chore");
|
|
8935
|
+
const queuedRealtime = scheduler.submit(async () => {
|
|
8936
|
+
order.push("queued-realtime");
|
|
8937
|
+
return 0;
|
|
8938
|
+
}, "realtime");
|
|
8939
|
+
now = elapsedMs;
|
|
8940
|
+
releaseActive();
|
|
8941
|
+
await Promise.all([activeRealtime, chore, queuedRealtime]);
|
|
8942
|
+
return order;
|
|
8943
|
+
};
|
|
8944
|
+
await expect(runCase(CHORE_STARVATION_CAP_MS - 1)).resolves.toEqual([
|
|
8945
|
+
"active-realtime",
|
|
8946
|
+
"queued-realtime",
|
|
8947
|
+
"chore",
|
|
8948
|
+
]);
|
|
8949
|
+
await expect(runCase(CHORE_STARVATION_CAP_MS)).resolves.toEqual([
|
|
8950
|
+
"active-realtime",
|
|
8951
|
+
"chore",
|
|
8952
|
+
"queued-realtime",
|
|
8953
|
+
]);
|
|
8954
|
+
});
|
|
8955
|
+
it("resumes a handed-off chore when a continuous realtime burst reaches the 15-minute cap", async () => {
|
|
8956
|
+
let now = 0;
|
|
8957
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
8958
|
+
const order = [];
|
|
8959
|
+
let releaseCheckpoint = () => { };
|
|
8960
|
+
const checkpointReady = new Promise((resolve) => { releaseCheckpoint = resolve; });
|
|
8961
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
8962
|
+
order.push("chore-before");
|
|
8963
|
+
await checkpointReady;
|
|
8964
|
+
await checkpoint();
|
|
8965
|
+
order.push("chore-after");
|
|
8966
|
+
return 0;
|
|
8967
|
+
}, "chore");
|
|
8968
|
+
await Promise.resolve();
|
|
8969
|
+
const firstRealtime = scheduler.submit(async () => {
|
|
8970
|
+
order.push("realtime-before-cap");
|
|
8971
|
+
now = CHORE_STARVATION_CAP_MS;
|
|
8972
|
+
return 0;
|
|
8973
|
+
}, "realtime");
|
|
8974
|
+
const laterRealtime = scheduler.submit(async () => {
|
|
8975
|
+
order.push("realtime-after-cap");
|
|
8976
|
+
return 0;
|
|
8977
|
+
}, "realtime");
|
|
8978
|
+
releaseCheckpoint();
|
|
8979
|
+
await Promise.all([chore, firstRealtime, laterRealtime]);
|
|
8980
|
+
expect(order).toEqual([
|
|
8981
|
+
"chore-before",
|
|
8982
|
+
"realtime-before-cap",
|
|
8983
|
+
"chore-after",
|
|
8984
|
+
"realtime-after-cap",
|
|
8985
|
+
]);
|
|
8986
|
+
});
|
|
8987
|
+
});
|
|
7786
8988
|
describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
|
|
7787
8989
|
const watchArgv = [
|
|
7788
8990
|
"--companies",
|
|
@@ -9946,6 +11148,9 @@ describe("runRunnerWithLoop — Phase 3 event-sync wiring (US-017/018/019)", ()
|
|
|
9946
11148
|
"/tmp/hq",
|
|
9947
11149
|
], {
|
|
9948
11150
|
runPass: runPass,
|
|
11151
|
+
...(opts.supportsCooperativeCheckpoint
|
|
11152
|
+
? { runPassSupportsCooperativeCheckpoint: true }
|
|
11153
|
+
: {}),
|
|
9949
11154
|
clock,
|
|
9950
11155
|
createWatcher: (options) => {
|
|
9951
11156
|
opts.onCreateWatcher?.({
|
|
@@ -10058,6 +11263,123 @@ describe("runRunnerWithLoop — Phase 3 event-sync wiring (US-017/018/019)", ()
|
|
|
10058
11263
|
shutdown();
|
|
10059
11264
|
await loop;
|
|
10060
11265
|
});
|
|
11266
|
+
it("records initial-pull writes before its cooperative checkpoint can admit watcher work", async () => {
|
|
11267
|
+
const markContentPublished = vi.fn();
|
|
11268
|
+
const startEventSync = vi.fn().mockResolvedValue({
|
|
11269
|
+
publishBatch: vi.fn(),
|
|
11270
|
+
markContentPublished,
|
|
11271
|
+
receiver: { start: async () => { }, dispose: async () => { }, connected: true },
|
|
11272
|
+
ownDeviceId: "dev-test",
|
|
11273
|
+
dispose: vi.fn().mockResolvedValue(undefined),
|
|
11274
|
+
});
|
|
11275
|
+
const watcher = makeBatchWatcherStub();
|
|
11276
|
+
let suppressionWasInstalledWhenRealtimeStarted;
|
|
11277
|
+
const runPass = vi.fn(async (argv, checkpoint, onJournalsReady, onAppliedUpserts) => {
|
|
11278
|
+
if (argv.includes("--scope-path")) {
|
|
11279
|
+
suppressionWasInstalledWhenRealtimeStarted = markContentPublished
|
|
11280
|
+
.mock.calls
|
|
11281
|
+
.some(([relativePath]) => relativePath === "companies/indigo/initial-pull.md");
|
|
11282
|
+
return completedPassOutcome();
|
|
11283
|
+
}
|
|
11284
|
+
onJournalsReady?.();
|
|
11285
|
+
await microtasks();
|
|
11286
|
+
watcher.emit(undefined, {
|
|
11287
|
+
paths: new Map([[
|
|
11288
|
+
"/tmp/hq/companies/indigo/watcher-arrived-during-pull.md",
|
|
11289
|
+
"companies/indigo/watcher-arrived-during-pull.md",
|
|
11290
|
+
]]),
|
|
11291
|
+
changes: new Map([[
|
|
11292
|
+
"/tmp/hq/companies/indigo/watcher-arrived-during-pull.md",
|
|
11293
|
+
{ kind: "change" },
|
|
11294
|
+
]]),
|
|
11295
|
+
});
|
|
11296
|
+
// Let the watcher submit its realtime work before the full pass reaches
|
|
11297
|
+
// its explicit hand-off. No timer is involved: the submission is a
|
|
11298
|
+
// promise-only scheduling boundary.
|
|
11299
|
+
await microtasks();
|
|
11300
|
+
onAppliedUpserts?.([{
|
|
11301
|
+
relativePath: "companies/indigo/initial-pull.md",
|
|
11302
|
+
contentHash: "sha256:initial-pull",
|
|
11303
|
+
}]);
|
|
11304
|
+
await checkpoint?.();
|
|
11305
|
+
return {
|
|
11306
|
+
exitCode: 0,
|
|
11307
|
+
result: {
|
|
11308
|
+
pushPathResults: [],
|
|
11309
|
+
appliedUpserts: [{
|
|
11310
|
+
relativePath: "companies/indigo/initial-pull.md",
|
|
11311
|
+
contentHash: "sha256:initial-pull",
|
|
11312
|
+
}],
|
|
11313
|
+
},
|
|
11314
|
+
};
|
|
11315
|
+
});
|
|
11316
|
+
const { loop, shutdown } = runLoop({
|
|
11317
|
+
claims: EMAIL_CLAIMS,
|
|
11318
|
+
startEventSync,
|
|
11319
|
+
runPass,
|
|
11320
|
+
watcher,
|
|
11321
|
+
supportsCooperativeCheckpoint: true,
|
|
11322
|
+
});
|
|
11323
|
+
await microtasks(16);
|
|
11324
|
+
expect(suppressionWasInstalledWhenRealtimeStarted).toBe(true);
|
|
11325
|
+
shutdown();
|
|
11326
|
+
await loop;
|
|
11327
|
+
});
|
|
11328
|
+
it("does not let a yielding warm-up catch-up overwrite a newer watcher unlink", async () => {
|
|
11329
|
+
const watcher = makeBatchWatcherStub();
|
|
11330
|
+
let resolveCatchup;
|
|
11331
|
+
watcher.collectWarmupCatchupAsync = vi.fn(() => new Promise((resolve) => {
|
|
11332
|
+
resolveCatchup = resolve;
|
|
11333
|
+
}));
|
|
11334
|
+
const runPass = vi.fn(async (_argv, _checkpoint, onJournalsReady) => {
|
|
11335
|
+
onJournalsReady?.();
|
|
11336
|
+
return completedPassOutcome();
|
|
11337
|
+
});
|
|
11338
|
+
const { loop, tick, shutdown } = runLoop({
|
|
11339
|
+
claims: EMAIL_CLAIMS,
|
|
11340
|
+
watcher,
|
|
11341
|
+
runPass,
|
|
11342
|
+
supportsCooperativeCheckpoint: true,
|
|
11343
|
+
});
|
|
11344
|
+
await microtasks(12);
|
|
11345
|
+
const absolutePath = "/tmp/hq/companies/indigo/recreated.md";
|
|
11346
|
+
const relativePath = "companies/indigo/recreated.md";
|
|
11347
|
+
watcher.emit(undefined, {
|
|
11348
|
+
paths: new Map([[absolutePath, relativePath]]),
|
|
11349
|
+
changes: new Map([[absolutePath, {
|
|
11350
|
+
kind: "unlink",
|
|
11351
|
+
deleteSnapshots: [{
|
|
11352
|
+
journalSlug: "indigo",
|
|
11353
|
+
journalPath: "recreated.md",
|
|
11354
|
+
deleteScopeRoot: "recreated.md",
|
|
11355
|
+
absolutePath,
|
|
11356
|
+
remoteEtag: "etag-before-recreate",
|
|
11357
|
+
localHash: "hash-before-recreate",
|
|
11358
|
+
localKind: "file",
|
|
11359
|
+
}],
|
|
11360
|
+
}]]),
|
|
11361
|
+
});
|
|
11362
|
+
// This is the older filesystem state collected before the live unlink.
|
|
11363
|
+
resolveCatchup?.({
|
|
11364
|
+
batch: {
|
|
11365
|
+
paths: new Map([[absolutePath, relativePath]]),
|
|
11366
|
+
changes: new Map([[absolutePath, { kind: "change" }]]),
|
|
11367
|
+
},
|
|
11368
|
+
scannedPaths: 1,
|
|
11369
|
+
});
|
|
11370
|
+
await microtasks(16);
|
|
11371
|
+
tick();
|
|
11372
|
+
await microtasks(16);
|
|
11373
|
+
const scopedPush = runPass.mock.calls.find((call) => call[0].includes("--scope-path"))?.[0];
|
|
11374
|
+
expect(scopedPush).toEqual(expect.arrayContaining([
|
|
11375
|
+
"--scope-path",
|
|
11376
|
+
"recreated.md",
|
|
11377
|
+
"--delete-scope-path",
|
|
11378
|
+
"recreated.md",
|
|
11379
|
+
]));
|
|
11380
|
+
shutdown();
|
|
11381
|
+
await loop;
|
|
11382
|
+
});
|
|
10061
11383
|
it("replays pull upsert marks after event-sync startup becomes live", async () => {
|
|
10062
11384
|
const markContentPublished = vi.fn();
|
|
10063
11385
|
let resolveStart;
|
|
@@ -12448,6 +13770,45 @@ describe("watcher delete intents without pre-seeded authorization", () => {
|
|
|
12448
13770
|
expect(Object.keys(readJournal("indigo").files)).toHaveLength(12);
|
|
12449
13771
|
await stopHarness(h);
|
|
12450
13772
|
});
|
|
13773
|
+
it("attributes a company-root unlinkDir to its journal while rejecting company-root metadata files", async () => {
|
|
13774
|
+
const keys = Array.from({ length: 12 }, (_, i) => `knowledge/company-root/f-${i}.md`);
|
|
13775
|
+
const pushPasses = [];
|
|
13776
|
+
const h = startDeleteHarness(keys, {
|
|
13777
|
+
direction: "push",
|
|
13778
|
+
onPass: (argv) => {
|
|
13779
|
+
if (argv[argv.indexOf("--direction") + 1] === "push") {
|
|
13780
|
+
pushPasses.push(argv);
|
|
13781
|
+
}
|
|
13782
|
+
},
|
|
13783
|
+
});
|
|
13784
|
+
await settle();
|
|
13785
|
+
pushPasses.length = 0;
|
|
13786
|
+
const childKey = keys[0];
|
|
13787
|
+
const childPath = path.join(h.companyRoot, childKey);
|
|
13788
|
+
fs.unlinkSync(childPath);
|
|
13789
|
+
expect(h.capture()(`companies/indigo/${childKey}`, "unlink")).toHaveLength(1);
|
|
13790
|
+
expect(readJournal("indigo").files[childKey]?.localDeleteIntent).toBeDefined();
|
|
13791
|
+
// The child intent is safe only until the enclosing whole-company removal
|
|
13792
|
+
// is classified. At that scope the exact denominator is the whole shard,
|
|
13793
|
+
// so the existing bulk guard must revoke it rather than lose attribution.
|
|
13794
|
+
fs.rmSync(h.companyRoot, { recursive: true, force: true });
|
|
13795
|
+
const rootSnapshots = h.capture()("companies/indigo", "unlinkDir");
|
|
13796
|
+
expect(rootSnapshots).toHaveLength(0);
|
|
13797
|
+
expect(Object.values(readJournal("indigo").files).filter((entry) => entry.localDeleteIntent)).toHaveLength(0);
|
|
13798
|
+
h.watcher.emit("companies/indigo", {
|
|
13799
|
+
paths: new Map([[h.companyRoot, "companies/indigo"]]),
|
|
13800
|
+
changes: new Map([[h.companyRoot, {
|
|
13801
|
+
kind: "unlinkDir",
|
|
13802
|
+
deleteSnapshots: rootSnapshots,
|
|
13803
|
+
}]]),
|
|
13804
|
+
});
|
|
13805
|
+
h.tick();
|
|
13806
|
+
await settle();
|
|
13807
|
+
expect(pushPasses).toEqual([
|
|
13808
|
+
expect.arrayContaining(["--company", "indigo", "--direction", "push"]),
|
|
13809
|
+
]);
|
|
13810
|
+
await stopHarness(h);
|
|
13811
|
+
});
|
|
12451
13812
|
// The native macOS/Windows backends make no child-before-parent guarantee,
|
|
12452
13813
|
// so the refusal has to hold for descendants that arrive after it too.
|
|
12453
13814
|
it("a descendant unlink arriving after a refused unlinkDir records no intent", async () => {
|