@indigoai-us/hq-cloud 6.16.71-alpha.0 → 6.16.71-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,7 +20,7 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
20
20
  import { setBandwidthGovernorForTesting } from "../bandwidth.js";
21
21
  import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
22
22
  import { FakeClock, TreeWatcher, } from "../watcher.js";
23
- import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
23
+ import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
24
24
  import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
25
25
  import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
26
26
  import { AreaLedger } from "../sync/area-ledger.js";
@@ -5850,6 +5850,86 @@ describe("runRunnerWithLoop — event-push wiring", () => {
5850
5850
  await loop;
5851
5851
  }
5852
5852
  });
5853
+ it("keeps logging reconcile decisions while a warmup catch-up drain is still running", async () => {
5854
+ const watcher = makeBatchWatcherStub();
5855
+ let markReady = () => { };
5856
+ watcher.onReady = (listener) => {
5857
+ markReady = listener;
5858
+ return () => { };
5859
+ };
5860
+ watcher.collectWarmupCatchup = vi.fn(() => ({
5861
+ batch: {
5862
+ paths: new Map([[
5863
+ "/tmp/hq/companies/indigo/knowledge/created-during-warmup.md",
5864
+ "companies/indigo/knowledge/created-during-warmup.md",
5865
+ ]]),
5866
+ changes: new Map([[
5867
+ "/tmp/hq/companies/indigo/knowledge/created-during-warmup.md",
5868
+ { kind: "add" },
5869
+ ]]),
5870
+ },
5871
+ scannedPaths: 1,
5872
+ }));
5873
+ const decisions = [];
5874
+ const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
5875
+ if (typeof chunk === "string" && chunk.startsWith("{")) {
5876
+ try {
5877
+ const parsed = JSON.parse(chunk);
5878
+ if (typeof parsed === "object" &&
5879
+ parsed !== null &&
5880
+ parsed.type === "full-reconcile-decision") {
5881
+ decisions.push(parsed);
5882
+ }
5883
+ }
5884
+ catch {
5885
+ /* non-JSON */
5886
+ }
5887
+ }
5888
+ return true;
5889
+ });
5890
+ let releaseScoped = () => { };
5891
+ const scopedGate = new Promise((resolve) => {
5892
+ releaseScoped = resolve;
5893
+ });
5894
+ const runPass = vi.fn(async (argv) => {
5895
+ if (Array.isArray(argv) && argv.includes("--scope-path")) {
5896
+ await scopedGate;
5897
+ }
5898
+ return completedPassOutcome();
5899
+ });
5900
+ const sleep = makeSteppableSleep();
5901
+ let shutdown = () => { };
5902
+ const loop = runRunnerWithLoop([
5903
+ "--company", "indigo", "--watch", "--event-push", "--direction", "both",
5904
+ "--hq-root", "/tmp/hq", "--poll-remote-ms", "60000",
5905
+ ], {
5906
+ runPass,
5907
+ createWatcher: () => watcher,
5908
+ sleep: sleep.sleep,
5909
+ onShutdownSignal: (handler) => {
5910
+ shutdown = handler;
5911
+ return () => { };
5912
+ },
5913
+ });
5914
+ try {
5915
+ await flushLoopMicrotasks();
5916
+ markReady();
5917
+ await flushLoopMicrotasks(20);
5918
+ const ticksBefore = decisions.length;
5919
+ sleep.tick();
5920
+ await flushLoopMicrotasks(20);
5921
+ sleep.tick();
5922
+ await flushLoopMicrotasks(20);
5923
+ expect(decisions.length).toBeGreaterThan(ticksBefore);
5924
+ expect(decisions.some((record) => record.mode === "catch-up" || record.mode === "idle" || record.mode === "full")).toBe(true);
5925
+ }
5926
+ finally {
5927
+ releaseScoped();
5928
+ spy.mockRestore();
5929
+ shutdown();
5930
+ await loop;
5931
+ }
5932
+ });
5853
5933
  it("hands the warm-up catch-up only journal rows that could still become deletes", async () => {
5854
5934
  // Session-log local retention deletes synced transcripts after 7 days and
5855
5935
  // deliberately KEEPS their journal rows; `isPersonalVaultDeletionExcluded`
@@ -9182,8 +9262,105 @@ describe("CooperativePassScheduler", () => {
9182
9262
  "realtime-after-cap",
9183
9263
  ]);
9184
9264
  });
9265
+ it("lets an in-flight realtime drain finish past the burst bound and admits no further items until the next checkpoint", async () => {
9266
+ expect(REALTIME_DRAIN_BURST_MS).toBe(REALTIME_ADMISSION_DEADLINE_MS);
9267
+ setRealtimeDrainBurstMsForTesting(40);
9268
+ let now = 0;
9269
+ const scheduler = new CooperativePassScheduler(() => now);
9270
+ const order = [];
9271
+ let releaseCheckpoint = () => { };
9272
+ const checkpointReady = new Promise((resolve) => {
9273
+ releaseCheckpoint = resolve;
9274
+ });
9275
+ const chore = scheduler.submit(async ({ checkpoint }) => {
9276
+ order.push("chore-before");
9277
+ await checkpointReady;
9278
+ await checkpoint();
9279
+ order.push("chore-mid");
9280
+ await checkpoint();
9281
+ order.push("chore-after");
9282
+ return 0;
9283
+ }, "chore");
9284
+ await Promise.resolve();
9285
+ const first = scheduler.submit(async ({ signal }) => {
9286
+ order.push("realtime-start");
9287
+ now = 45;
9288
+ await new Promise((resolve) => {
9289
+ setTimeout(resolve, 60);
9290
+ });
9291
+ expect(signal.aborted).toBe(false);
9292
+ order.push("realtime-done");
9293
+ return 0;
9294
+ }, "realtime");
9295
+ const second = scheduler.submit(async () => {
9296
+ order.push("realtime-second");
9297
+ return 0;
9298
+ }, "realtime");
9299
+ await new Promise((resolve) => setTimeout(resolve, 2));
9300
+ releaseCheckpoint();
9301
+ await expect(first).resolves.toBe(0);
9302
+ await chore;
9303
+ await second;
9304
+ expect(order).toEqual([
9305
+ "chore-before",
9306
+ "realtime-start",
9307
+ "realtime-done",
9308
+ "chore-mid",
9309
+ "realtime-second",
9310
+ "chore-after",
9311
+ ]);
9312
+ });
9313
+ it("does not abort or retry a 45s in-flight realtime PUT when the drain burst is 30s", async () => {
9314
+ expect(REALTIME_DRAIN_BURST_MS).toBe(30_000);
9315
+ vi.useFakeTimers();
9316
+ try {
9317
+ let now = 0;
9318
+ const scheduler = new CooperativePassScheduler(() => now);
9319
+ const order = [];
9320
+ let abortCount = 0;
9321
+ let releaseCheckpoint = () => { };
9322
+ const checkpointReady = new Promise((resolve) => {
9323
+ releaseCheckpoint = resolve;
9324
+ });
9325
+ const chore = scheduler.submit(async ({ checkpoint }) => {
9326
+ order.push("chore-before");
9327
+ await checkpointReady;
9328
+ await checkpoint();
9329
+ order.push("chore-after");
9330
+ return 0;
9331
+ }, "chore");
9332
+ await Promise.resolve();
9333
+ const put = scheduler.submit(async ({ signal }) => {
9334
+ order.push("put-start");
9335
+ signal.addEventListener("abort", () => {
9336
+ abortCount += 1;
9337
+ }, { once: true });
9338
+ now = 45_000;
9339
+ await new Promise((resolve) => {
9340
+ setTimeout(resolve, 45_000);
9341
+ });
9342
+ order.push("put-done");
9343
+ return 0;
9344
+ }, "realtime");
9345
+ releaseCheckpoint();
9346
+ const finished = Promise.all([chore, put]);
9347
+ await vi.advanceTimersByTimeAsync(45_000);
9348
+ await expect(finished).resolves.toEqual([0, 0]);
9349
+ expect(abortCount).toBe(0);
9350
+ expect(order).toEqual([
9351
+ "chore-before",
9352
+ "put-start",
9353
+ "put-done",
9354
+ "chore-after",
9355
+ ]);
9356
+ }
9357
+ finally {
9358
+ vi.useRealTimers();
9359
+ }
9360
+ });
9185
9361
  afterEach(() => {
9186
9362
  setRealtimeAdmissionDeadlineMsForTesting(undefined);
9363
+ setRealtimeDrainBurstMsForTesting(undefined);
9187
9364
  resetFullPassCapLogForTest();
9188
9365
  });
9189
9366
  it("commits queued realtime at a chore checkpoint well inside the admission deadline", async () => {
@@ -9256,6 +9433,128 @@ describe("CooperativePassScheduler", () => {
9256
9433
  await Promise.all([first, chore, retry]);
9257
9434
  expect(order).toEqual(["first-realtime", "retry-realtime", "chore"]);
9258
9435
  });
9436
+ });
9437
+ describe("first-tick realtime yield", () => {
9438
+ it("commits a watcher push at the first chore checkpoint within the admission deadline", async () => {
9439
+ const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
9440
+ const previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
9441
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
9442
+ process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
9443
+ const timings = [];
9444
+ const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
9445
+ const text = String(chunk);
9446
+ if (typeof chunk === "string" && chunk.startsWith("{")) {
9447
+ try {
9448
+ const parsed = JSON.parse(text);
9449
+ if (typeof parsed === "object" &&
9450
+ parsed !== null &&
9451
+ parsed.event === "sync.realtime.timing") {
9452
+ timings.push(parsed);
9453
+ }
9454
+ }
9455
+ catch {
9456
+ /* non-JSON */
9457
+ }
9458
+ }
9459
+ return true;
9460
+ });
9461
+ let choreRunning = false;
9462
+ let releaseCheckpoint = () => { };
9463
+ const checkpointGate = new Promise((resolve) => {
9464
+ releaseCheckpoint = resolve;
9465
+ });
9466
+ let releaseChore = () => { };
9467
+ const choreHold = new Promise((resolve) => {
9468
+ releaseChore = resolve;
9469
+ });
9470
+ const runPass = vi.fn(async (argv, checkpoint, onJournalsReady) => {
9471
+ const scoped = Array.isArray(argv) && argv.includes("--scope-path");
9472
+ if (!scoped && checkpoint) {
9473
+ choreRunning = true;
9474
+ onJournalsReady?.();
9475
+ await checkpointGate;
9476
+ await checkpoint();
9477
+ await choreHold;
9478
+ choreRunning = false;
9479
+ return completedPassOutcome();
9480
+ }
9481
+ return completedPassOutcome();
9482
+ });
9483
+ const watcher = makeBatchWatcherStub();
9484
+ const sleep = makeSteppableSleep();
9485
+ let triggerShutdown = () => { };
9486
+ const loop = runRunnerWithLoop([
9487
+ "--companies",
9488
+ "--watch",
9489
+ "--event-push",
9490
+ "--direction",
9491
+ "both",
9492
+ "--hq-root",
9493
+ "/tmp/hq",
9494
+ "--poll-remote-ms",
9495
+ "60000",
9496
+ ], {
9497
+ runPass: runPass,
9498
+ runPassSupportsCooperativeCheckpoint: true,
9499
+ clock: new FakeClock(),
9500
+ createWatcher: () => watcher,
9501
+ sleep: sleep.sleep,
9502
+ onShutdownSignal: (handler) => {
9503
+ triggerShutdown = handler;
9504
+ return () => { };
9505
+ },
9506
+ });
9507
+ try {
9508
+ await flushLoopMicrotasks(40);
9509
+ watcher.emit("companies/indigo/note.md", {
9510
+ paths: new Map([
9511
+ ["/tmp/hq/companies/indigo/note.md", "companies/indigo/note.md"],
9512
+ ]),
9513
+ });
9514
+ await flushLoopMicrotasks(20);
9515
+ await new Promise((resolve) => setTimeout(resolve, 20));
9516
+ await flushLoopMicrotasks(20);
9517
+ const started = Date.now();
9518
+ releaseCheckpoint();
9519
+ const deadline = started + 200;
9520
+ while (Date.now() < deadline) {
9521
+ if (timings.some((record) => record.outcome === "durable") ||
9522
+ runPass.mock.calls.some((call) => Array.isArray(call[0]) && call[0].includes("--scope-path"))) {
9523
+ break;
9524
+ }
9525
+ await new Promise((resolve) => setTimeout(resolve, 10));
9526
+ await flushLoopMicrotasks(10);
9527
+ }
9528
+ expect(choreRunning).toBe(true);
9529
+ expect(Date.now() - started).toBeLessThan(REALTIME_ADMISSION_DEADLINE_MS);
9530
+ expect(runPass.mock.calls.some((call) => Array.isArray(call[0]) && call[0].includes("--scope-path"))).toBe(true);
9531
+ expect(timings.some((record) => record.reason === "admission-timeout")).toBe(false);
9532
+ if (timings.length > 0) {
9533
+ expect(timings.some((record) => record.outcome === "durable")).toBe(true);
9534
+ }
9535
+ }
9536
+ finally {
9537
+ releaseChore();
9538
+ triggerShutdown();
9539
+ await loop;
9540
+ spy.mockRestore();
9541
+ if (previousCoordinator === undefined)
9542
+ delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
9543
+ else
9544
+ process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
9545
+ if (previousInterval === undefined)
9546
+ delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
9547
+ else
9548
+ process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
9549
+ }
9550
+ });
9551
+ });
9552
+ describe("CooperativePassScheduler cpu cap", () => {
9553
+ afterEach(() => {
9554
+ setRealtimeAdmissionDeadlineMsForTesting(undefined);
9555
+ setRealtimeDrainBurstMsForTesting(undefined);
9556
+ resetFullPassCapLogForTest();
9557
+ });
9259
9558
  it("sleeps a CPU-capped chore at the next checkpoint and admits queued realtime during that sleep", async () => {
9260
9559
  setRealtimeAdmissionDeadlineMsForTesting(250);
9261
9560
  let now = 0;