@indigoai-us/hq-cloud 6.16.72 → 6.16.74-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +37 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +182 -43
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.test.js +329 -14
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +46 -10
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/manifest-reconcile.test.js +111 -1
- package/dist/manifest-reconcile.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +6 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +13 -2
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +31 -4
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -20,7 +20,7 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
|
|
|
20
20
|
import { setBandwidthGovernorForTesting } from "../bandwidth.js";
|
|
21
21
|
import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
|
|
22
22
|
import { FakeClock, TreeWatcher, } from "../watcher.js";
|
|
23
|
-
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
23
|
+
import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
|
|
24
24
|
import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
|
|
25
25
|
import { AreaLedgerMigration, JournalStore, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalScoped, writeJournal, } from "../journal.js";
|
|
26
26
|
import { AreaLedger } from "../sync/area-ledger.js";
|
|
@@ -6191,7 +6191,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6191
6191
|
});
|
|
6192
6192
|
await expect(superseded).resolves.toBeUndefined();
|
|
6193
6193
|
expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
|
|
6194
|
-
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":
|
|
6194
|
+
expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":3,"source":"receiver","direction":"pull","outcome":"refused"');
|
|
6195
6195
|
releaseHeld();
|
|
6196
6196
|
await heldReceiverWork;
|
|
6197
6197
|
await replacement;
|
|
@@ -6207,6 +6207,237 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6207
6207
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6208
6208
|
}
|
|
6209
6209
|
});
|
|
6210
|
+
it("REGRESSION: a fast receiver pull overtakes a slow watcher push and records admitted occupancy", async () => {
|
|
6211
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6212
|
+
const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
|
|
6213
|
+
const previousFastBytes = process.env[FAST_LANE_MAX_BYTES_ENV];
|
|
6214
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6215
|
+
process.env.HQ_SYNC_AREA_STORAGE = "1";
|
|
6216
|
+
process.env[FAST_LANE_MAX_BYTES_ENV] = "32";
|
|
6217
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-receiver-fast-overtake-"));
|
|
6218
|
+
const watcher = makeBatchWatcherStub();
|
|
6219
|
+
let receiverSync;
|
|
6220
|
+
let shutdown = () => { };
|
|
6221
|
+
let releaseSlowPush = () => { };
|
|
6222
|
+
let markSlowPushStarted = () => { };
|
|
6223
|
+
const slowPushGate = new Promise((resolve) => { releaseSlowPush = resolve; });
|
|
6224
|
+
const slowPushStarted = new Promise((resolve) => { markSlowPushStarted = resolve; });
|
|
6225
|
+
let slowPushCompleted = false;
|
|
6226
|
+
let fastPullStartedAt;
|
|
6227
|
+
const timings = [];
|
|
6228
|
+
const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
|
|
6229
|
+
if (typeof chunk === "string" && chunk.startsWith("{")) {
|
|
6230
|
+
try {
|
|
6231
|
+
const record = JSON.parse(chunk);
|
|
6232
|
+
if (typeof record === "object" &&
|
|
6233
|
+
record !== null &&
|
|
6234
|
+
record.event === "sync.realtime.timing")
|
|
6235
|
+
timings.push(record);
|
|
6236
|
+
}
|
|
6237
|
+
catch {
|
|
6238
|
+
// Non-telemetry diagnostics are outside this assertion.
|
|
6239
|
+
}
|
|
6240
|
+
}
|
|
6241
|
+
return true;
|
|
6242
|
+
});
|
|
6243
|
+
const runPass = vi.fn(async (argv) => {
|
|
6244
|
+
const includesScope = (basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
|
|
6245
|
+
if (argv.includes("push") && includesScope("slow.bin")) {
|
|
6246
|
+
markSlowPushStarted();
|
|
6247
|
+
await slowPushGate;
|
|
6248
|
+
slowPushCompleted = true;
|
|
6249
|
+
return completedPassOutcome();
|
|
6250
|
+
}
|
|
6251
|
+
if (argv.includes("pull") && includesScope("fast.md")) {
|
|
6252
|
+
fastPullStartedAt = Date.now();
|
|
6253
|
+
return completedPassOutcome();
|
|
6254
|
+
}
|
|
6255
|
+
return completedPassOutcome();
|
|
6256
|
+
});
|
|
6257
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "both", "--hq-root", hqRoot], {
|
|
6258
|
+
runPass,
|
|
6259
|
+
createWatcher: () => watcher,
|
|
6260
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
6261
|
+
receiverSync = syncBatchFn;
|
|
6262
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6263
|
+
},
|
|
6264
|
+
sleep: () => new Promise(() => { }),
|
|
6265
|
+
onShutdownSignal: (handler) => {
|
|
6266
|
+
shutdown = handler;
|
|
6267
|
+
return () => { };
|
|
6268
|
+
},
|
|
6269
|
+
});
|
|
6270
|
+
const receiverEvent = (relativePath) => ({
|
|
6271
|
+
kind: "upsert",
|
|
6272
|
+
relativePath,
|
|
6273
|
+
contentHash: "sha256:remote",
|
|
6274
|
+
sizeBytes: 16,
|
|
6275
|
+
mtime: "2026-09-22T10:00:00.000Z",
|
|
6276
|
+
originDeviceId: "peer-device",
|
|
6277
|
+
originTenantId: "tenant-indigo",
|
|
6278
|
+
sequenceNumber: 1,
|
|
6279
|
+
eventTimestamp: "2026-09-22T10:00:00.000Z",
|
|
6280
|
+
});
|
|
6281
|
+
try {
|
|
6282
|
+
const slowAbsolutePath = path.join(hqRoot, "companies", "indigo", "slow.bin");
|
|
6283
|
+
fs.mkdirSync(path.dirname(slowAbsolutePath), { recursive: true });
|
|
6284
|
+
fs.writeFileSync(slowAbsolutePath, Buffer.alloc(1024));
|
|
6285
|
+
await flushLoopMicrotasks(30);
|
|
6286
|
+
runPass.mockClear();
|
|
6287
|
+
watcher.emit("companies/indigo/slow.bin", {
|
|
6288
|
+
paths: new Map([
|
|
6289
|
+
[slowAbsolutePath, "companies/indigo/slow.bin"],
|
|
6290
|
+
]),
|
|
6291
|
+
changes: new Map(),
|
|
6292
|
+
});
|
|
6293
|
+
await slowPushStarted;
|
|
6294
|
+
const deliveredAt = Date.now();
|
|
6295
|
+
const events = [receiverEvent("companies/indigo/fast.md")];
|
|
6296
|
+
const [fastBatch] = receiverSync.partition(events);
|
|
6297
|
+
expect(fastBatch).toEqual(events);
|
|
6298
|
+
const pull = receiverSync({
|
|
6299
|
+
events: fastBatch,
|
|
6300
|
+
signal: new AbortController().signal,
|
|
6301
|
+
});
|
|
6302
|
+
await flushLoopMicrotasks(50);
|
|
6303
|
+
expect(fastPullStartedAt).toBeDefined();
|
|
6304
|
+
expect(fastPullStartedAt - deliveredAt).toBeLessThan(30_000);
|
|
6305
|
+
await pull;
|
|
6306
|
+
expect(slowPushCompleted).toBe(false);
|
|
6307
|
+
const timing = timings.find((record) => record.source === "receiver" &&
|
|
6308
|
+
record.direction === "pull" &&
|
|
6309
|
+
record.outcome === "durable");
|
|
6310
|
+
expect(timing).toBeDefined();
|
|
6311
|
+
expect(timing?.occupancy).toBeGreaterThanOrEqual(1);
|
|
6312
|
+
}
|
|
6313
|
+
finally {
|
|
6314
|
+
releaseSlowPush();
|
|
6315
|
+
shutdown();
|
|
6316
|
+
await loop;
|
|
6317
|
+
stderr.mockRestore();
|
|
6318
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6319
|
+
if (previousCoordinator === undefined)
|
|
6320
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6321
|
+
else
|
|
6322
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6323
|
+
if (previousAreaStorage === undefined)
|
|
6324
|
+
delete process.env.HQ_SYNC_AREA_STORAGE;
|
|
6325
|
+
else
|
|
6326
|
+
process.env.HQ_SYNC_AREA_STORAGE = previousAreaStorage;
|
|
6327
|
+
if (previousFastBytes === undefined)
|
|
6328
|
+
delete process.env[FAST_LANE_MAX_BYTES_ENV];
|
|
6329
|
+
else
|
|
6330
|
+
process.env[FAST_LANE_MAX_BYTES_ENV] = previousFastBytes;
|
|
6331
|
+
}
|
|
6332
|
+
});
|
|
6333
|
+
it("REGRESSION: a same-path receiver pull stays behind the watcher lease", async () => {
|
|
6334
|
+
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6335
|
+
const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
|
|
6336
|
+
const previousFastBytes = process.env[FAST_LANE_MAX_BYTES_ENV];
|
|
6337
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
6338
|
+
process.env.HQ_SYNC_AREA_STORAGE = "1";
|
|
6339
|
+
process.env[FAST_LANE_MAX_BYTES_ENV] = "32";
|
|
6340
|
+
const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-receiver-path-lease-"));
|
|
6341
|
+
const watcher = makeBatchWatcherStub();
|
|
6342
|
+
let receiverSync;
|
|
6343
|
+
let shutdown = () => { };
|
|
6344
|
+
let releaseSlowPush = () => { };
|
|
6345
|
+
let markSlowPushStarted = () => { };
|
|
6346
|
+
const slowPushGate = new Promise((resolve) => { releaseSlowPush = resolve; });
|
|
6347
|
+
const slowPushStarted = new Promise((resolve) => { markSlowPushStarted = resolve; });
|
|
6348
|
+
let slowPushCompletedAt;
|
|
6349
|
+
let pullStartedAt;
|
|
6350
|
+
const runPass = vi.fn(async (argv) => {
|
|
6351
|
+
const includesScope = (basename) => argv.some((argument) => argument === basename || argument.endsWith(`/${basename}`));
|
|
6352
|
+
if (argv.includes("push") && includesScope("same.bin")) {
|
|
6353
|
+
markSlowPushStarted();
|
|
6354
|
+
await slowPushGate;
|
|
6355
|
+
slowPushCompletedAt = Date.now();
|
|
6356
|
+
return completedPassOutcome();
|
|
6357
|
+
}
|
|
6358
|
+
if (argv.includes("pull") && includesScope("same.bin")) {
|
|
6359
|
+
pullStartedAt = Date.now();
|
|
6360
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
6361
|
+
return completedPassOutcome();
|
|
6362
|
+
}
|
|
6363
|
+
return completedPassOutcome();
|
|
6364
|
+
});
|
|
6365
|
+
const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "both", "--hq-root", hqRoot], {
|
|
6366
|
+
runPass,
|
|
6367
|
+
createWatcher: () => watcher,
|
|
6368
|
+
createReceiver: ({ syncBatchFn }) => {
|
|
6369
|
+
receiverSync = syncBatchFn;
|
|
6370
|
+
return { connected: true, start: async () => { }, dispose: async () => { } };
|
|
6371
|
+
},
|
|
6372
|
+
sleep: () => new Promise(() => { }),
|
|
6373
|
+
onShutdownSignal: (handler) => {
|
|
6374
|
+
shutdown = handler;
|
|
6375
|
+
return () => { };
|
|
6376
|
+
},
|
|
6377
|
+
});
|
|
6378
|
+
try {
|
|
6379
|
+
const slowAbsolutePath = path.join(hqRoot, "companies", "indigo", "same.bin");
|
|
6380
|
+
fs.mkdirSync(path.dirname(slowAbsolutePath), { recursive: true });
|
|
6381
|
+
fs.writeFileSync(slowAbsolutePath, Buffer.alloc(1024));
|
|
6382
|
+
await flushLoopMicrotasks(30);
|
|
6383
|
+
runPass.mockClear();
|
|
6384
|
+
watcher.emit("companies/indigo/same.bin", {
|
|
6385
|
+
paths: new Map([
|
|
6386
|
+
[slowAbsolutePath, "companies/indigo/same.bin"],
|
|
6387
|
+
]),
|
|
6388
|
+
changes: new Map(),
|
|
6389
|
+
});
|
|
6390
|
+
await slowPushStarted;
|
|
6391
|
+
const events = [{
|
|
6392
|
+
kind: "upsert",
|
|
6393
|
+
relativePath: "companies/indigo/same.bin",
|
|
6394
|
+
contentHash: "sha256:remote",
|
|
6395
|
+
sizeBytes: 16,
|
|
6396
|
+
mtime: "2026-09-22T10:00:00.000Z",
|
|
6397
|
+
originDeviceId: "peer-device",
|
|
6398
|
+
originTenantId: "tenant-indigo",
|
|
6399
|
+
sequenceNumber: 1,
|
|
6400
|
+
eventTimestamp: "2026-09-22T10:00:00.000Z",
|
|
6401
|
+
}];
|
|
6402
|
+
const [fastBatch] = receiverSync.partition(events);
|
|
6403
|
+
const pull = receiverSync({
|
|
6404
|
+
events: fastBatch,
|
|
6405
|
+
signal: new AbortController().signal,
|
|
6406
|
+
});
|
|
6407
|
+
const settledPull = pull.then(() => ({ outcome: "durable" }), (error) => ({ outcome: "refused", error }));
|
|
6408
|
+
await flushLoopMicrotasks(50);
|
|
6409
|
+
expect(pullStartedAt).toBeUndefined();
|
|
6410
|
+
releaseSlowPush();
|
|
6411
|
+
const pullResult = await settledPull;
|
|
6412
|
+
if (pullResult.outcome === "durable") {
|
|
6413
|
+
expect(pullStartedAt).toBeDefined();
|
|
6414
|
+
expect(slowPushCompletedAt).toBeDefined();
|
|
6415
|
+
expect(pullStartedAt).toBeGreaterThanOrEqual(slowPushCompletedAt);
|
|
6416
|
+
}
|
|
6417
|
+
else {
|
|
6418
|
+
expect(pullResult.error).toBeInstanceOf(Error);
|
|
6419
|
+
expect(String(pullResult.error)).toContain("targeted pull failed");
|
|
6420
|
+
}
|
|
6421
|
+
}
|
|
6422
|
+
finally {
|
|
6423
|
+
releaseSlowPush();
|
|
6424
|
+
shutdown();
|
|
6425
|
+
await loop;
|
|
6426
|
+
fs.rmSync(hqRoot, { recursive: true, force: true });
|
|
6427
|
+
if (previousCoordinator === undefined)
|
|
6428
|
+
delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6429
|
+
else
|
|
6430
|
+
process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
|
|
6431
|
+
if (previousAreaStorage === undefined)
|
|
6432
|
+
delete process.env.HQ_SYNC_AREA_STORAGE;
|
|
6433
|
+
else
|
|
6434
|
+
process.env.HQ_SYNC_AREA_STORAGE = previousAreaStorage;
|
|
6435
|
+
if (previousFastBytes === undefined)
|
|
6436
|
+
delete process.env[FAST_LANE_MAX_BYTES_ENV];
|
|
6437
|
+
else
|
|
6438
|
+
process.env[FAST_LANE_MAX_BYTES_ENV] = previousFastBytes;
|
|
6439
|
+
}
|
|
6440
|
+
});
|
|
6210
6441
|
it("still rejects retryable coordinator receiver outcomes", async () => {
|
|
6211
6442
|
const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
|
|
6212
6443
|
process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
|
|
@@ -6756,7 +6987,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
6756
6987
|
signal: new AbortController().signal,
|
|
6757
6988
|
});
|
|
6758
6989
|
await flushLoopMicrotasks(20);
|
|
6759
|
-
expect(order).toEqual(["slow-held"]);
|
|
6990
|
+
expect(order).toEqual(["slow-held", "fast-small"]);
|
|
6760
6991
|
releaseSlow();
|
|
6761
6992
|
await held;
|
|
6762
6993
|
await partitioned;
|
|
@@ -9262,7 +9493,7 @@ describe("CooperativePassScheduler", () => {
|
|
|
9262
9493
|
"realtime-after-cap",
|
|
9263
9494
|
]);
|
|
9264
9495
|
});
|
|
9265
|
-
it("lets an in-flight realtime drain finish past the burst bound
|
|
9496
|
+
it("lets an in-flight realtime drain finish past the burst bound without making a new arrival wait for it", async () => {
|
|
9266
9497
|
expect(REALTIME_DRAIN_BURST_MS).toBe(REALTIME_ADMISSION_DEADLINE_MS);
|
|
9267
9498
|
setRealtimeDrainBurstMsForTesting(40);
|
|
9268
9499
|
let now = 0;
|
|
@@ -9272,6 +9503,10 @@ describe("CooperativePassScheduler", () => {
|
|
|
9272
9503
|
const checkpointReady = new Promise((resolve) => {
|
|
9273
9504
|
releaseCheckpoint = resolve;
|
|
9274
9505
|
});
|
|
9506
|
+
let releaseFirst = () => { };
|
|
9507
|
+
const firstGate = new Promise((resolve) => {
|
|
9508
|
+
releaseFirst = resolve;
|
|
9509
|
+
});
|
|
9275
9510
|
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
9276
9511
|
order.push("chore-before");
|
|
9277
9512
|
await checkpointReady;
|
|
@@ -9285,28 +9520,31 @@ describe("CooperativePassScheduler", () => {
|
|
|
9285
9520
|
const first = scheduler.submit(async ({ signal }) => {
|
|
9286
9521
|
order.push("realtime-start");
|
|
9287
9522
|
now = 45;
|
|
9288
|
-
await
|
|
9289
|
-
setTimeout(resolve, 60);
|
|
9290
|
-
});
|
|
9523
|
+
await firstGate;
|
|
9291
9524
|
expect(signal.aborted).toBe(false);
|
|
9292
9525
|
order.push("realtime-done");
|
|
9293
9526
|
return 0;
|
|
9294
9527
|
}, "realtime");
|
|
9528
|
+
releaseCheckpoint();
|
|
9529
|
+
await new Promise((resolve) => setTimeout(resolve, 2));
|
|
9530
|
+
expect(order).toContain("realtime-start");
|
|
9295
9531
|
const second = scheduler.submit(async () => {
|
|
9296
9532
|
order.push("realtime-second");
|
|
9297
9533
|
return 0;
|
|
9298
9534
|
}, "realtime");
|
|
9299
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
9300
|
-
|
|
9535
|
+
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
9536
|
+
expect(order).toContain("realtime-second");
|
|
9537
|
+
expect(order.indexOf("realtime-second")).toBeLessThan(order.includes("realtime-done") ? order.indexOf("realtime-done") : order.length);
|
|
9538
|
+
releaseFirst();
|
|
9301
9539
|
await expect(first).resolves.toBe(0);
|
|
9540
|
+
await expect(second).resolves.toBe(0);
|
|
9302
9541
|
await chore;
|
|
9303
|
-
await second;
|
|
9304
9542
|
expect(order).toEqual([
|
|
9305
9543
|
"chore-before",
|
|
9306
9544
|
"realtime-start",
|
|
9545
|
+
"realtime-second",
|
|
9307
9546
|
"realtime-done",
|
|
9308
9547
|
"chore-mid",
|
|
9309
|
-
"realtime-second",
|
|
9310
9548
|
"chore-after",
|
|
9311
9549
|
]);
|
|
9312
9550
|
});
|
|
@@ -9343,13 +9581,26 @@ describe("CooperativePassScheduler", () => {
|
|
|
9343
9581
|
return 0;
|
|
9344
9582
|
}, "realtime");
|
|
9345
9583
|
releaseCheckpoint();
|
|
9346
|
-
|
|
9347
|
-
|
|
9348
|
-
|
|
9584
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
9585
|
+
expect(order).toContain("put-start");
|
|
9586
|
+
const arrivedAt = now;
|
|
9587
|
+
let nextStartedAt = null;
|
|
9588
|
+
const next = scheduler.submit(async () => {
|
|
9589
|
+
nextStartedAt = now;
|
|
9590
|
+
order.push("next-start");
|
|
9591
|
+
return 0;
|
|
9592
|
+
}, "realtime");
|
|
9593
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
9594
|
+
expect(nextStartedAt).not.toBeNull();
|
|
9595
|
+
expect((nextStartedAt ?? Number.POSITIVE_INFINITY) - arrivedAt).toBeLessThanOrEqual(30_000);
|
|
9596
|
+
expect(order).not.toContain("put-done");
|
|
9597
|
+
await vi.advanceTimersByTimeAsync(15_000);
|
|
9598
|
+
await expect(Promise.all([chore, put, next])).resolves.toEqual([0, 0, 0]);
|
|
9349
9599
|
expect(abortCount).toBe(0);
|
|
9350
9600
|
expect(order).toEqual([
|
|
9351
9601
|
"chore-before",
|
|
9352
9602
|
"put-start",
|
|
9603
|
+
"next-start",
|
|
9353
9604
|
"put-done",
|
|
9354
9605
|
"chore-after",
|
|
9355
9606
|
]);
|
|
@@ -9358,6 +9609,70 @@ describe("CooperativePassScheduler", () => {
|
|
|
9358
9609
|
vi.useRealTimers();
|
|
9359
9610
|
}
|
|
9360
9611
|
});
|
|
9612
|
+
it("starts a 1-item realtime push within 30s while a 24-item execute is still in flight", async () => {
|
|
9613
|
+
expect(REALTIME_WORKER_OCCUPANCY_CAP).toBe(2);
|
|
9614
|
+
expect(REALTIME_DRAIN_BURST_MS).toBe(30_000);
|
|
9615
|
+
vi.useFakeTimers();
|
|
9616
|
+
try {
|
|
9617
|
+
let now = 0;
|
|
9618
|
+
const scheduler = new CooperativePassScheduler(() => now);
|
|
9619
|
+
const order = [];
|
|
9620
|
+
let abortCount = 0;
|
|
9621
|
+
let releaseCheckpoint = () => { };
|
|
9622
|
+
const checkpointReady = new Promise((resolve) => {
|
|
9623
|
+
releaseCheckpoint = resolve;
|
|
9624
|
+
});
|
|
9625
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
9626
|
+
order.push("chore-before");
|
|
9627
|
+
await checkpointReady;
|
|
9628
|
+
await checkpoint();
|
|
9629
|
+
order.push("chore-after");
|
|
9630
|
+
return 0;
|
|
9631
|
+
}, "chore");
|
|
9632
|
+
await Promise.resolve();
|
|
9633
|
+
const batchMs = 15 * 60_000;
|
|
9634
|
+
const batch = scheduler.submit(async ({ signal }) => {
|
|
9635
|
+
order.push("batch-start");
|
|
9636
|
+
signal.addEventListener("abort", () => {
|
|
9637
|
+
abortCount += 1;
|
|
9638
|
+
}, { once: true });
|
|
9639
|
+
await new Promise((resolve) => {
|
|
9640
|
+
setTimeout(resolve, batchMs);
|
|
9641
|
+
});
|
|
9642
|
+
expect(signal.aborted).toBe(false);
|
|
9643
|
+
order.push("batch-done");
|
|
9644
|
+
return 0;
|
|
9645
|
+
}, "realtime");
|
|
9646
|
+
releaseCheckpoint();
|
|
9647
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
9648
|
+
expect(order).toEqual(["chore-before", "batch-start"]);
|
|
9649
|
+
now = 2 * 60_000;
|
|
9650
|
+
const arrivedAt = now;
|
|
9651
|
+
let plantStartedAt = null;
|
|
9652
|
+
const plant = scheduler.submit(async () => {
|
|
9653
|
+
plantStartedAt = now;
|
|
9654
|
+
order.push("plant-start");
|
|
9655
|
+
return 0;
|
|
9656
|
+
}, "realtime");
|
|
9657
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
9658
|
+
expect(plantStartedAt).not.toBeNull();
|
|
9659
|
+
expect((plantStartedAt ?? Number.POSITIVE_INFINITY) - arrivedAt).toBeLessThanOrEqual(30_000);
|
|
9660
|
+
expect(order).not.toContain("batch-done");
|
|
9661
|
+
await vi.advanceTimersByTimeAsync(batchMs);
|
|
9662
|
+
await expect(Promise.all([chore, batch, plant])).resolves.toEqual([0, 0, 0]);
|
|
9663
|
+
expect(abortCount).toBe(0);
|
|
9664
|
+
expect(order).toEqual([
|
|
9665
|
+
"chore-before",
|
|
9666
|
+
"batch-start",
|
|
9667
|
+
"plant-start",
|
|
9668
|
+
"batch-done",
|
|
9669
|
+
"chore-after",
|
|
9670
|
+
]);
|
|
9671
|
+
}
|
|
9672
|
+
finally {
|
|
9673
|
+
vi.useRealTimers();
|
|
9674
|
+
}
|
|
9675
|
+
});
|
|
9361
9676
|
afterEach(() => {
|
|
9362
9677
|
setRealtimeAdmissionDeadlineMsForTesting(undefined);
|
|
9363
9678
|
setRealtimeDrainBurstMsForTesting(undefined);
|