@indigoai-us/hq-cloud 6.16.87-alpha.0 → 6.16.89-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +8 -2
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +26 -25
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +251 -183
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +22 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +5 -3
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +74 -509
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -6
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +108 -97
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +4 -177
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -34
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +17 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +138 -20
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +230 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +0 -7
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +7 -54
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +1 -20
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/object-io.d.ts +21 -9
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +133 -93
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +140 -108
- package/dist/object-io.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +30 -37
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +16 -104
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +3 -34
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +8 -15
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +4 -44
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +30 -7
- package/dist/s3.js.map +1 -1
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -7
- package/dist/scope-shrink.js.map +1 -1
- package/dist/sync/event-sync.d.ts +5 -19
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +14 -150
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +2 -386
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +0 -1
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -1
- package/dist/sync/parallel-sync-coordinator.js +8 -39
- package/dist/sync/parallel-sync-coordinator.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +1 -33
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts +0 -2
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +5 -15
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +1 -22
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +20 -114
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +24 -398
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +0 -15
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +0 -27
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +1 -14
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync/sync-work.d.ts +0 -5
- package/dist/sync/sync-work.d.ts.map +1 -1
- package/dist/sync/sync-work.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/watcher.d.ts +0 -4
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +4 -16
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +3 -50
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -11,13 +11,13 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
|
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
13
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
|
-
import {
|
|
14
|
+
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
18
18
|
import { clearLocalDeleteIntent, getStateDir, markLocalDeleteIntent, PERSONAL_VAULT_JOURNAL_SLUG, openJournalStoreSession, openJournalStoreSessionAsync, listJournalSlugs, listJournals, maintainAreaJournalSlice, disposeScopedJournalStores, releaseScopedJournalStore, releaseCachedAreaJournalMigrations, journalCacheCensusSizes, emitJournalCacheMetric, journalRows, readJournalScoped, writeJournal, } from "../journal.js";
|
|
19
19
|
import { isPersonalVaultDeletionExcluded } from "../personal-vault.js";
|
|
20
|
-
import { NoopPushReceiver,
|
|
20
|
+
import { NoopPushReceiver, } from "../sync/push-receiver.js";
|
|
21
21
|
import { resolveEventSync, startEventSync as defaultStartEventSync, } from "../sync/event-sync.js";
|
|
22
22
|
import { resolveAreaStorageEnabled, resolveEffectiveParallelSyncRollout, resolveParallelSyncRollout, } from "../sync/feature-flags.js";
|
|
23
23
|
import { resolveAreaLayoutAuthority } from "../sync/area-ledger-migration.js";
|
|
@@ -85,13 +85,6 @@ export class RealtimeDrainBurstError extends Error {
|
|
|
85
85
|
this.name = "RealtimeDrainBurst";
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
|
-
function receiverAdmissionRefusedError(error) {
|
|
89
|
-
if (error instanceof RealtimeAdmissionTimeoutError ||
|
|
90
|
-
error instanceof RealtimeDrainBurstError) {
|
|
91
|
-
return new ReceiverAdmissionRefusedError(error.message, { cause: error });
|
|
92
|
-
}
|
|
93
|
-
return null;
|
|
94
|
-
}
|
|
95
88
|
/**
|
|
96
89
|
* Coordinator path status for one sync-pass exit. Exit 19 (vanished company)
|
|
97
90
|
* is durable so a leftover local slug is not restored and retried forever;
|
|
@@ -160,6 +153,19 @@ export class CooperativePassScheduler {
|
|
|
160
153
|
activeFastRealtime = null;
|
|
161
154
|
stopped = false;
|
|
162
155
|
_handoffActive = false;
|
|
156
|
+
/** A chore's payload transfer is in flight while it lends the root lock. */
|
|
157
|
+
networkTransferActive = false;
|
|
158
|
+
detachedChore = null;
|
|
159
|
+
networkTransferClosing = false;
|
|
160
|
+
networkTransferCount = 0;
|
|
161
|
+
networkTransferEpoch = 0;
|
|
162
|
+
networkTransferWaiters = [];
|
|
163
|
+
networkTransferRelease = null;
|
|
164
|
+
resolveNetworkTransferRelease = null;
|
|
165
|
+
/** Successful callers still applying results after the network hand-off. */
|
|
166
|
+
networkTransferCompletionCount = 0;
|
|
167
|
+
networkTransferApplyRelease = null;
|
|
168
|
+
resolveNetworkTransferApplyRelease = null;
|
|
163
169
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
164
170
|
this.now = now;
|
|
165
171
|
this.starvationCapMs = starvationCapMs;
|
|
@@ -313,6 +319,7 @@ export class CooperativePassScheduler {
|
|
|
313
319
|
return;
|
|
314
320
|
if (this._handoffActive) {
|
|
315
321
|
this.promoteOccupancyArrivals();
|
|
322
|
+
this.admitHandoffBacklog();
|
|
316
323
|
return;
|
|
317
324
|
}
|
|
318
325
|
if (this.active !== null) {
|
|
@@ -422,15 +429,215 @@ export class CooperativePassScheduler {
|
|
|
422
429
|
execute(entry) {
|
|
423
430
|
if (entry.resourceCap)
|
|
424
431
|
this.options.caps?.beginChore();
|
|
432
|
+
const checkpoint = Object.assign(() => this.checkpoint(entry), {
|
|
433
|
+
shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
|
|
434
|
+
runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
|
|
435
|
+
});
|
|
425
436
|
return entry.task({
|
|
426
|
-
checkpoint
|
|
437
|
+
checkpoint,
|
|
427
438
|
signal: entry.controller.signal,
|
|
428
439
|
});
|
|
429
440
|
}
|
|
441
|
+
shouldYieldToRealtime(entry) {
|
|
442
|
+
return !this.stopped &&
|
|
443
|
+
entry.workClass === "chore" &&
|
|
444
|
+
this.active === entry &&
|
|
445
|
+
this.pending.some((pending) => pending.workClass === "realtime");
|
|
446
|
+
}
|
|
430
447
|
withinHandoffAdmission() {
|
|
431
448
|
if (this.handoffDeferredAt === null)
|
|
432
449
|
return false;
|
|
433
|
-
return this.
|
|
450
|
+
return this.networkTransferActive ||
|
|
451
|
+
this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Run payload I/O without owning the pass's durable state. Keep the chore
|
|
455
|
+
* slot reserved while realtime uses the root lock, then wait for active
|
|
456
|
+
* realtime writers before restoring chore ownership.
|
|
457
|
+
*/
|
|
458
|
+
async runChoreNetworkTransfer(entry, work) {
|
|
459
|
+
if (this.detachedChore === entry &&
|
|
460
|
+
this._handoffActive &&
|
|
461
|
+
this.networkTransferActive &&
|
|
462
|
+
!this.networkTransferClosing) {
|
|
463
|
+
return this.joinChoreNetworkTransfer(work);
|
|
464
|
+
}
|
|
465
|
+
if (this.detachedChore === entry &&
|
|
466
|
+
this._handoffActive &&
|
|
467
|
+
this.networkTransferRelease !== null) {
|
|
468
|
+
await this.networkTransferRelease;
|
|
469
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
470
|
+
}
|
|
471
|
+
if (this.active === entry && this.networkTransferApplyRelease !== null) {
|
|
472
|
+
await this.networkTransferApplyRelease;
|
|
473
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
474
|
+
}
|
|
475
|
+
if (this.stopped ||
|
|
476
|
+
entry.workClass !== "chore" ||
|
|
477
|
+
this.active !== entry) {
|
|
478
|
+
return { value: await work(), realtimeRan: false, complete: () => undefined };
|
|
479
|
+
}
|
|
480
|
+
this.active = null;
|
|
481
|
+
this._handoffActive = true;
|
|
482
|
+
this.networkTransferActive = true;
|
|
483
|
+
this.detachedChore = entry;
|
|
484
|
+
this.networkTransferClosing = false;
|
|
485
|
+
this.networkTransferCount = 1;
|
|
486
|
+
this.networkTransferCompletionCount = 1;
|
|
487
|
+
const applyReleased = new Promise((resolve) => {
|
|
488
|
+
this.resolveNetworkTransferApplyRelease = resolve;
|
|
489
|
+
});
|
|
490
|
+
this.networkTransferApplyRelease = applyReleased;
|
|
491
|
+
const released = new Promise((resolve) => {
|
|
492
|
+
this.resolveNetworkTransferRelease = resolve;
|
|
493
|
+
});
|
|
494
|
+
this.networkTransferRelease = released;
|
|
495
|
+
this.handoffDeferredAt = this.now();
|
|
496
|
+
this.handoffAdmitted = false;
|
|
497
|
+
this.handoffBacklog = null;
|
|
498
|
+
this.options.caps?.suspendForHandoff();
|
|
499
|
+
let value;
|
|
500
|
+
let failure;
|
|
501
|
+
let failed = false;
|
|
502
|
+
try {
|
|
503
|
+
// Admit an arrival that was queued just before the transfer began.
|
|
504
|
+
this.drain();
|
|
505
|
+
value = await work();
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
failed = true;
|
|
509
|
+
failure = err;
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
this.networkTransferClosing = true;
|
|
513
|
+
this.networkTransferCount -= 1;
|
|
514
|
+
this.notifyNetworkTransferChange();
|
|
515
|
+
while (this.networkTransferCount > 0) {
|
|
516
|
+
const seen = this.networkTransferEpoch;
|
|
517
|
+
if (this.networkTransferCount === 0)
|
|
518
|
+
break;
|
|
519
|
+
await this.waitNetworkTransferChange(seen);
|
|
520
|
+
}
|
|
521
|
+
this.networkTransferActive = false;
|
|
522
|
+
// A long transfer may outlive the ordinary realtime burst. Start a new
|
|
523
|
+
// bounded window for arrivals that queued safely during network I/O.
|
|
524
|
+
this.handoffDeferredAt = this.now();
|
|
525
|
+
try {
|
|
526
|
+
this.drain();
|
|
527
|
+
while (!this.stopped &&
|
|
528
|
+
(this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
529
|
+
if (this.realtimeOccupants.size === 0) {
|
|
530
|
+
this.admitHandoffBacklog();
|
|
531
|
+
if (this.realtimeOccupants.size === 0)
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
539
|
+
this.admitHandoffBacklog();
|
|
540
|
+
}
|
|
541
|
+
while (this.realtimeOccupants.size > 0) {
|
|
542
|
+
const seen = this.occupantEpoch;
|
|
543
|
+
if (this.realtimeOccupants.size === 0)
|
|
544
|
+
break;
|
|
545
|
+
await this.waitOccupantChange(seen);
|
|
546
|
+
this.promoteOccupancyArrivals();
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
this._handoffActive = false;
|
|
551
|
+
this.handoffDeferredAt = null;
|
|
552
|
+
this.handoffBacklog = null;
|
|
553
|
+
this.detachedChore = null;
|
|
554
|
+
if (!this.stopped)
|
|
555
|
+
this.active = entry;
|
|
556
|
+
this.options.caps?.resumeAfterHandoff();
|
|
557
|
+
const resolveRelease = this.resolveNetworkTransferRelease;
|
|
558
|
+
const realtimeRan = this.handoffAdmitted;
|
|
559
|
+
this.networkTransferRelease = null;
|
|
560
|
+
this.resolveNetworkTransferRelease = null;
|
|
561
|
+
this.networkTransferClosing = false;
|
|
562
|
+
resolveRelease?.(realtimeRan);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
const realtimeRan = await released;
|
|
566
|
+
if (failed) {
|
|
567
|
+
this.completeNetworkTransferParticipant();
|
|
568
|
+
throw failure;
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
value,
|
|
572
|
+
realtimeRan,
|
|
573
|
+
complete: this.createNetworkTransferCompletion(),
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
async joinChoreNetworkTransfer(work) {
|
|
577
|
+
const released = this.networkTransferRelease;
|
|
578
|
+
if (released === null) {
|
|
579
|
+
throw new Error("network transfer hand-off is missing its release signal");
|
|
580
|
+
}
|
|
581
|
+
this.networkTransferCount += 1;
|
|
582
|
+
this.networkTransferCompletionCount += 1;
|
|
583
|
+
let value;
|
|
584
|
+
let failure;
|
|
585
|
+
let failed = false;
|
|
586
|
+
try {
|
|
587
|
+
value = await work();
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
failed = true;
|
|
591
|
+
failure = err;
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
this.networkTransferCount -= 1;
|
|
595
|
+
this.notifyNetworkTransferChange();
|
|
596
|
+
}
|
|
597
|
+
const realtimeRan = await released;
|
|
598
|
+
if (failed) {
|
|
599
|
+
this.completeNetworkTransferParticipant();
|
|
600
|
+
throw failure;
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
value,
|
|
604
|
+
realtimeRan,
|
|
605
|
+
complete: this.createNetworkTransferCompletion(),
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
createNetworkTransferCompletion() {
|
|
609
|
+
let completed = false;
|
|
610
|
+
return () => {
|
|
611
|
+
if (completed)
|
|
612
|
+
return;
|
|
613
|
+
completed = true;
|
|
614
|
+
this.completeNetworkTransferParticipant();
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
completeNetworkTransferParticipant() {
|
|
618
|
+
if (this.networkTransferCompletionCount <= 0)
|
|
619
|
+
return;
|
|
620
|
+
this.networkTransferCompletionCount -= 1;
|
|
621
|
+
if (this.networkTransferCompletionCount > 0)
|
|
622
|
+
return;
|
|
623
|
+
const resolve = this.resolveNetworkTransferApplyRelease;
|
|
624
|
+
this.resolveNetworkTransferApplyRelease = null;
|
|
625
|
+
this.networkTransferApplyRelease = null;
|
|
626
|
+
resolve?.();
|
|
627
|
+
}
|
|
628
|
+
waitNetworkTransferChange(seenEpoch) {
|
|
629
|
+
if (this.networkTransferEpoch !== seenEpoch)
|
|
630
|
+
return Promise.resolve();
|
|
631
|
+
return new Promise((resolve) => {
|
|
632
|
+
this.networkTransferWaiters.push(resolve);
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
notifyNetworkTransferChange() {
|
|
636
|
+
this.networkTransferEpoch += 1;
|
|
637
|
+
const waiters = this.networkTransferWaiters;
|
|
638
|
+
this.networkTransferWaiters = [];
|
|
639
|
+
for (const resolve of waiters)
|
|
640
|
+
resolve();
|
|
434
641
|
}
|
|
435
642
|
waitOccupantChange(seenEpoch) {
|
|
436
643
|
if (this.occupantEpoch !== seenEpoch)
|
|
@@ -907,16 +1114,6 @@ export function resolveFullReconcileMs(env = process.env) {
|
|
|
907
1114
|
const ADAPTIVE_POLL_FLOOR_MS = 60_000;
|
|
908
1115
|
const ADAPTIVE_POLL_CEIL_MS = 600_000;
|
|
909
1116
|
const WATCH_IDLE_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
910
|
-
/**
|
|
911
|
-
* A failed watch pass is still retried in-process, but even an explicitly
|
|
912
|
-
* tiny poll interval must not turn a broken dependency into a hot loop.
|
|
913
|
-
* Three failures in thirty minutes escalate from this floor to exponential
|
|
914
|
-
* backoff; success or a quiet window starts a fresh budget.
|
|
915
|
-
*/
|
|
916
|
-
export const WATCH_PASS_EXIT_RETRY_WINDOW_MS = 30 * 60_000;
|
|
917
|
-
export const WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF = 3;
|
|
918
|
-
export const WATCH_PASS_EXIT_RETRY_INITIAL_MS = 60_000;
|
|
919
|
-
export const WATCH_PASS_EXIT_RETRY_MAX_MS = 10 * 60_000;
|
|
920
1117
|
/** Covers coarse timestamp filesystems and scheduling just before watcher start. */
|
|
921
1118
|
const WATCHER_WARMUP_SAFETY_MARGIN_MS = 2_000;
|
|
922
1119
|
/** Initial poll-only recovery delay after a watcher conclusively stands down. */
|
|
@@ -966,41 +1163,6 @@ export function advanceMonotonicDeadline(previousDueAt, now, intervalMs) {
|
|
|
966
1163
|
nextDueAt = now + intervalMs;
|
|
967
1164
|
return { nextDueAt, delayMs: intervalMs };
|
|
968
1165
|
}
|
|
969
|
-
/**
|
|
970
|
-
* Bounds in-process retries after a completed watch pass returned non-zero.
|
|
971
|
-
*
|
|
972
|
-
* This is deliberately separate from the adaptive poll cadence: cadence
|
|
973
|
-
* controls normal remote work, whereas this budget prevents a configured
|
|
974
|
-
* short poll from continuously re-running a pass that is already known to
|
|
975
|
-
* have failed. The caller supplies the monotonic scheduler clock so tests do
|
|
976
|
-
* not need wall-clock waits.
|
|
977
|
-
*/
|
|
978
|
-
export class WatchPassExitRetryBudget {
|
|
979
|
-
lastFailureAt = null;
|
|
980
|
-
failures = 0;
|
|
981
|
-
record(exitCode, now) {
|
|
982
|
-
if (exitCode === 0) {
|
|
983
|
-
this.lastFailureAt = null;
|
|
984
|
-
this.failures = 0;
|
|
985
|
-
return 0;
|
|
986
|
-
}
|
|
987
|
-
if (this.lastFailureAt === null ||
|
|
988
|
-
now < this.lastFailureAt ||
|
|
989
|
-
now - this.lastFailureAt >= WATCH_PASS_EXIT_RETRY_WINDOW_MS) {
|
|
990
|
-
this.lastFailureAt = now;
|
|
991
|
-
this.failures = 0;
|
|
992
|
-
}
|
|
993
|
-
this.failures += 1;
|
|
994
|
-
this.lastFailureAt = now;
|
|
995
|
-
if (this.failures < WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF) {
|
|
996
|
-
// Preserve the established normal-cadence retry for isolated failures.
|
|
997
|
-
// The bounded cooldown starts only after repeated exits in the window.
|
|
998
|
-
return 0;
|
|
999
|
-
}
|
|
1000
|
-
const exponent = Math.min(this.failures - WATCH_PASS_EXIT_FAILURES_BEFORE_BACKOFF, 4);
|
|
1001
|
-
return Math.min(WATCH_PASS_EXIT_RETRY_INITIAL_MS * 2 ** exponent, WATCH_PASS_EXIT_RETRY_MAX_MS);
|
|
1002
|
-
}
|
|
1003
|
-
}
|
|
1004
1166
|
/**
|
|
1005
1167
|
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
1006
1168
|
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
@@ -1320,7 +1482,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1320
1482
|
// repair/seed and before fanout. It starts ingress early, but never
|
|
1321
1483
|
// admits a second durable writer: queued realtime still waits for the
|
|
1322
1484
|
// existing scheduler's cooperative hand-off checkpoint.
|
|
1323
|
-
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
1485
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
|
|
1324
1486
|
};
|
|
1325
1487
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1326
1488
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -1513,8 +1675,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1513
1675
|
resolveInterruptWait = null;
|
|
1514
1676
|
resolve?.();
|
|
1515
1677
|
};
|
|
1516
|
-
const waitForNextPass = async (delayMs
|
|
1517
|
-
if (
|
|
1678
|
+
const waitForNextPass = async (delayMs) => {
|
|
1679
|
+
if (interruptGeneration !== observedInterruptGeneration) {
|
|
1518
1680
|
observedInterruptGeneration = interruptGeneration;
|
|
1519
1681
|
return;
|
|
1520
1682
|
}
|
|
@@ -1529,9 +1691,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1529
1691
|
}, idleHeartbeatIntervalMs);
|
|
1530
1692
|
heartbeatTimer.unref?.();
|
|
1531
1693
|
try {
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1694
|
+
await Promise.race([
|
|
1695
|
+
sleep(delayMs),
|
|
1696
|
+
new Promise((resolve) => {
|
|
1535
1697
|
resolveInterruptWait = resolve;
|
|
1536
1698
|
// Do not lose an event that arrived between the first generation
|
|
1537
1699
|
// check and registration of this wait.
|
|
@@ -1539,9 +1701,9 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1539
1701
|
resolveInterruptWait = null;
|
|
1540
1702
|
resolve();
|
|
1541
1703
|
}
|
|
1542
|
-
})
|
|
1543
|
-
|
|
1544
|
-
|
|
1704
|
+
}),
|
|
1705
|
+
stoppedSignal,
|
|
1706
|
+
]);
|
|
1545
1707
|
}
|
|
1546
1708
|
finally {
|
|
1547
1709
|
clearInterval(heartbeatTimer);
|
|
@@ -1628,32 +1790,19 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1628
1790
|
let coordinator = parallelCoordinatorEnabled ? createCoordinator() : null;
|
|
1629
1791
|
const runCoordinatorWork = async (work, task, options = {}) => {
|
|
1630
1792
|
const telemetry = work.class === "realtime" && parallelCoordinatorEnabled
|
|
1631
|
-
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr
|
|
1793
|
+
? new RealtimeTelemetrySpan(work.source, work.direction, work.paths.length, writeRealtimeTelemetryToStderr)
|
|
1632
1794
|
: undefined;
|
|
1633
1795
|
if (!coordinator) {
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
engineDispatched: true,
|
|
1645
|
-
};
|
|
1646
|
-
}
|
|
1647
|
-
catch (error) {
|
|
1648
|
-
if (work.class === "realtime" &&
|
|
1649
|
-
work.source === "receiver" &&
|
|
1650
|
-
work.direction === "pull") {
|
|
1651
|
-
const refused = receiverAdmissionRefusedError(error);
|
|
1652
|
-
if (refused)
|
|
1653
|
-
throw refused;
|
|
1654
|
-
}
|
|
1655
|
-
throw error;
|
|
1656
|
-
}
|
|
1796
|
+
return {
|
|
1797
|
+
result: await task({
|
|
1798
|
+
id: "legacy-direct",
|
|
1799
|
+
work,
|
|
1800
|
+
paths: [],
|
|
1801
|
+
signal: new AbortController().signal,
|
|
1802
|
+
}),
|
|
1803
|
+
accepted: true,
|
|
1804
|
+
coordinatorRefusedPaths: [],
|
|
1805
|
+
};
|
|
1657
1806
|
}
|
|
1658
1807
|
telemetry?.mark("worker");
|
|
1659
1808
|
const execution = {
|
|
@@ -1670,16 +1819,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1670
1819
|
const outcome = await coordinator.submit(work);
|
|
1671
1820
|
const executorSucceeded = execution.result !== undefined &&
|
|
1672
1821
|
coordinatorPathOutcome(passExitCode(execution.result)).status === "durable";
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
const requireEveryPathDurable = work.class === "realtime" &&
|
|
1678
|
-
work.source === "receiver" &&
|
|
1679
|
-
work.direction === "pull";
|
|
1680
|
-
const accepted = executorSucceeded && outcome.paths.every((path) => requireEveryPathDurable
|
|
1681
|
-
? path.status === "durable"
|
|
1682
|
-
: path.status === "durable" || path.status === "refused");
|
|
1822
|
+
const allRefused = outcome.paths.length > 0 && outcome.paths.every((path) => path.status === "refused");
|
|
1823
|
+
const accepted = executorSucceeded
|
|
1824
|
+
? outcome.paths.every((path) => path.status === "durable" || path.status === "refused")
|
|
1825
|
+
: allRefused;
|
|
1683
1826
|
if (telemetry) {
|
|
1684
1827
|
const outcomeLabel = executorSucceeded && outcome.paths.every((path) => path.status === "durable")
|
|
1685
1828
|
? "durable"
|
|
@@ -1689,9 +1832,8 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1689
1832
|
telemetry.finish(outcomeLabel, realtimeTelemetryReason(outcome.paths));
|
|
1690
1833
|
}
|
|
1691
1834
|
return {
|
|
1692
|
-
result: execution.result ?? 1,
|
|
1835
|
+
result: execution.result ?? (allRefused ? 0 : 1),
|
|
1693
1836
|
accepted,
|
|
1694
|
-
engineDispatched: execution.result !== undefined,
|
|
1695
1837
|
coordinatorRefusedPaths: normalizeCoordinatorPaths(outcome.paths
|
|
1696
1838
|
.filter((path) => path.status === "refused")
|
|
1697
1839
|
.map((path) => path.path)),
|
|
@@ -1703,13 +1845,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1703
1845
|
: error instanceof RealtimeDrainBurstError
|
|
1704
1846
|
? "drain-burst"
|
|
1705
1847
|
: undefined);
|
|
1706
|
-
if (work.class === "realtime" &&
|
|
1707
|
-
work.source === "receiver" &&
|
|
1708
|
-
work.direction === "pull") {
|
|
1709
|
-
const refused = receiverAdmissionRefusedError(error);
|
|
1710
|
-
if (refused)
|
|
1711
|
-
throw refused;
|
|
1712
|
-
}
|
|
1713
1848
|
throw error;
|
|
1714
1849
|
}
|
|
1715
1850
|
finally {
|
|
@@ -1717,17 +1852,16 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1717
1852
|
}
|
|
1718
1853
|
};
|
|
1719
1854
|
let nextCoordinatorCausality = 0;
|
|
1720
|
-
const realtimeWork = (source, direction, paths, lane
|
|
1855
|
+
const realtimeWork = (source, direction, paths, lane) => ({
|
|
1721
1856
|
class: "realtime",
|
|
1722
1857
|
source,
|
|
1723
1858
|
direction,
|
|
1724
1859
|
lane,
|
|
1725
1860
|
paths: normalizeCoordinatorPaths(paths),
|
|
1726
|
-
...(sequenceNumberByPath === undefined ? {} : { sequenceNumberByPath }),
|
|
1727
1861
|
causality: `${source}-${++nextCoordinatorCausality}`,
|
|
1728
1862
|
});
|
|
1729
1863
|
const choreWork = (kind) => ({ class: "chore", kind, areas: [] });
|
|
1730
|
-
const runRealtimePass = (source, direction, paths, lane, task
|
|
1864
|
+
const runRealtimePass = (source, direction, paths, lane, task) => runCoordinatorWork(realtimeWork(source, direction, paths, lane), task);
|
|
1731
1865
|
const enqueueChore = (kind, task, onSettled, options = { wholeMutation: true }) => {
|
|
1732
1866
|
void runCoordinatorWork(choreWork(kind), task, options).then(onSettled).catch((error) => {
|
|
1733
1867
|
process.stderr.write(`hq-sync-runner: coordinator ${kind} chore failed: ${describeError(error)}\n`);
|
|
@@ -3146,20 +3280,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3146
3280
|
// Submit every route in this lane before awaiting any result. In
|
|
3147
3281
|
// particular, a later route's fast receipt must never wait for an
|
|
3148
3282
|
// earlier route's slow pull to finish.
|
|
3149
|
-
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced)
|
|
3283
|
+
const completed = await Promise.all([...eventsByRoute.values()].map(({ route, events }) => runRealtimePass("receiver", "pull", events.map((event) => event.relativePath), lane, (slice, telemetry, coordinatorFenced) => runGuarded(buildTargetedPullArgv(route, passArgv, slice.paths.map((path) => path.canonicalPath)), undefined, lane, "realtime", lane === "fast", slice.signal, false, telemetry, coordinatorFenced))));
|
|
3150
3284
|
for (const execution of completed) {
|
|
3151
|
-
if (!execution.
|
|
3152
|
-
throw new ReceiverAdmissionRefusedError("targeted pull failed: refused before engine admission");
|
|
3153
|
-
}
|
|
3154
|
-
if (passExitCode(execution.result) !== 0) {
|
|
3285
|
+
if (!execution.accepted || passExitCode(execution.result) !== 0) {
|
|
3155
3286
|
throw new Error(`targeted pull failed with exit code ${passExitCode(execution.result)}`);
|
|
3156
3287
|
}
|
|
3157
|
-
if (!execution.accepted) {
|
|
3158
|
-
if (execution.coordinatorRefusedPaths.length > 0) {
|
|
3159
|
-
throw new ReceiverAdmissionRefusedError("targeted pull failed: paths were refused before engine admission");
|
|
3160
|
-
}
|
|
3161
|
-
throw new Error("targeted pull was not accepted by the coordinator");
|
|
3162
|
-
}
|
|
3163
3288
|
}
|
|
3164
3289
|
};
|
|
3165
3290
|
const receiverSyncBatchFn = async ({ events }) => {
|
|
@@ -3209,16 +3334,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3209
3334
|
}
|
|
3210
3335
|
for (const { route, paths } of pathsByRoute.values()) {
|
|
3211
3336
|
const targetedArgv = buildTargetedPullArgv(route, passArgv, paths);
|
|
3212
|
-
|
|
3213
|
-
try {
|
|
3214
|
-
result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
3215
|
-
}
|
|
3216
|
-
catch (error) {
|
|
3217
|
-
const refused = receiverAdmissionRefusedError(error);
|
|
3218
|
-
if (refused)
|
|
3219
|
-
throw refused;
|
|
3220
|
-
throw error;
|
|
3221
|
-
}
|
|
3337
|
+
const result = await runGuarded(targetedArgv, undefined, lane, "realtime");
|
|
3222
3338
|
if (passExitCode(result) !== 0) {
|
|
3223
3339
|
throw new Error(`targeted pull failed with exit code ${passExitCode(result)}`);
|
|
3224
3340
|
}
|
|
@@ -3316,14 +3432,11 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3316
3432
|
authToken: getAccessToken,
|
|
3317
3433
|
region: runtime.region,
|
|
3318
3434
|
});
|
|
3319
|
-
const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
|
|
3320
3435
|
const handles = await startES({
|
|
3321
3436
|
hqRoot,
|
|
3322
3437
|
apiUrl: runtime.apiUrl,
|
|
3323
3438
|
authToken: getAccessToken,
|
|
3324
|
-
deviceId:
|
|
3325
|
-
deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
|
|
3326
|
-
deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
|
|
3439
|
+
deviceId: getOrCreateMachineId(hqRoot),
|
|
3327
3440
|
resolveTenantId: async () => {
|
|
3328
3441
|
const persons = await eventSyncClient.entity.listByType("person");
|
|
3329
3442
|
const pick = pickCanonicalPersonEntity(persons);
|
|
@@ -3339,18 +3452,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3339
3452
|
publishedContentHashStateDir: getStateDir(),
|
|
3340
3453
|
publishedContentHashLookup,
|
|
3341
3454
|
onReceiveActivity,
|
|
3342
|
-
onReceiptOutcome: (receipt) => {
|
|
3343
|
-
// This is a bounded diagnostic trace, deliberately separate from
|
|
3344
|
-
// low-cardinality metrics. It includes only the irreversible hash
|
|
3345
|
-
// of the event path and the closed receipt outcome.
|
|
3346
|
-
process.stderr.write(`${JSON.stringify({
|
|
3347
|
-
event: "sync.realtime.receipt",
|
|
3348
|
-
schemaVersion: 1,
|
|
3349
|
-
source: "receiver",
|
|
3350
|
-
direction: "pull",
|
|
3351
|
-
...receipt,
|
|
3352
|
-
})}\n`);
|
|
3353
|
-
},
|
|
3354
3455
|
});
|
|
3355
3456
|
if (!handles)
|
|
3356
3457
|
return;
|
|
@@ -3626,15 +3727,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3626
3727
|
// The initial reconcile is due immediately. On-time passes advance from this
|
|
3627
3728
|
// deadline; overruns restart their cooldown from completion.
|
|
3628
3729
|
let nextDueAt = monotonicNow();
|
|
3629
|
-
const passExitRetryBudget = new WatchPassExitRetryBudget();
|
|
3630
|
-
let passExitRetryNotBefore = null;
|
|
3631
|
-
const recordWatchPassExit = (exitCode, now = monotonicNow()) => {
|
|
3632
|
-
const delayMs = passExitRetryBudget.record(exitCode, now);
|
|
3633
|
-
passExitRetryNotBefore = delayMs === 0 ? null : now + delayMs;
|
|
3634
|
-
if (delayMs > 0) {
|
|
3635
|
-
process.stderr.write(`hq-sync-runner: watch pass retry delayed ${delayMs}ms after exit=${exitCode}\n`);
|
|
3636
|
-
}
|
|
3637
|
-
};
|
|
3638
3730
|
let pollTick = 0;
|
|
3639
3731
|
let lastCompletedFullReconcileAt = null;
|
|
3640
3732
|
// Anchor for the elapsed-time cadence: the most recent full reconcile that
|
|
@@ -3664,17 +3756,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3664
3756
|
return RUNNER_HEAP_RECYCLE_EXIT;
|
|
3665
3757
|
if (coordinatorAuthRequired)
|
|
3666
3758
|
return runtime.authRequiredPassExit;
|
|
3667
|
-
if (passExitRetryNotBefore !== null) {
|
|
3668
|
-
const retryNow = monotonicNow();
|
|
3669
|
-
if (retryNow < passExitRetryNotBefore) {
|
|
3670
|
-
// A watcher/receiver interrupt must not turn a known failing pass
|
|
3671
|
-
// back into a tight loop. Pending work remains queued and runs when
|
|
3672
|
-
// this non-interruptible cooldown has elapsed.
|
|
3673
|
-
await waitForNextPass(passExitRetryNotBefore - retryNow, false);
|
|
3674
|
-
continue;
|
|
3675
|
-
}
|
|
3676
|
-
passExitRetryNotBefore = null;
|
|
3677
|
-
}
|
|
3678
3759
|
pollTick++;
|
|
3679
3760
|
areaAutoMigration.onTick();
|
|
3680
3761
|
const pending = takePendingWatcherChange();
|
|
@@ -3770,7 +3851,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3770
3851
|
};
|
|
3771
3852
|
let result;
|
|
3772
3853
|
let ranFullReconcile = false;
|
|
3773
|
-
let ranWatchPass = false;
|
|
3774
3854
|
let enqueuedFullChore = false;
|
|
3775
3855
|
if (shouldRunFull) {
|
|
3776
3856
|
emitDecision("full", reasons);
|
|
@@ -3790,9 +3870,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3790
3870
|
interruptWait();
|
|
3791
3871
|
return;
|
|
3792
3872
|
}
|
|
3793
|
-
recordWatchPassExit(exit);
|
|
3794
|
-
if (exit !== 0)
|
|
3795
|
-
interruptWait();
|
|
3796
3873
|
// Preserve the numeric success seam accepted by the prior
|
|
3797
3874
|
// coordinator while also accepting a completed RunnerPassOutcome
|
|
3798
3875
|
// (including a partial per-company pass) that proves journal
|
|
@@ -3837,7 +3914,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3837
3914
|
}
|
|
3838
3915
|
else {
|
|
3839
3916
|
ranFullReconcile = true;
|
|
3840
|
-
ranWatchPass = true;
|
|
3841
3917
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3842
3918
|
}
|
|
3843
3919
|
}
|
|
@@ -3867,7 +3943,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3867
3943
|
result = 0;
|
|
3868
3944
|
}
|
|
3869
3945
|
else {
|
|
3870
|
-
ranWatchPass = true;
|
|
3871
3946
|
result = overflowWithKnownRoutes
|
|
3872
3947
|
? await runOverflowRouteDrain(workingBatch)
|
|
3873
3948
|
: await runScopedDrain(workingBatch, pending.generation);
|
|
@@ -3885,7 +3960,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3885
3960
|
process.stderr.write(`hq-sync-runner: targeted event pass failed (${describeError(err)}); ` +
|
|
3886
3961
|
`falling back to a full reconcile\n`);
|
|
3887
3962
|
ranFullReconcile = true;
|
|
3888
|
-
ranWatchPass = true;
|
|
3889
3963
|
emitDecision("full", ["targeted-pass-failed"]);
|
|
3890
3964
|
result = await runGuarded(passArgv, undefined, "slow", "chore", false, undefined, true);
|
|
3891
3965
|
}
|
|
@@ -4031,8 +4105,6 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4031
4105
|
realtimeState.nextHighWaterAt = nextRealtimeHighWaterAt;
|
|
4032
4106
|
}
|
|
4033
4107
|
const now = monotonicNow();
|
|
4034
|
-
if (ranWatchPass)
|
|
4035
|
-
recordWatchPassExit(exitCode, now);
|
|
4036
4108
|
if (ranFullReconcile) {
|
|
4037
4109
|
// The awaited full pass finished on this tick; reuse the tick's clock
|
|
4038
4110
|
// sample so the cadence anchor never costs a second read.
|
|
@@ -4052,14 +4124,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
4052
4124
|
? { nextDueAt: now + nextPollMs, delayMs: nextPollMs }
|
|
4053
4125
|
: advanceMonotonicDeadline(nextDueAt, now, nextPollMs);
|
|
4054
4126
|
nextDueAt = schedule.nextDueAt;
|
|
4055
|
-
const retryDelayMs = passExitRetryNotBefore === null
|
|
4056
|
-
? 0
|
|
4057
|
-
: Math.max(0, passExitRetryNotBefore - now);
|
|
4058
|
-
const nextPassDelayMs = Math.max(schedule.delayMs, retryDelayMs);
|
|
4059
4127
|
const untilHighWaterMs = realtimeState.nextHighWaterAt === null
|
|
4060
|
-
?
|
|
4128
|
+
? schedule.delayMs
|
|
4061
4129
|
: Math.max(0, realtimeState.nextHighWaterAt - monotonicNow());
|
|
4062
|
-
await waitForNextPass(Math.min(
|
|
4130
|
+
await waitForNextPass(Math.min(schedule.delayMs, untilHighWaterMs));
|
|
4063
4131
|
}
|
|
4064
4132
|
return 0;
|
|
4065
4133
|
}
|