@indigoai-us/hq-cloud 6.18.3 → 6.18.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +56 -10
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +253 -17
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +103 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +56 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +15 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +31 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +228 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +31 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +120 -39
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +26 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +227 -62
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +252 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +21 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +140 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +274 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/lib/ontology-entity-push.d.ts +51 -0
- package/dist/lib/ontology-entity-push.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.js +125 -0
- package/dist/lib/ontology-entity-push.js.map +1 -0
- package/dist/lib/ontology-entity-push.test.d.ts +2 -0
- package/dist/lib/ontology-entity-push.test.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.test.js +149 -0
- package/dist/lib/ontology-entity-push.test.js.map +1 -0
- package/dist/object-io.d.ts +38 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +336 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +423 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +330 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +304 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +145 -14
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +266 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +7 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +39 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -11,7 +11,7 @@ import { acquireOperationLock, withOperationLock, withOperationLockSync, Operati
|
|
|
11
11
|
import { describeError } from "../lib/describe-error.js";
|
|
12
12
|
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
13
13
|
import { PARTIAL_SYNC_EXIT, UNRESOLVED_COMPANY_EXIT, } from "../lib/exit-codes.js";
|
|
14
|
-
import {
|
|
14
|
+
import { getOrCreateMachineIdMetadata } from "../lib/machine-id.js";
|
|
15
15
|
import { localPathForVaultKey } from "../local-path-codec.js";
|
|
16
16
|
import { TreeWatcher, resolveEventDebounceConfig, systemClock, } from "../watcher.js";
|
|
17
17
|
import { recoverHqRootLocalState } from "../hq-root-recovery.js";
|
|
@@ -153,6 +153,19 @@ export class CooperativePassScheduler {
|
|
|
153
153
|
activeFastRealtime = null;
|
|
154
154
|
stopped = false;
|
|
155
155
|
_handoffActive = false;
|
|
156
|
+
/** A chore's payload transfer is in flight while it lends the root lock. */
|
|
157
|
+
networkTransferActive = false;
|
|
158
|
+
detachedChore = null;
|
|
159
|
+
networkTransferClosing = false;
|
|
160
|
+
networkTransferCount = 0;
|
|
161
|
+
networkTransferEpoch = 0;
|
|
162
|
+
networkTransferWaiters = [];
|
|
163
|
+
networkTransferRelease = null;
|
|
164
|
+
resolveNetworkTransferRelease = null;
|
|
165
|
+
/** Successful callers still applying results after the network hand-off. */
|
|
166
|
+
networkTransferCompletionCount = 0;
|
|
167
|
+
networkTransferApplyRelease = null;
|
|
168
|
+
resolveNetworkTransferApplyRelease = null;
|
|
156
169
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
157
170
|
this.now = now;
|
|
158
171
|
this.starvationCapMs = starvationCapMs;
|
|
@@ -306,6 +319,7 @@ export class CooperativePassScheduler {
|
|
|
306
319
|
return;
|
|
307
320
|
if (this._handoffActive) {
|
|
308
321
|
this.promoteOccupancyArrivals();
|
|
322
|
+
this.admitHandoffBacklog();
|
|
309
323
|
return;
|
|
310
324
|
}
|
|
311
325
|
if (this.active !== null) {
|
|
@@ -415,15 +429,215 @@ export class CooperativePassScheduler {
|
|
|
415
429
|
execute(entry) {
|
|
416
430
|
if (entry.resourceCap)
|
|
417
431
|
this.options.caps?.beginChore();
|
|
432
|
+
const checkpoint = Object.assign(() => this.checkpoint(entry), {
|
|
433
|
+
shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
|
|
434
|
+
runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
|
|
435
|
+
});
|
|
418
436
|
return entry.task({
|
|
419
|
-
checkpoint
|
|
437
|
+
checkpoint,
|
|
420
438
|
signal: entry.controller.signal,
|
|
421
439
|
});
|
|
422
440
|
}
|
|
441
|
+
shouldYieldToRealtime(entry) {
|
|
442
|
+
return !this.stopped &&
|
|
443
|
+
entry.workClass === "chore" &&
|
|
444
|
+
this.active === entry &&
|
|
445
|
+
this.pending.some((pending) => pending.workClass === "realtime");
|
|
446
|
+
}
|
|
423
447
|
withinHandoffAdmission() {
|
|
424
448
|
if (this.handoffDeferredAt === null)
|
|
425
449
|
return false;
|
|
426
|
-
return this.
|
|
450
|
+
return this.networkTransferActive ||
|
|
451
|
+
this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Run payload I/O without owning the pass's durable state. Keep the chore
|
|
455
|
+
* slot reserved while realtime uses the root lock, then wait for active
|
|
456
|
+
* realtime writers before restoring chore ownership.
|
|
457
|
+
*/
|
|
458
|
+
async runChoreNetworkTransfer(entry, work) {
|
|
459
|
+
if (this.detachedChore === entry &&
|
|
460
|
+
this._handoffActive &&
|
|
461
|
+
this.networkTransferActive &&
|
|
462
|
+
!this.networkTransferClosing) {
|
|
463
|
+
return this.joinChoreNetworkTransfer(work);
|
|
464
|
+
}
|
|
465
|
+
if (this.detachedChore === entry &&
|
|
466
|
+
this._handoffActive &&
|
|
467
|
+
this.networkTransferRelease !== null) {
|
|
468
|
+
await this.networkTransferRelease;
|
|
469
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
470
|
+
}
|
|
471
|
+
if (this.active === entry && this.networkTransferApplyRelease !== null) {
|
|
472
|
+
await this.networkTransferApplyRelease;
|
|
473
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
474
|
+
}
|
|
475
|
+
if (this.stopped ||
|
|
476
|
+
entry.workClass !== "chore" ||
|
|
477
|
+
this.active !== entry) {
|
|
478
|
+
return { value: await work(), realtimeRan: false, complete: () => undefined };
|
|
479
|
+
}
|
|
480
|
+
this.active = null;
|
|
481
|
+
this._handoffActive = true;
|
|
482
|
+
this.networkTransferActive = true;
|
|
483
|
+
this.detachedChore = entry;
|
|
484
|
+
this.networkTransferClosing = false;
|
|
485
|
+
this.networkTransferCount = 1;
|
|
486
|
+
this.networkTransferCompletionCount = 1;
|
|
487
|
+
const applyReleased = new Promise((resolve) => {
|
|
488
|
+
this.resolveNetworkTransferApplyRelease = resolve;
|
|
489
|
+
});
|
|
490
|
+
this.networkTransferApplyRelease = applyReleased;
|
|
491
|
+
const released = new Promise((resolve) => {
|
|
492
|
+
this.resolveNetworkTransferRelease = resolve;
|
|
493
|
+
});
|
|
494
|
+
this.networkTransferRelease = released;
|
|
495
|
+
this.handoffDeferredAt = this.now();
|
|
496
|
+
this.handoffAdmitted = false;
|
|
497
|
+
this.handoffBacklog = null;
|
|
498
|
+
this.options.caps?.suspendForHandoff();
|
|
499
|
+
let value;
|
|
500
|
+
let failure;
|
|
501
|
+
let failed = false;
|
|
502
|
+
try {
|
|
503
|
+
// Admit an arrival that was queued just before the transfer began.
|
|
504
|
+
this.drain();
|
|
505
|
+
value = await work();
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
failed = true;
|
|
509
|
+
failure = err;
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
this.networkTransferClosing = true;
|
|
513
|
+
this.networkTransferCount -= 1;
|
|
514
|
+
this.notifyNetworkTransferChange();
|
|
515
|
+
while (this.networkTransferCount > 0) {
|
|
516
|
+
const seen = this.networkTransferEpoch;
|
|
517
|
+
if (this.networkTransferCount === 0)
|
|
518
|
+
break;
|
|
519
|
+
await this.waitNetworkTransferChange(seen);
|
|
520
|
+
}
|
|
521
|
+
this.networkTransferActive = false;
|
|
522
|
+
// A long transfer may outlive the ordinary realtime burst. Start a new
|
|
523
|
+
// bounded window for arrivals that queued safely during network I/O.
|
|
524
|
+
this.handoffDeferredAt = this.now();
|
|
525
|
+
try {
|
|
526
|
+
this.drain();
|
|
527
|
+
while (!this.stopped &&
|
|
528
|
+
(this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
529
|
+
if (this.realtimeOccupants.size === 0) {
|
|
530
|
+
this.admitHandoffBacklog();
|
|
531
|
+
if (this.realtimeOccupants.size === 0)
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
539
|
+
this.admitHandoffBacklog();
|
|
540
|
+
}
|
|
541
|
+
while (this.realtimeOccupants.size > 0) {
|
|
542
|
+
const seen = this.occupantEpoch;
|
|
543
|
+
if (this.realtimeOccupants.size === 0)
|
|
544
|
+
break;
|
|
545
|
+
await this.waitOccupantChange(seen);
|
|
546
|
+
this.promoteOccupancyArrivals();
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
this._handoffActive = false;
|
|
551
|
+
this.handoffDeferredAt = null;
|
|
552
|
+
this.handoffBacklog = null;
|
|
553
|
+
this.detachedChore = null;
|
|
554
|
+
if (!this.stopped)
|
|
555
|
+
this.active = entry;
|
|
556
|
+
this.options.caps?.resumeAfterHandoff();
|
|
557
|
+
const resolveRelease = this.resolveNetworkTransferRelease;
|
|
558
|
+
const realtimeRan = this.handoffAdmitted;
|
|
559
|
+
this.networkTransferRelease = null;
|
|
560
|
+
this.resolveNetworkTransferRelease = null;
|
|
561
|
+
this.networkTransferClosing = false;
|
|
562
|
+
resolveRelease?.(realtimeRan);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
const realtimeRan = await released;
|
|
566
|
+
if (failed) {
|
|
567
|
+
this.completeNetworkTransferParticipant();
|
|
568
|
+
throw failure;
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
value,
|
|
572
|
+
realtimeRan,
|
|
573
|
+
complete: this.createNetworkTransferCompletion(),
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
async joinChoreNetworkTransfer(work) {
|
|
577
|
+
const released = this.networkTransferRelease;
|
|
578
|
+
if (released === null) {
|
|
579
|
+
throw new Error("network transfer hand-off is missing its release signal");
|
|
580
|
+
}
|
|
581
|
+
this.networkTransferCount += 1;
|
|
582
|
+
this.networkTransferCompletionCount += 1;
|
|
583
|
+
let value;
|
|
584
|
+
let failure;
|
|
585
|
+
let failed = false;
|
|
586
|
+
try {
|
|
587
|
+
value = await work();
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
failed = true;
|
|
591
|
+
failure = err;
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
this.networkTransferCount -= 1;
|
|
595
|
+
this.notifyNetworkTransferChange();
|
|
596
|
+
}
|
|
597
|
+
const realtimeRan = await released;
|
|
598
|
+
if (failed) {
|
|
599
|
+
this.completeNetworkTransferParticipant();
|
|
600
|
+
throw failure;
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
value,
|
|
604
|
+
realtimeRan,
|
|
605
|
+
complete: this.createNetworkTransferCompletion(),
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
createNetworkTransferCompletion() {
|
|
609
|
+
let completed = false;
|
|
610
|
+
return () => {
|
|
611
|
+
if (completed)
|
|
612
|
+
return;
|
|
613
|
+
completed = true;
|
|
614
|
+
this.completeNetworkTransferParticipant();
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
completeNetworkTransferParticipant() {
|
|
618
|
+
if (this.networkTransferCompletionCount <= 0)
|
|
619
|
+
return;
|
|
620
|
+
this.networkTransferCompletionCount -= 1;
|
|
621
|
+
if (this.networkTransferCompletionCount > 0)
|
|
622
|
+
return;
|
|
623
|
+
const resolve = this.resolveNetworkTransferApplyRelease;
|
|
624
|
+
this.resolveNetworkTransferApplyRelease = null;
|
|
625
|
+
this.networkTransferApplyRelease = null;
|
|
626
|
+
resolve?.();
|
|
627
|
+
}
|
|
628
|
+
waitNetworkTransferChange(seenEpoch) {
|
|
629
|
+
if (this.networkTransferEpoch !== seenEpoch)
|
|
630
|
+
return Promise.resolve();
|
|
631
|
+
return new Promise((resolve) => {
|
|
632
|
+
this.networkTransferWaiters.push(resolve);
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
notifyNetworkTransferChange() {
|
|
636
|
+
this.networkTransferEpoch += 1;
|
|
637
|
+
const waiters = this.networkTransferWaiters;
|
|
638
|
+
this.networkTransferWaiters = [];
|
|
639
|
+
for (const resolve of waiters)
|
|
640
|
+
resolve();
|
|
427
641
|
}
|
|
428
642
|
waitOccupantChange(seenEpoch) {
|
|
429
643
|
if (this.occupantEpoch !== seenEpoch)
|
|
@@ -858,11 +1072,12 @@ export function takeEnumeratedBatchChunk(batch, limit) {
|
|
|
858
1072
|
};
|
|
859
1073
|
}
|
|
860
1074
|
/**
|
|
861
|
-
* Elapsed-time cadence for scheduled full reconciles:
|
|
1075
|
+
* Elapsed-time cadence for scheduled full reconciles: six hours by default,
|
|
862
1076
|
* measured from the last full pass (its dispatch or its completion, whichever
|
|
863
1077
|
* is later). `HQ_SYNC_FULL_RECONCILE_MS` overrides it (for example,
|
|
864
|
-
* 10_800_000 for three hours); an unset or invalid value falls back to
|
|
865
|
-
* hour
|
|
1078
|
+
* 10_800_000 for three hours); an unset or invalid value falls back to six
|
|
1079
|
+
* hours. The default was one hour until each routine pass's S3 list/HEAD and
|
|
1080
|
+
* API cost across every device outweighed the backstop's value at that rate.
|
|
866
1081
|
*
|
|
867
1082
|
* The previous default was "every `FULL_RECONCILE_EVERY_TICKS` ticks" with no
|
|
868
1083
|
* elapsed-time floor. Ticks fire on every watcher batch, not only on the
|
|
@@ -873,10 +1088,10 @@ export function takeEnumeratedBatchChunk(batch, limit) {
|
|
|
873
1088
|
* backstop behind event-push (a missed watcher event, a new directory the
|
|
874
1089
|
* native watch has not adopted yet); the surfaces-inactive, overflow, warm-up
|
|
875
1090
|
* and bare-wake triggers still force it immediately when live events cannot
|
|
876
|
-
* be trusted, so
|
|
1091
|
+
* be trusted, so six hours is the cadence of the backstop, not of realtime.
|
|
877
1092
|
*/
|
|
878
1093
|
export const FULL_RECONCILE_MS_ENV = "HQ_SYNC_FULL_RECONCILE_MS";
|
|
879
|
-
export const DEFAULT_FULL_RECONCILE_MS = 60 * 60_000;
|
|
1094
|
+
export const DEFAULT_FULL_RECONCILE_MS = 6 * 60 * 60_000;
|
|
880
1095
|
export function resolveFullReconcileMs(env = process.env) {
|
|
881
1096
|
const raw = env[FULL_RECONCILE_MS_ENV];
|
|
882
1097
|
if (raw === undefined || raw.trim() === "")
|
|
@@ -1269,7 +1484,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1269
1484
|
// repair/seed and before fanout. It starts ingress early, but never
|
|
1270
1485
|
// admits a second durable writer: queued realtime still waits for the
|
|
1271
1486
|
// existing scheduler's cooperative hand-off checkpoint.
|
|
1272
|
-
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
1487
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
|
|
1273
1488
|
};
|
|
1274
1489
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1275
1490
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -3219,11 +3434,14 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
3219
3434
|
authToken: getAccessToken,
|
|
3220
3435
|
region: runtime.region,
|
|
3221
3436
|
});
|
|
3437
|
+
const deviceIdentity = getOrCreateMachineIdMetadata(hqRoot);
|
|
3222
3438
|
const handles = await startES({
|
|
3223
3439
|
hqRoot,
|
|
3224
3440
|
apiUrl: runtime.apiUrl,
|
|
3225
3441
|
authToken: getAccessToken,
|
|
3226
|
-
deviceId:
|
|
3442
|
+
deviceId: deviceIdentity.machineId,
|
|
3443
|
+
deviceIdAgeBucket: deviceIdentity.deviceIdAgeBucket,
|
|
3444
|
+
deviceIdFileFreshlyCreatedThisRun: deviceIdentity.deviceIdFileFreshlyCreatedThisRun,
|
|
3227
3445
|
resolveTenantId: async () => {
|
|
3228
3446
|
const persons = await eventSyncClient.entity.listByType("person");
|
|
3229
3447
|
const pick = pickCanonicalPersonEntity(persons);
|