@indigoai-us/hq-cloud 6.17.1 → 6.18.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +45 -11
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +231 -18
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +124 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/outpost-session-host.d.ts +6 -0
- package/dist/bin/outpost-session-host.d.ts.map +1 -0
- package/dist/bin/outpost-session-host.js +39 -0
- package/dist/bin/outpost-session-host.js.map +1 -0
- package/dist/bin/outpost-session-host.test.d.ts +7 -0
- package/dist/bin/outpost-session-host.test.d.ts.map +1 -0
- package/dist/bin/outpost-session-host.test.js +37 -0
- package/dist/bin/outpost-session-host.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +6 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +26 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +217 -3
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +26 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +107 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -6
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +92 -47
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +3 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +17 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +155 -27
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +315 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +33 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +326 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +377 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +304 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +163 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +3 -2
|
@@ -153,6 +153,19 @@ export class CooperativePassScheduler {
|
|
|
153
153
|
activeFastRealtime = null;
|
|
154
154
|
stopped = false;
|
|
155
155
|
_handoffActive = false;
|
|
156
|
+
/** A chore's payload transfer is in flight while it lends the root lock. */
|
|
157
|
+
networkTransferActive = false;
|
|
158
|
+
detachedChore = null;
|
|
159
|
+
networkTransferClosing = false;
|
|
160
|
+
networkTransferCount = 0;
|
|
161
|
+
networkTransferEpoch = 0;
|
|
162
|
+
networkTransferWaiters = [];
|
|
163
|
+
networkTransferRelease = null;
|
|
164
|
+
resolveNetworkTransferRelease = null;
|
|
165
|
+
/** Successful callers still applying results after the network hand-off. */
|
|
166
|
+
networkTransferCompletionCount = 0;
|
|
167
|
+
networkTransferApplyRelease = null;
|
|
168
|
+
resolveNetworkTransferApplyRelease = null;
|
|
156
169
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
157
170
|
this.now = now;
|
|
158
171
|
this.starvationCapMs = starvationCapMs;
|
|
@@ -306,6 +319,7 @@ export class CooperativePassScheduler {
|
|
|
306
319
|
return;
|
|
307
320
|
if (this._handoffActive) {
|
|
308
321
|
this.promoteOccupancyArrivals();
|
|
322
|
+
this.admitHandoffBacklog();
|
|
309
323
|
return;
|
|
310
324
|
}
|
|
311
325
|
if (this.active !== null) {
|
|
@@ -415,15 +429,215 @@ export class CooperativePassScheduler {
|
|
|
415
429
|
execute(entry) {
|
|
416
430
|
if (entry.resourceCap)
|
|
417
431
|
this.options.caps?.beginChore();
|
|
432
|
+
const checkpoint = Object.assign(() => this.checkpoint(entry), {
|
|
433
|
+
shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
|
|
434
|
+
runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
|
|
435
|
+
});
|
|
418
436
|
return entry.task({
|
|
419
|
-
checkpoint
|
|
437
|
+
checkpoint,
|
|
420
438
|
signal: entry.controller.signal,
|
|
421
439
|
});
|
|
422
440
|
}
|
|
441
|
+
shouldYieldToRealtime(entry) {
|
|
442
|
+
return !this.stopped &&
|
|
443
|
+
entry.workClass === "chore" &&
|
|
444
|
+
this.active === entry &&
|
|
445
|
+
this.pending.some((pending) => pending.workClass === "realtime");
|
|
446
|
+
}
|
|
423
447
|
withinHandoffAdmission() {
|
|
424
448
|
if (this.handoffDeferredAt === null)
|
|
425
449
|
return false;
|
|
426
|
-
return this.
|
|
450
|
+
return this.networkTransferActive ||
|
|
451
|
+
this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Run payload I/O without owning the pass's durable state. Keep the chore
|
|
455
|
+
* slot reserved while realtime uses the root lock, then wait for active
|
|
456
|
+
* realtime writers before restoring chore ownership.
|
|
457
|
+
*/
|
|
458
|
+
async runChoreNetworkTransfer(entry, work) {
|
|
459
|
+
if (this.detachedChore === entry &&
|
|
460
|
+
this._handoffActive &&
|
|
461
|
+
this.networkTransferActive &&
|
|
462
|
+
!this.networkTransferClosing) {
|
|
463
|
+
return this.joinChoreNetworkTransfer(work);
|
|
464
|
+
}
|
|
465
|
+
if (this.detachedChore === entry &&
|
|
466
|
+
this._handoffActive &&
|
|
467
|
+
this.networkTransferRelease !== null) {
|
|
468
|
+
await this.networkTransferRelease;
|
|
469
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
470
|
+
}
|
|
471
|
+
if (this.active === entry && this.networkTransferApplyRelease !== null) {
|
|
472
|
+
await this.networkTransferApplyRelease;
|
|
473
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
474
|
+
}
|
|
475
|
+
if (this.stopped ||
|
|
476
|
+
entry.workClass !== "chore" ||
|
|
477
|
+
this.active !== entry) {
|
|
478
|
+
return { value: await work(), realtimeRan: false, complete: () => undefined };
|
|
479
|
+
}
|
|
480
|
+
this.active = null;
|
|
481
|
+
this._handoffActive = true;
|
|
482
|
+
this.networkTransferActive = true;
|
|
483
|
+
this.detachedChore = entry;
|
|
484
|
+
this.networkTransferClosing = false;
|
|
485
|
+
this.networkTransferCount = 1;
|
|
486
|
+
this.networkTransferCompletionCount = 1;
|
|
487
|
+
const applyReleased = new Promise((resolve) => {
|
|
488
|
+
this.resolveNetworkTransferApplyRelease = resolve;
|
|
489
|
+
});
|
|
490
|
+
this.networkTransferApplyRelease = applyReleased;
|
|
491
|
+
const released = new Promise((resolve) => {
|
|
492
|
+
this.resolveNetworkTransferRelease = resolve;
|
|
493
|
+
});
|
|
494
|
+
this.networkTransferRelease = released;
|
|
495
|
+
this.handoffDeferredAt = this.now();
|
|
496
|
+
this.handoffAdmitted = false;
|
|
497
|
+
this.handoffBacklog = null;
|
|
498
|
+
this.options.caps?.suspendForHandoff();
|
|
499
|
+
let value;
|
|
500
|
+
let failure;
|
|
501
|
+
let failed = false;
|
|
502
|
+
try {
|
|
503
|
+
// Admit an arrival that was queued just before the transfer began.
|
|
504
|
+
this.drain();
|
|
505
|
+
value = await work();
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
failed = true;
|
|
509
|
+
failure = err;
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
this.networkTransferClosing = true;
|
|
513
|
+
this.networkTransferCount -= 1;
|
|
514
|
+
this.notifyNetworkTransferChange();
|
|
515
|
+
while (this.networkTransferCount > 0) {
|
|
516
|
+
const seen = this.networkTransferEpoch;
|
|
517
|
+
if (this.networkTransferCount === 0)
|
|
518
|
+
break;
|
|
519
|
+
await this.waitNetworkTransferChange(seen);
|
|
520
|
+
}
|
|
521
|
+
this.networkTransferActive = false;
|
|
522
|
+
// A long transfer may outlive the ordinary realtime burst. Start a new
|
|
523
|
+
// bounded window for arrivals that queued safely during network I/O.
|
|
524
|
+
this.handoffDeferredAt = this.now();
|
|
525
|
+
try {
|
|
526
|
+
this.drain();
|
|
527
|
+
while (!this.stopped &&
|
|
528
|
+
(this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
529
|
+
if (this.realtimeOccupants.size === 0) {
|
|
530
|
+
this.admitHandoffBacklog();
|
|
531
|
+
if (this.realtimeOccupants.size === 0)
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
539
|
+
this.admitHandoffBacklog();
|
|
540
|
+
}
|
|
541
|
+
while (this.realtimeOccupants.size > 0) {
|
|
542
|
+
const seen = this.occupantEpoch;
|
|
543
|
+
if (this.realtimeOccupants.size === 0)
|
|
544
|
+
break;
|
|
545
|
+
await this.waitOccupantChange(seen);
|
|
546
|
+
this.promoteOccupancyArrivals();
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
this._handoffActive = false;
|
|
551
|
+
this.handoffDeferredAt = null;
|
|
552
|
+
this.handoffBacklog = null;
|
|
553
|
+
this.detachedChore = null;
|
|
554
|
+
if (!this.stopped)
|
|
555
|
+
this.active = entry;
|
|
556
|
+
this.options.caps?.resumeAfterHandoff();
|
|
557
|
+
const resolveRelease = this.resolveNetworkTransferRelease;
|
|
558
|
+
const realtimeRan = this.handoffAdmitted;
|
|
559
|
+
this.networkTransferRelease = null;
|
|
560
|
+
this.resolveNetworkTransferRelease = null;
|
|
561
|
+
this.networkTransferClosing = false;
|
|
562
|
+
resolveRelease?.(realtimeRan);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
const realtimeRan = await released;
|
|
566
|
+
if (failed) {
|
|
567
|
+
this.completeNetworkTransferParticipant();
|
|
568
|
+
throw failure;
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
value,
|
|
572
|
+
realtimeRan,
|
|
573
|
+
complete: this.createNetworkTransferCompletion(),
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
async joinChoreNetworkTransfer(work) {
|
|
577
|
+
const released = this.networkTransferRelease;
|
|
578
|
+
if (released === null) {
|
|
579
|
+
throw new Error("network transfer hand-off is missing its release signal");
|
|
580
|
+
}
|
|
581
|
+
this.networkTransferCount += 1;
|
|
582
|
+
this.networkTransferCompletionCount += 1;
|
|
583
|
+
let value;
|
|
584
|
+
let failure;
|
|
585
|
+
let failed = false;
|
|
586
|
+
try {
|
|
587
|
+
value = await work();
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
failed = true;
|
|
591
|
+
failure = err;
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
this.networkTransferCount -= 1;
|
|
595
|
+
this.notifyNetworkTransferChange();
|
|
596
|
+
}
|
|
597
|
+
const realtimeRan = await released;
|
|
598
|
+
if (failed) {
|
|
599
|
+
this.completeNetworkTransferParticipant();
|
|
600
|
+
throw failure;
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
value,
|
|
604
|
+
realtimeRan,
|
|
605
|
+
complete: this.createNetworkTransferCompletion(),
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
createNetworkTransferCompletion() {
|
|
609
|
+
let completed = false;
|
|
610
|
+
return () => {
|
|
611
|
+
if (completed)
|
|
612
|
+
return;
|
|
613
|
+
completed = true;
|
|
614
|
+
this.completeNetworkTransferParticipant();
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
completeNetworkTransferParticipant() {
|
|
618
|
+
if (this.networkTransferCompletionCount <= 0)
|
|
619
|
+
return;
|
|
620
|
+
this.networkTransferCompletionCount -= 1;
|
|
621
|
+
if (this.networkTransferCompletionCount > 0)
|
|
622
|
+
return;
|
|
623
|
+
const resolve = this.resolveNetworkTransferApplyRelease;
|
|
624
|
+
this.resolveNetworkTransferApplyRelease = null;
|
|
625
|
+
this.networkTransferApplyRelease = null;
|
|
626
|
+
resolve?.();
|
|
627
|
+
}
|
|
628
|
+
waitNetworkTransferChange(seenEpoch) {
|
|
629
|
+
if (this.networkTransferEpoch !== seenEpoch)
|
|
630
|
+
return Promise.resolve();
|
|
631
|
+
return new Promise((resolve) => {
|
|
632
|
+
this.networkTransferWaiters.push(resolve);
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
notifyNetworkTransferChange() {
|
|
636
|
+
this.networkTransferEpoch += 1;
|
|
637
|
+
const waiters = this.networkTransferWaiters;
|
|
638
|
+
this.networkTransferWaiters = [];
|
|
639
|
+
for (const resolve of waiters)
|
|
640
|
+
resolve();
|
|
427
641
|
}
|
|
428
642
|
waitOccupantChange(seenEpoch) {
|
|
429
643
|
if (this.occupantEpoch !== seenEpoch)
|
|
@@ -1268,7 +1482,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1268
1482
|
// repair/seed and before fanout. It starts ingress early, but never
|
|
1269
1483
|
// admits a second durable writer: queued realtime still waits for the
|
|
1270
1484
|
// existing scheduler's cooperative hand-off checkpoint.
|
|
1271
|
-
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
1485
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
|
|
1272
1486
|
};
|
|
1273
1487
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1274
1488
|
// admitting a pass and immediately after it returns. A pass that runs a long
|