@indigoai-us/hq-cloud 6.17.1 → 6.18.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/bandwidth.d.ts +45 -11
  2. package/dist/bandwidth.d.ts.map +1 -1
  3. package/dist/bandwidth.js +231 -18
  4. package/dist/bandwidth.js.map +1 -1
  5. package/dist/bandwidth.test.js +124 -4
  6. package/dist/bandwidth.test.js.map +1 -1
  7. package/dist/bin/outpost-session-host.d.ts +6 -0
  8. package/dist/bin/outpost-session-host.d.ts.map +1 -0
  9. package/dist/bin/outpost-session-host.js +39 -0
  10. package/dist/bin/outpost-session-host.js.map +1 -0
  11. package/dist/bin/outpost-session-host.test.d.ts +7 -0
  12. package/dist/bin/outpost-session-host.test.d.ts.map +1 -0
  13. package/dist/bin/outpost-session-host.test.js +37 -0
  14. package/dist/bin/outpost-session-host.test.js.map +1 -0
  15. package/dist/bin/sync-runner-company.d.ts +2 -0
  16. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-company.js +6 -0
  18. package/dist/bin/sync-runner-company.js.map +1 -1
  19. package/dist/bin/sync-runner-watch-loop.d.ts +26 -1
  20. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  21. package/dist/bin/sync-runner-watch-loop.js +217 -3
  22. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  23. package/dist/bin/sync-runner.d.ts +26 -2
  24. package/dist/bin/sync-runner.d.ts.map +1 -1
  25. package/dist/bin/sync-runner.js +14 -5
  26. package/dist/bin/sync-runner.js.map +1 -1
  27. package/dist/bin/sync-runner.test.js +107 -1
  28. package/dist/bin/sync-runner.test.js.map +1 -1
  29. package/dist/cli/share.d.ts +18 -6
  30. package/dist/cli/share.d.ts.map +1 -1
  31. package/dist/cli/share.js +92 -47
  32. package/dist/cli/share.js.map +1 -1
  33. package/dist/cli/share.test.js +3 -3
  34. package/dist/cli/share.test.js.map +1 -1
  35. package/dist/cli/sync.d.ts +17 -4
  36. package/dist/cli/sync.d.ts.map +1 -1
  37. package/dist/cli/sync.js +155 -27
  38. package/dist/cli/sync.js.map +1 -1
  39. package/dist/cli/sync.test.js +315 -1
  40. package/dist/cli/sync.test.js.map +1 -1
  41. package/dist/object-io.d.ts +33 -7
  42. package/dist/object-io.d.ts.map +1 -1
  43. package/dist/object-io.js +326 -140
  44. package/dist/object-io.js.map +1 -1
  45. package/dist/object-io.test.js +377 -1
  46. package/dist/object-io.test.js.map +1 -1
  47. package/dist/s3-download-stream-hash.test.d.ts +2 -0
  48. package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
  49. package/dist/s3-download-stream-hash.test.js +70 -0
  50. package/dist/s3-download-stream-hash.test.js.map +1 -0
  51. package/dist/s3.d.ts +8 -3
  52. package/dist/s3.d.ts.map +1 -1
  53. package/dist/s3.js +304 -27
  54. package/dist/s3.js.map +1 -1
  55. package/dist/s3.test.js +163 -0
  56. package/dist/s3.test.js.map +1 -1
  57. package/dist/sync/inbound-event-admission.d.ts.map +1 -1
  58. package/dist/sync/inbound-event-admission.js +3 -0
  59. package/dist/sync/inbound-event-admission.js.map +1 -1
  60. package/dist/sync/inbound-event-admission.test.js +6 -0
  61. package/dist/sync/inbound-event-admission.test.js.map +1 -1
  62. package/dist/sync-core.d.ts +2 -1
  63. package/dist/sync-core.d.ts.map +1 -1
  64. package/dist/sync-core.js +5 -10
  65. package/dist/sync-core.js.map +1 -1
  66. package/dist/sync-core.test.js +16 -0
  67. package/dist/sync-core.test.js.map +1 -1
  68. package/package.json +3 -2
@@ -153,6 +153,19 @@ export class CooperativePassScheduler {
153
153
  activeFastRealtime = null;
154
154
  stopped = false;
155
155
  _handoffActive = false;
156
+ /** A chore's payload transfer is in flight while it lends the root lock. */
157
+ networkTransferActive = false;
158
+ detachedChore = null;
159
+ networkTransferClosing = false;
160
+ networkTransferCount = 0;
161
+ networkTransferEpoch = 0;
162
+ networkTransferWaiters = [];
163
+ networkTransferRelease = null;
164
+ resolveNetworkTransferRelease = null;
165
+ /** Successful callers still applying results after the network hand-off. */
166
+ networkTransferCompletionCount = 0;
167
+ networkTransferApplyRelease = null;
168
+ resolveNetworkTransferApplyRelease = null;
156
169
  constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
157
170
  this.now = now;
158
171
  this.starvationCapMs = starvationCapMs;
@@ -306,6 +319,7 @@ export class CooperativePassScheduler {
306
319
  return;
307
320
  if (this._handoffActive) {
308
321
  this.promoteOccupancyArrivals();
322
+ this.admitHandoffBacklog();
309
323
  return;
310
324
  }
311
325
  if (this.active !== null) {
@@ -415,15 +429,215 @@ export class CooperativePassScheduler {
415
429
  execute(entry) {
416
430
  if (entry.resourceCap)
417
431
  this.options.caps?.beginChore();
432
+ const checkpoint = Object.assign(() => this.checkpoint(entry), {
433
+ shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
434
+ runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
435
+ });
418
436
  return entry.task({
419
- checkpoint: () => this.checkpoint(entry),
437
+ checkpoint,
420
438
  signal: entry.controller.signal,
421
439
  });
422
440
  }
441
+ shouldYieldToRealtime(entry) {
442
+ return !this.stopped &&
443
+ entry.workClass === "chore" &&
444
+ this.active === entry &&
445
+ this.pending.some((pending) => pending.workClass === "realtime");
446
+ }
423
447
  withinHandoffAdmission() {
424
448
  if (this.handoffDeferredAt === null)
425
449
  return false;
426
- return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
450
+ return this.networkTransferActive ||
451
+ this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
452
+ }
453
+ /**
454
+ * Run payload I/O without owning the pass's durable state. Keep the chore
455
+ * slot reserved while realtime uses the root lock, then wait for active
456
+ * realtime writers before restoring chore ownership.
457
+ */
458
+ async runChoreNetworkTransfer(entry, work) {
459
+ if (this.detachedChore === entry &&
460
+ this._handoffActive &&
461
+ this.networkTransferActive &&
462
+ !this.networkTransferClosing) {
463
+ return this.joinChoreNetworkTransfer(work);
464
+ }
465
+ if (this.detachedChore === entry &&
466
+ this._handoffActive &&
467
+ this.networkTransferRelease !== null) {
468
+ await this.networkTransferRelease;
469
+ return this.runChoreNetworkTransfer(entry, work);
470
+ }
471
+ if (this.active === entry && this.networkTransferApplyRelease !== null) {
472
+ await this.networkTransferApplyRelease;
473
+ return this.runChoreNetworkTransfer(entry, work);
474
+ }
475
+ if (this.stopped ||
476
+ entry.workClass !== "chore" ||
477
+ this.active !== entry) {
478
+ return { value: await work(), realtimeRan: false, complete: () => undefined };
479
+ }
480
+ this.active = null;
481
+ this._handoffActive = true;
482
+ this.networkTransferActive = true;
483
+ this.detachedChore = entry;
484
+ this.networkTransferClosing = false;
485
+ this.networkTransferCount = 1;
486
+ this.networkTransferCompletionCount = 1;
487
+ const applyReleased = new Promise((resolve) => {
488
+ this.resolveNetworkTransferApplyRelease = resolve;
489
+ });
490
+ this.networkTransferApplyRelease = applyReleased;
491
+ const released = new Promise((resolve) => {
492
+ this.resolveNetworkTransferRelease = resolve;
493
+ });
494
+ this.networkTransferRelease = released;
495
+ this.handoffDeferredAt = this.now();
496
+ this.handoffAdmitted = false;
497
+ this.handoffBacklog = null;
498
+ this.options.caps?.suspendForHandoff();
499
+ let value;
500
+ let failure;
501
+ let failed = false;
502
+ try {
503
+ // Admit an arrival that was queued just before the transfer began.
504
+ this.drain();
505
+ value = await work();
506
+ }
507
+ catch (err) {
508
+ failed = true;
509
+ failure = err;
510
+ }
511
+ finally {
512
+ this.networkTransferClosing = true;
513
+ this.networkTransferCount -= 1;
514
+ this.notifyNetworkTransferChange();
515
+ while (this.networkTransferCount > 0) {
516
+ const seen = this.networkTransferEpoch;
517
+ if (this.networkTransferCount === 0)
518
+ break;
519
+ await this.waitNetworkTransferChange(seen);
520
+ }
521
+ this.networkTransferActive = false;
522
+ // A long transfer may outlive the ordinary realtime burst. Start a new
523
+ // bounded window for arrivals that queued safely during network I/O.
524
+ this.handoffDeferredAt = this.now();
525
+ try {
526
+ this.drain();
527
+ while (!this.stopped &&
528
+ (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
529
+ if (this.realtimeOccupants.size === 0) {
530
+ this.admitHandoffBacklog();
531
+ if (this.realtimeOccupants.size === 0)
532
+ break;
533
+ }
534
+ const seen = this.occupantEpoch;
535
+ if (this.realtimeOccupants.size === 0)
536
+ break;
537
+ await this.waitOccupantChange(seen);
538
+ this.promoteOccupancyArrivals();
539
+ this.admitHandoffBacklog();
540
+ }
541
+ while (this.realtimeOccupants.size > 0) {
542
+ const seen = this.occupantEpoch;
543
+ if (this.realtimeOccupants.size === 0)
544
+ break;
545
+ await this.waitOccupantChange(seen);
546
+ this.promoteOccupancyArrivals();
547
+ }
548
+ }
549
+ finally {
550
+ this._handoffActive = false;
551
+ this.handoffDeferredAt = null;
552
+ this.handoffBacklog = null;
553
+ this.detachedChore = null;
554
+ if (!this.stopped)
555
+ this.active = entry;
556
+ this.options.caps?.resumeAfterHandoff();
557
+ const resolveRelease = this.resolveNetworkTransferRelease;
558
+ const realtimeRan = this.handoffAdmitted;
559
+ this.networkTransferRelease = null;
560
+ this.resolveNetworkTransferRelease = null;
561
+ this.networkTransferClosing = false;
562
+ resolveRelease?.(realtimeRan);
563
+ }
564
+ }
565
+ const realtimeRan = await released;
566
+ if (failed) {
567
+ this.completeNetworkTransferParticipant();
568
+ throw failure;
569
+ }
570
+ return {
571
+ value,
572
+ realtimeRan,
573
+ complete: this.createNetworkTransferCompletion(),
574
+ };
575
+ }
576
+ async joinChoreNetworkTransfer(work) {
577
+ const released = this.networkTransferRelease;
578
+ if (released === null) {
579
+ throw new Error("network transfer hand-off is missing its release signal");
580
+ }
581
+ this.networkTransferCount += 1;
582
+ this.networkTransferCompletionCount += 1;
583
+ let value;
584
+ let failure;
585
+ let failed = false;
586
+ try {
587
+ value = await work();
588
+ }
589
+ catch (err) {
590
+ failed = true;
591
+ failure = err;
592
+ }
593
+ finally {
594
+ this.networkTransferCount -= 1;
595
+ this.notifyNetworkTransferChange();
596
+ }
597
+ const realtimeRan = await released;
598
+ if (failed) {
599
+ this.completeNetworkTransferParticipant();
600
+ throw failure;
601
+ }
602
+ return {
603
+ value,
604
+ realtimeRan,
605
+ complete: this.createNetworkTransferCompletion(),
606
+ };
607
+ }
608
+ createNetworkTransferCompletion() {
609
+ let completed = false;
610
+ return () => {
611
+ if (completed)
612
+ return;
613
+ completed = true;
614
+ this.completeNetworkTransferParticipant();
615
+ };
616
+ }
617
+ completeNetworkTransferParticipant() {
618
+ if (this.networkTransferCompletionCount <= 0)
619
+ return;
620
+ this.networkTransferCompletionCount -= 1;
621
+ if (this.networkTransferCompletionCount > 0)
622
+ return;
623
+ const resolve = this.resolveNetworkTransferApplyRelease;
624
+ this.resolveNetworkTransferApplyRelease = null;
625
+ this.networkTransferApplyRelease = null;
626
+ resolve?.();
627
+ }
628
+ waitNetworkTransferChange(seenEpoch) {
629
+ if (this.networkTransferEpoch !== seenEpoch)
630
+ return Promise.resolve();
631
+ return new Promise((resolve) => {
632
+ this.networkTransferWaiters.push(resolve);
633
+ });
634
+ }
635
+ notifyNetworkTransferChange() {
636
+ this.networkTransferEpoch += 1;
637
+ const waiters = this.networkTransferWaiters;
638
+ this.networkTransferWaiters = [];
639
+ for (const resolve of waiters)
640
+ resolve();
427
641
  }
428
642
  waitOccupantChange(seenEpoch) {
429
643
  if (this.occupantEpoch !== seenEpoch)
@@ -1268,7 +1482,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1268
1482
  // repair/seed and before fanout. It starts ingress early, but never
1269
1483
  // admits a second durable writer: queued realtime still waits for the
1270
1484
  // existing scheduler's cooperative hand-off checkpoint.
1271
- return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
1485
+ return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
1272
1486
  };
1273
1487
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
1274
1488
  // admitting a pass and immediately after it returns. A pass that runs a long