@indigoai-us/hq-cloud 6.18.1-alpha.1 → 6.18.1-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/bandwidth.d.ts +45 -11
  2. package/dist/bandwidth.d.ts.map +1 -1
  3. package/dist/bandwidth.js +231 -18
  4. package/dist/bandwidth.js.map +1 -1
  5. package/dist/bandwidth.test.js +124 -4
  6. package/dist/bandwidth.test.js.map +1 -1
  7. package/dist/bin/sync-runner-company.d.ts +2 -0
  8. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-company.js +6 -9
  10. package/dist/bin/sync-runner-company.js.map +1 -1
  11. package/dist/bin/sync-runner-company.test.js +0 -56
  12. package/dist/bin/sync-runner-company.test.js.map +1 -1
  13. package/dist/bin/sync-runner-events.test.js +0 -15
  14. package/dist/bin/sync-runner-events.test.js.map +1 -1
  15. package/dist/bin/sync-runner-watch-loop.d.ts +32 -8
  16. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-watch-loop.js +228 -13
  18. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  19. package/dist/bin/sync-runner-watch-loop.test.js +63 -5
  20. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  21. package/dist/bin/sync-runner.d.ts +26 -7
  22. package/dist/bin/sync-runner.d.ts.map +1 -1
  23. package/dist/bin/sync-runner.js +14 -5
  24. package/dist/bin/sync-runner.js.map +1 -1
  25. package/dist/bin/sync-runner.test.js +107 -1
  26. package/dist/bin/sync-runner.test.js.map +1 -1
  27. package/dist/cli/share.d.ts +18 -6
  28. package/dist/cli/share.d.ts.map +1 -1
  29. package/dist/cli/share.js +92 -80
  30. package/dist/cli/share.js.map +1 -1
  31. package/dist/cli/share.test.js +3 -35
  32. package/dist/cli/share.test.js.map +1 -1
  33. package/dist/cli/sync.d.ts +17 -8
  34. package/dist/cli/sync.d.ts.map +1 -1
  35. package/dist/cli/sync.js +155 -30
  36. package/dist/cli/sync.js.map +1 -1
  37. package/dist/cli/sync.test.js +315 -1
  38. package/dist/cli/sync.test.js.map +1 -1
  39. package/dist/object-io.d.ts +34 -23
  40. package/dist/object-io.d.ts.map +1 -1
  41. package/dist/object-io.js +333 -258
  42. package/dist/object-io.js.map +1 -1
  43. package/dist/object-io.test.js +377 -245
  44. package/dist/object-io.test.js.map +1 -1
  45. package/dist/s3-download-stream-hash.test.d.ts +2 -0
  46. package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
  47. package/dist/s3-download-stream-hash.test.js +70 -0
  48. package/dist/s3-download-stream-hash.test.js.map +1 -0
  49. package/dist/s3.d.ts +8 -3
  50. package/dist/s3.d.ts.map +1 -1
  51. package/dist/s3.js +304 -27
  52. package/dist/s3.js.map +1 -1
  53. package/dist/s3.test.js +163 -0
  54. package/dist/s3.test.js.map +1 -1
  55. package/dist/sync/full-pass-caps.d.ts.map +1 -1
  56. package/dist/sync/full-pass-caps.js +3 -21
  57. package/dist/sync/full-pass-caps.js.map +1 -1
  58. package/dist/sync/full-pass-caps.test.js +1 -56
  59. package/dist/sync/full-pass-caps.test.js.map +1 -1
  60. package/dist/sync/inbound-event-admission.d.ts.map +1 -1
  61. package/dist/sync/inbound-event-admission.js +3 -0
  62. package/dist/sync/inbound-event-admission.js.map +1 -1
  63. package/dist/sync/inbound-event-admission.test.js +6 -0
  64. package/dist/sync/inbound-event-admission.test.js.map +1 -1
  65. package/dist/sync-core.d.ts +2 -1
  66. package/dist/sync-core.d.ts.map +1 -1
  67. package/dist/sync-core.js +5 -10
  68. package/dist/sync-core.js.map +1 -1
  69. package/dist/sync-core.test.js +16 -0
  70. package/dist/sync-core.test.js.map +1 -1
  71. package/dist/vault-client.d.ts +0 -5
  72. package/dist/vault-client.d.ts.map +1 -1
  73. package/dist/vault-client.js +0 -7
  74. package/dist/vault-client.js.map +1 -1
  75. package/dist/vault-client.test.js +0 -39
  76. package/dist/vault-client.test.js.map +1 -1
  77. package/package.json +1 -1
  78. package/dist/sync/presign-denial-backoff-flag.d.ts +0 -6
  79. package/dist/sync/presign-denial-backoff-flag.d.ts.map +0 -1
  80. package/dist/sync/presign-denial-backoff-flag.js +0 -11
  81. package/dist/sync/presign-denial-backoff-flag.js.map +0 -1
@@ -153,6 +153,19 @@ export class CooperativePassScheduler {
153
153
  activeFastRealtime = null;
154
154
  stopped = false;
155
155
  _handoffActive = false;
156
+ /** A chore's payload transfer is in flight while it lends the root lock. */
157
+ networkTransferActive = false;
158
+ detachedChore = null;
159
+ networkTransferClosing = false;
160
+ networkTransferCount = 0;
161
+ networkTransferEpoch = 0;
162
+ networkTransferWaiters = [];
163
+ networkTransferRelease = null;
164
+ resolveNetworkTransferRelease = null;
165
+ /** Successful callers still applying results after the network hand-off. */
166
+ networkTransferCompletionCount = 0;
167
+ networkTransferApplyRelease = null;
168
+ resolveNetworkTransferApplyRelease = null;
156
169
  constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
157
170
  this.now = now;
158
171
  this.starvationCapMs = starvationCapMs;
@@ -306,6 +319,7 @@ export class CooperativePassScheduler {
306
319
  return;
307
320
  if (this._handoffActive) {
308
321
  this.promoteOccupancyArrivals();
322
+ this.admitHandoffBacklog();
309
323
  return;
310
324
  }
311
325
  if (this.active !== null) {
@@ -415,15 +429,215 @@ export class CooperativePassScheduler {
415
429
  execute(entry) {
416
430
  if (entry.resourceCap)
417
431
  this.options.caps?.beginChore();
432
+ const checkpoint = Object.assign(() => this.checkpoint(entry), {
433
+ shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
434
+ runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
435
+ });
418
436
  return entry.task({
419
- checkpoint: () => this.checkpoint(entry),
437
+ checkpoint,
420
438
  signal: entry.controller.signal,
421
439
  });
422
440
  }
441
+ shouldYieldToRealtime(entry) {
442
+ return !this.stopped &&
443
+ entry.workClass === "chore" &&
444
+ this.active === entry &&
445
+ this.pending.some((pending) => pending.workClass === "realtime");
446
+ }
423
447
  withinHandoffAdmission() {
424
448
  if (this.handoffDeferredAt === null)
425
449
  return false;
426
- return this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
450
+ return this.networkTransferActive ||
451
+ this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
452
+ }
453
+ /**
454
+ * Run payload I/O without owning the pass's durable state. Keep the chore
455
+ * slot reserved while realtime uses the root lock, then wait for active
456
+ * realtime writers before restoring chore ownership.
457
+ */
458
+ async runChoreNetworkTransfer(entry, work) {
459
+ if (this.detachedChore === entry &&
460
+ this._handoffActive &&
461
+ this.networkTransferActive &&
462
+ !this.networkTransferClosing) {
463
+ return this.joinChoreNetworkTransfer(work);
464
+ }
465
+ if (this.detachedChore === entry &&
466
+ this._handoffActive &&
467
+ this.networkTransferRelease !== null) {
468
+ await this.networkTransferRelease;
469
+ return this.runChoreNetworkTransfer(entry, work);
470
+ }
471
+ if (this.active === entry && this.networkTransferApplyRelease !== null) {
472
+ await this.networkTransferApplyRelease;
473
+ return this.runChoreNetworkTransfer(entry, work);
474
+ }
475
+ if (this.stopped ||
476
+ entry.workClass !== "chore" ||
477
+ this.active !== entry) {
478
+ return { value: await work(), realtimeRan: false, complete: () => undefined };
479
+ }
480
+ this.active = null;
481
+ this._handoffActive = true;
482
+ this.networkTransferActive = true;
483
+ this.detachedChore = entry;
484
+ this.networkTransferClosing = false;
485
+ this.networkTransferCount = 1;
486
+ this.networkTransferCompletionCount = 1;
487
+ const applyReleased = new Promise((resolve) => {
488
+ this.resolveNetworkTransferApplyRelease = resolve;
489
+ });
490
+ this.networkTransferApplyRelease = applyReleased;
491
+ const released = new Promise((resolve) => {
492
+ this.resolveNetworkTransferRelease = resolve;
493
+ });
494
+ this.networkTransferRelease = released;
495
+ this.handoffDeferredAt = this.now();
496
+ this.handoffAdmitted = false;
497
+ this.handoffBacklog = null;
498
+ this.options.caps?.suspendForHandoff();
499
+ let value;
500
+ let failure;
501
+ let failed = false;
502
+ try {
503
+ // Admit an arrival that was queued just before the transfer began.
504
+ this.drain();
505
+ value = await work();
506
+ }
507
+ catch (err) {
508
+ failed = true;
509
+ failure = err;
510
+ }
511
+ finally {
512
+ this.networkTransferClosing = true;
513
+ this.networkTransferCount -= 1;
514
+ this.notifyNetworkTransferChange();
515
+ while (this.networkTransferCount > 0) {
516
+ const seen = this.networkTransferEpoch;
517
+ if (this.networkTransferCount === 0)
518
+ break;
519
+ await this.waitNetworkTransferChange(seen);
520
+ }
521
+ this.networkTransferActive = false;
522
+ // A long transfer may outlive the ordinary realtime burst. Start a new
523
+ // bounded window for arrivals that queued safely during network I/O.
524
+ this.handoffDeferredAt = this.now();
525
+ try {
526
+ this.drain();
527
+ while (!this.stopped &&
528
+ (this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
529
+ if (this.realtimeOccupants.size === 0) {
530
+ this.admitHandoffBacklog();
531
+ if (this.realtimeOccupants.size === 0)
532
+ break;
533
+ }
534
+ const seen = this.occupantEpoch;
535
+ if (this.realtimeOccupants.size === 0)
536
+ break;
537
+ await this.waitOccupantChange(seen);
538
+ this.promoteOccupancyArrivals();
539
+ this.admitHandoffBacklog();
540
+ }
541
+ while (this.realtimeOccupants.size > 0) {
542
+ const seen = this.occupantEpoch;
543
+ if (this.realtimeOccupants.size === 0)
544
+ break;
545
+ await this.waitOccupantChange(seen);
546
+ this.promoteOccupancyArrivals();
547
+ }
548
+ }
549
+ finally {
550
+ this._handoffActive = false;
551
+ this.handoffDeferredAt = null;
552
+ this.handoffBacklog = null;
553
+ this.detachedChore = null;
554
+ if (!this.stopped)
555
+ this.active = entry;
556
+ this.options.caps?.resumeAfterHandoff();
557
+ const resolveRelease = this.resolveNetworkTransferRelease;
558
+ const realtimeRan = this.handoffAdmitted;
559
+ this.networkTransferRelease = null;
560
+ this.resolveNetworkTransferRelease = null;
561
+ this.networkTransferClosing = false;
562
+ resolveRelease?.(realtimeRan);
563
+ }
564
+ }
565
+ const realtimeRan = await released;
566
+ if (failed) {
567
+ this.completeNetworkTransferParticipant();
568
+ throw failure;
569
+ }
570
+ return {
571
+ value,
572
+ realtimeRan,
573
+ complete: this.createNetworkTransferCompletion(),
574
+ };
575
+ }
576
+ async joinChoreNetworkTransfer(work) {
577
+ const released = this.networkTransferRelease;
578
+ if (released === null) {
579
+ throw new Error("network transfer hand-off is missing its release signal");
580
+ }
581
+ this.networkTransferCount += 1;
582
+ this.networkTransferCompletionCount += 1;
583
+ let value;
584
+ let failure;
585
+ let failed = false;
586
+ try {
587
+ value = await work();
588
+ }
589
+ catch (err) {
590
+ failed = true;
591
+ failure = err;
592
+ }
593
+ finally {
594
+ this.networkTransferCount -= 1;
595
+ this.notifyNetworkTransferChange();
596
+ }
597
+ const realtimeRan = await released;
598
+ if (failed) {
599
+ this.completeNetworkTransferParticipant();
600
+ throw failure;
601
+ }
602
+ return {
603
+ value,
604
+ realtimeRan,
605
+ complete: this.createNetworkTransferCompletion(),
606
+ };
607
+ }
608
+ createNetworkTransferCompletion() {
609
+ let completed = false;
610
+ return () => {
611
+ if (completed)
612
+ return;
613
+ completed = true;
614
+ this.completeNetworkTransferParticipant();
615
+ };
616
+ }
617
+ completeNetworkTransferParticipant() {
618
+ if (this.networkTransferCompletionCount <= 0)
619
+ return;
620
+ this.networkTransferCompletionCount -= 1;
621
+ if (this.networkTransferCompletionCount > 0)
622
+ return;
623
+ const resolve = this.resolveNetworkTransferApplyRelease;
624
+ this.resolveNetworkTransferApplyRelease = null;
625
+ this.networkTransferApplyRelease = null;
626
+ resolve?.();
627
+ }
628
+ waitNetworkTransferChange(seenEpoch) {
629
+ if (this.networkTransferEpoch !== seenEpoch)
630
+ return Promise.resolve();
631
+ return new Promise((resolve) => {
632
+ this.networkTransferWaiters.push(resolve);
633
+ });
634
+ }
635
+ notifyNetworkTransferChange() {
636
+ this.networkTransferEpoch += 1;
637
+ const waiters = this.networkTransferWaiters;
638
+ this.networkTransferWaiters = [];
639
+ for (const resolve of waiters)
640
+ resolve();
427
641
  }
428
642
  waitOccupantChange(seenEpoch) {
429
643
  if (this.occupantEpoch !== seenEpoch)
@@ -961,18 +1175,19 @@ export function journalEntriesForWatcherDelete(files, key, kind) {
961
1175
  return Object.entries(files).filter(([candidate]) => key === "" || candidate === key || candidate.startsWith(`${key}/`));
962
1176
  }
963
1177
  /**
964
- * Return a journal hash only when its row demonstrably predates this runner.
1178
+ * Return a journal hash when the content is already present in the cloud.
965
1179
  *
966
- * A scoped push writes the completed upload's hash before the runner hands the
967
- * path to PushEventEmitter. The lazy tier is a startup baseline, not a live
968
- * "has this runner just pushed it?" lookup, so every row stamped at or after
969
- * this session's start is deliberately a miss. Invalid timestamps are also a
970
- * miss: publishing an extra realtime event is safer than suppressing one.
1180
+ * A downloaded row is already cloud content, regardless of when this runner
1181
+ * recorded it. For uploads and rows without a direction, only a row that
1182
+ * predates this runner is a baseline; a current-session upload still needs its
1183
+ * local watcher event. Invalid timestamps on those rows are a miss.
971
1184
  */
972
- export function publishedContentHashFromJournalEntryBeforeSession(entry, sessionStartedAt) {
1185
+ export function publishedContentHashFromJournalEntry(entry, sessionStartedAt) {
973
1186
  if (entry === undefined || entry.removedAt !== undefined || entry.kind === "symlink" || entry.localDiverges) {
974
1187
  return undefined;
975
1188
  }
1189
+ if (entry.direction === "down")
1190
+ return `sha256:${entry.hash}`;
976
1191
  const syncedAt = Date.parse(entry.syncedAt);
977
1192
  if (!Number.isFinite(syncedAt) || syncedAt >= sessionStartedAt)
978
1193
  return undefined;
@@ -1268,7 +1483,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
1268
1483
  // repair/seed and before fanout. It starts ingress early, but never
1269
1484
  // admits a second durable writer: queued realtime still waits for the
1270
1485
  // existing scheduler's cooperative hand-off checkpoint.
1271
- return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
1486
+ return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
1272
1487
  };
1273
1488
  // Observe the heap at pass boundaries — a gated checkpoint immediately before
1274
1489
  // admitting a pass and immediately after it returns. A pass that runs a long
@@ -2415,10 +2630,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
2415
2630
  if (!coordinates)
2416
2631
  return undefined;
2417
2632
  // Do not capture a session-sized object in the emitter closure. A durable
2418
- // cache miss reads precisely this journal key; only a valid row from
2419
- // before this runner began can seed the historical baseline.
2633
+ // cache miss reads precisely this journal key; prior rows and downloads
2634
+ // already present in the cloud can seed the historical baseline.
2420
2635
  const entry = journalRows(readJournalScoped(coordinates.slug, [coordinates.key])).get(coordinates.key);
2421
- return publishedContentHashFromJournalEntryBeforeSession(entry, serviceStartedAt);
2636
+ return publishedContentHashFromJournalEntry(entry, serviceStartedAt);
2422
2637
  };
2423
2638
  const prepareWatcherChanges = (changes) => {
2424
2639
  const keysByJournal = new Map();