@indigoai-us/hq-cloud 6.18.1-alpha.1 → 6.18.1-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +45 -11
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +231 -18
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +124 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +6 -9
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -56
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +0 -15
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +32 -8
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +228 -13
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +63 -5
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +26 -7
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +107 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -6
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +92 -80
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +3 -35
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +17 -8
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +155 -30
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +315 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +34 -23
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +333 -258
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +377 -245
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +304 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +163 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/full-pass-caps.d.ts.map +1 -1
- package/dist/sync/full-pass-caps.js +3 -21
- package/dist/sync/full-pass-caps.js.map +1 -1
- package/dist/sync/full-pass-caps.test.js +1 -56
- package/dist/sync/full-pass-caps.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +0 -5
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +0 -7
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +0 -39
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/sync/presign-denial-backoff-flag.d.ts +0 -6
- package/dist/sync/presign-denial-backoff-flag.d.ts.map +0 -1
- package/dist/sync/presign-denial-backoff-flag.js +0 -11
- package/dist/sync/presign-denial-backoff-flag.js.map +0 -1
|
@@ -153,6 +153,19 @@ export class CooperativePassScheduler {
|
|
|
153
153
|
activeFastRealtime = null;
|
|
154
154
|
stopped = false;
|
|
155
155
|
_handoffActive = false;
|
|
156
|
+
/** A chore's payload transfer is in flight while it lends the root lock. */
|
|
157
|
+
networkTransferActive = false;
|
|
158
|
+
detachedChore = null;
|
|
159
|
+
networkTransferClosing = false;
|
|
160
|
+
networkTransferCount = 0;
|
|
161
|
+
networkTransferEpoch = 0;
|
|
162
|
+
networkTransferWaiters = [];
|
|
163
|
+
networkTransferRelease = null;
|
|
164
|
+
resolveNetworkTransferRelease = null;
|
|
165
|
+
/** Successful callers still applying results after the network hand-off. */
|
|
166
|
+
networkTransferCompletionCount = 0;
|
|
167
|
+
networkTransferApplyRelease = null;
|
|
168
|
+
resolveNetworkTransferApplyRelease = null;
|
|
156
169
|
constructor(now, starvationCapMs = CHORE_STARVATION_CAP_MS, options = {}) {
|
|
157
170
|
this.now = now;
|
|
158
171
|
this.starvationCapMs = starvationCapMs;
|
|
@@ -306,6 +319,7 @@ export class CooperativePassScheduler {
|
|
|
306
319
|
return;
|
|
307
320
|
if (this._handoffActive) {
|
|
308
321
|
this.promoteOccupancyArrivals();
|
|
322
|
+
this.admitHandoffBacklog();
|
|
309
323
|
return;
|
|
310
324
|
}
|
|
311
325
|
if (this.active !== null) {
|
|
@@ -415,15 +429,215 @@ export class CooperativePassScheduler {
|
|
|
415
429
|
execute(entry) {
|
|
416
430
|
if (entry.resourceCap)
|
|
417
431
|
this.options.caps?.beginChore();
|
|
432
|
+
const checkpoint = Object.assign(() => this.checkpoint(entry), {
|
|
433
|
+
shouldYieldToRealtime: () => this.shouldYieldToRealtime(entry),
|
|
434
|
+
runChoreNetworkTransfer: (work) => this.runChoreNetworkTransfer(entry, work),
|
|
435
|
+
});
|
|
418
436
|
return entry.task({
|
|
419
|
-
checkpoint
|
|
437
|
+
checkpoint,
|
|
420
438
|
signal: entry.controller.signal,
|
|
421
439
|
});
|
|
422
440
|
}
|
|
441
|
+
shouldYieldToRealtime(entry) {
|
|
442
|
+
return !this.stopped &&
|
|
443
|
+
entry.workClass === "chore" &&
|
|
444
|
+
this.active === entry &&
|
|
445
|
+
this.pending.some((pending) => pending.workClass === "realtime");
|
|
446
|
+
}
|
|
423
447
|
withinHandoffAdmission() {
|
|
424
448
|
if (this.handoffDeferredAt === null)
|
|
425
449
|
return false;
|
|
426
|
-
return this.
|
|
450
|
+
return this.networkTransferActive ||
|
|
451
|
+
this.now() - this.handoffDeferredAt < realtimeDrainBurstMs();
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Run payload I/O without owning the pass's durable state. Keep the chore
|
|
455
|
+
* slot reserved while realtime uses the root lock, then wait for active
|
|
456
|
+
* realtime writers before restoring chore ownership.
|
|
457
|
+
*/
|
|
458
|
+
async runChoreNetworkTransfer(entry, work) {
|
|
459
|
+
if (this.detachedChore === entry &&
|
|
460
|
+
this._handoffActive &&
|
|
461
|
+
this.networkTransferActive &&
|
|
462
|
+
!this.networkTransferClosing) {
|
|
463
|
+
return this.joinChoreNetworkTransfer(work);
|
|
464
|
+
}
|
|
465
|
+
if (this.detachedChore === entry &&
|
|
466
|
+
this._handoffActive &&
|
|
467
|
+
this.networkTransferRelease !== null) {
|
|
468
|
+
await this.networkTransferRelease;
|
|
469
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
470
|
+
}
|
|
471
|
+
if (this.active === entry && this.networkTransferApplyRelease !== null) {
|
|
472
|
+
await this.networkTransferApplyRelease;
|
|
473
|
+
return this.runChoreNetworkTransfer(entry, work);
|
|
474
|
+
}
|
|
475
|
+
if (this.stopped ||
|
|
476
|
+
entry.workClass !== "chore" ||
|
|
477
|
+
this.active !== entry) {
|
|
478
|
+
return { value: await work(), realtimeRan: false, complete: () => undefined };
|
|
479
|
+
}
|
|
480
|
+
this.active = null;
|
|
481
|
+
this._handoffActive = true;
|
|
482
|
+
this.networkTransferActive = true;
|
|
483
|
+
this.detachedChore = entry;
|
|
484
|
+
this.networkTransferClosing = false;
|
|
485
|
+
this.networkTransferCount = 1;
|
|
486
|
+
this.networkTransferCompletionCount = 1;
|
|
487
|
+
const applyReleased = new Promise((resolve) => {
|
|
488
|
+
this.resolveNetworkTransferApplyRelease = resolve;
|
|
489
|
+
});
|
|
490
|
+
this.networkTransferApplyRelease = applyReleased;
|
|
491
|
+
const released = new Promise((resolve) => {
|
|
492
|
+
this.resolveNetworkTransferRelease = resolve;
|
|
493
|
+
});
|
|
494
|
+
this.networkTransferRelease = released;
|
|
495
|
+
this.handoffDeferredAt = this.now();
|
|
496
|
+
this.handoffAdmitted = false;
|
|
497
|
+
this.handoffBacklog = null;
|
|
498
|
+
this.options.caps?.suspendForHandoff();
|
|
499
|
+
let value;
|
|
500
|
+
let failure;
|
|
501
|
+
let failed = false;
|
|
502
|
+
try {
|
|
503
|
+
// Admit an arrival that was queued just before the transfer began.
|
|
504
|
+
this.drain();
|
|
505
|
+
value = await work();
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
failed = true;
|
|
509
|
+
failure = err;
|
|
510
|
+
}
|
|
511
|
+
finally {
|
|
512
|
+
this.networkTransferClosing = true;
|
|
513
|
+
this.networkTransferCount -= 1;
|
|
514
|
+
this.notifyNetworkTransferChange();
|
|
515
|
+
while (this.networkTransferCount > 0) {
|
|
516
|
+
const seen = this.networkTransferEpoch;
|
|
517
|
+
if (this.networkTransferCount === 0)
|
|
518
|
+
break;
|
|
519
|
+
await this.waitNetworkTransferChange(seen);
|
|
520
|
+
}
|
|
521
|
+
this.networkTransferActive = false;
|
|
522
|
+
// A long transfer may outlive the ordinary realtime burst. Start a new
|
|
523
|
+
// bounded window for arrivals that queued safely during network I/O.
|
|
524
|
+
this.handoffDeferredAt = this.now();
|
|
525
|
+
try {
|
|
526
|
+
this.drain();
|
|
527
|
+
while (!this.stopped &&
|
|
528
|
+
(this.realtimeOccupants.size > 0 || this.withinHandoffAdmission())) {
|
|
529
|
+
if (this.realtimeOccupants.size === 0) {
|
|
530
|
+
this.admitHandoffBacklog();
|
|
531
|
+
if (this.realtimeOccupants.size === 0)
|
|
532
|
+
break;
|
|
533
|
+
}
|
|
534
|
+
const seen = this.occupantEpoch;
|
|
535
|
+
if (this.realtimeOccupants.size === 0)
|
|
536
|
+
break;
|
|
537
|
+
await this.waitOccupantChange(seen);
|
|
538
|
+
this.promoteOccupancyArrivals();
|
|
539
|
+
this.admitHandoffBacklog();
|
|
540
|
+
}
|
|
541
|
+
while (this.realtimeOccupants.size > 0) {
|
|
542
|
+
const seen = this.occupantEpoch;
|
|
543
|
+
if (this.realtimeOccupants.size === 0)
|
|
544
|
+
break;
|
|
545
|
+
await this.waitOccupantChange(seen);
|
|
546
|
+
this.promoteOccupancyArrivals();
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
finally {
|
|
550
|
+
this._handoffActive = false;
|
|
551
|
+
this.handoffDeferredAt = null;
|
|
552
|
+
this.handoffBacklog = null;
|
|
553
|
+
this.detachedChore = null;
|
|
554
|
+
if (!this.stopped)
|
|
555
|
+
this.active = entry;
|
|
556
|
+
this.options.caps?.resumeAfterHandoff();
|
|
557
|
+
const resolveRelease = this.resolveNetworkTransferRelease;
|
|
558
|
+
const realtimeRan = this.handoffAdmitted;
|
|
559
|
+
this.networkTransferRelease = null;
|
|
560
|
+
this.resolveNetworkTransferRelease = null;
|
|
561
|
+
this.networkTransferClosing = false;
|
|
562
|
+
resolveRelease?.(realtimeRan);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
const realtimeRan = await released;
|
|
566
|
+
if (failed) {
|
|
567
|
+
this.completeNetworkTransferParticipant();
|
|
568
|
+
throw failure;
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
value,
|
|
572
|
+
realtimeRan,
|
|
573
|
+
complete: this.createNetworkTransferCompletion(),
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
async joinChoreNetworkTransfer(work) {
|
|
577
|
+
const released = this.networkTransferRelease;
|
|
578
|
+
if (released === null) {
|
|
579
|
+
throw new Error("network transfer hand-off is missing its release signal");
|
|
580
|
+
}
|
|
581
|
+
this.networkTransferCount += 1;
|
|
582
|
+
this.networkTransferCompletionCount += 1;
|
|
583
|
+
let value;
|
|
584
|
+
let failure;
|
|
585
|
+
let failed = false;
|
|
586
|
+
try {
|
|
587
|
+
value = await work();
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
failed = true;
|
|
591
|
+
failure = err;
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
this.networkTransferCount -= 1;
|
|
595
|
+
this.notifyNetworkTransferChange();
|
|
596
|
+
}
|
|
597
|
+
const realtimeRan = await released;
|
|
598
|
+
if (failed) {
|
|
599
|
+
this.completeNetworkTransferParticipant();
|
|
600
|
+
throw failure;
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
value,
|
|
604
|
+
realtimeRan,
|
|
605
|
+
complete: this.createNetworkTransferCompletion(),
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
createNetworkTransferCompletion() {
|
|
609
|
+
let completed = false;
|
|
610
|
+
return () => {
|
|
611
|
+
if (completed)
|
|
612
|
+
return;
|
|
613
|
+
completed = true;
|
|
614
|
+
this.completeNetworkTransferParticipant();
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
completeNetworkTransferParticipant() {
|
|
618
|
+
if (this.networkTransferCompletionCount <= 0)
|
|
619
|
+
return;
|
|
620
|
+
this.networkTransferCompletionCount -= 1;
|
|
621
|
+
if (this.networkTransferCompletionCount > 0)
|
|
622
|
+
return;
|
|
623
|
+
const resolve = this.resolveNetworkTransferApplyRelease;
|
|
624
|
+
this.resolveNetworkTransferApplyRelease = null;
|
|
625
|
+
this.networkTransferApplyRelease = null;
|
|
626
|
+
resolve?.();
|
|
627
|
+
}
|
|
628
|
+
waitNetworkTransferChange(seenEpoch) {
|
|
629
|
+
if (this.networkTransferEpoch !== seenEpoch)
|
|
630
|
+
return Promise.resolve();
|
|
631
|
+
return new Promise((resolve) => {
|
|
632
|
+
this.networkTransferWaiters.push(resolve);
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
notifyNetworkTransferChange() {
|
|
636
|
+
this.networkTransferEpoch += 1;
|
|
637
|
+
const waiters = this.networkTransferWaiters;
|
|
638
|
+
this.networkTransferWaiters = [];
|
|
639
|
+
for (const resolve of waiters)
|
|
640
|
+
resolve();
|
|
427
641
|
}
|
|
428
642
|
waitOccupantChange(seenEpoch) {
|
|
429
643
|
if (this.occupantEpoch !== seenEpoch)
|
|
@@ -961,18 +1175,19 @@ export function journalEntriesForWatcherDelete(files, key, kind) {
|
|
|
961
1175
|
return Object.entries(files).filter(([candidate]) => key === "" || candidate === key || candidate.startsWith(`${key}/`));
|
|
962
1176
|
}
|
|
963
1177
|
/**
|
|
964
|
-
* Return a journal hash
|
|
1178
|
+
* Return a journal hash when the content is already present in the cloud.
|
|
965
1179
|
*
|
|
966
|
-
* A
|
|
967
|
-
*
|
|
968
|
-
*
|
|
969
|
-
*
|
|
970
|
-
* miss: publishing an extra realtime event is safer than suppressing one.
|
|
1180
|
+
* A downloaded row is already cloud content, regardless of when this runner
|
|
1181
|
+
* recorded it. For uploads and rows without a direction, only a row that
|
|
1182
|
+
* predates this runner is a baseline; a current-session upload still needs its
|
|
1183
|
+
* local watcher event. Invalid timestamps on those rows are a miss.
|
|
971
1184
|
*/
|
|
972
|
-
export function
|
|
1185
|
+
export function publishedContentHashFromJournalEntry(entry, sessionStartedAt) {
|
|
973
1186
|
if (entry === undefined || entry.removedAt !== undefined || entry.kind === "symlink" || entry.localDiverges) {
|
|
974
1187
|
return undefined;
|
|
975
1188
|
}
|
|
1189
|
+
if (entry.direction === "down")
|
|
1190
|
+
return `sha256:${entry.hash}`;
|
|
976
1191
|
const syncedAt = Date.parse(entry.syncedAt);
|
|
977
1192
|
if (!Number.isFinite(syncedAt) || syncedAt >= sessionStartedAt)
|
|
978
1193
|
return undefined;
|
|
@@ -1268,7 +1483,7 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
1268
1483
|
// repair/seed and before fanout. It starts ingress early, but never
|
|
1269
1484
|
// admits a second durable writer: queued realtime still waits for the
|
|
1270
1485
|
// existing scheduler's cooperative hand-off checkpoint.
|
|
1271
|
-
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts);
|
|
1486
|
+
return runPass(passArgvForRun, checkpoint, () => activateEventPushSurfaces(), recordDurableAppliedUpserts, workClass);
|
|
1272
1487
|
};
|
|
1273
1488
|
// Observe the heap at pass boundaries — a gated checkpoint immediately before
|
|
1274
1489
|
// admitting a pass and immediately after it returns. A pass that runs a long
|
|
@@ -2415,10 +2630,10 @@ export async function runWatchLoop(argv, parsed, deps, runtime) {
|
|
|
2415
2630
|
if (!coordinates)
|
|
2416
2631
|
return undefined;
|
|
2417
2632
|
// Do not capture a session-sized object in the emitter closure. A durable
|
|
2418
|
-
// cache miss reads precisely this journal key;
|
|
2419
|
-
//
|
|
2633
|
+
// cache miss reads precisely this journal key; prior rows and downloads
|
|
2634
|
+
// already present in the cloud can seed the historical baseline.
|
|
2420
2635
|
const entry = journalRows(readJournalScoped(coordinates.slug, [coordinates.key])).get(coordinates.key);
|
|
2421
|
-
return
|
|
2636
|
+
return publishedContentHashFromJournalEntry(entry, serviceStartedAt);
|
|
2422
2637
|
};
|
|
2423
2638
|
const prepareWatcherChanges = (changes) => {
|
|
2424
2639
|
const keysByJournal = new Map();
|