@korajs/sync 1.0.0-beta.2 → 1.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +283 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +82 -2
- package/dist/index.d.ts +82 -2
- package/dist/index.js +283 -41
- package/dist/index.js.map +1 -1
- package/dist/internal.d.cts +1 -1
- package/dist/internal.d.ts +1 -1
- package/dist/{transport-BGr-_eH8.d.cts → transport-DXmZ8kyI.d.cts} +52 -0
- package/dist/{transport-BGr-_eH8.d.ts → transport-DXmZ8kyI.d.ts} +52 -0
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -528,7 +528,8 @@ function toProtoEnvelope(message) {
|
|
|
528
528
|
})),
|
|
529
529
|
schemaVersion: message.schemaVersion,
|
|
530
530
|
authToken: message.authToken,
|
|
531
|
-
supportedWireFormats: message.supportedWireFormats
|
|
531
|
+
supportedWireFormats: message.supportedWireFormats,
|
|
532
|
+
...message.lastDeliverySequence !== void 0 ? { lastDeliverySequence: message.lastDeliverySequence } : {}
|
|
532
533
|
};
|
|
533
534
|
case "handshake-response":
|
|
534
535
|
return {
|
|
@@ -544,7 +545,8 @@ function toProtoEnvelope(message) {
|
|
|
544
545
|
rejectReason: message.rejectReason,
|
|
545
546
|
selectedWireFormat: message.selectedWireFormat,
|
|
546
547
|
serverTime: message.serverTime,
|
|
547
|
-
blobStorageEnabled: message.blobStorageEnabled
|
|
548
|
+
blobStorageEnabled: message.blobStorageEnabled,
|
|
549
|
+
...message.serverMaxDeliverySequence !== void 0 ? { serverMaxDeliverySequence: message.serverMaxDeliverySequence } : {}
|
|
548
550
|
};
|
|
549
551
|
case "operation-batch":
|
|
550
552
|
return {
|
|
@@ -552,14 +554,17 @@ function toProtoEnvelope(message) {
|
|
|
552
554
|
messageId: message.messageId,
|
|
553
555
|
operations: message.operations.map(serializeProtoOperation),
|
|
554
556
|
isFinal: message.isFinal,
|
|
555
|
-
batchIndex: message.batchIndex
|
|
557
|
+
batchIndex: message.batchIndex,
|
|
558
|
+
...message.baseDeliverySequence !== void 0 ? { baseDeliverySequence: message.baseDeliverySequence } : {},
|
|
559
|
+
...message.maxDeliverySequence !== void 0 ? { maxDeliverySequence: message.maxDeliverySequence } : {}
|
|
556
560
|
};
|
|
557
561
|
case "acknowledgment":
|
|
558
562
|
return {
|
|
559
563
|
type: message.type,
|
|
560
564
|
messageId: message.messageId,
|
|
561
565
|
acknowledgedMessageId: message.acknowledgedMessageId,
|
|
562
|
-
lastSequenceNumber: message.lastSequenceNumber
|
|
566
|
+
lastSequenceNumber: message.lastSequenceNumber,
|
|
567
|
+
...message.deliverySequence !== void 0 ? { deliverySequence: message.deliverySequence } : {}
|
|
563
568
|
};
|
|
564
569
|
case "error":
|
|
565
570
|
return {
|
|
@@ -624,7 +629,8 @@ function fromProtoEnvelope(envelope) {
|
|
|
624
629
|
authToken: envelope.authToken,
|
|
625
630
|
supportedWireFormats: envelope.supportedWireFormats?.filter(
|
|
626
631
|
(format) => format === "json" || format === "protobuf"
|
|
627
|
-
)
|
|
632
|
+
),
|
|
633
|
+
...envelope.lastDeliverySequence !== void 0 ? { lastDeliverySequence: envelope.lastDeliverySequence } : {}
|
|
628
634
|
};
|
|
629
635
|
case "handshake-response":
|
|
630
636
|
return {
|
|
@@ -641,7 +647,8 @@ function fromProtoEnvelope(envelope) {
|
|
|
641
647
|
// Preserve the server's wall-clock time so clock-skew detection and
|
|
642
648
|
// automatic timestamp rebase work over the protobuf wire, not just JSON.
|
|
643
649
|
...envelope.serverTime !== void 0 ? { serverTime: envelope.serverTime } : {},
|
|
644
|
-
...envelope.blobStorageEnabled !== void 0 ? { blobStorageEnabled: envelope.blobStorageEnabled } : {}
|
|
650
|
+
...envelope.blobStorageEnabled !== void 0 ? { blobStorageEnabled: envelope.blobStorageEnabled } : {},
|
|
651
|
+
...envelope.serverMaxDeliverySequence !== void 0 ? { serverMaxDeliverySequence: envelope.serverMaxDeliverySequence } : {}
|
|
645
652
|
};
|
|
646
653
|
case "operation-batch":
|
|
647
654
|
return {
|
|
@@ -649,14 +656,17 @@ function fromProtoEnvelope(envelope) {
|
|
|
649
656
|
messageId: envelope.messageId,
|
|
650
657
|
operations: (envelope.operations ?? []).map(deserializeProtoOperation),
|
|
651
658
|
isFinal: envelope.isFinal ?? false,
|
|
652
|
-
batchIndex: envelope.batchIndex ?? 0
|
|
659
|
+
batchIndex: envelope.batchIndex ?? 0,
|
|
660
|
+
...envelope.baseDeliverySequence !== void 0 ? { baseDeliverySequence: envelope.baseDeliverySequence } : {},
|
|
661
|
+
...envelope.maxDeliverySequence !== void 0 ? { maxDeliverySequence: envelope.maxDeliverySequence } : {}
|
|
653
662
|
};
|
|
654
663
|
case "acknowledgment":
|
|
655
664
|
return {
|
|
656
665
|
type: "acknowledgment",
|
|
657
666
|
messageId: envelope.messageId,
|
|
658
667
|
acknowledgedMessageId: envelope.acknowledgedMessageId ?? "",
|
|
659
|
-
lastSequenceNumber: envelope.lastSequenceNumber ?? 0
|
|
668
|
+
lastSequenceNumber: envelope.lastSequenceNumber ?? 0,
|
|
669
|
+
...envelope.deliverySequence !== void 0 ? { deliverySequence: envelope.deliverySequence } : {}
|
|
660
670
|
};
|
|
661
671
|
case "error":
|
|
662
672
|
return {
|
|
@@ -857,6 +867,15 @@ function encodeEnvelope(envelope) {
|
|
|
857
867
|
writer.uint32(218).string(envelope.collection);
|
|
858
868
|
if (envelope.recordId && envelope.recordId.length > 0)
|
|
859
869
|
writer.uint32(226).string(envelope.recordId);
|
|
870
|
+
if (envelope.lastDeliverySequence !== void 0)
|
|
871
|
+
writer.uint32(232).int64(envelope.lastDeliverySequence);
|
|
872
|
+
if (envelope.baseDeliverySequence !== void 0)
|
|
873
|
+
writer.uint32(240).int64(envelope.baseDeliverySequence);
|
|
874
|
+
if (envelope.maxDeliverySequence !== void 0)
|
|
875
|
+
writer.uint32(248).int64(envelope.maxDeliverySequence);
|
|
876
|
+
if (envelope.deliverySequence !== void 0) writer.uint32(256).int64(envelope.deliverySequence);
|
|
877
|
+
if (envelope.serverMaxDeliverySequence !== void 0)
|
|
878
|
+
writer.uint32(264).int64(envelope.serverMaxDeliverySequence);
|
|
860
879
|
return writer.finish();
|
|
861
880
|
}
|
|
862
881
|
function decodeEnvelope(bytes) {
|
|
@@ -952,6 +971,21 @@ function decodeEnvelope(bytes) {
|
|
|
952
971
|
case 28:
|
|
953
972
|
envelope.recordId = reader.string();
|
|
954
973
|
break;
|
|
974
|
+
case 29:
|
|
975
|
+
envelope.lastDeliverySequence = longToNumber(reader.int64());
|
|
976
|
+
break;
|
|
977
|
+
case 30:
|
|
978
|
+
envelope.baseDeliverySequence = longToNumber(reader.int64());
|
|
979
|
+
break;
|
|
980
|
+
case 31:
|
|
981
|
+
envelope.maxDeliverySequence = longToNumber(reader.int64());
|
|
982
|
+
break;
|
|
983
|
+
case 32:
|
|
984
|
+
envelope.deliverySequence = longToNumber(reader.int64());
|
|
985
|
+
break;
|
|
986
|
+
case 33:
|
|
987
|
+
envelope.serverMaxDeliverySequence = longToNumber(reader.int64());
|
|
988
|
+
break;
|
|
955
989
|
default:
|
|
956
990
|
reader.skipType(tag & 7);
|
|
957
991
|
}
|
|
@@ -1451,6 +1485,7 @@ var ChaosTransport = class {
|
|
|
1451
1485
|
reorderRate;
|
|
1452
1486
|
maxLatency;
|
|
1453
1487
|
random;
|
|
1488
|
+
dropPredicate;
|
|
1454
1489
|
messageHandler = null;
|
|
1455
1490
|
reorderBuffer = [];
|
|
1456
1491
|
pending = [];
|
|
@@ -1461,6 +1496,7 @@ var ChaosTransport = class {
|
|
|
1461
1496
|
this.reorderRate = config?.reorderRate ?? 0;
|
|
1462
1497
|
this.maxLatency = config?.maxLatency ?? 0;
|
|
1463
1498
|
this.random = config?.randomSource ?? Math.random;
|
|
1499
|
+
this.dropPredicate = config?.dropPredicate ?? null;
|
|
1464
1500
|
}
|
|
1465
1501
|
async connect(url, options) {
|
|
1466
1502
|
this.inner.onMessage((msg) => this.handleIncoming(msg));
|
|
@@ -1479,6 +1515,9 @@ var ChaosTransport = class {
|
|
|
1479
1515
|
return this.inner.disconnect();
|
|
1480
1516
|
}
|
|
1481
1517
|
send(message) {
|
|
1518
|
+
if (this.dropPredicate?.(message, "outgoing")) {
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1482
1521
|
if (this.random() < this.dropRate) {
|
|
1483
1522
|
return;
|
|
1484
1523
|
}
|
|
@@ -1506,6 +1545,9 @@ var ChaosTransport = class {
|
|
|
1506
1545
|
}
|
|
1507
1546
|
handleIncoming(message) {
|
|
1508
1547
|
if (!this.messageHandler) return;
|
|
1548
|
+
if (this.dropPredicate?.(message, "incoming")) {
|
|
1549
|
+
return;
|
|
1550
|
+
}
|
|
1509
1551
|
if (this.random() < this.dropRate) {
|
|
1510
1552
|
return;
|
|
1511
1553
|
}
|
|
@@ -2602,6 +2644,17 @@ var nextQuerySubsetId = 0;
|
|
|
2602
2644
|
function generateMessageId() {
|
|
2603
2645
|
return `msg-${Date.now()}-${nextMessageId++}`;
|
|
2604
2646
|
}
|
|
2647
|
+
var MAX_DELIVERY_VIEW_WATERMARKS = 64;
|
|
2648
|
+
function stableStringify(value) {
|
|
2649
|
+
if (value === null || typeof value !== "object") {
|
|
2650
|
+
return JSON.stringify(value) ?? "null";
|
|
2651
|
+
}
|
|
2652
|
+
if (Array.isArray(value)) {
|
|
2653
|
+
return `[${value.map(stableStringify).join(",")}]`;
|
|
2654
|
+
}
|
|
2655
|
+
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);
|
|
2656
|
+
return `{${entries.join(",")}}`;
|
|
2657
|
+
}
|
|
2605
2658
|
var SyncEngine = class {
|
|
2606
2659
|
state = "disconnected";
|
|
2607
2660
|
transport;
|
|
@@ -2651,6 +2704,25 @@ var SyncEngine = class {
|
|
|
2651
2704
|
/** Resume cursor for paginated initial sync (persisted across reconnects) */
|
|
2652
2705
|
resumeDeltaCursor = null;
|
|
2653
2706
|
initialSyncTotalBatches = 0;
|
|
2707
|
+
/**
|
|
2708
|
+
* The delivery watermark for the CURRENT sync view: the highest server delivery
|
|
2709
|
+
* sequence up to which every operation in the current view (scope + active query
|
|
2710
|
+
* subscriptions) has been contiguously applied. Advanced only through the server's
|
|
2711
|
+
* gap-free delivery stream (never through live relay), so it is a durable lower bound
|
|
2712
|
+
* on what this client holds for this view. Reported at handshake to resume the stream.
|
|
2713
|
+
*
|
|
2714
|
+
* The watermark is keyed by view: switching views (a scope or subscription change)
|
|
2715
|
+
* saves the current view's watermark and loads the target view's, so returning to a
|
|
2716
|
+
* previously-synced view resumes exactly where it left off instead of re-syncing. This
|
|
2717
|
+
* is correct because a view's watermark is a lower bound specific to that view's
|
|
2718
|
+
* filter; a widened view is simply a different (initially unsynced) view.
|
|
2719
|
+
*/
|
|
2720
|
+
deliveryWatermark = 0;
|
|
2721
|
+
/**
|
|
2722
|
+
* In-memory cache of watermark per view signature, preloaded from persistence at start
|
|
2723
|
+
* so a view switch resolves synchronously (no race with the reconnect it triggers).
|
|
2724
|
+
*/
|
|
2725
|
+
deliverySignatureWatermarks = /* @__PURE__ */ new Map();
|
|
2654
2726
|
constructor(options) {
|
|
2655
2727
|
this.transport = options.transport;
|
|
2656
2728
|
this.store = options.store;
|
|
@@ -2714,6 +2786,16 @@ var SyncEngine = class {
|
|
|
2714
2786
|
if (this.syncState.loadDeltaCursor) {
|
|
2715
2787
|
this.resumeDeltaCursor = await this.syncState.loadDeltaCursor();
|
|
2716
2788
|
}
|
|
2789
|
+
if (this.syncState.loadAllDeliveryWatermarks) {
|
|
2790
|
+
const all = await this.syncState.loadAllDeliveryWatermarks();
|
|
2791
|
+
for (const [signature, watermark] of Object.entries(all)) {
|
|
2792
|
+
this.deliverySignatureWatermarks.set(signature, watermark);
|
|
2793
|
+
}
|
|
2794
|
+
} else if (this.syncState.loadDeliveryWatermark) {
|
|
2795
|
+
this.deliverySignatureWatermarks.set("", await this.syncState.loadDeliveryWatermark(""));
|
|
2796
|
+
}
|
|
2797
|
+
this.deliveryWatermark = this.deliverySignatureWatermarks.get(this.deliverySignature()) ?? 0;
|
|
2798
|
+
this.evictColdViewWatermarks();
|
|
2717
2799
|
}
|
|
2718
2800
|
await this.reconcileOutboundFromOpLog();
|
|
2719
2801
|
await this.refreshPendingCount();
|
|
@@ -2743,7 +2825,11 @@ var SyncEngine = class {
|
|
|
2743
2825
|
supportedWireFormats: ["json", "protobuf"],
|
|
2744
2826
|
...this.config.scopeMap ? { syncScope: this.config.scopeMap } : {},
|
|
2745
2827
|
...activeQuerySubsets.length > 0 ? { syncQueries: activeQuerySubsets } : {},
|
|
2746
|
-
...this.resumeDeltaCursor ? { deltaCursor: encodeDeltaCursor(this.resumeDeltaCursor) } : {}
|
|
2828
|
+
...this.resumeDeltaCursor ? { deltaCursor: encodeDeltaCursor(this.resumeDeltaCursor) } : {},
|
|
2829
|
+
// Resume the server's gap-free delivery stream from our watermark. A server
|
|
2830
|
+
// that understands it drives server->client sync from delivery sequences; an
|
|
2831
|
+
// older server ignores it and falls back to the version-vector delta.
|
|
2832
|
+
lastDeliverySequence: this.deliveryWatermark
|
|
2747
2833
|
};
|
|
2748
2834
|
this.transport.send(handshake);
|
|
2749
2835
|
} catch (err) {
|
|
@@ -2780,7 +2866,7 @@ var SyncEngine = class {
|
|
|
2780
2866
|
* because they should remain local-only and not be sent to the server.
|
|
2781
2867
|
*/
|
|
2782
2868
|
async pushOperation(op) {
|
|
2783
|
-
if (!this.operationAllowedForSync(op)) {
|
|
2869
|
+
if (!await this.operationAllowedForSync(op)) {
|
|
2784
2870
|
return;
|
|
2785
2871
|
}
|
|
2786
2872
|
if (this.syncState) {
|
|
@@ -2959,8 +3045,84 @@ var SyncEngine = class {
|
|
|
2959
3045
|
* @param scopeMap - New per-collection scope filters, or undefined to remove scope
|
|
2960
3046
|
*/
|
|
2961
3047
|
updateScope(scopeMap) {
|
|
3048
|
+
const previousSignature = this.deliverySignature();
|
|
2962
3049
|
this.activeScope = scopeMap;
|
|
2963
3050
|
this.config.scopeMap = scopeMap;
|
|
3051
|
+
this.switchDeliveryView(previousSignature);
|
|
3052
|
+
}
|
|
3053
|
+
/**
|
|
3054
|
+
* A stable identifier for the current sync view: the auth/tenant scope plus the set of
|
|
3055
|
+
* active query subscriptions, canonicalized so the same view always produces the same
|
|
3056
|
+
* string. The delivery watermark is keyed by this, so each distinct view resumes from
|
|
3057
|
+
* its own last position.
|
|
3058
|
+
*/
|
|
3059
|
+
deliverySignature() {
|
|
3060
|
+
const subsets = this.getActiveQuerySubsets();
|
|
3061
|
+
if (!this.activeScope && subsets.length === 0) {
|
|
3062
|
+
return "";
|
|
3063
|
+
}
|
|
3064
|
+
const normalizedSubsets = subsets.map((s) => `${s.collection}:${stableStringify(s.where)}`).sort();
|
|
3065
|
+
return stableStringify({ scope: this.activeScope ?? null, subsets: normalizedSubsets });
|
|
3066
|
+
}
|
|
3067
|
+
/**
|
|
3068
|
+
* Switch the active delivery watermark to the current view after a scope or subscription
|
|
3069
|
+
* change. The prior view's watermark is saved so returning to it resumes; the new view's
|
|
3070
|
+
* watermark is restored from the in-memory cache (0 for a never-synced view, which then
|
|
3071
|
+
* does a one-time resync under that view). No data is lost: a view's watermark is a lower
|
|
3072
|
+
* bound for that specific view's filter.
|
|
3073
|
+
*/
|
|
3074
|
+
switchDeliveryView(previousSignature) {
|
|
3075
|
+
const nextSignature = this.deliverySignature();
|
|
3076
|
+
if (nextSignature === previousSignature) {
|
|
3077
|
+
return;
|
|
3078
|
+
}
|
|
3079
|
+
this.setViewWatermark(previousSignature, this.deliveryWatermark);
|
|
3080
|
+
void this.persistDeliveryWatermark(this.deliveryWatermark, previousSignature);
|
|
3081
|
+
this.deliveryWatermark = this.deliverySignatureWatermarks.get(nextSignature) ?? 0;
|
|
3082
|
+
}
|
|
3083
|
+
/**
|
|
3084
|
+
* Record a view's watermark in the in-memory cache as the most-recently-used entry (the
|
|
3085
|
+
* Map preserves insertion order, so re-inserting moves it to the tail), then evict cold
|
|
3086
|
+
* views past the retention cap. This is the single place the cache is written, so LRU
|
|
3087
|
+
* order and the size bound are always maintained together.
|
|
3088
|
+
*/
|
|
3089
|
+
setViewWatermark(signature, watermark) {
|
|
3090
|
+
this.deliverySignatureWatermarks.delete(signature);
|
|
3091
|
+
this.deliverySignatureWatermarks.set(signature, watermark);
|
|
3092
|
+
this.evictColdViewWatermarks();
|
|
3093
|
+
}
|
|
3094
|
+
/**
|
|
3095
|
+
* Trim the per-view watermark cache to `MAX_DELIVERY_VIEW_WATERMARKS`, evicting the
|
|
3096
|
+
* least-recently-used views. The default view ('') and the live view are never evicted
|
|
3097
|
+
* (evicting the live view would force an immediate re-scan of what we are actively
|
|
3098
|
+
* syncing). Each eviction also removes the persisted row, bounding storage. Safe by
|
|
3099
|
+
* construction: an evicted view simply back-fills from 0 (deduplicated) when next seen.
|
|
3100
|
+
*/
|
|
3101
|
+
evictColdViewWatermarks() {
|
|
3102
|
+
if (this.deliverySignatureWatermarks.size <= MAX_DELIVERY_VIEW_WATERMARKS) {
|
|
3103
|
+
return;
|
|
3104
|
+
}
|
|
3105
|
+
const liveSignature = this.deliverySignature();
|
|
3106
|
+
for (const signature of this.deliverySignatureWatermarks.keys()) {
|
|
3107
|
+
if (this.deliverySignatureWatermarks.size <= MAX_DELIVERY_VIEW_WATERMARKS) {
|
|
3108
|
+
break;
|
|
3109
|
+
}
|
|
3110
|
+
if (signature === "" || signature === liveSignature) {
|
|
3111
|
+
continue;
|
|
3112
|
+
}
|
|
3113
|
+
this.deliverySignatureWatermarks.delete(signature);
|
|
3114
|
+
void this.deleteDeliveryWatermark(signature);
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
/**
|
|
3118
|
+
* Remove a view's persisted watermark row (best-effort; a persistence layer that omits
|
|
3119
|
+
* the delete simply keeps the row, which is harmless).
|
|
3120
|
+
*/
|
|
3121
|
+
async deleteDeliveryWatermark(signature) {
|
|
3122
|
+
if (!this.syncState?.deleteDeliveryWatermark) {
|
|
3123
|
+
return;
|
|
3124
|
+
}
|
|
3125
|
+
await this.syncState.deleteDeliveryWatermark(signature);
|
|
2964
3126
|
}
|
|
2965
3127
|
/**
|
|
2966
3128
|
* Get the currently active scope map. Returns undefined if no scope is configured.
|
|
@@ -2974,10 +3136,14 @@ var SyncEngine = class {
|
|
|
2974
3136
|
*/
|
|
2975
3137
|
registerQuerySubset(subset) {
|
|
2976
3138
|
const id = `query-${nextQuerySubsetId++}`;
|
|
3139
|
+
const previousSignature = this.deliverySignature();
|
|
2977
3140
|
this.querySubsets.set(id, subset);
|
|
3141
|
+
this.switchDeliveryView(previousSignature);
|
|
2978
3142
|
this.scheduleQuerySubsetReconnect();
|
|
2979
3143
|
return () => {
|
|
3144
|
+
const sigBeforeRemove = this.deliverySignature();
|
|
2980
3145
|
this.querySubsets.delete(id);
|
|
3146
|
+
this.switchDeliveryView(sigBeforeRemove);
|
|
2981
3147
|
this.scheduleQuerySubsetReconnect();
|
|
2982
3148
|
};
|
|
2983
3149
|
}
|
|
@@ -3130,6 +3296,11 @@ var SyncEngine = class {
|
|
|
3130
3296
|
}
|
|
3131
3297
|
this.remoteVector = wireToVersionVector(msg.versionVector);
|
|
3132
3298
|
void this.persistLastAckedServerVector(this.remoteVector);
|
|
3299
|
+
if (typeof msg.serverMaxDeliverySequence === "number" && this.deliveryWatermark > msg.serverMaxDeliverySequence) {
|
|
3300
|
+
this.deliveryWatermark = 0;
|
|
3301
|
+
this.setViewWatermark(this.deliverySignature(), 0);
|
|
3302
|
+
await this.persistDeliveryWatermark(0);
|
|
3303
|
+
}
|
|
3133
3304
|
if (msg.selectedWireFormat) {
|
|
3134
3305
|
this.setSerializerWireFormat(msg.selectedWireFormat);
|
|
3135
3306
|
}
|
|
@@ -3195,7 +3366,7 @@ var SyncEngine = class {
|
|
|
3195
3366
|
async sendDelta() {
|
|
3196
3367
|
const localVector = this.store.getVersionVector();
|
|
3197
3368
|
const allMissingOps = await this.collectDelta(localVector, this.remoteVector);
|
|
3198
|
-
const missingOps =
|
|
3369
|
+
const missingOps = await this.filterAllowedForSync(allMissingOps);
|
|
3199
3370
|
this.deltaSentOpIds = missingOps.map((op) => op.id);
|
|
3200
3371
|
if (missingOps.length === 0) {
|
|
3201
3372
|
const messageId = generateMessageId();
|
|
@@ -3270,11 +3441,23 @@ var SyncEngine = class {
|
|
|
3270
3441
|
return missing;
|
|
3271
3442
|
}
|
|
3272
3443
|
async handleOperationBatch(msg) {
|
|
3444
|
+
const isDeliveryBatch = msg.maxDeliverySequence !== void 0;
|
|
3445
|
+
if (isDeliveryBatch) {
|
|
3446
|
+
const base = msg.baseDeliverySequence ?? 0;
|
|
3447
|
+
if (base > this.deliveryWatermark) {
|
|
3448
|
+
return;
|
|
3449
|
+
}
|
|
3450
|
+
if (base < this.deliveryWatermark) {
|
|
3451
|
+
this.sendDeliveryAck(msg.messageId, this.deliveryWatermark);
|
|
3452
|
+
return;
|
|
3453
|
+
}
|
|
3454
|
+
}
|
|
3273
3455
|
const deserialized = msg.operations.map((s) => this.serializer.decodeOperation(s));
|
|
3274
3456
|
const operations = this.encryptor ? await this.encryptor.decryptBatch(deserialized) : deserialized;
|
|
3275
|
-
const inScopeOps =
|
|
3457
|
+
const inScopeOps = await this.filterAllowedForSync(operations);
|
|
3276
3458
|
const targetSchemaVersion = this.config.schemaVersion ?? DEFAULT_SCHEMA_VERSION;
|
|
3277
3459
|
const transforms = this.config.operationTransforms ?? [];
|
|
3460
|
+
let fullyApplied = true;
|
|
3278
3461
|
for (const op of inScopeOps) {
|
|
3279
3462
|
const transformed = transforms.length > 0 ? applyOperationTransforms(op, targetSchemaVersion, transforms) : op;
|
|
3280
3463
|
if (transformed === null) {
|
|
@@ -3286,6 +3469,7 @@ var SyncEngine = class {
|
|
|
3286
3469
|
this.emitApplyFailure(transformed, result);
|
|
3287
3470
|
}
|
|
3288
3471
|
} catch (error) {
|
|
3472
|
+
fullyApplied = false;
|
|
3289
3473
|
if (error instanceof ClockDriftError) {
|
|
3290
3474
|
this.emitApplyFailure(transformed, "rejected", {
|
|
3291
3475
|
code: APPLY_FAILURE_CODES.CLOCK_DRIFT,
|
|
@@ -3296,11 +3480,8 @@ var SyncEngine = class {
|
|
|
3296
3480
|
}
|
|
3297
3481
|
const message = error instanceof Error ? error.message : "Apply failed";
|
|
3298
3482
|
const code = error instanceof SyncError4 ? error.code : error instanceof KoraError2 ? error.code : error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : APPLY_FAILURE_CODES.APPLY_FAILED;
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
message,
|
|
3302
|
-
retriable: code !== APPLY_FAILURE_CODES.REFERENTIAL_INTEGRITY
|
|
3303
|
-
});
|
|
3483
|
+
const retriable = code !== APPLY_FAILURE_CODES.REFERENTIAL_INTEGRITY;
|
|
3484
|
+
this.emitApplyFailure(transformed, "rejected", { code, message, retriable });
|
|
3304
3485
|
}
|
|
3305
3486
|
}
|
|
3306
3487
|
if (inScopeOps.length > 0) {
|
|
@@ -3311,18 +3492,27 @@ var SyncEngine = class {
|
|
|
3311
3492
|
batchSize: inScopeOps.length
|
|
3312
3493
|
});
|
|
3313
3494
|
}
|
|
3314
|
-
const
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3495
|
+
const deliveryFullyApplied = isDeliveryBatch && fullyApplied;
|
|
3496
|
+
if (!isDeliveryBatch || deliveryFullyApplied) {
|
|
3497
|
+
const lastOp = operations[operations.length - 1];
|
|
3498
|
+
const ack = {
|
|
3499
|
+
type: "acknowledgment",
|
|
3500
|
+
messageId: generateMessageId(),
|
|
3501
|
+
acknowledgedMessageId: msg.messageId,
|
|
3502
|
+
lastSequenceNumber: lastOp ? lastOp.sequenceNumber : 0,
|
|
3503
|
+
...deliveryFullyApplied && msg.maxDeliverySequence !== void 0 ? { deliverySequence: msg.maxDeliverySequence } : {}
|
|
3504
|
+
};
|
|
3505
|
+
this.transport.send(ack);
|
|
3506
|
+
this.emitter?.emit({
|
|
3507
|
+
type: "sync:acknowledged",
|
|
3508
|
+
sequenceNumber: lastOp ? lastOp.sequenceNumber : 0
|
|
3509
|
+
});
|
|
3510
|
+
}
|
|
3511
|
+
if (isDeliveryBatch && fullyApplied && msg.maxDeliverySequence !== void 0) {
|
|
3512
|
+
this.deliveryWatermark = msg.maxDeliverySequence;
|
|
3513
|
+
this.setViewWatermark(this.deliverySignature(), this.deliveryWatermark);
|
|
3514
|
+
await this.persistDeliveryWatermark(this.deliveryWatermark);
|
|
3515
|
+
}
|
|
3326
3516
|
if (this.state === "syncing") {
|
|
3327
3517
|
this.deltaBatchesReceived++;
|
|
3328
3518
|
const totalBatches = msg.totalBatches ?? this.initialSyncTotalBatches;
|
|
@@ -3330,15 +3520,17 @@ var SyncEngine = class {
|
|
|
3330
3520
|
this.initialSyncTotalBatches = msg.totalBatches;
|
|
3331
3521
|
}
|
|
3332
3522
|
this.metricsCollector.updateInitialSyncProgress(this.deltaBatchesReceived, totalBatches);
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3523
|
+
if (!isDeliveryBatch) {
|
|
3524
|
+
const cursorFromBatch = msg.cursor !== void 0 ? decodeDeltaCursor(msg.cursor) : createDeltaCursorFromBatch(
|
|
3525
|
+
inScopeOps.length > 0 ? inScopeOps : operations,
|
|
3526
|
+
msg.batchIndex
|
|
3527
|
+
);
|
|
3528
|
+
if (cursorFromBatch) {
|
|
3529
|
+
this.resumeDeltaCursor = cursorFromBatch;
|
|
3530
|
+
await this.persistDeltaCursor(cursorFromBatch);
|
|
3531
|
+
}
|
|
3340
3532
|
}
|
|
3341
|
-
if (msg.isFinal) {
|
|
3533
|
+
if (msg.isFinal && (!isDeliveryBatch || fullyApplied)) {
|
|
3342
3534
|
this.deltaReceiveComplete = true;
|
|
3343
3535
|
this.resumeDeltaCursor = null;
|
|
3344
3536
|
await this.persistDeltaCursor(null);
|
|
@@ -3347,6 +3539,17 @@ var SyncEngine = class {
|
|
|
3347
3539
|
}
|
|
3348
3540
|
}
|
|
3349
3541
|
}
|
|
3542
|
+
/** Acknowledge a delivery batch (used for duplicates) without re-applying it. */
|
|
3543
|
+
sendDeliveryAck(acknowledgedMessageId, deliverySequence) {
|
|
3544
|
+
const ack = {
|
|
3545
|
+
type: "acknowledgment",
|
|
3546
|
+
messageId: generateMessageId(),
|
|
3547
|
+
acknowledgedMessageId,
|
|
3548
|
+
lastSequenceNumber: 0,
|
|
3549
|
+
deliverySequence
|
|
3550
|
+
};
|
|
3551
|
+
this.transport.send(ack);
|
|
3552
|
+
}
|
|
3350
3553
|
handleAcknowledgment(msg) {
|
|
3351
3554
|
if (this.state === "syncing" && this.config.strictHandshake) {
|
|
3352
3555
|
this.pendingDeltaBatchAcks.delete(msg.acknowledgedMessageId);
|
|
@@ -3367,6 +3570,10 @@ var SyncEngine = class {
|
|
|
3367
3570
|
}
|
|
3368
3571
|
}
|
|
3369
3572
|
handleError(msg) {
|
|
3573
|
+
if (this.currentBatch) {
|
|
3574
|
+
this.outboundQueue.returnBatch(this.currentBatch.batchId);
|
|
3575
|
+
this.currentBatch = null;
|
|
3576
|
+
}
|
|
3370
3577
|
this.transitionTo("error");
|
|
3371
3578
|
if (msg.code === "AUTH_FAILED") {
|
|
3372
3579
|
this.emitter?.emit({ type: "sync:auth-failed", reason: msg.message });
|
|
@@ -3527,7 +3734,7 @@ var SyncEngine = class {
|
|
|
3527
3734
|
return;
|
|
3528
3735
|
}
|
|
3529
3736
|
const ops = await this.store.getOperationRange(localNodeId, 1, localSeq);
|
|
3530
|
-
const inScope =
|
|
3737
|
+
const inScope = await this.filterAllowedForSync(ops);
|
|
3531
3738
|
for (const op of inScope) {
|
|
3532
3739
|
await this.outboundQueue.enqueue(op);
|
|
3533
3740
|
}
|
|
@@ -3625,11 +3832,40 @@ var SyncEngine = class {
|
|
|
3625
3832
|
this.serializer.setWireFormat(format);
|
|
3626
3833
|
}
|
|
3627
3834
|
}
|
|
3628
|
-
operationAllowedForSync(op) {
|
|
3629
|
-
if (
|
|
3835
|
+
async operationAllowedForSync(op) {
|
|
3836
|
+
if (this.matchesScopeAndSubsets(op)) {
|
|
3837
|
+
return true;
|
|
3838
|
+
}
|
|
3839
|
+
const fullRecord = await this.readRecordForBackfill(op);
|
|
3840
|
+
if (!fullRecord) {
|
|
3841
|
+
return false;
|
|
3842
|
+
}
|
|
3843
|
+
return this.matchesScopeAndSubsets(op, fullRecord);
|
|
3844
|
+
}
|
|
3845
|
+
matchesScopeAndSubsets(op, fullRecord) {
|
|
3846
|
+
if (!operationMatchesScope(op, this.activeScope, fullRecord)) {
|
|
3630
3847
|
return false;
|
|
3631
3848
|
}
|
|
3632
|
-
return operationMatchesQuerySubsets(op, this.getActiveQuerySubsets());
|
|
3849
|
+
return operationMatchesQuerySubsets(op, this.getActiveQuerySubsets(), fullRecord);
|
|
3850
|
+
}
|
|
3851
|
+
async readRecordForBackfill(op) {
|
|
3852
|
+
if (!this.store.readRecordFields) {
|
|
3853
|
+
return null;
|
|
3854
|
+
}
|
|
3855
|
+
try {
|
|
3856
|
+
return await this.store.readRecordFields(op.collection, op.recordId);
|
|
3857
|
+
} catch {
|
|
3858
|
+
return null;
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
async filterAllowedForSync(ops) {
|
|
3862
|
+
const allowed = [];
|
|
3863
|
+
for (const op of ops) {
|
|
3864
|
+
if (await this.operationAllowedForSync(op)) {
|
|
3865
|
+
allowed.push(op);
|
|
3866
|
+
}
|
|
3867
|
+
}
|
|
3868
|
+
return allowed;
|
|
3633
3869
|
}
|
|
3634
3870
|
async persistDeltaCursor(cursor) {
|
|
3635
3871
|
if (!this.syncState?.saveDeltaCursor) {
|
|
@@ -3637,6 +3873,12 @@ var SyncEngine = class {
|
|
|
3637
3873
|
}
|
|
3638
3874
|
await this.syncState.saveDeltaCursor(cursor);
|
|
3639
3875
|
}
|
|
3876
|
+
async persistDeliveryWatermark(watermark, signature = this.deliverySignature()) {
|
|
3877
|
+
if (!this.syncState?.saveDeliveryWatermark) {
|
|
3878
|
+
return;
|
|
3879
|
+
}
|
|
3880
|
+
await this.syncState.saveDeliveryWatermark(signature, watermark);
|
|
3881
|
+
}
|
|
3640
3882
|
scheduleQuerySubsetReconnect() {
|
|
3641
3883
|
if (this.querySubsetReconnectTimer) {
|
|
3642
3884
|
clearTimeout(this.querySubsetReconnectTimer);
|