@korajs/sync 1.0.0-beta.4 → 1.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +283 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +82 -2
- package/dist/index.d.ts +82 -2
- package/dist/index.js +283 -41
- package/dist/index.js.map +1 -1
- package/dist/internal.d.cts +1 -1
- package/dist/internal.d.ts +1 -1
- package/dist/{transport-BGr-_eH8.d.cts → transport-DXmZ8kyI.d.cts} +52 -0
- package/dist/{transport-BGr-_eH8.d.ts → transport-DXmZ8kyI.d.ts} +52 -0
- package/package.json +3 -3
package/dist/index.cjs
CHANGED
|
@@ -620,7 +620,8 @@ function toProtoEnvelope(message) {
|
|
|
620
620
|
})),
|
|
621
621
|
schemaVersion: message.schemaVersion,
|
|
622
622
|
authToken: message.authToken,
|
|
623
|
-
supportedWireFormats: message.supportedWireFormats
|
|
623
|
+
supportedWireFormats: message.supportedWireFormats,
|
|
624
|
+
...message.lastDeliverySequence !== void 0 ? { lastDeliverySequence: message.lastDeliverySequence } : {}
|
|
624
625
|
};
|
|
625
626
|
case "handshake-response":
|
|
626
627
|
return {
|
|
@@ -636,7 +637,8 @@ function toProtoEnvelope(message) {
|
|
|
636
637
|
rejectReason: message.rejectReason,
|
|
637
638
|
selectedWireFormat: message.selectedWireFormat,
|
|
638
639
|
serverTime: message.serverTime,
|
|
639
|
-
blobStorageEnabled: message.blobStorageEnabled
|
|
640
|
+
blobStorageEnabled: message.blobStorageEnabled,
|
|
641
|
+
...message.serverMaxDeliverySequence !== void 0 ? { serverMaxDeliverySequence: message.serverMaxDeliverySequence } : {}
|
|
640
642
|
};
|
|
641
643
|
case "operation-batch":
|
|
642
644
|
return {
|
|
@@ -644,14 +646,17 @@ function toProtoEnvelope(message) {
|
|
|
644
646
|
messageId: message.messageId,
|
|
645
647
|
operations: message.operations.map(serializeProtoOperation),
|
|
646
648
|
isFinal: message.isFinal,
|
|
647
|
-
batchIndex: message.batchIndex
|
|
649
|
+
batchIndex: message.batchIndex,
|
|
650
|
+
...message.baseDeliverySequence !== void 0 ? { baseDeliverySequence: message.baseDeliverySequence } : {},
|
|
651
|
+
...message.maxDeliverySequence !== void 0 ? { maxDeliverySequence: message.maxDeliverySequence } : {}
|
|
648
652
|
};
|
|
649
653
|
case "acknowledgment":
|
|
650
654
|
return {
|
|
651
655
|
type: message.type,
|
|
652
656
|
messageId: message.messageId,
|
|
653
657
|
acknowledgedMessageId: message.acknowledgedMessageId,
|
|
654
|
-
lastSequenceNumber: message.lastSequenceNumber
|
|
658
|
+
lastSequenceNumber: message.lastSequenceNumber,
|
|
659
|
+
...message.deliverySequence !== void 0 ? { deliverySequence: message.deliverySequence } : {}
|
|
655
660
|
};
|
|
656
661
|
case "error":
|
|
657
662
|
return {
|
|
@@ -716,7 +721,8 @@ function fromProtoEnvelope(envelope) {
|
|
|
716
721
|
authToken: envelope.authToken,
|
|
717
722
|
supportedWireFormats: envelope.supportedWireFormats?.filter(
|
|
718
723
|
(format) => format === "json" || format === "protobuf"
|
|
719
|
-
)
|
|
724
|
+
),
|
|
725
|
+
...envelope.lastDeliverySequence !== void 0 ? { lastDeliverySequence: envelope.lastDeliverySequence } : {}
|
|
720
726
|
};
|
|
721
727
|
case "handshake-response":
|
|
722
728
|
return {
|
|
@@ -733,7 +739,8 @@ function fromProtoEnvelope(envelope) {
|
|
|
733
739
|
// Preserve the server's wall-clock time so clock-skew detection and
|
|
734
740
|
// automatic timestamp rebase work over the protobuf wire, not just JSON.
|
|
735
741
|
...envelope.serverTime !== void 0 ? { serverTime: envelope.serverTime } : {},
|
|
736
|
-
...envelope.blobStorageEnabled !== void 0 ? { blobStorageEnabled: envelope.blobStorageEnabled } : {}
|
|
742
|
+
...envelope.blobStorageEnabled !== void 0 ? { blobStorageEnabled: envelope.blobStorageEnabled } : {},
|
|
743
|
+
...envelope.serverMaxDeliverySequence !== void 0 ? { serverMaxDeliverySequence: envelope.serverMaxDeliverySequence } : {}
|
|
737
744
|
};
|
|
738
745
|
case "operation-batch":
|
|
739
746
|
return {
|
|
@@ -741,14 +748,17 @@ function fromProtoEnvelope(envelope) {
|
|
|
741
748
|
messageId: envelope.messageId,
|
|
742
749
|
operations: (envelope.operations ?? []).map(deserializeProtoOperation),
|
|
743
750
|
isFinal: envelope.isFinal ?? false,
|
|
744
|
-
batchIndex: envelope.batchIndex ?? 0
|
|
751
|
+
batchIndex: envelope.batchIndex ?? 0,
|
|
752
|
+
...envelope.baseDeliverySequence !== void 0 ? { baseDeliverySequence: envelope.baseDeliverySequence } : {},
|
|
753
|
+
...envelope.maxDeliverySequence !== void 0 ? { maxDeliverySequence: envelope.maxDeliverySequence } : {}
|
|
745
754
|
};
|
|
746
755
|
case "acknowledgment":
|
|
747
756
|
return {
|
|
748
757
|
type: "acknowledgment",
|
|
749
758
|
messageId: envelope.messageId,
|
|
750
759
|
acknowledgedMessageId: envelope.acknowledgedMessageId ?? "",
|
|
751
|
-
lastSequenceNumber: envelope.lastSequenceNumber ?? 0
|
|
760
|
+
lastSequenceNumber: envelope.lastSequenceNumber ?? 0,
|
|
761
|
+
...envelope.deliverySequence !== void 0 ? { deliverySequence: envelope.deliverySequence } : {}
|
|
752
762
|
};
|
|
753
763
|
case "error":
|
|
754
764
|
return {
|
|
@@ -949,6 +959,15 @@ function encodeEnvelope(envelope) {
|
|
|
949
959
|
writer.uint32(218).string(envelope.collection);
|
|
950
960
|
if (envelope.recordId && envelope.recordId.length > 0)
|
|
951
961
|
writer.uint32(226).string(envelope.recordId);
|
|
962
|
+
if (envelope.lastDeliverySequence !== void 0)
|
|
963
|
+
writer.uint32(232).int64(envelope.lastDeliverySequence);
|
|
964
|
+
if (envelope.baseDeliverySequence !== void 0)
|
|
965
|
+
writer.uint32(240).int64(envelope.baseDeliverySequence);
|
|
966
|
+
if (envelope.maxDeliverySequence !== void 0)
|
|
967
|
+
writer.uint32(248).int64(envelope.maxDeliverySequence);
|
|
968
|
+
if (envelope.deliverySequence !== void 0) writer.uint32(256).int64(envelope.deliverySequence);
|
|
969
|
+
if (envelope.serverMaxDeliverySequence !== void 0)
|
|
970
|
+
writer.uint32(264).int64(envelope.serverMaxDeliverySequence);
|
|
952
971
|
return writer.finish();
|
|
953
972
|
}
|
|
954
973
|
function decodeEnvelope(bytes) {
|
|
@@ -1044,6 +1063,21 @@ function decodeEnvelope(bytes) {
|
|
|
1044
1063
|
case 28:
|
|
1045
1064
|
envelope.recordId = reader.string();
|
|
1046
1065
|
break;
|
|
1066
|
+
case 29:
|
|
1067
|
+
envelope.lastDeliverySequence = longToNumber(reader.int64());
|
|
1068
|
+
break;
|
|
1069
|
+
case 30:
|
|
1070
|
+
envelope.baseDeliverySequence = longToNumber(reader.int64());
|
|
1071
|
+
break;
|
|
1072
|
+
case 31:
|
|
1073
|
+
envelope.maxDeliverySequence = longToNumber(reader.int64());
|
|
1074
|
+
break;
|
|
1075
|
+
case 32:
|
|
1076
|
+
envelope.deliverySequence = longToNumber(reader.int64());
|
|
1077
|
+
break;
|
|
1078
|
+
case 33:
|
|
1079
|
+
envelope.serverMaxDeliverySequence = longToNumber(reader.int64());
|
|
1080
|
+
break;
|
|
1047
1081
|
default:
|
|
1048
1082
|
reader.skipType(tag & 7);
|
|
1049
1083
|
}
|
|
@@ -1543,6 +1577,7 @@ var ChaosTransport = class {
|
|
|
1543
1577
|
reorderRate;
|
|
1544
1578
|
maxLatency;
|
|
1545
1579
|
random;
|
|
1580
|
+
dropPredicate;
|
|
1546
1581
|
messageHandler = null;
|
|
1547
1582
|
reorderBuffer = [];
|
|
1548
1583
|
pending = [];
|
|
@@ -1553,6 +1588,7 @@ var ChaosTransport = class {
|
|
|
1553
1588
|
this.reorderRate = config?.reorderRate ?? 0;
|
|
1554
1589
|
this.maxLatency = config?.maxLatency ?? 0;
|
|
1555
1590
|
this.random = config?.randomSource ?? Math.random;
|
|
1591
|
+
this.dropPredicate = config?.dropPredicate ?? null;
|
|
1556
1592
|
}
|
|
1557
1593
|
async connect(url, options) {
|
|
1558
1594
|
this.inner.onMessage((msg) => this.handleIncoming(msg));
|
|
@@ -1571,6 +1607,9 @@ var ChaosTransport = class {
|
|
|
1571
1607
|
return this.inner.disconnect();
|
|
1572
1608
|
}
|
|
1573
1609
|
send(message) {
|
|
1610
|
+
if (this.dropPredicate?.(message, "outgoing")) {
|
|
1611
|
+
return;
|
|
1612
|
+
}
|
|
1574
1613
|
if (this.random() < this.dropRate) {
|
|
1575
1614
|
return;
|
|
1576
1615
|
}
|
|
@@ -1598,6 +1637,9 @@ var ChaosTransport = class {
|
|
|
1598
1637
|
}
|
|
1599
1638
|
handleIncoming(message) {
|
|
1600
1639
|
if (!this.messageHandler) return;
|
|
1640
|
+
if (this.dropPredicate?.(message, "incoming")) {
|
|
1641
|
+
return;
|
|
1642
|
+
}
|
|
1601
1643
|
if (this.random() < this.dropRate) {
|
|
1602
1644
|
return;
|
|
1603
1645
|
}
|
|
@@ -2706,6 +2748,17 @@ var nextQuerySubsetId = 0;
|
|
|
2706
2748
|
function generateMessageId() {
|
|
2707
2749
|
return `msg-${Date.now()}-${nextMessageId++}`;
|
|
2708
2750
|
}
|
|
2751
|
+
var MAX_DELIVERY_VIEW_WATERMARKS = 64;
|
|
2752
|
+
function stableStringify(value) {
|
|
2753
|
+
if (value === null || typeof value !== "object") {
|
|
2754
|
+
return JSON.stringify(value) ?? "null";
|
|
2755
|
+
}
|
|
2756
|
+
if (Array.isArray(value)) {
|
|
2757
|
+
return `[${value.map(stableStringify).join(",")}]`;
|
|
2758
|
+
}
|
|
2759
|
+
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);
|
|
2760
|
+
return `{${entries.join(",")}}`;
|
|
2761
|
+
}
|
|
2709
2762
|
var SyncEngine = class {
|
|
2710
2763
|
state = "disconnected";
|
|
2711
2764
|
transport;
|
|
@@ -2755,6 +2808,25 @@ var SyncEngine = class {
|
|
|
2755
2808
|
/** Resume cursor for paginated initial sync (persisted across reconnects) */
|
|
2756
2809
|
resumeDeltaCursor = null;
|
|
2757
2810
|
initialSyncTotalBatches = 0;
|
|
2811
|
+
/**
|
|
2812
|
+
* The delivery watermark for the CURRENT sync view: the highest server delivery
|
|
2813
|
+
* sequence up to which every operation in the current view (scope + active query
|
|
2814
|
+
* subscriptions) has been contiguously applied. Advanced only through the server's
|
|
2815
|
+
* gap-free delivery stream (never through live relay), so it is a durable lower bound
|
|
2816
|
+
* on what this client holds for this view. Reported at handshake to resume the stream.
|
|
2817
|
+
*
|
|
2818
|
+
* The watermark is keyed by view: switching views (a scope or subscription change)
|
|
2819
|
+
* saves the current view's watermark and loads the target view's, so returning to a
|
|
2820
|
+
* previously-synced view resumes exactly where it left off instead of re-syncing. This
|
|
2821
|
+
* is correct because a view's watermark is a lower bound specific to that view's
|
|
2822
|
+
* filter; a widened view is simply a different (initially unsynced) view.
|
|
2823
|
+
*/
|
|
2824
|
+
deliveryWatermark = 0;
|
|
2825
|
+
/**
|
|
2826
|
+
* In-memory cache of watermark per view signature, preloaded from persistence at start
|
|
2827
|
+
* so a view switch resolves synchronously (no race with the reconnect it triggers).
|
|
2828
|
+
*/
|
|
2829
|
+
deliverySignatureWatermarks = /* @__PURE__ */ new Map();
|
|
2758
2830
|
constructor(options) {
|
|
2759
2831
|
this.transport = options.transport;
|
|
2760
2832
|
this.store = options.store;
|
|
@@ -2818,6 +2890,16 @@ var SyncEngine = class {
|
|
|
2818
2890
|
if (this.syncState.loadDeltaCursor) {
|
|
2819
2891
|
this.resumeDeltaCursor = await this.syncState.loadDeltaCursor();
|
|
2820
2892
|
}
|
|
2893
|
+
if (this.syncState.loadAllDeliveryWatermarks) {
|
|
2894
|
+
const all = await this.syncState.loadAllDeliveryWatermarks();
|
|
2895
|
+
for (const [signature, watermark] of Object.entries(all)) {
|
|
2896
|
+
this.deliverySignatureWatermarks.set(signature, watermark);
|
|
2897
|
+
}
|
|
2898
|
+
} else if (this.syncState.loadDeliveryWatermark) {
|
|
2899
|
+
this.deliverySignatureWatermarks.set("", await this.syncState.loadDeliveryWatermark(""));
|
|
2900
|
+
}
|
|
2901
|
+
this.deliveryWatermark = this.deliverySignatureWatermarks.get(this.deliverySignature()) ?? 0;
|
|
2902
|
+
this.evictColdViewWatermarks();
|
|
2821
2903
|
}
|
|
2822
2904
|
await this.reconcileOutboundFromOpLog();
|
|
2823
2905
|
await this.refreshPendingCount();
|
|
@@ -2847,7 +2929,11 @@ var SyncEngine = class {
|
|
|
2847
2929
|
supportedWireFormats: ["json", "protobuf"],
|
|
2848
2930
|
...this.config.scopeMap ? { syncScope: this.config.scopeMap } : {},
|
|
2849
2931
|
...activeQuerySubsets.length > 0 ? { syncQueries: activeQuerySubsets } : {},
|
|
2850
|
-
...this.resumeDeltaCursor ? { deltaCursor: encodeDeltaCursor(this.resumeDeltaCursor) } : {}
|
|
2932
|
+
...this.resumeDeltaCursor ? { deltaCursor: encodeDeltaCursor(this.resumeDeltaCursor) } : {},
|
|
2933
|
+
// Resume the server's gap-free delivery stream from our watermark. A server
|
|
2934
|
+
// that understands it drives server->client sync from delivery sequences; an
|
|
2935
|
+
// older server ignores it and falls back to the version-vector delta.
|
|
2936
|
+
lastDeliverySequence: this.deliveryWatermark
|
|
2851
2937
|
};
|
|
2852
2938
|
this.transport.send(handshake);
|
|
2853
2939
|
} catch (err) {
|
|
@@ -2884,7 +2970,7 @@ var SyncEngine = class {
|
|
|
2884
2970
|
* because they should remain local-only and not be sent to the server.
|
|
2885
2971
|
*/
|
|
2886
2972
|
async pushOperation(op) {
|
|
2887
|
-
if (!this.operationAllowedForSync(op)) {
|
|
2973
|
+
if (!await this.operationAllowedForSync(op)) {
|
|
2888
2974
|
return;
|
|
2889
2975
|
}
|
|
2890
2976
|
if (this.syncState) {
|
|
@@ -3063,8 +3149,84 @@ var SyncEngine = class {
|
|
|
3063
3149
|
* @param scopeMap - New per-collection scope filters, or undefined to remove scope
|
|
3064
3150
|
*/
|
|
3065
3151
|
updateScope(scopeMap) {
|
|
3152
|
+
const previousSignature = this.deliverySignature();
|
|
3066
3153
|
this.activeScope = scopeMap;
|
|
3067
3154
|
this.config.scopeMap = scopeMap;
|
|
3155
|
+
this.switchDeliveryView(previousSignature);
|
|
3156
|
+
}
|
|
3157
|
+
/**
|
|
3158
|
+
* A stable identifier for the current sync view: the auth/tenant scope plus the set of
|
|
3159
|
+
* active query subscriptions, canonicalized so the same view always produces the same
|
|
3160
|
+
* string. The delivery watermark is keyed by this, so each distinct view resumes from
|
|
3161
|
+
* its own last position.
|
|
3162
|
+
*/
|
|
3163
|
+
deliverySignature() {
|
|
3164
|
+
const subsets = this.getActiveQuerySubsets();
|
|
3165
|
+
if (!this.activeScope && subsets.length === 0) {
|
|
3166
|
+
return "";
|
|
3167
|
+
}
|
|
3168
|
+
const normalizedSubsets = subsets.map((s) => `${s.collection}:${stableStringify(s.where)}`).sort();
|
|
3169
|
+
return stableStringify({ scope: this.activeScope ?? null, subsets: normalizedSubsets });
|
|
3170
|
+
}
|
|
3171
|
+
/**
|
|
3172
|
+
* Switch the active delivery watermark to the current view after a scope or subscription
|
|
3173
|
+
* change. The prior view's watermark is saved so returning to it resumes; the new view's
|
|
3174
|
+
* watermark is restored from the in-memory cache (0 for a never-synced view, which then
|
|
3175
|
+
* does a one-time resync under that view). No data is lost: a view's watermark is a lower
|
|
3176
|
+
* bound for that specific view's filter.
|
|
3177
|
+
*/
|
|
3178
|
+
switchDeliveryView(previousSignature) {
|
|
3179
|
+
const nextSignature = this.deliverySignature();
|
|
3180
|
+
if (nextSignature === previousSignature) {
|
|
3181
|
+
return;
|
|
3182
|
+
}
|
|
3183
|
+
this.setViewWatermark(previousSignature, this.deliveryWatermark);
|
|
3184
|
+
void this.persistDeliveryWatermark(this.deliveryWatermark, previousSignature);
|
|
3185
|
+
this.deliveryWatermark = this.deliverySignatureWatermarks.get(nextSignature) ?? 0;
|
|
3186
|
+
}
|
|
3187
|
+
/**
|
|
3188
|
+
* Record a view's watermark in the in-memory cache as the most-recently-used entry (the
|
|
3189
|
+
* Map preserves insertion order, so re-inserting moves it to the tail), then evict cold
|
|
3190
|
+
* views past the retention cap. This is the single place the cache is written, so LRU
|
|
3191
|
+
* order and the size bound are always maintained together.
|
|
3192
|
+
*/
|
|
3193
|
+
setViewWatermark(signature, watermark) {
|
|
3194
|
+
this.deliverySignatureWatermarks.delete(signature);
|
|
3195
|
+
this.deliverySignatureWatermarks.set(signature, watermark);
|
|
3196
|
+
this.evictColdViewWatermarks();
|
|
3197
|
+
}
|
|
3198
|
+
/**
|
|
3199
|
+
* Trim the per-view watermark cache to `MAX_DELIVERY_VIEW_WATERMARKS`, evicting the
|
|
3200
|
+
* least-recently-used views. The default view ('') and the live view are never evicted
|
|
3201
|
+
* (evicting the live view would force an immediate re-scan of what we are actively
|
|
3202
|
+
* syncing). Each eviction also removes the persisted row, bounding storage. Safe by
|
|
3203
|
+
* construction: an evicted view simply back-fills from 0 (deduplicated) when next seen.
|
|
3204
|
+
*/
|
|
3205
|
+
evictColdViewWatermarks() {
|
|
3206
|
+
if (this.deliverySignatureWatermarks.size <= MAX_DELIVERY_VIEW_WATERMARKS) {
|
|
3207
|
+
return;
|
|
3208
|
+
}
|
|
3209
|
+
const liveSignature = this.deliverySignature();
|
|
3210
|
+
for (const signature of this.deliverySignatureWatermarks.keys()) {
|
|
3211
|
+
if (this.deliverySignatureWatermarks.size <= MAX_DELIVERY_VIEW_WATERMARKS) {
|
|
3212
|
+
break;
|
|
3213
|
+
}
|
|
3214
|
+
if (signature === "" || signature === liveSignature) {
|
|
3215
|
+
continue;
|
|
3216
|
+
}
|
|
3217
|
+
this.deliverySignatureWatermarks.delete(signature);
|
|
3218
|
+
void this.deleteDeliveryWatermark(signature);
|
|
3219
|
+
}
|
|
3220
|
+
}
|
|
3221
|
+
/**
|
|
3222
|
+
* Remove a view's persisted watermark row (best-effort; a persistence layer that omits
|
|
3223
|
+
* the delete simply keeps the row, which is harmless).
|
|
3224
|
+
*/
|
|
3225
|
+
async deleteDeliveryWatermark(signature) {
|
|
3226
|
+
if (!this.syncState?.deleteDeliveryWatermark) {
|
|
3227
|
+
return;
|
|
3228
|
+
}
|
|
3229
|
+
await this.syncState.deleteDeliveryWatermark(signature);
|
|
3068
3230
|
}
|
|
3069
3231
|
/**
|
|
3070
3232
|
* Get the currently active scope map. Returns undefined if no scope is configured.
|
|
@@ -3078,10 +3240,14 @@ var SyncEngine = class {
|
|
|
3078
3240
|
*/
|
|
3079
3241
|
registerQuerySubset(subset) {
|
|
3080
3242
|
const id = `query-${nextQuerySubsetId++}`;
|
|
3243
|
+
const previousSignature = this.deliverySignature();
|
|
3081
3244
|
this.querySubsets.set(id, subset);
|
|
3245
|
+
this.switchDeliveryView(previousSignature);
|
|
3082
3246
|
this.scheduleQuerySubsetReconnect();
|
|
3083
3247
|
return () => {
|
|
3248
|
+
const sigBeforeRemove = this.deliverySignature();
|
|
3084
3249
|
this.querySubsets.delete(id);
|
|
3250
|
+
this.switchDeliveryView(sigBeforeRemove);
|
|
3085
3251
|
this.scheduleQuerySubsetReconnect();
|
|
3086
3252
|
};
|
|
3087
3253
|
}
|
|
@@ -3234,6 +3400,11 @@ var SyncEngine = class {
|
|
|
3234
3400
|
}
|
|
3235
3401
|
this.remoteVector = wireToVersionVector(msg.versionVector);
|
|
3236
3402
|
void this.persistLastAckedServerVector(this.remoteVector);
|
|
3403
|
+
if (typeof msg.serverMaxDeliverySequence === "number" && this.deliveryWatermark > msg.serverMaxDeliverySequence) {
|
|
3404
|
+
this.deliveryWatermark = 0;
|
|
3405
|
+
this.setViewWatermark(this.deliverySignature(), 0);
|
|
3406
|
+
await this.persistDeliveryWatermark(0);
|
|
3407
|
+
}
|
|
3237
3408
|
if (msg.selectedWireFormat) {
|
|
3238
3409
|
this.setSerializerWireFormat(msg.selectedWireFormat);
|
|
3239
3410
|
}
|
|
@@ -3299,7 +3470,7 @@ var SyncEngine = class {
|
|
|
3299
3470
|
async sendDelta() {
|
|
3300
3471
|
const localVector = this.store.getVersionVector();
|
|
3301
3472
|
const allMissingOps = await this.collectDelta(localVector, this.remoteVector);
|
|
3302
|
-
const missingOps =
|
|
3473
|
+
const missingOps = await this.filterAllowedForSync(allMissingOps);
|
|
3303
3474
|
this.deltaSentOpIds = missingOps.map((op) => op.id);
|
|
3304
3475
|
if (missingOps.length === 0) {
|
|
3305
3476
|
const messageId = generateMessageId();
|
|
@@ -3374,11 +3545,23 @@ var SyncEngine = class {
|
|
|
3374
3545
|
return missing;
|
|
3375
3546
|
}
|
|
3376
3547
|
async handleOperationBatch(msg) {
|
|
3548
|
+
const isDeliveryBatch = msg.maxDeliverySequence !== void 0;
|
|
3549
|
+
if (isDeliveryBatch) {
|
|
3550
|
+
const base = msg.baseDeliverySequence ?? 0;
|
|
3551
|
+
if (base > this.deliveryWatermark) {
|
|
3552
|
+
return;
|
|
3553
|
+
}
|
|
3554
|
+
if (base < this.deliveryWatermark) {
|
|
3555
|
+
this.sendDeliveryAck(msg.messageId, this.deliveryWatermark);
|
|
3556
|
+
return;
|
|
3557
|
+
}
|
|
3558
|
+
}
|
|
3377
3559
|
const deserialized = msg.operations.map((s) => this.serializer.decodeOperation(s));
|
|
3378
3560
|
const operations = this.encryptor ? await this.encryptor.decryptBatch(deserialized) : deserialized;
|
|
3379
|
-
const inScopeOps =
|
|
3561
|
+
const inScopeOps = await this.filterAllowedForSync(operations);
|
|
3380
3562
|
const targetSchemaVersion = this.config.schemaVersion ?? DEFAULT_SCHEMA_VERSION;
|
|
3381
3563
|
const transforms = this.config.operationTransforms ?? [];
|
|
3564
|
+
let fullyApplied = true;
|
|
3382
3565
|
for (const op of inScopeOps) {
|
|
3383
3566
|
const transformed = transforms.length > 0 ? (0, import_core7.applyOperationTransforms)(op, targetSchemaVersion, transforms) : op;
|
|
3384
3567
|
if (transformed === null) {
|
|
@@ -3390,6 +3573,7 @@ var SyncEngine = class {
|
|
|
3390
3573
|
this.emitApplyFailure(transformed, result);
|
|
3391
3574
|
}
|
|
3392
3575
|
} catch (error) {
|
|
3576
|
+
fullyApplied = false;
|
|
3393
3577
|
if (error instanceof import_core7.ClockDriftError) {
|
|
3394
3578
|
this.emitApplyFailure(transformed, "rejected", {
|
|
3395
3579
|
code: import_core7.APPLY_FAILURE_CODES.CLOCK_DRIFT,
|
|
@@ -3400,11 +3584,8 @@ var SyncEngine = class {
|
|
|
3400
3584
|
}
|
|
3401
3585
|
const message = error instanceof Error ? error.message : "Apply failed";
|
|
3402
3586
|
const code = error instanceof import_core7.SyncError ? error.code : error instanceof import_core7.KoraError ? error.code : error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : import_core7.APPLY_FAILURE_CODES.APPLY_FAILED;
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
message,
|
|
3406
|
-
retriable: code !== import_core7.APPLY_FAILURE_CODES.REFERENTIAL_INTEGRITY
|
|
3407
|
-
});
|
|
3587
|
+
const retriable = code !== import_core7.APPLY_FAILURE_CODES.REFERENTIAL_INTEGRITY;
|
|
3588
|
+
this.emitApplyFailure(transformed, "rejected", { code, message, retriable });
|
|
3408
3589
|
}
|
|
3409
3590
|
}
|
|
3410
3591
|
if (inScopeOps.length > 0) {
|
|
@@ -3415,18 +3596,27 @@ var SyncEngine = class {
|
|
|
3415
3596
|
batchSize: inScopeOps.length
|
|
3416
3597
|
});
|
|
3417
3598
|
}
|
|
3418
|
-
const
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3599
|
+
const deliveryFullyApplied = isDeliveryBatch && fullyApplied;
|
|
3600
|
+
if (!isDeliveryBatch || deliveryFullyApplied) {
|
|
3601
|
+
const lastOp = operations[operations.length - 1];
|
|
3602
|
+
const ack = {
|
|
3603
|
+
type: "acknowledgment",
|
|
3604
|
+
messageId: generateMessageId(),
|
|
3605
|
+
acknowledgedMessageId: msg.messageId,
|
|
3606
|
+
lastSequenceNumber: lastOp ? lastOp.sequenceNumber : 0,
|
|
3607
|
+
...deliveryFullyApplied && msg.maxDeliverySequence !== void 0 ? { deliverySequence: msg.maxDeliverySequence } : {}
|
|
3608
|
+
};
|
|
3609
|
+
this.transport.send(ack);
|
|
3610
|
+
this.emitter?.emit({
|
|
3611
|
+
type: "sync:acknowledged",
|
|
3612
|
+
sequenceNumber: lastOp ? lastOp.sequenceNumber : 0
|
|
3613
|
+
});
|
|
3614
|
+
}
|
|
3615
|
+
if (isDeliveryBatch && fullyApplied && msg.maxDeliverySequence !== void 0) {
|
|
3616
|
+
this.deliveryWatermark = msg.maxDeliverySequence;
|
|
3617
|
+
this.setViewWatermark(this.deliverySignature(), this.deliveryWatermark);
|
|
3618
|
+
await this.persistDeliveryWatermark(this.deliveryWatermark);
|
|
3619
|
+
}
|
|
3430
3620
|
if (this.state === "syncing") {
|
|
3431
3621
|
this.deltaBatchesReceived++;
|
|
3432
3622
|
const totalBatches = msg.totalBatches ?? this.initialSyncTotalBatches;
|
|
@@ -3434,15 +3624,17 @@ var SyncEngine = class {
|
|
|
3434
3624
|
this.initialSyncTotalBatches = msg.totalBatches;
|
|
3435
3625
|
}
|
|
3436
3626
|
this.metricsCollector.updateInitialSyncProgress(this.deltaBatchesReceived, totalBatches);
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3627
|
+
if (!isDeliveryBatch) {
|
|
3628
|
+
const cursorFromBatch = msg.cursor !== void 0 ? decodeDeltaCursor(msg.cursor) : createDeltaCursorFromBatch(
|
|
3629
|
+
inScopeOps.length > 0 ? inScopeOps : operations,
|
|
3630
|
+
msg.batchIndex
|
|
3631
|
+
);
|
|
3632
|
+
if (cursorFromBatch) {
|
|
3633
|
+
this.resumeDeltaCursor = cursorFromBatch;
|
|
3634
|
+
await this.persistDeltaCursor(cursorFromBatch);
|
|
3635
|
+
}
|
|
3444
3636
|
}
|
|
3445
|
-
if (msg.isFinal) {
|
|
3637
|
+
if (msg.isFinal && (!isDeliveryBatch || fullyApplied)) {
|
|
3446
3638
|
this.deltaReceiveComplete = true;
|
|
3447
3639
|
this.resumeDeltaCursor = null;
|
|
3448
3640
|
await this.persistDeltaCursor(null);
|
|
@@ -3451,6 +3643,17 @@ var SyncEngine = class {
|
|
|
3451
3643
|
}
|
|
3452
3644
|
}
|
|
3453
3645
|
}
|
|
3646
|
+
/** Acknowledge a delivery batch (used for duplicates) without re-applying it. */
|
|
3647
|
+
sendDeliveryAck(acknowledgedMessageId, deliverySequence) {
|
|
3648
|
+
const ack = {
|
|
3649
|
+
type: "acknowledgment",
|
|
3650
|
+
messageId: generateMessageId(),
|
|
3651
|
+
acknowledgedMessageId,
|
|
3652
|
+
lastSequenceNumber: 0,
|
|
3653
|
+
deliverySequence
|
|
3654
|
+
};
|
|
3655
|
+
this.transport.send(ack);
|
|
3656
|
+
}
|
|
3454
3657
|
handleAcknowledgment(msg) {
|
|
3455
3658
|
if (this.state === "syncing" && this.config.strictHandshake) {
|
|
3456
3659
|
this.pendingDeltaBatchAcks.delete(msg.acknowledgedMessageId);
|
|
@@ -3471,6 +3674,10 @@ var SyncEngine = class {
|
|
|
3471
3674
|
}
|
|
3472
3675
|
}
|
|
3473
3676
|
handleError(msg) {
|
|
3677
|
+
if (this.currentBatch) {
|
|
3678
|
+
this.outboundQueue.returnBatch(this.currentBatch.batchId);
|
|
3679
|
+
this.currentBatch = null;
|
|
3680
|
+
}
|
|
3474
3681
|
this.transitionTo("error");
|
|
3475
3682
|
if (msg.code === "AUTH_FAILED") {
|
|
3476
3683
|
this.emitter?.emit({ type: "sync:auth-failed", reason: msg.message });
|
|
@@ -3631,7 +3838,7 @@ var SyncEngine = class {
|
|
|
3631
3838
|
return;
|
|
3632
3839
|
}
|
|
3633
3840
|
const ops = await this.store.getOperationRange(localNodeId, 1, localSeq);
|
|
3634
|
-
const inScope =
|
|
3841
|
+
const inScope = await this.filterAllowedForSync(ops);
|
|
3635
3842
|
for (const op of inScope) {
|
|
3636
3843
|
await this.outboundQueue.enqueue(op);
|
|
3637
3844
|
}
|
|
@@ -3729,11 +3936,40 @@ var SyncEngine = class {
|
|
|
3729
3936
|
this.serializer.setWireFormat(format);
|
|
3730
3937
|
}
|
|
3731
3938
|
}
|
|
3732
|
-
operationAllowedForSync(op) {
|
|
3733
|
-
if (
|
|
3939
|
+
async operationAllowedForSync(op) {
|
|
3940
|
+
if (this.matchesScopeAndSubsets(op)) {
|
|
3941
|
+
return true;
|
|
3942
|
+
}
|
|
3943
|
+
const fullRecord = await this.readRecordForBackfill(op);
|
|
3944
|
+
if (!fullRecord) {
|
|
3945
|
+
return false;
|
|
3946
|
+
}
|
|
3947
|
+
return this.matchesScopeAndSubsets(op, fullRecord);
|
|
3948
|
+
}
|
|
3949
|
+
matchesScopeAndSubsets(op, fullRecord) {
|
|
3950
|
+
if (!operationMatchesScope(op, this.activeScope, fullRecord)) {
|
|
3734
3951
|
return false;
|
|
3735
3952
|
}
|
|
3736
|
-
return operationMatchesQuerySubsets(op, this.getActiveQuerySubsets());
|
|
3953
|
+
return operationMatchesQuerySubsets(op, this.getActiveQuerySubsets(), fullRecord);
|
|
3954
|
+
}
|
|
3955
|
+
async readRecordForBackfill(op) {
|
|
3956
|
+
if (!this.store.readRecordFields) {
|
|
3957
|
+
return null;
|
|
3958
|
+
}
|
|
3959
|
+
try {
|
|
3960
|
+
return await this.store.readRecordFields(op.collection, op.recordId);
|
|
3961
|
+
} catch {
|
|
3962
|
+
return null;
|
|
3963
|
+
}
|
|
3964
|
+
}
|
|
3965
|
+
async filterAllowedForSync(ops) {
|
|
3966
|
+
const allowed = [];
|
|
3967
|
+
for (const op of ops) {
|
|
3968
|
+
if (await this.operationAllowedForSync(op)) {
|
|
3969
|
+
allowed.push(op);
|
|
3970
|
+
}
|
|
3971
|
+
}
|
|
3972
|
+
return allowed;
|
|
3737
3973
|
}
|
|
3738
3974
|
async persistDeltaCursor(cursor) {
|
|
3739
3975
|
if (!this.syncState?.saveDeltaCursor) {
|
|
@@ -3741,6 +3977,12 @@ var SyncEngine = class {
|
|
|
3741
3977
|
}
|
|
3742
3978
|
await this.syncState.saveDeltaCursor(cursor);
|
|
3743
3979
|
}
|
|
3980
|
+
async persistDeliveryWatermark(watermark, signature = this.deliverySignature()) {
|
|
3981
|
+
if (!this.syncState?.saveDeliveryWatermark) {
|
|
3982
|
+
return;
|
|
3983
|
+
}
|
|
3984
|
+
await this.syncState.saveDeliveryWatermark(signature, watermark);
|
|
3985
|
+
}
|
|
3744
3986
|
scheduleQuerySubsetReconnect() {
|
|
3745
3987
|
if (this.querySubsetReconnectTimer) {
|
|
3746
3988
|
clearTimeout(this.querySubsetReconnectTimer);
|