@powersync/service-core 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/routes/socket-route.js +1 -1
- package/dist/routes/socket-route.js.map +1 -1
- package/dist/routes/sync-stream.js +1 -1
- package/dist/routes/sync-stream.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +6 -1
- package/dist/storage/BucketStorage.js.map +1 -1
- package/dist/storage/ChecksumCache.d.ts +50 -0
- package/dist/storage/ChecksumCache.js +234 -0
- package/dist/storage/ChecksumCache.js.map +1 -0
- package/dist/storage/mongo/MongoSyncBucketStorage.d.ts +3 -1
- package/dist/storage/mongo/MongoSyncBucketStorage.js +26 -14
- package/dist/storage/mongo/MongoSyncBucketStorage.js.map +1 -1
- package/dist/sync/sync.js +32 -21
- package/dist/sync/sync.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -0
- package/dist/util/protocol-types.js +5 -1
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/utils.d.ts +6 -3
- package/dist/util/utils.js +32 -15
- package/dist/util/utils.js.map +1 -1
- package/package.json +6 -5
- package/src/routes/socket-route.ts +5 -1
- package/src/routes/sync-stream.ts +4 -1
- package/src/storage/BucketStorage.ts +6 -1
- package/src/storage/ChecksumCache.ts +294 -0
- package/src/storage/mongo/MongoSyncBucketStorage.ts +31 -15
- package/src/sync/sync.ts +44 -37
- package/src/util/protocol-types.ts +6 -1
- package/src/util/utils.ts +36 -16
- package/test/src/__snapshots__/sync.test.ts.snap +14 -2
- package/test/src/checksum_cache.test.ts +436 -0
- package/test/src/data_storage.test.ts +3 -3
- package/test/src/large_batch.test.ts +4 -4
- package/test/src/sync_rules.test.ts +11 -9
- package/tsconfig.tsbuildinfo +1 -1
package/dist/sync/sync.js
CHANGED
|
@@ -48,8 +48,8 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
48
48
|
// Bucket state of bucket id -> op_id.
|
|
49
49
|
// This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
|
|
50
50
|
let dataBuckets = new Map();
|
|
51
|
-
let
|
|
52
|
-
let
|
|
51
|
+
let lastChecksums = null;
|
|
52
|
+
let lastWriteCheckpoint = null;
|
|
53
53
|
const { raw_data, binary_data } = params;
|
|
54
54
|
if (params.buckets) {
|
|
55
55
|
for (let { name, after: start } of params.buckets) {
|
|
@@ -76,31 +76,35 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
76
76
|
// TODO: Limit number of buckets even before we get to this point
|
|
77
77
|
throw new Error(`Too many buckets: ${allBuckets.length}`);
|
|
78
78
|
}
|
|
79
|
-
let checksums = undefined;
|
|
80
79
|
let dataBucketsNew = new Map();
|
|
81
80
|
for (let bucket of allBuckets) {
|
|
82
81
|
dataBucketsNew.set(bucket, dataBuckets.get(bucket) ?? '0');
|
|
83
82
|
}
|
|
84
83
|
dataBuckets = dataBucketsNew;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
84
|
+
const bucketList = [...dataBuckets.keys()];
|
|
85
|
+
const checksumMap = await storage.getChecksums(checkpoint, bucketList);
|
|
86
|
+
// Subset of buckets for which there may be new data in this batch.
|
|
87
|
+
let bucketsToFetch;
|
|
88
|
+
if (lastChecksums) {
|
|
89
|
+
const diff = util.checksumsDiff(lastChecksums, checksumMap);
|
|
90
|
+
if (lastWriteCheckpoint == writeCheckpoint &&
|
|
91
|
+
diff.removedBuckets.length == 0 &&
|
|
92
|
+
diff.updatedBuckets.length == 0) {
|
|
91
93
|
// No changes - don't send anything to the client
|
|
92
94
|
continue;
|
|
93
95
|
}
|
|
96
|
+
bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
|
|
94
97
|
let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
|
|
95
98
|
message += `buckets: ${allBuckets.length} | `;
|
|
96
|
-
message += `updated: ${limitedBuckets(diff.
|
|
97
|
-
message += `removed: ${limitedBuckets(diff.
|
|
99
|
+
message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
|
|
100
|
+
message += `removed: ${limitedBuckets(diff.removedBuckets, 20)} | `;
|
|
98
101
|
micro.logger.info(message);
|
|
99
102
|
const checksum_line = {
|
|
100
103
|
checkpoint_diff: {
|
|
101
104
|
last_op_id: checkpoint,
|
|
102
105
|
write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
|
|
103
|
-
|
|
106
|
+
removed_buckets: diff.removedBuckets,
|
|
107
|
+
updated_buckets: diff.updatedBuckets
|
|
104
108
|
}
|
|
105
109
|
};
|
|
106
110
|
yield checksum_line;
|
|
@@ -109,24 +113,27 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
|
|
|
109
113
|
let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
|
|
110
114
|
message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
|
|
111
115
|
micro.logger.info(message);
|
|
116
|
+
bucketsToFetch = allBuckets;
|
|
112
117
|
const checksum_line = {
|
|
113
118
|
checkpoint: {
|
|
114
119
|
last_op_id: checkpoint,
|
|
115
120
|
write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
|
|
116
|
-
buckets:
|
|
121
|
+
buckets: [...checksumMap.values()]
|
|
117
122
|
}
|
|
118
123
|
};
|
|
119
124
|
yield checksum_line;
|
|
120
125
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
126
|
+
lastChecksums = checksumMap;
|
|
127
|
+
lastWriteCheckpoint = writeCheckpoint;
|
|
128
|
+
// This incrementally updates dataBuckets with each individual bucket position.
|
|
129
|
+
// At the end of this, we can be sure that all buckets have data up to the checkpoint.
|
|
130
|
+
yield* bucketDataInBatches({ storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal });
|
|
124
131
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
125
132
|
}
|
|
126
133
|
}
|
|
127
|
-
async function* bucketDataInBatches(
|
|
134
|
+
async function* bucketDataInBatches(request) {
|
|
128
135
|
let isDone = false;
|
|
129
|
-
while (!signal.aborted && !isDone) {
|
|
136
|
+
while (!request.signal.aborted && !isDone) {
|
|
130
137
|
// The code below is functionally the same as this for-await loop below.
|
|
131
138
|
// However, the for-await loop appears to have a memory leak, so we avoid it.
|
|
132
139
|
// for await (const { done, data } of bucketDataBatch(storage, checkpoint, dataBuckets, raw_data, signal)) {
|
|
@@ -136,7 +143,7 @@ async function* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data,
|
|
|
136
143
|
// }
|
|
137
144
|
// break;
|
|
138
145
|
// }
|
|
139
|
-
const iter = bucketDataBatch(
|
|
146
|
+
const iter = bucketDataBatch(request);
|
|
140
147
|
try {
|
|
141
148
|
while (true) {
|
|
142
149
|
const { value, done: iterDone } = await iter.next();
|
|
@@ -160,10 +167,14 @@ async function* bucketDataInBatches(storage, checkpoint, dataBuckets, raw_data,
|
|
|
160
167
|
/**
|
|
161
168
|
* Extracted as a separate internal function just to avoid memory leaks.
|
|
162
169
|
*/
|
|
163
|
-
async function* bucketDataBatch(
|
|
170
|
+
async function* bucketDataBatch(request) {
|
|
171
|
+
const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
|
|
164
172
|
const [_, release] = await syncSemaphore.acquire();
|
|
165
173
|
try {
|
|
166
|
-
|
|
174
|
+
// Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
|
|
175
|
+
// For the first batch, this will be all buckets.
|
|
176
|
+
const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)]));
|
|
177
|
+
const data = storage.getBucketDataBatch(checkpoint, filteredBuckets);
|
|
167
178
|
let has_more = false;
|
|
168
179
|
for await (let r of data) {
|
|
169
180
|
if (signal.aborted) {
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAc5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnF,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA0B,EAC1B,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAc5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnF,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA0B,EAC1B,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,2EAA2E;YAC3E,SAAS;QACX,CAAC;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC7B,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YAC9B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,CAAC;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B,CAAC;gBACD,iDAAiD;gBACjD,SAAS;YACX,CAAC;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,aAAa,eAAe,KAAK,CAAC;YACjF,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3B,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;QACtB,CAAC;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAEhH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAaD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1C,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM;gBACR,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,GAAG,IAAI,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEpG,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI,CAAC;QACH,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAEnD,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE,CAAC;gBAChB,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1B,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC1C,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE,CAAC;gBAC9B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACpC,CAAC;YACD,OAAO,CAAC,WAAW,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,GAAyC;gBACjD,mBAAmB,EAAE;oBACnB,UAAU,EAAE,UAAU;iBACvB;aACF,CAAC;YACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC,MAAM,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
|
|
@@ -76,6 +76,10 @@ export declare const StreamingSyncRequest: t.ObjectCodec<{
|
|
|
76
76
|
* Data is received in a serialized BSON Buffer
|
|
77
77
|
*/
|
|
78
78
|
binary_data: t.OptionalCodec<t.Codec<boolean, boolean, string, t.CodecProps>>;
|
|
79
|
+
/**
|
|
80
|
+
* Client parameters to be passed to the sync rules.
|
|
81
|
+
*/
|
|
82
|
+
parameters: t.OptionalCodec<t.Codec<Record<string, any>, Record<string, any>, string, t.CodecProps>>;
|
|
79
83
|
}>;
|
|
80
84
|
export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
|
|
81
85
|
export interface StreamingSyncCheckpoint {
|
|
@@ -26,7 +26,11 @@ export const StreamingSyncRequest = t.object({
|
|
|
26
26
|
/**
|
|
27
27
|
* Data is received in a serialized BSON Buffer
|
|
28
28
|
*/
|
|
29
|
-
binary_data: t.boolean.optional()
|
|
29
|
+
binary_data: t.boolean.optional(),
|
|
30
|
+
/**
|
|
31
|
+
* Client parameters to be passed to the sync rules.
|
|
32
|
+
*/
|
|
33
|
+
parameters: t.record(t.any).optional()
|
|
30
34
|
});
|
|
31
35
|
export function isContinueCheckpointRequest(request) {
|
|
32
36
|
return (Array.isArray(request.buckets) &&
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol-types.js","sourceRoot":"","sources":["../../src/util/protocol-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAwD9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM;IAEd;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,MAAM;CAChB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAE1C;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAElC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;
|
|
1
|
+
{"version":3,"file":"protocol-types.js","sourceRoot":"","sources":["../../src/util/protocol-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAwD9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM;IAEd;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,MAAM;CAChB,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C;;OAEG;IACH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IAE1C;;OAEG;IACH,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IAElC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEtC;;OAEG;IACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAE9B;;OAEG;IACH,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IAEjC;;OAEG;IACH,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAgGH,MAAM,UAAU,2BAA2B,CAAC,OAAoB;IAC9D,OAAO,CACL,KAAK,CAAC,OAAO,CAAE,OAAqC,CAAC,OAAO,CAAC;QAC7D,OAAQ,OAAqC,CAAC,gBAAgB,IAAI,QAAQ,CAC3E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,OAAoB;IAC7D,OAAO,OAAQ,OAAoC,CAAC,kBAAkB,IAAI,QAAQ,CAAC;AACrF,CAAC"}
|
package/dist/util/utils.d.ts
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
2
|
import * as storage from '../storage/storage-index.js';
|
|
3
3
|
import { BucketChecksum, OpId } from './protocol-types.js';
|
|
4
|
+
export type ChecksumMap = Map<string, BucketChecksum>;
|
|
4
5
|
export declare function hashData(type: string, id: string, data: string): number;
|
|
5
6
|
export declare function hashDelete(sourceKey: string): number;
|
|
6
7
|
export declare function timestampToOpId(ts: bigint): OpId;
|
|
7
|
-
export declare function checksumsDiff(previous:
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
export declare function checksumsDiff(previous: ChecksumMap, current: ChecksumMap): {
|
|
9
|
+
updatedBuckets: BucketChecksum[];
|
|
10
|
+
removedBuckets: string[];
|
|
10
11
|
};
|
|
12
|
+
export declare function addChecksums(a: number, b: number): number;
|
|
13
|
+
export declare function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null): BucketChecksum;
|
|
11
14
|
export declare function getClientCheckpoint(db: pgwire.PgClient, bucketStorage: storage.BucketStorageFactory, options?: {
|
|
12
15
|
timeout?: number;
|
|
13
16
|
}): Promise<OpId>;
|
package/dist/util/utils.js
CHANGED
|
@@ -23,29 +23,46 @@ export function timestampToOpId(ts) {
|
|
|
23
23
|
return ts.toString(10);
|
|
24
24
|
}
|
|
25
25
|
export function checksumsDiff(previous, current) {
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
// All changed ones
|
|
27
|
+
const updatedBuckets = new Map();
|
|
28
|
+
const toRemove = new Set(previous.keys());
|
|
29
|
+
for (let checksum of current.values()) {
|
|
30
|
+
const p = previous.get(checksum.bucket);
|
|
31
|
+
if (p == null) {
|
|
32
|
+
// Added
|
|
33
|
+
updatedBuckets.set(checksum.bucket, checksum);
|
|
34
34
|
}
|
|
35
35
|
else {
|
|
36
|
-
|
|
37
|
-
if (
|
|
38
|
-
|
|
36
|
+
toRemove.delete(checksum.bucket);
|
|
37
|
+
if (checksum.checksum != p.checksum || checksum.count != p.count) {
|
|
38
|
+
// Updated
|
|
39
|
+
updatedBuckets.set(checksum.bucket, checksum);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
// No change
|
|
39
43
|
}
|
|
40
|
-
previousBuckets.delete(checksum.bucket);
|
|
41
44
|
}
|
|
42
45
|
}
|
|
43
|
-
const removed_buckets = [...previousBuckets.keys()];
|
|
44
46
|
return {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
+
updatedBuckets: [...updatedBuckets.values()],
|
|
48
|
+
removedBuckets: [...toRemove]
|
|
47
49
|
};
|
|
48
50
|
}
|
|
51
|
+
export function addChecksums(a, b) {
|
|
52
|
+
return (a + b) & 0xffffffff;
|
|
53
|
+
}
|
|
54
|
+
export function addBucketChecksums(a, b) {
|
|
55
|
+
if (b == null) {
|
|
56
|
+
return a;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
return {
|
|
60
|
+
bucket: a.bucket,
|
|
61
|
+
count: a.count + b.count,
|
|
62
|
+
checksum: addChecksums(a.checksum, b.checksum)
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
49
66
|
export async function getClientCheckpoint(db, bucketStorage, options) {
|
|
50
67
|
const start = Date.now();
|
|
51
68
|
const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
|
package/dist/util/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAIrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAIjD,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAwB;IAC5E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;YACxB,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC;YAClB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,OAAO,EAAE,CAAC,UAAU,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,17 +5,18 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.1
|
|
8
|
+
"version": "0.2.1",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@journeyapps-platform/micro": "^17.0.1",
|
|
14
14
|
"@journeyapps-platform/micro-migrate": "^4.0.1",
|
|
15
|
+
"@js-sdsl/ordered-set": "^4.4.2",
|
|
15
16
|
"@opentelemetry/api": "~1.8.0",
|
|
16
|
-
"@opentelemetry/resources": "^1.24.1",
|
|
17
|
-
"@opentelemetry/exporter-prometheus": "^0.51.1",
|
|
18
17
|
"@opentelemetry/exporter-metrics-otlp-http": "^0.51.1",
|
|
18
|
+
"@opentelemetry/exporter-prometheus": "^0.51.1",
|
|
19
|
+
"@opentelemetry/resources": "^1.24.1",
|
|
19
20
|
"@opentelemetry/sdk-metrics": "1.24.1",
|
|
20
21
|
"async-mutex": "^0.5.0",
|
|
21
22
|
"bson": "^6.6.0",
|
|
@@ -35,8 +36,8 @@
|
|
|
35
36
|
"@powersync/service-jpgwire": "0.17.12",
|
|
36
37
|
"@powersync/service-jsonbig": "0.17.10",
|
|
37
38
|
"@powersync/service-rsocket-router": "0.0.7",
|
|
38
|
-
"@powersync/service-
|
|
39
|
-
"@powersync/service-
|
|
39
|
+
"@powersync/service-sync-rules": "0.17.10",
|
|
40
|
+
"@powersync/service-types": "0.1.0"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"@journeyapps-platform/micro-dev": "^1.6.7",
|
|
@@ -34,7 +34,11 @@ export const sync_stream_reactive: SocketRouteGenerator = (router) =>
|
|
|
34
34
|
|
|
35
35
|
const controller = new AbortController();
|
|
36
36
|
|
|
37
|
-
const syncParams: SyncParameters = normalizeTokenParameters(
|
|
37
|
+
const syncParams: SyncParameters = normalizeTokenParameters(
|
|
38
|
+
context.token_payload?.parameters ?? {},
|
|
39
|
+
params.parameters ?? {}
|
|
40
|
+
);
|
|
41
|
+
|
|
38
42
|
const storage = system.storage;
|
|
39
43
|
// Sanity check before we start the stream
|
|
40
44
|
const cp = await storage.getActiveCheckpoint();
|
|
@@ -30,7 +30,10 @@ export const syncStreamed: RouteGenerator = (router) =>
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
const params: util.StreamingSyncRequest = payload.params;
|
|
33
|
-
const syncParams: SyncParameters = normalizeTokenParameters(
|
|
33
|
+
const syncParams: SyncParameters = normalizeTokenParameters(
|
|
34
|
+
payload.context.token_payload!.parameters ?? {},
|
|
35
|
+
payload.params.parameters ?? {}
|
|
36
|
+
);
|
|
34
37
|
|
|
35
38
|
const storage = system.storage;
|
|
36
39
|
// Sanity check before we start the stream
|
|
@@ -230,7 +230,12 @@ export interface SyncRulesBucketStorage {
|
|
|
230
230
|
options?: BucketDataBatchOptions
|
|
231
231
|
): AsyncIterable<util.SyncBucketData>;
|
|
232
232
|
|
|
233
|
-
|
|
233
|
+
/**
|
|
234
|
+
* Compute checksums for a given list of buckets.
|
|
235
|
+
*
|
|
236
|
+
* Returns zero checksums for any buckets not found.
|
|
237
|
+
*/
|
|
238
|
+
getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
|
|
234
239
|
|
|
235
240
|
/**
|
|
236
241
|
* Terminate the sync rules.
|