@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/storage/MongoBucketStorage.d.ts +34 -16
- package/dist/storage/MongoBucketStorage.js +219 -70
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +42 -23
- package/dist/storage/implementation/MongoBucketBatch.js +61 -44
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +10 -9
- package/dist/storage/implementation/MongoChecksums.js +3 -111
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
- package/dist/storage/implementation/MongoCompactor.js +18 -9
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
- package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +5 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +41 -23
- package/dist/storage/implementation/MongoSyncBucketStorage.js +51 -43
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +1 -2
- package/dist/storage/implementation/common/PersistedBatch.js +5 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +9 -14
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +4 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +42 -26
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +15 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +47 -45
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +286 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +19 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +40 -28
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +232 -116
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +13 -12
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +29 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
- package/dist/storage/implementation/v3/bucket-format.js +58 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +28 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +64 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +2 -2
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +5 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +1 -1
- package/dist/utils/util.js +3 -2
- package/dist/utils/util.js.map +1 -1
- package/package.json +9 -19
- package/src/storage/MongoBucketStorage.ts +334 -87
- package/src/storage/implementation/MongoBucketBatch.ts +86 -58
- package/src/storage/implementation/MongoChecksums.ts +6 -138
- package/src/storage/implementation/MongoCompactor.ts +19 -9
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +5 -2
- package/src/storage/implementation/MongoSyncBucketStorage.ts +70 -56
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
- package/src/storage/implementation/common/PersistedBatch.ts +7 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +11 -19
- package/src/storage/implementation/models.ts +4 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +55 -34
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +70 -64
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +358 -314
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
- package/src/storage/implementation/v3/MongoCompactorV3.ts +646 -29
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +315 -157
- package/src/storage/implementation/v3/PersistedBatchV3.ts +62 -36
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +41 -36
- package/src/storage/implementation/v3/bucket-format.ts +70 -0
- package/src/storage/implementation/v3/chunking.ts +38 -0
- package/src/storage/implementation/v3/models.ts +93 -55
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/utils/test-utils.ts +6 -3
- package/src/utils/util.ts +3 -2
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
- package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
- package/test/src/storage.test.ts +11 -11
- package/test/src/storage_compacting.test.ts +1956 -6
- package/test/src/storage_sync.test.ts +1077 -68
- package/test/src/storeCurrentData.test.ts +17 -7
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
|
@@ -2,18 +2,20 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
|
2
2
|
import * as bson from 'bson';
|
|
3
3
|
import { mongoTableId } from '../../../utils/util.js';
|
|
4
4
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { serializeBucketData } from './bucket-format.js';
|
|
6
|
+
import { chunkBucketData } from './chunking.js';
|
|
7
|
+
import { serializeParameterLookup, taggedBucketParameterDocumentToTagged } from './models.js';
|
|
7
8
|
export class PersistedBatchV3 extends PersistedBatch {
|
|
8
9
|
currentData = [];
|
|
9
10
|
sourceTablePendingDeletes = new Map();
|
|
11
|
+
// Abstract override from PersistedBatch (V3-specific error message)
|
|
10
12
|
checkDefinitionId(definitionId) {
|
|
11
13
|
if (definitionId == null) {
|
|
12
|
-
// This is required for V3 storage.
|
|
13
14
|
throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
|
|
14
15
|
}
|
|
15
16
|
return definitionId;
|
|
16
17
|
}
|
|
18
|
+
// Concrete implementations from PersistedBatchShared
|
|
17
19
|
getBucketDefinitionId(bucketSource) {
|
|
18
20
|
return this.mapping.bucketSourceId(bucketSource);
|
|
19
21
|
}
|
|
@@ -22,13 +24,18 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
22
24
|
const remaining_lookups = new Map();
|
|
23
25
|
for (let lookup of data.existing_lookups) {
|
|
24
26
|
if (lookup.indexId == null) {
|
|
25
|
-
throw new ReplicationAssertionError('Expected
|
|
27
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
28
|
+
}
|
|
29
|
+
if (sourceTable.parameterLookupSourceIds != null && !sourceTable.parameterLookupSourceIds.has(lookup.indexId)) {
|
|
30
|
+
// The parameter index is not active anymore.
|
|
31
|
+
// We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
|
|
32
|
+
continue;
|
|
26
33
|
}
|
|
27
34
|
remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
|
|
28
35
|
}
|
|
29
36
|
for (let result of evaluated) {
|
|
30
37
|
const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
|
|
31
|
-
const binLookup =
|
|
38
|
+
const binLookup = serializeParameterLookup(result.lookup);
|
|
32
39
|
remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
|
|
33
40
|
const op_id = data.op_seq.next();
|
|
34
41
|
this.debugLastOpId = op_id;
|
|
@@ -52,7 +59,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
52
59
|
this.debugLastOpId = op_id;
|
|
53
60
|
const indexId = lookup.indexId;
|
|
54
61
|
if (indexId == null) {
|
|
55
|
-
throw new ReplicationAssertionError('Expected
|
|
62
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
56
63
|
}
|
|
57
64
|
const values = {
|
|
58
65
|
_id: op_id,
|
|
@@ -109,7 +116,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
109
116
|
upsertCurrentData(values) {
|
|
110
117
|
const buckets = values.buckets.map((bucket) => {
|
|
111
118
|
if (bucket.definitionId == null) {
|
|
112
|
-
throw new ReplicationAssertionError('Expected
|
|
119
|
+
throw new ReplicationAssertionError('Expected bucket when incrementalReprocessing is enabled');
|
|
113
120
|
}
|
|
114
121
|
return {
|
|
115
122
|
def: bucket.definitionId,
|
|
@@ -120,7 +127,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
120
127
|
});
|
|
121
128
|
const lookups = values.lookups.map((lookup) => {
|
|
122
129
|
if (lookup.indexId == null) {
|
|
123
|
-
throw new ReplicationAssertionError('Expected
|
|
130
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
124
131
|
}
|
|
125
132
|
return {
|
|
126
133
|
i: lookup.indexId,
|
|
@@ -149,6 +156,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
149
156
|
get currentDataCount() {
|
|
150
157
|
return this.currentData.length;
|
|
151
158
|
}
|
|
159
|
+
// Flush methods
|
|
152
160
|
async flushBucketData(session) {
|
|
153
161
|
const operationsByDefinition = new Map();
|
|
154
162
|
for (const document of this.bucketData) {
|
|
@@ -157,14 +165,29 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
157
165
|
operationsByDefinition.set(document.bucketKey.definitionId, existing);
|
|
158
166
|
}
|
|
159
167
|
for (const [definitionId, documents] of operationsByDefinition.entries()) {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
168
|
+
const operationsByBucket = new Map();
|
|
169
|
+
for (const document of documents) {
|
|
170
|
+
const existing = operationsByBucket.get(document.bucketKey.bucket) ?? [];
|
|
171
|
+
existing.push(document);
|
|
172
|
+
operationsByBucket.set(document.bucketKey.bucket, existing);
|
|
173
|
+
}
|
|
174
|
+
const inserts = [];
|
|
175
|
+
for (const [bucket, ops] of operationsByBucket.entries()) {
|
|
176
|
+
const chunks = chunkBucketData(ops);
|
|
177
|
+
for (const chunk of chunks) {
|
|
178
|
+
inserts.push({
|
|
179
|
+
insertOne: {
|
|
180
|
+
document: serializeBucketData(bucket, chunk)
|
|
181
|
+
}
|
|
182
|
+
});
|
|
163
183
|
}
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
184
|
+
}
|
|
185
|
+
if (inserts.length > 0) {
|
|
186
|
+
await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
|
|
187
|
+
session,
|
|
188
|
+
ordered: false
|
|
189
|
+
});
|
|
190
|
+
}
|
|
168
191
|
}
|
|
169
192
|
}
|
|
170
193
|
async flushBucketParameters(session) {
|
|
@@ -175,9 +198,9 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
175
198
|
operationsByIndex.set(document.index, existing);
|
|
176
199
|
}
|
|
177
200
|
for (const [indexId, documents] of operationsByIndex.entries()) {
|
|
178
|
-
await this.db.
|
|
201
|
+
await this.db.parameterIndex(this.group_id, indexId).bulkWrite(documents.map((document) => ({
|
|
179
202
|
insertOne: {
|
|
180
|
-
document:
|
|
203
|
+
document: taggedBucketParameterDocumentToTagged(document)
|
|
181
204
|
}
|
|
182
205
|
})), {
|
|
183
206
|
session,
|
|
@@ -208,18 +231,18 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
208
231
|
};
|
|
209
232
|
});
|
|
210
233
|
if (sourceTableUpdates.length > 0) {
|
|
211
|
-
await this.db.
|
|
234
|
+
await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
|
|
212
235
|
}
|
|
213
236
|
for (const operations of operationsBySourceTable.values()) {
|
|
214
237
|
const sourceTableId = operations[0].sourceTableId;
|
|
215
|
-
await this.db.
|
|
238
|
+
await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(operations.map((entry) => entry.operation), {
|
|
216
239
|
session,
|
|
217
240
|
ordered: true
|
|
218
241
|
});
|
|
219
242
|
}
|
|
220
243
|
}
|
|
221
244
|
async flushBucketStates(session) {
|
|
222
|
-
await this.db.
|
|
245
|
+
await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
|
|
223
246
|
session,
|
|
224
247
|
ordered: false
|
|
225
248
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatchV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/PersistedBatchV3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"PersistedBatchV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/PersistedBatchV3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAEL,cAAc,EAGf,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAIL,wBAAwB,EAExB,qCAAqC,EACtC,MAAM,aAAa,CAAC;AAGrB,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAClD,WAAW,GAAsG,EAAE,CAAC;IACpH,yBAAyB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAI5D,oEAAoE;IAEjD,iBAAiB,CAAC,YAAuC;QAC1E,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,yBAAyB,CAAC,4DAA4D,CAAC,CAAC;QACpG,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,qDAAqD;IAE3C,qBAAqB,CAAC,YAA8B;QAC5D,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAED,iBAAiB,CAAC,IAA8B;QAC9C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QAErE,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;YACjG,CAAC;YACD,IAAI,WAAW,CAAC,wBAAwB,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9G,6CAA6C;gBAC7C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACzF,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,wBAAwB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,iBAAiB,CAAC,MAAM,CAAC,GAAG,kBAAkB,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAElF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACb;gBACD,MAAM,EAAE,SAAS;gBACjB,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;aAC3C,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,kBAAkB;aAC1B,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC/B,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;YACjG,CAAC;YACD,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,KAAK;gBACV,GAAG,EAAE;oBACH,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC/B,CAAC,EAAE,SAAS;iBACb;gBACD,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,iBAAiB,EAAE,EAAE;aACtB,CAAC;YACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,GAAG,MAAM;gBACT,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qBAAqB,CAAC,aAA4B,EAAE,SAA4B;QAC9E,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa;YACb,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE;iBAC3B;aACF;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,qBAAqB,CACnB,aAA4B,EAC5B,SAA4B,EAC5B,qBAAmC;QAEnC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa;YACb,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE;oBAC1B,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,IAAI,EAAE,IAAI;4BACV,OAAO,EAAE,EAAE;4BACX,OAAO,EAAE,EAAE;4BACX,cAAc,EAAE,qBAAqB;yBACtC;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACF;SACF,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,qBAAqB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACjF,IAAI,qBAAqB,IAAI,IAAI,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnF,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,cAAc,EAAE,qBAAqB,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,MAAgC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAChC,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;YACjG,CAAC;YACD,OAAO;gBACL,GAAG,EAAE,MAAM,CAAC,YAAY;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5C,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,yBAAyB,CAAC,yDAAyD,CAAC,CAAC;YACjG,CAAC;YACD,OAAO;gBACL,CAAC,EAAE,MAAM,CAAC,OAAO;gBACjB,CAAC,EAAE,MAAM,CAAC,MAAM;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;YACpB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE;oBACjC,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,IAAI,EAAE,MAAM,CAAC,IAAI;4BACjB,OAAO;4BACP,OAAO;yBACR;wBACD,MAAM,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE;qBAC9B;oBACD,MAAM,EAAE,IAAI;iBACb;aACF;SACF,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,gBAAgB;IAEN,KAAK,CAAC,eAAe,CAAC,OAA4B;QAC1D,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAuC,CAAC;QAC9E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,KAAK,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,sBAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;YACzE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA2B,CAAC;YAC9D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACxB,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,OAAO,GAAwD,EAAE,CAAC;YACxE,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,SAAS,EAAE;4BACT,QAAQ,EAAE,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC;yBAC7C;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE;oBACvE,OAAO;oBACP,OAAO,EAAE,KAAK;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAES,KAAK,CAAC,qBAAqB,CAAC,OAA4B;QAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAwC,CAAC;QAC1E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,CAC5D,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC3B,SAAS,EAAE;oBACT,QAAQ,EAAE,qCAAqC,CAAC,QAAQ,CAAC;iBAC1D;aACF,CAAC,CAAC,EACH;gBACE,OAAO;gBACP,OAAO,EAAE,KAAK;aACf,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,OAA4B;QAC3D,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC3E,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;YAC5D,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAClE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,uBAAuB,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,kBAAkB,GAAyD;YAC/E,GAAG,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE;SAC5C,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACrB,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;oBACvC,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,qBAAqB,EAAE,KAAK;yBAC7B;qBACF;iBACF;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACvG,CAAC;QAED,KAAK,MAAM,UAAU,IAAI,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1D,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC;YACnD,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,SAAS,CACjE,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAC1C;gBACE,OAAO;gBACP,OAAO,EAAE,IAAI;aACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,OAA4B;QAC5D,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;YAC/E,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;IACL,CAAC;IAES,gBAAgB;QACxB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,CAAC;IACzC,CAAC;IAEO,qBAAqB;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAwB,EAAE,EAAE;YAC7E,IAAI,KAAK,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;gBAC/B,MAAM,IAAI,yBAAyB,CAAC,uEAAuE,CAAC,CAAC;YAC/G,CAAC;YACD,OAAO;gBACL,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,KAAK,CAAC,YAAY;4BACrB,CAAC,EAAE,KAAK,CAAC,MAAM;yBAChB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,OAAO,EAAE,KAAK,CAAC,MAAM;yBACtB;wBACD,IAAI,EAAE;4BACJ,8BAA8B,EAAE,KAAK,CAAC,cAAc;4BACpD,8BAA8B,EAAE,KAAK,CAAC,cAAc;yBACrD;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aAC2D,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import {
|
|
2
|
+
import { loadBucketDataDocument, serializeBucketData } from './bucket-format.js';
|
|
3
|
+
function extractPartialDocumentFields(doc) {
|
|
4
|
+
if (typeof doc !== 'object' || doc === null) {
|
|
5
|
+
throw new Error('Invalid partial document: expected object');
|
|
6
|
+
}
|
|
7
|
+
const d = doc;
|
|
8
|
+
const id = d._id;
|
|
9
|
+
if (typeof id !== 'object' || id === null || !('o' in id)) {
|
|
10
|
+
throw new Error('Invalid partial document: missing _id.o');
|
|
11
|
+
}
|
|
12
|
+
return d;
|
|
13
|
+
}
|
|
14
|
+
// MongoDB's MinKey/MaxKey are special sentinel values that don't match the bigint type
|
|
15
|
+
// for _id.o in BucketDataDocumentGenericId, so we need an explicit cast.
|
|
16
|
+
function minKeyForBucket(bucket) {
|
|
17
|
+
return {
|
|
18
|
+
b: bucket,
|
|
19
|
+
o: new mongo.MinKey()
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function maxKeyForBucket(bucket) {
|
|
23
|
+
return {
|
|
24
|
+
b: bucket,
|
|
25
|
+
o: new mongo.MaxKey()
|
|
26
|
+
};
|
|
27
|
+
}
|
|
3
28
|
export class SingleBucketStoreV3 {
|
|
4
29
|
db;
|
|
5
30
|
key;
|
|
@@ -7,10 +32,12 @@ export class SingleBucketStoreV3 {
|
|
|
7
32
|
constructor(db, key) {
|
|
8
33
|
this.db = db;
|
|
9
34
|
this.key = key;
|
|
10
|
-
|
|
35
|
+
// Cast from the version-specific collection type to the generic interface
|
|
36
|
+
// used across storage versions.
|
|
37
|
+
this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
|
|
11
38
|
}
|
|
12
39
|
docId(o) {
|
|
13
|
-
// `satisfies
|
|
40
|
+
// `satisfies BucketDataKey` checks that we use the correct type for V3 storage
|
|
14
41
|
// `as BucketDataDocumentGenericId` does a cast to get the interface virtual type
|
|
15
42
|
return {
|
|
16
43
|
b: this.key.bucket,
|
|
@@ -18,30 +45,39 @@ export class SingleBucketStoreV3 {
|
|
|
18
45
|
};
|
|
19
46
|
}
|
|
20
47
|
get minId() {
|
|
21
|
-
return
|
|
22
|
-
b: this.key.bucket,
|
|
23
|
-
o: new mongo.MinKey()
|
|
24
|
-
}; // No way to properly type this
|
|
48
|
+
return minKeyForBucket(this.key.bucket);
|
|
25
49
|
}
|
|
26
50
|
get maxId() {
|
|
27
|
-
return
|
|
28
|
-
b: this.key.bucket,
|
|
29
|
-
o: new mongo.MaxKey()
|
|
30
|
-
}; // No way to properly type this
|
|
51
|
+
return maxKeyForBucket(this.key.bucket);
|
|
31
52
|
}
|
|
32
53
|
toPersistedDocument(source) {
|
|
33
|
-
return
|
|
54
|
+
return serializeBucketData(this.key.bucket, [
|
|
55
|
+
{ bucketKey: this.key, ...source }
|
|
56
|
+
]);
|
|
34
57
|
}
|
|
35
58
|
fromPersistedDocument(doc) {
|
|
36
|
-
|
|
59
|
+
const generator = loadBucketDataDocument(this.key, doc);
|
|
60
|
+
const first = generator.next();
|
|
61
|
+
if (first.done) {
|
|
62
|
+
throw new Error('Empty ops array in BucketDataDocumentV3');
|
|
63
|
+
}
|
|
64
|
+
return first.value;
|
|
37
65
|
}
|
|
38
66
|
fromPartialPersistedDocument(doc) {
|
|
39
67
|
const document = doc;
|
|
40
|
-
const
|
|
68
|
+
const op = document.ops?.[0];
|
|
69
|
+
if (op == null) {
|
|
70
|
+
const fields = extractPartialDocumentFields(doc);
|
|
71
|
+
const { _id, ...rest } = fields;
|
|
72
|
+
return {
|
|
73
|
+
bucketKey: this.key,
|
|
74
|
+
o: _id.o,
|
|
75
|
+
...rest
|
|
76
|
+
};
|
|
77
|
+
}
|
|
41
78
|
return {
|
|
42
79
|
bucketKey: this.key,
|
|
43
|
-
|
|
44
|
-
...rest
|
|
80
|
+
...op
|
|
45
81
|
};
|
|
46
82
|
}
|
|
47
83
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SingleBucketStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SingleBucketStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"SingleBucketStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SingleBucketStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AASvD,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAIjF,SAAS,4BAA4B,CAAC,GAAY;IAChD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC;IACjB,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAmD,CAAC;AAC7D,CAAC;AAED,uFAAuF;AACvF,yEAAyE;AACzE,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO;QACL,CAAC,EAAE,MAAM;QACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;KACoB,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO;QACL,CAAC,EAAE,MAAM;QACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;KACoB,CAAC;AAC9C,CAAC;AAED,MAAM,OAAO,mBAAmB;IAIpB;IACQ;IAJF,UAAU,CAA8C;IAExE,YACU,EAA6B,EACrB,GAAc;QADtB,OAAE,GAAF,EAAE,CAA2B;QACrB,QAAG,GAAH,GAAG,CAAW;QAE9B,0EAA0E;QAC1E,gCAAgC;QAChC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAC7B,GAAG,CAAC,mBAAmB,EACvB,GAAG,CAAC,YAAY,CACyC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,CAAe;QACnB,+EAA+E;QAC/E,iFAAiF;QACjF,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC;SACqD,CAAC;IAC3D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,KAAK;QACP,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,mBAAmB,CAAC,MAAwC;QAC1D,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE;YAC1C,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE;SACnC,CAAyC,CAAC;IAC7C,CAAC;IAED,qBAAqB,CAAC,GAA8B;QAClD,MAAM,SAAS,GAAG,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAsC,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,4BAA4B,CAC1B,GAA+C;QAE/C,MAAM,QAAQ,GAAG,GAA2D,CAAC;QAC7E,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;YACjD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;YAChC,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,GAAG;gBACnB,CAAC,EAAE,GAAG,CAAC,CAAC;gBACR,GAAG,IAAI;aACsC,CAAC;QAClD,CAAC;QACD,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,GAAG,EAAE;SACwC,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { Logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import { BucketDefinitionMapping } from '@powersync/service-core';
|
|
3
4
|
import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
|
|
4
5
|
import * as bson from 'bson';
|
|
5
|
-
import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
|
|
6
6
|
import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
7
7
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
8
8
|
export declare class SourceRecordStoreV3 implements SourceRecordStore {
|
|
9
9
|
private readonly db;
|
|
10
|
-
private readonly
|
|
10
|
+
private readonly replicationStreamId;
|
|
11
11
|
private readonly mapping;
|
|
12
|
-
constructor(db: VersionedPowerSyncMongoV3,
|
|
12
|
+
constructor(db: VersionedPowerSyncMongoV3, replicationStreamId: number, mapping: BucketDefinitionMapping);
|
|
13
13
|
mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'];
|
|
14
14
|
mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'];
|
|
15
15
|
private createLoadedDocument;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
|
|
3
3
|
import { cacheKey } from '../OperationBatch.js';
|
|
4
|
-
import {
|
|
4
|
+
import { serializeParameterLookup } from './models.js';
|
|
5
5
|
export class SourceRecordStoreV3 {
|
|
6
6
|
db;
|
|
7
|
-
|
|
7
|
+
replicationStreamId;
|
|
8
8
|
mapping;
|
|
9
|
-
constructor(db,
|
|
9
|
+
constructor(db, replicationStreamId, mapping) {
|
|
10
10
|
this.db = db;
|
|
11
|
-
this.
|
|
11
|
+
this.replicationStreamId = replicationStreamId;
|
|
12
12
|
this.mapping = mapping;
|
|
13
13
|
}
|
|
14
14
|
mapEvaluatedBuckets(evaluated) {
|
|
@@ -22,7 +22,7 @@ export class SourceRecordStoreV3 {
|
|
|
22
22
|
mapParameterLookups(paramEvaluated) {
|
|
23
23
|
return paramEvaluated.map((entry) => ({
|
|
24
24
|
indexId: this.mapping.parameterLookupId(entry.lookup.source),
|
|
25
|
-
lookup:
|
|
25
|
+
lookup: serializeParameterLookup(entry.lookup)
|
|
26
26
|
}));
|
|
27
27
|
}
|
|
28
28
|
createLoadedDocument(sourceTableId, id, data, buckets, lookups) {
|
|
@@ -50,7 +50,7 @@ export class SourceRecordStoreV3 {
|
|
|
50
50
|
_id: { $in: replicaIds }
|
|
51
51
|
};
|
|
52
52
|
const sizeCursor = this.db
|
|
53
|
-
.
|
|
53
|
+
.sourceRecords(this.replicationStreamId, sourceTableId)
|
|
54
54
|
.aggregate([
|
|
55
55
|
{
|
|
56
56
|
$match: filter
|
|
@@ -75,7 +75,9 @@ export class SourceRecordStoreV3 {
|
|
|
75
75
|
const filter = {
|
|
76
76
|
_id: { $in: replicaIds }
|
|
77
77
|
};
|
|
78
|
-
const cursor = this.db
|
|
78
|
+
const cursor = this.db
|
|
79
|
+
.sourceRecords(this.replicationStreamId, sourceTableId)
|
|
80
|
+
.find(filter, { session, projection });
|
|
79
81
|
for await (const doc of cursor.stream()) {
|
|
80
82
|
const loaded = this.createLoadedDocument(sourceTableId, doc._id, idsOnly ? null : doc.data, idsOnly ? [] : doc.buckets, idsOnly ? [] : doc.lookups);
|
|
81
83
|
documents.set(loaded.cacheKey, loaded);
|
|
@@ -84,7 +86,7 @@ export class SourceRecordStoreV3 {
|
|
|
84
86
|
return documents;
|
|
85
87
|
}
|
|
86
88
|
async loadTruncateBatch(session, sourceTableId, limit) {
|
|
87
|
-
const cursor = this.db.
|
|
89
|
+
const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find({
|
|
88
90
|
pending_delete: { $exists: false }
|
|
89
91
|
}, {
|
|
90
92
|
projection: {
|
|
@@ -102,7 +104,7 @@ export class SourceRecordStoreV3 {
|
|
|
102
104
|
// Since there may be a lot (100+) of these collections in some cases, we track which
|
|
103
105
|
// ones have dirty deletes in source_tables.
|
|
104
106
|
const dirtySourceTables = await this.db
|
|
105
|
-
.
|
|
107
|
+
.sourceTables(this.replicationStreamId)
|
|
106
108
|
.find({
|
|
107
109
|
latest_pending_delete: { $exists: true }
|
|
108
110
|
}, {
|
|
@@ -112,7 +114,7 @@ export class SourceRecordStoreV3 {
|
|
|
112
114
|
let deletedCount = 0;
|
|
113
115
|
const sourceTableUpdates = [];
|
|
114
116
|
for (const sourceTable of dirtySourceTables) {
|
|
115
|
-
const collection = this.db.
|
|
117
|
+
const collection = this.db.sourceRecords(this.replicationStreamId, sourceTable._id);
|
|
116
118
|
const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
|
|
117
119
|
deletedCount += result.deletedCount;
|
|
118
120
|
if (sourceTable.latest_pending_delete != null && sourceTable.latest_pending_delete <= lastCheckpoint) {
|
|
@@ -133,7 +135,7 @@ export class SourceRecordStoreV3 {
|
|
|
133
135
|
}
|
|
134
136
|
}
|
|
135
137
|
if (sourceTableUpdates.length > 0) {
|
|
136
|
-
await this.db.
|
|
138
|
+
await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
|
|
137
139
|
}
|
|
138
140
|
if (deletedCount > 0) {
|
|
139
141
|
logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SourceRecordStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SourceRecordStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAM5D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"SourceRecordStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SourceRecordStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAM5D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAyB,wBAAwB,EAAyB,MAAM,aAAa,CAAC;AAErG,MAAM,OAAO,mBAAmB;IAEX;IACA;IACA;IAHnB,YACmB,EAA6B,EAC7B,mBAA2B,EAC3B,OAAgC;QAFhC,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,YAAO,GAAP,OAAO,CAAyB;IAChD,CAAC;IAEJ,mBAAmB,CAAC,SAAyB;QAC3C,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/B,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC;YACvD,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,EAAE,EAAE,KAAK,CAAC,EAAE;SACb,CAAC,CAAC,CAAC;IACN,CAAC;IAED,mBAAmB,CAAC,cAAqC;QACvD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC5D,MAAM,EAAE,wBAAwB,CAAC,KAAK,CAAC,MAAM,CAAC;SAC/C,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,oBAAoB,CAC1B,aAA4B,EAC5B,EAAqB,EACrB,IAAwB,EACxB,OAAyC,EACzC,OAAyC;QAEzC,OAAO;YACL,aAAa;YACb,SAAS,EAAE,EAAE;YACb,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,YAAY,EAAE,MAAM,CAAC,GAAG;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,OAAO,EAAE,MAAM,CAAC,CAAC;gBACjB,MAAM,EAAE,MAAM,CAAC,CAAC;aACjB,CAAC,CAAC;YACH,QAAQ,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAA4B,EAAE,OAAkC;QAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,KAAK,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,UAAmB,EAAE;aACgB,CAAC;YACpD,MAAM,UAAU,GAAsE,IAAI,CAAC,EAAE;iBAC1F,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC;iBACtD,SAAS,CACR;gBACE;oBACE,MAAM,EAAE,MAAM;iBACf;gBACD;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC5C,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA4B,EAC5B,OAAkC,EAClC,OAAgB;QAEhB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA8B,CAAC;QACxD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,KAAK,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,MAAM,MAAM,GAAG;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,UAAmB,EAAE;aACgB,CAAC;YACpD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;iBACnB,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC;iBACtD,IAAI,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;YACzC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CACtC,aAAa,EACb,GAAG,CAAC,GAAG,EACP,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EACzB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAC1B,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAC3B,CAAC;gBACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,OAA4B,EAC5B,aAA4B,EAC5B,KAAa;QAEb,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,IAAI,CAChF;YACE,cAAc,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACnC,EACD;YACE,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;gBACN,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;aACX;YACD,KAAK;YACL,OAAO;SACR,CACF,CAAC;QACF,OAAO,CAAC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAClF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,cAAsB,EAAE,MAAc;QAC5D,6DAA6D;QAC7D,qFAAqF;QACrF,4CAA4C;QAE5C,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,EAAE;aACpC,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC;aACtC,IAAI,CACH;YACE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACzC,EACD;YACE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAE;SACjD,CACF;aACA,OAAO,EAAE,CAAC;QAEb,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,kBAAkB,GAAyD,EAAE,CAAC;QACpF,KAAK,MAAM,WAAW,IAAI,iBAAiB,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;YACpF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YACpG,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAEpC,IAAI,WAAW,CAAC,qBAAqB,IAAI,IAAI,IAAI,WAAW,CAAC,qBAAqB,IAAI,cAAc,EAAE,CAAC;gBACrG,kBAAkB,CAAC,IAAI,CAAC;oBACtB,SAAS,EAAE;wBACT,MAAM,EAAE;4BACN,GAAG,EAAE,WAAW,CAAC,GAAG;4BACpB,oFAAoF;4BACpF,qBAAqB,EAAE,WAAW,CAAC,qBAAqB;yBACzD;wBACD,MAAM,EAAE;4BACN,MAAM,EAAE;gCACN,qBAAqB,EAAE,CAAC;6BACzB;yBACF;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,SAAS,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACzG,CAAC;QACD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,cAAc,YAAY,uDAAuD,cAAc,EAAE,CAAC,CAAC;QACjH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,UAAmD,EACnD,aAA4B,EAC5B,cAAsB,EACtB,MAAc;QAEd,OAAO,4BAA4B,CACjC,GAAG,EAAE,CACH,UAAU,CAAC,UAAU,CACnB;YACE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;SACxD,EACD;YACE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;SACzD,CACF,EACH;YACE,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,0BAA0B,GAAG,CAAC;YACzD,OAAO,EAAE,CAAC,CAAS,EAAE,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,wCAAwC,aAAa,iBAAiB,CAAC,CAAC;YACxG,CAAC;SACF,CACF,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,OAAkC;QACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC9D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACxD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -4,19 +4,20 @@ import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBa
|
|
|
4
4
|
import { CommonSourceTableDocument } from '../models.js';
|
|
5
5
|
import { BucketDataDocumentV3, BucketParameterDocumentV3, BucketStateDocumentV3, CurrentDataDocumentV3, SourceTableDocumentV3, SyncConfigDefinition } from './models.js';
|
|
6
6
|
export declare class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
7
|
+
constructor(upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0], storageConfig: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[1]);
|
|
7
8
|
get syncConfigDefinitions(): mongo.Collection<SyncConfigDefinition>;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId): Promise<void>;
|
|
9
|
+
sourceRecords(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3>;
|
|
10
|
+
listSourceRecordCollections(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]>;
|
|
11
|
+
initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId, session?: mongo.ClientSession): Promise<void>;
|
|
11
12
|
commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument>;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
indexId: ParameterIndexId;
|
|
13
|
+
bucketState(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3>;
|
|
14
|
+
parameterIndex(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3>;
|
|
15
|
+
sourceTables(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3>;
|
|
16
|
+
bucketData(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3>;
|
|
17
|
+
listBucketDataCollections(replicationStreamId: number): Promise<mongo.Collection<mongo.BSON.Document>[]>;
|
|
18
|
+
listParameterIndexCollections(replicationStreamId: number): Promise<{
|
|
19
|
+
collection: mongo.Collection<mongo.BSON.Document>;
|
|
20
|
+
indexId: string;
|
|
21
21
|
}[]>;
|
|
22
|
+
initializeStreamStorage(replicationStreamId: number): Promise<void>;
|
|
22
23
|
}
|
|
@@ -1,38 +1,56 @@
|
|
|
1
1
|
import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
|
|
2
2
|
export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
3
|
+
constructor(upstream, storageConfig) {
|
|
4
|
+
super(upstream, storageConfig);
|
|
5
|
+
}
|
|
3
6
|
get syncConfigDefinitions() {
|
|
4
7
|
return this.db.collection('sync_config');
|
|
5
8
|
}
|
|
6
|
-
|
|
9
|
+
sourceRecords(replicationStreamId, sourceTableId) {
|
|
7
10
|
const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
|
|
8
11
|
return this.db.collection(collectionName);
|
|
9
12
|
}
|
|
10
|
-
async
|
|
13
|
+
async listSourceRecordCollections(replicationStreamId) {
|
|
11
14
|
return this.listCollectionsByPrefix(`source_records_${replicationStreamId}_`);
|
|
12
15
|
}
|
|
13
|
-
async initializeSourceRecordsCollection(replicationStreamId, sourceTableId) {
|
|
14
|
-
await this.
|
|
16
|
+
async initializeSourceRecordsCollection(replicationStreamId, sourceTableId, session) {
|
|
17
|
+
await this.sourceRecords(replicationStreamId, sourceTableId).createIndex({
|
|
15
18
|
pending_delete: 1
|
|
16
19
|
}, {
|
|
17
20
|
partialFilterExpression: { pending_delete: { $exists: true } },
|
|
18
|
-
name: 'pending_delete'
|
|
21
|
+
name: 'pending_delete',
|
|
22
|
+
session
|
|
19
23
|
});
|
|
20
24
|
}
|
|
21
25
|
commonSourceTables(replicationStreamId) {
|
|
22
|
-
return this.
|
|
26
|
+
return this.sourceTables(replicationStreamId);
|
|
23
27
|
}
|
|
24
|
-
|
|
28
|
+
bucketState(replicationStreamId) {
|
|
25
29
|
return this.db.collection(`bucket_state_${replicationStreamId}`);
|
|
26
30
|
}
|
|
27
|
-
|
|
31
|
+
parameterIndex(replicationStreamId, indexId) {
|
|
28
32
|
return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
|
|
29
33
|
}
|
|
30
|
-
|
|
34
|
+
sourceTables(replicationStreamId) {
|
|
31
35
|
return this.db.collection(this.sourceTableCollectionName(replicationStreamId));
|
|
32
36
|
}
|
|
37
|
+
bucketData(replicationStreamId, definitionId) {
|
|
38
|
+
return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`);
|
|
39
|
+
}
|
|
40
|
+
listBucketDataCollections(replicationStreamId) {
|
|
41
|
+
return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
|
|
42
|
+
}
|
|
43
|
+
async listParameterIndexCollections(replicationStreamId) {
|
|
44
|
+
const prefix = `parameter_index_${replicationStreamId}_`;
|
|
45
|
+
const collections = await this.listCollectionsByPrefix(prefix);
|
|
46
|
+
return collections.map((collection) => ({
|
|
47
|
+
collection,
|
|
48
|
+
indexId: collection.collectionName.slice(prefix.length)
|
|
49
|
+
}));
|
|
50
|
+
}
|
|
33
51
|
async initializeStreamStorage(replicationStreamId) {
|
|
34
|
-
const sourceTables = this.
|
|
35
|
-
const bucketState = this.
|
|
52
|
+
const sourceTables = this.sourceTables(replicationStreamId);
|
|
53
|
+
const bucketState = this.bucketState(replicationStreamId);
|
|
36
54
|
await sourceTables.createIndex({
|
|
37
55
|
connection_id: 1,
|
|
38
56
|
schema_name: 1,
|
|
@@ -54,21 +72,5 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
|
54
72
|
'estimate_since_compact.count': -1
|
|
55
73
|
}, { name: 'dirty_count' });
|
|
56
74
|
}
|
|
57
|
-
bucketDataV3(replicationStreamId, definitionId) {
|
|
58
|
-
return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`);
|
|
59
|
-
}
|
|
60
|
-
listBucketDataCollectionsV3(replicationStreamId) {
|
|
61
|
-
return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
|
|
62
|
-
}
|
|
63
|
-
async listParameterIndexCollectionsV3(replicationStreamId) {
|
|
64
|
-
const prefix = `parameter_index_${replicationStreamId}_`;
|
|
65
|
-
const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
|
|
66
|
-
return collections
|
|
67
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
68
|
-
.map((collection) => ({
|
|
69
|
-
collection: this.db.collection(collection.name),
|
|
70
|
-
indexId: collection.name.slice(prefix.length)
|
|
71
|
-
}));
|
|
72
|
-
}
|
|
73
75
|
}
|
|
74
76
|
//# sourceMappingURL=VersionedPowerSyncMongoV3.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAWvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,IAAI,qBAAqB;QACvB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,aAAa,CAAC,CAAC;IACjE,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAWvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,YACE,QAAsE,EACtE,aAA2E;QAE3E,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,qBAAqB;QACvB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,aAAa,CAAC,CAAC;IACjE,CAAC;IAED,aAAa,CAAC,mBAA2B,EAAE,aAA6B;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,cAAc,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,mBAA2B;QAC3D,OAAO,IAAI,CAAC,uBAAuB,CAAwB,kBAAkB,mBAAmB,GAAG,CAAC,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,mBAA2B,EAC3B,aAA6B,EAC7B,OAA6B;QAE7B,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,WAAW,CACtE;YACE,cAAc,EAAE,CAAC;SAClB,EACD;YACE,uBAAuB,EAAE,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC9D,IAAI,EAAE,gBAAgB;YACtB,OAAO;SACR,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,mBAA2B;QAC5C,OAAO,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAuD,CAAC;IACtG,CAAC;IAED,WAAW,CAAC,mBAA2B;QACrC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,cAAc,CAAC,mBAA2B,EAAE,OAAyB;QACnE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,YAAY,CAAC,mBAA2B;QACtC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,UAAU,CAAC,mBAA2B,EAAE,YAAgC;QACtE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,eAAe,mBAAmB,IAAI,YAAY,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,IAAI,CAAC,uBAAuB,CAAC,eAAe,mBAAmB,GAAG,CAAC,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,mBAA2B;QAC7D,MAAM,MAAM,GAAG,mBAAmB,mBAAmB,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAE/D,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACtC,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SACxD,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,mBAA2B;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;SACf,EACD;YACE,IAAI,EAAE,eAAe;SACtB,CACF,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,qBAAqB,EAAE,CAAC;SACzB,EACD;YACE,uBAAuB,EAAE,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACrE,IAAI,EAAE,uBAAuB;SAC9B,CACF,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
2
|
+
import { BucketDataDocumentV3 } from './models.js';
|
|
3
|
+
export declare function serializeBucketData(bucket: string, operations: BucketDataDoc[]): BucketDataDocumentV3;
|
|
4
|
+
export declare function loadBucketDataDocument(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): Generator<BucketDataDoc>;
|