@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -13,23 +13,163 @@ export class MongoWriteCheckpointAPI {
|
|
|
13
13
|
setWriteCheckpointMode(mode) {
|
|
14
14
|
this._mode = mode;
|
|
15
15
|
}
|
|
16
|
-
async
|
|
16
|
+
async createManagedWriteCheckpoints(checkpoints) {
|
|
17
17
|
if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
|
|
18
18
|
throw new framework.ServiceAssertionError(`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`);
|
|
19
19
|
}
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
|
|
21
|
+
if (uniqueCheckpoints.length == 0) {
|
|
22
|
+
return { writeCheckpoints: new Map(), shouldAdvance: false };
|
|
23
|
+
}
|
|
24
|
+
let shouldAdvance = false;
|
|
25
|
+
const writeCheckpoints = new Map();
|
|
26
|
+
const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
|
|
27
|
+
const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
|
|
28
|
+
if (generatedCheckpoints.length == 1) {
|
|
29
|
+
// For the common case of a single checkpoint, we can do this in a single request.
|
|
30
|
+
const { user_id, heads: lsns } = generatedCheckpoints[0];
|
|
31
|
+
const doc = await this.db.write_checkpoints.findOneAndUpdate({
|
|
32
|
+
user_id
|
|
33
|
+
}, {
|
|
34
|
+
$set: {
|
|
35
|
+
lsns,
|
|
36
|
+
processed_at_lsn: null
|
|
37
|
+
},
|
|
38
|
+
// Unset rather than set to null, to keep the document out of the
|
|
39
|
+
// partial checkpoint_requested_at index used for retention cleanup.
|
|
40
|
+
$unset: {
|
|
41
|
+
checkpoint_requested_at: 1
|
|
42
|
+
},
|
|
43
|
+
$inc: {
|
|
44
|
+
client_id: 1n
|
|
45
|
+
}
|
|
46
|
+
}, { upsert: true, returnDocument: 'after' });
|
|
47
|
+
// Generated checkpoints always reset processed_at_lsn, so they are pending
|
|
48
|
+
// and the source marker must be forced.
|
|
49
|
+
shouldAdvance = true;
|
|
50
|
+
writeCheckpoints.set(doc.user_id, doc.client_id);
|
|
51
|
+
}
|
|
52
|
+
else if (generatedCheckpoints.length > 1) {
|
|
53
|
+
// For more than one generated checkpoint, this gives a constant 2 requests.
|
|
54
|
+
await this.db.write_checkpoints.bulkWrite(generatedCheckpoints.map(({ user_id, heads: lsns }) => ({
|
|
55
|
+
updateOne: {
|
|
56
|
+
filter: { user_id },
|
|
57
|
+
update: {
|
|
58
|
+
$set: {
|
|
59
|
+
lsns,
|
|
60
|
+
processed_at_lsn: null
|
|
61
|
+
},
|
|
62
|
+
$unset: {
|
|
63
|
+
checkpoint_requested_at: 1
|
|
64
|
+
},
|
|
65
|
+
$inc: {
|
|
66
|
+
client_id: 1n
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
upsert: true
|
|
70
|
+
}
|
|
71
|
+
})));
|
|
72
|
+
const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
|
|
73
|
+
const docs = await this.db.write_checkpoints
|
|
74
|
+
.find({
|
|
75
|
+
user_id: { $in: userIds }
|
|
76
|
+
}, {
|
|
77
|
+
projection: {
|
|
78
|
+
user_id: 1,
|
|
79
|
+
client_id: 1
|
|
80
|
+
}
|
|
81
|
+
})
|
|
82
|
+
.toArray();
|
|
83
|
+
shouldAdvance = true;
|
|
84
|
+
for (const doc of docs) {
|
|
85
|
+
writeCheckpoints.set(doc.user_id, doc.client_id);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (suppliedCheckpoints.length > 0) {
|
|
89
|
+
const suppliedResult = await this.createSuppliedManagedWriteCheckpoints(suppliedCheckpoints);
|
|
90
|
+
shouldAdvance ||= suppliedResult.shouldAdvance;
|
|
91
|
+
for (const [userId, writeCheckpoint] of suppliedResult.writeCheckpoints) {
|
|
92
|
+
writeCheckpoints.set(userId, writeCheckpoint);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return { writeCheckpoints, shouldAdvance };
|
|
96
|
+
}
|
|
97
|
+
async createSuppliedManagedWriteCheckpoints(checkpoints) {
|
|
98
|
+
const checkpointRequestedAt = new Date();
|
|
99
|
+
await this.db.write_checkpoints.bulkWrite(checkpoints.map((checkpoint) => {
|
|
100
|
+
const { user_id, heads: lsns } = checkpoint;
|
|
101
|
+
const checkpointRequestId = checkpoint.checkpoint_request_id;
|
|
102
|
+
// Supplied request ids are monotonic. updateMany preserves the existing
|
|
103
|
+
// upsert behavior for first checkpoint requests, while also advancing
|
|
104
|
+
// every duplicate row for the user_id that is lower than this request id.
|
|
105
|
+
const shouldApplyRequestId = {
|
|
106
|
+
$or: [
|
|
107
|
+
{ $eq: [{ $ifNull: ['$client_id', null] }, null] },
|
|
108
|
+
{ $lt: ['$client_id', { $literal: checkpointRequestId }] }
|
|
109
|
+
]
|
|
110
|
+
};
|
|
111
|
+
// Equal retries keep their original source head but refresh retention.
|
|
112
|
+
const shouldRefreshRequest = {
|
|
113
|
+
$or: [
|
|
114
|
+
{ $eq: [{ $ifNull: ['$client_id', null] }, null] },
|
|
115
|
+
{ $lte: ['$client_id', { $literal: checkpointRequestId }] }
|
|
116
|
+
]
|
|
117
|
+
};
|
|
118
|
+
return {
|
|
119
|
+
updateMany: {
|
|
120
|
+
filter: { user_id },
|
|
121
|
+
update: [
|
|
122
|
+
{
|
|
123
|
+
$set: {
|
|
124
|
+
user_id,
|
|
125
|
+
client_id: {
|
|
126
|
+
$cond: [shouldApplyRequestId, { $literal: checkpointRequestId }, '$client_id']
|
|
127
|
+
},
|
|
128
|
+
lsns: {
|
|
129
|
+
$cond: [shouldApplyRequestId, { $literal: lsns }, '$lsns']
|
|
130
|
+
},
|
|
131
|
+
processed_at_lsn: {
|
|
132
|
+
$cond: [shouldApplyRequestId, null, '$processed_at_lsn']
|
|
133
|
+
},
|
|
134
|
+
checkpoint_requested_at: {
|
|
135
|
+
$cond: [shouldRefreshRequest, { $literal: checkpointRequestedAt }, '$checkpoint_requested_at']
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
],
|
|
140
|
+
upsert: true
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}));
|
|
144
|
+
// Fetch the final ids separately so stale requests can still return the
|
|
145
|
+
// checkpoint currently stored. If duplicate rows already exist for a
|
|
146
|
+
// user_id, use the highest stored id instead of whichever document MongoDB
|
|
147
|
+
// happens to return first.
|
|
148
|
+
const userIds = checkpoints.map((checkpoint) => checkpoint.user_id);
|
|
149
|
+
const docs = await this.db.write_checkpoints
|
|
150
|
+
.find({
|
|
151
|
+
user_id: { $in: userIds }
|
|
23
152
|
}, {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
$inc: {
|
|
29
|
-
client_id: 1n
|
|
153
|
+
projection: {
|
|
154
|
+
user_id: 1,
|
|
155
|
+
client_id: 1,
|
|
156
|
+
processed_at_lsn: 1
|
|
30
157
|
}
|
|
31
|
-
}
|
|
32
|
-
|
|
158
|
+
})
|
|
159
|
+
.toArray();
|
|
160
|
+
const maxDocsByUser = new Map();
|
|
161
|
+
for (const doc of docs) {
|
|
162
|
+
const existing = maxDocsByUser.get(doc.user_id);
|
|
163
|
+
if (existing == null ||
|
|
164
|
+
doc.client_id > existing.client_id ||
|
|
165
|
+
(doc.client_id == existing.client_id && existing.processed_at_lsn != null && doc.processed_at_lsn == null)) {
|
|
166
|
+
maxDocsByUser.set(doc.user_id, doc);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return {
|
|
170
|
+
writeCheckpoints: new Map([...maxDocsByUser].map(([userId, doc]) => [userId, doc.client_id])),
|
|
171
|
+
shouldAdvance: [...maxDocsByUser.values()].some((doc) => doc.processed_at_lsn == null)
|
|
172
|
+
};
|
|
33
173
|
}
|
|
34
174
|
async lastWriteCheckpoint(filters) {
|
|
35
175
|
switch (this.writeCheckpointMode) {
|
|
@@ -130,18 +270,25 @@ export async function batchCreateCustomWriteCheckpoints(db, session, checkpoints
|
|
|
130
270
|
if (checkpoints.length == 0) {
|
|
131
271
|
return;
|
|
132
272
|
}
|
|
133
|
-
await db.custom_write_checkpoints.bulkWrite(checkpoints.map((checkpointOptions) =>
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
},
|
|
143
|
-
upsert: true
|
|
273
|
+
await db.custom_write_checkpoints.bulkWrite(checkpoints.map((checkpointOptions) => {
|
|
274
|
+
const set = {
|
|
275
|
+
checkpoint: checkpointOptions.checkpoint,
|
|
276
|
+
sync_rules_id: checkpointOptions.sync_rules_id,
|
|
277
|
+
op_id: opId
|
|
278
|
+
};
|
|
279
|
+
if (checkpointOptions.checkpoint_requested_at != null) {
|
|
280
|
+
set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
|
|
144
281
|
}
|
|
145
|
-
|
|
282
|
+
return {
|
|
283
|
+
updateOne: {
|
|
284
|
+
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
285
|
+
update: {
|
|
286
|
+
$set: set,
|
|
287
|
+
...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
|
|
288
|
+
},
|
|
289
|
+
upsert: true
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
}), { session });
|
|
146
293
|
}
|
|
147
294
|
//# sourceMappingURL=MongoWriteCheckpointAPI.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAS7F,MAAM,OAAO,uBAAuB;IACzB,EAAE,CAA0B;IAC7B,KAAK,CAA8B;IAE3C,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAS7F,MAAM,OAAO,uBAAuB;IACzB,EAAE,CAA0B;IAC7B,KAAK,CAA8B;IAE3C,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,6BAA6B,CACjC,WAAoD;QAEpD,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,iBAAiB,GAAG,OAAO,CAAC,6BAA6B,CAAC,WAAW,CAAC,CAAC;QAC7E,IAAI,iBAAiB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,gBAAgB,EAAE,IAAI,GAAG,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;QAED,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACnD,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,qBAAqB,IAAI,IAAI,CAAC,CAAC;QAChH,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,qBAAqB,IAAI,IAAI,CAAC,CAAC;QAE/G,IAAI,oBAAoB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACrC,kFAAkF;YAClF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;gBACE,OAAO;aACR,EACD;gBACE,IAAI,EAAE;oBACJ,IAAI;oBACJ,gBAAgB,EAAE,IAAI;iBACvB;gBACD,iEAAiE;gBACjE,oEAAoE;gBACpE,MAAM,EAAE;oBACN,uBAAuB,EAAE,CAAC;iBAC3B;gBACD,IAAI,EAAE;oBACJ,SAAS,EAAE,EAAE;iBACd;aACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;YAEF,2EAA2E;YAC3E,wCAAwC;YACxC,aAAa,GAAG,IAAI,CAAC;YACrB,gBAAgB,CAAC,GAAG,CAAC,GAAI,CAAC,OAAO,EAAE,GAAI,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,4EAA4E;YAC5E,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CACvC,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtD,SAAS,EAAE;oBACT,MAAM,EAAE,EAAE,OAAO,EAAE;oBACnB,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,IAAI;4BACJ,gBAAgB,EAAE,IAAI;yBACvB;wBACD,MAAM,EAAE;4BACN,uBAAuB,EAAE,CAAC;yBAC3B;wBACD,IAAI,EAAE;4BACJ,SAAS,EAAE,EAAE;yBACd;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC,CACJ,CAAC;YAEF,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAC7E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;iBACzC,IAAI,CACH;gBACE,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;aAC1B,EACD;gBACE,UAAU,EAAE;oBACV,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,CAAC;iBACb;aACF,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,aAAa,GAAG,IAAI,CAAC;YACrB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qCAAqC,CAAC,mBAAmB,CAAC,CAAC;YAC7F,aAAa,KAAK,cAAc,CAAC,aAAa,CAAC;YAC/C,KAAK,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;gBACxE,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,CAAC;IAC7C,CAAC;IAEO,KAAK,CAAC,qCAAqC,CAAC,WAAoD;QACtG,MAAM,qBAAqB,GAAG,IAAI,IAAI,EAAE,CAAC;QAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CACvC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC7B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;YAC5C,MAAM,mBAAmB,GAAG,UAAU,CAAC,qBAAsB,CAAC;YAE9D,wEAAwE;YACxE,sEAAsE;YACtE,0EAA0E;YAC1E,MAAM,oBAAoB,GAAG;gBAC3B,GAAG,EAAE;oBACH,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;oBAClD,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,EAAE;iBAC3D;aACF,CAAC;YACF,uEAAuE;YACvE,MAAM,oBAAoB,GAAG;gBAC3B,GAAG,EAAE;oBACH,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;oBAClD,EAAE,IAAI,EAAE,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,CAAC,EAAE;iBAC5D;aACF,CAAC;YAEF,OAAO;gBACL,UAAU,EAAE;oBACV,MAAM,EAAE,EAAE,OAAO,EAAE;oBACnB,MAAM,EAAE;wBACN;4BACE,IAAI,EAAE;gCACJ,OAAO;gCACP,SAAS,EAAE;oCACT,KAAK,EAAE,CAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE,EAAE,YAAY,CAAC;iCAC/E;gCACD,IAAI,EAAE;oCACJ,KAAK,EAAE,CAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC;iCAC3D;gCACD,gBAAgB,EAAE;oCAChB,KAAK,EAAE,CAAC,oBAAoB,EAAE,IAAI,EAAE,mBAAmB,CAAC;iCACzD;gCACD,uBAAuB,EAAE;oCACvB,KAAK,EAAE,CAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,qBAAqB,EAAE,EAAE,0BAA0B,CAAC;iCAC/F;6BACF;yBACF;qBACF;oBACD,MAAM,EAAE,IAAI;iBACb;aACF,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,wEAAwE;QACxE,qEAAqE;QACrE,2EAA2E;QAC3E,2BAA2B;QAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzC,IAAI,CACH;YACE,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;SAC1B,EACD;YACE,UAAU,EAAE;gBACV,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,CAAC;gBACZ,gBAAgB,EAAE,CAAC;aACpB;SACF,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,aAAa,GAAG,IAAI,GAAG,EAAiC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAChD,IACE,QAAQ,IAAI,IAAI;gBAChB,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS;gBAClC,CAAC,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,gBAAgB,IAAI,IAAI,IAAI,GAAG,CAAC,gBAAgB,IAAI,IAAI,CAAC,EAC1G,CAAC;gBACD,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,OAAO;YACL,gBAAgB,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7F,aAAa,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI,CAAC;SACvF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,yEAAyE,CAC1E,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,OAAoC;QAClE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,OAAO,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,CAAC;YACvD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,OAAO,IAAI,CAAC,gCAAgC,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC;YACzE,OAAO;YACP,aAAa;SACd,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,UAAU,IAAI,IAAI,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAAoC;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aAC5C,IAAI,CACH;YACE,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE;SACxF,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,2DAA2D;YAC3D,+DAA+D;YAC/D,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAAC,OAAoC;QAChF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB;aACnD,IAAI,CACH;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE;SAC3F,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,2DAA2D;YAC3D,+DAA+D;YAC/D,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAA2B,EAC3B,OAA4B,EAC5B,WAAmD,EACnD,IAAkB;IAElB,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CACzC,WAAW,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE;QACpC,MAAM,GAAG,GAA4B;YACnC,UAAU,EAAE,iBAAiB,CAAC,UAAU;YACxC,aAAa,EAAE,iBAAiB,CAAC,aAAa;YAC9C,KAAK,EAAE,IAAI;SACZ,CAAC;QACF,IAAI,iBAAiB,CAAC,uBAAuB,IAAI,IAAI,EAAE,CAAC;YACtD,GAAG,CAAC,uBAAuB,GAAG,iBAAiB,CAAC,uBAAuB,CAAC;QAC1E,CAAC;QAED,OAAO;YACL,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,aAAa,EAAE;gBAC9F,MAAM,EAAE;oBACN,IAAI,EAAE,GAAG;oBACT,GAAG,CAAC,iBAAiB,CAAC,uBAAuB,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,uBAAuB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACzG;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;IACJ,CAAC,CAAC,EACF,EAAE,OAAO,EAAE,CACZ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
export interface MongoSyncBucketStorageCheckpoint {
|
|
5
|
+
checkpoint: InternalOpId;
|
|
6
|
+
snapshotTime: bson.Timestamp;
|
|
7
|
+
clusterTime: mongo.ClusterTime;
|
|
8
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MongoSyncBucketStorageCheckpoint.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts"],"names":[],"mappings":""}
|
|
@@ -2,13 +2,14 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
import { Logger } from '@powersync/lib-services-framework';
|
|
5
|
-
import { InternalOpId, storage } from '@powersync/service-core';
|
|
6
|
-
import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
|
|
5
|
+
import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
|
|
7
6
|
import { MongoIdSequence } from '../MongoIdSequence.js';
|
|
8
7
|
import type { VersionedPowerSyncMongo } from '../db.js';
|
|
9
8
|
import { TaggedBucketParameterDocument } from '../models.js';
|
|
9
|
+
import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
|
|
10
10
|
import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
|
|
11
11
|
import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
|
|
12
|
+
export declare const DEFAULT_INLINE_THRESHOLD_BYTES: number;
|
|
12
13
|
export interface SaveBucketDataOptions {
|
|
13
14
|
op_seq: MongoIdSequence;
|
|
14
15
|
sourceKey: storage.ReplicaId;
|
|
@@ -32,6 +33,8 @@ export interface UpsertCurrentDataOptions {
|
|
|
32
33
|
}
|
|
33
34
|
export interface PersistedBatchOptions {
|
|
34
35
|
logger?: Logger;
|
|
36
|
+
objectStorage?: ObjectStorage;
|
|
37
|
+
inlineThresholdBytes?: number;
|
|
35
38
|
}
|
|
36
39
|
/**
|
|
37
40
|
* Keeps track of bulkwrite operations within a transaction.
|
|
@@ -47,6 +50,8 @@ export declare abstract class PersistedBatch {
|
|
|
47
50
|
bucketData: BucketDataDoc[];
|
|
48
51
|
bucketParameters: TaggedBucketParameterDocument[];
|
|
49
52
|
bucketStates: Map<string, BucketStateUpdate>;
|
|
53
|
+
protected readonly objectStorage?: ObjectStorage;
|
|
54
|
+
protected readonly inlineThresholdBytes: number;
|
|
50
55
|
/**
|
|
51
56
|
* For debug logging only.
|
|
52
57
|
*/
|
|
@@ -22,6 +22,7 @@ const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
|
|
|
22
22
|
* This has an effect on error message size in some cases.
|
|
23
23
|
*/
|
|
24
24
|
const MAX_TRANSACTION_DOC_COUNT = 2_000;
|
|
25
|
+
export const DEFAULT_INLINE_THRESHOLD_BYTES = 16 * 1024;
|
|
25
26
|
/**
|
|
26
27
|
* Keeps track of bulkwrite operations within a transaction.
|
|
27
28
|
*
|
|
@@ -36,6 +37,8 @@ export class PersistedBatch {
|
|
|
36
37
|
bucketData = [];
|
|
37
38
|
bucketParameters = [];
|
|
38
39
|
bucketStates = new Map();
|
|
40
|
+
objectStorage;
|
|
41
|
+
inlineThresholdBytes = DEFAULT_INLINE_THRESHOLD_BYTES;
|
|
39
42
|
/**
|
|
40
43
|
* For debug logging only.
|
|
41
44
|
*/
|
|
@@ -50,6 +53,10 @@ export class PersistedBatch {
|
|
|
50
53
|
this.mapping = mapping;
|
|
51
54
|
this.currentSize = writtenSize;
|
|
52
55
|
this.logger = options?.logger ?? defaultLogger;
|
|
56
|
+
this.objectStorage = options?.objectStorage;
|
|
57
|
+
if (options?.inlineThresholdBytes != null) {
|
|
58
|
+
this.inlineThresholdBytes = options.inlineThresholdBytes;
|
|
59
|
+
}
|
|
53
60
|
}
|
|
54
61
|
saveBucketData(options) {
|
|
55
62
|
const remaining_buckets = new Map();
|
|
@@ -60,6 +67,11 @@ export class PersistedBatch {
|
|
|
60
67
|
id: bucket.id,
|
|
61
68
|
table: bucket.table
|
|
62
69
|
};
|
|
70
|
+
if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId)) {
|
|
71
|
+
// The bucket definition is not active anymore.
|
|
72
|
+
// We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
63
75
|
remaining_buckets.set(currentBucketKey(mapped), mapped);
|
|
64
76
|
}
|
|
65
77
|
const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAQ9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAgCxD;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IAoBb;IACA;IACA;IArBrB,MAAM,CAAS;IACf,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEtC,aAAa,CAAiB;IAC9B,oBAAoB,GAAW,8BAA8B,CAAC;IAEjF;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAAgC,EACnD,WAAmB,EACnB,OAA+B;QAJZ,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAyB;QAInD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,OAAO,EAAE,oBAAoB,IAAI,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACzD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnG,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IA6BD,IAAc,eAAe;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAES,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAC5G,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAES,gBAAgB,CAAC,OAU1B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAES,mBAAmB,CAAC,OAQ7B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,IAAI,CAAC,gBAAgB,IAAI,yBAAyB;YAClD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA0C;QAClF,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,IAAI,CAAC,eAAe,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC/D,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,iBAAiB,EAAE,IAAI,CAAC,eAAe;wBACvC,oBAAoB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBAClD,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;wBACzC,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,IAAI,CAAC,eAAe,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC/D,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,iBAAiB,EAAE,IAAI,CAAC,eAAe;wBACvC,oBAAoB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBAClD,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;qBAC1C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -48,7 +48,7 @@ export class BaseVersionedPowerSyncMongo {
|
|
|
48
48
|
return this.upstream.sourceTableCollectionName(replicationStreamId);
|
|
49
49
|
}
|
|
50
50
|
async listCollectionsByPrefix(prefix) {
|
|
51
|
-
const collections = await this.db.listCollections({ name:
|
|
51
|
+
const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
|
|
52
52
|
return collections
|
|
53
53
|
.filter((collection) => collection.name.startsWith(prefix))
|
|
54
54
|
.map((collection) => this.db.collection(collection.name));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;CAKF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { storage } from '@powersync/service-core';
|
|
2
2
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
3
|
-
import {
|
|
3
|
+
import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
|
|
4
4
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
|
|
5
5
|
export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
|
|
6
6
|
export type { MongoSyncBucketStorage };
|
|
7
|
-
export declare function createMongoSyncBucketStorage(factory: MongoBucketStorage,
|
|
7
|
+
export declare function createMongoSyncBucketStorage(factory: MongoBucketStorage, replicationStreamId: number, replicationStream: MongoPersistedReplicationStream, replicationStreamName: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions): MongoSyncBucketStorage;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
|
|
2
2
|
import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
|
|
3
|
-
export function createMongoSyncBucketStorage(factory,
|
|
4
|
-
if (
|
|
5
|
-
return new MongoSyncBucketStorageV3(factory,
|
|
3
|
+
export function createMongoSyncBucketStorage(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
|
|
4
|
+
if (replicationStream.getStorageConfig().incrementalReprocessing) {
|
|
5
|
+
return new MongoSyncBucketStorageV3(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
6
6
|
}
|
|
7
|
-
return new MongoSyncBucketStorageV1(factory,
|
|
7
|
+
return new MongoSyncBucketStorageV1(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
8
8
|
}
|
|
9
9
|
//# sourceMappingURL=createMongoSyncBucketStorage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createMongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/implementation/createMongoSyncBucketStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAM5E,MAAM,UAAU,4BAA4B,CAC1C,OAA2B,EAC3B,
|
|
1
|
+
{"version":3,"file":"createMongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/implementation/createMongoSyncBucketStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAM5E,MAAM,UAAU,4BAA4B,CAC1C,OAA2B,EAC3B,mBAA2B,EAC3B,iBAAkD,EAClD,qBAA6B,EAC7B,mBAA4D,EAC5D,OAAsC;IAEtC,IAAI,iBAAiB,CAAC,gBAAgB,EAAE,CAAC,uBAAuB,EAAE,CAAC;QACjE,OAAO,IAAI,wBAAwB,CACjC,OAAO,EACP,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,OAAO,CACR,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,wBAAwB,CACjC,OAAO,EACP,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,OAAO,CACR,CAAC;AACJ,CAAC"}
|
|
@@ -41,7 +41,7 @@ export class PowerSyncMongo {
|
|
|
41
41
|
this.connection_report_events = this.db.collection('connection_report_events');
|
|
42
42
|
}
|
|
43
43
|
versioned(storageConfig) {
|
|
44
|
-
if (storageConfig.
|
|
44
|
+
if (storageConfig.version >= storage.STORAGE_VERSION_3) {
|
|
45
45
|
return new VersionedPowerSyncMongoV3(this, storageConfig);
|
|
46
46
|
}
|
|
47
47
|
return new VersionedPowerSyncMongoV1(this, storageConfig);
|
|
@@ -51,16 +51,13 @@ export class PowerSyncMongo {
|
|
|
51
51
|
*/
|
|
52
52
|
async listBucketDataCollectionsV3(groupId) {
|
|
53
53
|
const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
|
|
54
|
-
|
|
55
|
-
return collections
|
|
56
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
57
|
-
.map((collection) => this.db.collection(collection.name));
|
|
54
|
+
return this.collectionsByPrefix(prefix);
|
|
58
55
|
}
|
|
59
56
|
/**
|
|
60
57
|
* Not safe for user-provided prefix - only for hardcoded values.
|
|
61
58
|
*/
|
|
62
59
|
async collectionsByPrefix(prefix) {
|
|
63
|
-
const collections = await this.db.listCollections({ name:
|
|
60
|
+
const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
|
|
64
61
|
return collections
|
|
65
62
|
.filter((collection) => collection.name.startsWith(prefix))
|
|
66
63
|
.map((collection) => this.db.collection(collection.name));
|
|
@@ -91,14 +88,12 @@ export class PowerSyncMongo {
|
|
|
91
88
|
return `source_table_${replicationStreamId}`;
|
|
92
89
|
}
|
|
93
90
|
async listSourceTableCollections(replicationStreamId) {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
const collections = await this.db.listCollections(
|
|
99
|
-
return collections
|
|
100
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
101
|
-
.map((collection) => this.db.collection(collection.name));
|
|
91
|
+
if (replicationStreamId == null) {
|
|
92
|
+
return this.collectionsByPrefix('source_table_');
|
|
93
|
+
}
|
|
94
|
+
const name = this.sourceTableCollectionName(replicationStreamId);
|
|
95
|
+
const collections = await this.db.listCollections({ name }, { nameOnly: true }).toArray();
|
|
96
|
+
return collections.map((c) => this.db.collection(c.name));
|
|
102
97
|
}
|
|
103
98
|
/**
|
|
104
99
|
* Clear all collections.
|
|
@@ -145,7 +140,14 @@ export class PowerSyncMongo {
|
|
|
145
140
|
* This is used in a similar way to the Postgres NOTIFY functionality.
|
|
146
141
|
*/
|
|
147
142
|
async notifyCheckpoint() {
|
|
148
|
-
await this.checkpoint_events.insertOne({}, {
|
|
143
|
+
await this.checkpoint_events.insertOne({}, {
|
|
144
|
+
forceServerObjectId: true,
|
|
145
|
+
// No correctness issue if this write is missed / rolled back.
|
|
146
|
+
// It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
|
|
147
|
+
writeConcern: {
|
|
148
|
+
w: 1
|
|
149
|
+
}
|
|
150
|
+
});
|
|
149
151
|
}
|
|
150
152
|
/**
|
|
151
153
|
* Only use in migrations and tests.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS9E,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAyC;IACnD,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAKD,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS9E,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAyC;IACnD,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAKD,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACvD,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAC,OAAgB;QAChD,MAAM,MAAM,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,OAAO,GAAG,CAAC;QAC5E,OAAO,IAAI,CAAC,mBAAmB,CAAuB,MAAM,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAmC,MAAc;QAChF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kCAAkC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gCAAgC;QACpC,OAAO,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QACpF,OAAO,kBAAkB,mBAAmB,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;IAChF,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,gBAAgB,mBAAmB,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,mBAA4B;QAE5B,IAAI,mBAAmB,IAAI,IAAI,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC;YACvE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC;YACzE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,+BAA+B,EAAE,EAAE,CAAC;YACtE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,0BAA0B,EAAE,EAAE,CAAC;YACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE;YAChD,mBAAmB,EAAE,IAAI;YACzB,8DAA8D;YAC9D,qGAAqG;YACrG,YAAY,EAAE;gBACZ,CAAC,EAAE,CAAC;aACL;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAOD,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -3,7 +3,7 @@ import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules
|
|
|
3
3
|
import { event_types } from '@powersync/service-types';
|
|
4
4
|
import * as bson from 'bson';
|
|
5
5
|
import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
|
|
6
|
-
import type {
|
|
6
|
+
import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js';
|
|
7
7
|
/**
|
|
8
8
|
* Replica id uniquely identifying a row on the source database.
|
|
9
9
|
*
|
|
@@ -59,6 +59,11 @@ export interface BucketDataProperties {
|
|
|
59
59
|
row_id?: string;
|
|
60
60
|
checksum: bigint;
|
|
61
61
|
data: string | null;
|
|
62
|
+
/**
|
|
63
|
+
* V1-only.
|
|
64
|
+
*
|
|
65
|
+
* V3 stores this on the BucketDataDocumentV3 instead of the individual ops.
|
|
66
|
+
*/
|
|
62
67
|
target_op?: bigint | null;
|
|
63
68
|
}
|
|
64
69
|
export interface BucketDataDocumentBase extends BucketDataProperties {
|
|
@@ -176,7 +181,7 @@ export interface SyncRuleDocumentBase {
|
|
|
176
181
|
} | null;
|
|
177
182
|
storage_version?: number;
|
|
178
183
|
}
|
|
179
|
-
export interface SyncRuleCheckpointFields
|
|
184
|
+
export interface SyncRuleCheckpointFields {
|
|
180
185
|
/**
|
|
181
186
|
* The last consistent checkpoint.
|
|
182
187
|
*
|
|
@@ -191,15 +196,6 @@ export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint |
|
|
|
191
196
|
* If set, no new checkpoints may be created < this value.
|
|
192
197
|
*/
|
|
193
198
|
no_checkpoint_before: string | null;
|
|
194
|
-
/**
|
|
195
|
-
* Goes together with no_checkpoint_before.
|
|
196
|
-
*
|
|
197
|
-
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
198
|
-
* then the checkpoint must be equal to this keepalive_op.
|
|
199
|
-
*
|
|
200
|
-
* This is a string in V1, bigint in V3.
|
|
201
|
-
*/
|
|
202
|
-
keepalive_op: TKeepaliveOp;
|
|
203
199
|
}
|
|
204
200
|
export interface StorageConfig extends storage.StorageVersionConfig {
|
|
205
201
|
/**
|
|
@@ -230,6 +226,11 @@ export interface CustomWriteCheckpointDocument {
|
|
|
230
226
|
* This is not unique - multiple write checkpoints can have the same op_id.
|
|
231
227
|
*/
|
|
232
228
|
op_id?: InternalOpId;
|
|
229
|
+
/**
|
|
230
|
+
* Set when this checkpoint was created from a client-supplied checkpoint
|
|
231
|
+
* request rather than a persistent custom write checkpoint.
|
|
232
|
+
*/
|
|
233
|
+
checkpoint_requested_at?: Date | null;
|
|
233
234
|
}
|
|
234
235
|
export interface WriteCheckpointDocument {
|
|
235
236
|
_id: bson.ObjectId;
|
|
@@ -242,6 +243,12 @@ export interface WriteCheckpointDocument {
|
|
|
242
243
|
* between two checkpoints.
|
|
243
244
|
*/
|
|
244
245
|
processed_at_lsn: string | null;
|
|
246
|
+
/**
|
|
247
|
+
* Set when this checkpoint was created from a client-supplied checkpoint request
|
|
248
|
+
* rather than a server-generated id. A missing/null value means this is not a
|
|
249
|
+
* checkpoint request.
|
|
250
|
+
*/
|
|
251
|
+
checkpoint_requested_at?: Date | null;
|
|
245
252
|
}
|
|
246
253
|
export interface InstanceDocument {
|
|
247
254
|
_id: string;
|
|
@@ -249,6 +256,6 @@ export interface InstanceDocument {
|
|
|
249
256
|
export interface ClientConnectionDocument extends event_types.ClientConnection {
|
|
250
257
|
}
|
|
251
258
|
export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
|
|
252
|
-
export type CommonCurrentBucket = CurrentBucket
|
|
259
|
+
export type CommonCurrentBucket = CurrentBucket;
|
|
253
260
|
export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
|
|
254
|
-
export type CommonSourceTableDocument = SourceTableDocumentV1 |
|
|
261
|
+
export type CommonSourceTableDocument = Pick<SourceTableDocumentV1, '_id' | 'snapshot_status'>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;AA2BD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAoJpD,MAAM,8BAA8B,GAAG,CAAC,CAAC;AACzC,MAAM,wCAAwC,GAAG,OAAO,CAAC,iBAAiB,CAAC;AAE3E,MAAM,UAAU,qBAAqB,CAAC,cAAsB;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,cAAc,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,OAAO;QACL,GAAG,UAAU;QACb,aAAa,EAAE,cAAc,IAAI,8BAA8B;QAC/D,uBAAuB,EAAE,cAAc,IAAI,wCAAwC;KACpF,CAAC;AACJ,CAAC"}
|