@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,69 +1,95 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
-
import { InternalOpId, storage } from '@powersync/service-core';
|
|
3
|
+
import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
|
|
4
4
|
import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
5
5
|
import * as bson from 'bson';
|
|
6
6
|
import { mongoTableId } from '../../../utils/util.js';
|
|
7
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
7
8
|
import {
|
|
8
9
|
BucketStateUpdate,
|
|
9
10
|
PersistedBatch,
|
|
11
|
+
PersistedBatchOptions,
|
|
10
12
|
SaveParameterDataOptions,
|
|
11
13
|
UpsertCurrentDataOptions
|
|
12
14
|
} from '../common/PersistedBatch.js';
|
|
13
|
-
import {
|
|
15
|
+
import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
|
|
16
|
+
import { serializeBucketData } from './bucket-format.js';
|
|
17
|
+
import { chunkBucketData } from './chunking.js';
|
|
14
18
|
import {
|
|
15
|
-
|
|
19
|
+
BucketDataDocumentV3,
|
|
16
20
|
BucketStateDocumentV3,
|
|
17
21
|
CurrentDataDocumentV3,
|
|
18
|
-
|
|
22
|
+
serializeParameterLookup,
|
|
19
23
|
SourceTableDocumentV3,
|
|
20
|
-
|
|
24
|
+
taggedBucketParameterDocumentToTagged
|
|
21
25
|
} from './models.js';
|
|
22
|
-
import {
|
|
26
|
+
import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
|
|
23
27
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
24
28
|
|
|
25
29
|
export class PersistedBatchV3 extends PersistedBatch {
|
|
26
|
-
declare protected readonly db: VersionedPowerSyncMongoV3;
|
|
27
|
-
|
|
28
30
|
currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
|
|
29
31
|
sourceTablePendingDeletes = new Map<string, InternalOpId>();
|
|
32
|
+
protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
|
|
33
|
+
|
|
34
|
+
declare protected readonly db: VersionedPowerSyncMongoV3;
|
|
35
|
+
|
|
36
|
+
constructor(
|
|
37
|
+
db: VersionedPowerSyncMongoV3,
|
|
38
|
+
group_id: number,
|
|
39
|
+
mapping: BucketDefinitionMapping,
|
|
40
|
+
writtenSize: number,
|
|
41
|
+
options?: PersistedBatchOptions
|
|
42
|
+
) {
|
|
43
|
+
super(db, group_id, mapping, writtenSize, options);
|
|
44
|
+
if (this.objectStorage) {
|
|
45
|
+
this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
30
48
|
|
|
31
|
-
|
|
49
|
+
// Abstract override from PersistedBatch (V3-specific error message)
|
|
50
|
+
|
|
51
|
+
protected override checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
|
|
32
52
|
if (definitionId == null) {
|
|
33
|
-
// This is required for V3 storage.
|
|
34
53
|
throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
|
|
35
54
|
}
|
|
36
55
|
return definitionId;
|
|
37
56
|
}
|
|
38
57
|
|
|
58
|
+
// Concrete implementations from PersistedBatchShared
|
|
59
|
+
|
|
39
60
|
protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId {
|
|
40
61
|
return this.mapping.bucketSourceId(bucketSource);
|
|
41
62
|
}
|
|
42
63
|
|
|
43
64
|
saveParameterData(data: SaveParameterDataOptions) {
|
|
44
65
|
const { sourceTable, sourceKey, evaluated } = data;
|
|
45
|
-
const remaining_lookups = new Map<string,
|
|
66
|
+
const remaining_lookups = new Map<string, SourceRecordLookupState>();
|
|
46
67
|
|
|
47
68
|
for (let lookup of data.existing_lookups) {
|
|
48
69
|
if (lookup.indexId == null) {
|
|
49
|
-
throw new ReplicationAssertionError('Expected
|
|
70
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
71
|
+
}
|
|
72
|
+
if (sourceTable.parameterLookupSourceIds != null && !sourceTable.parameterLookupSourceIds.has(lookup.indexId)) {
|
|
73
|
+
// The parameter index is not active anymore.
|
|
74
|
+
// We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
|
|
75
|
+
continue;
|
|
50
76
|
}
|
|
51
77
|
remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
|
|
52
78
|
}
|
|
53
79
|
|
|
54
80
|
for (let result of evaluated) {
|
|
55
81
|
const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
|
|
56
|
-
const binLookup =
|
|
82
|
+
const binLookup = serializeParameterLookup(result.lookup);
|
|
57
83
|
remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
|
|
58
84
|
|
|
59
85
|
const op_id = data.op_seq.next();
|
|
60
86
|
this.debugLastOpId = op_id;
|
|
61
|
-
const values
|
|
87
|
+
const values = {
|
|
62
88
|
_id: op_id,
|
|
63
89
|
key: {
|
|
64
90
|
t: mongoTableId(sourceTable.id),
|
|
65
91
|
k: sourceKey
|
|
66
|
-
}
|
|
92
|
+
},
|
|
67
93
|
lookup: binLookup,
|
|
68
94
|
bucket_parameters: result.bucketParameters
|
|
69
95
|
};
|
|
@@ -80,14 +106,14 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
80
106
|
this.debugLastOpId = op_id;
|
|
81
107
|
const indexId = lookup.indexId;
|
|
82
108
|
if (indexId == null) {
|
|
83
|
-
throw new ReplicationAssertionError('Expected
|
|
109
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
84
110
|
}
|
|
85
|
-
const values
|
|
111
|
+
const values = {
|
|
86
112
|
_id: op_id,
|
|
87
113
|
key: {
|
|
88
114
|
t: mongoTableId(sourceTable.id),
|
|
89
115
|
k: sourceKey
|
|
90
|
-
}
|
|
116
|
+
},
|
|
91
117
|
lookup: lookup.lookup,
|
|
92
118
|
bucket_parameters: []
|
|
93
119
|
};
|
|
@@ -125,8 +151,8 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
125
151
|
update: {
|
|
126
152
|
$set: {
|
|
127
153
|
data: null,
|
|
128
|
-
buckets: []
|
|
129
|
-
lookups: []
|
|
154
|
+
buckets: [],
|
|
155
|
+
lookups: [],
|
|
130
156
|
pending_delete: checkpointGreaterThan
|
|
131
157
|
}
|
|
132
158
|
},
|
|
@@ -146,7 +172,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
146
172
|
upsertCurrentData(values: UpsertCurrentDataOptions) {
|
|
147
173
|
const buckets = values.buckets.map((bucket) => {
|
|
148
174
|
if (bucket.definitionId == null) {
|
|
149
|
-
throw new ReplicationAssertionError('Expected
|
|
175
|
+
throw new ReplicationAssertionError('Expected bucket when incrementalReprocessing is enabled');
|
|
150
176
|
}
|
|
151
177
|
return {
|
|
152
178
|
def: bucket.definitionId,
|
|
@@ -157,7 +183,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
157
183
|
});
|
|
158
184
|
const lookups = values.lookups.map((lookup) => {
|
|
159
185
|
if (lookup.indexId == null) {
|
|
160
|
-
throw new ReplicationAssertionError('Expected
|
|
186
|
+
throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
|
|
161
187
|
}
|
|
162
188
|
return {
|
|
163
189
|
i: lookup.indexId,
|
|
@@ -189,26 +215,81 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
189
215
|
return this.currentData.length;
|
|
190
216
|
}
|
|
191
217
|
|
|
218
|
+
// Flush methods
|
|
219
|
+
|
|
192
220
|
protected async flushBucketData(session: mongo.ClientSession) {
|
|
193
|
-
const operationsByDefinition = new Map<BucketDefinitionId,
|
|
221
|
+
const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
|
|
194
222
|
for (const document of this.bucketData) {
|
|
195
223
|
const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
|
|
196
224
|
existing.push(document);
|
|
197
225
|
operationsByDefinition.set(document.bucketKey.definitionId, existing);
|
|
198
226
|
}
|
|
199
227
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
228
|
+
let uploadCount = 0;
|
|
229
|
+
const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
|
|
230
|
+
const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
|
|
231
|
+
const lifecycle = this.objectStorage
|
|
232
|
+
? new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage)
|
|
233
|
+
: undefined;
|
|
234
|
+
const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
|
|
235
|
+
|
|
236
|
+
for (const [bucket, ops] of operationsByBucket) {
|
|
237
|
+
for (const chunk of chunkBucketData(ops)) {
|
|
238
|
+
const serialized = serializeBucketData(bucket, chunk);
|
|
239
|
+
if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
|
|
240
|
+
createInserts.push(async () => ({
|
|
241
|
+
insertOne: {
|
|
242
|
+
document: serialized
|
|
243
|
+
}
|
|
244
|
+
}));
|
|
245
|
+
continue;
|
|
205
246
|
}
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
247
|
+
|
|
248
|
+
uploadCount += 1;
|
|
249
|
+
createInserts.push(async () => {
|
|
250
|
+
const minOp = chunk[0].o;
|
|
251
|
+
const maxOp = chunk[chunk.length - 1].o;
|
|
252
|
+
const { ops: bucketOps, ...metadata } = serialized;
|
|
253
|
+
const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
|
|
254
|
+
const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!);
|
|
255
|
+
return {
|
|
256
|
+
insertOne: {
|
|
257
|
+
document: {
|
|
258
|
+
...metadata,
|
|
259
|
+
storage_ref: {
|
|
260
|
+
path,
|
|
261
|
+
file_size: fileSize
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
});
|
|
210
267
|
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
return { definitionId, createInserts };
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
const createAllInserts = () =>
|
|
274
|
+
Promise.all(
|
|
275
|
+
plans.map(async ({ definitionId, createInserts }) => ({
|
|
276
|
+
definitionId,
|
|
277
|
+
inserts: await Promise.all(createInserts.map((createInsert) => createInsert()))
|
|
278
|
+
}))
|
|
211
279
|
);
|
|
280
|
+
|
|
281
|
+
// S3ObjectStorage applies one shared concurrency limit across all callers,
|
|
282
|
+
// so replication can schedule its uploads together without creating a
|
|
283
|
+
// separate limiter here.
|
|
284
|
+
const writes = await createAllInserts();
|
|
285
|
+
|
|
286
|
+
for (const { definitionId, inserts } of writes) {
|
|
287
|
+
if (inserts.length > 0) {
|
|
288
|
+
await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
|
|
289
|
+
session,
|
|
290
|
+
ordered: false
|
|
291
|
+
});
|
|
292
|
+
}
|
|
212
293
|
}
|
|
213
294
|
}
|
|
214
295
|
|
|
@@ -221,10 +302,10 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
221
302
|
}
|
|
222
303
|
|
|
223
304
|
for (const [indexId, documents] of operationsByIndex.entries()) {
|
|
224
|
-
await this.db.
|
|
305
|
+
await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
|
|
225
306
|
documents.map((document) => ({
|
|
226
307
|
insertOne: {
|
|
227
|
-
document:
|
|
308
|
+
document: taggedBucketParameterDocumentToTagged(document)
|
|
228
309
|
}
|
|
229
310
|
})),
|
|
230
311
|
{
|
|
@@ -260,12 +341,12 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
260
341
|
});
|
|
261
342
|
|
|
262
343
|
if (sourceTableUpdates.length > 0) {
|
|
263
|
-
await this.db.
|
|
344
|
+
await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
|
|
264
345
|
}
|
|
265
346
|
|
|
266
347
|
for (const operations of operationsBySourceTable.values()) {
|
|
267
348
|
const sourceTableId = operations[0]!.sourceTableId;
|
|
268
|
-
await this.db.
|
|
349
|
+
await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
|
|
269
350
|
operations.map((entry) => entry.operation),
|
|
270
351
|
{
|
|
271
352
|
session,
|
|
@@ -276,7 +357,7 @@ export class PersistedBatchV3 extends PersistedBatch {
|
|
|
276
357
|
}
|
|
277
358
|
|
|
278
359
|
protected async flushBucketStates(session: mongo.ClientSession) {
|
|
279
|
-
await this.db.
|
|
360
|
+
await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
|
|
280
361
|
session,
|
|
281
362
|
ordered: false
|
|
282
363
|
});
|
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
3
|
import { Logger } from '@powersync/lib-services-framework';
|
|
4
|
-
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { BucketDefinitionMapping, storage } from '@powersync/service-core';
|
|
5
5
|
import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
|
|
6
6
|
import * as bson from 'bson';
|
|
7
7
|
import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
|
|
8
|
-
import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
|
|
9
8
|
import { cacheKey } from '../OperationBatch.js';
|
|
10
9
|
import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
11
|
-
import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
|
|
12
10
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
13
|
-
import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js';
|
|
11
|
+
import { CurrentDataDocumentV3, serializeParameterLookup, SourceTableDocumentV3 } from './models.js';
|
|
14
12
|
|
|
15
13
|
export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
16
14
|
constructor(
|
|
17
15
|
private readonly db: VersionedPowerSyncMongoV3,
|
|
18
|
-
private readonly
|
|
16
|
+
private readonly replicationStreamId: number,
|
|
19
17
|
private readonly mapping: BucketDefinitionMapping
|
|
20
18
|
) {}
|
|
21
19
|
|
|
@@ -31,7 +29,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
31
29
|
mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] {
|
|
32
30
|
return paramEvaluated.map((entry) => ({
|
|
33
31
|
indexId: this.mapping.parameterLookupId(entry.lookup.source),
|
|
34
|
-
lookup:
|
|
32
|
+
lookup: serializeParameterLookup(entry.lookup)
|
|
35
33
|
}));
|
|
36
34
|
}
|
|
37
35
|
|
|
@@ -67,7 +65,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
67
65
|
_id: { $in: replicaIds as any[] }
|
|
68
66
|
} as unknown as mongo.Filter<CurrentDataDocumentV3>;
|
|
69
67
|
const sizeCursor: mongo.AggregationCursor<CurrentDataDocumentV3 & { size: number }> = this.db
|
|
70
|
-
.
|
|
68
|
+
.sourceRecords(this.replicationStreamId, sourceTableId)
|
|
71
69
|
.aggregate(
|
|
72
70
|
[
|
|
73
71
|
{
|
|
@@ -100,7 +98,9 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
100
98
|
const filter = {
|
|
101
99
|
_id: { $in: replicaIds as any[] }
|
|
102
100
|
} as unknown as mongo.Filter<CurrentDataDocumentV3>;
|
|
103
|
-
const cursor = this.db
|
|
101
|
+
const cursor = this.db
|
|
102
|
+
.sourceRecords(this.replicationStreamId, sourceTableId)
|
|
103
|
+
.find(filter, { session, projection });
|
|
104
104
|
for await (const doc of cursor.stream()) {
|
|
105
105
|
const loaded = this.createLoadedDocument(
|
|
106
106
|
sourceTableId,
|
|
@@ -120,7 +120,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
120
120
|
sourceTableId: bson.ObjectId,
|
|
121
121
|
limit: number
|
|
122
122
|
): Promise<LoadedSourceRecord[]> {
|
|
123
|
-
const cursor = this.db.
|
|
123
|
+
const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find(
|
|
124
124
|
{
|
|
125
125
|
pending_delete: { $exists: false }
|
|
126
126
|
},
|
|
@@ -145,7 +145,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
145
145
|
// ones have dirty deletes in source_tables.
|
|
146
146
|
|
|
147
147
|
const dirtySourceTables = await this.db
|
|
148
|
-
.
|
|
148
|
+
.sourceTables(this.replicationStreamId)
|
|
149
149
|
.find(
|
|
150
150
|
{
|
|
151
151
|
latest_pending_delete: { $exists: true }
|
|
@@ -159,7 +159,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
159
159
|
let deletedCount = 0;
|
|
160
160
|
const sourceTableUpdates: mongo.AnyBulkWriteOperation<SourceTableDocumentV3>[] = [];
|
|
161
161
|
for (const sourceTable of dirtySourceTables) {
|
|
162
|
-
const collection = this.db.
|
|
162
|
+
const collection = this.db.sourceRecords(this.replicationStreamId, sourceTable._id);
|
|
163
163
|
const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
|
|
164
164
|
deletedCount += result.deletedCount;
|
|
165
165
|
|
|
@@ -182,7 +182,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
|
|
|
182
182
|
}
|
|
183
183
|
|
|
184
184
|
if (sourceTableUpdates.length > 0) {
|
|
185
|
-
await this.db.
|
|
185
|
+
await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
|
|
186
186
|
}
|
|
187
187
|
if (deletedCount > 0) {
|
|
188
188
|
logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);
|
|
@@ -7,58 +7,90 @@ import {
|
|
|
7
7
|
BucketParameterDocumentV3,
|
|
8
8
|
BucketStateDocumentV3,
|
|
9
9
|
CurrentDataDocumentV3,
|
|
10
|
+
ObjectStorageDeletionMarker,
|
|
10
11
|
SourceTableDocumentV3,
|
|
11
12
|
SyncConfigDefinition
|
|
12
13
|
} from './models.js';
|
|
13
14
|
|
|
14
15
|
export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
16
|
+
constructor(
|
|
17
|
+
upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
|
|
18
|
+
storageConfig: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[1]
|
|
19
|
+
) {
|
|
20
|
+
super(upstream, storageConfig);
|
|
21
|
+
}
|
|
22
|
+
|
|
15
23
|
get syncConfigDefinitions() {
|
|
16
24
|
return this.db.collection<SyncConfigDefinition>('sync_config');
|
|
17
25
|
}
|
|
18
26
|
|
|
19
|
-
|
|
27
|
+
sourceRecords(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
|
|
20
28
|
const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
|
|
21
29
|
return this.db.collection<CurrentDataDocumentV3>(collectionName);
|
|
22
30
|
}
|
|
23
31
|
|
|
24
|
-
async
|
|
32
|
+
async listSourceRecordCollections(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
|
|
25
33
|
return this.listCollectionsByPrefix<CurrentDataDocumentV3>(`source_records_${replicationStreamId}_`);
|
|
26
34
|
}
|
|
27
35
|
|
|
28
|
-
async initializeSourceRecordsCollection(
|
|
29
|
-
|
|
36
|
+
async initializeSourceRecordsCollection(
|
|
37
|
+
replicationStreamId: number,
|
|
38
|
+
sourceTableId: mongo.ObjectId,
|
|
39
|
+
session?: mongo.ClientSession
|
|
40
|
+
) {
|
|
41
|
+
await this.sourceRecords(replicationStreamId, sourceTableId).createIndex(
|
|
30
42
|
{
|
|
31
43
|
pending_delete: 1
|
|
32
44
|
},
|
|
33
45
|
{
|
|
34
46
|
partialFilterExpression: { pending_delete: { $exists: true } },
|
|
35
|
-
name: 'pending_delete'
|
|
47
|
+
name: 'pending_delete',
|
|
48
|
+
session
|
|
36
49
|
}
|
|
37
50
|
);
|
|
38
51
|
}
|
|
39
52
|
|
|
40
53
|
commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
|
|
41
|
-
return this.
|
|
54
|
+
return this.sourceTables(replicationStreamId) as any as mongo.Collection<CommonSourceTableDocument>;
|
|
42
55
|
}
|
|
43
56
|
|
|
44
|
-
|
|
57
|
+
bucketState(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
|
|
45
58
|
return this.db.collection(`bucket_state_${replicationStreamId}`);
|
|
46
59
|
}
|
|
47
60
|
|
|
48
|
-
|
|
49
|
-
replicationStreamId: number,
|
|
50
|
-
indexId: ParameterIndexId
|
|
51
|
-
): mongo.Collection<BucketParameterDocumentV3> {
|
|
61
|
+
parameterIndex(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3> {
|
|
52
62
|
return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
|
|
53
63
|
}
|
|
54
64
|
|
|
55
|
-
|
|
65
|
+
sourceTables(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
|
|
56
66
|
return this.db.collection<SourceTableDocumentV3>(this.sourceTableCollectionName(replicationStreamId));
|
|
57
67
|
}
|
|
58
68
|
|
|
69
|
+
bucketData(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3> {
|
|
70
|
+
return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
pendingObjectStorageDeletes(replicationStreamId: number): mongo.Collection<ObjectStorageDeletionMarker> {
|
|
74
|
+
return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
listBucketDataCollections(replicationStreamId: number) {
|
|
78
|
+
return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async listParameterIndexCollections(replicationStreamId: number) {
|
|
82
|
+
const prefix = `parameter_index_${replicationStreamId}_`;
|
|
83
|
+
const collections = await this.listCollectionsByPrefix(prefix);
|
|
84
|
+
|
|
85
|
+
return collections.map((collection) => ({
|
|
86
|
+
collection,
|
|
87
|
+
indexId: collection.collectionName.slice(prefix.length)
|
|
88
|
+
}));
|
|
89
|
+
}
|
|
90
|
+
|
|
59
91
|
async initializeStreamStorage(replicationStreamId: number) {
|
|
60
|
-
const sourceTables = this.
|
|
61
|
-
const bucketState = this.
|
|
92
|
+
const sourceTables = this.sourceTables(replicationStreamId);
|
|
93
|
+
const bucketState = this.bucketState(replicationStreamId);
|
|
62
94
|
await sourceTables.createIndex(
|
|
63
95
|
{
|
|
64
96
|
connection_id: 1,
|
|
@@ -92,26 +124,4 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
|
|
|
92
124
|
{ name: 'dirty_count' }
|
|
93
125
|
);
|
|
94
126
|
}
|
|
95
|
-
|
|
96
|
-
bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) {
|
|
97
|
-
return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
listBucketDataCollectionsV3(replicationStreamId: number) {
|
|
101
|
-
return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
async listParameterIndexCollectionsV3(
|
|
105
|
-
replicationStreamId: number
|
|
106
|
-
): Promise<{ collection: mongo.Collection<BucketParameterDocumentV3>; indexId: ParameterIndexId }[]> {
|
|
107
|
-
const prefix = `parameter_index_${replicationStreamId}_`;
|
|
108
|
-
const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
|
|
109
|
-
|
|
110
|
-
return collections
|
|
111
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
112
|
-
.map((collection) => ({
|
|
113
|
-
collection: this.db.collection<BucketParameterDocumentV3>(collection.name),
|
|
114
|
-
indexId: collection.name.slice(prefix.length)
|
|
115
|
-
}));
|
|
116
|
-
}
|
|
117
127
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { bson, InternalOpId } from '@powersync/service-core';
|
|
3
|
+
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
+
import { BucketDataDocumentV3, BucketOperation } from './models.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Serialize a non-empty list of operations in strictly ascending `o` order.
|
|
8
|
+
* Preserving that order establishes the {@link BucketDataDocumentV3} range
|
|
9
|
+
* invariants used by metadata-only queries.
|
|
10
|
+
*/
|
|
11
|
+
export function serializeBucketData(
|
|
12
|
+
bucket: string,
|
|
13
|
+
operations: BucketDataDoc[],
|
|
14
|
+
options?: { targetOp?: InternalOpId | null }
|
|
15
|
+
): BucketDataDocumentV3 {
|
|
16
|
+
const minOp = operations[0].o;
|
|
17
|
+
const maxOp = operations[operations.length - 1].o;
|
|
18
|
+
|
|
19
|
+
let totalChecksum = 0n;
|
|
20
|
+
let maxTargetOp: InternalOpId | null = options?.targetOp ?? null;
|
|
21
|
+
let hasClearOp = false;
|
|
22
|
+
|
|
23
|
+
const ops: BucketOperation[] = operations.map((op) => {
|
|
24
|
+
totalChecksum += op.checksum;
|
|
25
|
+
|
|
26
|
+
if (op.op == 'CLEAR') {
|
|
27
|
+
hasClearOp = true;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
o: op.o,
|
|
32
|
+
op: op.op,
|
|
33
|
+
source_table: op.source_table,
|
|
34
|
+
source_key: op.source_key,
|
|
35
|
+
table: op.table,
|
|
36
|
+
row_id: op.row_id,
|
|
37
|
+
checksum: op.checksum,
|
|
38
|
+
data: op.data
|
|
39
|
+
};
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const size = bson.calculateObjectSize(ops);
|
|
43
|
+
|
|
44
|
+
return {
|
|
45
|
+
_id: {
|
|
46
|
+
b: bucket,
|
|
47
|
+
o: maxOp
|
|
48
|
+
},
|
|
49
|
+
min_op: minOp,
|
|
50
|
+
checksum: totalChecksum,
|
|
51
|
+
count: operations.length,
|
|
52
|
+
size,
|
|
53
|
+
target_op: maxTargetOp,
|
|
54
|
+
...(hasClearOp ? { has_clear_op: true as const } : {}),
|
|
55
|
+
ops
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function* loadBucketDataDocument(
|
|
60
|
+
context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
|
|
61
|
+
doc: BucketDataDocumentV3
|
|
62
|
+
): Generator<BucketDataDoc> {
|
|
63
|
+
const { _id, ops } = doc;
|
|
64
|
+
if (!ops) {
|
|
65
|
+
throw new ServiceAssertionError(
|
|
66
|
+
`Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const bucketKey = {
|
|
70
|
+
...context,
|
|
71
|
+
bucket: _id.b
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
for (const op of ops) {
|
|
75
|
+
yield {
|
|
76
|
+
bucketKey,
|
|
77
|
+
o: op.o,
|
|
78
|
+
op: op.op,
|
|
79
|
+
source_table: op.source_table,
|
|
80
|
+
source_key: op.source_key,
|
|
81
|
+
table: op.table,
|
|
82
|
+
row_id: op.row_id,
|
|
83
|
+
checksum: op.checksum,
|
|
84
|
+
data: op.data
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null {
|
|
90
|
+
if (a == null) {
|
|
91
|
+
return b ?? null;
|
|
92
|
+
}
|
|
93
|
+
if (b == null) {
|
|
94
|
+
return a ?? null;
|
|
95
|
+
}
|
|
96
|
+
return a > b ? a : b;
|
|
97
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { bson } from '@powersync/service-core';
|
|
2
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Split an array of bucket data operations into chunks where each chunk's
|
|
8
|
+
* total data size stays close to `maxDocSizeBytes`.
|
|
9
|
+
*
|
|
10
|
+
* The threshold is a target, not a hard ceiling — a single operation whose
|
|
11
|
+
* `data` field exceeds the limit is placed in its own chunk and is NOT split.
|
|
12
|
+
*/
|
|
13
|
+
export function chunkBucketData(
|
|
14
|
+
operations: BucketDataDoc[],
|
|
15
|
+
maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES
|
|
16
|
+
): BucketDataDoc[][] {
|
|
17
|
+
const chunks: BucketDataDoc[][] = [];
|
|
18
|
+
let currentChunk: BucketDataDoc[] = [];
|
|
19
|
+
let currentSize = 0;
|
|
20
|
+
|
|
21
|
+
for (const op of operations) {
|
|
22
|
+
// Add 5 to account for array index up to 4 digits (9999) plus null byte.
|
|
23
|
+
const opSize = bson.calculateObjectSize(op) + 5;
|
|
24
|
+
|
|
25
|
+
if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
|
|
26
|
+
chunks.push(currentChunk);
|
|
27
|
+
currentChunk = [];
|
|
28
|
+
currentSize = 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
currentChunk.push(op);
|
|
32
|
+
currentSize += opSize;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (currentChunk.length > 0) {
|
|
36
|
+
chunks.push(currentChunk);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return chunks;
|
|
40
|
+
}
|