@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
export function sourceTableDesiredResolution(syncConfig, ref, mapping) {
|
|
4
|
+
const matchingSources = syncConfig.getMatchingSources(ref);
|
|
5
|
+
return {
|
|
6
|
+
bucketSourceById: new Map(matchingSources.bucketDataSources.map((source) => [mapping.bucketSourceId(source), source])),
|
|
7
|
+
parameterLookupSourceById: new Map(matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source])),
|
|
8
|
+
triggersEvent: syncConfig.tableTriggersEvent(ref)
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export function planSourceTableReconciliation(candidateDocs, context) {
|
|
12
|
+
return new SourceTableReconciliationPlanner(context).plan(candidateDocs);
|
|
13
|
+
}
|
|
14
|
+
function sameStringArray(left, right) {
|
|
15
|
+
return left.length == right.length && left.every((value, index) => value == right[index]);
|
|
16
|
+
}
|
|
17
|
+
function matchingSourcesFor(desired, memberships) {
|
|
18
|
+
return {
|
|
19
|
+
bucketDataSources: memberships.bucketDataSourceIds.map((id) => desired.bucketSourceById.get(id)),
|
|
20
|
+
parameterLookupSources: memberships.parameterLookupSourceIds.map((id) => desired.parameterLookupSourceById.get(id))
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
function intersectMembershipIds(doc, desired) {
|
|
24
|
+
return {
|
|
25
|
+
bucketDataSourceIds: doc.bucket_data_source_ids.filter((id) => desired.bucketSourceById.has(id)),
|
|
26
|
+
parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id))
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function hasMembershipIds(memberships) {
|
|
30
|
+
return memberships.bucketDataSourceIds.length > 0 || memberships.parameterLookupSourceIds.length > 0;
|
|
31
|
+
}
|
|
32
|
+
function sameMembershipIds(doc, memberships) {
|
|
33
|
+
return (sameStringArray(doc.bucket_data_source_ids, memberships.bucketDataSourceIds) &&
|
|
34
|
+
sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds));
|
|
35
|
+
}
|
|
36
|
+
class SourceTableReconciliationPlanner {
|
|
37
|
+
context;
|
|
38
|
+
coveredBucketDataSourceIds = new Set();
|
|
39
|
+
coveredParameterLookupSourceIds = new Set();
|
|
40
|
+
retainedDocIds = new Set();
|
|
41
|
+
tables = [];
|
|
42
|
+
narrowingUpdates = [];
|
|
43
|
+
retainedEventOnlyTable = false;
|
|
44
|
+
constructor(context) {
|
|
45
|
+
this.context = context;
|
|
46
|
+
}
|
|
47
|
+
plan(candidateDocs) {
|
|
48
|
+
const { identity } = this.context;
|
|
49
|
+
for (const doc of candidateDocs) {
|
|
50
|
+
if (matchingSourceTableIdentity(doc, identity)) {
|
|
51
|
+
this.retainDoc(doc);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
tables: this.tables,
|
|
56
|
+
narrowingUpdates: this.narrowingUpdates,
|
|
57
|
+
newTableMemberships: this.newTableMemberships(),
|
|
58
|
+
// Non-retained overlapping docs represent renames / relation-id changes /
|
|
59
|
+
// replica-identity changes. Same-identity stale docs are retained in production.
|
|
60
|
+
dropDocs: candidateDocs.filter((doc) => !this.retainedDocIds.has(doc._id.toHexString()) && !matchingSourceTableIdentity(doc, identity))
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
retainDoc(doc) {
|
|
64
|
+
const memberships = intersectMembershipIds(doc, this.context.desired);
|
|
65
|
+
const coversDesiredMembership = hasMembershipIds(memberships);
|
|
66
|
+
const coversEventOnlyTable = !this.desiredHasMembership() && this.context.desired.triggersEvent && !this.retainedEventOnlyTable;
|
|
67
|
+
this.recordCoverage(doc, memberships);
|
|
68
|
+
this.planNarrowingUpdate(doc, memberships, coversDesiredMembership, coversEventOnlyTable);
|
|
69
|
+
if (coversDesiredMembership || coversEventOnlyTable) {
|
|
70
|
+
this.retainedEventOnlyTable ||= coversEventOnlyTable;
|
|
71
|
+
this.retainedDocIds.add(doc._id.toHexString());
|
|
72
|
+
this.tables.push(this.sourceTableFor(doc, memberships));
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
desiredHasMembership() {
|
|
76
|
+
const { desired } = this.context;
|
|
77
|
+
return desired.bucketSourceById.size > 0 || desired.parameterLookupSourceById.size > 0;
|
|
78
|
+
}
|
|
79
|
+
recordCoverage(doc, memberships) {
|
|
80
|
+
this.addCoverage(doc, 'bucket data source', this.coveredBucketDataSourceIds, memberships.bucketDataSourceIds);
|
|
81
|
+
this.addCoverage(doc, 'parameter lookup source', this.coveredParameterLookupSourceIds, memberships.parameterLookupSourceIds);
|
|
82
|
+
}
|
|
83
|
+
// Membership sets must be pairwise disjoint across the docs of one physical table:
|
|
84
|
+
// each desired id is covered by exactly one doc, so each definition receives each
|
|
85
|
+
// row exactly once. The algorithm maintains this (new docs only get uncovered ids,
|
|
86
|
+
// narrowing only removes ids) - overlap means the persisted state is corrupt.
|
|
87
|
+
addCoverage(doc, kind, covered, ids) {
|
|
88
|
+
const { schema, name } = this.context.identity;
|
|
89
|
+
for (const id of ids) {
|
|
90
|
+
if (covered.has(id)) {
|
|
91
|
+
throw new ReplicationAssertionError(`Source table ${doc._id} duplicates coverage of ${kind} ${id} for ${schema}.${name}`);
|
|
92
|
+
}
|
|
93
|
+
covered.add(id);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
planNarrowingUpdate(doc, memberships, coversDesiredMembership, coversEventOnlyTable) {
|
|
97
|
+
const shouldNarrow = (coversDesiredMembership || coversEventOnlyTable) && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
|
|
98
|
+
if (!shouldNarrow) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
this.narrowingUpdates.push({
|
|
102
|
+
id: doc._id,
|
|
103
|
+
memberships
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
newTableMemberships() {
|
|
107
|
+
const { desired } = this.context;
|
|
108
|
+
const uncovered = {
|
|
109
|
+
bucketDataSourceIds: [...desired.bucketSourceById.keys()].filter((id) => !this.coveredBucketDataSourceIds.has(id)),
|
|
110
|
+
parameterLookupSourceIds: [...desired.parameterLookupSourceById.keys()].filter((id) => !this.coveredParameterLookupSourceIds.has(id))
|
|
111
|
+
};
|
|
112
|
+
if (hasMembershipIds(uncovered) || (desired.triggersEvent && this.tables.length == 0)) {
|
|
113
|
+
return uncovered;
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
sourceTableFor(doc, memberships) {
|
|
118
|
+
const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
|
|
119
|
+
const table = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, matchingSourcesFor(desired, memberships), memberships);
|
|
120
|
+
table.storeCurrentData = storeCurrentData;
|
|
121
|
+
return table;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
function sameReplicaIdColumns(left, right) {
|
|
125
|
+
return (left != null &&
|
|
126
|
+
left.length == right.length &&
|
|
127
|
+
left.every((column, index) => column.name == right[index].name && column.type == right[index].type && column.type_oid == right[index].type_oid));
|
|
128
|
+
}
|
|
129
|
+
function matchingSourceTableIdentity(doc, identity) {
|
|
130
|
+
return (doc.schema_name == identity.schema &&
|
|
131
|
+
doc.table_name == identity.name &&
|
|
132
|
+
(identity.objectId == null || doc.relation_id == identity.objectId) &&
|
|
133
|
+
sameReplicaIdColumns(doc.replica_id_columns, identity.replicaIdColumns));
|
|
134
|
+
}
|
|
135
|
+
export function overlappingSourceTableFilter(connectionId, identity) {
|
|
136
|
+
const clauses = [{ schema_name: identity.schema, table_name: identity.name }];
|
|
137
|
+
if (identity.objectId != null) {
|
|
138
|
+
clauses.push({ relation_id: identity.objectId });
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
connection_id: connectionId,
|
|
142
|
+
$or: clauses
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
export function createNewSourceTable(id, memberships, context) {
|
|
146
|
+
const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData } = context;
|
|
147
|
+
const doc = {
|
|
148
|
+
_id: id,
|
|
149
|
+
connection_id: connectionId,
|
|
150
|
+
relation_id: identity.objectId,
|
|
151
|
+
schema_name: identity.schema,
|
|
152
|
+
table_name: identity.name,
|
|
153
|
+
replica_id_columns: identity.replicaIdColumns,
|
|
154
|
+
snapshot_done: false,
|
|
155
|
+
snapshot_status: undefined,
|
|
156
|
+
bucket_data_source_ids: memberships.bucketDataSourceIds,
|
|
157
|
+
parameter_lookup_source_ids: memberships.parameterLookupSourceIds
|
|
158
|
+
};
|
|
159
|
+
const table = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, matchingSourcesFor(desired, memberships), memberships);
|
|
160
|
+
table.storeCurrentData = storeCurrentData;
|
|
161
|
+
return { doc, table };
|
|
162
|
+
}
|
|
163
|
+
export function designateEventCarrier(tables, triggersEvent) {
|
|
164
|
+
if (!triggersEvent) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const eventCarrier = tables.find((table) => table.snapshotComplete) ?? tables[0];
|
|
168
|
+
for (const table of tables) {
|
|
169
|
+
table.syncEvent = table === eventCarrier;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
function sourceTableMembershipsFromDocument(doc, syncConfig, mapping) {
|
|
173
|
+
const bucketDataSourceIds = new Set(doc.bucket_data_source_ids);
|
|
174
|
+
const parameterLookupSourceIds = new Set(doc.parameter_lookup_source_ids);
|
|
175
|
+
return {
|
|
176
|
+
bucketDataSources: syncConfig.bucketDataSources.filter((source) => bucketDataSourceIds.has(mapping.bucketSourceId(source))),
|
|
177
|
+
parameterLookupSources: syncConfig.bucketParameterLookupSources.filter((source) => parameterLookupSourceIds.has(mapping.parameterLookupId(source)))
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
export function sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, memberships, membershipIds) {
|
|
181
|
+
const resolvedMemberships = memberships ?? sourceTableMembershipsFromDocument(doc, syncConfig, mapping);
|
|
182
|
+
const resolvedMembershipIds = membershipIds ?? {
|
|
183
|
+
bucketDataSourceIds: resolvedMemberships.bucketDataSources.map((source) => mapping.bucketSourceId(source)),
|
|
184
|
+
parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) => mapping.parameterLookupId(source))
|
|
185
|
+
};
|
|
186
|
+
const table = new storage.SourceTable({
|
|
187
|
+
id: doc._id,
|
|
188
|
+
ref: {
|
|
189
|
+
connectionTag,
|
|
190
|
+
schema: doc.schema_name,
|
|
191
|
+
name: doc.table_name
|
|
192
|
+
},
|
|
193
|
+
objectId: doc.relation_id,
|
|
194
|
+
replicaIdColumns: doc.replica_id_columns.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })),
|
|
195
|
+
snapshotComplete: doc.snapshot_done,
|
|
196
|
+
bucketDataSources: resolvedMemberships.bucketDataSources,
|
|
197
|
+
parameterLookupSources: resolvedMemberships.parameterLookupSources,
|
|
198
|
+
bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
|
|
199
|
+
parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds)
|
|
200
|
+
});
|
|
201
|
+
table.syncData = table.bucketDataSources.length > 0;
|
|
202
|
+
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
203
|
+
table.syncEvent = syncConfig.tableTriggersEvent(table.ref);
|
|
204
|
+
table.snapshotStatus =
|
|
205
|
+
doc.snapshot_status == null
|
|
206
|
+
? undefined
|
|
207
|
+
: {
|
|
208
|
+
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
209
|
+
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
210
|
+
replicatedCount: doc.snapshot_status.replicated_count
|
|
211
|
+
};
|
|
212
|
+
return table;
|
|
213
|
+
}
|
|
214
|
+
//# sourceMappingURL=source-table-utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAiE7F,MAAM,UAAU,4BAA4B,CAC1C,UAA8B,EAC9B,GAAmB,EACnB,OAAgC;IAEhC,MAAM,eAAe,GAAG,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3D,OAAO;QACL,gBAAgB,EAAE,IAAI,GAAG,CACvB,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CACrG;QACD,yBAAyB,EAAE,IAAI,GAAG,CAChC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC7G;QACD,aAAa,EAAE,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,aAAsC,EACtC,OAAyC;IAEzC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqC,EACrC,WAAqC;IAErC,OAAO;QACL,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACjG,sBAAsB,EAAE,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;KACrH,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAA0B,EAC1B,OAAqC;IAErC,OAAO;QACL,mBAAmB,EAAE,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,wBAAwB,EAAE,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACpH,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAqC;IAC7D,OAAO,WAAW,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA0B,EAAE,WAAqC;IAC1F,OAAO,CACL,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,WAAW,CAAC,mBAAmB,CAAC;QAC5E,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,wBAAwB,CAAC,CACvF,CAAC;AACJ,CAAC;AAED,MAAM,gCAAgC;IAQP;IAPZ,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,+BAA+B,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,GAA0B,EAAE,CAAC;IACnC,gBAAgB,GAAkC,EAAE,CAAC;IAC9D,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YAA6B,OAAyC;QAAzC,YAAO,GAAP,OAAO,CAAkC;IAAG,CAAC;IAE1E,IAAI,CAAC,aAAsC;QACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,2BAA2B,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC/C,0EAA0E;YAC1E,iFAAiF;YACjF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAC5B,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,GAAG,EAAE,QAAQ,CAAC,CACxG;SACF,CAAC;IACJ,CAAC;IAEO,SAAS,CAAC,GAA0B;QAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GACxB,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC;QAErG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,uBAAuB,EAAE,oBAAoB,CAAC,CAAC;QAE1F,IAAI,uBAAuB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,sBAAsB,KAAK,oBAAoB,CAAC;YACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,CAAC;IACzF,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,CACd,GAAG,EACH,yBAAyB,EACzB,IAAI,CAAC,+BAA+B,EACpC,WAAW,CAAC,wBAAwB,CACrC,CAAC;IACJ,CAAC;IAED,mFAAmF;IACnF,kFAAkF;IAClF,mFAAmF;IACnF,8EAA8E;IACtE,WAAW,CAAC,GAA0B,EAAE,IAAY,EAAE,OAAoB,EAAE,GAAa;QAC/F,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CACjC,gBAAgB,GAAG,CAAC,GAAG,2BAA2B,IAAI,IAAI,EAAE,QAAQ,MAAM,IAAI,IAAI,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,GAA0B,EAC1B,WAAqC,EACrC,uBAAgC,EAChC,oBAA6B;QAE7B,MAAM,YAAY,GAChB,CAAC,uBAAuB,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAElH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACzB,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB;QACzB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,MAAM,SAAS,GAA6B;YAC1C,mBAAmB,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9D,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,EAAE,CAAC,CACjD;YACD,wBAAwB,EAAE,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5E,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,EAAE,CAAC,CACtD;SACF,CAAC;QACF,IAAI,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;QACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,IAAmC,EAAE,KAAwB;IACzF,OAAO,CACL,IAAI,IAAI,IAAI;QACZ,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM;QAC3B,IAAI,CAAC,KAAK,CACR,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CACnH,CACF,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,GAA0B,EAAE,QAA6B;IAC5F,OAAO,CACL,GAAG,CAAC,WAAW,IAAI,QAAQ,CAAC,MAAM;QAClC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI;QAC/B,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACnE,oBAAoB,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,YAAoB,EACpB,QAAmE;IAEnE,MAAM,OAAO,GAAG,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,CAA8B,CAAC;IAC3G,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,YAAY;QAC3B,GAAG,EAAE,OAAO;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAiB,EACjB,WAAqC,EACrC,OAAyC;IAEzC,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC;IAC1G,MAAM,GAAG,GAA0B;QACjC,GAAG,EAAE,EAAE;QACP,aAAa,EAAE,YAAY;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ;QAC9B,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;QAC7C,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,sBAAsB,EAAE,WAAW,CAAC,mBAAmB;QACvD,2BAA2B,EAAE,WAAW,CAAC,wBAAwB;KAClE,CAAC;IACF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;IACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B,EAAE,aAAsB;IACzF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,SAAS,GAAG,KAAK,KAAK,YAAY,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,kCAAkC,CACzC,GAA0B,EAC1B,UAA8B,EAC9B,OAAgC;IAEhC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAE1E,OAAO;QACL,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CACxD;QACD,sBAAsB,EAAE,UAAU,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChF,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAChE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,GAA0B,EAC1B,aAAqB,EACrB,UAA8B,EAC9B,OAAgC,EAChC,WAA6B,EAC7B,aAAwC;IAExC,MAAM,mBAAmB,GAAG,WAAW,IAAI,kCAAkC,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxG,MAAM,qBAAqB,GAAG,aAAa,IAAI;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1G,wBAAwB,EAAE,mBAAmB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAClF,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAClC;KACF,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;QACpC,EAAE,EAAE,GAAG,CAAC,GAAG;QACX,GAAG,EAAE;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB;QACD,QAAQ,EAAE,GAAG,CAAC,WAAW;QACzB,gBAAgB,EAAE,GAAG,CAAC,kBAAmB,CAAC,GAAG,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF;QACD,gBAAgB,EAAE,GAAG,CAAC,aAAa;QACnC,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;QACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;QAClE,mBAAmB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC;QACvE,wBAAwB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;KAClF,CAAC,CAAC;IACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D,KAAK,CAAC,cAAc;QAClB,GAAG,CAAC,eAAe,IAAI,IAAI;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;gBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;aACtD,CAAC;IACR,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
export * as test_utils from '../utils/test-utils.js';
|
|
2
2
|
export * from '../utils/util.js';
|
|
3
|
-
export * from './implementation/BucketDefinitionMapping.js';
|
|
4
3
|
export * from './implementation/common/PersistedBatch.js';
|
|
5
4
|
export * from './implementation/createMongoSyncBucketStorage.js';
|
|
6
5
|
export * from './implementation/db.js';
|
|
7
6
|
export * from './implementation/models.js';
|
|
8
7
|
export * from './implementation/MongoIdSequence.js';
|
|
9
|
-
export * from './implementation/
|
|
10
|
-
export * from './implementation/
|
|
8
|
+
export * from './implementation/MongoParsedSyncConfigSet.js';
|
|
9
|
+
export * from './implementation/MongoPersistedReplicationStream.js';
|
|
11
10
|
export * from './implementation/MongoStorageProvider.js';
|
|
12
11
|
export * from './implementation/MongoSyncRulesLock.js';
|
|
13
12
|
export * from './implementation/OperationBatch.js';
|
|
14
13
|
export * from './implementation/v1/models.js';
|
|
15
|
-
export
|
|
14
|
+
export { loadBucketDataDocument, serializeBucketData } from './implementation/v3/bucket-format.js';
|
|
15
|
+
export { BucketDataDocumentV3, BucketOperation, RecordedLookupV3, ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3, taggedBucketParameterDocumentToTagged } from './implementation/v3/models.js';
|
|
16
16
|
export * from './MongoBucketStorage.js';
|
|
17
17
|
export * from './MongoReportStorage.js';
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
export * as test_utils from '../utils/test-utils.js';
|
|
2
2
|
export * from '../utils/util.js';
|
|
3
|
-
export * from './implementation/BucketDefinitionMapping.js';
|
|
4
3
|
export * from './implementation/common/PersistedBatch.js';
|
|
5
4
|
export * from './implementation/createMongoSyncBucketStorage.js';
|
|
6
5
|
export * from './implementation/db.js';
|
|
7
6
|
export * from './implementation/models.js';
|
|
8
7
|
export * from './implementation/MongoIdSequence.js';
|
|
9
|
-
export * from './implementation/
|
|
10
|
-
export * from './implementation/
|
|
8
|
+
export * from './implementation/MongoParsedSyncConfigSet.js';
|
|
9
|
+
export * from './implementation/MongoPersistedReplicationStream.js';
|
|
11
10
|
export * from './implementation/MongoStorageProvider.js';
|
|
12
11
|
export * from './implementation/MongoSyncRulesLock.js';
|
|
13
12
|
export * from './implementation/OperationBatch.js';
|
|
14
13
|
export * from './implementation/v1/models.js';
|
|
15
|
-
export
|
|
14
|
+
export { loadBucketDataDocument, serializeBucketData } from './implementation/v3/bucket-format.js';
|
|
15
|
+
export { taggedBucketParameterDocumentToTagged } from './implementation/v3/models.js';
|
|
16
16
|
export * from './MongoBucketStorage.js';
|
|
17
17
|
export * from './MongoReportStorage.js';
|
|
18
18
|
//# sourceMappingURL=storage-index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,
|
|
1
|
+
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,8CAA8C,CAAC;AAC7D,cAAc,qDAAqD,CAAC;AACpE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AACnG,OAAO,EAOL,qCAAqC,EACtC,MAAM,+BAA+B,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -1,20 +1,66 @@
|
|
|
1
1
|
import * as service_types from '@powersync/service-types';
|
|
2
2
|
import * as t from 'ts-codec';
|
|
3
|
+
export declare const MongoStorageReadPreference: t.Union<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred", string, t.CodecProps>, t.LiteralCodec<"nearest">>;
|
|
4
|
+
export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
|
|
3
5
|
export declare const MongoStorageConfig: t.Intersection<t.Codec<{
|
|
4
|
-
uri: string;
|
|
5
6
|
type: "mongodb";
|
|
7
|
+
uri: string;
|
|
6
8
|
database?: string | undefined;
|
|
7
9
|
username?: string | undefined;
|
|
8
10
|
password?: string | undefined;
|
|
9
11
|
reject_ip_ranges?: string[] | undefined;
|
|
10
12
|
}, {
|
|
11
|
-
uri: string;
|
|
12
13
|
type: "mongodb";
|
|
14
|
+
uri: string;
|
|
13
15
|
database?: string | undefined;
|
|
14
16
|
username?: string | undefined;
|
|
15
17
|
password?: string | undefined;
|
|
16
18
|
reject_ip_ranges?: string[] | undefined;
|
|
17
|
-
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
19
|
+
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
20
|
+
/**
|
|
21
|
+
* Read preference for bulk checksum and bucket data reads.
|
|
22
|
+
*
|
|
23
|
+
* If unset, MongoDB driver defaults are used for backwards compatibility.
|
|
24
|
+
*
|
|
25
|
+
* This is an experimental option, and may be removed in a future release.
|
|
26
|
+
*/
|
|
27
|
+
bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
|
|
28
|
+
/**
|
|
29
|
+
* Throttle clear operations after sync config deploys, by pausing between batches.
|
|
30
|
+
* The rate produces a pause proportional to the previous batch duration.
|
|
31
|
+
*
|
|
32
|
+
* Increase this to reduce impact of clear operations on the storage cluster.
|
|
33
|
+
* Use 0 to clear as fast as possible.
|
|
34
|
+
*
|
|
35
|
+
* Defaults to 0.2.
|
|
36
|
+
*/
|
|
37
|
+
clear_batch_throttle_rate: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
|
|
38
|
+
object_storage: t.OptionalCodec<t.Codec<{
|
|
39
|
+
type: "s3";
|
|
40
|
+
bucket: string;
|
|
41
|
+
region?: string | undefined;
|
|
42
|
+
prefix?: string | undefined;
|
|
43
|
+
endpoint?: string | undefined;
|
|
44
|
+
force_path_style?: boolean | undefined;
|
|
45
|
+
access_key_id?: string | undefined;
|
|
46
|
+
secret_access_key?: string | undefined;
|
|
47
|
+
concurrency_limit?: number | undefined;
|
|
48
|
+
inline_threshold_bytes?: number | undefined;
|
|
49
|
+
}, {
|
|
50
|
+
type: "s3";
|
|
51
|
+
bucket: string;
|
|
52
|
+
region?: string | undefined;
|
|
53
|
+
prefix?: string | undefined;
|
|
54
|
+
endpoint?: string | undefined;
|
|
55
|
+
force_path_style?: boolean | undefined;
|
|
56
|
+
access_key_id?: string | undefined;
|
|
57
|
+
secret_access_key?: string | undefined;
|
|
58
|
+
concurrency_limit?: number | undefined;
|
|
59
|
+
inline_threshold_bytes?: number | undefined;
|
|
60
|
+
}, string, t.CodecProps>>;
|
|
61
|
+
}>>;
|
|
18
62
|
export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
|
|
19
63
|
export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
|
|
64
|
+
export declare const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
|
|
65
|
+
export declare function normalizeClearBatchThrottleRate(value: number | undefined): number;
|
|
20
66
|
export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;
|
package/dist/types/types.js
CHANGED
|
@@ -1,8 +1,55 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
3
|
import * as t from 'ts-codec';
|
|
4
|
+
const S3ObjectStorageConfig = t.object({
|
|
5
|
+
type: t.literal('s3'),
|
|
6
|
+
bucket: t.string,
|
|
7
|
+
region: t.string.optional(),
|
|
8
|
+
prefix: t.string.optional(),
|
|
9
|
+
endpoint: t.string.optional(),
|
|
10
|
+
force_path_style: t.boolean.optional(),
|
|
11
|
+
access_key_id: t.string.optional(),
|
|
12
|
+
secret_access_key: t.string.optional(),
|
|
13
|
+
concurrency_limit: t.number.optional(),
|
|
14
|
+
// Chunks whose BSON-serialized size falls below this byte threshold
|
|
15
|
+
// stay inline in MongoDB instead of being offloaded to S3. Default 1024.
|
|
16
|
+
inline_threshold_bytes: t.number.optional()
|
|
17
|
+
});
|
|
18
|
+
export const MongoStorageReadPreference = t
|
|
19
|
+
.literal('primary')
|
|
20
|
+
.or(t.literal('primaryPreferred'))
|
|
21
|
+
.or(t.literal('secondary'))
|
|
22
|
+
.or(t.literal('secondaryPreferred'))
|
|
23
|
+
.or(t.literal('nearest'));
|
|
3
24
|
export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
|
|
4
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Read preference for bulk checksum and bucket data reads.
|
|
27
|
+
*
|
|
28
|
+
* If unset, MongoDB driver defaults are used for backwards compatibility.
|
|
29
|
+
*
|
|
30
|
+
* This is an experimental option, and may be removed in a future release.
|
|
31
|
+
*/
|
|
32
|
+
bulk_read_preference: MongoStorageReadPreference.optional(),
|
|
33
|
+
/**
|
|
34
|
+
* Throttle clear operations after sync config deploys, by pausing between batches.
|
|
35
|
+
* The rate produces a pause proportional to the previous batch duration.
|
|
36
|
+
*
|
|
37
|
+
* Increase this to reduce impact of clear operations on the storage cluster.
|
|
38
|
+
* Use 0 to clear as fast as possible.
|
|
39
|
+
*
|
|
40
|
+
* Defaults to 0.2.
|
|
41
|
+
*/
|
|
42
|
+
clear_batch_throttle_rate: t.number.optional(),
|
|
43
|
+
object_storage: S3ObjectStorageConfig.optional()
|
|
5
44
|
}));
|
|
45
|
+
export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
|
|
46
|
+
export function normalizeClearBatchThrottleRate(value) {
|
|
47
|
+
const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
|
|
48
|
+
if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
|
|
49
|
+
throw new ServiceError(ErrorCode.PSYNC_S3201, 'storage.clear_batch_throttle_rate must be a finite number between 0 and 20');
|
|
50
|
+
}
|
|
51
|
+
return rate;
|
|
52
|
+
}
|
|
6
53
|
export function isMongoStorageConfig(config) {
|
|
7
54
|
return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
|
|
8
55
|
}
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IACtC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAClC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,oEAAoE;IACpE,yEAAyE;IACzE,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC3D;;;;;;;;OAQG;IACH,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9C,cAAc,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAKF,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAErD,MAAM,UAAU,+BAA+B,CAAC,KAAyB;IACvE,MAAM,IAAI,GAAG,KAAK,IAAI,iCAAiC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,4EAA4E,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
|
|
@@ -5,8 +5,7 @@ import { PowerSyncMongo } from '../storage/implementation/db.js';
|
|
|
5
5
|
export type MongoTestStorageOptions = {
|
|
6
6
|
url: string;
|
|
7
7
|
isCI: boolean;
|
|
8
|
-
|
|
9
|
-
};
|
|
8
|
+
} & Omit<MongoBucketStorageOptions, 'replicationStreamNamePrefix'>;
|
|
10
9
|
export declare function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions): {
|
|
11
10
|
factory: (options?: TestStorageOptions) => Promise<MongoBucketStorage>;
|
|
12
11
|
tableIdStrings: boolean;
|
package/dist/utils/test-utils.js
CHANGED
|
@@ -15,7 +15,13 @@ export function mongoTestStorageFactoryGenerator(factoryOptions) {
|
|
|
15
15
|
if (!options?.doNotClear) {
|
|
16
16
|
await db.clear();
|
|
17
17
|
}
|
|
18
|
-
return new MongoBucketStorage(db, {
|
|
18
|
+
return new MongoBucketStorage(db, {
|
|
19
|
+
replicationStreamNamePrefix: 'test_',
|
|
20
|
+
checksumOptions: factoryOptions.checksumOptions,
|
|
21
|
+
supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
|
|
22
|
+
objectStorage: factoryOptions.objectStorage,
|
|
23
|
+
inlineThresholdBytes: factoryOptions.inlineThresholdBytes
|
|
24
|
+
});
|
|
19
25
|
},
|
|
20
26
|
tableIdStrings: false
|
|
21
27
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;gBACvE,aAAa,EAAE,cAAc,CAAC,aAAa;gBAC3C,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
|
package/dist/utils/util.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
|
+
import { Logger } from '@powersync/lib-services-framework';
|
|
3
4
|
import { storage, utils } from '@powersync/service-core';
|
|
4
5
|
import * as bson from 'bson';
|
|
5
6
|
import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
|
|
8
|
+
_id: mongo.FilterOperators<mongo.InferIdType<T>>;
|
|
9
|
+
};
|
|
10
|
+
export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T>;
|
|
11
|
+
export declare function generateReplicationStreamName(prefix: string, replicationStreamId: number): string;
|
|
8
12
|
/**
|
|
9
13
|
* Read a single batch of data from a cursor, then close it.
|
|
10
14
|
*
|
|
@@ -30,6 +34,8 @@ export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise
|
|
|
30
34
|
retryDelayMs: number;
|
|
31
35
|
onRetry?: (retryCount: number) => void;
|
|
32
36
|
}): Promise<T>;
|
|
37
|
+
export declare function clearCollectionInIdRanges<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: ClearCollectionFilter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
|
|
38
|
+
export declare function clearCollectionInIdBatches<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: mongo.Filter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
|
|
33
39
|
export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
|
|
34
40
|
items: lib_mongo.mongo.WithId<T>[];
|
|
35
41
|
count: number;
|
|
@@ -37,3 +43,4 @@ export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(
|
|
|
37
43
|
cursor: any;
|
|
38
44
|
more: boolean;
|
|
39
45
|
}>;
|
|
46
|
+
export {};
|