@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
ServiceError
|
|
13
13
|
} from '@powersync/lib-services-framework';
|
|
14
14
|
import {
|
|
15
|
+
BucketDefinitionMapping,
|
|
15
16
|
BucketStorageMarkRecordUnavailable,
|
|
16
17
|
deserializeBson,
|
|
17
18
|
InternalOpId,
|
|
@@ -23,14 +24,15 @@ import {
|
|
|
23
24
|
} from '@powersync/service-core';
|
|
24
25
|
import * as timers from 'node:timers/promises';
|
|
25
26
|
import { mongoTableId } from '../../utils/util.js';
|
|
26
|
-
import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
|
|
27
27
|
import { PersistedBatch } from './common/PersistedBatch.js';
|
|
28
28
|
import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js';
|
|
29
29
|
import type { VersionedPowerSyncMongo } from './db.js';
|
|
30
30
|
import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js';
|
|
31
31
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
32
|
+
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
32
33
|
import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
|
|
33
34
|
import { OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
35
|
+
import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
|
|
34
36
|
|
|
35
37
|
// Currently, we can only have a single flush() at a time, since it locks the op_id sequence.
|
|
36
38
|
// While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex
|
|
@@ -41,15 +43,25 @@ const replicationMutex = new utils.Mutex();
|
|
|
41
43
|
|
|
42
44
|
export interface MongoBucketBatchOptions {
|
|
43
45
|
db: VersionedPowerSyncMongo;
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The parsed sync config set for this batch.
|
|
48
|
+
*
|
|
49
|
+
* The batch derives both the hydrated sync rules and the bucket definition mapping from
|
|
50
|
+
* this single set, so they always come from the same parse. Do not add separate
|
|
51
|
+
* syncRules/mapping options - pairing values from different parses is exactly the bug
|
|
52
|
+
* this shape prevents.
|
|
53
|
+
*/
|
|
54
|
+
parsedSyncConfig: MongoParsedSyncConfigSet;
|
|
55
|
+
replicationStreamId: number;
|
|
56
|
+
replicationStreamName: string;
|
|
57
|
+
syncConfigIds?: bson.ObjectId[];
|
|
58
|
+
/**
|
|
59
|
+
* Seeds the in-memory persisted-op tracking for v1 storage. Not used by v3 storage, which
|
|
60
|
+
* tracks the persisted-op head durably on the replication stream document instead.
|
|
61
|
+
*/
|
|
62
|
+
keepaliveOp?: InternalOpId | null;
|
|
50
63
|
resumeFromLsn: string | null;
|
|
51
64
|
storeCurrentData: boolean;
|
|
52
|
-
mapping: BucketDefinitionMapping;
|
|
53
65
|
/**
|
|
54
66
|
* Set to true for initial replication.
|
|
55
67
|
*/
|
|
@@ -60,6 +72,9 @@ export interface MongoBucketBatchOptions {
|
|
|
60
72
|
|
|
61
73
|
logger: Logger;
|
|
62
74
|
tracer?: PerformanceTracer<'storage' | 'evaluate'>;
|
|
75
|
+
|
|
76
|
+
objectStorage?: ObjectStorage;
|
|
77
|
+
inlineThresholdBytes?: number;
|
|
63
78
|
}
|
|
64
79
|
|
|
65
80
|
export abstract class MongoBucketBatch
|
|
@@ -70,13 +85,22 @@ export abstract class MongoBucketBatch
|
|
|
70
85
|
protected logger: Logger;
|
|
71
86
|
|
|
72
87
|
private readonly client: mongo.MongoClient;
|
|
73
|
-
|
|
88
|
+
readonly db: VersionedPowerSyncMongo;
|
|
74
89
|
public readonly session: mongo.ClientSession;
|
|
75
90
|
protected readonly sync_rules: HydratedSyncConfig;
|
|
76
91
|
|
|
77
|
-
protected readonly
|
|
92
|
+
protected readonly replicationStreamId: number;
|
|
78
93
|
|
|
79
|
-
private readonly
|
|
94
|
+
private readonly replicationStreamName: string;
|
|
95
|
+
/**
|
|
96
|
+
* Source-level setting for whether raw row data should be stored in current_data.
|
|
97
|
+
*
|
|
98
|
+
* Some sources always send complete rows (MongoDB, MySQL with binlog_row_image=full),
|
|
99
|
+
* in which case this is false for the whole batch. For sources where it depends on the
|
|
100
|
+
* table (Postgres REPLICA IDENTITY), this is true and the decision is refined per-table
|
|
101
|
+
* via SourceTable.storeCurrentData. The effective per-record value is the conjunction of
|
|
102
|
+
* the two.
|
|
103
|
+
*/
|
|
80
104
|
private readonly storeCurrentData: boolean;
|
|
81
105
|
public readonly skipExistingRows: boolean;
|
|
82
106
|
protected readonly mapping: BucketDefinitionMapping;
|
|
@@ -87,18 +111,7 @@ export abstract class MongoBucketBatch
|
|
|
87
111
|
private hooks: storage.StorageHooks | undefined;
|
|
88
112
|
private clearedError = false;
|
|
89
113
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Last LSN received associated with a checkpoint.
|
|
94
|
-
*
|
|
95
|
-
* This could be either:
|
|
96
|
-
* 1. A commit LSN.
|
|
97
|
-
* 2. A keepalive message LSN.
|
|
98
|
-
*/
|
|
99
|
-
protected last_checkpoint_lsn: string | null = null;
|
|
100
|
-
|
|
101
|
-
protected persisted_op: InternalOpId | null = null;
|
|
114
|
+
protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
|
|
102
115
|
|
|
103
116
|
/**
|
|
104
117
|
* Last written op, if any. This may not reflect a consistent checkpoint.
|
|
@@ -106,9 +119,9 @@ export abstract class MongoBucketBatch
|
|
|
106
119
|
public last_flushed_op: InternalOpId | null = null;
|
|
107
120
|
|
|
108
121
|
/**
|
|
109
|
-
*
|
|
122
|
+
* LSN to resume replication from.
|
|
110
123
|
*
|
|
111
|
-
*
|
|
124
|
+
* This is typically the last commit LSN, but there are cases where it differs:
|
|
112
125
|
* 1. After an initial snapshot, we don't have a consistent commit yet, but need to resume from the snapshot LSN.
|
|
113
126
|
* 2. If "no_checkpoint_before_lsn" is set far in advance, it may take a while to reach that point. We
|
|
114
127
|
* may want to resume at incremental points before that.
|
|
@@ -123,34 +136,28 @@ export abstract class MongoBucketBatch
|
|
|
123
136
|
this.options = options;
|
|
124
137
|
this.client = options.db.client;
|
|
125
138
|
this.db = options.db;
|
|
126
|
-
this.
|
|
127
|
-
this.last_checkpoint_lsn = options.lastCheckpointLsn;
|
|
139
|
+
this.replicationStreamId = options.replicationStreamId;
|
|
128
140
|
this.resumeFromLsn = options.resumeFromLsn;
|
|
129
141
|
this.session = this.client.startSession();
|
|
130
|
-
this.
|
|
131
|
-
this.sync_rules = options.
|
|
142
|
+
this.replicationStreamName = options.replicationStreamName;
|
|
143
|
+
this.sync_rules = options.parsedSyncConfig.hydratedSyncConfig;
|
|
132
144
|
this.storeCurrentData = options.storeCurrentData;
|
|
133
|
-
this.mapping = options.mapping;
|
|
145
|
+
this.mapping = options.parsedSyncConfig.mapping;
|
|
134
146
|
this.skipExistingRows = options.skipExistingRows;
|
|
135
147
|
this.markRecordUnavailable = options.markRecordUnavailable;
|
|
136
148
|
this.hooks = options.hooks;
|
|
137
149
|
this.batch = new OperationBatch();
|
|
138
150
|
|
|
139
|
-
this.persisted_op = options.keepaliveOp ?? null;
|
|
140
151
|
this.tracer = options.tracer ?? new PerformanceTracer('MongoDB storage');
|
|
141
152
|
}
|
|
142
153
|
|
|
143
154
|
addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void {
|
|
144
155
|
this.write_checkpoint_batch.push({
|
|
145
156
|
...checkpoint,
|
|
146
|
-
sync_rules_id: this.
|
|
157
|
+
sync_rules_id: this.replicationStreamId
|
|
147
158
|
});
|
|
148
159
|
}
|
|
149
160
|
|
|
150
|
-
get lastCheckpointLsn() {
|
|
151
|
-
return this.last_checkpoint_lsn;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
161
|
abstract resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
|
|
155
162
|
|
|
156
163
|
protected abstract createPersistedBatch(writtenSize: number): PersistedBatch;
|
|
@@ -195,14 +202,18 @@ export abstract class MongoBucketBatch
|
|
|
195
202
|
const batch = this.batch;
|
|
196
203
|
let last_op: InternalOpId | null = null;
|
|
197
204
|
let resumeBatch: OperationBatch | null = null;
|
|
205
|
+
let clearedError = false;
|
|
198
206
|
|
|
199
207
|
using _ = this.tracer.span('storage', 'flush');
|
|
200
208
|
|
|
201
209
|
await this.hooks?.beforeBatchFlush?.(this);
|
|
202
210
|
|
|
203
211
|
await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
|
|
212
|
+
clearedError = false;
|
|
204
213
|
if (batch != null) {
|
|
205
|
-
|
|
214
|
+
const result = await this.replicateBatch(session, batch, opSeq, options);
|
|
215
|
+
resumeBatch = result.resumeBatch;
|
|
216
|
+
clearedError ||= result.clearedError;
|
|
206
217
|
}
|
|
207
218
|
|
|
208
219
|
if (this.write_checkpoint_batch.length > 0) {
|
|
@@ -214,6 +225,10 @@ export abstract class MongoBucketBatch
|
|
|
214
225
|
last_op = opSeq.last();
|
|
215
226
|
});
|
|
216
227
|
|
|
228
|
+
if (clearedError) {
|
|
229
|
+
this.clearedError = true;
|
|
230
|
+
}
|
|
231
|
+
|
|
217
232
|
// null if done, set if we need another flush
|
|
218
233
|
this.batch = resumeBatch;
|
|
219
234
|
|
|
@@ -221,7 +236,7 @@ export abstract class MongoBucketBatch
|
|
|
221
236
|
throw new ReplicationAssertionError('Unexpected last_op == null');
|
|
222
237
|
}
|
|
223
238
|
|
|
224
|
-
this.
|
|
239
|
+
this.recordPersistedOp(last_op);
|
|
225
240
|
this.last_flushed_op = last_op;
|
|
226
241
|
await this.hooks?.afterBatchFlush?.(this);
|
|
227
242
|
return { flushed_op: last_op };
|
|
@@ -232,11 +247,15 @@ export abstract class MongoBucketBatch
|
|
|
232
247
|
batch: OperationBatch,
|
|
233
248
|
op_seq: MongoIdSequence,
|
|
234
249
|
options?: storage.BucketBatchCommitOptions
|
|
235
|
-
): Promise<OperationBatch | null> {
|
|
250
|
+
): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
|
|
236
251
|
let sizes: Map<string, number> | undefined = undefined;
|
|
237
252
|
using _ = this.tracer.span('storage', 'replicate_batch');
|
|
238
|
-
if
|
|
239
|
-
|
|
253
|
+
// Only look up current_data sizes if the batch stores current_data and at least one
|
|
254
|
+
// table in it does too (per-table can disable it, e.g. Postgres REPLICA IDENTITY FULL).
|
|
255
|
+
const anyTableStoresCurrentData =
|
|
256
|
+
this.storeCurrentData && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
|
|
257
|
+
if (anyTableStoresCurrentData && !this.skipExistingRows) {
|
|
258
|
+
// We skip this step if no tables store current_data, since the sizes will
|
|
240
259
|
// always be small in that case.
|
|
241
260
|
|
|
242
261
|
// With skipExistingRows, we don't load the full documents into memory,
|
|
@@ -249,10 +268,14 @@ export abstract class MongoBucketBatch
|
|
|
249
268
|
// (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
|
|
250
269
|
// the order of processing, which then becomes really tricky to manage.
|
|
251
270
|
// This now takes 2+ queries, but doesn't have any issues with order of operations.
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
271
|
+
// Within this branch this.storeCurrentData is true, so the per-table flag is the
|
|
272
|
+
// effective value - only look up sizes for tables that actually store current_data.
|
|
273
|
+
const sizeLookups = batch.batch
|
|
274
|
+
.filter((r) => r.record.sourceTable.storeCurrentData)
|
|
275
|
+
.map((r) => ({
|
|
276
|
+
sourceTableId: mongoTableId(r.record.sourceTable.id),
|
|
277
|
+
replicaId: r.beforeId
|
|
278
|
+
}));
|
|
256
279
|
|
|
257
280
|
sizes = await this.sourceRecordStore.loadSizes(session, sizeLookups);
|
|
258
281
|
}
|
|
@@ -329,12 +352,17 @@ export abstract class MongoBucketBatch
|
|
|
329
352
|
}
|
|
330
353
|
}
|
|
331
354
|
|
|
332
|
-
|
|
355
|
+
const clearedError = didFlush && !this.clearedError;
|
|
356
|
+
if (clearedError) {
|
|
357
|
+
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
333
358
|
using _ = this.tracer.span('storage', 'clear_error');
|
|
334
|
-
await this.clearError();
|
|
359
|
+
await this.clearError(this.session);
|
|
335
360
|
}
|
|
336
361
|
|
|
337
|
-
return
|
|
362
|
+
return {
|
|
363
|
+
resumeBatch: resumeBatch?.hasData() ? resumeBatch : null,
|
|
364
|
+
clearedError
|
|
365
|
+
};
|
|
338
366
|
}
|
|
339
367
|
|
|
340
368
|
private saveOperation(
|
|
@@ -348,6 +376,9 @@ export abstract class MongoBucketBatch
|
|
|
348
376
|
const afterId = operation.afterId;
|
|
349
377
|
let after = record.after;
|
|
350
378
|
const sourceTable = record.sourceTable;
|
|
379
|
+
// Effective per-record flag: store current_data only if both the batch (source-level,
|
|
380
|
+
// e.g. Postgres) and the table (e.g. non-FULL replica identity) require it.
|
|
381
|
+
const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
|
|
351
382
|
|
|
352
383
|
let existing_buckets: LoadedSourceRecord['buckets'] = [];
|
|
353
384
|
let new_buckets: LoadedSourceRecord['buckets'] = [];
|
|
@@ -376,7 +407,7 @@ export abstract class MongoBucketBatch
|
|
|
376
407
|
// Not an error if we re-apply a transaction
|
|
377
408
|
existing_buckets = [];
|
|
378
409
|
existing_lookups = [];
|
|
379
|
-
if (!isCompleteRow(
|
|
410
|
+
if (!isCompleteRow(storeCurrentData, after!)) {
|
|
380
411
|
if (this.markRecordUnavailable != null) {
|
|
381
412
|
// This will trigger a "resnapshot" of the record.
|
|
382
413
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -392,7 +423,7 @@ export abstract class MongoBucketBatch
|
|
|
392
423
|
} else {
|
|
393
424
|
existing_buckets = result.buckets;
|
|
394
425
|
existing_lookups = result.lookups;
|
|
395
|
-
if (
|
|
426
|
+
if (storeCurrentData && result.data != null) {
|
|
396
427
|
const data = deserializeBson(result.data.buffer) as SqliteRow;
|
|
397
428
|
after = storage.mergeToast<SqliteValue>(after!, data);
|
|
398
429
|
}
|
|
@@ -403,7 +434,9 @@ export abstract class MongoBucketBatch
|
|
|
403
434
|
// Not an error if we re-apply a transaction
|
|
404
435
|
existing_buckets = [];
|
|
405
436
|
existing_lookups = [];
|
|
406
|
-
// Log to help with debugging if there was a consistency issue
|
|
437
|
+
// Log to help with debugging if there was a consistency issue.
|
|
438
|
+
// Gate on the batch-level flag: FULL tables (per-record flag false) still get a
|
|
439
|
+
// current_data entry, so a missing record on DELETE is meaningful for them too.
|
|
407
440
|
if (this.storeCurrentData && this.markRecordUnavailable == null) {
|
|
408
441
|
this.logger.warn(
|
|
409
442
|
`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
@@ -416,7 +449,7 @@ export abstract class MongoBucketBatch
|
|
|
416
449
|
}
|
|
417
450
|
|
|
418
451
|
let afterData: bson.Binary | null = null;
|
|
419
|
-
if (afterId != null && !
|
|
452
|
+
if (afterId != null && !storeCurrentData) {
|
|
420
453
|
afterData = null;
|
|
421
454
|
} else if (afterId != null) {
|
|
422
455
|
try {
|
|
@@ -440,7 +473,7 @@ export abstract class MongoBucketBatch
|
|
|
440
473
|
{
|
|
441
474
|
level: errors.ErrorSeverity.WARNING,
|
|
442
475
|
metadata: {
|
|
443
|
-
replication_slot: this.
|
|
476
|
+
replication_slot: this.replicationStreamName,
|
|
444
477
|
table: record.sourceTable.qualifiedName
|
|
445
478
|
}
|
|
446
479
|
}
|
|
@@ -483,7 +516,7 @@ export abstract class MongoBucketBatch
|
|
|
483
516
|
// However, it will be valid by the end of the transaction.
|
|
484
517
|
//
|
|
485
518
|
// In this case, we don't save the op, but we do save the current data.
|
|
486
|
-
if (afterId && after && utils.isCompleteRow(
|
|
519
|
+
if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
|
|
487
520
|
// Insert or update
|
|
488
521
|
if (sourceTable.syncData) {
|
|
489
522
|
const { results, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
@@ -499,7 +532,7 @@ export abstract class MongoBucketBatch
|
|
|
499
532
|
{
|
|
500
533
|
level: errors.ErrorSeverity.WARNING,
|
|
501
534
|
metadata: {
|
|
502
|
-
replication_slot: this.
|
|
535
|
+
replication_slot: this.replicationStreamName,
|
|
503
536
|
table: record.sourceTable.qualifiedName
|
|
504
537
|
}
|
|
505
538
|
}
|
|
@@ -534,7 +567,7 @@ export abstract class MongoBucketBatch
|
|
|
534
567
|
{
|
|
535
568
|
level: errors.ErrorSeverity.WARNING,
|
|
536
569
|
metadata: {
|
|
537
|
-
replication_slot: this.
|
|
570
|
+
replication_slot: this.replicationStreamName,
|
|
538
571
|
table: record.sourceTable.qualifiedName
|
|
539
572
|
}
|
|
540
573
|
}
|
|
@@ -669,7 +702,7 @@ export abstract class MongoBucketBatch
|
|
|
669
702
|
|
|
670
703
|
await this.db.sync_rules.updateOne(
|
|
671
704
|
{
|
|
672
|
-
_id: this.
|
|
705
|
+
_id: this.replicationStreamId
|
|
673
706
|
},
|
|
674
707
|
{
|
|
675
708
|
$set: {
|
|
@@ -678,12 +711,41 @@ export abstract class MongoBucketBatch
|
|
|
678
711
|
},
|
|
679
712
|
{ session }
|
|
680
713
|
);
|
|
714
|
+
|
|
715
|
+
// Allow subclasses to persist additional flush-time state in the same transaction
|
|
716
|
+
// (e.g. v3 advances the stream-level last_persisted_op).
|
|
717
|
+
await this.onReplicationTransactionFlush(session, opSeq.last());
|
|
681
718
|
// We don't notify checkpoint here - we don't make any checkpoint updates directly
|
|
682
719
|
});
|
|
683
720
|
}
|
|
684
721
|
|
|
722
|
+
/**
|
|
723
|
+
* Hook called inside the replication flush transaction, after ops have been persisted.
|
|
724
|
+
*
|
|
725
|
+
* The base implementation does nothing; v3 storage overrides this to `$max` the stream-level
|
|
726
|
+
* `last_persisted_op` durably within the same transaction.
|
|
727
|
+
*/
|
|
728
|
+
protected async onReplicationTransactionFlush(_session: mongo.ClientSession, _lastOp: InternalOpId): Promise<void> {
|
|
729
|
+
// No-op by default (v1 behaviour unchanged).
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Called after a replication transaction has successfully committed, with the last persisted op id.
|
|
734
|
+
*
|
|
735
|
+
* v1 storage tracks this in memory to fold into the next checkpoint. v3 storage does not need it:
|
|
736
|
+
* the stream-level `last_persisted_op` is already advanced durably by
|
|
737
|
+
* {@link onReplicationTransactionFlush} within the same transaction, and checkpoints read it
|
|
738
|
+
* from the document.
|
|
739
|
+
*
|
|
740
|
+
* Keep calls to this adjacent to {@link withReplicationTransaction} usage - both must observe
|
|
741
|
+
* every path that persists ops.
|
|
742
|
+
*/
|
|
743
|
+
protected recordPersistedOp(_lastOp: InternalOpId): void {
|
|
744
|
+
// No-op by default.
|
|
745
|
+
}
|
|
746
|
+
|
|
685
747
|
async [Symbol.asyncDispose]() {
|
|
686
|
-
if (this.batch
|
|
748
|
+
if (this.batch?.hasData() || this.write_checkpoint_batch.length > 0) {
|
|
687
749
|
// We don't error here, since:
|
|
688
750
|
// 1. In error states, this is expected (we can't distinguish between disposing after success or error).
|
|
689
751
|
// 2. SuppressedError is messy to deal with.
|
|
@@ -699,19 +761,25 @@ export abstract class MongoBucketBatch
|
|
|
699
761
|
|
|
700
762
|
async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
|
|
701
763
|
const { after, before, sourceTable, tag } = record;
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
764
|
+
const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
|
|
765
|
+
// syncEvent is the per-table designation from resolveTables. With v3 storage, multiple
|
|
766
|
+
// SourceTables can exist for the same ref, with a row change saved once per table -
|
|
767
|
+
// only the designated event carrier may fire events, so each event fires once per row.
|
|
768
|
+
if (sourceTable.syncEvent) {
|
|
769
|
+
for (const event of this.getTableEvents(sourceTable)) {
|
|
770
|
+
this.iterateListeners((cb) =>
|
|
771
|
+
cb.replicationEvent?.({
|
|
772
|
+
batch: this,
|
|
773
|
+
table: sourceTable,
|
|
774
|
+
data: {
|
|
775
|
+
op: tag,
|
|
776
|
+
after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
|
|
777
|
+
before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
|
|
778
|
+
},
|
|
779
|
+
event
|
|
780
|
+
})
|
|
781
|
+
);
|
|
782
|
+
}
|
|
715
783
|
}
|
|
716
784
|
|
|
717
785
|
/**
|
|
@@ -744,7 +812,7 @@ export abstract class MongoBucketBatch
|
|
|
744
812
|
|
|
745
813
|
await this.withTransaction(async () => {
|
|
746
814
|
for (let table of sourceTables) {
|
|
747
|
-
await this.db.commonSourceTables(this.
|
|
815
|
+
await this.db.commonSourceTables(this.replicationStreamId).deleteOne({ _id: mongoTableId(table.id) });
|
|
748
816
|
}
|
|
749
817
|
});
|
|
750
818
|
|
|
@@ -761,7 +829,7 @@ export abstract class MongoBucketBatch
|
|
|
761
829
|
}
|
|
762
830
|
|
|
763
831
|
if (last_op) {
|
|
764
|
-
this.
|
|
832
|
+
this.recordPersistedOp(last_op);
|
|
765
833
|
return {
|
|
766
834
|
flushed_op: last_op
|
|
767
835
|
};
|
|
@@ -830,7 +898,7 @@ export abstract class MongoBucketBatch
|
|
|
830
898
|
copy.snapshotStatus = snapshotStatus;
|
|
831
899
|
|
|
832
900
|
await this.withTransaction(async () => {
|
|
833
|
-
await this.db.commonSourceTables(this.
|
|
901
|
+
await this.db.commonSourceTables(this.replicationStreamId).updateOne(
|
|
834
902
|
{ _id: mongoTableId(table.id) },
|
|
835
903
|
{
|
|
836
904
|
$set: {
|
|
@@ -848,24 +916,21 @@ export abstract class MongoBucketBatch
|
|
|
848
916
|
return copy;
|
|
849
917
|
}
|
|
850
918
|
|
|
851
|
-
protected async clearError(): Promise<void> {
|
|
852
|
-
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
853
|
-
if (this.clearedError) {
|
|
854
|
-
return;
|
|
855
|
-
}
|
|
856
|
-
|
|
919
|
+
protected async clearError(session: mongo.ClientSession): Promise<void> {
|
|
857
920
|
await this.db.sync_rules.updateOne(
|
|
858
921
|
{
|
|
859
|
-
_id: this.
|
|
922
|
+
_id: this.replicationStreamId
|
|
860
923
|
},
|
|
861
924
|
{
|
|
862
925
|
$set: {
|
|
863
926
|
last_fatal_error: null,
|
|
864
927
|
last_fatal_error_ts: null
|
|
865
928
|
}
|
|
929
|
+
},
|
|
930
|
+
{
|
|
931
|
+
session
|
|
866
932
|
}
|
|
867
933
|
);
|
|
868
|
-
this.clearedError = true;
|
|
869
934
|
}
|
|
870
935
|
|
|
871
936
|
/**
|