@powersync/service-module-mongodb-storage 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/dist/storage/MongoBucketStorage.d.ts +6 -4
- package/dist/storage/MongoBucketStorage.js +110 -36
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +4 -6
- package/dist/storage/implementation/BucketDefinitionMapping.js +3 -3
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -1
- package/dist/storage/implementation/CheckpointState.d.ts +20 -0
- package/dist/storage/implementation/CheckpointState.js +31 -0
- package/dist/storage/implementation/CheckpointState.js.map +1 -0
- package/dist/storage/implementation/MongoBucketBatch.d.ts +33 -22
- package/dist/storage/implementation/MongoBucketBatch.js +45 -271
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +2 -1
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +4 -4
- package/dist/storage/implementation/MongoPersistedSyncRules.js +11 -8
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +19 -5
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +53 -19
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +21 -10
- package/dist/storage/implementation/MongoSyncBucketStorage.js +18 -163
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +7 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
- package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -2
- package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +10 -2
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +31 -47
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +385 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +16 -7
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +77 -6
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +12 -1
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +17 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +429 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +1 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +1 -2
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +29 -7
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +117 -16
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -2
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +3 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +3 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +61 -3
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/package.json +6 -6
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +166 -44
- package/src/storage/implementation/BucketDefinitionMapping.ts +12 -9
- package/src/storage/implementation/CheckpointState.ts +59 -0
- package/src/storage/implementation/MongoBucketBatch.ts +81 -355
- package/src/storage/implementation/MongoChecksums.ts +2 -1
- package/src/storage/implementation/MongoCompactor.ts +1 -1
- package/src/storage/implementation/MongoPersistedSyncRules.ts +13 -7
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +69 -24
- package/src/storage/implementation/MongoSyncBucketStorage.ts +40 -215
- package/src/storage/implementation/MongoSyncRulesLock.ts +9 -3
- package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
- package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
- package/src/storage/implementation/common/PersistedBatch.ts +2 -2
- package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +2 -2
- package/src/storage/implementation/db.ts +5 -2
- package/src/storage/implementation/models.ts +35 -58
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +478 -1
- package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +111 -16
- package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
- package/src/storage/implementation/v1/models.ts +15 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +564 -1
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1 -1
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +1 -2
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +150 -22
- package/src/storage/implementation/v3/PersistedBatchV3.ts +1 -2
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +7 -2
- package/src/storage/implementation/v3/models.ts +70 -2
- package/test/src/storage_sync.test.ts +422 -6
- package/test/src/storeCurrentData.test.ts +211 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { storage } from '@powersync/service-core';
|
|
2
2
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
3
|
-
import {
|
|
3
|
+
import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
|
|
4
4
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
|
|
5
5
|
import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
|
|
6
6
|
import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
|
|
@@ -12,7 +12,7 @@ export type { MongoSyncBucketStorage };
|
|
|
12
12
|
export function createMongoSyncBucketStorage(
|
|
13
13
|
factory: MongoBucketStorage,
|
|
14
14
|
group_id: number,
|
|
15
|
-
sync_rules:
|
|
15
|
+
sync_rules: MongoPersistedSyncRulesContentV1,
|
|
16
16
|
slot_name: string,
|
|
17
17
|
writeCheckpointMode: storage.WriteCheckpointMode | undefined,
|
|
18
18
|
options: MongoSyncBucketStorageOptions
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
InstanceDocument,
|
|
15
15
|
SourceTableDocument,
|
|
16
16
|
StorageConfig,
|
|
17
|
-
|
|
17
|
+
SyncRuleDocumentBase,
|
|
18
18
|
WriteCheckpointDocument
|
|
19
19
|
} from './models.js';
|
|
20
20
|
import {
|
|
@@ -41,7 +41,7 @@ export class PowerSyncMongo {
|
|
|
41
41
|
readonly bucket_data: mongo.Collection<BucketDataDocumentV1>;
|
|
42
42
|
readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
|
|
43
43
|
readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
|
|
44
|
-
readonly sync_rules: mongo.Collection<
|
|
44
|
+
readonly sync_rules: mongo.Collection<SyncRuleDocumentBase>;
|
|
45
45
|
readonly source_tables: mongo.Collection<SourceTableDocument>;
|
|
46
46
|
readonly custom_write_checkpoints: mongo.Collection<CustomWriteCheckpointDocument>;
|
|
47
47
|
readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
|
|
@@ -77,6 +77,9 @@ export class PowerSyncMongo {
|
|
|
77
77
|
this.connection_report_events = this.db.collection('connection_report_events');
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
+
versioned(storageConfig: StorageConfig & { incrementalReprocessing: true }): VersionedPowerSyncMongoV3;
|
|
81
|
+
versioned(storageConfig: StorageConfig & { incrementalReprocessing: false }): VersionedPowerSyncMongoV1;
|
|
82
|
+
versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo;
|
|
80
83
|
versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo {
|
|
81
84
|
if (storageConfig.incrementalReprocessing) {
|
|
82
85
|
return new VersionedPowerSyncMongoV3(this, storageConfig);
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
-
import { InternalOpId,
|
|
3
|
-
import { SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
2
|
+
import { InternalOpId, storage } from '@powersync/service-core';
|
|
3
|
+
import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
4
4
|
import { event_types } from '@powersync/service-types';
|
|
5
5
|
import * as bson from 'bson';
|
|
6
|
-
import { ParameterIndexId } from './BucketDefinitionMapping.js';
|
|
7
6
|
import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
|
|
8
7
|
import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
|
|
9
8
|
|
|
@@ -157,18 +156,14 @@ export interface IdSequenceDocument {
|
|
|
157
156
|
op_id: bigint;
|
|
158
157
|
}
|
|
159
158
|
|
|
160
|
-
|
|
159
|
+
/**
|
|
160
|
+
* Base for sync_rules collection.
|
|
161
|
+
*/
|
|
162
|
+
export interface SyncRuleDocumentBase {
|
|
161
163
|
_id: number;
|
|
162
164
|
|
|
163
165
|
state: storage.SyncRuleState;
|
|
164
166
|
|
|
165
|
-
/**
|
|
166
|
-
* True if initial snapshot has been replicated.
|
|
167
|
-
*
|
|
168
|
-
* Can only be false if state == PROCESSING.
|
|
169
|
-
*/
|
|
170
|
-
snapshot_done: boolean;
|
|
171
|
-
|
|
172
167
|
/**
|
|
173
168
|
* This is now used for "resumeLsn".
|
|
174
169
|
*
|
|
@@ -182,31 +177,6 @@ export interface SyncRuleDocument {
|
|
|
182
177
|
*/
|
|
183
178
|
snapshot_lsn: string | undefined;
|
|
184
179
|
|
|
185
|
-
/**
|
|
186
|
-
* The last consistent checkpoint.
|
|
187
|
-
*
|
|
188
|
-
* There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
|
|
189
|
-
*/
|
|
190
|
-
last_checkpoint: bigint | null;
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* The LSN associated with the last consistent checkpoint.
|
|
194
|
-
*/
|
|
195
|
-
last_checkpoint_lsn: string | null;
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
* If set, no new checkpoints may be created < this value.
|
|
199
|
-
*/
|
|
200
|
-
no_checkpoint_before: string | null;
|
|
201
|
-
|
|
202
|
-
/**
|
|
203
|
-
* Goes together with no_checkpoint_before.
|
|
204
|
-
*
|
|
205
|
-
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
206
|
-
* then the checkpoint must be equal to this keepalive_op.
|
|
207
|
-
*/
|
|
208
|
-
keepalive_op: string | null;
|
|
209
|
-
|
|
210
180
|
slot_name: string | null;
|
|
211
181
|
|
|
212
182
|
/**
|
|
@@ -230,23 +200,6 @@ export interface SyncRuleDocument {
|
|
|
230
200
|
|
|
231
201
|
last_fatal_error_ts: Date | null;
|
|
232
202
|
|
|
233
|
-
content: string;
|
|
234
|
-
serialized_plan?: SerializedSyncPlan | null;
|
|
235
|
-
|
|
236
|
-
/**
|
|
237
|
-
* Required for V3+ storage.
|
|
238
|
-
*/
|
|
239
|
-
rule_mapping?: {
|
|
240
|
-
/**
|
|
241
|
-
* Map of uniqueName -> id, unique per replication stream.
|
|
242
|
-
*/
|
|
243
|
-
definitions: Record<string, string>;
|
|
244
|
-
/**
|
|
245
|
-
* Map of (lookupName, queryId) -> id, unique per replication stream.
|
|
246
|
-
*/
|
|
247
|
-
parameter_indexes: Record<string, string>;
|
|
248
|
-
};
|
|
249
|
-
|
|
250
203
|
lock?: {
|
|
251
204
|
id: string;
|
|
252
205
|
expires_at: Date;
|
|
@@ -255,6 +208,35 @@ export interface SyncRuleDocument {
|
|
|
255
208
|
storage_version?: number;
|
|
256
209
|
}
|
|
257
210
|
|
|
211
|
+
export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint | null> {
|
|
212
|
+
/**
|
|
213
|
+
* The last consistent checkpoint.
|
|
214
|
+
*
|
|
215
|
+
* There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
|
|
216
|
+
*/
|
|
217
|
+
last_checkpoint: bigint | null;
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The LSN associated with the last consistent checkpoint.
|
|
221
|
+
*/
|
|
222
|
+
last_checkpoint_lsn: string | null;
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* If set, no new checkpoints may be created < this value.
|
|
226
|
+
*/
|
|
227
|
+
no_checkpoint_before: string | null;
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Goes together with no_checkpoint_before.
|
|
231
|
+
*
|
|
232
|
+
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
233
|
+
* then the checkpoint must be equal to this keepalive_op.
|
|
234
|
+
*
|
|
235
|
+
* This is a string in V1, bigint in V3.
|
|
236
|
+
*/
|
|
237
|
+
keepalive_op: TKeepaliveOp;
|
|
238
|
+
}
|
|
239
|
+
|
|
258
240
|
export interface StorageConfig extends storage.StorageVersionConfig {
|
|
259
241
|
/**
|
|
260
242
|
* When true, bucket_data.checksum is guaranteed to be persisted as a Long.
|
|
@@ -289,11 +271,6 @@ export interface CheckpointEventDocument {
|
|
|
289
271
|
_id: bson.ObjectId;
|
|
290
272
|
}
|
|
291
273
|
|
|
292
|
-
export type SyncRuleCheckpointState = Pick<
|
|
293
|
-
SyncRuleDocument,
|
|
294
|
-
'last_checkpoint' | 'last_checkpoint_lsn' | '_id' | 'state'
|
|
295
|
-
>;
|
|
296
|
-
|
|
297
274
|
export interface CustomWriteCheckpointDocument {
|
|
298
275
|
_id: bson.ObjectId;
|
|
299
276
|
user_id: string;
|
|
@@ -1,15 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ColumnDescriptor, SourceTable, storage } from '@powersync/service-core';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { mongoTableId } from '../../../utils/util.js';
|
|
5
|
+
import { calculateCheckpointState } from '../CheckpointState.js';
|
|
2
6
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
3
7
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
4
8
|
import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
5
9
|
import { PersistedBatchV1 } from './PersistedBatchV1.js';
|
|
6
10
|
import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
|
|
7
11
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
12
|
+
import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
|
|
8
13
|
|
|
9
14
|
export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
10
15
|
declare public readonly db: VersionedPowerSyncMongoV1;
|
|
11
16
|
|
|
12
17
|
private readonly store: SourceRecordStore;
|
|
18
|
+
private needsActivation = true;
|
|
19
|
+
private lastWaitingLogThrottled = 0;
|
|
13
20
|
|
|
14
21
|
constructor(options: MongoBucketBatchOptions) {
|
|
15
22
|
super(options);
|
|
@@ -29,4 +36,474 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
29
36
|
protected async cleanupDroppedSourceTables(_tables: SourceTable[]) {
|
|
30
37
|
// No-op for V1: source records live in a shared collection.
|
|
31
38
|
}
|
|
39
|
+
|
|
40
|
+
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
41
|
+
const syncRules = options.syncRules ?? this.sync_rules;
|
|
42
|
+
const { connection_id, source } = options;
|
|
43
|
+
const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
44
|
+
|
|
45
|
+
const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
|
|
46
|
+
name: column.name,
|
|
47
|
+
type: column.type,
|
|
48
|
+
type_oid: column.typeId
|
|
49
|
+
}));
|
|
50
|
+
|
|
51
|
+
let result: storage.ResolveTablesResult | null = null;
|
|
52
|
+
await this.db.client.withSession(async (session) => {
|
|
53
|
+
const col = this.db.commonSourceTables(this.group_id);
|
|
54
|
+
const filter: any = {
|
|
55
|
+
group_id: this.group_id,
|
|
56
|
+
connection_id,
|
|
57
|
+
schema_name: schema,
|
|
58
|
+
table_name: name,
|
|
59
|
+
replica_id_columns2: normalizedReplicaIdColumns
|
|
60
|
+
};
|
|
61
|
+
if (objectId != null) {
|
|
62
|
+
filter.relation_id = objectId;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
let doc = await col.findOne(filter, { session });
|
|
66
|
+
if (doc == null) {
|
|
67
|
+
doc = {
|
|
68
|
+
_id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
|
|
69
|
+
group_id: this.group_id,
|
|
70
|
+
connection_id,
|
|
71
|
+
relation_id: objectId,
|
|
72
|
+
schema_name: schema,
|
|
73
|
+
table_name: name,
|
|
74
|
+
replica_id_columns: null,
|
|
75
|
+
replica_id_columns2: normalizedReplicaIdColumns,
|
|
76
|
+
snapshot_done: false,
|
|
77
|
+
snapshot_status: undefined
|
|
78
|
+
};
|
|
79
|
+
await col.insertOne(doc, { session });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const sourceTable = new storage.SourceTable({
|
|
83
|
+
id: doc._id,
|
|
84
|
+
ref: source,
|
|
85
|
+
objectId,
|
|
86
|
+
replicaIdColumns,
|
|
87
|
+
snapshotComplete: doc.snapshot_done ?? true,
|
|
88
|
+
...syncRules.getMatchingSources(source)
|
|
89
|
+
});
|
|
90
|
+
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
91
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
92
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
93
|
+
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
94
|
+
sourceTable.snapshotStatus =
|
|
95
|
+
doc.snapshot_status == null
|
|
96
|
+
? undefined
|
|
97
|
+
: {
|
|
98
|
+
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
99
|
+
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
100
|
+
replicatedCount: doc.snapshot_status.replicated_count
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
|
|
104
|
+
if (objectId != null) {
|
|
105
|
+
truncateFilter.push({ relation_id: objectId });
|
|
106
|
+
}
|
|
107
|
+
const truncate = await col
|
|
108
|
+
.find(
|
|
109
|
+
{
|
|
110
|
+
group_id: this.group_id,
|
|
111
|
+
connection_id,
|
|
112
|
+
_id: { $ne: doc._id },
|
|
113
|
+
$or: truncateFilter
|
|
114
|
+
},
|
|
115
|
+
{ session }
|
|
116
|
+
)
|
|
117
|
+
.toArray();
|
|
118
|
+
const dropTables = truncate.map((dropDoc) => {
|
|
119
|
+
const ref = {
|
|
120
|
+
connectionTag,
|
|
121
|
+
schema: dropDoc.schema_name,
|
|
122
|
+
name: dropDoc.table_name
|
|
123
|
+
};
|
|
124
|
+
const table = new storage.SourceTable({
|
|
125
|
+
id: dropDoc._id,
|
|
126
|
+
ref,
|
|
127
|
+
objectId: dropDoc.relation_id,
|
|
128
|
+
replicaIdColumns:
|
|
129
|
+
dropDoc.replica_id_columns2?.map(
|
|
130
|
+
(c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
|
|
131
|
+
) ?? [],
|
|
132
|
+
snapshotComplete: dropDoc.snapshot_done ?? true,
|
|
133
|
+
...syncRules.getMatchingSources(ref)
|
|
134
|
+
});
|
|
135
|
+
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
136
|
+
table.syncData = table.bucketDataSources.length > 0;
|
|
137
|
+
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
138
|
+
return table;
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
result = { tables: [sourceTable], dropTables };
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
return result!;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
|
|
148
|
+
const doc = (await this.db.commonSourceTables(this.group_id).findOne(
|
|
149
|
+
{
|
|
150
|
+
group_id: this.group_id,
|
|
151
|
+
_id: mongoTableId(table.id)
|
|
152
|
+
},
|
|
153
|
+
{ session: this.session }
|
|
154
|
+
)) as SourceTableDocumentV1 | null;
|
|
155
|
+
if (doc == null) {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const ref = {
|
|
160
|
+
connectionTag: table.ref.connectionTag,
|
|
161
|
+
schema: doc.schema_name,
|
|
162
|
+
name: doc.table_name
|
|
163
|
+
};
|
|
164
|
+
const sourceTable = new storage.SourceTable({
|
|
165
|
+
id: doc._id,
|
|
166
|
+
ref,
|
|
167
|
+
objectId: doc.relation_id,
|
|
168
|
+
replicaIdColumns:
|
|
169
|
+
doc.replica_id_columns2?.map(
|
|
170
|
+
(c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
|
|
171
|
+
) ?? [],
|
|
172
|
+
snapshotComplete: doc.snapshot_done ?? true,
|
|
173
|
+
...this.sync_rules.getMatchingSources(ref)
|
|
174
|
+
});
|
|
175
|
+
sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
|
|
176
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
177
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
178
|
+
sourceTable.snapshotStatus =
|
|
179
|
+
doc.snapshot_status == null
|
|
180
|
+
? undefined
|
|
181
|
+
: {
|
|
182
|
+
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
183
|
+
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
184
|
+
replicatedCount: doc.snapshot_status.replicated_count
|
|
185
|
+
};
|
|
186
|
+
return sourceTable;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
|
|
190
|
+
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
191
|
+
|
|
192
|
+
await this.flush(options);
|
|
193
|
+
|
|
194
|
+
const now = new Date();
|
|
195
|
+
|
|
196
|
+
await this.db.write_checkpoints.updateMany(
|
|
197
|
+
{
|
|
198
|
+
processed_at_lsn: null,
|
|
199
|
+
'lsns.1': { $lte: lsn }
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
$set: {
|
|
203
|
+
processed_at_lsn: lsn
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
session: this.session
|
|
208
|
+
}
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
const can_checkpoint = {
|
|
212
|
+
$and: [
|
|
213
|
+
{ $eq: ['$snapshot_done', true] },
|
|
214
|
+
{
|
|
215
|
+
$or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
|
|
216
|
+
},
|
|
217
|
+
{
|
|
218
|
+
$or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
|
|
219
|
+
}
|
|
220
|
+
]
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
const new_keepalive_op = {
|
|
224
|
+
$cond: [
|
|
225
|
+
can_checkpoint,
|
|
226
|
+
{ $literal: null },
|
|
227
|
+
{
|
|
228
|
+
$toString: {
|
|
229
|
+
$max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
]
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
const new_last_checkpoint = {
|
|
236
|
+
$cond: [
|
|
237
|
+
can_checkpoint,
|
|
238
|
+
{
|
|
239
|
+
$max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
|
|
240
|
+
},
|
|
241
|
+
'$last_checkpoint'
|
|
242
|
+
]
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
|
|
246
|
+
{ _id: this.group_id },
|
|
247
|
+
[
|
|
248
|
+
{
|
|
249
|
+
$set: {
|
|
250
|
+
_can_checkpoint: can_checkpoint,
|
|
251
|
+
_not_empty: createEmptyCheckpoints
|
|
252
|
+
? true
|
|
253
|
+
: {
|
|
254
|
+
$or: [
|
|
255
|
+
{ $literal: createEmptyCheckpoints },
|
|
256
|
+
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
257
|
+
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
258
|
+
]
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
$set: {
|
|
264
|
+
last_checkpoint_lsn: {
|
|
265
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
266
|
+
},
|
|
267
|
+
last_checkpoint_ts: {
|
|
268
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
269
|
+
},
|
|
270
|
+
last_keepalive_ts: { $literal: now },
|
|
271
|
+
last_fatal_error: { $literal: null },
|
|
272
|
+
last_fatal_error_ts: { $literal: null },
|
|
273
|
+
keepalive_op: new_keepalive_op,
|
|
274
|
+
last_checkpoint: new_last_checkpoint,
|
|
275
|
+
snapshot_lsn: {
|
|
276
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
$unset: ['_can_checkpoint', '_not_empty']
|
|
282
|
+
}
|
|
283
|
+
],
|
|
284
|
+
{
|
|
285
|
+
session: this.session,
|
|
286
|
+
returnDocument: 'before',
|
|
287
|
+
projection: {
|
|
288
|
+
snapshot_done: 1,
|
|
289
|
+
last_checkpoint_lsn: 1,
|
|
290
|
+
no_checkpoint_before: 1,
|
|
291
|
+
keepalive_op: 1,
|
|
292
|
+
last_checkpoint: 1
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
)) as SyncRuleDocumentV1;
|
|
296
|
+
|
|
297
|
+
if (preUpdateDocument == null) {
|
|
298
|
+
throw new ReplicationAssertionError(
|
|
299
|
+
'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.group_id
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const checkpointState = calculateCheckpointState({
|
|
304
|
+
lsn,
|
|
305
|
+
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
306
|
+
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
307
|
+
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
308
|
+
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
309
|
+
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
310
|
+
persistedOp: this.persisted_op,
|
|
311
|
+
createEmptyCheckpoints
|
|
312
|
+
});
|
|
313
|
+
if (checkpointState.checkpointBlocked) {
|
|
314
|
+
if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
|
|
315
|
+
this.logger.info(
|
|
316
|
+
`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify(
|
|
317
|
+
{
|
|
318
|
+
snapshot_done: preUpdateDocument.snapshot_done,
|
|
319
|
+
last_checkpoint_lsn: preUpdateDocument.last_checkpoint_lsn,
|
|
320
|
+
no_checkpoint_before: preUpdateDocument.no_checkpoint_before
|
|
321
|
+
}
|
|
322
|
+
)}`
|
|
323
|
+
);
|
|
324
|
+
this.lastWaitingLogThrottled = Date.now();
|
|
325
|
+
}
|
|
326
|
+
} else {
|
|
327
|
+
if (checkpointState.checkpointCreated) {
|
|
328
|
+
this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
|
|
329
|
+
}
|
|
330
|
+
await this.autoActivate(lsn);
|
|
331
|
+
await this.db.notifyCheckpoint();
|
|
332
|
+
this.persisted_op = null;
|
|
333
|
+
this.last_checkpoint_lsn = lsn;
|
|
334
|
+
if (checkpointState.newLastCheckpoint != null) {
|
|
335
|
+
await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return {
|
|
340
|
+
checkpointBlocked: checkpointState.checkpointBlocked,
|
|
341
|
+
checkpointCreated: checkpointState.checkpointCreated
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async keepalive(lsn: string): Promise<storage.CheckpointResult> {
|
|
346
|
+
return await this.commit(lsn, { createEmptyCheckpoints: true });
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
async setResumeLsn(lsn: string): Promise<void> {
|
|
350
|
+
await this.db.sync_rules.updateOne(
|
|
351
|
+
{
|
|
352
|
+
_id: this.group_id
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
$set: {
|
|
356
|
+
snapshot_lsn: lsn
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
{ session: this.session }
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void> {
|
|
364
|
+
await this.db.sync_rules.updateOne(
|
|
365
|
+
{
|
|
366
|
+
_id: this.group_id
|
|
367
|
+
},
|
|
368
|
+
{
|
|
369
|
+
$set: {
|
|
370
|
+
snapshot_done: true,
|
|
371
|
+
last_keepalive_ts: new Date()
|
|
372
|
+
},
|
|
373
|
+
$max: {
|
|
374
|
+
no_checkpoint_before: no_checkpoint_before_lsn
|
|
375
|
+
}
|
|
376
|
+
},
|
|
377
|
+
{ session: this.session }
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
382
|
+
await this.withTransaction(async () => {
|
|
383
|
+
// Protect against race conditions
|
|
384
|
+
const count = await this.db.commonSourceTables(this.group_id).countDocuments(
|
|
385
|
+
{
|
|
386
|
+
group_id: this.group_id,
|
|
387
|
+
snapshot_done: false
|
|
388
|
+
},
|
|
389
|
+
{ session: this.session }
|
|
390
|
+
);
|
|
391
|
+
if (count > 0) {
|
|
392
|
+
if (options?.throwOnConflict ?? true) {
|
|
393
|
+
throw new ReplicationAssertionError(
|
|
394
|
+
`Cannot mark snapshot done while ${count} source table${count == 1 ? '' : 's'} still require snapshotting`
|
|
395
|
+
);
|
|
396
|
+
} else {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
await this.markAllSnapshotDone(no_checkpoint_before_lsn);
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
async markTableSnapshotRequired(_table: storage.SourceTable): Promise<void> {
|
|
406
|
+
await this.db.sync_rules.updateOne(
|
|
407
|
+
{
|
|
408
|
+
_id: this.group_id
|
|
409
|
+
},
|
|
410
|
+
{
|
|
411
|
+
$set: {
|
|
412
|
+
snapshot_done: false
|
|
413
|
+
}
|
|
414
|
+
},
|
|
415
|
+
{ session: this.session }
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
async markTableSnapshotDone(
|
|
420
|
+
tables: storage.SourceTable[],
|
|
421
|
+
no_checkpoint_before_lsn?: string
|
|
422
|
+
): Promise<storage.SourceTable[]> {
|
|
423
|
+
const session = this.session;
|
|
424
|
+
const ids = tables.map((table) => mongoTableId(table.id));
|
|
425
|
+
|
|
426
|
+
await this.withTransaction(async () => {
|
|
427
|
+
await this.db.commonSourceTables(this.group_id).updateMany(
|
|
428
|
+
{ _id: { $in: ids } },
|
|
429
|
+
{
|
|
430
|
+
$set: {
|
|
431
|
+
snapshot_done: true
|
|
432
|
+
},
|
|
433
|
+
$unset: {
|
|
434
|
+
snapshot_status: 1
|
|
435
|
+
}
|
|
436
|
+
},
|
|
437
|
+
{ session }
|
|
438
|
+
);
|
|
439
|
+
|
|
440
|
+
if (no_checkpoint_before_lsn != null) {
|
|
441
|
+
await this.db.sync_rules.updateOne(
|
|
442
|
+
{
|
|
443
|
+
_id: this.group_id
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
$set: {
|
|
447
|
+
last_keepalive_ts: new Date()
|
|
448
|
+
},
|
|
449
|
+
$max: {
|
|
450
|
+
no_checkpoint_before: no_checkpoint_before_lsn
|
|
451
|
+
}
|
|
452
|
+
},
|
|
453
|
+
{ session: this.session }
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
});
|
|
457
|
+
return tables.map((table) => {
|
|
458
|
+
const copy = table.clone();
|
|
459
|
+
copy.snapshotComplete = true;
|
|
460
|
+
return copy;
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
private async autoActivate(lsn: string): Promise<void> {
|
|
465
|
+
if (!this.needsActivation) {
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const session = this.session;
|
|
470
|
+
let activated = false;
|
|
471
|
+
await session.withTransaction(async () => {
|
|
472
|
+
const doc = (await this.db.sync_rules.findOne({ _id: this.group_id }, { session })) as SyncRuleDocumentV1;
|
|
473
|
+
if (doc && doc.state == storage.SyncRuleState.PROCESSING && doc.snapshot_done && doc.last_checkpoint != null) {
|
|
474
|
+
await this.db.sync_rules.updateOne(
|
|
475
|
+
{
|
|
476
|
+
_id: this.group_id
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
$set: {
|
|
480
|
+
state: storage.SyncRuleState.ACTIVE
|
|
481
|
+
}
|
|
482
|
+
},
|
|
483
|
+
{ session }
|
|
484
|
+
);
|
|
485
|
+
|
|
486
|
+
await this.db.sync_rules.updateMany(
|
|
487
|
+
{
|
|
488
|
+
_id: { $ne: this.group_id },
|
|
489
|
+
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
490
|
+
},
|
|
491
|
+
{
|
|
492
|
+
$set: {
|
|
493
|
+
state: storage.SyncRuleState.STOP
|
|
494
|
+
}
|
|
495
|
+
},
|
|
496
|
+
{ session }
|
|
497
|
+
);
|
|
498
|
+
activated = true;
|
|
499
|
+
} else if (doc?.state != storage.SyncRuleState.PROCESSING) {
|
|
500
|
+
this.needsActivation = false;
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
if (activated) {
|
|
504
|
+
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
505
|
+
await this.db.notifyCheckpoint();
|
|
506
|
+
this.needsActivation = false;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
32
509
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
|
-
import { BucketDefinitionId } from '
|
|
4
|
+
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
5
5
|
import { SingleBucketStore } from '../common/SingleBucketStore.js';
|
|
6
6
|
import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
|
|
7
7
|
import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
|