@powersync/service-module-mongodb 0.20.1 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MongoRouteAPIAdapter.js +62 -142
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/api/infer-collection-schema.d.ts +7 -0
- package/dist/api/infer-collection-schema.js +118 -0
- package/dist/api/infer-collection-schema.js.map +1 -0
- package/dist/replication/ChangeStream.js +2 -4
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/MongoSnapshotter.js +6 -6
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +3 -11
- package/dist/replication/RawChangeStream.js +0 -17
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/types/types.js +2 -2
- package/dist/types/types.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MongoRouteAPIAdapter.ts +68 -126
- package/src/api/infer-collection-schema.ts +127 -0
- package/src/replication/ChangeStream.ts +2 -4
- package/src/replication/MongoSnapshotter.ts +6 -6
- package/src/replication/RawChangeStream.ts +3 -30
- package/src/types/types.ts +2 -2
- package/test/src/change_stream.test.ts +73 -0
- package/test/src/documentdb_mode.test.ts +1 -2
- package/test/src/raw_change_stream.test.ts +0 -63
- package/test/src/schema.test.ts +180 -0
- package/test/src/util.ts +11 -29
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -8,10 +8,12 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
8
8
|
import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js';
|
|
9
9
|
import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
|
|
10
10
|
import { MongoManager } from '../replication/MongoManager.js';
|
|
11
|
-
import { constructAfterRecord } from '../replication/MongoRelation.js';
|
|
12
11
|
import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js';
|
|
13
12
|
import * as types from '../types/types.js';
|
|
14
13
|
import { escapeRegExp } from '../utils.js';
|
|
14
|
+
import { inferCollectionSchema } from './infer-collection-schema.js';
|
|
15
|
+
|
|
16
|
+
const SCHEMA_INFERENCE_CONCURRENCY = 4;
|
|
15
17
|
|
|
16
18
|
export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
17
19
|
protected client: mongo.MongoClient;
|
|
@@ -21,6 +23,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
21
23
|
defaultSchema: string;
|
|
22
24
|
|
|
23
25
|
private checkpointImplementation: CheckpointImplementation | null = null;
|
|
26
|
+
private documentDbDetection: Promise<boolean> | null = null;
|
|
24
27
|
|
|
25
28
|
constructor(protected config: types.ResolvedConnectionConfig) {
|
|
26
29
|
const manager = new MongoManager(config);
|
|
@@ -206,9 +209,18 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
206
209
|
return checkpointImplementation.createReplicationHead(callback);
|
|
207
210
|
}
|
|
208
211
|
|
|
212
|
+
private isDocumentDb(): Promise<boolean> {
|
|
213
|
+
// Share in-flight detection and cache its result for this adapter's lifetime.
|
|
214
|
+
return (this.documentDbDetection ??= detectDocumentDb(this.db).catch((error) => {
|
|
215
|
+
// Allow a later request to retry after a transient connection error.
|
|
216
|
+
this.documentDbDetection = null;
|
|
217
|
+
throw error;
|
|
218
|
+
}));
|
|
219
|
+
}
|
|
220
|
+
|
|
209
221
|
private async getCheckpointImplementation(): Promise<CheckpointImplementation> {
|
|
210
222
|
if (this.checkpointImplementation == null) {
|
|
211
|
-
const isDocumentDb = await
|
|
223
|
+
const isDocumentDb = await this.isDocumentDb();
|
|
212
224
|
this.checkpointImplementation = createCheckpointImplementation(isDocumentDb, {
|
|
213
225
|
client: this.client,
|
|
214
226
|
db: this.db,
|
|
@@ -226,32 +238,38 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
226
238
|
}
|
|
227
239
|
|
|
228
240
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
229
|
-
const
|
|
230
|
-
|
|
241
|
+
const isDocumentDb = await this.isDocumentDb();
|
|
231
242
|
const databases = await this.db.admin().listDatabases({ nameOnly: true });
|
|
232
243
|
const filteredDatabases = databases.databases.filter((db) => {
|
|
233
244
|
return !['local', 'admin', 'config'].includes(db.name);
|
|
234
245
|
});
|
|
235
|
-
const databaseSchemas =
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
}
|
|
250
|
-
throw e;
|
|
246
|
+
const databaseSchemas: service_types.DatabaseSchema[] = [];
|
|
247
|
+
for (const db of filteredDatabases) {
|
|
248
|
+
/**
|
|
249
|
+
* Filtering the list of database with `authorizedDatabases: true`
|
|
250
|
+
* does not produce the full list of databases under some circumstances.
|
|
251
|
+
* This catches any potential auth errors.
|
|
252
|
+
*/
|
|
253
|
+
let collections: mongo.CollectionInfo[];
|
|
254
|
+
try {
|
|
255
|
+
collections = await this.client.db(db.name).listCollections().toArray();
|
|
256
|
+
} catch (e) {
|
|
257
|
+
if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
|
|
258
|
+
// Ignore databases we're not authorized to query
|
|
259
|
+
continue;
|
|
251
260
|
}
|
|
261
|
+
throw e;
|
|
262
|
+
}
|
|
252
263
|
|
|
253
|
-
|
|
254
|
-
|
|
264
|
+
const tables: (service_types.TableSchema | undefined)[] = new Array(collections.length);
|
|
265
|
+
const pendingCollections = collections.entries();
|
|
266
|
+
let failed = false;
|
|
267
|
+
const inferCollections = async () => {
|
|
268
|
+
// Reading from the iterator automatically manages the queue
|
|
269
|
+
for (const [index, collection] of pendingCollections) {
|
|
270
|
+
if (failed) {
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
255
273
|
if ([CHECKPOINTS_COLLECTION].includes(collection.name)) {
|
|
256
274
|
continue;
|
|
257
275
|
}
|
|
@@ -264,121 +282,45 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
264
282
|
continue;
|
|
265
283
|
}
|
|
266
284
|
try {
|
|
267
|
-
const
|
|
268
|
-
.collection(collection.name)
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const columns = this.getColumnsFromDocuments(sampleDocuments);
|
|
274
|
-
|
|
275
|
-
tables.push({
|
|
276
|
-
name: collection.name,
|
|
277
|
-
// Since documents are sampled in a random order, we need to sort
|
|
278
|
-
// to get a consistent order
|
|
279
|
-
columns: columns.sort((a, b) => a.name.localeCompare(b.name))
|
|
280
|
-
});
|
|
281
|
-
} else {
|
|
282
|
-
tables.push({
|
|
283
|
-
name: collection.name,
|
|
284
|
-
columns: []
|
|
285
|
-
});
|
|
286
|
-
}
|
|
285
|
+
const columns = await inferCollectionSchema(
|
|
286
|
+
this.client.db(db.name).collection(collection.name),
|
|
287
|
+
isDocumentDb
|
|
288
|
+
);
|
|
289
|
+
// Preserve collection order even when queries finish out of order.
|
|
290
|
+
tables[index] = { name: collection.name, columns };
|
|
287
291
|
} catch (e) {
|
|
288
292
|
if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
|
|
289
293
|
// Ignore collections we're not authorized to query
|
|
290
294
|
continue;
|
|
291
295
|
}
|
|
296
|
+
// Fail the whole request on unexpected errors: the response cannot indicate
|
|
297
|
+
// partial results, so omitting a collection would make an incomplete schema
|
|
298
|
+
// appear complete and could produce an incorrect generated client schema.
|
|
299
|
+
failed = true;
|
|
292
300
|
throw e;
|
|
293
301
|
}
|
|
294
302
|
}
|
|
295
|
-
|
|
296
|
-
return {
|
|
297
|
-
name: db.name,
|
|
298
|
-
tables: tables
|
|
299
|
-
} satisfies service_types.DatabaseSchema;
|
|
300
|
-
})
|
|
301
|
-
);
|
|
302
|
-
return databaseSchemas.filter((schema) => !!schema);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
private getColumnsFromDocuments(documents: mongo.BSON.Document[]) {
|
|
306
|
-
let columns = new Map<string, { sqliteType: sync_rules.ExpressionType; bsonTypes: Set<string> }>();
|
|
307
|
-
for (const document of documents) {
|
|
308
|
-
const parsed = constructAfterRecord(document);
|
|
309
|
-
for (const key in parsed) {
|
|
310
|
-
const value = parsed[key];
|
|
311
|
-
const type = sync_rules.sqliteTypeOf(value);
|
|
312
|
-
const sqliteType = sync_rules.ExpressionType.fromTypeText(type);
|
|
313
|
-
let entry = columns.get(key);
|
|
314
|
-
if (entry == null) {
|
|
315
|
-
entry = { sqliteType, bsonTypes: new Set() };
|
|
316
|
-
columns.set(key, entry);
|
|
317
|
-
} else {
|
|
318
|
-
entry.sqliteType = entry.sqliteType.or(sqliteType);
|
|
319
|
-
}
|
|
320
|
-
const bsonType = this.getBsonType(document[key]);
|
|
321
|
-
if (bsonType != null) {
|
|
322
|
-
entry.bsonTypes.add(bsonType);
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
return [...columns.entries()].map(([key, value]) => {
|
|
327
|
-
const internal_type = value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ');
|
|
328
|
-
return {
|
|
329
|
-
name: key,
|
|
330
|
-
type: internal_type,
|
|
331
|
-
sqlite_type: value.sqliteType.typeFlags,
|
|
332
|
-
internal_type,
|
|
333
|
-
pg_type: internal_type
|
|
334
303
|
};
|
|
335
|
-
});
|
|
336
|
-
}
|
|
337
304
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
305
|
+
// Each worker holds only schema metadata, with a bounded number of source queries.
|
|
306
|
+
const workers = Array.from(
|
|
307
|
+
{ length: Math.min(SCHEMA_INFERENCE_CONCURRENCY, collections.length) },
|
|
308
|
+
inferCollections
|
|
309
|
+
);
|
|
310
|
+
try {
|
|
311
|
+
await Promise.all(workers);
|
|
312
|
+
} catch (e) {
|
|
313
|
+
// Finish in-flight queries before rejecting the request. Workers stop taking
|
|
314
|
+
// new collections as soon as one encounters a non-authorization error.
|
|
315
|
+
await Promise.allSettled(workers);
|
|
316
|
+
throw e;
|
|
349
317
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
return 'ObjectId';
|
|
356
|
-
} else if (data instanceof mongo.UUID) {
|
|
357
|
-
return 'UUID';
|
|
358
|
-
} else if (data instanceof Date) {
|
|
359
|
-
return 'Date';
|
|
360
|
-
} else if (data instanceof mongo.Timestamp) {
|
|
361
|
-
return 'Timestamp';
|
|
362
|
-
} else if (data instanceof mongo.Binary) {
|
|
363
|
-
return 'Binary';
|
|
364
|
-
} else if (data instanceof mongo.Long) {
|
|
365
|
-
return 'Long';
|
|
366
|
-
} else if (data instanceof RegExp) {
|
|
367
|
-
return 'RegExp';
|
|
368
|
-
} else if (data instanceof mongo.MinKey) {
|
|
369
|
-
return 'MinKey';
|
|
370
|
-
} else if (data instanceof mongo.MaxKey) {
|
|
371
|
-
return 'MaxKey';
|
|
372
|
-
} else if (data instanceof mongo.Decimal128) {
|
|
373
|
-
return 'Decimal';
|
|
374
|
-
} else if (Array.isArray(data)) {
|
|
375
|
-
return 'Array';
|
|
376
|
-
} else if (data instanceof Uint8Array) {
|
|
377
|
-
return 'Binary';
|
|
378
|
-
} else if (typeof data == 'object') {
|
|
379
|
-
return 'Object';
|
|
380
|
-
} else {
|
|
381
|
-
return null;
|
|
318
|
+
|
|
319
|
+
databaseSchemas.push({
|
|
320
|
+
name: db.name,
|
|
321
|
+
tables: tables.filter((table) => table != null)
|
|
322
|
+
});
|
|
382
323
|
}
|
|
324
|
+
return databaseSchemas;
|
|
383
325
|
}
|
|
384
326
|
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ExpressionType } from '@powersync/service-sync-rules';
|
|
3
|
+
import { TableSchema } from '@powersync/service-types';
|
|
4
|
+
|
|
5
|
+
// Match the types exposed after BSON deserialization and conversion to sync rules values.
|
|
6
|
+
const BSON_TYPES: Record<string, { name: string; sqliteType: ExpressionType }> = {
|
|
7
|
+
int: { name: 'Integer', sqliteType: ExpressionType.INTEGER },
|
|
8
|
+
long: { name: 'Long', sqliteType: ExpressionType.INTEGER },
|
|
9
|
+
double: { name: 'Double', sqliteType: ExpressionType.REAL },
|
|
10
|
+
decimal: { name: 'Decimal', sqliteType: ExpressionType.TEXT },
|
|
11
|
+
string: { name: 'String', sqliteType: ExpressionType.TEXT },
|
|
12
|
+
symbol: { name: 'String', sqliteType: ExpressionType.TEXT },
|
|
13
|
+
bool: { name: 'Boolean', sqliteType: ExpressionType.INTEGER },
|
|
14
|
+
objectId: { name: 'ObjectId', sqliteType: ExpressionType.TEXT },
|
|
15
|
+
uuid: { name: 'UUID', sqliteType: ExpressionType.TEXT },
|
|
16
|
+
binData: { name: 'Binary', sqliteType: ExpressionType.BLOB },
|
|
17
|
+
date: { name: 'Date', sqliteType: ExpressionType.TEXT },
|
|
18
|
+
timestamp: { name: 'Timestamp', sqliteType: ExpressionType.INTEGER },
|
|
19
|
+
regex: { name: 'RegExp', sqliteType: ExpressionType.TEXT },
|
|
20
|
+
object: { name: 'Object', sqliteType: ExpressionType.TEXT },
|
|
21
|
+
array: { name: 'Array', sqliteType: ExpressionType.TEXT },
|
|
22
|
+
javascript: { name: 'Object', sqliteType: ExpressionType.TEXT },
|
|
23
|
+
javascriptWithScope: { name: 'Object', sqliteType: ExpressionType.TEXT },
|
|
24
|
+
dbPointer: { name: 'Object', sqliteType: ExpressionType.TEXT },
|
|
25
|
+
null: { name: 'Null', sqliteType: ExpressionType.NONE },
|
|
26
|
+
undefined: { name: 'Null', sqliteType: ExpressionType.NONE },
|
|
27
|
+
minKey: { name: 'MinKey', sqliteType: ExpressionType.NONE },
|
|
28
|
+
maxKey: { name: 'MaxKey', sqliteType: ExpressionType.NONE }
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Infer top-level fields without transferring sampled document values to the service.
|
|
33
|
+
* Memory on the service scales with the inferred schema, not the size of those values.
|
|
34
|
+
*/
|
|
35
|
+
export async function inferCollectionSchema(
|
|
36
|
+
collection: mongo.Collection,
|
|
37
|
+
isDocumentDb: boolean
|
|
38
|
+
): Promise<TableSchema['columns']> {
|
|
39
|
+
const fields = await collection
|
|
40
|
+
.aggregate<{ _id: string; types: string[] }>(
|
|
41
|
+
[
|
|
42
|
+
// Keep this first so MongoDB can use its random cursor on large collections.
|
|
43
|
+
{ $sample: { size: 50 } },
|
|
44
|
+
{
|
|
45
|
+
$project: {
|
|
46
|
+
_id: 0,
|
|
47
|
+
fields: {
|
|
48
|
+
$map: {
|
|
49
|
+
input: { $objectToArray: '$$ROOT' },
|
|
50
|
+
as: 'field',
|
|
51
|
+
in: {
|
|
52
|
+
name: '$$field.k',
|
|
53
|
+
type: {
|
|
54
|
+
$let: {
|
|
55
|
+
vars: { bsonType: { $type: '$$field.v' } },
|
|
56
|
+
in: {
|
|
57
|
+
$switch: {
|
|
58
|
+
branches: [
|
|
59
|
+
{
|
|
60
|
+
case: { $eq: ['$$bsonType', 'double'] },
|
|
61
|
+
// Whole-number doubles are replicated as SQLite integers.
|
|
62
|
+
then: { $cond: [{ $eq: [{ $mod: ['$$field.v', 1] }, 0] }, 'int', 'double'] }
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
case: { $eq: ['$$bsonType', 'binData'] },
|
|
66
|
+
then: {
|
|
67
|
+
$cond: [
|
|
68
|
+
// BinData compares by length, then subtype, then bytes. Only
|
|
69
|
+
// 16-byte values of subtype 4 fall within this UUID range.
|
|
70
|
+
// This works without transferring binary values or requiring
|
|
71
|
+
// the newer MongoDB binary conversion operators.
|
|
72
|
+
// https://www.mongodb.com/docs/manual/reference/bson-type-comparison-order/#bindata
|
|
73
|
+
{
|
|
74
|
+
$and: [
|
|
75
|
+
{ $gte: ['$$field.v', new mongo.UUID('00000000-0000-0000-0000-000000000000')] },
|
|
76
|
+
{ $lte: ['$$field.v', new mongo.UUID('ffffffff-ffff-ffff-ffff-ffffffffffff')] }
|
|
77
|
+
]
|
|
78
|
+
},
|
|
79
|
+
'uuid',
|
|
80
|
+
'binData'
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
],
|
|
85
|
+
default: '$$bsonType'
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
// Discard values before unwinding so large documents aren't duplicated per field.
|
|
96
|
+
{ $unwind: '$fields' },
|
|
97
|
+
{ $group: { _id: '$fields.name', types: { $addToSet: '$fields.type' } } }
|
|
98
|
+
],
|
|
99
|
+
{
|
|
100
|
+
// Bound execution per collection below the default 60-second socket timeout
|
|
101
|
+
// so MongoDB can return a query timeout before the connection times out.
|
|
102
|
+
maxTimeMS: 30_000,
|
|
103
|
+
// Small collections can require $sample to sort full documents. Disable
|
|
104
|
+
// disk spill explicitly so concurrent schema queries fail at the memory
|
|
105
|
+
// limit without adding temporary-file I/O on the source database.
|
|
106
|
+
allowDiskUse: false,
|
|
107
|
+
// Field names are case-sensitive even when the collection's default collation isn't.
|
|
108
|
+
// DocumentDB rejects the collation option, including simple collation.
|
|
109
|
+
...(isDocumentDb ? {} : { collation: { locale: 'simple' } })
|
|
110
|
+
}
|
|
111
|
+
)
|
|
112
|
+
.toArray();
|
|
113
|
+
|
|
114
|
+
return fields
|
|
115
|
+
.map(({ _id: name, types }) => {
|
|
116
|
+
let sqliteType = ExpressionType.NONE;
|
|
117
|
+
const bsonTypes = new Set<string>();
|
|
118
|
+
for (const type of types) {
|
|
119
|
+
const inferred = BSON_TYPES[type];
|
|
120
|
+
sqliteType = sqliteType.or(inferred.sqliteType);
|
|
121
|
+
bsonTypes.add(inferred.name);
|
|
122
|
+
}
|
|
123
|
+
const internal_type = [...bsonTypes].sort().join(' | ');
|
|
124
|
+
return { name, type: internal_type, sqlite_type: sqliteType.typeFlags, internal_type, pg_type: internal_type };
|
|
125
|
+
})
|
|
126
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
127
|
+
}
|
|
@@ -213,7 +213,7 @@ export class ChangeStream {
|
|
|
213
213
|
this.isDocumentDb = await detectDocumentDb(this.defaultDb);
|
|
214
214
|
if (this.isDocumentDb) {
|
|
215
215
|
this.logger.warn(
|
|
216
|
-
'Azure DocumentDB support is
|
|
216
|
+
'Azure DocumentDB support is in alpha. APIs and behavior may change, and long-term stability is not yet guaranteed.'
|
|
217
217
|
);
|
|
218
218
|
}
|
|
219
219
|
this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
|
|
@@ -595,9 +595,6 @@ export class ChangeStream {
|
|
|
595
595
|
return rawChangeStream(watchDb, pipeline, {
|
|
596
596
|
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
597
597
|
maxAwaitTimeMS,
|
|
598
|
-
// maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
|
|
599
|
-
// In that case there is no client-side wait to emulate for DocumentDB.
|
|
600
|
-
clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
|
|
601
598
|
maxTimeMS: this.changeStreamTimeout,
|
|
602
599
|
|
|
603
600
|
signal: options.signal,
|
|
@@ -633,6 +630,7 @@ export class ChangeStream {
|
|
|
633
630
|
// We get a complete postimage for every change, so we don't need to store the current data.
|
|
634
631
|
storeCurrentData: false,
|
|
635
632
|
hooks: this.storageHooks,
|
|
633
|
+
signal: this.abortSignal,
|
|
636
634
|
tracer
|
|
637
635
|
},
|
|
638
636
|
async (batch) => {
|
|
@@ -212,6 +212,7 @@ export class MongoSnapshotter {
|
|
|
212
212
|
defaultSchema: this.defaultDb.databaseName,
|
|
213
213
|
storeCurrentData: false,
|
|
214
214
|
skipExistingRows: true,
|
|
215
|
+
signal: this.abortSignal,
|
|
215
216
|
tracer: new PerformanceTracer('MongoDB initial snapshot setup')
|
|
216
217
|
});
|
|
217
218
|
if (snapshotLsn == null) {
|
|
@@ -345,10 +346,10 @@ export class MongoSnapshotter {
|
|
|
345
346
|
return;
|
|
346
347
|
}
|
|
347
348
|
|
|
348
|
-
//
|
|
349
|
+
// Compact storage _after_ initial replication, but _before_ we switch to this replication stream.
|
|
349
350
|
// Keeping snapshot_done false until this completes makes this resumable after interruption.
|
|
350
351
|
// No checkpoint exists yet - storage defaults to its highest persisted op id.
|
|
351
|
-
await this.storage.
|
|
352
|
+
await this.storage.compactInitialReplication({
|
|
352
353
|
signal: this.abortSignal
|
|
353
354
|
});
|
|
354
355
|
|
|
@@ -361,7 +362,8 @@ export class MongoSnapshotter {
|
|
|
361
362
|
zeroLSN: MongoLSN.ZERO.comparable,
|
|
362
363
|
defaultSchema: this.defaultDb.databaseName,
|
|
363
364
|
storeCurrentData: false,
|
|
364
|
-
skipExistingRows: true
|
|
365
|
+
skipExistingRows: true,
|
|
366
|
+
signal: this.abortSignal
|
|
365
367
|
});
|
|
366
368
|
|
|
367
369
|
// The checkpoint here is a marker - we need to replicate up to at least this
|
|
@@ -391,6 +393,7 @@ export class MongoSnapshotter {
|
|
|
391
393
|
storeCurrentData: false,
|
|
392
394
|
skipExistingRows: true,
|
|
393
395
|
hooks: this.storageHooks,
|
|
396
|
+
signal: this.abortSignal,
|
|
394
397
|
tracer: new PerformanceTracer('MongoDB snapshot table')
|
|
395
398
|
});
|
|
396
399
|
// Get fresh table info, in case it was updated while queuing.
|
|
@@ -782,9 +785,6 @@ export class MongoSnapshotter {
|
|
|
782
785
|
return rawChangeStream(watchDb, pipeline, {
|
|
783
786
|
batchSize: options.batchSize ?? this.snapshotChunkLength,
|
|
784
787
|
maxAwaitTimeMS,
|
|
785
|
-
// maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
|
|
786
|
-
// In that case there is no client-side wait to emulate for DocumentDB.
|
|
787
|
-
clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
|
|
788
788
|
maxTimeMS: this.changeStreamTimeout,
|
|
789
789
|
signal: options.signal,
|
|
790
790
|
logger: this.logger,
|
|
@@ -6,14 +6,8 @@ import {
|
|
|
6
6
|
ReplicationAssertionError
|
|
7
7
|
} from '@powersync/lib-services-framework';
|
|
8
8
|
import { PerformanceTracer } from '@powersync/service-core';
|
|
9
|
-
import { performance } from 'node:perf_hooks';
|
|
10
|
-
import { setTimeout as delay } from 'node:timers/promises';
|
|
11
9
|
import { ChangeStreamInvalidatedError } from './ChangeStream.js';
|
|
12
10
|
|
|
13
|
-
// Keep the DocumentDB idle-poll workaround from adding the full maxAwaitTimeMS
|
|
14
|
-
// as local latency when an update arrives just after an empty batch.
|
|
15
|
-
const CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS = 1_000;
|
|
16
|
-
|
|
17
11
|
export interface RawChangeStreamOptions {
|
|
18
12
|
signal?: AbortSignal;
|
|
19
13
|
|
|
@@ -21,21 +15,12 @@ export interface RawChangeStreamOptions {
|
|
|
21
15
|
* How long to wait for new data per batch (max time for long-polling).
|
|
22
16
|
* This is sent as maxTimeMS for the getMore command.
|
|
23
17
|
*
|
|
24
|
-
* A value of 0
|
|
25
|
-
*
|
|
18
|
+
* A value of 0 removes the explicit await limit and leaves the server's
|
|
19
|
+
* default awaitData behavior in effect; it does not make getMore return
|
|
20
|
+
* immediately. Snapshot probes use 0 for this purpose.
|
|
26
21
|
*/
|
|
27
22
|
maxAwaitTimeMS: number;
|
|
28
23
|
|
|
29
|
-
/**
|
|
30
|
-
* Also enforce maxAwaitTimeMS on the client for empty getMore batches.
|
|
31
|
-
*
|
|
32
|
-
* Azure DocumentDB currently returns idle getMore calls before maxTimeMS. When
|
|
33
|
-
* this is enabled, empty batches are delayed locally (capped at 1s) to avoid
|
|
34
|
-
* tight polling. We still send maxTimeMS so this remains compatible with
|
|
35
|
-
* servers that handle maxAwaitTimeMS correctly.
|
|
36
|
-
*/
|
|
37
|
-
clientSideMaxAwaitTimeMS?: boolean;
|
|
38
|
-
|
|
39
24
|
/**
|
|
40
25
|
* Timeout for the initial aggregate command.
|
|
41
26
|
*/
|
|
@@ -230,17 +215,12 @@ async function* rawChangeStreamInner(
|
|
|
230
215
|
options.signal?.throwIfAborted();
|
|
231
216
|
|
|
232
217
|
using commandSpan = options.tracer?.span('changestream', 'getmore');
|
|
233
|
-
const getMoreStartedAt = performance.now();
|
|
234
218
|
const getMoreCommand: mongo.Document = {
|
|
235
219
|
getMore: cursorId,
|
|
236
220
|
collection: nsCollection,
|
|
237
221
|
batchSize: batchSizer.next(),
|
|
238
222
|
maxTimeMS: options.maxAwaitTimeMS
|
|
239
223
|
};
|
|
240
|
-
// Azure DocumentDB currently returns empty getMore batches before
|
|
241
|
-
// maxTimeMS expires. Keep maxTimeMS for forward compatibility with the
|
|
242
|
-
// server-side behavior, and when client-side mode is enabled, enforce the
|
|
243
|
-
// capped idle wait locally for empty batches below.
|
|
244
224
|
const getMoreResult: mongo.Document = await db.command(getMoreCommand, { session, raw: true }).catch((e) => {
|
|
245
225
|
if (isMongoServerError(e) && e.codeName == 'CursorKilled') {
|
|
246
226
|
// This may be due to the killCursors command issued when aborting.
|
|
@@ -268,13 +248,6 @@ async function* rawChangeStreamInner(
|
|
|
268
248
|
// postBatchResumeToken is returned in MongoDB 4.0.7 and later, and we support 6.0+
|
|
269
249
|
throw new ReplicationAssertionError(`postBatchResumeToken from aggregate response`);
|
|
270
250
|
}
|
|
271
|
-
if (options.clientSideMaxAwaitTimeMS && nextBatch.length == 0) {
|
|
272
|
-
const remainingMaxAwaitTimeMS = Math.ceil(options.maxAwaitTimeMS - (performance.now() - getMoreStartedAt));
|
|
273
|
-
if (remainingMaxAwaitTimeMS > 0) {
|
|
274
|
-
const clientSideDelayMs = Math.min(remainingMaxAwaitTimeMS, CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS);
|
|
275
|
-
await delay(clientSideDelayMs, undefined, { signal: options.signal });
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
251
|
yield {
|
|
279
252
|
events: nextBatch,
|
|
280
253
|
resumeToken: cursor.postBatchResumeToken,
|
package/src/types/types.ts
CHANGED
|
@@ -67,11 +67,11 @@ export interface NormalizedMongoConnectionConfig {
|
|
|
67
67
|
export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
|
|
68
68
|
t.object({
|
|
69
69
|
// Replication specific settings
|
|
70
|
-
post_images:
|
|
70
|
+
post_images: service_types.enumLiteral('off', 'auto_configure', 'read_only').optional(),
|
|
71
71
|
/**
|
|
72
72
|
* Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
|
|
73
73
|
*/
|
|
74
|
-
heartbeat_interval_seconds:
|
|
74
|
+
heartbeat_interval_seconds: service_types.orNull(t.number).optional()
|
|
75
75
|
})
|
|
76
76
|
);
|
|
77
77
|
|
|
@@ -622,6 +622,79 @@ bucket_definitions:
|
|
|
622
622
|
expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
|
|
623
623
|
});
|
|
624
624
|
|
|
625
|
+
test.runIf(storageVersion >= 3)(
|
|
626
|
+
'does not resnapshot an unchanged event definition during incremental reprocessing',
|
|
627
|
+
async (testContext) => {
|
|
628
|
+
const eventDefinition = `
|
|
629
|
+
event_definitions:
|
|
630
|
+
checkpoint_requests:
|
|
631
|
+
payloads:
|
|
632
|
+
- SELECT user_id, checkpoint, client_id, CASE WHEN checkpoint_requested_at IS NULL THEN true ELSE false END AS is_legacy FROM checkpoints
|
|
633
|
+
`;
|
|
634
|
+
const syncConfig = (deploymentComment: string) => `
|
|
635
|
+
config:
|
|
636
|
+
edition: 3
|
|
637
|
+
storage_version: 3
|
|
638
|
+
# ${deploymentComment}
|
|
639
|
+
|
|
640
|
+
${eventDefinition}
|
|
641
|
+
streams:
|
|
642
|
+
global:
|
|
643
|
+
auto_subscribe: true
|
|
644
|
+
queries:
|
|
645
|
+
- SELECT *, _id AS id FROM lists
|
|
646
|
+
- SELECT *, _id AS id FROM todos
|
|
647
|
+
`;
|
|
648
|
+
|
|
649
|
+
let replicationStreamId: number;
|
|
650
|
+
{
|
|
651
|
+
await using context = await openContext();
|
|
652
|
+
const firstStorage = await context.updateSyncRules(syncConfig('initial deployment'));
|
|
653
|
+
if (
|
|
654
|
+
!('incrementalReprocessing' in firstStorage.storageConfig) ||
|
|
655
|
+
firstStorage.storageConfig.incrementalReprocessing !== true
|
|
656
|
+
) {
|
|
657
|
+
testContext.skip('storage does not support incremental reprocessing');
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const { db } = context;
|
|
661
|
+
await db.collection('lists').insertOne({ name: 'list' });
|
|
662
|
+
await db.collection('todos').insertOne({ description: 'todo' });
|
|
663
|
+
await db.collection('checkpoints').insertOne({
|
|
664
|
+
user_id: 'user-1',
|
|
665
|
+
checkpoint: 1n,
|
|
666
|
+
client_id: 'client-1',
|
|
667
|
+
checkpoint_requested_at: null
|
|
668
|
+
});
|
|
669
|
+
|
|
670
|
+
replicationStreamId = firstStorage.replicationStreamId;
|
|
671
|
+
await context.replicateSnapshot();
|
|
672
|
+
context.startStreaming();
|
|
673
|
+
await context.getCheckpoint();
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
const resnapshottedCollections: string[] = [];
|
|
677
|
+
await using context = await openContext({
|
|
678
|
+
doNotClear: true,
|
|
679
|
+
streamOptions: {
|
|
680
|
+
snapshotHooks: {
|
|
681
|
+
beforeSnapshotStarted: async (table) => {
|
|
682
|
+
resnapshottedCollections.push(table.name);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
const processingStorage = await context.updateSyncRules(syncConfig('redeployed without behavioral changes'));
|
|
689
|
+
expect(processingStorage.replicationStreamId).toBe(replicationStreamId);
|
|
690
|
+
expect(await context.factory.getDeployingSyncConfig()).not.toBeNull();
|
|
691
|
+
|
|
692
|
+
await context.replicateSnapshot();
|
|
693
|
+
|
|
694
|
+
expect(resnapshottedCollections).toEqual([]);
|
|
695
|
+
}
|
|
696
|
+
);
|
|
697
|
+
|
|
625
698
|
test('coalesces standalone checkpoints when backlog is buffered', async () => {
|
|
626
699
|
await using context = await openContext();
|
|
627
700
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
@@ -759,8 +759,7 @@ bucket_definitions:
|
|
|
759
759
|
expect(JSON.parse(lastOp.data as string)).toMatchObject({ description: 'after_keepalive' });
|
|
760
760
|
});
|
|
761
761
|
|
|
762
|
-
|
|
763
|
-
test.skip('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
|
|
762
|
+
test('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
|
|
764
763
|
const maxAwaitTimeMS = 2_000;
|
|
765
764
|
|
|
766
765
|
await using context = await openContext({
|