@powersync/service-module-mongodb 0.20.1 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,10 +8,12 @@ import { logger } from '@powersync/lib-services-framework';
8
8
  import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js';
9
9
  import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
10
10
  import { MongoManager } from '../replication/MongoManager.js';
11
- import { constructAfterRecord } from '../replication/MongoRelation.js';
12
11
  import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js';
13
12
  import * as types from '../types/types.js';
14
13
  import { escapeRegExp } from '../utils.js';
14
+ import { inferCollectionSchema } from './infer-collection-schema.js';
15
+
16
+ const SCHEMA_INFERENCE_CONCURRENCY = 4;
15
17
 
16
18
  export class MongoRouteAPIAdapter implements api.RouteAPI {
17
19
  protected client: mongo.MongoClient;
@@ -21,6 +23,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
21
23
  defaultSchema: string;
22
24
 
23
25
  private checkpointImplementation: CheckpointImplementation | null = null;
26
+ private documentDbDetection: Promise<boolean> | null = null;
24
27
 
25
28
  constructor(protected config: types.ResolvedConnectionConfig) {
26
29
  const manager = new MongoManager(config);
@@ -206,9 +209,18 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
206
209
  return checkpointImplementation.createReplicationHead(callback);
207
210
  }
208
211
 
212
+ private isDocumentDb(): Promise<boolean> {
213
+ // Share in-flight detection and cache its result for this adapter's lifetime.
214
+ return (this.documentDbDetection ??= detectDocumentDb(this.db).catch((error) => {
215
+ // Allow a later request to retry after a transient connection error.
216
+ this.documentDbDetection = null;
217
+ throw error;
218
+ }));
219
+ }
220
+
209
221
  private async getCheckpointImplementation(): Promise<CheckpointImplementation> {
210
222
  if (this.checkpointImplementation == null) {
211
- const isDocumentDb = await detectDocumentDb(this.db);
223
+ const isDocumentDb = await this.isDocumentDb();
212
224
  this.checkpointImplementation = createCheckpointImplementation(isDocumentDb, {
213
225
  client: this.client,
214
226
  db: this.db,
@@ -226,32 +238,38 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
226
238
  }
227
239
 
228
240
  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
229
- const sampleSize = 50;
230
-
241
+ const isDocumentDb = await this.isDocumentDb();
231
242
  const databases = await this.db.admin().listDatabases({ nameOnly: true });
232
243
  const filteredDatabases = databases.databases.filter((db) => {
233
244
  return !['local', 'admin', 'config'].includes(db.name);
234
245
  });
235
- const databaseSchemas = await Promise.all(
236
- filteredDatabases.map(async (db) => {
237
- /**
238
- * Filtering the list of database with `authorizedDatabases: true`
239
- * does not produce the full list of databases under some circumstances.
240
- * This catches any potential auth errors.
241
- */
242
- let collections: mongo.CollectionInfo[];
243
- try {
244
- collections = await this.client.db(db.name).listCollections().toArray();
245
- } catch (e) {
246
- if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
247
- // Ignore databases we're not authorized to query
248
- return null;
249
- }
250
- throw e;
246
+ const databaseSchemas: service_types.DatabaseSchema[] = [];
247
+ for (const db of filteredDatabases) {
248
+ /**
249
+ * Filtering the list of database with `authorizedDatabases: true`
250
+ * does not produce the full list of databases under some circumstances.
251
+ * This catches any potential auth errors.
252
+ */
253
+ let collections: mongo.CollectionInfo[];
254
+ try {
255
+ collections = await this.client.db(db.name).listCollections().toArray();
256
+ } catch (e) {
257
+ if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
258
+ // Ignore databases we're not authorized to query
259
+ continue;
251
260
  }
261
+ throw e;
262
+ }
252
263
 
253
- let tables: service_types.TableSchema[] = [];
254
- for (let collection of collections) {
264
+ const tables: (service_types.TableSchema | undefined)[] = new Array(collections.length);
265
+ const pendingCollections = collections.entries();
266
+ let failed = false;
267
+ const inferCollections = async () => {
268
+ // Reading from the iterator automatically manages the queue
269
+ for (const [index, collection] of pendingCollections) {
270
+ if (failed) {
271
+ return;
272
+ }
255
273
  if ([CHECKPOINTS_COLLECTION].includes(collection.name)) {
256
274
  continue;
257
275
  }
@@ -264,121 +282,45 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
264
282
  continue;
265
283
  }
266
284
  try {
267
- const sampleDocuments = await this.db
268
- .collection(collection.name)
269
- .aggregate([{ $sample: { size: sampleSize } }])
270
- .toArray();
271
-
272
- if (sampleDocuments.length > 0) {
273
- const columns = this.getColumnsFromDocuments(sampleDocuments);
274
-
275
- tables.push({
276
- name: collection.name,
277
- // Since documents are sampled in a random order, we need to sort
278
- // to get a consistent order
279
- columns: columns.sort((a, b) => a.name.localeCompare(b.name))
280
- });
281
- } else {
282
- tables.push({
283
- name: collection.name,
284
- columns: []
285
- });
286
- }
285
+ const columns = await inferCollectionSchema(
286
+ this.client.db(db.name).collection(collection.name),
287
+ isDocumentDb
288
+ );
289
+ // Preserve collection order even when queries finish out of order.
290
+ tables[index] = { name: collection.name, columns };
287
291
  } catch (e) {
288
292
  if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
289
293
  // Ignore collections we're not authorized to query
290
294
  continue;
291
295
  }
296
+ // Fail the whole request on unexpected errors: the response cannot indicate
297
+ // partial results, so omitting a collection would make an incomplete schema
298
+ // appear complete and could produce an incorrect generated client schema.
299
+ failed = true;
292
300
  throw e;
293
301
  }
294
302
  }
295
-
296
- return {
297
- name: db.name,
298
- tables: tables
299
- } satisfies service_types.DatabaseSchema;
300
- })
301
- );
302
- return databaseSchemas.filter((schema) => !!schema);
303
- }
304
-
305
- private getColumnsFromDocuments(documents: mongo.BSON.Document[]) {
306
- let columns = new Map<string, { sqliteType: sync_rules.ExpressionType; bsonTypes: Set<string> }>();
307
- for (const document of documents) {
308
- const parsed = constructAfterRecord(document);
309
- for (const key in parsed) {
310
- const value = parsed[key];
311
- const type = sync_rules.sqliteTypeOf(value);
312
- const sqliteType = sync_rules.ExpressionType.fromTypeText(type);
313
- let entry = columns.get(key);
314
- if (entry == null) {
315
- entry = { sqliteType, bsonTypes: new Set() };
316
- columns.set(key, entry);
317
- } else {
318
- entry.sqliteType = entry.sqliteType.or(sqliteType);
319
- }
320
- const bsonType = this.getBsonType(document[key]);
321
- if (bsonType != null) {
322
- entry.bsonTypes.add(bsonType);
323
- }
324
- }
325
- }
326
- return [...columns.entries()].map(([key, value]) => {
327
- const internal_type = value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ');
328
- return {
329
- name: key,
330
- type: internal_type,
331
- sqlite_type: value.sqliteType.typeFlags,
332
- internal_type,
333
- pg_type: internal_type
334
303
  };
335
- });
336
- }
337
304
 
338
- private getBsonType(data: any): string | null {
339
- if (data == null) {
340
- // null or undefined
341
- return 'Null';
342
- } else if (typeof data == 'string') {
343
- return 'String';
344
- } else if (typeof data == 'number') {
345
- if (Number.isInteger(data)) {
346
- return 'Integer';
347
- } else {
348
- return 'Double';
305
+ // Each worker holds only schema metadata, with a bounded number of source queries.
306
+ const workers = Array.from(
307
+ { length: Math.min(SCHEMA_INFERENCE_CONCURRENCY, collections.length) },
308
+ inferCollections
309
+ );
310
+ try {
311
+ await Promise.all(workers);
312
+ } catch (e) {
313
+ // Finish in-flight queries before rejecting the request. Workers stop taking
314
+ // new collections as soon as one encounters a non-authorization error.
315
+ await Promise.allSettled(workers);
316
+ throw e;
349
317
  }
350
- } else if (typeof data == 'bigint') {
351
- return 'Long';
352
- } else if (typeof data == 'boolean') {
353
- return 'Boolean';
354
- } else if (data instanceof mongo.ObjectId) {
355
- return 'ObjectId';
356
- } else if (data instanceof mongo.UUID) {
357
- return 'UUID';
358
- } else if (data instanceof Date) {
359
- return 'Date';
360
- } else if (data instanceof mongo.Timestamp) {
361
- return 'Timestamp';
362
- } else if (data instanceof mongo.Binary) {
363
- return 'Binary';
364
- } else if (data instanceof mongo.Long) {
365
- return 'Long';
366
- } else if (data instanceof RegExp) {
367
- return 'RegExp';
368
- } else if (data instanceof mongo.MinKey) {
369
- return 'MinKey';
370
- } else if (data instanceof mongo.MaxKey) {
371
- return 'MaxKey';
372
- } else if (data instanceof mongo.Decimal128) {
373
- return 'Decimal';
374
- } else if (Array.isArray(data)) {
375
- return 'Array';
376
- } else if (data instanceof Uint8Array) {
377
- return 'Binary';
378
- } else if (typeof data == 'object') {
379
- return 'Object';
380
- } else {
381
- return null;
318
+
319
+ databaseSchemas.push({
320
+ name: db.name,
321
+ tables: tables.filter((table) => table != null)
322
+ });
382
323
  }
324
+ return databaseSchemas;
383
325
  }
384
326
  }
@@ -0,0 +1,127 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { ExpressionType } from '@powersync/service-sync-rules';
3
+ import { TableSchema } from '@powersync/service-types';
4
+
5
+ // Match the types exposed after BSON deserialization and conversion to sync rules values.
6
+ const BSON_TYPES: Record<string, { name: string; sqliteType: ExpressionType }> = {
7
+ int: { name: 'Integer', sqliteType: ExpressionType.INTEGER },
8
+ long: { name: 'Long', sqliteType: ExpressionType.INTEGER },
9
+ double: { name: 'Double', sqliteType: ExpressionType.REAL },
10
+ decimal: { name: 'Decimal', sqliteType: ExpressionType.TEXT },
11
+ string: { name: 'String', sqliteType: ExpressionType.TEXT },
12
+ symbol: { name: 'String', sqliteType: ExpressionType.TEXT },
13
+ bool: { name: 'Boolean', sqliteType: ExpressionType.INTEGER },
14
+ objectId: { name: 'ObjectId', sqliteType: ExpressionType.TEXT },
15
+ uuid: { name: 'UUID', sqliteType: ExpressionType.TEXT },
16
+ binData: { name: 'Binary', sqliteType: ExpressionType.BLOB },
17
+ date: { name: 'Date', sqliteType: ExpressionType.TEXT },
18
+ timestamp: { name: 'Timestamp', sqliteType: ExpressionType.INTEGER },
19
+ regex: { name: 'RegExp', sqliteType: ExpressionType.TEXT },
20
+ object: { name: 'Object', sqliteType: ExpressionType.TEXT },
21
+ array: { name: 'Array', sqliteType: ExpressionType.TEXT },
22
+ javascript: { name: 'Object', sqliteType: ExpressionType.TEXT },
23
+ javascriptWithScope: { name: 'Object', sqliteType: ExpressionType.TEXT },
24
+ dbPointer: { name: 'Object', sqliteType: ExpressionType.TEXT },
25
+ null: { name: 'Null', sqliteType: ExpressionType.NONE },
26
+ undefined: { name: 'Null', sqliteType: ExpressionType.NONE },
27
+ minKey: { name: 'MinKey', sqliteType: ExpressionType.NONE },
28
+ maxKey: { name: 'MaxKey', sqliteType: ExpressionType.NONE }
29
+ };
30
+
31
+ /**
32
+ * Infer top-level fields without transferring sampled document values to the service.
33
+ * Memory on the service scales with the inferred schema, not the size of those values.
34
+ */
35
+ export async function inferCollectionSchema(
36
+ collection: mongo.Collection,
37
+ isDocumentDb: boolean
38
+ ): Promise<TableSchema['columns']> {
39
+ const fields = await collection
40
+ .aggregate<{ _id: string; types: string[] }>(
41
+ [
42
+ // Keep this first so MongoDB can use its random cursor on large collections.
43
+ { $sample: { size: 50 } },
44
+ {
45
+ $project: {
46
+ _id: 0,
47
+ fields: {
48
+ $map: {
49
+ input: { $objectToArray: '$$ROOT' },
50
+ as: 'field',
51
+ in: {
52
+ name: '$$field.k',
53
+ type: {
54
+ $let: {
55
+ vars: { bsonType: { $type: '$$field.v' } },
56
+ in: {
57
+ $switch: {
58
+ branches: [
59
+ {
60
+ case: { $eq: ['$$bsonType', 'double'] },
61
+ // Whole-number doubles are replicated as SQLite integers.
62
+ then: { $cond: [{ $eq: [{ $mod: ['$$field.v', 1] }, 0] }, 'int', 'double'] }
63
+ },
64
+ {
65
+ case: { $eq: ['$$bsonType', 'binData'] },
66
+ then: {
67
+ $cond: [
68
+ // BinData compares by length, then subtype, then bytes. Only
69
+ // 16-byte values of subtype 4 fall within this UUID range.
70
+ // This works without transferring binary values or requiring
71
+ // the newer MongoDB binary conversion operators.
72
+ // https://www.mongodb.com/docs/manual/reference/bson-type-comparison-order/#bindata
73
+ {
74
+ $and: [
75
+ { $gte: ['$$field.v', new mongo.UUID('00000000-0000-0000-0000-000000000000')] },
76
+ { $lte: ['$$field.v', new mongo.UUID('ffffffff-ffff-ffff-ffff-ffffffffffff')] }
77
+ ]
78
+ },
79
+ 'uuid',
80
+ 'binData'
81
+ ]
82
+ }
83
+ }
84
+ ],
85
+ default: '$$bsonType'
86
+ }
87
+ }
88
+ }
89
+ }
90
+ }
91
+ }
92
+ }
93
+ }
94
+ },
95
+ // Discard values before unwinding so large documents aren't duplicated per field.
96
+ { $unwind: '$fields' },
97
+ { $group: { _id: '$fields.name', types: { $addToSet: '$fields.type' } } }
98
+ ],
99
+ {
100
+ // Bound execution per collection below the default 60-second socket timeout
101
+ // so MongoDB can return a query timeout before the connection times out.
102
+ maxTimeMS: 30_000,
103
+ // Small collections can require $sample to sort full documents. Disable
104
+ // disk spill explicitly so concurrent schema queries fail at the memory
105
+ // limit without adding temporary-file I/O on the source database.
106
+ allowDiskUse: false,
107
+ // Field names are case-sensitive even when the collection's default collation isn't.
108
+ // DocumentDB rejects the collation option, including simple collation.
109
+ ...(isDocumentDb ? {} : { collation: { locale: 'simple' } })
110
+ }
111
+ )
112
+ .toArray();
113
+
114
+ return fields
115
+ .map(({ _id: name, types }) => {
116
+ let sqliteType = ExpressionType.NONE;
117
+ const bsonTypes = new Set<string>();
118
+ for (const type of types) {
119
+ const inferred = BSON_TYPES[type];
120
+ sqliteType = sqliteType.or(inferred.sqliteType);
121
+ bsonTypes.add(inferred.name);
122
+ }
123
+ const internal_type = [...bsonTypes].sort().join(' | ');
124
+ return { name, type: internal_type, sqlite_type: sqliteType.typeFlags, internal_type, pg_type: internal_type };
125
+ })
126
+ .sort((a, b) => a.name.localeCompare(b.name));
127
+ }
@@ -213,7 +213,7 @@ export class ChangeStream {
213
213
  this.isDocumentDb = await detectDocumentDb(this.defaultDb);
214
214
  if (this.isDocumentDb) {
215
215
  this.logger.warn(
216
- 'Azure DocumentDB support is experimental. APIs and behavior may change, and long-term stability is not yet guaranteed.'
216
+ 'Azure DocumentDB support is in alpha. APIs and behavior may change, and long-term stability is not yet guaranteed.'
217
217
  );
218
218
  }
219
219
  this._checkpointImplementation = createCheckpointImplementation(this.isDocumentDb, {
@@ -595,9 +595,6 @@ export class ChangeStream {
595
595
  return rawChangeStream(watchDb, pipeline, {
596
596
  batchSize: options.batchSize ?? this.snapshotChunkLength,
597
597
  maxAwaitTimeMS,
598
- // maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
599
- // In that case there is no client-side wait to emulate for DocumentDB.
600
- clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
601
598
  maxTimeMS: this.changeStreamTimeout,
602
599
 
603
600
  signal: options.signal,
@@ -633,6 +630,7 @@ export class ChangeStream {
633
630
  // We get a complete postimage for every change, so we don't need to store the current data.
634
631
  storeCurrentData: false,
635
632
  hooks: this.storageHooks,
633
+ signal: this.abortSignal,
636
634
  tracer
637
635
  },
638
636
  async (batch) => {
@@ -212,6 +212,7 @@ export class MongoSnapshotter {
212
212
  defaultSchema: this.defaultDb.databaseName,
213
213
  storeCurrentData: false,
214
214
  skipExistingRows: true,
215
+ signal: this.abortSignal,
215
216
  tracer: new PerformanceTracer('MongoDB initial snapshot setup')
216
217
  });
217
218
  if (snapshotLsn == null) {
@@ -345,10 +346,10 @@ export class MongoSnapshotter {
345
346
  return;
346
347
  }
347
348
 
348
- // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream.
349
+ // Compact storage _after_ initial replication, but _before_ we switch to this replication stream.
349
350
  // Keeping snapshot_done false until this completes makes this resumable after interruption.
350
351
  // No checkpoint exists yet - storage defaults to its highest persisted op id.
351
- await this.storage.populatePersistentChecksumCache({
352
+ await this.storage.compactInitialReplication({
352
353
  signal: this.abortSignal
353
354
  });
354
355
 
@@ -361,7 +362,8 @@ export class MongoSnapshotter {
361
362
  zeroLSN: MongoLSN.ZERO.comparable,
362
363
  defaultSchema: this.defaultDb.databaseName,
363
364
  storeCurrentData: false,
364
- skipExistingRows: true
365
+ skipExistingRows: true,
366
+ signal: this.abortSignal
365
367
  });
366
368
 
367
369
  // The checkpoint here is a marker - we need to replicate up to at least this
@@ -391,6 +393,7 @@ export class MongoSnapshotter {
391
393
  storeCurrentData: false,
392
394
  skipExistingRows: true,
393
395
  hooks: this.storageHooks,
396
+ signal: this.abortSignal,
394
397
  tracer: new PerformanceTracer('MongoDB snapshot table')
395
398
  });
396
399
  // Get fresh table info, in case it was updated while queuing.
@@ -782,9 +785,6 @@ export class MongoSnapshotter {
782
785
  return rawChangeStream(watchDb, pipeline, {
783
786
  batchSize: options.batchSize ?? this.snapshotChunkLength,
784
787
  maxAwaitTimeMS,
785
- // maxAwaitTimeMS can be 0 for probe-style streams that do not want an idle wait.
786
- // In that case there is no client-side wait to emulate for DocumentDB.
787
- clientSideMaxAwaitTimeMS: this.isDocumentDb && maxAwaitTimeMS > 0,
788
788
  maxTimeMS: this.changeStreamTimeout,
789
789
  signal: options.signal,
790
790
  logger: this.logger,
@@ -6,14 +6,8 @@ import {
6
6
  ReplicationAssertionError
7
7
  } from '@powersync/lib-services-framework';
8
8
  import { PerformanceTracer } from '@powersync/service-core';
9
- import { performance } from 'node:perf_hooks';
10
- import { setTimeout as delay } from 'node:timers/promises';
11
9
  import { ChangeStreamInvalidatedError } from './ChangeStream.js';
12
10
 
13
- // Keep the DocumentDB idle-poll workaround from adding the full maxAwaitTimeMS
14
- // as local latency when an update arrives just after an empty batch.
15
- const CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS = 1_000;
16
-
17
11
  export interface RawChangeStreamOptions {
18
12
  signal?: AbortSignal;
19
13
 
@@ -21,21 +15,12 @@ export interface RawChangeStreamOptions {
21
15
  * How long to wait for new data per batch (max time for long-polling).
22
16
  * This is sent as maxTimeMS for the getMore command.
23
17
  *
24
- * A value of 0 is allowed for probe-style streams that do not want an idle
25
- * wait; in that case PowerSync also skips the local empty-batch delay.
18
+ * A value of 0 removes the explicit await limit and leaves the server's
19
+ * default awaitData behavior in effect; it does not make getMore return
20
+ * immediately. Snapshot probes use 0 for this purpose.
26
21
  */
27
22
  maxAwaitTimeMS: number;
28
23
 
29
- /**
30
- * Also enforce maxAwaitTimeMS on the client for empty getMore batches.
31
- *
32
- * Azure DocumentDB currently returns idle getMore calls before maxTimeMS. When
33
- * this is enabled, empty batches are delayed locally (capped at 1s) to avoid
34
- * tight polling. We still send maxTimeMS so this remains compatible with
35
- * servers that handle maxAwaitTimeMS correctly.
36
- */
37
- clientSideMaxAwaitTimeMS?: boolean;
38
-
39
24
  /**
40
25
  * Timeout for the initial aggregate command.
41
26
  */
@@ -230,17 +215,12 @@ async function* rawChangeStreamInner(
230
215
  options.signal?.throwIfAborted();
231
216
 
232
217
  using commandSpan = options.tracer?.span('changestream', 'getmore');
233
- const getMoreStartedAt = performance.now();
234
218
  const getMoreCommand: mongo.Document = {
235
219
  getMore: cursorId,
236
220
  collection: nsCollection,
237
221
  batchSize: batchSizer.next(),
238
222
  maxTimeMS: options.maxAwaitTimeMS
239
223
  };
240
- // Azure DocumentDB currently returns empty getMore batches before
241
- // maxTimeMS expires. Keep maxTimeMS for forward compatibility with the
242
- // server-side behavior, and when client-side mode is enabled, enforce the
243
- // capped idle wait locally for empty batches below.
244
224
  const getMoreResult: mongo.Document = await db.command(getMoreCommand, { session, raw: true }).catch((e) => {
245
225
  if (isMongoServerError(e) && e.codeName == 'CursorKilled') {
246
226
  // This may be due to the killCursors command issued when aborting.
@@ -268,13 +248,6 @@ async function* rawChangeStreamInner(
268
248
  // postBatchResumeToken is returned in MongoDB 4.0.7 and later, and we support 6.0+
269
249
  throw new ReplicationAssertionError(`postBatchResumeToken from aggregate response`);
270
250
  }
271
- if (options.clientSideMaxAwaitTimeMS && nextBatch.length == 0) {
272
- const remainingMaxAwaitTimeMS = Math.ceil(options.maxAwaitTimeMS - (performance.now() - getMoreStartedAt));
273
- if (remainingMaxAwaitTimeMS > 0) {
274
- const clientSideDelayMs = Math.min(remainingMaxAwaitTimeMS, CLIENT_SIDE_MAX_AWAIT_TIME_MS_DELAY_CAP_MS);
275
- await delay(clientSideDelayMs, undefined, { signal: options.signal });
276
- }
277
- }
278
251
  yield {
279
252
  events: nextBatch,
280
253
  resumeToken: cursor.postBatchResumeToken,
@@ -67,11 +67,11 @@ export interface NormalizedMongoConnectionConfig {
67
67
  export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_mongo.BaseMongoConfig).and(
68
68
  t.object({
69
69
  // Replication specific settings
70
- post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional(),
70
+ post_images: service_types.enumLiteral('off', 'auto_configure', 'read_only').optional(),
71
71
  /**
72
72
  * Interval in seconds between source connection heartbeats. Null or omitted defaults to 60 seconds.
73
73
  */
74
- heartbeat_interval_seconds: t.number.or(t.Null).optional()
74
+ heartbeat_interval_seconds: service_types.orNull(t.number).optional()
75
75
  })
76
76
  );
77
77
 
@@ -622,6 +622,79 @@ bucket_definitions:
622
622
  expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
623
623
  });
624
624
 
625
+ test.runIf(storageVersion >= 3)(
626
+ 'does not resnapshot an unchanged event definition during incremental reprocessing',
627
+ async (testContext) => {
628
+ const eventDefinition = `
629
+ event_definitions:
630
+ checkpoint_requests:
631
+ payloads:
632
+ - SELECT user_id, checkpoint, client_id, CASE WHEN checkpoint_requested_at IS NULL THEN true ELSE false END AS is_legacy FROM checkpoints
633
+ `;
634
+ const syncConfig = (deploymentComment: string) => `
635
+ config:
636
+ edition: 3
637
+ storage_version: 3
638
+ # ${deploymentComment}
639
+
640
+ ${eventDefinition}
641
+ streams:
642
+ global:
643
+ auto_subscribe: true
644
+ queries:
645
+ - SELECT *, _id AS id FROM lists
646
+ - SELECT *, _id AS id FROM todos
647
+ `;
648
+
649
+ let replicationStreamId: number;
650
+ {
651
+ await using context = await openContext();
652
+ const firstStorage = await context.updateSyncRules(syncConfig('initial deployment'));
653
+ if (
654
+ !('incrementalReprocessing' in firstStorage.storageConfig) ||
655
+ firstStorage.storageConfig.incrementalReprocessing !== true
656
+ ) {
657
+ testContext.skip('storage does not support incremental reprocessing');
658
+ }
659
+
660
+ const { db } = context;
661
+ await db.collection('lists').insertOne({ name: 'list' });
662
+ await db.collection('todos').insertOne({ description: 'todo' });
663
+ await db.collection('checkpoints').insertOne({
664
+ user_id: 'user-1',
665
+ checkpoint: 1n,
666
+ client_id: 'client-1',
667
+ checkpoint_requested_at: null
668
+ });
669
+
670
+ replicationStreamId = firstStorage.replicationStreamId;
671
+ await context.replicateSnapshot();
672
+ context.startStreaming();
673
+ await context.getCheckpoint();
674
+ }
675
+
676
+ const resnapshottedCollections: string[] = [];
677
+ await using context = await openContext({
678
+ doNotClear: true,
679
+ streamOptions: {
680
+ snapshotHooks: {
681
+ beforeSnapshotStarted: async (table) => {
682
+ resnapshottedCollections.push(table.name);
683
+ }
684
+ }
685
+ }
686
+ });
687
+
688
+ const processingStorage = await context.updateSyncRules(syncConfig('redeployed without behavioral changes'));
689
+ expect(processingStorage.replicationStreamId).toBe(replicationStreamId);
690
+ expect(await context.factory.getDeployingSyncConfig()).not.toBeNull();
691
+
692
+ await context.replicateSnapshot();
693
+
694
+ expect(resnapshottedCollections).toEqual([]);
695
+ }
696
+ );
697
+
625
698
  test('coalesces standalone checkpoints when backlog is buffered', async () => {
626
699
  await using context = await openContext();
627
700
  await context.updateSyncRules(BASIC_SYNC_RULES);
@@ -759,8 +759,7 @@ bucket_definitions:
759
759
  expect(JSON.parse(lastOp.data as string)).toMatchObject({ description: 'after_keepalive' });
760
760
  });
761
761
 
762
- // Skipped until Azure DocumentDB ships the server-side getMore maxAwaitTimeMS fix.
763
- test.skip('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
762
+ test('respects maxAwaitTimeMS for idle getMore calls in documentDbMode', async () => {
764
763
  const maxAwaitTimeMS = 2_000;
765
764
 
766
765
  await using context = await openContext({