@powersync/service-module-mongodb 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716

package/src/api/MongoRouteAPIAdapter.ts

@@ -8,10 +8,12 @@ import { logger } from '@powersync/lib-services-framework';
 import { CheckpointImplementation } from '../replication/checkpoints/CheckpointImplementation.js';
 import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
 import { MongoManager } from '../replication/MongoManager.js';
-import { constructAfterRecord } from '../replication/MongoRelation.js';
 import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js';
 import * as types from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
+import { inferCollectionSchema, isSchemaInferenceFailure } from './infer-collection-schema.js';
+
+const SCHEMA_INFERENCE_CONCURRENCY = 4;
 
 export class MongoRouteAPIAdapter implements api.RouteAPI {
   protected client: mongo.MongoClient;
@@ -21,6 +23,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
   defaultSchema: string;
 
   private checkpointImplementation: CheckpointImplementation | null = null;
+  private documentDbDetection: Promise<boolean> | null = null;
 
   constructor(protected config: types.ResolvedConnectionConfig) {
     const manager = new MongoManager(config);
@@ -147,7 +150,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
           };
           let errors: service_types.ReplicationError[] = [];
           if (collection.type == 'view') {
-            errors.push({ level: 'warning', message: `Collection ${schema}.${tablePattern.name} is a view` });
+            errors.push({ level: 'warning', message: `Collection ${schema}.${collection.name} is a view` });
           } else {
             errors.push(...validatePostImages(schema, collection));
           }
@@ -206,9 +209,18 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
     return checkpointImplementation.createReplicationHead(callback);
   }
 
+  private isDocumentDb(): Promise<boolean> {
+    // Share in-flight detection and cache its result for this adapter's lifetime.
+    return (this.documentDbDetection ??= detectDocumentDb(this.db).catch((error) => {
+      // Allow a later request to retry after a transient connection error.
+      this.documentDbDetection = null;
+      throw error;
+    }));
+  }
+
   private async getCheckpointImplementation(): Promise<CheckpointImplementation> {
     if (this.checkpointImplementation == null) {
-      const isDocumentDb = await detectDocumentDb(this.db);
+      const isDocumentDb = await this.isDocumentDb();
       this.checkpointImplementation = createCheckpointImplementation(isDocumentDb, {
         client: this.client,
         db: this.db,
@@ -226,32 +238,38 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
   }
 
   async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
-    const sampleSize = 50;
-
+    const isDocumentDb = await this.isDocumentDb();
     const databases = await this.db.admin().listDatabases({ nameOnly: true });
     const filteredDatabases = databases.databases.filter((db) => {
       return !['local', 'admin', 'config'].includes(db.name);
     });
-    const databaseSchemas = await Promise.all(
-      filteredDatabases.map(async (db) => {
-        /**
-         * Filtering the list of database with `authorizedDatabases: true`
-         * does not produce the full list of databases under some circumstances.
-         * This catches any potential auth errors.
-         */
-        let collections: mongo.CollectionInfo[];
-        try {
-          collections = await this.client.db(db.name).listCollections().toArray();
-        } catch (e) {
-          if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
-            // Ignore databases we're not authorized to query
-            return null;
-          }
-          throw e;
+    const databaseSchemas: service_types.DatabaseSchema[] = [];
+    for (const db of filteredDatabases) {
+      /**
+       * Filtering the list of database with `authorizedDatabases: true`
+       * does not produce the full list of databases under some circumstances.
+       * This catches any potential auth errors.
+       */
+      let collections: mongo.CollectionInfo[];
+      try {
+        collections = await this.client.db(db.name).listCollections().toArray();
+      } catch (e) {
+        if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
+          // Ignore databases we're not authorized to query
+          continue;
         }
+        throw e;
+      }
 
-        let tables: service_types.TableSchema[] = [];
-        for (let collection of collections) {
+      const tables: (service_types.TableSchema | undefined)[] = new Array(collections.length);
+      const pendingCollections = collections.entries();
+      let failed = false;
+      const inferCollections = async () => {
+        // Reading from the iterator automatically manages the queue
+        for (const [index, collection] of pendingCollections) {
+          if (failed) {
+            return;
+          }
           if ([CHECKPOINTS_COLLECTION].includes(collection.name)) {
             continue;
           }
@@ -264,121 +282,55 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
             continue;
           }
           try {
-            const sampleDocuments = await this.db
-              .collection(collection.name)
-              .aggregate([{ $sample: { size: sampleSize } }])
-              .toArray();
-
-            if (sampleDocuments.length > 0) {
-              const columns = this.getColumnsFromDocuments(sampleDocuments);
-
-              tables.push({
-                name: collection.name,
-                // Since documents are sampled in a random order, we need to sort
-                // to get a consistent order
-                columns: columns.sort((a, b) => a.name.localeCompare(b.name))
-              });
-            } else {
-              tables.push({
-                name: collection.name,
-                columns: []
-              });
-            }
+            const columns = await inferCollectionSchema(
+              this.client.db(db.name).collection(collection.name),
+              isDocumentDb
+            );
+            // Preserve collection order even when queries finish out of order.
+            tables[index] = { name: collection.name, columns };
           } catch (e) {
             if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') {
               // Ignore collections we're not authorized to query
               continue;
             }
+            if (isSchemaInferenceFailure(e)) {
+              // We know the collection is there, we just could not read its fields. Report it
+              // without columns rather than failing: one collection we cannot sample should not
+              // block validation and deployment for every sync config on the instance. Queries
+              // naming its columns warn about unknown columns; the result is indistinguishable
+              // from an empty collection, so a `SELECT *` query still validates cleanly.
+              logger.warn(`Could not infer the schema of ${db.name}.${collection.name}`, e);
+              tables[index] = { name: collection.name, columns: [] };
+              continue;
+            }
+            // Fail the whole request on unexpected errors: the response cannot indicate
+            // partial results, so omitting a collection would make an incomplete schema
+            // appear complete and could produce an incorrect generated client schema.
+            failed = true;
             throw e;
           }
         }
-
-        return {
-          name: db.name,
-          tables: tables
-        } satisfies service_types.DatabaseSchema;
-      })
-    );
-    return databaseSchemas.filter((schema) => !!schema);
-  }
-
-  private getColumnsFromDocuments(documents: mongo.BSON.Document[]) {
-    let columns = new Map<string, { sqliteType: sync_rules.ExpressionType; bsonTypes: Set<string> }>();
-    for (const document of documents) {
-      const parsed = constructAfterRecord(document);
-      for (const key in parsed) {
-        const value = parsed[key];
-        const type = sync_rules.sqliteTypeOf(value);
-        const sqliteType = sync_rules.ExpressionType.fromTypeText(type);
-        let entry = columns.get(key);
-        if (entry == null) {
-          entry = { sqliteType, bsonTypes: new Set() };
-          columns.set(key, entry);
-        } else {
-          entry.sqliteType = entry.sqliteType.or(sqliteType);
-        }
-        const bsonType = this.getBsonType(document[key]);
-        if (bsonType != null) {
-          entry.bsonTypes.add(bsonType);
-        }
-      }
-    }
-    return [...columns.entries()].map(([key, value]) => {
-      const internal_type = value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ');
-      return {
-        name: key,
-        type: internal_type,
-        sqlite_type: value.sqliteType.typeFlags,
-        internal_type,
-        pg_type: internal_type
       };
-    });
-  }
 
-  private getBsonType(data: any): string | null {
-    if (data == null) {
-      // null or undefined
-      return 'Null';
-    } else if (typeof data == 'string') {
-      return 'String';
-    } else if (typeof data == 'number') {
-      if (Number.isInteger(data)) {
-        return 'Integer';
-      } else {
-        return 'Double';
+      // Each worker holds only schema metadata, with a bounded number of source queries.
+      const workers = Array.from(
+        { length: Math.min(SCHEMA_INFERENCE_CONCURRENCY, collections.length) },
+        inferCollections
+      );
+      try {
+        await Promise.all(workers);
+      } catch (e) {
+        // Finish in-flight queries before rejecting the request. Workers stop taking
+        // new collections as soon as one encounters a non-authorization error.
+        await Promise.allSettled(workers);
+        throw e;
       }
-    } else if (typeof data == 'bigint') {
-      return 'Long';
-    } else if (typeof data == 'boolean') {
-      return 'Boolean';
-    } else if (data instanceof mongo.ObjectId) {
-      return 'ObjectId';
-    } else if (data instanceof mongo.UUID) {
-      return 'UUID';
-    } else if (data instanceof Date) {
-      return 'Date';
-    } else if (data instanceof mongo.Timestamp) {
-      return 'Timestamp';
-    } else if (data instanceof mongo.Binary) {
-      return 'Binary';
-    } else if (data instanceof mongo.Long) {
-      return 'Long';
-    } else if (data instanceof RegExp) {
-      return 'RegExp';
-    } else if (data instanceof mongo.MinKey) {
-      return 'MinKey';
-    } else if (data instanceof mongo.MaxKey) {
-      return 'MaxKey';
-    } else if (data instanceof mongo.Decimal128) {
-      return 'Decimal';
-    } else if (Array.isArray(data)) {
-      return 'Array';
-    } else if (data instanceof Uint8Array) {
-      return 'Binary';
-    } else if (typeof data == 'object') {
-      return 'Object';
-    } else {
-      return null;
+
+      databaseSchemas.push({
+        name: db.name,
+        tables: tables.filter((table) => table != null)
+      });
     }
+    return databaseSchemas;
   }
 }

package/src/api/infer-collection-schema.ts

@@ -0,0 +1,153 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ExpressionType } from '@powersync/service-sync-rules';
+import { TableSchema } from '@powersync/service-types';
+
+/**
+ * Server errors that mean this one collection could not be inspected, rather than the
+ * connection being unusable.
+ *
+ * These come from the memory ceiling {@link inferCollectionSchema} imposes on itself by
+ * refusing to spill to disk, and are deterministic per collection. Timeouts are
+ * deliberately excluded: they usually indicate an overloaded source, where degrading
+ * would silently empty the schema of every collection at once.
+ */
+const SCHEMA_INFERENCE_FAILURE_CODES = new Set([
+  // ExceededMemoryLimit - a blocking stage ran out of memory with disk spill disabled.
+  146,
+  // QueryExceededMemoryLimitNoDiskUseAllowed - as above, reported by the sort $sample
+  // uses on collections too small for its random cursor.
+  292
+]);
+
+/**
+ * Whether the error means we could not determine this collection's fields, as opposed to
+ * something that makes the rest of the schema unreliable too.
+ */
+export function isSchemaInferenceFailure(e: unknown): boolean {
+  return lib_mongo.isMongoServerError(e) && SCHEMA_INFERENCE_FAILURE_CODES.has(e.code as number);
+}
+
+// Match the types exposed after BSON deserialization and conversion to sync rules values.
+const BSON_TYPES: Record<string, { name: string; sqliteType: ExpressionType }> = {
+  int: { name: 'Integer', sqliteType: ExpressionType.INTEGER },
+  long: { name: 'Long', sqliteType: ExpressionType.INTEGER },
+  double: { name: 'Double', sqliteType: ExpressionType.REAL },
+  decimal: { name: 'Decimal', sqliteType: ExpressionType.TEXT },
+  string: { name: 'String', sqliteType: ExpressionType.TEXT },
+  symbol: { name: 'String', sqliteType: ExpressionType.TEXT },
+  bool: { name: 'Boolean', sqliteType: ExpressionType.INTEGER },
+  objectId: { name: 'ObjectId', sqliteType: ExpressionType.TEXT },
+  uuid: { name: 'UUID', sqliteType: ExpressionType.TEXT },
+  binData: { name: 'Binary', sqliteType: ExpressionType.BLOB },
+  date: { name: 'Date', sqliteType: ExpressionType.TEXT },
+  timestamp: { name: 'Timestamp', sqliteType: ExpressionType.INTEGER },
+  regex: { name: 'RegExp', sqliteType: ExpressionType.TEXT },
+  object: { name: 'Object', sqliteType: ExpressionType.TEXT },
+  array: { name: 'Array', sqliteType: ExpressionType.TEXT },
+  javascript: { name: 'Object', sqliteType: ExpressionType.TEXT },
+  javascriptWithScope: { name: 'Object', sqliteType: ExpressionType.TEXT },
+  dbPointer: { name: 'Object', sqliteType: ExpressionType.TEXT },
+  null: { name: 'Null', sqliteType: ExpressionType.NONE },
+  undefined: { name: 'Null', sqliteType: ExpressionType.NONE },
+  minKey: { name: 'MinKey', sqliteType: ExpressionType.NONE },
+  maxKey: { name: 'MaxKey', sqliteType: ExpressionType.NONE }
+};
+
+/**
+ * Infer top-level fields without transferring sampled document values to the service.
+ * Memory on the service scales with the inferred schema, not the size of those values.
+ */
+export async function inferCollectionSchema(
+  collection: mongo.Collection,
+  isDocumentDb: boolean
+): Promise<TableSchema['columns']> {
+  const fields = await collection
+    .aggregate<{ _id: string; types: string[] }>(
+      [
+        // Keep this first so MongoDB can use its random cursor on large collections.
+        { $sample: { size: 50 } },
+        {
+          $project: {
+            _id: 0,
+            fields: {
+              $map: {
+                input: { $objectToArray: '$$ROOT' },
+                as: 'field',
+                in: {
+                  name: '$$field.k',
+                  type: {
+                    $let: {
+                      vars: { bsonType: { $type: '$$field.v' } },
+                      in: {
+                        $switch: {
+                          branches: [
+                            {
+                              case: { $eq: ['$$bsonType', 'double'] },
+                              // Whole-number doubles are replicated as SQLite integers.
+                              then: { $cond: [{ $eq: [{ $mod: ['$$field.v', 1] }, 0] }, 'int', 'double'] }
+                            },
+                            {
+                              case: { $eq: ['$$bsonType', 'binData'] },
+                              then: {
+                                $cond: [
+                                  // BinData compares by length, then subtype, then bytes. Only
+                                  // 16-byte values of subtype 4 fall within this UUID range.
+                                  // This works without transferring binary values or requiring
+                                  // the newer MongoDB binary conversion operators.
+                                  // https://www.mongodb.com/docs/manual/reference/bson-type-comparison-order/#bindata
+                                  {
+                                    $and: [
+                                      { $gte: ['$$field.v', new mongo.UUID('00000000-0000-0000-0000-000000000000')] },
+                                      { $lte: ['$$field.v', new mongo.UUID('ffffffff-ffff-ffff-ffff-ffffffffffff')] }
+                                    ]
+                                  },
+                                  'uuid',
+                                  'binData'
+                                ]
+                              }
+                            }
+                          ],
+                          default: '$$bsonType'
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        // Discard values before unwinding so large documents aren't duplicated per field.
+        { $unwind: '$fields' },
+        { $group: { _id: '$fields.name', types: { $addToSet: '$fields.type' } } }
+      ],
+      {
+        // Bound execution per collection below the default 60-second socket timeout
+        // so MongoDB can return a query timeout before the connection times out.
+        maxTimeMS: 30_000,
+        // Small collections can require $sample to sort full documents. Disable
+        // disk spill explicitly so concurrent schema queries fail at the memory
+        // limit without adding temporary-file I/O on the source database.
+        allowDiskUse: false,
+        // Field names are case-sensitive even when the collection's default collation isn't.
+        // DocumentDB rejects the collation option, including simple collation.
+        ...(isDocumentDb ? {} : { collation: { locale: 'simple' } })
+      }
+    )
+    .toArray();
+
+  return fields
+    .map(({ _id: name, types }) => {
+      let sqliteType = ExpressionType.NONE;
+      const bsonTypes = new Set<string>();
+      for (const type of types) {
+        const inferred = BSON_TYPES[type];
+        sqliteType = sqliteType.or(inferred.sqliteType);
+        bsonTypes.add(inferred.name);
+      }
+      const internal_type = [...bsonTypes].sort().join(' | ');
+      return { name, type: internal_type, sqlite_type: sqliteType.typeFlags, internal_type, pg_type: internal_type };
+    })
+    .sort((a, b) => a.name.localeCompare(b.name));
+}

package/src/index.ts

@@ -1,4 +1,5 @@
 export * from './api/MongoRouteAPIAdapter.js';
 export * from './module/MongoModule.js';
 export * from './replication/replication-index.js';
+export * as test_utils from './test-utils/test-utils-index.js';
 export * from './types/types.js';

package/src/module/MongoModule.ts

@@ -44,12 +44,10 @@ export class MongoModule extends replication.ReplicationModule<types.MongoConnec
     });
   }
 
-  async onInitialized(context: system.ServiceContextContainer): Promise<void> {}
-
   /**
    * Combines base config with normalized connection settings
    */
-  private resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig {
+  protected resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig {
     return {
       ...config,
       ...types.normalizeConnectionConfig(config)