@powersync/service-module-mongodb 0.0.0-dev-20241001150444 → 0.0.0-dev-20241002180742

package/src/api/MongoRouteAPIAdapter.ts

@@ -1,15 +1,16 @@
-import { api, ParseSyncRulesOptions } from '@powersync/service-core';
+import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core';
 import * as mongo from 'mongodb';
 
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
 import * as types from '../types/types.js';
 import { MongoManager } from '../replication/MongoManager.js';
-import { createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js';
+import { constructAfterRecord, createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js';
+import { escapeRegExp } from '../utils.js';
 
 export class MongoRouteAPIAdapter implements api.RouteAPI {
   protected client: mongo.MongoClient;
-  private db: mongo.Db;
+  public db: mongo.Db;
 
   connectionTag: string;
   defaultSchema: string;
@@ -37,11 +38,21 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
   }
 
   async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
-    // TODO: Implement
     const base = {
       id: this.config.id,
       uri: types.baseUri(this.config)
     };
+
+    try {
+      await this.client.connect();
+      await this.db.command({ hello: 1 });
+    } catch (e) {
+      return {
+        ...base,
+        connected: false,
+        errors: [{ level: 'fatal', message: e.message }]
+      };
+    }
     return {
       ...base,
       connected: true,
@@ -64,14 +75,100 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
     tablePatterns: sync_rules.TablePattern[],
     sqlSyncRules: sync_rules.SqlSyncRules
   ): Promise<api.PatternResult[]> {
-    // TODO: Implement
-    return [];
-  }
+    let result: api.PatternResult[] = [];
+    for (let tablePattern of tablePatterns) {
+      const schema = tablePattern.schema;
+
+      let patternResult: api.PatternResult = {
+        schema: schema,
+        pattern: tablePattern.tablePattern,
+        wildcard: tablePattern.isWildcard
+      };
+      result.push(patternResult);
+
+      let nameFilter: RegExp | string;
+      if (tablePattern.isWildcard) {
+        nameFilter = new RegExp('^' + escapeRegExp(tablePattern.tablePrefix));
+      } else {
+        nameFilter = tablePattern.name;
+      }
+
+      // Check if the collection exists
+      const collections = await this.client
+        .db(schema)
+        .listCollections(
+          {
+            name: nameFilter
+          },
+          { nameOnly: true }
+        )
+        .toArray();
 
-  async getReplicationLag(syncRulesId: string): Promise<number> {
-    // TODO: Implement
+      if (tablePattern.isWildcard) {
+        patternResult.tables = [];
+        for (let collection of collections) {
+          const sourceTable = new SourceTable(
+            0,
+            this.connectionTag,
+            collection.name,
+            schema,
+            collection.name,
+            [],
+            true
+          );
+          const syncData = sqlSyncRules.tableSyncsData(sourceTable);
+          const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable);
+          patternResult.tables.push({
+            schema,
+            name: collection.name,
+            replication_id: ['_id'],
+            data_queries: syncData,
+            parameter_queries: syncParameters,
+            errors: []
+          });
+        }
+      } else {
+        const sourceTable = new SourceTable(
+          0,
+          this.connectionTag,
+          tablePattern.name,
+          schema,
+          tablePattern.name,
+          [],
+          true
+        );
 
-    return 0;
+        const syncData = sqlSyncRules.tableSyncsData(sourceTable);
+        const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable);
+
+        if (collections.length == 1) {
+          patternResult.table = {
+            schema,
+            name: tablePattern.name,
+            replication_id: ['_id'],
+            data_queries: syncData,
+            parameter_queries: syncParameters,
+            errors: []
+          };
+        } else {
+          patternResult.table = {
+            schema,
+            name: tablePattern.name,
+            replication_id: ['_id'],
+            data_queries: syncData,
+            parameter_queries: syncParameters,
+            errors: [{ level: 'warning', message: `Collection ${schema}.${tablePattern.name} not found` }]
+          };
+        }
+      }
+    }
+    return result;
+  }
+
+  async getReplicationLag(syncRulesId: string): Promise<number | undefined> {
+    // There is no fast way to get replication lag in bytes in MongoDB.
+    // We can get replication lag in seconds, but need a different API for that.
+    return undefined;
   }
 
   async getReplicationHead(): Promise<string> {
@@ -79,8 +176,117 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
   }
 
   async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
-    // TODO: Implement
+    const sampleSize = 50;
+
+    const databases = await this.db.admin().listDatabases({ authorizedDatabases: true, nameOnly: true });
+    const filteredDatabases = databases.databases.filter((db) => {
+      return !['local', 'admin', 'config'].includes(db.name);
+    });
+    return await Promise.all(
+      filteredDatabases.map(async (db) => {
+        const collections = await this.client.db(db.name).listCollections().toArray();
+        const filtered = collections.filter((c) => {
+          return !['_powersync_checkpoints'].includes(c.name);
+        });
+
+        const tables = await Promise.all(
+          filtered.map(async (collection) => {
+            const sampleDocuments = await this.db
+              .collection(collection.name)
+              .aggregate([{ $sample: { size: sampleSize } }])
+              .toArray();
+
+            if (sampleDocuments.length > 0) {
+              const columns = this.getColumnsFromDocuments(sampleDocuments);
+
+              return {
+                name: collection.name,
+                // Since documents are sampled in a random order, we need to sort
+                // to get a consistent order
+                columns: columns.sort((a, b) => a.name.localeCompare(b.name))
+              } satisfies service_types.TableSchema;
+            } else {
+              return {
+                name: collection.name,
+                columns: []
+              } satisfies service_types.TableSchema;
+            }
+          })
+        );
+        return {
+          name: db.name,
+          tables: tables
+        } satisfies service_types.DatabaseSchema;
+      })
+    );
+  }
+
+  private getColumnsFromDocuments(documents: mongo.BSON.Document[]) {
+    let columns = new Map<string, { sqliteType: sync_rules.ExpressionType; bsonTypes: Set<string> }>();
+    for (const document of documents) {
+      const parsed = constructAfterRecord(document);
+      for (const key in parsed) {
+        const value = parsed[key];
+        const type = sync_rules.sqliteTypeOf(value);
+        const sqliteType = sync_rules.ExpressionType.fromTypeText(type);
+        let entry = columns.get(key);
+        if (entry == null) {
+          entry = { sqliteType, bsonTypes: new Set() };
+          columns.set(key, entry);
+        } else {
+          entry.sqliteType = entry.sqliteType.or(sqliteType);
+        }
+        const bsonType = this.getBsonType(document[key]);
+        if (bsonType != null) {
+          entry.bsonTypes.add(bsonType);
+        }
+      }
+    }
+    return [...columns.entries()].map(([key, value]) => {
+      return {
+        name: key,
+        sqlite_type: value.sqliteType.typeFlags,
+        internal_type: value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ')
+      };
+    });
+  }
 
-    return [];
+  private getBsonType(data: any): string | null {
+    if (data == null) {
+      // null or undefined
+      return 'Null';
+    } else if (typeof data == 'string') {
+      return 'String';
+    } else if (typeof data == 'number') {
+      if (Number.isInteger(data)) {
+        return 'Integer';
+      } else {
+        return 'Double';
+      }
+    } else if (typeof data == 'bigint') {
+      return 'Long';
+    } else if (typeof data == 'boolean') {
+      return 'Boolean';
+    } else if (data instanceof mongo.ObjectId) {
+      return 'ObjectId';
+    } else if (data instanceof mongo.UUID) {
+      return 'UUID';
+    } else if (data instanceof Date) {
+      return 'Date';
+    } else if (data instanceof mongo.Timestamp) {
+      return 'Timestamp';
+    } else if (data instanceof mongo.Binary) {
+      return 'Binary';
+    } else if (data instanceof mongo.Long) {
+      return 'Long';
+    } else if (Array.isArray(data)) {
+      return 'Array';
+    } else if (data instanceof Uint8Array) {
+      return 'Binary';
+    } else if (typeof data == 'object') {
+      return 'Object';
+    } else {
+      return null;
+    }
   }
 }

package/src/replication/ChangeStream.ts

@@ -10,6 +10,7 @@ import {
   getMongoRelation,
   mongoLsnToTimestamp
 } from './MongoRelation.js';
+import { escapeRegExp } from '../utils.js';
 
 export const ZERO_LSN = '0000000000000000';
 
@@ -496,8 +497,3 @@ async function touch() {
   // or reduce PING_INTERVAL here.
   return container.probes.touch();
 }
-
-function escapeRegExp(string: string) {
-  // https://stackoverflow.com/a/3561711/214837
-  return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
-}

package/src/utils.ts

@@ -0,0 +1,4 @@
+export function escapeRegExp(string: string) {
+  // https://stackoverflow.com/a/3561711/214837
+  return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&');
+}

package/test/src/mongo_test.test.ts

@@ -1,9 +1,10 @@
+import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
 import { ChangeStream } from '@module/replication/ChangeStream.js';
+import { constructAfterRecord } from '@module/replication/MongoRelation.js';
+import { SqliteRow } from '@powersync/service-sync-rules';
 import * as mongo from 'mongodb';
 import { describe, expect, test } from 'vitest';
-import { clearTestDb, connectMongoData } from './util.js';
-import { SqliteRow } from '@powersync/service-sync-rules';
-import { constructAfterRecord } from '@module/replication/MongoRelation.js';
+import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js';
 
 describe('mongo data types', () => {
   async function setupTable(db: mongo.Db) {
@@ -202,6 +203,48 @@ describe('mongo data types', () => {
       await client.close();
     }
   });
+
+  test('connection schema', async () => {
+    const adapter = new MongoRouteAPIAdapter({
+      type: 'mongodb',
+      ...TEST_CONNECTION_OPTIONS
+    });
+    try {
+      const db = adapter.db;
+      await clearTestDb(db);
+
+      const collection = db.collection('test_data');
+      await setupTable(db);
+      await insert(collection);
+
+      const schema = await adapter.getConnectionSchema();
+      const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
+      expect(dbSchema).not.toBeNull();
+      expect(dbSchema.tables).toEqual([
+        {
+          name: 'test_data',
+          columns: [
+            { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
+            { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' },
+            { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' },
+            { name: 'date', sqlite_type: 2, internal_type: 'Date' },
+            { name: 'float', sqlite_type: 8, internal_type: 'Double' },
+            { name: 'int2', sqlite_type: 4, internal_type: 'Integer' },
+            { name: 'int4', sqlite_type: 4, internal_type: 'Integer' },
+            { name: 'int8', sqlite_type: 4, internal_type: 'Long' },
+            { name: 'nested', sqlite_type: 2, internal_type: 'Object' },
+            { name: 'null', sqlite_type: 0, internal_type: 'Null' },
+            { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' },
+            { name: 'text', sqlite_type: 2, internal_type: 'String' },
+            { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' },
+            { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
+          ]
+        }
+      ]);
+    } finally {
+      await adapter.shutdown();
+    }
+  });
 });
 
 /**