@powersync/service-core 0.0.0-dev-20240930091342 → 0.0.0-dev-20241001150444

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.0-dev-20240930091342",
+  "version": "0.0.0-dev-20241001150444",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -29,14 +29,15 @@
     "mongodb": "^6.7.0",
     "node-fetch": "^3.3.2",
     "ts-codec": "^1.2.2",
+    "uri-js": "^4.4.1",
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.0.0-dev-20240930091342",
+    "@powersync/lib-services-framework": "0.0.0-dev-20241001150444",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-rsocket-router": "0.0.0-dev-20240930091342",
-    "@powersync/service-sync-rules": "0.0.0-dev-20240930091342",
-    "@powersync/service-types": "0.0.0-dev-20240930091342"
+    "@powersync/service-rsocket-router": "0.0.0-dev-20241001150444",
+    "@powersync/service-sync-rules": "0.0.0-dev-20241001150444",
+    "@powersync/service-types": "0.0.0-dev-20241001150444"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
@@ -44,14 +45,14 @@
     "@types/uuid": "^9.0.4",
     "fastify": "4.23.2",
     "fastify-plugin": "^4.5.1",
-    "typescript": "^5.6.2",
+    "typescript": "^5.2.2",
     "vite-tsconfig-paths": "^4.3.2",
-    "vitest": "^2.1.1"
+    "vitest": "^0.34.6"
   },
   "scripts": {
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
-    "test": "vitest",
+    "test": "vitest --no-threads",
     "clean": "rm -rf ./lib && tsc -b --clean"
   }
 }

package/src/auth/RemoteJWKSCollector.ts

@@ -22,10 +22,7 @@ export type RemoteJWKSCollectorOptions = {
 export class RemoteJWKSCollector implements KeyCollector {
   private url: URL;
 
-  constructor(
-    url: string,
-    protected options?: RemoteJWKSCollectorOptions
-  ) {
+  constructor(url: string, protected options?: RemoteJWKSCollectorOptions) {
     try {
       this.url = new URL(url);
     } catch (e) {

package/src/db/mongo.ts

@@ -2,6 +2,7 @@ import * as mongo from 'mongodb';
 import * as timers from 'timers/promises';
 
 import { configFile } from '@powersync/service-types';
+import { normalizeMongoConfig } from '../storage/storage-index.js';
 
 /**
  * Time for new connection to timeout.
@@ -23,10 +24,11 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000;
 export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
 
 export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
-  return new mongo.MongoClient(config.uri, {
+  const normalized = normalizeMongoConfig(config);
+  return new mongo.MongoClient(normalized.uri, {
     auth: {
-      username: config.username,
-      password: config.password
+      username: normalized.username,
+      password: normalized.password
     },
     // Time for connection to timeout
     connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS,

package/src/storage/BucketStorage.ts

@@ -339,6 +339,11 @@ export interface BucketStorageBatch {
    */
   keepalive(lsn: string): Promise<boolean>;
 
+  /**
+   * Get the last checkpoint LSN, from either commit or keepalive.
+   */
+  lastCheckpointLsn: string | null;
+
   markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
 }
 

package/src/storage/mongo/MongoBucketBatch.ts

@@ -7,7 +7,7 @@ import * as util from '../../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
 import { PowerSyncMongo } from './db.js';
-import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js';
+import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
@@ -73,6 +73,10 @@ export class MongoBucketBatch implements BucketStorageBatch {
     this.no_checkpoint_before_lsn = no_checkpoint_before_lsn;
   }
 
+  get lastCheckpointLsn() {
+    return this.last_checkpoint_lsn;
+  }
+
   async flush(): Promise<FlushedResult | null> {
     let result: FlushedResult | null = null;
     // One flush may be split over multiple transactions.
@@ -535,7 +539,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
   async commit(lsn: string): Promise<boolean> {
     await this.flush();
 
-    if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) {
+    if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
       // When re-applying transactions, don't create a new checkpoint until
       // we are past the last transaction.
       logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`);

package/src/storage/mongo/MongoCompactor.ts

@@ -58,11 +58,7 @@ export class MongoCompactor {
   private maxOpId: bigint | undefined;
   private buckets: string[] | undefined;
 
-  constructor(
-    private db: PowerSyncMongo,
-    private group_id: number,
-    options?: MongoCompactOptions
-  ) {
+  constructor(private db: PowerSyncMongo, private group_id: number, options?: MongoCompactOptions) {
     this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
     this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
     this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;

package/src/storage/mongo/MongoPersistedSyncRulesContent.ts

@@ -19,10 +19,7 @@ export class MongoPersistedSyncRulesContent implements PersistedSyncRulesContent
 
   public current_lock: MongoSyncRulesLock | null = null;
 
-  constructor(
-    private db: PowerSyncMongo,
-    doc: mongo.WithId<SyncRuleDocument>
-  ) {
+  constructor(private db: PowerSyncMongo, doc: mongo.WithId<SyncRuleDocument>) {
     this.id = doc._id;
     this.sync_rules_content = doc.content;
     this.last_checkpoint_lsn = doc.last_checkpoint_lsn;

package/src/storage/mongo/MongoSyncRulesLock.ts

@@ -9,7 +9,7 @@ import { logger } from '@powersync/lib-services-framework';
  * replicates those sync rules at a time.
  */
 export class MongoSyncRulesLock implements ReplicationLock {
-  private readonly refreshInterval: NodeJS.Timeout;
+  private readonly refreshInterval: NodeJS.Timer;
 
   static async createLock(db: PowerSyncMongo, sync_rules: PersistedSyncRulesContent): Promise<MongoSyncRulesLock> {
     const lockId = crypto.randomBytes(8).toString('hex');
@@ -35,11 +35,7 @@ export class MongoSyncRulesLock implements ReplicationLock {
     return new MongoSyncRulesLock(db, sync_rules.id, lockId);
   }
 
-  constructor(
-    private db: PowerSyncMongo,
-    public sync_rules_id: number,
-    private lock_id: string
-  ) {
+  constructor(private db: PowerSyncMongo, public sync_rules_id: number, private lock_id: string) {
     this.refreshInterval = setInterval(async () => {
       try {
         await this.refresh();

package/src/storage/mongo/PersistedBatch.ts

@@ -54,10 +54,7 @@ export class PersistedBatch {
    */
   currentSize = 0;
 
-  constructor(
-    private group_id: number,
-    writtenSize: number
-  ) {
+  constructor(private group_id: number, writtenSize: number) {
     this.currentSize = writtenSize;
   }
 

package/src/storage/mongo/config.ts

@@ -0,0 +1,40 @@
+import * as urijs from 'uri-js';
+
+export interface MongoConnectionConfig {
+  uri: string;
+  username?: string;
+  password?: string;
+  database?: string;
+}
+
+/**
+ * Validate and normalize connection options.
+ *
+ * Returns destructured options.
+ *
+ * For use by both storage and mongo module.
+ */
+export function normalizeMongoConfig(options: MongoConnectionConfig) {
+  let uri = urijs.parse(options.uri);
+
+  const database = options.database ?? uri.path?.substring(1) ?? '';
+
+  const userInfo = uri.userinfo?.split(':');
+
+  const username = options.username ?? userInfo?.[0];
+  const password = options.password ?? userInfo?.[1];
+
+  if (database == '') {
+    throw new Error(`database required`);
+  }
+
+  delete uri.userinfo;
+
+  return {
+    uri: urijs.serialize(uri),
+    database,
+
+    username,
+    password
+  };
+}

package/src/storage/storage-index.ts

@@ -16,3 +16,4 @@ export * from './mongo/MongoSyncRulesLock.js';
 export * from './mongo/OperationBatch.js';
 export * from './mongo/PersistedBatch.js';
 export * from './mongo/util.js';
+export * from './mongo/config.js';

package/test/src/__snapshots__/sync.test.ts.snap

@@ -56,7 +56,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
       "data": [
         {
           "checksum": 1859363232n,
-          "data": "{"id":"t1","description":"Test 1b"}",
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1b\\"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -65,7 +65,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
         },
         {
           "checksum": 3028503153n,
-          "data": "{"id":"t2","description":"Test 2b"}",
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2b\\"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
@@ -146,7 +146,7 @@ exports[`sync - mongodb > sync global data 1`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -155,7 +155,7 @@ exports[`sync - mongodb > sync global data 1`] = `
         },
         {
           "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
@@ -199,7 +199,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "checksum": 3442149460n,
           "data": {
             "description": "Test
-"string"",
+\\"string\\"",
             "id": "t1",
             "large_num": 12345678901234567890n,
           },
@@ -268,7 +268,7 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{"id":"t1","description":"Test 1"}",
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -311,7 +311,7 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
       "data": [
         {
           "checksum": 3280762209n,
-          "data": "{"id":"t2","description":"Test 2"}",
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",