@powersync/service-core 0.0.0-dev-20241015210820 → 0.0.0-dev-20241016143203

package/src/routes/configure-rsocket.ts

@@ -8,7 +8,7 @@ import { ServiceContext } from '../system/ServiceContext.js';
 import { generateContext, getTokenFromHeader } from './auth.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
-import { Context, RouterServiceContext } from './router.js';
+import { Context } from './router.js';
 
 export type RSockerRouterConfig = {
   service_context: ServiceContext;
@@ -36,17 +36,12 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
           if (context?.token_payload == null) {
             throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
           }
-
-          if (!service_context.routerEngine) {
-            throw new Error(`RouterEngine has not been registered`);
-          }
-
           return {
             token,
             user_agent,
             ...context,
             token_errors: token_errors,
-            service_context: service_context as RouterServiceContext
+            service_context
           };
         } else {
           throw new errors.AuthorizationError('No token provided');

package/src/routes/endpoints/admin.ts

@@ -137,8 +137,8 @@ export const reprocess = routeDefinition({
       connections: [
         {
           // Previously the connection was asserted with `!`
-          tag: baseConfig.tag,
-          id: baseConfig.id,
+          tag: baseConfig!.tag!,
+          id: baseConfig!.id,
           slot_name: new_rules.slot_name
         }
       ]

package/src/routes/endpoints/sync-rules.ts

@@ -3,7 +3,6 @@ import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
 import type { FastifyPluginAsync } from 'fastify';
 import * as t from 'ts-codec';
 
-import * as system from '../../system/system-index.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
 import { RouteAPI } from '../../api/RouteAPI.js';

package/src/routes/router.ts

@@ -1,19 +1,13 @@
 import { router } from '@powersync/lib-services-framework';
 import * as auth from '../auth/auth-index.js';
 import { ServiceContext } from '../system/ServiceContext.js';
-import { RouterEngine } from './RouterEngine.js';
 
-/**
- * The {@link RouterEngine} must be provided for these routes
- */
-export type RouterServiceContext = ServiceContext & { routerEngine: RouterEngine };
 /**
  * Common context for routes
  */
 export type Context = {
   user_id?: string;
-
-  service_context: RouterServiceContext;
+  service_context: ServiceContext;
 
   token_payload?: auth.JwtPayload;
   token_errors?: string[];

package/src/storage/BucketStorage.ts

@@ -106,22 +106,20 @@ export interface BucketStorageFactory
   getPowerSyncInstanceId(): Promise<string>;
 }
 
-export interface Checkpoint {
+export interface WriteCheckpoint {
+  base: ActiveCheckpoint;
+  writeCheckpoint: bigint | null;
+}
+
+export interface ActiveCheckpoint {
   readonly checkpoint: util.OpId;
   readonly lsn: string | null;
-}
 
-export interface ActiveCheckpoint extends Checkpoint {
   hasSyncRules(): boolean;
 
   getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
 }
 
-export interface WriteCheckpoint {
-  base: ActiveCheckpoint;
-  writeCheckpoint: bigint | null;
-}
-
 export interface StorageMetrics {
   /**
    * Size of operations (bucket_data)
@@ -221,7 +219,7 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getCheckpoint(): Promise<Checkpoint>;
+  getCheckpoint(): Promise<{ checkpoint: util.OpId }>;
 
   getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
 
@@ -267,8 +265,6 @@ export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRul
    */
   clear(): Promise<void>;
 
-  setSnapshotDone(lsn: string): Promise<void>;
-
   autoActivate(): Promise<void>;
 
   /**
@@ -389,14 +385,8 @@ export type SaveOp = 'insert' | 'update' | 'delete';
 
 export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete;
 
-export enum SaveOperationTag {
-  INSERT = 'insert',
-  UPDATE = 'update',
-  DELETE = 'delete'
-}
-
 export interface SaveInsert {
-  tag: SaveOperationTag.INSERT;
+  tag: 'insert';
   sourceTable: SourceTable;
   before?: undefined;
   beforeReplicaId?: undefined;
@@ -405,7 +395,7 @@ export interface SaveInsert {
 }
 
 export interface SaveUpdate {
-  tag: SaveOperationTag.UPDATE;
+  tag: 'update';
   sourceTable: SourceTable;
 
   /**
@@ -424,7 +414,7 @@ export interface SaveUpdate {
 }
 
 export interface SaveDelete {
-  tag: SaveOperationTag.DELETE;
+  tag: 'delete';
   sourceTable: SourceTable;
   before?: SqliteRow;
   beforeReplicaId: ReplicaId;

package/src/storage/MongoBucketStorage.ts

@@ -331,39 +331,56 @@ export class MongoBucketStorage
   }
 
   async getStorageMetrics(): Promise<StorageMetrics> {
+    const ignoreNotExiting = (e: unknown) => {
+      if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
+        // Collection doesn't exist - return 0
+        return [{ storageStats: { size: 0 } }];
+      } else {
+        return Promise.reject(e);
+      }
+    };
+
+    const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
+    if (active_sync_rules == null) {
+      return {
+        operations_size_bytes: 0,
+        parameters_size_bytes: 0,
+        replication_size_bytes: 0
+      };
+    }
     const operations_aggregate = await this.db.bucket_data
 
       .aggregate([
         {
           $collStats: {
-            storageStats: {},
-            count: {}
+            storageStats: {}
           }
         }
       ])
-      .toArray();
+      .toArray()
+      .catch(ignoreNotExiting);
 
     const parameters_aggregate = await this.db.bucket_parameters
       .aggregate([
         {
           $collStats: {
-            storageStats: {},
-            count: {}
+            storageStats: {}
           }
         }
       ])
-      .toArray();
+      .toArray()
+      .catch(ignoreNotExiting);
 
     const replication_aggregate = await this.db.current_data
       .aggregate([
         {
           $collStats: {
-            storageStats: {},
-            count: {}
+            storageStats: {}
           }
         }
       ])
-      .toArray();
+      .toArray()
+      .catch(ignoreNotExiting);
 
     return {
       operations_size_bytes: operations_aggregate[0].storageStats.size,

package/src/storage/mongo/MongoBucketBatch.ts

@@ -81,7 +81,6 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     this.session = this.client.startSession();
     this.slot_name = slot_name;
     this.sync_rules = sync_rules;
-    this.batch = new OperationBatch();
   }
 
   addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void {

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -8,7 +8,6 @@ import * as util from '../../util/util-index.js';
 import {
   BucketDataBatchOptions,
   BucketStorageBatch,
-  Checkpoint,
   CompactOptions,
   DEFAULT_DOCUMENT_BATCH_LIMIT,
   DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES,
@@ -32,6 +31,8 @@ import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './m
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js';
+import { logger } from '@powersync/lib-services-framework';
+import * as timers from 'timers/promises';
 
 export class MongoSyncBucketStorage
   extends DisposableObserver<SyncRulesBucketStorageListener>
@@ -61,16 +62,15 @@ export class MongoSyncBucketStorage
     return this.parsedSyncRulesCache;
   }
 
-  async getCheckpoint(): Promise<Checkpoint> {
+  async getCheckpoint() {
     const doc = await this.db.sync_rules.findOne(
       { _id: this.group_id },
       {
-        projection: { last_checkpoint: 1, last_checkpoint_lsn: 1 }
+        projection: { last_checkpoint: 1 }
       }
     );
     return {
-      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
-      lsn: doc?.last_checkpoint_lsn ?? null
+      checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n)
     };
   }
 
@@ -461,10 +461,28 @@ export class MongoSyncBucketStorage
   }
 
   async clear(): Promise<void> {
+    while (true) {
+      try {
+        await this.clearIteration();
+        return;
+      } catch (e: unknown) {
+        if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') {
+          logger.info(
+            `Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.`
+          );
+          await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5);
+          continue;
+        } else {
+          throw e;
+        }
+      }
+    }
+  }
+
+  private async clearIteration(): Promise<void> {
     // Individual operations here may time out with the maxTimeMS option.
     // It is expected to still make progress, and continue on the next try.
 
-    // TODO: Transactional?
     await this.db.sync_rules.updateOne(
       {
         _id: this.group_id
@@ -478,48 +496,33 @@ export class MongoSyncBucketStorage
           no_checkpoint_before: null
         }
       },
-      { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
+      { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
     await this.db.bucket_data.deleteMany(
       {
         _id: idPrefixFilter<BucketDataKey>({ g: this.group_id }, ['b', 'o'])
       },
-      { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
+      { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
     await this.db.bucket_parameters.deleteMany(
       {
         key: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
       },
-      { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
+      { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
 
     await this.db.current_data.deleteMany(
       {
         _id: idPrefixFilter<SourceKey>({ g: this.group_id }, ['t', 'k'])
       },
-      { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
+      { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
 
     await this.db.source_tables.deleteMany(
       {
         group_id: this.group_id
       },
-      { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS }
-    );
-  }
-
-  async setSnapshotDone(lsn: string): Promise<void> {
-    await this.db.sync_rules.updateOne(
-      {
-        _id: this.group_id
-      },
-      {
-        $set: {
-          snapshot_done: true,
-          persisted_lsn: lsn,
-          last_checkpoint_ts: new Date()
-        }
-      }
+      { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
     );
   }
 

package/src/storage/mongo/db.ts

@@ -62,6 +62,9 @@ export class PowerSyncMongo {
     this.locks = this.db.collection('locks');
   }
 
+  /**
+   * Clear all collections.
+   */
   async clear() {
     await this.current_data.deleteMany({});
     await this.bucket_data.deleteMany({});
@@ -73,4 +76,13 @@ export class PowerSyncMongo {
     await this.instance.deleteOne({});
     await this.locks.deleteMany({});
   }
+
+  /**
+   * Drop the entire database.
+   *
+   * Primarily for tests.
+   */
+  async drop() {
+    await this.db.dropDatabase();
+  }
 }

package/src/util/protocol-types.ts

@@ -88,7 +88,7 @@ export type StreamingSyncLine =
  */
 export type OpId = string;
 
-interface Checkpoint {
+export interface Checkpoint {
   last_op_id: OpId;
   write_checkpoint?: OpId;
   buckets: BucketChecksum[];

package/test/src/compacting.test.ts

@@ -1,9 +1,11 @@
-import { SaveOperationTag } from '@/storage/BucketStorage.js';
 import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { validateCompactedBucket } from './bucket_validation.js';
 import { oneFromAsync } from './stream_utils.js';
-import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
+import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -29,7 +31,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -38,7 +40,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -47,7 +49,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
+        tag: 'update',
         after: {
           id: 't2'
         },
@@ -126,7 +128,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -135,7 +137,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -144,7 +146,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't1'
         },
@@ -153,7 +155,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
+        tag: 'update',
         after: {
           id: 't2'
         },
@@ -231,7 +233,7 @@ bucket_definitions:
     const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't1'
         },
@@ -240,7 +242,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
+        tag: 'insert',
         after: {
           id: 't2'
         },
@@ -249,7 +251,7 @@ bucket_definitions:
 
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't1'
         },
@@ -263,7 +265,7 @@ bucket_definitions:
     const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
       await batch.save({
         sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
+        tag: 'delete',
         before: {
           id: 't2'
         },