@powersync/service-core 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606

package/src/routes/endpoints/checkpointing.ts

@@ -1,7 +1,6 @@
 import { logger, router, schema } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
 
-import * as framework from '@powersync/lib-services-framework';
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
@@ -25,14 +24,15 @@ export const writeCheckpoint = routeDefinition({
     // Since we don't use LSNs anymore, the only way to get that is to wait.
     const start = Date.now();
 
-    const head = await apiHandler.getReplicationHead();
+    const head = await apiHandler.createReplicationHead(async (head) => head);
 
     const timeout = 50_000;
 
     logger.info(`Waiting for LSN checkpoint: ${head}`);
     while (Date.now() - start < timeout) {
-      const cp = await service_context.storageEngine.activeBucketStorage.getActiveCheckpoint();
-      if (!cp.hasSyncRules()) {
+      const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
+      const cp = await bucketStorage?.getCheckpoint();
+      if (cp == null) {
         throw new Error('No sync rules available');
       }
       if (cp.lsn && cp.lsn >= head) {
@@ -56,25 +56,14 @@ export const writeCheckpoint2 = routeDefinition({
 
     const apiHandler = service_context.routerEngine!.getAPI();
 
-    const client_id = payload.params.client_id;
-    const full_user_id = util.checkpointUserId(user_id, client_id);
-
-    const currentCheckpoint = await apiHandler.getReplicationHead();
-    const {
-      storageEngine: { activeBucketStorage }
-    } = service_context;
-
-    const activeSyncRules = await activeBucketStorage.getActiveSyncRulesContent();
-    if (!activeSyncRules) {
-      throw new framework.errors.ValidationError(`Cannot create Write Checkpoint since no sync rules are active.`);
-    }
-
-    using syncBucketStorage = activeBucketStorage.getInstance(activeSyncRules);
-    const writeCheckpoint = await syncBucketStorage.createManagedWriteCheckpoint({
-      user_id: full_user_id,
-      heads: { '1': currentCheckpoint }
+    const { replicationHead, writeCheckpoint } = await util.createWriteCheckpoint({
+      userId: user_id,
+      clientId: payload.params.client_id,
+      api: apiHandler,
+      storage: service_context.storageEngine.activeBucketStorage
     });
-    logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
+
+    logger.info(`Write checkpoint for ${user_id}/${payload.params.client_id}: ${writeCheckpoint} | ${replicationHead}`);
 
     return {
       write_checkpoint: String(writeCheckpoint)

package/src/routes/endpoints/socket-route.ts

@@ -13,7 +13,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => {
       const { service_context } = context;
-      const { routerEngine } = service_context;
+      const { routerEngine, syncContext } = service_context;
 
       // Create our own controller that we can abort directly
       const controller = new AbortController();
@@ -49,9 +49,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const {
         storageEngine: { activeBucketStorage }
       } = service_context;
-      // Sanity check before we start the stream
-      const cp = await activeBucketStorage.getActiveCheckpoint();
-      if (!cp.hasSyncRules()) {
+
+      const bucketStorage = await activeBucketStorage.getActiveStorage();
+      if (bucketStorage == null) {
         responder.onError(
           new errors.ServiceError({
             status: 500,
@@ -63,6 +63,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         return;
       }
 
+      const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
+
       const removeStopHandler = routerEngine!.addStopHandler(() => {
         controller.abort();
       });
@@ -71,8 +73,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       const tracker = new sync.RequestTracker();
       try {
         for await (const data of sync.streamResponse({
-          storage: activeBucketStorage,
-          parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
+          syncContext: syncContext,
+          bucketStorage: bucketStorage,
+          syncRules: syncRules,
           params: {
             ...params,
             binary_data: true // always true for web sockets

package/src/routes/endpoints/sync-rules.ts

@@ -69,7 +69,9 @@ export const deploySyncRules = routeDefinition({
     }
 
     const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules({
-      content: content
+      content: content,
+      // Aready validated above
+      validate: false
     });
 
     return {
@@ -167,7 +169,10 @@ export const reprocessSyncRules = routeDefinition({
     }
 
     const new_rules = await activeBucketStorage.updateSyncRules({
-      content: sync_rules.sync_rules.content
+      content: sync_rules.sync_rules.content,
+      // These sync rules already passed validation. But if the rules are not valid anymore due
+      // to a service change, we do want to report the error here.
+      validate: true
     });
     return {
       slot_name: new_rules.slot_name

package/src/routes/endpoints/sync-stream.ts

@@ -20,7 +20,7 @@ export const syncStreamed = routeDefinition({
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const { service_context } = payload.context;
-    const { routerEngine, storageEngine } = service_context;
+    const { routerEngine, storageEngine, syncContext } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
@@ -36,15 +36,18 @@ export const syncStreamed = routeDefinition({
     const params: util.StreamingSyncRequest = payload.params;
     const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
 
-    // Sanity check before we start the stream
-    const cp = await storageEngine.activeBucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
+    const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
+
+    if (bucketStorage == null) {
       throw new errors.ServiceError({
         status: 500,
         code: ErrorCode.PSYNC_S2302,
         description: 'No sync rules available'
       });
     }
+
+    const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions());
+
     const controller = new AbortController();
     const tracker = new sync.RequestTracker();
     try {
@@ -53,8 +56,9 @@ export const syncStreamed = routeDefinition({
         sync.transformToBytesTracked(
           sync.ndjson(
             sync.streamResponse({
-              storage: storageEngine.activeBucketStorage,
-              parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(),
+              syncContext: syncContext,
+              bucketStorage,
+              syncRules: syncRules,
               params,
               syncParams,
               token: payload.context.token_payload!,

package/src/runner/teardown.ts

@@ -51,7 +51,7 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
 
       // Mark the sync rules as terminated
       for (let syncRules of combinedSyncRules) {
-        using syncRulesStorage = storageFactory.getInstance(syncRules);
+        const syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here
         await syncRulesStorage.terminate({ clearStorage: false });
       }