@powersync/service-core 1.21.0 → 1.23.0

package/src/routes/endpoints/admin.ts

@@ -1,5 +1,13 @@
+import * as sqlite from 'node:sqlite';
+
 import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
-import { SourceSchema, SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
+import {
+  HydratedSyncConfig,
+  nodeSqlite,
+  SourceSchema,
+  SqlSyncRules,
+  StaticSchema
+} from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
@@ -63,20 +71,35 @@ export const diagnostics = routeDefinition({
     const {
       storageEngine: { activeBucketStorage }
     } = service_context;
-    const active = await activeBucketStorage.getActiveSyncRulesContent();
-    const next = await activeBucketStorage.getNextSyncRulesContent();
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    const deploying = await activeBucketStorage.getDeployingSyncConfig();
 
-    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
-      include_content,
-      check_connection: status.connected,
-      live_status: true
-    });
+    const active_status = await api.getSyncRulesStatus(
+      apiHandler,
+      active?.content ?? null,
+      {
+        include_content,
+        check_connection: status.connected,
+        live_status: true,
+        active: true
+      },
+      active?.storage
+    );
 
-    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
-      include_content,
-      check_connection: status.connected,
-      live_status: true
-    });
+    const deploying_status =
+      deploying == null
+        ? undefined
+        : await api.getSyncRulesStatus(
+            apiHandler,
+            deploying.content,
+            {
+              include_content,
+              check_connection: status.connected,
+              live_status: true,
+              active: false
+            },
+            deploying.storage
+          );
 
     return internal_routes.DiagnosticsResponse.encode({
       connections: [
@@ -87,7 +110,7 @@ export const diagnostics = routeDefinition({
         }
       ],
       active_sync_rules: active_status,
-      deploying_sync_rules: next_status
+      deploying_sync_rules: deploying_status
     });
   }
 });
@@ -117,12 +140,16 @@ export const reprocess = routeDefinition({
       storageEngine: { activeBucketStorage }
     } = service_context;
     const apiHandler = service_context.routerEngine.getAPI();
-    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
+    const next = await activeBucketStorage.getDeployingSyncConfig();
     if (next != null) {
-      throw new Error(`Busy processing sync config - cannot reprocess`);
+      throw new errors.ServiceError({
+        status: 409,
+        code: ErrorCode.PSYNC_S4106,
+        description: 'Busy processing sync config - cannot reprocess'
+      });
     }
 
-    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
+    const active = await activeBucketStorage.getActiveSyncConfig();
     if (active == null) {
       throw new errors.ServiceError({
         status: 422,
@@ -130,13 +157,12 @@ export const reprocess = routeDefinition({
         description: 'No active sync config'
       });
     }
-
     // There are some differences between this and using asUpdateOptions():
     // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
     // 2. If the source does not set the storage version, this will update it do the current version.
     // We can consider tweaking this behavior in the future.
     const new_rules = await activeBucketStorage.updateSyncRules(
-      storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
+      storage.updateSyncRulesFromYaml(active.content.sync_rules_content, {
         // This sync config already passed validation. But if the config is not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
@@ -151,41 +177,47 @@ export const reprocess = routeDefinition({
           // Previously the connection was asserted with `!`
           tag: baseConfig.tag,
           id: baseConfig.id,
-          slot_name: new_rules.slot_name
+          slot_name: new_rules.replicationStreamName
         }
       ]
     });
   }
 });
 
-class FakeSyncRulesContentForValidation extends storage.PersistedSyncRulesContent {
+class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent {
   constructor(
     private readonly apiHandler: api.RouteAPI,
     private readonly schema: SourceSchema,
-    data: storage.PersistedSyncRulesContentData
+    data: storage.PersistedSyncConfigContentData
   ) {
     super(data);
   }
 
-  current_lock: storage.ReplicationLock | null = null;
-
-  async lock(): Promise<storage.ReplicationLock> {
-    throw new Error('Lock not implemented');
-  }
+  parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
+    const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, {
+      ...this.apiHandler.getParseSyncRulesOptions(),
+      schema: this.schema
+    });
 
-  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
+    let hydrated: HydratedSyncConfig | undefined;
     return {
-      ...this,
-      sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, {
-        ...this.apiHandler.getParseSyncRulesOptions(),
-        schema: this.schema
-      }),
+      replicationStreamId: this.replicationStreamId,
+      replicationStreamName: this.replicationStreamName,
+      syncConfigs: [syncConfig],
       hydrationState: DEFAULT_HYDRATION_STATE,
-      hydratedSyncRules() {
-        return this.sync_rules.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+      get hydratedSyncConfig(): HydratedSyncConfig {
+        hydrated ??= syncConfig.config.hydrate({
+          hydrationState: DEFAULT_HYDRATION_STATE,
+          sqlite: nodeSqlite(sqlite)
+        });
+        return hydrated;
       }
     };
   }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    return null;
+  }
 }
 
 export const validate = routeDefinition({
@@ -205,10 +237,8 @@ export const validate = routeDefinition({
 
     const sync_rules = new FakeSyncRulesContentForValidation(apiHandler, schema, {
       // Dummy values
-      id: 0,
-      slot_name: '',
-      active: false,
-      last_checkpoint_lsn: '',
+      replicationStreamId: 0,
+      replicationStreamName: '',
       storageVersion: storage.LEGACY_STORAGE_VERSION,
       sync_rules_content: content,
       compiled_plan: null
@@ -222,16 +252,12 @@ export const validate = routeDefinition({
       });
     }
 
-    const status = (await api.getSyncRulesStatus(
-      service_context.storageEngine.activeBucketStorage,
-      apiHandler,
-      sync_rules,
-      {
-        include_content: false,
-        check_connection: connectionStatus.connected,
-        live_status: false
-      }
-    ))!;
+    const status = (await api.getSyncRulesStatus(apiHandler, sync_rules, {
+      include_content: false,
+      check_connection: connectionStatus.connected,
+      live_status: false,
+      active: false
+    }))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured', ts: new Date().toISOString() });

package/src/routes/endpoints/checkpointing.ts

@@ -30,7 +30,7 @@ export const writeCheckpoint = routeDefinition({
 
     logger.info(`Waiting for LSN checkpoint: ${head}`);
     while (Date.now() - start < timeout) {
-      const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
+      const bucketStorage = (await service_context.storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
       const cp = await bucketStorage?.getCheckpoint();
       if (cp == null) {
         throw new Error('No sync config available');

package/src/routes/endpoints/socket-route.ts

@@ -71,7 +71,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         storageEngine: { activeBucketStorage }
       } = service_context;
 
-      const bucketStorage = await activeBucketStorage.getActiveStorage();
+      const bucketStorage = (await activeBucketStorage.getActiveSyncConfig())?.storage;
       if (bucketStorage == null) {
         responder.onError(
           new errors.ServiceError({

package/src/routes/endpoints/sync-rules.ts

@@ -73,7 +73,7 @@ export const deploySyncRules = routeDefinition({
     const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
 
     return {
-      slot_name: sync_rules.slot_name
+      slot_name: sync_rules.replicationStreamName
     };
   }
 });
@@ -110,8 +110,8 @@ export const currentSyncRules = routeDefinition({
       storageEngine: { activeBucketStorage }
     } = service_context;
 
-    const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
-    if (!sync_rules) {
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    if (!active) {
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
@@ -119,15 +119,16 @@ export const currentSyncRules = routeDefinition({
       });
     }
 
+    const sync_rules = active.content;
     const apiHandler = service_context.routerEngine.getAPI();
     const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content);
-    const next = await activeBucketStorage.getNextSyncRulesContent();
+    const next = await activeBucketStorage.getDeployingSyncConfig();
 
-    const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null;
+    const next_info = next ? await debugSyncRules(apiHandler, next.content.sync_rules_content) : null;
 
     const response = {
       current: {
-        slot_name: sync_rules.slot_name,
+        slot_name: sync_rules.replicationStreamName,
         content: sync_rules.sync_rules_content,
         ...info
       },
@@ -135,8 +136,8 @@ export const currentSyncRules = routeDefinition({
         next == null
           ? null
           : {
-              slot_name: next.slot_name,
-              content: next.sync_rules_content,
+              slot_name: next.content.replicationStreamName,
+              content: next.content.sync_rules_content,
               ...next_info
             }
     };
@@ -156,9 +157,17 @@ export const reprocessSyncRules = routeDefinition({
     const {
       storageEngine: { activeBucketStorage }
     } = payload.context.service_context;
-    const apiHandler = payload.context.service_context.routerEngine.getAPI();
-    const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
-    if (sync_rules == null) {
+    const next = await activeBucketStorage.getDeployingSyncConfig();
+    if (next != null) {
+      throw new errors.ServiceError({
+        status: 409,
+        code: ErrorCode.PSYNC_S4106,
+        description: 'Busy processing sync config - cannot reprocess'
+      });
+    }
+
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    if (active == null) {
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
@@ -166,15 +175,16 @@ export const reprocessSyncRules = routeDefinition({
       });
     }
 
+    const sync_rules = active.content;
     const new_rules = await activeBucketStorage.updateSyncRules(
-      updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
+      updateSyncRulesFromYaml(sync_rules.sync_rules_content, {
         // This sync config already passed validation. But if the rules are not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
       })
     );
     return {
-      slot_name: new_rules.slot_name
+      slot_name: new_rules.replicationStreamName
     };
   }
 });

package/src/routes/endpoints/sync-stream.ts

@@ -62,7 +62,7 @@ export const syncStreamed = routeDefinition({
       });
     }
 
-    const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
+    const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
 
     if (bucketStorage == null) {
       throw new errors.ServiceError({

package/src/routes/route-register.ts

@@ -1,7 +1,8 @@
 import type fastify from 'fastify';
+import * as zlib from 'node:zlib';
 import * as uuid from 'uuid';
 
-import { errors, HTTPMethod, logger, RouteNotFound, router, ServiceError } from '@powersync/lib-services-framework';
+import { errors, HTTPMethod, logger, RouteNotFound, router } from '@powersync/lib-services-framework';
 import { FastifyReply } from 'fastify';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
@@ -105,18 +106,86 @@ export function registerFastifyNotFoundHandler(app: fastify.FastifyInstance) {
   });
 }
 
-function serviceErrorToResponse(error: ServiceError): router.RouterResponse {
+/** Registers a custom error handler that emits service-error JSON honouring any negotiated `Content-Encoding`. */
+export function registerFastifyErrorHandler(app: fastify.FastifyInstance) {
+  app.setErrorHandler<Error>(async (error, _request, reply) => {
+    if (reply.raw.headersSent) {
+      // Headers already flushed - reply.code/header would throw ERR_HTTP_HEADERS_SENT.
+      reply.raw.destroy(error);
+      return;
+    }
+
+    const { status, data } = fastifyErrorToResponse(error);
+    if (status >= 500) {
+      logger.error('Request failed', error);
+    } else {
+      logger.warn(`Request failed: ${error.message}`, {
+        status,
+        code: data.error.code
+      });
+    }
+
+    const json = JSON.stringify(data);
+
+    // Fastify's default handler leaves `content-encoding` intact when it rewrites the body.
+    const encoding = reply.getHeader('content-encoding');
+    let body: string | Buffer = json;
+    if (encoding === 'gzip') {
+      body = zlib.gzipSync(json);
+    } else if (encoding === 'zstd') {
+      body = zlib.zstdCompressSync(json);
+    } else {
+      reply.removeHeader('content-encoding');
+    }
+
+    reply
+      .code(status)
+      .header('content-type', 'application/json; charset=utf-8')
+      .header('content-length', Buffer.byteLength(body))
+      .send(body);
+  });
+}
+
+type ErrorResponseData = { error: errors.ErrorData };
+
+function serviceErrorToResponse(serviceError: errors.ServiceError): router.RouterResponse<ErrorResponseData> {
   return new router.RouterResponse({
-    status: error.errorData.status || 500,
+    status: serviceError.errorData.status || 500,
     headers: {
       'Content-Type': 'application/json'
     },
     data: {
-      error: error.errorData
+      error: serviceError.errorData
     }
   });
 }
 
+function fastifyErrorToResponse(error: any): router.RouterResponse<ErrorResponseData> {
+  // Preserve 4xx status from Fastify built-ins (validation, invalid JSON body, etc.) instead of collapsing to 500.
+  if (
+    typeof error?.statusCode === 'number' &&
+    error.statusCode >= 400 &&
+    error.statusCode < 500 &&
+    typeof error.code === 'string'
+  ) {
+    return new router.RouterResponse({
+      status: error.statusCode,
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      data: {
+        error: {
+          name: error.name,
+          code: error.code,
+          status: error.statusCode,
+          description: error.message
+        }
+      }
+    });
+  }
+  return serviceErrorToResponse(errors.asServiceError(error));
+}
+
 async function respond(reply: FastifyReply, response: router.RouterResponse) {
   Object.keys(response.headers).forEach((key) => {
     reply.header(key, response.headers[key]);

package/src/runner/teardown.ts

@@ -40,22 +40,22 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
   const locks: storage.ReplicationLock[] = [];
   while (Date.now() - start < 120_000) {
     let retry = false;
-    const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
+    const replicatingStreams = await storageFactory.getReplicatingReplicationStreams();
     // Lock all the replicating replication streams
-    for (const replicatingSyncRule of replicatingSyncRules) {
-      const lock = await replicatingSyncRule.lock();
+    for (const replicatingStream of replicatingStreams) {
+      const lock = await replicatingStream.lock();
       locks.push(lock);
     }
 
-    const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
-    const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
+    const stoppedStreams = await storageFactory.getStoppedReplicationStreams();
+    const combinedStreams = [...replicatingStreams, ...stoppedStreams];
     try {
       // Clean up any module specific configuration for the replication stream
-      await moduleManager.tearDown({ syncRules: combinedSyncRules });
+      await moduleManager.tearDown({ replicationStreams: combinedStreams, syncRules: combinedStreams });
 
       // Mark the replication stream as terminated
-      for (let syncRules of combinedSyncRules) {
-        const syncRulesStorage = storageFactory.getInstance(syncRules);
+      for (let replicationStream of combinedStreams) {
+        const syncRulesStorage = storageFactory.getInstance(replicationStream);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here
         await syncRulesStorage.terminate({ clearStorage: false });
       }

package/src/storage/BucketStorageBatch.ts

@@ -5,6 +5,7 @@ import { InternalOpId } from '../util/utils.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
+import { ResolveTablesOptions, ResolveTablesResult } from './SyncRulesBucketStorage.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
   createEmptyCheckpoints: true,
@@ -22,6 +23,11 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   last_flushed_op: InternalOpId | null;
 
+  /**
+   * True for snapshot batches that should skip rows already present in current_data.
+   */
+  readonly skipExistingRows: boolean;
+
   /**
    * Save an op, and potentially flush.
    *
@@ -77,11 +83,6 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   setResumeLsn(lsn: string): Promise<void>;
 
-  /**
-   * Get the last checkpoint LSN, from either commit or keepalive.
-   */
-  lastCheckpointLsn: string | null;
-
   /**
    * LSN to resume from.
    *
@@ -91,10 +92,36 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
 
   markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
   markTableSnapshotRequired(table: SourceTable): Promise<void>;
+
+  /**
+   * Mark the full replication snapshot as done without validating individual source table snapshot state.
+   *
+   * This is primarily intended for storage tests and setup helpers that manually construct storage state.
+   * Replicators should use `markSnapshotDone()` instead, because that validates that all known source tables
+   * have completed snapshotting before allowing checkpoints to be unblocked.
+   */
   markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
 
+  /**
+   * Mark the full replication snapshot as done after validating that all known source tables have completed snapshotting.
+   *
+   * Replicators should use this method when completing an initial snapshot. The validation prevents races where
+   * new source tables are marked as requiring a snapshot while global snapshot finalization is running.
+   *
+   * If `throwOnConflict` is false, this will instead return early without throwing if there are source tables that still require snapshotting.
+   * Use that only in cases where concurrency is expected, and can automatically retry/continue.
+   */
+  markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void>;
+
   updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
 
+  /**
+   * Get the current status for an existing source table without creating or resolving a replacement table.
+   */
+  getSourceTableStatus(table: SourceTable): Promise<SourceTable | null>;
+
+  resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
+
   /**
    * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
    */
@@ -184,8 +211,6 @@ export interface CheckpointResult {
   /**
    * True if a checkpoint was actually created by this operation. This can be false even if checkpointBlocked is false,
    * if the checkpoint was empty.
-   *
-   * This is primarily used for testing.
    */
   checkpointCreated: boolean;
 }