@powersync/service-core 1.20.5 → 1.22.0

package/src/routes/route-register.ts

@@ -1,7 +1,8 @@
 import type fastify from 'fastify';
+import * as zlib from 'node:zlib';
 import * as uuid from 'uuid';
 
-import { errors, HTTPMethod, logger, RouteNotFound, router, ServiceError } from '@powersync/lib-services-framework';
+import { errors, HTTPMethod, logger, RouteNotFound, router } from '@powersync/lib-services-framework';
 import { FastifyReply } from 'fastify';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
@@ -105,18 +106,86 @@ export function registerFastifyNotFoundHandler(app: fastify.FastifyInstance) {
   });
 }
 
-function serviceErrorToResponse(error: ServiceError): router.RouterResponse {
+/** Registers a custom error handler that emits service-error JSON honouring any negotiated `Content-Encoding`. */
+export function registerFastifyErrorHandler(app: fastify.FastifyInstance) {
+  app.setErrorHandler<Error>(async (error, _request, reply) => {
+    if (reply.raw.headersSent) {
+      // Headers already flushed - reply.code/header would throw ERR_HTTP_HEADERS_SENT.
+      reply.raw.destroy(error);
+      return;
+    }
+
+    const { status, data } = fastifyErrorToResponse(error);
+    if (status >= 500) {
+      logger.error('Request failed', error);
+    } else {
+      logger.warn(`Request failed: ${error.message}`, {
+        status,
+        code: data.error.code
+      });
+    }
+
+    const json = JSON.stringify(data);
+
+    // Fastify's default handler leaves `content-encoding` intact when it rewrites the body.
+    const encoding = reply.getHeader('content-encoding');
+    let body: string | Buffer = json;
+    if (encoding === 'gzip') {
+      body = zlib.gzipSync(json);
+    } else if (encoding === 'zstd') {
+      body = zlib.zstdCompressSync(json);
+    } else {
+      reply.removeHeader('content-encoding');
+    }
+
+    reply
+      .code(status)
+      .header('content-type', 'application/json; charset=utf-8')
+      .header('content-length', Buffer.byteLength(body))
+      .send(body);
+  });
+}
+
+type ErrorResponseData = { error: errors.ErrorData };
+
+function serviceErrorToResponse(serviceError: errors.ServiceError): router.RouterResponse<ErrorResponseData> {
   return new router.RouterResponse({
-    status: error.errorData.status || 500,
+    status: serviceError.errorData.status || 500,
     headers: {
       'Content-Type': 'application/json'
     },
     data: {
-      error: error.errorData
+      error: serviceError.errorData
     }
   });
 }
 
+function fastifyErrorToResponse(error: any): router.RouterResponse<ErrorResponseData> {
+  // Preserve 4xx status from Fastify built-ins (validation, invalid JSON body, etc.) instead of collapsing to 500.
+  if (
+    typeof error?.statusCode === 'number' &&
+    error.statusCode >= 400 &&
+    error.statusCode < 500 &&
+    typeof error.code === 'string'
+  ) {
+    return new router.RouterResponse({
+      status: error.statusCode,
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      data: {
+        error: {
+          name: error.name,
+          code: error.code,
+          status: error.statusCode,
+          description: error.message
+        }
+      }
+    });
+  }
+  return serviceErrorToResponse(errors.asServiceError(error));
+}
+
 async function respond(reply: FastifyReply, response: router.RouterResponse) {
   Object.keys(response.headers).forEach((key) => {
     reply.header(key, response.headers[key]);

package/src/runner/teardown.ts

@@ -35,13 +35,13 @@ export async function teardown(runnerConfig: utils.RunnerConfig) {
 }
 
 async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
-  logger.info(`Terminating sync rules...`);
+  logger.info(`Terminating replication stream...`);
   const start = Date.now();
   const locks: storage.ReplicationLock[] = [];
   while (Date.now() - start < 120_000) {
     let retry = false;
     const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
-    // Lock all the replicating sync rules
+    // Lock all the replicating replication streams
     for (const replicatingSyncRule of replicatingSyncRules) {
       const lock = await replicatingSyncRule.lock();
       locks.push(lock);
@@ -50,10 +50,10 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
     const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
     const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
     try {
-      // Clean up any module specific configuration for the sync rules
+      // Clean up any module specific configuration for the replication stream
       await moduleManager.tearDown({ syncRules: combinedSyncRules });
 
-      // Mark the sync rules as terminated
+      // Mark the replication stream as terminated
       for (let syncRules of combinedSyncRules) {
         const syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here

package/src/storage/BucketStorage.ts

@@ -2,36 +2,36 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 
 export enum SyncRuleState {
   /**
-   * New sync rules - needs to be processed (initial replication).
+   * New replication stream - needs to be processed (initial replication).
    *
-   * While multiple sets of sync rules _can_ be in PROCESSING,
+   * While multiple replication streams _can_ be in PROCESSING,
    * it's generally pointless, so we only keep one in that state.
    */
   PROCESSING = 'PROCESSING',
 
   /**
-   * Sync rule processing is done, and can be used for sync.
+   * Intial processing is done, and can be used for sync.
    *
-   * Only one set of sync rules should be in ACTIVE or ERRORED state.
+   * Only one replication stream should be in ACTIVE or ERRORED state.
    */
   ACTIVE = 'ACTIVE',
   /**
-   * This state is used when the sync rules has been replaced,
+   * This state is used when the replication stream has been replaced,
    * and replication is or should be stopped.
    */
   STOP = 'STOP',
   /**
-   * After sync rules have been stopped, the data needs to be
+   * After replication stream has been stopped, the data needs to be
    * deleted. Once deleted, the state is TERMINATED.
    */
   TERMINATED = 'TERMINATED',
 
   /**
-   * Sync rules has run into a permanent replication error. It
-   * is still the "active" sync rules for syncing to users,
+   * Replication stream has run into a permanent replication error. It
+   * is still the "active" replication stram for syncing to users,
    * but should not replicate anymore.
    *
-   * It will transition to STOP when a new sync rules is activated.
+   * It will transition to STOP when a new replication stream is activated.
    */
   ERRORED = 'ERRORED'
 }

package/src/storage/BucketStorageBatch.ts

@@ -5,6 +5,7 @@ import { InternalOpId } from '../util/utils.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
+import { ResolveTablesOptions, ResolveTablesResult } from './SyncRulesBucketStorage.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
   createEmptyCheckpoints: true,
@@ -22,6 +23,11 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   last_flushed_op: InternalOpId | null;
 
+  /**
+   * True for snapshot batches that should skip rows already present in current_data.
+   */
+  readonly skipExistingRows: boolean;
+
   /**
    * Save an op, and potentially flush.
    *
@@ -91,10 +97,36 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
 
   markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
   markTableSnapshotRequired(table: SourceTable): Promise<void>;
+
+  /**
+   * Mark the full replication snapshot as done without validating individual source table snapshot state.
+   *
+   * This is primarily intended for storage tests and setup helpers that manually construct storage state.
+   * Replicators should use `markSnapshotDone()` instead, because that validates that all known source tables
+   * have completed snapshotting before allowing checkpoints to be unblocked.
+   */
   markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
 
+  /**
+   * Mark the full replication snapshot as done after validating that all known source tables have completed snapshotting.
+   *
+   * Replicators should use this method when completing an initial snapshot. The validation prevents races where
+   * new source tables are marked as requiring a snapshot while global snapshot finalization is running.
+   *
+   * If `throwOnConflict` is false, this will instead return early without throwing if there are source tables that still require snapshotting.
+   * Use that only in cases where concurrency is expected, and can automatically retry/continue.
+   */
+  markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void>;
+
   updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
 
+  /**
+   * Get the current status for an existing source table without creating or resolving a replacement table.
+   */
+  getSourceTableStatus(table: SourceTable): Promise<SourceTable | null>;
+
+  resolveTables(options: ResolveTablesOptions): Promise<ResolveTablesResult>;
+
   /**
    * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
    */

package/src/storage/BucketStorageFactory.ts

@@ -17,17 +17,17 @@ import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 /**
  * Represents a configured storage provider.
  *
- * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
- * This is to handle replication of a new version of sync rules, while the old version is still active.
+ * The provider can handle multiple replication streams concurrently, each with their own storage.
+ * This is to handle replication of a new version of sync config, while the old replication stream is still active.
  *
- * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
+ * Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
  */
 export abstract class BucketStorageFactory
   extends BaseObserver<BucketStorageFactoryListener>
   implements AsyncDisposable
 {
   /**
-   * Update sync rules from configuration, if changed.
+   * Update sync config from configuration, if changed.
    */
   async configureSyncRules(
     options: UpdateSyncRulesOptions
@@ -36,42 +36,42 @@ export abstract class BucketStorageFactory
     const active = await this.getActiveSyncRulesContent();
 
     if (next?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else if (next == null && active?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else {
-      logger.info('Sync rules updated from configuration');
+      logger.info('Sync config updated');
       const persisted_sync_rules = await this.updateSyncRules(options);
       return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
     }
   }
 
   /**
-   * Get a storage instance to query sync data for specific sync rules.
+   * Get a storage instance to query sync data for specific sync config.
    */
   abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
 
   /**
-   * Deploy new sync rules.
+   * Deploy new sync config.
    */
   abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
 
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
-   * a new sync rules instance.
+   * a new replication stream.
    *
    * This is roughly the same as deploying a new version of the current sync
-   * rules, but also accounts for cases where the current sync rules are not
-   * the latest ones.
+   * config, but also accounts for cases where the current sync config is not
+   * the latest one.
    *
    * Replication should be restarted after this.
    */
   abstract restartReplication(sync_rules_group_id: number): Promise<void>;
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getActiveSyncRulesContent();
@@ -79,12 +79,12 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getNextSyncRulesContent();
@@ -92,17 +92,17 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
+   * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
    */
   abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
   /**
-   * Get all sync rules stopped but not terminated yet.
+   * Get all sync config stopped but not terminated yet.
    */
   abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
@@ -112,12 +112,12 @@ export abstract class BucketStorageFactory
   abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
 
   /**
-   * Get storage size of active sync rules.
+   * Get storage size of active replication stream.
    */
   abstract getStorageMetrics(): Promise<StorageMetrics>;
 
   /**
-   * Get the unique identifier for this instance of Powersync
+   * Get the unique identifier for this instance of Powersync.
    */
   abstract getPowerSyncInstanceId(): Promise<string>;
 
@@ -161,9 +161,20 @@ export interface UpdateSyncRulesOptions {
      * compiler.
      */
     plan: SerializedSyncPlan | null;
+
+    /**
+     * Parsed sync config, primarily to generate a definition mapping.
+     * Not persisted, and the defaultSchema used for parsing is not relevant.
+     */
+    parsed: SyncConfigWithErrors;
   };
   lock?: boolean;
   storageVersion?: number;
+
+  /**
+   * Only relevant if the result is used. This does not affect the persisted config.
+   */
+  defaultSchema?: string;
 }
 
 export interface SerializedSyncPlan {
@@ -190,7 +201,7 @@ export function updateSyncRulesFromYaml(
   const config = SqlSyncRules.fromYaml(content, {
     // No schema-based validation at this point
     schema: undefined,
-    defaultSchema: 'not_applicable', // Not needed for validation
+    defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
     throwOnError: options?.validate ?? false
   });
 
@@ -198,10 +209,11 @@ export function updateSyncRulesFromYaml(
 }
 
 export function updateSyncRulesFromConfig(
-  { config, errors }: SyncConfigWithErrors,
+  parsed: SyncConfigWithErrors,
   options?: Omit<UpdateSyncRulesOptions, 'config'>
 ): UpdateSyncRulesOptions {
   let plan: SerializedSyncPlan | null = null;
+  const { config, errors } = parsed;
   if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
     for (const event of config.eventDescriptors) {
@@ -216,7 +228,7 @@ export function updateSyncRulesFromConfig(
     };
   }
 
-  return { config: { yaml: config.content, plan }, ...options };
+  return { config: { yaml: config.content, plan, parsed }, ...options };
 }
 
 export interface GetIntanceOptions {

package/src/storage/PersistedSyncRulesContent.ts

@@ -1,13 +1,13 @@
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
 import {
   CompatibilityContext,
   CompatibilityOption,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
   ErrorLocation,
-  HydratedSyncRules,
+  HydratedSyncConfig,
   HydrationState,
-  javaScriptExpressionEngine,
+  nodeSqlite,
   PrecompiledSyncConfig,
   SqlEventDescriptor,
   SqlSyncRules,
@@ -15,6 +15,7 @@ import {
   versionedHydrationState,
   YamlError
 } from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
@@ -29,7 +30,7 @@ export interface PersistedSyncRulesContentData {
   readonly compiled_plan: SerializedSyncPlan | null;
   readonly slot_name: string;
   /**
-   * True if this is the "active" copy of the sync rules.
+   * True if this is the "active" copy of the sync config.
    */
   readonly active: boolean;
   readonly storageVersion: number;
@@ -49,6 +50,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
   readonly slot_name!: string;
   readonly active!: boolean;
   readonly storageVersion!: number;
+  readonly logger: Logger;
 
   readonly last_checkpoint_lsn!: string | null;
 
@@ -61,6 +63,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
   constructor(data: PersistedSyncRulesContentData) {
     Object.assign(this, data);
+    this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
   }
 
   /**
@@ -73,7 +76,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
     if (storageConfig == null) {
       throw new ServiceError(
         ErrorCode.PSYNC_S1005,
-        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+        `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
       );
     }
     return storageConfig;
@@ -99,10 +102,13 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
       const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
         defaultSchema: options.defaultSchema,
-        engine: javaScriptExpressionEngine(compatibility),
         sourceText: this.sync_rules_content
       });
 
+      // Note: If the original content did not define a storage version, this will still set the storage version.
+      // This means asUpdateOptions will not change the storage version, even if the default changes.
+      precompiled.storageVersion = this.storageVersion;
+
       const errors: YamlError[] = [];
       if (this.compiled_plan.errors) {
         for (const error of this.compiled_plan.errors) {
@@ -135,17 +141,19 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
     return {
       id: this.id,
       slot_name: this.slot_name,
-      sync_rules: config,
+      syncConfigWithErrors: config,
       hydrationState,
-      hydratedSyncRules: () => {
-        return config.config.hydrate({ hydrationState });
+      hydratedSyncConfig: () => {
+        return config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
       }
     };
   }
 
   asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
+    // defaultSchema is not relevant for the parsed version here
+    const parsed = this.parsed({ defaultSchema: 'not_applicable' });
     return {
-      config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
+      config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.syncConfigWithErrors },
       ...options
     };
   }
@@ -155,12 +163,12 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
 export interface PersistedSyncRules {
   readonly id: number;
-  readonly sync_rules: SyncConfigWithErrors;
+  readonly syncConfigWithErrors: SyncConfigWithErrors;
   readonly slot_name: string;
   /**
    * For testing only.
    */
   readonly hydrationState: HydrationState;
 
-  hydratedSyncRules(): HydratedSyncRules;
+  hydratedSyncConfig(): HydratedSyncConfig;
 }

package/src/storage/SourceEntity.ts

@@ -1,3 +1,5 @@
+import { SourceTableRef } from '@powersync/service-sync-rules';
+
 export interface ColumnDescriptor {
   name: string;
   /**
@@ -10,7 +12,7 @@ export interface ColumnDescriptor {
   typeId?: number;
 }
 
-export interface SourceEntityDescriptor {
+export interface SourceEntityDescriptor extends SourceTableRef {
   /**
    * The internal id of the source entity structure in the database.
    * If undefined, the schema and name are used as the identifier.
@@ -23,4 +25,10 @@ export interface SourceEntityDescriptor {
    *  The columns that are used to uniquely identify a record in the source entity.
    */
   replicaIdColumns: ColumnDescriptor[];
+  /**
+   * Whether the source always sends complete row data with each operation (e.g. Postgres REPLICA
+   * IDENTITY FULL). When true, no current_data copy is needed. Undefined means the source does not
+   * report this, in which case we default to keeping a copy.
+   */
+  sendsCompleteRows?: boolean;
 }