@powersync/service-core 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844

package/src/replication/AbstractReplicator.ts

@@ -32,19 +32,19 @@ export interface AbstractReplicatorOptions {
 /**
  *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
  *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
- *   It also handles any changes to the sync rules.
+ *   It also handles any changes to the sync config.
  */
 export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
   protected logger: winston.Logger;
   private lockAlerted: boolean = false;
   /**
-   *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
-   *  transitioning to a new set of sync rules.
+   *  Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
+   *  transitioning to a new replication stream.
    */
   private replicationJobs = new Map<number, T>();
 
   /**
-   * Map of sync rule ids to promises that are clearing the sync rule configuration.
+   * Map of replciation stream ids to promises that are clearing the replication stream.
    *
    * We primarily do this to keep track of what we're currently clearing, but don't currently
    * use the Promise value.
@@ -68,8 +68,8 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   abstract createJob(options: CreateJobOptions): T;
 
   /**
-   *  Clean up any configuration or state for the specified sync rule on the datasource.
-   *  Should be a no-op if the configuration has already been cleared
+   * Clean up any configuration or state for the specified replication stream on the datasource.
+   * Should be a no-op if the replication stream has already been cleared
    */
   abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
 
@@ -100,22 +100,24 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   public async start(): Promise<void> {
     this.abortController = new AbortController();
     this.runLoop().catch((e) => {
-      this.logger.error('Data source fatal replication error', e);
+      this.logger.error('Fatal replication error', e);
       container.reporter.captureException(e);
       setTimeout(() => {
         process.exit(1);
       }, 1000);
     });
     this.metrics.getObservableGauge(ReplicationMetric.REPLICATION_LAG_SECONDS).setValueProvider(async () => {
-      const lag = await this.getReplicationLagMillis().catch((e) => {
+      try {
+        const lag = this.getReplicationLagMillis();
+        if (lag == null) {
+          return undefined;
+        }
+        // ms to seconds
+        return Math.round(lag / 1000);
+      } catch (e) {
         this.logger.error('Failed to get replication lag', e);
         return undefined;
-      });
-      if (lag == null) {
-        return undefined;
       }
-      // ms to seconds
-      return Math.round(lag / 1000);
     });
   }
 
@@ -133,9 +135,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
     let configuredLock: storage.ReplicationLock | undefined = undefined;
     if (syncRules != null) {
-      this.logger.info('Loaded sync rules');
+      this.logger.info('Loaded sync config');
       try {
-        // Configure new sync rules, if they have changed.
+        // Configure new sync config, if they have changed.
         // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
 
         const { lock } = await this.storage.configureSyncRules(
@@ -147,11 +149,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       } catch (e) {
         // Log and re-raise to exit.
         // Should only reach this due to validation errors if exit_on_error is true.
-        this.logger.error(`Failed to update sync rules from configuration`, e);
+        this.logger.error(`Failed to update sync config`, e);
         throw e;
       }
     } else {
-      this.logger.info('No sync rules configured - configure via API');
+      this.logger.info('No sync streams or rules configured - configure via API');
     }
     while (!this.stopped) {
       await container.probes.touch();
@@ -204,7 +206,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         // Remove from the list. Next refresh call will restart the job.
         existingJobs.delete(syncRules.id);
       } else {
-        // New sync rules were found (or resume after restart)
+        // New sync config was found (or resume after restart)
         try {
           let lock: storage.ReplicationLock;
           if (configuredLock?.sync_rules_id == syncRules.id) {
@@ -227,15 +229,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         } catch (e) {
           if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
             if (!this.lockAlerted) {
-              this.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
+              syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
               this.lockAlerted = true;
             }
           } else {
-            // Could be a sync rules parse error,
+            // Could be a sync config parse error,
             // for example from stricter validation that was added.
             // This will be retried every couple of seconds.
-            // When new (valid) sync rules are deployed and processed, this one be disabled.
-            this.logger.error('Failed to start replication for new sync rules', e);
+            // When new (valid) sync config is deployed and processed, this one be disabled.
+            syncRules.logger.error('Failed to start replication for new sync config', e);
           }
         }
       }
@@ -244,7 +246,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     this.replicationJobs = newJobs;
     this.activeReplicationJob = activeJob;
 
-    // Stop any orphaned jobs that no longer have sync rules.
+    // Stop any orphaned jobs that no longer have a replication stream.
     // Termination happens below
     for (let job of existingJobs.values()) {
       // Old - stop and clean up
@@ -252,11 +254,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
         await job.stop();
       } catch (e) {
         // This will be retried
-        this.logger.warn('Failed to stop old replication job}', e);
+        job.storage.logger.warn('Failed to stop old replication job', e);
       }
     }
 
-    // Sync rules stopped previously, including by a different process.
+    // Replication stream stopped previously, including by a different process.
     const stopped = await this.storage.getStoppedSyncRules();
     for (let syncRules of stopped) {
       if (this.clearingJobs.has(syncRules.id)) {
@@ -266,11 +268,11 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
       // We clear storage asynchronously.
       // It is important to be able to continue running the refresh loop, otherwise we cannot
-      // retry locked sync rules, for example.
+      // retry locked replication stream, for example.
       const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
       const promise = this.terminateSyncRules(syncRuleStorage)
         .catch((e) => {
-          this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
+          syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
         })
         .finally(() => {
           this.clearingJobs.delete(syncRules.id);
@@ -284,12 +286,12 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   }
 
   protected async terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage) {
-    this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
+    syncRuleStorage.logger.info(`Terminating replication stream...`);
     // This deletes postgres replication slots - should complete quickly.
     // It is safe to do before or after clearing the data in the storage.
     await this.cleanUp(syncRuleStorage);
     await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
-    this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
+    syncRuleStorage.logger.info(`Successfully terminated replication stream`);
   }
 
   abstract testConnection(): Promise<ConnectionTestResult>;
@@ -312,7 +314,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
    *
    * "processing" replication streams are not taken into account for this metric.
    */
-  async getReplicationLagMillis(): Promise<number | undefined> {
+  getReplicationLagMillis(): number | undefined {
     return this.activeReplicationJob?.getReplicationLagMillis();
   }
 }

package/src/replication/ReplicationLagTracker.ts

@@ -0,0 +1,86 @@
+import { RollingBucketMax } from '../metrics/RollingBucketMax.js';
+
+/**
+ * Tracks replication lag across the current in-flight transaction and a rolling
+ * max of recently observed lag values.
+ */
+export class ReplicationLagTracker {
+  private readonly rollingReplicationLag = new RollingBucketMax();
+  private _oldestUncommittedChange: Date | null = null;
+  private _isStartingReplication = true;
+
+  /**
+   * The oldest source timestamp still part of the current in-flight work.
+   */
+  get oldestUncommittedChange(): Date | null {
+    return this._oldestUncommittedChange;
+  }
+
+  /**
+   * True until replication has seen its first completed commit or equivalent keepalive.
+   */
+  get isStartingReplication(): boolean {
+    return this._isStartingReplication;
+  }
+
+  /**
+   * Registers the first source timestamp for the current in-flight work,
+   * for example the start of a transaction
+   */
+  trackUncommittedChange(timestamp: Date | null | undefined): void {
+    if (this._oldestUncommittedChange == null && timestamp != null) {
+      this._oldestUncommittedChange = timestamp;
+    }
+  }
+
+  /**
+   * Clears the current in-flight timestamp without changing startup state.
+   */
+  clearUncommittedChange(): void {
+    this._oldestUncommittedChange = null;
+  }
+
+  /**
+   * Marks replication as started even if no committed transaction lag was recorded.
+   */
+  markStarted(): void {
+    this._isStartingReplication = false;
+  }
+
+  /**
+   * Mark the current pending changes as "committed".
+   *
+   * Records the current in-flight lag into the rolling window and clears it.
+   * The current lag is calculated as the differnence between current time and the oldest change,
+   * as marked by trackUncommittedChange.
+   */
+  markCommitted(timestampMs = Date.now()): void {
+    if (this._oldestUncommittedChange != null) {
+      this.rollingReplicationLag.report(timestampMs - this._oldestUncommittedChange.getTime(), timestampMs);
+    }
+    this.clearUncommittedChange();
+    this.markStarted();
+  }
+
+  /**
+   * Returns the lag for the current in-flight work.
+   *
+   * 0 if idle (no pending changes to replicate).
+   *
+   * undefined when replication is still starting up.
+   */
+  getCurrentLagMillis(timestampMs = Date.now()): number | undefined {
+    if (this._oldestUncommittedChange == null) {
+      return this._isStartingReplication ? undefined : 0;
+    }
+    return timestampMs - this._oldestUncommittedChange.getTime();
+  }
+
+  /**
+   * Returns the rolling lag metric value, including the current in-flight lag when present.
+   */
+  getLagMillis(timestampMs = Date.now()): number | undefined {
+    this.rollingReplicationLag.report(this.getCurrentLagMillis(timestampMs), timestampMs);
+    return this.rollingReplicationLag.getRollingMax(timestampMs);
+  }
+}

package/src/replication/replication-index.ts

@@ -1,7 +1,8 @@
 export * from './AbstractReplicationJob.js';
 export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
+export * from './RelationCache.js';
+export * from './replication-metrics.js';
 export * from './ReplicationEngine.js';
+export * from './ReplicationLagTracker.js';
 export * from './ReplicationModule.js';
-export * from './replication-metrics.js';
-export * from './RelationCache.js';

package/src/replication/replication-metrics.ts

@@ -1,5 +1,5 @@
-import { MetricsEngine } from '../metrics/metrics-index.js';
 import { ReplicationMetric } from '@powersync/service-types';
+import { MetricsEngine } from '../metrics/metrics-index.js';
 
 /**
  *  Create and register the core replication metrics.

package/src/routes/endpoints/admin.ts

@@ -119,7 +119,7 @@ export const reprocess = routeDefinition({
     const apiHandler = service_context.routerEngine.getAPI();
     const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
     if (next != null) {
-      throw new Error(`Busy processing sync rules - cannot reprocess`);
+      throw new Error(`Busy processing sync config - cannot reprocess`);
     }
 
     const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
@@ -127,13 +127,17 @@ export const reprocess = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
-        description: 'No active sync rules'
+        description: 'No active sync config'
       });
     }
 
+    // There are some differences between this and using asUpdateOptions():
+    // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
+    // 2. If the source does not set the storage version, this will update it do the current version.
+    // We can consider tweaking this behavior in the future.
     const new_rules = await activeBucketStorage.updateSyncRules(
       storage.updateSyncRulesFromYaml(active.sync_rules.config.content, {
-        // These sync rules already passed validation. But if the rules are not valid anymore due
+        // This sync config already passed validation. But if the config is not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
       })

package/src/routes/endpoints/checkpointing.ts

@@ -33,7 +33,7 @@ export const writeCheckpoint = routeDefinition({
       const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
       const cp = await bucketStorage?.getCheckpoint();
       if (cp == null) {
-        throw new Error('No sync rules available');
+        throw new Error('No sync config available');
       }
       if (cp.lsn && cp.lsn >= head) {
         logger.info(`Got write checkpoint: ${head} : ${cp.checkpoint}`);

package/src/routes/endpoints/socket-route.ts

@@ -77,7 +77,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
           new errors.ServiceError({
             status: 500,
             code: ErrorCode.PSYNC_S2302,
-            description: 'No sync rules available'
+            description: 'No sync config available'
           })
         );
         responder.onComplete();

package/src/routes/endpoints/sync-rules.ts

@@ -4,9 +4,9 @@ import type { FastifyPluginAsync } from 'fastify';
 import * as t from 'ts-codec';
 
 import { RouteAPI } from '../../api/RouteAPI.js';
+import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 
 const DeploySyncRulesRequest = t.object({
   content: t.string
@@ -43,12 +43,12 @@ export const deploySyncRules = routeDefinition({
     const { storageEngine } = service_context;
 
     if (service_context.configuration.sync_rules.present) {
-      // If sync rules are configured via the config, disable deploy via the API.
+      // If sync config is configured via the service config, disable deploy via the API.
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4105,
-        description: 'Sync rules API disabled',
-        details: 'Use the management API to deploy sync rules'
+        description: 'Sync config API disabled',
+        details: 'Update sync config in the service configuration'
       });
     }
     const content = payload.params.content;
@@ -56,7 +56,7 @@ export const deploySyncRules = routeDefinition({
 
     try {
       const apiHandler = service_context.routerEngine.getAPI();
-      syncConfig = SqlSyncRules.fromYaml(payload.params.content, {
+      syncConfig = SqlSyncRules.fromYaml(content, {
         ...apiHandler.getParseSyncRulesOptions(),
         // We don't do any schema-level validation at this point
         schema: undefined
@@ -65,14 +65,12 @@ export const deploySyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_R0001,
-        description: 'Sync rules parsing failed',
+        description: 'Sync config parsing failed',
         details: e.message
       });
     }
 
-    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(
-      updateSyncRulesFromConfig(syncConfig.config)
-    );
+    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
 
     return {
       slot_name: sync_rules.slot_name
@@ -117,7 +115,7 @@ export const currentSyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
-        description: 'No active sync rules'
+        description: 'No active sync config'
       });
     }
 
@@ -164,13 +162,13 @@ export const reprocessSyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
-        description: 'No active sync rules'
+        description: 'No active sync config'
       });
     }
 
     const new_rules = await activeBucketStorage.updateSyncRules(
       updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
-        // These sync rules already passed validation. But if the rules are not valid anymore due
+        // This sync config already passed validation. But if the rules are not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
       })

package/src/routes/endpoints/sync-stream.ts

@@ -68,7 +68,7 @@ export const syncStreamed = routeDefinition({
       throw new errors.ServiceError({
         status: 500,
         code: ErrorCode.PSYNC_S2302,
-        description: 'No sync rules available'
+        description: 'No sync config available'
       });
     }
 
@@ -121,7 +121,7 @@ export const syncStreamed = routeDefinition({
       });
 
       stream.on('end', () => {
-        // Auth failure or switch to new sync rules
+        // Auth failure or switch to new sync config
         closeReason ??= 'service closing stream';
       });
 

package/src/routes/hooks.ts

@@ -1,6 +1,6 @@
-import type fastify from 'fastify';
-import a from 'async';
 import { logger } from '@powersync/lib-services-framework';
+import a from 'async';
+import type fastify from 'fastify';
 
 export type CreateRequestQueueParams = {
   max_queue_depth: number;

package/src/routes/route-register.ts

@@ -1,17 +1,9 @@
 import type fastify from 'fastify';
 import * as uuid from 'uuid';
 
-import {
-  ErrorCode,
-  errors,
-  HTTPMethod,
-  logger,
-  RouteNotFound,
-  router,
-  ServiceError
-} from '@powersync/lib-services-framework';
-import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
+import { errors, HTTPMethod, logger, RouteNotFound, router, ServiceError } from '@powersync/lib-services-framework';
 import { FastifyReply } from 'fastify';
+import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
   parse?: boolean;

package/src/runner/teardown.ts

@@ -35,13 +35,13 @@ export async function teardown(runnerConfig: utils.RunnerConfig) {
 }
 
 async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
-  logger.info(`Terminating sync rules...`);
+  logger.info(`Terminating replication stream...`);
   const start = Date.now();
   const locks: storage.ReplicationLock[] = [];
   while (Date.now() - start < 120_000) {
     let retry = false;
     const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
-    // Lock all the replicating sync rules
+    // Lock all the replicating replication streams
     for (const replicatingSyncRule of replicatingSyncRules) {
       const lock = await replicatingSyncRule.lock();
       locks.push(lock);
@@ -50,10 +50,10 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
     const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
     const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
     try {
-      // Clean up any module specific configuration for the sync rules
+      // Clean up any module specific configuration for the replication stream
       await moduleManager.tearDown({ syncRules: combinedSyncRules });
 
-      // Mark the sync rules as terminated
+      // Mark the replication stream as terminated
       for (let syncRules of combinedSyncRules) {
         const syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here

package/src/storage/BucketStorage.ts

@@ -2,36 +2,36 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 
 export enum SyncRuleState {
   /**
-   * New sync rules - needs to be processed (initial replication).
+   * New replication stream - needs to be processed (initial replication).
    *
-   * While multiple sets of sync rules _can_ be in PROCESSING,
+   * While multiple replication streams _can_ be in PROCESSING,
    * it's generally pointless, so we only keep one in that state.
    */
   PROCESSING = 'PROCESSING',
 
   /**
-   * Sync rule processing is done, and can be used for sync.
+   * Intial processing is done, and can be used for sync.
    *
-   * Only one set of sync rules should be in ACTIVE or ERRORED state.
+   * Only one replication stream should be in ACTIVE or ERRORED state.
    */
   ACTIVE = 'ACTIVE',
   /**
-   * This state is used when the sync rules has been replaced,
+   * This state is used when the replication stream has been replaced,
    * and replication is or should be stopped.
    */
   STOP = 'STOP',
   /**
-   * After sync rules have been stopped, the data needs to be
+   * After replication stream has been stopped, the data needs to be
    * deleted. Once deleted, the state is TERMINATED.
    */
   TERMINATED = 'TERMINATED',
 
   /**
-   * Sync rules has run into a permanent replication error. It
-   * is still the "active" sync rules for syncing to users,
+   * Replication stream has run into a permanent replication error. It
+   * is still the "active" replication stram for syncing to users,
    * but should not replicate anymore.
    *
-   * It will transition to STOP when a new sync rules is activated.
+   * It will transition to STOP when a new replication stream is activated.
    */
   ERRORED = 'ERRORED'
 }

package/src/storage/BucketStorageBatch.ts

@@ -1,10 +1,10 @@
 import { ObserverClient } from '@powersync/lib-services-framework';
 import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
+import { InternalOpId } from '../util/utils.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
-import { InternalOpId } from '../util/utils.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
   createEmptyCheckpoints: true,