@powersync/service-core 1.22.0 → 1.23.1

package/src/routes/endpoints/admin.ts

@@ -1,7 +1,13 @@
 import * as sqlite from 'node:sqlite';
 
 import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
-import { nodeSqlite, SourceSchema, SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
+import {
+  HydratedSyncConfig,
+  nodeSqlite,
+  SourceSchema,
+  SqlSyncRules,
+  StaticSchema
+} from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
@@ -65,20 +71,35 @@ export const diagnostics = routeDefinition({
     const {
       storageEngine: { activeBucketStorage }
     } = service_context;
-    const active = await activeBucketStorage.getActiveSyncRulesContent();
-    const next = await activeBucketStorage.getNextSyncRulesContent();
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    const deploying = await activeBucketStorage.getDeployingSyncConfig();
 
-    const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, {
-      include_content,
-      check_connection: status.connected,
-      live_status: true
-    });
+    const active_status = await api.getSyncRulesStatus(
+      apiHandler,
+      active?.content ?? null,
+      {
+        include_content,
+        check_connection: status.connected,
+        live_status: true,
+        active: true
+      },
+      active?.storage
+    );
 
-    const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, {
-      include_content,
-      check_connection: status.connected,
-      live_status: true
-    });
+    const deploying_status =
+      deploying == null
+        ? undefined
+        : await api.getSyncRulesStatus(
+            apiHandler,
+            deploying.content,
+            {
+              include_content,
+              check_connection: status.connected,
+              live_status: true,
+              active: false
+            },
+            deploying.storage
+          );
 
     return internal_routes.DiagnosticsResponse.encode({
       connections: [
@@ -89,7 +110,7 @@ export const diagnostics = routeDefinition({
         }
       ],
       active_sync_rules: active_status,
-      deploying_sync_rules: next_status
+      deploying_sync_rules: deploying_status
     });
   }
 });
@@ -119,12 +140,16 @@ export const reprocess = routeDefinition({
       storageEngine: { activeBucketStorage }
     } = service_context;
     const apiHandler = service_context.routerEngine.getAPI();
-    const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions());
+    const next = await activeBucketStorage.getDeployingSyncConfig();
     if (next != null) {
-      throw new Error(`Busy processing sync config - cannot reprocess`);
+      throw new errors.ServiceError({
+        status: 409,
+        code: ErrorCode.PSYNC_S4106,
+        description: 'Busy processing sync config - cannot reprocess'
+      });
     }
 
-    const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
+    const active = await activeBucketStorage.getActiveSyncConfig();
     if (active == null) {
       throw new errors.ServiceError({
         status: 422,
@@ -132,13 +157,12 @@ export const reprocess = routeDefinition({
         description: 'No active sync config'
       });
     }
-
     // There are some differences between this and using asUpdateOptions():
     // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan.
     // 2. If the source does not set the storage version, this will update it do the current version.
     // We can consider tweaking this behavior in the future.
     const new_rules = await activeBucketStorage.updateSyncRules(
-      storage.updateSyncRulesFromYaml(active.syncConfigWithErrors.config.content, {
+      storage.updateSyncRulesFromYaml(active.content.sync_rules_content, {
         // This sync config already passed validation. But if the config is not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
@@ -153,44 +177,47 @@ export const reprocess = routeDefinition({
           // Previously the connection was asserted with `!`
           tag: baseConfig.tag,
           id: baseConfig.id,
-          slot_name: new_rules.slot_name
+          slot_name: new_rules.replicationStreamName
         }
       ]
     });
   }
 });
 
-class FakeSyncRulesContentForValidation extends storage.PersistedSyncRulesContent {
+class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent {
   constructor(
     private readonly apiHandler: api.RouteAPI,
     private readonly schema: SourceSchema,
-    data: storage.PersistedSyncRulesContentData
+    data: storage.PersistedSyncConfigContentData
   ) {
     super(data);
   }
 
-  current_lock: storage.ReplicationLock | null = null;
-
-  async lock(): Promise<storage.ReplicationLock> {
-    throw new Error('Lock not implemented');
-  }
+  parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
+    const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, {
+      ...this.apiHandler.getParseSyncRulesOptions(),
+      schema: this.schema
+    });
 
-  parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules {
+    let hydrated: HydratedSyncConfig | undefined;
     return {
-      ...this,
-      syncConfigWithErrors: SqlSyncRules.fromYaml(this.sync_rules_content, {
-        ...this.apiHandler.getParseSyncRulesOptions(),
-        schema: this.schema
-      }),
+      replicationStreamId: this.replicationStreamId,
+      replicationStreamName: this.replicationStreamName,
+      syncConfigs: [syncConfig],
       hydrationState: DEFAULT_HYDRATION_STATE,
-      hydratedSyncConfig() {
-        return this.syncConfigWithErrors.config.hydrate({
+      get hydratedSyncConfig(): HydratedSyncConfig {
+        hydrated ??= syncConfig.config.hydrate({
           hydrationState: DEFAULT_HYDRATION_STATE,
           sqlite: nodeSqlite(sqlite)
         });
+        return hydrated;
       }
     };
   }
+
+  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
+    return null;
+  }
 }
 
 export const validate = routeDefinition({
@@ -210,13 +237,12 @@ export const validate = routeDefinition({
 
     const sync_rules = new FakeSyncRulesContentForValidation(apiHandler, schema, {
       // Dummy values
-      id: 0,
-      slot_name: '',
-      active: false,
-      last_checkpoint_lsn: '',
+      replicationStreamId: 0,
+      replicationStreamName: '',
       storageVersion: storage.LEGACY_STORAGE_VERSION,
       sync_rules_content: content,
-      compiled_plan: null
+      compiled_plan: null,
+      syncConfigState: storage.SyncRuleState.PROCESSING
     });
 
     const connectionStatus = await apiHandler.getConnectionStatus();
@@ -227,16 +253,12 @@ export const validate = routeDefinition({
       });
     }
 
-    const status = (await api.getSyncRulesStatus(
-      service_context.storageEngine.activeBucketStorage,
-      apiHandler,
-      sync_rules,
-      {
-        include_content: false,
-        check_connection: connectionStatus.connected,
-        live_status: false
-      }
-    ))!;
+    const status = (await api.getSyncRulesStatus(apiHandler, sync_rules, {
+      include_content: false,
+      check_connection: connectionStatus.connected,
+      live_status: false,
+      active: false
+    }))!;
 
     if (connectionStatus == null) {
       status.errors.push({ level: 'fatal', message: 'No connection configured', ts: new Date().toISOString() });

package/src/routes/endpoints/checkpointing.ts

@@ -30,7 +30,7 @@ export const writeCheckpoint = routeDefinition({
 
     logger.info(`Waiting for LSN checkpoint: ${head}`);
     while (Date.now() - start < timeout) {
-      const bucketStorage = await service_context.storageEngine.activeBucketStorage.getActiveStorage();
+      const bucketStorage = (await service_context.storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
       const cp = await bucketStorage?.getCheckpoint();
       if (cp == null) {
         throw new Error('No sync config available');

package/src/routes/endpoints/socket-route.ts

@@ -71,7 +71,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         storageEngine: { activeBucketStorage }
       } = service_context;
 
-      const bucketStorage = await activeBucketStorage.getActiveStorage();
+      const bucketStorage = (await activeBucketStorage.getActiveSyncConfig())?.storage;
       if (bucketStorage == null) {
         responder.onError(
           new errors.ServiceError({

package/src/routes/endpoints/sync-rules.ts

@@ -73,7 +73,7 @@ export const deploySyncRules = routeDefinition({
     const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
 
     return {
-      slot_name: sync_rules.slot_name
+      slot_name: sync_rules.replicationStreamName
     };
   }
 });
@@ -110,8 +110,8 @@ export const currentSyncRules = routeDefinition({
       storageEngine: { activeBucketStorage }
     } = service_context;
 
-    const sync_rules = await activeBucketStorage.getActiveSyncRulesContent();
-    if (!sync_rules) {
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    if (!active) {
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
@@ -119,15 +119,16 @@ export const currentSyncRules = routeDefinition({
       });
     }
 
+    const sync_rules = active.content;
     const apiHandler = service_context.routerEngine.getAPI();
     const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content);
-    const next = await activeBucketStorage.getNextSyncRulesContent();
+    const next = await activeBucketStorage.getDeployingSyncConfig();
 
-    const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null;
+    const next_info = next ? await debugSyncRules(apiHandler, next.content.sync_rules_content) : null;
 
     const response = {
       current: {
-        slot_name: sync_rules.slot_name,
+        slot_name: sync_rules.replicationStreamName,
         content: sync_rules.sync_rules_content,
         ...info
       },
@@ -135,8 +136,8 @@ export const currentSyncRules = routeDefinition({
         next == null
           ? null
           : {
-              slot_name: next.slot_name,
-              content: next.sync_rules_content,
+              slot_name: next.content.replicationStreamName,
+              content: next.content.sync_rules_content,
               ...next_info
             }
     };
@@ -156,9 +157,17 @@ export const reprocessSyncRules = routeDefinition({
     const {
       storageEngine: { activeBucketStorage }
     } = payload.context.service_context;
-    const apiHandler = payload.context.service_context.routerEngine.getAPI();
-    const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions());
-    if (sync_rules == null) {
+    const next = await activeBucketStorage.getDeployingSyncConfig();
+    if (next != null) {
+      throw new errors.ServiceError({
+        status: 409,
+        code: ErrorCode.PSYNC_S4106,
+        description: 'Busy processing sync config - cannot reprocess'
+      });
+    }
+
+    const active = await activeBucketStorage.getActiveSyncConfig();
+    if (active == null) {
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
@@ -166,15 +175,16 @@ export const reprocessSyncRules = routeDefinition({
       });
     }
 
+    const sync_rules = active.content;
     const new_rules = await activeBucketStorage.updateSyncRules(
-      updateSyncRulesFromYaml(sync_rules.syncConfigWithErrors.config.content, {
+      updateSyncRulesFromYaml(sync_rules.sync_rules_content, {
         // This sync config already passed validation. But if the rules are not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
       })
     );
     return {
-      slot_name: new_rules.slot_name
+      slot_name: new_rules.replicationStreamName
     };
   }
 });

package/src/routes/endpoints/sync-stream.ts

@@ -62,7 +62,7 @@ export const syncStreamed = routeDefinition({
       });
     }
 
-    const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage();
+    const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
 
     if (bucketStorage == null) {
       throw new errors.ServiceError({

package/src/runner/teardown.ts

@@ -40,22 +40,22 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
   const locks: storage.ReplicationLock[] = [];
   while (Date.now() - start < 120_000) {
     let retry = false;
-    const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
+    const replicatingStreams = await storageFactory.getReplicatingReplicationStreams();
     // Lock all the replicating replication streams
-    for (const replicatingSyncRule of replicatingSyncRules) {
-      const lock = await replicatingSyncRule.lock();
+    for (const replicatingStream of replicatingStreams) {
+      const lock = await replicatingStream.lock();
       locks.push(lock);
     }
 
-    const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
-    const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
+    const stoppedStreams = await storageFactory.getStoppedReplicationStreams();
+    const combinedStreams = [...replicatingStreams, ...stoppedStreams];
     try {
       // Clean up any module specific configuration for the replication stream
-      await moduleManager.tearDown({ syncRules: combinedSyncRules });
+      await moduleManager.tearDown({ replicationStreams: combinedStreams, syncRules: combinedStreams });
 
       // Mark the replication stream as terminated
-      for (let syncRules of combinedSyncRules) {
-        const syncRulesStorage = storageFactory.getInstance(syncRules);
+      for (let replicationStream of combinedStreams) {
+        const syncRulesStorage = storageFactory.getInstance(replicationStream);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here
         await syncRulesStorage.terminate({ clearStorage: false });
       }

package/src/storage/BucketStorageBatch.ts

@@ -83,11 +83,6 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
    */
   setResumeLsn(lsn: string): Promise<void>;
 
-  /**
-   * Get the last checkpoint LSN, from either commit or keepalive.
-   */
-  lastCheckpointLsn: string | null;
-
   /**
    * LSN to resume from.
    *
@@ -216,8 +211,6 @@ export interface CheckpointResult {
   /**
    * True if a checkpoint was actually created by this operation. This can be false even if checkpointBlocked is false,
    * if the checkpoint was empty.
-   *
-   * This is primarily used for testing.
    */
   checkpointCreated: boolean;
 }

package/src/storage/BucketStorageFactory.ts

@@ -2,13 +2,15 @@ import { BaseObserver, logger } from '@powersync/lib-services-framework';
 import {
   PrecompiledSyncConfig,
   SerializedCompatibilityContext,
+  SerializedSyncPlanV1,
   serializeSyncPlan,
   SqlSyncRules,
   SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
 import { ReplicationError } from '@powersync/service-types';
 import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { PersistedReplicationStream } from './PersistedReplicationStream.js';
+import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { ReportStorage } from './ReportStorage.js';
@@ -31,14 +33,14 @@ export abstract class BucketStorageFactory
    */
   async configureSyncRules(
     options: UpdateSyncRulesOptions
-  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> {
-    const next = await this.getNextSyncRulesContent();
-    const active = await this.getActiveSyncRulesContent();
+  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedReplicationStream; lock?: ReplicationLock }> {
+    const deploying = await this.getDeployingSyncConfig();
+    const active = await this.getActiveSyncConfig();
 
-    if (next?.sync_rules_content == options.config.yaml) {
+    if (deploying?.content.sync_rules_content == options.config.yaml) {
       logger.info('Sync config unchanged');
       return { updated: false };
-    } else if (next == null && active?.sync_rules_content == options.config.yaml) {
+    } else if (deploying == null && active?.content.sync_rules_content == options.config.yaml) {
       logger.info('Sync config unchanged');
       return { updated: false };
     } else {
@@ -51,12 +53,15 @@ export abstract class BucketStorageFactory
   /**
    * Get a storage instance to query sync data for specific sync config.
    */
-  abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
+  abstract getInstance(
+    replicationStream: PersistedReplicationStream,
+    options?: GetIntanceOptions
+  ): SyncRulesBucketStorage;
 
   /**
    * Deploy new sync config.
    */
-  abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
+  abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
 
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
@@ -68,48 +73,27 @@ export abstract class BucketStorageFactory
    *
    * Replication should be restarted after this.
    */
-  abstract restartReplication(sync_rules_group_id: number): Promise<void>;
+  abstract restartReplication(replicationStreamId: number): Promise<void>;
 
   /**
-   * Get the sync config used for querying.
+   * Get the sync config and storage used for querying.
    */
-  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
-    const content = await this.getActiveSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
+  abstract getActiveSyncConfig(): Promise<ResolvedSyncConfig | null>;
 
   /**
-   * Get the sync config used for querying.
+   * Get the sync config and storage that is still deploying.
    */
-  abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+  abstract getDeployingSyncConfig(): Promise<ResolvedSyncConfig | null>;
 
   /**
-   * Get the sync config that will be active next once done with initial replicatino.
+   * Get all replication streams currently replicating.
    */
-  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
-    const content = await this.getNextSyncRulesContent();
-    return content?.parsed(options) ?? null;
-  }
+  abstract getReplicatingReplicationStreams(): Promise<PersistedReplicationStream[]>;
 
   /**
-   * Get the sync config that will be active next once done with initial replicatino.
+   * Get all replication streams stopped but not terminated yet.
    */
-  abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
-
-  /**
-   * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
-   */
-  abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
-
-  /**
-   * Get all sync config stopped but not terminated yet.
-   */
-  abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
-
-  /**
-   * Get the active storage instance.
-   */
-  abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
+  abstract getStoppedReplicationStreams(): Promise<PersistedReplicationStream[]>;
 
   /**
    * Get storage size of active replication stream.
@@ -134,6 +118,12 @@ export interface BucketStorageFactoryListener {
   replicationEvent: (event: ReplicationEventPayload) => void;
 }
 
+export interface ResolvedSyncConfig {
+  content: PersistedSyncConfigContent;
+  replicationStream: PersistedReplicationStream;
+  storage: SyncRulesBucketStorage;
+}
+
 export interface StorageMetrics {
   /**
    * Size of operations (bucket_data)
@@ -181,7 +171,7 @@ export interface SerializedSyncPlan {
   /**
    * The serialized plan, from {@link serializeSyncPlan}.
    */
-  plan: unknown;
+  plan: SerializedSyncPlanV1;
   compatibility: SerializedCompatibilityContext;
   /**
    * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler
@@ -278,4 +268,5 @@ export interface TestStorageConfig {
   factory: TestStorageFactory;
   tableIdStrings: boolean;
   storageVersion?: number;
+  compressedBucketStorage?: boolean;
 }

package/src/storage/ParsedSyncConfigSet.ts

@@ -0,0 +1,25 @@
+import { HydratedSyncConfig, HydrationState, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+
+/**
+ * A parsed sync config set is the identity boundary for parsed sync config state.
+ *
+ * All values on a single instance - the parsed configs, hydration state and hydrated
+ * sync config - are derived from the same parse, and may be associated with each other
+ * by object identity. Values from two different instances must never be mixed, even if
+ * they were parsed from the same persisted content.
+ */
+export interface ParsedSyncConfigSet {
+  readonly replicationStreamId: number;
+  readonly replicationStreamName: string;
+  readonly syncConfigs: SyncConfigWithErrors[];
+  /**
+   * For testing only.
+   */
+  readonly hydrationState: HydrationState;
+
+  /**
+   * The hydrated sync config for this parse. This is a stable property: repeated reads
+   * return the same instance.
+   */
+  readonly hydratedSyncConfig: HydratedSyncConfig;
+}

package/src/storage/PersistedReplicationStream.ts

@@ -0,0 +1,55 @@
+import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { Logger } from 'winston';
+import { SyncRuleState } from './BucketStorage.js';
+import type { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import type { ParseSyncConfigOptions, PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
+
+export abstract class PersistedReplicationStream implements PersistedReplicationStreamData {
+  readonly replicationStreamId: number;
+  readonly replicationJobId: string;
+  readonly replicationStreamName: string;
+  readonly state: SyncRuleState;
+  readonly storageVersion: number;
+  readonly logger: Logger;
+
+  abstract readonly syncConfigContent: readonly PersistedSyncConfigContent[];
+  abstract readonly current_lock: ReplicationLock | null;
+
+  constructor(data: PersistedReplicationStreamData) {
+    this.replicationStreamId = data.replicationStreamId;
+    this.replicationJobId = data.replicationJobId ?? String(data.replicationStreamId);
+    this.replicationStreamName = data.replicationStreamName;
+    this.state = data.state;
+    this.storageVersion = data.storageVersion;
+    this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
+  }
+
+  getStorageConfig(): StorageVersionConfig {
+    const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
+      );
+    }
+    return storageConfig;
+  }
+
+  abstract lock(): Promise<ReplicationLock>;
+
+  abstract parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+}
+export interface PersistedReplicationStreamData {
+  readonly replicationStreamId: number;
+  readonly replicationStreamName: string;
+  readonly state: SyncRuleState;
+  readonly storageVersion: number;
+  /**
+   * Uniquely identifies a job, as a combination of replication stream and processing sync configs.
+   *
+   * Used in-memory only.
+   */
+  readonly replicationJobId?: string;
+}