@powersync/service-core 1.22.0 → 1.23.0

package/src/storage/PersistedSyncConfigContent.ts

@@ -0,0 +1,188 @@
+import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import {
+  CompatibilityContext,
+  CompatibilityOption,
+  DEFAULT_HYDRATION_STATE,
+  deserializeSyncPlan,
+  ErrorLocation,
+  HydratedSyncConfig,
+  HydrationState,
+  nodeSqlite,
+  PrecompiledSyncConfig,
+  SqlEventDescriptor,
+  SqlSyncRules,
+  SyncConfigWithErrors,
+  versionedHydrationState,
+  YamlError
+} from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
+import { Logger } from 'winston';
+import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
+import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import { PersistedSyncConfigStatus } from './PersistedSyncConfigStatus.js';
+import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
+
+export interface ParsePersistedSyncConfigContentOptions {
+  content: string;
+  compiledPlan: SerializedSyncPlan | null;
+  storageVersion: number;
+  parseOptions: ParseSyncConfigOptions;
+}
+
+export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfigContentOptions): SyncConfigWithErrors {
+  const { content, compiledPlan, storageVersion, parseOptions } = options;
+
+  if (compiledPlan == null) {
+    // Fallback: Only parse from YAML if no compiled plan is available.
+    return SqlSyncRules.fromYaml(content, parseOptions);
+  }
+
+  const plan = deserializeSyncPlan(compiledPlan.plan);
+  const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility);
+  const eventDefinitions: SqlEventDescriptor[] = [];
+  for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) {
+    const descriptor = new SqlEventDescriptor(name, compatibility);
+    for (const query of queries) {
+      descriptor.addSourceQuery(query, parseOptions);
+    }
+
+    eventDefinitions.push(descriptor);
+  }
+
+  const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
+    defaultSchema: parseOptions.defaultSchema,
+    sourceText: content
+  });
+
+  // Note: If the original content did not define a storage version, this will still set the storage version.
+  // This means asUpdateOptions will not change the storage version, even if the default changes.
+  precompiled.storageVersion = storageVersion;
+
+  const errors: YamlError[] = [];
+  if (compiledPlan.errors) {
+    for (const error of compiledPlan.errors) {
+      const location: ErrorLocation | undefined = error.location && {
+        start: error.location.start_offset,
+        end: error.location.end_offset
+      };
+      const asYamlError = new YamlError(new Error(error.message), location);
+      asYamlError.type = error.level;
+
+      errors.push(asYamlError);
+    }
+  }
+
+  return { config: precompiled, errors };
+}
+
+/**
+ * Immutable sync config content for one sync config inside a replication stream.
+ *
+ * This represents the persisted config content. Fetch per-config status with
+ * {@link getSyncConfigStatus}. Use {@link PersistedReplicationStream} for stream
+ * lifecycle concerns such as locking and termination.
+ */
+
+export abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData {
+  readonly replicationStreamId: number;
+  readonly sync_rules_content: string;
+  readonly compiled_plan: SerializedSyncPlan | null;
+  readonly replicationStreamName: string;
+  readonly storageVersion: number;
+  readonly logger: Logger;
+  readonly syncConfigId: PersistedSyncConfigId | null;
+
+  constructor(data: PersistedSyncConfigContentData) {
+    this.replicationStreamId = data.replicationStreamId;
+    this.sync_rules_content = data.sync_rules_content;
+    this.compiled_plan = data.compiled_plan;
+    this.replicationStreamName = data.replicationStreamName;
+    this.storageVersion = data.storageVersion;
+    this.syncConfigId = data.syncConfigId ?? null;
+    this.logger = defaultLogger.child({ prefix: `[${this.replicationStreamName}] ` });
+  }
+
+  /**
+   * Load the storage config.
+   *
+   * This may throw if the persisted storage version is not supported.
+   */
+  getStorageConfig(): StorageVersionConfig {
+    const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion];
+    if (storageConfig == null) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S1005,
+        `Unsupported storage version ${this.storageVersion} for replication stream ${this.replicationStreamId}`
+      );
+    }
+    return storageConfig;
+  }
+
+  /**
+   * Parse only this config's content into a single {@link SyncConfigWithErrors}.
+   *
+   * This does not depend on any other configs in the same replication stream.
+   */
+  protected parseSingleConfig(options: ParseSyncConfigOptions): SyncConfigWithErrors {
+    return parsePersistedSyncConfigContent({
+      content: this.sync_rules_content,
+      compiledPlan: this.compiled_plan,
+      storageVersion: this.storageVersion,
+      parseOptions: options
+    });
+  }
+
+  parsed(options: ParseSyncConfigOptions): ParsedSyncConfigSet {
+    let hydrationState: HydrationState;
+    const config = this.parseSingleConfig(options);
+
+    const storageConfig = this.getStorageConfig();
+    if (
+      storageConfig.versionedBuckets ||
+      config.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)
+    ) {
+      hydrationState = versionedHydrationState(this.replicationStreamId);
+    } else {
+      hydrationState = DEFAULT_HYDRATION_STATE;
+    }
+
+    let hydrated: HydratedSyncConfig | undefined;
+    return {
+      replicationStreamId: this.replicationStreamId,
+      replicationStreamName: this.replicationStreamName,
+      syncConfigs: [config],
+      hydrationState,
+      get hydratedSyncConfig(): HydratedSyncConfig {
+        hydrated ??= config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
+        return hydrated;
+      }
+    };
+  }
+
+  asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
+    // defaultSchema is not relevant for the parsed version here
+    const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' });
+    return {
+      config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed },
+      ...options
+    };
+  }
+
+  /**
+   * Fetch the current persisted state for this exact sync config.
+   */
+  abstract getSyncConfigStatus(): Promise<PersistedSyncConfigStatus | null>;
+}
+export interface PersistedSyncConfigContentData {
+  readonly replicationStreamId: number;
+  readonly sync_rules_content: string;
+  readonly compiled_plan: SerializedSyncPlan | null;
+  readonly replicationStreamName: string;
+  readonly storageVersion: number;
+
+  readonly syncConfigId?: PersistedSyncConfigId | null;
+}
+export type PersistedSyncConfigId = string;
+export interface ParseSyncConfigOptions {
+  defaultSchema: string;
+}

package/src/storage/PersistedSyncConfigStatus.ts

@@ -0,0 +1,14 @@
+import { SyncRuleState } from './BucketStorage.js';
+import { PersistedSyncConfigId } from './PersistedSyncConfigContent.js';
+
+export interface PersistedSyncConfigStatus {
+  readonly id: PersistedSyncConfigId;
+  readonly replicationStreamId: number;
+  readonly state: SyncRuleState;
+  readonly snapshot_done?: boolean;
+  readonly last_checkpoint_lsn: string | null;
+  readonly last_fatal_error?: string | null;
+  readonly last_fatal_error_ts?: Date | null;
+  readonly last_keepalive_ts?: Date | null;
+  readonly last_checkpoint_ts?: Date | null;
+}

package/src/storage/SourceTable.ts

@@ -1,6 +1,8 @@
 import {
   BucketDataSource,
+  BucketDefinitionId,
   DEFAULT_TAG,
+  ParameterIndexId,
   ParameterIndexLookupCreator,
   SourceTableRef
 } from '@powersync/service-sync-rules';
@@ -21,6 +23,8 @@ export interface SourceTableOptions {
   snapshotComplete: boolean;
   bucketDataSources: BucketDataSource[];
   parameterLookupSources: ParameterIndexLookupCreator[];
+  bucketDataSourceIds?: Set<BucketDefinitionId>;
+  parameterLookupSourceIds?: Set<ParameterIndexId>;
 }
 
 export interface TableSnapshotStatus {
@@ -57,9 +61,11 @@ export class SourceTable {
   public syncParameters = true;
 
   /**
-   * True if the table is used in sync config for events.
+   * True if this table should fire events for row changes.
    *
-   * This value is resolved externally, and cached here.
+   * This value is resolved externally, and cached here. When multiple SourceTables exist
+   * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them
+   * as the event carrier, so that a row change saved once per table fires each event once.
    *
    * Defaults to true for tests.
    */
@@ -122,6 +128,14 @@ export class SourceTable {
     return this.options.parameterLookupSources;
   }
 
+  get bucketDataSourceIds() {
+    return this.options.bucketDataSourceIds;
+  }
+
+  get parameterLookupSourceIds() {
+    return this.options.parameterLookupSourceIds;
+  }
+
   /**
    * Sanitized name of the entity in the format of "{schema}.{entity name}".
    * Suitable for safe use in Postgres queries.
@@ -145,7 +159,10 @@ export class SourceTable {
       replicaIdColumns: this.replicaIdColumns,
       snapshotComplete: this.snapshotComplete,
       bucketDataSources: this.bucketDataSources,
-      parameterLookupSources: this.parameterLookupSources
+      parameterLookupSources: this.parameterLookupSources,
+      bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds),
+      parameterLookupSourceIds:
+        this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds)
     });
     copy.syncData = this.syncData;
     copy.syncParameters = this.syncParameters;

package/src/storage/SyncRulesBucketStorage.ts

@@ -10,7 +10,8 @@ import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
-import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
+import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js';
+import { ParseSyncConfigOptions } from './PersistedSyncConfigContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
 import { StorageVersionConfig } from './StorageVersionConfig.js';
@@ -22,8 +23,8 @@ import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 export interface SyncRulesBucketStorage
   extends ObserverClient<SyncRulesBucketStorageListener>,
     SyncStorageWriteCheckpointAPI {
-  readonly group_id: number;
-  readonly slot_name: string;
+  readonly replicationStreamId: number;
+  readonly replicationStreamName: string;
   readonly storageConfig: StorageVersionConfig;
 
   readonly factory: BucketStorageFactory;
@@ -44,7 +45,20 @@ export interface SyncRulesBucketStorage
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncConfig;
+  /**
+   * Get the canonical parsed sync config set for the given parse options.
+   *
+   * Repeated calls with the same options return the same instance for the lifetime of
+   * this storage instance. This is the identity boundary for parsed sync config state:
+   * all operations against this storage instance must use parsed objects from sets
+   * returned here, never from an independent parse of the same content.
+   */
+  getParsedSyncConfigSet(options: ParseSyncConfigOptions): ParsedSyncConfigSet;
+
+  /**
+   * Shortcut for `getParsedSyncConfigSet(options).hydratedSyncConfig`.
+   */
+  getParsedSyncRules(options: ParseSyncConfigOptions): HydratedSyncConfig;
 
   /**
    * Terminate the replication stream.
@@ -55,7 +69,7 @@ export interface SyncRulesBucketStorage
    */
   terminate(options?: TerminateOptions): Promise<void>;
 
-  getStatus(): Promise<SyncRuleStatus>;
+  getStatus(): Promise<ReplicationStreamStatus>;
 
   /**
    * Clear the storage, without changing state.
@@ -133,6 +147,12 @@ export interface SyncRulesBucketStorage
    * Clear checksum cache. Primarily intended for tests.
    */
   clearChecksumCache(): void;
+
+  /**
+   * Optional storage-provider cleanup for sync configs that have been stopped
+   * inside this replication stream.
+   */
+  cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise<CleanupStoppedSyncConfigsResult>;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -149,15 +169,15 @@ export interface BucketChecksumRequest {
   source: BucketDataSource;
 }
 
-export interface SyncRuleStatus {
-  checkpoint_lsn: string | null;
-  active: boolean;
-  snapshot_done: boolean;
-  snapshot_lsn: string | null;
+export interface ReplicationStreamStatus {
+  /**
+   * Source position to resume replication from.
+   */
+  resumeLsn: string | null;
   /**
-   * Last persisted operation that must be included in the next checkpoint once checkpointing is unblocked.
+   * True if _every_ active/processing sync config has a client-visible checkpoint.
    */
-  keepalive_op: util.InternalOpId | null;
+  snapshotDone: boolean;
 }
 export interface ResolveTablesOptions {
   connection_id: number;
@@ -167,9 +187,9 @@ export interface ResolveTablesOptions {
    */
   idGenerator?: () => string | bson.ObjectId;
   /**
-   * For tests only - override the sync rules used.
+   * For tests only - override the parsed sync config set used.
    */
-  syncRules?: HydratedSyncConfig;
+  parsedSyncConfig?: ParsedSyncConfigSet;
 }
 
 export interface ResolveTablesResult {
@@ -177,7 +197,7 @@ export interface ResolveTablesResult {
   dropTables: SourceTable[];
 }
 
-export interface CreateWriterOptions extends ParseSyncRulesOptions {
+export interface CreateWriterOptions extends ParseSyncConfigOptions {
   zeroLSN: string;
   /**
    * Whether or not to store a copy of the current data.
@@ -262,6 +282,9 @@ export interface CompactOptions {
   /** Minimum of 1 */
   moveBatchQueryLimit?: number;
 
+  /** Byte cap per read batch for streaming compaction. Default 16MB. */
+  moveBatchByteLimit?: number;
+
   /**
    * Minimum number new operations in a bucket to trigger compaction of that bucket.
    *
@@ -287,7 +310,14 @@ export interface CompactOptions {
 }
 
 export interface PopulateChecksumCacheOptions {
-  maxOpId: util.InternalOpId;
+  /**
+   * Compute checksums up to this op id.
+   *
+   * Defaults to the highest persisted op id for the replication stream, which covers
+   * the common case of populating the cache right after initial replication, before
+   * the first checkpoint exists.
+   */
+  maxOpId?: util.InternalOpId;
   minBucketChanges?: number;
   signal?: AbortSignal;
 }
@@ -303,6 +333,24 @@ export interface ClearStorageOptions {
   signal?: AbortSignal;
 }
 
+export interface CleanupStoppedSyncConfigsOptions {
+  signal?: AbortSignal;
+  logger?: Logger;
+  defaultSchema: string;
+  sourceConnectionTag: string;
+}
+
+export interface CleanupStoppedSyncConfigsResult {
+  stoppedSyncConfigsRemoved: number;
+  bucketDataCollectionsDropped: number;
+  parameterIndexCollectionsDropped: number;
+  /** Best-effort number, not accurate if the operation takes longer than the timeout. */
+  bucketStateDocumentsDeleted: number;
+  sourceRecordCollectionsDropped: number;
+  sourceTablesUpdated: number;
+  sourceTablesDeleted: number;
+}
+
 export interface TerminateOptions extends ClearStorageOptions {
   /**
    * If true, also clear the storage before terminating.