@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/storage/BucketStorage.ts

@@ -7,10 +7,10 @@ import {
   SqliteRow,
   ToastableSqliteRow
 } from '@powersync/service-sync-rules';
-
-import * as replication from '../replication/replication-index.js';
 import * as util from '../util/util-index.js';
 import { SourceTable } from './SourceTable.js';
+import { SourceEntityDescriptor } from './SourceEntity.js';
+import { ReplicaId } from './storage-index.js';
 
 export interface BucketStorageFactory {
   /**
@@ -24,7 +24,7 @@ export interface BucketStorageFactory {
   /**
    * Get a storage instance to query sync data for specific sync rules.
    */
-  getInstance(options: PersistedSyncRules): SyncRulesBucketStorage;
+  getInstance(options: PersistedSyncRulesContent): SyncRulesBucketStorage;
 
   /**
    * Deploy new sync rules.
@@ -48,7 +48,7 @@ export interface BucketStorageFactory {
   /**
    * Get the sync rules used for querying.
    */
-  getActiveSyncRules(): Promise<PersistedSyncRules | null>;
+  getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
 
   /**
    * Get the sync rules used for querying.
@@ -58,7 +58,7 @@ export interface BucketStorageFactory {
   /**
    * Get the sync rules that will be active next once done with initial replicatino.
    */
-  getNextSyncRules(): Promise<PersistedSyncRules | null>;
+  getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
 
   /**
    * Get the sync rules that will be active next once done with initial replicatino.
@@ -105,7 +105,7 @@ export interface WriteCheckpoint {
 
 export interface ActiveCheckpoint {
   readonly checkpoint: util.OpId;
-  readonly lsn: string;
+  readonly lsn: string | null;
 
   hasSyncRules(): boolean;
 
@@ -131,6 +131,10 @@ export interface StorageMetrics {
   replication_size_bytes: number;
 }
 
+export interface ParseSyncRulesOptions {
+  defaultSchema: string;
+}
+
 export interface PersistedSyncRulesContent {
   readonly id: number;
   readonly sync_rules_content: string;
@@ -140,7 +144,7 @@ export interface PersistedSyncRulesContent {
   readonly last_keepalive_ts?: Date | null;
   readonly last_checkpoint_ts?: Date | null;
 
-  parsed(): PersistedSyncRules;
+  parsed(options: ParseSyncRulesOptions): PersistedSyncRules;
 
   lock(): Promise<ReplicationLock>;
 }
@@ -157,18 +161,6 @@ export interface PersistedSyncRules {
   readonly slot_name: string;
 }
 
-export class DefaultPersistedSyncRules implements PersistedSyncRules {
-  public readonly checkpoint_lsn: string | null;
-
-  constructor(public readonly id: number, public readonly sync_rules: SqlSyncRules, checkpoint_lsn: string | null) {
-    this.checkpoint_lsn = checkpoint_lsn;
-  }
-
-  get slot_name(): string {
-    return `powersync_${this.id}`;
-  }
-}
-
 export interface UpdateSyncRulesOptions {
   content: string;
   lock?: boolean;
@@ -198,8 +190,11 @@ export interface BucketDataBatchOptions {
   chunkLimitBytes?: number;
 }
 
+export interface StartBatchOptions extends ParseSyncRulesOptions {
+  zeroLSN: string;
+}
+
 export interface SyncRulesBucketStorage {
-  readonly sync_rules: SqlSyncRules;
   readonly group_id: number;
   readonly slot_name: string;
 
@@ -207,9 +202,14 @@ export interface SyncRulesBucketStorage {
 
   resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
 
-  startBatch(options: {}, callback: (batch: BucketStorageBatch) => Promise<void>): Promise<FlushedResult | null>;
+  startBatch(
+    options: StartBatchOptions,
+    callback: (batch: BucketStorageBatch) => Promise<void>
+  ): Promise<FlushedResult | null>;
 
-  getCheckpoint(): Promise<{ checkpoint: util.OpId; lsn: string }>;
+  getCheckpoint(): Promise<{ checkpoint: util.OpId }>;
+
+  getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
 
   getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
 
@@ -228,7 +228,7 @@ export interface SyncRulesBucketStorage {
     checkpoint: util.OpId,
     dataBuckets: Map<string, string>,
     options?: BucketDataBatchOptions
-  ): AsyncIterable<util.SyncBucketData>;
+  ): AsyncIterable<SyncBucketDataBatch>;
 
   /**
    * Compute checksums for a given list of buckets.
@@ -244,7 +244,7 @@ export interface SyncRulesBucketStorage {
    *
    * Must only be called on stopped sync rules.
    */
-  terminate(): Promise<void>;
+  terminate(options?: TerminateOptions): Promise<void>;
 
   getStatus(): Promise<SyncRuleStatus>;
 
@@ -266,6 +266,8 @@ export interface SyncRulesBucketStorage {
    * Errors are cleared on commit.
    */
   reportError(e: any): Promise<void>;
+
+  compact(options?: CompactOptions): Promise<void>;
 }
 
 export interface SyncRuleStatus {
@@ -277,7 +279,7 @@ export interface ResolveTableOptions {
   group_id: number;
   connection_id: number;
   connection_tag: string;
-  relation: replication.PgRelation;
+  entity_descriptor: SourceEntityDescriptor;
 
   sync_rules: SqlSyncRules;
 }
@@ -361,7 +363,9 @@ export interface SaveInsert {
   tag: 'insert';
   sourceTable: SourceTable;
   before?: undefined;
+  beforeReplicaId?: undefined;
   after: SqliteRow;
+  afterReplicaId: ReplicaId;
 }
 
 export interface SaveUpdate {
@@ -372,6 +376,7 @@ export interface SaveUpdate {
    * This is only present when the id has changed, and will only contain replica identity columns.
    */
   before?: SqliteRow;
+  beforeReplicaId?: ReplicaId;
 
   /**
    * A null value means null column.
@@ -379,13 +384,21 @@ export interface SaveUpdate {
    * An undefined value means it's a TOAST value - must be copied from another record.
    */
   after: ToastableSqliteRow;
+  afterReplicaId: ReplicaId;
 }
 
 export interface SaveDelete {
   tag: 'delete';
   sourceTable: SourceTable;
-  before: SqliteRow;
+  before?: SqliteRow;
+  beforeReplicaId: ReplicaId;
   after?: undefined;
+  afterReplicaId?: undefined;
+}
+
+export interface SyncBucketDataBatch {
+  batch: util.SyncBucketData;
+  targetOp: bigint | null;
 }
 
 export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqliteRow): ToastableSqliteRow {
@@ -399,3 +412,38 @@ export function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqlit
   }
   return newRecord;
 }
+
+export interface CompactOptions {
+  /**
+   * Heap memory limit for the compact process.
+   *
+   * Add around 64MB to this to determine the "--max-old-space-size" argument.
+   * Add another 80MB to get RSS usage / memory limits.
+   */
+  memoryLimitMB?: number;
+
+  /**
+   * If specified, ignore any operations newer than this when compacting.
+   *
+   * This is primarily for tests, where we want to test compacting at a specific
+   * point.
+   *
+   * This can also be used to create a "safe buffer" of recent operations that should
+   * not be compacted, to avoid invalidating checkpoints in use.
+   */
+  maxOpId?: bigint;
+
+  /**
+   * If specified, compact only the specific buckets.
+   *
+   * If not specified, compacts all buckets.
+   */
+  compactBuckets?: string[];
+}
+
+export interface TerminateOptions {
+  /**
+   * If true, also clear the storage before terminating.
+   */
+  clearStorage: boolean;
+}

package/src/storage/MongoBucketStorage.ts

@@ -1,17 +1,19 @@
-import * as mongo from 'mongodb';
-import * as timers from 'timers/promises';
-import { LRUCache } from 'lru-cache/min';
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { LRUCache } from 'lru-cache/min';
+import * as mongo from 'mongodb';
+import * as timers from 'timers/promises';
 
-import * as replication from '../replication/replication-index.js';
+import * as locks from '../locks/locks-index.js';
 import * as sync from '../sync/sync-index.js';
 import * as util from '../util/util-index.js';
-import * as locks from '../locks/locks-index.js';
 
+import { logger } from '@powersync/lib-services-framework';
+import { v4 as uuid } from 'uuid';
 import {
   ActiveCheckpoint,
   BucketStorageFactory,
+  ParseSyncRulesOptions,
   PersistedSyncRules,
   PersistedSyncRulesContent,
   StorageMetrics,
@@ -23,14 +25,13 @@ import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
 import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js';
 import { SyncRuleDocument, SyncRuleState } from './mongo/models.js';
 import { generateSlotName } from './mongo/util.js';
-import { v4 as uuid } from 'uuid';
-import { logger } from '@powersync/lib-services-framework';
 
 export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {}
 
 export class MongoBucketStorage implements BucketStorageFactory {
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
+  // TODO: This is still Postgres specific and needs to be reworked
   public readonly slot_name_prefix: string;
 
   private readonly storageCache = new LRUCache<number, MongoSyncBucketStorage>({
@@ -47,8 +48,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
         return undefined;
       }
       const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
-      const storage = this.getInstance(rules.parsed());
-      return storage;
+      return this.getInstance(rules);
     }
   });
 
@@ -61,12 +61,12 @@ export class MongoBucketStorage implements BucketStorageFactory {
     this.slot_name_prefix = options.slot_name_prefix;
   }
 
-  getInstance(options: PersistedSyncRules): MongoSyncBucketStorage {
-    let { id, sync_rules, slot_name } = options;
+  getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage {
+    let { id, slot_name } = options;
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    return new MongoSyncBucketStorage(this, id, sync_rules, slot_name);
+    return new MongoSyncBucketStorage(this, id, options, slot_name);
   }
 
   async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) {
@@ -136,7 +136,12 @@ export class MongoBucketStorage implements BucketStorageFactory {
 
   async updateSyncRules(options: UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
     // Parse and validate before applying any changes
-    const parsed = SqlSyncRules.fromYaml(options.content);
+    const parsed = SqlSyncRules.fromYaml(options.content, {
+      // No schema-based validation at this point
+      schema: undefined,
+      defaultSchema: 'not_applicable', // Not needed for validation
+      throwOnError: true
+    });
 
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
@@ -204,9 +209,9 @@ export class MongoBucketStorage implements BucketStorageFactory {
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getActiveSyncRules(): Promise<PersistedSyncRules | null> {
+  async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getActiveSyncRulesContent();
-    return content?.parsed() ?? null;
+    return content?.parsed(options) ?? null;
   }
 
   async getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
@@ -223,9 +228,9 @@ export class MongoBucketStorage implements BucketStorageFactory {
     return new MongoPersistedSyncRulesContent(this.db, doc);
   }
 
-  async getNextSyncRules(): Promise<PersistedSyncRules | null> {
+  async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getNextSyncRulesContent();
-    return content?.parsed() ?? null;
+    return content?.parsed(options) ?? null;
   }
 
   async getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]> {
@@ -294,14 +299,6 @@ export class MongoBucketStorage implements BucketStorageFactory {
   }
 
   async getStorageMetrics(): Promise<StorageMetrics> {
-    const active_sync_rules = await this.getActiveSyncRules();
-    if (active_sync_rules == null) {
-      return {
-        operations_size_bytes: 0,
-        parameters_size_bytes: 0,
-        replication_size_bytes: 0
-      };
-    }
     const operations_aggregate = await this.db.bucket_data
 
       .aggregate([
@@ -370,7 +367,7 @@ export class MongoBucketStorage implements BucketStorageFactory {
   private makeActiveCheckpoint(doc: SyncRuleDocument | null) {
     return {
       checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
-      lsn: doc?.last_checkpoint_lsn ?? replication.ZERO_LSN,
+      lsn: doc?.last_checkpoint_lsn ?? null,
       hasSyncRules() {
         return doc != null;
       },

package/src/storage/SourceEntity.ts

@@ -0,0 +1,22 @@
+export interface ColumnDescriptor {
+  name: string;
+  /**
+   *  The type of the column ie VARCHAR, INT, etc
+   */
+  type?: string;
+  /**
+   *  Some data sources have a type id that can be used to identify the type of the column
+   */
+  typeId?: number;
+}
+
+// TODO: This needs to be consolidated with SourceTable into something new.
+export interface SourceEntityDescriptor {
+  /**
+   *  The internal id of the data source structure in the database
+   */
+  objectId: number | string;
+  schema: string;
+  name: string;
+  replicationColumns: ColumnDescriptor[];
+}

package/src/storage/SourceTable.ts

@@ -1,10 +1,8 @@
-import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules';
-
-import * as replication from '../replication/replication-index.js';
+import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
+import { ColumnDescriptor } from './SourceEntity.js';
 
 export class SourceTable {
-  static readonly DEFAULT_SCHEMA = DEFAULT_SCHEMA;
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
@@ -28,11 +26,11 @@ export class SourceTable {
   constructor(
     public readonly id: any,
     public readonly connectionTag: string,
-    public readonly relationId: number,
+    public readonly objectId: number | string,
     public readonly schema: string,
     public readonly table: string,
 
-    public readonly replicaIdColumns: replication.ReplicationColumn[],
+    public readonly replicaIdColumns: ColumnDescriptor[],
     public readonly snapshotComplete: boolean
   ) {}
 

package/src/storage/StorageEngine.ts

@@ -0,0 +1,55 @@
+import { ResolvedPowerSyncConfig } from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+import { BucketStorageProvider, ActiveStorage } from './StorageProvider.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export type StorageEngineOptions = {
+  configuration: ResolvedPowerSyncConfig;
+};
+
+export class StorageEngine {
+  // TODO: This will need to revisited when we actually support multiple storage providers.
+  private storageProviders: Map<string, BucketStorageProvider> = new Map();
+  private currentActiveStorage: ActiveStorage | null = null;
+
+  constructor(private options: StorageEngineOptions) {}
+
+  get activeBucketStorage(): BucketStorageFactory {
+    return this.activeStorage.storage;
+  }
+
+  get activeStorage(): ActiveStorage {
+    if (!this.currentActiveStorage) {
+      throw new Error(`No storage provider has been initialized yet.`);
+    }
+
+    return this.currentActiveStorage;
+  }
+
+  /**
+   * Register a provider which generates a {@link BucketStorageFactory}
+   * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
+   */
+  registerProvider(provider: BucketStorageProvider) {
+    this.storageProviders.set(provider.type, provider);
+  }
+
+  public async start(): Promise<void> {
+    logger.info('Starting Storage Engine...');
+    const { configuration } = this.options;
+    this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({
+      resolvedConfig: configuration
+    });
+    logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
+    logger.info('Successfully started Storage Engine.');
+  }
+
+  /**
+   *  Shutdown the storage engine, safely shutting down any activated storage providers.
+   */
+  public async shutDown(): Promise<void> {
+    logger.info('Shutting down Storage Engine...');
+    await this.currentActiveStorage?.shutDown();
+    logger.info('Successfully shut down Storage Engine.');
+  }
+}

package/src/storage/StorageProvider.ts

@@ -0,0 +1,27 @@
+import { BucketStorageFactory } from './BucketStorage.js';
+import * as util from '../util/util-index.js';
+
+export interface ActiveStorage {
+  storage: BucketStorageFactory;
+  shutDown(): Promise<void>;
+
+  /**
+   *  Tear down / drop the storage permanently
+   */
+  tearDown(): Promise<boolean>;
+}
+
+export interface GetStorageOptions {
+  // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage
+  resolvedConfig: util.ResolvedPowerSyncConfig;
+}
+
+export interface BucketStorageProvider {
+  /**
+   *  The storage type that this provider provides.
+   *  The type should match the `type` field in the config.
+   */
+  type: string;
+
+  getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
+}

package/src/storage/mongo/MongoBucketBatch.ts

@@ -2,9 +2,8 @@ import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import * as mongo from 'mongodb';
 
-import * as util from '../../util/util-index.js';
-import * as replication from '../../replication/replication-index.js';
 import { container, errors, logger } from '@powersync/lib-services-framework';
+import * as util from '../../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
 import { PowerSyncMongo } from './db.js';
@@ -12,7 +11,7 @@ import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
-import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, serializeLookup } from './util.js';
+import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
 
 /**
  * 15MB
@@ -25,6 +24,7 @@ const MAX_ROW_SIZE = 15 * 1024 * 1024;
 //
 // In the future, we can investigate allowing multiple replication streams operating independently.
 const replicationMutex = new util.Mutex();
+
 export class MongoBucketBatch implements BucketStorageBatch {
   private readonly client: mongo.MongoClient;
   public readonly db: PowerSyncMongo;
@@ -61,7 +61,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
     group_id: number,
     slot_name: string,
     last_checkpoint_lsn: string | null,
-    no_checkpoint_before_lsn: string | null
+    no_checkpoint_before_lsn: string
   ) {
     this.db = db;
     this.client = db.client;
@@ -70,7 +70,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
     this.slot_name = slot_name;
     this.session = this.client.startSession();
     this.last_checkpoint_lsn = last_checkpoint_lsn;
-    this.no_checkpoint_before_lsn = no_checkpoint_before_lsn ?? replication.ZERO_LSN;
+    this.no_checkpoint_before_lsn = no_checkpoint_before_lsn;
   }
 
   async flush(): Promise<FlushedResult | null> {
@@ -301,7 +301,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
     }
 
     // 2. Save bucket data
-    if (beforeId != null && (afterId == null || !beforeId.equals(afterId))) {
+    if (beforeId != null && (afterId == null || !replicaIdEquals(beforeId, afterId))) {
       // Source ID updated
       if (sourceTable.syncData) {
         // Delete old record
@@ -431,7 +431,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
       };
     }
 
-    if (afterId == null || !beforeId.equals(afterId)) {
+    if (afterId == null || !replicaIdEquals(beforeId, afterId)) {
       // Either a delete (afterId == null), or replaced the old replication id
       batch.deleteCurrentData(before_key);
     }
@@ -743,7 +743,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
       const copy = new SourceTable(
         table.id,
         table.connectionTag,
-        table.relationId,
+        table.objectId,
         table.schema,
         table.table,
         table.replicaIdColumns,