@powersync/service-core 0.18.1 → 1.7.0

package/src/storage/BucketStorageBatch.ts

@@ -0,0 +1,158 @@
+import { ObserverClient } from '@powersync/lib-services-framework';
+import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
+import { BSON } from 'bson';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { SourceTable } from './SourceTable.js';
+import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
+
+export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
+  createEmptyCheckpoints: true
+};
+
+export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
+  /**
+   * Save an op, and potentially flush.
+   *
+   * This can be an insert, update or delete op.
+   */
+  save(record: SaveOptions): Promise<FlushedResult | null>;
+
+  /**
+   * Replicate a truncate op - deletes all data in the specified tables.
+   */
+  truncate(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
+
+  /**
+   * Drop one or more tables.
+   *
+   * This is the same as truncate, but additionally removes the SourceTable record.
+   */
+  drop(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
+
+  /**
+   * Explicitly flush all pending changes in the batch.
+   *
+   * This does not create a new checkpoint until `commit()` is called. This means it's
+   * safe to flush multiple times in the middle of a large transaction.
+   *
+   * @returns null if there are no changes to flush.
+   */
+  flush(): Promise<FlushedResult | null>;
+
+  /**
+   * Flush and commit any saved ops. This creates a new checkpoint by default.
+   *
+   * Only call this after a transaction.
+   */
+  commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
+
+  /**
+   * Advance the checkpoint LSN position, without any associated op.
+   *
+   * This must only be called when not inside a transaction.
+   *
+   * @returns true if the checkpoint was advanced, false if this was a no-op
+   */
+  keepalive(lsn: string): Promise<boolean>;
+
+  /**
+   * Get the last checkpoint LSN, from either commit or keepalive.
+   */
+  lastCheckpointLsn: string | null;
+
+  markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
+
+  /**
+   * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
+   */
+  addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
+}
+
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = BSON.UUID | BSON.Document | any;
+
+export interface SaveParameterData {
+  sourceTable: SourceTable;
+  /** UUID */
+  sourceKey: string;
+  evaluated: EvaluatedParameters[];
+}
+
+export interface SaveBucketData {
+  sourceTable: SourceTable;
+  /** UUID */
+  sourceKey: string;
+
+  evaluated: EvaluatedRow[];
+}
+
+export type SaveOp = 'insert' | 'update' | 'delete';
+
+export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete;
+
+export enum SaveOperationTag {
+  INSERT = 'insert',
+  UPDATE = 'update',
+  DELETE = 'delete'
+}
+
+export interface SaveInsert {
+  tag: SaveOperationTag.INSERT;
+  sourceTable: SourceTable;
+  before?: undefined;
+  beforeReplicaId?: undefined;
+  after: SqliteRow;
+  afterReplicaId: ReplicaId;
+}
+
+export interface SaveUpdate {
+  tag: SaveOperationTag.UPDATE;
+  sourceTable: SourceTable;
+
+  /**
+   * This is only present when the id has changed, and will only contain replica identity columns.
+   */
+  before?: SqliteRow;
+  beforeReplicaId?: ReplicaId;
+
+  /**
+   * A null value means null column.
+   *
+   * An undefined value means it's a TOAST value - must be copied from another record.
+   */
+  after: ToastableSqliteRow;
+  afterReplicaId: ReplicaId;
+}
+
+export interface SaveDelete {
+  tag: SaveOperationTag.DELETE;
+  sourceTable: SourceTable;
+  before?: SqliteRow;
+  beforeReplicaId: ReplicaId;
+  after?: undefined;
+  afterReplicaId?: undefined;
+}
+
+export interface BucketBatchStorageListener {
+  replicationEvent: (payload: ReplicationEventPayload) => void;
+}
+
+export interface FlushedResult {
+  flushed_op: string;
+}
+
+export interface BucketBatchCommitOptions {
+  /**
+   * Creates a new checkpoint even if there were no persisted operations.
+   * Defaults to true.
+   */
+  createEmptyCheckpoints?: boolean;
+}
+
+export type ResolvedBucketBatchCommitOptions = Required<BucketBatchCommitOptions>;

package/src/storage/BucketStorageFactory.ts

@@ -0,0 +1,156 @@
+import { ObserverClient } from '@powersync/lib-services-framework';
+import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
+
+/**
+ * Represents a configured storage provider.
+ *
+ * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
+ * This is to handle replication of a new version of sync rules, while the old version is still active.
+ *
+ * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
+ */
+export interface BucketStorageFactory extends ObserverClient<BucketStorageFactoryListener>, AsyncDisposable {
+  /**
+   * Update sync rules from configuration, if changed.
+   */
+  configureSyncRules(
+    options: UpdateSyncRulesOptions
+  ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>;
+
+  /**
+   * Get a storage instance to query sync data for specific sync rules.
+   */
+  getInstance(options: PersistedSyncRulesContent): SyncRulesBucketStorage;
+
+  /**
+   * Deploy new sync rules.
+   */
+  updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
+
+  /**
+   * Indicate that a slot was removed, and we should re-sync by creating
+   * a new sync rules instance.
+   *
+   * This is roughly the same as deploying a new version of the current sync
+   * rules, but also accounts for cases where the current sync rules are not
+   * the latest ones.
+   *
+   * Replication should be restarted after this.
+   *
+   * @param slot_name The removed slot
+   */
+  slotRemoved(slot_name: string): Promise<void>;
+
+  /**
+   * Get the sync rules used for querying.
+   */
+  getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+
+  /**
+   * Get the sync rules used for querying.
+   */
+  getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+
+  /**
+   * Get the sync rules that will be active next once done with initial replicatino.
+   */
+  getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null>;
+
+  /**
+   * Get the sync rules that will be active next once done with initial replicatino.
+   */
+  getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
+
+  /**
+   * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
+   */
+  getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
+
+  /**
+   * Get all sync rules stopped but not terminated yet.
+   */
+  getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
+
+  /**
+   * Get the active storage instance.
+   */
+  getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
+
+  /**
+   * Get storage size of active sync rules.
+   */
+  getStorageMetrics(): Promise<StorageMetrics>;
+
+  /**
+   * Get the unique identifier for this instance of Powersync
+   */
+  getPowerSyncInstanceId(): Promise<string>;
+
+  /**
+   * Get a unique identifier for the system used for storage.
+   */
+  getSystemIdentifier(): Promise<BucketStorageSystemIdentifier>;
+}
+
+export interface BucketStorageFactoryListener {
+  syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
+  replicationEvent: (event: ReplicationEventPayload) => void;
+}
+
+export interface StorageMetrics {
+  /**
+   * Size of operations (bucket_data)
+   */
+  operations_size_bytes: number;
+
+  /**
+   * Size of parameter storage.
+   *
+   * Replication storage -> raw data as received from Postgres.
+   */
+  parameters_size_bytes: number;
+
+  /**
+   * Size of current_data.
+   */
+  replication_size_bytes: number;
+}
+
+export interface UpdateSyncRulesOptions {
+  content: string;
+  lock?: boolean;
+  validate?: boolean;
+}
+
+export interface BucketStorageSystemIdentifier {
+  /**
+   * A unique identifier for the system used for storage.
+   * For Postgres this can be the cluster `system_identifier` and database name.
+   * For MongoDB this can be the replica set name.
+   */
+  id: string;
+  /**
+   * A unique type for the storage implementation.
+   * e.g. `mongodb`, `postgresql`.
+   */
+  type: string;
+}
+
+/**
+ * Helper for tests.
+ * This is not in the `service-core-tests` package in order for storage modules
+ * to provide relevant factories without requiring `service-core-tests` as a direct dependency.
+ */
+export interface TestStorageOptions {
+  /**
+   * By default, collections are only cleared/
+   * Setting this to true will drop the collections completely.
+   */
+  dropAll?: boolean;
+
+  doNotClear?: boolean;
+}
+export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;

package/src/storage/ChecksumCache.ts

@@ -2,6 +2,7 @@ import { BucketChecksum, OpId } from '../util/protocol-types.js';
 import { ChecksumMap, addBucketChecksums } from '../util/utils.js';
 import { LRUCache } from 'lru-cache/min';
 import { OrderedSet } from '@js-sdsl/ordered-set';
+import { BucketPriority } from '@powersync/service-sync-rules';
 
 interface ChecksumFetchContext {
   fetch(bucket: string): Promise<BucketChecksum>;

package/src/storage/PersistedSyncRulesContent.ts

@@ -0,0 +1,26 @@
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { ReplicationLock } from './ReplicationLock.js';
+
+export interface ParseSyncRulesOptions {
+  defaultSchema: string;
+}
+
+export interface PersistedSyncRulesContent {
+  readonly id: number;
+  readonly sync_rules_content: string;
+  readonly slot_name: string;
+
+  readonly last_fatal_error?: string | null;
+  readonly last_keepalive_ts?: Date | null;
+  readonly last_checkpoint_ts?: Date | null;
+
+  parsed(options: ParseSyncRulesOptions): PersistedSyncRules;
+
+  lock(): Promise<ReplicationLock>;
+}
+
+export interface PersistedSyncRules {
+  readonly id: number;
+  readonly sync_rules: SqlSyncRules;
+  readonly slot_name: string;
+}

package/src/storage/ReplicationEventPayload.ts

@@ -1,6 +1,6 @@
 import * as sync_rules from '@powersync/service-sync-rules';
-import { BucketStorageBatch, SaveOp } from './BucketStorage.js';
 import { SourceTable } from './SourceTable.js';
+import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
 
 export type EventData = {
   op: SaveOp;

package/src/storage/ReplicationLock.ts

@@ -0,0 +1,5 @@
+export interface ReplicationLock {
+  sync_rules_id: number;
+
+  release(): Promise<void>;
+}

package/src/storage/StorageEngine.ts

@@ -1,17 +1,17 @@
-import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { BaseObserver, logger } from '@powersync/lib-services-framework';
 import { ResolvedPowerSyncConfig } from '../util/util-index.js';
-import { BucketStorageFactory } from './BucketStorage.js';
 import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
+import { BucketStorageFactory } from './BucketStorageFactory.js';
 
 export type StorageEngineOptions = {
   configuration: ResolvedPowerSyncConfig;
 };
 
-export interface StorageEngineListener extends DisposableListener {
+export interface StorageEngineListener {
   storageActivated: (storage: BucketStorageFactory) => void;
 }
 
-export class StorageEngine extends DisposableObserver<StorageEngineListener> {
+export class StorageEngine extends BaseObserver<StorageEngineListener> {
   // TODO: This will need to revisited when we actually support multiple storage providers.
   private storageProviders: Map<string, BucketStorageProvider> = new Map();
   private currentActiveStorage: ActiveStorage | null = null;

package/src/storage/StorageProvider.ts

@@ -1,5 +1,5 @@
 import * as util from '../util/util-index.js';
-import { BucketStorageFactory } from './BucketStorage.js';
+import { BucketStorageFactory } from './BucketStorageFactory.js';
 
 export interface ActiveStorage {
   storage: BucketStorageFactory;
@@ -16,6 +16,9 @@ export interface GetStorageOptions {
   resolvedConfig: util.ResolvedPowerSyncConfig;
 }
 
+/**
+ * Represents a provider that can create a storage instance for a specific storage type from configuration.
+ */
 export interface BucketStorageProvider {
   /**
    *  The storage type that this provider provides.

package/src/storage/SyncRulesBucketStorage.ts

@@ -0,0 +1,256 @@
+import { ObserverClient } from '@powersync/lib-services-framework';
+import { SqlSyncRules, SqliteJsonRow, SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as util from '../util/util-index.js';
+import { BucketStorageBatch, FlushedResult } from './BucketStorageBatch.js';
+import { BucketStorageFactory } from './BucketStorageFactory.js';
+import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
+import { SourceEntityDescriptor } from './SourceEntity.js';
+import { SourceTable } from './SourceTable.js';
+import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
+
+/**
+ * Storage for a specific copy of sync rules.
+ */
+export interface SyncRulesBucketStorage
+  extends ObserverClient<SyncRulesBucketStorageListener>,
+    SyncStorageWriteCheckpointAPI {
+  readonly group_id: number;
+  readonly slot_name: string;
+
+  readonly factory: BucketStorageFactory;
+
+  /**
+   * Resolve a table, keeping track of it internally.
+   */
+  resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
+
+  /**
+   * Use this to get access to update storage data.
+   */
+  startBatch(
+    options: StartBatchOptions,
+    callback: (batch: BucketStorageBatch) => Promise<void>
+  ): Promise<FlushedResult | null>;
+
+  getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
+
+  /**
+   * Terminate the sync rules.
+   *
+   * This clears the storage, and sets state to TERMINATED.
+   *
+   * Must only be called on stopped sync rules.
+   */
+  terminate(options?: TerminateOptions): Promise<void>;
+
+  getStatus(): Promise<SyncRuleStatus>;
+
+  /**
+   * Clear the storage, without changing state.
+   */
+  clear(): Promise<void>;
+
+  autoActivate(): Promise<void>;
+
+  /**
+   * Record a replication error.
+   *
+   * This could be a recoverable error (e.g. temporary network failure),
+   * or a permanent error (e.g. missing toast data).
+   *
+   * Errors are cleared on commit.
+   */
+  reportError(e: any): Promise<void>;
+
+  compact(options?: CompactOptions): Promise<void>;
+
+  // ## Read operations
+
+  getCheckpoint(): Promise<ReplicationCheckpoint>;
+
+  /**
+   * Used to resolve "dynamic" parameter queries.
+   */
+  getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
+
+  getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<CheckpointChanges>;
+
+  /**
+   * Yields the latest user write checkpoint whenever the sync checkpoint updates.
+   *
+   * The stream stops or errors if this is not the active sync rules (anymore).
+   */
+  watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<StorageCheckpointUpdate>;
+
+  /**
+   * Get a "batch" of data for a checkpoint.
+   *
+   * The results will be split into separate SyncBucketData chunks to:
+   * 1. Separate buckets.
+   * 2. Limit the size of each individual chunk according to options.batchSizeLimitBytes.
+   *
+   * @param checkpoint the checkpoint
+   * @param dataBuckets current bucket states
+   * @param options batch size options
+   */
+  getBucketDataBatch(
+    checkpoint: util.OpId,
+    dataBuckets: Map<string, string>,
+    options?: BucketDataBatchOptions
+  ): AsyncIterable<SyncBucketDataBatch>;
+
+  /**
+   * Compute checksums for a given list of buckets.
+   *
+   * Returns zero checksums for any buckets not found.
+   */
+  getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
+}
+
+export interface SyncRulesBucketStorageListener {
+  batchStarted: (batch: BucketStorageBatch) => void;
+}
+
+export interface SyncRuleStatus {
+  checkpoint_lsn: string | null;
+  active: boolean;
+  snapshot_done: boolean;
+}
+export interface ResolveTableOptions {
+  group_id: number;
+  connection_id: number;
+  connection_tag: string;
+  entity_descriptor: SourceEntityDescriptor;
+
+  sync_rules: SqlSyncRules;
+}
+
+export interface ResolveTableResult {
+  table: SourceTable;
+  dropTables: SourceTable[];
+}
+
+export interface StartBatchOptions extends ParseSyncRulesOptions {
+  zeroLSN: string;
+  /**
+   * Whether or not to store a copy of the current data.
+   *
+   * This is needed if we need to apply partial updates, for example
+   * when we get TOAST values from Postgres.
+   *
+   * This is not needed when we get the full document from the source
+   * database, for example from MongoDB.
+   */
+  storeCurrentData: boolean;
+
+  /**
+   * Set to true for initial replication.
+   *
+   * This will avoid creating new operations for rows previously replicated.
+   */
+  skipExistingRows?: boolean;
+}
+
+export interface CompactOptions {
+  /**
+   * Heap memory limit for the compact process.
+   *
+   * Add around 64MB to this to determine the "--max-old-space-size" argument.
+   * Add another 80MB to get RSS usage / memory limits.
+   */
+  memoryLimitMB?: number;
+
+  /**
+   * If specified, ignore any operations newer than this when compacting.
+   *
+   * This is primarily for tests, where we want to test compacting at a specific
+   * point.
+   *
+   * This can also be used to create a "safe buffer" of recent operations that should
+   * not be compacted, to avoid invalidating checkpoints in use.
+   */
+  maxOpId?: bigint;
+
+  /**
+   * If specified, compact only the specific buckets.
+   *
+   * If not specified, compacts all buckets.
+   *
+   * These can be individual bucket names, or bucket definition names.
+   */
+  compactBuckets?: string[];
+}
+
+export interface TerminateOptions {
+  /**
+   * If true, also clear the storage before terminating.
+   */
+  clearStorage: boolean;
+}
+
+export interface BucketDataBatchOptions {
+  /** Limit number of documents returned. Defaults to 1000. */
+  limit?: number;
+
+  /**
+   * Limit size of chunks returned. Defaults to 1MB.
+   *
+   * This is a lower bound, not an upper bound. As soon as the chunk size goes over this limit,
+   * it is returned.
+   *
+   * Note that an individual data row can be close to 16MB in size, so this does not help in
+   * extreme cases.
+   */
+  chunkLimitBytes?: number;
+}
+
+export interface SyncBucketDataBatch {
+  batch: util.SyncBucketData;
+  targetOp: bigint | null;
+}
+
+export interface ReplicationCheckpoint {
+  readonly checkpoint: util.OpId;
+  readonly lsn: string | null;
+}
+
+export interface WatchWriteCheckpointOptions {
+  /** user_id and client_id combined. */
+  user_id: string;
+
+  signal: AbortSignal;
+}
+
+export interface WatchFilterEvent {
+  changedDataBucket?: string;
+  changedParameterBucketDefinition?: string;
+  invalidate?: boolean;
+}
+
+export interface WriteCheckpoint {
+  base: ReplicationCheckpoint;
+  writeCheckpoint: bigint | null;
+}
+
+export interface StorageCheckpointUpdate extends WriteCheckpoint {
+  update: CheckpointChanges;
+}
+
+export interface GetCheckpointChangesOptions {
+  lastCheckpoint: util.OpId;
+  nextCheckpoint: util.OpId;
+}
+
+export interface CheckpointChanges {
+  updatedDataBuckets: string[];
+  invalidateDataBuckets: boolean;
+  updatedParameterBucketDefinitions: string[];
+  invalidateParameterBuckets: boolean;
+}
+
+export const CHECKPOINT_INVALIDATE_ALL: CheckpointChanges = {
+  updatedDataBuckets: [],
+  invalidateDataBuckets: true,
+  updatedParameterBucketDefinitions: [],
+  invalidateParameterBuckets: true
+};