@powersync/service-core 1.20.5 → 1.22.0

package/src/storage/SourceTable.ts

@@ -1,7 +1,12 @@
-import { DEFAULT_TAG } from '@powersync/service-sync-rules';
+import {
+  BucketDataSource,
+  DEFAULT_TAG,
+  ParameterIndexLookupCreator,
+  SourceTableRef
+} from '@powersync/service-sync-rules';
 import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
-import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
+import { ColumnDescriptor } from './SourceEntity.js';
 
 /**
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
@@ -10,12 +15,12 @@ export type SourceTableId = string | bson.ObjectId;
 
 export interface SourceTableOptions {
   id: SourceTableId;
-  connectionTag: string;
+  ref: SourceTableRef;
   objectId: number | string | undefined;
-  schema: string;
-  name: string;
   replicaIdColumns: ColumnDescriptor[];
   snapshotComplete: boolean;
+  bucketDataSources: BucketDataSource[];
+  parameterLookupSources: ParameterIndexLookupCreator[];
 }
 
 export interface TableSnapshotStatus {
@@ -24,11 +29,17 @@ export interface TableSnapshotStatus {
   lastKey: Uint8Array | null;
 }
 
-export class SourceTable implements SourceEntityDescriptor {
+/**
+ * Represents a resolved source table.
+ *
+ * There could be multiple of these for the same SourceTableRef.
+ * For that reason, we do not implement the SourceTableRef interface, to ensure that the two are not used interchangably.
+ */
+export class SourceTable {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -37,7 +48,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncData = true;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -46,7 +57,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncParameters = true;
 
   /**
-   * True if the table is used in sync rules for events.
+   * True if the table is used in sync config for events.
    *
    * This value is resolved externally, and cached here.
    *
@@ -54,6 +65,19 @@ export class SourceTable implements SourceEntityDescriptor {
    */
   public syncEvent = true;
 
+  /**
+   * True if raw data should be stored in current_data collection.
+   *
+   * This is needed when the source sends partial row data (e.g. TOAST values).
+   * When REPLICA IDENTITY FULL is configured, complete rows are always sent,
+   * so we don't need to store raw data.
+   *
+   * This value is resolved externally based on table configuration.
+   *
+   * Defaults to true for tests (conservative approach).
+   */
+  public storeCurrentData = true;
+
   /**
    * Always undefined if snapshotComplete = true.
    *
@@ -71,31 +95,39 @@ export class SourceTable implements SourceEntityDescriptor {
     return this.options.id;
   }
 
-  get connectionTag() {
-    return this.options.connectionTag;
-  }
-
   get objectId() {
     return this.options.objectId;
   }
 
   get schema() {
-    return this.options.schema;
+    return this.options.ref.schema;
   }
   get name() {
-    return this.options.name;
+    return this.options.ref.name;
+  }
+
+  get ref() {
+    return this.options.ref;
   }
 
   get replicaIdColumns() {
     return this.options.replicaIdColumns;
   }
 
+  get bucketDataSources() {
+    return this.options.bucketDataSources;
+  }
+
+  get parameterLookupSources() {
+    return this.options.parameterLookupSources;
+  }
+
   /**
-   *  Sanitized name of the entity in the format of "{schema}.{entity name}"
-   *  Suitable for safe use in Postgres queries.
+   * Sanitized name of the entity in the format of "{schema}.{entity name}".
+   * Suitable for safe use in Postgres queries.
    */
   get qualifiedName() {
-    return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.name)}`;
+    return util.qualifiedName(this.ref);
   }
 
   get syncAny() {
@@ -108,15 +140,17 @@ export class SourceTable implements SourceEntityDescriptor {
   clone() {
     const copy = new SourceTable({
       id: this.id,
-      connectionTag: this.connectionTag,
+      ref: this.options.ref,
       objectId: this.objectId,
-      schema: this.schema,
-      name: this.name,
       replicaIdColumns: this.replicaIdColumns,
-      snapshotComplete: this.snapshotComplete
+      snapshotComplete: this.snapshotComplete,
+      bucketDataSources: this.bucketDataSources,
+      parameterLookupSources: this.parameterLookupSources
     });
     copy.syncData = this.syncData;
     copy.syncParameters = this.syncParameters;
+    copy.syncEvent = this.syncEvent;
+    copy.storeCurrentData = this.storeCurrentData;
     copy.snapshotStatus = this.snapshotStatus;
     return copy;
   }

package/src/storage/StorageVersionConfig.ts

@@ -45,7 +45,7 @@ export const STORAGE_VERSION_3 = 3;
 export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
 
 /**
- * Default storage version for newly persisted sync rules.
+ * Default storage version for newly persisted replication streams.
  */
 export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
 

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,33 +1,33 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
 import {
   BucketDataSource,
-  HydratedSyncRules,
-  ScopedParameterLookup,
-  SqliteJsonRow
+  HydratedSyncConfig,
+  ParameterLookupRows,
+  ScopedParameterLookup
 } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
 import { ParseSyncRulesOptions } from './PersistedSyncRulesContent.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
+import { StorageVersionConfig } from './StorageVersionConfig.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
 /**
- * Storage for a specific copy of sync rules.
+ * Storage for a specific replication stream.
  */
 export interface SyncRulesBucketStorage
   extends ObserverClient<SyncRulesBucketStorageListener>,
     SyncStorageWriteCheckpointAPI {
   readonly group_id: number;
   readonly slot_name: string;
+  readonly storageConfig: StorageVersionConfig;
 
   readonly factory: BucketStorageFactory;
-
-  /**
-   * Resolve a table, keeping track of it internally.
-   */
-  resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
+  readonly logger: Logger;
 
   /**
    * Create a new writer.
@@ -44,14 +44,14 @@ export interface SyncRulesBucketStorage
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
+  getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncConfig;
 
   /**
-   * Terminate the sync rules.
+   * Terminate the replication stream.
    *
    * This clears the storage, and sets state to TERMINATED.
    *
-   * Must only be called on stopped sync rules.
+   * Must only be called on stopped replication streams.
    */
   terminate(options?: TerminateOptions): Promise<void>;
 
@@ -98,7 +98,7 @@ export interface SyncRulesBucketStorage
   /**
    * Yields the latest user write checkpoint whenever the sync checkpoint updates.
    *
-   * The stream stops or errors if this is not the active sync rules (anymore).
+   * The stream stops or errors if this is not the active sync config (anymore).
    */
   watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable<StorageCheckpointUpdate>;
 
@@ -154,18 +154,26 @@ export interface SyncRuleStatus {
   active: boolean;
   snapshot_done: boolean;
   snapshot_lsn: string | null;
+  /**
+   * Last persisted operation that must be included in the next checkpoint once checkpointing is unblocked.
+   */
+  keepalive_op: util.InternalOpId | null;
 }
-export interface ResolveTableOptions {
-  group_id: number;
+export interface ResolveTablesOptions {
   connection_id: number;
-  connection_tag: string;
-  entity_descriptor: SourceEntityDescriptor;
-
-  sync_rules: HydratedSyncRules;
+  source: SourceEntityDescriptor;
+  /**
+   * For tests only - custom id generator for stable ids.
+   */
+  idGenerator?: () => string | bson.ObjectId;
+  /**
+   * For tests only - override the sync rules used.
+   */
+  syncRules?: HydratedSyncConfig;
 }
 
-export interface ResolveTableResult {
-  table: SourceTable;
+export interface ResolveTablesResult {
+  tables: SourceTable[];
   dropTables: SourceTable[];
 }
 
@@ -196,9 +204,18 @@ export interface CreateWriterOptions extends ParseSyncRulesOptions {
    */
   markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
 
+  hooks?: StorageHooks;
+
+  tracer?: PerformanceTracer<'storage' | 'evaluate'>;
+
   logger?: Logger;
 }
 
+export interface StorageHooks {
+  beforeBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+  afterBatchFlush?: (batch: BucketStorageBatch) => Promise<void>;
+}
+
 /**
  * @deprecated Use `CreateWriterOptions`.
  */
@@ -265,6 +282,8 @@ export interface CompactOptions {
   compactParameterCacheLimit?: number;
 
   signal?: AbortSignal;
+
+  logger?: Logger;
 }
 
 export interface PopulateChecksumCacheOptions {
@@ -320,8 +339,41 @@ export interface ReplicationCheckpoint {
    * Used to resolve "dynamic" parameter queries.
    *
    * This gets parameter sets specific to this checkpoint.
+   *
+   * @throws {@link ParameterSetLimitExceededError}
+   * Thrown if resolved lookups in bucket storage exceed the `limit` parameter.
+   */
+  getParameterSets(lookups: ScopedParameterLookup[], limit: number): Promise<ParameterLookupRows[]>;
+}
+
+/**
+ * An exception thrown by {@link ReplicationCheckpoint} implementations if there are too many parameter results.
+ *
+ * This is not a suitable exception to show to users, `BucketParameterState` adds additional context.
+ */
+export class ParameterSetLimitExceededError extends Error {
+  constructor(
+    readonly limit: number,
+    readonly breakdown?: ParameterQueryInvocationLog[]
+  ) {
+    super(`Too many parameter results (limit was ${limit})`);
+  }
+}
+
+export interface ParameterQueryInvocationLog {
+  /**
+   * The definition for which a parameter query was invoked.
+   *
+   * The exact format of definition is unspecified, it's shown to users to help them debug this failure.
+   */
+  definition: string;
+  /**
+   * If {@link didExceedLimit} is false, the amount of rows returned by the invocation.
+   *
+   * Otherwise, the maximum amount of rows this invocation was allowed to return.
    */
-  getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]>;
+  resultsOrLimit: number;
+  didExceedLimit: boolean;
 }
 
 export interface WatchWriteCheckpointOptions {

package/src/storage/WriteCheckpointAPI.ts

@@ -1,7 +1,7 @@
 export enum WriteCheckpointMode {
   /**
    * Raw mappings of `user_id` to `write_checkpoint`s should
-   * be supplied for each set of sync rules.
+   * be supplied for each replication stream.
    */
   CUSTOM = 'custom',
   /**
@@ -21,7 +21,7 @@ export interface BaseWriteCheckpointIdentifier {
 
 export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
   /**
-   * Sync rules which were active when this checkpoint was created.
+   * Replication stream which was active when this checkpoint was created.
    */
   sync_rules_id: number;
 }
@@ -35,7 +35,7 @@ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpoint
 
 export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
   /**
-   * Sync rules which were active when this checkpoint was created.
+   * Replication stream which was active when this checkpoint was created.
    */
   sync_rules_id: number;
 }
@@ -63,8 +63,8 @@ export interface BaseWriteCheckpointAPI {
 
 /**
  * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
- * This storage corresponds with a set of sync rules. These APIs don't require specifying a
- * sync rules id.
+ * This storage corresponds with a replication stream. These APIs don't require specifying a
+ * replication stream id.
  */
 export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
   lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
@@ -72,7 +72,7 @@ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
 
 /**
  * Write Checkpoint API which is interfaced directly with the storage layer. This requires
- * sync rules identifiers for custom write checkpoints.
+ * replication stream identifiers for custom write checkpoints.
  */
 export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
   lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;

package/src/storage/bson.ts

@@ -40,11 +40,6 @@ export const deserializeParameterLookup = (lookup: bson.Binary) => {
   return parsed;
 };
 
-export const getLookupBucketDefinitionName = (lookup: bson.Binary) => {
-  const parsed = deserializeParameterLookup(lookup);
-  return parsed[0] as string;
-};
-
 /**
  * True if this is a bson.UUID.
  *