@powersync/service-core 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,5 +1,11 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
-import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
+import {
+  BucketDataSource,
+  HydratedSyncRules,
+  ScopedParameterLookup,
+  SqliteJsonRow
+} from '@powersync/service-sync-rules';
+import { PerformanceTracer } from '../tracing/PerformanceTracer.js';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
@@ -9,7 +15,7 @@ import { SourceTable } from './SourceTable.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
 /**
- * Storage for a specific copy of sync rules.
+ * Storage for a specific replication stream.
  */
 export interface SyncRulesBucketStorage
   extends ObserverClient<SyncRulesBucketStorageListener>,
@@ -18,6 +24,7 @@ export interface SyncRulesBucketStorage
   readonly slot_name: string;
 
   readonly factory: BucketStorageFactory;
+  readonly logger: Logger;
 
   /**
    * Resolve a table, keeping track of it internally.
@@ -25,21 +32,28 @@ export interface SyncRulesBucketStorage
   resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
 
   /**
-   * Use this to get access to update storage data.
+   * Create a new writer.
+   *
+   * The writer must be flushed and disposed when done.
+   */
+  createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
+
+  /**
+   * @deprecated Use `createWriter()` with `await using` instead.
    */
   startBatch(
-    options: StartBatchOptions,
+    options: CreateWriterOptions,
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
   getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
 
   /**
-   * Terminate the sync rules.
+   * Terminate the replication stream.
    *
    * This clears the storage, and sets state to TERMINATED.
    *
-   * Must only be called on stopped sync rules.
+   * Must only be called on stopped replication streams.
    */
   terminate(options?: TerminateOptions): Promise<void>;
 
@@ -86,7 +100,7 @@ export interface SyncRulesBucketStorage
   /**
    * Yields the latest user write checkpoint whenever the sync checkpoint updates.
    *
-   * The stream stops or errors if this is not the active sync rules (anymore).
+   * The stream stops or errors if this is not the active sync config (anymore).
    */
   watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable<StorageCheckpointUpdate>;
 
@@ -103,7 +117,7 @@ export interface SyncRulesBucketStorage
    */
   getBucketDataBatch(
     checkpoint: util.InternalOpId,
-    dataBuckets: Map<string, util.InternalOpId>,
+    dataBuckets: BucketDataRequest[],
     options?: BucketDataBatchOptions
   ): AsyncIterable<SyncBucketDataChunk>;
 
@@ -115,7 +129,7 @@ export interface SyncRulesBucketStorage
    * This may be slow, depending on the size of the buckets.
    * The checksums are cached internally to compensate for this, but does not cover all cases.
    */
-  getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+  getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
 
   /**
    * Clear checksum cache. Primarily intended for tests.
@@ -127,6 +141,16 @@ export interface SyncRulesBucketStorageListener {
   batchStarted: (batch: BucketStorageBatch) => void;
 }
 
+export interface BucketDataRequest {
+  bucket: string;
+  start: util.InternalOpId;
+  source: BucketDataSource;
+}
+export interface BucketChecksumRequest {
+  bucket: string;
+  source: BucketDataSource;
+}
+
 export interface SyncRuleStatus {
   checkpoint_lsn: string | null;
   active: boolean;
@@ -147,7 +171,7 @@ export interface ResolveTableResult {
   dropTables: SourceTable[];
 }
 
-export interface StartBatchOptions extends ParseSyncRulesOptions {
+export interface CreateWriterOptions extends ParseSyncRulesOptions {
   zeroLSN: string;
   /**
    * Whether or not to store a copy of the current data.
@@ -174,9 +198,16 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    */
   markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
 
+  tracer?: PerformanceTracer<'storage' | 'evaluate'>;
+
   logger?: Logger;
 }
 
+/**
+ * @deprecated Use `CreateWriterOptions`.
+ */
+export interface StartBatchOptions extends CreateWriterOptions {}
+
 export interface CompactOptions {
   /**
    * Heap memory limit for the compact process.
@@ -238,6 +269,8 @@ export interface CompactOptions {
   compactParameterCacheLimit?: number;
 
   signal?: AbortSignal;
+
+  logger?: Logger;
 }
 
 export interface PopulateChecksumCacheOptions {
@@ -293,8 +326,41 @@ export interface ReplicationCheckpoint {
    * Used to resolve "dynamic" parameter queries.
    *
    * This gets parameter sets specific to this checkpoint.
+   *
+   * @throws {@link ParameterSetLimitExceededError}
+   * Thrown if resolved lookups in bucket storage exceed the `limit` parameter.
+   */
+  getParameterSets(lookups: ScopedParameterLookup[], limit: number): Promise<SqliteJsonRow[]>;
+}
+
+/**
+ * An exception thrown by {@link ReplicationCheckpoint} implementations if there are too many parameter results.
+ *
+ * This is not a suitable exception to show to users, `BucketParameterState` adds additional context.
+ */
+export class ParameterSetLimitExceededError extends Error {
+  constructor(
+    readonly limit: number,
+    readonly breakdown?: ParameterQueryInvocationLog[]
+  ) {
+    super(`Too many parameter results (limit was ${limit})`);
+  }
+}
+
+export interface ParameterQueryInvocationLog {
+  /**
+   * The definition for which a parameter query was invoked.
+   *
+   * The exact format of definition is unspecified, it's shown to users to help them debug this failure.
+   */
+  definition: string;
+  /**
+   * If {@link didExceedLimit} is false, the amount of rows returned by the invocation.
+   *
+   * Otherwise, the maximum amount of rows this invocation was allowed to return.
    */
-  getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]>;
+  resultsOrLimit: number;
+  didExceedLimit: boolean;
 }
 
 export interface WatchWriteCheckpointOptions {

package/src/storage/WriteCheckpointAPI.ts

@@ -1,7 +1,7 @@
 export enum WriteCheckpointMode {
   /**
    * Raw mappings of `user_id` to `write_checkpoint`s should
-   * be supplied for each set of sync rules.
+   * be supplied for each replication stream.
    */
   CUSTOM = 'custom',
   /**
@@ -21,7 +21,7 @@ export interface BaseWriteCheckpointIdentifier {
 
 export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
   /**
-   * Sync rules which were active when this checkpoint was created.
+   * Replication stream which was active when this checkpoint was created.
    */
   sync_rules_id: number;
 }
@@ -35,7 +35,7 @@ export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpoint
 
 export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
   /**
-   * Sync rules which were active when this checkpoint was created.
+   * Replication stream which was active when this checkpoint was created.
    */
   sync_rules_id: number;
 }
@@ -63,8 +63,8 @@ export interface BaseWriteCheckpointAPI {
 
 /**
  * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
- * This storage corresponds with a set of sync rules. These APIs don't require specifying a
- * sync rules id.
+ * This storage corresponds with a replication stream. These APIs don't require specifying a
+ * replication stream id.
  */
 export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
   lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
@@ -72,7 +72,7 @@ export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
 
 /**
  * Write Checkpoint API which is interfaced directly with the storage layer. This requires
- * sync rules identifiers for custom write checkpoints.
+ * replication stream identifiers for custom write checkpoints.
  */
 export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
   lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;

package/src/storage/bson.ts

@@ -40,11 +40,6 @@ export const deserializeParameterLookup = (lookup: bson.Binary) => {
   return parsed;
 };
 
-export const getLookupBucketDefinitionName = (lookup: bson.Binary) => {
-  const parsed = deserializeParameterLookup(lookup);
-  return parsed[0] as string;
-};
-
 /**
  * True if this is a bson.UUID.
  *

package/src/storage/storage-index.ts

@@ -1,17 +1,17 @@
 export * from './bson.js';
 export * from './BucketStorage.js';
+export * from './BucketStorageBatch.js';
+export * from './BucketStorageFactory.js';
 export * from './ChecksumCache.js';
+export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationEventPayload.js';
+export * from './ReplicationLock.js';
+export * from './ReportStorage.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
+export * from './storage-metrics.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
-export * from './storage-metrics.js';
-export * from './WriteCheckpointAPI.js';
-export * from './BucketStorageFactory.js';
-export * from './BucketStorageBatch.js';
-export * from './SyncRulesBucketStorage.js';
-export * from './PersistedSyncRulesContent.js';
-export * from './ReplicationLock.js';
-export * from './ReportStorage.js';
 export * from './StorageVersionConfig.js';
+export * from './SyncRulesBucketStorage.js';
+export * from './WriteCheckpointAPI.js';

package/src/storage/storage-metrics.ts

@@ -1,7 +1,7 @@
-import { MetricsEngine } from '../metrics/MetricsEngine.js';
 import { logger } from '@powersync/lib-services-framework';
-import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 import { StorageMetric } from '@powersync/service-types';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 
 export function createCoreStorageMetrics(engine: MetricsEngine): void {
   engine.createObservableGauge({

package/src/streams/streams-index.ts

@@ -1,3 +1,3 @@
-export * from './merge.js';
-export * from './LastValueSink.js';
 export * from './BroadcastIterable.js';
+export * from './LastValueSink.js';
+export * from './merge.js';