@powersync/service-core 1.7.1 → 1.8.0

package/src/storage/ChecksumCache.ts

@@ -1,12 +1,11 @@
-import { BucketChecksum, OpId } from '../util/protocol-types.js';
-import { ChecksumMap, addBucketChecksums } from '../util/utils.js';
-import { LRUCache } from 'lru-cache/min';
 import { OrderedSet } from '@js-sdsl/ordered-set';
-import { BucketPriority } from '@powersync/service-sync-rules';
+import { LRUCache } from 'lru-cache/min';
+import { BucketChecksum } from '../util/protocol-types.js';
+import { addBucketChecksums, ChecksumMap, InternalOpId } from '../util/utils.js';
 
 interface ChecksumFetchContext {
   fetch(bucket: string): Promise<BucketChecksum>;
-  checkpoint: bigint;
+  checkpoint: InternalOpId;
 }
 
 export interface PartialChecksum {
@@ -28,10 +27,11 @@ export interface PartialChecksum {
    */
   isFullChecksum: boolean;
 }
+
 export interface FetchPartialBucketChecksum {
   bucket: string;
-  start?: OpId;
-  end: OpId;
+  start?: InternalOpId;
+  end: InternalOpId;
 }
 
 export type PartialChecksumMap = Map<string, PartialChecksum>;
@@ -101,8 +101,7 @@ export class ChecksumCache {
 
       dispose: (value, key) => {
         // Remove from the set of cached checkpoints for the bucket
-        const { checkpointString } = parseCacheKey(key);
-        const checkpoint = BigInt(checkpointString);
+        const { checkpoint } = parseCacheKey(key);
         const checkpointSet = this.bucketCheckpoints.get(value.bucket);
         if (checkpointSet == null) {
           return;
@@ -128,7 +127,7 @@ export class ChecksumCache {
     });
   }
 
-  async getChecksums(checkpoint: OpId, buckets: string[]): Promise<BucketChecksum[]> {
+  async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
     const checksums = await this.getChecksumMap(checkpoint, buckets);
     // Return results in the same order as the request
     return buckets.map((bucket) => checksums.get(bucket)!);
@@ -141,7 +140,7 @@ export class ChecksumCache {
    *
    * @returns a Map with exactly one entry for each bucket requested
    */
-  async getChecksumMap(checkpoint: OpId, buckets: string[]): Promise<ChecksumMap> {
+  async getChecksumMap(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
     // Buckets that don't have a cached checksum for this checkpoint yet
     let toFetch = new Set<string>();
 
@@ -235,7 +234,7 @@ export class ChecksumCache {
                 // Partial checksum found - make a partial checksum request
                 bucketRequest = {
                   bucket,
-                  start: cp.toString(),
+                  start: cp,
                   end: checkpoint
                 };
                 add.set(bucket, cached);
@@ -315,11 +314,11 @@ export class ChecksumCache {
   }
 }
 
-function makeCacheKey(checkpoint: bigint | string, bucket: string) {
+function makeCacheKey(checkpoint: InternalOpId | string, bucket: string) {
   return `${checkpoint}/${bucket}`;
 }
 
 function parseCacheKey(key: string) {
   const index = key.indexOf('/');
-  return { checkpointString: key.substring(0, index), bucket: key.substring(index + 1) };
+  return { checkpoint: BigInt(key.substring(0, index)), bucket: key.substring(index + 1) };
 }

package/src/storage/SyncRulesBucketStorage.ts

@@ -71,7 +71,7 @@ export interface SyncRulesBucketStorage
   /**
    * Used to resolve "dynamic" parameter queries.
    */
-  getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
+  getParameterSets(checkpoint: util.InternalOpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]>;
 
   getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<CheckpointChanges>;
 
@@ -94,8 +94,8 @@ export interface SyncRulesBucketStorage
    * @param options batch size options
    */
   getBucketDataBatch(
-    checkpoint: util.OpId,
-    dataBuckets: Map<string, string>,
+    checkpoint: util.InternalOpId,
+    dataBuckets: Map<string, util.InternalOpId>,
     options?: BucketDataBatchOptions
   ): AsyncIterable<SyncBucketDataBatch>;
 
@@ -104,7 +104,7 @@ export interface SyncRulesBucketStorage
    *
    * Returns zero checksums for any buckets not found.
    */
-  getChecksums(checkpoint: util.OpId, buckets: string[]): Promise<util.ChecksumMap>;
+  getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -169,7 +169,7 @@ export interface CompactOptions {
    * This can also be used to create a "safe buffer" of recent operations that should
    * not be compacted, to avoid invalidating checkpoints in use.
    */
-  maxOpId?: bigint;
+  maxOpId?: util.InternalOpId;
 
   /**
    * If specified, compact only the specific buckets.
@@ -215,11 +215,11 @@ export interface BucketDataBatchOptions {
 
 export interface SyncBucketDataBatch {
   batch: util.SyncBucketData;
-  targetOp: bigint | null;
+  targetOp: util.InternalOpId | null;
 }
 
 export interface ReplicationCheckpoint {
-  readonly checkpoint: util.OpId;
+  readonly checkpoint: util.InternalOpId;
   readonly lsn: string | null;
 }
 
@@ -238,7 +238,7 @@ export interface WatchFilterEvent {
 
 export interface WriteCheckpoint {
   base: ReplicationCheckpoint;
-  writeCheckpoint: bigint | null;
+  writeCheckpoint: util.InternalOpId | null;
 }
 
 export interface StorageCheckpointUpdate extends WriteCheckpoint {
@@ -246,8 +246,8 @@ export interface StorageCheckpointUpdate extends WriteCheckpoint {
 }
 
 export interface GetCheckpointChangesOptions {
-  lastCheckpoint: util.OpId;
-  nextCheckpoint: util.OpId;
+  lastCheckpoint: util.InternalOpId;
+  nextCheckpoint: util.InternalOpId;
 }
 
 export interface CheckpointChanges {

package/src/sync/BucketChecksumState.ts

@@ -6,12 +6,14 @@ import * as util from '../util/util-index.js';
 import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
 import { BucketSyncState } from './sync.js';
+import { SyncContext } from './SyncContext.js';
 
 export interface BucketChecksumStateOptions {
+  syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
   syncRules: SqlSyncRules;
   syncParams: RequestParameters;
-  initialBucketPositions?: { name: string; after: string }[];
+  initialBucketPositions?: { name: string; after: util.InternalOpId }[];
 }
 
 /**
@@ -20,6 +22,7 @@ export interface BucketChecksumStateOptions {
  * Handles incrementally re-computing checkpoints.
  */
 export class BucketChecksumState {
+  private readonly context: SyncContext;
   private readonly bucketStorage: BucketChecksumStateStorage;
 
   /**
@@ -43,8 +46,14 @@ export class BucketChecksumState {
   private pendingBucketDownloads = new Set<string>();
 
   constructor(options: BucketChecksumStateOptions) {
+    this.context = options.syncContext;
     this.bucketStorage = options.bucketStorage;
-    this.parameterState = new BucketParameterState(options.bucketStorage, options.syncRules, options.syncParams);
+    this.parameterState = new BucketParameterState(
+      options.syncContext,
+      options.bucketStorage,
+      options.syncRules,
+      options.syncParams
+    );
     this.bucketDataPositions = new Map();
 
     for (let { name, after: start } of options.initialBucketPositions ?? []) {
@@ -69,10 +78,16 @@ export class BucketChecksumState {
     for (let bucket of allBuckets) {
       dataBucketsNew.set(bucket.bucket, {
         description: bucket,
-        start_op_id: this.bucketDataPositions.get(bucket.bucket)?.start_op_id ?? '0'
+        start_op_id: this.bucketDataPositions.get(bucket.bucket)?.start_op_id ?? 0n
       });
     }
     this.bucketDataPositions = dataBucketsNew;
+    if (dataBucketsNew.size > this.context.maxBuckets) {
+      throw new ServiceError(
+        ErrorCode.PSYNC_S2305,
+        `Too many buckets: ${dataBucketsNew.size} (limit of ${this.context.maxBuckets})`
+      );
+    }
 
     let checksumMap: util.ChecksumMap;
     if (updatedBuckets != null) {
@@ -165,7 +180,7 @@ export class BucketChecksumState {
 
       checkpointLine = {
         checkpoint_diff: {
-          last_op_id: base.checkpoint,
+          last_op_id: util.internalToExternalOpId(base.checkpoint),
           write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
           removed_buckets: diff.removedBuckets,
           updated_buckets: updatedBucketDescriptions
@@ -178,7 +193,7 @@ export class BucketChecksumState {
       bucketsToFetch = allBuckets;
       checkpointLine = {
         checkpoint: {
-          last_op_id: base.checkpoint,
+          last_op_id: util.internalToExternalOpId(base.checkpoint),
           write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined,
           buckets: [...checksumMap.values()].map((e) => ({
             ...e,
@@ -204,8 +219,8 @@ export class BucketChecksumState {
    * @param bucketsToFetch List of buckets to fetch, typically from buildNextCheckpointLine, or a subset of that
    * @returns
    */
-  getFilteredBucketPositions(bucketsToFetch: BucketDescription[]): Map<string, string> {
-    const filtered = new Map<string, string>();
+  getFilteredBucketPositions(bucketsToFetch: BucketDescription[]): Map<string, util.InternalOpId> {
+    const filtered = new Map<string, util.InternalOpId>();
     for (let bucket of bucketsToFetch) {
       const state = this.bucketDataPositions.get(bucket.bucket);
       if (state) {
@@ -221,7 +236,7 @@ export class BucketChecksumState {
    * @param bucket the bucket name
    * @param nextAfter sync operations >= this value in the next batch
    */
-  updateBucketPosition(options: { bucket: string; nextAfter: string; hasMore: boolean }) {
+  updateBucketPosition(options: { bucket: string; nextAfter: util.InternalOpId; hasMore: boolean }) {
     const state = this.bucketDataPositions.get(options.bucket);
     if (state) {
       state.start_op_id = options.nextAfter;
@@ -247,13 +262,20 @@ export interface CheckpointUpdate {
 }
 
 export class BucketParameterState {
+  private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
   public readonly syncRules: SqlSyncRules;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   private readonly staticBuckets: Map<string, BucketDescription>;
 
-  constructor(bucketStorage: BucketChecksumStateStorage, syncRules: SqlSyncRules, syncParams: RequestParameters) {
+  constructor(
+    context: SyncContext,
+    bucketStorage: BucketChecksumStateStorage,
+    syncRules: SqlSyncRules,
+    syncParams: RequestParameters
+  ) {
+    this.context = context;
     this.bucketStorage = bucketStorage;
     this.syncRules = syncRules;
     this.syncParams = syncParams;
@@ -275,9 +297,13 @@ export class BucketParameterState {
       return null;
     }
 
-    if (update.buckets.length > 1000) {
-      // TODO: Limit number of buckets even before we get to this point
-      const error = new ServiceError(ErrorCode.PSYNC_S2305, `Too many buckets: ${update.buckets.length}`);
+    if (update.buckets.length > this.context.maxParameterQueryResults) {
+      // TODO: Limit number of results even before we get to this point
+      // This limit applies _before_ we get the unique set
+      const error = new ServiceError(
+        ErrorCode.PSYNC_S2305,
+        `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
+      );
       logger.error(error.message, {
         checkpoint: checkpoint,
         user_id: this.syncParams.user_id,

package/src/sync/SyncContext.ts

@@ -0,0 +1,36 @@
+import { Semaphore, SemaphoreInterface, withTimeout } from 'async-mutex';
+
+export interface SyncContextOptions {
+  maxBuckets: number;
+  maxParameterQueryResults: number;
+  maxDataFetchConcurrency: number;
+}
+
+/**
+ * Maximum duration to wait for the mutex to become available.
+ *
+ * This gives an explicit error if there are mutex issues, rather than just hanging.
+ */
+const MUTEX_ACQUIRE_TIMEOUT = 30_000;
+
+/**
+ * Represents the context in which sync happens.
+ *
+ * This is global to all sync requests, not per request.
+ */
+export class SyncContext {
+  readonly maxBuckets: number;
+  readonly maxParameterQueryResults: number;
+
+  readonly syncSemaphore: SemaphoreInterface;
+
+  constructor(options: SyncContextOptions) {
+    this.maxBuckets = options.maxBuckets;
+    this.maxParameterQueryResults = options.maxParameterQueryResults;
+    this.syncSemaphore = withTimeout(
+      new Semaphore(options.maxDataFetchConcurrency),
+      MUTEX_ACQUIRE_TIMEOUT,
+      new Error(`Timeout while waiting for data`)
+    );
+  }
+}

package/src/sync/sync-index.ts

@@ -6,3 +6,4 @@ export * from './safeRace.js';
 export * from './sync.js';
 export * from './util.js';
 export * from './BucketChecksumState.js';
+export * from './SyncContext.js';

package/src/sync/sync.ts

@@ -1,6 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { BucketDescription, BucketPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
-import { Semaphore, withTimeout } from 'async-mutex';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -11,28 +10,12 @@ import * as util from '../util/util-index.js';
 import { logger } from '@powersync/lib-services-framework';
 import { BucketChecksumState } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from './merge.js';
-import { RequestTracker } from './RequestTracker.js';
 import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
-
-/**
- * Maximum number of connections actively fetching data.
- */
-const MAX_ACTIVE_CONNECTIONS = 10;
-
-/**
- * Maximum duration to wait for the mutex to become available.
- *
- * This gives an explicit error if there are mutex issues, rather than just hanging.
- */
-const MUTEX_ACQUIRE_TIMEOUT = 30_000;
-
-const syncSemaphore = withTimeout(
-  new Semaphore(MAX_ACTIVE_CONNECTIONS),
-  MUTEX_ACQUIRE_TIMEOUT,
-  new Error(`Timeout while waiting for data`)
-);
+import { SyncContext } from './SyncContext.js';
+import { RequestTracker } from './RequestTracker.js';
 
 export interface SyncStreamParameters {
+  syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
   syncRules: SqlSyncRules;
   params: util.StreamingSyncRequest;
@@ -50,7 +33,8 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
+  const { syncContext, bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } =
+    options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -66,7 +50,15 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(bucketStorage, syncRules, params, syncParams, tracker, controller.signal);
+  const stream = streamResponseInner(
+    syncContext,
+    bucketStorage,
+    syncRules,
+    params,
+    syncParams,
+    tracker,
+    controller.signal
+  );
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -87,10 +79,11 @@ export async function* streamResponse(
 
 export type BucketSyncState = {
   description?: BucketDescription; // Undefined if the bucket has not yet been resolved by us.
-  start_op_id: string;
+  start_op_id: util.InternalOpId;
 };
 
 async function* streamResponseInner(
+  syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
   syncRules: SqlSyncRules,
   params: util.StreamingSyncRequest,
@@ -103,10 +96,14 @@ async function* streamResponseInner(
   const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id);
 
   const checksumState = new BucketChecksumState({
+    syncContext,
     bucketStorage,
     syncRules,
     syncParams,
-    initialBucketPositions: params.buckets
+    initialBucketPositions: params.buckets?.map((bucket) => ({
+      name: bucket.name,
+      after: BigInt(bucket.after)
+    }))
   });
   const stream = bucketStorage.watchWriteCheckpoint({
     user_id: checkpointUserId,
@@ -195,6 +192,7 @@ async function* streamResponseInner(
         }
 
         yield* bucketDataInBatches({
+          syncContext: syncContext,
           bucketStorage: bucketStorage,
           checkpoint: next.value.value.base.checkpoint,
           bucketsToFetch: buckets,
@@ -221,8 +219,9 @@ async function* streamResponseInner(
 }
 
 interface BucketDataRequest {
+  syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  checkpoint: string;
+  checkpoint: util.InternalOpId;
   bucketsToFetch: BucketDescription[];
   /** Contains current bucket state. Modified by the request as data is sent.  */
   checksumState: BucketChecksumState;
@@ -282,6 +281,7 @@ interface BucketDataBatchResult {
  */
 async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
   const {
+    syncContext,
     bucketStorage: storage,
     checkpoint,
     bucketsToFetch,
@@ -293,13 +293,12 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     onRowsSent
   } = request;
 
-  const checkpointOp = BigInt(checkpoint);
   let checkpointInvalidated = false;
 
-  if (syncSemaphore.isLocked()) {
+  if (syncContext.syncSemaphore.isLocked()) {
     logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
   }
-  const acquired = await acquireSemaphoreAbortable(syncSemaphore, AbortSignal.any([abort_batch]));
+  const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
   if (acquired === 'aborted') {
     return;
   }
@@ -329,7 +328,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       if (r.has_more) {
         has_more = true;
       }
-      if (targetOp != null && targetOp > checkpointOp) {
+      if (targetOp != null && targetOp > checkpoint) {
         checkpointInvalidated = true;
       }
       if (r.data.length == 0) {
@@ -365,7 +364,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
       }
       onRowsSent(r.data.length);
 
-      checksumState.updateBucketPosition({ bucket: r.bucket, nextAfter: r.next_after, hasMore: r.has_more });
+      checksumState.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
 
       // Check if syncing bucket data is supposed to stop before fetching more data
       // from storage.
@@ -384,7 +383,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         if (request.forPriority != null) {
           const line: util.StreamingSyncCheckpointPartiallyComplete = {
             partial_checkpoint_complete: {
-              last_op_id: checkpoint,
+              last_op_id: util.internalToExternalOpId(checkpoint),
               priority: request.forPriority
             }
           };
@@ -392,7 +391,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         } else {
           const line: util.StreamingSyncCheckpointComplete = {
             checkpoint_complete: {
-              last_op_id: checkpoint
+              last_op_id: util.internalToExternalOpId(checkpoint)
             }
           };
           yield { data: line, done: true };

package/src/system/ServiceContext.ts

@@ -7,6 +7,7 @@ import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as utils from '../util/util-index.js';
+import { SyncContext } from '../sync/SyncContext.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
@@ -16,6 +17,7 @@ export interface ServiceContext {
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
   migrations: PowerSyncMigrationManager;
+  syncContext: SyncContext;
 }
 
 /**
@@ -26,6 +28,7 @@ export interface ServiceContext {
 export class ServiceContextContainer implements ServiceContext {
   lifeCycleEngine: LifeCycledSystem;
   storageEngine: storage.StorageEngine;
+  syncContext: SyncContext;
 
   constructor(public configuration: utils.ResolvedPowerSyncConfig) {
     this.lifeCycleEngine = new LifeCycledSystem();
@@ -34,6 +37,12 @@ export class ServiceContextContainer implements ServiceContext {
       configuration
     });
 
+    this.syncContext = new SyncContext({
+      maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
+      maxBuckets: configuration.api_parameters.max_buckets_per_connection,
+      maxParameterQueryResults: configuration.api_parameters.max_parameter_query_results
+    });
+
     const migrationManager = new MigrationManager();
     container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
 

package/src/util/config/compound-config-collector.ts

@@ -10,6 +10,13 @@ import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { SyncRulesCollector } from './sync-rules/sync-collector.js';
 import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
+import {
+  DEFAULT_MAX_BUCKETS_PER_CONNECTION,
+  DEFAULT_MAX_CONCURRENT_CONNECTIONS,
+  DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
+  DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
+  DEFAULT_MAX_POOL_SIZE
+} from './defaults.js';
 
 export type CompoundConfigCollectorOptions = {
   /**
@@ -124,7 +131,12 @@ export class CompoundConfigCollector {
     let config: ResolvedPowerSyncConfig = {
       base_config: baseConfig,
       connections: baseConfig.replication?.connections || [],
-      storage: baseConfig.storage,
+      storage: {
+        ...baseConfig.storage,
+        parameters: {
+          max_pool_size: baseConfig.storage?.parameters?.max_pool_size ?? DEFAULT_MAX_POOL_SIZE
+        }
+      },
       client_keystore: keyStore,
       // Dev tokens only use the static keys, no external key sources
       // We may restrict this even further to only the powersync-dev key.
@@ -146,6 +158,17 @@ export class CompoundConfigCollector {
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
       },
+      api_parameters: {
+        max_buckets_per_connection:
+          baseConfig.api?.parameters?.max_buckets_per_connection ?? DEFAULT_MAX_BUCKETS_PER_CONNECTION,
+
+        max_parameter_query_results:
+          baseConfig.api?.parameters?.max_parameter_query_results ?? DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
+        max_concurrent_connections:
+          baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
+        max_data_fetch_concurrency:
+          baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
+      },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
       slot_name_prefix: 'powersync_',

package/src/util/config/defaults.ts

@@ -0,0 +1,5 @@
+export const DEFAULT_MAX_POOL_SIZE = 8;
+export const DEFAULT_MAX_CONCURRENT_CONNECTIONS = 200;
+export const DEFAULT_MAX_DATA_FETCH_CONCURRENCY = 10;
+export const DEFAULT_MAX_BUCKETS_PER_CONNECTION = 1000;
+export const DEFAULT_MAX_PARAMETER_QUERY_RESULTS = 1000;

package/src/util/config/types.ts

@@ -1,5 +1,4 @@
 import { configFile } from '@powersync/service-types';
-import { PowerSyncConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
 import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
 import { KeySpec } from '../../auth/KeySpec.js';
 import { KeyStore } from '../../auth/KeyStore.js';
@@ -30,7 +29,7 @@ export type SyncRulesConfig = {
 };
 
 export type ResolvedPowerSyncConfig = {
-  base_config: PowerSyncConfig;
+  base_config: configFile.PowerSyncConfig;
   connections?: configFile.GenericDataSourceConfig[];
   storage: configFile.GenericStorageConfig;
   dev: {
@@ -60,6 +59,13 @@ export type ResolvedPowerSyncConfig = {
     internal_service_endpoint: string;
   };
 
+  api_parameters: {
+    max_concurrent_connections: number;
+    max_data_fetch_concurrency: number;
+    max_buckets_per_connection: number;
+    max_parameter_query_results: number;
+  };
+
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
   parameters: Record<string, number | string | boolean | null>;

package/src/util/protocol-types.ts

@@ -57,8 +57,8 @@ export interface StreamingSyncCheckpoint {
 
 export interface StreamingSyncCheckpointDiff {
   checkpoint_diff: {
-    last_op_id: OpId;
-    write_checkpoint?: OpId;
+    last_op_id: ProtocolOpId;
+    write_checkpoint?: ProtocolOpId;
     updated_buckets: BucketChecksumWithDescription[];
     removed_buckets: string[];
   };
@@ -70,13 +70,13 @@ export interface StreamingSyncData {
 
 export interface StreamingSyncCheckpointComplete {
   checkpoint_complete: {
-    last_op_id: OpId;
+    last_op_id: ProtocolOpId;
   };
 }
 
 export interface StreamingSyncCheckpointPartiallyComplete {
   partial_checkpoint_complete: {
-    last_op_id: OpId;
+    last_op_id: ProtocolOpId;
     priority: BucketPriority;
   };
 }
@@ -96,11 +96,11 @@ export type StreamingSyncLine =
 /**
  * 64-bit unsigned number, as a base-10 string.
  */
-export type OpId = string;
+export type ProtocolOpId = string;
 
 export interface Checkpoint {
-  last_op_id: OpId;
-  write_checkpoint?: OpId;
+  last_op_id: ProtocolOpId;
+  write_checkpoint?: ProtocolOpId;
   buckets: BucketChecksumWithDescription[];
 }
 
@@ -123,15 +123,15 @@ export interface SyncBucketData {
   /**
    * The `after` specified in the request.
    */
-  after: OpId;
+  after: ProtocolOpId;
   /**
    * Use this for the next request.
    */
-  next_after: OpId;
+  next_after: ProtocolOpId;
 }
 
 export interface OplogEntry {
-  op_id: OpId;
+  op_id: ProtocolOpId;
   op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
   object_type?: string;
   object_id?: string;