@powersync/service-core 0.0.0-dev-20250227082606 → 0.0.0-dev-20250303114151

package/src/routes/endpoints/sync-stream.ts

@@ -20,7 +20,7 @@ export const syncStreamed = routeDefinition({
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const { service_context } = payload.context;
-    const { routerEngine, storageEngine, syncContext } = service_context;
+    const { routerEngine, storageEngine } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
@@ -56,7 +56,6 @@ export const syncStreamed = routeDefinition({
         sync.transformToBytesTracked(
           sync.ndjson(
             sync.streamResponse({
-              syncContext: syncContext,
               bucketStorage,
               syncRules: syncRules,
               params,

package/src/sync/BucketChecksumState.ts

@@ -6,10 +6,8 @@ import * as util from '../util/util-index.js';
 import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
 import { BucketSyncState } from './sync.js';
-import { SyncContext } from './SyncContext.js';
 
 export interface BucketChecksumStateOptions {
-  syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
   syncRules: SqlSyncRules;
   syncParams: RequestParameters;
@@ -22,7 +20,6 @@ export interface BucketChecksumStateOptions {
  * Handles incrementally re-computing checkpoints.
  */
 export class BucketChecksumState {
-  private readonly context: SyncContext;
   private readonly bucketStorage: BucketChecksumStateStorage;
 
   /**
@@ -46,14 +43,8 @@ export class BucketChecksumState {
   private pendingBucketDownloads = new Set<string>();
 
   constructor(options: BucketChecksumStateOptions) {
-    this.context = options.syncContext;
     this.bucketStorage = options.bucketStorage;
-    this.parameterState = new BucketParameterState(
-      options.syncContext,
-      options.bucketStorage,
-      options.syncRules,
-      options.syncParams
-    );
+    this.parameterState = new BucketParameterState(options.bucketStorage, options.syncRules, options.syncParams);
     this.bucketDataPositions = new Map();
 
     for (let { name, after: start } of options.initialBucketPositions ?? []) {
@@ -82,12 +73,6 @@ export class BucketChecksumState {
       });
     }
     this.bucketDataPositions = dataBucketsNew;
-    if (dataBucketsNew.size > this.context.maxBuckets) {
-      throw new ServiceError(
-        ErrorCode.PSYNC_S2305,
-        `Too many buckets: ${dataBucketsNew.size} (limit of ${this.context.maxBuckets})`
-      );
-    }
 
     let checksumMap: util.ChecksumMap;
     if (updatedBuckets != null) {
@@ -262,20 +247,13 @@ export interface CheckpointUpdate {
 }
 
 export class BucketParameterState {
-  private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
   public readonly syncRules: SqlSyncRules;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   private readonly staticBuckets: Map<string, BucketDescription>;
 
-  constructor(
-    context: SyncContext,
-    bucketStorage: BucketChecksumStateStorage,
-    syncRules: SqlSyncRules,
-    syncParams: RequestParameters
-  ) {
-    this.context = context;
+  constructor(bucketStorage: BucketChecksumStateStorage, syncRules: SqlSyncRules, syncParams: RequestParameters) {
     this.bucketStorage = bucketStorage;
     this.syncRules = syncRules;
     this.syncParams = syncParams;
@@ -297,13 +275,9 @@ export class BucketParameterState {
       return null;
     }
 
-    if (update.buckets.length > this.context.maxParameterQueryResults) {
-      // TODO: Limit number of results even before we get to this point
-      // This limit applies _before_ we get the unique set
-      const error = new ServiceError(
-        ErrorCode.PSYNC_S2305,
-        `Too many parameter query results: ${update.buckets.length} (limit of ${this.context.maxParameterQueryResults})`
-      );
+    if (update.buckets.length > 1000) {
+      // TODO: Limit number of buckets even before we get to this point
+      const error = new ServiceError(ErrorCode.PSYNC_S2305, `Too many buckets: ${update.buckets.length}`);
       logger.error(error.message, {
         checkpoint: checkpoint,
         user_id: this.syncParams.user_id,

package/src/sync/sync-index.ts

@@ -6,4 +6,3 @@ export * from './safeRace.js';
 export * from './sync.js';
 export * from './util.js';
 export * from './BucketChecksumState.js';
-export * from './SyncContext.js';

package/src/sync/sync.ts

@@ -1,5 +1,6 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { BucketDescription, BucketPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import { Semaphore, withTimeout } from 'async-mutex';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -10,12 +11,28 @@ import * as util from '../util/util-index.js';
 import { logger } from '@powersync/lib-services-framework';
 import { BucketChecksumState } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from './merge.js';
-import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
-import { SyncContext } from './SyncContext.js';
 import { RequestTracker } from './RequestTracker.js';
+import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
+
+/**
+ * Maximum number of connections actively fetching data.
+ */
+const MAX_ACTIVE_CONNECTIONS = 10;
+
+/**
+ * Maximum duration to wait for the mutex to become available.
+ *
+ * This gives an explicit error if there are mutex issues, rather than just hanging.
+ */
+const MUTEX_ACQUIRE_TIMEOUT = 30_000;
+
+const syncSemaphore = withTimeout(
+  new Semaphore(MAX_ACTIVE_CONNECTIONS),
+  MUTEX_ACQUIRE_TIMEOUT,
+  new Error(`Timeout while waiting for data`)
+);
 
 export interface SyncStreamParameters {
-  syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
   syncRules: SqlSyncRules;
   params: util.StreamingSyncRequest;
@@ -33,8 +50,7 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { syncContext, bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } =
-    options;
+  const { bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -50,15 +66,7 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(
-    syncContext,
-    bucketStorage,
-    syncRules,
-    params,
-    syncParams,
-    tracker,
-    controller.signal
-  );
+  const stream = streamResponseInner(bucketStorage, syncRules, params, syncParams, tracker, controller.signal);
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -83,7 +91,6 @@ export type BucketSyncState = {
 };
 
 async function* streamResponseInner(
-  syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
   syncRules: SqlSyncRules,
   params: util.StreamingSyncRequest,
@@ -96,7 +103,6 @@ async function* streamResponseInner(
   const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id);
 
   const checksumState = new BucketChecksumState({
-    syncContext,
     bucketStorage,
     syncRules,
     syncParams,
@@ -189,7 +195,6 @@ async function* streamResponseInner(
         }
 
         yield* bucketDataInBatches({
-          syncContext: syncContext,
           bucketStorage: bucketStorage,
           checkpoint: next.value.value.base.checkpoint,
           bucketsToFetch: buckets,
@@ -216,7 +221,6 @@ async function* streamResponseInner(
 }
 
 interface BucketDataRequest {
-  syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
   checkpoint: string;
   bucketsToFetch: BucketDescription[];
@@ -278,7 +282,6 @@ interface BucketDataBatchResult {
  */
 async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
   const {
-    syncContext,
     bucketStorage: storage,
     checkpoint,
     bucketsToFetch,
@@ -293,10 +296,10 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
   const checkpointOp = BigInt(checkpoint);
   let checkpointInvalidated = false;
 
-  if (syncContext.syncSemaphore.isLocked()) {
+  if (syncSemaphore.isLocked()) {
     logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
   }
-  const acquired = await acquireSemaphoreAbortable(syncContext.syncSemaphore, AbortSignal.any([abort_batch]));
+  const acquired = await acquireSemaphoreAbortable(syncSemaphore, AbortSignal.any([abort_batch]));
   if (acquired === 'aborted') {
     return;
   }

package/src/system/ServiceContext.ts

@@ -7,7 +7,6 @@ import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as utils from '../util/util-index.js';
-import { SyncContext } from '../sync/SyncContext.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
@@ -17,7 +16,6 @@ export interface ServiceContext {
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
   migrations: PowerSyncMigrationManager;
-  syncContext: SyncContext;
 }
 
 /**
@@ -28,7 +26,6 @@ export interface ServiceContext {
 export class ServiceContextContainer implements ServiceContext {
   lifeCycleEngine: LifeCycledSystem;
   storageEngine: storage.StorageEngine;
-  syncContext: SyncContext;
 
   constructor(public configuration: utils.ResolvedPowerSyncConfig) {
     this.lifeCycleEngine = new LifeCycledSystem();
@@ -37,12 +34,6 @@ export class ServiceContextContainer implements ServiceContext {
       configuration
     });
 
-    this.syncContext = new SyncContext({
-      maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
-      maxBuckets: configuration.api_parameters.max_buckets_per_connection,
-      maxParameterQueryResults: configuration.api_parameters.max_parameter_query_results
-    });
-
     const migrationManager = new MigrationManager();
     container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
 

package/src/util/config/compound-config-collector.ts

@@ -10,13 +10,6 @@ import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { SyncRulesCollector } from './sync-rules/sync-collector.js';
 import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
-import {
-  DEFAULT_MAX_BUCKETS_PER_CONNECTION,
-  DEFAULT_MAX_CONCURRENT_CONNECTIONS,
-  DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
-  DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
-  DEFAULT_MAX_POOL_SIZE
-} from './defaults.js';
 
 export type CompoundConfigCollectorOptions = {
   /**
@@ -131,12 +124,7 @@ export class CompoundConfigCollector {
     let config: ResolvedPowerSyncConfig = {
       base_config: baseConfig,
       connections: baseConfig.replication?.connections || [],
-      storage: {
-        ...baseConfig.storage,
-        parameters: {
-          max_pool_size: baseConfig.storage?.parameters?.max_pool_size ?? DEFAULT_MAX_POOL_SIZE
-        }
-      },
+      storage: baseConfig.storage,
       client_keystore: keyStore,
       // Dev tokens only use the static keys, no external key sources
       // We may restrict this even further to only the powersync-dev key.
@@ -158,17 +146,6 @@ export class CompoundConfigCollector {
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
       },
-      api_parameters: {
-        max_buckets_per_connection:
-          baseConfig.api?.parameters?.max_buckets_per_connection ?? DEFAULT_MAX_BUCKETS_PER_CONNECTION,
-
-        max_parameter_query_results:
-          baseConfig.api?.parameters?.max_parameter_query_results ?? DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
-        max_concurrent_connections:
-          baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
-        max_data_fetch_concurrency:
-          baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
-      },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
       slot_name_prefix: 'powersync_',

package/src/util/config/types.ts

@@ -1,4 +1,5 @@
 import { configFile } from '@powersync/service-types';
+import { PowerSyncConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
 import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
 import { KeySpec } from '../../auth/KeySpec.js';
 import { KeyStore } from '../../auth/KeyStore.js';
@@ -29,7 +30,7 @@ export type SyncRulesConfig = {
 };
 
 export type ResolvedPowerSyncConfig = {
-  base_config: configFile.PowerSyncConfig;
+  base_config: PowerSyncConfig;
   connections?: configFile.GenericDataSourceConfig[];
   storage: configFile.GenericStorageConfig;
   dev: {
@@ -59,13 +60,6 @@ export type ResolvedPowerSyncConfig = {
     internal_service_endpoint: string;
   };
 
-  api_parameters: {
-    max_concurrent_connections: number;
-    max_data_fetch_concurrency: number;
-    max_buckets_per_connection: number;
-    max_parameter_query_results: number;
-  };
-
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
   parameters: Record<string, number | string | boolean | null>;

package/test/src/sync/BucketChecksumState.test.ts

@@ -5,7 +5,6 @@ import {
   CHECKPOINT_INVALIDATE_ALL,
   ChecksumMap,
   OpId,
-  SyncContext,
   WatchFilterEvent
 } from '@/index.js';
 import { RequestParameters, SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
@@ -47,19 +46,12 @@ bucket_definitions:
     { defaultSchema: 'public' }
   );
 
-  const syncContext = new SyncContext({
-    maxBuckets: 100,
-    maxParameterQueryResults: 100,
-    maxDataFetchConcurrency: 10
-  });
-
   test('global bucket with update', async () => {
     const storage = new MockBucketChecksumStateStorage();
     // Set intial state
     storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: '' }, {}),
       syncRules: SYNC_RULES_GLOBAL,
       bucketStorage: storage
@@ -123,7 +115,6 @@ bucket_definitions:
     storage.updateTestChecksum({ bucket: 'global[]', checksum: 1, count: 1 });
 
     const state = new BucketChecksumState({
-      syncContext,
       // Client sets the initial state here
       initialBucketPositions: [{ name: 'global[]', after: '1' }],
       syncParams: new RequestParameters({ sub: '' }, {}),
@@ -160,7 +151,6 @@ bucket_definitions:
     storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: '' }, {}),
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
@@ -224,7 +214,6 @@ bucket_definitions:
     const storage = new MockBucketChecksumStateStorage();
 
     const state = new BucketChecksumState({
-      syncContext,
       // Client sets the initial state here
       initialBucketPositions: [{ name: 'something_here[]', after: '1' }],
       syncParams: new RequestParameters({ sub: '' }, {}),
@@ -264,7 +253,6 @@ bucket_definitions:
     storage.updateTestChecksum({ bucket: 'global[2]', checksum: 1, count: 1 });
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: '' }, {}),
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
@@ -316,7 +304,6 @@ bucket_definitions:
     const storage = new MockBucketChecksumStateStorage();
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: '' }, {}),
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
@@ -368,7 +355,6 @@ bucket_definitions:
     storage.updateTestChecksum({ bucket: 'global[2]', checksum: 3, count: 3 });
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: '' }, {}),
       syncRules: SYNC_RULES_GLOBAL_TWO,
       bucketStorage: storage
@@ -466,7 +452,6 @@ bucket_definitions:
     storage.updateTestChecksum({ bucket: 'by_project[3]', checksum: 1, count: 1 });
 
     const state = new BucketChecksumState({
-      syncContext,
       syncParams: new RequestParameters({ sub: 'u1' }, {}),
       syncRules: SYNC_RULES_DYNAMIC,
       bucketStorage: storage