@powersync/service-core 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606

package/src/util/config/compound-config-collector.ts

@@ -10,6 +10,13 @@ import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { SyncRulesCollector } from './sync-rules/sync-collector.js';
 import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
+import {
+  DEFAULT_MAX_BUCKETS_PER_CONNECTION,
+  DEFAULT_MAX_CONCURRENT_CONNECTIONS,
+  DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
+  DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
+  DEFAULT_MAX_POOL_SIZE
+} from './defaults.js';
 
 export type CompoundConfigCollectorOptions = {
   /**
@@ -124,7 +131,12 @@ export class CompoundConfigCollector {
     let config: ResolvedPowerSyncConfig = {
       base_config: baseConfig,
       connections: baseConfig.replication?.connections || [],
-      storage: baseConfig.storage,
+      storage: {
+        ...baseConfig.storage,
+        parameters: {
+          max_pool_size: baseConfig.storage?.parameters?.max_pool_size ?? DEFAULT_MAX_POOL_SIZE
+        }
+      },
       client_keystore: keyStore,
       // Dev tokens only use the static keys, no external key sources
       // We may restrict this even further to only the powersync-dev key.
@@ -146,6 +158,17 @@ export class CompoundConfigCollector {
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
       },
+      api_parameters: {
+        max_buckets_per_connection:
+          baseConfig.api?.parameters?.max_buckets_per_connection ?? DEFAULT_MAX_BUCKETS_PER_CONNECTION,
+
+        max_parameter_query_results:
+          baseConfig.api?.parameters?.max_parameter_query_results ?? DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
+        max_concurrent_connections:
+          baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
+        max_data_fetch_concurrency:
+          baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
+      },
       // TODO maybe move this out of the connection or something
       // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
       slot_name_prefix: 'powersync_',
@@ -196,7 +219,8 @@ export class CompoundConfigCollector {
       }
     }
     return {
-      present: false
+      present: false,
+      exit_on_error: true
     };
   }
 }

package/src/util/config/defaults.ts

@@ -0,0 +1,5 @@
+export const DEFAULT_MAX_POOL_SIZE = 8;
+export const DEFAULT_MAX_CONCURRENT_CONNECTIONS = 200;
+export const DEFAULT_MAX_DATA_FETCH_CONCURRENCY = 10;
+export const DEFAULT_MAX_BUCKETS_PER_CONNECTION = 1000;
+export const DEFAULT_MAX_PARAMETER_QUERY_RESULTS = 1000;

package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts

@@ -15,6 +15,7 @@ export class Base64SyncRulesCollector extends SyncRulesCollector {
 
     return {
       present: true,
+      exit_on_error: baseConfig.sync_rules?.exit_on_error ?? true,
       content: Buffer.from(sync_rules_base64, 'base64').toString()
     };
   }

package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts

@@ -20,6 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
     // Only persist the path here, and load on demand using `loadSyncRules()`.
     return {
       present: true,
+      exit_on_error: baseConfig.sync_rules?.exit_on_error ?? true,
       path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
     };
   }

package/src/util/config/sync-rules/impl/inline-sync-rules-collector.ts

@@ -15,6 +15,7 @@ export class InlineSyncRulesCollector extends SyncRulesCollector {
 
     return {
       present: true,
+      exit_on_error: true,
       ...baseConfig.sync_rules
     };
   }

package/src/util/config/sync-rules/sync-rules-provider.ts

@@ -3,6 +3,8 @@ import fs from 'fs/promises';
 
 export interface SyncRulesProvider {
   get(): Promise<string | undefined>;
+
+  readonly exitOnError: boolean;
 }
 
 export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
@@ -15,4 +17,8 @@ export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
       return await fs.readFile(this.config.path, 'utf-8');
     }
   }
+
+  get exitOnError() {
+    return this.config.exit_on_error;
+  }
 }

package/src/util/config/types.ts

@@ -1,5 +1,4 @@
 import { configFile } from '@powersync/service-types';
-import { PowerSyncConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
 import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
 import { KeySpec } from '../../auth/KeySpec.js';
 import { KeyStore } from '../../auth/KeyStore.js';
@@ -26,10 +25,11 @@ export type SyncRulesConfig = {
   present: boolean;
   content?: string;
   path?: string;
+  exit_on_error: boolean;
 };
 
 export type ResolvedPowerSyncConfig = {
-  base_config: PowerSyncConfig;
+  base_config: configFile.PowerSyncConfig;
   connections?: configFile.GenericDataSourceConfig[];
   storage: configFile.GenericStorageConfig;
   dev: {
@@ -59,6 +59,13 @@ export type ResolvedPowerSyncConfig = {
     internal_service_endpoint: string;
   };
 
+  api_parameters: {
+    max_concurrent_connections: number;
+    max_data_fetch_concurrency: number;
+    max_buckets_per_connection: number;
+    max_parameter_query_results: number;
+  };
+
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
   parameters: Record<string, number | string | boolean | null>;

package/src/util/memory-tracking.ts

@@ -25,8 +25,8 @@ export function trackMemoryUsage() {
   for (let key of Object.keys(bufferMemory)) {
     const typedKey = key as keyof typeof bufferMemory;
     const originalFunction = Buffer[typedKey] as (...args: any[]) => Buffer;
-    Buffer[typedKey] = function (...args: any[]) {
-      const buffer = originalFunction.apply(this, args);
+    Buffer[typedKey] = function <TArrayBuffer extends ArrayBufferLike = ArrayBufferLike>(...args: any[]) {
+      const buffer = originalFunction.apply(this, args) as Buffer<TArrayBuffer>;
       bufferMemory[typedKey] += buffer.byteLength;
       bufferRegistry.register(buffer, [typedKey, buffer.byteLength]);
       return buffer;

package/src/util/protocol-types.ts

@@ -1,5 +1,5 @@
 import * as t from 'ts-codec';
-import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import { BucketDescription, BucketPriority, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 export const BucketRequest = t.object({
   name: t.string,
@@ -59,7 +59,7 @@ export interface StreamingSyncCheckpointDiff {
   checkpoint_diff: {
     last_op_id: OpId;
     write_checkpoint?: OpId;
-    updated_buckets: BucketChecksum[];
+    updated_buckets: BucketChecksumWithDescription[];
     removed_buckets: string[];
   };
 }
@@ -74,13 +74,23 @@ export interface StreamingSyncCheckpointComplete {
   };
 }
 
-export interface StreamingSyncKeepalive {}
+export interface StreamingSyncCheckpointPartiallyComplete {
+  partial_checkpoint_complete: {
+    last_op_id: OpId;
+    priority: BucketPriority;
+  };
+}
+
+export interface StreamingSyncKeepalive {
+  token_expires_in: number;
+}
 
 export type StreamingSyncLine =
   | StreamingSyncData
   | StreamingSyncCheckpoint
   | StreamingSyncCheckpointDiff
   | StreamingSyncCheckpointComplete
+  | StreamingSyncCheckpointPartiallyComplete
   | StreamingSyncKeepalive;
 
 /**
@@ -91,7 +101,7 @@ export type OpId = string;
 export interface Checkpoint {
   last_op_id: OpId;
   write_checkpoint?: OpId;
-  buckets: BucketChecksum[];
+  buckets: BucketChecksumWithDescription[];
 }
 
 export interface BucketState {
@@ -142,3 +152,5 @@ export interface BucketChecksum {
    */
   count: number;
 }
+
+export interface BucketChecksumWithDescription extends BucketChecksum, BucketDescription {}

package/src/util/util-index.ts

@@ -5,6 +5,7 @@ export * from './Mutex.js';
 export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
+export * from './checkpointing.js';
 
 export * from './config.js';
 export * from './config/compound-config-collector.js';

package/src/util/utils.ts

@@ -145,16 +145,6 @@ export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqlit
   return !hasToastedValues(row);
 }
 
-export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {
-  if (user_id == null) {
-    throw new Error('user_id is required');
-  }
-  if (client_id == null) {
-    return user_id;
-  }
-  return `${user_id}/${client_id}`;
-}
-
 /**
  * Reduce a bucket to the final state as stored on the client.
  *

package/test/src/auth.test.ts

@@ -274,7 +274,7 @@ describe('JWT Auth', () => {
     ).rejects.toThrow('Token must expire in a maximum of');
   });
 
-  test('http', async () => {
+  test('http', { timeout: 20_000 }, async () => {
     // Not ideal to rely on an external endpoint for tests, but it is good to test that this
     // one actually works.
     const remote = new RemoteJWKSCollector(
@@ -290,9 +290,9 @@ describe('JWT Auth', () => {
         reject_ip_ranges: ['local']
       }
     });
-    expect(invalid.getKeys()).rejects.toThrow('IPs in this range are not supported');
+    await expect(invalid.getKeys()).rejects.toThrow('IPs in this range are not supported');
 
-    // IPS throw an error immediately
+    // IPs throw an error immediately
     expect(
       () =>
         new RemoteJWKSCollector('https://127.0.0.1/.well-known/jwks.json', {
@@ -315,11 +315,11 @@ describe('JWT Auth', () => {
 
     const invalid = new RemoteJWKSCollector('https://127.0.0.1/.well-known/jwks.json');
     // Should try and fetch
-    expect(invalid.getKeys()).rejects.toThrow();
+    await expect(invalid.getKeys()).rejects.toThrow();
 
     const invalid2 = new RemoteJWKSCollector('https://localhost/.well-known/jwks.json');
     // Should try and fetch
-    expect(invalid2.getKeys()).rejects.toThrow();
+    await expect(invalid2.getKeys()).rejects.toThrow();
   });
 
   test('caching', async () => {