@powersync/service-core 0.0.0-dev-20250827091123 → 0.0.0-dev-20250828090417

package/src/storage/StorageProvider.ts

@@ -1,11 +1,9 @@
 import { ServiceError } from '@powersync/lib-services-framework';
 import * as util from '../util/util-index.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
-import { ReportStorage } from './ReportStorage.js';
 
 export interface ActiveStorage {
   storage: BucketStorageFactory;
-  reportStorage: ReportStorage;
   shutDown(): Promise<void>;
 
   /**
@@ -24,7 +22,7 @@ export interface GetStorageOptions {
 /**
  * Represents a provider that can create a storage instance for a specific storage type from configuration.
  */
-export interface StorageProvider {
+export interface BucketStorageProvider {
   /**
    *  The storage type that this provider provides.
    *  The type should match the `type` field in the config.

package/src/storage/SyncRulesBucketStorage.ts

@@ -62,6 +62,11 @@ export interface SyncRulesBucketStorage
 
   compact(options?: CompactOptions): Promise<void>;
 
+  /**
+   * Lightweight "compact" process to populate the checksum cache, if any.
+   */
+  populatePersistentChecksumCache(options?: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
+
   // ## Read operations
 
   getCheckpoint(): Promise<ReplicationCheckpoint>;
@@ -108,6 +113,11 @@ export interface SyncRulesBucketStorage
    * Returns zero checksums for any buckets not found.
    */
   getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+
+  /**
+   * Clear checksum cache. Primarily intended for tests.
+   */
+  clearChecksumCache(): void;
 }
 
 export interface SyncRulesBucketStorageListener {
@@ -208,6 +218,8 @@ export interface CompactOptions {
    * Internal/testing use: Cache size for compacting parameters.
    */
   compactParameterCacheLimit?: number;
+
+  signal?: AbortSignal;
 }
 
 export interface ClearStorageOptions {

package/src/storage/storage-index.ts

@@ -13,4 +13,3 @@ export * from './BucketStorageBatch.js';
 export * from './SyncRulesBucketStorage.js';
 export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationLock.js';
-export * from './ReportStorage.js';

package/src/sync/BucketChecksumState.ts

@@ -24,10 +24,15 @@ import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-ru
 import { SyncContext } from './SyncContext.js';
 import { getIntersection, hasIntersection } from './util.js';
 
+export interface VersionedSyncRules {
+  syncRules: SqlSyncRules;
+  version: number;
+}
+
 export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
   bucketStorage: BucketChecksumStateStorage;
-  syncRules: SqlSyncRules;
+  syncRules: VersionedSyncRules;
   tokenPayload: RequestJwtPayload;
   syncRequest: util.StreamingSyncRequest;
   logger?: Logger;
@@ -248,7 +253,7 @@ export class BucketChecksumState {
       const streamNameToIndex = new Map<string, number>();
       this.streamNameToIndex = streamNameToIndex;
 
-      for (const source of this.parameterState.syncRules.bucketSources) {
+      for (const source of this.parameterState.syncRules.syncRules.bucketSources) {
         if (this.parameterState.isSubscribedToStream(source)) {
           streamNameToIndex.set(source.name, subscriptions.length);
 
@@ -376,7 +381,7 @@ export interface CheckpointUpdate {
 export class BucketParameterState {
   private readonly context: SyncContext;
   public readonly bucketStorage: BucketChecksumStateStorage;
-  public readonly syncRules: SqlSyncRules;
+  public readonly syncRules: VersionedSyncRules;
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   /**
@@ -399,7 +404,7 @@ export class BucketParameterState {
   constructor(
     context: SyncContext,
     bucketStorage: BucketChecksumStateStorage,
-    syncRules: SqlSyncRules,
+    syncRules: VersionedSyncRules,
     tokenPayload: RequestJwtPayload,
     request: util.StreamingSyncRequest,
     logger: Logger
@@ -431,10 +436,11 @@ export class BucketParameterState {
     this.includeDefaultStreams = subscriptions?.include_defaults ?? true;
     this.explicitStreamSubscriptions = explicitStreamSubscriptions;
 
-    const { querier, errors } = syncRules.getBucketParameterQuerier({
+    const { querier, errors } = syncRules.syncRules.getBucketParameterQuerier({
       globalParameters: this.syncParams,
       hasDefaultStreams: this.includeDefaultStreams,
-      streams: streamsByName
+      streams: streamsByName,
+      bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`)
     });
     this.querier = querier;
     this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record<string, QuerierError[]>;

package/src/sync/RequestTracker.ts

@@ -2,6 +2,7 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
 import { APIMetric } from '@powersync/service-types';
 import { SyncBucketData } from '../util/protocol-types.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 /**
  * Record sync stats per request stream.
@@ -9,9 +10,12 @@ import { SyncBucketData } from '../util/protocol-types.js';
 export class RequestTracker {
   operationsSynced = 0;
   dataSyncedBytes = 0;
+  dataSentBytes = 0;
   operationCounts: OperationCounts = { put: 0, remove: 0, move: 0, clear: 0 };
   largeBuckets: Record<string, number> = {};
 
+  private encoding: string | undefined = undefined;
+
   constructor(private metrics: MetricsEngine) {
     this.metrics = metrics;
   }
@@ -29,18 +33,39 @@ export class RequestTracker {
     this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED).add(operations.total);
   }
 
-  addDataSynced(bytes: number) {
+  setCompressed(encoding: string) {
+    this.encoding = encoding;
+  }
+
+  addPlaintextDataSynced(bytes: number) {
     this.dataSyncedBytes += bytes;
 
     this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes);
+
+    if (this.encoding == null) {
+      // This avoids having to create a separate stream just to track this
+      this.dataSentBytes += bytes;
+
+      this.metrics.getCounter(APIMetric.DATA_SENT_BYTES).add(bytes);
+    }
+  }
+
+  addCompressedDataSent(bytes: number) {
+    if (this.encoding == null) {
+      throw new ServiceAssertionError('No compression encoding set');
+    }
+    this.dataSentBytes += bytes;
+    this.metrics.getCounter(APIMetric.DATA_SENT_BYTES).add(bytes);
   }
 
   getLogMeta() {
     return {
       operations_synced: this.operationsSynced,
       data_synced_bytes: this.dataSyncedBytes,
+      data_sent_bytes: this.dataSentBytes,
       operation_counts: this.operationCounts,
-      large_buckets: this.largeBuckets
+      large_buckets: this.largeBuckets,
+      encoding: this.encoding
     };
   }
 }

package/src/sync/sync.ts

@@ -14,7 +14,7 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
-import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
+import { BucketChecksumState, CheckpointLine, VersionedSyncRules } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
 import { SyncContext } from './SyncContext.js';
@@ -23,7 +23,7 @@ import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTra
 export interface SyncStreamParameters {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
-  syncRules: SqlSyncRules;
+  syncRules: VersionedSyncRules;
   params: util.StreamingSyncRequest;
   token: auth.JwtPayload;
   logger?: Logger;
@@ -100,7 +100,7 @@ export async function* streamResponse(
 async function* streamResponseInner(
   syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
-  syncRules: SqlSyncRules,
+  syncRules: VersionedSyncRules,
   params: util.StreamingSyncRequest,
   tokenPayload: RequestJwtPayload,
   tracker: RequestTracker,

package/src/sync/util.ts

@@ -125,7 +125,7 @@ export async function* transformToBytesTracked(
       encoded = data;
     }
 
-    tracker.addDataSynced(encoded.length);
+    tracker.addPlaintextDataSynced(encoded.length);
     yield encoded;
   }
 }

package/src/system/ServiceContext.ts

@@ -1,4 +1,4 @@
-import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
+import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
 import { framework } from '../index.js';
 import * as metrics from '../metrics/MetricsEngine.js';
@@ -8,7 +8,6 @@ import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import { SyncContext } from '../sync/SyncContext.js';
 import * as utils from '../util/util-index.js';
-import { EventsEngine } from '../events/EventsEngine.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
@@ -20,7 +19,6 @@ export interface ServiceContext {
   migrations: PowerSyncMigrationManager;
   syncContext: SyncContext;
   serviceMode: ServiceContextMode;
-  eventsEngine: EventsEngine;
 }
 
 export enum ServiceContextMode {
@@ -47,7 +45,6 @@ export class ServiceContextContainer implements ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
   storageEngine: storage.StorageEngine;
-  eventsEngine: EventsEngine;
   syncContext: SyncContext;
   routerEngine: routes.RouterEngine;
   serviceMode: ServiceContextMode;
@@ -69,11 +66,6 @@ export class ServiceContextContainer implements ServiceContext {
       }
     });
 
-    this.eventsEngine = new EventsEngine();
-    this.lifeCycleEngine.withLifecycle(this.eventsEngine, {
-      stop: (emitterEngine) => emitterEngine.shutDown()
-    });
-
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),
       stop: (storageEngine) => storageEngine.shutDown()
@@ -97,10 +89,6 @@ export class ServiceContextContainer implements ServiceContext {
       // Migrations should be executed before the system starts
       start: () => migrationManager[Symbol.asyncDispose]()
     });
-
-    this.lifeCycleEngine.withLifecycle(this.eventsEngine, {
-      stop: (emitterEngine) => emitterEngine.shutDown()
-    });
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {

package/src/util/utils.ts

@@ -6,11 +6,26 @@ import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
-import { PartialChecksum } from '../storage/ChecksumCache.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+/**
+ * A partial checksum can never be used on its own - must always be combined with a full BucketChecksum.
+ */
+export interface PartialChecksum {
+  bucket: string;
+  /**
+   * 32-bit unsigned hash.
+   */
+  partialChecksum: number;
+
+  /**
+   * Count of operations - informational only.
+   */
+  partialCount: number;
+}
+
 /**
  * op_id as used internally, for individual operations and checkpoints.
  *
@@ -83,20 +98,49 @@ export function addChecksums(a: number, b: number) {
   return (a + b) & 0xffffffff;
 }
 
-export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null): BucketChecksum {
-  if (b == null) {
-    return a;
-  } else if (b.isFullChecksum) {
+export function isPartialChecksum(c: PartialChecksum | BucketChecksum): c is PartialChecksum {
+  return 'partialChecksum' in c;
+}
+
+export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | BucketChecksum | null): BucketChecksum {
+  const checksum = addPartialChecksums(a.bucket, a, b);
+  if (isPartialChecksum(checksum)) {
+    // Should not happen since a != null
+    throw new ServiceAssertionError('Expected full checksum');
+  }
+  return checksum;
+}
+
+export function addPartialChecksums(
+  bucket: string,
+  a: BucketChecksum | null,
+  b: PartialChecksum | BucketChecksum | null
+): PartialChecksum | BucketChecksum {
+  if (a != null && b != null) {
+    if (!isPartialChecksum(b)) {
+      // Replaces preState
+      return b;
+    }
+    // merge
+    return {
+      bucket,
+      checksum: addChecksums(a.checksum, b.partialChecksum),
+      count: a.count + b.partialCount
+    };
+  } else if (a != null) {
     return {
-      bucket: b.bucket,
-      count: b.partialCount,
-      checksum: b.partialChecksum
+      bucket,
+      checksum: a.checksum,
+      count: a.count
     };
+  } else if (b != null) {
+    return b;
   } else {
+    // No data found (may still have a previously-cached checksum).
     return {
-      bucket: a.bucket,
-      count: a.count + b.partialCount,
-      checksum: addChecksums(a.checksum, b.partialChecksum)
+      bucket,
+      partialChecksum: 0,
+      partialCount: 0
     };
   }
 }

package/test/src/checksum_cache.test.ts

@@ -1,5 +1,5 @@
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
-import { addChecksums, InternalOpId } from '@/util/util-index.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import { addChecksums, BucketChecksum, InternalOpId, PartialChecksum } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
 import { describe, expect, it } from 'vitest';
 
@@ -12,22 +12,20 @@ function testHash(bucket: string, checkpoint: InternalOpId) {
   return hash;
 }
 
-function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum {
+function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum | BucketChecksum {
   if (request.start) {
     const a = testHash(request.bucket, request.start);
     const b = testHash(request.bucket, request.end);
     return {
       bucket: request.bucket,
       partialCount: Number(request.end) - Number(request.start),
-      partialChecksum: addChecksums(b, -a),
-      isFullChecksum: false
+      partialChecksum: addChecksums(b, -a)
     };
   } else {
     return {
       bucket: request.bucket,
-      partialChecksum: testHash(request.bucket, request.end),
-      partialCount: Number(request.end),
-      isFullChecksum: true
+      checksum: testHash(request.bucket, request.end),
+      count: Number(request.end)
     };
   }
 }

package/test/src/routes/mocks.ts

@@ -0,0 +1,59 @@
+import {
+  BucketStorageFactory,
+  createCoreAPIMetrics,
+  MetricsEngine,
+  OpenTelemetryMetricsFactory,
+  RouteAPI,
+  RouterEngine,
+  ServiceContext,
+  StorageEngine,
+  SyncContext,
+  SyncRulesBucketStorage
+} from '@/index.js';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+
+export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
+  // This is very incomplete - just enough to get the current tests passing.
+
+  const storageEngine: StorageEngine = {
+    activeBucketStorage: {
+      async getActiveStorage() {
+        return storage;
+      }
+    } as Partial<BucketStorageFactory>
+  } as any;
+
+  const meterProvider = new MeterProvider({
+    readers: []
+  });
+  const meter = meterProvider.getMeter('powersync-tests');
+  const metricsEngine = new MetricsEngine({
+    disable_telemetry_sharing: true,
+    factory: new OpenTelemetryMetricsFactory(meter)
+  });
+  createCoreAPIMetrics(metricsEngine);
+  const service_context: Partial<ServiceContext> = {
+    syncContext: new SyncContext({ maxBuckets: 1, maxDataFetchConcurrency: 1, maxParameterQueryResults: 1 }),
+    routerEngine: {
+      getAPI() {
+        return {
+          getParseSyncRulesOptions() {
+            return { defaultSchema: 'public' };
+          }
+        } as Partial<RouteAPI>;
+      },
+      addStopHandler() {
+        return () => {};
+      }
+    } as Partial<RouterEngine> as any,
+    storageEngine,
+    metricsEngine: metricsEngine,
+    // Not used
+    configuration: null as any,
+    lifeCycleEngine: null as any,
+    migrations: null as any,
+    replicationEngine: null as any,
+    serviceMode: null as any
+  };
+  return service_context as ServiceContext;
+}

package/test/src/routes/stream.test.ts

@@ -0,0 +1,84 @@
+import { BasicRouterRequest, Context, SyncRulesBucketStorage } from '@/index.js';
+import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { Readable, Writable } from 'stream';
+import { pipeline } from 'stream/promises';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
+import { mockServiceContext } from './mocks.js';
+
+describe('Stream Route', () => {
+  describe('compressed stream', () => {
+    it('handles missing sync rules', async () => {
+      const context: Context = {
+        logger: logger,
+        service_context: mockServiceContext(null)
+      };
+
+      const request: BasicRouterRequest = {
+        headers: {},
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const error = (await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>).catch(
+        (e) => e
+      )) as ServiceError;
+
+      expect(error.errorData.status).toEqual(500);
+      expect(error.errorData.code).toEqual('PSYNC_S2302');
+    });
+
+    it('handles a stream error with compression', async () => {
+      // This primarily tests that an underlying storage error doesn't result in an uncaught error
+      // when compressing the stream.
+
+      const storage = {
+        getParsedSyncRules() {
+          return new SqlSyncRules('bucket_definitions: {}');
+        },
+        watchCheckpointChanges: async function* (options) {
+          throw new Error('Simulated storage error');
+        }
+      } as Partial<SyncRulesBucketStorage>;
+      const serviceContext = mockServiceContext(storage);
+
+      const context: Context = {
+        logger: logger,
+        service_context: serviceContext,
+        token_payload: {
+          exp: new Date().getTime() / 1000 + 10000,
+          iat: new Date().getTime() / 1000 - 10000,
+          sub: 'test-user'
+        }
+      };
+
+      // It may be worth eventually doing this via Fastify to test the full stack
+
+      const request: BasicRouterRequest = {
+        headers: {
+          'accept-encoding': 'gzip'
+        },
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const response = await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>);
+      expect(response.status).toEqual(200);
+      const stream = response.data as Readable;
+      const r = await drainWithTimeout(stream).catch((error) => error);
+      expect(r.message).toContain('Simulated storage error');
+    });
+  });
+});
+
+export async function drainWithTimeout(readable: Readable, ms = 2_000) {
+  const devNull = new Writable({
+    write(_chunk, _enc, cb) {
+      cb();
+    } // discard everything
+  });
+
+  // Throws AbortError if it takes longer than ms, and destroys the stream
+  await pipeline(readable, devNull, { signal: AbortSignal.timeout(ms) });
+}