@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/test/src/storage/SourceTable.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, test } from 'vitest';
+import * as storage from '../../../src/storage/storage-index.js';
+
+/**
+ * Build a SourceTable with the current `ref`-based options shape.
+ * Data/parameter sources are not relevant to the storeCurrentData behaviour, so they default to empty.
+ */
+function makeTable(options: Partial<storage.SourceTableOptions> = {}): storage.SourceTable {
+  return new storage.SourceTable({
+    id: 'test-id',
+    ref: { connectionTag: 'test', schema: 'public', name: 'test_table' },
+    objectId: 123,
+    replicaIdColumns: [{ name: 'id' }],
+    snapshotComplete: true,
+    bucketDataSources: [],
+    parameterLookupSources: [],
+    ...options
+  });
+}
+
+describe('SourceTable', () => {
+  describe('storeCurrentData property', () => {
+    test('defaults to true', () => {
+      const table = makeTable();
+      expect(table.storeCurrentData).toBe(true);
+    });
+
+    test('can be set to false', () => {
+      const table = makeTable();
+      table.storeCurrentData = false;
+      expect(table.storeCurrentData).toBe(false);
+    });
+
+    test('is preserved when cloning', () => {
+      const table = makeTable();
+      table.storeCurrentData = false;
+      const cloned = table.clone();
+
+      expect(cloned.storeCurrentData).toBe(false);
+      expect(cloned).not.toBe(table);
+    });
+
+    test('clone preserves all properties including storeCurrentData', () => {
+      const table = makeTable({
+        replicaIdColumns: [{ name: 'id', type: 'int4' }],
+        snapshotComplete: false,
+        eventDefinitionIds: new Set(['event-1'])
+      });
+
+      table.syncData = false;
+      table.syncParameters = false;
+      table.syncEvent = false;
+      table.storeCurrentData = false;
+      table.snapshotStatus = {
+        totalEstimatedCount: 100,
+        replicatedCount: 50,
+        lastKey: Buffer.from('test')
+      };
+
+      const cloned = table.clone();
+
+      expect(cloned.syncData).toBe(false);
+      expect(cloned.syncParameters).toBe(false);
+      expect(cloned.syncEvent).toBe(false);
+      expect(cloned.storeCurrentData).toBe(false);
+      expect(cloned.snapshotStatus).toEqual(table.snapshotStatus);
+      expect(cloned.eventDefinitionIds).toEqual(new Set(['event-1']));
+      expect(cloned.eventDefinitionIds).not.toBe(table.eventDefinitionIds);
+    });
+  });
+
+  describe('integration with other properties', () => {
+    test('storeCurrentData does not affect syncAny', () => {
+      const table = makeTable();
+
+      table.storeCurrentData = false;
+      table.syncData = true;
+      table.syncParameters = false;
+      table.syncEvent = false;
+
+      expect(table.syncAny).toBe(true); // Should still be true
+    });
+
+    test('storeCurrentData is independent of snapshot status', () => {
+      const table = makeTable({ snapshotComplete: false });
+
+      table.storeCurrentData = false;
+      expect(table.snapshotComplete).toBe(false);
+      expect(table.storeCurrentData).toBe(false);
+    });
+  });
+});

package/test/src/storage-metrics.test.ts

@@ -0,0 +1,148 @@
+import { MetricsEngine } from '@/metrics/MetricsEngine.js';
+import {
+  MetricMetadata,
+  MetricsFactory,
+  ObservableGauge,
+  ObservableGaugeObservation
+} from '@/metrics/metrics-interfaces.js';
+import { BucketStorageFactory, StorageMetrics, StorageSyncConfigMetrics } from '@/storage/BucketStorageFactory.js';
+import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '@/storage/storage-metrics.js';
+import { StorageMetric } from '@powersync/service-types';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+type ValueProvider = () => Promise<number | ObservableGaugeObservation[] | undefined>;
+
+function testEngine() {
+  const providers = new Map<string, ValueProvider>();
+  const factory: MetricsFactory = {
+    createCounter: () => ({ add: () => {} }),
+    createUpDownCounter: () => ({ add: () => {} }),
+    createObservableGauge: (metadata: MetricMetadata): ObservableGauge => ({
+      setValueProvider: (valueProvider) => providers.set(metadata.name, valueProvider)
+    })
+  };
+  const engine = new MetricsEngine({ factory, disable_telemetry_sharing: true });
+  createCoreStorageMetrics(engine);
+  return { engine, providers };
+}
+
+function syncConfigMetrics(id: string, state: string, bytes: number, versionLabel?: string): StorageSyncConfigMetrics {
+  return {
+    sync_config_id: id,
+    sync_config_state: state,
+    version_label: versionLabel,
+    attributed_bucket_data_bytes: bytes,
+    attributed_parameter_indexes_bytes: bytes,
+    attributed_source_records_bytes: bytes,
+    attributed_object_storage_bytes: bytes
+  };
+}
+
+function storageMetrics(sync_config_metrics: StorageSyncConfigMetrics[]): StorageMetrics {
+  return {
+    operations_size_bytes: 0,
+    parameters_size_bytes: 0,
+    replication_size_bytes: 0,
+    object_storage_size_bytes: 0,
+    sync_config_metrics
+  };
+}
+
+describe('storage metrics', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.setSystemTime(0);
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it('reports retired sync configs as zero, then stops reporting them', async () => {
+    let current = storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 100)]);
+    const storage = {
+      getStorageMetrics: async () => current
+    } as unknown as BucketStorageFactory;
+
+    const { engine, providers } = testEngine();
+    initializeCoreStorageMetrics(engine, storage);
+    const provider = providers.get(StorageMetric.ATTRIBUTED_BUCKET_DATA_BYTES)!;
+
+    expect(await provider()).toEqual([
+      { value: 100, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
+    ]);
+
+    // The sync config is pruned. Its series must go to zero rather than keep its last value,
+    // since the SDK carries the last reported value forward under cumulative temporality.
+    current = storageMetrics([syncConfigMetrics('config-b', 'ACTIVE', 50)]);
+    vi.setSystemTime(61_000);
+
+    expect(await provider()).toEqual([
+      { value: 50, attributes: { sync_config_id: 'config-b', sync_config_state: 'ACTIVE' } },
+      { value: 0, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
+    ]);
+
+    // Once every reader has had a chance to observe the zero, stop reporting it entirely.
+    vi.setSystemTime(61_000 + 16 * 60_000);
+
+    expect(await provider()).toEqual([
+      { value: 50, attributes: { sync_config_id: 'config-b', sync_config_state: 'ACTIVE' } }
+    ]);
+  });
+
+  it('treats a state transition as a retired attribute set', async () => {
+    let current = storageMetrics([syncConfigMetrics('config-a', 'PROCESSING', 100, 'v1')]);
+    const storage = {
+      getStorageMetrics: async () => current
+    } as unknown as BucketStorageFactory;
+
+    const { engine, providers } = testEngine();
+    initializeCoreStorageMetrics(engine, storage);
+    const provider = providers.get(StorageMetric.ATTRIBUTED_SOURCE_RECORDS_BYTES)!;
+
+    await provider();
+
+    current = storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 120, 'v1')]);
+    vi.setSystemTime(61_000);
+
+    expect(await provider()).toEqual([
+      {
+        value: 120,
+        attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE', version_label: 'v1' }
+      },
+      {
+        value: 0,
+        attributes: { sync_config_id: 'config-a', sync_config_state: 'PROCESSING', version_label: 'v1' }
+      }
+    ]);
+  });
+
+  it('does not report anything as retired when reading the metrics fails', async () => {
+    let fail = false;
+    const storage = {
+      getStorageMetrics: async () => {
+        if (fail) {
+          throw new Error('storage unavailable');
+        }
+        return storageMetrics([syncConfigMetrics('config-a', 'ACTIVE', 100)]);
+      }
+    } as unknown as BucketStorageFactory;
+
+    const { engine, providers } = testEngine();
+    initializeCoreStorageMetrics(engine, storage);
+    const provider = providers.get(StorageMetric.ATTRIBUTED_OBJECT_STORAGE_BYTES)!;
+
+    await provider();
+
+    fail = true;
+    vi.setSystemTime(61_000);
+    expect(await provider()).toBeUndefined();
+
+    // The failure must not be interpreted as the sync config having been pruned.
+    fail = false;
+    vi.setSystemTime(122_000);
+    expect(await provider()).toEqual([
+      { value: 100, attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' } }
+    ]);
+  });
+});

package/test/src/sync/BucketChecksumState.test.ts

@@ -137,6 +137,7 @@ bucket_definitions:
         priority: 3
       }
     ]);
+    expect(line.bucketDataRequestHint).toEqual('bulk');
     // This is the bucket data to be fetched
     expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
 
@@ -170,6 +171,45 @@ bucket_definitions:
       }
     });
     expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
+    expect(line2.bucketDataRequestHint).toEqual('incremental');
+  });
+
+  test('unfinished bulk bucket does not make checkpoint diff bulk', async () => {
+    const storage = new MockBucketChecksumStateStorage();
+    storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
+
+    const state = new BucketChecksumState({
+      syncContext,
+      syncRequest,
+      tokenPayload,
+      syncRules: SYNC_RULES_GLOBAL,
+      bucketStorage: storage
+    });
+
+    const line = (await state.buildNextCheckpointLine({
+      base: storage.makeCheckpoint(1n),
+      writeCheckpoint: null,
+      update: CHECKPOINT_INVALIDATE_ALL
+    }))!;
+    line.advance();
+    line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: true });
+
+    storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
+
+    const line2 = (await state.buildNextCheckpointLine({
+      base: storage.makeCheckpoint(2n),
+      writeCheckpoint: null,
+      update: {
+        updatedDataBuckets: new Set(['1#global[]']),
+        invalidateDataBuckets: false,
+        updatedParameterLookups: new Set(),
+        invalidateParameterBuckets: false
+      }
+    }))!;
+    line2.advance();
+
+    expect(line2.bucketDataRequestHint).toEqual('incremental');
+    expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
   });
 
   test('global bucket with initial state', async () => {
@@ -1143,7 +1183,7 @@ streams:
       ).rejects.toThrow('Too many buckets: 60 (limit of 50');
 
       // Verify error log includes breakdown
-      expect(errorMessages[0]).toContain('Buckets by definition:');
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by sync stream:');
       expect(errorMessages[0]).toContain('projects: 30');
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
@@ -1157,15 +1197,60 @@ streams:
       });
     });
 
-    test('limits bucket breakdown to top 10 definitions', async () => {
-      // Create sync streams with 15 different definitions with dynamic parameters
+    test('labels the bucket breakdown as bucket definitions for legacy sync rules', async () => {
+      const legacySyncRules = SqlSyncRules.fromYaml(
+        `
+bucket_definitions:
+  projects:
+    parameters: SELECT id FROM projects WHERE user_id = request.user_id()
+    data: []
+    `,
+        { defaultSchema: 'public' }
+      ).config.hydrate({ hydrationState: versionedHydrationState(5), sqlite: nodeSqlite(sqlite) });
+
+      const storage = new MockBucketChecksumStateStorage();
+      const errorMessages: string[] = [];
+      const state = new BucketChecksumState({
+        syncContext: new SyncContext({
+          maxBuckets: 50,
+          maxParameterQueryResults: 100,
+          maxDataFetchConcurrency: 10
+        }),
+        tokenPayload: new JwtPayload({ sub: 'u1' }),
+        syncRequest,
+        syncRules: legacySyncRules,
+        bucketStorage: storage,
+        logger: {
+          info: () => {},
+          error: (message: string) => errorMessages.push(message),
+          warn: () => {},
+          debug: () => {}
+        } as any
+      });
+
+      await expect(
+        state.buildNextCheckpointLine({
+          base: storage.makeCheckpoint(1n, (lookups) => [
+            { lookup: lookups[0], rows: Array.from({ length: 60 }, (_, id) => ({ id })) }
+          ]),
+          writeCheckpoint: null,
+          update: CHECKPOINT_INVALIDATE_ALL
+        })
+      ).rejects.toThrow('Too many buckets: 60 (limit of 50)');
+
+      expect(errorMessages[0]).toContain('(limit of 50)\nBuckets by bucket definition:');
+      expect(errorMessages[0]).toContain('projects: 60');
+    });
+
+    test('limits bucket breakdown to top 100 sync streams', async () => {
+      // Create 105 sync streams with dynamic parameters.
       let yamlDefinitions = `
 config:
   edition: 3
 
 streams:
 `;
-      for (let i = 1; i <= 15; i++) {
+      for (let i = 1; i <= 105; i++) {
         yamlDefinitions += `  def${i}:\n`;
         yamlDefinitions += `    auto_subscribe: true\n`;
         yamlDefinitions += `    query: SELECT * FROM tbl WHERE b = auth.parameter('${i}')\n`;
@@ -1189,7 +1274,7 @@ streams:
       };
 
       const smallContext = new SyncContext({
-        maxBuckets: 10,
+        maxBuckets: 100,
         maxParameterQueryResults: 10,
         maxDataFetchConcurrency: 10
       });
@@ -1198,7 +1283,7 @@ streams:
         syncContext: smallContext,
         tokenPayload: new JwtPayload({
           sub: 'u1',
-          ...Object.fromEntries(Array.from({ length: 30 }, (_, i) => [i.toString(), BigInt(i)]))
+          ...Object.fromEntries(Array.from({ length: 106 }, (_, i) => [i.toString(), BigInt(i)]))
         }),
         syncRequest,
         syncRules: SYNC_RULES_MANY,
@@ -1212,15 +1297,16 @@ streams:
           writeCheckpoint: null,
           update: CHECKPOINT_INVALIDATE_ALL
         })
-      ).rejects.toThrow('Too many buckets: 15 (limit of 10)');
+      ).rejects.toThrow('Too many buckets: 105 (limit of 100)');
 
-      // Verify only top 10 are shown
+      // Verify only the first 100 are shown.
       const errorMessage = errorMessages[0];
-      expect(errorMessage).toContain('... and 5 more results from 5 definitions');
+      expect(errorMessage).toContain('Buckets by sync stream:');
+      expect(errorMessage).toContain('... and 5 more buckets from 5 sync streams');
 
-      // Count how many definitions are listed (should be exactly 10)
+      // Count how many sync streams are listed (should be exactly 100).
       const defMatches = errorMessage.match(/def\d+:/g);
-      expect(defMatches?.length).toBe(10);
+      expect(defMatches?.length).toBe(100);
     });
   });
 });
@@ -1240,7 +1326,7 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
     this.filter?.({ invalidate: true });
   }
 
-  async getChecksums(_checkpoint: InternalOpId, buckets: { bucket: string }[]): Promise<ChecksumMap> {
+  async getChecksums(_checkpoint: ReplicationCheckpoint, buckets: { bucket: string }[]): Promise<ChecksumMap> {
     return new Map<string, BucketChecksum>(
       buckets.map(({ bucket }) => {
         const checksum = this.state.get(bucket);

package/test/src/sync/util.test.ts

@@ -1,7 +1,19 @@
-import { acquireSemaphoreAbortable } from '@/index.js';
+import { acquireSemaphoreAbortable, isAbortError } from '@/index.js';
 import { Semaphore, SemaphoreInterface } from 'async-mutex';
+import { AbortError } from 'ix/aborterror.js';
 import { describe, expect, test, vi } from 'vitest';
 
+describe('isAbortError', () => {
+  test('recognizes ix and native abort errors', () => {
+    const controller = new AbortController();
+    controller.abort();
+
+    expect(isAbortError(new AbortError())).toBe(true);
+    expect(isAbortError(controller.signal.reason)).toBe(true);
+    expect(isAbortError(new Error('not aborted'))).toBe(false);
+  });
+});
+
 describe('acquireSemaphoreAbortable', () => {
   test('can acquire', async () => {
     const semaphore = new Semaphore(1);
@@ -31,4 +43,19 @@ describe('acquireSemaphoreAbortable', () => {
     // Releasing the semaphore should not invoke resolve again
     release();
   });
+
+  test('does not wait when the signal is already aborted', async () => {
+    const semaphore = new Semaphore(1);
+    const controller = new AbortController();
+
+    // Hold the only slot, so acquiring would block indefinitely.
+    const result = await acquireSemaphoreAbortable(semaphore, controller.signal);
+    expect(result).not.toBe('aborted');
+    const [, release] = result as [number, SemaphoreInterface.Releaser];
+
+    controller.abort();
+    expect(await acquireSemaphoreAbortable(semaphore, controller.signal)).toBe('aborted');
+
+    release();
+  });
 });