@powersync/service-module-postgres 0.0.0-dev-20250304151813 → 0.0.0-dev-20250306152715

package/test/src/slow_tests.test.ts

@@ -1,5 +1,5 @@
 import * as bson from 'bson';
-import { afterEach, describe, expect, test } from 'vitest';
+import { afterEach, beforeAll, describe, expect, test } from 'vitest';
 import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js';
 import { env } from './env.js';
 import {
@@ -15,8 +15,8 @@ import * as pgwire from '@powersync/service-jpgwire';
 import { SqliteRow } from '@powersync/service-sync-rules';
 
 import { PgManager } from '@module/replication/PgManager.js';
-import { storage } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import { createCoreReplicationMetrics, initializeCoreReplicationMetrics, storage } from '@powersync/service-core';
+import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import * as timers from 'node:timers/promises';
@@ -49,6 +49,11 @@ function defineSlowTests(factory: storage.TestStorageFactory) {
   let abortController: AbortController | undefined;
   let streamPromise: Promise<void> | undefined;
 
+  beforeAll(async () => {
+    createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+    initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+  });
+
   afterEach(async () => {
     // This cleans up, similar to WalStreamTestContext.dispose().
     // These tests are a little more complex than what is supported by WalStreamTestContext.
@@ -98,7 +103,8 @@ bucket_definitions:
     const options: WalStreamOptions = {
       abort_signal: abortController.signal,
       connections,
-      storage: storage
+      storage: storage,
+      metrics: METRICS_HELPER.metricsEngine
     };
     walStream = new WalStream(options);
 
@@ -178,7 +184,7 @@ bucket_definitions:
             break;
           }
 
-          const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint);
+          const checkpoint = (await storage.getCheckpoint()).checkpoint;
           if (f instanceof mongo_storage.storage.MongoBucketStorage) {
             const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray())
               .filter((row) => row._id.o <= checkpoint)
@@ -344,13 +350,14 @@ bucket_definitions:
       const connections = new PgManager(TEST_CONNECTION_OPTIONS, {});
       const replicationConnection = await connections.replicationConnection();
 
-      abortController = new AbortController();
-      const options: WalStreamOptions = {
-        abort_signal: abortController.signal,
-        connections,
-        storage: storage
-      };
-      walStream = new WalStream(options);
+        abortController = new AbortController();
+        const options: WalStreamOptions = {
+          abort_signal: abortController.signal,
+          connections,
+          storage: storage,
+          metrics: METRICS_HELPER.metricsEngine
+        };
+        walStream = new WalStream(options);
 
       await storage.clear();
 
@@ -403,7 +410,7 @@ bucket_definitions:
         getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }),
         streamPromise
       ]);
-      if (typeof checkpoint == undefined) {
+      if (checkpoint == null) {
         // This indicates an issue with the test setup - streamingPromise completed instead
         // of getClientCheckpoint()
         throw new Error('Test failure - streamingPromise completed');

package/test/src/util.ts

@@ -2,7 +2,7 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'
 import * as types from '@module/types/types.js';
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { logger } from '@powersync/lib-services-framework';
-import { BucketStorageFactory, OpId } from '@powersync/service-core';
+import { BucketStorageFactory, InternalOpId, TestStorageOptions } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
@@ -64,7 +64,7 @@ export async function getClientCheckpoint(
   db: pgwire.PgClient,
   storageFactory: BucketStorageFactory,
   options?: { timeout?: number }
-): Promise<OpId> {
+): Promise<InternalOpId> {
   const start = Date.now();
 
   const api = new PostgresRouteAPIAdapter(db);

package/test/src/wal_stream.test.ts

@@ -1,12 +1,13 @@
 import { MissingReplicationSlotError } from '@module/replication/WalStream.js';
-import { Metrics, storage } from '@powersync/service-core';
-import { putOp, removeOp } from '@powersync/service-core-tests';
+import { storage } from '@powersync/service-core';
+import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import * as crypto from 'crypto';
 import { describe, expect, test } from 'vitest';
 import { env } from './env.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
 import { WalStreamTestContext } from './wal_stream_utils.js';
+import { ReplicationMetric } from '@powersync/service-types';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:
@@ -40,9 +41,9 @@ bucket_definitions:
 
     await context.replicateSnapshot();
 
-    const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const startTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
 
     context.startStreaming();
 
@@ -55,9 +56,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })]);
-    const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const endTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
     expect(endRowCount - startRowCount).toEqual(1);
     expect(endTxCount - startTxCount).toEqual(1);
   });
@@ -77,9 +78,9 @@ bucket_definitions:
 
     await context.replicateSnapshot();
 
-    const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const startTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
 
     context.startStreaming();
 
@@ -90,9 +91,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]);
-    const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const endTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
     expect(endRowCount - startRowCount).toEqual(1);
     expect(endTxCount - startTxCount).toEqual(1);
   });
@@ -274,9 +275,9 @@ bucket_definitions:
 
     await context.replicateSnapshot();
 
-    const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const startTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
 
     context.startStreaming();
 
@@ -287,9 +288,9 @@ bucket_definitions:
     const data = await context.getBucketData('global[]');
 
     expect(data).toMatchObject([]);
-    const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0;
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0;
     const endTxCount =
-      (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0;
+      (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0;
 
     // There was a transaction, but we should not replicate any actual data
     expect(endRowCount - startRowCount).toEqual(0);

package/test/src/wal_stream_utils.ts

@@ -1,7 +1,15 @@
 import { PgManager } from '@module/replication/PgManager.js';
 import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js';
-import { BucketStorageFactory, OplogEntry, storage, SyncRulesBucketStorage } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import {
+  BucketStorageFactory,
+  createCoreReplicationMetrics,
+  initializeCoreReplicationMetrics,
+  InternalOpId,
+  OplogEntry,
+  storage,
+  SyncRulesBucketStorage
+} from '@powersync/service-core';
+import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import * as pgwire from '@powersync/service-jpgwire';
 import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
 
@@ -35,7 +43,10 @@ export class WalStreamTestContext implements AsyncDisposable {
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: PgManager
-  ) {}
+  ) {
+    createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+    initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+  }
 
   async [Symbol.asyncDispose]() {
     await this.dispose();
@@ -95,6 +106,7 @@ export class WalStreamTestContext implements AsyncDisposable {
     }
     const options: WalStreamOptions = {
       storage: this.storage,
+      metrics: METRICS_HELPER.metricsEngine,
       connections: this.connectionManager,
       abort_signal: this.abortController.signal
     };
@@ -120,27 +132,30 @@ export class WalStreamTestContext implements AsyncDisposable {
       getClientCheckpoint(this.pool, this.factory, { timeout: options?.timeout ?? 15_000 }),
       this.streamPromise
     ]);
-    if (typeof checkpoint == undefined) {
+    if (checkpoint == null) {
       // This indicates an issue with the test setup - streamingPromise completed instead
       // of getClientCheckpoint()
       throw new Error('Test failure - streamingPromise completed');
     }
-    return checkpoint as string;
+    return checkpoint;
   }
 
-  async getBucketsDataBatch(buckets: Record<string, string>, options?: { timeout?: number }) {
+  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, string>(Object.entries(buckets));
+    const map = new Map<string, InternalOpId>(Object.entries(buckets));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
   /**
    * This waits for a client checkpoint.
    */
-  async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) {
-    start ??= '0';
+  async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
     const checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, string>([[bucket, start]]);
+    const map = new Map<string, InternalOpId>([[bucket, start]]);
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -150,7 +165,7 @@ export class WalStreamTestContext implements AsyncDisposable {
       if (batches.length == 0 || !batches[0]!.batch.has_more) {
         break;
       }
-      map.set(bucket, batches[0]!.batch.next_after);
+      map.set(bucket, BigInt(batches[0]!.batch.next_after));
     }
     return data;
   }
@@ -158,10 +173,13 @@ export class WalStreamTestContext implements AsyncDisposable {
   /**
    * This does not wait for a client checkpoint.
    */
-  async getCurrentBucketData(bucket: string, start?: string) {
-    start ??= '0';
+  async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
     const { checkpoint } = await this.storage!.getCheckpoint();
-    const map = new Map<string, string>([[bucket, start]]);
+    const map = new Map<string, InternalOpId>([[bucket, start]]);
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await test_utils.fromAsync(batch);
     return batches[0]?.batch.data ?? [];