@powersync/service-module-postgres 0.0.0-dev-20241001150444 → 0.0.0-dev-20241002180742

package/src/replication/WalStream.ts

@@ -196,15 +196,22 @@ export class WalStream {
           // We peek a large number of changes here, to make it more likely to pick up replication slot errors.
           // For example, "publication does not exist" only occurs here if the peek actually includes changes related
           // to the slot.
-          await this.connections.pool.query({
-            statement: `SELECT *
-                    FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1',
-                                                                        'publication_names', $2)`,
+          logger.info(`Checking ${slotName}`);
+
+          // The actual results can be quite large, so we don't actually return everything
+          // due to memory and processing overhead that would create.
+          const cursor = await this.connections.pool.stream({
+            statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
             params: [
               { type: 'varchar', value: slotName },
               { type: 'varchar', value: PUBLICATION_NAME }
             ]
           });
+
+          for await (let _chunk of cursor) {
+            // No-op, just exhaust the cursor
+          }
+
           // Success
           logger.info(`Slot ${slotName} appears healthy`);
           return { needsInitialSync: false };

package/src/utils/populate_test_data.ts

@@ -1,5 +1,4 @@
-import * as crypto from 'crypto';
-import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
+import { Worker } from 'node:worker_threads';
 
 import * as pgwire from '@powersync/service-jpgwire';
 
@@ -12,49 +11,10 @@ export interface PopulateDataOptions {
   size: number;
 }
 
-if (isMainThread || parentPort == null) {
-  // Not a worker - ignore
-} else {
-  try {
-    const options = workerData as PopulateDataOptions;
-
-    const result = await populateDataInner(options);
-    parentPort.postMessage(result);
-    process.exit(0);
-  } catch (e) {
-    // This is a bug, not a connection issue
-    console.error(e);
-    // Only closes the Worker thread
-    process.exit(2);
-  }
-}
-
-async function populateDataInner(options: PopulateDataOptions) {
-  // Dedicated connection so we can release the memory easily
-  const initialDb = await pgwire.connectPgWire(options.connection, { type: 'standard' });
-  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
-  let operation_count = 0;
-  for (let i = 0; i < options.num_transactions; i++) {
-    const prefix = `test${i}K`;
-
-    await initialDb.query({
-      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
-      params: [
-        { type: 'varchar', value: prefix },
-        { type: 'varchar', value: largeDescription },
-        { type: 'int4', value: options.per_transaction }
-      ]
-    });
-    operation_count += options.per_transaction;
-  }
-  await initialDb.end();
-  return operation_count;
-}
-
 export async function populateData(options: PopulateDataOptions) {
   const WORKER_TIMEOUT = 30_000;
 
-  const worker = new Worker(new URL('./populate_test_data.js', import.meta.url), {
+  const worker = new Worker(new URL('./populate_test_data_worker.js', import.meta.url), {
     workerData: options
   });
   const timeout = setTimeout(() => {

package/src/utils/populate_test_data_worker.ts

@@ -0,0 +1,50 @@
+import * as crypto from 'crypto';
+import { isMainThread, parentPort, workerData } from 'node:worker_threads';
+
+import * as pgwire from '@powersync/service-jpgwire';
+import type { PopulateDataOptions } from './populate_test_data.js';
+
+// This util is actually for tests only, but we need it compiled to JS for the service to work, so it's placed in the service.
+
+if (isMainThread || parentPort == null) {
+  // Must not be imported - only expected to run in a worker
+  throw new Error('Do not import this file');
+} else {
+  try {
+    const options = workerData as PopulateDataOptions;
+    if (options == null) {
+      throw new Error('loaded worker without options');
+    }
+
+    const result = await populateDataInner(options);
+    parentPort.postMessage(result);
+    process.exit(0);
+  } catch (e) {
+    // This is a bug, not a connection issue
+    console.error(e);
+    // Only closes the Worker thread
+    process.exit(2);
+  }
+}
+
+async function populateDataInner(options: PopulateDataOptions) {
+  // Dedicated connection so we can release the memory easily
+  const initialDb = await pgwire.connectPgWire(options.connection, { type: 'standard' });
+  const largeDescription = crypto.randomBytes(options.size / 2).toString('hex');
+  let operation_count = 0;
+  for (let i = 0; i < options.num_transactions; i++) {
+    const prefix = `test${i}K`;
+
+    await initialDb.query({
+      statement: `INSERT INTO test_data(id, description, other) SELECT $1 || i, $2, 'foo' FROM generate_series(1, $3) i`,
+      params: [
+        { type: 'varchar', value: prefix },
+        { type: 'varchar', value: largeDescription },
+        { type: 'int4', value: options.per_transaction }
+      ]
+    });
+    operation_count += options.per_transaction;
+  }
+  await initialDb.end();
+  return operation_count;
+}

package/test/src/wal_stream_utils.ts

@@ -36,7 +36,10 @@ export class WalStreamTestContext {
   public storage?: SyncRulesBucketStorage;
   private replicationConnection?: pgwire.PgConnection;
 
-  constructor(public factory: BucketStorageFactory, public connectionManager: PgManager) {}
+  constructor(
+    public factory: BucketStorageFactory,
+    public connectionManager: PgManager
+  ) {}
 
   async dispose() {
     this.abortController.abort();