@powersync/service-module-postgres 0.4.0 → 0.5.1

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.4.0",
+  "version": "0.5.1",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -27,19 +27,19 @@
     "ts-codec": "^1.3.0",
     "uuid": "^9.0.1",
     "uri-js": "^4.4.1",
-    "@powersync/lib-services-framework": "0.4.0",
-    "@powersync/lib-service-postgres": "0.1.0",
-    "@powersync/service-core": "0.15.0",
-    "@powersync/service-jpgwire": "0.18.5",
+    "@powersync/lib-services-framework": "0.5.0",
+    "@powersync/lib-service-postgres": "0.2.0",
+    "@powersync/service-core": "0.16.1",
+    "@powersync/service-jpgwire": "0.19.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.23.1",
-    "@powersync/service-types": "0.7.0"
+    "@powersync/service-sync-rules": "0.23.3",
+    "@powersync/service-types": "0.7.1"
   },
   "devDependencies": {
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.3.0",
-    "@powersync/service-module-mongodb-storage": "0.3.0",
-    "@powersync/service-module-postgres-storage": "0.1.0"
+    "@powersync/service-core-tests": "0.3.2",
+    "@powersync/service-module-mongodb-storage": "0.3.2",
+    "@powersync/service-module-postgres-storage": "0.1.2"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/PostgresRouteAPIAdapter.ts

@@ -1,4 +1,5 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { api, ParseSyncRulesOptions } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import * as sync_rules from '@powersync/service-sync-rules';
@@ -228,7 +229,11 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
       return Number(row.lsn_distance);
     }
 
-    throw new Error(`Could not determine replication lag for slot ${slotName}`);
+    throw new ServiceError({
+      status: 500,
+      code: ErrorCode.PSYNC_S4001,
+      description: `Could not determine replication lag for slot ${slotName}`
+    });
   }
 
   async getReplicationHead(): Promise<string> {

package/src/module/PostgresModule.ts

@@ -1,4 +1,12 @@
-import { api, auth, ConfigurationFileSyncRulesProvider, modules, replication, system } from '@powersync/service-core';
+import {
+  api,
+  auth,
+  ConfigurationFileSyncRulesProvider,
+  ConnectionTestResult,
+  modules,
+  replication,
+  system
+} from '@powersync/service-core';
 import * as jpgwire from '@powersync/service-jpgwire';
 import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js';
 import { SupabaseKeyCollector } from '../auth/SupabaseKeyCollector.js';
@@ -10,6 +18,7 @@ import { PUBLICATION_NAME } from '../replication/WalStream.js';
 import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
 import * as types from '../types/types.js';
 import { PostgresConnectionConfig } from '../types/types.js';
+import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres';
 
 export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
   constructor() {
@@ -124,18 +133,26 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
       });
   }
 
-  async testConnection(config: PostgresConnectionConfig): Promise<void> {
+  async testConnection(config: PostgresConnectionConfig): Promise<ConnectionTestResult> {
     this.decodeConfig(config);
-    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
-    const connectionManager = new PgManager(normalisedConfig, {
+    const normalizedConfig = this.resolveConfig(this.decodedConfig!);
+    return await this.testConnection(normalizedConfig);
+  }
+
+  static async testConnection(normalizedConfig: NormalizedBasePostgresConnectionConfig): Promise<ConnectionTestResult> {
+    // FIXME: This is not a complete implementation yet.
+    const connectionManager = new PgManager(normalizedConfig, {
       idleTimeout: 30_000,
       maxSize: 1
     });
     const connection = await connectionManager.snapshotConnection();
     try {
-      return await checkSourceConfiguration(connection, PUBLICATION_NAME);
+      await checkSourceConfiguration(connection, PUBLICATION_NAME);
     } finally {
       await connectionManager.end();
     }
+    return {
+      connectionDescription: baseUri(normalizedConfig)
+    };
   }
 }

package/src/replication/ConnectionManagerFactory.ts

@@ -5,7 +5,7 @@ import { logger } from '@powersync/lib-services-framework';
 
 export class ConnectionManagerFactory {
   private readonly connectionManagers: PgManager[];
-  private readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
+  public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
 
   constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig) {
     this.dbConnectionConfig = dbConnectionConfig;

package/src/replication/PgManager.ts

@@ -50,6 +50,7 @@ export class PgManager {
     // for the full 6 minutes.
     // This we are constantly using the connection, we don't need any
     // custom keepalives.
+
     (connection as any)._socket.setTimeout(SNAPSHOT_SOCKET_TIMEOUT);
 
     // Disable statement timeout for snapshot queries.

package/src/replication/PgRelation.ts

@@ -1,3 +1,4 @@
+import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { PgoutputRelation } from '@powersync/service-jpgwire';
 
@@ -16,7 +17,7 @@ export function getRelId(source: PgoutputRelation): number {
   // Source types are wrong here
   const relId = (source as any).relationOid as number;
   if (!relId) {
-    throw new Error(`No relation id!`);
+    throw new ReplicationAssertionError(`No relation id found`);
   }
   return relId;
 }

package/src/replication/WalStream.ts

@@ -1,5 +1,11 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { container, errors, logger } from '@powersync/lib-services-framework';
+import {
+  container,
+  errors,
+  logger,
+  ReplicationAbortedError,
+  ReplicationAssertionError
+} from '@powersync/lib-services-framework';
 import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
 import * as pgwire from '@powersync/service-jpgwire';
 import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
@@ -133,7 +139,7 @@ export class WalStream {
     for (let row of tableRows) {
       const name = row.table_name as string;
       if (typeof row.relid != 'bigint') {
-        throw new Error(`missing relid for ${name}`);
+        throw new ReplicationAssertionError(`Missing relid for ${name}`);
       }
       const relid = Number(row.relid as bigint);
 
@@ -294,7 +300,7 @@ export class WalStream {
       }
     }
 
-    throw new Error('Unreachable');
+    throw new ReplicationAssertionError('Unreachable');
   }
 
   async estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string> {
@@ -415,7 +421,7 @@ WHERE  oid = $1::regclass`,
         lastLogIndex = at;
       }
       if (this.abort_signal.aborted) {
-        throw new Error(`Aborted initial replication of ${this.slot_name}`);
+        throw new ReplicationAbortedError(`Aborted initial replication of ${this.slot_name}`);
       }
 
       for (const record of WalStream.getQueryData(rows)) {
@@ -441,7 +447,7 @@ WHERE  oid = $1::regclass`,
 
   async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) {
     if (!descriptor.objectId && typeof descriptor.objectId != 'number') {
-      throw new Error('objectId expected');
+      throw new ReplicationAssertionError(`objectId expected, got ${typeof descriptor.objectId}`);
     }
     const result = await this.storage.resolveTable({
       group_id: this.group_id,
@@ -484,6 +490,7 @@ WHERE  oid = $1::regclass`,
           await db.query('COMMIT');
         } catch (e) {
           await db.query('ROLLBACK');
+          // TODO: Wrap with custom error type
           throw e;
         }
       } finally {
@@ -501,7 +508,7 @@ WHERE  oid = $1::regclass`,
     if (table == null) {
       // We should always receive a replication message before the relation is used.
       // If we can't find it, it's a bug.
-      throw new Error(`Missing relation cache for ${relationId}`);
+      throw new ReplicationAssertionError(`Missing relation cache for ${relationId}`);
     }
     return table;
   }

package/src/replication/WalStreamReplicator.ts

@@ -2,6 +2,7 @@ import { replication, storage } from '@powersync/service-core';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
 import { cleanUpReplicationSlot } from './replication-utils.js';
 import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
+import { PostgresModule } from '../module/PostgresModule.js';
 
 export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;
@@ -42,4 +43,8 @@ export class WalStreamReplicator extends replication.AbstractReplicator<WalStrea
     await super.stop();
     await this.connectionFactory.shutdown();
   }
+
+  async testConnection() {
+    return await PostgresModule.testConnection(this.connectionFactory.dbConnectionConfig);
+  }
 }

package/src/replication/replication-utils.ts

@@ -1,7 +1,7 @@
 import * as pgwire from '@powersync/service-jpgwire';
 
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { PatternResult, storage } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
@@ -117,17 +117,22 @@ $$ LANGUAGE plpgsql;`
   });
   const row = pgwire.pgwireRows(rs)[0];
   if (row == null) {
-    throw new Error(
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1141,
       `Publication '${publicationName}' does not exist. Run: \`CREATE PUBLICATION ${publicationName} FOR ALL TABLES\`, or read the documentation for details.`
     );
   }
   if (row.pubinsert == false || row.pubupdate == false || row.pubdelete == false || row.pubtruncate == false) {
-    throw new Error(
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1142,
       `Publication '${publicationName}' does not publish all changes. Create a publication using \`WITH (publish = "insert, update, delete, truncate")\` (the default).`
     );
   }
   if (row.pubviaroot) {
-    throw new Error(`'${publicationName}' uses publish_via_partition_root, which is not supported.`);
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1143,
+      `'${publicationName}' uses publish_via_partition_root, which is not supported.`
+    );
   }
 }
 

package/src/utils/migration_lib.ts

@@ -1,3 +1,4 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import * as pgwire from '@powersync/service-jpgwire';
 
 export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
@@ -14,7 +15,7 @@ export class Migrations {
 
   add(id: number, name: string, up: MigrationFunction) {
     if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
-      throw new Error('Migration ids must be strictly incrementing');
+      throw new ServiceAssertionError('Migration ids must be strictly incrementing');
     }
     this.migrations.push({ id, up, name });
   }

package/src/utils/pgwire_utils.ts

@@ -1,7 +1,7 @@
 // Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
 
 import * as pgwire from '@powersync/service-jpgwire';
-import { SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
+import { DatabaseInputRow, SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
 
 /**
  * pgwire message -> SQLite row.
@@ -10,7 +10,7 @@ import { SqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
 export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteRow {
   const rawData = (message as any).afterRaw;
 
-  const record = pgwire.decodeTuple(message.relation, rawData);
+  const record = decodeTuple(message.relation, rawData);
   return toSyncRulesRow(record);
 }
 
@@ -23,6 +23,24 @@ export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.Pg
   if (rawData == null) {
     return undefined;
   }
-  const record = pgwire.decodeTuple(message.relation, rawData);
+  const record = decodeTuple(message.relation, rawData);
   return toSyncRulesRow(record);
 }
+
+/**
+ * We need a high level of control over how values are decoded, to make sure there is no loss
+ * of precision in the process.
+ */
+export function decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record<string, any>): DatabaseInputRow {
+  let result: Record<string, any> = {};
+  for (let columnName in tupleRaw) {
+    const rawval = tupleRaw[columnName];
+    const typeOid = (relation as any)._tupleDecoder._typeOids.get(columnName);
+    if (typeof rawval == 'string' && typeOid) {
+      result[columnName] = pgwire.PgType.decode(rawval, typeOid);
+    } else {
+      result[columnName] = rawval;
+    }
+  }
+  return result;
+}