@powersync/service-module-postgres 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +67 -11
  2. package/dist/api/PostgresRouteAPIAdapter.d.ts +2 -1
  3. package/dist/api/PostgresRouteAPIAdapter.js +22 -10
  4. package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
  5. package/dist/auth/SupabaseKeyCollector.js +6 -5
  6. package/dist/auth/SupabaseKeyCollector.js.map +1 -1
  7. package/dist/module/PostgresModule.d.ts +4 -2
  8. package/dist/module/PostgresModule.js +13 -5
  9. package/dist/module/PostgresModule.js.map +1 -1
  10. package/dist/replication/ConnectionManagerFactory.d.ts +1 -1
  11. package/dist/replication/ConnectionManagerFactory.js +2 -0
  12. package/dist/replication/ConnectionManagerFactory.js.map +1 -1
  13. package/dist/replication/PgManager.d.ts +5 -0
  14. package/dist/replication/PgManager.js +17 -2
  15. package/dist/replication/PgManager.js.map +1 -1
  16. package/dist/replication/PgRelation.js +2 -1
  17. package/dist/replication/PgRelation.js.map +1 -1
  18. package/dist/replication/PostgresErrorRateLimiter.js +5 -7
  19. package/dist/replication/PostgresErrorRateLimiter.js.map +1 -1
  20. package/dist/replication/WalStream.d.ts +18 -3
  21. package/dist/replication/WalStream.js +133 -22
  22. package/dist/replication/WalStream.js.map +1 -1
  23. package/dist/replication/WalStreamReplicationJob.js +7 -5
  24. package/dist/replication/WalStreamReplicationJob.js.map +1 -1
  25. package/dist/replication/WalStreamReplicator.d.ts +1 -0
  26. package/dist/replication/WalStreamReplicator.js +6 -1
  27. package/dist/replication/WalStreamReplicator.js.map +1 -1
  28. package/dist/replication/replication-utils.js +4 -4
  29. package/dist/replication/replication-utils.js.map +1 -1
  30. package/dist/utils/migration_lib.js +3 -4
  31. package/dist/utils/migration_lib.js.map +1 -1
  32. package/dist/utils/populate_test_data.js +1 -1
  33. package/dist/utils/populate_test_data.js.map +1 -1
  34. package/package.json +14 -12
  35. package/src/api/PostgresRouteAPIAdapter.ts +19 -9
  36. package/src/module/PostgresModule.ts +22 -5
  37. package/src/replication/ConnectionManagerFactory.ts +1 -1
  38. package/src/replication/PgManager.ts +10 -0
  39. package/src/replication/PgRelation.ts +2 -1
  40. package/src/replication/WalStream.ts +160 -26
  41. package/src/replication/WalStreamReplicationJob.ts +3 -3
  42. package/src/replication/WalStreamReplicator.ts +5 -0
  43. package/src/replication/replication-utils.ts +9 -4
  44. package/src/utils/migration_lib.ts +2 -1
  45. package/test/src/checkpoints.test.ts +70 -0
  46. package/test/src/storage_combination.test.ts +35 -0
  47. package/test/src/util.ts +1 -1
  48. package/test/src/wal_stream_utils.ts +1 -1
  49. package/tsconfig.tsbuildinfo +1 -1
package/package.json CHANGED
@@ -5,7 +5,7 @@
5
5
  "publishConfig": {
6
6
  "access": "public"
7
7
  },
8
- "version": "0.0.0-dev-20250117095455",
8
+ "version": "0.0.0-dev-20250214100224",
9
9
  "main": "dist/index.js",
10
10
  "license": "FSL-1.1-Apache-2.0",
11
11
  "type": "module",
@@ -22,24 +22,26 @@
22
22
  }
23
23
  },
24
24
  "dependencies": {
25
- "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
26
25
  "jose": "^4.15.1",
26
+ "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
27
+ "semver": "^7.5.4",
27
28
  "ts-codec": "^1.3.0",
28
- "uuid": "^9.0.1",
29
29
  "uri-js": "^4.4.1",
30
- "@powersync/lib-services-framework": "0.0.0-dev-20250117095455",
31
- "@powersync/lib-service-postgres": "0.0.0-dev-20250117095455",
32
- "@powersync/service-core": "0.0.0-dev-20250117095455",
33
- "@powersync/service-jpgwire": "0.0.0-dev-20250117095455",
30
+ "uuid": "^9.0.1",
31
+ "@powersync/lib-service-postgres": "0.3.1",
32
+ "@powersync/lib-services-framework": "0.5.1",
33
+ "@powersync/service-core": "0.0.0-dev-20250214100224",
34
+ "@powersync/service-jpgwire": "0.19.0",
34
35
  "@powersync/service-jsonbig": "0.17.10",
35
- "@powersync/service-sync-rules": "0.23.1",
36
- "@powersync/service-types": "0.0.0-dev-20250117095455"
36
+ "@powersync/service-sync-rules": "0.23.4",
37
+ "@powersync/service-types": "0.8.0"
37
38
  },
38
39
  "devDependencies": {
40
+ "@types/semver": "^7.5.4",
39
41
  "@types/uuid": "^9.0.4",
40
- "@powersync/service-core-tests": "0.0.0-dev-20250117095455",
41
- "@powersync/service-module-mongodb-storage": "0.0.0-dev-20250117095455",
42
- "@powersync/service-module-postgres-storage": "0.0.0-dev-20250117095455"
42
+ "@powersync/service-core-tests": "0.0.0-dev-20250214100224",
43
+ "@powersync/service-module-mongodb-storage": "0.0.0-dev-20250214100224",
44
+ "@powersync/service-module-postgres-storage": "0.0.0-dev-20250214100224"
43
45
  },
44
46
  "scripts": {
45
47
  "build": "tsc -b",
@@ -1,11 +1,12 @@
1
1
  import * as lib_postgres from '@powersync/lib-service-postgres';
2
- import { api, ParseSyncRulesOptions } from '@powersync/service-core';
2
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
+ import { api, ParseSyncRulesOptions, ReplicationHeadCallback } from '@powersync/service-core';
3
4
  import * as pgwire from '@powersync/service-jpgwire';
4
5
  import * as sync_rules from '@powersync/service-sync-rules';
5
6
  import * as service_types from '@powersync/service-types';
6
7
  import * as replication_utils from '../replication/replication-utils.js';
7
8
  import { getDebugTableInfo } from '../replication/replication-utils.js';
8
- import { PUBLICATION_NAME } from '../replication/WalStream.js';
9
+ import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
9
10
  import * as types from '../types/types.js';
10
11
 
11
12
  export class PostgresRouteAPIAdapter implements api.RouteAPI {
@@ -228,7 +229,11 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
228
229
  return Number(row.lsn_distance);
229
230
  }
230
231
 
231
- throw new Error(`Could not determine replication lag for slot ${slotName}`);
232
+ throw new ServiceError({
233
+ status: 500,
234
+ code: ErrorCode.PSYNC_S4001,
235
+ description: `Could not determine replication lag for slot ${slotName}`
236
+ });
232
237
  }
233
238
 
234
239
  async getReplicationHead(): Promise<string> {
@@ -236,17 +241,22 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
236
241
  // However, on Aurora (Postgres compatible), it can return an entirely different LSN,
237
242
  // causing the write checkpoints to never be replicated back to the client.
238
243
  // For those, we need to use pg_current_wal_lsn() instead.
239
- const { results } = await lib_postgres.retriedQuery(
240
- this.pool,
241
- { statement: `SELECT pg_current_wal_lsn() as lsn` },
242
- { statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` }
243
- );
244
+ const { results } = await lib_postgres.retriedQuery(this.pool, `SELECT pg_current_wal_lsn() as lsn`);
244
245
 
245
- // Specifically use the lsn from the first statement, not the second one.
246
246
  const lsn = results[0].rows[0][0];
247
247
  return String(lsn);
248
248
  }
249
249
 
250
+ async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
251
+ const currentLsn = await this.getReplicationHead();
252
+
253
+ const r = await callback(currentLsn);
254
+
255
+ await lib_postgres.retriedQuery(this.pool, KEEPALIVE_STATEMENT);
256
+
257
+ return r;
258
+ }
259
+
250
260
  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
251
261
  // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
252
262
  const results = await lib_postgres.retriedQuery(
@@ -1,4 +1,12 @@
1
- import { api, auth, ConfigurationFileSyncRulesProvider, modules, replication, system } from '@powersync/service-core';
1
+ import {
2
+ api,
3
+ auth,
4
+ ConfigurationFileSyncRulesProvider,
5
+ ConnectionTestResult,
6
+ modules,
7
+ replication,
8
+ system
9
+ } from '@powersync/service-core';
2
10
  import * as jpgwire from '@powersync/service-jpgwire';
3
11
  import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js';
4
12
  import { SupabaseKeyCollector } from '../auth/SupabaseKeyCollector.js';
@@ -10,6 +18,7 @@ import { PUBLICATION_NAME } from '../replication/WalStream.js';
10
18
  import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
11
19
  import * as types from '../types/types.js';
12
20
  import { PostgresConnectionConfig } from '../types/types.js';
21
+ import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres';
13
22
 
14
23
  export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
15
24
  constructor() {
@@ -124,18 +133,26 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
124
133
  });
125
134
  }
126
135
 
127
- async testConnection(config: PostgresConnectionConfig): Promise<void> {
136
+ async testConnection(config: PostgresConnectionConfig): Promise<ConnectionTestResult> {
128
137
  this.decodeConfig(config);
129
- const normalisedConfig = this.resolveConfig(this.decodedConfig!);
130
- const connectionManager = new PgManager(normalisedConfig, {
138
+ const normalizedConfig = this.resolveConfig(this.decodedConfig!);
139
+ return await this.testConnection(normalizedConfig);
140
+ }
141
+
142
+ static async testConnection(normalizedConfig: NormalizedBasePostgresConnectionConfig): Promise<ConnectionTestResult> {
143
+ // FIXME: This is not a complete implementation yet.
144
+ const connectionManager = new PgManager(normalizedConfig, {
131
145
  idleTimeout: 30_000,
132
146
  maxSize: 1
133
147
  });
134
148
  const connection = await connectionManager.snapshotConnection();
135
149
  try {
136
- return await checkSourceConfiguration(connection, PUBLICATION_NAME);
150
+ await checkSourceConfiguration(connection, PUBLICATION_NAME);
137
151
  } finally {
138
152
  await connectionManager.end();
139
153
  }
154
+ return {
155
+ connectionDescription: baseUri(normalizedConfig)
156
+ };
140
157
  }
141
158
  }
@@ -5,7 +5,7 @@ import { logger } from '@powersync/lib-services-framework';
5
5
 
6
6
  export class ConnectionManagerFactory {
7
7
  private readonly connectionManagers: PgManager[];
8
- private readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
8
+ public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
9
9
 
10
10
  constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig) {
11
11
  this.dbConnectionConfig = dbConnectionConfig;
@@ -1,4 +1,5 @@
1
1
  import * as pgwire from '@powersync/service-jpgwire';
2
+ import semver from 'semver';
2
3
  import { NormalizedPostgresConnectionConfig } from '../types/types.js';
3
4
 
4
5
  /**
@@ -35,6 +36,15 @@ export class PgManager {
35
36
  return await p;
36
37
  }
37
38
 
39
+ /**
40
+ * @returns The Postgres server version in a parsed Semver instance
41
+ */
42
+ async getServerVersion(): Promise<semver.SemVer | null> {
43
+ const result = await this.pool.query(`SHOW server_version;`);
44
+ // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
45
+ return semver.coerce(result.rows[0][0].split(' ')[0]);
46
+ }
47
+
38
48
  /**
39
49
  * Create a new standard connection, used for initial snapshot.
40
50
  *
@@ -1,3 +1,4 @@
1
+ import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
1
2
  import { storage } from '@powersync/service-core';
2
3
  import { PgoutputRelation } from '@powersync/service-jpgwire';
3
4
 
@@ -16,7 +17,7 @@ export function getRelId(source: PgoutputRelation): number {
16
17
  // Source types are wrong here
17
18
  const relId = (source as any).relationOid as number;
18
19
  if (!relId) {
19
- throw new Error(`No relation id!`);
20
+ throw new ReplicationAssertionError(`No relation id found`);
20
21
  }
21
22
  return relId;
22
23
  }
@@ -1,5 +1,13 @@
1
1
  import * as lib_postgres from '@powersync/lib-service-postgres';
2
- import { container, errors, logger } from '@powersync/lib-services-framework';
2
+ import {
3
+ container,
4
+ DatabaseConnectionError,
5
+ ErrorCode,
6
+ errors,
7
+ logger,
8
+ ReplicationAbortedError,
9
+ ReplicationAssertionError
10
+ } from '@powersync/lib-services-framework';
3
11
  import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
4
12
  import * as pgwire from '@powersync/service-jpgwire';
5
13
  import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
@@ -9,10 +17,6 @@ import { PgManager } from './PgManager.js';
9
17
  import { getPgOutputRelation, getRelId } from './PgRelation.js';
10
18
  import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js';
11
19
 
12
- export const ZERO_LSN = '00000000/00000000';
13
- export const PUBLICATION_NAME = 'powersync';
14
- export const POSTGRES_DEFAULT_SCHEMA = 'public';
15
-
16
20
  export interface WalStreamOptions {
17
21
  connections: PgManager;
18
22
  storage: storage.SyncRulesBucketStorage;
@@ -26,6 +30,35 @@ interface InitResult {
26
30
  needsNewSlot: boolean;
27
31
  }
28
32
 
33
+ export const ZERO_LSN = '00000000/00000000';
34
+ export const PUBLICATION_NAME = 'powersync';
35
+ export const POSTGRES_DEFAULT_SCHEMA = 'public';
36
+
37
+ export const KEEPALIVE_CONTENT = 'ping';
38
+ export const KEEPALIVE_BUFFER = Buffer.from(KEEPALIVE_CONTENT);
39
+ export const KEEPALIVE_STATEMENT: pgwire.Statement = {
40
+ statement: /* sql */ `
41
+ SELECT
42
+ *
43
+ FROM
44
+ pg_logical_emit_message(FALSE, 'powersync', $1)
45
+ `,
46
+ params: [{ type: 'varchar', value: KEEPALIVE_CONTENT }]
47
+ } as const;
48
+
49
+ export const isKeepAliveMessage = (msg: pgwire.PgoutputMessage) => {
50
+ return (
51
+ msg.tag == 'message' &&
52
+ msg.prefix == 'powersync' &&
53
+ msg.content &&
54
+ Buffer.from(msg.content).equals(KEEPALIVE_BUFFER)
55
+ );
56
+ };
57
+
58
+ export const sendKeepAlive = async (db: pgwire.PgClient) => {
59
+ await lib_postgres.retriedQuery(db, KEEPALIVE_STATEMENT);
60
+ };
61
+
29
62
  export class MissingReplicationSlotError extends Error {
30
63
  constructor(message: string) {
31
64
  super(message);
@@ -65,10 +98,7 @@ export class WalStream {
65
98
  // Ping to speed up cancellation of streaming replication
66
99
  // We're not using pg_snapshot here, since it could be in the middle of
67
100
  // an initial replication transaction.
68
- const promise = lib_postgres.retriedQuery(
69
- this.connections.pool,
70
- `SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`
71
- );
101
+ const promise = sendKeepAlive(this.connections.pool);
72
102
  promise.catch((e) => {
73
103
  // Failures here are okay - this only speeds up stopping the process.
74
104
  logger.warn('Failed to ping connection', e);
@@ -133,7 +163,7 @@ export class WalStream {
133
163
  for (let row of tableRows) {
134
164
  const name = row.table_name as string;
135
165
  if (typeof row.relid != 'bigint') {
136
- throw new Error(`missing relid for ${name}`);
166
+ throw new ReplicationAssertionError(`Missing relid for ${name}`);
137
167
  }
138
168
  const relid = Number(row.relid as bigint);
139
169
 
@@ -174,6 +204,7 @@ export class WalStream {
174
204
 
175
205
  async initSlot(): Promise<InitResult> {
176
206
  await checkSourceConfiguration(this.connections.pool, PUBLICATION_NAME);
207
+ await this.ensureStorageCompatibility();
177
208
 
178
209
  const slotName = this.slot_name;
179
210
 
@@ -294,7 +325,7 @@ export class WalStream {
294
325
  }
295
326
  }
296
327
 
297
- throw new Error('Unreachable');
328
+ throw new ReplicationAssertionError('Unreachable');
298
329
  }
299
330
 
300
331
  async estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string> {
@@ -376,6 +407,15 @@ WHERE oid = $1::regclass`,
376
407
  await batch.commit(ZERO_LSN);
377
408
  }
378
409
  );
410
+ /**
411
+ * Send a keepalive message after initial replication.
412
+ * In some edge cases we wait for a keepalive after the initial snapshot.
413
+ * If we don't explicitly check the contents of keepalive messages then a keepalive is detected
414
+ * rather quickly after initial replication - perhaps due to other WAL events.
415
+ * If we do explicitly check the contents of messages, we need an actual keepalive payload in order
416
+ * to advance the active sync rules LSN.
417
+ */
418
+ await sendKeepAlive(db);
379
419
  }
380
420
 
381
421
  static *getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow> {
@@ -415,7 +455,7 @@ WHERE oid = $1::regclass`,
415
455
  lastLogIndex = at;
416
456
  }
417
457
  if (this.abort_signal.aborted) {
418
- throw new Error(`Aborted initial replication of ${this.slot_name}`);
458
+ throw new ReplicationAbortedError(`Aborted initial replication of ${this.slot_name}`);
419
459
  }
420
460
 
421
461
  for (const record of WalStream.getQueryData(rows)) {
@@ -441,7 +481,7 @@ WHERE oid = $1::regclass`,
441
481
 
442
482
  async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) {
443
483
  if (!descriptor.objectId && typeof descriptor.objectId != 'number') {
444
- throw new Error('objectId expected');
484
+ throw new ReplicationAssertionError(`objectId expected, got ${typeof descriptor.objectId}`);
445
485
  }
446
486
  const result = await this.storage.resolveTable({
447
487
  group_id: this.group_id,
@@ -484,6 +524,7 @@ WHERE oid = $1::regclass`,
484
524
  await db.query('COMMIT');
485
525
  } catch (e) {
486
526
  await db.query('ROLLBACK');
527
+ // TODO: Wrap with custom error type
487
528
  throw e;
488
529
  }
489
530
  } finally {
@@ -501,7 +542,7 @@ WHERE oid = $1::regclass`,
501
542
  if (table == null) {
502
543
  // We should always receive a replication message before the relation is used.
503
544
  // If we can't find it, it's a bug.
504
- throw new Error(`Missing relation cache for ${relationId}`);
545
+ throw new ReplicationAssertionError(`Missing relation cache for ${relationId}`);
505
546
  }
506
547
  return table;
507
548
  }
@@ -592,13 +633,33 @@ WHERE oid = $1::regclass`,
592
633
  async streamChanges(replicationConnection: pgwire.PgConnection) {
593
634
  // When changing any logic here, check /docs/wal-lsns.md.
594
635
 
636
+ const { createEmptyCheckpoints } = await this.ensureStorageCompatibility();
637
+
638
+ const replicationOptions: Record<string, string> = {
639
+ proto_version: '1',
640
+ publication_names: PUBLICATION_NAME
641
+ };
642
+
643
+ /**
644
+ * Viewing the contents of logical messages emitted with `pg_logical_emit_message`
645
+ * is only supported on Postgres >= 14.0.
646
+ * https://www.postgresql.org/docs/14/protocol-logical-replication.html
647
+ */
648
+ const exposesLogicalMessages = await this.checkLogicalMessageSupport();
649
+ if (exposesLogicalMessages) {
650
+ /**
651
+ * Only add this option if the Postgres server supports it.
652
+ * Adding the option to a server that doesn't support it will throw an exception when starting logical replication.
653
+ * Error: `unrecognized pgoutput option: messages`
654
+ */
655
+ replicationOptions['messages'] = 'true';
656
+ }
657
+
595
658
  const replicationStream = replicationConnection.logicalReplication({
596
659
  slot: this.slot_name,
597
- options: {
598
- proto_version: '1',
599
- publication_names: PUBLICATION_NAME
600
- }
660
+ options: replicationOptions
601
661
  });
662
+
602
663
  this.startedStreaming = true;
603
664
 
604
665
  // Auto-activate as soon as initial replication is done
@@ -621,6 +682,15 @@ WHERE oid = $1::regclass`,
621
682
  // chunkLastLsn may come from normal messages in the chunk,
622
683
  // or from a PrimaryKeepalive message.
623
684
  const { messages, lastLsn: chunkLastLsn } = chunk;
685
+
686
+ /**
687
+ * We can check if an explicit keepalive was sent if `exposesLogicalMessages == true`.
688
+ * If we can't check the logical messages, we should assume a keepalive if we
689
+ * receive an empty array of messages in a replication event.
690
+ */
691
+ const assumeKeepAlive = !exposesLogicalMessages;
692
+ let keepAliveDetected = false;
693
+
624
694
  for (const msg of messages) {
625
695
  if (msg.tag == 'relation') {
626
696
  await this.handleRelation(batch, getPgOutputRelation(msg), true);
@@ -629,27 +699,44 @@ WHERE oid = $1::regclass`,
629
699
  } else if (msg.tag == 'commit') {
630
700
  Metrics.getInstance().transactions_replicated_total.add(1);
631
701
  inTx = false;
632
- await batch.commit(msg.lsn!);
702
+ await batch.commit(msg.lsn!, { createEmptyCheckpoints });
633
703
  await this.ack(msg.lsn!, replicationStream);
634
704
  } else {
635
705
  if (count % 100 == 0) {
636
706
  logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`);
637
707
  }
638
708
 
709
+ /**
710
+ * If we can see the contents of logical messages, then we can check if a keepalive
711
+ * message is present. We only perform a keepalive (below) if we explicitly detect a keepalive message.
712
+ * If we can't see the contents of logical messages, then we should assume a keepalive is required
713
+ * due to the default value of `assumeKeepalive`.
714
+ */
715
+ if (exposesLogicalMessages && isKeepAliveMessage(msg)) {
716
+ keepAliveDetected = true;
717
+ }
718
+
639
719
  count += 1;
640
720
  await this.writeChange(batch, msg);
641
721
  }
642
722
  }
643
723
 
644
724
  if (!inTx) {
645
- // In a transaction, we ack and commit according to the transaction progress.
646
- // Outside transactions, we use the PrimaryKeepalive messages to advance progress.
647
- // Big caveat: This _must not_ be used to skip individual messages, since this LSN
648
- // may be in the middle of the next transaction.
649
- // It must only be used to associate checkpoints with LSNs.
650
- if (await batch.keepalive(chunkLastLsn)) {
651
- await this.ack(chunkLastLsn, replicationStream);
725
+ if (assumeKeepAlive || keepAliveDetected) {
726
+ // Reset the detection flag.
727
+ keepAliveDetected = false;
728
+
729
+ // In a transaction, we ack and commit according to the transaction progress.
730
+ // Outside transactions, we use the PrimaryKeepalive messages to advance progress.
731
+ // Big caveat: This _must not_ be used to skip individual messages, since this LSN
732
+ // may be in the middle of the next transaction.
733
+ // It must only be used to associate checkpoints with LSNs.
734
+ await batch.keepalive(chunkLastLsn);
652
735
  }
736
+
737
+ // We receive chunks with empty messages often (about each second).
738
+ // Acknowledging here progresses the slot past these and frees up resources.
739
+ await this.ack(chunkLastLsn, replicationStream);
653
740
  }
654
741
 
655
742
  Metrics.getInstance().chunks_replicated_total.add(1);
@@ -665,6 +752,53 @@ WHERE oid = $1::regclass`,
665
752
 
666
753
  replicationStream.ack(lsn);
667
754
  }
755
+
756
+ /**
757
+ * Ensures that the storage is compatible with the replication connection.
758
+ * @throws {DatabaseConnectionError} If the storage is not compatible with the replication connection.
759
+ */
760
+ protected async ensureStorageCompatibility(): Promise<storage.ResolvedBucketBatchCommitOptions> {
761
+ const supportsLogicalMessages = await this.checkLogicalMessageSupport();
762
+
763
+ const storageIdentifier = await this.storage.factory.getSystemIdentifier();
764
+ if (storageIdentifier.type != lib_postgres.POSTGRES_CONNECTION_TYPE) {
765
+ return {
766
+ // Keep the same behaviour as before allowing Postgres storage.
767
+ createEmptyCheckpoints: true
768
+ };
769
+ }
770
+
771
+ const parsedStorageIdentifier = lib_postgres.utils.decodePostgresSystemIdentifier(storageIdentifier.id);
772
+ /**
773
+ * Check if the same server is being used for both the sync bucket storage and the logical replication.
774
+ */
775
+ const replicationIdentifier = await lib_postgres.utils.queryPostgresSystemIdentifier(this.connections.pool);
776
+
777
+ if (!supportsLogicalMessages && replicationIdentifier.server_id == parsedStorageIdentifier.server_id) {
778
+ throw new DatabaseConnectionError(
779
+ ErrorCode.PSYNC_S1144,
780
+ `Separate Postgres servers are required for the replication source and sync bucket storage when using Postgres versions below 14.0.`,
781
+ new Error('Postgres version is below 14')
782
+ );
783
+ }
784
+
785
+ return {
786
+ /**
787
+ * Don't create empty checkpoints if the same Postgres database is used for the data source
788
+ * and sync bucket storage. Creating empty checkpoints will cause WAL feedback loops.
789
+ */
790
+ createEmptyCheckpoints: replicationIdentifier.database_name != parsedStorageIdentifier.database_name
791
+ };
792
+ }
793
+
794
+ /**
795
+ * Check if the replication connection Postgres server supports
796
+ * viewing the contents of logical replication messages.
797
+ */
798
+ protected async checkLogicalMessageSupport() {
799
+ const version = await this.connections.getServerVersion();
800
+ return version ? version.compareMain('14.0.0') >= 0 : false;
801
+ }
668
802
  }
669
803
 
670
804
  async function touch() {
@@ -1,6 +1,6 @@
1
1
  import { container } from '@powersync/lib-services-framework';
2
2
  import { PgManager } from './PgManager.js';
3
- import { MissingReplicationSlotError, WalStream } from './WalStream.js';
3
+ import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
4
4
 
5
5
  import { replication } from '@powersync/service-core';
6
6
  import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
@@ -37,7 +37,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
37
37
  */
38
38
  async keepAlive() {
39
39
  try {
40
- await this.connectionManager.pool.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
40
+ await sendKeepAlive(this.connectionManager.pool);
41
41
  } catch (e) {
42
42
  this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
43
43
  }
@@ -99,7 +99,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
99
99
  });
100
100
  await stream.replicate();
101
101
  } catch (e) {
102
- this.logger.error(`Replication error`, e);
102
+ this.logger.error(`${this.slotName} Replication error`, e);
103
103
  if (e.cause != null) {
104
104
  // Example:
105
105
  // PgError.conn_ended: Unable to do postgres query on ended connection
@@ -2,6 +2,7 @@ import { replication, storage } from '@powersync/service-core';
2
2
  import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
3
3
  import { cleanUpReplicationSlot } from './replication-utils.js';
4
4
  import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
5
+ import { PostgresModule } from '../module/PostgresModule.js';
5
6
 
6
7
  export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
7
8
  connectionFactory: ConnectionManagerFactory;
@@ -42,4 +43,8 @@ export class WalStreamReplicator extends replication.AbstractReplicator<WalStrea
42
43
  await super.stop();
43
44
  await this.connectionFactory.shutdown();
44
45
  }
46
+
47
+ async testConnection() {
48
+ return await PostgresModule.testConnection(this.connectionFactory.dbConnectionConfig);
49
+ }
45
50
  }
@@ -1,7 +1,7 @@
1
1
  import * as pgwire from '@powersync/service-jpgwire';
2
2
 
3
3
  import * as lib_postgres from '@powersync/lib-service-postgres';
4
- import { logger } from '@powersync/lib-services-framework';
4
+ import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
5
5
  import { PatternResult, storage } from '@powersync/service-core';
6
6
  import * as sync_rules from '@powersync/service-sync-rules';
7
7
  import * as service_types from '@powersync/service-types';
@@ -117,17 +117,22 @@ $$ LANGUAGE plpgsql;`
117
117
  });
118
118
  const row = pgwire.pgwireRows(rs)[0];
119
119
  if (row == null) {
120
- throw new Error(
120
+ throw new ServiceError(
121
+ ErrorCode.PSYNC_S1141,
121
122
  `Publication '${publicationName}' does not exist. Run: \`CREATE PUBLICATION ${publicationName} FOR ALL TABLES\`, or read the documentation for details.`
122
123
  );
123
124
  }
124
125
  if (row.pubinsert == false || row.pubupdate == false || row.pubdelete == false || row.pubtruncate == false) {
125
- throw new Error(
126
+ throw new ServiceError(
127
+ ErrorCode.PSYNC_S1142,
126
128
  `Publication '${publicationName}' does not publish all changes. Create a publication using \`WITH (publish = "insert, update, delete, truncate")\` (the default).`
127
129
  );
128
130
  }
129
131
  if (row.pubviaroot) {
130
- throw new Error(`'${publicationName}' uses publish_via_partition_root, which is not supported.`);
132
+ throw new ServiceError(
133
+ ErrorCode.PSYNC_S1143,
134
+ `'${publicationName}' uses publish_via_partition_root, which is not supported.`
135
+ );
131
136
  }
132
137
  }
133
138
 
@@ -1,3 +1,4 @@
1
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
1
2
  import * as pgwire from '@powersync/service-jpgwire';
2
3
 
3
4
  export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
@@ -14,7 +15,7 @@ export class Migrations {
14
15
 
15
16
  add(id: number, name: string, up: MigrationFunction) {
16
17
  if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
17
- throw new Error('Migration ids must be strictly incrementing');
18
+ throw new ServiceAssertionError('Migration ids must be strictly incrementing');
18
19
  }
19
20
  this.migrations.push({ id, up, name });
20
21
  }