@powersync/service-module-postgres 0.0.0-dev-20250117095455 → 0.0.0-dev-20250214100224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -11
- package/dist/api/PostgresRouteAPIAdapter.d.ts +2 -1
- package/dist/api/PostgresRouteAPIAdapter.js +22 -10
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/auth/SupabaseKeyCollector.js +6 -5
- package/dist/auth/SupabaseKeyCollector.js.map +1 -1
- package/dist/module/PostgresModule.d.ts +4 -2
- package/dist/module/PostgresModule.js +13 -5
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.d.ts +1 -1
- package/dist/replication/ConnectionManagerFactory.js +2 -0
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/PgManager.d.ts +5 -0
- package/dist/replication/PgManager.js +17 -2
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/PgRelation.js +2 -1
- package/dist/replication/PgRelation.js.map +1 -1
- package/dist/replication/PostgresErrorRateLimiter.js +5 -7
- package/dist/replication/PostgresErrorRateLimiter.js.map +1 -1
- package/dist/replication/WalStream.d.ts +18 -3
- package/dist/replication/WalStream.js +133 -22
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/replication/WalStreamReplicationJob.js +7 -5
- package/dist/replication/WalStreamReplicationJob.js.map +1 -1
- package/dist/replication/WalStreamReplicator.d.ts +1 -0
- package/dist/replication/WalStreamReplicator.js +6 -1
- package/dist/replication/WalStreamReplicator.js.map +1 -1
- package/dist/replication/replication-utils.js +4 -4
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/utils/migration_lib.js +3 -4
- package/dist/utils/migration_lib.js.map +1 -1
- package/dist/utils/populate_test_data.js +1 -1
- package/dist/utils/populate_test_data.js.map +1 -1
- package/package.json +14 -12
- package/src/api/PostgresRouteAPIAdapter.ts +19 -9
- package/src/module/PostgresModule.ts +22 -5
- package/src/replication/ConnectionManagerFactory.ts +1 -1
- package/src/replication/PgManager.ts +10 -0
- package/src/replication/PgRelation.ts +2 -1
- package/src/replication/WalStream.ts +160 -26
- package/src/replication/WalStreamReplicationJob.ts +3 -3
- package/src/replication/WalStreamReplicator.ts +5 -0
- package/src/replication/replication-utils.ts +9 -4
- package/src/utils/migration_lib.ts +2 -1
- package/test/src/checkpoints.test.ts +70 -0
- package/test/src/storage_combination.test.ts +35 -0
- package/test/src/util.ts +1 -1
- package/test/src/wal_stream_utils.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20250214100224",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -22,24 +22,26 @@
|
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
|
|
26
25
|
"jose": "^4.15.1",
|
|
26
|
+
"pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
|
|
27
|
+
"semver": "^7.5.4",
|
|
27
28
|
"ts-codec": "^1.3.0",
|
|
28
|
-
"uuid": "^9.0.1",
|
|
29
29
|
"uri-js": "^4.4.1",
|
|
30
|
-
"
|
|
31
|
-
"@powersync/lib-service-postgres": "0.
|
|
32
|
-
"@powersync/
|
|
33
|
-
"@powersync/service-
|
|
30
|
+
"uuid": "^9.0.1",
|
|
31
|
+
"@powersync/lib-service-postgres": "0.3.1",
|
|
32
|
+
"@powersync/lib-services-framework": "0.5.1",
|
|
33
|
+
"@powersync/service-core": "0.0.0-dev-20250214100224",
|
|
34
|
+
"@powersync/service-jpgwire": "0.19.0",
|
|
34
35
|
"@powersync/service-jsonbig": "0.17.10",
|
|
35
|
-
"@powersync/service-sync-rules": "0.23.
|
|
36
|
-
"@powersync/service-types": "0.
|
|
36
|
+
"@powersync/service-sync-rules": "0.23.4",
|
|
37
|
+
"@powersync/service-types": "0.8.0"
|
|
37
38
|
},
|
|
38
39
|
"devDependencies": {
|
|
40
|
+
"@types/semver": "^7.5.4",
|
|
39
41
|
"@types/uuid": "^9.0.4",
|
|
40
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
41
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
42
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
42
|
+
"@powersync/service-core-tests": "0.0.0-dev-20250214100224",
|
|
43
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20250214100224",
|
|
44
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20250214100224"
|
|
43
45
|
},
|
|
44
46
|
"scripts": {
|
|
45
47
|
"build": "tsc -b",
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import {
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { api, ParseSyncRulesOptions, ReplicationHeadCallback } from '@powersync/service-core';
|
|
3
4
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
4
5
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
6
|
import * as service_types from '@powersync/service-types';
|
|
6
7
|
import * as replication_utils from '../replication/replication-utils.js';
|
|
7
8
|
import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
8
|
-
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
|
+
import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
10
|
import * as types from '../types/types.js';
|
|
10
11
|
|
|
11
12
|
export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
@@ -228,7 +229,11 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
228
229
|
return Number(row.lsn_distance);
|
|
229
230
|
}
|
|
230
231
|
|
|
231
|
-
throw new
|
|
232
|
+
throw new ServiceError({
|
|
233
|
+
status: 500,
|
|
234
|
+
code: ErrorCode.PSYNC_S4001,
|
|
235
|
+
description: `Could not determine replication lag for slot ${slotName}`
|
|
236
|
+
});
|
|
232
237
|
}
|
|
233
238
|
|
|
234
239
|
async getReplicationHead(): Promise<string> {
|
|
@@ -236,17 +241,22 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
236
241
|
// However, on Aurora (Postgres compatible), it can return an entirely different LSN,
|
|
237
242
|
// causing the write checkpoints to never be replicated back to the client.
|
|
238
243
|
// For those, we need to use pg_current_wal_lsn() instead.
|
|
239
|
-
const { results } = await lib_postgres.retriedQuery(
|
|
240
|
-
this.pool,
|
|
241
|
-
{ statement: `SELECT pg_current_wal_lsn() as lsn` },
|
|
242
|
-
{ statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` }
|
|
243
|
-
);
|
|
244
|
+
const { results } = await lib_postgres.retriedQuery(this.pool, `SELECT pg_current_wal_lsn() as lsn`);
|
|
244
245
|
|
|
245
|
-
// Specifically use the lsn from the first statement, not the second one.
|
|
246
246
|
const lsn = results[0].rows[0][0];
|
|
247
247
|
return String(lsn);
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
+
async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
|
|
251
|
+
const currentLsn = await this.getReplicationHead();
|
|
252
|
+
|
|
253
|
+
const r = await callback(currentLsn);
|
|
254
|
+
|
|
255
|
+
await lib_postgres.retriedQuery(this.pool, KEEPALIVE_STATEMENT);
|
|
256
|
+
|
|
257
|
+
return r;
|
|
258
|
+
}
|
|
259
|
+
|
|
250
260
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
251
261
|
// https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
|
|
252
262
|
const results = await lib_postgres.retriedQuery(
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
api,
|
|
3
|
+
auth,
|
|
4
|
+
ConfigurationFileSyncRulesProvider,
|
|
5
|
+
ConnectionTestResult,
|
|
6
|
+
modules,
|
|
7
|
+
replication,
|
|
8
|
+
system
|
|
9
|
+
} from '@powersync/service-core';
|
|
2
10
|
import * as jpgwire from '@powersync/service-jpgwire';
|
|
3
11
|
import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js';
|
|
4
12
|
import { SupabaseKeyCollector } from '../auth/SupabaseKeyCollector.js';
|
|
@@ -10,6 +18,7 @@ import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
|
10
18
|
import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
|
|
11
19
|
import * as types from '../types/types.js';
|
|
12
20
|
import { PostgresConnectionConfig } from '../types/types.js';
|
|
21
|
+
import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres';
|
|
13
22
|
|
|
14
23
|
export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
|
|
15
24
|
constructor() {
|
|
@@ -124,18 +133,26 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
124
133
|
});
|
|
125
134
|
}
|
|
126
135
|
|
|
127
|
-
async testConnection(config: PostgresConnectionConfig): Promise<
|
|
136
|
+
async testConnection(config: PostgresConnectionConfig): Promise<ConnectionTestResult> {
|
|
128
137
|
this.decodeConfig(config);
|
|
129
|
-
const
|
|
130
|
-
|
|
138
|
+
const normalizedConfig = this.resolveConfig(this.decodedConfig!);
|
|
139
|
+
return await this.testConnection(normalizedConfig);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
static async testConnection(normalizedConfig: NormalizedBasePostgresConnectionConfig): Promise<ConnectionTestResult> {
|
|
143
|
+
// FIXME: This is not a complete implementation yet.
|
|
144
|
+
const connectionManager = new PgManager(normalizedConfig, {
|
|
131
145
|
idleTimeout: 30_000,
|
|
132
146
|
maxSize: 1
|
|
133
147
|
});
|
|
134
148
|
const connection = await connectionManager.snapshotConnection();
|
|
135
149
|
try {
|
|
136
|
-
|
|
150
|
+
await checkSourceConfiguration(connection, PUBLICATION_NAME);
|
|
137
151
|
} finally {
|
|
138
152
|
await connectionManager.end();
|
|
139
153
|
}
|
|
154
|
+
return {
|
|
155
|
+
connectionDescription: baseUri(normalizedConfig)
|
|
156
|
+
};
|
|
140
157
|
}
|
|
141
158
|
}
|
|
@@ -5,7 +5,7 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
5
5
|
|
|
6
6
|
export class ConnectionManagerFactory {
|
|
7
7
|
private readonly connectionManagers: PgManager[];
|
|
8
|
-
|
|
8
|
+
public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
|
|
9
9
|
|
|
10
10
|
constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig) {
|
|
11
11
|
this.dbConnectionConfig = dbConnectionConfig;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
+
import semver from 'semver';
|
|
2
3
|
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -35,6 +36,15 @@ export class PgManager {
|
|
|
35
36
|
return await p;
|
|
36
37
|
}
|
|
37
38
|
|
|
39
|
+
/**
|
|
40
|
+
* @returns The Postgres server version in a parsed Semver instance
|
|
41
|
+
*/
|
|
42
|
+
async getServerVersion(): Promise<semver.SemVer | null> {
|
|
43
|
+
const result = await this.pool.query(`SHOW server_version;`);
|
|
44
|
+
// The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)"
|
|
45
|
+
return semver.coerce(result.rows[0][0].split(' ')[0]);
|
|
46
|
+
}
|
|
47
|
+
|
|
38
48
|
/**
|
|
39
49
|
* Create a new standard connection, used for initial snapshot.
|
|
40
50
|
*
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
1
2
|
import { storage } from '@powersync/service-core';
|
|
2
3
|
import { PgoutputRelation } from '@powersync/service-jpgwire';
|
|
3
4
|
|
|
@@ -16,7 +17,7 @@ export function getRelId(source: PgoutputRelation): number {
|
|
|
16
17
|
// Source types are wrong here
|
|
17
18
|
const relId = (source as any).relationOid as number;
|
|
18
19
|
if (!relId) {
|
|
19
|
-
throw new
|
|
20
|
+
throw new ReplicationAssertionError(`No relation id found`);
|
|
20
21
|
}
|
|
21
22
|
return relId;
|
|
22
23
|
}
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
container,
|
|
4
|
+
DatabaseConnectionError,
|
|
5
|
+
ErrorCode,
|
|
6
|
+
errors,
|
|
7
|
+
logger,
|
|
8
|
+
ReplicationAbortedError,
|
|
9
|
+
ReplicationAssertionError
|
|
10
|
+
} from '@powersync/lib-services-framework';
|
|
3
11
|
import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
|
|
4
12
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
5
13
|
import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
@@ -9,10 +17,6 @@ import { PgManager } from './PgManager.js';
|
|
|
9
17
|
import { getPgOutputRelation, getRelId } from './PgRelation.js';
|
|
10
18
|
import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js';
|
|
11
19
|
|
|
12
|
-
export const ZERO_LSN = '00000000/00000000';
|
|
13
|
-
export const PUBLICATION_NAME = 'powersync';
|
|
14
|
-
export const POSTGRES_DEFAULT_SCHEMA = 'public';
|
|
15
|
-
|
|
16
20
|
export interface WalStreamOptions {
|
|
17
21
|
connections: PgManager;
|
|
18
22
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -26,6 +30,35 @@ interface InitResult {
|
|
|
26
30
|
needsNewSlot: boolean;
|
|
27
31
|
}
|
|
28
32
|
|
|
33
|
+
export const ZERO_LSN = '00000000/00000000';
|
|
34
|
+
export const PUBLICATION_NAME = 'powersync';
|
|
35
|
+
export const POSTGRES_DEFAULT_SCHEMA = 'public';
|
|
36
|
+
|
|
37
|
+
export const KEEPALIVE_CONTENT = 'ping';
|
|
38
|
+
export const KEEPALIVE_BUFFER = Buffer.from(KEEPALIVE_CONTENT);
|
|
39
|
+
export const KEEPALIVE_STATEMENT: pgwire.Statement = {
|
|
40
|
+
statement: /* sql */ `
|
|
41
|
+
SELECT
|
|
42
|
+
*
|
|
43
|
+
FROM
|
|
44
|
+
pg_logical_emit_message(FALSE, 'powersync', $1)
|
|
45
|
+
`,
|
|
46
|
+
params: [{ type: 'varchar', value: KEEPALIVE_CONTENT }]
|
|
47
|
+
} as const;
|
|
48
|
+
|
|
49
|
+
export const isKeepAliveMessage = (msg: pgwire.PgoutputMessage) => {
|
|
50
|
+
return (
|
|
51
|
+
msg.tag == 'message' &&
|
|
52
|
+
msg.prefix == 'powersync' &&
|
|
53
|
+
msg.content &&
|
|
54
|
+
Buffer.from(msg.content).equals(KEEPALIVE_BUFFER)
|
|
55
|
+
);
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export const sendKeepAlive = async (db: pgwire.PgClient) => {
|
|
59
|
+
await lib_postgres.retriedQuery(db, KEEPALIVE_STATEMENT);
|
|
60
|
+
};
|
|
61
|
+
|
|
29
62
|
export class MissingReplicationSlotError extends Error {
|
|
30
63
|
constructor(message: string) {
|
|
31
64
|
super(message);
|
|
@@ -65,10 +98,7 @@ export class WalStream {
|
|
|
65
98
|
// Ping to speed up cancellation of streaming replication
|
|
66
99
|
// We're not using pg_snapshot here, since it could be in the middle of
|
|
67
100
|
// an initial replication transaction.
|
|
68
|
-
const promise =
|
|
69
|
-
this.connections.pool,
|
|
70
|
-
`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`
|
|
71
|
-
);
|
|
101
|
+
const promise = sendKeepAlive(this.connections.pool);
|
|
72
102
|
promise.catch((e) => {
|
|
73
103
|
// Failures here are okay - this only speeds up stopping the process.
|
|
74
104
|
logger.warn('Failed to ping connection', e);
|
|
@@ -133,7 +163,7 @@ export class WalStream {
|
|
|
133
163
|
for (let row of tableRows) {
|
|
134
164
|
const name = row.table_name as string;
|
|
135
165
|
if (typeof row.relid != 'bigint') {
|
|
136
|
-
throw new
|
|
166
|
+
throw new ReplicationAssertionError(`Missing relid for ${name}`);
|
|
137
167
|
}
|
|
138
168
|
const relid = Number(row.relid as bigint);
|
|
139
169
|
|
|
@@ -174,6 +204,7 @@ export class WalStream {
|
|
|
174
204
|
|
|
175
205
|
async initSlot(): Promise<InitResult> {
|
|
176
206
|
await checkSourceConfiguration(this.connections.pool, PUBLICATION_NAME);
|
|
207
|
+
await this.ensureStorageCompatibility();
|
|
177
208
|
|
|
178
209
|
const slotName = this.slot_name;
|
|
179
210
|
|
|
@@ -294,7 +325,7 @@ export class WalStream {
|
|
|
294
325
|
}
|
|
295
326
|
}
|
|
296
327
|
|
|
297
|
-
throw new
|
|
328
|
+
throw new ReplicationAssertionError('Unreachable');
|
|
298
329
|
}
|
|
299
330
|
|
|
300
331
|
async estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string> {
|
|
@@ -376,6 +407,15 @@ WHERE oid = $1::regclass`,
|
|
|
376
407
|
await batch.commit(ZERO_LSN);
|
|
377
408
|
}
|
|
378
409
|
);
|
|
410
|
+
/**
|
|
411
|
+
* Send a keepalive message after initial replication.
|
|
412
|
+
* In some edge cases we wait for a keepalive after the initial snapshot.
|
|
413
|
+
* If we don't explicitly check the contents of keepalive messages then a keepalive is detected
|
|
414
|
+
* rather quickly after initial replication - perhaps due to other WAL events.
|
|
415
|
+
* If we do explicitly check the contents of messages, we need an actual keepalive payload in order
|
|
416
|
+
* to advance the active sync rules LSN.
|
|
417
|
+
*/
|
|
418
|
+
await sendKeepAlive(db);
|
|
379
419
|
}
|
|
380
420
|
|
|
381
421
|
static *getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow> {
|
|
@@ -415,7 +455,7 @@ WHERE oid = $1::regclass`,
|
|
|
415
455
|
lastLogIndex = at;
|
|
416
456
|
}
|
|
417
457
|
if (this.abort_signal.aborted) {
|
|
418
|
-
throw new
|
|
458
|
+
throw new ReplicationAbortedError(`Aborted initial replication of ${this.slot_name}`);
|
|
419
459
|
}
|
|
420
460
|
|
|
421
461
|
for (const record of WalStream.getQueryData(rows)) {
|
|
@@ -441,7 +481,7 @@ WHERE oid = $1::regclass`,
|
|
|
441
481
|
|
|
442
482
|
async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) {
|
|
443
483
|
if (!descriptor.objectId && typeof descriptor.objectId != 'number') {
|
|
444
|
-
throw new
|
|
484
|
+
throw new ReplicationAssertionError(`objectId expected, got ${typeof descriptor.objectId}`);
|
|
445
485
|
}
|
|
446
486
|
const result = await this.storage.resolveTable({
|
|
447
487
|
group_id: this.group_id,
|
|
@@ -484,6 +524,7 @@ WHERE oid = $1::regclass`,
|
|
|
484
524
|
await db.query('COMMIT');
|
|
485
525
|
} catch (e) {
|
|
486
526
|
await db.query('ROLLBACK');
|
|
527
|
+
// TODO: Wrap with custom error type
|
|
487
528
|
throw e;
|
|
488
529
|
}
|
|
489
530
|
} finally {
|
|
@@ -501,7 +542,7 @@ WHERE oid = $1::regclass`,
|
|
|
501
542
|
if (table == null) {
|
|
502
543
|
// We should always receive a replication message before the relation is used.
|
|
503
544
|
// If we can't find it, it's a bug.
|
|
504
|
-
throw new
|
|
545
|
+
throw new ReplicationAssertionError(`Missing relation cache for ${relationId}`);
|
|
505
546
|
}
|
|
506
547
|
return table;
|
|
507
548
|
}
|
|
@@ -592,13 +633,33 @@ WHERE oid = $1::regclass`,
|
|
|
592
633
|
async streamChanges(replicationConnection: pgwire.PgConnection) {
|
|
593
634
|
// When changing any logic here, check /docs/wal-lsns.md.
|
|
594
635
|
|
|
636
|
+
const { createEmptyCheckpoints } = await this.ensureStorageCompatibility();
|
|
637
|
+
|
|
638
|
+
const replicationOptions: Record<string, string> = {
|
|
639
|
+
proto_version: '1',
|
|
640
|
+
publication_names: PUBLICATION_NAME
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
/**
|
|
644
|
+
* Viewing the contents of logical messages emitted with `pg_logical_emit_message`
|
|
645
|
+
* is only supported on Postgres >= 14.0.
|
|
646
|
+
* https://www.postgresql.org/docs/14/protocol-logical-replication.html
|
|
647
|
+
*/
|
|
648
|
+
const exposesLogicalMessages = await this.checkLogicalMessageSupport();
|
|
649
|
+
if (exposesLogicalMessages) {
|
|
650
|
+
/**
|
|
651
|
+
* Only add this option if the Postgres server supports it.
|
|
652
|
+
* Adding the option to a server that doesn't support it will throw an exception when starting logical replication.
|
|
653
|
+
* Error: `unrecognized pgoutput option: messages`
|
|
654
|
+
*/
|
|
655
|
+
replicationOptions['messages'] = 'true';
|
|
656
|
+
}
|
|
657
|
+
|
|
595
658
|
const replicationStream = replicationConnection.logicalReplication({
|
|
596
659
|
slot: this.slot_name,
|
|
597
|
-
options:
|
|
598
|
-
proto_version: '1',
|
|
599
|
-
publication_names: PUBLICATION_NAME
|
|
600
|
-
}
|
|
660
|
+
options: replicationOptions
|
|
601
661
|
});
|
|
662
|
+
|
|
602
663
|
this.startedStreaming = true;
|
|
603
664
|
|
|
604
665
|
// Auto-activate as soon as initial replication is done
|
|
@@ -621,6 +682,15 @@ WHERE oid = $1::regclass`,
|
|
|
621
682
|
// chunkLastLsn may come from normal messages in the chunk,
|
|
622
683
|
// or from a PrimaryKeepalive message.
|
|
623
684
|
const { messages, lastLsn: chunkLastLsn } = chunk;
|
|
685
|
+
|
|
686
|
+
/**
|
|
687
|
+
* We can check if an explicit keepalive was sent if `exposesLogicalMessages == true`.
|
|
688
|
+
* If we can't check the logical messages, we should assume a keepalive if we
|
|
689
|
+
* receive an empty array of messages in a replication event.
|
|
690
|
+
*/
|
|
691
|
+
const assumeKeepAlive = !exposesLogicalMessages;
|
|
692
|
+
let keepAliveDetected = false;
|
|
693
|
+
|
|
624
694
|
for (const msg of messages) {
|
|
625
695
|
if (msg.tag == 'relation') {
|
|
626
696
|
await this.handleRelation(batch, getPgOutputRelation(msg), true);
|
|
@@ -629,27 +699,44 @@ WHERE oid = $1::regclass`,
|
|
|
629
699
|
} else if (msg.tag == 'commit') {
|
|
630
700
|
Metrics.getInstance().transactions_replicated_total.add(1);
|
|
631
701
|
inTx = false;
|
|
632
|
-
await batch.commit(msg.lsn
|
|
702
|
+
await batch.commit(msg.lsn!, { createEmptyCheckpoints });
|
|
633
703
|
await this.ack(msg.lsn!, replicationStream);
|
|
634
704
|
} else {
|
|
635
705
|
if (count % 100 == 0) {
|
|
636
706
|
logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`);
|
|
637
707
|
}
|
|
638
708
|
|
|
709
|
+
/**
|
|
710
|
+
* If we can see the contents of logical messages, then we can check if a keepalive
|
|
711
|
+
* message is present. We only perform a keepalive (below) if we explicitly detect a keepalive message.
|
|
712
|
+
* If we can't see the contents of logical messages, then we should assume a keepalive is required
|
|
713
|
+
* due to the default value of `assumeKeepalive`.
|
|
714
|
+
*/
|
|
715
|
+
if (exposesLogicalMessages && isKeepAliveMessage(msg)) {
|
|
716
|
+
keepAliveDetected = true;
|
|
717
|
+
}
|
|
718
|
+
|
|
639
719
|
count += 1;
|
|
640
720
|
await this.writeChange(batch, msg);
|
|
641
721
|
}
|
|
642
722
|
}
|
|
643
723
|
|
|
644
724
|
if (!inTx) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
725
|
+
if (assumeKeepAlive || keepAliveDetected) {
|
|
726
|
+
// Reset the detection flag.
|
|
727
|
+
keepAliveDetected = false;
|
|
728
|
+
|
|
729
|
+
// In a transaction, we ack and commit according to the transaction progress.
|
|
730
|
+
// Outside transactions, we use the PrimaryKeepalive messages to advance progress.
|
|
731
|
+
// Big caveat: This _must not_ be used to skip individual messages, since this LSN
|
|
732
|
+
// may be in the middle of the next transaction.
|
|
733
|
+
// It must only be used to associate checkpoints with LSNs.
|
|
734
|
+
await batch.keepalive(chunkLastLsn);
|
|
652
735
|
}
|
|
736
|
+
|
|
737
|
+
// We receive chunks with empty messages often (about each second).
|
|
738
|
+
// Acknowledging here progresses the slot past these and frees up resources.
|
|
739
|
+
await this.ack(chunkLastLsn, replicationStream);
|
|
653
740
|
}
|
|
654
741
|
|
|
655
742
|
Metrics.getInstance().chunks_replicated_total.add(1);
|
|
@@ -665,6 +752,53 @@ WHERE oid = $1::regclass`,
|
|
|
665
752
|
|
|
666
753
|
replicationStream.ack(lsn);
|
|
667
754
|
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Ensures that the storage is compatible with the replication connection.
|
|
758
|
+
* @throws {DatabaseConnectionError} If the storage is not compatible with the replication connection.
|
|
759
|
+
*/
|
|
760
|
+
protected async ensureStorageCompatibility(): Promise<storage.ResolvedBucketBatchCommitOptions> {
|
|
761
|
+
const supportsLogicalMessages = await this.checkLogicalMessageSupport();
|
|
762
|
+
|
|
763
|
+
const storageIdentifier = await this.storage.factory.getSystemIdentifier();
|
|
764
|
+
if (storageIdentifier.type != lib_postgres.POSTGRES_CONNECTION_TYPE) {
|
|
765
|
+
return {
|
|
766
|
+
// Keep the same behaviour as before allowing Postgres storage.
|
|
767
|
+
createEmptyCheckpoints: true
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
const parsedStorageIdentifier = lib_postgres.utils.decodePostgresSystemIdentifier(storageIdentifier.id);
|
|
772
|
+
/**
|
|
773
|
+
* Check if the same server is being used for both the sync bucket storage and the logical replication.
|
|
774
|
+
*/
|
|
775
|
+
const replicationIdentifier = await lib_postgres.utils.queryPostgresSystemIdentifier(this.connections.pool);
|
|
776
|
+
|
|
777
|
+
if (!supportsLogicalMessages && replicationIdentifier.server_id == parsedStorageIdentifier.server_id) {
|
|
778
|
+
throw new DatabaseConnectionError(
|
|
779
|
+
ErrorCode.PSYNC_S1144,
|
|
780
|
+
`Separate Postgres servers are required for the replication source and sync bucket storage when using Postgres versions below 14.0.`,
|
|
781
|
+
new Error('Postgres version is below 14')
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
return {
|
|
786
|
+
/**
|
|
787
|
+
* Don't create empty checkpoints if the same Postgres database is used for the data source
|
|
788
|
+
* and sync bucket storage. Creating empty checkpoints will cause WAL feedback loops.
|
|
789
|
+
*/
|
|
790
|
+
createEmptyCheckpoints: replicationIdentifier.database_name != parsedStorageIdentifier.database_name
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/**
|
|
795
|
+
* Check if the replication connection Postgres server supports
|
|
796
|
+
* viewing the contents of logical replication messages.
|
|
797
|
+
*/
|
|
798
|
+
protected async checkLogicalMessageSupport() {
|
|
799
|
+
const version = await this.connections.getServerVersion();
|
|
800
|
+
return version ? version.compareMain('14.0.0') >= 0 : false;
|
|
801
|
+
}
|
|
668
802
|
}
|
|
669
803
|
|
|
670
804
|
async function touch() {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { container } from '@powersync/lib-services-framework';
|
|
2
2
|
import { PgManager } from './PgManager.js';
|
|
3
|
-
import { MissingReplicationSlotError, WalStream } from './WalStream.js';
|
|
3
|
+
import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
|
|
4
4
|
|
|
5
5
|
import { replication } from '@powersync/service-core';
|
|
6
6
|
import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
|
|
@@ -37,7 +37,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
37
37
|
*/
|
|
38
38
|
async keepAlive() {
|
|
39
39
|
try {
|
|
40
|
-
await this.connectionManager.pool
|
|
40
|
+
await sendKeepAlive(this.connectionManager.pool);
|
|
41
41
|
} catch (e) {
|
|
42
42
|
this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
|
|
43
43
|
}
|
|
@@ -99,7 +99,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
99
99
|
});
|
|
100
100
|
await stream.replicate();
|
|
101
101
|
} catch (e) {
|
|
102
|
-
this.logger.error(
|
|
102
|
+
this.logger.error(`${this.slotName} Replication error`, e);
|
|
103
103
|
if (e.cause != null) {
|
|
104
104
|
// Example:
|
|
105
105
|
// PgError.conn_ended: Unable to do postgres query on ended connection
|
|
@@ -2,6 +2,7 @@ import { replication, storage } from '@powersync/service-core';
|
|
|
2
2
|
import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
|
|
3
3
|
import { cleanUpReplicationSlot } from './replication-utils.js';
|
|
4
4
|
import { WalStreamReplicationJob } from './WalStreamReplicationJob.js';
|
|
5
|
+
import { PostgresModule } from '../module/PostgresModule.js';
|
|
5
6
|
|
|
6
7
|
export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
|
|
7
8
|
connectionFactory: ConnectionManagerFactory;
|
|
@@ -42,4 +43,8 @@ export class WalStreamReplicator extends replication.AbstractReplicator<WalStrea
|
|
|
42
43
|
await super.stop();
|
|
43
44
|
await this.connectionFactory.shutdown();
|
|
44
45
|
}
|
|
46
|
+
|
|
47
|
+
async testConnection() {
|
|
48
|
+
return await PostgresModule.testConnection(this.connectionFactory.dbConnectionConfig);
|
|
49
|
+
}
|
|
45
50
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
2
|
|
|
3
3
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
4
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
4
|
+
import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
|
|
5
5
|
import { PatternResult, storage } from '@powersync/service-core';
|
|
6
6
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
7
7
|
import * as service_types from '@powersync/service-types';
|
|
@@ -117,17 +117,22 @@ $$ LANGUAGE plpgsql;`
|
|
|
117
117
|
});
|
|
118
118
|
const row = pgwire.pgwireRows(rs)[0];
|
|
119
119
|
if (row == null) {
|
|
120
|
-
throw new
|
|
120
|
+
throw new ServiceError(
|
|
121
|
+
ErrorCode.PSYNC_S1141,
|
|
121
122
|
`Publication '${publicationName}' does not exist. Run: \`CREATE PUBLICATION ${publicationName} FOR ALL TABLES\`, or read the documentation for details.`
|
|
122
123
|
);
|
|
123
124
|
}
|
|
124
125
|
if (row.pubinsert == false || row.pubupdate == false || row.pubdelete == false || row.pubtruncate == false) {
|
|
125
|
-
throw new
|
|
126
|
+
throw new ServiceError(
|
|
127
|
+
ErrorCode.PSYNC_S1142,
|
|
126
128
|
`Publication '${publicationName}' does not publish all changes. Create a publication using \`WITH (publish = "insert, update, delete, truncate")\` (the default).`
|
|
127
129
|
);
|
|
128
130
|
}
|
|
129
131
|
if (row.pubviaroot) {
|
|
130
|
-
throw new
|
|
132
|
+
throw new ServiceError(
|
|
133
|
+
ErrorCode.PSYNC_S1143,
|
|
134
|
+
`'${publicationName}' uses publish_via_partition_root, which is not supported.`
|
|
135
|
+
);
|
|
131
136
|
}
|
|
132
137
|
}
|
|
133
138
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
1
2
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
3
|
|
|
3
4
|
export type MigrationFunction = (db: pgwire.PgConnection) => Promise<void>;
|
|
@@ -14,7 +15,7 @@ export class Migrations {
|
|
|
14
15
|
|
|
15
16
|
add(id: number, name: string, up: MigrationFunction) {
|
|
16
17
|
if (this.migrations.length > 0 && this.migrations[this.migrations.length - 1].id >= id) {
|
|
17
|
-
throw new
|
|
18
|
+
throw new ServiceAssertionError('Migration ids must be strictly incrementing');
|
|
18
19
|
}
|
|
19
20
|
this.migrations.push({ id, up, name });
|
|
20
21
|
}
|