@powersync/service-module-postgres 0.2.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/api/PostgresRouteAPIAdapter.js +9 -9
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/auth/SupabaseKeyCollector.js +2 -2
- package/dist/auth/SupabaseKeyCollector.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/module/PostgresModule.js +1 -1
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/WalStream.js +2 -1
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/replication/replication-utils.d.ts +1 -1
- package/dist/replication/replication-utils.js +11 -13
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/types/types.d.ts +52 -51
- package/dist/types/types.js +11 -95
- package/dist/types/types.js.map +1 -1
- package/dist/utils/pgwire_utils.d.ts +1 -5
- package/dist/utils/pgwire_utils.js +0 -45
- package/dist/utils/pgwire_utils.js.map +1 -1
- package/package.json +12 -8
- package/src/api/PostgresRouteAPIAdapter.ts +9 -10
- package/src/auth/SupabaseKeyCollector.ts +2 -2
- package/src/index.ts +2 -0
- package/src/module/PostgresModule.ts +1 -1
- package/src/replication/WalStream.ts +3 -1
- package/src/replication/replication-utils.ts +12 -14
- package/src/types/types.ts +18 -132
- package/src/utils/pgwire_utils.ts +1 -46
- package/test/src/__snapshots__/schema_changes.test.ts.snap +5 -0
- package/test/src/env.ts +5 -1
- package/test/src/large_batch.test.ts +21 -7
- package/test/src/schema_changes.test.ts +19 -13
- package/test/src/setup.ts +8 -2
- package/test/src/slow_tests.test.ts +132 -39
- package/test/src/util.ts +14 -30
- package/test/src/validation.test.ts +4 -2
- package/test/src/wal_stream.test.ts +12 -9
- package/test/src/wal_stream_utils.ts +7 -7
- package/tsconfig.json +3 -0
- package/tsconfig.tsbuildinfo +1 -1
package/dist/types/types.js
CHANGED
|
@@ -1,108 +1,24 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import * as service_types from '@powersync/service-types';
|
|
2
3
|
import * as t from 'ts-codec';
|
|
3
|
-
|
|
4
|
-
export const
|
|
5
|
-
export const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
/** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
|
|
10
|
-
tag: t.string.optional(),
|
|
11
|
-
uri: t.string.optional(),
|
|
12
|
-
hostname: t.string.optional(),
|
|
13
|
-
port: service_types.configFile.portCodec.optional(),
|
|
14
|
-
username: t.string.optional(),
|
|
15
|
-
password: t.string.optional(),
|
|
16
|
-
database: t.string.optional(),
|
|
17
|
-
/** Defaults to verify-full */
|
|
18
|
-
sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(),
|
|
19
|
-
/** Required for verify-ca, optional for verify-full */
|
|
20
|
-
cacert: t.string.optional(),
|
|
21
|
-
client_certificate: t.string.optional(),
|
|
22
|
-
client_private_key: t.string.optional(),
|
|
23
|
-
/** Expose database credentials */
|
|
24
|
-
demo_database: t.boolean.optional(),
|
|
25
|
-
/**
|
|
26
|
-
* Prefix for the slot name. Defaults to "powersync_"
|
|
27
|
-
*/
|
|
28
|
-
slot_name_prefix: t.string.optional()
|
|
4
|
+
// Maintain backwards compatibility by exporting these
|
|
5
|
+
export const validatePort = lib_postgres.validatePort;
|
|
6
|
+
export const baseUri = lib_postgres.baseUri;
|
|
7
|
+
export const POSTGRES_CONNECTION_TYPE = lib_postgres.POSTGRES_CONNECTION_TYPE;
|
|
8
|
+
export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and(lib_postgres.BasePostgresConnectionConfig).and(t.object({
|
|
9
|
+
// Add any replication connection specific config here in future
|
|
29
10
|
}));
|
|
11
|
+
export function isPostgresConfig(config) {
|
|
12
|
+
return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE;
|
|
13
|
+
}
|
|
30
14
|
/**
|
|
31
15
|
* Validate and normalize connection options.
|
|
32
16
|
*
|
|
33
17
|
* Returns destructured options.
|
|
34
18
|
*/
|
|
35
19
|
export function normalizeConnectionConfig(options) {
|
|
36
|
-
let uri;
|
|
37
|
-
if (options.uri) {
|
|
38
|
-
uri = urijs.parse(options.uri);
|
|
39
|
-
if (uri.scheme != 'postgresql' && uri.scheme != 'postgres') {
|
|
40
|
-
`Invalid URI - protocol must be postgresql, got ${uri.scheme}`;
|
|
41
|
-
}
|
|
42
|
-
else if (uri.scheme != 'postgresql') {
|
|
43
|
-
uri.scheme = 'postgresql';
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
else {
|
|
47
|
-
uri = urijs.parse('postgresql:///');
|
|
48
|
-
}
|
|
49
|
-
const hostname = options.hostname ?? uri.host ?? '';
|
|
50
|
-
const port = validatePort(options.port ?? uri.port ?? 5432);
|
|
51
|
-
const database = options.database ?? uri.path?.substring(1) ?? '';
|
|
52
|
-
const [uri_username, uri_password] = (uri.userinfo ?? '').split(':');
|
|
53
|
-
const username = options.username ?? uri_username ?? '';
|
|
54
|
-
const password = options.password ?? uri_password ?? '';
|
|
55
|
-
const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI
|
|
56
|
-
const cacert = options.cacert;
|
|
57
|
-
if (sslmode == 'verify-ca' && cacert == null) {
|
|
58
|
-
throw new Error('Explicit cacert is required for sslmode=verify-ca');
|
|
59
|
-
}
|
|
60
|
-
if (hostname == '') {
|
|
61
|
-
throw new Error(`hostname required`);
|
|
62
|
-
}
|
|
63
|
-
if (username == '') {
|
|
64
|
-
throw new Error(`username required`);
|
|
65
|
-
}
|
|
66
|
-
if (password == '') {
|
|
67
|
-
throw new Error(`password required`);
|
|
68
|
-
}
|
|
69
|
-
if (database == '') {
|
|
70
|
-
throw new Error(`database required`);
|
|
71
|
-
}
|
|
72
20
|
return {
|
|
73
|
-
|
|
74
|
-
tag: options.tag ?? 'default',
|
|
75
|
-
hostname,
|
|
76
|
-
port,
|
|
77
|
-
database,
|
|
78
|
-
username,
|
|
79
|
-
password,
|
|
80
|
-
sslmode,
|
|
81
|
-
cacert,
|
|
82
|
-
client_certificate: options.client_certificate ?? undefined,
|
|
83
|
-
client_private_key: options.client_private_key ?? undefined
|
|
21
|
+
...lib_postgres.normalizeConnectionConfig(options)
|
|
84
22
|
};
|
|
85
23
|
}
|
|
86
|
-
/**
|
|
87
|
-
* Check whether the port is in a "safe" range.
|
|
88
|
-
*
|
|
89
|
-
* We do not support connecting to "privileged" ports.
|
|
90
|
-
*/
|
|
91
|
-
export function validatePort(port) {
|
|
92
|
-
if (typeof port == 'string') {
|
|
93
|
-
port = parseInt(port);
|
|
94
|
-
}
|
|
95
|
-
if (port < 1024) {
|
|
96
|
-
throw new Error(`Port ${port} not supported`);
|
|
97
|
-
}
|
|
98
|
-
return port;
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Construct a postgres URI, without username, password or ssl options.
|
|
102
|
-
*
|
|
103
|
-
* Only contains hostname, port, database.
|
|
104
|
-
*/
|
|
105
|
-
export function baseUri(options) {
|
|
106
|
-
return `postgresql://${options.hostname}:${options.port}/${options.database}`;
|
|
107
|
-
}
|
|
108
24
|
//# sourceMappingURL=types.js.map
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,sDAAsD;AACtD,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;AACtD,MAAM,CAAC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;AAE5C,MAAM,CAAC,MAAM,wBAAwB,GAAG,YAAY,CAAC,wBAAwB,CAAC;AAE9E,MAAM,CAAC,MAAM,wBAAwB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CACnF,YAAY,CAAC,4BAA4B,CAC1C,CAAC,GAAG,CACH,CAAC,CAAC,MAAM,CAAC;AACP,gEAAgE;CACjE,CAAC,CACH,CAAC;AAYF,MAAM,UAAU,gBAAgB,CAC9B,MAAiD;IAEjD,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAiC;IACzE,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,OAAO,CAAC;KACN,CAAC;AACjD,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
-
import {
|
|
2
|
+
import { SqliteRow } from '@powersync/service-sync-rules';
|
|
3
3
|
/**
|
|
4
4
|
* pgwire message -> SQLite row.
|
|
5
5
|
* @param message
|
|
@@ -10,7 +10,3 @@ export declare function constructAfterRecord(message: pgwire.PgoutputInsert | pg
|
|
|
10
10
|
* @param message
|
|
11
11
|
*/
|
|
12
12
|
export declare function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteRow | undefined;
|
|
13
|
-
export declare function escapeIdentifier(identifier: string): string;
|
|
14
|
-
export declare function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam;
|
|
15
|
-
export declare function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise<pgwire.PgResult>;
|
|
16
|
-
export declare function retriedQuery(db: pgwire.PgClient, query: string): Promise<pgwire.PgResult>;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
|
|
2
2
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
3
3
|
import { toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
4
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
5
4
|
/**
|
|
6
5
|
* pgwire message -> SQLite row.
|
|
7
6
|
* @param message
|
|
@@ -23,48 +22,4 @@ export function constructBeforeRecord(message) {
|
|
|
23
22
|
const record = pgwire.decodeTuple(message.relation, rawData);
|
|
24
23
|
return toSyncRulesRow(record);
|
|
25
24
|
}
|
|
26
|
-
export function escapeIdentifier(identifier) {
|
|
27
|
-
return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
|
|
28
|
-
}
|
|
29
|
-
export function autoParameter(arg) {
|
|
30
|
-
if (arg == null) {
|
|
31
|
-
return { type: 'varchar', value: null };
|
|
32
|
-
}
|
|
33
|
-
else if (typeof arg == 'string') {
|
|
34
|
-
return { type: 'varchar', value: arg };
|
|
35
|
-
}
|
|
36
|
-
else if (typeof arg == 'number') {
|
|
37
|
-
if (Number.isInteger(arg)) {
|
|
38
|
-
return { type: 'int8', value: arg };
|
|
39
|
-
}
|
|
40
|
-
else {
|
|
41
|
-
return { type: 'float8', value: arg };
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
else if (typeof arg == 'boolean') {
|
|
45
|
-
return { type: 'bool', value: arg };
|
|
46
|
-
}
|
|
47
|
-
else if (typeof arg == 'bigint') {
|
|
48
|
-
return { type: 'int8', value: arg };
|
|
49
|
-
}
|
|
50
|
-
else {
|
|
51
|
-
throw new Error(`Unsupported query parameter: ${typeof arg}`);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Retry a simple query - up to 2 attempts total.
|
|
56
|
-
*/
|
|
57
|
-
export async function retriedQuery(db, ...args) {
|
|
58
|
-
for (let tries = 2;; tries--) {
|
|
59
|
-
try {
|
|
60
|
-
return await db.query(...args);
|
|
61
|
-
}
|
|
62
|
-
catch (e) {
|
|
63
|
-
if (tries == 1) {
|
|
64
|
-
throw e;
|
|
65
|
-
}
|
|
66
|
-
logger.warn('Query error, retrying', e);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
25
|
//# sourceMappingURL=pgwire_utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pgwire_utils.js","sourceRoot":"","sources":["../../src/utils/pgwire_utils.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAE1G,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,
|
|
1
|
+
{"version":3,"file":"pgwire_utils.js","sourceRoot":"","sources":["../../src/utils/pgwire_utils.ts"],"names":[],"mappings":"AAAA,0GAA0G;AAE1G,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAa,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAE1E;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsD;IACzF,MAAM,OAAO,GAAI,OAAe,CAAC,QAAQ,CAAC;IAE1C,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAsD;IAC1F,MAAM,OAAO,GAAI,OAAe,CAAC,SAAS,CAAC;IAC3C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.
|
|
8
|
+
"version": "0.4.0",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-Apache-2.0",
|
|
11
11
|
"type": "module",
|
|
@@ -24,18 +24,22 @@
|
|
|
24
24
|
"dependencies": {
|
|
25
25
|
"pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
|
|
26
26
|
"jose": "^4.15.1",
|
|
27
|
-
"ts-codec": "^1.
|
|
27
|
+
"ts-codec": "^1.3.0",
|
|
28
28
|
"uuid": "^9.0.1",
|
|
29
29
|
"uri-js": "^4.4.1",
|
|
30
|
-
"@powersync/
|
|
31
|
-
"@powersync/service-
|
|
32
|
-
"@powersync/service-
|
|
33
|
-
"@powersync/service-
|
|
30
|
+
"@powersync/lib-services-framework": "0.4.0",
|
|
31
|
+
"@powersync/lib-service-postgres": "0.1.0",
|
|
32
|
+
"@powersync/service-core": "0.15.0",
|
|
33
|
+
"@powersync/service-jpgwire": "0.18.5",
|
|
34
34
|
"@powersync/service-jsonbig": "0.17.10",
|
|
35
|
-
"@powersync/
|
|
35
|
+
"@powersync/service-sync-rules": "0.23.1",
|
|
36
|
+
"@powersync/service-types": "0.7.0"
|
|
36
37
|
},
|
|
37
38
|
"devDependencies": {
|
|
38
|
-
"@types/uuid": "^9.0.4"
|
|
39
|
+
"@types/uuid": "^9.0.4",
|
|
40
|
+
"@powersync/service-core-tests": "0.3.0",
|
|
41
|
+
"@powersync/service-module-mongodb-storage": "0.3.0",
|
|
42
|
+
"@powersync/service-module-postgres-storage": "0.1.0"
|
|
39
43
|
},
|
|
40
44
|
"scripts": {
|
|
41
45
|
"build": "tsc -b",
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import { api, ParseSyncRulesOptions } from '@powersync/service-core';
|
|
2
3
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
3
|
-
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as service_types from '@powersync/service-types';
|
|
6
6
|
import * as replication_utils from '../replication/replication-utils.js';
|
|
7
|
-
import * as types from '../types/types.js';
|
|
8
|
-
import * as pg_utils from '../utils/pgwire_utils.js';
|
|
9
7
|
import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
10
8
|
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
|
+
import * as types from '../types/types.js';
|
|
11
10
|
|
|
12
11
|
export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
13
12
|
connectionTag: string;
|
|
@@ -53,7 +52,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
53
52
|
};
|
|
54
53
|
|
|
55
54
|
try {
|
|
56
|
-
await
|
|
55
|
+
await lib_postgres.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`);
|
|
57
56
|
} catch (e) {
|
|
58
57
|
return {
|
|
59
58
|
...base,
|
|
@@ -94,7 +93,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
94
93
|
try {
|
|
95
94
|
const result = await this.pool.query({
|
|
96
95
|
statement: query,
|
|
97
|
-
params: params.map(
|
|
96
|
+
params: params.map(lib_postgres.autoParameter)
|
|
98
97
|
});
|
|
99
98
|
|
|
100
99
|
return service_types.internal_routes.ExecuteSqlResponse.encode({
|
|
@@ -146,7 +145,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
146
145
|
if (tablePattern.isWildcard) {
|
|
147
146
|
patternResult.tables = [];
|
|
148
147
|
const prefix = tablePattern.tablePrefix;
|
|
149
|
-
const results = await
|
|
148
|
+
const results = await lib_postgres.retriedQuery(this.pool, {
|
|
150
149
|
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
151
150
|
FROM pg_class c
|
|
152
151
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
@@ -169,7 +168,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
169
168
|
patternResult.tables.push(details);
|
|
170
169
|
}
|
|
171
170
|
} else {
|
|
172
|
-
const results = await
|
|
171
|
+
const results = await lib_postgres.retriedQuery(this.pool, {
|
|
173
172
|
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
174
173
|
FROM pg_class c
|
|
175
174
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
@@ -215,7 +214,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
215
214
|
async getReplicationLag(options: api.ReplicationLagOptions): Promise<number | undefined> {
|
|
216
215
|
const { bucketStorage } = options;
|
|
217
216
|
const slotName = bucketStorage.slot_name;
|
|
218
|
-
const results = await
|
|
217
|
+
const results = await lib_postgres.retriedQuery(this.pool, {
|
|
219
218
|
statement: `SELECT
|
|
220
219
|
slot_name,
|
|
221
220
|
confirmed_flush_lsn,
|
|
@@ -237,7 +236,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
237
236
|
// However, on Aurora (Postgres compatible), it can return an entirely different LSN,
|
|
238
237
|
// causing the write checkpoints to never be replicated back to the client.
|
|
239
238
|
// For those, we need to use pg_current_wal_lsn() instead.
|
|
240
|
-
const { results } = await
|
|
239
|
+
const { results } = await lib_postgres.retriedQuery(
|
|
241
240
|
this.pool,
|
|
242
241
|
{ statement: `SELECT pg_current_wal_lsn() as lsn` },
|
|
243
242
|
{ statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` }
|
|
@@ -250,7 +249,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
250
249
|
|
|
251
250
|
async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
|
|
252
251
|
// https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
|
|
253
|
-
const results = await
|
|
252
|
+
const results = await lib_postgres.retriedQuery(
|
|
254
253
|
this.pool,
|
|
255
254
|
`SELECT
|
|
256
255
|
tbl.schemaname,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import { auth } from '@powersync/service-core';
|
|
2
3
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
3
4
|
import * as jose from 'jose';
|
|
4
5
|
|
|
5
6
|
import * as types from '../types/types.js';
|
|
6
|
-
import * as pgwire_utils from '../utils/pgwire_utils.js';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Fetches key from the Supabase database.
|
|
@@ -39,7 +39,7 @@ export class SupabaseKeyCollector implements auth.KeyCollector {
|
|
|
39
39
|
let row: { jwt_secret: string };
|
|
40
40
|
try {
|
|
41
41
|
const rows = pgwire.pgwireRows(
|
|
42
|
-
await
|
|
42
|
+
await lib_postgres.retriedQuery(this.pool, `SELECT current_setting('app.settings.jwt_secret') as jwt_secret`)
|
|
43
43
|
);
|
|
44
44
|
row = rows[0] as any;
|
|
45
45
|
} catch (e) {
|
package/src/index.ts
CHANGED
|
@@ -6,10 +6,10 @@ import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactor
|
|
|
6
6
|
import { PgManager } from '../replication/PgManager.js';
|
|
7
7
|
import { PostgresErrorRateLimiter } from '../replication/PostgresErrorRateLimiter.js';
|
|
8
8
|
import { checkSourceConfiguration, cleanUpReplicationSlot } from '../replication/replication-utils.js';
|
|
9
|
+
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
10
|
import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
|
|
10
11
|
import * as types from '../types/types.js';
|
|
11
12
|
import { PostgresConnectionConfig } from '../types/types.js';
|
|
12
|
-
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
13
13
|
|
|
14
14
|
export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
|
|
15
15
|
constructor() {
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import { container, errors, logger } from '@powersync/lib-services-framework';
|
|
2
3
|
import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core';
|
|
3
4
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
4
5
|
import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
5
6
|
import * as pg_utils from '../utils/pgwire_utils.js';
|
|
7
|
+
|
|
6
8
|
import { PgManager } from './PgManager.js';
|
|
7
9
|
import { getPgOutputRelation, getRelId } from './PgRelation.js';
|
|
8
10
|
import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js';
|
|
@@ -63,7 +65,7 @@ export class WalStream {
|
|
|
63
65
|
// Ping to speed up cancellation of streaming replication
|
|
64
66
|
// We're not using pg_snapshot here, since it could be in the middle of
|
|
65
67
|
// an initial replication transaction.
|
|
66
|
-
const promise =
|
|
68
|
+
const promise = lib_postgres.retriedQuery(
|
|
67
69
|
this.connections.pool,
|
|
68
70
|
`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`
|
|
69
71
|
);
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
2
|
|
|
3
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
4
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
3
5
|
import { PatternResult, storage } from '@powersync/service-core';
|
|
4
|
-
import * as pgwire_utils from '../utils/pgwire_utils.js';
|
|
5
|
-
import { ReplicationIdentity } from './PgRelation.js';
|
|
6
6
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
7
7
|
import * as service_types from '@powersync/service-types';
|
|
8
|
-
import
|
|
9
|
-
import * as util from '../utils/pgwire_utils.js';
|
|
10
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
8
|
+
import { ReplicationIdentity } from './PgRelation.js';
|
|
11
9
|
|
|
12
10
|
export interface ReplicaIdentityResult {
|
|
13
11
|
replicationColumns: storage.ColumnDescriptor[];
|
|
@@ -20,7 +18,7 @@ export async function getPrimaryKeyColumns(
|
|
|
20
18
|
mode: 'primary' | 'replident'
|
|
21
19
|
): Promise<storage.ColumnDescriptor[]> {
|
|
22
20
|
const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
|
|
23
|
-
const attrRows = await
|
|
21
|
+
const attrRows = await lib_postgres.retriedQuery(db, {
|
|
24
22
|
statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum
|
|
25
23
|
FROM pg_index i
|
|
26
24
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey)
|
|
@@ -41,7 +39,7 @@ export async function getPrimaryKeyColumns(
|
|
|
41
39
|
}
|
|
42
40
|
|
|
43
41
|
export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<storage.ColumnDescriptor[]> {
|
|
44
|
-
const attrRows = await
|
|
42
|
+
const attrRows = await lib_postgres.retriedQuery(db, {
|
|
45
43
|
statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum
|
|
46
44
|
FROM pg_attribute a
|
|
47
45
|
JOIN pg_type t ON a.atttypid = t.oid
|
|
@@ -62,7 +60,7 @@ export async function getReplicationIdentityColumns(
|
|
|
62
60
|
db: pgwire.PgClient,
|
|
63
61
|
relationId: number
|
|
64
62
|
): Promise<ReplicaIdentityResult> {
|
|
65
|
-
const rows = await
|
|
63
|
+
const rows = await lib_postgres.retriedQuery(db, {
|
|
66
64
|
statement: `SELECT CASE relreplident
|
|
67
65
|
WHEN 'd' THEN 'default'
|
|
68
66
|
WHEN 'n' THEN 'nothing'
|
|
@@ -95,7 +93,7 @@ WHERE oid = $1::oid LIMIT 1`,
|
|
|
95
93
|
|
|
96
94
|
export async function checkSourceConfiguration(db: pgwire.PgClient, publicationName: string): Promise<void> {
|
|
97
95
|
// Check basic config
|
|
98
|
-
await
|
|
96
|
+
await lib_postgres.retriedQuery(
|
|
99
97
|
db,
|
|
100
98
|
`DO $$
|
|
101
99
|
BEGIN
|
|
@@ -113,7 +111,7 @@ $$ LANGUAGE plpgsql;`
|
|
|
113
111
|
);
|
|
114
112
|
|
|
115
113
|
// Check that publication exists
|
|
116
|
-
const rs = await
|
|
114
|
+
const rs = await lib_postgres.retriedQuery(db, {
|
|
117
115
|
statement: `SELECT * FROM pg_publication WHERE pubname = $1`,
|
|
118
116
|
params: [{ type: 'varchar', value: publicationName }]
|
|
119
117
|
});
|
|
@@ -158,7 +156,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr
|
|
|
158
156
|
if (tablePattern.isWildcard) {
|
|
159
157
|
patternResult.tables = [];
|
|
160
158
|
const prefix = tablePattern.tablePrefix;
|
|
161
|
-
const results = await
|
|
159
|
+
const results = await lib_postgres.retriedQuery(db, {
|
|
162
160
|
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
163
161
|
FROM pg_class c
|
|
164
162
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
@@ -189,7 +187,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr
|
|
|
189
187
|
patternResult.tables.push(details);
|
|
190
188
|
}
|
|
191
189
|
} else {
|
|
192
|
-
const results = await
|
|
190
|
+
const results = await lib_postgres.retriedQuery(db, {
|
|
193
191
|
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
194
192
|
FROM pg_class c
|
|
195
193
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
@@ -284,14 +282,14 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
|
|
|
284
282
|
|
|
285
283
|
let selectError = null;
|
|
286
284
|
try {
|
|
287
|
-
await
|
|
285
|
+
await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`);
|
|
288
286
|
} catch (e) {
|
|
289
287
|
selectError = { level: 'fatal', message: e.message };
|
|
290
288
|
}
|
|
291
289
|
|
|
292
290
|
let replicateError = null;
|
|
293
291
|
|
|
294
|
-
const publications = await
|
|
292
|
+
const publications = await lib_postgres.retriedQuery(db, {
|
|
295
293
|
statement: `SELECT tablename FROM pg_publication_tables WHERE pubname = $1 AND schemaname = $2 AND tablename = $3`,
|
|
296
294
|
params: [
|
|
297
295
|
{ type: 'varchar', value: publicationName },
|
package/src/types/types.ts
CHANGED
|
@@ -1,56 +1,18 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
1
2
|
import * as service_types from '@powersync/service-types';
|
|
2
3
|
import * as t from 'ts-codec';
|
|
3
|
-
import * as urijs from 'uri-js';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
hostname: string;
|
|
12
|
-
port: number;
|
|
13
|
-
database: string;
|
|
14
|
-
|
|
15
|
-
username: string;
|
|
16
|
-
password: string;
|
|
17
|
-
|
|
18
|
-
sslmode: 'verify-full' | 'verify-ca' | 'disable';
|
|
19
|
-
cacert: string | undefined;
|
|
20
|
-
|
|
21
|
-
client_certificate: string | undefined;
|
|
22
|
-
client_private_key: string | undefined;
|
|
23
|
-
}
|
|
5
|
+
// Maintain backwards compatibility by exporting these
|
|
6
|
+
export const validatePort = lib_postgres.validatePort;
|
|
7
|
+
export const baseUri = lib_postgres.baseUri;
|
|
8
|
+
export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig;
|
|
9
|
+
export const POSTGRES_CONNECTION_TYPE = lib_postgres.POSTGRES_CONNECTION_TYPE;
|
|
24
10
|
|
|
25
11
|
export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and(
|
|
12
|
+
lib_postgres.BasePostgresConnectionConfig
|
|
13
|
+
).and(
|
|
26
14
|
t.object({
|
|
27
|
-
|
|
28
|
-
/** Unique identifier for the connection - optional when a single connection is present. */
|
|
29
|
-
id: t.string.optional(),
|
|
30
|
-
/** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */
|
|
31
|
-
tag: t.string.optional(),
|
|
32
|
-
uri: t.string.optional(),
|
|
33
|
-
hostname: t.string.optional(),
|
|
34
|
-
port: service_types.configFile.portCodec.optional(),
|
|
35
|
-
username: t.string.optional(),
|
|
36
|
-
password: t.string.optional(),
|
|
37
|
-
database: t.string.optional(),
|
|
38
|
-
|
|
39
|
-
/** Defaults to verify-full */
|
|
40
|
-
sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(),
|
|
41
|
-
/** Required for verify-ca, optional for verify-full */
|
|
42
|
-
cacert: t.string.optional(),
|
|
43
|
-
|
|
44
|
-
client_certificate: t.string.optional(),
|
|
45
|
-
client_private_key: t.string.optional(),
|
|
46
|
-
|
|
47
|
-
/** Expose database credentials */
|
|
48
|
-
demo_database: t.boolean.optional(),
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Prefix for the slot name. Defaults to "powersync_"
|
|
52
|
-
*/
|
|
53
|
-
slot_name_prefix: t.string.optional()
|
|
15
|
+
// Add any replication connection specific config here in future
|
|
54
16
|
})
|
|
55
17
|
);
|
|
56
18
|
|
|
@@ -64,95 +26,19 @@ export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig
|
|
|
64
26
|
*/
|
|
65
27
|
export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig;
|
|
66
28
|
|
|
29
|
+
export function isPostgresConfig(
|
|
30
|
+
config: service_types.configFile.DataSourceConfig
|
|
31
|
+
): config is PostgresConnectionConfig {
|
|
32
|
+
return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE;
|
|
33
|
+
}
|
|
34
|
+
|
|
67
35
|
/**
|
|
68
36
|
* Validate and normalize connection options.
|
|
69
37
|
*
|
|
70
38
|
* Returns destructured options.
|
|
71
39
|
*/
|
|
72
|
-
export function normalizeConnectionConfig(options: PostgresConnectionConfig)
|
|
73
|
-
let uri: urijs.URIComponents;
|
|
74
|
-
if (options.uri) {
|
|
75
|
-
uri = urijs.parse(options.uri);
|
|
76
|
-
if (uri.scheme != 'postgresql' && uri.scheme != 'postgres') {
|
|
77
|
-
`Invalid URI - protocol must be postgresql, got ${uri.scheme}`;
|
|
78
|
-
} else if (uri.scheme != 'postgresql') {
|
|
79
|
-
uri.scheme = 'postgresql';
|
|
80
|
-
}
|
|
81
|
-
} else {
|
|
82
|
-
uri = urijs.parse('postgresql:///');
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
const hostname = options.hostname ?? uri.host ?? '';
|
|
86
|
-
const port = validatePort(options.port ?? uri.port ?? 5432);
|
|
87
|
-
|
|
88
|
-
const database = options.database ?? uri.path?.substring(1) ?? '';
|
|
89
|
-
|
|
90
|
-
const [uri_username, uri_password] = (uri.userinfo ?? '').split(':');
|
|
91
|
-
|
|
92
|
-
const username = options.username ?? uri_username ?? '';
|
|
93
|
-
const password = options.password ?? uri_password ?? '';
|
|
94
|
-
|
|
95
|
-
const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI
|
|
96
|
-
const cacert = options.cacert;
|
|
97
|
-
|
|
98
|
-
if (sslmode == 'verify-ca' && cacert == null) {
|
|
99
|
-
throw new Error('Explicit cacert is required for sslmode=verify-ca');
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
if (hostname == '') {
|
|
103
|
-
throw new Error(`hostname required`);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (username == '') {
|
|
107
|
-
throw new Error(`username required`);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
if (password == '') {
|
|
111
|
-
throw new Error(`password required`);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
if (database == '') {
|
|
115
|
-
throw new Error(`database required`);
|
|
116
|
-
}
|
|
117
|
-
|
|
40
|
+
export function normalizeConnectionConfig(options: PostgresConnectionConfig) {
|
|
118
41
|
return {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
hostname,
|
|
123
|
-
port,
|
|
124
|
-
database,
|
|
125
|
-
|
|
126
|
-
username,
|
|
127
|
-
password,
|
|
128
|
-
sslmode,
|
|
129
|
-
cacert,
|
|
130
|
-
|
|
131
|
-
client_certificate: options.client_certificate ?? undefined,
|
|
132
|
-
client_private_key: options.client_private_key ?? undefined
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/**
|
|
137
|
-
* Check whether the port is in a "safe" range.
|
|
138
|
-
*
|
|
139
|
-
* We do not support connecting to "privileged" ports.
|
|
140
|
-
*/
|
|
141
|
-
export function validatePort(port: string | number): number {
|
|
142
|
-
if (typeof port == 'string') {
|
|
143
|
-
port = parseInt(port);
|
|
144
|
-
}
|
|
145
|
-
if (port < 1024) {
|
|
146
|
-
throw new Error(`Port ${port} not supported`);
|
|
147
|
-
}
|
|
148
|
-
return port;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* Construct a postgres URI, without username, password or ssl options.
|
|
153
|
-
*
|
|
154
|
-
* Only contains hostname, port, database.
|
|
155
|
-
*/
|
|
156
|
-
export function baseUri(options: NormalizedPostgresConnectionConfig) {
|
|
157
|
-
return `postgresql://${options.hostname}:${options.port}/${options.database}`;
|
|
42
|
+
...lib_postgres.normalizeConnectionConfig(options)
|
|
43
|
+
} satisfies NormalizedPostgresConnectionConfig;
|
|
158
44
|
}
|