@powersync/service-module-postgres 0.0.0-dev-20251111093449 → 0.0.0-dev-20251124070259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -5
- package/dist/api/PostgresRouteAPIAdapter.js +1 -2
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/module/PostgresModule.d.ts +0 -1
- package/dist/module/PostgresModule.js +4 -9
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.d.ts +3 -5
- package/dist/replication/ConnectionManagerFactory.js +11 -9
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/PgManager.d.ts +6 -4
- package/dist/replication/PgManager.js +17 -7
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/PostgresErrorRateLimiter.js +6 -1
- package/dist/replication/PostgresErrorRateLimiter.js.map +1 -1
- package/dist/replication/WalStream.js +5 -1
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/replication/WalStreamReplicationJob.d.ts +1 -2
- package/dist/replication/WalStreamReplicationJob.js +47 -70
- package/dist/replication/WalStreamReplicationJob.js.map +1 -1
- package/dist/types/resolver.d.ts +9 -3
- package/dist/types/resolver.js +26 -5
- package/dist/types/resolver.js.map +1 -1
- package/dist/types/types.d.ts +1 -4
- package/dist/types/types.js.map +1 -1
- package/package.json +11 -11
- package/src/api/PostgresRouteAPIAdapter.ts +1 -1
- package/src/module/PostgresModule.ts +4 -9
- package/src/replication/ConnectionManagerFactory.ts +14 -13
- package/src/replication/PgManager.ts +22 -11
- package/src/replication/PostgresErrorRateLimiter.ts +5 -1
- package/src/replication/WalStream.ts +5 -1
- package/src/replication/WalStreamReplicationJob.ts +48 -68
- package/src/types/resolver.ts +27 -6
- package/src/types/types.ts +1 -5
- package/test/src/pg_test.test.ts +2 -2
- package/test/src/slow_tests.test.ts +2 -2
- package/test/src/wal_stream.test.ts +14 -3
- package/test/src/wal_stream_utils.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,21 +1,15 @@
|
|
|
1
|
-
import { container, logger
|
|
1
|
+
import { container, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { MissingReplicationSlotError, sendKeepAlive, WalStream } from './WalStream.js';
|
|
3
3
|
import { replication } from '@powersync/service-core';
|
|
4
4
|
import { getApplicationName } from '../utils/application-name.js';
|
|
5
5
|
export class WalStreamReplicationJob extends replication.AbstractReplicationJob {
|
|
6
6
|
connectionFactory;
|
|
7
|
-
connectionManager;
|
|
7
|
+
connectionManager = null;
|
|
8
8
|
lastStream = null;
|
|
9
9
|
constructor(options) {
|
|
10
10
|
super(options);
|
|
11
11
|
this.logger = logger.child({ prefix: `[${this.slotName}] ` });
|
|
12
12
|
this.connectionFactory = options.connectionFactory;
|
|
13
|
-
this.connectionManager = this.connectionFactory.create({
|
|
14
|
-
// Pool connections are only used intermittently.
|
|
15
|
-
idleTimeout: 30_000,
|
|
16
|
-
maxSize: 2,
|
|
17
|
-
applicationName: getApplicationName()
|
|
18
|
-
});
|
|
19
13
|
}
|
|
20
14
|
/**
|
|
21
15
|
* Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
|
|
@@ -30,11 +24,13 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
30
24
|
* **This may be a bug in pgwire or how we're using it.
|
|
31
25
|
*/
|
|
32
26
|
async keepAlive() {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
27
|
+
if (this.connectionManager) {
|
|
28
|
+
try {
|
|
29
|
+
await sendKeepAlive(this.connectionManager.pool);
|
|
30
|
+
}
|
|
31
|
+
catch (e) {
|
|
32
|
+
this.logger.warn(`KeepAlive failed, unable to post to WAL`, e);
|
|
33
|
+
}
|
|
38
34
|
}
|
|
39
35
|
}
|
|
40
36
|
get slotName() {
|
|
@@ -42,33 +38,55 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
42
38
|
}
|
|
43
39
|
async replicate() {
|
|
44
40
|
try {
|
|
45
|
-
await this.
|
|
41
|
+
await this.replicateOnce();
|
|
46
42
|
}
|
|
47
43
|
catch (e) {
|
|
48
44
|
// Fatal exception
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
45
|
+
if (!this.isStopped) {
|
|
46
|
+
// Ignore aborted errors
|
|
47
|
+
this.logger.error(`Replication error`, e);
|
|
48
|
+
if (e.cause != null) {
|
|
49
|
+
// Example:
|
|
50
|
+
// PgError.conn_ended: Unable to do postgres query on ended connection
|
|
51
|
+
// at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
|
|
52
|
+
// at stream.next (<anonymous>)
|
|
53
|
+
// at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
|
|
54
|
+
// at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
|
|
55
|
+
// at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
|
|
56
|
+
// ...
|
|
57
|
+
// cause: TypeError: match is not iterable
|
|
58
|
+
// at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
|
|
59
|
+
// at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
|
|
60
|
+
// at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
|
|
61
|
+
// at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
|
|
62
|
+
// at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
|
|
63
|
+
// at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
|
|
64
|
+
// at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
|
|
65
|
+
// [Symbol(pg.ErrorCode)]: 'conn_ended',
|
|
66
|
+
// [Symbol(pg.ErrorResponse)]: undefined
|
|
67
|
+
// }
|
|
68
|
+
// Without this additional log, the cause would not be visible in the logs.
|
|
69
|
+
this.logger.error(`cause`, e.cause);
|
|
52
70
|
}
|
|
53
|
-
|
|
54
|
-
|
|
71
|
+
// Report the error if relevant, before retrying
|
|
72
|
+
container.reporter.captureException(e, {
|
|
73
|
+
metadata: {
|
|
74
|
+
replication_slot: this.slotName
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
// This sets the retry delay
|
|
78
|
+
this.rateLimiter.reportError(e);
|
|
79
|
+
}
|
|
55
80
|
if (e instanceof MissingReplicationSlotError) {
|
|
56
81
|
// This stops replication on this slot and restarts with a new slot
|
|
57
82
|
await this.options.storage.factory.restartReplication(this.storage.group_id);
|
|
58
83
|
}
|
|
84
|
+
// No need to rethrow - the error is already logged, and retry behavior is the same on error
|
|
59
85
|
}
|
|
60
86
|
finally {
|
|
61
87
|
this.abortController.abort();
|
|
62
88
|
}
|
|
63
89
|
}
|
|
64
|
-
async replicateLoop() {
|
|
65
|
-
while (!this.isStopped) {
|
|
66
|
-
await this.replicateOnce();
|
|
67
|
-
if (!this.isStopped) {
|
|
68
|
-
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
90
|
async replicateOnce() {
|
|
73
91
|
// New connections on every iteration (every error with retry),
|
|
74
92
|
// otherwise we risk repeating errors related to the connection,
|
|
@@ -79,6 +97,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
79
97
|
maxSize: 2,
|
|
80
98
|
applicationName: getApplicationName()
|
|
81
99
|
});
|
|
100
|
+
this.connectionManager = connectionManager;
|
|
82
101
|
try {
|
|
83
102
|
await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
|
|
84
103
|
if (this.isStopped) {
|
|
@@ -94,50 +113,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob
|
|
|
94
113
|
this.lastStream = stream;
|
|
95
114
|
await stream.replicate();
|
|
96
115
|
}
|
|
97
|
-
catch (e) {
|
|
98
|
-
if (this.isStopped && e instanceof ReplicationAbortedError) {
|
|
99
|
-
// Ignore aborted errors
|
|
100
|
-
return;
|
|
101
|
-
}
|
|
102
|
-
this.logger.error(`Replication error`, e);
|
|
103
|
-
if (e.cause != null) {
|
|
104
|
-
// Example:
|
|
105
|
-
// PgError.conn_ended: Unable to do postgres query on ended connection
|
|
106
|
-
// at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
|
|
107
|
-
// at stream.next (<anonymous>)
|
|
108
|
-
// at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
|
|
109
|
-
// at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
|
|
110
|
-
// at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
|
|
111
|
-
// ...
|
|
112
|
-
// cause: TypeError: match is not iterable
|
|
113
|
-
// at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
|
|
114
|
-
// at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
|
|
115
|
-
// at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
|
|
116
|
-
// at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
|
|
117
|
-
// at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
|
|
118
|
-
// at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
|
|
119
|
-
// at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
|
|
120
|
-
// [Symbol(pg.ErrorCode)]: 'conn_ended',
|
|
121
|
-
// [Symbol(pg.ErrorResponse)]: undefined
|
|
122
|
-
// }
|
|
123
|
-
// Without this additional log, the cause would not be visible in the logs.
|
|
124
|
-
this.logger.error(`cause`, e.cause);
|
|
125
|
-
}
|
|
126
|
-
if (e instanceof MissingReplicationSlotError) {
|
|
127
|
-
throw e;
|
|
128
|
-
}
|
|
129
|
-
else {
|
|
130
|
-
// Report the error if relevant, before retrying
|
|
131
|
-
container.reporter.captureException(e, {
|
|
132
|
-
metadata: {
|
|
133
|
-
replication_slot: this.slotName
|
|
134
|
-
}
|
|
135
|
-
});
|
|
136
|
-
// This sets the retry delay
|
|
137
|
-
this.rateLimiter?.reportError(e);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
116
|
finally {
|
|
117
|
+
this.connectionManager = null;
|
|
141
118
|
await connectionManager.end();
|
|
142
119
|
}
|
|
143
120
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WalStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/WalStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"WalStreamReplicationJob.js","sourceRoot":"","sources":["../../src/replication/WalStreamReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAA2B,MAAM,mCAAmC,CAAC;AAE/F,OAAO,EAAE,2BAA2B,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEvF,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAMlE,MAAM,OAAO,uBAAwB,SAAQ,WAAW,CAAC,sBAAsB;IACrE,iBAAiB,CAA2B;IAC5C,iBAAiB,GAAqB,IAAI,CAAC;IAC3C,UAAU,GAAqB,IAAI,CAAC;IAE5C,YAAY,OAAuC;QACjD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAElB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,wBAAwB;gBAExB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;oBACpB,WAAW;oBACX,sEAAsE;oBACtE,2KAA2K;oBAC3K,mCAAmC;oBACnC,4KAA4K;oBAC5K,0KAA0K;oBAC1K,yHAAyH;oBACzH,UAAU;oBACV,4CAA4C;oBAC5C,4FAA4F;oBAC5F,6FAA6F;oBAC7F,iGAAiG;oBACjG,oLAAoL;oBACpL,qLAAqL;oBACrL,sFAAsF;oBACtF,qLAAqL;oBACrL,0CAA0C;oBAC1C,0CAA0C;oBAC1C,IAAI;oBACJ,2EAA2E;oBAC3E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBACD,gDAAgD;gBAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACrC,QAAQ,EAAE;wBACR,gBAAgB,EAAE,IAAI,CAAC,QAAQ;qBAChC;iBACF,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,mEAAmE;gBACnE,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC/E,CAAC;YAED,4FAA4F;QAC9F,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACtD,iDAAiD;YACjD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,OAAO,IAAI,CAAC,UAAU,EAAE,uBAAuB,EAAE,CAAC;IACpD,CAAC;CACF"}
|
package/dist/types/resolver.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DatabaseInputRow, SqliteInputRow } from '@powersync/service-sync-rules';
|
|
2
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
+
import { DatabaseInputRow, SqliteInputRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import { CustomTypeRegistry } from './registry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Resolves descriptions used to decode values for custom postgres types.
|
|
@@ -7,10 +7,10 @@ import { CustomTypeRegistry } from './registry.js';
|
|
|
7
7
|
* Custom types are resolved from the source database, which also involves crawling inner types (e.g. for composites).
|
|
8
8
|
*/
|
|
9
9
|
export declare class PostgresTypeResolver {
|
|
10
|
-
readonly registry: CustomTypeRegistry;
|
|
11
10
|
private readonly pool;
|
|
12
11
|
private cachedVersion;
|
|
13
|
-
|
|
12
|
+
readonly registry: CustomTypeRegistry;
|
|
13
|
+
constructor(pool: pgwire.PgClient);
|
|
14
14
|
private fetchVersion;
|
|
15
15
|
/**
|
|
16
16
|
* @returns Whether the Postgres instance this type cache is connected to has support for the multirange type (which
|
|
@@ -38,10 +38,16 @@ export declare class PostgresTypeResolver {
|
|
|
38
38
|
* @param message
|
|
39
39
|
*/
|
|
40
40
|
constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteInputRow | undefined;
|
|
41
|
+
constructRowRecord(columnMap: Record<string, number>, tupleRaw: Record<string, any>): SqliteInputRow;
|
|
41
42
|
/**
|
|
42
43
|
* We need a high level of control over how values are decoded, to make sure there is no loss
|
|
43
44
|
* of precision in the process.
|
|
44
45
|
*/
|
|
45
46
|
decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record<string, any>): DatabaseInputRow;
|
|
47
|
+
/**
|
|
48
|
+
* We need a high level of control over how values are decoded, to make sure there is no loss
|
|
49
|
+
* of precision in the process.
|
|
50
|
+
*/
|
|
51
|
+
private decodeTupleForTable;
|
|
46
52
|
private static minVersionForMultirange;
|
|
47
53
|
}
|
package/dist/types/resolver.js
CHANGED
|
@@ -1,19 +1,18 @@
|
|
|
1
|
-
import { toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
2
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
3
|
-
import {
|
|
2
|
+
import { toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
4
3
|
import semver from 'semver';
|
|
5
4
|
import { getServerVersion } from '../utils/postgres_version.js';
|
|
5
|
+
import { CustomTypeRegistry } from './registry.js';
|
|
6
6
|
/**
|
|
7
7
|
* Resolves descriptions used to decode values for custom postgres types.
|
|
8
8
|
*
|
|
9
9
|
* Custom types are resolved from the source database, which also involves crawling inner types (e.g. for composites).
|
|
10
10
|
*/
|
|
11
11
|
export class PostgresTypeResolver {
|
|
12
|
-
registry;
|
|
13
12
|
pool;
|
|
14
13
|
cachedVersion = null;
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
registry;
|
|
15
|
+
constructor(pool) {
|
|
17
16
|
this.pool = pool;
|
|
18
17
|
this.registry = new CustomTypeRegistry();
|
|
19
18
|
}
|
|
@@ -168,6 +167,10 @@ WHERE a.attnum > 0
|
|
|
168
167
|
const record = this.decodeTuple(message.relation, rawData);
|
|
169
168
|
return toSyncRulesRow(record);
|
|
170
169
|
}
|
|
170
|
+
constructRowRecord(columnMap, tupleRaw) {
|
|
171
|
+
const record = this.decodeTupleForTable(columnMap, tupleRaw);
|
|
172
|
+
return toSyncRulesRow(record);
|
|
173
|
+
}
|
|
171
174
|
/**
|
|
172
175
|
* We need a high level of control over how values are decoded, to make sure there is no loss
|
|
173
176
|
* of precision in the process.
|
|
@@ -186,6 +189,24 @@ WHERE a.attnum > 0
|
|
|
186
189
|
}
|
|
187
190
|
return result;
|
|
188
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* We need a high level of control over how values are decoded, to make sure there is no loss
|
|
194
|
+
* of precision in the process.
|
|
195
|
+
*/
|
|
196
|
+
decodeTupleForTable(columnMap, tupleRaw) {
|
|
197
|
+
let result = {};
|
|
198
|
+
for (let columnName in tupleRaw) {
|
|
199
|
+
const rawval = tupleRaw[columnName];
|
|
200
|
+
const typeOid = columnMap[columnName];
|
|
201
|
+
if (typeof rawval == 'string' && typeOid) {
|
|
202
|
+
result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid);
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
result[columnName] = rawval;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return result;
|
|
209
|
+
}
|
|
189
210
|
static minVersionForMultirange = semver.parse('14.0.0');
|
|
190
211
|
}
|
|
191
212
|
//# sourceMappingURL=resolver.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../src/types/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoC,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACjG,OAAO,
|
|
1
|
+
{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../src/types/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAoC,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACjG,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAIF;IAHrB,aAAa,GAAyB,IAAI,CAAC;IAC1C,QAAQ,CAAqB;IAEtC,YAA6B,IAAqB;QAArB,SAAI,GAAJ,IAAI,CAAiB;QAChD,IAAI,CAAC,QAAQ,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,aAAa,GAAG,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,IAAI,CAAC,aAAc,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,UAAU,CAAC,IAAc;QACpC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE3D,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5D,2FAA2F;QAC3F,MAAM,cAAc,GAAG,yGAAyG,CAAC;QACjI,MAAM,SAAS,GAAG;;;;;;;;;;;;UAYZ,iBAAiB,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE;;;;;CAKhD,CAAC;QAEE,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YAC7F,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,EAAE;gBAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACvF,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAElC,QAAQ,GAAG,CAAC,OAAO,EAAE,CAAC;oBACpB,KAAK,GAAG;wBACN,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;wBAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;4BAC9B,gDAAgD;4BAChD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;4BACnC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gCACf,2FAA2F;gCAC3F,WAAW,CAAC,KAAK,CAAC,CAAC;gCACnB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;oCACrB,IAAI,EAAE,OAAO;oCACb,OAAO,EAAE,KAAK;oCACd,iBAAiB,EAAG,KAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;oCAClD,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;iCAC5C,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;wBACD,MAAM;oBACR,KAAK,GAAG;wBACN,wDAAwD;wBACxD,MAAM,QAAQ,GAAuC,EAAE,CAAC;wBACxD,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;4BAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;4BAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;4BAChC,WAAW,CAAC,MAAM,CAAC,CAAC;wBACtB,CAAC;wBAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,QAAQ;4BACjB,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;yBAC5C,CAAC,CAAC;wBACH,MAAM;oBACR,KAAK,GAAG;wBACN,wGAAwG;wBACxG,wCAAwC;wBACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAChC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACxC,WAAW,CAAC,KAAK,CAAC,CAAC;wBACnB,MAAM;oBACR,KAAK,GAAG,CAAC;oBACT,KAAK,GAAG,CAAC,CAAC,CAAC;wBACT,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;4BACrB,IAAI,EAAE,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY;4BACjD,OAAO,EAAE,KAAK;4BACd,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB;yBAC5C,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,GAAG,YAAY,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,mBAAmB;QAC9B,MAAM,GAAG,GAAG;;;;;;;;;;KAUX,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACxD,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,OAAsD;QACzE,MAAM,OAAO,GAAI,OAAe,CAAC,QAAQ,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,OAAsD;QAC1E,MAAM,OAAO,GAAI,OAAe,CAAC,SAAS,CAAC;QAC3C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED,kBAAkB,CAAC,SAAiC,EAAE,QAA6B;QACjF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC7D,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,QAAiC,EAAE,QAA6B;QAC1E,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,OAAO,GAAI,QAAgB,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1E,IAAI,OAAO,MAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,mBAAmB,CAAC,SAAiC,EAAE,QAA6B;QAC1F,IAAI,MAAM,GAAwB,EAAE,CAAC;QACrC,KAAK,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;YACtC,IAAI,OAAO,MAAM,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACzC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,uBAAuB,GAAkB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
2
|
import * as service_types from '@powersync/service-types';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
|
-
import { CustomTypeRegistry } from './registry.js';
|
|
5
4
|
export declare const validatePort: typeof lib_postgres.validatePort;
|
|
6
5
|
export declare const baseUri: typeof lib_postgres.baseUri;
|
|
7
6
|
export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig;
|
|
@@ -60,9 +59,7 @@ export type PostgresConnectionConfig = t.Decoded<typeof PostgresConnectionConfig
|
|
|
60
59
|
/**
|
|
61
60
|
* Resolved version of {@link PostgresConnectionConfig}
|
|
62
61
|
*/
|
|
63
|
-
export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig
|
|
64
|
-
typeRegistry: CustomTypeRegistry;
|
|
65
|
-
};
|
|
62
|
+
export type ResolvedConnectionConfig = PostgresConnectionConfig & NormalizedPostgresConnectionConfig;
|
|
66
63
|
export declare function isPostgresConfig(config: service_types.configFile.DataSourceConfig): config is PostgresConnectionConfig;
|
|
67
64
|
/**
|
|
68
65
|
* Validate and normalize connection options.
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,sDAAsD;AACtD,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;AACtD,MAAM,CAAC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;AAE5C,MAAM,CAAC,MAAM,wBAAwB,GAAG,YAAY,CAAC,wBAAwB,CAAC;AAE9E,MAAM,CAAC,MAAM,wBAAwB,GAAG,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CACnF,YAAY,CAAC,4BAA4B,CAC1C,CAAC,GAAG,CACH,CAAC,CAAC,MAAM,CAAC;AACP,gEAAgE;CACjE,CAAC,CACH,CAAC;AAYF,MAAM,UAAU,gBAAgB,CAC9B,MAAiD;IAEjD,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAiC;IACzE,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,OAAO,CAAC;KACN,CAAC;AACjD,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "0.0.0-dev-
|
|
8
|
+
"version": "0.0.0-dev-20251124070259",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-ALv2",
|
|
11
11
|
"type": "module",
|
|
@@ -28,20 +28,20 @@
|
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
29
|
"uri-js": "^4.4.1",
|
|
30
30
|
"uuid": "^11.1.0",
|
|
31
|
-
"@powersync/lib-service-postgres": "0.0.0-dev-
|
|
32
|
-
"@powersync/lib-services-framework": "0.7.
|
|
33
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
34
|
-
"@powersync/service-jpgwire": "0.21.
|
|
31
|
+
"@powersync/lib-service-postgres": "0.0.0-dev-20251124070259",
|
|
32
|
+
"@powersync/lib-services-framework": "0.7.10",
|
|
33
|
+
"@powersync/service-core": "0.0.0-dev-20251124070259",
|
|
34
|
+
"@powersync/service-jpgwire": "0.21.6",
|
|
35
35
|
"@powersync/service-jsonbig": "0.17.12",
|
|
36
|
-
"@powersync/service-sync-rules": "0.29.
|
|
37
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
36
|
+
"@powersync/service-sync-rules": "0.29.7",
|
|
37
|
+
"@powersync/service-types": "0.0.0-dev-20251124070259"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@types/semver": "^7.5.4",
|
|
41
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
42
|
-
"@powersync/service-module-mongodb-storage": "0.0.0-dev-
|
|
43
|
-
"@powersync/lib-service-postgres": "0.0.0-dev-
|
|
44
|
-
"@powersync/service-module-postgres-storage": "0.0.0-dev-
|
|
41
|
+
"@powersync/service-core-tests": "0.0.0-dev-20251124070259",
|
|
42
|
+
"@powersync/service-module-mongodb-storage": "0.0.0-dev-20251124070259",
|
|
43
|
+
"@powersync/lib-service-postgres": "0.0.0-dev-20251124070259",
|
|
44
|
+
"@powersync/service-module-postgres-storage": "0.0.0-dev-20251124070259"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc -b",
|
|
@@ -34,7 +34,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
34
34
|
connectionTag?: string,
|
|
35
35
|
private config?: types.ResolvedConnectionConfig
|
|
36
36
|
) {
|
|
37
|
-
this.typeCache = new PostgresTypeResolver(
|
|
37
|
+
this.typeCache = new PostgresTypeResolver(pool);
|
|
38
38
|
this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
|
|
39
39
|
}
|
|
40
40
|
|
|
@@ -22,8 +22,6 @@ import { getApplicationName } from '../utils/application-name.js';
|
|
|
22
22
|
import { CustomTypeRegistry } from '../types/registry.js';
|
|
23
23
|
|
|
24
24
|
export class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
|
|
25
|
-
private customTypes: CustomTypeRegistry = new CustomTypeRegistry();
|
|
26
|
-
|
|
27
25
|
constructor() {
|
|
28
26
|
super({
|
|
29
27
|
name: 'Postgres',
|
|
@@ -51,7 +49,7 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
51
49
|
protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
|
|
52
50
|
const normalisedConfig = this.resolveConfig(this.decodedConfig!);
|
|
53
51
|
const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
|
|
54
|
-
const connectionFactory = new ConnectionManagerFactory(normalisedConfig
|
|
52
|
+
const connectionFactory = new ConnectionManagerFactory(normalisedConfig);
|
|
55
53
|
|
|
56
54
|
return new WalStreamReplicator({
|
|
57
55
|
id: this.getDefaultId(normalisedConfig.database),
|
|
@@ -69,8 +67,7 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
69
67
|
private resolveConfig(config: types.PostgresConnectionConfig): types.ResolvedConnectionConfig {
|
|
70
68
|
return {
|
|
71
69
|
...config,
|
|
72
|
-
...types.normalizeConnectionConfig(config)
|
|
73
|
-
typeRegistry: this.customTypes
|
|
70
|
+
...types.normalizeConnectionConfig(config)
|
|
74
71
|
};
|
|
75
72
|
}
|
|
76
73
|
|
|
@@ -79,8 +76,7 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
79
76
|
const connectionManager = new PgManager(normalisedConfig, {
|
|
80
77
|
idleTimeout: 30_000,
|
|
81
78
|
maxSize: 1,
|
|
82
|
-
applicationName: getApplicationName()
|
|
83
|
-
registry: this.customTypes
|
|
79
|
+
applicationName: getApplicationName()
|
|
84
80
|
});
|
|
85
81
|
|
|
86
82
|
try {
|
|
@@ -111,8 +107,7 @@ export class PostgresModule extends replication.ReplicationModule<types.Postgres
|
|
|
111
107
|
const connectionManager = new PgManager(normalizedConfig, {
|
|
112
108
|
idleTimeout: 30_000,
|
|
113
109
|
maxSize: 1,
|
|
114
|
-
applicationName: getApplicationName()
|
|
115
|
-
registry: new CustomTypeRegistry()
|
|
110
|
+
applicationName: getApplicationName()
|
|
116
111
|
});
|
|
117
112
|
const connection = await connectionManager.snapshotConnection();
|
|
118
113
|
try {
|
|
@@ -1,30 +1,31 @@
|
|
|
1
|
-
import { PgManager } from './PgManager.js';
|
|
2
|
-
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
3
|
-
import { PgPoolOptions } from '@powersync/service-jpgwire';
|
|
4
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
-
import {
|
|
2
|
+
import { PgPoolOptions } from '@powersync/service-jpgwire';
|
|
3
|
+
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
4
|
+
import { PgManager } from './PgManager.js';
|
|
6
5
|
|
|
7
6
|
export class ConnectionManagerFactory {
|
|
8
|
-
private readonly connectionManagers
|
|
7
|
+
private readonly connectionManagers = new Set<PgManager>();
|
|
9
8
|
public readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
|
|
10
9
|
|
|
11
|
-
constructor(
|
|
12
|
-
dbConnectionConfig: NormalizedPostgresConnectionConfig,
|
|
13
|
-
private readonly registry: CustomTypeRegistry
|
|
14
|
-
) {
|
|
10
|
+
constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig) {
|
|
15
11
|
this.dbConnectionConfig = dbConnectionConfig;
|
|
16
|
-
this.connectionManagers = [];
|
|
17
12
|
}
|
|
18
13
|
|
|
19
14
|
create(poolOptions: PgPoolOptions) {
|
|
20
|
-
const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions
|
|
21
|
-
this.connectionManagers.
|
|
15
|
+
const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions });
|
|
16
|
+
this.connectionManagers.add(manager);
|
|
17
|
+
|
|
18
|
+
manager.registerListener({
|
|
19
|
+
onEnded: () => {
|
|
20
|
+
this.connectionManagers.delete(manager);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
22
23
|
return manager;
|
|
23
24
|
}
|
|
24
25
|
|
|
25
26
|
async shutdown() {
|
|
26
27
|
logger.info('Shutting down Postgres connection Managers...');
|
|
27
|
-
for (const manager of this.connectionManagers) {
|
|
28
|
+
for (const manager of [...this.connectionManagers]) {
|
|
28
29
|
await manager.end();
|
|
29
30
|
}
|
|
30
31
|
logger.info('Postgres connection Managers shutdown completed.');
|
|
@@ -1,21 +1,23 @@
|
|
|
1
|
+
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
1
2
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
3
|
import semver from 'semver';
|
|
4
|
+
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
3
5
|
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
4
6
|
import { getApplicationName } from '../utils/application-name.js';
|
|
5
|
-
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
6
7
|
import { getServerVersion } from '../utils/postgres_version.js';
|
|
7
|
-
import { CustomTypeRegistry } from '../types/registry.js';
|
|
8
8
|
|
|
9
|
-
export interface PgManagerOptions extends pgwire.PgPoolOptions {
|
|
10
|
-
registry: CustomTypeRegistry;
|
|
11
|
-
}
|
|
9
|
+
export interface PgManagerOptions extends pgwire.PgPoolOptions {}
|
|
12
10
|
|
|
13
11
|
/**
|
|
14
12
|
* Shorter timeout for snapshot connections than for replication connections.
|
|
15
13
|
*/
|
|
16
14
|
const SNAPSHOT_SOCKET_TIMEOUT = 30_000;
|
|
17
15
|
|
|
18
|
-
export
|
|
16
|
+
export interface PgManagerListener {
|
|
17
|
+
onEnded(): void;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export class PgManager extends BaseObserver<PgManagerListener> {
|
|
19
21
|
/**
|
|
20
22
|
* Do not use this for any transactions.
|
|
21
23
|
*/
|
|
@@ -29,9 +31,10 @@ export class PgManager {
|
|
|
29
31
|
public options: NormalizedPostgresConnectionConfig,
|
|
30
32
|
public poolOptions: PgManagerOptions
|
|
31
33
|
) {
|
|
34
|
+
super();
|
|
32
35
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
33
36
|
this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
|
|
34
|
-
this.types = new PostgresTypeResolver(
|
|
37
|
+
this.types = new PostgresTypeResolver(this.pool);
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
public get connectionTag() {
|
|
@@ -83,8 +86,9 @@ export class PgManager {
|
|
|
83
86
|
for (let result of await Promise.allSettled([
|
|
84
87
|
this.pool.end(),
|
|
85
88
|
...this.connectionPromises.map(async (promise) => {
|
|
86
|
-
|
|
87
|
-
|
|
89
|
+
// Wait for connection attempts to finish, but do not throw connection errors here
|
|
90
|
+
const connection = await promise.catch((_) => {});
|
|
91
|
+
return await connection?.end();
|
|
88
92
|
})
|
|
89
93
|
])) {
|
|
90
94
|
// Throw the first error, if any
|
|
@@ -92,14 +96,18 @@ export class PgManager {
|
|
|
92
96
|
throw result.reason;
|
|
93
97
|
}
|
|
94
98
|
}
|
|
99
|
+
this.iterateListeners((listener) => {
|
|
100
|
+
listener.onEnded?.();
|
|
101
|
+
});
|
|
95
102
|
}
|
|
96
103
|
|
|
97
104
|
async destroy() {
|
|
98
105
|
this.pool.destroy();
|
|
99
106
|
for (let result of await Promise.allSettled([
|
|
100
107
|
...this.connectionPromises.map(async (promise) => {
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
// Wait for connection attempts to finish, but do not throw connection errors here
|
|
109
|
+
const connection = await promise.catch((_) => {});
|
|
110
|
+
return connection?.destroy();
|
|
103
111
|
})
|
|
104
112
|
])) {
|
|
105
113
|
// Throw the first error, if any
|
|
@@ -107,5 +115,8 @@ export class PgManager {
|
|
|
107
115
|
throw result.reason;
|
|
108
116
|
}
|
|
109
117
|
}
|
|
118
|
+
this.iterateListeners((listener) => {
|
|
119
|
+
listener.onEnded?.();
|
|
120
|
+
});
|
|
110
121
|
}
|
|
111
122
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { setTimeout } from 'timers/promises';
|
|
2
2
|
import { ErrorRateLimiter } from '@powersync/service-core';
|
|
3
|
+
import { MissingReplicationSlotError } from './WalStream.js';
|
|
3
4
|
|
|
4
5
|
export class PostgresErrorRateLimiter implements ErrorRateLimiter {
|
|
5
6
|
nextAllowed: number = Date.now();
|
|
@@ -17,7 +18,10 @@ export class PostgresErrorRateLimiter implements ErrorRateLimiter {
|
|
|
17
18
|
|
|
18
19
|
reportError(e: any): void {
|
|
19
20
|
const message = (e.message as string) ?? '';
|
|
20
|
-
if (
|
|
21
|
+
if (e instanceof MissingReplicationSlotError) {
|
|
22
|
+
// Short delay for a retrying (re-creating the slot)
|
|
23
|
+
this.setDelay(2_000);
|
|
24
|
+
} else if (message.includes('password authentication failed')) {
|
|
21
25
|
// Wait 15 minutes, to avoid triggering Supabase's fail2ban
|
|
22
26
|
this.setDelay(900_000);
|
|
23
27
|
} else if (message.includes('ENOTFOUND')) {
|
|
@@ -546,6 +546,7 @@ WHERE oid = $1::regclass`,
|
|
|
546
546
|
await q.initialize();
|
|
547
547
|
|
|
548
548
|
let columns: { i: number; name: string }[] = [];
|
|
549
|
+
let columnMap: Record<string, number> = {};
|
|
549
550
|
let hasRemainingData = true;
|
|
550
551
|
while (hasRemainingData) {
|
|
551
552
|
// Fetch 10k at a time.
|
|
@@ -565,6 +566,9 @@ WHERE oid = $1::regclass`,
|
|
|
565
566
|
columns = chunk.payload.map((c) => {
|
|
566
567
|
return { i: i++, name: c.name };
|
|
567
568
|
});
|
|
569
|
+
for (let column of chunk.payload) {
|
|
570
|
+
columnMap[column.name] = column.typeOid;
|
|
571
|
+
}
|
|
568
572
|
continue;
|
|
569
573
|
}
|
|
570
574
|
|
|
@@ -580,7 +584,7 @@ WHERE oid = $1::regclass`,
|
|
|
580
584
|
}
|
|
581
585
|
|
|
582
586
|
for (const inputRecord of WalStream.getQueryData(rows)) {
|
|
583
|
-
const record = this.syncRulesRecord(inputRecord);
|
|
587
|
+
const record = this.syncRulesRecord(this.connections.types.constructRowRecord(columnMap, inputRecord));
|
|
584
588
|
// This auto-flushes when the batch reaches its size limit
|
|
585
589
|
await batch.save({
|
|
586
590
|
tag: storage.SaveOperationTag.INSERT,
|