@powersync/service-module-postgres 0.16.10 → 0.16.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/api/PostgresRouteAPIAdapter.js +1 -2
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/module/PostgresModule.d.ts +0 -1
- package/dist/module/PostgresModule.js +4 -9
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/ConnectionManagerFactory.d.ts +3 -5
- package/dist/replication/ConnectionManagerFactory.js +11 -9
- package/dist/replication/ConnectionManagerFactory.js.map +1 -1
- package/dist/replication/PgManager.d.ts +6 -4
- package/dist/replication/PgManager.js +17 -7
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/PostgresErrorRateLimiter.js +6 -1
- package/dist/replication/PostgresErrorRateLimiter.js.map +1 -1
- package/dist/replication/WalStream.d.ts +2 -5
- package/dist/replication/WalStream.js +69 -96
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/replication/WalStreamReplicationJob.d.ts +1 -2
- package/dist/replication/WalStreamReplicationJob.js +47 -70
- package/dist/replication/WalStreamReplicationJob.js.map +1 -1
- package/dist/types/resolver.d.ts +9 -3
- package/dist/types/resolver.js +26 -5
- package/dist/types/resolver.js.map +1 -1
- package/dist/types/types.d.ts +1 -4
- package/dist/types/types.js.map +1 -1
- package/package.json +10 -10
- package/src/api/PostgresRouteAPIAdapter.ts +1 -1
- package/src/module/PostgresModule.ts +4 -9
- package/src/replication/ConnectionManagerFactory.ts +14 -13
- package/src/replication/PgManager.ts +22 -11
- package/src/replication/PostgresErrorRateLimiter.ts +5 -1
- package/src/replication/WalStream.ts +76 -117
- package/src/replication/WalStreamReplicationJob.ts +48 -68
- package/src/types/resolver.ts +27 -6
- package/src/types/types.ts +1 -5
- package/test/src/pg_test.test.ts +2 -2
- package/test/src/slow_tests.test.ts +2 -2
- package/test/src/wal_stream.test.ts +29 -10
- package/test/src/wal_stream_utils.ts +24 -5
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @powersync/service-module-postgres
|
|
2
2
|
|
|
3
|
+
## 0.16.12
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- fff0024: [Postgres] Fix custom type parsing on initial replication
|
|
8
|
+
- Updated dependencies [fff0024]
|
|
9
|
+
- @powersync/service-sync-rules@0.29.7
|
|
10
|
+
- @powersync/service-core@1.16.3
|
|
11
|
+
- @powersync/lib-services-framework@0.7.10
|
|
12
|
+
- @powersync/service-jpgwire@0.21.6
|
|
13
|
+
- @powersync/lib-service-postgres@0.4.14
|
|
14
|
+
|
|
15
|
+
## 0.16.11
|
|
16
|
+
|
|
17
|
+
### Patch Changes
|
|
18
|
+
|
|
19
|
+
- d889219: Fix memory leaks when retrying replication after errors.
|
|
20
|
+
- 7eb7957: [Postgres] Remove usage of pg_logical_slot_peek_binary_changes due to performance issues in some cases
|
|
21
|
+
- Updated dependencies [b364581]
|
|
22
|
+
- Updated dependencies [d889219]
|
|
23
|
+
- Updated dependencies [0ace0d3]
|
|
24
|
+
- Updated dependencies [7eb7957]
|
|
25
|
+
- Updated dependencies [b364581]
|
|
26
|
+
- @powersync/service-core@1.16.2
|
|
27
|
+
|
|
3
28
|
## 0.16.10
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
|
@@ -8,7 +8,6 @@ import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
|
8
8
|
import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
9
|
import * as types from '../types/types.js';
|
|
10
10
|
import { getApplicationName } from '../utils/application-name.js';
|
|
11
|
-
import { CustomTypeRegistry } from '../types/registry.js';
|
|
12
11
|
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
13
12
|
export class PostgresRouteAPIAdapter {
|
|
14
13
|
pool;
|
|
@@ -30,7 +29,7 @@ export class PostgresRouteAPIAdapter {
|
|
|
30
29
|
constructor(pool, connectionTag, config) {
|
|
31
30
|
this.pool = pool;
|
|
32
31
|
this.config = config;
|
|
33
|
-
this.typeCache = new PostgresTypeResolver(
|
|
32
|
+
this.typeCache = new PostgresTypeResolver(pool);
|
|
34
33
|
this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
|
|
35
34
|
}
|
|
36
35
|
getParseSyncRulesOptions() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE5D,MAAM,OAAO,uBAAuB;IAkBtB;IAEF;IAnBF,SAAS,CAAuB;IACxC,aAAa,CAAS;IACtB,oDAAoD;IACpD,eAAe,GAAG,gBAAgB,CAAC;IAEnC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,MAAM;YACnB,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAE/C,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC;aAC/C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,iBAAiB,CAC3C,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAClC,UAAU,CAAC,oBAAoB,CAAC,4BAA4B,CAC7D,CAAC;4BACF,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;gBACxC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;8BAKS;oBACpB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;oBAC3F,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzD,SAAS,EAAE;;;;;2BAKM;oBACjB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC7B,kBAAkB;oBAClB,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC1G,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAC/B,YAAqC,EACrC,IAAY,EACZ,UAAyB,EACzB,SAAkC;QAElC,OAAO,iBAAiB,CAAC;YACvB,EAAE,EAAE,IAAI,CAAC,IAAI;YACb,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,OAAkC;QAC7D,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACzD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,YAAY,CAAC;YACrB,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,SAAS,CAAC,WAAW;YAC3B,WAAW,EAAE,gDAAgD,QAAQ,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAErG,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEnD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;QAErC,sFAAsF;QACtF,qGAAqG;QACrG,6FAA6F;QAC7F,wCAAwC;QACxC,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAEhE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAwCsC,CACvC,CAAC;QACF,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9E,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,SAAS;oBACrF,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}
|
|
@@ -3,7 +3,6 @@ import { api, ConnectionTestResult, modules, replication, system } from '@powers
|
|
|
3
3
|
import * as types from '../types/types.js';
|
|
4
4
|
import { PostgresConnectionConfig } from '../types/types.js';
|
|
5
5
|
export declare class PostgresModule extends replication.ReplicationModule<types.PostgresConnectionConfig> {
|
|
6
|
-
private customTypes;
|
|
7
6
|
constructor();
|
|
8
7
|
onInitialized(context: system.ServiceContextContainer): Promise<void>;
|
|
9
8
|
protected createRouteAPIAdapter(): api.RouteAPI;
|
|
@@ -11,9 +11,7 @@ import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
|
11
11
|
import { WalStreamReplicator } from '../replication/WalStreamReplicator.js';
|
|
12
12
|
import * as types from '../types/types.js';
|
|
13
13
|
import { getApplicationName } from '../utils/application-name.js';
|
|
14
|
-
import { CustomTypeRegistry } from '../types/registry.js';
|
|
15
14
|
export class PostgresModule extends replication.ReplicationModule {
|
|
16
|
-
customTypes = new CustomTypeRegistry();
|
|
17
15
|
constructor() {
|
|
18
16
|
super({
|
|
19
17
|
name: 'Postgres',
|
|
@@ -38,7 +36,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
38
36
|
createReplicator(context) {
|
|
39
37
|
const normalisedConfig = this.resolveConfig(this.decodedConfig);
|
|
40
38
|
const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
|
|
41
|
-
const connectionFactory = new ConnectionManagerFactory(normalisedConfig
|
|
39
|
+
const connectionFactory = new ConnectionManagerFactory(normalisedConfig);
|
|
42
40
|
return new WalStreamReplicator({
|
|
43
41
|
id: this.getDefaultId(normalisedConfig.database),
|
|
44
42
|
syncRuleProvider: syncRuleProvider,
|
|
@@ -54,8 +52,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
54
52
|
resolveConfig(config) {
|
|
55
53
|
return {
|
|
56
54
|
...config,
|
|
57
|
-
...types.normalizeConnectionConfig(config)
|
|
58
|
-
typeRegistry: this.customTypes
|
|
55
|
+
...types.normalizeConnectionConfig(config)
|
|
59
56
|
};
|
|
60
57
|
}
|
|
61
58
|
async teardown(options) {
|
|
@@ -63,8 +60,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
63
60
|
const connectionManager = new PgManager(normalisedConfig, {
|
|
64
61
|
idleTimeout: 30_000,
|
|
65
62
|
maxSize: 1,
|
|
66
|
-
applicationName: getApplicationName()
|
|
67
|
-
registry: this.customTypes
|
|
63
|
+
applicationName: getApplicationName()
|
|
68
64
|
});
|
|
69
65
|
try {
|
|
70
66
|
if (options.syncRules) {
|
|
@@ -94,8 +90,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
94
90
|
const connectionManager = new PgManager(normalizedConfig, {
|
|
95
91
|
idleTimeout: 30_000,
|
|
96
92
|
maxSize: 1,
|
|
97
|
-
applicationName: getApplicationName()
|
|
98
|
-
registry: new CustomTypeRegistry()
|
|
93
|
+
applicationName: getApplicationName()
|
|
99
94
|
});
|
|
100
95
|
const connection = await connectionManager.snapshotConnection();
|
|
101
96
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA0C,MAAM,iCAAiC,CAAC;AAClG,OAAO,EAEL,kCAAkC,EAGlC,WAAW,EAEZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA0C,MAAM,iCAAiC,CAAC;AAClG,OAAO,EAEL,kCAAkC,EAGlC,WAAW,EAEZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAGlE,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAuC;QACzD,sGAAsG;QACtG,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC9B,OAAO,CAAC,kBAAkB,CAAC;gBACzB,YAAY,CAAC,KAAK;oBAChB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvF,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAES,qBAAqB;QAC7B,OAAO,uBAAuB,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IACrF,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,mBAAmB,CAAC;YAC7B,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC;YAChD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,wBAAwB,EAAE;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAsC;QAC1D,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAgC;QAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,sHAAsH;gBACtH,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,sBAAsB,CAAC,SAAS,CAAC,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBAC5E,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,gGAAgG;wBAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;oBACpG,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAgC;QACnD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAwD;QAClF,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAC/D,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,OAAO,CAAC,gBAAgB,CAAC;SACjD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
import { PgManager } from './PgManager.js';
|
|
2
|
-
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
3
1
|
import { PgPoolOptions } from '@powersync/service-jpgwire';
|
|
4
|
-
import {
|
|
2
|
+
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
3
|
+
import { PgManager } from './PgManager.js';
|
|
5
4
|
export declare class ConnectionManagerFactory {
|
|
6
|
-
private readonly registry;
|
|
7
5
|
private readonly connectionManagers;
|
|
8
6
|
readonly dbConnectionConfig: NormalizedPostgresConnectionConfig;
|
|
9
|
-
constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig
|
|
7
|
+
constructor(dbConnectionConfig: NormalizedPostgresConnectionConfig);
|
|
10
8
|
create(poolOptions: PgPoolOptions): PgManager;
|
|
11
9
|
shutdown(): Promise<void>;
|
|
12
10
|
}
|
|
@@ -1,22 +1,24 @@
|
|
|
1
|
-
import { PgManager } from './PgManager.js';
|
|
2
1
|
import { logger } from '@powersync/lib-services-framework';
|
|
2
|
+
import { PgManager } from './PgManager.js';
|
|
3
3
|
export class ConnectionManagerFactory {
|
|
4
|
-
|
|
5
|
-
connectionManagers;
|
|
4
|
+
connectionManagers = new Set();
|
|
6
5
|
dbConnectionConfig;
|
|
7
|
-
constructor(dbConnectionConfig
|
|
8
|
-
this.registry = registry;
|
|
6
|
+
constructor(dbConnectionConfig) {
|
|
9
7
|
this.dbConnectionConfig = dbConnectionConfig;
|
|
10
|
-
this.connectionManagers = [];
|
|
11
8
|
}
|
|
12
9
|
create(poolOptions) {
|
|
13
|
-
const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions
|
|
14
|
-
this.connectionManagers.
|
|
10
|
+
const manager = new PgManager(this.dbConnectionConfig, { ...poolOptions });
|
|
11
|
+
this.connectionManagers.add(manager);
|
|
12
|
+
manager.registerListener({
|
|
13
|
+
onEnded: () => {
|
|
14
|
+
this.connectionManagers.delete(manager);
|
|
15
|
+
}
|
|
16
|
+
});
|
|
15
17
|
return manager;
|
|
16
18
|
}
|
|
17
19
|
async shutdown() {
|
|
18
20
|
logger.info('Shutting down Postgres connection Managers...');
|
|
19
|
-
for (const manager of this.connectionManagers) {
|
|
21
|
+
for (const manager of [...this.connectionManagers]) {
|
|
20
22
|
await manager.end();
|
|
21
23
|
}
|
|
22
24
|
logger.info('Postgres connection Managers shutdown completed.');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAG3D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,OAAO,wBAAwB;IAClB,kBAAkB,GAAG,IAAI,GAAG,EAAa,CAAC;IAC3C,kBAAkB,CAAqC;IAEvE,YAAY,kBAAsD;QAChE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,WAA0B;QAC/B,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAErC,OAAO,CAAC,gBAAgB,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE;gBACZ,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC7D,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;QACtB,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IAClE,CAAC;CACF"}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
1
2
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
3
|
import semver from 'semver';
|
|
3
|
-
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
4
4
|
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
5
|
-
import {
|
|
5
|
+
import { NormalizedPostgresConnectionConfig } from '../types/types.js';
|
|
6
6
|
export interface PgManagerOptions extends pgwire.PgPoolOptions {
|
|
7
|
-
registry: CustomTypeRegistry;
|
|
8
7
|
}
|
|
9
|
-
export
|
|
8
|
+
export interface PgManagerListener {
|
|
9
|
+
onEnded(): void;
|
|
10
|
+
}
|
|
11
|
+
export declare class PgManager extends BaseObserver<PgManagerListener> {
|
|
10
12
|
options: NormalizedPostgresConnectionConfig;
|
|
11
13
|
poolOptions: PgManagerOptions;
|
|
12
14
|
/**
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import { BaseObserver } from '@powersync/lib-services-framework';
|
|
1
2
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
-
import { getApplicationName } from '../utils/application-name.js';
|
|
3
3
|
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
4
|
+
import { getApplicationName } from '../utils/application-name.js';
|
|
4
5
|
import { getServerVersion } from '../utils/postgres_version.js';
|
|
5
6
|
/**
|
|
6
7
|
* Shorter timeout for snapshot connections than for replication connections.
|
|
7
8
|
*/
|
|
8
9
|
const SNAPSHOT_SOCKET_TIMEOUT = 30_000;
|
|
9
|
-
export class PgManager {
|
|
10
|
+
export class PgManager extends BaseObserver {
|
|
10
11
|
options;
|
|
11
12
|
poolOptions;
|
|
12
13
|
/**
|
|
@@ -16,11 +17,12 @@ export class PgManager {
|
|
|
16
17
|
types;
|
|
17
18
|
connectionPromises = [];
|
|
18
19
|
constructor(options, poolOptions) {
|
|
20
|
+
super();
|
|
19
21
|
this.options = options;
|
|
20
22
|
this.poolOptions = poolOptions;
|
|
21
23
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
22
24
|
this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
|
|
23
|
-
this.types = new PostgresTypeResolver(
|
|
25
|
+
this.types = new PostgresTypeResolver(this.pool);
|
|
24
26
|
}
|
|
25
27
|
get connectionTag() {
|
|
26
28
|
return this.options.tag;
|
|
@@ -63,8 +65,9 @@ export class PgManager {
|
|
|
63
65
|
for (let result of await Promise.allSettled([
|
|
64
66
|
this.pool.end(),
|
|
65
67
|
...this.connectionPromises.map(async (promise) => {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
+
// Wait for connection attempts to finish, but do not throw connection errors here
|
|
69
|
+
const connection = await promise.catch((_) => { });
|
|
70
|
+
return await connection?.end();
|
|
68
71
|
})
|
|
69
72
|
])) {
|
|
70
73
|
// Throw the first error, if any
|
|
@@ -72,13 +75,17 @@ export class PgManager {
|
|
|
72
75
|
throw result.reason;
|
|
73
76
|
}
|
|
74
77
|
}
|
|
78
|
+
this.iterateListeners((listener) => {
|
|
79
|
+
listener.onEnded?.();
|
|
80
|
+
});
|
|
75
81
|
}
|
|
76
82
|
async destroy() {
|
|
77
83
|
this.pool.destroy();
|
|
78
84
|
for (let result of await Promise.allSettled([
|
|
79
85
|
...this.connectionPromises.map(async (promise) => {
|
|
80
|
-
|
|
81
|
-
|
|
86
|
+
// Wait for connection attempts to finish, but do not throw connection errors here
|
|
87
|
+
const connection = await promise.catch((_) => { });
|
|
88
|
+
return connection?.destroy();
|
|
82
89
|
})
|
|
83
90
|
])) {
|
|
84
91
|
// Throw the first error, if any
|
|
@@ -86,6 +93,9 @@ export class PgManager {
|
|
|
86
93
|
throw result.reason;
|
|
87
94
|
}
|
|
88
95
|
}
|
|
96
|
+
this.iterateListeners((listener) => {
|
|
97
|
+
listener.onEnded?.();
|
|
98
|
+
});
|
|
89
99
|
}
|
|
90
100
|
}
|
|
91
101
|
//# sourceMappingURL=PgManager.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAIhE;;GAEG;AACH,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAMvC,MAAM,OAAO,SAAU,SAAQ,YAA+B;IAWnD;IACA;IAXT;;OAEG;IACa,IAAI,CAAkB;IAEtB,KAAK,CAAuB;IAEpC,kBAAkB,GAAmC,EAAE,CAAC;IAEhE,YACS,OAA2C,EAC3C,WAA6B;QAEpC,KAAK,EAAE,CAAC;QAHD,YAAO,GAAP,OAAO,CAAoC;QAC3C,gBAAW,GAAX,WAAW,CAAkB;QAGpC,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAChE,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAC7G,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QACpB,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC;QAE3B,mDAAmD;QACnD,iEAAiE;QACjE,0BAA0B;QAC1B,iEAAiE;QACjE,qBAAqB;QAEpB,UAAkB,CAAC,OAAO,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAC;QAEhE,kDAAkD;QAClD,yCAAyC;QACzC,MAAM,UAAU,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAE5D,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,kFAAkF;gBAClF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,GAAE,CAAC,CAAC,CAAC;gBAClD,OAAO,MAAM,UAAU,EAAE,GAAG,EAAE,CAAC;YACjC,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,kFAAkF;gBAClF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,GAAE,CAAC,CAAC,CAAC;gBAClD,OAAO,UAAU,EAAE,OAAO,EAAE,CAAC;YAC/B,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { setTimeout } from 'timers/promises';
|
|
2
|
+
import { MissingReplicationSlotError } from './WalStream.js';
|
|
2
3
|
export class PostgresErrorRateLimiter {
|
|
3
4
|
nextAllowed = Date.now();
|
|
4
5
|
async waitUntilAllowed(options) {
|
|
@@ -12,7 +13,11 @@ export class PostgresErrorRateLimiter {
|
|
|
12
13
|
}
|
|
13
14
|
reportError(e) {
|
|
14
15
|
const message = e.message ?? '';
|
|
15
|
-
if (
|
|
16
|
+
if (e instanceof MissingReplicationSlotError) {
|
|
17
|
+
// Short delay for a retrying (re-creating the slot)
|
|
18
|
+
this.setDelay(2_000);
|
|
19
|
+
}
|
|
20
|
+
else if (message.includes('password authentication failed')) {
|
|
16
21
|
// Wait 15 minutes, to avoid triggering Supabase's fail2ban
|
|
17
22
|
this.setDelay(900_000);
|
|
18
23
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/PostgresErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/PostgresErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,2BAA2B,EAAE,MAAM,gBAAgB,CAAC;AAE7D,MAAM,OAAO,wBAAwB;IACnC,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;YAC7C,oDAAoD;YACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE,CAAC;YAC9D,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACzC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D,CAAC;YACD,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}
|
|
@@ -33,7 +33,7 @@ export declare const KEEPALIVE_STATEMENT: pgwire.Statement;
|
|
|
33
33
|
export declare const isKeepAliveMessage: (msg: pgwire.PgoutputMessage) => boolean;
|
|
34
34
|
export declare const sendKeepAlive: (db: pgwire.PgClient) => Promise<void>;
|
|
35
35
|
export declare class MissingReplicationSlotError extends Error {
|
|
36
|
-
constructor(message: string);
|
|
36
|
+
constructor(message: string, cause?: any);
|
|
37
37
|
}
|
|
38
38
|
export declare class WalStream {
|
|
39
39
|
sync_rules: SqlSyncRules;
|
|
@@ -62,10 +62,6 @@ export declare class WalStream {
|
|
|
62
62
|
get stopped(): boolean;
|
|
63
63
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
|
|
64
64
|
initSlot(): Promise<InitResult>;
|
|
65
|
-
/**
|
|
66
|
-
* If a replication slot exists, check that it is healthy.
|
|
67
|
-
*/
|
|
68
|
-
private checkReplicationSlot;
|
|
69
65
|
estimatedCountNumber(db: pgwire.PgConnection, table: storage.SourceTable): Promise<number>;
|
|
70
66
|
/**
|
|
71
67
|
* Start initial replication.
|
|
@@ -100,6 +96,7 @@ export declare class WalStream {
|
|
|
100
96
|
replicate(): Promise<void>;
|
|
101
97
|
initReplication(replicationConnection: pgwire.PgConnection): Promise<void>;
|
|
102
98
|
streamChanges(replicationConnection: pgwire.PgConnection): Promise<void>;
|
|
99
|
+
private streamChangesInternal;
|
|
103
100
|
ack(lsn: string, replicationStream: pgwire.ReplicationStream): Promise<void>;
|
|
104
101
|
/**
|
|
105
102
|
* Ensures that the storage is compatible with the replication connection.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { container, DatabaseConnectionError, logger as defaultLogger, ErrorCode,
|
|
2
|
+
import { container, DatabaseConnectionError, logger as defaultLogger, ErrorCode, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { getUuidReplicaIdentityBson, RelationCache, storage } from '@powersync/service-core';
|
|
4
4
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
5
5
|
import { applyValueContext, CompatibilityContext, toSyncRulesRow } from '@powersync/service-sync-rules';
|
|
@@ -31,8 +31,9 @@ export const sendKeepAlive = async (db) => {
|
|
|
31
31
|
await lib_postgres.retriedQuery(db, KEEPALIVE_STATEMENT);
|
|
32
32
|
};
|
|
33
33
|
export class MissingReplicationSlotError extends Error {
|
|
34
|
-
constructor(message) {
|
|
34
|
+
constructor(message, cause) {
|
|
35
35
|
super(message);
|
|
36
|
+
this.cause = cause;
|
|
36
37
|
}
|
|
37
38
|
}
|
|
38
39
|
export class WalStream {
|
|
@@ -196,113 +197,50 @@ export class WalStream {
|
|
|
196
197
|
statement: 'SELECT * FROM pg_replication_slots WHERE slot_name = $1',
|
|
197
198
|
params: [{ type: 'varchar', value: slotName }]
|
|
198
199
|
}))[0];
|
|
200
|
+
// Previously we also used pg_catalog.pg_logical_slot_peek_binary_changes to confirm that we can query the slot.
|
|
201
|
+
// However, there were some edge cases where the query times out, repeating the query, ultimately
|
|
202
|
+
// causing high load on the source database and never recovering automatically.
|
|
203
|
+
// We now instead jump straight to replication if the wal_status is not "lost", rather detecting those
|
|
204
|
+
// errors during streaming replication, which is a little more robust.
|
|
205
|
+
// We can have:
|
|
206
|
+
// 1. needsInitialSync: true, lost slot -> MissingReplicationSlotError (starts new sync rules version).
|
|
207
|
+
// Theoretically we could handle this the same as (2).
|
|
208
|
+
// 2. needsInitialSync: true, no slot -> create new slot
|
|
209
|
+
// 3. needsInitialSync: true, valid slot -> resume initial sync
|
|
210
|
+
// 4. needsInitialSync: false, lost slot -> MissingReplicationSlotError (starts new sync rules version)
|
|
211
|
+
// 5. needsInitialSync: false, no slot -> MissingReplicationSlotError (starts new sync rules version)
|
|
212
|
+
// 6. needsInitialSync: false, valid slot -> resume streaming replication
|
|
213
|
+
// The main advantage of MissingReplicationSlotError are:
|
|
214
|
+
// 1. If there was a complete snapshot already (cases 4/5), users can still sync from that snapshot while
|
|
215
|
+
// we do the reprocessing under a new slot name.
|
|
216
|
+
// 2. If there was a partial snapshot (case 1), we can start with the new slot faster by not waiting for
|
|
217
|
+
// the partial data to be cleared.
|
|
199
218
|
if (slot != null) {
|
|
200
219
|
// This checks that the slot is still valid
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
//
|
|
209
|
-
// needsInitialSync: false, needsNewSlot: true -> handled above
|
|
210
|
-
// needsInitialSync: false, needsNewSlot: false -> resume streaming replication
|
|
220
|
+
// wal_status is present in postgres 13+
|
|
221
|
+
// invalidation_reason is present in postgres 17+
|
|
222
|
+
const lost = slot.wal_status == 'lost';
|
|
223
|
+
if (lost) {
|
|
224
|
+
// Case 1 / 4
|
|
225
|
+
throw new MissingReplicationSlotError(`Replication slot ${slotName} is not valid anymore. invalidation_reason: ${slot.invalidation_reason ?? 'unknown'}`);
|
|
226
|
+
}
|
|
227
|
+
// Case 3 / 6
|
|
211
228
|
return {
|
|
212
229
|
needsInitialSync: !snapshotDone,
|
|
213
|
-
needsNewSlot:
|
|
230
|
+
needsNewSlot: false
|
|
214
231
|
};
|
|
215
232
|
}
|
|
216
233
|
else {
|
|
217
234
|
if (snapshotDone) {
|
|
235
|
+
// Case 5
|
|
218
236
|
// This will create a new slot, while keeping the current sync rules active
|
|
219
237
|
throw new MissingReplicationSlotError(`Replication slot ${slotName} is missing`);
|
|
220
238
|
}
|
|
221
|
-
//
|
|
239
|
+
// Case 2
|
|
240
|
+
// This will clear data (if any) and re-create the same slot
|
|
222
241
|
return { needsInitialSync: true, needsNewSlot: true };
|
|
223
242
|
}
|
|
224
243
|
}
|
|
225
|
-
/**
|
|
226
|
-
* If a replication slot exists, check that it is healthy.
|
|
227
|
-
*/
|
|
228
|
-
async checkReplicationSlot(slot) {
|
|
229
|
-
// Start with a placeholder error, should be replaced if there is an actual issue.
|
|
230
|
-
let last_error = new ReplicationAssertionError(`Slot health check failed to execute`);
|
|
231
|
-
const slotName = this.slot_name;
|
|
232
|
-
const lost = slot.wal_status == 'lost';
|
|
233
|
-
if (lost) {
|
|
234
|
-
this.logger.warn(`Replication slot ${slotName} is invalidated. invalidation_reason: ${slot.invalidation_reason ?? 'unknown'}`);
|
|
235
|
-
return {
|
|
236
|
-
needsNewSlot: true
|
|
237
|
-
};
|
|
238
|
-
}
|
|
239
|
-
// Check that replication slot exists, trying for up to 2 minutes.
|
|
240
|
-
const startAt = performance.now();
|
|
241
|
-
while (performance.now() - startAt < 120_000) {
|
|
242
|
-
this.touch();
|
|
243
|
-
try {
|
|
244
|
-
// We peek a large number of changes here, to make it more likely to pick up replication slot errors.
|
|
245
|
-
// For example, "publication does not exist" only occurs here if the peek actually includes changes related
|
|
246
|
-
// to the slot.
|
|
247
|
-
this.logger.info(`Checking ${slotName}`);
|
|
248
|
-
// The actual results can be quite large, so we don't actually return everything
|
|
249
|
-
// due to memory and processing overhead that would create.
|
|
250
|
-
const cursor = await this.connections.pool.stream({
|
|
251
|
-
statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
|
|
252
|
-
params: [
|
|
253
|
-
{ type: 'varchar', value: slotName },
|
|
254
|
-
{ type: 'varchar', value: PUBLICATION_NAME }
|
|
255
|
-
]
|
|
256
|
-
});
|
|
257
|
-
for await (let _chunk of cursor) {
|
|
258
|
-
// No-op, just exhaust the cursor
|
|
259
|
-
}
|
|
260
|
-
// Success
|
|
261
|
-
this.logger.info(`Slot ${slotName} appears healthy`);
|
|
262
|
-
return { needsNewSlot: false };
|
|
263
|
-
}
|
|
264
|
-
catch (e) {
|
|
265
|
-
last_error = e;
|
|
266
|
-
this.logger.warn(`Replication slot error`, e);
|
|
267
|
-
if (this.stopped) {
|
|
268
|
-
throw e;
|
|
269
|
-
}
|
|
270
|
-
if (/incorrect prev-link/.test(e.message) ||
|
|
271
|
-
/replication slot.*does not exist/.test(e.message) ||
|
|
272
|
-
/publication.*does not exist/.test(e.message) ||
|
|
273
|
-
// Postgres 18 - exceeded max_slot_wal_keep_size
|
|
274
|
-
/can no longer access replication slot/.test(e.message) ||
|
|
275
|
-
// Postgres 17 - exceeded max_slot_wal_keep_size
|
|
276
|
-
/can no longer get changes from replication slot/.test(e.message)) {
|
|
277
|
-
// Fatal error. In most cases since Postgres 13+, the `wal_status == 'lost'` check should pick this up, but this
|
|
278
|
-
// works as a fallback.
|
|
279
|
-
container.reporter.captureException(e, {
|
|
280
|
-
level: errors.ErrorSeverity.WARNING,
|
|
281
|
-
metadata: {
|
|
282
|
-
replication_slot: slotName
|
|
283
|
-
}
|
|
284
|
-
});
|
|
285
|
-
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
286
|
-
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
287
|
-
//
|
|
288
|
-
// Sample: publication "powersync" does not exist
|
|
289
|
-
// Happens when publication deleted or never created.
|
|
290
|
-
// Slot must be re-created in this case.
|
|
291
|
-
this.logger.info(`${slotName} is not valid anymore`);
|
|
292
|
-
return { needsNewSlot: true };
|
|
293
|
-
}
|
|
294
|
-
// Try again after a pause
|
|
295
|
-
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
container.reporter.captureException(last_error, {
|
|
299
|
-
level: errors.ErrorSeverity.ERROR,
|
|
300
|
-
metadata: {
|
|
301
|
-
replication_slot: slotName
|
|
302
|
-
}
|
|
303
|
-
});
|
|
304
|
-
throw last_error;
|
|
305
|
-
}
|
|
306
244
|
async estimatedCountNumber(db, table) {
|
|
307
245
|
const results = await db.query({
|
|
308
246
|
statement: `SELECT reltuples::bigint AS estimate
|
|
@@ -470,6 +408,7 @@ WHERE oid = $1::regclass`,
|
|
|
470
408
|
}
|
|
471
409
|
await q.initialize();
|
|
472
410
|
let columns = [];
|
|
411
|
+
let columnMap = {};
|
|
473
412
|
let hasRemainingData = true;
|
|
474
413
|
while (hasRemainingData) {
|
|
475
414
|
// Fetch 10k at a time.
|
|
@@ -489,6 +428,9 @@ WHERE oid = $1::regclass`,
|
|
|
489
428
|
columns = chunk.payload.map((c) => {
|
|
490
429
|
return { i: i++, name: c.name };
|
|
491
430
|
});
|
|
431
|
+
for (let column of chunk.payload) {
|
|
432
|
+
columnMap[column.name] = column.typeOid;
|
|
433
|
+
}
|
|
492
434
|
continue;
|
|
493
435
|
}
|
|
494
436
|
const rows = chunk.rows.map((row) => {
|
|
@@ -502,7 +444,7 @@ WHERE oid = $1::regclass`,
|
|
|
502
444
|
hasRemainingData = true;
|
|
503
445
|
}
|
|
504
446
|
for (const inputRecord of WalStream.getQueryData(rows)) {
|
|
505
|
-
const record = this.syncRulesRecord(inputRecord);
|
|
447
|
+
const record = this.syncRulesRecord(this.connections.types.constructRowRecord(columnMap, inputRecord));
|
|
506
448
|
// This auto-flushes when the batch reaches its size limit
|
|
507
449
|
await batch.save({
|
|
508
450
|
tag: storage.SaveOperationTag.INSERT,
|
|
@@ -724,6 +666,17 @@ WHERE oid = $1::regclass`,
|
|
|
724
666
|
}
|
|
725
667
|
}
|
|
726
668
|
async streamChanges(replicationConnection) {
|
|
669
|
+
try {
|
|
670
|
+
await this.streamChangesInternal(replicationConnection);
|
|
671
|
+
}
|
|
672
|
+
catch (e) {
|
|
673
|
+
if (isReplicationSlotInvalidError(e)) {
|
|
674
|
+
throw new MissingReplicationSlotError(e.message, e);
|
|
675
|
+
}
|
|
676
|
+
throw e;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
async streamChangesInternal(replicationConnection) {
|
|
727
680
|
// When changing any logic here, check /docs/wal-lsns.md.
|
|
728
681
|
const { createEmptyCheckpoints } = await this.ensureStorageCompatibility();
|
|
729
682
|
const replicationOptions = {
|
|
@@ -956,4 +909,24 @@ WHERE oid = $1::regclass`,
|
|
|
956
909
|
});
|
|
957
910
|
}
|
|
958
911
|
}
|
|
912
|
+
function isReplicationSlotInvalidError(e) {
|
|
913
|
+
// We could access the error code from pgwire using this:
|
|
914
|
+
// e[Symbol.for('pg.ErrorCode')]
|
|
915
|
+
// However, we typically get a generic code such as 42704 (undefined_object), which does not
|
|
916
|
+
// help much. So we check the actual error message.
|
|
917
|
+
const message = e.message ?? '';
|
|
918
|
+
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
919
|
+
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
920
|
+
//
|
|
921
|
+
// Sample: publication "powersync" does not exist
|
|
922
|
+
// Happens when publication deleted or never created.
|
|
923
|
+
// Slot must be re-created in this case.
|
|
924
|
+
return (/incorrect prev-link/.test(message) ||
|
|
925
|
+
/replication slot.*does not exist/.test(message) ||
|
|
926
|
+
/publication.*does not exist/.test(message) ||
|
|
927
|
+
// Postgres 18 - exceeded max_slot_wal_keep_size
|
|
928
|
+
/can no longer access replication slot/.test(message) ||
|
|
929
|
+
// Postgres 17 - exceeded max_slot_wal_keep_size
|
|
930
|
+
/can no longer get changes from replication slot/.test(message));
|
|
931
|
+
}
|
|
959
932
|
//# sourceMappingURL=WalStream.js.map
|