@powersync/service-module-postgres 0.0.0-dev-20241128134723 → 0.0.0-dev-20241210162153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -5
- package/dist/api/PostgresRouteAPIAdapter.d.ts +6 -2
- package/dist/api/PostgresRouteAPIAdapter.js +24 -10
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/module/PostgresModule.js +10 -3
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/PgManager.js +15 -1
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/WalStream.d.ts +9 -2
- package/dist/replication/WalStream.js +122 -114
- package/dist/replication/WalStream.js.map +1 -1
- package/package.json +6 -6
- package/src/api/PostgresRouteAPIAdapter.ts +31 -12
- package/src/module/PostgresModule.ts +12 -3
- package/src/replication/PgManager.ts +19 -1
- package/src/replication/WalStream.ts +138 -125
- package/test/src/large_batch.test.ts +333 -148
- package/test/src/schema_changes.test.ts +562 -513
- package/test/src/slow_tests.test.ts +2 -1
- package/test/src/util.ts +3 -1
- package/test/src/validation.test.ts +45 -48
- package/test/src/wal_stream.test.ts +224 -249
- package/test/src/wal_stream_utils.ts +61 -22
- package/tsconfig.tsbuildinfo +1 -1
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
package/CHANGELOG.md
CHANGED
|
@@ -1,13 +1,66 @@
|
|
|
1
1
|
# @powersync/service-module-postgres
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20241210162153
|
|
4
4
|
|
|
5
5
|
### Patch Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
- 34601ce: Revert Postgres snapshot strategy.
|
|
8
|
+
|
|
9
|
+
## 0.2.1
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 889ac46: Fix "BSONObj size is invalid" error during replication.
|
|
14
|
+
- Updated dependencies [889ac46]
|
|
15
|
+
- @powersync/service-core@0.12.1
|
|
16
|
+
|
|
17
|
+
## 0.2.0
|
|
18
|
+
|
|
19
|
+
### Minor Changes
|
|
20
|
+
|
|
21
|
+
- f1e9ef3: Improve timeouts and table snapshots for Postgres initial replication.
|
|
22
|
+
|
|
23
|
+
### Patch Changes
|
|
24
|
+
|
|
25
|
+
- Updated dependencies [ebc62ff]
|
|
26
|
+
- Updated dependencies [f1e9ef3]
|
|
27
|
+
- @powersync/service-core@0.12.0
|
|
28
|
+
- @powersync/service-types@0.5.0
|
|
29
|
+
- @powersync/service-jpgwire@0.18.3
|
|
30
|
+
|
|
31
|
+
## 0.1.0
|
|
32
|
+
|
|
33
|
+
### Minor Changes
|
|
34
|
+
|
|
35
|
+
- 62e97f3: Support resuming initial replication for Postgres.
|
|
36
|
+
|
|
37
|
+
### Patch Changes
|
|
38
|
+
|
|
39
|
+
- 15b2d8e: Disable SupabaseKeyCollector when a specific secret is configured.
|
|
40
|
+
- 0fa01ee: Fix replication lag diagnostics for Postgres.
|
|
41
|
+
- Updated dependencies [62e97f3]
|
|
42
|
+
- Updated dependencies [a235c9f]
|
|
43
|
+
- Updated dependencies [8c6ce90]
|
|
44
|
+
- @powersync/service-core@0.11.0
|
|
45
|
+
- @powersync/service-sync-rules@0.22.0
|
|
46
|
+
- @powersync/service-jpgwire@0.18.2
|
|
47
|
+
|
|
48
|
+
## 0.0.4
|
|
49
|
+
|
|
50
|
+
### Patch Changes
|
|
51
|
+
|
|
52
|
+
- Updated dependencies [2a4f020]
|
|
53
|
+
- @powersync/service-core@0.10.1
|
|
54
|
+
|
|
55
|
+
## 0.0.3
|
|
56
|
+
|
|
57
|
+
### Patch Changes
|
|
58
|
+
|
|
59
|
+
- Updated dependencies [2c18ad2]
|
|
60
|
+
- Updated dependencies [35c267f]
|
|
61
|
+
- @powersync/service-core@0.10.0
|
|
62
|
+
- @powersync/service-types@0.4.0
|
|
63
|
+
- @powersync/service-jpgwire@0.18.1
|
|
11
64
|
|
|
12
65
|
## 0.0.2
|
|
13
66
|
|
|
@@ -4,11 +4,15 @@ import * as sync_rules from '@powersync/service-sync-rules';
|
|
|
4
4
|
import * as service_types from '@powersync/service-types';
|
|
5
5
|
import * as types from '../types/types.js';
|
|
6
6
|
export declare class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
7
|
-
protected config: types.ResolvedConnectionConfig;
|
|
8
7
|
protected pool: pgwire.PgClient;
|
|
8
|
+
private config?;
|
|
9
9
|
connectionTag: string;
|
|
10
10
|
publicationName: string;
|
|
11
|
-
|
|
11
|
+
static withConfig(config: types.ResolvedConnectionConfig): PostgresRouteAPIAdapter;
|
|
12
|
+
/**
|
|
13
|
+
* @param config - Required for the service; optional for tests.
|
|
14
|
+
*/
|
|
15
|
+
constructor(pool: pgwire.PgClient, connectionTag?: string, config?: types.ResolvedConnectionConfig | undefined);
|
|
12
16
|
getParseSyncRulesOptions(): ParseSyncRulesOptions;
|
|
13
17
|
shutdown(): Promise<void>;
|
|
14
18
|
getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig>;
|
|
@@ -7,14 +7,21 @@ import * as pg_utils from '../utils/pgwire_utils.js';
|
|
|
7
7
|
import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
8
8
|
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
9
|
export class PostgresRouteAPIAdapter {
|
|
10
|
-
|
|
10
|
+
static withConfig(config) {
|
|
11
|
+
const pool = pgwire.connectPgWirePool(config, {
|
|
12
|
+
idleTimeout: 30000
|
|
13
|
+
});
|
|
14
|
+
return new PostgresRouteAPIAdapter(pool, config.tag, config);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* @param config - Required for the service; optional for tests.
|
|
18
|
+
*/
|
|
19
|
+
constructor(pool, connectionTag, config) {
|
|
20
|
+
this.pool = pool;
|
|
11
21
|
this.config = config;
|
|
12
22
|
// TODO this should probably be configurable one day
|
|
13
23
|
this.publicationName = PUBLICATION_NAME;
|
|
14
|
-
this.
|
|
15
|
-
idleTimeout: 30000
|
|
16
|
-
});
|
|
17
|
-
this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG;
|
|
24
|
+
this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
|
|
18
25
|
}
|
|
19
26
|
getParseSyncRulesOptions() {
|
|
20
27
|
return {
|
|
@@ -29,8 +36,8 @@ export class PostgresRouteAPIAdapter {
|
|
|
29
36
|
}
|
|
30
37
|
async getConnectionStatus() {
|
|
31
38
|
const base = {
|
|
32
|
-
id: this.config
|
|
33
|
-
uri: types.baseUri(this.config)
|
|
39
|
+
id: this.config?.id ?? '',
|
|
40
|
+
uri: this.config == null ? '' : types.baseUri(this.config)
|
|
34
41
|
};
|
|
35
42
|
try {
|
|
36
43
|
await pg_utils.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`);
|
|
@@ -59,7 +66,7 @@ export class PostgresRouteAPIAdapter {
|
|
|
59
66
|
};
|
|
60
67
|
}
|
|
61
68
|
async executeQuery(query, params) {
|
|
62
|
-
if (!this.config
|
|
69
|
+
if (!this.config?.debug_api) {
|
|
63
70
|
return service_types.internal_routes.ExecuteSqlResponse.encode({
|
|
64
71
|
results: {
|
|
65
72
|
columns: [],
|
|
@@ -180,7 +187,8 @@ export class PostgresRouteAPIAdapter {
|
|
|
180
187
|
});
|
|
181
188
|
}
|
|
182
189
|
async getReplicationLag(options) {
|
|
183
|
-
const { bucketStorage
|
|
190
|
+
const { bucketStorage } = options;
|
|
191
|
+
const slotName = bucketStorage.slot_name;
|
|
184
192
|
const results = await pg_utils.retriedQuery(this.pool, {
|
|
185
193
|
statement: `SELECT
|
|
186
194
|
slot_name,
|
|
@@ -197,7 +205,13 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
197
205
|
throw new Error(`Could not determine replication lag for slot ${slotName}`);
|
|
198
206
|
}
|
|
199
207
|
async getReplicationHead() {
|
|
200
|
-
|
|
208
|
+
// On most Postgres versions, pg_logical_emit_message() returns the correct LSN.
|
|
209
|
+
// However, on Aurora (Postgres compatible), it can return an entirely different LSN,
|
|
210
|
+
// causing the write checkpoints to never be replicated back to the client.
|
|
211
|
+
// For those, we need to use pg_current_wal_lsn() instead.
|
|
212
|
+
const { results } = await pg_utils.retriedQuery(this.pool, { statement: `SELECT pg_current_wal_lsn() as lsn` }, { statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` });
|
|
213
|
+
// Specifically use the lsn from the first statement, not the second one.
|
|
214
|
+
const lsn = results[0].rows[0][0];
|
|
201
215
|
return String(lsn);
|
|
202
216
|
}
|
|
203
217
|
async getConnectionSchema() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,uBAAuB;
|
|
1
|
+
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,uBAAuB;IAKlC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,KAAM;SACpB,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAhBjD,oDAAoD;QACpD,oBAAe,GAAG,gBAAgB,CAAC;QAiBjC,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;aAC3C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;4BACpD,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;gBACxC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACrD,SAAS,EAAE;;;;;8BAKS;oBACpB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;oBAC3F,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACrD,SAAS,EAAE;;;;;2BAKM;oBACjB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC7B,kBAAkB;oBAClB,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC1G,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAC/B,YAAqC,EACrC,IAAY,EACZ,UAAyB,EACzB,SAAkC;QAElC,OAAO,iBAAiB,CAAC;YACvB,EAAE,EAAE,IAAI,CAAC,IAAI;YACb,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAkC;QACxD,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACrD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT,EAAE,SAAS,EAAE,oCAAoC,EAAE,EACnD,EAAE,SAAS,EAAE,4DAA4D,EAAE,CAC5E,CAAC;QAEF,yEAAyE;QACzE,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,mBAAmB;;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CACzC,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAuCsC,CACvC,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,MAAC,GAAG,CAAC,UAAU,MAAtB,OAAO,OAAqB;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,EAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,SAAS;oBACzE,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}
|
|
@@ -19,7 +19,14 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
19
19
|
}
|
|
20
20
|
async initialize(context) {
|
|
21
21
|
await super.initialize(context);
|
|
22
|
-
|
|
22
|
+
const client_auth = context.configuration.base_config.client_auth;
|
|
23
|
+
if (client_auth?.supabase && client_auth?.supabase_jwt_secret == null) {
|
|
24
|
+
// Only use the deprecated SupabaseKeyCollector when there is no
|
|
25
|
+
// secret hardcoded. Hardcoded secrets are handled elsewhere, using
|
|
26
|
+
// StaticSupabaseKeyCollector.
|
|
27
|
+
// Support for SupabaseKeyCollector is deprecated and support will be
|
|
28
|
+
// completely removed by Supabase soon. We can keep support a while
|
|
29
|
+
// longer for self-hosted setups, before also removing that on our side.
|
|
23
30
|
this.registerSupabaseAuth(context);
|
|
24
31
|
}
|
|
25
32
|
// Record replicated bytes using global jpgwire metrics.
|
|
@@ -32,7 +39,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
32
39
|
}
|
|
33
40
|
}
|
|
34
41
|
createRouteAPIAdapter() {
|
|
35
|
-
return
|
|
42
|
+
return PostgresRouteAPIAdapter.withConfig(this.resolveConfig(this.decodedConfig));
|
|
36
43
|
}
|
|
37
44
|
createReplicator(context) {
|
|
38
45
|
const normalisedConfig = this.resolveConfig(this.decodedConfig);
|
|
@@ -114,7 +121,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
114
121
|
});
|
|
115
122
|
const connection = await connectionManager.snapshotConnection();
|
|
116
123
|
try {
|
|
117
|
-
return checkSourceConfiguration(connection, PUBLICATION_NAME);
|
|
124
|
+
return await checkSourceConfiguration(connection, PUBLICATION_NAME);
|
|
118
125
|
}
|
|
119
126
|
finally {
|
|
120
127
|
await connectionManager.end();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,IAAI,EAAE,kCAAkC,EAAW,WAAW,EAAU,MAAM,yBAAyB,CAAC;AACtH,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAuC;QACtD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEhC,
|
|
1
|
+
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,IAAI,EAAE,kCAAkC,EAAW,WAAW,EAAU,MAAM,yBAAyB,CAAC;AACtH,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAuC;QACtD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEhC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW,CAAC;QAElE,IAAI,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,mBAAmB,IAAI,IAAI,EAAE,CAAC;YACtE,gEAAgE;YAChE,mEAAmE;YACnE,8BAA8B;YAE9B,qEAAqE;YACrE,mEAAmE;YACnE,wEAAwE;YACxE,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,wDAAwD;QACxD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,kBAAkB,CAAC;gBACzB,YAAY,CAAC,KAAK;oBAChB,OAAO,CAAC,OAAQ,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACpD,CAAC;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAES,qBAAqB;QAC7B,OAAO,uBAAuB,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IACrF,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,mBAAmB,CAAC;YAC7B,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC;YAChD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,wBAAwB,EAAE;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAsC;QAC1D,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAgC;QAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,KAAM;YACnB,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,sHAAsH;gBACtH,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,sBAAsB,CAAC,SAAS,CAAC,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBAC5E,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,gGAAgG;wBAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;oBACpG,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,gGAAgG;IACxF,oBAAoB,CAAC,OAAuC;QAClE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,yCAAyC;QACzC,aAAa,CAAC,WAAW;YACvB,EAAE,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YACnB,IAAI,UAAU,CAAC,IAAI,IAAI,KAAK,CAAC,wBAAwB,EAAE,CAAC;gBACtD,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,wBAAwB,CAAC,MAAM,CAAC,UAAiB,CAAC,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;aAClB,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAClB,MAAM,YAAY,GAAG,IAAI,oBAAoB,CAAC,MAAO,CAAC,CAAC;YACvD,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,YAAY,EAAE;gBAClD,0BAA0B;gBAC1B,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE;aAC1C,CAAC,CAAC;YACH,aAAa,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAgC;QACnD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,KAAM;YACnB,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,OAAO,MAAM,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QACtE,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
2
|
+
/**
|
|
3
|
+
* Shorter timeout for snapshot connections than for replication connections.
|
|
4
|
+
*/
|
|
5
|
+
const SNAPSHOT_SOCKET_TIMEOUT = 30000;
|
|
2
6
|
export class PgManager {
|
|
3
7
|
constructor(options, poolOptions) {
|
|
4
8
|
this.options = options;
|
|
@@ -26,7 +30,17 @@ export class PgManager {
|
|
|
26
30
|
async snapshotConnection() {
|
|
27
31
|
const p = pgwire.connectPgWire(this.options, { type: 'standard' });
|
|
28
32
|
this.connectionPromises.push(p);
|
|
29
|
-
|
|
33
|
+
const connection = await p;
|
|
34
|
+
// Use an shorter timeout for snapshot connections.
|
|
35
|
+
// This is to detect broken connections early, instead of waiting
|
|
36
|
+
// for the full 6 minutes.
|
|
37
|
+
// This we are constantly using the connection, we don't need any
|
|
38
|
+
// custom keepalives.
|
|
39
|
+
connection._socket.setTimeout(SNAPSHOT_SOCKET_TIMEOUT);
|
|
40
|
+
// Disable statement timeout for snapshot queries.
|
|
41
|
+
// On Supabase, the default is 2 minutes.
|
|
42
|
+
await connection.query(`set session statement_timeout = 0`);
|
|
43
|
+
return connection;
|
|
30
44
|
}
|
|
31
45
|
async end() {
|
|
32
46
|
for (let result of await Promise.allSettled([
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAGrD,MAAM,OAAO,SAAS;IAQpB,YACS,OAA2C,EAC3C,WAAiC;QADjC,YAAO,GAAP,OAAO,CAAoC;QAC3C,gBAAW,GAAX,WAAW,CAAsB;QAJlC,uBAAkB,GAAmC,EAAE,CAAC;QAM9D,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAGrD;;GAEG;AACH,MAAM,uBAAuB,GAAG,KAAM,CAAC;AAEvC,MAAM,OAAO,SAAS;IAQpB,YACS,OAA2C,EAC3C,WAAiC;QADjC,YAAO,GAAP,OAAO,CAAoC;QAC3C,gBAAW,GAAX,WAAW,CAAsB;QAJlC,uBAAkB,GAAmC,EAAE,CAAC;QAM9D,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC;QAE3B,mDAAmD;QACnD,iEAAiE;QACjE,0BAA0B;QAC1B,iEAAiE;QACjE,qBAAqB;QACpB,UAAkB,CAAC,OAAO,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAC;QAEhE,kDAAkD;QAClD,yCAAyC;QACzC,MAAM,UAAU,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAE5D,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -11,7 +11,10 @@ export interface WalStreamOptions {
|
|
|
11
11
|
abort_signal: AbortSignal;
|
|
12
12
|
}
|
|
13
13
|
interface InitResult {
|
|
14
|
+
/** True if initial snapshot is not yet done. */
|
|
14
15
|
needsInitialSync: boolean;
|
|
16
|
+
/** True if snapshot must be started from scratch with a new slot. */
|
|
17
|
+
needsNewSlot: boolean;
|
|
15
18
|
}
|
|
16
19
|
export declare class MissingReplicationSlotError extends Error {
|
|
17
20
|
constructor(message: string);
|
|
@@ -30,6 +33,10 @@ export declare class WalStream {
|
|
|
30
33
|
get stopped(): boolean;
|
|
31
34
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
|
|
32
35
|
initSlot(): Promise<InitResult>;
|
|
36
|
+
/**
|
|
37
|
+
* If a replication slot exists, check that it is healthy.
|
|
38
|
+
*/
|
|
39
|
+
private checkReplicationSlot;
|
|
33
40
|
estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string>;
|
|
34
41
|
/**
|
|
35
42
|
* Start initial replication.
|
|
@@ -37,8 +44,8 @@ export declare class WalStream {
|
|
|
37
44
|
* If (partial) replication was done before on this slot, this clears the state
|
|
38
45
|
* and starts again from scratch.
|
|
39
46
|
*/
|
|
40
|
-
startInitialReplication(replicationConnection: pgwire.PgConnection): Promise<void>;
|
|
41
|
-
initialReplication(db: pgwire.PgConnection
|
|
47
|
+
startInitialReplication(replicationConnection: pgwire.PgConnection, status: InitResult): Promise<void>;
|
|
48
|
+
initialReplication(db: pgwire.PgConnection): Promise<void>;
|
|
42
49
|
static getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow>;
|
|
43
50
|
private snapshotTable;
|
|
44
51
|
handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean): Promise<storage.SourceTable>;
|
|
@@ -118,75 +118,100 @@ export class WalStream {
|
|
|
118
118
|
await checkSourceConfiguration(this.connections.pool, PUBLICATION_NAME);
|
|
119
119
|
const slotName = this.slot_name;
|
|
120
120
|
const status = await this.storage.getStatus();
|
|
121
|
-
|
|
121
|
+
const snapshotDone = status.snapshot_done && status.checkpoint_lsn != null;
|
|
122
|
+
if (snapshotDone) {
|
|
123
|
+
// Snapshot is done, but we still need to check the replication slot status
|
|
122
124
|
logger.info(`${slotName} Initial replication already done`);
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
125
|
+
}
|
|
126
|
+
// Check if replication slot exists
|
|
127
|
+
const rs = await this.connections.pool.query({
|
|
128
|
+
statement: 'SELECT 1 FROM pg_replication_slots WHERE slot_name = $1',
|
|
129
|
+
params: [{ type: 'varchar', value: slotName }]
|
|
130
|
+
});
|
|
131
|
+
const slotExists = rs.rows.length > 0;
|
|
132
|
+
if (slotExists) {
|
|
133
|
+
// This checks that the slot is still valid
|
|
134
|
+
const r = await this.checkReplicationSlot();
|
|
135
|
+
return {
|
|
136
|
+
needsInitialSync: !snapshotDone,
|
|
137
|
+
needsNewSlot: r.needsNewSlot
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
return { needsInitialSync: true, needsNewSlot: true };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* If a replication slot exists, check that it is healthy.
|
|
146
|
+
*/
|
|
147
|
+
async checkReplicationSlot() {
|
|
148
|
+
let last_error = null;
|
|
149
|
+
const slotName = this.slot_name;
|
|
150
|
+
// Check that replication slot exists
|
|
151
|
+
for (let i = 120; i >= 0; i--) {
|
|
152
|
+
await touch();
|
|
153
|
+
if (i == 0) {
|
|
154
|
+
container.reporter.captureException(last_error, {
|
|
155
|
+
level: errors.ErrorSeverity.ERROR,
|
|
156
|
+
metadata: {
|
|
157
|
+
replication_slot: slotName
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
throw last_error;
|
|
161
|
+
}
|
|
162
|
+
try {
|
|
163
|
+
// We peek a large number of changes here, to make it more likely to pick up replication slot errors.
|
|
164
|
+
// For example, "publication does not exist" only occurs here if the peek actually includes changes related
|
|
165
|
+
// to the slot.
|
|
166
|
+
logger.info(`Checking ${slotName}`);
|
|
167
|
+
// The actual results can be quite large, so we don't actually return everything
|
|
168
|
+
// due to memory and processing overhead that would create.
|
|
169
|
+
const cursor = await this.connections.pool.stream({
|
|
170
|
+
statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
|
|
171
|
+
params: [
|
|
172
|
+
{ type: 'varchar', value: slotName },
|
|
173
|
+
{ type: 'varchar', value: PUBLICATION_NAME }
|
|
174
|
+
]
|
|
175
|
+
});
|
|
176
|
+
for await (let _chunk of cursor) {
|
|
177
|
+
// No-op, just exhaust the cursor
|
|
178
|
+
}
|
|
179
|
+
// Success
|
|
180
|
+
logger.info(`Slot ${slotName} appears healthy`);
|
|
181
|
+
return { needsInitialSync: false, needsNewSlot: false };
|
|
182
|
+
}
|
|
183
|
+
catch (e) {
|
|
184
|
+
last_error = e;
|
|
185
|
+
logger.warn(`${slotName} Replication slot error`, e);
|
|
186
|
+
if (this.stopped) {
|
|
187
|
+
throw e;
|
|
188
|
+
}
|
|
189
|
+
// Could also be `publication "powersync" does not exist`, although this error may show up much later
|
|
190
|
+
// in some cases.
|
|
191
|
+
if (/incorrect prev-link/.test(e.message) ||
|
|
192
|
+
/replication slot.*does not exist/.test(e.message) ||
|
|
193
|
+
/publication.*does not exist/.test(e.message)) {
|
|
194
|
+
container.reporter.captureException(e, {
|
|
195
|
+
level: errors.ErrorSeverity.WARNING,
|
|
130
196
|
metadata: {
|
|
197
|
+
try_index: i,
|
|
131
198
|
replication_slot: slotName
|
|
132
199
|
}
|
|
133
200
|
});
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
logger.info(
|
|
141
|
-
|
|
142
|
-
// due to memory and processing overhead that would create.
|
|
143
|
-
const cursor = await this.connections.pool.stream({
|
|
144
|
-
statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
|
|
145
|
-
params: [
|
|
146
|
-
{ type: 'varchar', value: slotName },
|
|
147
|
-
{ type: 'varchar', value: PUBLICATION_NAME }
|
|
148
|
-
]
|
|
149
|
-
});
|
|
150
|
-
for await (let _chunk of cursor) {
|
|
151
|
-
// No-op, just exhaust the cursor
|
|
152
|
-
}
|
|
153
|
-
// Success
|
|
154
|
-
logger.info(`Slot ${slotName} appears healthy`);
|
|
155
|
-
return { needsInitialSync: false };
|
|
156
|
-
}
|
|
157
|
-
catch (e) {
|
|
158
|
-
last_error = e;
|
|
159
|
-
logger.warn(`${slotName} Replication slot error`, e);
|
|
160
|
-
if (this.stopped) {
|
|
161
|
-
throw e;
|
|
162
|
-
}
|
|
163
|
-
// Could also be `publication "powersync" does not exist`, although this error may show up much later
|
|
164
|
-
// in some cases.
|
|
165
|
-
if (/incorrect prev-link/.test(e.message) ||
|
|
166
|
-
/replication slot.*does not exist/.test(e.message) ||
|
|
167
|
-
/publication.*does not exist/.test(e.message)) {
|
|
168
|
-
container.reporter.captureException(e, {
|
|
169
|
-
level: errors.ErrorSeverity.WARNING,
|
|
170
|
-
metadata: {
|
|
171
|
-
try_index: i,
|
|
172
|
-
replication_slot: slotName
|
|
173
|
-
}
|
|
174
|
-
});
|
|
175
|
-
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
176
|
-
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
177
|
-
//
|
|
178
|
-
// Sample: publication "powersync" does not exist
|
|
179
|
-
// Happens when publication deleted or never created.
|
|
180
|
-
// Slot must be re-created in this case.
|
|
181
|
-
logger.info(`${slotName} does not exist anymore, will create new slot`);
|
|
182
|
-
throw new MissingReplicationSlotError(`Replication slot ${slotName} does not exist anymore`);
|
|
183
|
-
}
|
|
184
|
-
// Try again after a pause
|
|
185
|
-
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
201
|
+
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
202
|
+
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
203
|
+
//
|
|
204
|
+
// Sample: publication "powersync" does not exist
|
|
205
|
+
// Happens when publication deleted or never created.
|
|
206
|
+
// Slot must be re-created in this case.
|
|
207
|
+
logger.info(`${slotName} does not exist anymore, will create new slot`);
|
|
208
|
+
return { needsInitialSync: true, needsNewSlot: true };
|
|
186
209
|
}
|
|
210
|
+
// Try again after a pause
|
|
211
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
187
212
|
}
|
|
188
213
|
}
|
|
189
|
-
|
|
214
|
+
throw new Error('Unreachable');
|
|
190
215
|
}
|
|
191
216
|
async estimatedCount(db, table) {
|
|
192
217
|
const results = await db.query({
|
|
@@ -209,69 +234,51 @@ WHERE oid = $1::regclass`,
|
|
|
209
234
|
* If (partial) replication was done before on this slot, this clears the state
|
|
210
235
|
* and starts again from scratch.
|
|
211
236
|
*/
|
|
212
|
-
async startInitialReplication(replicationConnection) {
|
|
237
|
+
async startInitialReplication(replicationConnection, status) {
|
|
213
238
|
// If anything here errors, the entire replication process is aborted,
|
|
214
|
-
// and all connections closed, including this one.
|
|
239
|
+
// and all connections are closed, including this one.
|
|
215
240
|
const db = await this.connections.snapshotConnection();
|
|
216
241
|
const slotName = this.slot_name;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
// with streaming replication.
|
|
233
|
-
const lsn = pgwire.lsnMakeComparable(row[1]);
|
|
234
|
-
const snapshot = row[2];
|
|
235
|
-
logger.info(`Created replication slot ${slotName} at ${lsn} with snapshot ${snapshot}`);
|
|
236
|
-
// https://stackoverflow.com/questions/70160769/postgres-logical-replication-starting-from-given-lsn
|
|
237
|
-
await db.query('BEGIN');
|
|
238
|
-
// Use the snapshot exported above.
|
|
239
|
-
// Using SERIALIZABLE isolation level may give stronger guarantees, but that complicates
|
|
240
|
-
// the replication slot + snapshot above. And we still won't have SERIALIZABLE
|
|
241
|
-
// guarantees with streaming replication.
|
|
242
|
-
// See: ./docs/serializability.md for details.
|
|
243
|
-
//
|
|
244
|
-
// Another alternative here is to use the same pgwire connection for initial replication as well,
|
|
245
|
-
// instead of synchronizing a separate transaction to the snapshot.
|
|
246
|
-
try {
|
|
247
|
-
await db.query(`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ`);
|
|
248
|
-
await db.query(`SET TRANSACTION READ ONLY`);
|
|
249
|
-
await db.query(`SET TRANSACTION SNAPSHOT '${snapshot}'`);
|
|
250
|
-
// Disable statement timeout for the duration of this transaction.
|
|
251
|
-
// On Supabase, the default is 2 minutes.
|
|
252
|
-
await db.query(`set local statement_timeout = 0`);
|
|
253
|
-
logger.info(`${slotName} Starting initial replication`);
|
|
254
|
-
await this.initialReplication(db, lsn);
|
|
255
|
-
logger.info(`${slotName} Initial replication done`);
|
|
256
|
-
await db.query('COMMIT');
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
await db.query('ROLLBACK');
|
|
260
|
-
throw e;
|
|
242
|
+
if (status.needsNewSlot) {
|
|
243
|
+
// This happens when there is no existing replication slot, or if the
|
|
244
|
+
// existing one is unhealthy.
|
|
245
|
+
// In those cases, we have to start replication from scratch.
|
|
246
|
+
// If there is an existing healthy slot, we can skip this and continue
|
|
247
|
+
// initial replication where we left off.
|
|
248
|
+
await this.storage.clear();
|
|
249
|
+
await db.query({
|
|
250
|
+
statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
|
|
251
|
+
params: [{ type: 'varchar', value: slotName }]
|
|
252
|
+
});
|
|
253
|
+
// We use the replication connection here, not a pool.
|
|
254
|
+
// The replication slot must be created before we start snapshotting tables.
|
|
255
|
+
await replicationConnection.query(`CREATE_REPLICATION_SLOT ${slotName} LOGICAL pgoutput`);
|
|
256
|
+
logger.info(`Created replication slot ${slotName}`);
|
|
261
257
|
}
|
|
258
|
+
await this.initialReplication(db);
|
|
262
259
|
}
|
|
263
|
-
async initialReplication(db
|
|
260
|
+
async initialReplication(db) {
|
|
264
261
|
const sourceTables = this.sync_rules.getSourceTables();
|
|
265
|
-
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true }, async (batch) => {
|
|
262
|
+
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true, skipExistingRows: true }, async (batch) => {
|
|
266
263
|
for (let tablePattern of sourceTables) {
|
|
267
264
|
const tables = await this.getQualifiedTableNames(batch, db, tablePattern);
|
|
268
265
|
for (let table of tables) {
|
|
266
|
+
if (table.snapshotComplete) {
|
|
267
|
+
logger.info(`${this.slot_name} Skipping ${table.qualifiedName} - snapshot already done`);
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
269
270
|
await this.snapshotTable(batch, db, table);
|
|
270
|
-
await
|
|
271
|
+
const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
|
|
272
|
+
const tableLsnNotBefore = rs.rows[0][0];
|
|
273
|
+
await batch.markSnapshotDone([table], tableLsnNotBefore);
|
|
271
274
|
await touch();
|
|
272
275
|
}
|
|
273
276
|
}
|
|
274
|
-
|
|
277
|
+
// Always commit the initial snapshot at zero.
|
|
278
|
+
// This makes sure we don't skip any changes applied before starting this snapshot,
|
|
279
|
+
// in the case of snapshot retries.
|
|
280
|
+
// We could alternatively commit at the replication slot LSN.
|
|
281
|
+
await batch.commit(ZERO_LSN);
|
|
275
282
|
});
|
|
276
283
|
}
|
|
277
284
|
static *getQueryData(results) {
|
|
@@ -356,12 +363,13 @@ WHERE oid = $1::regclass`,
|
|
|
356
363
|
try {
|
|
357
364
|
await db.query('BEGIN');
|
|
358
365
|
try {
|
|
366
|
+
await this.snapshotTable(batch, db, result.table);
|
|
359
367
|
// Get the current LSN.
|
|
360
368
|
// The data will only be consistent once incremental replication
|
|
361
369
|
// has passed that point.
|
|
370
|
+
// We have to get this LSN _after_ we have started the snapshot query.
|
|
362
371
|
const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
|
|
363
372
|
lsn = rs.rows[0][0];
|
|
364
|
-
await this.snapshotTable(batch, db, result.table);
|
|
365
373
|
await db.query('COMMIT');
|
|
366
374
|
}
|
|
367
375
|
catch (e) {
|
|
@@ -462,7 +470,7 @@ WHERE oid = $1::regclass`,
|
|
|
462
470
|
async initReplication(replicationConnection) {
|
|
463
471
|
const result = await this.initSlot();
|
|
464
472
|
if (result.needsInitialSync) {
|
|
465
|
-
await this.startInitialReplication(replicationConnection);
|
|
473
|
+
await this.startInitialReplication(replicationConnection, result);
|
|
466
474
|
}
|
|
467
475
|
}
|
|
468
476
|
async streamChanges(replicationConnection) {
|
|
@@ -477,7 +485,7 @@ WHERE oid = $1::regclass`,
|
|
|
477
485
|
this.startedStreaming = true;
|
|
478
486
|
// Auto-activate as soon as initial replication is done
|
|
479
487
|
await this.storage.autoActivate();
|
|
480
|
-
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true }, async (batch) => {
|
|
488
|
+
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true, skipExistingRows: false }, async (batch) => {
|
|
481
489
|
// Replication never starts in the middle of a transaction
|
|
482
490
|
let inTx = false;
|
|
483
491
|
let count = 0;
|