@powersync/service-module-postgres 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/api/PostgresRouteAPIAdapter.d.ts +6 -2
- package/dist/api/PostgresRouteAPIAdapter.js +24 -10
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/module/PostgresModule.js +9 -2
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/PgManager.js +5 -1
- package/dist/replication/PgManager.js.map +1 -1
- package/dist/replication/WalStream.d.ts +9 -2
- package/dist/replication/WalStream.js +120 -114
- package/dist/replication/WalStream.js.map +1 -1
- package/package.json +4 -4
- package/src/api/PostgresRouteAPIAdapter.ts +31 -12
- package/src/module/PostgresModule.ts +11 -2
- package/src/replication/PgManager.ts +5 -1
- package/src/replication/WalStream.ts +136 -125
- package/test/src/large_batch.test.ts +268 -148
- package/test/src/schema_changes.test.ts +562 -513
- package/test/src/slow_tests.test.ts +2 -1
- package/test/src/util.ts +3 -1
- package/test/src/validation.test.ts +45 -48
- package/test/src/wal_stream.test.ts +224 -249
- package/test/src/wal_stream_utils.ts +61 -22
- package/tsconfig.tsbuildinfo +1 -1
- package/test/src/__snapshots__/pg_test.test.ts.snap +0 -256
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# @powersync/service-module-postgres
|
|
2
2
|
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 62e97f3: Support resuming initial replication for Postgres.
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- 15b2d8e: Disable SupabaseKeyCollector when a specific secret is configured.
|
|
12
|
+
- 0fa01ee: Fix replication lag diagnostics for Postgres.
|
|
13
|
+
- Updated dependencies [62e97f3]
|
|
14
|
+
- Updated dependencies [a235c9f]
|
|
15
|
+
- Updated dependencies [8c6ce90]
|
|
16
|
+
- @powersync/service-core@0.11.0
|
|
17
|
+
- @powersync/service-sync-rules@0.22.0
|
|
18
|
+
- @powersync/service-jpgwire@0.18.2
|
|
19
|
+
|
|
3
20
|
## 0.0.4
|
|
4
21
|
|
|
5
22
|
### Patch Changes
|
|
@@ -4,11 +4,15 @@ import * as sync_rules from '@powersync/service-sync-rules';
|
|
|
4
4
|
import * as service_types from '@powersync/service-types';
|
|
5
5
|
import * as types from '../types/types.js';
|
|
6
6
|
export declare class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
7
|
-
protected config: types.ResolvedConnectionConfig;
|
|
8
7
|
protected pool: pgwire.PgClient;
|
|
8
|
+
private config?;
|
|
9
9
|
connectionTag: string;
|
|
10
10
|
publicationName: string;
|
|
11
|
-
|
|
11
|
+
static withConfig(config: types.ResolvedConnectionConfig): PostgresRouteAPIAdapter;
|
|
12
|
+
/**
|
|
13
|
+
* @param config - Required for the service; optional for tests.
|
|
14
|
+
*/
|
|
15
|
+
constructor(pool: pgwire.PgClient, connectionTag?: string, config?: types.ResolvedConnectionConfig | undefined);
|
|
12
16
|
getParseSyncRulesOptions(): ParseSyncRulesOptions;
|
|
13
17
|
shutdown(): Promise<void>;
|
|
14
18
|
getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig>;
|
|
@@ -7,14 +7,21 @@ import * as pg_utils from '../utils/pgwire_utils.js';
|
|
|
7
7
|
import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
8
8
|
import { PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
9
9
|
export class PostgresRouteAPIAdapter {
|
|
10
|
-
|
|
10
|
+
static withConfig(config) {
|
|
11
|
+
const pool = pgwire.connectPgWirePool(config, {
|
|
12
|
+
idleTimeout: 30000
|
|
13
|
+
});
|
|
14
|
+
return new PostgresRouteAPIAdapter(pool, config.tag, config);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* @param config - Required for the service; optional for tests.
|
|
18
|
+
*/
|
|
19
|
+
constructor(pool, connectionTag, config) {
|
|
20
|
+
this.pool = pool;
|
|
11
21
|
this.config = config;
|
|
12
22
|
// TODO this should probably be configurable one day
|
|
13
23
|
this.publicationName = PUBLICATION_NAME;
|
|
14
|
-
this.
|
|
15
|
-
idleTimeout: 30000
|
|
16
|
-
});
|
|
17
|
-
this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG;
|
|
24
|
+
this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG;
|
|
18
25
|
}
|
|
19
26
|
getParseSyncRulesOptions() {
|
|
20
27
|
return {
|
|
@@ -29,8 +36,8 @@ export class PostgresRouteAPIAdapter {
|
|
|
29
36
|
}
|
|
30
37
|
async getConnectionStatus() {
|
|
31
38
|
const base = {
|
|
32
|
-
id: this.config
|
|
33
|
-
uri: types.baseUri(this.config)
|
|
39
|
+
id: this.config?.id ?? '',
|
|
40
|
+
uri: this.config == null ? '' : types.baseUri(this.config)
|
|
34
41
|
};
|
|
35
42
|
try {
|
|
36
43
|
await pg_utils.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`);
|
|
@@ -59,7 +66,7 @@ export class PostgresRouteAPIAdapter {
|
|
|
59
66
|
};
|
|
60
67
|
}
|
|
61
68
|
async executeQuery(query, params) {
|
|
62
|
-
if (!this.config
|
|
69
|
+
if (!this.config?.debug_api) {
|
|
63
70
|
return service_types.internal_routes.ExecuteSqlResponse.encode({
|
|
64
71
|
results: {
|
|
65
72
|
columns: [],
|
|
@@ -180,7 +187,8 @@ export class PostgresRouteAPIAdapter {
|
|
|
180
187
|
});
|
|
181
188
|
}
|
|
182
189
|
async getReplicationLag(options) {
|
|
183
|
-
const { bucketStorage
|
|
190
|
+
const { bucketStorage } = options;
|
|
191
|
+
const slotName = bucketStorage.slot_name;
|
|
184
192
|
const results = await pg_utils.retriedQuery(this.pool, {
|
|
185
193
|
statement: `SELECT
|
|
186
194
|
slot_name,
|
|
@@ -197,7 +205,13 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
197
205
|
throw new Error(`Could not determine replication lag for slot ${slotName}`);
|
|
198
206
|
}
|
|
199
207
|
async getReplicationHead() {
|
|
200
|
-
|
|
208
|
+
// On most Postgres versions, pg_logical_emit_message() returns the correct LSN.
|
|
209
|
+
// However, on Aurora (Postgres compatible), it can return an entirely different LSN,
|
|
210
|
+
// causing the write checkpoints to never be replicated back to the client.
|
|
211
|
+
// For those, we need to use pg_current_wal_lsn() instead.
|
|
212
|
+
const { results } = await pg_utils.retriedQuery(this.pool, { statement: `SELECT pg_current_wal_lsn() as lsn` }, { statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` });
|
|
213
|
+
// Specifically use the lsn from the first statement, not the second one.
|
|
214
|
+
const lsn = results[0].rows[0][0];
|
|
201
215
|
return String(lsn);
|
|
202
216
|
}
|
|
203
217
|
async getConnectionSchema() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,uBAAuB;
|
|
1
|
+
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAErD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,uBAAuB;IAKlC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,KAAM;SACpB,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAhBjD,oDAAoD;QACpD,oBAAe,GAAG,gBAAgB,CAAC;QAiBjC,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;aAC3C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BACvB,MAAM,QAAQ,GAAG,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;4BACpD,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;gBACxC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACrD,SAAS,EAAE;;;;;8BAKS;oBACpB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBAEH,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;oBAC3F,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;oBACrD,SAAS,EAAE;;;;;2BAKM;oBACjB,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBAClC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,YAAY,EAAE;qBACtD;iBACF,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC7B,kBAAkB;oBAClB,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC1G,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,UAAoB,CAAC;oBACtC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAe,CAAC;oBACvC,aAAa,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAC/B,YAAqC,EACrC,IAAY,EACZ,UAAyB,EACzB,SAAkC;QAElC,OAAO,iBAAiB,CAAC;YACvB,EAAE,EAAE,IAAI,CAAC,IAAI;YACb,IAAI,EAAE,IAAI;YACV,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAAkC;QACxD,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACrD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT,EAAE,SAAS,EAAE,oCAAoC,EAAE,EACnD,EAAE,SAAS,EAAE,4DAA4D,EAAE,CAC5E,CAAC;QAEF,yEAAyE;QACzE,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,mBAAmB;;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,CACzC,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAuCsC,CACvC,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,MAAC,GAAG,CAAC,UAAU,MAAtB,OAAO,OAAqB;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,EAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC,SAAS;oBACzE,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}
|
|
@@ -19,7 +19,14 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
19
19
|
}
|
|
20
20
|
async initialize(context) {
|
|
21
21
|
await super.initialize(context);
|
|
22
|
-
|
|
22
|
+
const client_auth = context.configuration.base_config.client_auth;
|
|
23
|
+
if (client_auth?.supabase && client_auth?.supabase_jwt_secret == null) {
|
|
24
|
+
// Only use the deprecated SupabaseKeyCollector when there is no
|
|
25
|
+
// secret hardcoded. Hardcoded secrets are handled elsewhere, using
|
|
26
|
+
// StaticSupabaseKeyCollector.
|
|
27
|
+
// Support for SupabaseKeyCollector is deprecated and support will be
|
|
28
|
+
// completely removed by Supabase soon. We can keep support a while
|
|
29
|
+
// longer for self-hosted setups, before also removing that on our side.
|
|
23
30
|
this.registerSupabaseAuth(context);
|
|
24
31
|
}
|
|
25
32
|
// Record replicated bytes using global jpgwire metrics.
|
|
@@ -32,7 +39,7 @@ export class PostgresModule extends replication.ReplicationModule {
|
|
|
32
39
|
}
|
|
33
40
|
}
|
|
34
41
|
createRouteAPIAdapter() {
|
|
35
|
-
return
|
|
42
|
+
return PostgresRouteAPIAdapter.withConfig(this.resolveConfig(this.decodedConfig));
|
|
36
43
|
}
|
|
37
44
|
createReplicator(context) {
|
|
38
45
|
const normalisedConfig = this.resolveConfig(this.decodedConfig);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,IAAI,EAAE,kCAAkC,EAAW,WAAW,EAAU,MAAM,yBAAyB,CAAC;AACtH,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAuC;QACtD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEhC,
|
|
1
|
+
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,IAAI,EAAE,kCAAkC,EAAW,WAAW,EAAU,MAAM,yBAAyB,CAAC;AACtH,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAuC;QACtD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEhC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW,CAAC;QAElE,IAAI,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,mBAAmB,IAAI,IAAI,EAAE,CAAC;YACtE,gEAAgE;YAChE,mEAAmE;YACnE,8BAA8B;YAE9B,qEAAqE;YACrE,mEAAmE;YACnE,wEAAwE;YACxE,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,wDAAwD;QACxD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,kBAAkB,CAAC;gBACzB,YAAY,CAAC,KAAK;oBAChB,OAAO,CAAC,OAAQ,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACpD,CAAC;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAES,qBAAqB;QAC7B,OAAO,uBAAuB,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IACrF,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,mBAAmB,CAAC;YAC7B,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC;YAChD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,wBAAwB,EAAE;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAsC;QAC1D,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAgC;QAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,KAAM;YACnB,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,sHAAsH;gBACtH,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,sBAAsB,CAAC,SAAS,CAAC,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBAC5E,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,gGAAgG;wBAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;oBACpG,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,gGAAgG;IACxF,oBAAoB,CAAC,OAAuC;QAClE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,yCAAyC;QACzC,aAAa,CAAC,WAAW;YACvB,EAAE,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YACnB,IAAI,UAAU,CAAC,IAAI,IAAI,KAAK,CAAC,wBAAwB,EAAE,CAAC;gBACtD,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,wBAAwB,CAAC,MAAM,CAAC,UAAiB,CAAC,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;aAClB,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAClB,MAAM,YAAY,GAAG,IAAI,oBAAoB,CAAC,MAAO,CAAC,CAAC;YACvD,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,YAAY,EAAE;gBAClD,0BAA0B;gBAC1B,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE;aAC1C,CAAC,CAAC;YACH,aAAa,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAgC;QACnD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,KAAM;YACnB,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,OAAO,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAChE,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -26,7 +26,11 @@ export class PgManager {
|
|
|
26
26
|
async snapshotConnection() {
|
|
27
27
|
const p = pgwire.connectPgWire(this.options, { type: 'standard' });
|
|
28
28
|
this.connectionPromises.push(p);
|
|
29
|
-
|
|
29
|
+
const connection = await p;
|
|
30
|
+
// Disable statement timeout for snapshot queries.
|
|
31
|
+
// On Supabase, the default is 2 minutes.
|
|
32
|
+
await connection.query(`set session statement_timeout = 0`);
|
|
33
|
+
return connection;
|
|
30
34
|
}
|
|
31
35
|
async end() {
|
|
32
36
|
for (let result of await Promise.allSettled([
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAGrD,MAAM,OAAO,SAAS;IAQpB,YACS,OAA2C,EAC3C,WAAiC;QADjC,YAAO,GAAP,OAAO,CAAoC;QAC3C,gBAAW,GAAX,WAAW,CAAsB;QAJlC,uBAAkB,GAAmC,EAAE,CAAC;QAM9D,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,
|
|
1
|
+
{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAGrD,MAAM,OAAO,SAAS;IAQpB,YACS,OAA2C,EAC3C,WAAiC;QADjC,YAAO,GAAP,OAAO,CAAoC;QAC3C,gBAAW,GAAX,WAAW,CAAsB;QAJlC,uBAAkB,GAAmC,EAAE,CAAC;QAM9D,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC;QAC3B,kDAAkD;QAClD,yCAAyC;QACzC,MAAM,UAAU,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC5D,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC,CAAC;SACH,CAAC,EAAE,CAAC;YACH,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,MAAM,MAAM,CAAC,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -11,7 +11,10 @@ export interface WalStreamOptions {
|
|
|
11
11
|
abort_signal: AbortSignal;
|
|
12
12
|
}
|
|
13
13
|
interface InitResult {
|
|
14
|
+
/** True if initial snapshot is not yet done. */
|
|
14
15
|
needsInitialSync: boolean;
|
|
16
|
+
/** True if snapshot must be started from scratch with a new slot. */
|
|
17
|
+
needsNewSlot: boolean;
|
|
15
18
|
}
|
|
16
19
|
export declare class MissingReplicationSlotError extends Error {
|
|
17
20
|
constructor(message: string);
|
|
@@ -30,6 +33,10 @@ export declare class WalStream {
|
|
|
30
33
|
get stopped(): boolean;
|
|
31
34
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
|
|
32
35
|
initSlot(): Promise<InitResult>;
|
|
36
|
+
/**
|
|
37
|
+
* If a replication slot exists, check that it is healthy.
|
|
38
|
+
*/
|
|
39
|
+
private checkReplicationSlot;
|
|
33
40
|
estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string>;
|
|
34
41
|
/**
|
|
35
42
|
* Start initial replication.
|
|
@@ -37,8 +44,8 @@ export declare class WalStream {
|
|
|
37
44
|
* If (partial) replication was done before on this slot, this clears the state
|
|
38
45
|
* and starts again from scratch.
|
|
39
46
|
*/
|
|
40
|
-
startInitialReplication(replicationConnection: pgwire.PgConnection): Promise<void>;
|
|
41
|
-
initialReplication(db: pgwire.PgConnection
|
|
47
|
+
startInitialReplication(replicationConnection: pgwire.PgConnection, status: InitResult): Promise<void>;
|
|
48
|
+
initialReplication(db: pgwire.PgConnection): Promise<void>;
|
|
42
49
|
static getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow>;
|
|
43
50
|
private snapshotTable;
|
|
44
51
|
handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean): Promise<storage.SourceTable>;
|
|
@@ -118,75 +118,100 @@ export class WalStream {
|
|
|
118
118
|
await checkSourceConfiguration(this.connections.pool, PUBLICATION_NAME);
|
|
119
119
|
const slotName = this.slot_name;
|
|
120
120
|
const status = await this.storage.getStatus();
|
|
121
|
-
|
|
121
|
+
const snapshotDone = status.snapshot_done && status.checkpoint_lsn != null;
|
|
122
|
+
if (snapshotDone) {
|
|
123
|
+
// Snapshot is done, but we still need to check the replication slot status
|
|
122
124
|
logger.info(`${slotName} Initial replication already done`);
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
125
|
+
}
|
|
126
|
+
// Check if replication slot exists
|
|
127
|
+
const rs = await this.connections.pool.query({
|
|
128
|
+
statement: 'SELECT 1 FROM pg_replication_slots WHERE slot_name = $1',
|
|
129
|
+
params: [{ type: 'varchar', value: slotName }]
|
|
130
|
+
});
|
|
131
|
+
const slotExists = rs.rows.length > 0;
|
|
132
|
+
if (slotExists) {
|
|
133
|
+
// This checks that the slot is still valid
|
|
134
|
+
const r = await this.checkReplicationSlot();
|
|
135
|
+
return {
|
|
136
|
+
needsInitialSync: !snapshotDone,
|
|
137
|
+
needsNewSlot: r.needsNewSlot
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
return { needsInitialSync: true, needsNewSlot: true };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* If a replication slot exists, check that it is healthy.
|
|
146
|
+
*/
|
|
147
|
+
async checkReplicationSlot() {
|
|
148
|
+
let last_error = null;
|
|
149
|
+
const slotName = this.slot_name;
|
|
150
|
+
// Check that replication slot exists
|
|
151
|
+
for (let i = 120; i >= 0; i--) {
|
|
152
|
+
await touch();
|
|
153
|
+
if (i == 0) {
|
|
154
|
+
container.reporter.captureException(last_error, {
|
|
155
|
+
level: errors.ErrorSeverity.ERROR,
|
|
156
|
+
metadata: {
|
|
157
|
+
replication_slot: slotName
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
throw last_error;
|
|
161
|
+
}
|
|
162
|
+
try {
|
|
163
|
+
// We peek a large number of changes here, to make it more likely to pick up replication slot errors.
|
|
164
|
+
// For example, "publication does not exist" only occurs here if the peek actually includes changes related
|
|
165
|
+
// to the slot.
|
|
166
|
+
logger.info(`Checking ${slotName}`);
|
|
167
|
+
// The actual results can be quite large, so we don't actually return everything
|
|
168
|
+
// due to memory and processing overhead that would create.
|
|
169
|
+
const cursor = await this.connections.pool.stream({
|
|
170
|
+
statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
|
|
171
|
+
params: [
|
|
172
|
+
{ type: 'varchar', value: slotName },
|
|
173
|
+
{ type: 'varchar', value: PUBLICATION_NAME }
|
|
174
|
+
]
|
|
175
|
+
});
|
|
176
|
+
for await (let _chunk of cursor) {
|
|
177
|
+
// No-op, just exhaust the cursor
|
|
178
|
+
}
|
|
179
|
+
// Success
|
|
180
|
+
logger.info(`Slot ${slotName} appears healthy`);
|
|
181
|
+
return { needsInitialSync: false, needsNewSlot: false };
|
|
182
|
+
}
|
|
183
|
+
catch (e) {
|
|
184
|
+
last_error = e;
|
|
185
|
+
logger.warn(`${slotName} Replication slot error`, e);
|
|
186
|
+
if (this.stopped) {
|
|
187
|
+
throw e;
|
|
188
|
+
}
|
|
189
|
+
// Could also be `publication "powersync" does not exist`, although this error may show up much later
|
|
190
|
+
// in some cases.
|
|
191
|
+
if (/incorrect prev-link/.test(e.message) ||
|
|
192
|
+
/replication slot.*does not exist/.test(e.message) ||
|
|
193
|
+
/publication.*does not exist/.test(e.message)) {
|
|
194
|
+
container.reporter.captureException(e, {
|
|
195
|
+
level: errors.ErrorSeverity.WARNING,
|
|
130
196
|
metadata: {
|
|
197
|
+
try_index: i,
|
|
131
198
|
replication_slot: slotName
|
|
132
199
|
}
|
|
133
200
|
});
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
logger.info(
|
|
141
|
-
|
|
142
|
-
// due to memory and processing overhead that would create.
|
|
143
|
-
const cursor = await this.connections.pool.stream({
|
|
144
|
-
statement: `SELECT 1 FROM pg_catalog.pg_logical_slot_peek_binary_changes($1, NULL, 1000, 'proto_version', '1', 'publication_names', $2)`,
|
|
145
|
-
params: [
|
|
146
|
-
{ type: 'varchar', value: slotName },
|
|
147
|
-
{ type: 'varchar', value: PUBLICATION_NAME }
|
|
148
|
-
]
|
|
149
|
-
});
|
|
150
|
-
for await (let _chunk of cursor) {
|
|
151
|
-
// No-op, just exhaust the cursor
|
|
152
|
-
}
|
|
153
|
-
// Success
|
|
154
|
-
logger.info(`Slot ${slotName} appears healthy`);
|
|
155
|
-
return { needsInitialSync: false };
|
|
156
|
-
}
|
|
157
|
-
catch (e) {
|
|
158
|
-
last_error = e;
|
|
159
|
-
logger.warn(`${slotName} Replication slot error`, e);
|
|
160
|
-
if (this.stopped) {
|
|
161
|
-
throw e;
|
|
162
|
-
}
|
|
163
|
-
// Could also be `publication "powersync" does not exist`, although this error may show up much later
|
|
164
|
-
// in some cases.
|
|
165
|
-
if (/incorrect prev-link/.test(e.message) ||
|
|
166
|
-
/replication slot.*does not exist/.test(e.message) ||
|
|
167
|
-
/publication.*does not exist/.test(e.message)) {
|
|
168
|
-
container.reporter.captureException(e, {
|
|
169
|
-
level: errors.ErrorSeverity.WARNING,
|
|
170
|
-
metadata: {
|
|
171
|
-
try_index: i,
|
|
172
|
-
replication_slot: slotName
|
|
173
|
-
}
|
|
174
|
-
});
|
|
175
|
-
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
176
|
-
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
177
|
-
//
|
|
178
|
-
// Sample: publication "powersync" does not exist
|
|
179
|
-
// Happens when publication deleted or never created.
|
|
180
|
-
// Slot must be re-created in this case.
|
|
181
|
-
logger.info(`${slotName} does not exist anymore, will create new slot`);
|
|
182
|
-
throw new MissingReplicationSlotError(`Replication slot ${slotName} does not exist anymore`);
|
|
183
|
-
}
|
|
184
|
-
// Try again after a pause
|
|
185
|
-
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
201
|
+
// Sample: record with incorrect prev-link 10000/10000 at 0/18AB778
|
|
202
|
+
// Seen during development. Some internal error, fixed by re-creating slot.
|
|
203
|
+
//
|
|
204
|
+
// Sample: publication "powersync" does not exist
|
|
205
|
+
// Happens when publication deleted or never created.
|
|
206
|
+
// Slot must be re-created in this case.
|
|
207
|
+
logger.info(`${slotName} does not exist anymore, will create new slot`);
|
|
208
|
+
return { needsInitialSync: true, needsNewSlot: true };
|
|
186
209
|
}
|
|
210
|
+
// Try again after a pause
|
|
211
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
187
212
|
}
|
|
188
213
|
}
|
|
189
|
-
|
|
214
|
+
throw new Error('Unreachable');
|
|
190
215
|
}
|
|
191
216
|
async estimatedCount(db, table) {
|
|
192
217
|
const results = await db.query({
|
|
@@ -209,69 +234,49 @@ WHERE oid = $1::regclass`,
|
|
|
209
234
|
* If (partial) replication was done before on this slot, this clears the state
|
|
210
235
|
* and starts again from scratch.
|
|
211
236
|
*/
|
|
212
|
-
async startInitialReplication(replicationConnection) {
|
|
237
|
+
async startInitialReplication(replicationConnection, status) {
|
|
213
238
|
// If anything here errors, the entire replication process is aborted,
|
|
214
|
-
// and all connections closed, including this one.
|
|
239
|
+
// and all connections are closed, including this one.
|
|
215
240
|
const db = await this.connections.snapshotConnection();
|
|
216
241
|
const slotName = this.slot_name;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
// with streaming replication.
|
|
233
|
-
const lsn = pgwire.lsnMakeComparable(row[1]);
|
|
234
|
-
const snapshot = row[2];
|
|
235
|
-
logger.info(`Created replication slot ${slotName} at ${lsn} with snapshot ${snapshot}`);
|
|
236
|
-
// https://stackoverflow.com/questions/70160769/postgres-logical-replication-starting-from-given-lsn
|
|
237
|
-
await db.query('BEGIN');
|
|
238
|
-
// Use the snapshot exported above.
|
|
239
|
-
// Using SERIALIZABLE isolation level may give stronger guarantees, but that complicates
|
|
240
|
-
// the replication slot + snapshot above. And we still won't have SERIALIZABLE
|
|
241
|
-
// guarantees with streaming replication.
|
|
242
|
-
// See: ./docs/serializability.md for details.
|
|
243
|
-
//
|
|
244
|
-
// Another alternative here is to use the same pgwire connection for initial replication as well,
|
|
245
|
-
// instead of synchronizing a separate transaction to the snapshot.
|
|
246
|
-
try {
|
|
247
|
-
await db.query(`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ`);
|
|
248
|
-
await db.query(`SET TRANSACTION READ ONLY`);
|
|
249
|
-
await db.query(`SET TRANSACTION SNAPSHOT '${snapshot}'`);
|
|
250
|
-
// Disable statement timeout for the duration of this transaction.
|
|
251
|
-
// On Supabase, the default is 2 minutes.
|
|
252
|
-
await db.query(`set local statement_timeout = 0`);
|
|
253
|
-
logger.info(`${slotName} Starting initial replication`);
|
|
254
|
-
await this.initialReplication(db, lsn);
|
|
255
|
-
logger.info(`${slotName} Initial replication done`);
|
|
256
|
-
await db.query('COMMIT');
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
await db.query('ROLLBACK');
|
|
260
|
-
throw e;
|
|
242
|
+
if (status.needsNewSlot) {
|
|
243
|
+
// This happens when there is no existing replication slot, or if the
|
|
244
|
+
// existing one is unhealthy.
|
|
245
|
+
// In those cases, we have to start replication from scratch.
|
|
246
|
+
// If there is an existing healthy slot, we can skip this and continue
|
|
247
|
+
// initial replication where we left off.
|
|
248
|
+
await this.storage.clear();
|
|
249
|
+
await db.query({
|
|
250
|
+
statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
|
|
251
|
+
params: [{ type: 'varchar', value: slotName }]
|
|
252
|
+
});
|
|
253
|
+
// We use the replication connection here, not a pool.
|
|
254
|
+
// The replication slot must be created before we start snapshotting tables.
|
|
255
|
+
await replicationConnection.query(`CREATE_REPLICATION_SLOT ${slotName} LOGICAL pgoutput`);
|
|
256
|
+
logger.info(`Created replication slot ${slotName}`);
|
|
261
257
|
}
|
|
258
|
+
await this.initialReplication(db);
|
|
262
259
|
}
|
|
263
|
-
async initialReplication(db
|
|
260
|
+
async initialReplication(db) {
|
|
264
261
|
const sourceTables = this.sync_rules.getSourceTables();
|
|
265
|
-
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true }, async (batch) => {
|
|
262
|
+
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true, skipExistingRows: true }, async (batch) => {
|
|
263
|
+
const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
|
|
264
|
+
const startLsn = rs.rows[0][0];
|
|
266
265
|
for (let tablePattern of sourceTables) {
|
|
267
266
|
const tables = await this.getQualifiedTableNames(batch, db, tablePattern);
|
|
268
267
|
for (let table of tables) {
|
|
268
|
+
if (table.snapshotComplete) {
|
|
269
|
+
logger.info(`${this.slot_name} Skipping ${table.qualifiedName} - snapshot already done`);
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
269
272
|
await this.snapshotTable(batch, db, table);
|
|
270
|
-
await
|
|
273
|
+
const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
|
|
274
|
+
const tableLsnNotBefore = rs.rows[0][0];
|
|
275
|
+
await batch.markSnapshotDone([table], tableLsnNotBefore);
|
|
271
276
|
await touch();
|
|
272
277
|
}
|
|
273
278
|
}
|
|
274
|
-
await batch.commit(
|
|
279
|
+
await batch.commit(startLsn);
|
|
275
280
|
});
|
|
276
281
|
}
|
|
277
282
|
static *getQueryData(results) {
|
|
@@ -356,12 +361,13 @@ WHERE oid = $1::regclass`,
|
|
|
356
361
|
try {
|
|
357
362
|
await db.query('BEGIN');
|
|
358
363
|
try {
|
|
364
|
+
await this.snapshotTable(batch, db, result.table);
|
|
359
365
|
// Get the current LSN.
|
|
360
366
|
// The data will only be consistent once incremental replication
|
|
361
367
|
// has passed that point.
|
|
368
|
+
// We have to get this LSN _after_ we have started the snapshot query.
|
|
362
369
|
const rs = await db.query(`select pg_current_wal_lsn() as lsn`);
|
|
363
370
|
lsn = rs.rows[0][0];
|
|
364
|
-
await this.snapshotTable(batch, db, result.table);
|
|
365
371
|
await db.query('COMMIT');
|
|
366
372
|
}
|
|
367
373
|
catch (e) {
|
|
@@ -462,7 +468,7 @@ WHERE oid = $1::regclass`,
|
|
|
462
468
|
async initReplication(replicationConnection) {
|
|
463
469
|
const result = await this.initSlot();
|
|
464
470
|
if (result.needsInitialSync) {
|
|
465
|
-
await this.startInitialReplication(replicationConnection);
|
|
471
|
+
await this.startInitialReplication(replicationConnection, result);
|
|
466
472
|
}
|
|
467
473
|
}
|
|
468
474
|
async streamChanges(replicationConnection) {
|
|
@@ -477,7 +483,7 @@ WHERE oid = $1::regclass`,
|
|
|
477
483
|
this.startedStreaming = true;
|
|
478
484
|
// Auto-activate as soon as initial replication is done
|
|
479
485
|
await this.storage.autoActivate();
|
|
480
|
-
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true }, async (batch) => {
|
|
486
|
+
await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA, storeCurrentData: true, skipExistingRows: false }, async (batch) => {
|
|
481
487
|
// Replication never starts in the middle of a transaction
|
|
482
488
|
let inTx = false;
|
|
483
489
|
let count = 0;
|