@powersync/service-module-postgres 0.19.2 → 0.19.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/PostgresRouteAPIAdapter.d.ts +1 -1
- package/dist/api/PostgresRouteAPIAdapter.js +63 -72
- package/dist/api/PostgresRouteAPIAdapter.js.map +1 -1
- package/dist/module/PostgresModule.js.map +1 -1
- package/dist/replication/MissingReplicationSlotError.d.ts +41 -0
- package/dist/replication/MissingReplicationSlotError.js +33 -0
- package/dist/replication/MissingReplicationSlotError.js.map +1 -0
- package/dist/replication/PostgresErrorRateLimiter.js +1 -1
- package/dist/replication/PostgresErrorRateLimiter.js.map +1 -1
- package/dist/replication/SnapshotQuery.js +2 -2
- package/dist/replication/SnapshotQuery.js.map +1 -1
- package/dist/replication/WalStream.d.ts +37 -14
- package/dist/replication/WalStream.js +145 -41
- package/dist/replication/WalStream.js.map +1 -1
- package/dist/replication/WalStreamReplicationJob.d.ts +1 -1
- package/dist/replication/WalStreamReplicationJob.js +7 -4
- package/dist/replication/WalStreamReplicationJob.js.map +1 -1
- package/dist/replication/WalStreamReplicator.d.ts +0 -1
- package/dist/replication/WalStreamReplicator.js +0 -22
- package/dist/replication/WalStreamReplicator.js.map +1 -1
- package/dist/replication/replication-index.d.ts +3 -1
- package/dist/replication/replication-index.js +3 -1
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-utils.d.ts +3 -11
- package/dist/replication/replication-utils.js +101 -164
- package/dist/replication/replication-utils.js.map +1 -1
- package/dist/replication/wal-budget-utils.d.ts +23 -0
- package/dist/replication/wal-budget-utils.js +57 -0
- package/dist/replication/wal-budget-utils.js.map +1 -0
- package/dist/types/registry.js +1 -1
- package/dist/types/registry.js.map +1 -1
- package/package.json +15 -11
- package/sql/check-source-configuration.plpgsql +13 -0
- package/sql/debug-tables-info-batched.plpgsql +230 -0
- package/CHANGELOG.md +0 -843
- package/src/api/PostgresRouteAPIAdapter.ts +0 -356
- package/src/index.ts +0 -1
- package/src/module/PostgresModule.ts +0 -122
- package/src/replication/ConnectionManagerFactory.ts +0 -33
- package/src/replication/PgManager.ts +0 -122
- package/src/replication/PgRelation.ts +0 -41
- package/src/replication/PostgresErrorRateLimiter.ts +0 -48
- package/src/replication/SnapshotQuery.ts +0 -213
- package/src/replication/WalStream.ts +0 -1157
- package/src/replication/WalStreamReplicationJob.ts +0 -138
- package/src/replication/WalStreamReplicator.ts +0 -79
- package/src/replication/replication-index.ts +0 -5
- package/src/replication/replication-utils.ts +0 -398
- package/src/types/registry.ts +0 -275
- package/src/types/resolver.ts +0 -227
- package/src/types/types.ts +0 -44
- package/src/utils/application-name.ts +0 -8
- package/src/utils/migration_lib.ts +0 -80
- package/src/utils/populate_test_data.ts +0 -37
- package/src/utils/populate_test_data_worker.ts +0 -53
- package/src/utils/postgres_version.ts +0 -8
- package/test/src/checkpoints.test.ts +0 -86
- package/test/src/chunked_snapshots.test.ts +0 -161
- package/test/src/env.ts +0 -11
- package/test/src/large_batch.test.ts +0 -241
- package/test/src/pg_test.test.ts +0 -729
- package/test/src/resuming_snapshots.test.ts +0 -160
- package/test/src/route_api_adapter.test.ts +0 -62
- package/test/src/schema_changes.test.ts +0 -655
- package/test/src/setup.ts +0 -12
- package/test/src/slow_tests.test.ts +0 -519
- package/test/src/storage_combination.test.ts +0 -35
- package/test/src/types/registry.test.ts +0 -149
- package/test/src/util.ts +0 -151
- package/test/src/validation.test.ts +0 -63
- package/test/src/wal_stream.test.ts +0 -607
- package/test/src/wal_stream_utils.ts +0 -284
- package/test/tsconfig.json +0 -27
- package/tsconfig.json +0 -34
- package/tsconfig.tsbuildinfo +0 -1
- package/vitest.config.ts +0 -3
|
@@ -20,8 +20,8 @@ export declare class PostgresRouteAPIAdapter implements api.RouteAPI {
|
|
|
20
20
|
getConnectionStatus(): Promise<service_types.ConnectionStatusV2>;
|
|
21
21
|
executeQuery(query: string, params: any[]): Promise<service_types.internal_routes.ExecuteSqlResponse>;
|
|
22
22
|
getDebugTablesInfo(tablePatterns: sync_rules.TablePattern[], sqlSyncRules: sync_rules.SqlSyncRules): Promise<api.PatternResult[]>;
|
|
23
|
-
protected getDebugTableInfo(tablePattern: sync_rules.TablePattern, name: string, relationId: number | null, syncRules: sync_rules.SqlSyncRules): Promise<service_types.TableInfo>;
|
|
24
23
|
getReplicationLagBytes(options: api.ReplicationLagOptions): Promise<number | undefined>;
|
|
24
|
+
getSlotWalBudget(options: api.SlotWalBudgetOptions): Promise<api.SlotWalBudgetInfo | undefined>;
|
|
25
25
|
getReplicationHead(): Promise<string>;
|
|
26
26
|
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
27
27
|
getConnectionSchema(): Promise<service_types.DatabaseSchema[]>;
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ErrorCode, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
4
4
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
5
5
|
import * as service_types from '@powersync/service-types';
|
|
6
6
|
import * as replication_utils from '../replication/replication-utils.js';
|
|
7
|
-
import { getDebugTableInfo } from '../replication/replication-utils.js';
|
|
8
7
|
import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js';
|
|
8
|
+
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
9
9
|
import * as types from '../types/types.js';
|
|
10
10
|
import { getApplicationName } from '../utils/application-name.js';
|
|
11
|
-
import { PostgresTypeResolver } from '../types/resolver.js';
|
|
12
11
|
export class PostgresRouteAPIAdapter {
|
|
13
12
|
pool;
|
|
14
13
|
config;
|
|
@@ -124,77 +123,25 @@ export class PostgresRouteAPIAdapter {
|
|
|
124
123
|
}
|
|
125
124
|
}
|
|
126
125
|
async getDebugTablesInfo(tablePatterns, sqlSyncRules) {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const schema = tablePattern.schema;
|
|
130
|
-
let patternResult = {
|
|
131
|
-
schema: schema,
|
|
132
|
-
pattern: tablePattern.tablePattern,
|
|
133
|
-
wildcard: tablePattern.isWildcard
|
|
134
|
-
};
|
|
135
|
-
result.push(patternResult);
|
|
136
|
-
if (tablePattern.isWildcard) {
|
|
137
|
-
patternResult.tables = [];
|
|
138
|
-
const prefix = tablePattern.tablePrefix;
|
|
139
|
-
const results = await lib_postgres.retriedQuery(this.pool, {
|
|
140
|
-
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
141
|
-
FROM pg_class c
|
|
142
|
-
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
143
|
-
WHERE n.nspname = $1
|
|
144
|
-
AND c.relkind = 'r'
|
|
145
|
-
AND c.relname LIKE $2`,
|
|
146
|
-
params: [
|
|
147
|
-
{ type: 'varchar', value: schema },
|
|
148
|
-
{ type: 'varchar', value: tablePattern.tablePattern }
|
|
149
|
-
]
|
|
150
|
-
});
|
|
151
|
-
for (let row of pgwire.pgwireRows(results)) {
|
|
152
|
-
const name = row.table_name;
|
|
153
|
-
const relationId = row.relid;
|
|
154
|
-
if (!name.startsWith(prefix)) {
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
const details = await this.getDebugTableInfo(tablePattern, name, relationId, sqlSyncRules);
|
|
158
|
-
patternResult.tables.push(details);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
else {
|
|
162
|
-
const results = await lib_postgres.retriedQuery(this.pool, {
|
|
163
|
-
statement: `SELECT c.oid AS relid, c.relname AS table_name
|
|
164
|
-
FROM pg_class c
|
|
165
|
-
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
166
|
-
WHERE n.nspname = $1
|
|
167
|
-
AND c.relkind = 'r'
|
|
168
|
-
AND c.relname = $2`,
|
|
169
|
-
params: [
|
|
170
|
-
{ type: 'varchar', value: schema },
|
|
171
|
-
{ type: 'varchar', value: tablePattern.tablePattern }
|
|
172
|
-
]
|
|
173
|
-
});
|
|
174
|
-
if (results.rows.length == 0) {
|
|
175
|
-
// Table not found
|
|
176
|
-
patternResult.table = await this.getDebugTableInfo(tablePattern, tablePattern.name, null, sqlSyncRules);
|
|
177
|
-
}
|
|
178
|
-
else {
|
|
179
|
-
const row = pgwire.pgwireRows(results)[0];
|
|
180
|
-
const name = row.table_name;
|
|
181
|
-
const relationId = row.relid;
|
|
182
|
-
patternResult.table = await this.getDebugTableInfo(tablePattern, name, relationId, sqlSyncRules);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
126
|
+
if (!this.config) {
|
|
127
|
+
throw new ServiceAssertionError(`config is required`);
|
|
185
128
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
return getDebugTableInfo({
|
|
190
|
-
db: this.pool,
|
|
191
|
-
name: name,
|
|
192
|
-
publicationName: this.publicationName,
|
|
193
|
-
connectionTag: this.connectionTag,
|
|
194
|
-
tablePattern: tablePattern,
|
|
195
|
-
relationId: relationId,
|
|
196
|
-
syncRules: syncRules
|
|
129
|
+
const connection = await pgwire.connectPgWire(this.config, {
|
|
130
|
+
type: 'standard',
|
|
131
|
+
applicationName: getApplicationName()
|
|
197
132
|
});
|
|
133
|
+
try {
|
|
134
|
+
return await replication_utils.getDebugTablesInfo({
|
|
135
|
+
db: connection,
|
|
136
|
+
publicationName: this.publicationName,
|
|
137
|
+
connectionTag: this.connectionTag,
|
|
138
|
+
tablePatterns,
|
|
139
|
+
syncRules: sqlSyncRules
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
finally {
|
|
143
|
+
await connection.end();
|
|
144
|
+
}
|
|
198
145
|
}
|
|
199
146
|
async getReplicationLagBytes(options) {
|
|
200
147
|
const { bucketStorage } = options;
|
|
@@ -218,6 +165,50 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
|
|
|
218
165
|
description: `Could not determine replication lag for slot ${slotName}`
|
|
219
166
|
});
|
|
220
167
|
}
|
|
168
|
+
async getSlotWalBudget(options) {
|
|
169
|
+
const { slotName } = options;
|
|
170
|
+
// Query slot status
|
|
171
|
+
const slotResult = await lib_postgres.retriedQuery(this.pool, {
|
|
172
|
+
statement: 'SELECT * FROM pg_replication_slots WHERE slot_name = $1',
|
|
173
|
+
params: [{ type: 'varchar', value: slotName }]
|
|
174
|
+
});
|
|
175
|
+
const [slot] = pgwire.pgwireRows(slotResult);
|
|
176
|
+
if (!slot) {
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
const walStatus = slot.wal_status;
|
|
180
|
+
if (walStatus == null) {
|
|
181
|
+
// PG < 13 — wal_status column doesn't exist
|
|
182
|
+
return undefined;
|
|
183
|
+
}
|
|
184
|
+
const result = {
|
|
185
|
+
wal_status: String(walStatus)
|
|
186
|
+
};
|
|
187
|
+
// Query max_slot_wal_keep_size
|
|
188
|
+
try {
|
|
189
|
+
const settingsResult = await lib_postgres.retriedQuery(this.pool, {
|
|
190
|
+
statement: `SELECT setting, unit FROM pg_settings WHERE name = 'max_slot_wal_keep_size'`
|
|
191
|
+
});
|
|
192
|
+
const [row] = pgwire.pgwireRows(settingsResult);
|
|
193
|
+
if (row) {
|
|
194
|
+
const setting = Number(row.setting);
|
|
195
|
+
if (setting >= 0) {
|
|
196
|
+
const unit = String(row.unit ?? 'MB');
|
|
197
|
+
const multiplier = unit === 'kB' ? 1024 : unit === '8kB' ? 8192 : 1024 * 1024; // default MB
|
|
198
|
+
result.max_slot_wal_keep_size = setting * multiplier;
|
|
199
|
+
}
|
|
200
|
+
// setting < 0 means unlimited — leave undefined
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
catch (e) {
|
|
204
|
+
// Best-effort — budget info is informational
|
|
205
|
+
}
|
|
206
|
+
// safe_wal_size (PG 13+, only populated when max_slot_wal_keep_size is set)
|
|
207
|
+
if (slot.safe_wal_size != null) {
|
|
208
|
+
result.safe_wal_size = Number(slot.safe_wal_size);
|
|
209
|
+
}
|
|
210
|
+
return result;
|
|
211
|
+
}
|
|
221
212
|
async getReplicationHead() {
|
|
222
213
|
// On most Postgres versions, pg_logical_emit_message() returns the correct LSN.
|
|
223
214
|
// However, on Aurora (Postgres compatible), it can return an entirely different LSN,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/PostgresRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAEnG,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,iBAAiB,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,OAAO,uBAAuB;IAkBtB;IAEF;IAnBF,SAAS,CAAuB;IACxC,aAAa,CAAS;IACtB,oDAAoD;IACpD,eAAe,GAAG,gBAAgB,CAAC;IAEnC,MAAM,CAAC,UAAU,CAAC,MAAsC;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC5C,WAAW,EAAE,MAAM;YACnB,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,OAAO,IAAI,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,YACY,IAAqB,EAC/B,aAAsB,EACd,MAAuC;QAFrC,SAAI,GAAJ,IAAI,CAAiB;QAEvB,WAAM,GAAN,MAAM,CAAiC;QAE/C,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,UAAU,CAAC,WAAW,CAAC;IAC/D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,QAAQ;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SAC3D,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,iBAAiB,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,IAAI;gBACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;YAC5B,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,6BAA6B;aACrC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBACnC,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC;aAC/C,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1C,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;4BAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;4BAChE,MAAM,QAAQ,GAAG,UAAU,CAAC,iBAAiB,CAC3C,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAClC,UAAU,CAAC,oBAAoB,CAAC,4BAA4B,CAC7D,CAAC;4BACF,IAAI,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC;gCAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;4BACvB,CAAC;iCAAM,IAAI,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;gCAC5C,OAAO,QAAQ,CAAC;4BAClB,CAAC;iCAAM,CAAC;gCACN,OAAO,IAAI,CAAC;4BACd,CAAC;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC;iBACH;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;gBAC7D,OAAO,EAAE;oBACP,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,EAAE;iBACT;gBACD,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAqC;QAErC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE;YACzD,IAAI,EAAE,UAAU;YAChB,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,OAAO,MAAM,iBAAiB,CAAC,kBAAkB,CAAC;gBAChD,EAAE,EAAE,UAAU;gBACd,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,aAAa;gBACb,SAAS,EAAE,YAAY;aACxB,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,OAAkC;QAC7D,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YACzD,SAAS,EAAE;;;;;wDAKuC;YAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,GAAG,EAAE,CAAC;YACR,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,YAAY,CAAC;YACrB,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,SAAS,CAAC,WAAW;YAC3B,WAAW,EAAE,gDAAgD,QAAQ,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAiC;QACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAE7B,oBAAoB;QACpB,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;YAC5D,SAAS,EAAE,yDAAyD;YACpE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC/C,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,4CAA4C;YAC5C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAA0B;YACpC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC;SAC9B,CAAC;QAEF,+BAA+B;QAC/B,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;gBAChE,SAAS,EAAE,6EAA6E;aACzF,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAChD,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;oBACtC,MAAM,UAAU,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,aAAa;oBAC5F,MAAM,CAAC,sBAAsB,GAAG,OAAO,GAAG,UAAU,CAAC;gBACvD,CAAC;gBACD,gDAAgD;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,6CAA6C;QAC/C,CAAC;QAED,4EAA4E;QAC5E,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YAC/B,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,0DAA0D;QAC1D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;QAErG,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEnD,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;QAErC,sFAAsF;QACtF,qGAAqG;QACrG,6FAA6F;QAC7F,wCAAwC;QACxC,MAAM,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAEhE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,iHAAiH;QACjH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAC7C,IAAI,CAAC,IAAI,EACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4CAwCsC,CACvC,CAAC;QACF,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,OAAO,GAAiD,EAAE,CAAC;QAE/D,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK;gBAC1C,IAAI,EAAE,GAAG,CAAC,UAAU;gBACpB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;YACH,MAAM,KAAK,GAA8B;gBACvC,IAAI,EAAE,GAAG,CAAC,SAAS;gBACnB,OAAO,EAAE,EAAW;aACrB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;gBACvC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9E,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;oBACjB,IAAI,EAAE,MAAM,CAAC,OAAO;oBACpB,WAAW,EAAE,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,SAAS;oBACrF,IAAI,EAAE,MAAM,CAAC,SAAS;oBACtB,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA0C,MAAM,iCAAiC,CAAC;AAClG,OAAO,EAEL,kCAAkC,EAGlC,WAAW,EAEZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"PostgresModule.js","sourceRoot":"","sources":["../../src/module/PostgresModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA0C,MAAM,iCAAiC,CAAC;AAClG,OAAO,EAEL,kCAAkC,EAGlC,WAAW,EAEZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AACvG,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,OAAO,cAAe,SAAQ,WAAW,CAAC,iBAAiD;IAC/F;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,KAAK,CAAC,wBAAwB;YACpC,YAAY,EAAE,KAAK,CAAC,wBAAwB;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAuC;QACzD,sGAAsG;QACtG,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC9B,OAAO,CAAC,kBAAkB,CAAC;gBACzB,YAAY,CAAC,KAAK;oBAChB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvF,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAES,qBAAqB;QAC7B,OAAO,uBAAuB,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IACrF,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,mBAAmB,CAAC;YAC7B,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC;YAChD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,wBAAwB,EAAE;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAsC;QAC1D,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAgC;QAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,sHAAsH;gBACtH,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,sBAAsB,CAAC,SAAS,CAAC,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;oBAC5E,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,gGAAgG;wBAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;oBACpG,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAgC;QACnD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAwD;QAClF,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE;YACxD,WAAW,EAAE,MAAM;YACnB,OAAO,EAAE,CAAC;YACV,eAAe,EAAE,kBAAkB,EAAE;SACtC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAC/D,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,OAAO,CAAC,gBAAgB,CAAC;SACjD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Replication slot WAL status from `pg_replication_slots.wal_status` (PG 13+).
|
|
3
|
+
* - `reserved`: WAL needed by the slot is within `max_wal_size`
|
|
4
|
+
* - `extended`: WAL needed exceeds `max_wal_size` but is protected by `max_slot_wal_keep_size`
|
|
5
|
+
* - `unreserved`: WAL may be removed at next checkpoint — slot is at risk
|
|
6
|
+
* - `lost`: WAL has been removed — slot is invalidated and unusable
|
|
7
|
+
* - `missing`: synthetic value — the slot row is absent from `pg_replication_slots`
|
|
8
|
+
*/
|
|
9
|
+
export type WalStatus = 'reserved' | 'extended' | 'unreserved' | 'lost' | 'missing';
|
|
10
|
+
/**
|
|
11
|
+
* Replication phase when the error was detected.
|
|
12
|
+
* - `snapshot`: during an active full-table scan (snapshotTable) — long-running, retry may be futile
|
|
13
|
+
* - `streaming`: during WAL streaming, at startup, or between replication cycles — retry is safe
|
|
14
|
+
*/
|
|
15
|
+
export type ReplicationPhase = 'snapshot' | 'streaming';
|
|
16
|
+
export declare class MissingReplicationSlotError extends Error {
|
|
17
|
+
/** Slot WAL status at the time the error was detected. */
|
|
18
|
+
walStatus: WalStatus;
|
|
19
|
+
/** Replication phase when the error occurred — controls retry behavior. */
|
|
20
|
+
phase: ReplicationPhase;
|
|
21
|
+
/** PG 14+ invalidation reason from `pg_replication_slots.invalidation_reason`. */
|
|
22
|
+
invalidationReason?: string;
|
|
23
|
+
constructor(message: string, options: {
|
|
24
|
+
walStatus: WalStatus;
|
|
25
|
+
phase: ReplicationPhase;
|
|
26
|
+
invalidationReason?: string;
|
|
27
|
+
cause?: any;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Determines whether replication should be retried after a slot invalidation.
|
|
32
|
+
*
|
|
33
|
+
* Returns false when retry would be futile (e.g. slot lost during snapshot
|
|
34
|
+
* due to WAL budget exhaustion — retrying would repeat the same long snapshot
|
|
35
|
+
* and likely fail again).
|
|
36
|
+
*
|
|
37
|
+
* Blocks retry when walStatus is 'lost' during snapshot phase (unless the
|
|
38
|
+
* invalidation reason is 'rows_removed', which is not a WAL budget issue).
|
|
39
|
+
* Allows retry in all other cases.
|
|
40
|
+
*/
|
|
41
|
+
export declare function shouldRetryReplication(error: MissingReplicationSlotError): boolean;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export class MissingReplicationSlotError extends Error {
|
|
2
|
+
/** Slot WAL status at the time the error was detected. */
|
|
3
|
+
walStatus;
|
|
4
|
+
/** Replication phase when the error occurred — controls retry behavior. */
|
|
5
|
+
phase;
|
|
6
|
+
/** PG 14+ invalidation reason from `pg_replication_slots.invalidation_reason`. */
|
|
7
|
+
invalidationReason;
|
|
8
|
+
constructor(message, options) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.cause = options.cause;
|
|
11
|
+
this.walStatus = options.walStatus;
|
|
12
|
+
this.phase = options.phase;
|
|
13
|
+
this.invalidationReason = options.invalidationReason;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Determines whether replication should be retried after a slot invalidation.
|
|
18
|
+
*
|
|
19
|
+
* Returns false when retry would be futile (e.g. slot lost during snapshot
|
|
20
|
+
* due to WAL budget exhaustion — retrying would repeat the same long snapshot
|
|
21
|
+
* and likely fail again).
|
|
22
|
+
*
|
|
23
|
+
* Blocks retry when walStatus is 'lost' during snapshot phase (unless the
|
|
24
|
+
* invalidation reason is 'rows_removed', which is not a WAL budget issue).
|
|
25
|
+
* Allows retry in all other cases.
|
|
26
|
+
*/
|
|
27
|
+
export function shouldRetryReplication(error) {
|
|
28
|
+
if (error.walStatus === 'lost' && error.phase === 'snapshot' && error.invalidationReason !== 'rows_removed') {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=MissingReplicationSlotError.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MissingReplicationSlotError.js","sourceRoot":"","sources":["../../src/replication/MissingReplicationSlotError.ts"],"names":[],"mappings":"AAiBA,MAAM,OAAO,2BAA4B,SAAQ,KAAK;IACpD,0DAA0D;IAC1D,SAAS,CAAY;IACrB,2EAA2E;IAC3E,KAAK,CAAmB;IACxB,kFAAkF;IAClF,kBAAkB,CAAU;IAE5B,YACE,OAAe,EACf,OAKC;QAED,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IACvD,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAkC;IACvE,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,kBAAkB,KAAK,cAAc,EAAE,CAAC;QAC5G,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { setTimeout } from 'timers/promises';
|
|
2
|
-
import { MissingReplicationSlotError } from './
|
|
2
|
+
import { MissingReplicationSlotError } from './MissingReplicationSlotError.js';
|
|
3
3
|
export class PostgresErrorRateLimiter {
|
|
4
4
|
nextAllowed = Date.now();
|
|
5
5
|
async waitUntilAllowed(options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/PostgresErrorRateLimiter.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"PostgresErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/PostgresErrorRateLimiter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAE/E,MAAM,OAAO,wBAAwB;IACnC,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;YAC7C,oDAAoD;YACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE,CAAC;YAC9D,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACzC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D,CAAC;YACD,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { bson } from '@powersync/service-core';
|
|
2
|
-
import { PgType, PgTypeOid } from '@powersync/service-jpgwire';
|
|
3
1
|
import { escapeIdentifier } from '@powersync/lib-service-postgres';
|
|
4
2
|
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { bson } from '@powersync/service-core';
|
|
4
|
+
import { PgType, PgTypeOid } from '@powersync/service-jpgwire';
|
|
5
5
|
/**
|
|
6
6
|
* Snapshot query using a plain SELECT * FROM table; chunked using
|
|
7
7
|
* DECLARE CURSOR / FETCH.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SnapshotQuery.js","sourceRoot":"","sources":["../../src/replication/SnapshotQuery.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"SnapshotQuery.js","sourceRoot":"","sources":["../../src/replication/SnapshotQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAiC,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAyB,MAAM,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAoBtF;;;;;;GAMG;AACH,MAAM,OAAO,mBAAmB;IAEX;IACA;IACA;IAHnB,YACmB,UAAwB,EACxB,KAAkB,EAClB,YAAoB,MAAM;QAF1B,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAiB;IAC1C,CAAC;IAEG,KAAK,CAAC,UAAU;QACrB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,oDAAoD,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;IAC9G,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,SAAS,uBAAuB,CAAC,CAAC;IAChF,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAoB;IA6BZ;IACA;IACA;IA9BnB;;;;;OAKG;IACH,MAAM,CAAC,eAAe,GAAG;QACvB,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,OAAO;QACjB,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;QACd,SAAS,CAAC,IAAI;KACf,CAAC;IAEF,MAAM,CAAC,QAAQ,CAAC,KAAkB;QAChC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE7C,OAAO,UAAU,CAAC,MAAM,IAAI,IAAI,IAAI,oBAAoB,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/G,CAAC;IAEgB,GAAG,CAAmB;IACvC,OAAO,GAA2B,IAAI,CAAC;IAEvC,YACmB,UAAwB,EACxB,KAAkB,EAClB,YAAoB,MAAM,EAC3C,iBAAoC;QAHnB,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAiB;QAG3C,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAErC,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,UAAU;QACrB,QAAQ;IACV,CAAC;IAEO,cAAc,CAAC,GAAe;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,qBAAqB,CAAC,wBAAwB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,oBAAoB;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;IAEM,KAAK,CAAC,CAAC,SAAS;QACrB,IAAI,MAAsC,CAAC;QAC3C,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAC7B,iBAAiB,IAAI,CAAC,KAAK,CAAC,aAAa,aAAa,cAAc,UAAU,IAAI,CAAC,SAAS,EAAE,CAC/F,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC9B,SAAS,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,aAAa,UAAU,cAAc,kBAAkB,cAAc,UAAU,IAAI,CAAC,SAAS,EAAE;gBACtI,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;aACxC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,eAAe,GAAW,CAAC,CAAC,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,EAAE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,GAAG,IAAI,gBAAgB,EAAE,CAAC;gBAClC,+DAA+D;gBAC/D,mBAAmB;gBACnB,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnE,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBACX,MAAM,IAAI,KAAK,CACb,kCAAkC,IAAI,CAAC,GAAG,sBAAsB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9G,CAAC;gBACJ,CAAC;gBACD,eAAe,GAAG,EAAE,CAAC;gBACrB,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC;YACtC,CAAC;YAED,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,OAAO,CAGpF,CAAC;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;;AAGH;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IAWP;IACA;IACA;IAZX,QAAQ,GAAG,KAAK,CAAC;IAEzB,MAAM,CAAC,QAAQ,CAAC,KAAkB;QAChC,yDAAyD;QACzD,sEAAsE;QACtE,qBAAqB;QACrB,OAAO,oBAAoB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,YACmB,UAAwB,EACxB,KAAkB,EAClB,IAAuB;QAFvB,eAAU,GAAV,UAAU,CAAc;QACxB,UAAK,GAAL,KAAK,CAAa;QAClB,SAAI,GAAJ,IAAI,CAAmB;IACvC,CAAC;IAEG,KAAK,CAAC,UAAU;QACrB,QAAQ;IACV,CAAC;IAEM,KAAK,CAAC,CAAC,SAAS;QACrB,yBAAyB;QACzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,MAAO,CAAC,CAAC;QACxD,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC5B,SAAS,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,aAAa,UAAU,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,YAAY;YAC9G,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,GAAG;iBACX;aACF;SACF,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -18,6 +18,18 @@ export interface WalStreamOptions {
|
|
|
18
18
|
* Note that queries are streamed, so we don't actually keep that much data in memory.
|
|
19
19
|
*/
|
|
20
20
|
snapshotChunkLength?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Called after each snapshot chunk is flushed, for testing.
|
|
23
|
+
* This allows tests to perform actions (like generating WAL) synchronously
|
|
24
|
+
* with the snapshot's chunk processing.
|
|
25
|
+
*/
|
|
26
|
+
onSnapshotChunkFlushed?: () => Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Interval for slot health checks during snapshot, in milliseconds.
|
|
29
|
+
* Set to 0 for every-chunk checking (useful for testing).
|
|
30
|
+
* Defaults to 120_000 (2 minutes).
|
|
31
|
+
*/
|
|
32
|
+
slotHealthCheckIntervalMs?: number;
|
|
21
33
|
}
|
|
22
34
|
interface InitResult {
|
|
23
35
|
/** True if initial snapshot is not yet done. */
|
|
@@ -33,9 +45,6 @@ export declare const KEEPALIVE_BUFFER: Buffer<ArrayBuffer>;
|
|
|
33
45
|
export declare const KEEPALIVE_STATEMENT: pgwire.Statement;
|
|
34
46
|
export declare const isKeepAliveMessage: (msg: pgwire.PgoutputMessage) => boolean;
|
|
35
47
|
export declare const sendKeepAlive: (db: pgwire.PgClient) => Promise<void>;
|
|
36
|
-
export declare class MissingReplicationSlotError extends Error {
|
|
37
|
-
constructor(message: string, cause?: any);
|
|
38
|
-
}
|
|
39
48
|
export declare class WalStream {
|
|
40
49
|
sync_rules: HydratedSyncRules;
|
|
41
50
|
group_id: number;
|
|
@@ -49,21 +58,35 @@ export declare class WalStream {
|
|
|
49
58
|
private relationCache;
|
|
50
59
|
private startedStreaming;
|
|
51
60
|
private snapshotChunkLength;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
* This is used to determine the replication lag.
|
|
55
|
-
*/
|
|
56
|
-
private oldestUncommittedChange;
|
|
57
|
-
/**
|
|
58
|
-
* Keep track of whether we have done a commit or keepalive yet.
|
|
59
|
-
* We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
|
|
60
|
-
*/
|
|
61
|
-
private isStartingReplication;
|
|
61
|
+
private onSnapshotChunkFlushed?;
|
|
62
|
+
private replicationLag;
|
|
62
63
|
private initialSnapshotPromise;
|
|
64
|
+
private slotHealthCheckIntervalMs;
|
|
65
|
+
/** Cached max_slot_wal_keep_size in bytes. null = not yet queried. */
|
|
66
|
+
private maxSlotWalKeepSize;
|
|
67
|
+
/** Whether max_slot_wal_keep_size has been queried (distinguishes "not
|
|
68
|
+
* queried" from "queried and found unlimited"). */
|
|
69
|
+
private maxSlotWalKeepSizeQueried;
|
|
70
|
+
/** Previous safe_wal_size sample for rate calculation. */
|
|
71
|
+
private prevWalBudgetSample;
|
|
72
|
+
/** Timestamp of last slot health check. */
|
|
73
|
+
private lastSlotHealthCheckTime;
|
|
63
74
|
constructor(options: WalStreamOptions);
|
|
64
75
|
get stopped(): boolean;
|
|
65
76
|
getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
|
|
66
77
|
initSlot(): Promise<InitResult>;
|
|
78
|
+
private queryMaxSlotWalKeepSize;
|
|
79
|
+
private logWalBudget;
|
|
80
|
+
/**
|
|
81
|
+
* Check if the replication slot is still valid. Called after each chunk
|
|
82
|
+
* flush during snapshot to detect slot invalidation early.
|
|
83
|
+
*
|
|
84
|
+
* The query hits pg_replication_slots (shared memory, not a table scan)
|
|
85
|
+
* and costs ~1-2ms per round-trip — negligible next to the per-chunk
|
|
86
|
+
* storage flush.
|
|
87
|
+
*/
|
|
88
|
+
private checkSlotHealth;
|
|
89
|
+
private formatWalBudgetContext;
|
|
67
90
|
estimatedCountNumber(db: pgwire.PgConnection, table: storage.SourceTable): Promise<number>;
|
|
68
91
|
/**
|
|
69
92
|
* Start initial replication.
|
|
@@ -116,7 +139,7 @@ export declare class WalStream {
|
|
|
116
139
|
* viewing the contents of logical replication messages.
|
|
117
140
|
*/
|
|
118
141
|
protected checkLogicalMessageSupport(): Promise<boolean>;
|
|
119
|
-
getReplicationLagMillis():
|
|
142
|
+
getReplicationLagMillis(): number | undefined;
|
|
120
143
|
private touch;
|
|
121
144
|
}
|
|
122
145
|
export {};
|