@powersync/service-module-postgres-storage 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +12 -12
- package/dist/storage/PostgresBucketStorageFactory.js +57 -52
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +1 -1
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +16 -8
- package/dist/storage/PostgresSyncRulesStorage.js +48 -196
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +13 -5
- package/dist/storage/batch/PostgresBucketBatch.js +280 -29
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +89 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
- package/dist/utils/test-utils.js +2 -2
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -22
- package/src/storage/PostgresBucketStorageFactory.ts +64 -54
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresStorageProvider.ts +1 -1
- package/src/storage/PostgresSyncRulesStorage.ts +51 -211
- package/src/storage/batch/PostgresBucketBatch.ts +325 -45
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +97 -0
- package/src/utils/test-utils.ts +2 -2
- package/test/src/migrations.test.ts +1 -1
- package/test/src/storage.test.ts +7 -2
- package/test/src/storage_compacting.test.ts +6 -4
- package/test/src/storage_sync.test.ts +2 -1
- package/test/tsconfig.json +0 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -0
- package/dist/.tsbuildinfo +0 -1
- package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
- /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
- /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +0 -0
- /package/dist/{@types/storage → storage}/current-data-store.d.ts +0 -0
- /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
- /package/dist/{@types/types → types}/codecs.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
- /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
- /package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/json.d.ts +0 -0
- /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
- /package/dist/{@types/types → types}/types.d.ts +0 -0
- /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
- /package/dist/{@types/utils → utils}/db.d.ts +0 -0
- /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
- /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
- /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import {
|
|
13
13
|
BucketStorageMarkRecordUnavailable,
|
|
14
14
|
CheckpointResult,
|
|
15
|
+
ColumnDescriptor,
|
|
15
16
|
deserializeReplicaId,
|
|
16
17
|
InternalOpId,
|
|
17
18
|
storage,
|
|
@@ -20,8 +21,10 @@ import {
|
|
|
20
21
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
21
22
|
import * as timers from 'timers/promises';
|
|
22
23
|
import * as t from 'ts-codec';
|
|
24
|
+
import * as uuid from 'uuid';
|
|
23
25
|
import { bigint } from '../../types/codecs.js';
|
|
24
26
|
import { CurrentBucket, V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
|
|
27
|
+
import { SourceTableDecoded, StoredRelationId } from '../../types/models/SourceTable.js';
|
|
25
28
|
import { models, RequiredOperationBatchLimits } from '../../types/types.js';
|
|
26
29
|
import { NOTIFICATION_CHANNEL } from '../../utils/db.js';
|
|
27
30
|
import { pick } from '../../utils/ts-codec.js';
|
|
@@ -34,9 +37,9 @@ import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
|
|
|
34
37
|
export interface PostgresBucketBatchOptions {
|
|
35
38
|
logger: Logger;
|
|
36
39
|
db: lib_postgres.DatabaseClient;
|
|
37
|
-
sync_rules: sync_rules.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
sync_rules: sync_rules.HydratedSyncConfig;
|
|
41
|
+
replicationStreamId: number;
|
|
42
|
+
replicationStreamName: string;
|
|
40
43
|
last_checkpoint_lsn: string | null;
|
|
41
44
|
store_current_data: boolean;
|
|
42
45
|
keep_alive_op?: InternalOpId | null;
|
|
@@ -48,6 +51,7 @@ export interface PostgresBucketBatchOptions {
|
|
|
48
51
|
batch_limits: RequiredOperationBatchLimits;
|
|
49
52
|
|
|
50
53
|
markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
54
|
+
hooks: storage.StorageHooks | undefined;
|
|
51
55
|
storageConfig: storage.StorageVersionConfig;
|
|
52
56
|
}
|
|
53
57
|
|
|
@@ -85,6 +89,7 @@ export class PostgresBucketBatch
|
|
|
85
89
|
public last_flushed_op: InternalOpId | null = null;
|
|
86
90
|
|
|
87
91
|
public resumeFromLsn: string | null;
|
|
92
|
+
public readonly skipExistingRows: boolean;
|
|
88
93
|
|
|
89
94
|
protected db: lib_postgres.DatabaseClient;
|
|
90
95
|
protected group_id: number;
|
|
@@ -93,10 +98,11 @@ export class PostgresBucketBatch
|
|
|
93
98
|
protected persisted_op: InternalOpId | null;
|
|
94
99
|
|
|
95
100
|
protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
|
|
96
|
-
protected readonly sync_rules: sync_rules.
|
|
101
|
+
protected readonly sync_rules: sync_rules.HydratedSyncConfig;
|
|
97
102
|
protected batch: OperationBatch | null;
|
|
98
103
|
private lastWaitingLogThrottled = 0;
|
|
99
104
|
private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
105
|
+
private hooks: storage.StorageHooks | undefined;
|
|
100
106
|
private needsActivation = true;
|
|
101
107
|
private clearedError = false;
|
|
102
108
|
private readonly storageConfig: storage.StorageVersionConfig;
|
|
@@ -106,12 +112,14 @@ export class PostgresBucketBatch
|
|
|
106
112
|
super();
|
|
107
113
|
this.logger = options.logger;
|
|
108
114
|
this.db = options.db;
|
|
109
|
-
this.group_id = options.
|
|
115
|
+
this.group_id = options.replicationStreamId;
|
|
110
116
|
this.last_checkpoint_lsn = options.last_checkpoint_lsn;
|
|
111
117
|
this.resumeFromLsn = options.resumeFromLsn;
|
|
118
|
+
this.skipExistingRows = options.skip_existing_rows;
|
|
112
119
|
this.write_checkpoint_batch = [];
|
|
113
120
|
this.sync_rules = options.sync_rules;
|
|
114
121
|
this.markRecordUnavailable = options.markRecordUnavailable;
|
|
122
|
+
this.hooks = options.hooks;
|
|
115
123
|
this.batch = null;
|
|
116
124
|
this.persisted_op = null;
|
|
117
125
|
this.storageConfig = options.storageConfig;
|
|
@@ -121,10 +129,6 @@ export class PostgresBucketBatch
|
|
|
121
129
|
}
|
|
122
130
|
}
|
|
123
131
|
|
|
124
|
-
get lastCheckpointLsn() {
|
|
125
|
-
return this.last_checkpoint_lsn;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
132
|
async [Symbol.asyncDispose]() {
|
|
129
133
|
if (this.batch != null || this.write_checkpoint_batch.length > 0) {
|
|
130
134
|
// We don't error here, since:
|
|
@@ -139,22 +143,237 @@ export class PostgresBucketBatch
|
|
|
139
143
|
await this[Symbol.asyncDispose]();
|
|
140
144
|
}
|
|
141
145
|
|
|
146
|
+
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
147
|
+
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
148
|
+
const { connection_id, source } = options;
|
|
149
|
+
const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
150
|
+
|
|
151
|
+
const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
|
|
152
|
+
name: column.name,
|
|
153
|
+
type: column.type,
|
|
154
|
+
type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
|
|
155
|
+
}));
|
|
156
|
+
|
|
157
|
+
return this.db.transaction(async (db) => {
|
|
158
|
+
let sourceTableRow: SourceTableDecoded | null;
|
|
159
|
+
if (objectId != null) {
|
|
160
|
+
sourceTableRow = await db.sql`
|
|
161
|
+
SELECT
|
|
162
|
+
*
|
|
163
|
+
FROM
|
|
164
|
+
source_tables
|
|
165
|
+
WHERE
|
|
166
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
167
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
168
|
+
AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
169
|
+
AND schema_name = ${{ type: 'varchar', value: schema }}
|
|
170
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
171
|
+
AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
172
|
+
`
|
|
173
|
+
.decoded(models.SourceTable)
|
|
174
|
+
.first();
|
|
175
|
+
} else {
|
|
176
|
+
sourceTableRow = await db.sql`
|
|
177
|
+
SELECT
|
|
178
|
+
*
|
|
179
|
+
FROM
|
|
180
|
+
source_tables
|
|
181
|
+
WHERE
|
|
182
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
183
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
184
|
+
AND schema_name = ${{ type: 'varchar', value: schema }}
|
|
185
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
186
|
+
AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
187
|
+
`
|
|
188
|
+
.decoded(models.SourceTable)
|
|
189
|
+
.first();
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (sourceTableRow == null) {
|
|
193
|
+
const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
|
|
194
|
+
sourceTableRow = await db.sql`
|
|
195
|
+
INSERT INTO
|
|
196
|
+
source_tables (
|
|
197
|
+
id,
|
|
198
|
+
group_id,
|
|
199
|
+
connection_id,
|
|
200
|
+
relation_id,
|
|
201
|
+
schema_name,
|
|
202
|
+
table_name,
|
|
203
|
+
replica_id_columns
|
|
204
|
+
)
|
|
205
|
+
VALUES
|
|
206
|
+
(
|
|
207
|
+
${{ type: 'varchar', value: id }},
|
|
208
|
+
${{ type: 'int4', value: this.group_id }},
|
|
209
|
+
${{ type: 'int4', value: connection_id }},
|
|
210
|
+
${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
|
|
211
|
+
${{ type: 'varchar', value: schema }},
|
|
212
|
+
${{ type: 'varchar', value: table }},
|
|
213
|
+
${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
214
|
+
)
|
|
215
|
+
RETURNING
|
|
216
|
+
*
|
|
217
|
+
`
|
|
218
|
+
.decoded(models.SourceTable)
|
|
219
|
+
.first();
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const sourceTable = new storage.SourceTable({
|
|
223
|
+
id: sourceTableRow!.id,
|
|
224
|
+
ref: source,
|
|
225
|
+
objectId,
|
|
226
|
+
replicaIdColumns,
|
|
227
|
+
snapshotComplete: sourceTableRow!.snapshot_done ?? true,
|
|
228
|
+
...syncRules.getMatchingSources(source)
|
|
229
|
+
});
|
|
230
|
+
if (!sourceTable.snapshotComplete) {
|
|
231
|
+
sourceTable.snapshotStatus = {
|
|
232
|
+
totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
|
|
233
|
+
replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
|
|
234
|
+
lastKey: sourceTableRow!.snapshot_last_key
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
238
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
239
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
240
|
+
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
241
|
+
|
|
242
|
+
let truncatedTables: SourceTableDecoded[] = [];
|
|
243
|
+
if (objectId != null) {
|
|
244
|
+
truncatedTables = await db.sql`
|
|
245
|
+
SELECT
|
|
246
|
+
*
|
|
247
|
+
FROM
|
|
248
|
+
source_tables
|
|
249
|
+
WHERE
|
|
250
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
251
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
252
|
+
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
253
|
+
AND (
|
|
254
|
+
relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
255
|
+
OR (
|
|
256
|
+
schema_name = ${{ type: 'varchar', value: schema }}
|
|
257
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
258
|
+
)
|
|
259
|
+
)
|
|
260
|
+
`
|
|
261
|
+
.decoded(models.SourceTable)
|
|
262
|
+
.rows();
|
|
263
|
+
} else {
|
|
264
|
+
truncatedTables = await db.sql`
|
|
265
|
+
SELECT
|
|
266
|
+
*
|
|
267
|
+
FROM
|
|
268
|
+
source_tables
|
|
269
|
+
WHERE
|
|
270
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
271
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
272
|
+
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
273
|
+
AND (
|
|
274
|
+
schema_name = ${{ type: 'varchar', value: schema }}
|
|
275
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
276
|
+
)
|
|
277
|
+
`
|
|
278
|
+
.decoded(models.SourceTable)
|
|
279
|
+
.rows();
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
return {
|
|
283
|
+
tables: [sourceTable],
|
|
284
|
+
dropTables: truncatedTables.map((doc) => {
|
|
285
|
+
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
286
|
+
const dropTable = new storage.SourceTable({
|
|
287
|
+
id: doc.id,
|
|
288
|
+
ref,
|
|
289
|
+
objectId: doc.relation_id?.object_id ?? 0,
|
|
290
|
+
replicaIdColumns:
|
|
291
|
+
doc.replica_id_columns?.map(
|
|
292
|
+
(c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
|
|
293
|
+
) ?? [],
|
|
294
|
+
snapshotComplete: doc.snapshot_done ?? true,
|
|
295
|
+
...syncRules.getMatchingSources(ref)
|
|
296
|
+
});
|
|
297
|
+
dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
298
|
+
dropTable.syncData = dropTable.bucketDataSources.length > 0;
|
|
299
|
+
dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
|
|
300
|
+
return dropTable;
|
|
301
|
+
})
|
|
302
|
+
};
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
|
|
307
|
+
const row = await this.db.sql`
|
|
308
|
+
SELECT
|
|
309
|
+
*
|
|
310
|
+
FROM
|
|
311
|
+
source_tables
|
|
312
|
+
WHERE
|
|
313
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
314
|
+
AND id = ${{ type: 'varchar', value: table.id.toString() }}
|
|
315
|
+
`
|
|
316
|
+
.decoded(models.SourceTable)
|
|
317
|
+
.first();
|
|
318
|
+
|
|
319
|
+
if (row == null) {
|
|
320
|
+
return null;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
return this.sourceTableFromRow(row, table.ref.connectionTag, this.sync_rules);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
private sourceTableFromRow(
|
|
327
|
+
row: SourceTableDecoded,
|
|
328
|
+
connectionTag: string,
|
|
329
|
+
syncRules: sync_rules.HydratedSyncConfig
|
|
330
|
+
): storage.SourceTable {
|
|
331
|
+
const ref = { connectionTag, schema: row.schema_name, name: row.table_name };
|
|
332
|
+
const sourceTable = new storage.SourceTable({
|
|
333
|
+
id: row.id,
|
|
334
|
+
ref,
|
|
335
|
+
objectId: row.relation_id?.object_id,
|
|
336
|
+
replicaIdColumns:
|
|
337
|
+
row.replica_id_columns?.map(
|
|
338
|
+
(c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
|
|
339
|
+
) ?? [],
|
|
340
|
+
snapshotComplete: row.snapshot_done ?? true,
|
|
341
|
+
...syncRules.getMatchingSources(ref)
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
if (!sourceTable.snapshotComplete) {
|
|
345
|
+
sourceTable.snapshotStatus = {
|
|
346
|
+
totalEstimatedCount: Number(row.snapshot_total_estimated_count ?? -1n),
|
|
347
|
+
replicatedCount: Number(row.snapshot_replicated_count ?? 0n),
|
|
348
|
+
lastKey: row.snapshot_last_key
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
sourceTable.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
352
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
353
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
354
|
+
return sourceTable;
|
|
355
|
+
}
|
|
356
|
+
|
|
142
357
|
async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
|
|
143
358
|
// TODO maybe share with abstract class
|
|
144
359
|
const { after, before, sourceTable, tag } = record;
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
360
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
361
|
+
// Only the table designated as event carrier by resolveTables may fire events.
|
|
362
|
+
if (sourceTable.syncEvent) {
|
|
363
|
+
for (const event of this.getTableEvents(sourceTable)) {
|
|
364
|
+
this.iterateListeners((cb) =>
|
|
365
|
+
cb.replicationEvent?.({
|
|
366
|
+
batch: this,
|
|
367
|
+
table: sourceTable,
|
|
368
|
+
data: {
|
|
369
|
+
op: tag,
|
|
370
|
+
after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
|
|
371
|
+
before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
|
|
372
|
+
},
|
|
373
|
+
event
|
|
374
|
+
})
|
|
375
|
+
);
|
|
376
|
+
}
|
|
158
377
|
}
|
|
159
378
|
/**
|
|
160
379
|
* Return if the table is just an event table
|
|
@@ -296,6 +515,8 @@ export class PostgresBucketBatch
|
|
|
296
515
|
return null;
|
|
297
516
|
}
|
|
298
517
|
|
|
518
|
+
await this.hooks?.beforeBatchFlush?.(this);
|
|
519
|
+
|
|
299
520
|
let resumeBatch: OperationBatch | null = null;
|
|
300
521
|
|
|
301
522
|
const lastOp = await this.withReplicationTransaction(async (db) => {
|
|
@@ -313,6 +534,7 @@ export class PostgresBucketBatch
|
|
|
313
534
|
|
|
314
535
|
this.persisted_op = lastOp;
|
|
315
536
|
this.last_flushed_op = lastOp;
|
|
537
|
+
await this.hooks?.afterBatchFlush?.(this);
|
|
316
538
|
return { flushed_op: lastOp };
|
|
317
539
|
}
|
|
318
540
|
|
|
@@ -536,6 +758,49 @@ export class PostgresBucketBatch
|
|
|
536
758
|
});
|
|
537
759
|
}
|
|
538
760
|
|
|
761
|
+
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
762
|
+
await this.db.transaction(async (db) => {
|
|
763
|
+
const blockingTables = await db.sql`
|
|
764
|
+
SELECT
|
|
765
|
+
schema_name,
|
|
766
|
+
table_name
|
|
767
|
+
FROM
|
|
768
|
+
source_tables
|
|
769
|
+
WHERE
|
|
770
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
771
|
+
AND snapshot_done = FALSE
|
|
772
|
+
`
|
|
773
|
+
.decoded(t.object({ schema_name: t.string, table_name: t.string }))
|
|
774
|
+
.rows();
|
|
775
|
+
if (blockingTables.length > 0) {
|
|
776
|
+
if (options?.throwOnConflict ?? true) {
|
|
777
|
+
throw new ReplicationAssertionError(
|
|
778
|
+
`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
|
|
779
|
+
);
|
|
780
|
+
} else {
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
await db.sql`
|
|
786
|
+
UPDATE sync_rules
|
|
787
|
+
SET
|
|
788
|
+
snapshot_done = TRUE,
|
|
789
|
+
last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }},
|
|
790
|
+
no_checkpoint_before = CASE
|
|
791
|
+
WHEN no_checkpoint_before IS NULL
|
|
792
|
+
OR no_checkpoint_before < ${{ type: 'varchar', value: no_checkpoint_before_lsn }} THEN ${{
|
|
793
|
+
type: 'varchar',
|
|
794
|
+
value: no_checkpoint_before_lsn
|
|
795
|
+
}}
|
|
796
|
+
ELSE no_checkpoint_before
|
|
797
|
+
END
|
|
798
|
+
WHERE
|
|
799
|
+
id = ${{ type: 'int4', value: this.group_id }}
|
|
800
|
+
`.execute();
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
|
|
539
804
|
async markTableSnapshotRequired(table: storage.SourceTable): Promise<void> {
|
|
540
805
|
await this.db.sql`
|
|
541
806
|
UPDATE sync_rules
|
|
@@ -628,8 +893,12 @@ export class PostgresBucketBatch
|
|
|
628
893
|
|
|
629
894
|
protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) {
|
|
630
895
|
let sizes: Map<string, number> | undefined = undefined;
|
|
631
|
-
if
|
|
632
|
-
|
|
896
|
+
// Check if any table in this batch needs to store current_data
|
|
897
|
+
const anyTableStoresCurrentData =
|
|
898
|
+
this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
|
|
899
|
+
|
|
900
|
+
if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
|
|
901
|
+
// We skip this step if no tables store current_data, since the sizes will
|
|
633
902
|
// always be small in that case.
|
|
634
903
|
|
|
635
904
|
// With skipExistingRows, we don't load the full documents into memory,
|
|
@@ -637,15 +906,19 @@ export class PostgresBucketBatch
|
|
|
637
906
|
|
|
638
907
|
// Find sizes of current_data documents, to assist in intelligent batching without
|
|
639
908
|
// exceeding memory limits.
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
909
|
+
// Within this branch the batch stores current_data, so the per-table flag is the
|
|
910
|
+
// effective value - only look up sizes for tables that actually store current_data.
|
|
911
|
+
const sizeLookups = batch.batch
|
|
912
|
+
.filter((r) => r.record.sourceTable.storeCurrentData)
|
|
913
|
+
.map((r) => {
|
|
914
|
+
return {
|
|
915
|
+
source_table: postgresTableId(r.record.sourceTable.id),
|
|
916
|
+
/**
|
|
917
|
+
* Encode to hex in order to pass a jsonb
|
|
918
|
+
*/
|
|
919
|
+
source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
|
|
920
|
+
};
|
|
921
|
+
});
|
|
649
922
|
|
|
650
923
|
sizes = new Map<string, number>();
|
|
651
924
|
|
|
@@ -764,6 +1037,9 @@ export class PostgresBucketBatch
|
|
|
764
1037
|
const afterId = operation.afterId;
|
|
765
1038
|
let after = record.after;
|
|
766
1039
|
const sourceTable = record.sourceTable;
|
|
1040
|
+
// Effective per-record flag: store current_data only if both the batch (source-level)
|
|
1041
|
+
// and the table (e.g. non-FULL replica identity) require it.
|
|
1042
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
767
1043
|
|
|
768
1044
|
let existingBuckets: CurrentBucket[] = [];
|
|
769
1045
|
let newBuckets: CurrentBucket[] = [];
|
|
@@ -792,7 +1068,7 @@ export class PostgresBucketBatch
|
|
|
792
1068
|
existingLookups = [];
|
|
793
1069
|
// Log to help with debugging if there was a consistency issue
|
|
794
1070
|
|
|
795
|
-
if (
|
|
1071
|
+
if (storeCurrentData) {
|
|
796
1072
|
if (this.markRecordUnavailable != null) {
|
|
797
1073
|
// This will trigger a "resnapshot" of the record.
|
|
798
1074
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -808,7 +1084,7 @@ export class PostgresBucketBatch
|
|
|
808
1084
|
} else {
|
|
809
1085
|
existingBuckets = result.buckets;
|
|
810
1086
|
existingLookups = result.lookups;
|
|
811
|
-
if (
|
|
1087
|
+
if (storeCurrentData) {
|
|
812
1088
|
const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
|
|
813
1089
|
after = storage.mergeToast(after!, data);
|
|
814
1090
|
}
|
|
@@ -819,7 +1095,9 @@ export class PostgresBucketBatch
|
|
|
819
1095
|
// Not an error if we re-apply a transaction
|
|
820
1096
|
existingBuckets = [];
|
|
821
1097
|
existingLookups = [];
|
|
822
|
-
// Log to help with debugging if there was a consistency issue
|
|
1098
|
+
// Log to help with debugging if there was a consistency issue.
|
|
1099
|
+
// Gate on the batch-level flag: FULL tables (per-record flag false) still get a
|
|
1100
|
+
// current_data entry, so a missing record on DELETE is meaningful for them too.
|
|
823
1101
|
if (this.options.store_current_data && this.markRecordUnavailable == null) {
|
|
824
1102
|
this.logger.warn(
|
|
825
1103
|
`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
@@ -832,7 +1110,7 @@ export class PostgresBucketBatch
|
|
|
832
1110
|
}
|
|
833
1111
|
|
|
834
1112
|
let afterData: Buffer<ArrayBuffer> | undefined;
|
|
835
|
-
if (afterId != null && !
|
|
1113
|
+
if (afterId != null && !storeCurrentData) {
|
|
836
1114
|
afterData = storage.serializeBson({});
|
|
837
1115
|
} else if (afterId != null) {
|
|
838
1116
|
try {
|
|
@@ -854,7 +1132,7 @@ export class PostgresBucketBatch
|
|
|
854
1132
|
{
|
|
855
1133
|
level: errors.ErrorSeverity.WARNING,
|
|
856
1134
|
metadata: {
|
|
857
|
-
replication_slot: this.options.
|
|
1135
|
+
replication_slot: this.options.replicationStreamName,
|
|
858
1136
|
table: record.sourceTable.qualifiedName
|
|
859
1137
|
}
|
|
860
1138
|
}
|
|
@@ -895,12 +1173,13 @@ export class PostgresBucketBatch
|
|
|
895
1173
|
// However, it will be valid by the end of the transaction.
|
|
896
1174
|
//
|
|
897
1175
|
// In this case, we don't save the op, but we do save the current data.
|
|
898
|
-
if (afterId && after && utils.isCompleteRow(
|
|
1176
|
+
if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
|
|
899
1177
|
// Insert or update
|
|
900
1178
|
if (sourceTable.syncData) {
|
|
901
1179
|
const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
902
1180
|
record: after,
|
|
903
|
-
sourceTable
|
|
1181
|
+
sourceTable: sourceTable.ref,
|
|
1182
|
+
bucketDataSources: sourceTable.bucketDataSources
|
|
904
1183
|
});
|
|
905
1184
|
|
|
906
1185
|
for (const error of syncErrors) {
|
|
@@ -909,7 +1188,7 @@ export class PostgresBucketBatch
|
|
|
909
1188
|
{
|
|
910
1189
|
level: errors.ErrorSeverity.WARNING,
|
|
911
1190
|
metadata: {
|
|
912
|
-
replication_slot: this.options.
|
|
1191
|
+
replication_slot: this.options.replicationStreamName,
|
|
913
1192
|
table: record.sourceTable.qualifiedName
|
|
914
1193
|
}
|
|
915
1194
|
}
|
|
@@ -939,8 +1218,9 @@ export class PostgresBucketBatch
|
|
|
939
1218
|
if (sourceTable.syncParameters) {
|
|
940
1219
|
// Parameters
|
|
941
1220
|
const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors(
|
|
942
|
-
sourceTable,
|
|
943
|
-
after
|
|
1221
|
+
sourceTable.ref,
|
|
1222
|
+
after,
|
|
1223
|
+
{ parameterLookupSources: sourceTable.parameterLookupSources }
|
|
944
1224
|
);
|
|
945
1225
|
|
|
946
1226
|
for (let error of paramErrors) {
|
|
@@ -949,7 +1229,7 @@ export class PostgresBucketBatch
|
|
|
949
1229
|
{
|
|
950
1230
|
level: errors.ErrorSeverity.WARNING,
|
|
951
1231
|
metadata: {
|
|
952
|
-
replication_slot: this.options.
|
|
1232
|
+
replication_slot: this.options.replicationStreamName,
|
|
953
1233
|
table: record.sourceTable.qualifiedName
|
|
954
1234
|
}
|
|
955
1235
|
}
|
|
@@ -1064,7 +1344,7 @@ export class PostgresBucketBatch
|
|
|
1064
1344
|
*/
|
|
1065
1345
|
protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
|
|
1066
1346
|
return this.sync_rules.eventDescriptors.filter((evt) =>
|
|
1067
|
-
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
|
|
1347
|
+
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
|
|
1068
1348
|
);
|
|
1069
1349
|
}
|
|
1070
1350
|
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { models } from '../../types/types.js';
|
|
5
|
+
|
|
6
|
+
export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
|
|
7
|
+
constructor(
|
|
8
|
+
private db: lib_postgres.DatabaseClient,
|
|
9
|
+
row: models.SyncRulesDecoded
|
|
10
|
+
) {
|
|
11
|
+
super({
|
|
12
|
+
replicationStreamId: Number(row.id),
|
|
13
|
+
sync_rules_content: row.content,
|
|
14
|
+
compiled_plan: row.sync_plan,
|
|
15
|
+
replicationStreamName: row.slot_name,
|
|
16
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
21
|
+
const row = await this.db.sql`
|
|
22
|
+
SELECT
|
|
23
|
+
*
|
|
24
|
+
FROM
|
|
25
|
+
sync_rules
|
|
26
|
+
WHERE
|
|
27
|
+
id = ${{ value: this.replicationStreamId, type: 'int4' }}
|
|
28
|
+
`
|
|
29
|
+
.decoded(models.SyncRules)
|
|
30
|
+
.first();
|
|
31
|
+
|
|
32
|
+
return row == null ? null : syncConfigStatusFromRow(row);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
|
|
37
|
+
return {
|
|
38
|
+
id: String(row.id),
|
|
39
|
+
replicationStreamId: Number(row.id),
|
|
40
|
+
state: row.state as storage.SyncRuleState,
|
|
41
|
+
snapshot_done: row.snapshot_done,
|
|
42
|
+
last_checkpoint_lsn: row.last_checkpoint_lsn,
|
|
43
|
+
last_fatal_error: row.last_fatal_error,
|
|
44
|
+
last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
|
|
45
|
+
last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
50
|
+
current_lock: storage.ReplicationLock | null = null;
|
|
51
|
+
readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
private db: lib_postgres.DatabaseClient,
|
|
55
|
+
private readonly row: models.SyncRulesDecoded
|
|
56
|
+
) {
|
|
57
|
+
super({
|
|
58
|
+
replicationStreamId: Number(row.id),
|
|
59
|
+
replicationStreamName: row.slot_name,
|
|
60
|
+
state: row.state as storage.SyncRuleState,
|
|
61
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
62
|
+
});
|
|
63
|
+
this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
67
|
+
return this.syncConfigContent[0].parsed(options);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async lock(): Promise<storage.ReplicationLock> {
|
|
71
|
+
const manager = new lib_postgres.PostgresLockManager({
|
|
72
|
+
db: this.db,
|
|
73
|
+
name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
|
|
74
|
+
});
|
|
75
|
+
const lockHandle = await manager.acquire();
|
|
76
|
+
if (!lockHandle) {
|
|
77
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const interval = setInterval(async () => {
|
|
81
|
+
try {
|
|
82
|
+
await lockHandle.refresh();
|
|
83
|
+
} catch (e) {
|
|
84
|
+
this.logger.error('Failed to refresh lock', e);
|
|
85
|
+
clearInterval(interval);
|
|
86
|
+
}
|
|
87
|
+
}, 30_130);
|
|
88
|
+
|
|
89
|
+
return (this.current_lock = {
|
|
90
|
+
sync_rules_id: this.replicationStreamId,
|
|
91
|
+
release: async () => {
|
|
92
|
+
clearInterval(interval);
|
|
93
|
+
return lockHandle.release();
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/utils/test-utils.ts
CHANGED
|
@@ -73,7 +73,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
|
73
73
|
|
|
74
74
|
await using storageFactory = new PostgresBucketStorageFactory({
|
|
75
75
|
config: TEST_CONNECTION_OPTIONS,
|
|
76
|
-
|
|
76
|
+
replicationStreamNamePrefix: 'test_'
|
|
77
77
|
});
|
|
78
78
|
await truncateTables(storageFactory.db);
|
|
79
79
|
};
|
|
@@ -102,7 +102,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
|
102
102
|
|
|
103
103
|
return new PostgresBucketStorageFactory({
|
|
104
104
|
config: TEST_CONNECTION_OPTIONS,
|
|
105
|
-
|
|
105
|
+
replicationStreamNamePrefix: 'test_'
|
|
106
106
|
});
|
|
107
107
|
} catch (ex) {
|
|
108
108
|
// Vitest does not display these errors nicely when using the `await using` syntx
|
|
@@ -18,7 +18,7 @@ describe('Migrations', () => {
|
|
|
18
18
|
const setup = POSTGRES_STORAGE_SETUP;
|
|
19
19
|
await using factory = new PostgresBucketStorageFactory({
|
|
20
20
|
config: TEST_CONNECTION_OPTIONS,
|
|
21
|
-
|
|
21
|
+
replicationStreamNamePrefix: 'test_'
|
|
22
22
|
});
|
|
23
23
|
|
|
24
24
|
await dropTables(factory.db);
|