@powersync/service-module-postgres-storage 0.13.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/@types/storage/PostgresSyncRulesStorage.d.ts +4 -4
- package/dist/@types/storage/batch/PostgresBucketBatch.d.ts +12 -3
- package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js +1 -1
- package/dist/storage/PostgresBucketStorageFactory.js +5 -5
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.js +78 -197
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.js +265 -15
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +3 -3
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +1 -1
- package/package.json +11 -11
- package/src/migrations/scripts/1771424826685-current-data-pending-deletes.ts +1 -1
- package/src/storage/PostgresBucketStorageFactory.ts +6 -5
- package/src/storage/PostgresSyncRulesStorage.ts +90 -209
- package/src/storage/batch/PostgresBucketBatch.ts +308 -26
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +3 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +3 -6
- package/test/tsconfig.json +0 -1
- package/dist/@types/storage/current-data-table.d.ts +0 -9
- package/dist/storage/current-data-table.js +0 -22
- package/dist/storage/current-data-table.js.map +0 -1
- package/src/storage/current-data-table.ts +0 -26
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import {
|
|
13
13
|
BucketStorageMarkRecordUnavailable,
|
|
14
14
|
CheckpointResult,
|
|
15
|
+
ColumnDescriptor,
|
|
15
16
|
deserializeReplicaId,
|
|
16
17
|
InternalOpId,
|
|
17
18
|
storage,
|
|
@@ -20,8 +21,10 @@ import {
|
|
|
20
21
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
21
22
|
import * as timers from 'timers/promises';
|
|
22
23
|
import * as t from 'ts-codec';
|
|
24
|
+
import * as uuid from 'uuid';
|
|
23
25
|
import { bigint } from '../../types/codecs.js';
|
|
24
26
|
import { CurrentBucket, V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
|
|
27
|
+
import { SourceTableDecoded, StoredRelationId } from '../../types/models/SourceTable.js';
|
|
25
28
|
import { models, RequiredOperationBatchLimits } from '../../types/types.js';
|
|
26
29
|
import { NOTIFICATION_CHANNEL } from '../../utils/db.js';
|
|
27
30
|
import { pick } from '../../utils/ts-codec.js';
|
|
@@ -34,7 +37,7 @@ import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
|
|
|
34
37
|
export interface PostgresBucketBatchOptions {
|
|
35
38
|
logger: Logger;
|
|
36
39
|
db: lib_postgres.DatabaseClient;
|
|
37
|
-
sync_rules: sync_rules.
|
|
40
|
+
sync_rules: sync_rules.HydratedSyncConfig;
|
|
38
41
|
group_id: number;
|
|
39
42
|
slot_name: string;
|
|
40
43
|
last_checkpoint_lsn: string | null;
|
|
@@ -48,11 +51,12 @@ export interface PostgresBucketBatchOptions {
|
|
|
48
51
|
batch_limits: RequiredOperationBatchLimits;
|
|
49
52
|
|
|
50
53
|
markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
54
|
+
hooks: storage.StorageHooks | undefined;
|
|
51
55
|
storageConfig: storage.StorageVersionConfig;
|
|
52
56
|
}
|
|
53
57
|
|
|
54
58
|
/**
|
|
55
|
-
* Intermediate type which helps for only watching the active
|
|
59
|
+
* Intermediate type which helps for only watching the active replication stream
|
|
56
60
|
* via the Postgres NOTIFY protocol.
|
|
57
61
|
*/
|
|
58
62
|
const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
|
|
@@ -85,6 +89,7 @@ export class PostgresBucketBatch
|
|
|
85
89
|
public last_flushed_op: InternalOpId | null = null;
|
|
86
90
|
|
|
87
91
|
public resumeFromLsn: string | null;
|
|
92
|
+
public readonly skipExistingRows: boolean;
|
|
88
93
|
|
|
89
94
|
protected db: lib_postgres.DatabaseClient;
|
|
90
95
|
protected group_id: number;
|
|
@@ -93,10 +98,11 @@ export class PostgresBucketBatch
|
|
|
93
98
|
protected persisted_op: InternalOpId | null;
|
|
94
99
|
|
|
95
100
|
protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[];
|
|
96
|
-
protected readonly sync_rules: sync_rules.
|
|
101
|
+
protected readonly sync_rules: sync_rules.HydratedSyncConfig;
|
|
97
102
|
protected batch: OperationBatch | null;
|
|
98
103
|
private lastWaitingLogThrottled = 0;
|
|
99
104
|
private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
|
|
105
|
+
private hooks: storage.StorageHooks | undefined;
|
|
100
106
|
private needsActivation = true;
|
|
101
107
|
private clearedError = false;
|
|
102
108
|
private readonly storageConfig: storage.StorageVersionConfig;
|
|
@@ -109,9 +115,11 @@ export class PostgresBucketBatch
|
|
|
109
115
|
this.group_id = options.group_id;
|
|
110
116
|
this.last_checkpoint_lsn = options.last_checkpoint_lsn;
|
|
111
117
|
this.resumeFromLsn = options.resumeFromLsn;
|
|
118
|
+
this.skipExistingRows = options.skip_existing_rows;
|
|
112
119
|
this.write_checkpoint_batch = [];
|
|
113
120
|
this.sync_rules = options.sync_rules;
|
|
114
121
|
this.markRecordUnavailable = options.markRecordUnavailable;
|
|
122
|
+
this.hooks = options.hooks;
|
|
115
123
|
this.batch = null;
|
|
116
124
|
this.persisted_op = null;
|
|
117
125
|
this.storageConfig = options.storageConfig;
|
|
@@ -139,9 +147,221 @@ export class PostgresBucketBatch
|
|
|
139
147
|
await this[Symbol.asyncDispose]();
|
|
140
148
|
}
|
|
141
149
|
|
|
150
|
+
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
151
|
+
const syncRules = options.syncRules ?? this.sync_rules;
|
|
152
|
+
const { connection_id, source } = options;
|
|
153
|
+
const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
154
|
+
|
|
155
|
+
const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
|
|
156
|
+
name: column.name,
|
|
157
|
+
type: column.type,
|
|
158
|
+
type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
|
|
159
|
+
}));
|
|
160
|
+
|
|
161
|
+
return this.db.transaction(async (db) => {
|
|
162
|
+
let sourceTableRow: SourceTableDecoded | null;
|
|
163
|
+
if (objectId != null) {
|
|
164
|
+
sourceTableRow = await db.sql`
|
|
165
|
+
SELECT
|
|
166
|
+
*
|
|
167
|
+
FROM
|
|
168
|
+
source_tables
|
|
169
|
+
WHERE
|
|
170
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
171
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
172
|
+
AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
173
|
+
AND schema_name = ${{ type: 'varchar', value: schema }}
|
|
174
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
175
|
+
AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
176
|
+
`
|
|
177
|
+
.decoded(models.SourceTable)
|
|
178
|
+
.first();
|
|
179
|
+
} else {
|
|
180
|
+
sourceTableRow = await db.sql`
|
|
181
|
+
SELECT
|
|
182
|
+
*
|
|
183
|
+
FROM
|
|
184
|
+
source_tables
|
|
185
|
+
WHERE
|
|
186
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
187
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
188
|
+
AND schema_name = ${{ type: 'varchar', value: schema }}
|
|
189
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
190
|
+
AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
191
|
+
`
|
|
192
|
+
.decoded(models.SourceTable)
|
|
193
|
+
.first();
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (sourceTableRow == null) {
|
|
197
|
+
const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
|
|
198
|
+
sourceTableRow = await db.sql`
|
|
199
|
+
INSERT INTO
|
|
200
|
+
source_tables (
|
|
201
|
+
id,
|
|
202
|
+
group_id,
|
|
203
|
+
connection_id,
|
|
204
|
+
relation_id,
|
|
205
|
+
schema_name,
|
|
206
|
+
table_name,
|
|
207
|
+
replica_id_columns
|
|
208
|
+
)
|
|
209
|
+
VALUES
|
|
210
|
+
(
|
|
211
|
+
${{ type: 'varchar', value: id }},
|
|
212
|
+
${{ type: 'int4', value: this.group_id }},
|
|
213
|
+
${{ type: 'int4', value: connection_id }},
|
|
214
|
+
${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
|
|
215
|
+
${{ type: 'varchar', value: schema }},
|
|
216
|
+
${{ type: 'varchar', value: table }},
|
|
217
|
+
${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
218
|
+
)
|
|
219
|
+
RETURNING
|
|
220
|
+
*
|
|
221
|
+
`
|
|
222
|
+
.decoded(models.SourceTable)
|
|
223
|
+
.first();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const sourceTable = new storage.SourceTable({
|
|
227
|
+
id: sourceTableRow!.id,
|
|
228
|
+
ref: source,
|
|
229
|
+
objectId,
|
|
230
|
+
replicaIdColumns,
|
|
231
|
+
snapshotComplete: sourceTableRow!.snapshot_done ?? true,
|
|
232
|
+
...syncRules.getMatchingSources(source)
|
|
233
|
+
});
|
|
234
|
+
if (!sourceTable.snapshotComplete) {
|
|
235
|
+
sourceTable.snapshotStatus = {
|
|
236
|
+
totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
|
|
237
|
+
replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
|
|
238
|
+
lastKey: sourceTableRow!.snapshot_last_key
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
242
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
243
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
244
|
+
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
245
|
+
|
|
246
|
+
let truncatedTables: SourceTableDecoded[] = [];
|
|
247
|
+
if (objectId != null) {
|
|
248
|
+
truncatedTables = await db.sql`
|
|
249
|
+
SELECT
|
|
250
|
+
*
|
|
251
|
+
FROM
|
|
252
|
+
source_tables
|
|
253
|
+
WHERE
|
|
254
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
255
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
256
|
+
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
257
|
+
AND (
|
|
258
|
+
relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
259
|
+
OR (
|
|
260
|
+
schema_name = ${{ type: 'varchar', value: schema }}
|
|
261
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
262
|
+
)
|
|
263
|
+
)
|
|
264
|
+
`
|
|
265
|
+
.decoded(models.SourceTable)
|
|
266
|
+
.rows();
|
|
267
|
+
} else {
|
|
268
|
+
truncatedTables = await db.sql`
|
|
269
|
+
SELECT
|
|
270
|
+
*
|
|
271
|
+
FROM
|
|
272
|
+
source_tables
|
|
273
|
+
WHERE
|
|
274
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
275
|
+
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
276
|
+
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
277
|
+
AND (
|
|
278
|
+
schema_name = ${{ type: 'varchar', value: schema }}
|
|
279
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
280
|
+
)
|
|
281
|
+
`
|
|
282
|
+
.decoded(models.SourceTable)
|
|
283
|
+
.rows();
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
return {
|
|
287
|
+
tables: [sourceTable],
|
|
288
|
+
dropTables: truncatedTables.map((doc) => {
|
|
289
|
+
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
290
|
+
const dropTable = new storage.SourceTable({
|
|
291
|
+
id: doc.id,
|
|
292
|
+
ref,
|
|
293
|
+
objectId: doc.relation_id?.object_id ?? 0,
|
|
294
|
+
replicaIdColumns:
|
|
295
|
+
doc.replica_id_columns?.map(
|
|
296
|
+
(c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
|
|
297
|
+
) ?? [],
|
|
298
|
+
snapshotComplete: doc.snapshot_done ?? true,
|
|
299
|
+
...syncRules.getMatchingSources(ref)
|
|
300
|
+
});
|
|
301
|
+
dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
302
|
+
dropTable.syncData = dropTable.bucketDataSources.length > 0;
|
|
303
|
+
dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
|
|
304
|
+
return dropTable;
|
|
305
|
+
})
|
|
306
|
+
};
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
|
|
311
|
+
const row = await this.db.sql`
|
|
312
|
+
SELECT
|
|
313
|
+
*
|
|
314
|
+
FROM
|
|
315
|
+
source_tables
|
|
316
|
+
WHERE
|
|
317
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
318
|
+
AND id = ${{ type: 'varchar', value: table.id.toString() }}
|
|
319
|
+
`
|
|
320
|
+
.decoded(models.SourceTable)
|
|
321
|
+
.first();
|
|
322
|
+
|
|
323
|
+
if (row == null) {
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
return this.sourceTableFromRow(row, table.ref.connectionTag, this.sync_rules);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
private sourceTableFromRow(
|
|
331
|
+
row: SourceTableDecoded,
|
|
332
|
+
connectionTag: string,
|
|
333
|
+
syncRules: sync_rules.HydratedSyncConfig
|
|
334
|
+
): storage.SourceTable {
|
|
335
|
+
const ref = { connectionTag, schema: row.schema_name, name: row.table_name };
|
|
336
|
+
const sourceTable = new storage.SourceTable({
|
|
337
|
+
id: row.id,
|
|
338
|
+
ref,
|
|
339
|
+
objectId: row.relation_id?.object_id,
|
|
340
|
+
replicaIdColumns:
|
|
341
|
+
row.replica_id_columns?.map(
|
|
342
|
+
(c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
|
|
343
|
+
) ?? [],
|
|
344
|
+
snapshotComplete: row.snapshot_done ?? true,
|
|
345
|
+
...syncRules.getMatchingSources(ref)
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
if (!sourceTable.snapshotComplete) {
|
|
349
|
+
sourceTable.snapshotStatus = {
|
|
350
|
+
totalEstimatedCount: Number(row.snapshot_total_estimated_count ?? -1n),
|
|
351
|
+
replicatedCount: Number(row.snapshot_replicated_count ?? 0n),
|
|
352
|
+
lastKey: row.snapshot_last_key
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
sourceTable.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
356
|
+
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
357
|
+
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
358
|
+
return sourceTable;
|
|
359
|
+
}
|
|
360
|
+
|
|
142
361
|
async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
|
|
143
362
|
// TODO maybe share with abstract class
|
|
144
363
|
const { after, before, sourceTable, tag } = record;
|
|
364
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
145
365
|
for (const event of this.getTableEvents(sourceTable)) {
|
|
146
366
|
this.iterateListeners((cb) =>
|
|
147
367
|
cb.replicationEvent?.({
|
|
@@ -149,8 +369,8 @@ export class PostgresBucketBatch
|
|
|
149
369
|
table: sourceTable,
|
|
150
370
|
data: {
|
|
151
371
|
op: tag,
|
|
152
|
-
after: after && utils.isCompleteRow(
|
|
153
|
-
before: before && utils.isCompleteRow(
|
|
372
|
+
after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
|
|
373
|
+
before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
|
|
154
374
|
},
|
|
155
375
|
event
|
|
156
376
|
})
|
|
@@ -296,6 +516,8 @@ export class PostgresBucketBatch
|
|
|
296
516
|
return null;
|
|
297
517
|
}
|
|
298
518
|
|
|
519
|
+
await this.hooks?.beforeBatchFlush?.(this);
|
|
520
|
+
|
|
299
521
|
let resumeBatch: OperationBatch | null = null;
|
|
300
522
|
|
|
301
523
|
const lastOp = await this.withReplicationTransaction(async (db) => {
|
|
@@ -313,6 +535,7 @@ export class PostgresBucketBatch
|
|
|
313
535
|
|
|
314
536
|
this.persisted_op = lastOp;
|
|
315
537
|
this.last_flushed_op = lastOp;
|
|
538
|
+
await this.hooks?.afterBatchFlush?.(this);
|
|
316
539
|
return { flushed_op: lastOp };
|
|
317
540
|
}
|
|
318
541
|
|
|
@@ -536,6 +759,50 @@ export class PostgresBucketBatch
|
|
|
536
759
|
});
|
|
537
760
|
}
|
|
538
761
|
|
|
762
|
+
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
763
|
+
await this.db.transaction(async (db) => {
|
|
764
|
+
const snapshotRequiredCount = await db.sql`
|
|
765
|
+
SELECT
|
|
766
|
+
COUNT(*) AS count
|
|
767
|
+
FROM
|
|
768
|
+
source_tables
|
|
769
|
+
WHERE
|
|
770
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
771
|
+
AND snapshot_done = FALSE
|
|
772
|
+
`
|
|
773
|
+
.decoded(t.object({ count: bigint }))
|
|
774
|
+
.first();
|
|
775
|
+
if ((snapshotRequiredCount?.count ?? 0n) > 0n) {
|
|
776
|
+
if (options?.throwOnConflict ?? true) {
|
|
777
|
+
throw new ReplicationAssertionError(
|
|
778
|
+
`Cannot mark snapshot done while ${snapshotRequiredCount?.count} source table${
|
|
779
|
+
snapshotRequiredCount?.count == 1n ? '' : 's'
|
|
780
|
+
} still require snapshotting`
|
|
781
|
+
);
|
|
782
|
+
} else {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
await db.sql`
|
|
788
|
+
UPDATE sync_rules
|
|
789
|
+
SET
|
|
790
|
+
snapshot_done = TRUE,
|
|
791
|
+
last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }},
|
|
792
|
+
no_checkpoint_before = CASE
|
|
793
|
+
WHEN no_checkpoint_before IS NULL
|
|
794
|
+
OR no_checkpoint_before < ${{ type: 'varchar', value: no_checkpoint_before_lsn }} THEN ${{
|
|
795
|
+
type: 'varchar',
|
|
796
|
+
value: no_checkpoint_before_lsn
|
|
797
|
+
}}
|
|
798
|
+
ELSE no_checkpoint_before
|
|
799
|
+
END
|
|
800
|
+
WHERE
|
|
801
|
+
id = ${{ type: 'int4', value: this.group_id }}
|
|
802
|
+
`.execute();
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
|
|
539
806
|
async markTableSnapshotRequired(table: storage.SourceTable): Promise<void> {
|
|
540
807
|
await this.db.sql`
|
|
541
808
|
UPDATE sync_rules
|
|
@@ -628,8 +895,12 @@ export class PostgresBucketBatch
|
|
|
628
895
|
|
|
629
896
|
protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) {
|
|
630
897
|
let sizes: Map<string, number> | undefined = undefined;
|
|
631
|
-
if
|
|
632
|
-
|
|
898
|
+
// Check if any table in this batch needs to store current_data
|
|
899
|
+
const anyTableStoresCurrentData =
|
|
900
|
+
this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
|
|
901
|
+
|
|
902
|
+
if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
|
|
903
|
+
// We skip this step if no tables store current_data, since the sizes will
|
|
633
904
|
// always be small in that case.
|
|
634
905
|
|
|
635
906
|
// With skipExistingRows, we don't load the full documents into memory,
|
|
@@ -637,15 +908,19 @@ export class PostgresBucketBatch
|
|
|
637
908
|
|
|
638
909
|
// Find sizes of current_data documents, to assist in intelligent batching without
|
|
639
910
|
// exceeding memory limits.
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
911
|
+
// Within this branch the batch stores current_data, so the per-table flag is the
|
|
912
|
+
// effective value - only look up sizes for tables that actually store current_data.
|
|
913
|
+
const sizeLookups = batch.batch
|
|
914
|
+
.filter((r) => r.record.sourceTable.storeCurrentData)
|
|
915
|
+
.map((r) => {
|
|
916
|
+
return {
|
|
917
|
+
source_table: postgresTableId(r.record.sourceTable.id),
|
|
918
|
+
/**
|
|
919
|
+
* Encode to hex in order to pass a jsonb
|
|
920
|
+
*/
|
|
921
|
+
source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
|
|
922
|
+
};
|
|
923
|
+
});
|
|
649
924
|
|
|
650
925
|
sizes = new Map<string, number>();
|
|
651
926
|
|
|
@@ -764,6 +1039,9 @@ export class PostgresBucketBatch
|
|
|
764
1039
|
const afterId = operation.afterId;
|
|
765
1040
|
let after = record.after;
|
|
766
1041
|
const sourceTable = record.sourceTable;
|
|
1042
|
+
// Effective per-record flag: store current_data only if both the batch (source-level)
|
|
1043
|
+
// and the table (e.g. non-FULL replica identity) require it.
|
|
1044
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
767
1045
|
|
|
768
1046
|
let existingBuckets: CurrentBucket[] = [];
|
|
769
1047
|
let newBuckets: CurrentBucket[] = [];
|
|
@@ -792,7 +1070,7 @@ export class PostgresBucketBatch
|
|
|
792
1070
|
existingLookups = [];
|
|
793
1071
|
// Log to help with debugging if there was a consistency issue
|
|
794
1072
|
|
|
795
|
-
if (
|
|
1073
|
+
if (storeCurrentData) {
|
|
796
1074
|
if (this.markRecordUnavailable != null) {
|
|
797
1075
|
// This will trigger a "resnapshot" of the record.
|
|
798
1076
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -808,7 +1086,7 @@ export class PostgresBucketBatch
|
|
|
808
1086
|
} else {
|
|
809
1087
|
existingBuckets = result.buckets;
|
|
810
1088
|
existingLookups = result.lookups;
|
|
811
|
-
if (
|
|
1089
|
+
if (storeCurrentData) {
|
|
812
1090
|
const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
|
|
813
1091
|
after = storage.mergeToast(after!, data);
|
|
814
1092
|
}
|
|
@@ -819,7 +1097,9 @@ export class PostgresBucketBatch
|
|
|
819
1097
|
// Not an error if we re-apply a transaction
|
|
820
1098
|
existingBuckets = [];
|
|
821
1099
|
existingLookups = [];
|
|
822
|
-
// Log to help with debugging if there was a consistency issue
|
|
1100
|
+
// Log to help with debugging if there was a consistency issue.
|
|
1101
|
+
// Gate on the batch-level flag: FULL tables (per-record flag false) still get a
|
|
1102
|
+
// current_data entry, so a missing record on DELETE is meaningful for them too.
|
|
823
1103
|
if (this.options.store_current_data && this.markRecordUnavailable == null) {
|
|
824
1104
|
this.logger.warn(
|
|
825
1105
|
`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
@@ -832,7 +1112,7 @@ export class PostgresBucketBatch
|
|
|
832
1112
|
}
|
|
833
1113
|
|
|
834
1114
|
let afterData: Buffer<ArrayBuffer> | undefined;
|
|
835
|
-
if (afterId != null && !
|
|
1115
|
+
if (afterId != null && !storeCurrentData) {
|
|
836
1116
|
afterData = storage.serializeBson({});
|
|
837
1117
|
} else if (afterId != null) {
|
|
838
1118
|
try {
|
|
@@ -895,12 +1175,13 @@ export class PostgresBucketBatch
|
|
|
895
1175
|
// However, it will be valid by the end of the transaction.
|
|
896
1176
|
//
|
|
897
1177
|
// In this case, we don't save the op, but we do save the current data.
|
|
898
|
-
if (afterId && after && utils.isCompleteRow(
|
|
1178
|
+
if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
|
|
899
1179
|
// Insert or update
|
|
900
1180
|
if (sourceTable.syncData) {
|
|
901
1181
|
const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
902
1182
|
record: after,
|
|
903
|
-
sourceTable
|
|
1183
|
+
sourceTable: sourceTable.ref,
|
|
1184
|
+
bucketDataSources: sourceTable.bucketDataSources
|
|
904
1185
|
});
|
|
905
1186
|
|
|
906
1187
|
for (const error of syncErrors) {
|
|
@@ -939,8 +1220,9 @@ export class PostgresBucketBatch
|
|
|
939
1220
|
if (sourceTable.syncParameters) {
|
|
940
1221
|
// Parameters
|
|
941
1222
|
const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors(
|
|
942
|
-
sourceTable,
|
|
943
|
-
after
|
|
1223
|
+
sourceTable.ref,
|
|
1224
|
+
after,
|
|
1225
|
+
{ parameterLookupSources: sourceTable.parameterLookupSources }
|
|
944
1226
|
);
|
|
945
1227
|
|
|
946
1228
|
for (let error of paramErrors) {
|
|
@@ -1054,7 +1336,7 @@ export class PostgresBucketBatch
|
|
|
1054
1336
|
}
|
|
1055
1337
|
});
|
|
1056
1338
|
if (didActivate) {
|
|
1057
|
-
this.logger.info(`Activated new
|
|
1339
|
+
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
1058
1340
|
}
|
|
1059
1341
|
}
|
|
1060
1342
|
|
|
@@ -1064,7 +1346,7 @@ export class PostgresBucketBatch
|
|
|
1064
1346
|
*/
|
|
1065
1347
|
protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
|
|
1066
1348
|
return this.sync_rules.eventDescriptors.filter((evt) =>
|
|
1067
|
-
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table))
|
|
1349
|
+
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
|
|
1068
1350
|
);
|
|
1069
1351
|
}
|
|
1070
1352
|
|
|
@@ -64,7 +64,9 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
64
64
|
switch (this.writeCheckpointMode) {
|
|
65
65
|
case storage.WriteCheckpointMode.CUSTOM:
|
|
66
66
|
if (false == 'sync_rules_id' in filters) {
|
|
67
|
-
throw new framework.errors.ValidationError(
|
|
67
|
+
throw new framework.errors.ValidationError(
|
|
68
|
+
`Replication stream ID is required for custom Write Checkpoint filtering`
|
|
69
|
+
);
|
|
68
70
|
}
|
|
69
71
|
return this.lastCustomWriteCheckpoint(filters as storage.CustomWriteCheckpointFilters);
|
|
70
72
|
case storage.WriteCheckpointMode.MANAGED:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
-
import { ErrorCode,
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
import { models } from '../../types/types.js';
|
|
5
5
|
|
|
@@ -31,17 +31,14 @@ export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRule
|
|
|
31
31
|
});
|
|
32
32
|
const lockHandle = await manager.acquire();
|
|
33
33
|
if (!lockHandle) {
|
|
34
|
-
throw new ServiceError(
|
|
35
|
-
ErrorCode.PSYNC_S1003,
|
|
36
|
-
`Sync rules: ${this.id} have been locked by another process for replication.`
|
|
37
|
-
);
|
|
34
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
38
35
|
}
|
|
39
36
|
|
|
40
37
|
const interval = setInterval(async () => {
|
|
41
38
|
try {
|
|
42
39
|
await lockHandle.refresh();
|
|
43
40
|
} catch (e) {
|
|
44
|
-
logger.error('Failed to refresh lock', e);
|
|
41
|
+
this.logger.error('Failed to refresh lock', e);
|
|
45
42
|
clearInterval(interval);
|
|
46
43
|
}
|
|
47
44
|
}, 30_130);
|
package/test/tsconfig.json
CHANGED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { storage } from '@powersync/service-core';
|
|
2
|
-
export declare const V1_CURRENT_DATA_TABLE = "current_data";
|
|
3
|
-
export declare const V3_CURRENT_DATA_TABLE = "v3_current_data";
|
|
4
|
-
/**
|
|
5
|
-
* The table used by a specific storage version for general current_data access.
|
|
6
|
-
*/
|
|
7
|
-
export declare function getCommonCurrentDataTable(storageConfig: storage.StorageVersionConfig): "current_data" | "v3_current_data";
|
|
8
|
-
export declare function getV1CurrentDataTable(storageConfig: storage.StorageVersionConfig): string;
|
|
9
|
-
export declare function getV3CurrentDataTable(storageConfig: storage.StorageVersionConfig): string;
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
-
export const V1_CURRENT_DATA_TABLE = 'current_data';
|
|
3
|
-
export const V3_CURRENT_DATA_TABLE = 'v3_current_data';
|
|
4
|
-
/**
|
|
5
|
-
* The table used by a specific storage version for general current_data access.
|
|
6
|
-
*/
|
|
7
|
-
export function getCommonCurrentDataTable(storageConfig) {
|
|
8
|
-
return storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE;
|
|
9
|
-
}
|
|
10
|
-
export function getV1CurrentDataTable(storageConfig) {
|
|
11
|
-
if (storageConfig.softDeleteCurrentData) {
|
|
12
|
-
throw new ServiceAssertionError('current_data table cannot be used when softDeleteCurrentData is enabled');
|
|
13
|
-
}
|
|
14
|
-
return V1_CURRENT_DATA_TABLE;
|
|
15
|
-
}
|
|
16
|
-
export function getV3CurrentDataTable(storageConfig) {
|
|
17
|
-
if (!storageConfig.softDeleteCurrentData) {
|
|
18
|
-
throw new ServiceAssertionError('v3_current_data table cannot be used when softDeleteCurrentData is disabled');
|
|
19
|
-
}
|
|
20
|
-
return V3_CURRENT_DATA_TABLE;
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=current-data-table.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"current-data-table.js","sourceRoot":"","sources":["../../src/storage/current-data-table.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,cAAc,CAAC;AACpD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAEvD;;GAEG;AACH,MAAM,UAAU,yBAAyB,CAAC,aAA2C;IACnF,OAAO,aAAa,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,aAA2C;IAC/E,IAAI,aAAa,CAAC,qBAAqB,EAAE,CAAC;QACxC,MAAM,IAAI,qBAAqB,CAAC,yEAAyE,CAAC,CAAC;IAC7G,CAAC;IACD,OAAO,qBAAqB,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,aAA2C;IAC/E,IAAI,CAAC,aAAa,CAAC,qBAAqB,EAAE,CAAC;QACzC,MAAM,IAAI,qBAAqB,CAAC,6EAA6E,CAAC,CAAC;IACjH,CAAC;IACD,OAAO,qBAAqB,CAAC;AAC/B,CAAC"}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
3
|
-
|
|
4
|
-
export const V1_CURRENT_DATA_TABLE = 'current_data';
|
|
5
|
-
export const V3_CURRENT_DATA_TABLE = 'v3_current_data';
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* The table used by a specific storage version for general current_data access.
|
|
9
|
-
*/
|
|
10
|
-
export function getCommonCurrentDataTable(storageConfig: storage.StorageVersionConfig) {
|
|
11
|
-
return storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export function getV1CurrentDataTable(storageConfig: storage.StorageVersionConfig) {
|
|
15
|
-
if (storageConfig.softDeleteCurrentData) {
|
|
16
|
-
throw new ServiceAssertionError('current_data table cannot be used when softDeleteCurrentData is enabled');
|
|
17
|
-
}
|
|
18
|
-
return V1_CURRENT_DATA_TABLE;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export function getV3CurrentDataTable(storageConfig: storage.StorageVersionConfig) {
|
|
22
|
-
if (!storageConfig.softDeleteCurrentData) {
|
|
23
|
-
throw new ServiceAssertionError('v3_current_data table cannot be used when softDeleteCurrentData is disabled');
|
|
24
|
-
}
|
|
25
|
-
return V3_CURRENT_DATA_TABLE;
|
|
26
|
-
}
|