@powersync/service-module-postgres-storage 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +12 -12
- package/dist/storage/PostgresBucketStorageFactory.js +57 -52
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +1 -1
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +16 -7
- package/dist/storage/PostgresSyncRulesStorage.js +48 -45
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +2 -3
- package/dist/storage/batch/PostgresBucketBatch.js +26 -25
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +89 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
- package/dist/utils/test-utils.js +2 -2
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -22
- package/src/storage/PostgresBucketStorageFactory.ts +64 -54
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresStorageProvider.ts +1 -1
- package/src/storage/PostgresSyncRulesStorage.ts +51 -52
- package/src/storage/batch/PostgresBucketBatch.ts +30 -32
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +97 -0
- package/src/utils/test-utils.ts +2 -2
- package/test/src/migrations.test.ts +1 -1
- package/test/src/storage.test.ts +7 -2
- package/test/src/storage_compacting.test.ts +6 -4
- package/test/src/storage_sync.test.ts +2 -1
- package/test/tsconfig.json +0 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -0
- package/dist/.tsbuildinfo +0 -1
- package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
- /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
- /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +0 -0
- /package/dist/{@types/storage → storage}/current-data-store.d.ts +0 -0
- /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
- /package/dist/{@types/types → types}/codecs.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
- /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
- /package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/json.d.ts +0 -0
- /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
- /package/dist/{@types/types → types}/types.d.ts +0 -0
- /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
- /package/dist/{@types/utils → utils}/db.d.ts +0 -0
- /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
- /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
- /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
|
@@ -41,7 +41,7 @@ import { PostgresCompactor } from './PostgresCompactor.js';
|
|
|
41
41
|
export type PostgresSyncRulesStorageOptions = {
|
|
42
42
|
factory: PostgresBucketStorageFactory;
|
|
43
43
|
db: lib_postgres.DatabaseClient;
|
|
44
|
-
|
|
44
|
+
replicationStream: storage.PersistedReplicationStream;
|
|
45
45
|
write_checkpoint_mode?: storage.WriteCheckpointMode;
|
|
46
46
|
batchLimits: RequiredOperationBatchLimits;
|
|
47
47
|
};
|
|
@@ -52,9 +52,10 @@ export class PostgresSyncRulesStorage
|
|
|
52
52
|
{
|
|
53
53
|
[framework.DO_NOT_LOG] = true;
|
|
54
54
|
|
|
55
|
-
public readonly
|
|
56
|
-
public readonly
|
|
57
|
-
public readonly
|
|
55
|
+
public readonly replicationStreamId: number;
|
|
56
|
+
public readonly replicationStream: storage.PersistedReplicationStream;
|
|
57
|
+
public readonly sync_rules: storage.PersistedSyncConfigContent;
|
|
58
|
+
public readonly replicationStreamName: string;
|
|
58
59
|
public readonly factory: PostgresBucketStorageFactory;
|
|
59
60
|
public readonly storageConfig: StorageVersionConfig;
|
|
60
61
|
public readonly logger: framework.Logger;
|
|
@@ -65,22 +66,27 @@ export class PostgresSyncRulesStorage
|
|
|
65
66
|
protected writeCheckpointAPI: PostgresWriteCheckpointAPI;
|
|
66
67
|
private readonly currentDataStore: PostgresCurrentDataStore;
|
|
67
68
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
/**
|
|
70
|
+
* Canonical parsed sync config sets, keyed by defaultSchema. Entries are never evicted,
|
|
71
|
+
* so each parse options value maps to exactly one parsed set for the lifetime of this
|
|
72
|
+
* storage instance.
|
|
73
|
+
*
|
|
74
|
+
* TODO we might be able to share this in an abstract class
|
|
75
|
+
*/
|
|
76
|
+
private readonly parsedSyncConfigSets = new Map<string, storage.ParsedSyncConfigSet>();
|
|
72
77
|
private _checksumCache: storage.ChecksumCache | undefined;
|
|
73
78
|
|
|
74
79
|
constructor(protected options: PostgresSyncRulesStorageOptions) {
|
|
75
80
|
super();
|
|
76
|
-
this.
|
|
81
|
+
this.replicationStream = options.replicationStream;
|
|
82
|
+
this.replicationStreamId = options.replicationStream.replicationStreamId;
|
|
77
83
|
this.db = options.db;
|
|
78
|
-
this.sync_rules = options.
|
|
79
|
-
this.
|
|
84
|
+
this.sync_rules = options.replicationStream.syncConfigContent[0];
|
|
85
|
+
this.replicationStreamName = options.replicationStream.replicationStreamName;
|
|
80
86
|
this.factory = options.factory;
|
|
81
|
-
this.storageConfig = options.
|
|
87
|
+
this.storageConfig = options.replicationStream.getStorageConfig();
|
|
82
88
|
this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
|
|
83
|
-
this.logger = options.
|
|
89
|
+
this.logger = options.replicationStream.logger;
|
|
84
90
|
|
|
85
91
|
this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
|
|
86
92
|
db: this.db,
|
|
@@ -106,18 +112,17 @@ export class PostgresSyncRulesStorage
|
|
|
106
112
|
return this.writeCheckpointAPI.writeCheckpointMode;
|
|
107
113
|
}
|
|
108
114
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
* Parse sync config if the options are different or if there is no cached value.
|
|
115
|
-
*/
|
|
116
|
-
if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
|
|
117
|
-
this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncConfig(), options };
|
|
115
|
+
getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
116
|
+
let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
|
|
117
|
+
if (parsed == null) {
|
|
118
|
+
parsed = this.replicationStream.parsed(options);
|
|
119
|
+
this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
|
|
118
120
|
}
|
|
121
|
+
return parsed;
|
|
122
|
+
}
|
|
119
123
|
|
|
120
|
-
|
|
124
|
+
getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig {
|
|
125
|
+
return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
|
|
121
126
|
}
|
|
122
127
|
|
|
123
128
|
async reportError(e: any): Promise<void> {
|
|
@@ -127,7 +132,7 @@ export class PostgresSyncRulesStorage
|
|
|
127
132
|
SET
|
|
128
133
|
last_fatal_error = ${{ type: 'varchar', value: message }}
|
|
129
134
|
WHERE
|
|
130
|
-
id = ${{ type: 'int4', value: this.
|
|
135
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }};
|
|
131
136
|
`.execute();
|
|
132
137
|
}
|
|
133
138
|
|
|
@@ -139,7 +144,7 @@ export class PostgresSyncRulesStorage
|
|
|
139
144
|
maxOpId = checkpoint.checkpoint;
|
|
140
145
|
}
|
|
141
146
|
|
|
142
|
-
return new PostgresCompactor(this.db, this.
|
|
147
|
+
return new PostgresCompactor(this.db, this.replicationStreamId, {
|
|
143
148
|
...options,
|
|
144
149
|
maxOpId,
|
|
145
150
|
logger: this.logger
|
|
@@ -154,7 +159,7 @@ export class PostgresSyncRulesStorage
|
|
|
154
159
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
|
|
155
160
|
return this.writeCheckpointAPI.lastWriteCheckpoint({
|
|
156
161
|
...filters,
|
|
157
|
-
sync_rules_id: this.
|
|
162
|
+
sync_rules_id: this.replicationStreamId
|
|
158
163
|
});
|
|
159
164
|
}
|
|
160
165
|
|
|
@@ -174,7 +179,7 @@ export class PostgresSyncRulesStorage
|
|
|
174
179
|
FROM
|
|
175
180
|
sync_rules
|
|
176
181
|
WHERE
|
|
177
|
-
id = ${{ type: 'int4', value: this.
|
|
182
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
178
183
|
`
|
|
179
184
|
.decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn']))
|
|
180
185
|
.first();
|
|
@@ -196,7 +201,7 @@ export class PostgresSyncRulesStorage
|
|
|
196
201
|
FROM
|
|
197
202
|
sync_rules
|
|
198
203
|
WHERE
|
|
199
|
-
id = ${{ type: 'int4', value: this.
|
|
204
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
200
205
|
`
|
|
201
206
|
.decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op', 'snapshot_lsn']))
|
|
202
207
|
.first();
|
|
@@ -206,9 +211,9 @@ export class PostgresSyncRulesStorage
|
|
|
206
211
|
const writer = new PostgresBucketBatch({
|
|
207
212
|
logger: options.logger ?? this.logger,
|
|
208
213
|
db: this.db,
|
|
209
|
-
sync_rules: this.
|
|
210
|
-
|
|
211
|
-
|
|
214
|
+
sync_rules: this.getParsedSyncRules(options),
|
|
215
|
+
replicationStreamId: this.replicationStreamId,
|
|
216
|
+
replicationStreamName: this.replicationStreamName,
|
|
212
217
|
last_checkpoint_lsn: checkpoint_lsn,
|
|
213
218
|
keep_alive_op: syncRules?.keepalive_op,
|
|
214
219
|
resumeFromLsn: maxLsn(syncRules?.snapshot_lsn, checkpoint_lsn),
|
|
@@ -254,7 +259,7 @@ export class PostgresSyncRulesStorage
|
|
|
254
259
|
value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
|
|
255
260
|
}}) WITH ORDINALITY AS requested (value, index)
|
|
256
261
|
WHERE
|
|
257
|
-
group_id = ${{ type: 'int4', value: this.
|
|
262
|
+
group_id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
258
263
|
AND lookup = decode((requested.value ->> 0)::text, 'hex') -- Decode the hex string to bytea
|
|
259
264
|
AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
|
|
260
265
|
ORDER BY
|
|
@@ -381,7 +386,7 @@ export class PostgresSyncRulesStorage
|
|
|
381
386
|
LIMIT
|
|
382
387
|
$3;`,
|
|
383
388
|
params: [
|
|
384
|
-
{ type: 'int4', value: this.
|
|
389
|
+
{ type: 'int4', value: this.replicationStreamId },
|
|
385
390
|
{ type: 'int8', value: end },
|
|
386
391
|
{ type: 'int4', value: batchRowLimit },
|
|
387
392
|
...filters.flatMap((f) => [
|
|
@@ -497,26 +502,23 @@ export class PostgresSyncRulesStorage
|
|
|
497
502
|
state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }},
|
|
498
503
|
snapshot_done = ${{ type: 'bool', value: false }}
|
|
499
504
|
WHERE
|
|
500
|
-
id = ${{ type: 'int4', value: this.
|
|
505
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
501
506
|
`.execute();
|
|
502
507
|
}
|
|
503
508
|
|
|
504
|
-
async getStatus(): Promise<storage.
|
|
509
|
+
async getStatus(): Promise<storage.ReplicationStreamStatus> {
|
|
505
510
|
const syncRulesRow = await this.db.sql`
|
|
506
511
|
SELECT
|
|
507
512
|
snapshot_done,
|
|
508
513
|
snapshot_lsn,
|
|
509
514
|
last_checkpoint_lsn,
|
|
510
|
-
state
|
|
511
|
-
keepalive_op
|
|
515
|
+
state
|
|
512
516
|
FROM
|
|
513
517
|
sync_rules
|
|
514
518
|
WHERE
|
|
515
|
-
id = ${{ type: 'int4', value: this.
|
|
519
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
516
520
|
`
|
|
517
|
-
.decoded(
|
|
518
|
-
pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state', 'snapshot_lsn', 'keepalive_op'])
|
|
519
|
-
)
|
|
521
|
+
.decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'snapshot_lsn']))
|
|
520
522
|
.first();
|
|
521
523
|
|
|
522
524
|
if (syncRulesRow == null) {
|
|
@@ -524,11 +526,8 @@ export class PostgresSyncRulesStorage
|
|
|
524
526
|
}
|
|
525
527
|
|
|
526
528
|
return {
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null,
|
|
530
|
-
snapshot_lsn: syncRulesRow.snapshot_lsn ?? null,
|
|
531
|
-
keepalive_op: syncRulesRow.keepalive_op ?? null
|
|
529
|
+
snapshotDone: syncRulesRow.snapshot_done && syncRulesRow.last_checkpoint_lsn != null,
|
|
530
|
+
resumeLsn: maxLsn(syncRulesRow.snapshot_lsn, syncRulesRow.last_checkpoint_lsn)
|
|
532
531
|
};
|
|
533
532
|
}
|
|
534
533
|
|
|
@@ -542,27 +541,27 @@ export class PostgresSyncRulesStorage
|
|
|
542
541
|
last_checkpoint = NULL,
|
|
543
542
|
no_checkpoint_before = NULL
|
|
544
543
|
WHERE
|
|
545
|
-
id = ${{ type: 'int4', value: this.
|
|
544
|
+
id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
546
545
|
`.execute();
|
|
547
546
|
|
|
548
547
|
await this.db.sql`
|
|
549
548
|
DELETE FROM bucket_data
|
|
550
549
|
WHERE
|
|
551
|
-
group_id = ${{ type: 'int4', value: this.
|
|
550
|
+
group_id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
552
551
|
`.execute();
|
|
553
552
|
|
|
554
553
|
await this.db.sql`
|
|
555
554
|
DELETE FROM bucket_parameters
|
|
556
555
|
WHERE
|
|
557
|
-
group_id = ${{ type: 'int4', value: this.
|
|
556
|
+
group_id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
558
557
|
`.execute();
|
|
559
558
|
|
|
560
|
-
await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.
|
|
559
|
+
await this.currentDataStore.deleteGroupRows(this.db, { groupId: this.replicationStreamId });
|
|
561
560
|
|
|
562
561
|
await this.db.sql`
|
|
563
562
|
DELETE FROM source_tables
|
|
564
563
|
WHERE
|
|
565
|
-
group_id = ${{ type: 'int4', value: this.
|
|
564
|
+
group_id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
566
565
|
`.execute();
|
|
567
566
|
}
|
|
568
567
|
|
|
@@ -603,7 +602,7 @@ export class PostgresSyncRulesStorage
|
|
|
603
602
|
AND b.op_id > f.start_op_id
|
|
604
603
|
AND b.op_id <= f.end_op_id
|
|
605
604
|
WHERE
|
|
606
|
-
b.group_id = ${{ type: 'int4', value: this.
|
|
605
|
+
b.group_id = ${{ type: 'int4', value: this.replicationStreamId }}
|
|
607
606
|
GROUP BY
|
|
608
607
|
b.bucket_name;
|
|
609
608
|
`.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>();
|
|
@@ -38,8 +38,8 @@ export interface PostgresBucketBatchOptions {
|
|
|
38
38
|
logger: Logger;
|
|
39
39
|
db: lib_postgres.DatabaseClient;
|
|
40
40
|
sync_rules: sync_rules.HydratedSyncConfig;
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
replicationStreamId: number;
|
|
42
|
+
replicationStreamName: string;
|
|
43
43
|
last_checkpoint_lsn: string | null;
|
|
44
44
|
store_current_data: boolean;
|
|
45
45
|
keep_alive_op?: InternalOpId | null;
|
|
@@ -112,7 +112,7 @@ export class PostgresBucketBatch
|
|
|
112
112
|
super();
|
|
113
113
|
this.logger = options.logger;
|
|
114
114
|
this.db = options.db;
|
|
115
|
-
this.group_id = options.
|
|
115
|
+
this.group_id = options.replicationStreamId;
|
|
116
116
|
this.last_checkpoint_lsn = options.last_checkpoint_lsn;
|
|
117
117
|
this.resumeFromLsn = options.resumeFromLsn;
|
|
118
118
|
this.skipExistingRows = options.skip_existing_rows;
|
|
@@ -129,10 +129,6 @@ export class PostgresBucketBatch
|
|
|
129
129
|
}
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
-
get lastCheckpointLsn() {
|
|
133
|
-
return this.last_checkpoint_lsn;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
132
|
async [Symbol.asyncDispose]() {
|
|
137
133
|
if (this.batch != null || this.write_checkpoint_batch.length > 0) {
|
|
138
134
|
// We don't error here, since:
|
|
@@ -148,7 +144,7 @@ export class PostgresBucketBatch
|
|
|
148
144
|
}
|
|
149
145
|
|
|
150
146
|
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
151
|
-
const syncRules = options.
|
|
147
|
+
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
152
148
|
const { connection_id, source } = options;
|
|
153
149
|
const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
154
150
|
|
|
@@ -362,19 +358,22 @@ export class PostgresBucketBatch
|
|
|
362
358
|
// TODO maybe share with abstract class
|
|
363
359
|
const { after, before, sourceTable, tag } = record;
|
|
364
360
|
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
361
|
+
// Only the table designated as event carrier by resolveTables may fire events.
|
|
362
|
+
if (sourceTable.syncEvent) {
|
|
363
|
+
for (const event of this.getTableEvents(sourceTable)) {
|
|
364
|
+
this.iterateListeners((cb) =>
|
|
365
|
+
cb.replicationEvent?.({
|
|
366
|
+
batch: this,
|
|
367
|
+
table: sourceTable,
|
|
368
|
+
data: {
|
|
369
|
+
op: tag,
|
|
370
|
+
after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
|
|
371
|
+
before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
|
|
372
|
+
},
|
|
373
|
+
event
|
|
374
|
+
})
|
|
375
|
+
);
|
|
376
|
+
}
|
|
378
377
|
}
|
|
379
378
|
/**
|
|
380
379
|
* Return if the table is just an event table
|
|
@@ -761,23 +760,22 @@ export class PostgresBucketBatch
|
|
|
761
760
|
|
|
762
761
|
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
763
762
|
await this.db.transaction(async (db) => {
|
|
764
|
-
const
|
|
763
|
+
const blockingTables = await db.sql`
|
|
765
764
|
SELECT
|
|
766
|
-
|
|
765
|
+
schema_name,
|
|
766
|
+
table_name
|
|
767
767
|
FROM
|
|
768
768
|
source_tables
|
|
769
769
|
WHERE
|
|
770
770
|
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
771
771
|
AND snapshot_done = FALSE
|
|
772
772
|
`
|
|
773
|
-
.decoded(t.object({
|
|
774
|
-
.
|
|
775
|
-
if (
|
|
773
|
+
.decoded(t.object({ schema_name: t.string, table_name: t.string }))
|
|
774
|
+
.rows();
|
|
775
|
+
if (blockingTables.length > 0) {
|
|
776
776
|
if (options?.throwOnConflict ?? true) {
|
|
777
777
|
throw new ReplicationAssertionError(
|
|
778
|
-
`Cannot mark snapshot done while ${
|
|
779
|
-
snapshotRequiredCount?.count == 1n ? '' : 's'
|
|
780
|
-
} still require snapshotting`
|
|
778
|
+
`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
|
|
781
779
|
);
|
|
782
780
|
} else {
|
|
783
781
|
return;
|
|
@@ -1134,7 +1132,7 @@ export class PostgresBucketBatch
|
|
|
1134
1132
|
{
|
|
1135
1133
|
level: errors.ErrorSeverity.WARNING,
|
|
1136
1134
|
metadata: {
|
|
1137
|
-
replication_slot: this.options.
|
|
1135
|
+
replication_slot: this.options.replicationStreamName,
|
|
1138
1136
|
table: record.sourceTable.qualifiedName
|
|
1139
1137
|
}
|
|
1140
1138
|
}
|
|
@@ -1190,7 +1188,7 @@ export class PostgresBucketBatch
|
|
|
1190
1188
|
{
|
|
1191
1189
|
level: errors.ErrorSeverity.WARNING,
|
|
1192
1190
|
metadata: {
|
|
1193
|
-
replication_slot: this.options.
|
|
1191
|
+
replication_slot: this.options.replicationStreamName,
|
|
1194
1192
|
table: record.sourceTable.qualifiedName
|
|
1195
1193
|
}
|
|
1196
1194
|
}
|
|
@@ -1231,7 +1229,7 @@ export class PostgresBucketBatch
|
|
|
1231
1229
|
{
|
|
1232
1230
|
level: errors.ErrorSeverity.WARNING,
|
|
1233
1231
|
metadata: {
|
|
1234
|
-
replication_slot: this.options.
|
|
1232
|
+
replication_slot: this.options.replicationStreamName,
|
|
1235
1233
|
table: record.sourceTable.qualifiedName
|
|
1236
1234
|
}
|
|
1237
1235
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { models } from '../../types/types.js';
|
|
5
|
+
|
|
6
|
+
export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
|
|
7
|
+
constructor(
|
|
8
|
+
private db: lib_postgres.DatabaseClient,
|
|
9
|
+
row: models.SyncRulesDecoded
|
|
10
|
+
) {
|
|
11
|
+
super({
|
|
12
|
+
replicationStreamId: Number(row.id),
|
|
13
|
+
sync_rules_content: row.content,
|
|
14
|
+
compiled_plan: row.sync_plan,
|
|
15
|
+
replicationStreamName: row.slot_name,
|
|
16
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
21
|
+
const row = await this.db.sql`
|
|
22
|
+
SELECT
|
|
23
|
+
*
|
|
24
|
+
FROM
|
|
25
|
+
sync_rules
|
|
26
|
+
WHERE
|
|
27
|
+
id = ${{ value: this.replicationStreamId, type: 'int4' }}
|
|
28
|
+
`
|
|
29
|
+
.decoded(models.SyncRules)
|
|
30
|
+
.first();
|
|
31
|
+
|
|
32
|
+
return row == null ? null : syncConfigStatusFromRow(row);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
|
|
37
|
+
return {
|
|
38
|
+
id: String(row.id),
|
|
39
|
+
replicationStreamId: Number(row.id),
|
|
40
|
+
state: row.state as storage.SyncRuleState,
|
|
41
|
+
snapshot_done: row.snapshot_done,
|
|
42
|
+
last_checkpoint_lsn: row.last_checkpoint_lsn,
|
|
43
|
+
last_fatal_error: row.last_fatal_error,
|
|
44
|
+
last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
|
|
45
|
+
last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
50
|
+
current_lock: storage.ReplicationLock | null = null;
|
|
51
|
+
readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
|
|
52
|
+
|
|
53
|
+
constructor(
|
|
54
|
+
private db: lib_postgres.DatabaseClient,
|
|
55
|
+
private readonly row: models.SyncRulesDecoded
|
|
56
|
+
) {
|
|
57
|
+
super({
|
|
58
|
+
replicationStreamId: Number(row.id),
|
|
59
|
+
replicationStreamName: row.slot_name,
|
|
60
|
+
state: row.state as storage.SyncRuleState,
|
|
61
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
62
|
+
});
|
|
63
|
+
this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
67
|
+
return this.syncConfigContent[0].parsed(options);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async lock(): Promise<storage.ReplicationLock> {
|
|
71
|
+
const manager = new lib_postgres.PostgresLockManager({
|
|
72
|
+
db: this.db,
|
|
73
|
+
name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
|
|
74
|
+
});
|
|
75
|
+
const lockHandle = await manager.acquire();
|
|
76
|
+
if (!lockHandle) {
|
|
77
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const interval = setInterval(async () => {
|
|
81
|
+
try {
|
|
82
|
+
await lockHandle.refresh();
|
|
83
|
+
} catch (e) {
|
|
84
|
+
this.logger.error('Failed to refresh lock', e);
|
|
85
|
+
clearInterval(interval);
|
|
86
|
+
}
|
|
87
|
+
}, 30_130);
|
|
88
|
+
|
|
89
|
+
return (this.current_lock = {
|
|
90
|
+
sync_rules_id: this.replicationStreamId,
|
|
91
|
+
release: async () => {
|
|
92
|
+
clearInterval(interval);
|
|
93
|
+
return lockHandle.release();
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/utils/test-utils.ts
CHANGED
|
@@ -73,7 +73,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
|
73
73
|
|
|
74
74
|
await using storageFactory = new PostgresBucketStorageFactory({
|
|
75
75
|
config: TEST_CONNECTION_OPTIONS,
|
|
76
|
-
|
|
76
|
+
replicationStreamNamePrefix: 'test_'
|
|
77
77
|
});
|
|
78
78
|
await truncateTables(storageFactory.db);
|
|
79
79
|
};
|
|
@@ -102,7 +102,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
|
102
102
|
|
|
103
103
|
return new PostgresBucketStorageFactory({
|
|
104
104
|
config: TEST_CONNECTION_OPTIONS,
|
|
105
|
-
|
|
105
|
+
replicationStreamNamePrefix: 'test_'
|
|
106
106
|
});
|
|
107
107
|
} catch (ex) {
|
|
108
108
|
// Vitest does not display these errors nicely when using the `await using` syntx
|
|
@@ -18,7 +18,7 @@ describe('Migrations', () => {
|
|
|
18
18
|
const setup = POSTGRES_STORAGE_SETUP;
|
|
19
19
|
await using factory = new PostgresBucketStorageFactory({
|
|
20
20
|
config: TEST_CONNECTION_OPTIONS,
|
|
21
|
-
|
|
21
|
+
replicationStreamNamePrefix: 'test_'
|
|
22
22
|
});
|
|
23
23
|
|
|
24
24
|
await dropTables(factory.db);
|
package/test/src/storage.test.ts
CHANGED
|
@@ -10,7 +10,11 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
10
10
|
register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
|
|
11
11
|
|
|
12
12
|
describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
|
|
13
|
-
register.registerDataStorageDataTests({
|
|
13
|
+
register.registerDataStorageDataTests({
|
|
14
|
+
...POSTGRES_STORAGE_FACTORY,
|
|
15
|
+
storageVersion,
|
|
16
|
+
compressedBucketStorage: false
|
|
17
|
+
}));
|
|
14
18
|
|
|
15
19
|
describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
|
|
16
20
|
register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
|
|
@@ -39,7 +43,8 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
39
43
|
)
|
|
40
44
|
);
|
|
41
45
|
const bucketStorage = factory.getInstance(syncRules);
|
|
42
|
-
const
|
|
46
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
47
|
+
const globalBucket = bucketRequest(syncRulesContent, 'global[]');
|
|
43
48
|
|
|
44
49
|
const result = await (async () => {
|
|
45
50
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -17,6 +17,7 @@ bucket_definitions:
|
|
|
17
17
|
`)
|
|
18
18
|
);
|
|
19
19
|
const bucketStorage = factory.getInstance(syncRules);
|
|
20
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
20
21
|
|
|
21
22
|
const result = await (async () => {
|
|
22
23
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -44,12 +45,12 @@ bucket_definitions:
|
|
|
44
45
|
// Compact with an explicit bucket name — exercises the this.buckets
|
|
45
46
|
// iteration path, NOT the compactAllBuckets discovery path.
|
|
46
47
|
await bucketStorage.compact({
|
|
47
|
-
compactBuckets: [bucketRequest(
|
|
48
|
+
compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket],
|
|
48
49
|
minBucketChanges: 1
|
|
49
50
|
});
|
|
50
51
|
|
|
51
52
|
const batch = await test_utils.oneFromAsync(
|
|
52
|
-
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(
|
|
53
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
|
|
53
54
|
);
|
|
54
55
|
|
|
55
56
|
expect(batch.chunkData.data).toMatchObject([
|
|
@@ -70,7 +71,8 @@ bucket_definitions:
|
|
|
70
71
|
`)
|
|
71
72
|
);
|
|
72
73
|
const bucketStorage = factory.getInstance(syncRules);
|
|
73
|
-
const
|
|
74
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
75
|
+
const request = bucketRequest(syncRulesContent, 'global[]');
|
|
74
76
|
|
|
75
77
|
const result = await (async () => {
|
|
76
78
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
@@ -110,7 +112,7 @@ bucket_definitions:
|
|
|
110
112
|
const dataBefore = test_utils.getBatchData(rowsBefore);
|
|
111
113
|
const clearToOpId = BigInt(dataBefore[2].op_id);
|
|
112
114
|
|
|
113
|
-
const compactor = new PostgresCompactor(factory.db, bucketStorage.
|
|
115
|
+
const compactor = new PostgresCompactor(factory.db, bucketStorage.replicationStreamId, {});
|
|
114
116
|
// Trigger the private method directly
|
|
115
117
|
await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
|
|
116
118
|
/Unexpected PUT operation/
|
|
@@ -34,7 +34,8 @@ function registerStorageVersionTests(storageVersion: number) {
|
|
|
34
34
|
)
|
|
35
35
|
);
|
|
36
36
|
const bucketStorage = factory.getInstance(syncRules);
|
|
37
|
-
const
|
|
37
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
38
|
+
const globalBucket = bucketRequest(syncRulesContent, 'global[]');
|
|
38
39
|
|
|
39
40
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
40
41
|
|
package/test/tsconfig.json
CHANGED
package/tsconfig.json
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"extends": "../../tsconfig.base.json",
|
|
3
3
|
"compilerOptions": {
|
|
4
|
-
"outDir": "dist",
|
|
5
|
-
"declarationDir": "dist/@types",
|
|
6
|
-
"tsBuildInfoFile": "dist/.tsbuildinfo",
|
|
7
4
|
"rootDir": "src",
|
|
8
|
-
"
|
|
5
|
+
"outDir": "dist"
|
|
9
6
|
},
|
|
10
7
|
"include": ["src"],
|
|
11
8
|
"references": [
|