@powersync/service-module-postgres-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -8
- package/dist/migrations/scripts/1784900000000-source-metadata.d.ts +3 -0
- package/dist/migrations/scripts/1784900000000-source-metadata.js +22 -0
- package/dist/migrations/scripts/1784900000000-source-metadata.js.map +1 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.d.ts +3 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.js +12 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.js.map +1 -0
- package/dist/storage/PostgresBucketStorageFactory.d.ts +2 -1
- package/dist/storage/PostgresBucketStorageFactory.js +27 -14
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +3 -2
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.d.ts +4 -3
- package/dist/storage/PostgresSyncRulesStorage.js +17 -5
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.d.ts +2 -2
- package/dist/storage/batch/PostgresBucketBatch.js +63 -94
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +2 -2
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +3 -2
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +12 -5
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -1
- package/dist/types/models/ActiveCheckpoint.d.ts +2 -2
- package/dist/types/models/ActiveCheckpoint.js +3 -2
- package/dist/types/models/ActiveCheckpoint.js.map +1 -1
- package/dist/types/models/ActiveCheckpointNotification.d.ts +2 -2
- package/dist/types/models/BucketData.d.ts +6 -6
- package/dist/types/models/BucketData.js +7 -6
- package/dist/types/models/BucketData.js.map +1 -1
- package/dist/types/models/CurrentData.d.ts +1 -1
- package/dist/types/models/CurrentData.js +2 -1
- package/dist/types/models/CurrentData.js.map +1 -1
- package/dist/types/models/SdkReporting.d.ts +3 -3
- package/dist/types/models/SdkReporting.js +3 -5
- package/dist/types/models/SdkReporting.js.map +1 -1
- package/dist/types/models/SourceTable.d.ts +13 -8
- package/dist/types/models/SourceTable.js +11 -6
- package/dist/types/models/SourceTable.js.map +1 -1
- package/dist/types/models/SyncRules.d.ts +11 -10
- package/dist/types/models/SyncRules.js +12 -11
- package/dist/types/models/SyncRules.js.map +1 -1
- package/dist/types/models/WriteCheckpoint.d.ts +2 -2
- package/dist/types/models/WriteCheckpoint.js +3 -2
- package/dist/types/models/WriteCheckpoint.js.map +1 -1
- package/dist/types/types.d.ts +2 -0
- package/dist/utils/test-utils.d.ts +2 -1
- package/dist/utils/test-utils.js +3 -3
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +8 -8
- package/src/migrations/scripts/1784900000000-source-metadata.ts +31 -0
- package/src/migrations/scripts/1788000000000-sync-config-version-label.ts +18 -0
- package/src/storage/PostgresBucketStorageFactory.ts +30 -14
- package/src/storage/PostgresStorageProvider.ts +9 -5
- package/src/storage/PostgresSyncRulesStorage.ts +21 -6
- package/src/storage/batch/PostgresBucketBatch.ts +67 -98
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +2 -2
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +24 -11
- package/src/types/models/ActiveCheckpoint.ts +3 -2
- package/src/types/models/BucketData.ts +7 -6
- package/src/types/models/CurrentData.ts +2 -1
- package/src/types/models/SdkReporting.ts +4 -4
- package/src/types/models/SourceTable.ts +12 -6
- package/src/types/models/SyncRules.ts +12 -11
- package/src/types/models/WriteCheckpoint.ts +3 -2
- package/src/utils/test-utils.ts +23 -9
- package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -995
- package/test/src/checkpoint_notifications.test.ts +8 -2
- package/test/src/migrations.test.ts +8 -4
- package/test/src/storage.test.ts +12 -4
- package/test/src/storage_compacting.test.ts +2 -2
- package/test/src/storage_sync.test.ts +44 -0
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -15,6 +15,7 @@ export type PostgresBucketStorageOptions = {
|
|
|
15
15
|
config: NormalizedPostgresStorageConfig;
|
|
16
16
|
replicationStreamNamePrefix: string;
|
|
17
17
|
checksumCacheTtlMs?: number;
|
|
18
|
+
defaultStorageVersion?: number;
|
|
18
19
|
};
|
|
19
20
|
|
|
20
21
|
export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
@@ -24,8 +25,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
24
25
|
|
|
25
26
|
private activeStorageCache: storage.SyncRulesBucketStorage | undefined;
|
|
26
27
|
|
|
27
|
-
constructor(
|
|
28
|
-
|
|
28
|
+
constructor(
|
|
29
|
+
protected options: PostgresBucketStorageOptions,
|
|
30
|
+
syncConfigParser: storage.SyncConfigParser
|
|
31
|
+
) {
|
|
32
|
+
super({ syncConfigParser });
|
|
29
33
|
this.db = new lib_postgres.DatabaseClient({
|
|
30
34
|
config: options.config,
|
|
31
35
|
schema: STORAGE_SCHEMA_NAME,
|
|
@@ -40,7 +44,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
40
44
|
}
|
|
41
45
|
|
|
42
46
|
async [Symbol.asyncDispose]() {
|
|
43
|
-
|
|
47
|
+
try {
|
|
48
|
+
await super[Symbol.asyncDispose]();
|
|
49
|
+
} finally {
|
|
50
|
+
await this.db[Symbol.asyncDispose]();
|
|
51
|
+
}
|
|
44
52
|
}
|
|
45
53
|
|
|
46
54
|
async prepareStatements(connection: pg_wire.PgConnection) {
|
|
@@ -78,7 +86,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
78
86
|
return {
|
|
79
87
|
operations_size_bytes: 0,
|
|
80
88
|
parameters_size_bytes: 0,
|
|
81
|
-
replication_size_bytes: 0
|
|
89
|
+
replication_size_bytes: 0,
|
|
90
|
+
object_storage_size_bytes: 0
|
|
82
91
|
};
|
|
83
92
|
}
|
|
84
93
|
|
|
@@ -104,7 +113,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
104
113
|
return {
|
|
105
114
|
operations_size_bytes: Number(sizes!.operations_size_bytes),
|
|
106
115
|
parameters_size_bytes: Number(sizes!.parameter_size_bytes),
|
|
107
|
-
replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes)
|
|
116
|
+
replication_size_bytes: Number(sizes!.v1_current_size_bytes) + Number(sizes!.v3_current_size_bytes),
|
|
117
|
+
object_storage_size_bytes: 0
|
|
108
118
|
};
|
|
109
119
|
}
|
|
110
120
|
|
|
@@ -156,12 +166,15 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
156
166
|
|
|
157
167
|
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
|
|
158
168
|
const storageVersion =
|
|
159
|
-
options.storageVersion ??
|
|
169
|
+
options.storageVersion ??
|
|
170
|
+
options.config.parsed.config.storageVersion ??
|
|
171
|
+
this.options.defaultStorageVersion ??
|
|
172
|
+
storage.CURRENT_STORAGE_VERSION;
|
|
160
173
|
const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
|
|
161
|
-
if (storageConfig == null) {
|
|
174
|
+
if (storageConfig == null || storageVersion >= storage.STORAGE_VERSION_4) {
|
|
162
175
|
throw new framework.ServiceError(
|
|
163
176
|
framework.ErrorCode.PSYNC_S1005,
|
|
164
|
-
`Unsupported storage version ${storageVersion}`
|
|
177
|
+
`Unsupported storage version ${storageVersion} for PostgreSQL storage`
|
|
165
178
|
);
|
|
166
179
|
}
|
|
167
180
|
await this.initializeStorageVersion(storageConfig);
|
|
@@ -174,6 +187,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
174
187
|
state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
|
|
175
188
|
`.execute();
|
|
176
189
|
|
|
190
|
+
// Uses the legacy sequence name "sync_rules_id_sequence" for backwards-compatibility. See AGENTS.md.
|
|
177
191
|
const newSyncRulesRow = await db.sql`
|
|
178
192
|
WITH
|
|
179
193
|
next_id AS (
|
|
@@ -187,7 +201,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
187
201
|
sync_plan,
|
|
188
202
|
state,
|
|
189
203
|
slot_name,
|
|
190
|
-
storage_version
|
|
204
|
+
storage_version,
|
|
205
|
+
version_label
|
|
191
206
|
)
|
|
192
207
|
VALUES
|
|
193
208
|
(
|
|
@@ -211,7 +226,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
211
226
|
'_',
|
|
212
227
|
${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
|
|
213
228
|
),
|
|
214
|
-
${{ type: 'int4', value: storageVersion }}
|
|
229
|
+
${{ type: 'int4', value: storageVersion }},
|
|
230
|
+
${{ type: 'varchar', value: options.version_label ?? null }}
|
|
215
231
|
)
|
|
216
232
|
RETURNING
|
|
217
233
|
*
|
|
@@ -219,7 +235,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
219
235
|
.decoded(models.SyncRules)
|
|
220
236
|
.first();
|
|
221
237
|
|
|
222
|
-
return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow
|
|
238
|
+
return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!, this.syncConfigParser);
|
|
223
239
|
});
|
|
224
240
|
}
|
|
225
241
|
|
|
@@ -353,7 +369,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
353
369
|
.decoded(models.SyncRules)
|
|
354
370
|
.rows();
|
|
355
371
|
|
|
356
|
-
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
|
|
372
|
+
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser));
|
|
357
373
|
}
|
|
358
374
|
|
|
359
375
|
async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
|
|
@@ -368,14 +384,14 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
368
384
|
.decoded(models.SyncRules)
|
|
369
385
|
.rows();
|
|
370
386
|
|
|
371
|
-
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
|
|
387
|
+
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser));
|
|
372
388
|
}
|
|
373
389
|
|
|
374
390
|
private resolvedSyncConfigFromRow(
|
|
375
391
|
row: models.SyncRulesDecoded,
|
|
376
392
|
options: { cacheActiveStorage?: boolean } = {}
|
|
377
393
|
): storage.ResolvedSyncConfig {
|
|
378
|
-
const stream = new PostgresPersistedReplicationStream(this.db, row);
|
|
394
|
+
const stream = new PostgresPersistedReplicationStream(this.db, row, this.syncConfigParser);
|
|
379
395
|
const content = stream.syncConfigContent[0];
|
|
380
396
|
const thisFactory = this;
|
|
381
397
|
|
|
@@ -25,11 +25,15 @@ export class PostgresStorageProvider implements storage.StorageProvider {
|
|
|
25
25
|
|
|
26
26
|
const decodedConfig = PostgresStorageConfig.decode(storage);
|
|
27
27
|
const normalizedConfig = normalizePostgresStorageConfig(decodedConfig);
|
|
28
|
-
const storageFactory = new PostgresBucketStorageFactory(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
28
|
+
const storageFactory = new PostgresBucketStorageFactory(
|
|
29
|
+
{
|
|
30
|
+
config: normalizedConfig,
|
|
31
|
+
replicationStreamNamePrefix: options.resolvedConfig.slot_name_prefix,
|
|
32
|
+
checksumCacheTtlMs: options.resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
|
|
33
|
+
defaultStorageVersion: decodedConfig.default_storage_version
|
|
34
|
+
},
|
|
35
|
+
options.syncConfigParser
|
|
36
|
+
);
|
|
33
37
|
|
|
34
38
|
const reportStorageFactory = new PostgresReportStorage({
|
|
35
39
|
config: normalizedConfig
|
|
@@ -4,6 +4,8 @@ import {
|
|
|
4
4
|
BucketChecksum,
|
|
5
5
|
CHECKPOINT_INVALIDATE_ALL,
|
|
6
6
|
CheckpointChanges,
|
|
7
|
+
CompactInitialReplicationOptions,
|
|
8
|
+
CompactInitialReplicationResults,
|
|
7
9
|
GetCheckpointChangesOptions,
|
|
8
10
|
InternalOpId,
|
|
9
11
|
internalToExternalOpId,
|
|
@@ -11,11 +13,10 @@ import {
|
|
|
11
13
|
maxLsn,
|
|
12
14
|
ParameterSetLimitExceededError,
|
|
13
15
|
PartialChecksum,
|
|
14
|
-
PopulateChecksumCacheOptions,
|
|
15
|
-
PopulateChecksumCacheResults,
|
|
16
16
|
ReplicationCheckpoint,
|
|
17
17
|
storage,
|
|
18
18
|
StorageVersionConfig,
|
|
19
|
+
SyncRuleState,
|
|
19
20
|
utils,
|
|
20
21
|
WatchWriteCheckpointOptions
|
|
21
22
|
} from '@powersync/service-core';
|
|
@@ -94,7 +95,7 @@ export class PostgresSyncRulesStorage
|
|
|
94
95
|
this.factory = options.factory;
|
|
95
96
|
this.storageConfig = options.replicationStream.getStorageConfig();
|
|
96
97
|
this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
|
|
97
|
-
this.logger =
|
|
98
|
+
this.logger = this.sync_rules.logger;
|
|
98
99
|
|
|
99
100
|
this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
|
|
100
101
|
db: this.db,
|
|
@@ -145,7 +146,19 @@ export class PostgresSyncRulesStorage
|
|
|
145
146
|
`.execute();
|
|
146
147
|
}
|
|
147
148
|
|
|
149
|
+
supportsIncrementalParameterCompaction(): boolean {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
|
|
148
153
|
async compact(options?: storage.CompactOptions): Promise<void> {
|
|
154
|
+
if (options?.incrementalOnly) {
|
|
155
|
+
// Not supported yet
|
|
156
|
+
this.logger.info('Incremental compacting is not supported on Postgres storage yet.');
|
|
157
|
+
return;
|
|
158
|
+
} else if (this.replicationStream.state != SyncRuleState.ACTIVE) {
|
|
159
|
+
this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
149
162
|
let maxOpId = options?.maxOpId;
|
|
150
163
|
if (maxOpId == null) {
|
|
151
164
|
const checkpoint = await this.getCheckpoint();
|
|
@@ -160,7 +173,9 @@ export class PostgresSyncRulesStorage
|
|
|
160
173
|
}).compact();
|
|
161
174
|
}
|
|
162
175
|
|
|
163
|
-
async
|
|
176
|
+
async compactInitialReplication(
|
|
177
|
+
_options: CompactInitialReplicationOptions
|
|
178
|
+
): Promise<CompactInitialReplicationResults> {
|
|
164
179
|
// no-op - checksum cache is not implemented for Postgres yet
|
|
165
180
|
return { buckets: 0 };
|
|
166
181
|
}
|
|
@@ -172,8 +187,8 @@ export class PostgresSyncRulesStorage
|
|
|
172
187
|
});
|
|
173
188
|
}
|
|
174
189
|
|
|
175
|
-
setWriteCheckpointMode(
|
|
176
|
-
return this.writeCheckpointAPI.setWriteCheckpointMode(
|
|
190
|
+
setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
|
|
191
|
+
return this.writeCheckpointAPI.setWriteCheckpointMode(config);
|
|
177
192
|
}
|
|
178
193
|
|
|
179
194
|
createManagedWriteCheckpoints(
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
utils
|
|
20
20
|
} from '@powersync/service-core';
|
|
21
21
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
22
|
+
import { orNull } from '@powersync/service-types';
|
|
22
23
|
import * as timers from 'timers/promises';
|
|
23
24
|
import * as t from 'ts-codec';
|
|
24
25
|
import * as uuid from 'uuid';
|
|
@@ -64,10 +65,10 @@ const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(
|
|
|
64
65
|
const CheckpointWithStatus = StatefulCheckpoint.and(
|
|
65
66
|
t.object({
|
|
66
67
|
snapshot_done: t.boolean,
|
|
67
|
-
no_checkpoint_before: t.string
|
|
68
|
+
no_checkpoint_before: orNull(t.string),
|
|
68
69
|
can_checkpoint: t.boolean,
|
|
69
|
-
keepalive_op: bigint
|
|
70
|
-
new_last_checkpoint: bigint
|
|
70
|
+
keepalive_op: orNull(bigint),
|
|
71
|
+
new_last_checkpoint: orNull(bigint),
|
|
71
72
|
created_checkpoint: t.boolean
|
|
72
73
|
})
|
|
73
74
|
);
|
|
@@ -144,6 +145,7 @@ export class PostgresBucketBatch
|
|
|
144
145
|
|
|
145
146
|
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
146
147
|
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
148
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
147
149
|
const { connection_id, source } = options;
|
|
148
150
|
const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
149
151
|
|
|
@@ -154,9 +156,10 @@ export class PostgresBucketBatch
|
|
|
154
156
|
}));
|
|
155
157
|
|
|
156
158
|
return this.db.transaction(async (db) => {
|
|
157
|
-
|
|
159
|
+
// Find records that overlap by name or relation id.
|
|
160
|
+
let candidateRows: SourceTableDecoded[];
|
|
158
161
|
if (objectId != null) {
|
|
159
|
-
|
|
162
|
+
candidateRows = await db.sql`
|
|
160
163
|
SELECT
|
|
161
164
|
*
|
|
162
165
|
FROM
|
|
@@ -164,15 +167,18 @@ export class PostgresBucketBatch
|
|
|
164
167
|
WHERE
|
|
165
168
|
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
166
169
|
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
167
|
-
AND
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
170
|
+
AND (
|
|
171
|
+
relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
172
|
+
OR (
|
|
173
|
+
schema_name = ${{ type: 'varchar', value: schema }}
|
|
174
|
+
AND table_name = ${{ type: 'varchar', value: table }}
|
|
175
|
+
)
|
|
176
|
+
)
|
|
171
177
|
`
|
|
172
178
|
.decoded(models.SourceTable)
|
|
173
|
-
.
|
|
179
|
+
.rows();
|
|
174
180
|
} else {
|
|
175
|
-
|
|
181
|
+
candidateRows = await db.sql`
|
|
176
182
|
SELECT
|
|
177
183
|
*
|
|
178
184
|
FROM
|
|
@@ -182,15 +188,42 @@ export class PostgresBucketBatch
|
|
|
182
188
|
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
183
189
|
AND schema_name = ${{ type: 'varchar', value: schema }}
|
|
184
190
|
AND table_name = ${{ type: 'varchar', value: table }}
|
|
185
|
-
AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
186
191
|
`
|
|
187
192
|
.decoded(models.SourceTable)
|
|
188
|
-
.
|
|
193
|
+
.rows();
|
|
189
194
|
}
|
|
190
195
|
|
|
191
|
-
|
|
196
|
+
const candidateTables = candidateRows.map((row) => {
|
|
197
|
+
const table = this.sourceTableFromRow(row, connectionTag, syncRules);
|
|
198
|
+
table.storeCurrentData = sendsCompleteRows !== true;
|
|
199
|
+
return table;
|
|
200
|
+
});
|
|
201
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
202
|
+
const resolution = await reconcile({ source, candidates });
|
|
203
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
204
|
+
|
|
205
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
206
|
+
await db.sql`
|
|
207
|
+
UPDATE source_tables
|
|
208
|
+
SET
|
|
209
|
+
source_metadata = ${{ type: 'jsonb', value: sourceMetadata ?? null }}
|
|
210
|
+
WHERE
|
|
211
|
+
id = ${{ type: 'varchar', value: id.toString() }}
|
|
212
|
+
`.execute();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
216
|
+
const compatibleTables = materializedResolution.compatibleTables;
|
|
217
|
+
|
|
218
|
+
// Keep one record, preferring one that has already been snapshotted.
|
|
219
|
+
const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
|
|
220
|
+
|
|
221
|
+
let sourceTable: storage.SourceTable;
|
|
222
|
+
if (reuse != null) {
|
|
223
|
+
sourceTable = reuse;
|
|
224
|
+
} else {
|
|
192
225
|
const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
|
|
193
|
-
|
|
226
|
+
const insertedRow = await db.sql`
|
|
194
227
|
INSERT INTO
|
|
195
228
|
source_tables (
|
|
196
229
|
id,
|
|
@@ -199,7 +232,8 @@ export class PostgresBucketBatch
|
|
|
199
232
|
relation_id,
|
|
200
233
|
schema_name,
|
|
201
234
|
table_name,
|
|
202
|
-
replica_id_columns
|
|
235
|
+
replica_id_columns,
|
|
236
|
+
source_metadata
|
|
203
237
|
)
|
|
204
238
|
VALUES
|
|
205
239
|
(
|
|
@@ -209,95 +243,27 @@ export class PostgresBucketBatch
|
|
|
209
243
|
${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
|
|
210
244
|
${{ type: 'varchar', value: schema }},
|
|
211
245
|
${{ type: 'varchar', value: table }},
|
|
212
|
-
${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
|
|
246
|
+
${{ type: 'jsonb', value: normalizedReplicaIdColumns }},
|
|
247
|
+
${{ type: 'jsonb', value: resolution.newTableValues.sourceMetadata ?? null }}
|
|
213
248
|
)
|
|
214
249
|
RETURNING
|
|
215
250
|
*
|
|
216
251
|
`
|
|
217
252
|
.decoded(models.SourceTable)
|
|
218
253
|
.first();
|
|
254
|
+
sourceTable = this.sourceTableFromRow(insertedRow!, connectionTag, syncRules);
|
|
255
|
+
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
219
256
|
}
|
|
220
257
|
|
|
221
|
-
const
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
snapshotComplete: sourceTableRow!.snapshot_done ?? true,
|
|
227
|
-
...syncRules.getMatchingSources(source)
|
|
228
|
-
});
|
|
229
|
-
if (!sourceTable.snapshotComplete) {
|
|
230
|
-
sourceTable.snapshotStatus = {
|
|
231
|
-
totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
|
|
232
|
-
replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
|
|
233
|
-
lastKey: sourceTableRow!.snapshot_last_key
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
237
|
-
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
238
|
-
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
239
|
-
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
240
|
-
|
|
241
|
-
let truncatedTables: SourceTableDecoded[] = [];
|
|
242
|
-
if (objectId != null) {
|
|
243
|
-
truncatedTables = await db.sql`
|
|
244
|
-
SELECT
|
|
245
|
-
*
|
|
246
|
-
FROM
|
|
247
|
-
source_tables
|
|
248
|
-
WHERE
|
|
249
|
-
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
250
|
-
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
251
|
-
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
252
|
-
AND (
|
|
253
|
-
relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
|
|
254
|
-
OR (
|
|
255
|
-
schema_name = ${{ type: 'varchar', value: schema }}
|
|
256
|
-
AND table_name = ${{ type: 'varchar', value: table }}
|
|
257
|
-
)
|
|
258
|
-
)
|
|
259
|
-
`
|
|
260
|
-
.decoded(models.SourceTable)
|
|
261
|
-
.rows();
|
|
262
|
-
} else {
|
|
263
|
-
truncatedTables = await db.sql`
|
|
264
|
-
SELECT
|
|
265
|
-
*
|
|
266
|
-
FROM
|
|
267
|
-
source_tables
|
|
268
|
-
WHERE
|
|
269
|
-
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
270
|
-
AND connection_id = ${{ type: 'int4', value: connection_id }}
|
|
271
|
-
AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
|
|
272
|
-
AND (
|
|
273
|
-
schema_name = ${{ type: 'varchar', value: schema }}
|
|
274
|
-
AND table_name = ${{ type: 'varchar', value: table }}
|
|
275
|
-
)
|
|
276
|
-
`
|
|
277
|
-
.decoded(models.SourceTable)
|
|
278
|
-
.rows();
|
|
279
|
-
}
|
|
258
|
+
const dropTables = [
|
|
259
|
+
...materializedResolution.incompatibleTables,
|
|
260
|
+
// PostgreSQL storage only keeps one SourceTable per physical table.
|
|
261
|
+
...compatibleTables.filter((candidate) => !storage.sourceTableIdEquals(candidate.id, sourceTable.id))
|
|
262
|
+
];
|
|
280
263
|
|
|
281
264
|
return {
|
|
282
265
|
tables: [sourceTable],
|
|
283
|
-
dropTables
|
|
284
|
-
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
285
|
-
const dropTable = new storage.SourceTable({
|
|
286
|
-
id: doc.id,
|
|
287
|
-
ref,
|
|
288
|
-
objectId: doc.relation_id?.object_id ?? 0,
|
|
289
|
-
replicaIdColumns:
|
|
290
|
-
doc.replica_id_columns?.map(
|
|
291
|
-
(c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
|
|
292
|
-
) ?? [],
|
|
293
|
-
snapshotComplete: doc.snapshot_done ?? true,
|
|
294
|
-
...syncRules.getMatchingSources(ref)
|
|
295
|
-
});
|
|
296
|
-
dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
297
|
-
dropTable.syncData = dropTable.bucketDataSources.length > 0;
|
|
298
|
-
dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
|
|
299
|
-
return dropTable;
|
|
300
|
-
})
|
|
266
|
+
dropTables
|
|
301
267
|
};
|
|
302
268
|
});
|
|
303
269
|
}
|
|
@@ -334,9 +300,10 @@ export class PostgresBucketBatch
|
|
|
334
300
|
objectId: row.relation_id?.object_id,
|
|
335
301
|
replicaIdColumns:
|
|
336
302
|
row.replica_id_columns?.map(
|
|
337
|
-
(
|
|
303
|
+
(column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
|
|
338
304
|
) ?? [],
|
|
339
305
|
snapshotComplete: row.snapshot_done ?? true,
|
|
306
|
+
sourceMetadata: row.source_metadata,
|
|
340
307
|
...syncRules.getMatchingSources(ref)
|
|
341
308
|
});
|
|
342
309
|
|
|
@@ -357,7 +324,9 @@ export class PostgresBucketBatch
|
|
|
357
324
|
// TODO maybe share with abstract class
|
|
358
325
|
const { after, before, sourceTable, tag } = record;
|
|
359
326
|
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
360
|
-
//
|
|
327
|
+
// PostgreSQL storage resolves each physical table to one SourceTable. syncEvent records whether
|
|
328
|
+
// that table matches any event definition; each matching definition is evaluated once here and
|
|
329
|
+
// may produce multiple payloads from its matching queries.
|
|
361
330
|
if (sourceTable.syncEvent) {
|
|
362
331
|
for (const event of this.getTableEvents(sourceTable)) {
|
|
363
332
|
this.iterateListeners((cb) =>
|
|
@@ -1368,10 +1337,10 @@ export class PostgresBucketBatch
|
|
|
1368
1337
|
}
|
|
1369
1338
|
|
|
1370
1339
|
/**
|
|
1371
|
-
* Gets relevant {@link
|
|
1340
|
+
* Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
|
|
1372
1341
|
* TODO maybe share this with an abstract class
|
|
1373
1342
|
*/
|
|
1374
|
-
protected getTableEvents(table: storage.SourceTable): sync_rules.
|
|
1343
|
+
protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] {
|
|
1375
1344
|
return this.sync_rules.eventDescriptors.filter((evt) =>
|
|
1376
1345
|
[...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
|
|
1377
1346
|
);
|
|
@@ -21,8 +21,8 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
21
21
|
return this._mode;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
setWriteCheckpointMode(
|
|
25
|
-
this._mode = mode;
|
|
24
|
+
setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
|
|
25
|
+
this._mode = config.mode;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
async batchCreateCustomWriteCheckpoints(
|
|
@@ -6,16 +6,21 @@ import { models } from '../../types/types.js';
|
|
|
6
6
|
export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
|
|
7
7
|
constructor(
|
|
8
8
|
private db: lib_postgres.DatabaseClient,
|
|
9
|
-
row: models.SyncRulesDecoded
|
|
9
|
+
row: models.SyncRulesDecoded,
|
|
10
|
+
syncConfigParser: storage.SyncConfigParser
|
|
10
11
|
) {
|
|
11
|
-
super(
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
super(
|
|
13
|
+
{
|
|
14
|
+
replicationStreamId: Number(row.id),
|
|
15
|
+
sync_rules_content: row.content,
|
|
16
|
+
compiled_plan: row.sync_plan,
|
|
17
|
+
replicationStreamName: row.slot_name,
|
|
18
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
|
|
19
|
+
syncConfigState: row.state as storage.SyncRuleState,
|
|
20
|
+
version_label: row.version_label ?? undefined
|
|
21
|
+
},
|
|
22
|
+
syncConfigParser
|
|
23
|
+
);
|
|
19
24
|
}
|
|
20
25
|
|
|
21
26
|
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
@@ -53,7 +58,8 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
|
|
|
53
58
|
|
|
54
59
|
constructor(
|
|
55
60
|
private db: lib_postgres.DatabaseClient,
|
|
56
|
-
private readonly row: models.SyncRulesDecoded
|
|
61
|
+
private readonly row: models.SyncRulesDecoded,
|
|
62
|
+
private readonly syncConfigParser: storage.SyncConfigParser
|
|
57
63
|
) {
|
|
58
64
|
super({
|
|
59
65
|
replicationStreamId: Number(row.id),
|
|
@@ -61,7 +67,13 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
|
|
|
61
67
|
state: row.state as storage.SyncRuleState,
|
|
62
68
|
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
63
69
|
});
|
|
64
|
-
|
|
70
|
+
if (this.storageVersion >= storage.STORAGE_VERSION_4) {
|
|
71
|
+
throw new ServiceError(
|
|
72
|
+
ErrorCode.PSYNC_S1005,
|
|
73
|
+
`Unsupported storage version ${this.storageVersion} for PostgreSQL storage`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row, this.syncConfigParser)];
|
|
65
77
|
}
|
|
66
78
|
|
|
67
79
|
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
@@ -71,6 +83,7 @@ export class PostgresPersistedReplicationStream extends storage.PersistedReplica
|
|
|
71
83
|
async lock(): Promise<storage.ReplicationLock> {
|
|
72
84
|
const manager = new lib_postgres.PostgresLockManager({
|
|
73
85
|
db: this.db,
|
|
86
|
+
// Uses the legacy "sync_rules_" lock name prefix for backwards-compatibility. See AGENTS.md.
|
|
74
87
|
name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
|
|
75
88
|
});
|
|
76
89
|
const lockHandle = await manager.acquire();
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -7,8 +8,8 @@ import { bigint, pgwire_number } from '../codecs.js';
|
|
|
7
8
|
*/
|
|
8
9
|
export const ActiveCheckpoint = t.object({
|
|
9
10
|
id: pgwire_number,
|
|
10
|
-
last_checkpoint:
|
|
11
|
-
last_checkpoint_lsn:
|
|
11
|
+
last_checkpoint: orNull(bigint),
|
|
12
|
+
last_checkpoint_lsn: orNull(t.string)
|
|
12
13
|
});
|
|
13
14
|
|
|
14
15
|
export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -13,13 +14,13 @@ export const BucketData = t.object({
|
|
|
13
14
|
bucket_name: t.string,
|
|
14
15
|
op_id: bigint,
|
|
15
16
|
op: t.Enum(OpType),
|
|
16
|
-
source_table:
|
|
17
|
-
source_key:
|
|
18
|
-
table_name: t.string
|
|
19
|
-
row_id: t.string
|
|
17
|
+
source_table: orNull(t.string),
|
|
18
|
+
source_key: orNull(hexBuffer),
|
|
19
|
+
table_name: orNull(t.string),
|
|
20
|
+
row_id: orNull(t.string),
|
|
20
21
|
checksum: bigint,
|
|
21
|
-
data:
|
|
22
|
-
target_op:
|
|
22
|
+
data: orNull(t.string),
|
|
23
|
+
target_op: orNull(bigint)
|
|
23
24
|
});
|
|
24
25
|
|
|
25
26
|
export type BucketData = t.Encoded<typeof BucketData>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -26,7 +27,7 @@ export const V3CurrentData = t.object({
|
|
|
26
27
|
lookups: t.array(hexBuffer),
|
|
27
28
|
source_key: hexBuffer,
|
|
28
29
|
source_table: t.string,
|
|
29
|
-
pending_delete:
|
|
30
|
+
pending_delete: orNull(bigint)
|
|
30
31
|
});
|
|
31
32
|
|
|
32
33
|
export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, jsonb } from '../codecs.js';
|
|
3
4
|
|
|
@@ -11,12 +12,11 @@ export type Sdks = t.Encoded<typeof Sdks>;
|
|
|
11
12
|
|
|
12
13
|
export const SdkReporting = t.object({
|
|
13
14
|
users: bigint,
|
|
14
|
-
sdks:
|
|
15
|
-
.object({
|
|
15
|
+
sdks: orNull(
|
|
16
|
+
t.object({
|
|
16
17
|
data: jsonb<Sdks[]>(t.array(Sdks))
|
|
17
18
|
})
|
|
18
|
-
|
|
19
|
-
.or(t.Null)
|
|
19
|
+
).optional()
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
export type SdkReporting = t.Encoded<typeof SdkReporting>;
|