@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +179 -6
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
- package/dist/migrations/scripts/1784900000000-source-metadata.d.ts +3 -0
- package/dist/migrations/scripts/1784900000000-source-metadata.js +22 -0
- package/dist/migrations/scripts/1784900000000-source-metadata.js.map +1 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.d.ts +3 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.js +12 -0
- package/dist/migrations/scripts/1788000000000-sync-config-version-label.js.map +1 -0
- package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +14 -12
- package/dist/storage/PostgresBucketStorageFactory.js +73 -62
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +3 -1
- package/dist/storage/PostgresCompactor.js +21 -1
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +3 -1
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +39 -13
- package/dist/storage/PostgresSyncRulesStorage.js +266 -181
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +10 -26
- package/dist/storage/batch/PostgresBucketBatch.js +290 -286
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +2 -2
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +163 -26
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/{@types/storage → storage}/current-data-store.d.ts +8 -2
- package/dist/storage/current-data-store.js +66 -11
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +94 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
- package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +2 -2
- package/dist/types/models/ActiveCheckpoint.js +3 -2
- package/dist/types/models/ActiveCheckpoint.js.map +1 -1
- package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +2 -2
- package/dist/types/models/BucketData.d.ts +22 -0
- package/dist/types/models/BucketData.js +7 -6
- package/dist/types/models/BucketData.js.map +1 -1
- package/dist/{@types/types → types}/models/CurrentData.d.ts +1 -1
- package/dist/types/models/CurrentData.js +2 -1
- package/dist/types/models/CurrentData.js.map +1 -1
- package/dist/{@types/types → types}/models/SdkReporting.d.ts +3 -3
- package/dist/types/models/SdkReporting.js +3 -5
- package/dist/types/models/SdkReporting.js.map +1 -1
- package/dist/{@types/types → types}/models/SourceTable.d.ts +13 -8
- package/dist/types/models/SourceTable.js +11 -6
- package/dist/types/models/SourceTable.js.map +1 -1
- package/dist/{@types/types → types}/models/SyncRules.d.ts +11 -10
- package/dist/types/models/SyncRules.js +12 -11
- package/dist/types/models/SyncRules.js.map +1 -1
- package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +2 -0
- package/dist/types/models/WriteCheckpoint.js +6 -2
- package/dist/types/models/WriteCheckpoint.js.map +1 -1
- package/dist/{@types/types → types}/types.d.ts +2 -0
- package/dist/utils/checkpoints.d.ts +9 -0
- package/dist/utils/checkpoints.js +26 -0
- package/dist/utils/checkpoints.js.map +1 -0
- package/dist/utils/test-utils.js +2 -2
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -22
- package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
- package/src/migrations/scripts/1784900000000-source-metadata.ts +31 -0
- package/src/migrations/scripts/1788000000000-sync-config-version-label.ts +18 -0
- package/src/storage/PostgresBucketStorageFactory.ts +82 -65
- package/src/storage/PostgresCompactor.ts +25 -1
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresStorageProvider.ts +3 -1
- package/src/storage/PostgresSyncRulesStorage.ts +295 -196
- package/src/storage/batch/PostgresBucketBatch.ts +321 -310
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +176 -26
- package/src/storage/current-data-store.ts +66 -11
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +105 -0
- package/src/types/models/ActiveCheckpoint.ts +3 -2
- package/src/types/models/BucketData.ts +7 -6
- package/src/types/models/CurrentData.ts +2 -1
- package/src/types/models/SdkReporting.ts +4 -4
- package/src/types/models/SourceTable.ts +12 -6
- package/src/types/models/SyncRules.ts +12 -11
- package/src/types/models/WriteCheckpoint.ts +6 -2
- package/src/utils/checkpoints.ts +31 -0
- package/src/utils/test-utils.ts +2 -2
- package/test/src/__snapshots__/storage_sync.test.ts.snap +44 -1621
- package/test/src/checkpoint_notifications.test.ts +528 -0
- package/test/src/migrations.test.ts +1 -1
- package/test/src/storage.test.ts +235 -11
- package/test/src/storage_compacting.test.ts +17 -9
- package/test/src/storage_sync.test.ts +55 -8
- package/test/src/util.ts +1 -1
- package/test/tsconfig.json +1 -3
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -0
- package/dist/.tsbuildinfo +0 -1
- package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
- package/dist/@types/types/models/BucketData.d.ts +0 -22
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
- /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
- /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
- /package/dist/{@types/types → types}/codecs.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
- /package/dist/{@types/types → types}/models/json.d.ts +0 -0
- /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
- /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
- /package/dist/{@types/utils → utils}/db.d.ts +0 -0
- /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
- /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
- /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
|
@@ -21,8 +21,8 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
21
21
|
return this._mode;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
setWriteCheckpointMode(
|
|
25
|
-
this._mode = mode;
|
|
24
|
+
setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
|
|
25
|
+
this._mode = config.mode;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
async batchCreateCustomWriteCheckpoints(
|
|
@@ -32,32 +32,172 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
32
32
|
return batchCreateCustomWriteCheckpoints(this.db, checkpoints);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
async
|
|
35
|
+
async createManagedWriteCheckpoints(
|
|
36
|
+
checkpoints: storage.ManagedWriteCheckpointOptions[]
|
|
37
|
+
): Promise<storage.CreateManagedWriteCheckpointsResult> {
|
|
36
38
|
if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) {
|
|
37
39
|
throw new framework.errors.ValidationError(
|
|
38
40
|
`Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"`
|
|
39
41
|
);
|
|
40
42
|
}
|
|
41
43
|
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
lsns
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
44
|
+
const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints);
|
|
45
|
+
if (uniqueCheckpoints.length == 0) {
|
|
46
|
+
return { writeCheckpoints: new Map(), shouldAdvance: false };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const writeCheckpoints = new Map<string, bigint>();
|
|
50
|
+
const generatedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id == null);
|
|
51
|
+
const suppliedCheckpoints = uniqueCheckpoints.filter((checkpoint) => checkpoint.checkpoint_request_id != null);
|
|
52
|
+
|
|
53
|
+
if (generatedCheckpoints.length > 0) {
|
|
54
|
+
const mappedCheckpoints = generatedCheckpoints.map((checkpoint) => ({
|
|
55
|
+
user_id: checkpoint.user_id,
|
|
56
|
+
lsns: checkpoint.heads
|
|
57
|
+
}));
|
|
58
|
+
|
|
59
|
+
const generatedRows = await this.db.sql`
|
|
60
|
+
WITH
|
|
61
|
+
json_data AS (
|
|
62
|
+
SELECT
|
|
63
|
+
CHECKPOINT ->> 'user_id' AS user_id,
|
|
64
|
+
CHECKPOINT -> 'lsns' AS lsns
|
|
65
|
+
FROM
|
|
66
|
+
jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
|
|
67
|
+
CHECKPOINT
|
|
68
|
+
)
|
|
69
|
+
INSERT INTO
|
|
70
|
+
write_checkpoints (
|
|
71
|
+
user_id,
|
|
72
|
+
lsns,
|
|
73
|
+
write_checkpoint,
|
|
74
|
+
checkpoint_requested_at
|
|
75
|
+
)
|
|
76
|
+
SELECT
|
|
77
|
+
user_id,
|
|
78
|
+
lsns,
|
|
79
|
+
1,
|
|
80
|
+
NULL
|
|
81
|
+
FROM
|
|
82
|
+
json_data
|
|
83
|
+
ON CONFLICT (user_id) DO UPDATE
|
|
84
|
+
SET
|
|
85
|
+
write_checkpoint = write_checkpoints.write_checkpoint + 1,
|
|
86
|
+
lsns = EXCLUDED.lsns,
|
|
87
|
+
checkpoint_requested_at = NULL
|
|
88
|
+
RETURNING
|
|
89
|
+
*;
|
|
90
|
+
`
|
|
91
|
+
.decoded(models.WriteCheckpoint)
|
|
92
|
+
.rows();
|
|
93
|
+
|
|
94
|
+
for (const row of generatedRows) {
|
|
95
|
+
writeCheckpoints.set(row.user_id, row.write_checkpoint);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (suppliedCheckpoints.length > 0) {
|
|
100
|
+
// Supplied request ids are monotonic. Greater values update the checkpoint
|
|
101
|
+
// id and heads, while equal retries only refresh the retention timestamp.
|
|
102
|
+
// Stale requests return the stored id without changing the row.
|
|
103
|
+
const mappedCheckpoints = suppliedCheckpoints.map((checkpoint) => ({
|
|
104
|
+
user_id: checkpoint.user_id,
|
|
105
|
+
lsns: checkpoint.heads,
|
|
106
|
+
checkpoint_request_id: String(checkpoint.checkpoint_request_id)
|
|
107
|
+
}));
|
|
108
|
+
|
|
109
|
+
const suppliedRows = await this.db.sql`
|
|
110
|
+
WITH
|
|
111
|
+
json_data AS (
|
|
112
|
+
SELECT
|
|
113
|
+
CHECKPOINT ->> 'user_id' AS user_id,
|
|
114
|
+
CHECKPOINT -> 'lsns' AS lsns,
|
|
115
|
+
(
|
|
116
|
+
CHECKPOINT ->> 'checkpoint_request_id'
|
|
117
|
+
)::int8 AS checkpoint_request_id
|
|
118
|
+
FROM
|
|
119
|
+
jsonb_array_elements(${{ type: 'jsonb', value: mappedCheckpoints }}) AS
|
|
120
|
+
CHECKPOINT
|
|
121
|
+
)
|
|
122
|
+
INSERT INTO
|
|
123
|
+
write_checkpoints (
|
|
124
|
+
user_id,
|
|
125
|
+
lsns,
|
|
126
|
+
write_checkpoint,
|
|
127
|
+
checkpoint_requested_at
|
|
128
|
+
)
|
|
129
|
+
SELECT
|
|
130
|
+
user_id,
|
|
131
|
+
lsns,
|
|
132
|
+
checkpoint_request_id,
|
|
133
|
+
NOW()
|
|
134
|
+
FROM
|
|
135
|
+
json_data
|
|
136
|
+
ON CONFLICT (user_id) DO UPDATE
|
|
137
|
+
SET
|
|
138
|
+
write_checkpoint = CASE
|
|
139
|
+
WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.write_checkpoint
|
|
140
|
+
ELSE write_checkpoints.write_checkpoint
|
|
141
|
+
END,
|
|
142
|
+
lsns = CASE
|
|
143
|
+
WHEN EXCLUDED.write_checkpoint > write_checkpoints.write_checkpoint THEN EXCLUDED.lsns
|
|
144
|
+
ELSE write_checkpoints.lsns
|
|
145
|
+
END,
|
|
146
|
+
checkpoint_requested_at = NOW()
|
|
147
|
+
WHERE
|
|
148
|
+
EXCLUDED.write_checkpoint >= write_checkpoints.write_checkpoint
|
|
149
|
+
RETURNING
|
|
150
|
+
*;
|
|
151
|
+
`
|
|
152
|
+
.decoded(models.WriteCheckpoint)
|
|
153
|
+
.rows();
|
|
154
|
+
|
|
155
|
+
for (const row of suppliedRows) {
|
|
156
|
+
writeCheckpoints.set(row.user_id, row.write_checkpoint);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// RETURNING only excludes stale requests rejected by the monotonic
|
|
160
|
+
// conflict condition. Those requests still need the stored id.
|
|
161
|
+
const returnedSuppliedUserIds = new Set(suppliedRows.map((row) => row.user_id));
|
|
162
|
+
const unchangedUserIds = suppliedCheckpoints
|
|
163
|
+
.map((checkpoint) => checkpoint.user_id)
|
|
164
|
+
.filter((userId) => !returnedSuppliedUserIds.has(userId));
|
|
165
|
+
|
|
166
|
+
if (unchangedUserIds.length > 0) {
|
|
167
|
+
const mappedUserIds = unchangedUserIds.map((user_id) => ({ user_id }));
|
|
168
|
+
const unchangedRows = await this.db.sql`
|
|
169
|
+
WITH
|
|
170
|
+
json_data AS (
|
|
171
|
+
SELECT
|
|
172
|
+
CHECKPOINT ->> 'user_id' AS user_id
|
|
173
|
+
FROM
|
|
174
|
+
jsonb_array_elements(${{ type: 'jsonb', value: mappedUserIds }}) AS
|
|
175
|
+
CHECKPOINT
|
|
176
|
+
)
|
|
177
|
+
SELECT
|
|
178
|
+
write_checkpoints.*
|
|
179
|
+
FROM
|
|
180
|
+
write_checkpoints
|
|
181
|
+
JOIN json_data ON write_checkpoints.user_id = json_data.user_id;
|
|
182
|
+
`
|
|
183
|
+
.decoded(models.WriteCheckpoint)
|
|
184
|
+
.rows();
|
|
185
|
+
|
|
186
|
+
for (const row of unchangedRows) {
|
|
187
|
+
writeCheckpoints.set(row.user_id, row.write_checkpoint);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Postgres storage does not track a per-row processed indicator: a write
|
|
193
|
+
// checkpoint is considered processed at read time by comparing its lsns
|
|
194
|
+
// against the replicated head (see lastManagedWriteCheckpoint). We therefore
|
|
195
|
+
// force the source marker whenever any checkpoint was matched, which also
|
|
196
|
+
// covers stale or duplicate requests whose stored checkpoint may still be
|
|
197
|
+
// pending. Forcing a marker for an already-processed checkpoint is wasteful
|
|
198
|
+
// but harmless.
|
|
199
|
+
const shouldAdvance = generatedCheckpoints.length > 0 || suppliedCheckpoints.length > 0;
|
|
200
|
+
return { writeCheckpoints, shouldAdvance };
|
|
61
201
|
}
|
|
62
202
|
|
|
63
203
|
async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise<bigint | null> {
|
|
@@ -133,7 +273,8 @@ export async function batchCreateCustomWriteCheckpoints(
|
|
|
133
273
|
// Cannot encode bigint directly using JSON.stringify.
|
|
134
274
|
// The ::int8 in the query below will take care of casting back to a number
|
|
135
275
|
checkpoint: String(cp.checkpoint),
|
|
136
|
-
sync_rules_id: cp.sync_rules_id
|
|
276
|
+
sync_rules_id: cp.sync_rules_id,
|
|
277
|
+
checkpoint_requested_at: cp.checkpoint_requested_at?.toISOString() ?? null
|
|
137
278
|
};
|
|
138
279
|
});
|
|
139
280
|
|
|
@@ -145,7 +286,12 @@ export async function batchCreateCustomWriteCheckpoints(
|
|
|
145
286
|
CHECKPOINT
|
|
146
287
|
)
|
|
147
288
|
INSERT INTO
|
|
148
|
-
custom_write_checkpoints (
|
|
289
|
+
custom_write_checkpoints (
|
|
290
|
+
user_id,
|
|
291
|
+
write_checkpoint,
|
|
292
|
+
sync_rules_id,
|
|
293
|
+
checkpoint_requested_at
|
|
294
|
+
)
|
|
149
295
|
SELECT
|
|
150
296
|
CHECKPOINT ->> 'user_id'::varchar,
|
|
151
297
|
(
|
|
@@ -153,11 +299,15 @@ export async function batchCreateCustomWriteCheckpoints(
|
|
|
153
299
|
)::int8,
|
|
154
300
|
(
|
|
155
301
|
CHECKPOINT ->> 'sync_rules_id'
|
|
156
|
-
)::int4
|
|
302
|
+
)::int4,
|
|
303
|
+
(
|
|
304
|
+
CHECKPOINT ->> 'checkpoint_requested_at'
|
|
305
|
+
)::timestamptz
|
|
157
306
|
FROM
|
|
158
307
|
json_data
|
|
159
308
|
ON CONFLICT (user_id, sync_rules_id) DO UPDATE
|
|
160
309
|
SET
|
|
161
|
-
write_checkpoint = EXCLUDED.write_checkpoint
|
|
310
|
+
write_checkpoint = EXCLUDED.write_checkpoint,
|
|
311
|
+
checkpoint_requested_at = EXCLUDED.checkpoint_requested_at;
|
|
162
312
|
`.execute();
|
|
163
313
|
}
|
|
@@ -301,19 +301,74 @@ export class PostgresCurrentDataStore {
|
|
|
301
301
|
`.execute();
|
|
302
302
|
}
|
|
303
303
|
|
|
304
|
-
|
|
304
|
+
/**
|
|
305
|
+
* Delete up to `limit` rows for the group, returning the number of candidate
|
|
306
|
+
* rows found by the scan (see PostgresSyncRulesStorage#deleteGroupBatch for
|
|
307
|
+
* why the scan is counted rather than the delete).
|
|
308
|
+
*/
|
|
309
|
+
async deleteGroupRowsBatch(db: Queryable, options: { groupId: number; limit: number }): Promise<bigint> {
|
|
305
310
|
if (this.softDeleteEnabled) {
|
|
306
|
-
await db.sql`
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
+
const result = await db.sql`
|
|
312
|
+
WITH
|
|
313
|
+
batch AS (
|
|
314
|
+
SELECT
|
|
315
|
+
ctid
|
|
316
|
+
FROM
|
|
317
|
+
v3_current_data
|
|
318
|
+
WHERE
|
|
319
|
+
group_id = ${{ type: 'int4', value: options.groupId }}
|
|
320
|
+
LIMIT
|
|
321
|
+
${{ type: 'int4', value: options.limit }}
|
|
322
|
+
),
|
|
323
|
+
deleted AS (
|
|
324
|
+
DELETE FROM v3_current_data
|
|
325
|
+
WHERE
|
|
326
|
+
ctid IN (
|
|
327
|
+
SELECT
|
|
328
|
+
ctid
|
|
329
|
+
FROM
|
|
330
|
+
batch
|
|
331
|
+
)
|
|
332
|
+
RETURNING
|
|
333
|
+
1
|
|
334
|
+
)
|
|
335
|
+
SELECT
|
|
336
|
+
COUNT(*) AS count
|
|
337
|
+
FROM
|
|
338
|
+
batch
|
|
339
|
+
`.first<{ count: bigint }>();
|
|
340
|
+
return result?.count ?? 0n;
|
|
311
341
|
} else {
|
|
312
|
-
await db.sql`
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
342
|
+
const result = await db.sql`
|
|
343
|
+
WITH
|
|
344
|
+
batch AS (
|
|
345
|
+
SELECT
|
|
346
|
+
ctid
|
|
347
|
+
FROM
|
|
348
|
+
current_data
|
|
349
|
+
WHERE
|
|
350
|
+
group_id = ${{ type: 'int4', value: options.groupId }}
|
|
351
|
+
LIMIT
|
|
352
|
+
${{ type: 'int4', value: options.limit }}
|
|
353
|
+
),
|
|
354
|
+
deleted AS (
|
|
355
|
+
DELETE FROM current_data
|
|
356
|
+
WHERE
|
|
357
|
+
ctid IN (
|
|
358
|
+
SELECT
|
|
359
|
+
ctid
|
|
360
|
+
FROM
|
|
361
|
+
batch
|
|
362
|
+
)
|
|
363
|
+
RETURNING
|
|
364
|
+
1
|
|
365
|
+
)
|
|
366
|
+
SELECT
|
|
367
|
+
COUNT(*) AS count
|
|
368
|
+
FROM
|
|
369
|
+
batch
|
|
370
|
+
`.first<{ count: bigint }>();
|
|
371
|
+
return result?.count ?? 0n;
|
|
317
372
|
}
|
|
318
373
|
}
|
|
319
374
|
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { models } from '../../types/types.js';
|
|
5
|
+
|
|
6
|
+
export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
|
|
7
|
+
constructor(
|
|
8
|
+
private db: lib_postgres.DatabaseClient,
|
|
9
|
+
row: models.SyncRulesDecoded
|
|
10
|
+
) {
|
|
11
|
+
super({
|
|
12
|
+
replicationStreamId: Number(row.id),
|
|
13
|
+
sync_rules_content: row.content,
|
|
14
|
+
compiled_plan: row.sync_plan,
|
|
15
|
+
replicationStreamName: row.slot_name,
|
|
16
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
|
|
17
|
+
syncConfigState: row.state as storage.SyncRuleState,
|
|
18
|
+
version_label: row.version_label ?? undefined
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
23
|
+
const row = await this.db.sql`
|
|
24
|
+
SELECT
|
|
25
|
+
*
|
|
26
|
+
FROM
|
|
27
|
+
sync_rules
|
|
28
|
+
WHERE
|
|
29
|
+
id = ${{ value: this.replicationStreamId, type: 'int4' }}
|
|
30
|
+
`
|
|
31
|
+
.decoded(models.SyncRules)
|
|
32
|
+
.first();
|
|
33
|
+
|
|
34
|
+
return row == null ? null : syncConfigStatusFromRow(row);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function syncConfigStatusFromRow(row: models.SyncRulesDecoded): storage.PersistedSyncConfigStatus {
|
|
39
|
+
return {
|
|
40
|
+
id: String(row.id),
|
|
41
|
+
replicationStreamId: Number(row.id),
|
|
42
|
+
state: row.state as storage.SyncRuleState,
|
|
43
|
+
snapshot_done: row.snapshot_done,
|
|
44
|
+
last_checkpoint_lsn: row.last_checkpoint_lsn,
|
|
45
|
+
last_fatal_error: row.last_fatal_error,
|
|
46
|
+
last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
|
|
47
|
+
last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
52
|
+
current_lock: storage.ReplicationLock | null = null;
|
|
53
|
+
readonly syncConfigContent: readonly PostgresPersistedSyncConfigContent[];
|
|
54
|
+
|
|
55
|
+
constructor(
|
|
56
|
+
private db: lib_postgres.DatabaseClient,
|
|
57
|
+
private readonly row: models.SyncRulesDecoded
|
|
58
|
+
) {
|
|
59
|
+
super({
|
|
60
|
+
replicationStreamId: Number(row.id),
|
|
61
|
+
replicationStreamName: row.slot_name,
|
|
62
|
+
state: row.state as storage.SyncRuleState,
|
|
63
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
64
|
+
});
|
|
65
|
+
if (this.storageVersion >= storage.STORAGE_VERSION_4) {
|
|
66
|
+
throw new ServiceError(
|
|
67
|
+
ErrorCode.PSYNC_S1005,
|
|
68
|
+
`Unsupported storage version ${this.storageVersion} for PostgreSQL storage`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet {
|
|
75
|
+
return this.syncConfigContent[0].parsed(options);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async lock(): Promise<storage.ReplicationLock> {
|
|
79
|
+
const manager = new lib_postgres.PostgresLockManager({
|
|
80
|
+
db: this.db,
|
|
81
|
+
name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
|
|
82
|
+
});
|
|
83
|
+
const lockHandle = await manager.acquire();
|
|
84
|
+
if (!lockHandle) {
|
|
85
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const interval = setInterval(async () => {
|
|
89
|
+
try {
|
|
90
|
+
await lockHandle.refresh();
|
|
91
|
+
} catch (e) {
|
|
92
|
+
this.logger.error('Failed to refresh lock', e);
|
|
93
|
+
clearInterval(interval);
|
|
94
|
+
}
|
|
95
|
+
}, 30_130);
|
|
96
|
+
|
|
97
|
+
return (this.current_lock = {
|
|
98
|
+
sync_rules_id: this.replicationStreamId,
|
|
99
|
+
release: async () => {
|
|
100
|
+
clearInterval(interval);
|
|
101
|
+
return lockHandle.release();
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -7,8 +8,8 @@ import { bigint, pgwire_number } from '../codecs.js';
|
|
|
7
8
|
*/
|
|
8
9
|
export const ActiveCheckpoint = t.object({
|
|
9
10
|
id: pgwire_number,
|
|
10
|
-
last_checkpoint:
|
|
11
|
-
last_checkpoint_lsn:
|
|
11
|
+
last_checkpoint: orNull(bigint),
|
|
12
|
+
last_checkpoint_lsn: orNull(t.string)
|
|
12
13
|
});
|
|
13
14
|
|
|
14
15
|
export type ActiveCheckpoint = t.Encoded<typeof ActiveCheckpoint>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, hexBuffer, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -13,13 +14,13 @@ export const BucketData = t.object({
|
|
|
13
14
|
bucket_name: t.string,
|
|
14
15
|
op_id: bigint,
|
|
15
16
|
op: t.Enum(OpType),
|
|
16
|
-
source_table:
|
|
17
|
-
source_key:
|
|
18
|
-
table_name: t.string
|
|
19
|
-
row_id: t.string
|
|
17
|
+
source_table: orNull(t.string),
|
|
18
|
+
source_key: orNull(hexBuffer),
|
|
19
|
+
table_name: orNull(t.string),
|
|
20
|
+
row_id: orNull(t.string),
|
|
20
21
|
checksum: bigint,
|
|
21
|
-
data:
|
|
22
|
-
target_op:
|
|
22
|
+
data: orNull(t.string),
|
|
23
|
+
target_op: orNull(bigint)
|
|
23
24
|
});
|
|
24
25
|
|
|
25
26
|
export type BucketData = t.Encoded<typeof BucketData>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
|
|
3
4
|
|
|
@@ -26,7 +27,7 @@ export const V3CurrentData = t.object({
|
|
|
26
27
|
lookups: t.array(hexBuffer),
|
|
27
28
|
source_key: hexBuffer,
|
|
28
29
|
source_table: t.string,
|
|
29
|
-
pending_delete:
|
|
30
|
+
pending_delete: orNull(bigint)
|
|
30
31
|
});
|
|
31
32
|
|
|
32
33
|
export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { orNull } from '@powersync/service-types';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, jsonb } from '../codecs.js';
|
|
3
4
|
|
|
@@ -11,12 +12,11 @@ export type Sdks = t.Encoded<typeof Sdks>;
|
|
|
11
12
|
|
|
12
13
|
export const SdkReporting = t.object({
|
|
13
14
|
users: bigint,
|
|
14
|
-
sdks:
|
|
15
|
-
.object({
|
|
15
|
+
sdks: orNull(
|
|
16
|
+
t.object({
|
|
16
17
|
data: jsonb<Sdks[]>(t.array(Sdks))
|
|
17
18
|
})
|
|
18
|
-
|
|
19
|
-
.or(t.Null)
|
|
19
|
+
).optional()
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
export type SdkReporting = t.Encoded<typeof SdkReporting>;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { JsonValue } from '@powersync/service-core';
|
|
2
|
+
import { orNull } from '@powersync/service-types';
|
|
1
3
|
import * as t from 'ts-codec';
|
|
2
4
|
import { bigint, hexBuffer, jsonb, jsonb_raw, pgwire_number } from '../codecs.js';
|
|
3
5
|
|
|
@@ -14,21 +16,25 @@ export const ColumnDescriptor = t.object({
|
|
|
14
16
|
/**
|
|
15
17
|
* Some data sources have a type id that can be used to identify the type of the column
|
|
16
18
|
*/
|
|
17
|
-
|
|
19
|
+
type_oid: t.number.optional()
|
|
18
20
|
});
|
|
19
21
|
|
|
20
22
|
export const SourceTable = t.object({
|
|
21
23
|
id: t.string,
|
|
22
24
|
group_id: pgwire_number,
|
|
23
25
|
connection_id: bigint,
|
|
24
|
-
relation_id:
|
|
26
|
+
relation_id: orNull(jsonb_raw<StoredRelationId>()),
|
|
25
27
|
schema_name: t.string,
|
|
26
28
|
table_name: t.string,
|
|
27
|
-
replica_id_columns:
|
|
29
|
+
replica_id_columns: orNull(jsonb(t.array(ColumnDescriptor))),
|
|
28
30
|
snapshot_done: t.boolean,
|
|
29
|
-
snapshot_total_estimated_count:
|
|
30
|
-
snapshot_replicated_count:
|
|
31
|
-
snapshot_last_key:
|
|
31
|
+
snapshot_total_estimated_count: orNull(bigint),
|
|
32
|
+
snapshot_replicated_count: orNull(bigint),
|
|
33
|
+
snapshot_last_key: orNull(hexBuffer),
|
|
34
|
+
/**
|
|
35
|
+
* Source-specific metadata. Null for legacy records.
|
|
36
|
+
*/
|
|
37
|
+
source_metadata: orNull(jsonb_raw<JsonValue>())
|
|
32
38
|
});
|
|
33
39
|
|
|
34
40
|
export type SourceTable = t.Encoded<typeof SourceTable>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { framework, storage } from '@powersync/service-core';
|
|
2
|
-
import { ReplicationError } from '@powersync/service-types';
|
|
2
|
+
import { orNull, ReplicationError } from '@powersync/service-types';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
4
|
import { bigint, pgwire_number } from '../codecs.js';
|
|
5
5
|
import { jsonContainerObject } from './json.js';
|
|
@@ -16,42 +16,43 @@ export const SyncRules = t.object({
|
|
|
16
16
|
/**
|
|
17
17
|
* May be set if snapshot_done = false, if the replication stream requires it.
|
|
18
18
|
*/
|
|
19
|
-
snapshot_lsn:
|
|
19
|
+
snapshot_lsn: orNull(t.string),
|
|
20
20
|
/**
|
|
21
21
|
* The last consistent checkpoint.
|
|
22
22
|
*
|
|
23
23
|
* There may be higher OpIds used in the database if we're in the middle of replicating a large transaction.
|
|
24
24
|
*/
|
|
25
|
-
last_checkpoint:
|
|
25
|
+
last_checkpoint: orNull(bigint),
|
|
26
26
|
/**
|
|
27
27
|
* The LSN associated with the last consistent checkpoint.
|
|
28
28
|
*/
|
|
29
|
-
last_checkpoint_lsn:
|
|
29
|
+
last_checkpoint_lsn: orNull(t.string),
|
|
30
30
|
/**
|
|
31
31
|
* If set, no new checkpoints may be created < this value.
|
|
32
32
|
*/
|
|
33
|
-
no_checkpoint_before:
|
|
33
|
+
no_checkpoint_before: orNull(t.string),
|
|
34
34
|
slot_name: t.string,
|
|
35
35
|
/**
|
|
36
36
|
* Last time we persisted a checkpoint.
|
|
37
37
|
*
|
|
38
38
|
* This may be old if no data is incoming.
|
|
39
39
|
*/
|
|
40
|
-
last_checkpoint_ts:
|
|
40
|
+
last_checkpoint_ts: orNull(framework.codecs.date),
|
|
41
41
|
/**
|
|
42
42
|
* Last time we persisted a checkpoint or keepalive.
|
|
43
43
|
*
|
|
44
44
|
* This should stay fairly current while replicating.
|
|
45
45
|
*/
|
|
46
|
-
last_keepalive_ts:
|
|
46
|
+
last_keepalive_ts: orNull(framework.codecs.date),
|
|
47
47
|
/**
|
|
48
48
|
* If an error is stopping replication, it will be stored here.
|
|
49
49
|
*/
|
|
50
|
-
last_fatal_error:
|
|
51
|
-
keepalive_op:
|
|
52
|
-
storage_version:
|
|
50
|
+
last_fatal_error: orNull(t.string),
|
|
51
|
+
keepalive_op: orNull(bigint),
|
|
52
|
+
storage_version: orNull(pgwire_number).optional(),
|
|
53
|
+
version_label: orNull(t.string).optional(),
|
|
53
54
|
content: t.string,
|
|
54
|
-
sync_plan:
|
|
55
|
+
sync_plan: orNull(
|
|
55
56
|
jsonContainerObject(
|
|
56
57
|
t.object({
|
|
57
58
|
plan: t.any,
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import { framework } from '@powersync/service-core';
|
|
2
|
+
import { orNull } from '@powersync/service-types';
|
|
1
3
|
import * as t from 'ts-codec';
|
|
2
4
|
import { bigint, jsonb } from '../codecs.js';
|
|
3
5
|
|
|
4
6
|
export const WriteCheckpoint = t.object({
|
|
5
7
|
user_id: t.string,
|
|
6
8
|
lsns: jsonb(t.record(t.string)),
|
|
7
|
-
write_checkpoint: bigint
|
|
9
|
+
write_checkpoint: bigint,
|
|
10
|
+
checkpoint_requested_at: orNull(framework.codecs.date)
|
|
8
11
|
});
|
|
9
12
|
|
|
10
13
|
export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
|
|
@@ -13,7 +16,8 @@ export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
|
|
|
13
16
|
export const CustomWriteCheckpoint = t.object({
|
|
14
17
|
user_id: t.string,
|
|
15
18
|
write_checkpoint: bigint,
|
|
16
|
-
sync_rules_id: bigint
|
|
19
|
+
sync_rules_id: bigint,
|
|
20
|
+
checkpoint_requested_at: orNull(framework.codecs.date)
|
|
17
21
|
});
|
|
18
22
|
|
|
19
23
|
export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { storage } from '@powersync/service-core';
|
|
3
|
+
import { models } from '../types/types.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Gets the latest active checkpoint document.
|
|
7
|
+
* This is mainly exported for mocking in tests.
|
|
8
|
+
*/
|
|
9
|
+
export async function getActiveCheckpointDocument({
|
|
10
|
+
db
|
|
11
|
+
}: {
|
|
12
|
+
db: lib_postgres.DatabaseClient;
|
|
13
|
+
}): Promise<models.ActiveCheckpointDecoded | null> {
|
|
14
|
+
return db.sql`
|
|
15
|
+
SELECT
|
|
16
|
+
id,
|
|
17
|
+
last_checkpoint,
|
|
18
|
+
last_checkpoint_lsn
|
|
19
|
+
FROM
|
|
20
|
+
sync_rules
|
|
21
|
+
WHERE
|
|
22
|
+
state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
|
|
23
|
+
OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
24
|
+
ORDER BY
|
|
25
|
+
id DESC
|
|
26
|
+
LIMIT
|
|
27
|
+
1
|
|
28
|
+
`
|
|
29
|
+
.decoded(models.ActiveCheckpoint)
|
|
30
|
+
.first();
|
|
31
|
+
}
|