@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -6
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
- package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +13 -12
- package/dist/storage/PostgresBucketStorageFactory.js +59 -55
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +3 -1
- package/dist/storage/PostgresCompactor.js +21 -1
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +2 -1
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +36 -11
- package/dist/storage/PostgresSyncRulesStorage.js +250 -177
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +8 -24
- package/dist/storage/batch/PostgresBucketBatch.js +229 -194
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +161 -24
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/{@types/storage → storage}/current-data-store.d.ts +8 -2
- package/dist/storage/current-data-store.js +66 -11
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +90 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
- package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +2 -0
- package/dist/types/models/WriteCheckpoint.js +5 -2
- package/dist/types/models/WriteCheckpoint.js.map +1 -1
- package/dist/utils/checkpoints.d.ts +9 -0
- package/dist/utils/checkpoints.js +26 -0
- package/dist/utils/checkpoints.js.map +1 -0
- package/dist/utils/test-utils.js +2 -2
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -22
- package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
- package/src/storage/PostgresBucketStorageFactory.ts +67 -58
- package/src/storage/PostgresCompactor.ts +25 -1
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresStorageProvider.ts +2 -1
- package/src/storage/PostgresSyncRulesStorage.ts +275 -191
- package/src/storage/batch/PostgresBucketBatch.ts +256 -214
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +174 -24
- package/src/storage/current-data-store.ts +66 -11
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +98 -0
- package/src/types/models/WriteCheckpoint.ts +5 -2
- package/src/utils/checkpoints.ts +31 -0
- package/src/utils/test-utils.ts +2 -2
- package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
- package/test/src/checkpoint_notifications.test.ts +522 -0
- package/test/src/migrations.test.ts +1 -1
- package/test/src/storage.test.ts +226 -10
- package/test/src/storage_compacting.test.ts +15 -7
- package/test/src/storage_sync.test.ts +11 -8
- package/test/tsconfig.json +1 -3
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -0
- package/dist/.tsbuildinfo +0 -1
- package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
- /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
- /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
- /package/dist/{@types/types → types}/codecs.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
- /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
- /package/dist/{@types/types → types}/models/json.d.ts +0 -0
- /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
- /package/dist/{@types/types → types}/types.d.ts +0 -0
- /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
- /package/dist/{@types/utils → utils}/db.d.ts +0 -0
- /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
- /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
- /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { models } from '../../types/types.js';
|
|
5
|
+
export class PostgresPersistedSyncConfigContent extends storage.PersistedSyncConfigContent {
|
|
6
|
+
db;
|
|
7
|
+
constructor(db, row) {
|
|
8
|
+
super({
|
|
9
|
+
replicationStreamId: Number(row.id),
|
|
10
|
+
sync_rules_content: row.content,
|
|
11
|
+
compiled_plan: row.sync_plan,
|
|
12
|
+
replicationStreamName: row.slot_name,
|
|
13
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION,
|
|
14
|
+
syncConfigState: row.state
|
|
15
|
+
});
|
|
16
|
+
this.db = db;
|
|
17
|
+
}
|
|
18
|
+
async getSyncConfigStatus() {
|
|
19
|
+
const row = await this.db.sql `
|
|
20
|
+
SELECT
|
|
21
|
+
*
|
|
22
|
+
FROM
|
|
23
|
+
sync_rules
|
|
24
|
+
WHERE
|
|
25
|
+
id = ${{ value: this.replicationStreamId, type: 'int4' }}
|
|
26
|
+
`
|
|
27
|
+
.decoded(models.SyncRules)
|
|
28
|
+
.first();
|
|
29
|
+
return row == null ? null : syncConfigStatusFromRow(row);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function syncConfigStatusFromRow(row) {
|
|
33
|
+
return {
|
|
34
|
+
id: String(row.id),
|
|
35
|
+
replicationStreamId: Number(row.id),
|
|
36
|
+
state: row.state,
|
|
37
|
+
snapshot_done: row.snapshot_done,
|
|
38
|
+
last_checkpoint_lsn: row.last_checkpoint_lsn,
|
|
39
|
+
last_fatal_error: row.last_fatal_error,
|
|
40
|
+
last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null,
|
|
41
|
+
last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream {
|
|
45
|
+
db;
|
|
46
|
+
row;
|
|
47
|
+
current_lock = null;
|
|
48
|
+
syncConfigContent;
|
|
49
|
+
constructor(db, row) {
|
|
50
|
+
super({
|
|
51
|
+
replicationStreamId: Number(row.id),
|
|
52
|
+
replicationStreamName: row.slot_name,
|
|
53
|
+
state: row.state,
|
|
54
|
+
storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION
|
|
55
|
+
});
|
|
56
|
+
this.db = db;
|
|
57
|
+
this.row = row;
|
|
58
|
+
this.syncConfigContent = [new PostgresPersistedSyncConfigContent(this.db, this.row)];
|
|
59
|
+
}
|
|
60
|
+
parsed(options) {
|
|
61
|
+
return this.syncConfigContent[0].parsed(options);
|
|
62
|
+
}
|
|
63
|
+
async lock() {
|
|
64
|
+
const manager = new lib_postgres.PostgresLockManager({
|
|
65
|
+
db: this.db,
|
|
66
|
+
name: `sync_rules_${this.replicationStreamId}_${this.replicationStreamName}`
|
|
67
|
+
});
|
|
68
|
+
const lockHandle = await manager.acquire();
|
|
69
|
+
if (!lockHandle) {
|
|
70
|
+
throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
|
|
71
|
+
}
|
|
72
|
+
const interval = setInterval(async () => {
|
|
73
|
+
try {
|
|
74
|
+
await lockHandle.refresh();
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
this.logger.error('Failed to refresh lock', e);
|
|
78
|
+
clearInterval(interval);
|
|
79
|
+
}
|
|
80
|
+
}, 30_130);
|
|
81
|
+
return (this.current_lock = {
|
|
82
|
+
sync_rules_id: this.replicationStreamId,
|
|
83
|
+
release: async () => {
|
|
84
|
+
clearInterval(interval);
|
|
85
|
+
return lockHandle.release();
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=PostgresPersistedSyncConfigContent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PostgresPersistedSyncConfigContent.js","sourceRoot":"","sources":["../../../src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,MAAM,OAAO,kCAAmC,SAAQ,OAAO,CAAC,0BAA0B;IAE9E;IADV,YACU,EAA+B,EACvC,GAA4B;QAE5B,KAAK,CAAC;YACJ,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,kBAAkB,EAAE,GAAG,CAAC,OAAO;YAC/B,aAAa,EAAE,GAAG,CAAC,SAAS;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB;YACrE,eAAe,EAAE,GAAG,CAAC,KAA8B;SACpD,CAAC,CAAC;QAVK,OAAE,GAAF,EAAE,CAA6B;IAWzC,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;eAMlB,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;KAC3D;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,GAA4B;IAC3D,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,EAAE,GAAG,CAAC,KAA8B;QACzC,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB;QAC5C,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI;QACjF,kBAAkB,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;KACrF,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,kCAAmC,SAAQ,OAAO,CAAC,0BAA0B;IAK9E;IACS;IALnB,YAAY,GAAmC,IAAI,CAAC;IAC3C,iBAAiB,CAAgD;IAE1E,YACU,EAA+B,EACtB,GAA4B;QAE7C,KAAK,CAAC;YACJ,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,KAAK,EAAE,GAAG,CAAC,KAA8B;YACzC,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,sBAAsB;SACtE,CAAC,CAAC;QARK,OAAE,GAAF,EAAE,CAA6B;QACtB,QAAG,GAAH,GAAG,CAAyB;QAQ7C,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,CAAC,OAAuC;QAC5C,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC;YACnD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,cAAc,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,EAAE;SAC7E,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+DAA+D,CAAC,CAAC;QACjH,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;gBAC/C,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,EAAE,MAAM,CAAC,CAAC;QAEX,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG;YAC1B,aAAa,EAAE,IAAI,CAAC,mBAAmB;YACvC,OAAO,EAAE,KAAK,IAAI,EAAE;gBAClB,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACxB,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC;SACF,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -3,6 +3,7 @@ export declare const WriteCheckpoint: t.ObjectCodec<{
|
|
|
3
3
|
user_id: t.IdentityCodec<t.CodecType.String>;
|
|
4
4
|
lsns: t.Codec<Record<string, string>, string, string, t.CodecProps>;
|
|
5
5
|
write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
|
|
6
|
+
checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
|
|
6
7
|
}>;
|
|
7
8
|
export type WriteCheckpoint = t.Encoded<typeof WriteCheckpoint>;
|
|
8
9
|
export type WriteCheckpointDecoded = t.Decoded<typeof WriteCheckpoint>;
|
|
@@ -10,6 +11,7 @@ export declare const CustomWriteCheckpoint: t.ObjectCodec<{
|
|
|
10
11
|
user_id: t.IdentityCodec<t.CodecType.String>;
|
|
11
12
|
write_checkpoint: t.Codec<bigint, string | number, string, t.CodecProps>;
|
|
12
13
|
sync_rules_id: t.Codec<bigint, string | number, string, t.CodecProps>;
|
|
14
|
+
checkpoint_requested_at: t.Union<t.Codec<null, null, string, t.CodecProps>, t.Codec<Date, string | import("@powersync/service-sync-rules").DateTimeValue, string, t.CodecProps>>;
|
|
13
15
|
}>;
|
|
14
16
|
export type CustomWriteCheckpoint = t.Encoded<typeof CustomWriteCheckpoint>;
|
|
15
17
|
export type CustomWriteCheckpointDecoded = t.Decoded<typeof CustomWriteCheckpoint>;
|
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
import { framework } from '@powersync/service-core';
|
|
1
2
|
import * as t from 'ts-codec';
|
|
2
3
|
import { bigint, jsonb } from '../codecs.js';
|
|
3
4
|
export const WriteCheckpoint = t.object({
|
|
4
5
|
user_id: t.string,
|
|
5
6
|
lsns: jsonb(t.record(t.string)),
|
|
6
|
-
write_checkpoint: bigint
|
|
7
|
+
write_checkpoint: bigint,
|
|
8
|
+
checkpoint_requested_at: t.Null.or(framework.codecs.date)
|
|
7
9
|
});
|
|
8
10
|
export const CustomWriteCheckpoint = t.object({
|
|
9
11
|
user_id: t.string,
|
|
10
12
|
write_checkpoint: bigint,
|
|
11
|
-
sync_rules_id: bigint
|
|
13
|
+
sync_rules_id: bigint,
|
|
14
|
+
checkpoint_requested_at: t.Null.or(framework.codecs.date)
|
|
12
15
|
});
|
|
13
16
|
//# sourceMappingURL=WriteCheckpoint.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;IACxB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;IACrB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
2
|
+
import { models } from '../types/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Gets the latest active checkpoint document.
|
|
5
|
+
* This is mainly exported for mocking in tests.
|
|
6
|
+
*/
|
|
7
|
+
export declare function getActiveCheckpointDocument({ db }: {
|
|
8
|
+
db: lib_postgres.DatabaseClient;
|
|
9
|
+
}): Promise<models.ActiveCheckpointDecoded | null>;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { models } from '../types/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Gets the latest active checkpoint document.
|
|
5
|
+
* This is mainly exported for mocking in tests.
|
|
6
|
+
*/
|
|
7
|
+
export async function getActiveCheckpointDocument({ db }) {
|
|
8
|
+
return db.sql `
|
|
9
|
+
SELECT
|
|
10
|
+
id,
|
|
11
|
+
last_checkpoint,
|
|
12
|
+
last_checkpoint_lsn
|
|
13
|
+
FROM
|
|
14
|
+
sync_rules
|
|
15
|
+
WHERE
|
|
16
|
+
state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
|
|
17
|
+
OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
18
|
+
ORDER BY
|
|
19
|
+
id DESC
|
|
20
|
+
LIMIT
|
|
21
|
+
1
|
|
22
|
+
`
|
|
23
|
+
.decoded(models.ActiveCheckpoint)
|
|
24
|
+
.first();
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=checkpoints.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/utils/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,EAAE,EAGH;IACC,OAAO,EAAE,CAAC,GAAG,CAAA;;;;;;;;gBAQC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;mBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;GAKzE;SACE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;SAChC,KAAK,EAAE,CAAC;AACb,CAAC"}
|
package/dist/utils/test-utils.js
CHANGED
|
@@ -54,7 +54,7 @@ export function postgresTestSetup(factoryOptions) {
|
|
|
54
54
|
await runMigrations({ down: false, up: true });
|
|
55
55
|
await using storageFactory = new PostgresBucketStorageFactory({
|
|
56
56
|
config: TEST_CONNECTION_OPTIONS,
|
|
57
|
-
|
|
57
|
+
replicationStreamNamePrefix: 'test_'
|
|
58
58
|
});
|
|
59
59
|
await truncateTables(storageFactory.db);
|
|
60
60
|
};
|
|
@@ -81,7 +81,7 @@ export function postgresTestSetup(factoryOptions) {
|
|
|
81
81
|
}
|
|
82
82
|
return new PostgresBucketStorageFactory({
|
|
83
83
|
config: TEST_CONNECTION_OPTIONS,
|
|
84
|
-
|
|
84
|
+
replicationStreamNamePrefix: 'test_'
|
|
85
85
|
});
|
|
86
86
|
}
|
|
87
87
|
catch (ex) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAWzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,
|
|
1
|
+
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAWzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,2BAA2B,EAAE,OAAO;SACrC,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,2BAA2B,EAAE,OAAO;iBACrC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,44 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-postgres-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"
|
|
5
|
-
"version": "0.0.0-dev-20260602073133",
|
|
6
|
-
"main": "dist/index.js",
|
|
4
|
+
"version": "0.0.0-dev-20260803144812",
|
|
7
5
|
"license": "FSL-1.1-ALv2",
|
|
8
6
|
"type": "module",
|
|
9
7
|
"publishConfig": {
|
|
10
8
|
"access": "public"
|
|
11
9
|
},
|
|
12
10
|
"exports": {
|
|
13
|
-
".":
|
|
14
|
-
|
|
15
|
-
"import": "./dist/index.js",
|
|
16
|
-
"require": "./dist/index.js",
|
|
17
|
-
"default": "./dist/index.js"
|
|
18
|
-
},
|
|
19
|
-
"./types": {
|
|
20
|
-
"types": "./dist/@types/index.d.ts",
|
|
21
|
-
"import": "./dist/types/types.js",
|
|
22
|
-
"require": "./dist/types/types.js",
|
|
23
|
-
"default": "./dist/types/types.js"
|
|
24
|
-
}
|
|
11
|
+
".": "./dist/index.js",
|
|
12
|
+
"./types": "./dist/types/types.js"
|
|
25
13
|
},
|
|
26
14
|
"dependencies": {
|
|
27
15
|
"ix": "^5.0.0",
|
|
28
16
|
"lru-cache": "^10.2.2",
|
|
29
17
|
"ts-codec": "^1.3.0",
|
|
30
18
|
"uuid": "^14.0.0",
|
|
31
|
-
"@powersync/lib-service-postgres": "0.0.0-dev-
|
|
32
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
33
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
34
|
-
"@powersync/service-types": "0.
|
|
35
|
-
"@powersync/service-jpgwire": "0.0.0-dev-
|
|
19
|
+
"@powersync/lib-service-postgres": "0.0.0-dev-20260803144812",
|
|
20
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20260803144812",
|
|
21
|
+
"@powersync/service-core": "0.0.0-dev-20260803144812",
|
|
22
|
+
"@powersync/service-types": "0.0.0-dev-20260803144812",
|
|
23
|
+
"@powersync/service-jpgwire": "0.0.0-dev-20260803144812",
|
|
36
24
|
"@powersync/service-jsonbig": "0.17.13",
|
|
37
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
25
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20260803144812"
|
|
38
26
|
},
|
|
39
27
|
"devDependencies": {
|
|
40
28
|
"typescript": "~6.0.3",
|
|
41
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
29
|
+
"@powersync/service-core-tests": "0.0.0-dev-20260803144812"
|
|
42
30
|
},
|
|
43
31
|
"scripts": {
|
|
44
32
|
"build": "tsc -b",
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { migrations } from '@powersync/service-core';
|
|
2
|
+
|
|
3
|
+
import { openMigrationDB } from '../migration-utils.js';
|
|
4
|
+
|
|
5
|
+
export const up: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
6
|
+
const {
|
|
7
|
+
service_context: { configuration }
|
|
8
|
+
} = context;
|
|
9
|
+
await using client = openMigrationDB(configuration.storage);
|
|
10
|
+
|
|
11
|
+
await client.transaction(async (db) => {
|
|
12
|
+
await db.sql`
|
|
13
|
+
ALTER TABLE write_checkpoints
|
|
14
|
+
ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
|
|
15
|
+
`.execute();
|
|
16
|
+
await db.sql`
|
|
17
|
+
ALTER TABLE custom_write_checkpoints
|
|
18
|
+
ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
|
|
19
|
+
`.execute();
|
|
20
|
+
await db.sql`
|
|
21
|
+
CREATE INDEX write_checkpoints_checkpoint_requested_at ON write_checkpoints (checkpoint_requested_at)
|
|
22
|
+
WHERE
|
|
23
|
+
checkpoint_requested_at IS NOT NULL
|
|
24
|
+
`.execute();
|
|
25
|
+
await db.sql`
|
|
26
|
+
CREATE INDEX custom_write_checkpoints_checkpoint_requested_at ON custom_write_checkpoints (checkpoint_requested_at)
|
|
27
|
+
WHERE
|
|
28
|
+
checkpoint_requested_at IS NOT NULL
|
|
29
|
+
`.execute();
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export const down: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
34
|
+
const {
|
|
35
|
+
service_context: { configuration }
|
|
36
|
+
} = context;
|
|
37
|
+
await using client = openMigrationDB(configuration.storage);
|
|
38
|
+
|
|
39
|
+
await client.transaction(async (db) => {
|
|
40
|
+
// Also drops the write_checkpoints_checkpoint_requested_at index.
|
|
41
|
+
await db.sql`
|
|
42
|
+
ALTER TABLE write_checkpoints
|
|
43
|
+
DROP COLUMN checkpoint_requested_at
|
|
44
|
+
`.execute();
|
|
45
|
+
// Also drops the custom_write_checkpoints_checkpoint_requested_at index.
|
|
46
|
+
await db.sql`
|
|
47
|
+
ALTER TABLE custom_write_checkpoints
|
|
48
|
+
DROP COLUMN checkpoint_requested_at
|
|
49
|
+
`.execute();
|
|
50
|
+
});
|
|
51
|
+
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { framework, GetIntanceOptions, storage
|
|
1
|
+
import { framework, GetIntanceOptions, storage } from '@powersync/service-core';
|
|
2
2
|
import * as pg_wire from '@powersync/service-jpgwire';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
4
|
import * as uuid from 'uuid';
|
|
@@ -8,19 +8,19 @@ import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
|
|
|
8
8
|
|
|
9
9
|
import { getStorageApplicationName } from '../utils/application-name.js';
|
|
10
10
|
import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
|
|
11
|
-
import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
|
|
12
11
|
import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
|
|
13
|
-
import {
|
|
12
|
+
import { PostgresPersistedReplicationStream } from './sync-rules/PostgresPersistedSyncConfigContent.js';
|
|
14
13
|
|
|
15
14
|
export type PostgresBucketStorageOptions = {
|
|
16
15
|
config: NormalizedPostgresStorageConfig;
|
|
17
|
-
|
|
16
|
+
replicationStreamNamePrefix: string;
|
|
17
|
+
checksumCacheTtlMs?: number;
|
|
18
18
|
};
|
|
19
19
|
|
|
20
20
|
export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
21
21
|
[framework.DO_NOT_LOG] = true;
|
|
22
22
|
readonly db: lib_postgres.DatabaseClient;
|
|
23
|
-
public readonly
|
|
23
|
+
public readonly replicationStreamNamePrefix: string;
|
|
24
24
|
|
|
25
25
|
private activeStorageCache: storage.SyncRulesBucketStorage | undefined;
|
|
26
26
|
|
|
@@ -32,7 +32,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
32
32
|
notificationChannels: [NOTIFICATION_CHANNEL],
|
|
33
33
|
applicationName: getStorageApplicationName()
|
|
34
34
|
});
|
|
35
|
-
this.
|
|
35
|
+
this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
|
|
36
36
|
|
|
37
37
|
this.db.registerListener({
|
|
38
38
|
connectionCreated: async (connection) => this.prepareStatements(connection)
|
|
@@ -49,31 +49,32 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
getInstance(
|
|
52
|
-
|
|
52
|
+
replicationStream: storage.PersistedReplicationStream,
|
|
53
53
|
options?: GetIntanceOptions
|
|
54
54
|
): storage.SyncRulesBucketStorage {
|
|
55
|
-
const
|
|
55
|
+
const syncRuleStorage = new PostgresSyncRulesStorage({
|
|
56
56
|
factory: this,
|
|
57
57
|
db: this.db,
|
|
58
|
-
|
|
59
|
-
batchLimits: this.options.config.batch_limits
|
|
58
|
+
replicationStream,
|
|
59
|
+
batchLimits: this.options.config.batch_limits,
|
|
60
|
+
checksumCacheTtlMs: this.options.checksumCacheTtlMs
|
|
60
61
|
});
|
|
61
62
|
if (!options?.skipLifecycleHooks) {
|
|
62
|
-
this.iterateListeners((cb) => cb.syncStorageCreated?.(
|
|
63
|
+
this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage));
|
|
63
64
|
}
|
|
64
|
-
|
|
65
|
+
syncRuleStorage.registerListener({
|
|
65
66
|
batchStarted: (batch) => {
|
|
66
67
|
batch.registerListener({
|
|
67
68
|
replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
|
|
68
69
|
});
|
|
69
70
|
}
|
|
70
71
|
});
|
|
71
|
-
return
|
|
72
|
+
return syncRuleStorage;
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
async getStorageMetrics(): Promise<storage.StorageMetrics> {
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
76
|
+
const activeSyncConfig = await this.getActiveSyncConfig();
|
|
77
|
+
if (activeSyncConfig == null) {
|
|
77
78
|
return {
|
|
78
79
|
operations_size_bytes: 0,
|
|
79
80
|
parameters_size_bytes: 0,
|
|
@@ -153,7 +154,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
153
154
|
};
|
|
154
155
|
}
|
|
155
156
|
|
|
156
|
-
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<
|
|
157
|
+
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
|
|
157
158
|
const storageVersion =
|
|
158
159
|
options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
|
|
159
160
|
const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
|
|
@@ -200,7 +201,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
200
201
|
${{ type: 'json', value: options.config.plan }},
|
|
201
202
|
${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
|
|
202
203
|
CONCAT(
|
|
203
|
-
${{ type: 'varchar', value: this.
|
|
204
|
+
${{ type: 'varchar', value: this.replicationStreamNamePrefix }},
|
|
204
205
|
(
|
|
205
206
|
SELECT
|
|
206
207
|
id
|
|
@@ -218,9 +219,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
218
219
|
.decoded(models.SyncRules)
|
|
219
220
|
.first();
|
|
220
221
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
return new PostgresPersistedSyncRulesContent(this.db, newSyncRulesRow!);
|
|
222
|
+
return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
|
|
224
223
|
});
|
|
225
224
|
}
|
|
226
225
|
|
|
@@ -252,28 +251,28 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
252
251
|
`.execute();
|
|
253
252
|
}
|
|
254
253
|
|
|
255
|
-
async restartReplication(
|
|
256
|
-
const next = await this.
|
|
257
|
-
const active = await this.
|
|
254
|
+
async restartReplication(replicationStreamId: number): Promise<void> {
|
|
255
|
+
const next = await this.getDeployingSyncConfig();
|
|
256
|
+
const active = await this.getActiveSyncConfig();
|
|
258
257
|
|
|
259
258
|
// In both the below cases, we create a new replication stream.
|
|
260
259
|
// The current one will continue serving sync requests until the next one has finished processing.
|
|
261
|
-
if (next != null && next.
|
|
260
|
+
if (next != null && next.content.replicationStreamId == replicationStreamId) {
|
|
262
261
|
// We need to redo the "next" replication stream
|
|
263
262
|
|
|
264
|
-
await this.updateSyncRules(next.asUpdateOptions());
|
|
263
|
+
await this.updateSyncRules(next.content.asUpdateOptions());
|
|
265
264
|
// Pro-actively stop replicating
|
|
266
265
|
await this.db.sql`
|
|
267
266
|
UPDATE sync_rules
|
|
268
267
|
SET
|
|
269
268
|
state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }}
|
|
270
269
|
WHERE
|
|
271
|
-
id = ${{ value: next.
|
|
270
|
+
id = ${{ value: next.content.replicationStreamId, type: 'int4' }}
|
|
272
271
|
AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }}
|
|
273
272
|
`.execute();
|
|
274
|
-
} else if (next == null && active?.
|
|
273
|
+
} else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
|
|
275
274
|
// Slot removed for "active" replication stream, while there is no "next" one.
|
|
276
|
-
await this.updateSyncRules(active.asUpdateOptions());
|
|
275
|
+
await this.updateSyncRules(active.content.asUpdateOptions());
|
|
277
276
|
|
|
278
277
|
// Pro-actively stop replicating, but still serve clients with existing data
|
|
279
278
|
await this.db.sql`
|
|
@@ -281,10 +280,10 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
281
280
|
SET
|
|
282
281
|
state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
283
282
|
WHERE
|
|
284
|
-
id = ${{ value: active.
|
|
283
|
+
id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
|
|
285
284
|
AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
|
|
286
285
|
`.execute();
|
|
287
|
-
} else if (next != null && active?.
|
|
286
|
+
} else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
|
|
288
287
|
// Already have "next" replication stream - don't update any.
|
|
289
288
|
|
|
290
289
|
// Pro-actively stop replicating, but still serve clients with existing data
|
|
@@ -293,13 +292,13 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
293
292
|
SET
|
|
294
293
|
state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
295
294
|
WHERE
|
|
296
|
-
id = ${{ value: active.
|
|
295
|
+
id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
|
|
297
296
|
AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
|
|
298
297
|
`.execute();
|
|
299
298
|
}
|
|
300
299
|
}
|
|
301
300
|
|
|
302
|
-
async
|
|
301
|
+
async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
|
|
303
302
|
const activeRow = await this.db.sql`
|
|
304
303
|
SELECT
|
|
305
304
|
*
|
|
@@ -319,11 +318,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
319
318
|
return null;
|
|
320
319
|
}
|
|
321
320
|
|
|
322
|
-
return
|
|
321
|
+
return this.resolvedSyncConfigFromRow(activeRow, { cacheActiveStorage: true });
|
|
323
322
|
}
|
|
324
323
|
|
|
325
|
-
async
|
|
326
|
-
const
|
|
324
|
+
async getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
|
|
325
|
+
const row = await this.db.sql`
|
|
327
326
|
SELECT
|
|
328
327
|
*
|
|
329
328
|
FROM
|
|
@@ -337,14 +336,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
337
336
|
`
|
|
338
337
|
.decoded(models.SyncRules)
|
|
339
338
|
.first();
|
|
340
|
-
if (!nextRow) {
|
|
341
|
-
return null;
|
|
342
|
-
}
|
|
343
339
|
|
|
344
|
-
return
|
|
340
|
+
return row == null ? null : this.resolvedSyncConfigFromRow(row);
|
|
345
341
|
}
|
|
346
342
|
|
|
347
|
-
async
|
|
343
|
+
async getReplicatingReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
|
|
348
344
|
const rows = await this.db.sql`
|
|
349
345
|
SELECT
|
|
350
346
|
*
|
|
@@ -357,10 +353,10 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
357
353
|
.decoded(models.SyncRules)
|
|
358
354
|
.rows();
|
|
359
355
|
|
|
360
|
-
return rows.map((row) => new
|
|
356
|
+
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
|
|
361
357
|
}
|
|
362
358
|
|
|
363
|
-
async
|
|
359
|
+
async getStoppedReplicationStreams(): Promise<storage.PersistedReplicationStream[]> {
|
|
364
360
|
const rows = await this.db.sql`
|
|
365
361
|
SELECT
|
|
366
362
|
*
|
|
@@ -372,24 +368,37 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
372
368
|
.decoded(models.SyncRules)
|
|
373
369
|
.rows();
|
|
374
370
|
|
|
375
|
-
return rows.map((row) => new
|
|
371
|
+
return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row));
|
|
376
372
|
}
|
|
377
373
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
374
|
+
private resolvedSyncConfigFromRow(
|
|
375
|
+
row: models.SyncRulesDecoded,
|
|
376
|
+
options: { cacheActiveStorage?: boolean } = {}
|
|
377
|
+
): storage.ResolvedSyncConfig {
|
|
378
|
+
const stream = new PostgresPersistedReplicationStream(this.db, row);
|
|
379
|
+
const content = stream.syncConfigContent[0];
|
|
380
|
+
const thisFactory = this;
|
|
383
381
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
382
|
+
return {
|
|
383
|
+
content,
|
|
384
|
+
replicationStream: stream,
|
|
385
|
+
get storage() {
|
|
386
|
+
// It is important that this instance is cached.
|
|
387
|
+
// Not for the instance construction itself, but to ensure that internal caches on the instance
|
|
388
|
+
// are re-used properly.
|
|
389
|
+
if (options.cacheActiveStorage) {
|
|
390
|
+
const activeStorageCache = thisFactory.activeStorageCache;
|
|
391
|
+
if (activeStorageCache?.replicationStreamId == stream.replicationStreamId) {
|
|
392
|
+
return activeStorageCache;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const instance = thisFactory.getInstance(stream);
|
|
397
|
+
if (options.cacheActiveStorage) {
|
|
398
|
+
thisFactory.activeStorageCache = instance;
|
|
399
|
+
}
|
|
400
|
+
return instance;
|
|
401
|
+
}
|
|
402
|
+
};
|
|
394
403
|
}
|
|
395
404
|
}
|
|
@@ -53,6 +53,7 @@ export class PostgresCompactor {
|
|
|
53
53
|
private clearBatchLimit: number;
|
|
54
54
|
private maxOpId: InternalOpId;
|
|
55
55
|
private buckets: string[] | undefined;
|
|
56
|
+
private deleteCheckpointRequestsBefore: Date | undefined;
|
|
56
57
|
|
|
57
58
|
constructor(
|
|
58
59
|
private db: lib_postgres.DatabaseClient,
|
|
@@ -65,14 +66,17 @@ export class PostgresCompactor {
|
|
|
65
66
|
this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
66
67
|
this.maxOpId = options.maxOpId ?? 0n;
|
|
67
68
|
this.buckets = options.compactBuckets;
|
|
69
|
+
this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
|
|
68
70
|
}
|
|
69
71
|
|
|
70
72
|
/**
|
|
71
73
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
72
74
|
*
|
|
73
|
-
* See /docs/compacting-operations.md for details.
|
|
75
|
+
* See /docs/storage/compacting-operations.md for details.
|
|
74
76
|
*/
|
|
75
77
|
async compact() {
|
|
78
|
+
await this.deleteOldCheckpointRequests();
|
|
79
|
+
|
|
76
80
|
if (this.buckets) {
|
|
77
81
|
for (let bucket of this.buckets) {
|
|
78
82
|
await this.compactSingleBucket(bucket);
|
|
@@ -82,6 +86,26 @@ export class PostgresCompactor {
|
|
|
82
86
|
}
|
|
83
87
|
}
|
|
84
88
|
|
|
89
|
+
private async deleteOldCheckpointRequests() {
|
|
90
|
+
if (this.deleteCheckpointRequestsBefore == null) {
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
await this.db.sql`
|
|
95
|
+
DELETE FROM write_checkpoints
|
|
96
|
+
WHERE
|
|
97
|
+
checkpoint_requested_at IS NOT NULL
|
|
98
|
+
AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
|
|
99
|
+
`.execute();
|
|
100
|
+
|
|
101
|
+
await this.db.sql`
|
|
102
|
+
DELETE FROM custom_write_checkpoints
|
|
103
|
+
WHERE
|
|
104
|
+
checkpoint_requested_at IS NOT NULL
|
|
105
|
+
AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
|
|
106
|
+
`.execute();
|
|
107
|
+
}
|
|
108
|
+
|
|
85
109
|
private async compactAllBuckets() {
|
|
86
110
|
const DISCOVERY_BATCH_SIZE = 200;
|
|
87
111
|
let lastBucket = '';
|