@powersync/service-module-postgres-storage 0.16.3 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
- package/dist/storage/PostgresBucketStorageFactory.js +0 -2
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresCompactor.d.ts +2 -0
- package/dist/storage/PostgresCompactor.js +20 -0
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.d.ts +16 -1
- package/dist/storage/PostgresSyncRulesStorage.js +200 -131
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.d.ts +2 -20
- package/dist/storage/batch/PostgresBucketBatch.js +162 -151
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +156 -36
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/current-data-store.d.ts +8 -2
- package/dist/storage/current-data-store.js +66 -11
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/types/models/WriteCheckpoint.d.ts +2 -0
- package/dist/types/models/WriteCheckpoint.js +5 -2
- package/dist/types/models/WriteCheckpoint.js.map +1 -1
- package/dist/utils/checkpoints.d.ts +9 -0
- package/dist/utils/checkpoints.js +26 -0
- package/dist/utils/checkpoints.js.map +1 -0
- package/package.json +8 -8
- package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
- package/src/storage/PostgresBucketStorageFactory.ts +0 -3
- package/src/storage/PostgresCompactor.ts +24 -0
- package/src/storage/PostgresSyncRulesStorage.ts +219 -137
- package/src/storage/batch/PostgresBucketBatch.ts +169 -154
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +165 -37
- package/src/storage/current-data-store.ts +66 -11
- package/src/types/models/WriteCheckpoint.ts +5 -2
- package/src/utils/checkpoints.ts +31 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
- package/test/src/checkpoint_notifications.test.ts +522 -0
- package/test/src/storage.test.ts +214 -5
- package/test/src/storage_compacting.test.ts +3 -3
- package/test/src/storage_sync.test.ts +4 -4
- package/test/tsconfig.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
# @powersync/service-module-postgres-storage
|
|
2
2
|
|
|
3
|
+
## 0.17.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
|
|
8
|
+
|
|
9
|
+
This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 294f334: Clear the replication error flag outside the flush transaction. Clearing it inside the REPEATABLE READ replication transaction conflicts with concurrent keepalive updates on the same sync_rules row, causing serialization failures that retry the entire flush when multiple writers are active.
|
|
14
|
+
- 0a511c2: Delete rows in batches when clearing storage for a replication stream. A single DELETE covering the entire group can run for hours on large deployments and never complete once it exceeds statement or socket timeouts; batched deletes make durable progress and can be safely retried or aborted.
|
|
15
|
+
- c4860c9: Improve Postgres sync throughput by reading bucket data with ordered, per-bucket index range scans that stop once the requested batch is full.
|
|
16
|
+
- 63707af: Stability and performance fixes for MongoDB storage V3.
|
|
17
|
+
- 123c801: Reconnect PostgreSQL notification connections and restore `LISTEN` subscriptions when the underlying connection is terminated. Harden connection-slot retries and lease handling, and publish checkpoint notifications atomically with checkpoint updates.
|
|
18
|
+
- Updated dependencies [087b61e]
|
|
19
|
+
- Updated dependencies [2189250]
|
|
20
|
+
- Updated dependencies [922f974]
|
|
21
|
+
- Updated dependencies [c4860c9]
|
|
22
|
+
- Updated dependencies [483415d]
|
|
23
|
+
- Updated dependencies [8daa300]
|
|
24
|
+
- Updated dependencies [aab068b]
|
|
25
|
+
- Updated dependencies [123c801]
|
|
26
|
+
- Updated dependencies [37591e9]
|
|
27
|
+
- Updated dependencies [be42e25]
|
|
28
|
+
- Updated dependencies [cb4c627]
|
|
29
|
+
- @powersync/service-core@1.24.0
|
|
30
|
+
- @powersync/lib-services-framework@0.10.0
|
|
31
|
+
- @powersync/service-types@0.17.0
|
|
32
|
+
- @powersync/service-sync-rules@0.40.0
|
|
33
|
+
- @powersync/lib-service-postgres@0.5.3
|
|
34
|
+
- @powersync/service-jpgwire@0.21.22
|
|
35
|
+
|
|
3
36
|
## 0.16.3
|
|
4
37
|
|
|
5
38
|
### Patch Changes
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { openMigrationDB } from '../migration-utils.js';
|
|
2
|
+
export const up = async (context) => {
|
|
3
|
+
const { service_context: { configuration } } = context;
|
|
4
|
+
await using client = openMigrationDB(configuration.storage);
|
|
5
|
+
await client.transaction(async (db) => {
|
|
6
|
+
await db.sql `
|
|
7
|
+
ALTER TABLE write_checkpoints
|
|
8
|
+
ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
|
|
9
|
+
`.execute();
|
|
10
|
+
await db.sql `
|
|
11
|
+
ALTER TABLE custom_write_checkpoints
|
|
12
|
+
ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
|
|
13
|
+
`.execute();
|
|
14
|
+
await db.sql `
|
|
15
|
+
CREATE INDEX write_checkpoints_checkpoint_requested_at ON write_checkpoints (checkpoint_requested_at)
|
|
16
|
+
WHERE
|
|
17
|
+
checkpoint_requested_at IS NOT NULL
|
|
18
|
+
`.execute();
|
|
19
|
+
await db.sql `
|
|
20
|
+
CREATE INDEX custom_write_checkpoints_checkpoint_requested_at ON custom_write_checkpoints (checkpoint_requested_at)
|
|
21
|
+
WHERE
|
|
22
|
+
checkpoint_requested_at IS NOT NULL
|
|
23
|
+
`.execute();
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
export const down = async (context) => {
|
|
27
|
+
const { service_context: { configuration } } = context;
|
|
28
|
+
await using client = openMigrationDB(configuration.storage);
|
|
29
|
+
await client.transaction(async (db) => {
|
|
30
|
+
// Also drops the write_checkpoints_checkpoint_requested_at index.
|
|
31
|
+
await db.sql `
|
|
32
|
+
ALTER TABLE write_checkpoints
|
|
33
|
+
DROP COLUMN checkpoint_requested_at
|
|
34
|
+
`.execute();
|
|
35
|
+
// Also drops the custom_write_checkpoints_checkpoint_requested_at index.
|
|
36
|
+
await db.sql `
|
|
37
|
+
ALTER TABLE custom_write_checkpoints
|
|
38
|
+
DROP COLUMN checkpoint_requested_at
|
|
39
|
+
`.execute();
|
|
40
|
+
});
|
|
41
|
+
};
|
|
42
|
+
//# sourceMappingURL=1782950400000-checkpoint-requested-at.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"1782950400000-checkpoint-requested-at.js","sourceRoot":"","sources":["../../../src/migrations/scripts/1782950400000-checkpoint-requested-at.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,YAAY,MAAM,GAAG,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAE5D,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACpC,MAAM,EAAE,CAAC,GAAG,CAAA;;;KAGX,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;KAGX,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;;KAIX,CAAC,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;;KAIX,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,YAAY,MAAM,GAAG,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAE5D,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACpC,kEAAkE;QAClE,MAAM,EAAE,CAAC,GAAG,CAAA;;;KAGX,CAAC,OAAO,EAAE,CAAC;QACZ,yEAAyE;QACzE,MAAM,EAAE,CAAC,GAAG,CAAA;;;KAGX,CAAC,OAAO,EAAE,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
|
|
@@ -5,7 +5,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
|
5
5
|
import { models } from '../types/types.js';
|
|
6
6
|
import { getStorageApplicationName } from '../utils/application-name.js';
|
|
7
7
|
import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
|
|
8
|
-
import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
|
|
9
8
|
import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
|
|
10
9
|
import { PostgresPersistedReplicationStream } from './sync-rules/PostgresPersistedSyncConfigContent.js';
|
|
11
10
|
export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
@@ -183,7 +182,6 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
183
182
|
`
|
|
184
183
|
.decoded(models.SyncRules)
|
|
185
184
|
.first();
|
|
186
|
-
await notifySyncRulesUpdate(this.db, newSyncRulesRow);
|
|
187
185
|
return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow);
|
|
188
186
|
});
|
|
189
187
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresBucketStorageFactory.js","sourceRoot":"","sources":["../../src/storage/PostgresBucketStorageFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEhF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,MAAM,EAAmC,MAAM,mBAAmB,CAAC;AAE5E,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"PostgresBucketStorageFactory.js","sourceRoot":"","sources":["../../src/storage/PostgresBucketStorageFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEhF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,MAAM,EAAmC,MAAM,mBAAmB,CAAC;AAE5E,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kCAAkC,EAAE,MAAM,oDAAoD,CAAC;AAQxG,MAAM,OAAO,4BAA6B,SAAQ,OAAO,CAAC,oBAAoB;IAOtD;IANtB,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IACrB,EAAE,CAA8B;IACzB,2BAA2B,CAAS;IAE5C,kBAAkB,CAA6C;IAEvE,YAAsB,OAAqC;QACzD,KAAK,EAAE,CAAC;QADY,YAAO,GAAP,OAAO,CAA8B;QAEzD,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,mBAAmB;YAC3B,oBAAoB,EAAE,CAAC,oBAAoB,CAAC;YAC5C,eAAe,EAAE,yBAAyB,EAAE;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,2BAA2B,GAAG,OAAO,CAAC,2BAA2B,CAAC;QAEvE,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC;YACvB,iBAAiB,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC;SAC5E,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,UAAgC;QACtD,+EAA+E;QAC/E,qCAAqC;IACvC,CAAC;IAED,WAAW,CACT,iBAAqD,EACrD,OAA2B;QAE3B,MAAM,eAAe,GAAG,IAAI,wBAAwB,CAAC;YACnD,OAAO,EAAE,IAAI;YACb,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,iBAAiB;YACjB,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY;YAC7C,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB;SACpD,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,eAAe,CAAC,gBAAgB,CAAC;YAC/B,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1D,IAAI,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAC7B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;KAY9B,CAAC,KAAK,EAKH,CAAC;QAEL,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,KAAM,CAAC,qBAAqB,CAAC;YAC3D,qBAAqB,EAAE,MAAM,CAAC,KAAM,CAAC,oBAAoB,CAAC;YAC1D,sBAAsB,EAAE,MAAM,CAAC,KAAM,CAAC,qBAAqB,CAAC,GAAG,MAAM,CAAC,KAAM,CAAC,qBAAqB,CAAC;SACpG,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;KAKpC;aACE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxB,KAAK,EAAE,CAAC;QACX,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO,WAAW,CAAC,EAAE,CAAC;QACxB,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC;YACvD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,4BAA4B;SACnC,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;aAIV,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE;OAC3C,CAAC,OAAO,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;KAKvC;aACE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxB,KAAK,EAAE,CAAC;QACX,OAAO,cAAe,CAAC,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,8BAA8B,CAC1D,MAAM,YAAY,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CACrE,CAAC;QAEF,OAAO;YACL,EAAE;YACF,IAAI,EAAE,YAAY,CAAC,wBAAwB;SAC5C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,MAAM,cAAc,GAClB,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,uBAAuB,CAAC;QAC3G,MAAM,aAAa,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QACrE,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,SAAS,CAAC,YAAY,CAC9B,SAAS,CAAC,SAAS,CAAC,WAAW,EAC/B,+BAA+B,cAAc,EAAE,CAChD,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,CAAC,GAAG,CAAA;;;oBAGE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE;;oBAEtD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE;OACzE,CAAC,OAAO,EAAE,CAAC;YAEZ,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;cAuB5B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;cAC/C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE;cAC5C,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE;;gBAE1D,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,2BAA2B,EAAE;;;;;;;;gBAQ5D,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;;cAEnE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE;;;;OAI9C;iBACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;iBACzB,KAAK,EAAE,CAAC;YAEX,OAAO,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,eAAgB,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,aAA2C;QAChF,IAAI,CAAC,aAAa,CAAC,qBAAqB,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;KAWhB,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;KAIhB,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAEhD,+DAA+D;QAC/D,kGAAkG;QAClG,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,IAAI,mBAAmB,EAAE,CAAC;YAC5E,gDAAgD;YAEhD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YAC3D,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;oBAGH,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;;iBAEzD,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;wBAClD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE;OAC7E,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,OAAO,CAAC,mBAAmB,IAAI,mBAAmB,EAAE,CAAC;YACtF,8EAA8E;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YAE7D,4EAA4E;YAC5E,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;oBAGH,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;iBAE5D,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;wBACpD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;OACzE,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,OAAO,CAAC,mBAAmB,IAAI,mBAAmB,EAAE,CAAC;YACtF,6DAA6D;YAE7D,4EAA4E;YAC5E,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;oBAGH,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;iBAE5D,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;wBACpD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;OACzE,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;kBAMrB,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;qBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;KAKzE;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,KAAK,EAAE,CAAC;QACX,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;kBAMf,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;KAKzE;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,gCAAgC;QACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;kBAMhB,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;qBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE;KAC5E;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,IAAI,EAAE,CAAC;QAEV,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,4BAA4B;QAChC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;kBAMhB,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;KACnE;aACE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aACzB,IAAI,EAAE,CAAC;QAEV,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IACjF,CAAC;IAEO,yBAAyB,CAC/B,GAA4B,EAC5B,UAA4C,EAAE;QAE9C,MAAM,MAAM,GAAG,IAAI,kCAAkC,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC;QAEzB,OAAO;YACL,OAAO;YACP,iBAAiB,EAAE,MAAM;YACzB,IAAI,OAAO;gBACT,gDAAgD;gBAChD,+FAA+F;gBAC/F,wBAAwB;gBACxB,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;oBAC/B,MAAM,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC;oBAC1D,IAAI,kBAAkB,EAAE,mBAAmB,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;wBAC1E,OAAO,kBAAkB,CAAC;oBAC5B,CAAC;gBACH,CAAC;gBAED,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBACjD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;oBAC/B,WAAW,CAAC,kBAAkB,GAAG,QAAQ,CAAC;gBAC5C,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -15,6 +15,7 @@ export declare class PostgresCompactor {
|
|
|
15
15
|
private clearBatchLimit;
|
|
16
16
|
private maxOpId;
|
|
17
17
|
private buckets;
|
|
18
|
+
private deleteCheckpointRequestsBefore;
|
|
18
19
|
constructor(db: lib_postgres.DatabaseClient, group_id: number, options: PostgresCompactOptions);
|
|
19
20
|
/**
|
|
20
21
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
@@ -22,6 +23,7 @@ export declare class PostgresCompactor {
|
|
|
22
23
|
* See /docs/storage/compacting-operations.md for details.
|
|
23
24
|
*/
|
|
24
25
|
compact(): Promise<void>;
|
|
26
|
+
private deleteOldCheckpointRequests;
|
|
25
27
|
private compactAllBuckets;
|
|
26
28
|
private compactSingleBucket;
|
|
27
29
|
private flush;
|
|
@@ -20,6 +20,7 @@ export class PostgresCompactor {
|
|
|
20
20
|
clearBatchLimit;
|
|
21
21
|
maxOpId;
|
|
22
22
|
buckets;
|
|
23
|
+
deleteCheckpointRequestsBefore;
|
|
23
24
|
constructor(db, group_id, options) {
|
|
24
25
|
this.db = db;
|
|
25
26
|
this.group_id = group_id;
|
|
@@ -29,6 +30,7 @@ export class PostgresCompactor {
|
|
|
29
30
|
this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
|
|
30
31
|
this.maxOpId = options.maxOpId ?? 0n;
|
|
31
32
|
this.buckets = options.compactBuckets;
|
|
33
|
+
this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
|
|
32
34
|
}
|
|
33
35
|
/**
|
|
34
36
|
* Compact buckets by converting operations into MOVE and/or CLEAR operations.
|
|
@@ -36,6 +38,7 @@ export class PostgresCompactor {
|
|
|
36
38
|
* See /docs/storage/compacting-operations.md for details.
|
|
37
39
|
*/
|
|
38
40
|
async compact() {
|
|
41
|
+
await this.deleteOldCheckpointRequests();
|
|
39
42
|
if (this.buckets) {
|
|
40
43
|
for (let bucket of this.buckets) {
|
|
41
44
|
await this.compactSingleBucket(bucket);
|
|
@@ -45,6 +48,23 @@ export class PostgresCompactor {
|
|
|
45
48
|
await this.compactAllBuckets();
|
|
46
49
|
}
|
|
47
50
|
}
|
|
51
|
+
async deleteOldCheckpointRequests() {
|
|
52
|
+
if (this.deleteCheckpointRequestsBefore == null) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
await this.db.sql `
|
|
56
|
+
DELETE FROM write_checkpoints
|
|
57
|
+
WHERE
|
|
58
|
+
checkpoint_requested_at IS NOT NULL
|
|
59
|
+
AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
|
|
60
|
+
`.execute();
|
|
61
|
+
await this.db.sql `
|
|
62
|
+
DELETE FROM custom_write_checkpoints
|
|
63
|
+
WHERE
|
|
64
|
+
checkpoint_requested_at IS NOT NULL
|
|
65
|
+
AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
|
|
66
|
+
`.execute();
|
|
67
|
+
}
|
|
48
68
|
async compactAllBuckets() {
|
|
49
69
|
const DISCOVERY_BATCH_SIZE = 200;
|
|
50
70
|
let lastBucket = '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresCompactor.js","sourceRoot":"","sources":["../../src/storage/PostgresCompactor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA8B5D,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,iBAAiB;
|
|
1
|
+
{"version":3,"file":"PostgresCompactor.js","sourceRoot":"","sources":["../../src/storage/PostgresCompactor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA8B5D,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,iBAAiB;IAYlB;IACA;IAZF,OAAO,GAAuB,EAAE,CAAC;IAEjC,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAe;IACtB,OAAO,CAAuB;IAC9B,8BAA8B,CAAmB;IAEzD,YACU,EAA+B,EAC/B,QAAgB,EACxB,OAA+B;QAFvB,OAAE,GAAF,EAAE,CAA6B;QAC/B,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACrF,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,8BAA8B,CAAC;QACzF,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;QAC5E,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC;QACtC,IAAI,CAAC,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC;IAC/E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEzC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B;QACvC,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,EAAE,CAAC;YAChD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;wCAImB,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,8BAA8B,CAAC,WAAW,EAAE,EAAE;KAC3G,CAAC,OAAO,EAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;wCAImB,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,8BAA8B,CAAC,WAAW,EAAE,EAAE;KAC3G,CAAC,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,MAAM,oBAAoB,GAAG,GAAG,CAAC;QACjC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;uBAMpB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;8BAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE;;;;YAIxD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE;OAClD,CAAC,IAAI,EAAE,CAA8B,CAAC;YAEvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM;YACR,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAClD,CAAC;YAED,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAAuB;YACrC,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;SACf,CAAC;QAEF,IAAI,cAAc,GAAG,UAAU,CAAC;QAEhC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;uBAYd,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;8BAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBACxC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE;;;;YAInD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE;OACtD;iBACE,OAAO,CACN,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAC9G;iBACA,IAAI,EAAE,CAAC;YAEV,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9C,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACrG,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC3D,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAA;;;;8BAIH,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;;;;;;6BAOlC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;oCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,EAAE;8BACjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE;aACnD,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACpE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,mBAAmB,CAAC,MAAc,EAAE,EAAgB;QAClD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAgB;QACxD;;;WAGG;QACH,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,OAAO,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxC;;;mBAGG;gBACH,MAAM,EAAE,CAAC,GAAG,CAAA,mDAAmD,CAAC,OAAO,EAAE,CAAC;gBAE1E,IAAI,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAwB,IAAI,CAAC;oBACzC,IAAI,QAAQ,GAAwB,IAAI,CAAC;oBACzC,IAAI,OAAO,GAAG,KAAK,CAAC;oBAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;oBAC9G,IAAI,KAAK,EAAE,MAAM,UAAU,IAAI,EAAE,CAAC,UAAU,CAA0B,GAAG,CAAA;;;;;;;;;;;2BAWxD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;;;;gBAIxC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE;WAClD,CAAC,EAAE,CAAC;wBACH,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC1D,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;4BAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gCACpF,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;gCAC7D,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC;gCACpB,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oCACjC,OAAO,GAAG,IAAI,CAAC;gCACjB,CAAC;gCACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;oCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;wCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;oCAC1B,CAAC;gCACH,CAAC;4BACH,CAAC;iCAAM,CAAC;gCACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAC1E,CAAC;4BACJ,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;wBAC/B,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;oBAE7C,MAAM,EAAE,CAAC,GAAG,CAAA;;;2BAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;WACnD,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;kBAYJ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kBACtC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;kBAClC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;kBAC/C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;WAExC,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,MAAM,EAAE,CAAC,GAAG,CAAA,UAAU,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
|
@@ -13,6 +13,10 @@ export type PostgresSyncRulesStorageOptions = {
|
|
|
13
13
|
batchLimits: RequiredOperationBatchLimits;
|
|
14
14
|
checksumCacheTtlMs?: number;
|
|
15
15
|
};
|
|
16
|
+
/**
|
|
17
|
+
* Number of rows deleted per batch when clearing storage for a replication stream.
|
|
18
|
+
*/
|
|
19
|
+
export declare const CLEAR_BATCH_LIMIT = 50000;
|
|
16
20
|
export declare class PostgresSyncRulesStorage extends framework.BaseObserver<storage.SyncRulesBucketStorageListener> implements storage.SyncRulesBucketStorage {
|
|
17
21
|
protected options: PostgresSyncRulesStorageOptions;
|
|
18
22
|
[framework.DO_NOT_LOG]: boolean;
|
|
@@ -51,7 +55,7 @@ export declare class PostgresSyncRulesStorage extends framework.BaseObserver<sto
|
|
|
51
55
|
populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults>;
|
|
52
56
|
lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
|
|
53
57
|
setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void;
|
|
54
|
-
createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<
|
|
58
|
+
createManagedWriteCheckpoints(checkpoints: storage.ManagedWriteCheckpointOptions[]): Promise<storage.CreateManagedWriteCheckpointsResult>;
|
|
55
59
|
getCheckpoint(): Promise<storage.ReplicationCheckpoint>;
|
|
56
60
|
createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
|
|
57
61
|
/**
|
|
@@ -65,6 +69,17 @@ export declare class PostgresSyncRulesStorage extends framework.BaseObserver<sto
|
|
|
65
69
|
terminate(options?: storage.TerminateOptions): Promise<void>;
|
|
66
70
|
getStatus(): Promise<storage.ReplicationStreamStatus>;
|
|
67
71
|
clear(options?: storage.ClearStorageOptions): Promise<void>;
|
|
72
|
+
private clearBatched;
|
|
73
|
+
/**
|
|
74
|
+
* Delete up to {@link CLEAR_BATCH_LIMIT} rows for this group from the given table,
|
|
75
|
+
* returning the number of candidate rows found by the scan.
|
|
76
|
+
*
|
|
77
|
+
* The count is taken from the scan (`batch`) rather than the delete: with a
|
|
78
|
+
* concurrent process deleting the same rows, the delete may remove fewer rows
|
|
79
|
+
* than the scan found, and counting deleted rows could stop the loop while
|
|
80
|
+
* rows remain.
|
|
81
|
+
*/
|
|
82
|
+
private deleteGroupBatch;
|
|
68
83
|
private getChecksumsInternal;
|
|
69
84
|
getActiveCheckpoint(): Promise<storage.ReplicationCheckpoint>;
|
|
70
85
|
watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate>;
|