@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/storage/MongoBucketStorage.d.ts +31 -3
- package/dist/storage/MongoBucketStorage.js +365 -206
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +15 -4
- package/dist/storage/implementation/MongoBucketBatch.js +35 -9
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +19 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +29 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +4 -2
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +3 -1
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +27 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +78 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +21 -9
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +7 -28
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +27 -0
- package/dist/storage/implementation/common/PersistedBatch.js +43 -2
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +14 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +6 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +80 -84
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +36 -5
- package/dist/storage/implementation/v1/MongoCompactorV1.js +229 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +65 -16
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +120 -39
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +77 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +585 -141
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +91 -31
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +2 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.js +52 -11
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +89 -5
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +8 -4
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +3 -5
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +10 -2
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +177 -23
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +23 -1
- package/dist/types/types.js +16 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +495 -268
- package/src/storage/implementation/MongoBucketBatch.ts +54 -14
- package/src/storage/implementation/MongoCompactor.ts +35 -346
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +34 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +4 -2
- package/src/storage/implementation/MongoStorageProvider.ts +3 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +114 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +22 -9
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +12 -44
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +54 -2
- package/src/storage/implementation/db.ts +21 -2
- package/src/storage/implementation/models.ts +38 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +107 -96
- package/src/storage/implementation/v1/MongoCompactorV1.ts +303 -19
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +87 -27
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +159 -42
- package/src/storage/implementation/v3/MongoCompactorV3.ts +744 -190
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +111 -41
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +57 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +96 -3
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +16 -8
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +15 -4
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +257 -27
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +24 -7
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +605 -16
- package/test/src/cleanup-stopped-sync-configs.test.ts +40 -13
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +273 -22
- package/test/src/storage_compacting.test.ts +714 -244
- package/test/src/storage_s3_checksums.test.ts +3 -4
- package/test/src/storage_s3_compaction_lifecycle.test.ts +138 -12
- package/test/src/storage_s3_reading.test.ts +243 -1
- package/test/src/storage_s3_writing.test.ts +1 -0
- package/test/src/storage_sync.test.ts +365 -41
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
3
2
|
/**
|
|
4
|
-
* Update pipeline to
|
|
3
|
+
* Update pipeline to stop a deploying replication stream, covering all storage versions.
|
|
5
4
|
*
|
|
6
5
|
* Roughly equivalent to:
|
|
7
6
|
* $set: {
|
|
8
|
-
* state:
|
|
9
|
-
* 'sync_configs.$[].state':
|
|
7
|
+
* state: STOP,
|
|
8
|
+
* 'sync_configs.$[].state': STOP
|
|
10
9
|
* }
|
|
11
10
|
*
|
|
12
11
|
* The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
|
|
12
|
+
*
|
|
13
|
+
* This can be used for specific use cases:
|
|
14
|
+
* 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
|
|
15
|
+
* 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
|
|
16
|
+
*
|
|
17
|
+
* Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
|
|
18
|
+
* In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
|
|
13
19
|
*/
|
|
14
|
-
export declare function
|
|
20
|
+
export declare function stopReplicationStreamPipeline(): mongo.Document[];
|
|
@@ -1,19 +1,27 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
1
2
|
/**
|
|
2
|
-
* Update pipeline to
|
|
3
|
+
* Update pipeline to stop a deploying replication stream, covering all storage versions.
|
|
3
4
|
*
|
|
4
5
|
* Roughly equivalent to:
|
|
5
6
|
* $set: {
|
|
6
|
-
* state:
|
|
7
|
-
* 'sync_configs.$[].state':
|
|
7
|
+
* state: STOP,
|
|
8
|
+
* 'sync_configs.$[].state': STOP
|
|
8
9
|
* }
|
|
9
10
|
*
|
|
10
11
|
* The difference is that this also handles v1 storage cases, where `sync_configs` is not present.
|
|
12
|
+
*
|
|
13
|
+
* This can be used for specific use cases:
|
|
14
|
+
* 1. Stop an existing PROCESSING stream, when it is being replaced by a new PROCESSING one.
|
|
15
|
+
* 2. Stop an existing ACTIVE or ERRORED stream, when it is being replaced by a new ACTIVE one.
|
|
16
|
+
*
|
|
17
|
+
* Do not use this when an ACTIVE stream transitions to ERRORED, and is being replaced by a new PROCESSING one.
|
|
18
|
+
* In that case, we need to maintain the ERRORED state, which this pipeline does not cover.
|
|
11
19
|
*/
|
|
12
|
-
export function
|
|
20
|
+
export function stopReplicationStreamPipeline() {
|
|
13
21
|
return [
|
|
14
22
|
{
|
|
15
23
|
$set: {
|
|
16
|
-
state,
|
|
24
|
+
state: storage.SyncRuleState.STOP,
|
|
17
25
|
sync_configs: {
|
|
18
26
|
$cond: [
|
|
19
27
|
{ $isArray: '$sync_configs' },
|
|
@@ -22,7 +30,7 @@ export function syncRuleStateUpdatePipeline(state) {
|
|
|
22
30
|
input: '$sync_configs',
|
|
23
31
|
as: 'config',
|
|
24
32
|
in: {
|
|
25
|
-
$mergeObjects: ['$$config', { state }]
|
|
33
|
+
$mergeObjects: ['$$config', { state: storage.SyncRuleState.STOP }]
|
|
26
34
|
}
|
|
27
35
|
}
|
|
28
36
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SyncRuleStateUpdate.js","sourceRoot":"","sources":["../../../src/storage/implementation/SyncRuleStateUpdate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SyncRuleStateUpdate.js","sourceRoot":"","sources":["../../../src/storage/implementation/SyncRuleStateUpdate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,6BAA6B;IAC3C,OAAO;QACL;YACE,IAAI,EAAE;gBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;gBACjC,YAAY,EAAE;oBACZ,KAAK,EAAE;wBACL,EAAE,QAAQ,EAAE,eAAe,EAAE;wBAC7B;4BACE,IAAI,EAAE;gCACJ,KAAK,EAAE,eAAe;gCACtB,EAAE,EAAE,QAAQ;gCACZ,EAAE,EAAE;oCACF,aAAa,EAAE,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;iCACnE;6BACF;yBACF;wBACD,UAAU;qBACX;iBACF;aACF;SACF;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1,8 +1,28 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
2
|
+
import { InternalOpId, ReplicationCheckpoint } from '@powersync/service-core';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
|
-
export interface MongoSyncBucketStorageCheckpoint {
|
|
4
|
+
export interface MongoSyncBucketStorageCheckpoint extends ReplicationCheckpoint {
|
|
5
5
|
checkpoint: InternalOpId;
|
|
6
6
|
snapshotTime: bson.Timestamp;
|
|
7
7
|
clusterTime: mongo.ClusterTime;
|
|
8
|
+
/**
|
|
9
|
+
* The stream's `parameter_compaction.checkpoint_changes_invalid_before` boundary, read in the
|
|
10
|
+
* same snapshot as {@link checkpoint} and {@link snapshotTime}.
|
|
11
|
+
*
|
|
12
|
+
* Parameter compaction may have deleted parameter entries below this boundary, so checkpoint
|
|
13
|
+
* change detection cannot enumerate individual lookup changes from a checkpoint below it.
|
|
14
|
+
*
|
|
15
|
+
* 0n for streams that have never been compacted.
|
|
16
|
+
*/
|
|
17
|
+
parameterChangesInvalidBefore: InternalOpId;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* MongoDB-specific version of GetCheckpointChangesOptions.
|
|
21
|
+
*
|
|
22
|
+
* The next checkpoint carries the snapshot and invalidation boundary that change detection must
|
|
23
|
+
* be evaluated against. Only `checkpoint` is used from the previous one.
|
|
24
|
+
*/
|
|
25
|
+
export interface MongoGetCheckpointChangesOptions {
|
|
26
|
+
lastCheckpoint: ReplicationCheckpoint;
|
|
27
|
+
nextCheckpoint: MongoSyncBucketStorageCheckpoint;
|
|
8
28
|
}
|
|
@@ -35,6 +35,11 @@ export interface PersistedBatchOptions {
|
|
|
35
35
|
logger?: Logger;
|
|
36
36
|
objectStorage?: ObjectStorage;
|
|
37
37
|
inlineThresholdBytes?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Aborts in-flight object storage uploads when replication stops.
|
|
40
|
+
*/
|
|
41
|
+
signal?: AbortSignal;
|
|
42
|
+
objectStorageUsageWriterId?: string;
|
|
38
43
|
}
|
|
39
44
|
/**
|
|
40
45
|
* Keeps track of bulkwrite operations within a transaction.
|
|
@@ -52,6 +57,8 @@ export declare abstract class PersistedBatch {
|
|
|
52
57
|
bucketStates: Map<string, BucketStateUpdate>;
|
|
53
58
|
protected readonly objectStorage?: ObjectStorage;
|
|
54
59
|
protected readonly inlineThresholdBytes: number;
|
|
60
|
+
protected readonly signal?: AbortSignal;
|
|
61
|
+
protected readonly objectStorageUsageWriterId?: string;
|
|
55
62
|
/**
|
|
56
63
|
* For debug logging only.
|
|
57
64
|
*/
|
|
@@ -74,14 +81,32 @@ export declare abstract class PersistedBatch {
|
|
|
74
81
|
protected abstract resetCurrentData(): void;
|
|
75
82
|
protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
|
|
76
83
|
protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
|
|
84
|
+
/**
|
|
85
|
+
* New bucket operations persist protocol subkeys during replication. Older
|
|
86
|
+
* V1 documents may not have one, so sync reads retain a fallback.
|
|
87
|
+
*/
|
|
88
|
+
protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string;
|
|
77
89
|
protected get bucketDataCount(): number;
|
|
78
90
|
protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* V3 persists operations in chunks. Keep this separate from incrementBucket:
|
|
93
|
+
* operation counts are known while evaluating rows, while chunk counts and
|
|
94
|
+
* exact persisted bytes are only known after the writer has chunked a flush.
|
|
95
|
+
*/
|
|
96
|
+
protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number): void;
|
|
97
|
+
/**
|
|
98
|
+
* V3's compactor calculates byte deltas from persisted chunk metadata.
|
|
99
|
+
* Replace the writer's per-operation estimate with those exact sizes before
|
|
100
|
+
* flushing the corresponding bucket state.
|
|
101
|
+
*/
|
|
102
|
+
protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string): void;
|
|
79
103
|
protected addBucketDataPut(options: {
|
|
80
104
|
op_id: InternalOpId;
|
|
81
105
|
bucketKey: BucketKey;
|
|
82
106
|
bucket: string;
|
|
83
107
|
sourceTableId: storage.SourceTable['id'];
|
|
84
108
|
sourceKey: storage.ReplicaId;
|
|
109
|
+
subkey: string;
|
|
85
110
|
table: string;
|
|
86
111
|
rowId: string;
|
|
87
112
|
checksum: bigint;
|
|
@@ -92,6 +117,7 @@ export declare abstract class PersistedBatch {
|
|
|
92
117
|
bucketKey: BucketKey;
|
|
93
118
|
sourceTableId: storage.SourceTable['id'];
|
|
94
119
|
sourceKey: storage.ReplicaId;
|
|
120
|
+
subkey: string;
|
|
95
121
|
table: string;
|
|
96
122
|
rowId: string;
|
|
97
123
|
checksum: bigint;
|
|
@@ -110,4 +136,5 @@ export interface BucketStateUpdate {
|
|
|
110
136
|
lastOp: InternalOpId;
|
|
111
137
|
incrementCount: number;
|
|
112
138
|
incrementBytes: number;
|
|
139
|
+
incrementChunks: number;
|
|
113
140
|
}
|
|
@@ -39,6 +39,8 @@ export class PersistedBatch {
|
|
|
39
39
|
bucketStates = new Map();
|
|
40
40
|
objectStorage;
|
|
41
41
|
inlineThresholdBytes = DEFAULT_INLINE_THRESHOLD_BYTES;
|
|
42
|
+
signal;
|
|
43
|
+
objectStorageUsageWriterId;
|
|
42
44
|
/**
|
|
43
45
|
* For debug logging only.
|
|
44
46
|
*/
|
|
@@ -54,6 +56,8 @@ export class PersistedBatch {
|
|
|
54
56
|
this.currentSize = writtenSize;
|
|
55
57
|
this.logger = options?.logger ?? defaultLogger;
|
|
56
58
|
this.objectStorage = options?.objectStorage;
|
|
59
|
+
this.signal = options?.signal;
|
|
60
|
+
this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
|
|
57
61
|
if (options?.inlineThresholdBytes != null) {
|
|
58
62
|
this.inlineThresholdBytes = options.inlineThresholdBytes;
|
|
59
63
|
}
|
|
@@ -74,7 +78,8 @@ export class PersistedBatch {
|
|
|
74
78
|
}
|
|
75
79
|
remaining_buckets.set(currentBucketKey(mapped), mapped);
|
|
76
80
|
}
|
|
77
|
-
const
|
|
81
|
+
const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
|
|
82
|
+
const dchecksum = BigInt(utils.hashDelete(subkey));
|
|
78
83
|
for (const evaluated of options.evaluated) {
|
|
79
84
|
const definitionId = this.getBucketDefinitionId(evaluated.source);
|
|
80
85
|
const key = currentBucketKey({
|
|
@@ -104,6 +109,7 @@ export class PersistedBatch {
|
|
|
104
109
|
bucket: evaluated.bucket,
|
|
105
110
|
sourceTableId: options.table.id,
|
|
106
111
|
sourceKey: options.sourceKey,
|
|
112
|
+
subkey,
|
|
107
113
|
table: evaluated.table,
|
|
108
114
|
rowId: evaluated.id,
|
|
109
115
|
checksum: BigInt(checksum),
|
|
@@ -124,6 +130,7 @@ export class PersistedBatch {
|
|
|
124
130
|
op_id,
|
|
125
131
|
sourceTableId: options.table.id,
|
|
126
132
|
sourceKey: options.sourceKey,
|
|
133
|
+
subkey,
|
|
127
134
|
table: bucket.table,
|
|
128
135
|
rowId: bucket.id,
|
|
129
136
|
checksum: dchecksum
|
|
@@ -132,6 +139,13 @@ export class PersistedBatch {
|
|
|
132
139
|
this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
|
|
133
140
|
}
|
|
134
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* New bucket operations persist protocol subkeys during replication. Older
|
|
144
|
+
* V1 documents may not have one, so sync reads retain a fallback.
|
|
145
|
+
*/
|
|
146
|
+
persistedSubkey(table, key) {
|
|
147
|
+
return replicaIdToSubkey(table, key);
|
|
148
|
+
}
|
|
135
149
|
get bucketDataCount() {
|
|
136
150
|
return this.bucketData.length;
|
|
137
151
|
}
|
|
@@ -149,10 +163,35 @@ export class PersistedBatch {
|
|
|
149
163
|
bucket,
|
|
150
164
|
lastOp: op_id,
|
|
151
165
|
incrementCount: 1,
|
|
152
|
-
incrementBytes: bytes
|
|
166
|
+
incrementBytes: bytes,
|
|
167
|
+
incrementChunks: 0
|
|
153
168
|
});
|
|
154
169
|
}
|
|
155
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* V3 persists operations in chunks. Keep this separate from incrementBucket:
|
|
173
|
+
* operation counts are known while evaluating rows, while chunk counts and
|
|
174
|
+
* exact persisted bytes are only known after the writer has chunked a flush.
|
|
175
|
+
*/
|
|
176
|
+
incrementBucketPersistedChunk(definitionId, bucket, bytes) {
|
|
177
|
+
const key = `${definitionId ?? ''}:${bucket}`;
|
|
178
|
+
const existingState = this.bucketStates.get(key);
|
|
179
|
+
if (existingState != null) {
|
|
180
|
+
existingState.incrementChunks += 1;
|
|
181
|
+
existingState.incrementBytes += bytes;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* V3's compactor calculates byte deltas from persisted chunk metadata.
|
|
186
|
+
* Replace the writer's per-operation estimate with those exact sizes before
|
|
187
|
+
* flushing the corresponding bucket state.
|
|
188
|
+
*/
|
|
189
|
+
resetBucketPersistedBytes(definitionId, bucket) {
|
|
190
|
+
const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
|
|
191
|
+
if (state != null) {
|
|
192
|
+
state.incrementBytes = 0;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
156
195
|
addBucketDataPut(options) {
|
|
157
196
|
this.bucketData.push({
|
|
158
197
|
bucketKey: options.bucketKey,
|
|
@@ -160,6 +199,7 @@ export class PersistedBatch {
|
|
|
160
199
|
op: 'PUT',
|
|
161
200
|
source_table: mongoTableId(options.sourceTableId),
|
|
162
201
|
source_key: options.sourceKey,
|
|
202
|
+
subkey: options.subkey,
|
|
163
203
|
table: options.table,
|
|
164
204
|
row_id: options.rowId,
|
|
165
205
|
checksum: options.checksum,
|
|
@@ -173,6 +213,7 @@ export class PersistedBatch {
|
|
|
173
213
|
op: 'REMOVE',
|
|
174
214
|
source_table: mongoTableId(options.sourceTableId),
|
|
175
215
|
source_key: options.sourceKey,
|
|
216
|
+
subkey: options.subkey,
|
|
176
217
|
table: options.table,
|
|
177
218
|
row_id: options.rowId,
|
|
178
219
|
checksum: options.checksum,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAQ9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAQ9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAqCxD;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IAsBb;IACA;IACA;IAvBrB,MAAM,CAAS;IACf,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEtC,aAAa,CAAiB;IAC9B,oBAAoB,GAAW,8BAA8B,CAAC;IAC9D,MAAM,CAAe;IACrB,0BAA0B,CAAU;IAEvD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAAgC,EACnD,WAAmB,EACnB,OAA+B;QAJZ,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAyB;QAInD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,0BAA0B,GAAG,OAAO,EAAE,0BAA0B,CAAC;QACtE,IAAI,OAAO,EAAE,oBAAoB,IAAI,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACzD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IA6BD;;;OAGG;IACO,eAAe,CAAC,KAA4B,EAAE,GAAsB;QAC5E,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,IAAc,eAAe;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAES,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAC5G,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,6BAA6B,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAa;QACrG,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,eAAe,IAAI,CAAC,CAAC;YACnC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,yBAAyB,CAAC,YAAgC,EAAE,MAAc;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAES,gBAAgB,CAAC,OAW1B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAES,mBAAmB,CAAC,OAS7B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,IAAI,CAAC,gBAAgB,IAAI,yBAAyB;YAClD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA0C;QAClF,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;wBAC3C,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;qBAC5C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -6,7 +6,8 @@ import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBas
|
|
|
6
6
|
import { CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, StorageConfig, SyncRuleDocumentBase, WriteCheckpointDocument } from './models.js';
|
|
7
7
|
import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './v1/models.js';
|
|
8
8
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
9
|
-
import { BucketDataDocumentV3 } from './v3/models.js';
|
|
9
|
+
import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
|
|
10
|
+
import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
|
|
10
11
|
import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
11
12
|
export interface PowerSyncMongoOptions {
|
|
12
13
|
/**
|
|
@@ -62,6 +63,8 @@ export declare class PowerSyncMongo {
|
|
|
62
63
|
listAllBucketStateCollectionsV3(): Promise<mongo.Collection<never>[]>;
|
|
63
64
|
sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId): string;
|
|
64
65
|
sourceTableCollectionName(replicationStreamId: number): string;
|
|
66
|
+
customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier): string;
|
|
67
|
+
listCustomCheckpointRequestCollections(replicationStreamId?: number): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]>;
|
|
65
68
|
listSourceTableCollections(replicationStreamId?: number): Promise<mongo.Collection<CommonSourceTableDocument>[]>;
|
|
66
69
|
/**
|
|
67
70
|
* Clear all collections.
|
|
@@ -2,7 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
3
3
|
import { DO_NOT_LOG } from '@powersync/lib-services-framework';
|
|
4
4
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
5
|
-
import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
5
|
+
import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
6
6
|
export class PowerSyncMongo {
|
|
7
7
|
[DO_NOT_LOG] = true;
|
|
8
8
|
current_data;
|
|
@@ -87,6 +87,15 @@ export class PowerSyncMongo {
|
|
|
87
87
|
sourceTableCollectionName(replicationStreamId) {
|
|
88
88
|
return `source_table_${replicationStreamId}`;
|
|
89
89
|
}
|
|
90
|
+
customCheckpointRequestCollectionName(options) {
|
|
91
|
+
return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
|
|
92
|
+
}
|
|
93
|
+
async listCustomCheckpointRequestCollections(replicationStreamId) {
|
|
94
|
+
const prefix = replicationStreamId == null
|
|
95
|
+
? 'custom_checkpoint_requests_'
|
|
96
|
+
: `custom_checkpoint_requests_${replicationStreamId}_`;
|
|
97
|
+
return this.collectionsByPrefix(prefix);
|
|
98
|
+
}
|
|
90
99
|
async listSourceTableCollections(replicationStreamId) {
|
|
91
100
|
if (replicationStreamId == null) {
|
|
92
101
|
return this.collectionsByPrefix('source_table_');
|
|
@@ -125,6 +134,10 @@ export class PowerSyncMongo {
|
|
|
125
134
|
await this.locks.deleteMany({});
|
|
126
135
|
await this.bucket_state.deleteMany({});
|
|
127
136
|
await this.custom_write_checkpoints.deleteMany({});
|
|
137
|
+
await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
|
|
138
|
+
for (const collection of await this.listCustomCheckpointRequestCollections()) {
|
|
139
|
+
await collection.drop();
|
|
140
|
+
}
|
|
128
141
|
}
|
|
129
142
|
/**
|
|
130
143
|
* Drop the entire database.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,EAAE,+BAA+B,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS/G,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAyC;IACnD,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAKD,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACvD,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAC,OAAgB;QAChD,MAAM,MAAM,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,OAAO,GAAG,CAAC;QAC5E,OAAO,IAAI,CAAC,mBAAmB,CAAuB,MAAM,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAmC,MAAc;QAChF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kCAAkC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gCAAgC;QACpC,OAAO,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QACpF,OAAO,kBAAkB,mBAAmB,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;IAChF,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,gBAAgB,mBAAmB,EAAE,CAAC;IAC/C,CAAC;IAED,qCAAqC,CAAC,OAAmD;QACvF,OAAO,8BAA8B,OAAO,CAAC,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,sCAAsC,CAC1C,mBAA4B;QAE5B,MAAM,MAAM,GACV,mBAAmB,IAAI,IAAI;YACzB,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,8BAA8B,mBAAmB,GAAG,CAAC;QAC3D,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,mBAA4B;QAE5B,IAAI,mBAAmB,IAAI,IAAI,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC;YACvE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC;YACzE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,+BAA+B,EAAE,EAAE,CAAC;YACtE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,0BAA0B,EAAE,EAAE,CAAC;YACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzE,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,sCAAsC,EAAE,EAAE,CAAC;YAC7E,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE;YAChD,mBAAmB,EAAE,IAAI;YACzB,8DAA8D;YAC9D,qGAAqG;YACrG,YAAY,EAAE;gBACZ,CAAC,EAAE,CAAC;aACL;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAOD,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -55,6 +55,12 @@ export interface BucketDataProperties {
|
|
|
55
55
|
op: OpType;
|
|
56
56
|
source_table?: bson.ObjectId;
|
|
57
57
|
source_key?: ReplicaId;
|
|
58
|
+
/**
|
|
59
|
+
* Stable identifier for the source record in the client protocol.
|
|
60
|
+
*
|
|
61
|
+
* This is derived from (source_table, source_key), so persistence is optional but preferred.
|
|
62
|
+
*/
|
|
63
|
+
subkey?: string;
|
|
58
64
|
table?: string;
|
|
59
65
|
row_id?: string;
|
|
60
66
|
checksum: bigint;
|
|
@@ -93,6 +99,10 @@ export interface SourceTableDocument {
|
|
|
93
99
|
}[] | undefined;
|
|
94
100
|
snapshot_done: boolean | undefined;
|
|
95
101
|
snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
|
|
102
|
+
/**
|
|
103
|
+
* Source-specific metadata. Absent for legacy records.
|
|
104
|
+
*/
|
|
105
|
+
source_metadata?: storage.JsonValue;
|
|
96
106
|
}
|
|
97
107
|
export interface SourceTableDocumentSnapshotStatus {
|
|
98
108
|
total_estimated_count: number;
|
|
@@ -180,6 +190,31 @@ export interface SyncRuleDocumentBase {
|
|
|
180
190
|
expires_at: Date;
|
|
181
191
|
} | null;
|
|
182
192
|
storage_version?: number;
|
|
193
|
+
/** Optional operator-supplied label identifying the sync config version. */
|
|
194
|
+
version_label?: string | null;
|
|
195
|
+
/**
|
|
196
|
+
* Incremental parameter compaction state for the replication stream.
|
|
197
|
+
*
|
|
198
|
+
* Operation ids are allocated across all parameter indexes of a stream, so this is
|
|
199
|
+
* stream-level state, shared by all sync configs of the stream.
|
|
200
|
+
*/
|
|
201
|
+
parameter_compaction?: {
|
|
202
|
+
/**
|
|
203
|
+
* The exclusive operation-id boundary through which every parameter index in this stream has
|
|
204
|
+
* been compacted.
|
|
205
|
+
*/
|
|
206
|
+
compacted_before: InternalOpId;
|
|
207
|
+
/**
|
|
208
|
+
* Parameter entries below this boundary may no longer be available for checkpoint change
|
|
209
|
+
* detection, since compaction may have deleted them.
|
|
210
|
+
*
|
|
211
|
+
* This is advanced before the first delete of a compaction pass, while
|
|
212
|
+
* {@link compacted_before} is only advanced after every delete of the pass completed.
|
|
213
|
+
* A checkpoint transition starting below this boundary must conservatively invalidate all
|
|
214
|
+
* parameter buckets, since the individual changes may no longer be available.
|
|
215
|
+
*/
|
|
216
|
+
checkpoint_changes_invalid_before?: InternalOpId;
|
|
217
|
+
};
|
|
183
218
|
}
|
|
184
219
|
export interface SyncRuleCheckpointFields {
|
|
185
220
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;AAiCD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAoLpD,MAAM,8BAA8B,GAAG,CAAC,CAAC;AACzC,MAAM,wCAAwC,GAAG,OAAO,CAAC,iBAAiB,CAAC;AAE3E,MAAM,UAAU,qBAAqB,CAAC,cAAsB;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,cAAc,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,OAAO;QACL,GAAG,UAAU;QACb,aAAa,EAAE,cAAc,IAAI,8BAA8B;QAC/D,uBAAuB,EAAE,cAAc,IAAI,wCAAwC;KACpF,CAAC;AACJ,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { InternalOpId, SourceTable, storage } from '@powersync/service-core';
|
|
2
3
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
3
4
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
@@ -20,7 +21,12 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
20
21
|
protected createPersistedBatch(writtenSize: number): PersistedBatch;
|
|
21
22
|
protected get sourceRecordStore(): SourceRecordStore;
|
|
22
23
|
protected cleanupDroppedSourceTables(_tables: SourceTable[]): Promise<void>;
|
|
24
|
+
protected batchCreateCustomWriteCheckpoints(session: mongo.ClientSession, opId: InternalOpId): Promise<void>;
|
|
23
25
|
resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
|
|
26
|
+
/**
|
|
27
|
+
* Build a SourceTable from its persisted v1 record.
|
|
28
|
+
*/
|
|
29
|
+
private hydrateSourceTable;
|
|
24
30
|
getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
|
|
25
31
|
commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
|
|
26
32
|
keepalive(lsn: string): Promise<storage.CheckpointResult>;
|