@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +1 -1
- package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +1 -1
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +3 -3
- package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
- package/dist/migrations/db/migrations/1770213298299-storage-version.js.map +1 -1
- package/dist/storage/MongoBucketStorage.d.ts +5 -3
- package/dist/storage/MongoBucketStorage.js +50 -36
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/MongoReportStorage.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +17 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +58 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/MongoBucketBatch.d.ts +16 -14
- package/dist/storage/implementation/MongoBucketBatch.js +80 -115
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatchShared.d.ts +5 -0
- package/dist/storage/implementation/MongoBucketBatchShared.js +8 -0
- package/dist/storage/implementation/MongoBucketBatchShared.js.map +1 -0
- package/dist/storage/implementation/MongoChecksums.d.ts +28 -17
- package/dist/storage/implementation/MongoChecksums.js +13 -72
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +98 -58
- package/dist/storage/implementation/MongoCompactor.js +229 -296
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +11 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +11 -8
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +14 -0
- package/dist/storage/implementation/MongoPersistedSyncRules.js +64 -0
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +3 -0
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +9 -0
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +49 -30
- package/dist/storage/implementation/MongoSyncBucketStorage.js +96 -388
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js +12 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/OperationBatch.js +1 -1
- package/dist/storage/implementation/common/BucketDataDoc.d.ts +35 -0
- package/dist/storage/implementation/common/BucketDataDoc.js +2 -0
- package/dist/storage/implementation/common/BucketDataDoc.js.map +1 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +13 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +108 -0
- package/dist/storage/implementation/common/PersistedBatch.js +237 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -0
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +54 -0
- package/dist/storage/implementation/common/SingleBucketStore.js +3 -0
- package/dist/storage/implementation/common/SingleBucketStore.js.map +1 -0
- package/dist/storage/implementation/common/SourceRecordStore.d.ts +36 -0
- package/dist/storage/implementation/common/SourceRecordStore.js +2 -0
- package/dist/storage/implementation/common/SourceRecordStore.js.map +1 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +27 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +57 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -0
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +7 -0
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +9 -0
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -0
- package/dist/storage/implementation/db.d.ts +34 -34
- package/dist/storage/implementation/db.js +78 -98
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +63 -34
- package/dist/storage/implementation/models.js +21 -2
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +13 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +22 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -0
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +12 -0
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +56 -0
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -0
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +23 -0
- package/dist/storage/implementation/v1/MongoCompactorV1.js +52 -0
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -0
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +9 -0
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -0
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -0
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +41 -0
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +283 -0
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -0
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +26 -0
- package/dist/storage/implementation/v1/PersistedBatchV1.js +183 -0
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -0
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +18 -0
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +57 -0
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -0
- package/dist/storage/implementation/v1/SourceRecordStoreV1.d.ts +19 -0
- package/dist/storage/implementation/v1/SourceRecordStoreV1.js +105 -0
- package/dist/storage/implementation/v1/SourceRecordStoreV1.js.map +1 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +12 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +20 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -0
- package/dist/storage/implementation/v1/models.d.ts +34 -0
- package/dist/storage/implementation/v1/models.js +37 -0
- package/dist/storage/implementation/v1/models.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +13 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +34 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +15 -0
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +84 -0
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +23 -0
- package/dist/storage/implementation/v3/MongoCompactorV3.js +68 -0
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +9 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +5 -0
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +9 -0
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +41 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +407 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +29 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.js +259 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -0
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +18 -0
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +48 -0
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -0
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +22 -0
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +164 -0
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +21 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +71 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +43 -0
- package/dist/storage/implementation/v3/models.js +34 -0
- package/dist/storage/implementation/v3/models.js.map +1 -0
- package/dist/storage/storage-index.d.ts +8 -5
- package/dist/storage/storage-index.js +8 -5
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/utils/util.d.ts +11 -4
- package/dist/utils/util.js +25 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +9 -9
- package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +1 -1
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -7
- package/src/migrations/db/migrations/1770213298299-storage-version.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +97 -62
- package/src/storage/MongoReportStorage.ts +2 -2
- package/src/storage/implementation/BucketDefinitionMapping.ts +72 -0
- package/src/storage/implementation/MongoBucketBatch.ts +110 -144
- package/src/storage/implementation/MongoBucketBatchShared.ts +11 -0
- package/src/storage/implementation/MongoChecksums.ts +53 -76
- package/src/storage/implementation/MongoCompactor.ts +374 -404
- package/src/storage/implementation/MongoParameterCompactor.ts +37 -24
- package/src/storage/implementation/MongoPersistedSyncRules.ts +76 -0
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +18 -1
- package/src/storage/implementation/MongoStorageProvider.ts +1 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +190 -457
- package/src/storage/implementation/MongoSyncRulesLock.ts +12 -14
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +4 -2
- package/src/storage/implementation/OperationBatch.ts +1 -1
- package/src/storage/implementation/common/BucketDataDoc.ts +37 -0
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +15 -0
- package/src/storage/implementation/common/PersistedBatch.ts +364 -0
- package/src/storage/implementation/common/SingleBucketStore.ts +63 -0
- package/src/storage/implementation/common/SourceRecordStore.ts +49 -0
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +80 -0
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +25 -0
- package/src/storage/implementation/db.ts +107 -128
- package/src/storage/implementation/models.ts +84 -38
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +32 -0
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +75 -0
- package/src/storage/implementation/v1/MongoCompactorV1.ts +93 -0
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +26 -0
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +448 -0
- package/src/storage/implementation/v1/PersistedBatchV1.ts +230 -0
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +74 -0
- package/src/storage/implementation/v1/SourceRecordStoreV1.ts +156 -0
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +28 -0
- package/src/storage/implementation/v1/models.ts +84 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +44 -0
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +120 -0
- package/src/storage/implementation/v3/MongoCompactorV3.ts +107 -0
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +24 -0
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +12 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +550 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +318 -0
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +68 -0
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +226 -0
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +112 -0
- package/src/storage/implementation/v3/models.ts +96 -0
- package/src/storage/storage-index.ts +8 -5
- package/src/utils/util.ts +36 -7
- package/test/src/__snapshots__/storage_sync.test.ts.snap +282 -0
- package/test/src/connection-report-storage.test.ts +3 -3
- package/test/src/setup.ts +1 -1
- package/test/src/storage.test.ts +2 -2
- package/test/src/storage_compacting.test.ts +57 -29
- package/test/src/storage_sync.test.ts +351 -5
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/PersistedBatch.d.ts +0 -71
- package/dist/storage/implementation/PersistedBatch.js +0 -354
- package/dist/storage/implementation/PersistedBatch.js.map +0 -1
- package/src/storage/implementation/PersistedBatch.ts +0 -432
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
|
|
4
|
+
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
5
|
+
import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, TaggedBucketParameterDocument } from '../models.js';
|
|
6
|
+
export interface CurrentBucketV3 extends CurrentBucket {
|
|
7
|
+
def: BucketDefinitionId;
|
|
8
|
+
}
|
|
9
|
+
export interface RecordedLookupV3 {
|
|
10
|
+
i: ParameterIndexId;
|
|
11
|
+
l: bson.Binary;
|
|
12
|
+
}
|
|
13
|
+
export interface CurrentDataDocumentV3 {
|
|
14
|
+
_id: ReplicaId;
|
|
15
|
+
data: bson.Binary | null;
|
|
16
|
+
buckets: CurrentBucketV3[];
|
|
17
|
+
lookups: RecordedLookupV3[];
|
|
18
|
+
/**
|
|
19
|
+
* If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
|
|
20
|
+
*
|
|
21
|
+
* This must only be set if buckets = [], lookups = [].
|
|
22
|
+
*/
|
|
23
|
+
pending_delete?: bigint;
|
|
24
|
+
}
|
|
25
|
+
export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {
|
|
26
|
+
}
|
|
27
|
+
export type BucketDataKeyV3 = BucketDataKey;
|
|
28
|
+
export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
|
|
29
|
+
_id: BucketDataKeyV3;
|
|
30
|
+
}
|
|
31
|
+
export declare function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3;
|
|
32
|
+
export declare function loadBucketDataDocumentV3(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): BucketDataDoc;
|
|
33
|
+
export declare function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
|
|
34
|
+
export interface SourceTableDocumentV3 extends SourceTableDocument {
|
|
35
|
+
bucket_data_source_ids: BucketDefinitionId[];
|
|
36
|
+
parameter_lookup_source_ids: ParameterIndexId[];
|
|
37
|
+
latest_pending_delete?: InternalOpId | undefined;
|
|
38
|
+
}
|
|
39
|
+
export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
|
|
40
|
+
_id: BucketStateDocumentBase['_id'] & {
|
|
41
|
+
d: BucketDefinitionId;
|
|
42
|
+
};
|
|
43
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export function serializeBucketDataV3(document) {
|
|
2
|
+
const { bucketKey, o } = document;
|
|
3
|
+
return {
|
|
4
|
+
_id: {
|
|
5
|
+
b: bucketKey.bucket,
|
|
6
|
+
o: o
|
|
7
|
+
},
|
|
8
|
+
// List fields directly, so that we don't accidentally persist any unknown fields
|
|
9
|
+
op: document.op,
|
|
10
|
+
source_table: document.source_table,
|
|
11
|
+
source_key: document.source_key,
|
|
12
|
+
table: document.table,
|
|
13
|
+
row_id: document.row_id,
|
|
14
|
+
checksum: document.checksum,
|
|
15
|
+
data: document.data,
|
|
16
|
+
target_op: document.target_op
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export function loadBucketDataDocumentV3(context, doc) {
|
|
20
|
+
const { _id, ...rest } = doc;
|
|
21
|
+
return {
|
|
22
|
+
bucketKey: {
|
|
23
|
+
...context,
|
|
24
|
+
bucket: _id.b
|
|
25
|
+
},
|
|
26
|
+
o: _id.o,
|
|
27
|
+
...rest
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function taggedBucketParameterDocumentToV3(document) {
|
|
31
|
+
const { index: _index, ...rest } = document;
|
|
32
|
+
return rest;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=models.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AA8CA,MAAM,UAAU,qBAAqB,CAAC,QAAuB;IAC3D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC;IAClC,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,SAAS,CAAC,MAAM;YACnB,CAAC,EAAE,CAAC;SACL;QACD,iFAAiF;QACjF,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAC7B,OAAO;QACL,SAAS,EAAE;YACT,GAAG,OAAO;YACV,MAAM,EAAE,GAAG,CAAC,CAAC;SACd;QACD,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
export * as test_utils from '../utils/test-utils.js';
|
|
2
|
+
export * from '../utils/util.js';
|
|
3
|
+
export * from './implementation/BucketDefinitionMapping.js';
|
|
4
|
+
export * from './implementation/common/PersistedBatch.js';
|
|
5
|
+
export * from './implementation/createMongoSyncBucketStorage.js';
|
|
1
6
|
export * from './implementation/db.js';
|
|
2
7
|
export * from './implementation/models.js';
|
|
3
|
-
export * from './implementation/MongoBucketBatch.js';
|
|
4
8
|
export * from './implementation/MongoIdSequence.js';
|
|
9
|
+
export * from './implementation/MongoPersistedSyncRules.js';
|
|
5
10
|
export * from './implementation/MongoPersistedSyncRulesContent.js';
|
|
6
11
|
export * from './implementation/MongoStorageProvider.js';
|
|
7
|
-
export * from './implementation/MongoSyncBucketStorage.js';
|
|
8
12
|
export * from './implementation/MongoSyncRulesLock.js';
|
|
9
13
|
export * from './implementation/OperationBatch.js';
|
|
10
|
-
export * from './implementation/
|
|
11
|
-
export * from '
|
|
14
|
+
export * from './implementation/v1/models.js';
|
|
15
|
+
export * from './implementation/v3/models.js';
|
|
12
16
|
export * from './MongoBucketStorage.js';
|
|
13
17
|
export * from './MongoReportStorage.js';
|
|
14
|
-
export * as test_utils from '../utils/test-utils.js';
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
export * as test_utils from '../utils/test-utils.js';
|
|
2
|
+
export * from '../utils/util.js';
|
|
3
|
+
export * from './implementation/BucketDefinitionMapping.js';
|
|
4
|
+
export * from './implementation/common/PersistedBatch.js';
|
|
5
|
+
export * from './implementation/createMongoSyncBucketStorage.js';
|
|
1
6
|
export * from './implementation/db.js';
|
|
2
7
|
export * from './implementation/models.js';
|
|
3
|
-
export * from './implementation/MongoBucketBatch.js';
|
|
4
8
|
export * from './implementation/MongoIdSequence.js';
|
|
9
|
+
export * from './implementation/MongoPersistedSyncRules.js';
|
|
5
10
|
export * from './implementation/MongoPersistedSyncRulesContent.js';
|
|
6
11
|
export * from './implementation/MongoStorageProvider.js';
|
|
7
|
-
export * from './implementation/MongoSyncBucketStorage.js';
|
|
8
12
|
export * from './implementation/MongoSyncRulesLock.js';
|
|
9
13
|
export * from './implementation/OperationBatch.js';
|
|
10
|
-
export * from './implementation/
|
|
11
|
-
export * from '
|
|
14
|
+
export * from './implementation/v1/models.js';
|
|
15
|
+
export * from './implementation/v3/models.js';
|
|
12
16
|
export * from './MongoBucketStorage.js';
|
|
13
17
|
export * from './MongoReportStorage.js';
|
|
14
|
-
export * as test_utils from '../utils/test-utils.js';
|
|
15
18
|
//# sourceMappingURL=storage-index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,6CAA6C,CAAC;AAC5D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
|
package/dist/utils/util.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import * as
|
|
1
|
+
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
3
|
import { storage, utils } from '@powersync/service-core';
|
|
4
|
-
import
|
|
4
|
+
import * as bson from 'bson';
|
|
5
|
+
import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
|
|
5
6
|
export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
|
|
6
7
|
export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
|
|
7
8
|
/**
|
|
@@ -19,12 +20,18 @@ export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Pro
|
|
|
19
20
|
data: T[];
|
|
20
21
|
hasMore: boolean;
|
|
21
22
|
}>;
|
|
22
|
-
export declare function mapOpEntry(row:
|
|
23
|
+
export declare function mapOpEntry(row: BucketDataDoc): utils.OplogEntry;
|
|
23
24
|
export declare function replicaIdToSubkey(table: storage.SourceTableId, id: storage.ReplicaId): string;
|
|
24
25
|
export declare function mongoTableId(table: storage.SourceTableId): bson.ObjectId;
|
|
25
26
|
export declare function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp): void;
|
|
27
|
+
export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise<T>, options: {
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
abortMessage?: string;
|
|
30
|
+
retryDelayMs: number;
|
|
31
|
+
onRetry?: (retryCount: number) => void;
|
|
32
|
+
}): Promise<T>;
|
|
26
33
|
export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
|
|
27
|
-
items: mongo.WithId<T>[];
|
|
34
|
+
items: lib_mongo.mongo.WithId<T>[];
|
|
28
35
|
count: number;
|
|
29
36
|
/** Setting the cursor to the connected at date of the last item in the list */
|
|
30
37
|
cursor: any;
|
package/dist/utils/util.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { storage, utils } from '@powersync/service-core';
|
|
1
4
|
import * as bson from 'bson';
|
|
2
5
|
import * as crypto from 'crypto';
|
|
6
|
+
import * as timers from 'node:timers/promises';
|
|
3
7
|
import * as uuid from 'uuid';
|
|
4
|
-
import { storage, utils } from '@powersync/service-core';
|
|
5
|
-
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
6
8
|
export function idPrefixFilter(prefix, rest) {
|
|
7
9
|
let filter = {
|
|
8
10
|
$gte: {
|
|
@@ -65,7 +67,7 @@ export async function readSingleBatch(cursor) {
|
|
|
65
67
|
export function mapOpEntry(row) {
|
|
66
68
|
if (row.op == 'PUT' || row.op == 'REMOVE') {
|
|
67
69
|
return {
|
|
68
|
-
op_id: utils.internalToExternalOpId(row.
|
|
70
|
+
op_id: utils.internalToExternalOpId(row.o),
|
|
69
71
|
op: row.op,
|
|
70
72
|
object_type: row.table,
|
|
71
73
|
object_id: row.row_id,
|
|
@@ -77,7 +79,7 @@ export function mapOpEntry(row) {
|
|
|
77
79
|
else {
|
|
78
80
|
// MOVE, CLEAR
|
|
79
81
|
return {
|
|
80
|
-
op_id: utils.internalToExternalOpId(row.
|
|
82
|
+
op_id: utils.internalToExternalOpId(row.o),
|
|
81
83
|
op: row.op,
|
|
82
84
|
checksum: Number(row.checksum)
|
|
83
85
|
};
|
|
@@ -120,6 +122,25 @@ export function setSessionSnapshotTime(session, time) {
|
|
|
120
122
|
throw new ServiceAssertionError(`Session snapshotTime is already set`);
|
|
121
123
|
}
|
|
122
124
|
}
|
|
125
|
+
export async function retryOnMongoMaxTimeMSExpired(operation, options) {
|
|
126
|
+
let retryCount = 0;
|
|
127
|
+
while (true) {
|
|
128
|
+
if (options.signal?.aborted) {
|
|
129
|
+
throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
return await operation();
|
|
133
|
+
}
|
|
134
|
+
catch (e) {
|
|
135
|
+
if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
|
|
136
|
+
throw e;
|
|
137
|
+
}
|
|
138
|
+
retryCount += 1;
|
|
139
|
+
options.onRetry?.(retryCount);
|
|
140
|
+
await timers.setTimeout(options.retryDelayMs);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
123
144
|
export const createPaginatedConnectionQuery = async (query, collection, limit, cursor) => {
|
|
124
145
|
const createQuery = (cursor) => {
|
|
125
146
|
if (!cursor) {
|
package/dist/utils/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,EAAE,CAAC;gBAC1E,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.16.0",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-ALv2",
|
|
8
8
|
"type": "module",
|
|
@@ -26,16 +26,16 @@
|
|
|
26
26
|
"ix": "^5.0.0",
|
|
27
27
|
"lru-cache": "^10.2.2",
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
|
-
"uuid": "^
|
|
30
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
31
|
-
"@powersync/lib-services-framework": "0.9.
|
|
32
|
-
"@powersync/service-core": "1.
|
|
33
|
-
"@powersync/service-jsonbig": "0.17.
|
|
34
|
-
"@powersync/service-sync-rules": "0.
|
|
35
|
-
"@powersync/service-types": "0.15.
|
|
29
|
+
"uuid": "^14.0.0",
|
|
30
|
+
"@powersync/lib-service-mongodb": "0.6.25",
|
|
31
|
+
"@powersync/lib-services-framework": "0.9.4",
|
|
32
|
+
"@powersync/service-core": "1.21.0",
|
|
33
|
+
"@powersync/service-jsonbig": "0.17.13",
|
|
34
|
+
"@powersync/service-sync-rules": "0.36.0",
|
|
35
|
+
"@powersync/service-types": "0.15.2"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@powersync/service-core-tests": "0.15.
|
|
38
|
+
"@powersync/service-core-tests": "0.15.5"
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "tsc -b",
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
3
|
import { storage as core_storage, migrations } from '@powersync/service-core';
|
|
3
4
|
import * as storage from '../../../storage/storage-index.js';
|
|
4
5
|
import { MongoStorageConfig } from '../../../types/types.js';
|
|
5
|
-
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
6
6
|
|
|
7
7
|
interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
|
|
8
8
|
/**
|
|
9
|
-
* True if this is the active
|
|
9
|
+
* True if this is the active replication stream.
|
|
10
10
|
* requires `snapshot_done == true` and `replicating == true`.
|
|
11
11
|
*/
|
|
12
12
|
active?: boolean;
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
|
-
* True if this
|
|
15
|
+
* True if this replication stream should be used for replication.
|
|
16
16
|
*
|
|
17
|
-
* During reprocessing, there is one
|
|
17
|
+
* During reprocessing, there is one replication stream with `replicating = true, active = true`,
|
|
18
18
|
* and one with `replicating = true, active = false, auto_activate = true`.
|
|
19
19
|
*/
|
|
20
20
|
replicating?: boolean;
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
|
-
* True if the
|
|
23
|
+
* True if the replication stream should set `active = true` when `snapshot_done` = true.
|
|
24
24
|
*/
|
|
25
25
|
auto_activate?: boolean;
|
|
26
26
|
}
|
|
@@ -35,7 +35,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
|
35
35
|
try {
|
|
36
36
|
// We keep the old flags for existing deployments still shutting down.
|
|
37
37
|
|
|
38
|
-
// 1. New
|
|
38
|
+
// 1. New replication stream: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
|
|
39
39
|
await db.sync_rules.updateMany(
|
|
40
40
|
{
|
|
41
41
|
active: { $ne: true },
|
|
@@ -66,7 +66,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
|
|
|
66
66
|
const remaining = await db.sync_rules.find({ state: null as any }).toArray();
|
|
67
67
|
if (remaining.length > 0) {
|
|
68
68
|
const slots = remaining.map((doc) => doc.slot_name).join(', ');
|
|
69
|
-
throw new ServiceAssertionError(`Invalid state for
|
|
69
|
+
throw new ServiceAssertionError(`Invalid state for replication stream: ${slots}`);
|
|
70
70
|
}
|
|
71
71
|
} finally {
|
|
72
72
|
await db.client.close();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { storage as core_storage, migrations } from '@powersync/service-core';
|
|
2
2
|
import * as mongo_storage from '../../../storage/storage-index.js';
|
|
3
3
|
import { MongoStorageConfig } from '../../../types/types.js';
|
|
4
4
|
|
|
@@ -1,23 +1,27 @@
|
|
|
1
1
|
import { GetIntanceOptions, storage } from '@powersync/service-core';
|
|
2
2
|
|
|
3
|
-
import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
|
|
4
4
|
import { v4 as uuid } from 'uuid';
|
|
5
5
|
|
|
6
6
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
7
7
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
8
8
|
|
|
9
|
+
import { generateSlotName } from '../utils/util.js';
|
|
10
|
+
import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
|
|
11
|
+
import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
|
|
12
|
+
import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
|
|
9
13
|
import { PowerSyncMongo } from './implementation/db.js';
|
|
10
14
|
import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
|
|
11
|
-
import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
|
|
12
|
-
import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
|
|
13
|
-
import { generateSlotName } from '../utils/util.js';
|
|
14
15
|
import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
|
|
16
|
+
import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
|
|
15
17
|
|
|
16
18
|
export interface MongoBucketStorageOptions {
|
|
17
|
-
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
|
|
19
|
+
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
23
|
+
[DO_NOT_LOG] = true;
|
|
24
|
+
|
|
21
25
|
private readonly client: mongo.MongoClient;
|
|
22
26
|
private readonly session: mongo.ClientSession;
|
|
23
27
|
// TODO: This is still Postgres specific and needs to be reworked
|
|
@@ -51,7 +55,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
51
55
|
id = Number(id);
|
|
52
56
|
}
|
|
53
57
|
const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
|
|
54
|
-
const storage =
|
|
58
|
+
const storage = createMongoSyncBucketStorage(
|
|
55
59
|
this,
|
|
56
60
|
id,
|
|
57
61
|
syncRules as MongoPersistedSyncRulesContent,
|
|
@@ -98,7 +102,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
98
102
|
const active = await this.getActiveSyncRulesContent();
|
|
99
103
|
|
|
100
104
|
if (next != null && next.id == sync_rules_group_id) {
|
|
101
|
-
// We need to redo the "next"
|
|
105
|
+
// We need to redo the "next" replication stream
|
|
102
106
|
await this.updateSyncRules(next.asUpdateOptions());
|
|
103
107
|
// Pro-actively stop replicating
|
|
104
108
|
await this.db.sync_rules.updateOne(
|
|
@@ -114,7 +118,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
114
118
|
);
|
|
115
119
|
await this.db.notifyCheckpoint();
|
|
116
120
|
} else if (next == null && active?.id == sync_rules_group_id) {
|
|
117
|
-
// Slot removed for "active"
|
|
121
|
+
// Slot removed for "active" replication stream, while there is no "next" one.
|
|
118
122
|
await this.updateSyncRules(active.asUpdateOptions());
|
|
119
123
|
|
|
120
124
|
// In this case we keep the old one as active for clients, so that that existing clients
|
|
@@ -134,7 +138,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
134
138
|
);
|
|
135
139
|
await this.db.notifyCheckpoint();
|
|
136
140
|
} else if (next != null && active?.id == sync_rules_group_id) {
|
|
137
|
-
// Already have next
|
|
141
|
+
// Already have next replication stream, but need to stop replicating the active one.
|
|
138
142
|
|
|
139
143
|
await this.db.sync_rules.updateOne(
|
|
140
144
|
{
|
|
@@ -152,14 +156,14 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
152
156
|
}
|
|
153
157
|
|
|
154
158
|
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
|
|
155
|
-
const storageVersion =
|
|
159
|
+
const storageVersion =
|
|
160
|
+
options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
|
|
156
161
|
const storageConfig = getMongoStorageConfig(storageVersion);
|
|
157
|
-
await this.db.initializeStorageVersion(storageConfig);
|
|
158
162
|
|
|
159
163
|
let rules: MongoPersistedSyncRulesContent | undefined = undefined;
|
|
160
164
|
|
|
161
165
|
await this.session.withTransaction(async () => {
|
|
162
|
-
// Only have a single
|
|
166
|
+
// Only have a single replication stream with PROCESSING.
|
|
163
167
|
await this.db.sync_rules.updateMany(
|
|
164
168
|
{
|
|
165
169
|
state: storage.SyncRuleState.PROCESSING
|
|
@@ -203,6 +207,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
203
207
|
last_fatal_error_ts: null,
|
|
204
208
|
last_keepalive_ts: null
|
|
205
209
|
};
|
|
210
|
+
if (storageConfig.incrementalReprocessing) {
|
|
211
|
+
const parsed = options.config.parsed;
|
|
212
|
+
doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize();
|
|
213
|
+
}
|
|
206
214
|
await this.db.sync_rules.insertOne(doc);
|
|
207
215
|
await this.db.notifyCheckpoint();
|
|
208
216
|
rules = new MongoPersistedSyncRulesContent(this.db, doc);
|
|
@@ -294,63 +302,90 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
294
302
|
}
|
|
295
303
|
};
|
|
296
304
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
return {
|
|
300
|
-
operations_size_bytes: 0,
|
|
301
|
-
parameters_size_bytes: 0,
|
|
302
|
-
replication_size_bytes: 0
|
|
303
|
-
};
|
|
304
|
-
}
|
|
305
|
-
const operations_aggregate = await this.db.bucket_data
|
|
305
|
+
// For now, we get storage metrics over all v1 and v3 collections.
|
|
306
|
+
// In the future, we may split these metrics to report separately for active replication streams versus processing streams.
|
|
306
307
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
308
|
+
const aggregateStaticCollection = async <T extends mongo.Document>(collection: mongo.Collection<T>) => {
|
|
309
|
+
// We check whether the collection exists before getting the statistics. This avoids repeated
|
|
310
|
+
// errors in the MongoDB logs if the collection hasn't been created yet.
|
|
311
|
+
const exists =
|
|
312
|
+
(await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length >
|
|
313
|
+
0;
|
|
314
|
+
if (!exists) {
|
|
315
|
+
return [{ storageStats: { size: 0 } }];
|
|
316
|
+
}
|
|
316
317
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
318
|
+
return collection
|
|
319
|
+
.aggregate([
|
|
320
|
+
{
|
|
321
|
+
$collStats: {
|
|
322
|
+
storageStats: {}
|
|
323
|
+
}
|
|
322
324
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
325
|
+
])
|
|
326
|
+
.toArray()
|
|
327
|
+
.catch(ignoreNotExisting);
|
|
328
|
+
};
|
|
327
329
|
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
330
|
+
const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data);
|
|
331
|
+
const v3_operation_aggregates = await Promise.all(
|
|
332
|
+
(await this.db.listBucketDataCollectionsV3()).map((collection) =>
|
|
333
|
+
collection
|
|
334
|
+
.aggregate([
|
|
335
|
+
{
|
|
336
|
+
$collStats: {
|
|
337
|
+
storageStats: {}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
])
|
|
341
|
+
.toArray()
|
|
342
|
+
.catch(ignoreNotExisting)
|
|
343
|
+
)
|
|
344
|
+
);
|
|
338
345
|
|
|
339
|
-
const
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
346
|
+
const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters);
|
|
347
|
+
|
|
348
|
+
const v3_parameter_aggregates = await Promise.all(
|
|
349
|
+
(await this.db.listAllParameterIndexCollectionsV3()).map((collection) =>
|
|
350
|
+
collection
|
|
351
|
+
.aggregate([
|
|
352
|
+
{
|
|
353
|
+
$collStats: {
|
|
354
|
+
storageStats: {}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
])
|
|
358
|
+
.toArray()
|
|
359
|
+
.catch(ignoreNotExisting)
|
|
360
|
+
)
|
|
361
|
+
);
|
|
362
|
+
|
|
363
|
+
const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data);
|
|
364
|
+
|
|
365
|
+
const source_record_aggregates = await Promise.all(
|
|
366
|
+
(await this.db.listAllSourceRecordCollectionsV3()).map((collection) =>
|
|
367
|
+
collection
|
|
368
|
+
.aggregate([
|
|
369
|
+
{
|
|
370
|
+
$collStats: {
|
|
371
|
+
storageStats: {}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
])
|
|
375
|
+
.toArray()
|
|
376
|
+
.catch(ignoreNotExisting)
|
|
377
|
+
)
|
|
378
|
+
);
|
|
349
379
|
return {
|
|
350
|
-
operations_size_bytes:
|
|
351
|
-
|
|
380
|
+
operations_size_bytes:
|
|
381
|
+
Number(operations_aggregate[0].storageStats.size) +
|
|
382
|
+
v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
|
|
383
|
+
parameters_size_bytes:
|
|
384
|
+
Number(parameters_aggregate[0].storageStats.size) +
|
|
385
|
+
v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
|
|
352
386
|
replication_size_bytes:
|
|
353
|
-
Number(
|
|
387
|
+
Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) +
|
|
388
|
+
source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0)
|
|
354
389
|
};
|
|
355
390
|
}
|
|
356
391
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
1
2
|
import { storage } from '@powersync/service-core';
|
|
2
3
|
import { event_types } from '@powersync/service-types';
|
|
3
|
-
import { PowerSyncMongo } from './implementation/db.js';
|
|
4
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
5
4
|
import { createPaginatedConnectionQuery } from '../utils/util.js';
|
|
5
|
+
import { PowerSyncMongo } from './implementation/db.js';
|
|
6
6
|
|
|
7
7
|
export class MongoReportStorage implements storage.ReportStorage {
|
|
8
8
|
public readonly db: PowerSyncMongo;
|