@powersync/service-module-mongodb-storage 0.18.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +5 -1
- package/dist/storage/MongoBucketStorage.js +5 -1
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +3 -0
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +15 -19
- package/dist/storage/implementation/MongoCompactor.js +68 -260
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +26 -3
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +10 -4
- package/dist/storage/implementation/MongoSyncBucketStorage.js +13 -12
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +146 -34
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +6 -0
- package/dist/storage/implementation/common/PersistedBatch.js +15 -8
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +4 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +58 -84
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +250 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +19 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +35 -23
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +1 -0
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +59 -103
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +23 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +354 -219
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +5 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +12 -11
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +5 -3
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +108 -106
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +5 -2
- package/dist/storage/implementation/v3/PersistedBatchV3.js +53 -15
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +2 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +3 -0
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +10 -1
- package/dist/storage/implementation/v3/bucket-format.js +28 -9
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/chunking.js +3 -1
- package/dist/storage/implementation/v3/chunking.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +43 -2
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +9 -1
- package/dist/storage/implementation/v3/source-table-utils.js +20 -21
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/types/types.d.ts +35 -0
- package/dist/types/types.js +35 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.js +3 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +8 -1
- package/dist/utils/util.js +125 -2
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -7
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +10 -2
- package/src/storage/implementation/MongoBucketBatch.ts +4 -0
- package/src/storage/implementation/MongoCompactor.ts +90 -314
- package/src/storage/implementation/MongoStorageProvider.ts +32 -3
- package/src/storage/implementation/MongoSyncBucketStorage.ts +23 -22
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +164 -37
- package/src/storage/implementation/common/PersistedBatch.ts +22 -10
- package/src/storage/implementation/models.ts +20 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +76 -96
- package/src/storage/implementation/v1/MongoCompactorV1.ts +292 -8
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +53 -40
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +40 -23
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +75 -105
- package/src/storage/implementation/v3/MongoCompactorV3.ts +456 -258
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +21 -19
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +122 -126
- package/src/storage/implementation/v3/PersistedBatchV3.ts +69 -16
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +5 -0
- package/src/storage/implementation/v3/bucket-format.ts +36 -9
- package/src/storage/implementation/v3/chunking.ts +3 -1
- package/src/storage/implementation/v3/models.ts +45 -1
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +43 -35
- package/src/types/types.ts +42 -1
- package/src/utils/test-utils.ts +3 -1
- package/src/utils/util.ts +182 -4
- package/test/src/__snapshots__/storage_sync.test.ts.snap +1 -583
- package/test/src/cleanup-stopped-sync-configs.test.ts +24 -1
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +216 -2
- package/test/src/storage_compacting.test.ts +176 -187
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +124 -5
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -84
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -108
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
-
import { BucketDataProperties } from '../models.js';
|
|
4
|
-
import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
|
|
5
|
-
declare const GENERIC_ID: unique symbol;
|
|
6
|
-
export type BucketDataDocumentGenericId = {
|
|
7
|
-
b: string;
|
|
8
|
-
o: InternalOpId;
|
|
9
|
-
[GENERIC_ID]: true;
|
|
10
|
-
};
|
|
11
|
-
/**
|
|
12
|
-
* This document is never actually constructed - we use it as a "virtual" type.
|
|
13
|
-
*
|
|
14
|
-
* The actual implementations are BucketDataDocumentV1 or BucketDataDocumentV3.
|
|
15
|
-
* They don't fully satisfy this interface, but this works to share common implementations.
|
|
16
|
-
*
|
|
17
|
-
* The idea is that we can have a common implementation between V1 & V3, using this type,
|
|
18
|
-
* and operate on MongoDB collections.
|
|
19
|
-
*
|
|
20
|
-
* This interface serves two primary purposes:
|
|
21
|
-
* 1. Captures properties that exist on both V1 and V3 storage models.
|
|
22
|
-
* 2. Gives a common reference when querying or modifying collections.
|
|
23
|
-
*
|
|
24
|
-
* Generics would've been ideal, but they don't play well with MongoDB collections.
|
|
25
|
-
*/
|
|
26
|
-
export interface BucketDataDocumentGeneric extends BucketDataProperties {
|
|
27
|
-
_id: BucketDataDocumentGenericId;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Represent read/write access for a single bucket.
|
|
31
|
-
*
|
|
32
|
-
* This does not implement the actual collection operations, but supports the required conversions
|
|
33
|
-
* between in-memory BucketDataDoc and the specific storage formats.
|
|
34
|
-
*/
|
|
35
|
-
export interface SingleBucketStore {
|
|
36
|
-
readonly key: BucketKey;
|
|
37
|
-
readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
|
|
38
|
-
docId(o: InternalOpId): BucketDataDocumentGenericId;
|
|
39
|
-
readonly minId: BucketDataDocumentGenericId;
|
|
40
|
-
readonly maxId: BucketDataDocumentGenericId;
|
|
41
|
-
/**
|
|
42
|
-
* Convert in-memory document -> persisted document.
|
|
43
|
-
*/
|
|
44
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric;
|
|
45
|
-
/**
|
|
46
|
-
* Convert persisted document -> in-memory document.
|
|
47
|
-
*/
|
|
48
|
-
fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc;
|
|
49
|
-
/**
|
|
50
|
-
* Convert partial persisted document -> partial in-memory document.
|
|
51
|
-
*/
|
|
52
|
-
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(doc: Pick<BucketDataDocumentGeneric, '_id' | T>): Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
|
|
53
|
-
}
|
|
54
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SingleBucketStore.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/SingleBucketStore.ts"],"names":[],"mappings":"AAKA,MAAM,UAAU,GAAG,MAAM,CAAC,6BAA6B,CAAC,CAAC"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
-
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
-
import { BucketDataDocumentGeneric, BucketDataDocumentGenericId, SingleBucketStore } from '../common/SingleBucketStore.js';
|
|
5
|
-
import { BucketDataProperties } from '../models.js';
|
|
6
|
-
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
7
|
-
export declare class SingleBucketStoreV3 implements SingleBucketStore {
|
|
8
|
-
private db;
|
|
9
|
-
readonly key: BucketKey;
|
|
10
|
-
readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
|
|
11
|
-
constructor(db: VersionedPowerSyncMongoV3, key: BucketKey);
|
|
12
|
-
docId(o: InternalOpId): BucketDataDocumentGenericId;
|
|
13
|
-
get minId(): BucketDataDocumentGenericId;
|
|
14
|
-
get maxId(): BucketDataDocumentGenericId;
|
|
15
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric;
|
|
16
|
-
fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc;
|
|
17
|
-
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(doc: Pick<BucketDataDocumentGeneric, '_id' | T>): Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
|
|
18
|
-
}
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { loadBucketDataDocument, serializeBucketData } from './bucket-format.js';
|
|
3
|
-
function extractPartialDocumentFields(doc) {
|
|
4
|
-
if (typeof doc !== 'object' || doc === null) {
|
|
5
|
-
throw new Error('Invalid partial document: expected object');
|
|
6
|
-
}
|
|
7
|
-
const d = doc;
|
|
8
|
-
const id = d._id;
|
|
9
|
-
if (typeof id !== 'object' || id === null || !('o' in id)) {
|
|
10
|
-
throw new Error('Invalid partial document: missing _id.o');
|
|
11
|
-
}
|
|
12
|
-
return d;
|
|
13
|
-
}
|
|
14
|
-
// MongoDB's MinKey/MaxKey are special sentinel values that don't match the bigint type
|
|
15
|
-
// for _id.o in BucketDataDocumentGenericId, so we need an explicit cast.
|
|
16
|
-
function minKeyForBucket(bucket) {
|
|
17
|
-
return {
|
|
18
|
-
b: bucket,
|
|
19
|
-
o: new mongo.MinKey()
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
function maxKeyForBucket(bucket) {
|
|
23
|
-
return {
|
|
24
|
-
b: bucket,
|
|
25
|
-
o: new mongo.MaxKey()
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
export class SingleBucketStoreV3 {
|
|
29
|
-
db;
|
|
30
|
-
key;
|
|
31
|
-
collection;
|
|
32
|
-
constructor(db, key) {
|
|
33
|
-
this.db = db;
|
|
34
|
-
this.key = key;
|
|
35
|
-
// Cast from the version-specific collection type to the generic interface
|
|
36
|
-
// used across storage versions.
|
|
37
|
-
this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
|
|
38
|
-
}
|
|
39
|
-
docId(o) {
|
|
40
|
-
// `satisfies BucketDataKey` checks that we use the correct type for V3 storage
|
|
41
|
-
// `as BucketDataDocumentGenericId` does a cast to get the interface virtual type
|
|
42
|
-
return {
|
|
43
|
-
b: this.key.bucket,
|
|
44
|
-
o
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
get minId() {
|
|
48
|
-
return minKeyForBucket(this.key.bucket);
|
|
49
|
-
}
|
|
50
|
-
get maxId() {
|
|
51
|
-
return maxKeyForBucket(this.key.bucket);
|
|
52
|
-
}
|
|
53
|
-
toPersistedDocument(source) {
|
|
54
|
-
return serializeBucketData(this.key.bucket, [
|
|
55
|
-
{ bucketKey: this.key, ...source }
|
|
56
|
-
]);
|
|
57
|
-
}
|
|
58
|
-
fromPersistedDocument(doc) {
|
|
59
|
-
const generator = loadBucketDataDocument(this.key, doc);
|
|
60
|
-
const first = generator.next();
|
|
61
|
-
if (first.done) {
|
|
62
|
-
throw new Error('Empty ops array in BucketDataDocumentV3');
|
|
63
|
-
}
|
|
64
|
-
return first.value;
|
|
65
|
-
}
|
|
66
|
-
fromPartialPersistedDocument(doc) {
|
|
67
|
-
const document = doc;
|
|
68
|
-
const op = document.ops?.[0];
|
|
69
|
-
if (op == null) {
|
|
70
|
-
const fields = extractPartialDocumentFields(doc);
|
|
71
|
-
const { _id, ...rest } = fields;
|
|
72
|
-
return {
|
|
73
|
-
bucketKey: this.key,
|
|
74
|
-
o: _id.o,
|
|
75
|
-
...rest
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
return {
|
|
79
|
-
bucketKey: this.key,
|
|
80
|
-
...op
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
//# sourceMappingURL=SingleBucketStoreV3.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SingleBucketStoreV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/SingleBucketStoreV3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AASvD,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAIjF,SAAS,4BAA4B,CAAC,GAAY;IAChD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC;IACjB,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAmD,CAAC;AAC7D,CAAC;AAED,uFAAuF;AACvF,yEAAyE;AACzE,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO;QACL,CAAC,EAAE,MAAM;QACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;KACoB,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO;QACL,CAAC,EAAE,MAAM;QACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;KACoB,CAAC;AAC9C,CAAC;AAED,MAAM,OAAO,mBAAmB;IAIpB;IACQ;IAJF,UAAU,CAA8C;IAExE,YACU,EAA6B,EACrB,GAAc;QADtB,OAAE,GAAF,EAAE,CAA2B;QACrB,QAAG,GAAH,GAAG,CAAW;QAE9B,0EAA0E;QAC1E,gCAAgC;QAChC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAC7B,GAAG,CAAC,mBAAmB,EACvB,GAAG,CAAC,YAAY,CACyC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,CAAe;QACnB,+EAA+E;QAC/E,iFAAiF;QACjF,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC;SACqD,CAAC;IAC3D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,KAAK;QACP,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,mBAAmB,CAAC,MAAwC;QAC1D,OAAO,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE;YAC1C,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE;SACnC,CAAyC,CAAC;IAC7C,CAAC;IAED,qBAAqB,CAAC,GAA8B;QAClD,MAAM,SAAS,GAAG,sBAAsB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAsC,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,4BAA4B,CAC1B,GAA+C;QAE/C,MAAM,QAAQ,GAAG,GAA2D,CAAC;QAC7E,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;YACjD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;YAChC,OAAO;gBACL,SAAS,EAAE,IAAI,CAAC,GAAG;gBACnB,CAAC,EAAE,GAAG,CAAC,CAAC;gBACR,GAAG,IAAI;aACsC,CAAC;QAClD,CAAC;QACD,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,GAAG,EAAE;SACwC,CAAC;IAClD,CAAC;CACF"}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
-
import { BucketDataProperties } from '../models.js';
|
|
4
|
-
import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
|
|
5
|
-
|
|
6
|
-
const GENERIC_ID = Symbol('BucketDataDocumentGenericId');
|
|
7
|
-
export type BucketDataDocumentGenericId = {
|
|
8
|
-
b: string;
|
|
9
|
-
o: InternalOpId;
|
|
10
|
-
// Hack to ensure this can't be constructed directly
|
|
11
|
-
[GENERIC_ID]: true;
|
|
12
|
-
};
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* This document is never actually constructed - we use it as a "virtual" type.
|
|
16
|
-
*
|
|
17
|
-
* The actual implementations are BucketDataDocumentV1 or BucketDataDocumentV3.
|
|
18
|
-
* They don't fully satisfy this interface, but this works to share common implementations.
|
|
19
|
-
*
|
|
20
|
-
* The idea is that we can have a common implementation between V1 & V3, using this type,
|
|
21
|
-
* and operate on MongoDB collections.
|
|
22
|
-
*
|
|
23
|
-
* This interface serves two primary purposes:
|
|
24
|
-
* 1. Captures properties that exist on both V1 and V3 storage models.
|
|
25
|
-
* 2. Gives a common reference when querying or modifying collections.
|
|
26
|
-
*
|
|
27
|
-
* Generics would've been ideal, but they don't play well with MongoDB collections.
|
|
28
|
-
*/
|
|
29
|
-
export interface BucketDataDocumentGeneric extends BucketDataProperties {
|
|
30
|
-
_id: BucketDataDocumentGenericId;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Represent read/write access for a single bucket.
|
|
35
|
-
*
|
|
36
|
-
* This does not implement the actual collection operations, but supports the required conversions
|
|
37
|
-
* between in-memory BucketDataDoc and the specific storage formats.
|
|
38
|
-
*/
|
|
39
|
-
export interface SingleBucketStore {
|
|
40
|
-
readonly key: BucketKey;
|
|
41
|
-
|
|
42
|
-
readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
|
|
43
|
-
docId(o: InternalOpId): BucketDataDocumentGenericId;
|
|
44
|
-
readonly minId: BucketDataDocumentGenericId;
|
|
45
|
-
readonly maxId: BucketDataDocumentGenericId;
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Convert in-memory document -> persisted document.
|
|
49
|
-
*/
|
|
50
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric;
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* Convert persisted document -> in-memory document.
|
|
54
|
-
*/
|
|
55
|
-
fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc;
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Convert partial persisted document -> partial in-memory document.
|
|
59
|
-
*/
|
|
60
|
-
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
|
|
61
|
-
doc: Pick<BucketDataDocumentGeneric, '_id' | T>
|
|
62
|
-
): Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
|
|
63
|
-
}
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
-
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
-
import {
|
|
5
|
-
BucketDataDocumentGeneric,
|
|
6
|
-
BucketDataDocumentGenericId,
|
|
7
|
-
SingleBucketStore
|
|
8
|
-
} from '../common/SingleBucketStore.js';
|
|
9
|
-
import { BucketDataKey, BucketDataProperties } from '../models.js';
|
|
10
|
-
import { loadBucketDataDocument, serializeBucketData } from './bucket-format.js';
|
|
11
|
-
import { BucketDataDocumentV3 } from './models.js';
|
|
12
|
-
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
13
|
-
|
|
14
|
-
function extractPartialDocumentFields(doc: unknown): { _id: { o: bigint }; [key: string]: unknown } {
|
|
15
|
-
if (typeof doc !== 'object' || doc === null) {
|
|
16
|
-
throw new Error('Invalid partial document: expected object');
|
|
17
|
-
}
|
|
18
|
-
const d = doc as Record<string, unknown>;
|
|
19
|
-
const id = d._id;
|
|
20
|
-
if (typeof id !== 'object' || id === null || !('o' in id)) {
|
|
21
|
-
throw new Error('Invalid partial document: missing _id.o');
|
|
22
|
-
}
|
|
23
|
-
return d as { _id: { o: bigint }; [key: string]: unknown };
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
// MongoDB's MinKey/MaxKey are special sentinel values that don't match the bigint type
|
|
27
|
-
// for _id.o in BucketDataDocumentGenericId, so we need an explicit cast.
|
|
28
|
-
function minKeyForBucket(bucket: string): BucketDataDocumentGenericId {
|
|
29
|
-
return {
|
|
30
|
-
b: bucket,
|
|
31
|
-
o: new mongo.MinKey()
|
|
32
|
-
} as unknown as BucketDataDocumentGenericId;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function maxKeyForBucket(bucket: string): BucketDataDocumentGenericId {
|
|
36
|
-
return {
|
|
37
|
-
b: bucket,
|
|
38
|
-
o: new mongo.MaxKey()
|
|
39
|
-
} as unknown as BucketDataDocumentGenericId;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export class SingleBucketStoreV3 implements SingleBucketStore {
|
|
43
|
-
public readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
|
|
44
|
-
|
|
45
|
-
constructor(
|
|
46
|
-
private db: VersionedPowerSyncMongoV3,
|
|
47
|
-
public readonly key: BucketKey
|
|
48
|
-
) {
|
|
49
|
-
// Cast from the version-specific collection type to the generic interface
|
|
50
|
-
// used across storage versions.
|
|
51
|
-
this.collection = db.bucketData(
|
|
52
|
-
key.replicationStreamId,
|
|
53
|
-
key.definitionId
|
|
54
|
-
) as unknown as mongo.Collection<BucketDataDocumentGeneric>;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
docId(o: InternalOpId): BucketDataDocumentGenericId {
|
|
58
|
-
// `satisfies BucketDataKey` checks that we use the correct type for V3 storage
|
|
59
|
-
// `as BucketDataDocumentGenericId` does a cast to get the interface virtual type
|
|
60
|
-
return {
|
|
61
|
-
b: this.key.bucket,
|
|
62
|
-
o
|
|
63
|
-
} satisfies BucketDataKey as BucketDataDocumentGenericId;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
get minId(): BucketDataDocumentGenericId {
|
|
67
|
-
return minKeyForBucket(this.key.bucket);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
get maxId(): BucketDataDocumentGenericId {
|
|
71
|
-
return maxKeyForBucket(this.key.bucket);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric {
|
|
75
|
-
return serializeBucketData(this.key.bucket, [
|
|
76
|
-
{ bucketKey: this.key, ...source }
|
|
77
|
-
]) as unknown as BucketDataDocumentGeneric;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc {
|
|
81
|
-
const generator = loadBucketDataDocument(this.key, doc as unknown as BucketDataDocumentV3);
|
|
82
|
-
const first = generator.next();
|
|
83
|
-
if (first.done) {
|
|
84
|
-
throw new Error('Empty ops array in BucketDataDocumentV3');
|
|
85
|
-
}
|
|
86
|
-
return first.value;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
|
|
90
|
-
doc: Pick<BucketDataDocumentGeneric, '_id' | T>
|
|
91
|
-
): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
|
|
92
|
-
const document = doc as unknown as Pick<BucketDataDocumentV3, '_id' | 'ops'>;
|
|
93
|
-
const op = document.ops?.[0];
|
|
94
|
-
if (op == null) {
|
|
95
|
-
const fields = extractPartialDocumentFields(doc);
|
|
96
|
-
const { _id, ...rest } = fields;
|
|
97
|
-
return {
|
|
98
|
-
bucketKey: this.key,
|
|
99
|
-
o: _id.o,
|
|
100
|
-
...rest
|
|
101
|
-
} as Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
|
|
102
|
-
}
|
|
103
|
-
return {
|
|
104
|
-
bucketKey: this.key,
|
|
105
|
-
...op
|
|
106
|
-
} as Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
|
|
107
|
-
}
|
|
108
|
-
}
|